diff --git a/.github/workflows/README.md b/.github/workflows/README.md index d4363fe3..e1c68c53 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -409,6 +409,58 @@ gh workflow run skill-code-review.yml -f pr_number=42 - `MAINTAINERS` — authorized users list - `.github/workflows/skill-code-review.yml` — workflow definition +### 6. `mcp-tool-validation.yml` - MCP Tool Validation + +**Purpose**: Validates that `allowed-tools` declarations in SKILL.md frontmatter match the actual tools exposed by MCP servers defined in each pack's `mcps.json`. + +**Triggers**: +- **Pull requests** → Validates only packs with changed `mcps.json` or `skills/*/SKILL.md` files +- **Pushes to main** → Validates all packs +- **Manual dispatch** → Optionally specify a single pack name to validate + +**What it validates**: +- ✅ Starts each container-based MCP server via `podman` +- ✅ Queries tools via JSON-RPC (`initialize` + `tools/list`) +- ✅ Cross-references declared `allowed-tools` against actual tool names +- ✅ Suggests corrections for misspelled tool names (Levenshtein distance) + +**Classification**: +- **PASS** — All declared tools found in started MCP servers +- **WARN** — Tools could not be verified because their MCP server is non-container (`npx`, `uvx`, empty command) or failed to start. Does not block the PR +- **SKIP** — Skill has no `allowed-tools` declared +- **FAIL** — Tools missing from MCP servers that were successfully started. Blocks the PR + +**How to run locally**: +```bash +# Validate all packs +python scripts/validate_mcp_tools.py + +# Validate specific packs +python scripts/validate_mcp_tools.py rh-sre ocp-admin rh-virt +``` + +**Expected output**: +``` +VALIDATION SUMMARY +------------------------------------------------------------------ + Total skills: 71 + Passed: 31 + Warned (unverifiable): 31 + Skipped (no allowed-tools): 9 + Failed: 0 + +PASSED WITH WARNINGS - some tools could not be verified (MCP servers not started) +``` + +**Prerequisites**: +- `podman` installed +- `KUBECONFIG` set (or `~/.kube/config` present) — a dummy kubeconfig is created in CI + +**Related files**: +- `scripts/validate_mcp_tools.py` — validation script +- `*/mcps.json` — MCP server configurations per pack +- `*/skills/*/SKILL.md` — skill definitions with `allowed-tools` frontmatter + ## Adding New Workflows When adding new workflows: @@ -482,5 +534,5 @@ This README should be updated when: - New validation levels are introduced - Troubleshooting patterns emerge -**Last Updated**: 2026-05-06 -**Workflows Count**: 5 (skill-spec-report.yml, compliance-check.yml, deploy-pages.yml, skill-security-scan.yml, skill-code-review.yml) +**Last Updated**: 2026-05-26 +**Workflows Count**: 6 (skill-spec-report.yml, compliance-check.yml, deploy-pages.yml, skill-security-scan.yml, skill-code-review.yml, mcp-tool-validation.yml) diff --git a/.github/workflows/mcp-tool-validation.yml b/.github/workflows/mcp-tool-validation.yml index c1908697..152309d8 100644 --- a/.github/workflows/mcp-tool-validation.yml +++ b/.github/workflows/mcp-tool-validation.yml @@ -57,16 +57,26 @@ jobs: - cluster: server: https://localhost:6443 name: mcp-validation + - cluster: + server: https://localhost:6444 + name: mcp-validation-2 contexts: - context: cluster: mcp-validation user: mcp-validation name: mcp-validation + - context: + cluster: mcp-validation-2 + user: mcp-validation-2 + name: mcp-validation-2 current-context: mcp-validation users: - name: mcp-validation user: token: dummy-token-for-tool-listing + - name: mcp-validation-2 + user: + token: dummy-token-for-tool-listing-2 KUBECONFIG echo "KUBECONFIG=$HOME/.kube/config" >> "$GITHUB_ENV" diff --git a/ocp-admin/skills/cluster-creator/SKILL.md b/ocp-admin/skills/cluster-creator/SKILL.md index 0858279f..2bf0d057 100644 --- a/ocp-admin/skills/cluster-creator/SKILL.md +++ b/ocp-admin/skills/cluster-creator/SKILL.md @@ -20,6 +20,7 @@ description: | license: Apache-2.0 model: inherit color: green +allowed-tools: list_versions create_cluster cluster_info set_cluster_vips set_host_role cluster_iso_download_url install_cluster cluster_credentials_download_url cluster_logs_download_url list_static_network_config generate_nmstate_yaml validate_nmstate_yaml alter_static_network_config_nmstate_for_host metadata: mcp_server: openshift-self-managed mcp_tools_priority: true diff --git a/ocp-admin/skills/cluster-inventory/SKILL.md b/ocp-admin/skills/cluster-inventory/SKILL.md index 8cf86f0d..0cc5d71e 100644 --- a/ocp-admin/skills/cluster-inventory/SKILL.md +++ b/ocp-admin/skills/cluster-inventory/SKILL.md @@ -16,6 +16,7 @@ description: | license: Apache-2.0 model: inherit color: cyan +allowed-tools: list_clusters cluster_info cluster_events cluster_logs_download_url metadata: mcp_servers: - openshift-self-managed diff --git a/ocp-admin/skills/cluster-report/SKILL.md b/ocp-admin/skills/cluster-report/SKILL.md index e25660a4..c4a8db28 100644 --- a/ocp-admin/skills/cluster-report/SKILL.md +++ b/ocp-admin/skills/cluster-report/SKILL.md @@ -16,6 +16,7 @@ description: | license: Apache-2.0 model: inherit color: cyan +allowed-tools: configuration_contexts_list resources_get nodes_top resources_list namespaces_list pods_list metadata: mcp_server: openshift-administration mcp_tools_priority: true diff --git a/rh-ai-engineer/skills/ai-observability/SKILL.md b/rh-ai-engineer/skills/ai-observability/SKILL.md index fe11567a..3e4b2240 100644 --- a/rh-ai-engineer/skills/ai-observability/SKILL.md +++ b/rh-ai-engineer/skills/ai-observability/SKILL.md @@ -18,6 +18,7 @@ description: | model: inherit color: cyan license: Apache-2.0 +allowed-tools: list_models list_vllm_namespaces get_gpu_info get_deployment_info analyze_vllm chat_vllm analyze_openshift list_openshift_metric_groups list_openshift_namespaces query_tempo_tool get_trace_details_tool search_metrics execute_promql korrel8r_get_correlated list_data_science_projects list_inference_services get_inference_service resources_get resources_list pods_list --- # /ai-observability Skill diff --git a/rh-ai-engineer/skills/debug-inference/SKILL.md b/rh-ai-engineer/skills/debug-inference/SKILL.md index 0539ed12..7dcf0328 100644 --- a/rh-ai-engineer/skills/debug-inference/SKILL.md +++ b/rh-ai-engineer/skills/debug-inference/SKILL.md @@ -17,6 +17,7 @@ description: | model: inherit color: yellow license: Apache-2.0 +allowed-tools: resources_get resources_list pods_list pods_log events_list list_inference_services get_inference_service get_model_endpoint get_deployment_info analyze_vllm chat_vllm get_gpu_info analyze_openshift query_tempo_tool get_trace_details_tool execute_promql korrel8r_get_correlated --- # /debug-inference Skill diff --git a/rh-ai-engineer/skills/ds-project-setup/SKILL.md b/rh-ai-engineer/skills/ds-project-setup/SKILL.md index cf5ca9be..0c30759f 100644 --- a/rh-ai-engineer/skills/ds-project-setup/SKILL.md +++ b/rh-ai-engineer/skills/ds-project-setup/SKILL.md @@ -21,6 +21,7 @@ metadata: author: "Red Hat Ecosystem Engineering" version: "1.0" license: Apache-2.0 +allowed-tools: resources_get resources_list resources_create_or_update list_data_science_projects create_data_science_project get_project_details get_project_status create_s3_data_connection list_data_connections get_pipeline_server set_model_serving_mode --- # /ds-project-setup Skill diff --git a/rh-ai-engineer/skills/guardrails-config/SKILL.md b/rh-ai-engineer/skills/guardrails-config/SKILL.md index 8361c46c..f0d1e4c1 100644 --- a/rh-ai-engineer/skills/guardrails-config/SKILL.md +++ b/rh-ai-engineer/skills/guardrails-config/SKILL.md @@ -18,6 +18,7 @@ description: | model: inherit color: blue license: Apache-2.0 +allowed-tools: resources_get resources_list resources_create_or_update resources_delete pods_list pods_log events_list list_inference_services get_inference_service get_model_endpoint test_model_endpoint deploy_model list_serving_runtimes recommend_serving_runtime execute_promql analyze_vllm --- # /guardrails-config Skill diff --git a/rh-ai-engineer/skills/model-deploy/SKILL.md b/rh-ai-engineer/skills/model-deploy/SKILL.md index 4fa28044..9627483b 100644 --- a/rh-ai-engineer/skills/model-deploy/SKILL.md +++ b/rh-ai-engineer/skills/model-deploy/SKILL.md @@ -17,6 +17,7 @@ description: | model: inherit color: green license: Apache-2.0 +allowed-tools: resources_get resources_list resources_create_or_update pods_list pods_log events_list deploy_model list_inference_services get_inference_service get_model_endpoint list_serving_runtimes list_data_science_projects list_data_connections get_gpu_info get_deployment_info analyze_vllm --- # /model-deploy Skill diff --git a/rh-ai-engineer/skills/model-monitor/SKILL.md b/rh-ai-engineer/skills/model-monitor/SKILL.md index 438a9125..a75e393e 100644 --- a/rh-ai-engineer/skills/model-monitor/SKILL.md +++ b/rh-ai-engineer/skills/model-monitor/SKILL.md @@ -18,6 +18,7 @@ description: | model: inherit color: blue license: Apache-2.0 +allowed-tools: resources_get resources_list resources_create_or_update pods_list pods_log events_list list_inference_services get_inference_service list_data_science_projects execute_promql --- # /model-monitor Skill diff --git a/rh-ai-engineer/skills/model-registry/SKILL.md b/rh-ai-engineer/skills/model-registry/SKILL.md index 195ef7ce..06217366 100644 --- a/rh-ai-engineer/skills/model-registry/SKILL.md +++ b/rh-ai-engineer/skills/model-registry/SKILL.md @@ -17,6 +17,7 @@ description: | color: cyan model: inherit license: Apache-2.0 +allowed-tools: resources_create_or_update resources_get resources_list list_registered_models get_registered_model list_model_versions get_model_version get_model_artifacts get_model_benchmarks get_catalog_model_artifacts list_data_science_projects list_data_connections --- # /model-registry Skill diff --git a/rh-ai-engineer/skills/nim-setup/SKILL.md b/rh-ai-engineer/skills/nim-setup/SKILL.md index 02643cf6..7dc34eaf 100644 --- a/rh-ai-engineer/skills/nim-setup/SKILL.md +++ b/rh-ai-engineer/skills/nim-setup/SKILL.md @@ -16,6 +16,7 @@ description: | model: inherit color: blue license: Apache-2.0 +allowed-tools: resources_get resources_list resources_create_or_update events_list list_data_science_projects list_serving_runtimes get_gpu_info --- # /nim-setup Skill diff --git a/rh-ai-engineer/skills/pipeline-manage/SKILL.md b/rh-ai-engineer/skills/pipeline-manage/SKILL.md index f1862661..bba1d7bb 100644 --- a/rh-ai-engineer/skills/pipeline-manage/SKILL.md +++ b/rh-ai-engineer/skills/pipeline-manage/SKILL.md @@ -19,6 +19,7 @@ description: | color: green model: inherit license: Apache-2.0 +allowed-tools: resources_create_or_update resources_list resources_get resources_delete events_list pods_list pods_log list_data_science_projects get_pipeline_server delete_pipeline_server list_resources get_resource list_resource_names resource_status diagnose_resource list_data_connections project_summary --- # /pipeline-manage Skill diff --git a/rh-ai-engineer/skills/serving-runtime-config/SKILL.md b/rh-ai-engineer/skills/serving-runtime-config/SKILL.md index 59dc783d..3970a74f 100644 --- a/rh-ai-engineer/skills/serving-runtime-config/SKILL.md +++ b/rh-ai-engineer/skills/serving-runtime-config/SKILL.md @@ -17,6 +17,7 @@ description: | model: inherit color: blue license: Apache-2.0 +allowed-tools: resources_get resources_list resources_create_or_update list_serving_runtimes create_serving_runtime list_data_science_projects list_models --- # /serving-runtime-config Skill diff --git a/rh-ai-engineer/skills/workbench-manage/SKILL.md b/rh-ai-engineer/skills/workbench-manage/SKILL.md index b5a556a3..cde5aa74 100644 --- a/rh-ai-engineer/skills/workbench-manage/SKILL.md +++ b/rh-ai-engineer/skills/workbench-manage/SKILL.md @@ -21,6 +21,7 @@ metadata: author: "Red Hat Ecosystem Engineering" version: "1.0" license: Apache-2.0 +allowed-tools: resources_get resources_list resources_create_or_update resources_delete events_list pods_list list_data_science_projects list_workbenches get_workbench create_workbench start_workbench stop_workbench delete_workbench get_workbench_url list_storage create_storage delete_storage list_data_connections --- # /workbench-manage Skill diff --git a/rh-automation/skills/aap-mcp-validator/SKILL.md b/rh-automation/skills/aap-mcp-validator/SKILL.md index 5af6e630..a4055c20 100644 --- a/rh-automation/skills/aap-mcp-validator/SKILL.md +++ b/rh-automation/skills/aap-mcp-validator/SKILL.md @@ -12,6 +12,7 @@ description: | model: inherit color: blue license: Apache-2.0 +allowed-tools: job_templates_list inventories_list notification_templates_list credentials_list instance_groups_list users_list --- # AAP MCP Validator diff --git a/rh-automation/skills/execution-risk-analyzer/SKILL.md b/rh-automation/skills/execution-risk-analyzer/SKILL.md index c8c455a1..3b1ebae9 100644 --- a/rh-automation/skills/execution-risk-analyzer/SKILL.md +++ b/rh-automation/skills/execution-risk-analyzer/SKILL.md @@ -13,6 +13,7 @@ description: | model: inherit color: yellow license: Apache-2.0 +allowed-tools: job_templates_list job_templates_retrieve job_templates_launch_retrieve jobs_list jobs_job_events_list workflow_job_templates_list inventories_list hosts_list --- # Execution Risk Analyzer diff --git a/rh-automation/skills/execution-summary/SKILL.md b/rh-automation/skills/execution-summary/SKILL.md index ac550f16..d8778502 100644 --- a/rh-automation/skills/execution-summary/SKILL.md +++ b/rh-automation/skills/execution-summary/SKILL.md @@ -13,6 +13,7 @@ description: | model: inherit color: green license: Apache-2.0 +allowed-tools: --- # Execution Summary diff --git a/rh-automation/skills/forensic-troubleshooter/SKILL.md b/rh-automation/skills/forensic-troubleshooter/SKILL.md index 52d9c019..c8373c8a 100644 --- a/rh-automation/skills/forensic-troubleshooter/SKILL.md +++ b/rh-automation/skills/forensic-troubleshooter/SKILL.md @@ -12,6 +12,7 @@ description: | model: inherit color: yellow license: Apache-2.0 +allowed-tools: job_templates_list jobs_retrieve jobs_job_events_list jobs_job_host_summaries_list jobs_stdout_retrieve inventories_list hosts_list hosts_variable_data_retrieve notification_templates_list credentials_list instance_groups_list users_list --- # Forensic Troubleshooter diff --git a/rh-automation/skills/governance-assessor/SKILL.md b/rh-automation/skills/governance-assessor/SKILL.md index 8cbc8629..4e2cb917 100644 --- a/rh-automation/skills/governance-assessor/SKILL.md +++ b/rh-automation/skills/governance-assessor/SKILL.md @@ -23,6 +23,7 @@ description: | model: inherit color: red license: Apache-2.0 +allowed-tools: job_templates_list workflow_job_templates_list inventories_list hosts_list notification_templates_list execution_environments_list notification_templates_create execution_environments_create credentials_list credential_types_list credentials_create instance_groups_list activity_stream_list instance_groups_create users_list teams_list role_user_assignments_list role_team_assignments_list authenticators_list teams_create role_user_assignments_create authenticators_create --- # Governance Assessor diff --git a/rh-automation/skills/governance-executor/SKILL.md b/rh-automation/skills/governance-executor/SKILL.md index f1fd1566..052f4279 100644 --- a/rh-automation/skills/governance-executor/SKILL.md +++ b/rh-automation/skills/governance-executor/SKILL.md @@ -12,6 +12,7 @@ description: | model: inherit color: red license: Apache-2.0 +allowed-tools: job_templates_list job_templates_retrieve job_templates_launch_retrieve job_templates_launch_create jobs_list jobs_retrieve jobs_job_events_list jobs_job_host_summaries_list jobs_relaunch_create workflow_job_templates_list inventories_list hosts_list notification_templates_list credentials_list instance_groups_list users_list --- # Governance Executor diff --git a/rh-automation/skills/governance-readiness-assessor/SKILL.md b/rh-automation/skills/governance-readiness-assessor/SKILL.md index e7b2973d..5674ae28 100644 --- a/rh-automation/skills/governance-readiness-assessor/SKILL.md +++ b/rh-automation/skills/governance-readiness-assessor/SKILL.md @@ -14,6 +14,7 @@ description: | model: inherit color: red license: Apache-2.0 +allowed-tools: workflow_job_templates_list job_templates_list notification_templates_list execution_environments_list notification_templates_create execution_environments_create users_list teams_list role_user_assignments_list role_team_assignments_list authenticators_list teams_create role_user_assignments_create authenticators_create credentials_list credential_types_list credentials_create instance_groups_list activity_stream_list instance_groups_create inventories_list hosts_list --- # Governance Readiness Assessor diff --git a/rh-automation/skills/governed-job-launcher/SKILL.md b/rh-automation/skills/governed-job-launcher/SKILL.md index 23e3a4b4..2dbe0fd1 100644 --- a/rh-automation/skills/governed-job-launcher/SKILL.md +++ b/rh-automation/skills/governed-job-launcher/SKILL.md @@ -13,6 +13,7 @@ description: | model: inherit color: red license: Apache-2.0 +allowed-tools: job_templates_launch_create jobs_retrieve jobs_job_events_list jobs_job_host_summaries_list jobs_relaunch_create --- # Governed Job Launcher diff --git a/rh-automation/skills/host-fact-inspector/SKILL.md b/rh-automation/skills/host-fact-inspector/SKILL.md index 4a2ac58d..375a94df 100644 --- a/rh-automation/skills/host-fact-inspector/SKILL.md +++ b/rh-automation/skills/host-fact-inspector/SKILL.md @@ -13,6 +13,7 @@ description: | model: inherit color: blue license: Apache-2.0 +allowed-tools: hosts_list hosts_variable_data_retrieve --- # Host Fact Inspector diff --git a/rh-automation/skills/job-failure-analyzer/SKILL.md b/rh-automation/skills/job-failure-analyzer/SKILL.md index 40f9104d..f9a771fb 100644 --- a/rh-automation/skills/job-failure-analyzer/SKILL.md +++ b/rh-automation/skills/job-failure-analyzer/SKILL.md @@ -12,6 +12,7 @@ description: | model: inherit color: yellow license: Apache-2.0 +allowed-tools: jobs_retrieve jobs_job_events_list jobs_job_host_summaries_list jobs_stdout_retrieve --- # Job Failure Analyzer diff --git a/rh-automation/skills/resolution-advisor/SKILL.md b/rh-automation/skills/resolution-advisor/SKILL.md index b151801f..6d2ba580 100644 --- a/rh-automation/skills/resolution-advisor/SKILL.md +++ b/rh-automation/skills/resolution-advisor/SKILL.md @@ -13,6 +13,7 @@ description: | model: inherit color: green license: Apache-2.0 +allowed-tools: --- # Resolution Advisor diff --git a/rh-basic/skills/red-hat-cve-explainer/SKILL.md b/rh-basic/skills/red-hat-cve-explainer/SKILL.md index ac2e4cf3..41253346 100644 --- a/rh-basic/skills/red-hat-cve-explainer/SKILL.md +++ b/rh-basic/skills/red-hat-cve-explainer/SKILL.md @@ -5,6 +5,7 @@ license: Apache-2.0 user_invocable: true model: inherit color: cyan +allowed-tools: cve-detail map-cve-advisories get-advisory-solution summarize-advisory --- # Red Hat CVE Explainer diff --git a/rh-basic/skills/red-hat-diagnostics/SKILL.md b/rh-basic/skills/red-hat-diagnostics/SKILL.md index 90a0df0c..1b62aeaa 100644 --- a/rh-basic/skills/red-hat-diagnostics/SKILL.md +++ b/rh-basic/skills/red-hat-diagnostics/SKILL.md @@ -5,6 +5,7 @@ license: Apache-2.0 user_invocable: true model: inherit color: cyan +allowed-tools: --- # Red Hat Diagnostic Information Gathering diff --git a/rh-basic/skills/red-hat-get-started/SKILL.md b/rh-basic/skills/red-hat-get-started/SKILL.md index 11ce61f5..830093b8 100644 --- a/rh-basic/skills/red-hat-get-started/SKILL.md +++ b/rh-basic/skills/red-hat-get-started/SKILL.md @@ -5,6 +5,7 @@ license: Apache-2.0 user_invocable: true model: inherit color: yellow +allowed-tools: --- # Red Hat Skills Installer diff --git a/rh-basic/skills/red-hat-product-lifecycle/SKILL.md b/rh-basic/skills/red-hat-product-lifecycle/SKILL.md index 672cf7d1..dc927be6 100644 --- a/rh-basic/skills/red-hat-product-lifecycle/SKILL.md +++ b/rh-basic/skills/red-hat-product-lifecycle/SKILL.md @@ -5,6 +5,7 @@ license: Apache-2.0 user_invocable: true model: inherit color: cyan +allowed-tools: --- # Red Hat Product Lifecycle Advisor diff --git a/rh-basic/skills/red-hat-security-mcp-setup/SKILL.md b/rh-basic/skills/red-hat-security-mcp-setup/SKILL.md index c45fc5a5..b38e18cc 100644 --- a/rh-basic/skills/red-hat-security-mcp-setup/SKILL.md +++ b/rh-basic/skills/red-hat-security-mcp-setup/SKILL.md @@ -5,6 +5,7 @@ license: Apache-2.0 user_invocable: true model: inherit color: blue +allowed-tools: --- # Red Hat Security MCP Setup diff --git a/rh-basic/skills/red-hat-support-severity/SKILL.md b/rh-basic/skills/red-hat-support-severity/SKILL.md index 85bb55cb..b2a17f96 100644 --- a/rh-basic/skills/red-hat-support-severity/SKILL.md +++ b/rh-basic/skills/red-hat-support-severity/SKILL.md @@ -5,6 +5,7 @@ license: Apache-2.0 user_invocable: true model: inherit color: cyan +allowed-tools: cve-detail --- # Red Hat Support Ticket Severity Helper diff --git a/rh-developer/skills/containerize-deploy/SKILL.md b/rh-developer/skills/containerize-deploy/SKILL.md index df3c4a8d..901c653f 100644 --- a/rh-developer/skills/containerize-deploy/SKILL.md +++ b/rh-developer/skills/containerize-deploy/SKILL.md @@ -5,6 +5,7 @@ description: | model: inherit color: green license: Apache-2.0 +allowed-tools: resources_list resources_get resources_create_or_update pods_list pods_list_in_namespace pods_log events_list get_file_contents helm_list helm_install inventory__find_host_by_name vulnerability__get_system_cves planning__get_rhel_lifecycle metadata: user_invocable: "true" --- diff --git a/rh-developer/skills/debug-build/SKILL.md b/rh-developer/skills/debug-build/SKILL.md index 5af70052..52d5eb76 100644 --- a/rh-developer/skills/debug-build/SKILL.md +++ b/rh-developer/skills/debug-build/SKILL.md @@ -5,6 +5,7 @@ description: | model: inherit color: cyan license: Apache-2.0 +allowed-tools: resources_list resources_get pods_log metadata: user_invocable: "true" --- diff --git a/rh-developer/skills/debug-container/SKILL.md b/rh-developer/skills/debug-container/SKILL.md index ba9d960e..853e4ae1 100644 --- a/rh-developer/skills/debug-container/SKILL.md +++ b/rh-developer/skills/debug-container/SKILL.md @@ -5,6 +5,7 @@ description: | model: inherit color: cyan license: Apache-2.0 +allowed-tools: container_list container_inspect container_logs image_list metadata: user_invocable: "true" --- diff --git a/rh-developer/skills/debug-network/SKILL.md b/rh-developer/skills/debug-network/SKILL.md index 7a5ec991..dc506df7 100644 --- a/rh-developer/skills/debug-network/SKILL.md +++ b/rh-developer/skills/debug-network/SKILL.md @@ -5,6 +5,7 @@ description: | model: inherit color: cyan license: Apache-2.0 +allowed-tools: resources_list resources_get pods_list metadata: user_invocable: "true" --- diff --git a/rh-developer/skills/debug-pipeline/SKILL.md b/rh-developer/skills/debug-pipeline/SKILL.md index 693f9d12..678a8f6d 100644 --- a/rh-developer/skills/debug-pipeline/SKILL.md +++ b/rh-developer/skills/debug-pipeline/SKILL.md @@ -5,6 +5,7 @@ description: | model: inherit color: cyan license: Apache-2.0 +allowed-tools: resources_list resources_get pods_log metadata: user_invocable: "true" --- diff --git a/rh-developer/skills/debug-pod/SKILL.md b/rh-developer/skills/debug-pod/SKILL.md index 3a426e38..a215ffeb 100644 --- a/rh-developer/skills/debug-pod/SKILL.md +++ b/rh-developer/skills/debug-pod/SKILL.md @@ -5,6 +5,7 @@ description: | model: inherit color: cyan license: Apache-2.0 +allowed-tools: pods_list resources_get events_list pods_log metadata: user_invocable: "true" --- diff --git a/rh-developer/skills/debug-rhel/SKILL.md b/rh-developer/skills/debug-rhel/SKILL.md index 1b1b65cf..d9505131 100644 --- a/rh-developer/skills/debug-rhel/SKILL.md +++ b/rh-developer/skills/debug-rhel/SKILL.md @@ -5,6 +5,7 @@ description: | model: inherit color: cyan license: Apache-2.0 +allowed-tools: inventory__find_host_by_name vulnerability__get_system_cves advisor__get_active_rules advisor__get_rule_by_text_search metadata: user_invocable: "true" --- diff --git a/rh-developer/skills/deploy/SKILL.md b/rh-developer/skills/deploy/SKILL.md index 27a90642..11d1f2bc 100644 --- a/rh-developer/skills/deploy/SKILL.md +++ b/rh-developer/skills/deploy/SKILL.md @@ -5,6 +5,7 @@ description: | model: inherit color: green license: Apache-2.0 +allowed-tools: resources_create_or_update pods_list pods_log metadata: user_invocable: "true" --- diff --git a/rh-developer/skills/detect-project/SKILL.md b/rh-developer/skills/detect-project/SKILL.md index d1a18f72..a06644c4 100644 --- a/rh-developer/skills/detect-project/SKILL.md +++ b/rh-developer/skills/detect-project/SKILL.md @@ -5,6 +5,7 @@ description: | model: inherit color: cyan license: Apache-2.0 +allowed-tools: get_file_contents metadata: user_invocable: "true" --- diff --git a/rh-developer/skills/helm-deploy/SKILL.md b/rh-developer/skills/helm-deploy/SKILL.md index ed7b5d75..d7a2f43f 100644 --- a/rh-developer/skills/helm-deploy/SKILL.md +++ b/rh-developer/skills/helm-deploy/SKILL.md @@ -5,6 +5,7 @@ description: | model: inherit color: green license: Apache-2.0 +allowed-tools: helm_list helm_install pods_list_in_namespace resources_create_or_update metadata: user_invocable: "true" --- diff --git a/rh-developer/skills/recommend-image/SKILL.md b/rh-developer/skills/recommend-image/SKILL.md index e0d0cf1c..6e2cb42c 100644 --- a/rh-developer/skills/recommend-image/SKILL.md +++ b/rh-developer/skills/recommend-image/SKILL.md @@ -5,6 +5,7 @@ description: | model: inherit color: cyan license: Apache-2.0 +allowed-tools: metadata: user_invocable: "true" --- diff --git a/rh-developer/skills/rhel-deploy/SKILL.md b/rh-developer/skills/rhel-deploy/SKILL.md index c0b9592a..dcb57dac 100644 --- a/rh-developer/skills/rhel-deploy/SKILL.md +++ b/rh-developer/skills/rhel-deploy/SKILL.md @@ -5,6 +5,7 @@ description: | model: inherit color: yellow license: Apache-2.0 +allowed-tools: inventory__find_host_by_name vulnerability__get_system_cves planning__get_rhel_lifecycle metadata: user_invocable: "true" --- diff --git a/rh-developer/skills/s2i-build/SKILL.md b/rh-developer/skills/s2i-build/SKILL.md index f32e8711..90f11d3b 100644 --- a/rh-developer/skills/s2i-build/SKILL.md +++ b/rh-developer/skills/s2i-build/SKILL.md @@ -5,6 +5,7 @@ description: | model: inherit color: green license: Apache-2.0 +allowed-tools: resources_list resources_create_or_update pods_log metadata: user_invocable: "true" --- diff --git a/rh-developer/skills/validate-environment/SKILL.md b/rh-developer/skills/validate-environment/SKILL.md index d635628d..2b5ae8a5 100644 --- a/rh-developer/skills/validate-environment/SKILL.md +++ b/rh-developer/skills/validate-environment/SKILL.md @@ -5,6 +5,7 @@ description: | model: inherit color: cyan license: Apache-2.0 +allowed-tools: metadata: user_invocable: "true" --- diff --git a/rh-sre/skills/cve-impact/SKILL.md b/rh-sre/skills/cve-impact/SKILL.md index 40a68c32..896af539 100644 --- a/rh-sre/skills/cve-impact/SKILL.md +++ b/rh-sre/skills/cve-impact/SKILL.md @@ -11,6 +11,7 @@ description: | model: inherit color: blue license: Apache-2.0 +allowed-tools: vulnerability__get_cves vulnerability__get_cve vulnerability__get_cve_systems vulnerability__get_system_cves inventory__find_host_by_name inventory__get_host_details --- # CVE Impact Analysis Skill diff --git a/rh-sre/skills/cve-validation/SKILL.md b/rh-sre/skills/cve-validation/SKILL.md index 03388664..13b0b113 100644 --- a/rh-sre/skills/cve-validation/SKILL.md +++ b/rh-sre/skills/cve-validation/SKILL.md @@ -20,6 +20,7 @@ description: | model: inherit color: blue license: Apache-2.0 +allowed-tools: vulnerability__get_cve --- # CVE Validation Skill diff --git a/rh-sre/skills/execution-summary/SKILL.md b/rh-sre/skills/execution-summary/SKILL.md index b3811263..53a8edaf 100644 --- a/rh-sre/skills/execution-summary/SKILL.md +++ b/rh-sre/skills/execution-summary/SKILL.md @@ -12,6 +12,7 @@ description: | model: haiku color: blue license: Apache-2.0 +allowed-tools: --- # Execution Summary Skill diff --git a/rh-sre/skills/fleet-inventory/SKILL.md b/rh-sre/skills/fleet-inventory/SKILL.md index 25d75e14..17ed2a79 100644 --- a/rh-sre/skills/fleet-inventory/SKILL.md +++ b/rh-sre/skills/fleet-inventory/SKILL.md @@ -19,6 +19,7 @@ description: | model: inherit color: blue license: Apache-2.0 +allowed-tools: inventory__get_host_details vulnerability__get_cve_systems --- # Fleet Inventory Skill diff --git a/rh-sre/skills/job-template-creator/SKILL.md b/rh-sre/skills/job-template-creator/SKILL.md index fc88c23a..6a72a99c 100644 --- a/rh-sre/skills/job-template-creator/SKILL.md +++ b/rh-sre/skills/job-template-creator/SKILL.md @@ -11,6 +11,7 @@ description: | model: inherit color: blue license: Apache-2.0 +allowed-tools: job_templates_list job_templates_retrieve projects_list inventories_list job_templates_launch_retrieve --- # AAP Job Template Creator Skill diff --git a/rh-sre/skills/job-template-remediation-validator/SKILL.md b/rh-sre/skills/job-template-remediation-validator/SKILL.md index 50fa2912..cfea4a36 100644 --- a/rh-sre/skills/job-template-remediation-validator/SKILL.md +++ b/rh-sre/skills/job-template-remediation-validator/SKILL.md @@ -13,6 +13,7 @@ description: | model: inherit color: blue license: Apache-2.0 +allowed-tools: job_templates_list job_templates_retrieve projects_list inventories_list --- # AAP Job Template Remediation Validator diff --git a/rh-sre/skills/mcp-aap-validator/SKILL.md b/rh-sre/skills/mcp-aap-validator/SKILL.md index e25e98eb..b97247fa 100644 --- a/rh-sre/skills/mcp-aap-validator/SKILL.md +++ b/rh-sre/skills/mcp-aap-validator/SKILL.md @@ -5,6 +5,7 @@ description: | model: haiku color: yellow license: Apache-2.0 +allowed-tools: job_templates_list inventories_list --- # MCP AAP Validator diff --git a/rh-sre/skills/mcp-lightspeed-validator/SKILL.md b/rh-sre/skills/mcp-lightspeed-validator/SKILL.md index 6621cc7e..a0b44717 100644 --- a/rh-sre/skills/mcp-lightspeed-validator/SKILL.md +++ b/rh-sre/skills/mcp-lightspeed-validator/SKILL.md @@ -5,6 +5,7 @@ description: | model: haiku color: yellow license: Apache-2.0 +allowed-tools: vulnerability__get_cves --- # MCP Lightspeed Validator diff --git a/rh-sre/skills/playbook-executor/SKILL.md b/rh-sre/skills/playbook-executor/SKILL.md index 319bce1b..d751547c 100644 --- a/rh-sre/skills/playbook-executor/SKILL.md +++ b/rh-sre/skills/playbook-executor/SKILL.md @@ -15,6 +15,7 @@ description: | model: inherit color: red license: Apache-2.0 +allowed-tools: job_templates_list job_templates_retrieve projects_list job_templates_launch_retrieve jobs_retrieve jobs_stdout_retrieve jobs_job_events_list jobs_job_host_summaries_list jobs_relaunch_retrieve inventories_list hosts_list --- # AAP Playbook Executor Skill diff --git a/rh-sre/skills/playbook-generator/SKILL.md b/rh-sre/skills/playbook-generator/SKILL.md index 041e1036..4352531a 100644 --- a/rh-sre/skills/playbook-generator/SKILL.md +++ b/rh-sre/skills/playbook-generator/SKILL.md @@ -19,6 +19,7 @@ description: | model: inherit color: yellow license: Apache-2.0 +allowed-tools: remediations__create_vuln_playbook --- # Ansible Playbook Generator Skill diff --git a/rh-sre/skills/remediation-verifier/SKILL.md b/rh-sre/skills/remediation-verifier/SKILL.md index 8f184964..351deb06 100644 --- a/rh-sre/skills/remediation-verifier/SKILL.md +++ b/rh-sre/skills/remediation-verifier/SKILL.md @@ -17,6 +17,7 @@ description: | model: inherit color: blue license: Apache-2.0 +allowed-tools: vulnerability__get_cve vulnerability__get_cve_systems inventory__get_host_details --- # Remediation Verification Skill diff --git a/rh-sre/skills/remediation/SKILL.md b/rh-sre/skills/remediation/SKILL.md index 206e48c9..d467081f 100644 --- a/rh-sre/skills/remediation/SKILL.md +++ b/rh-sre/skills/remediation/SKILL.md @@ -23,6 +23,7 @@ metadata: author: "Red Hat Ecosystem Engineering" priority: "high" license: Apache-2.0 +allowed-tools: vulnerability__get_cves vulnerability__get_cve vulnerability__get_cve_systems vulnerability__get_system_cves inventory__find_host_by_name inventory__get_host_details remediations__create_vuln_playbook job_templates_list job_templates_retrieve projects_list job_templates_launch_retrieve jobs_retrieve jobs_stdout_retrieve jobs_job_events_list jobs_job_host_summaries_list jobs_relaunch_retrieve inventories_list hosts_list --- # Remediation Skill diff --git a/rh-sre/skills/system-context/SKILL.md b/rh-sre/skills/system-context/SKILL.md index 44cda074..dd273255 100644 --- a/rh-sre/skills/system-context/SKILL.md +++ b/rh-sre/skills/system-context/SKILL.md @@ -17,6 +17,7 @@ description: | model: inherit color: blue license: Apache-2.0 +allowed-tools: vulnerability__get_cve_systems inventory__get_host_details --- # System Context Gathering Skill diff --git a/rh-virt/skills/vm-clone/SKILL.md b/rh-virt/skills/vm-clone/SKILL.md index f19f3719..1584f8f0 100644 --- a/rh-virt/skills/vm-clone/SKILL.md +++ b/rh-virt/skills/vm-clone/SKILL.md @@ -16,6 +16,7 @@ description: | license: Apache-2.0 model: inherit color: blue +allowed-tools: resources_get resources_create_or_update resources_list --- # /vm-clone Skill diff --git a/rh-virt/skills/vm-create/SKILL.md b/rh-virt/skills/vm-create/SKILL.md index 8afe1430..01445014 100644 --- a/rh-virt/skills/vm-create/SKILL.md +++ b/rh-virt/skills/vm-create/SKILL.md @@ -16,6 +16,7 @@ description: | license: Apache-2.0 model: inherit color: green +allowed-tools: vm_create resources_get resources_list namespaces_list events_list vm_lifecycle resources_create_or_update --- # /vm-create Skill diff --git a/rh-virt/skills/vm-delete/SKILL.md b/rh-virt/skills/vm-delete/SKILL.md index e6d2d5f2..e30dc658 100644 --- a/rh-virt/skills/vm-delete/SKILL.md +++ b/rh-virt/skills/vm-delete/SKILL.md @@ -16,6 +16,7 @@ description: | license: Apache-2.0 model: inherit color: red +allowed-tools: resources_get resources_delete resources_list resources_create_or_update vm_lifecycle pods_list_in_namespace --- # /vm-delete Skill diff --git a/rh-virt/skills/vm-inventory/SKILL.md b/rh-virt/skills/vm-inventory/SKILL.md index 8a292540..2d746656 100644 --- a/rh-virt/skills/vm-inventory/SKILL.md +++ b/rh-virt/skills/vm-inventory/SKILL.md @@ -16,6 +16,7 @@ description: | license: Apache-2.0 model: inherit color: cyan +allowed-tools: resources_list resources_get --- # /vm-inventory Skill diff --git a/rh-virt/skills/vm-lifecycle-manager/SKILL.md b/rh-virt/skills/vm-lifecycle-manager/SKILL.md index b7f8ed71..0f86b4f6 100644 --- a/rh-virt/skills/vm-lifecycle-manager/SKILL.md +++ b/rh-virt/skills/vm-lifecycle-manager/SKILL.md @@ -16,6 +16,7 @@ description: | license: Apache-2.0 model: inherit color: blue +allowed-tools: vm_lifecycle resources_get --- # /vm-lifecycle-manager Skill diff --git a/rh-virt/skills/vm-rebalance/SKILL.md b/rh-virt/skills/vm-rebalance/SKILL.md index af1e04ee..f972e5e7 100644 --- a/rh-virt/skills/vm-rebalance/SKILL.md +++ b/rh-virt/skills/vm-rebalance/SKILL.md @@ -16,6 +16,7 @@ description: | license: Apache-2.0 model: inherit color: yellow +allowed-tools: resources_list resources_get resources_create_or_update vm_lifecycle nodes_top pods_top nodes_stats_summary --- # /vm-rebalance Skill diff --git a/rh-virt/skills/vm-snapshot-create/SKILL.md b/rh-virt/skills/vm-snapshot-create/SKILL.md index 855d3d43..3935e145 100644 --- a/rh-virt/skills/vm-snapshot-create/SKILL.md +++ b/rh-virt/skills/vm-snapshot-create/SKILL.md @@ -15,7 +15,7 @@ description: | license: Apache-2.0 model: inherit color: green -allowed-tools: mcp__openshift-virtualization__resources_create_or_update mcp__openshift-virtualization__resources_get mcp__openshift-virtualization__resources_list +allowed-tools: resources_create_or_update resources_get resources_list vm_lifecycle --- # /vm-snapshot-create Skill diff --git a/rh-virt/skills/vm-snapshot-delete/SKILL.md b/rh-virt/skills/vm-snapshot-delete/SKILL.md index f7c292ad..d598e71f 100644 --- a/rh-virt/skills/vm-snapshot-delete/SKILL.md +++ b/rh-virt/skills/vm-snapshot-delete/SKILL.md @@ -15,6 +15,7 @@ description: | license: Apache-2.0 model: inherit color: yellow +allowed-tools: resources_get resources_list resources_delete --- # /vm-snapshot-delete Skill diff --git a/rh-virt/skills/vm-snapshot-list/SKILL.md b/rh-virt/skills/vm-snapshot-list/SKILL.md index e8f52b6b..1ace800d 100644 --- a/rh-virt/skills/vm-snapshot-list/SKILL.md +++ b/rh-virt/skills/vm-snapshot-list/SKILL.md @@ -15,6 +15,7 @@ description: | license: Apache-2.0 model: inherit color: cyan +allowed-tools: resources_list resources_get --- # /vm-snapshot-list Skill diff --git a/rh-virt/skills/vm-snapshot-restore/SKILL.md b/rh-virt/skills/vm-snapshot-restore/SKILL.md index cd3f395f..b5929b22 100644 --- a/rh-virt/skills/vm-snapshot-restore/SKILL.md +++ b/rh-virt/skills/vm-snapshot-restore/SKILL.md @@ -15,6 +15,7 @@ description: | license: Apache-2.0 model: inherit color: red +allowed-tools: resources_create_or_update resources_get vm_lifecycle --- # /vm-snapshot-restore Skill diff --git a/scripts/validate_mcp_tools.py b/scripts/validate_mcp_tools.py index c1339d14..3b2875ca 100644 --- a/scripts/validate_mcp_tools.py +++ b/scripts/validate_mcp_tools.py @@ -67,9 +67,12 @@ class ValidationResult: total_skills: int = 0 passed: int = 0 skipped: int = 0 + warned: int = 0 failed: int = 0 findings: list[Finding] = field(default_factory=list) + warnings: list[Finding] = field(default_factory=list) skipped_servers: list[str] = field(default_factory=list) + has_skipped_servers: bool = False @property def success(self) -> bool: @@ -356,8 +359,12 @@ def validate_pack(pack: str, repo_root: Path, kubeconfig: str) -> ValidationResu continue print(f" {len(tools)} tools available") + for t in sorted(tools): + print(f" - {t}") all_available_tools.update(tools) + result.has_skipped_servers = len(result.skipped_servers) > 0 + if all_available_tools: print(f" Combined tool pool: {len(all_available_tools)} unique tools") @@ -386,6 +393,23 @@ def validate_pack(pack: str, repo_root: Path, kubeconfig: str) -> ValidationResu if not missing: result.passed += 1 print(f" PASS {pack}/{skill_name}: all {len(declared_tools)} tools validated") + elif result.has_skipped_servers: + verified = [t for t in declared_tools if t in all_available_tools] + result.warned += 1 + rel_path = str(skill_file.relative_to(repo_root)) + for tool in missing: + suggestion = suggest_tool(tool, all_available_tools) + finding = Finding( + skill=skill_name, + pack=pack, + tool=tool, + file_path=rel_path, + line_number=line_number, + suggestion=suggestion, + ) + result.warnings.append(finding) + print(f" WARN {pack}/{skill_name}: {len(verified)}/{len(declared_tools)} tools verified, " + f"{len(missing)} unverifiable (MCP server not started)") else: result.failed += 1 rel_path = str(skill_file.relative_to(repo_root)) @@ -463,8 +487,10 @@ def main(packs: list[str] | None = None) -> int: combined.total_skills += result.total_skills combined.passed += result.passed combined.skipped += result.skipped + combined.warned += result.warned combined.failed += result.failed combined.findings.extend(result.findings) + combined.warnings.extend(result.warnings) combined.skipped_servers.extend(result.skipped_servers) print() @@ -474,26 +500,35 @@ def main(packs: list[str] | None = None) -> int: print("-" * 66) print(f" Total skills: {combined.total_skills}") print(f" Passed: {combined.passed}") + print(f" Warned (unverifiable): {combined.warned}") print(f" Skipped (no allowed-tools): {combined.skipped}") print(f" Failed: {combined.failed}") print() if combined.skipped_servers: - print("Skipped MCP servers:") + print("Skipped MCP servers (non-container or unreachable):") for s in combined.skipped_servers: print(f" - {s}") print() + if combined.warnings: + print("WARNINGS (tools from skipped MCP servers, cannot verify):") + for w in combined.warnings: + print(f" - {w}") + print() + if combined.findings: - print("FINDINGS:") + print("FAILURES (tools missing from started MCP servers):") for f in combined.findings: print(f" - {f}") print() if not combined.success: - print("VALIDATION FAILED - tool name mismatches detected") + print("VALIDATION FAILED - tool name mismatches detected in started servers") print("Check mcps.json for the correct tool names.") return 1 + elif combined.warned > 0: + print("PASSED WITH WARNINGS - some tools could not be verified (MCP servers not started)") elif combined.skipped > 0: print("PASSED WITH WARNINGS - some skills have no allowed-tools") else: