From 7fad8bea727efd2b2d89f8847feb5a5f1c6c9c1a Mon Sep 17 00:00:00 2001 From: Jonathan Haas Date: Wed, 20 May 2026 21:35:30 -0700 Subject: [PATCH] ci: lock CodeQL out of org defaults --- .github/contracts/org-control-plane.yml | 35 +++++++++++++++++++ .../verify-org-control-plane-contract.rb | 35 +++++++++++++++++++ SECURITY.md | 5 +-- profile/GITHUB_ACTIONS_QUOTA.md | 4 ++- .../verify_org_control_plane_contract_test.rb | 14 ++++++++ 5 files changed, 90 insertions(+), 3 deletions(-) diff --git a/.github/contracts/org-control-plane.yml b/.github/contracts/org-control-plane.yml index 57b95d9..aceb429 100644 --- a/.github/contracts/org-control-plane.yml +++ b/.github/contracts/org-control-plane.yml @@ -71,6 +71,38 @@ requirements: checked_by: - .github/scripts/validate-services-catalog.rb - test/validate_services_catalog_test.rb + - id: no-codeql-org-defaults + title: GitHub CodeQL and default code scanning stay disabled + source: + path: SECURITY.md + lines: 26-44 + evidence_fields: + - source_id + - decision_id + - output_id + checked_by: + - .github/scripts/verify-org-control-plane-contract.rb + - .github/workflows/codex-rails-check.yml +github_security_configuration: + id: 245233 + name: EvalOps Blacksmith recommended + default_for_new_repos: all + required_settings: + advanced_security: secret_protection + code_scanning_default_setup: disabled + dependency_graph: enabled + dependency_graph_autosubmit_action: disabled + dependabot_alerts: enabled + secret_scanning: enabled + secret_scanning_push_protection: enabled + forbidden_workflows: + checked_in_path_globs: + - .github/workflows/*codeql* + - .github/workflow-templates/*codeql* + generated_paths: + - dynamic/github-code-scanning/codeql + actions: + - github/codeql-action provenance: stable_id_pattern: "evalops.github..." source_records: @@ -83,6 +115,9 @@ provenance: - id: evalops.github.org-defaults.source.service-catalog path: services.yaml digest: sha256 + - id: evalops.github.org-defaults.source.security-policy + path: SECURITY.md + digest: sha256 derived_decisions: - id: evalops.github.org-defaults.decision.codex-rails path: .github/workflows/codex-rails-check.yml diff --git a/.github/scripts/verify-org-control-plane-contract.rb b/.github/scripts/verify-org-control-plane-contract.rb index 118a827..955ca38 100644 --- a/.github/scripts/verify-org-control-plane-contract.rb +++ b/.github/scripts/verify-org-control-plane-contract.rb @@ -107,6 +107,39 @@ def check_slo_gates(contract, errors) end end + def check_github_security_configuration(contract, errors) + config = contract["github_security_configuration"] || {} + errors << "github_security_configuration is required" if config.empty? + + errors << "github_security_configuration.id must be 245233" unless config["id"] == 245_233 + errors << "github_security_configuration.default_for_new_repos must be all" unless config["default_for_new_repos"] == "all" + + required = config["required_settings"] || {} + { + "advanced_security" => "secret_protection", + "code_scanning_default_setup" => "disabled", + "dependency_graph" => "enabled", + "dependency_graph_autosubmit_action" => "disabled", + "dependabot_alerts" => "enabled", + "secret_scanning" => "enabled", + "secret_scanning_push_protection" => "enabled" + }.each do |key, expected| + errors << "github_security_configuration.required_settings.#{key} must be #{expected}" unless required[key] == expected + end + + forbidden = config["forbidden_workflows"] || {} + actions = Array(forbidden["actions"]) + generated_paths = Array(forbidden["generated_paths"]) + checked_in_globs = Array(forbidden["checked_in_path_globs"]) + errors << "github_security_configuration.forbidden_workflows.actions must include github/codeql-action" unless actions.include?("github/codeql-action") + unless generated_paths.include?("dynamic/github-code-scanning/codeql") + errors << "github_security_configuration.forbidden_workflows.generated_paths must include dynamic/github-code-scanning/codeql" + end + unless checked_in_globs.any? { |glob| glob.include?("codeql") } + errors << "github_security_configuration.forbidden_workflows.checked_in_path_globs must include a codeql glob" + end + end + def check_golden_workflows(contract, root, errors, warnings) workflows = Array(contract["golden_workflows"]) errors << "at least one golden_workflow is required" if workflows.empty? @@ -164,6 +197,7 @@ def verify(contract, root: Dir.pwd, generated_at: Time.now.utc) check_requirements(contract, root, errors, warnings) check_provenance(contract, root, errors, warnings) check_slo_gates(contract, errors) + check_github_security_configuration(contract, errors) check_golden_workflows(contract, root, errors, warnings) check_adversarial_fixtures(contract, root, errors, warnings) @@ -180,6 +214,7 @@ def verify(contract, root: Dir.pwd, generated_at: Time.now.utc) "derived_decisions" => Array(contract.dig("provenance", "derived_decisions")).length, "emitted_outputs" => Array(contract.dig("provenance", "emitted_outputs")).length, "slo_gates" => Array(contract["slo_gates"]).length, + "github_security_configuration" => contract["github_security_configuration"] ? 1 : 0, "golden_workflows" => Array(contract["golden_workflows"]).length, "adversarial_fixtures" => Array(contract["adversarial_fixtures"]).length }, diff --git a/SECURITY.md b/SECURITY.md index 842cd66..ae9276d 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -27,8 +27,9 @@ This policy applies to all repositories in the [evalops](https://github.com/eval EvalOps does not use GitHub CodeQL or GitHub default code scanning. Every repository is attached to the **EvalOps Blacksmith recommended** code security -configuration (`id=245233`), which sets `code_scanning_default_setup: -disabled` and is the default for new repositories. +configuration (`id=245233`), which sets `advanced_security: +secret_protection` and `code_scanning_default_setup: disabled`, and is the +default for new repositories. Security signal should come from bounded, owned checks: diff --git a/profile/GITHUB_ACTIONS_QUOTA.md b/profile/GITHUB_ACTIONS_QUOTA.md index d0f3452..f2a3244 100644 --- a/profile/GITHUB_ACTIONS_QUOTA.md +++ b/profile/GITHUB_ACTIONS_QUOTA.md @@ -7,7 +7,9 @@ Actions minute or artifact quotas block unrelated pull requests. Do not add CodeQL or GitHub default code-scanning workflows. EvalOps does not use them, and they should not become required checks, scheduled jobs, or -generated default-setup runs. +generated default-setup runs. The org security configuration should keep +GitHub Advanced Security limited to secret protection and keep default code +scanning disabled. Security checks need an owner and a runtime budget before they belong in CI. Prefer narrow repository-owned checks that answer a concrete question: diff --git a/test/verify_org_control_plane_contract_test.rb b/test/verify_org_control_plane_contract_test.rb index aaa0768..1220d3b 100644 --- a/test/verify_org_control_plane_contract_test.rb +++ b/test/verify_org_control_plane_contract_test.rb @@ -18,6 +18,7 @@ def test_repo_contract_passes_and_emits_evidence assert_equal "pass", report.fetch("status") assert_equal "evalops.github.org-defaults", report.fetch("contract_id") assert_operator report.dig("metrics", "requirements_checked"), :>=, 4 + assert_equal 1, report.dig("metrics", "github_security_configuration") assert_operator report.dig("metrics", "adversarial_fixtures"), :>=, 3 assert report.fetch("evidence").all? { |item| item.fetch("sha256").match?(/\A[0-9a-f]{64}\z/) } @@ -52,6 +53,19 @@ def test_adversarial_fixture_must_fail_closed_or_degrade_safely end end + def test_codeql_org_defaults_must_stay_disabled + contract = EvalOpsOrgControlPlaneContract.load_contract(".github/contracts/org-control-plane.yml") + contract["github_security_configuration"]["required_settings"]["code_scanning_default_setup"] = "enabled" + + report = EvalOpsOrgControlPlaneContract.verify(contract, root: Dir.pwd) + + assert_equal "fail", report.fetch("status") + assert_includes( + report.fetch("errors"), + "github_security_configuration.required_settings.code_scanning_default_setup must be disabled" + ) + end + private def write_minimal_repo(root)