Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions .github/workflows/hypatia-scan.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,11 @@ permissions:
contents: read
# security-events: write serves two purposes (write implies read):
# 1. read — lets the built-in GITHUB_TOKEN query this repo's own
# Dependabot alerts via the Hypatia DependabotAlerts rule
# (DA001-DA004). Without read, `scan_from_path` gets HTTP 403
# and the rule silently returns no findings.
# Dependabot alerts (DependabotAlerts rule, DA001-DA004),
# secret-scanning alerts (SecretScanningAlerts, SSA001-SSA004),
# and code-scanning alerts (CodeScanningAlerts, CSA001-CSA004).
# Without read, `scan_from_path` gets HTTP 403 and the rule
# silently returns no findings.
# See 007-lang/audits/audit-dependabot-automation-gap-2026-04-17.md.
# 2. write — lets the "Upload SARIF to code scanning" step publish
# Hypatia findings to the Security → Code scanning page so they
Expand Down
62 changes: 61 additions & 1 deletion lib/hypatia/cli.ex
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ defmodule Hypatia.CLI do
Available: root_hygiene,honest_completion,workflow_audit,
cicd_rules,code_safety,migration_rules,scorecard,
green_web,git_state,dependabot_alerts,
secret_scanning_alerts,code_scanning_alerts,
structural_drift
--format <fmt> Output format: json (default), text, github
--severity <lvl> Minimum severity to report: critical, high, medium (default), low, info
Expand All @@ -47,6 +48,8 @@ defmodule Hypatia.CLI do
:green_web,
:git_state,
:dependabot_alerts,
:secret_scanning_alerts,
:code_scanning_alerts,
:structural_drift
]

Expand Down Expand Up @@ -636,6 +639,60 @@ defmodule Hypatia.CLI do
results
end

# Secret Scanning Alerts
results =
if :secret_scanning_alerts in rules do
case Hypatia.Rules.SecretScanningAlerts.scan_from_path(repo_path) do
{:ok, %{findings: findings}} ->
normalized =
Enum.map(findings, fn f ->
%{
rule_module: "secret_scanning_alerts",
severity: to_string(f.severity),
type: f.rule,
file: Map.get(f, :file, ""),
reason: f.reason,
action: to_string(f.action)
}
end)

results ++ normalized

{:error, reason} ->
IO.puts(:stderr, "Warning: Secret-scanning alerts unavailable: #{reason}")
results
end
else
results
end

# Code Scanning Alerts
results =
if :code_scanning_alerts in rules do
case Hypatia.Rules.CodeScanningAlerts.scan_from_path(repo_path) do
{:ok, %{findings: findings}} ->
normalized =
Enum.map(findings, fn f ->
%{
rule_module: "code_scanning_alerts",
severity: to_string(f.severity),
type: f.rule,
file: Map.get(f, :file, ""),
reason: f.reason,
action: to_string(f.action)
}
end)

results ++ normalized

{:error, reason} ->
IO.puts(:stderr, "Warning: Code-scanning alerts unavailable: #{reason}")
results
end
else
results
end

# Structural Drift
results =
if :structural_drift in rules do
Expand Down Expand Up @@ -1042,6 +1099,8 @@ defmodule Hypatia.CLI do
defp format_module_name("green_web"), do: "Green Web Foundation"
defp format_module_name("git_state"), do: "Git State Sync"
defp format_module_name("dependabot_alerts"), do: "Dependabot Alerts"
defp format_module_name("secret_scanning_alerts"), do: "Secret Scanning Alerts"
defp format_module_name("code_scanning_alerts"), do: "Code Scanning Alerts"
defp format_module_name(other), do: other

defp print_usage do
Expand All @@ -1062,7 +1121,8 @@ defmodule Hypatia.CLI do
Available: root_hygiene,honest_completion,
workflow_audit,cicd_rules,code_safety,
migration_rules,scorecard,green_web,
git_state,dependabot_alerts
git_state,dependabot_alerts,
secret_scanning_alerts,code_scanning_alerts
--format, -f <fmt> Output format: json (default), text, github
--severity, -s <lvl> Minimum severity: critical, high, medium (default), low
--path, -p <dir> Path to scan (alternative to positional arg)
Expand Down
185 changes: 185 additions & 0 deletions lib/mix/tasks/hypatia.recipe_health.ex
Original file line number Diff line number Diff line change
@@ -0,0 +1,185 @@
# SPDX-License-Identifier: MPL-2.0
# Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) <j.d.a.jewell@open.ac.uk>

defmodule Mix.Tasks.Hypatia.RecipeHealth do
@moduledoc """
Per-recipe health report driven by `Hypatia.OutcomeTracker.recipe_health/1`.

Surfaces recipes whose re-scan verification rate is low (potential
false-fix candidates) or insufficient (verification was not attempted
often enough to draw conclusions). Output is sorted so the most
actionable rows -- quarantine candidates and degraded recipes -- are
at the top.

Status legend:
healthy -- verification rate >= 0.70
degraded -- verification rate < 0.70 (review)
quarantine_cand -- verification rate < 0.30 (auto-quarantine candidate)
insufficient -- fewer than --min-attempts verifiable outcomes
no_data -- recipe has outcomes but none were verified

Options:
--format text|json (default: text)
--min-attempts N fewer than this and the recipe is "insufficient"
--degraded N.NN threshold below "healthy" (default 0.70)
--quarantine N.NN threshold below "degraded" (default 0.30)
--only-actionable hide healthy + insufficient + no_data rows

## Examples

mix hypatia.recipe_health
mix hypatia.recipe_health --only-actionable
mix hypatia.recipe_health --format json > recipe-health.json
"""

use Mix.Task

@shortdoc "Show per-recipe success + verification health"

@switches [
format: :string,
min_attempts: :integer,
degraded: :float,
quarantine: :float,
only_actionable: :boolean
]

@impl Mix.Task
def run(argv) do
{opts, _, _} = OptionParser.parse(argv, switches: @switches)

format = Keyword.get(opts, :format, "text")
min_attempts = Keyword.get(opts, :min_attempts, 5)
degraded = Keyword.get(opts, :degraded, 0.70)
quarantine = Keyword.get(opts, :quarantine, 0.30)
only_actionable = Keyword.get(opts, :only_actionable, false)

rows =
Hypatia.OutcomeTracker.recipe_health(
min_attempts: min_attempts,
degraded_threshold: degraded,
quarantine_threshold: quarantine
)

rows =
if only_actionable do
Enum.filter(rows, fn r -> r.status in [:degraded, :quarantine_candidate] end)
else
rows
end

case format do
"json" -> emit_json(rows)
_ -> emit_text(rows)
end
end

defp emit_text([]) do
Mix.shell().info("No recipes match the filter (or no outcomes recorded yet).")
end

defp emit_text(rows) do
headers = ["recipe_id", "disp", "succ", "fail", "fp", "verified", "still", "scan_fail", "rate", "status"]
width = column_widths(rows, headers)

Mix.shell().info(format_row(headers, width))
Mix.shell().info(format_row(Enum.map(width, fn w -> String.duplicate("-", w) end), width))

Enum.each(rows, fn r ->
row = [
r.recipe_id,
Integer.to_string(r.dispatches),
Integer.to_string(r.successes),
Integer.to_string(r.failures),
Integer.to_string(r.false_positives),
Integer.to_string(r.verification.verified),
Integer.to_string(r.verification.still_present),
Integer.to_string(r.verification.scan_failed),
format_rate(r.verification.rate),
Atom.to_string(r.status)
]

Mix.shell().info(format_row(row, width))
end)

Mix.shell().info("")

Mix.shell().info(
"#{length(rows)} recipe(s). " <>
"Quarantine threshold #{quarantine_msg(rows)}, " <>
"degraded threshold #{degraded_msg(rows)}."
)
end

defp emit_json(rows) do
payload = %{
"generated_at" => DateTime.utc_now() |> DateTime.to_iso8601(),
"rows" =>
Enum.map(rows, fn r ->
%{
"recipe_id" => r.recipe_id,
"dispatches" => r.dispatches,
"successes" => r.successes,
"failures" => r.failures,
"false_positives" => r.false_positives,
"success_rate" => to_jsonable(r.success_rate),
"verification" => %{
"verified" => r.verification.verified,
"still_present" => r.verification.still_present,
"scan_failed" => r.verification.scan_failed,
"unverified" => r.verification.unverified,
"verifiable" => r.verification.verifiable,
"rate" => to_jsonable(r.verification.rate)
},
"status" => Atom.to_string(r.status)
}
end)
}

IO.puts(Jason.encode!(payload, pretty: true))
end

defp column_widths(rows, headers) do
initial = Enum.map(headers, &String.length/1)

Enum.reduce(rows, initial, fn r, widths ->
lengths = [
String.length(r.recipe_id),
String.length(Integer.to_string(r.dispatches)),
String.length(Integer.to_string(r.successes)),
String.length(Integer.to_string(r.failures)),
String.length(Integer.to_string(r.false_positives)),
String.length(Integer.to_string(r.verification.verified)),
String.length(Integer.to_string(r.verification.still_present)),
String.length(Integer.to_string(r.verification.scan_failed)),
String.length(format_rate(r.verification.rate)),
String.length(Atom.to_string(r.status))
]

Enum.zip_with([widths, lengths], fn [a, b] -> max(a, b) end)
end)
end

defp format_row(cells, widths) do
Enum.zip(cells, widths)
|> Enum.map_join(" ", fn {cell, w} -> String.pad_trailing(cell, w) end)
end

defp format_rate(:no_data), do: "—"
defp format_rate(:insufficient_data), do: "?"
defp format_rate(r) when is_float(r), do: :erlang.float_to_binary(r, decimals: 2)

defp to_jsonable(:no_data), do: nil
defp to_jsonable(:insufficient_data), do: "insufficient_data"
defp to_jsonable(r) when is_float(r), do: r

defp quarantine_msg(rows) do
count = Enum.count(rows, &(&1.status == :quarantine_candidate))
"#{count} recipe(s)"
end

defp degraded_msg(rows) do
count = Enum.count(rows, &(&1.status == :degraded))
"#{count} recipe(s)"
end
end
Loading
Loading