Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 62 additions & 0 deletions .github/actions/collect-fuzz-stats/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
name: Collect fuzz stats
description: Dump coverage summaries (project-wide + per-harness) and corpus file counts into stats-<variant>/ for the report job.

inputs:
project:
description: Project name.
required: true
variant:
description: Variant label — "baseline" or "current".
required: true
sha:
description: Commit SHA measured in this variant.
required: true
has_project:
description: String "true" if the project was present and built for this variant; otherwise only meta.json is written.
required: true
out_base:
description: Directory containing the oss-fuzz out tree (e.g. build/out for oss-fuzz template, /tmp/oss-fuzz/build/out for upstream).
required: true

runs:
using: composite
steps:
- shell: bash
env:
PROJECT: ${{ inputs.project }}
VARIANT: ${{ inputs.variant }}
SHA: ${{ inputs.sha }}
HAS_PROJECT: ${{ inputs.has_project }}
OUT_BASE: ${{ inputs.out_base }}
run: |
OUT="stats-$VARIANT"
mkdir -p "$OUT/harness"

jq -n \
--arg variant "$VARIANT" \
--arg sha "${SHA:-}" \
--arg project "$PROJECT" \
--arg has_project "${HAS_PROJECT:-false}" \
'{variant: $variant, sha: $sha, project: $project, has_project: ($has_project == "true")}' \
> "$OUT/meta.json"

if [ "$HAS_PROJECT" != "true" ]; then
echo "No project at this variant; stats collection skipped"
exit 0
fi

PROJ_SUM=$(find "$OUT_BASE/$PROJECT/report/linux" -maxdepth 1 -name summary.json 2>/dev/null | head -1)
if [ -n "$PROJ_SUM" ]; then
cp "$PROJ_SUM" "$OUT/project.summary.json"
fi

if [ -d "$OUT_BASE/$PROJECT/report_target" ]; then
for d in "$OUT_BASE/$PROJECT/report_target"/*/linux; do
[ -d "$d" ] || continue
HNAME=$(basename "$(dirname "$d")")
[ -f "$d/summary.json" ] && cp "$d/summary.json" "$OUT/harness/$HNAME.summary.json"
done
fi

echo "---- $OUT ----"
find "$OUT" -type f -printf '%p (%s bytes)\n'
33 changes: 0 additions & 33 deletions .github/actions/invariance_tests/action.yml

This file was deleted.

65 changes: 65 additions & 0 deletions .github/actions/post-fuzz-report/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
name: Post fuzz coverage report
description: Download baseline/current stats artifacts, render markdown, post or update a sticky PR comment.

inputs:
footer:
description: Footer flavor — "generic" for oss-fuzz, "upstream" for upstream source PRs.
required: false
default: generic
fuzz_seconds:
description: Total fuzz budget used (for display only).
required: true
github_token:
description: Token with pull-requests:write to post the comment.
required: true

runs:
using: composite
steps:
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
with:
name: stats-baseline
path: stats/baseline
continue-on-error: true

- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
with:
name: stats-current
path: stats/current
continue-on-error: true

- name: Render comment
shell: bash
env:
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
FUZZ_SECONDS: ${{ inputs.fuzz_seconds }}
FOOTER: ${{ inputs.footer }}
STATS_ROOT: stats
run: |
python3 "$GITHUB_ACTION_PATH/render_comment.py" > comment.md
echo "---- comment.md ----"
cat comment.md

- name: Post or update PR comment
shell: bash
env:
GH_TOKEN: ${{ inputs.github_token }}
REPO: ${{ github.repository }}
BRANCH: ${{ github.ref_name }}
run: |
PR=$(gh pr list --repo "$REPO" --head "$BRANCH" --state open --json number --jq '.[0].number // empty')
if [ -z "$PR" ]; then
echo "No open PR for branch $BRANCH — skipping comment"
exit 0
fi
MARKER='<!-- fuzz-coverage-report -->'
EXISTING=$(gh api "repos/$REPO/issues/$PR/comments" --paginate \
--jq ".[] | select(.body | contains(\"$MARKER\")) | .id" | head -1)
jq -Rs '{body: .}' < comment.md > payload.json
if [ -n "$EXISTING" ]; then
gh api --method PATCH "repos/$REPO/issues/comments/$EXISTING" --input payload.json > /dev/null
echo "Updated existing comment $EXISTING on PR #$PR"
else
gh api --method POST "repos/$REPO/issues/$PR/comments" --input payload.json > /dev/null
echo "Created new comment on PR #$PR"
fi
188 changes: 188 additions & 0 deletions .github/actions/post-fuzz-report/render_comment.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,188 @@
# Copyright 2026 fuzz-for-me contributors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Render before/after coverage comparison as a sticky PR comment body.

Reads artifacts downloaded by the calling workflow:
stats/baseline/meta.json, project.summary.json, corpus.json, harness/*.summary.json
stats/current/ ... (same layout)

Writes the markdown body to stdout.
"""

import datetime
import json
import os
import pathlib
import sys

MARKER = "<!-- fuzz-coverage-report -->"

FOOTER_GENERIC = (
"<sub>Per-harness data from `report_target/&lt;fuzzer&gt;/linux/summary.json`. "
"Full HTML reports in the workflow artifacts.</sub>")
FOOTER_UPSTREAM = ("<sub>Same harness config applied to both sides "
"(baseline = base source + PR harness). " +
FOOTER_GENERIC[5:])


def _load_json(path: pathlib.Path):
if not path.exists():
return None
try:
return json.loads(path.read_text())
except json.JSONDecodeError:
return None


def _load_variant(base: pathlib.Path) -> dict:
harness: dict = {}
hd = base / "harness"
if hd.is_dir():
for f in sorted(hd.glob("*.summary.json")):
data = _load_json(f)
if data is not None:
harness[f.name.removesuffix(".summary.json")] = data
return {
"meta": _load_json(base / "meta.json"),
"project": _load_json(base / "project.summary.json"),
"harness": harness,
}


def _totals(summary):
if not summary:
return None
t = summary["data"][0]["totals"]
return {
"lines":
(t["lines"]["covered"], t["lines"]["count"], t["lines"]["percent"]),
"branches": (
t["branches"]["covered"],
t["branches"]["count"],
t["branches"]["percent"],
),
"functions": (
t["functions"]["covered"],
t["functions"]["count"],
t["functions"]["percent"],
),
}


def _fmt_cov(tot, key):
if not tot:
return "—"
cov, n, pct = tot[key]
return f"{pct:.1f}% ({cov}/{n})"


def _fmt_delta(b, a, key):
if not b and not a:
return "—"
if not b:
return "**new**"
if not a:
return "**removed**"
d = a[key][2] - b[key][2]
sign = "+" if d >= 0 else ""
return f"**{sign}{d:.1f} pp**"


def render(
stats_root: pathlib.Path,
run_url: str,
fuzz_seconds: str,
now_utc: str,
footer: str,
) -> str:
b = _load_variant(stats_root / "baseline")
c = _load_variant(stats_root / "current")

b_meta = b["meta"] or {}
c_meta = c["meta"] or {}
b_sha_full = b_meta.get("sha") or ""
c_sha_full = c_meta.get("sha") or ""
b_sha = b_sha_full[:7] if b_sha_full else "unknown"
c_sha = c_sha_full[:7] if c_sha_full else "unknown"
project = c_meta.get("project") or b_meta.get("project") or "?"
b_has = bool(b_meta.get("has_project"))
c_has = bool(c_meta.get("has_project"))

out = [MARKER, "", "## Fuzzing Coverage Report", ""]

tested = f"**Tested:** project `{project}` · base `{b_sha}`"
if not b_has:
tested += (
" _(no baseline — project not present at base or baseline build failed)_"
)
tested += f" → head `{c_sha}`"
if not c_has:
tested += " _(current measurement failed)_"
tested += (f" · {fuzz_seconds}s total fuzz budget"
f" · updated {now_utc}"
f" · [workflow run]({run_url})")
out += [tested, ""]

bt = _totals(b["project"])
ct = _totals(c["project"])
if bt or ct:
out += [
"| Metric | Before | After | Delta |",
"|---|---|---|---|",
f"| Line coverage | {_fmt_cov(bt, 'lines')} | {_fmt_cov(ct, 'lines')} | {_fmt_delta(bt, ct, 'lines')} |",
f"| Branch coverage | {_fmt_cov(bt, 'branches')} | {_fmt_cov(ct, 'branches')} | {_fmt_delta(bt, ct, 'branches')} |",
f"| Function coverage | {_fmt_cov(bt, 'functions')} | {_fmt_cov(ct, 'functions')} | {_fmt_delta(bt, ct, 'functions')} |",
"",
]

all_h = sorted(set(b["harness"].keys()) | set(c["harness"].keys()))
if all_h:
out += [
"### Per-harness",
"",
"| Harness | Lines before | Lines after | Δ |",
"|---|---|---|---|",
]
for h in all_h:
bh = _totals(b["harness"].get(h))
ch = _totals(c["harness"].get(h))
out.append(
f"| `{h}` | {_fmt_cov(bh, 'lines')} | {_fmt_cov(ch, 'lines')} | "
f"{_fmt_delta(bh, ch, 'lines')} |")
out.append("")

if not (bt or ct or all_h):
out += [
"_No coverage data collected. Check the workflow run for build errors._",
"",
]

out.append(footer)
return "\n".join(out)


def main():
stats_root = pathlib.Path(os.environ.get("STATS_ROOT", "stats"))
run_url = os.environ["RUN_URL"]
fuzz_seconds = os.environ.get("FUZZ_SECONDS", "300")
footer_kind = os.environ.get("FOOTER", "generic")
now_utc = datetime.datetime.now(
datetime.timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
footer = FOOTER_UPSTREAM if footer_kind == "upstream" else FOOTER_GENERIC
sys.stdout.write(render(stats_root, run_url, fuzz_seconds, now_utc, footer))
sys.stdout.write("\n")


if __name__ == "__main__":
main()
37 changes: 0 additions & 37 deletions .github/actions/regression_tests/action.yml

This file was deleted.

Loading
Loading