Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
f2c7302
initial proptest
shahan-khatchadourian-anchorage Mar 11, 2026
c9bb956
Next step using broader constructions
shahan-khatchadourian-anchorage Mar 11, 2026
d3029af
pipeline tests
shahan-khatchadourian-anchorage Mar 12, 2026
d610b92
check ok a bit more
shahan-khatchadourian-anchorage Mar 12, 2026
998fb1c
check good discriminators with good args are ok
shahan-khatchadourian-anchorage Mar 12, 2026
dd65009
fuzz all idls
shahan-khatchadourian-anchorage Mar 12, 2026
7c61fb8
update to anchor solana parser dependency
shahan-khatchadourian-anchorage Mar 13, 2026
0d6395e
add proptest-based fuzz and pipeline integration tests for IDL parsing
shahan-khatchadourian-anchorage Mar 13, 2026
9620125
Add cargo fuzz targets for visualsign-solana
shahan-khatchadourian-anchorage Mar 14, 2026
ed1c414
Add proptest and fuzz label-triggered CI jobs
shahan-khatchadourian-anchorage Mar 14, 2026
633b31b
Add clippy steps to proptest and fuzz CI jobs
shahan-khatchadourian-anchorage Mar 14, 2026
e2b9ee8
Fix formatting and remove redundant clippy steps from CI
shahan-khatchadourian-anchorage Mar 14, 2026
fae97c2
Update Cargo.lock for solana_parser git dependency
shahan-khatchadourian-anchorage Mar 14, 2026
2568ac7
Update Cargo.lock with solana_parser git source
shahan-khatchadourian-anchorage Mar 14, 2026
4888982
Post fuzz crash summary as PR comment on failure
shahan-khatchadourian-anchorage Mar 14, 2026
cd4b7d2
Split proptest and fuzz workflows into separate files
shahan-khatchadourian-anchorage Mar 14, 2026
c767a98
Add reusable post-failure-comment action tagging @copilot
shahan-khatchadourian-anchorage Mar 14, 2026
7210c4b
Initial plan
Copilot Mar 15, 2026
d955ae7
Fix panics in decode_instructions on malformed fuzz inputs
Copilot Mar 15, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 33 additions & 0 deletions .github/actions/post-failure-comment/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
name: Post Failure Comment
description: 'Post a PR comment with failure details and tag @copilot to fix the issue'

inputs:
pr-number:
description: Pull request number
required: true
title:
description: Short title describing what failed
required: true
body:
description: Failure output to include in the comment
required: true
gh-token:
description: GitHub token with pull-requests write permission
required: true

runs:
using: composite
steps:
- name: Post comment
shell: bash
env:
GH_TOKEN: ${{ inputs.gh-token }}
COMMENT_BODY: ${{ inputs.body }}
run: |
gh pr comment ${{ inputs.pr-number }} --body "### ${{ inputs.title }}

\`\`\`
${COMMENT_BODY}
\`\`\`

@copilot please investigate the failure above and fix the issue in this PR."
99 changes: 99 additions & 0 deletions .github/workflows/fuzz.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
name: Fuzz Testing

on:
pull_request:
types: [opened, synchronize, reopened, labeled]

jobs:
fuzz:
if: contains(github.event.pull_request.labels.*.name, 'fuzz')
runs-on: ubuntu-latest-4-cores
permissions:
pull-requests: write
steps:
- name: git checkout
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
token: ${{ secrets.GITHUB_TOKEN }}
- name: Install Rust (nightly)
uses: actions-rust-lang/setup-rust-toolchain@fb51252c7ba57d633bc668f941da052e410add48 # v1.13.0
with:
toolchain: nightly
- name: Install cargo-fuzz
run: cargo install cargo-fuzz
- name: Cache Rust dependencies
uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
with:
path: |
~/.cargo/registry/index/
~/.cargo/registry/cache/
~/.cargo/git/db/
src/target/
key: ${{ runner.os }}-cargo-fuzz-${{ hashFiles('src/Cargo.lock') }}
restore-keys: |
${{ runner.os }}-cargo-fuzz-
${{ runner.os }}-cargo-
- name: install protoc
uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0
with:
version: "21.4"
repo-token: ${{ secrets.GITHUB_TOKEN }}
- name: free disk space
run: |
sudo swapoff -a
sudo rm -f /swapfile
sudo apt clean
df -h
- name: Run codegen
run: make -C src generated
- name: Fuzz fuzz_transaction_string (30s)
id: fuzz_transaction_string
continue-on-error: true
run: |
cargo +nightly fuzz run fuzz_transaction_string -- -max_total_time=30 2>&1 | tee /tmp/fuzz_transaction_string.txt
exit ${PIPESTATUS[0]}
working-directory: src/chain_parsers/visualsign-solana/fuzz
- name: Extract fuzz_transaction_string crash output
id: extract_fuzz_transaction_string
if: steps.fuzz_transaction_string.outcome == 'failure'
run: |
body=$(awk '/^─+$/{found=1} found{print}' /tmp/fuzz_transaction_string.txt)
delim=$(openssl rand -hex 8)
echo "body<<${delim}" >> "$GITHUB_OUTPUT"
echo "${body}" >> "$GITHUB_OUTPUT"
echo "${delim}" >> "$GITHUB_OUTPUT"
- name: Post fuzz_transaction_string crash comment
if: steps.fuzz_transaction_string.outcome == 'failure'
uses: ./.github/actions/post-failure-comment
with:
pr-number: ${{ github.event.pull_request.number }}
title: "Fuzz crash: `fuzz_transaction_string`"
body: ${{ steps.extract_fuzz_transaction_string.outputs.body }}
gh-token: ${{ secrets.GITHUB_TOKEN }}
- name: Fuzz fuzz_versioned_transaction (30s)
id: fuzz_versioned_transaction
continue-on-error: true
run: |
cargo +nightly fuzz run fuzz_versioned_transaction -- -max_total_time=30 2>&1 | tee /tmp/fuzz_versioned_transaction.txt
exit ${PIPESTATUS[0]}
working-directory: src/chain_parsers/visualsign-solana/fuzz
- name: Extract fuzz_versioned_transaction crash output
id: extract_fuzz_versioned_transaction
if: steps.fuzz_versioned_transaction.outcome == 'failure'
run: |
body=$(awk '/^─+$/{found=1} found{print}' /tmp/fuzz_versioned_transaction.txt)
delim=$(openssl rand -hex 8)
echo "body<<${delim}" >> "$GITHUB_OUTPUT"
echo "${body}" >> "$GITHUB_OUTPUT"
echo "${delim}" >> "$GITHUB_OUTPUT"
- name: Post fuzz_versioned_transaction crash comment
if: steps.fuzz_versioned_transaction.outcome == 'failure'
uses: ./.github/actions/post-failure-comment
with:
pr-number: ${{ github.event.pull_request.number }}
title: "Fuzz crash: `fuzz_versioned_transaction`"
body: ${{ steps.extract_fuzz_versioned_transaction.outputs.body }}
gh-token: ${{ secrets.GITHUB_TOKEN }}
- name: Fail if any fuzz target crashed
if: steps.fuzz_transaction_string.outcome == 'failure' || steps.fuzz_versioned_transaction.outcome == 'failure'
run: exit 1
6 changes: 4 additions & 2 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,13 @@ on:
branches:
- main
pull_request:
branches:
- main
types: [opened, synchronize, reopened]

jobs:
ubuntu:
if: |
github.ref == 'refs/heads/main' ||
github.base_ref == 'main'
runs-on: ubuntu-latest-4-cores
steps:
- name: git checkout
Expand Down
73 changes: 73 additions & 0 deletions .github/workflows/proptest.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
name: Property Tests

on:
pull_request:
types: [opened, synchronize, reopened, labeled]

jobs:
proptest:
if: contains(github.event.pull_request.labels.*.name, 'proptest')
runs-on: ubuntu-latest-4-cores
permissions:
pull-requests: write
steps:
- name: git checkout
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
token: ${{ secrets.GITHUB_TOKEN }}
- name: Install Rust
uses: actions-rust-lang/setup-rust-toolchain@fb51252c7ba57d633bc668f941da052e410add48 # v1.13.0
with:
components: clippy, rustfmt
- name: Cache Rust dependencies
uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
with:
path: |
~/.cargo/registry/index/
~/.cargo/registry/cache/
~/.cargo/git/db/
src/target/
key: ${{ runner.os }}-cargo-proptest-${{ hashFiles('src/Cargo.lock') }}
restore-keys: |
${{ runner.os }}-cargo-proptest-
${{ runner.os }}-cargo-
- name: install protoc
uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0
with:
version: "21.4"
repo-token: ${{ secrets.GITHUB_TOKEN }}
- name: free disk space
run: |
sudo swapoff -a
sudo rm -f /swapfile
sudo apt clean
df -h
- name: Run codegen
run: make -C src generated
- name: Run proptest tests
id: proptest
continue-on-error: true
run: |
cargo test -p visualsign-solana 2>&1 | tee /tmp/proptest.txt
exit ${PIPESTATUS[0]}
working-directory: src
- name: Extract proptest failure output
id: extract_proptest
if: steps.proptest.outcome == 'failure'
run: |
body=$(grep -A 50 'FAILED\|proptest\|thread.*panicked' /tmp/proptest.txt | head -100)
delim=$(openssl rand -hex 8)
echo "body<<${delim}" >> "$GITHUB_OUTPUT"
echo "${body}" >> "$GITHUB_OUTPUT"
echo "${delim}" >> "$GITHUB_OUTPUT"
- name: Post proptest failure comment
if: steps.proptest.outcome == 'failure'
uses: ./.github/actions/post-failure-comment
with:
pr-number: ${{ github.event.pull_request.number }}
title: "Property test failure"
body: ${{ steps.extract_proptest.outputs.body }}
gh-token: ${{ secrets.GITHUB_TOKEN }}
- name: Fail if proptest failed
if: steps.proptest.outcome == 'failure'
run: exit 1
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
**/target
out
src/.cargo/
151 changes: 151 additions & 0 deletions scripts/fuzz_all_idls.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,151 @@
#!/usr/bin/env bash
# fuzz_all_idls.sh — run IDL fuzz tests against every embedded Solana IDL.
#
# The embedded IDLs live in the solana_parser git dependency:
#
# ape_pro.json 6 instructions 4 types (Ape Pro)
# cndy.json 7 instructions 4 types (Metaplex Candy Machine)
# collision.json 1 instruction 2 types (test fixture: duplicate type names)
# cyclic.json 1 instruction 2 types (test fixture: cyclic type references)
# drift.json 199 instructions 81 types (Drift Protocol V2)
# jupiter.json 34 instructions 8 types (Jupiter Swap)
# jupiter_agg_v6.json 14 instructions 9 types (Jupiter Aggregator V6)
# jupiter_limit.json 8 instructions 12 types (Jupiter Limit)
# kamino.json 36 instructions 51 types (Kamino)
# lifinity.json 3 instructions 4 types (Lifinity Swap V2)
# meteora.json 64 instructions 38 types (Meteora)
# openbook.json 29 instructions 32 types (Openbook)
# orca.json 49 instructions 11 types (Orca Whirlpool)
# raydium.json 10 instructions 5 types (Raydium)
# stabble.json 17 instructions 8 types (Stabble)
#
# For each IDL the script runs two test functions from fuzz_idl_parsing.rs:
#
# real_idl_never_panics
# — 50/50 valid/random discriminator mix; on Ok asserts correct dispatch.
#
# real_idl_valid_data_always_parses_ok
# — generates borsh-correct bytes for every instruction; asserts is_ok().
#
# Usage:
# ./scripts/fuzz_all_idls.sh
# PROPTEST_CASES=1000 ./scripts/fuzz_all_idls.sh
# ./scripts/fuzz_all_idls.sh /path/to/extra.json ... # append extra IDLs
#
# Requirements: cargo, python3

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
WORKSPACE_TOML="$SCRIPT_DIR/../src/Cargo.toml"
CASES="${PROPTEST_CASES:-256}"

# ── Locate the solana_parser IDL directory via cargo metadata ─────────────────

IDL_DIR="$(python3 - "$WORKSPACE_TOML" <<'PY'
import json, os, subprocess, sys

manifest = sys.argv[1]
result = subprocess.run(
["cargo", "metadata", "--manifest-path", manifest, "--format-version", "1"],
capture_output=True, text=True, check=True,
)
data = json.loads(result.stdout)
for pkg in data["packages"]:
if pkg["name"] == "solana_parser":
idl_dir = os.path.join(os.path.dirname(pkg["manifest_path"]), "src", "solana", "idls")
if os.path.isdir(idl_dir):
print(idl_dir)
sys.exit(0)
print("error: solana_parser IDL directory not found", file=sys.stderr)
sys.exit(1)
PY
)"

# ── Collect IDL files: embedded + any extras passed as arguments ──────────────

IDL_FILES=("$IDL_DIR"/*.json)
for extra in "${@}"; do
IDL_FILES+=("$extra")
done

# ── Build once so the loop doesn't pay compilation cost each iteration ─────────

echo "Building test binary..."
cargo test \
--manifest-path "$WORKSPACE_TOML" \
-p visualsign-solana \
--test fuzz_idl_parsing \
--no-run \
2>&1 | grep -E "^( Compiling| Finished|error)" || true
echo ""

# ── Run tests for each IDL ────────────────────────────────────────────────────

PASS=0
FAIL=0
FAILED_IDLS=()

printf "%-30s %13s %7s %s\n" "IDL" "Instructions" "Types" "Result"
printf "%-30s %13s %7s %s\n" "───────────────────────────" "────────────" "─────" "──────"

for idl_file in "${IDL_FILES[@]}"; do
name="$(basename "$idl_file" .json)"

# Get instruction/type counts
read -r inst_count type_count < <(python3 -c "
import json, sys
try:
d = json.load(open(sys.argv[1]))
print(len(d.get('instructions', [])), len(d.get('types', [])))
except Exception:
print(0, 0)
" "$idl_file")

printf "%-30s %13s %7s " "$name" "$inst_count" "$type_count"

# Run both real_idl_* tests for this IDL.
output=$(IDL_FILE="$idl_file" PROPTEST_CASES="$CASES" \
cargo test \
--manifest-path "$WORKSPACE_TOML" \
-p visualsign-solana \
--test fuzz_idl_parsing \
real_idl \
--quiet \
2>&1)

# Extract "N passed; M failed" directly from cargo's summary line.
summary=$(echo "$output" | grep -oE "[0-9]+ passed; [0-9]+ failed" | head -1)

if [ -z "$summary" ]; then
echo "FAIL (no test result)"
FAIL=$(( FAIL + 1 ))
FAILED_IDLS+=("$name ($idl_file)")
else
failed_count=$(echo "$summary" | grep -oE "^[0-9]+ failed" | grep -oE "^[0-9]+" || \
echo "$summary" | grep -oE "[0-9]+ failed" | grep -oE "[0-9]+")
if [ "${failed_count:-0}" -gt 0 ]; then
echo "FAIL ($summary)"
FAIL=$(( FAIL + 1 ))
FAILED_IDLS+=("$name ($idl_file)")
else
echo "PASS ($summary)"
PASS=$(( PASS + 1 ))
fi
fi
done

echo ""
echo "Results: $PASS passed, $FAIL failed (PROPTEST_CASES=$CASES)"

if (( FAIL > 0 )); then
echo ""
echo "Failed:"
for entry in "${FAILED_IDLS[@]}"; do
echo " $entry"
done
echo ""
echo "Re-run a single IDL with full output:"
echo " IDL_FILE=<path> cargo test --manifest-path src/Cargo.toml -p visualsign-solana --test fuzz_idl_parsing real_idl"
exit 1
fi
Loading