Skip to content
Open
122 changes: 111 additions & 11 deletions crates/forge/src/cmd/test/mod.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use super::{install, test::filter::ProjectPathsAwareFilter, watch::WatchArgs};
use super::{install, watch::WatchArgs};
use crate::{
MultiContractRunner, MultiContractRunnerBuilder,
decode::decode_console_logs,
Expand Down Expand Up @@ -66,7 +66,7 @@ use yansi::Paint;
mod filter;
mod summary;
use crate::{result::TestKind, traces::render_trace_arena_inner};
pub use filter::FilterArgs;
pub use filter::{FilterArgs, ProjectPathsAwareFilter};
use quick_junit::{NonSuccessKind, Report, TestCase, TestCaseStatus, TestSuite};
use summary::{TestSummaryReport, format_invariant_metrics_table};

Expand Down Expand Up @@ -447,6 +447,39 @@ impl TestArgs {
bail!("`fuzz.run` must be greater than 0");
}

// Mutation testing has bespoke orchestration (per-mutant temp
// workspaces, baseline + N mutants, aggregated mutation report). It is
// not compatible with the single-run debug / flame / list / junit
// modes — running them together would either mix incompatible output
// formats, or run the secondary mode against the baseline tests and
// then silently continue into mutation testing. Reject up front with a
// clear error rather than do the wrong thing.
if self.mutate.is_some() {
let mut conflicts = Vec::new();
if self.list {
conflicts.push("--list");
}
if self.debug {
conflicts.push("--debug");
}
if self.flamegraph {
conflicts.push("--flamegraph");
}
if self.flamechart {
conflicts.push("--flamechart");
}
if self.junit {
conflicts.push("--junit");
}
if !conflicts.is_empty() {
bail!(
"`--mutate` cannot be combined with: {}. Re-run without those flags to use \
mutation testing.",
conflicts.join(", ")
);
}
}

// Explicitly enable isolation for gas reports for more correct gas accounting.
if self.gas_report {
evm_opts.isolate = true;
Expand Down Expand Up @@ -644,6 +677,19 @@ impl TestArgs {
eyre::bail!("Cannot run mutation testing with failed tests");
}

// A green baseline that ran *zero* tests is not a useful baseline:
// every compileable mutant would be reported as `Alive` (no test
// failed, so nothing killed it), which produces a wildly
// misleading mutation report. Hard-error so users get an actual
// signal that their filter / path / setup matched nothing.
if outcome.tests().next().is_none() {
eyre::bail!(
"Mutation testing requires at least one matching baseline test; the current \
filter/path selection matched zero tests. Loosen `--match-test` / \
`--match-contract` / `--match-path` or check the project layout."
);
}

// Explicit paths on --mutate cannot be combined with the --mutate-path
// glob filter: clap can't express this directly because --mutate takes
// an optional list of paths.
Expand Down Expand Up @@ -676,9 +722,6 @@ impl TestArgs {
// Detect both up front so users aren't surprised by races or
// corruption of their real dependency tree.
use foundry_config::fs_permissions::FsAccessPermission;
let has_broad_write = config_for_mutation.fs_permissions.permissions.iter().any(|p| {
matches!(p.access, FsAccessPermission::Write | FsAccessPermission::ReadWrite)
});
if config_for_mutation.ffi {
eyre::bail!(
"Mutation testing is unsafe with `ffi = true`: per-mutant workspaces share \
Expand All @@ -687,13 +730,58 @@ impl TestArgs {
Disable ffi in your foundry.toml to run mutation tests."
);
}
if has_broad_write {

// Only refuse write-capable `fs_permissions` whose path can actually
// reach one of the symlinked dependency trees. Scoped writes (e.g.
// `./out`, `./snapshots`) are safe because they target paths that
// never resolve into the shared `lib`/`node_modules`/`dependencies`
// trees.
let root = &config_for_mutation.root;
let mut shared_dep_dirs: Vec<PathBuf> = config_for_mutation.libs.clone();
shared_dep_dirs.push(root.join("node_modules"));
shared_dep_dirs.push(root.join("dependencies"));
let shared_dep_dirs: Vec<PathBuf> =
shared_dep_dirs.into_iter().map(|p| dunce::canonicalize(&p).unwrap_or(p)).collect();

let touches_shared_dep = |perm_path: &Path| -> bool {
let resolved = if perm_path.is_absolute() {
perm_path.to_path_buf()
} else {
root.join(perm_path)
};
let canon = dunce::canonicalize(&resolved).unwrap_or(resolved);
shared_dep_dirs.iter().any(|dep| {
// Either the permission path is inside a shared dep dir
// (write directly into deps), or it is an ancestor of one
// (broad permission like `./` covers them transitively).
canon.starts_with(dep) || dep.starts_with(&canon)
})
};

let unsafe_write_paths: Vec<&Path> = config_for_mutation
.fs_permissions
.permissions
.iter()
.filter(|p| {
matches!(p.access, FsAccessPermission::Write | FsAccessPermission::ReadWrite)
})
.filter(|p| touches_shared_dep(&p.path))
.map(|p| p.path.as_path())
.collect();

if !unsafe_write_paths.is_empty() {
let paths = unsafe_write_paths
.iter()
.map(|p| format!(" - {}", p.display()))
.collect::<Vec<_>>()
.join("\n");
eyre::bail!(
"Mutation testing is unsafe with write-capable `fs_permissions`: per-mutant \
workspaces share symlinked dependency directories, and `vm.writeFile` \
calls can race against or corrupt the real `lib`/`node_modules`/\
`dependencies` trees. Restrict `fs_permissions` to read-only (or scope it \
away from dependency paths) to run mutation tests."
"Mutation testing is unsafe with write-capable `fs_permissions` that can \
reach the symlinked dependency trees (`lib`/`node_modules`/`dependencies`); \
per-mutant workspaces share those trees, so `vm.writeFile` calls would race \
against or corrupt your real dependencies. Restrict the following \
`fs_permissions` entries to read-only or scope them away from dependency \
paths:\n{paths}"
);
}

Expand All @@ -706,6 +794,18 @@ impl TestArgs {
num_workers: self.mutation_jobs.unwrap_or(0),
show_progress: self.show_progress,
json_output,
// Carry the same filter args (--match-test, --match-contract,
// --match-path, positional path shorthand, --rerun, ...) and
// isolation flag the baseline actually used, so every mutant
// exercises the exact same test set under the same execution
// model. We pull from the materialized `filter`, not the raw
// CLI flags on `self`, because the baseline applies extras:
// the positional `forge test <path>` shorthand is folded into
// `path_pattern`, and `--rerun` injects last-run failures
// into `test_pattern`. Using `self.filter.clone()` would lose
// those and let mutant runs silently diverge from baseline.
filter_args: filter.args().clone(),
isolate: evm_opts_for_mutation.isolate,
};

let result = run_mutation_testing(
Expand Down
30 changes: 25 additions & 5 deletions crates/forge/src/mutation/mod.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
use std::{
collections::{HashMap, HashSet},
collections::{BTreeMap, HashSet},
path::PathBuf,
sync::Arc,
};
Expand Down Expand Up @@ -146,16 +146,33 @@ impl MutationsSummary {
if valid_mutants == 0 { 0.0 } else { self.dead.len() as f64 / valid_mutants as f64 * 100.0 }
}

/// Convert to JSON output format
/// Convert to JSON output format.
///
/// Output is sorted deterministically: files in lexicographic order
/// (`BTreeMap` keys), and survived mutants within each file sorted by
/// `(span.lo, span.hi, mutation_text)`. Without this, parallel worker
/// completion order leaks into the JSON and breaks downstream diffing,
/// snapshot tests, and reproducibility.
pub fn to_json_output(&self, duration_secs: f64) -> MutationJsonOutput {
let mut survived_mutants: HashMap<String, Vec<SurvivedMutantJson>> = HashMap::new();
let mut survived_mutants: BTreeMap<String, Vec<SurvivedMutantJson>> = BTreeMap::new();

for mutant in &self.survived {
let file_path = mutant.relative_path();
let entry = survived_mutants.entry(file_path).or_default();
entry.push(SurvivedMutantJson::from_mutant(mutant));
}

for entries in survived_mutants.values_mut() {
entries.sort_by(|a, b| {
(a.line, a.column, &a.original, &a.mutant).cmp(&(
b.line,
b.column,
&b.original,
&b.mutant,
))
});
}

MutationJsonOutput {
summary: MutationSummaryJson {
total: self.total_mutants(),
Expand All @@ -172,11 +189,14 @@ impl MutationsSummary {
}
}

/// JSON output for mutation testing results
/// JSON output for mutation testing results.
///
/// Uses [`BTreeMap`] for `survived_mutants` so file ordering in the emitted
/// JSON is deterministic.
#[derive(Debug, Clone, Serialize)]
pub struct MutationJsonOutput {
pub summary: MutationSummaryJson,
pub survived_mutants: HashMap<String, Vec<SurvivedMutantJson>>,
pub survived_mutants: BTreeMap<String, Vec<SurvivedMutantJson>>,
}

/// Summary section of JSON output
Expand Down
11 changes: 10 additions & 1 deletion crates/forge/src/mutation/mutant.rs
Original file line number Diff line number Diff line change
Expand Up @@ -108,12 +108,20 @@ pub enum OwnedStrKind {

#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum OwnedLiteral {
Str { kind: OwnedStrKind, text: String },
Str {
kind: OwnedStrKind,
text: String,
},
Number(alloy_primitives::U256),
Rational(String),
Address(String),
Bool(bool),
Err(String),
/// Signed-negation of a numeric literal (e.g. `-123`). We cannot represent
/// negative values inside `Number(U256)` (the cast wraps via two's
/// complement and renders as a huge unsigned literal), so we carry the
/// negation textually and render it as `-{val}`.
NegatedNumber(alloy_primitives::U256),
}

impl From<&LitKind<'_>> for OwnedLiteral {
Expand Down Expand Up @@ -142,6 +150,7 @@ impl Display for OwnedLiteral {
match self {
Self::Bool(val) => write!(f, "{val}"),
Self::Number(val) => write!(f, "{val}"),
Self::NegatedNumber(val) => write!(f, "-{val}"),
Self::Rational(s) => write!(f, "{s}"),
Self::Address(s) => write!(f, "{s}"),
Self::Str { kind, text } => match kind {
Expand Down
10 changes: 9 additions & 1 deletion crates/forge/src/mutation/mutators/assignment_mutator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -47,10 +47,14 @@ impl Mutator for AssignmentMutator {
line_number,
column_number,
},
// Negation of a numeric literal must be carried textually:
// applying `-*val` on a `U256` wraps via two's complement
// and would render as a huge unsigned literal (e.g. `1`
// becomes `2^256 - 1`), producing wrong-source mutants.
Mutant {
span: replacement_span,
mutation: MutationType::Assignment(AssignVarTypes::Literal(
OwnedLiteral::Number(-*val),
OwnedLiteral::NegatedNumber(*val),
)),
path: context.path.clone(),
original,
Expand All @@ -63,6 +67,10 @@ impl Mutator for AssignmentMutator {
OwnedLiteral::Rational(_) => Ok(vec![]),
OwnedLiteral::Address(_) => Ok(vec![]),
OwnedLiteral::Err(_) => Ok(vec![]),
// `NegatedNumber` is only ever constructed *as* a mutant; it
// does not appear as an original literal in the source AST,
// so there is nothing to mutate here.
OwnedLiteral::NegatedNumber(_) => Ok(vec![]),
},
AssignVarTypes::Identifier(ref ident) => Ok(vec![
Mutant {
Expand Down
33 changes: 20 additions & 13 deletions crates/forge/src/mutation/mutators/binary_op_mutator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ pub struct BinaryOpMutator;
impl Mutator for BinaryOpMutator {
fn generate_mutants(&self, context: &MutationContext<'_>) -> Result<Vec<Mutant>> {
let expr = context.expr.ok_or_eyre("BinaryOpMutator: no expression")?;
let (bin_op, _op_span, lhs, rhs) = get_bin_op_parts(expr)?;
let (bin_op, _op_span, lhs, rhs, compound_assignment) = get_bin_op_parts(expr)?;
let op = bin_op.kind;

let operations_bools = vec![
Expand Down Expand Up @@ -47,8 +47,11 @@ impl Mutator for BinaryOpMutator {
let rhs_text = extract_span_text(source, rhs.span);
let op_str = op.to_str();

// Build original expression: "lhs op rhs"
let original_expr = format!("{lhs_text} {op_str} {rhs_text}");
let original_expr = if compound_assignment {
format!("{lhs_text} {op_str}= {rhs_text}")
} else {
format!("{lhs_text} {op_str} {rhs_text}")
};

// Use the full expression span for the mutation (not just the operator span)
let expr_span = context.span;
Expand All @@ -62,8 +65,11 @@ impl Mutator for BinaryOpMutator {
.into_iter()
.filter(|&kind| kind != op)
.map(|kind| {
// Build mutated expression: "lhs new_op rhs"
let mutated_expr = format!("{} {} {}", lhs_text, kind.to_str(), rhs_text);
let mutated_expr = if compound_assignment {
format!("{} {}= {}", lhs_text, kind.to_str(), rhs_text)
} else {
format!("{} {} {}", lhs_text, kind.to_str(), rhs_text)
};
Mutant {
span: expr_span,
mutation: MutationType::BinaryOpExpr { new_op: kind, mutated_expr },
Expand All @@ -82,19 +88,20 @@ impl Mutator for BinaryOpMutator {
return false;
}

match ctxt.expr.unwrap().kind {
ExprKind::Binary(_, _, _) => true,
ExprKind::Assign(_, bin_op, _) => bin_op.is_some(),
_ => false,
}
matches!(
ctxt.expr.unwrap().kind,
ExprKind::Binary(_, _, _) | ExprKind::Assign(_, Some(_), _)
)
}
}

/// Extract the binary operator, its span, and LHS/RHS expressions
fn get_bin_op_parts<'a>(expr: &'a Expr<'a>) -> Result<(BinOp, Span, &'a Expr<'a>, &'a Expr<'a>)> {
fn get_bin_op_parts<'a>(
expr: &'a Expr<'a>,
) -> Result<(BinOp, Span, &'a Expr<'a>, &'a Expr<'a>, bool)> {
match &expr.kind {
ExprKind::Assign(lhs, Some(bin_op), rhs) => Ok((*bin_op, bin_op.span, lhs, rhs)),
ExprKind::Binary(lhs, op, rhs) => Ok((*op, op.span, lhs, rhs)),
ExprKind::Assign(lhs, Some(op), rhs) => Ok((*op, op.span, lhs, rhs, true)),
ExprKind::Binary(lhs, op, rhs) => Ok((*op, op.span, lhs, rhs, false)),
_ => eyre::bail!("BinaryOpMutator: unexpected expression kind"),
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,14 @@ use crate::mutation::mutators::{
assignment_mutator::AssignmentMutator, tests::helper::mutator_tests,
};

// Each emitted mutation only carries the *replacement text* for the RHS
// span — not the full statement. So `x = 123` mutates to `0` (zero) and
// `-123` (signed-negation), not `x = 0` / `x = -123`.
mutator_tests!(AssignmentMutator;
assign_lit: "x = y" => Some(vec!["x = 0", "x = -y"]);
assign_number: "x = 123" => Some(vec!["x = 0", "x = -123"]);
assign_true: "x = true" => Some(vec!["x = false"]);
assign_false: "x = false" => Some(vec!["x = true"]);
assign_declare: "uint256 x = 123" => Some(vec!["uint256 x = 0", "uint256 x = -123"]);
assign_lit: "x = y" => Some(vec!["0", "-y"]);
assign_number: "x = 123" => Some(vec!["0", "-123"]);
assign_true: "x = true" => Some(vec!["false"]);
assign_false: "x = false" => Some(vec!["true"]);
assign_declare: "uint256 x = 123" => Some(vec!["0", "-123"]);
non_assign: "a = b + c" => None;
);
Original file line number Diff line number Diff line change
Expand Up @@ -14,4 +14,7 @@ mutator_tests!(BinaryOpMutator;
bit_or: "x | y" => Some(vec!["x + y", "x - y", "x * y", "x / y", "x % y", "x ** y", "x << y", "x >> y", "x >>> y", "x & y", "x ^ y"]);
bit_xor: "x ^ y" => Some(vec!["x + y", "x - y", "x * y", "x / y", "x % y", "x ** y", "x << y", "x >> y", "x >>> y", "x & y", "x | y"]);
non_binary: "a = true" => None;
compound_assign_add: "a += b" => Some(vec!["a >>= b", "a <<= b", "a >>>= b", "a &= b", "a |= b", "a ^= b", "a -= b", "a **= b", "a *= b", "a /= b", "a %= b"]);
compound_assign_sub: "a -= b" => Some(vec!["a >>= b", "a <<= b", "a >>>= b", "a &= b", "a |= b", "a ^= b", "a += b", "a **= b", "a *= b", "a /= b", "a %= b"]);
compound_assign_mul: "a *= b" => Some(vec!["a >>= b", "a <<= b", "a >>>= b", "a &= b", "a |= b", "a ^= b", "a += b", "a -= b", "a **= b", "a /= b", "a %= b"]);
);
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,10 @@ use crate::mutation::mutators::{
delete_expression_mutator::DeleteExpressionMutator, tests::helper::mutator_tests,
};

// `delete x` is replaced by `assert(true)` (a no-op statement) — the test
// expects the mutation's *replacement text*, not the original expression
// stripped of the `delete` keyword.
mutator_tests!(DeleteExpressionMutator;
delete_expr: "delete x" => Some(vec!["x"]);
delete_expr: "delete x" => Some(vec!["assert(true)"]);
non_delete: "a = b + c" => None;
);
Loading
Loading