diff --git a/CHANGELOG.md b/CHANGELOG.md index b90c4f6..bfe6e01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,20 @@ Format follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). This pr --- +## [v2.6.15] — 2026-05-26 + +### Fixed +- Worker goroutines no longer crash with a raw Go stack trace when `--exec` is given a bad path or encounters an I/O error; the error is now printed cleanly to stderr and the mutant is recorded as errored. +- Diff output is now printed under the report mutex, eliminating interleaved diff/PASS/FAIL lines when running with multiple workers. +- `--dry-run` output now notes that the count is an upper bound (identical mutations across files are deduplicated in a real run); the `--dry-run` help text says the same. +- README blacklist section corrected: checksums are only printed with `--debug`, not during a normal run. +- README `--exec` example path no longer contains a spurious `/v2/` segment that caused a not-found panic. +- Dead struct fields `TimeOutCount`, `MutationCodeCoverage`, and `Timeouted` (which were never populated) removed from `Stats` and `Report`; the README no longer documents them as meaningful. +- `--logger-agentic-json` now writes `msi` in the same 0–1 ratio scale as `--logger-summary-json`, eliminating the prior inconsistency where agentic JSON used 0–100. +- Skipped mutants (mutations that did not compile) are now tracked in `Report.Skipped` and included in the per-mutator breakdown table, making the breakdown consistent with the headline MSI. + +--- + ## [v2.6.14] — 2026-05-26 ### Fixed @@ -311,3 +325,4 @@ Format follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). This pr [v2.6.12]: https://github.com/jonbaldie/go-mutesting/compare/v2.6.11...v2.6.12 [v2.6.13]: https://github.com/jonbaldie/go-mutesting/compare/v2.6.12...v2.6.13 [v2.6.14]: https://github.com/jonbaldie/go-mutesting/compare/v2.6.13...v2.6.14 +[v2.6.15]: https://github.com/jonbaldie/go-mutesting/compare/v2.6.14...v2.6.15 diff --git a/README.md b/README.md index 152fe46..d3cdf08 100644 --- a/README.md +++ b/README.md @@ -128,7 +128,7 @@ Every mutation has to be tested using an [exec command](#write-mutation-exec-com Alternatively the `--exec` argument can be used to invoke an external exec command. The [/scripts/exec](/scripts/exec) directory holds basic exec commands for Go projects. The [test-mutated-package.sh](/scripts/exec/test-mutated-package.sh) script implements all steps and almost all features of the built-in exec command. It can be for example used to test the [github.com/jonbaldie/go-mutesting/v2/example](/example) package. ```bash -go-mutesting --exec "$GOPATH/src/github.com/jonbaldie/go-mutesting/v2/scripts/exec/test-mutated-package.sh" github.com/jonbaldie/go-mutesting/v2/example +go-mutesting --exec "$GOPATH/src/github.com/jonbaldie/go-mutesting/scripts/exec/test-mutated-package.sh" github.com/jonbaldie/go-mutesting/v2/example ``` The execution will print the following output. @@ -177,7 +177,7 @@ Mutation testing can produce false positives when the mutated code path is never Use `--blacklist` with a file that lists the MD5 checksum of each mutation to ignore (one per line). Checksums are derived from only the lines that actually changed, not the whole file, so they stay valid when unrelated code in the same file is edited. -To get the checksum for a mutation, run go-mutesting normally and copy the hex string printed next to the mutation. For example, if a mutation's checksum is `a1b2c3d4...`, create a file: +To get the checksum for a mutation, run go-mutesting with `--debug` and copy the hex string printed next to the mutation. For example, if a mutation's checksum is `a1b2c3d4...`, create a file: ``` a1b2c3d4e5f6... @@ -359,9 +359,7 @@ Writes `go-mutesting-summary.json` after each run. Useful for badges, dashboards "errorCount": 0, "skippedCount": 2, "notCoveredCount": 0, - "timeOutCount": 0, "msi": 0.8333, - "mutationCodeCoverage": 0, "coveredCodeMsi": 0.9211 } ``` @@ -374,9 +372,7 @@ Writes `go-mutesting-summary.json` after each run. Useful for badges, dashboards | `errorCount` | int | Mutations that caused a build or test error | | `skippedCount` | int | Mutations skipped (blacklisted or annotated) | | `notCoveredCount` | int | Mutations on lines with no coverage (requires `--coverage`) | -| `timeOutCount` | int | Mutations that timed out during testing | | `msi` | float | Mutation Score Indicator: killed / total, range 0–1 | -| `mutationCodeCoverage` | int | Lines covered by the coverage profile | | `coveredCodeMsi` | float | MSI restricted to covered lines, range 0–1 | #### `--logger-agentic-json` @@ -386,7 +382,7 @@ Writes `go-mutesting-agentic.json` — a richer payload designed for LLM consump ```json { "generated_at": "2026-05-19T08:13:38Z", - "msi": 58.57, + "msi": 0.5857, "escaped_count": 5, "reminder": "A mutant is an example of how this code could be wrong...", "mutants": [ diff --git a/cmd/go-mutesting/main.go b/cmd/go-mutesting/main.go index 7237958..984839d 100644 --- a/cmd/go-mutesting/main.go +++ b/cmd/go-mutesting/main.go @@ -203,7 +203,7 @@ func mainCmd(args []string) int { tmpDir, err := os.MkdirTemp("", "go-mutesting-") if err != nil { - panic(err) + return exitError("Cannot create temp directory: %v", err) } console.Verbose(opts, "Save mutations into %q", tmpDir) @@ -622,13 +622,13 @@ func processMutationFile( } if err = os.MkdirAll(tmpDir+"/"+filepath.Dir(file), 0755); err != nil { - panic(err) + return 0, exitError("Cannot create mutation directory: %v", err) } tmpFile := tmpDir + "/" + file originalFile := fmt.Sprintf("%s.original", tmpFile) if err = osutil.CopyFile(file, originalFile); err != nil { - panic(err) + return 0, exitError("Cannot copy original file: %v", err) } console.Debug(opts, "Save original into %q", originalFile) @@ -663,9 +663,10 @@ func shutdownAndCleanup(opts *models.Options, jobs chan execJob, jobWg *sync.Wai } if !opts.General.DoNotRemoveTmpFolder { if err := os.RemoveAll(tmpDir); err != nil { - panic(err) + fmt.Fprintf(os.Stderr, "go-mutesting: cannot remove %s: %v\n", tmpDir, err) + } else { + console.Debug(opts, "Remove %q", tmpDir) } - console.Debug(opts, "Remove %q", tmpDir) } } @@ -683,6 +684,8 @@ func printDryRunReport(total int, totals map[string]int) { } } fmt.Printf("\nTotal: %d mutation(s) would be generated. No files written, no tests run.\n", total) + fmt.Println("Note: this count is an upper bound. Identical mutations across files are deduplicated during an actual run.") + } // handleBaselineUpdate writes the current escaped mutants to the baseline file. @@ -816,9 +819,6 @@ func checkQualityGates(opts *models.Options, report *models.Report, bl *baseline return returnOk } - msiPct := report.Stats.Msi * 100 - covMsiPct := report.Stats.CoveredCodeMsi * 100 - // CLI flag is -1 when not provided; config file defaults to 0 when not set. // CLI always wins when explicitly set (>= 0); fall back to config otherwise. minMsi := opts.Score.MinMsi @@ -843,6 +843,8 @@ func checkQualityGates(opts *models.Options, report *models.Report, bl *baseline failed = true } } + msiPct := report.Stats.Msi * 100 + covMsiPct := report.Stats.CoveredCodeMsi * 100 if minMsi >= 0 && msiPct < minMsi { fmt.Fprintf(os.Stderr, "MSI %.2f%% is below minimum required %.2f%%\n", msiPct, minMsi) failed = true @@ -1082,6 +1084,9 @@ func recordMutantResult(opts *models.Options, stats *models.Report, mutant model if statusVisible(opts, 'k') { console.PrintPass(out) } + if opts.General.Debug && !opts.General.NoDiffs && mutant.Diff != "" { + console.PrintDiff([]byte(mutant.Diff)) + } mutant.ProcessOutput = out stats.Killed = append(stats.Killed, mutant) stats.Stats.KilledCount++ @@ -1090,6 +1095,9 @@ func recordMutantResult(opts *models.Options, stats *models.Report, mutant model if statusVisible(opts, 'e') { console.PrintFail(out) } + if !opts.General.NoDiffs && statusVisible(opts, 'e') && mutant.Diff != "" { + console.PrintDiff([]byte(mutant.Diff)) + } mutant.ProcessOutput = out stats.Escaped = append(stats.Escaped, mutant) stats.Stats.EscapedCount++ @@ -1098,12 +1106,22 @@ func recordMutantResult(opts *models.Options, stats *models.Report, mutant model if statusVisible(opts, 's') { console.PrintSkip(out) } + if opts.General.Verbose { + fmt.Println("Mutation did not compile") + } + if opts.General.Debug && !opts.General.NoDiffs && mutant.Diff != "" { + console.PrintDiff([]byte(mutant.Diff)) + } mutant.ProcessOutput = out + stats.Skipped = append(stats.Skipped, mutant) stats.Stats.SkippedCount++ default: out := fmt.Sprintf("UNKNOWN exit code for %s\n", msg) if statusVisible(opts, 'x') { console.PrintUnknown(out) + if !opts.General.NoDiffs && mutant.Diff != "" { + console.PrintDiff([]byte(mutant.Diff)) + } } mutant.ProcessOutput = out stats.Errored = append(stats.Errored, mutant) @@ -1151,11 +1169,12 @@ func runBuiltinExec( } else if e, ok := err.(*exec.ExitError); ok { diffExitCode = e.Sys().(syscall.WaitStatus).ExitStatus() } else { - panic(err) + fmt.Fprintf(os.Stderr, "go-mutesting: diff error: %v\n", err) + return 3 } if diffExitCode != 0 && diffExitCode != 1 { - fmt.Printf("%s\n", diff) - panic("Could not execute diff on mutation file") + fmt.Fprintf(os.Stderr, "go-mutesting: diff exited with code %d\n", diffExitCode) + return 3 } absOrig, _ := filepath.Abs(file) @@ -1166,12 +1185,14 @@ func runBuiltinExec( overlayFile, err := os.CreateTemp("", "go-mutesting-overlay-*.json") if err != nil { - panic(err) + fmt.Fprintf(os.Stderr, "go-mutesting: cannot create overlay file: %v\n", err) + return 3 } if _, err := overlayFile.Write(overlayData); err != nil { overlayFile.Close() os.Remove(overlayFile.Name()) - panic(err) + fmt.Fprintf(os.Stderr, "go-mutesting: cannot write overlay file: %v\n", err) + return 3 } overlayFile.Close() defer os.Remove(overlayFile.Name()) @@ -1205,7 +1226,8 @@ func runBuiltinExec( } else if e, ok := err.(*exec.ExitError); ok { execExitCode = e.Sys().(syscall.WaitStatus).ExitStatus() } else { - panic(err) + fmt.Fprintf(os.Stderr, "go-mutesting: go test error: %v\n", err) + return 3 } if opts.General.Debug { @@ -1213,37 +1235,12 @@ func runBuiltinExec( } mutant.Diff = string(diff) - return interpretBuiltinExitCode(opts, execExitCode, diff) -} - -// interpretBuiltinExitCode maps the go test exit code to a mutation result code -// and prints diff output when appropriate. -func interpretBuiltinExitCode(opts *models.Options, execExitCode int, diff []byte) int { - switch execExitCode { - case 0: // Tests passed → mutation escaped - if !opts.General.NoDiffs && statusVisible(opts, 'e') { - console.PrintDiff(diff) - } + // Map go test exit codes: 0 (tests passed) means the mutation escaped; 1 (tests failed) means killed. + if execExitCode == 0 { return 1 - case 1: // Tests failed → mutation killed - if opts.General.Debug && !opts.General.NoDiffs { - console.PrintDiff(diff) - } + } + if execExitCode == 1 { return 0 - case 2: // Did not compile → skip - if opts.General.Verbose { - fmt.Println("Mutation did not compile") - } - if opts.General.Debug && !opts.General.NoDiffs { - console.PrintDiff(diff) - } - default: - if statusVisible(opts, 'x') { - fmt.Println("Unknown exit code") - if !opts.General.NoDiffs { - console.PrintDiff(diff) - } - } } return execExitCode } @@ -1272,7 +1269,8 @@ func runCustomExec(opts *models.Options, pkg *types.Package, file string, mutati } if err := execCommand.Start(); err != nil { - panic(err) + fmt.Fprintf(os.Stderr, "go-mutesting: cannot start %q: %v\n", execs[0], err) + return 3 } err := execCommand.Wait() @@ -1282,7 +1280,8 @@ func runCustomExec(opts *models.Options, pkg *types.Package, file string, mutati } else if e, ok := err.(*exec.ExitError); ok { execExitCode = e.Sys().(syscall.WaitStatus).ExitStatus() } else { - panic(err) + fmt.Fprintf(os.Stderr, "go-mutesting: exec wait error: %v\n", err) + return 3 } return execExitCode } diff --git a/cmd/go-mutesting/main_test.go b/cmd/go-mutesting/main_test.go index 77c172c..79094ef 100644 --- a/cmd/go-mutesting/main_test.go +++ b/cmd/go-mutesting/main_test.go @@ -157,7 +157,6 @@ func TestMainJSONReport(t *testing.T) { // Collection lengths must match the stat fields. assert.Equal(t, int(s.EscapedCount), len(mutationReport.Escaped)) assert.Equal(t, int(s.KilledCount), len(mutationReport.Killed)) - assert.Nil(t, mutationReport.Timeouted) assert.Nil(t, mutationReport.Errored) for i := 0; i < len(mutationReport.Escaped); i++ { diff --git a/internal/models/options.go b/internal/models/options.go index eb62bb7..c0af317 100644 --- a/internal/models/options.go +++ b/internal/models/options.go @@ -7,7 +7,7 @@ type Options struct { DoNotRemoveTmpFolder bool `long:"do-not-remove-tmp-folder" description:"Do not remove the tmp folder where all mutations are saved to"` Help bool `long:"help" description:"Show this help message"` Noop bool `long:"noop" description:"Run the test suite once without any mutations first; exit with an error if it fails"` - DryRun bool `long:"dry-run" description:"Count mutations per file and mutator without generating files or running tests; prints a summary table and exits 0"` + DryRun bool `long:"dry-run" description:"Count mutations per file and mutator without generating files or running tests; prints a summary table and exits 0. The count is an upper bound — identical mutations across files are deduplicated in a real run."` NoDiffs bool `long:"no-diffs" description:"Suppress diff output for all mutation results (useful in CI where diffs are noisy and the JSON report is consumed instead)"` OutputStatuses string `long:"output-statuses" description:"Show only these result statuses in the terminal: k=killed e=escaped s=skipped n=not-covered x=errored (e.g. --output-statuses=ke). Does not affect JSON reports. Overrides --quiet when set."` Quiet bool `long:"quiet" description:"Only print escaped mutants and the summary (suppress killed/skipped output). Combine with --no-diffs to also suppress escaped-mutant diffs."` diff --git a/internal/models/report.go b/internal/models/report.go index a2d370e..5b85094 100644 --- a/internal/models/report.go +++ b/internal/models/report.go @@ -20,8 +20,8 @@ type Report struct { Stats Stats `json:"stats"` MutatorStats []MutatorStats `json:"mutatorStats,omitempty"` Escaped []Mutant `json:"escaped"` - Timeouted []Mutant `json:"timeouted"` Killed []Mutant `json:"killed"` + Skipped []Mutant `json:"skipped,omitempty"` Errored []Mutant `json:"errored"` NotCovered []Mutant `json:"notCovered,omitempty"` // HasCoverage is true when a coverage profile was loaded before mutation. @@ -37,11 +37,9 @@ type Stats struct { NotCoveredCount int64 `json:"notCoveredCount"` EscapedCount int64 `json:"escapedCount"` ErrorCount int64 `json:"errorCount"` - SkippedCount int64 `json:"skippedCount"` - TimeOutCount int64 `json:"timeOutCount"` - Msi float64 `json:"msi"` - MutationCodeCoverage int64 `json:"mutationCodeCoverage"` - CoveredCodeMsi float64 `json:"coveredCodeMsi"` + SkippedCount int64 `json:"skippedCount"` + Msi float64 `json:"msi"` + CoveredCodeMsi float64 `json:"coveredCodeMsi"` DuplicatedCount int64 `json:"-"` } @@ -128,6 +126,7 @@ func (report *Report) computeMutatorStats() []MutatorStats { } add(report.Killed, func(s *MutatorStats) { s.Killed++ }) add(report.Escaped, func(s *MutatorStats) { s.Escaped++ }) + add(report.Skipped, func(s *MutatorStats) { s.Skipped++ }) add(report.Errored, func(s *MutatorStats) { s.Killed++ }) // errors count as kills result := make([]MutatorStats, 0, len(counts)) diff --git a/internal/reportmaker/maker.go b/internal/reportmaker/maker.go index a3dbdcb..7a1cfc2 100644 --- a/internal/reportmaker/maker.go +++ b/internal/reportmaker/maker.go @@ -96,7 +96,7 @@ var funcMap = template.FuncMap{ // MakeHTMLReport is a function for creating an HTML report based on a stripped-down version of the models.Report model (not all fields are used) func MakeHTMLReport(report models.Report) error { - // MSI in percent + // Convert 0–1 ratio to percentage for the HTML template. report.Stats.Msi = math.Round(report.Stats.Msi*10_000) / 100 groupedMutants := groupEscapedMutants(report.Escaped) @@ -241,7 +241,6 @@ func generateInstanceDescription(mutatorName, diff string) string { // data designed for LLM consumption: stable IDs, context lines, test file paths, // mutator descriptions, and heuristic test-writing hints. func MakeAgenticJSONReport(report models.Report, moduleRoot string) error { - msi := math.Round(report.Stats.Msi*10_000) / 100 mutants := make([]AgenticMutant, 0, len(report.Escaped)) for _, m := range report.Escaped { relFile := toRelPath(m.Mutator.OriginalFilePath, moduleRoot) @@ -264,7 +263,7 @@ func MakeAgenticJSONReport(report models.Report, moduleRoot string) error { doc := agenticReport{ GeneratedAt: time.Now().UTC().Format(time.RFC3339), - Msi: msi, + Msi: report.Stats.Msi, EscapedCount: len(report.Escaped), Reminder: agenticReminder, Mutants: mutants,