test/goroot: fix bug449 runoutput handling

test/goroot/runner_test.go

@@ -573,7 +573,7 @@ func runCase(t *testing.T, repoRoot, goroot, goCmd, llgoBin string, tc testCase,
 	case "run", "buildrun":
 		return runSingleFileCase(t, repoRoot, goroot, goCmd, llgoBin, tc, opts, buildTimeout)
 	case "runoutput":
-		return runOutputCase(t, repoRoot, goroot, goCmd, llgoBin, tc, opts)
+		return runOutputCase(t, repoRoot, goroot, goCmd, llgoBin, tc, opts, buildTimeout)
 	case "rundir":
 		return runDirCase(t, repoRoot, goroot, goCmd, llgoBin, tc, opts, false, buildTimeout)
 	case "runindir":
@@ -1120,7 +1120,7 @@ func runSingleFileCase(t *testing.T, repoRoot, goroot, goCmd, llgoBin string, tc
 	return compareOutputs(goStdout, goStderr, goExit, llgoStdout, llgoStderr, llgoExit)
 }
 
-func runOutputCase(t *testing.T, repoRoot, goroot, goCmd, llgoBin string, tc testCase, opts directiveOptions) error {
+func runOutputCase(t *testing.T, repoRoot, goroot, goCmd, llgoBin string, tc testCase, opts directiveOptions, buildTimeout time.Duration) error {
 	t.Helper()
 	ws, err := prepareCaseWorkspace(repoRoot)
 	if err != nil {
@@ -1133,37 +1133,39 @@ func runOutputCase(t *testing.T, repoRoot, goroot, goCmd, llgoBin string, tc tes
 	env := runnerEnv(repoRoot, goroot, ws.gopath, opts.ExtraEnv)
 	metrics := caseMetrics{}
 
-	goWS, err := stageRunOutputWorkspace(ws, "go", tc.Dir, sourceFiles)
+	genWS, err := stageRunOutputWorkspace(ws, "go", tc.Dir, sourceFiles)
 	if err != nil {
 		return err
 	}
-	llgoWS, err := stageRunOutputWorkspace(ws, "llgo", tc.Dir, sourceFiles)
+
+	goModVersion, err := toolchainGoModVersion(goroot)
 	if err != nil {
 		return err
 	}
 
-	goModVersion, err := toolchainGoModVersion(goroot)
+	goGen, goGenRun, err := generateRunOutput(genWS, goCmd, env, sourceFiles, programArgs, opts.Timeout, false, "go", goModVersion)
+	metrics.goRun += goGenRun
 	if err != nil {
 		return err
 	}
 
-	goGen, goGenRun, err := generateRunOutput(goWS, goCmd, env, sourceFiles, programArgs, opts.Timeout, false, "go", goModVersion)
-	metrics.goRun += goGenRun
+	goWS, err := stageRunOutputWorkspace(ws, "go-generated", genWS.workDir, []string{goGen})
 	if err != nil {
 		return err
 	}
-	llgoGen, llgoGenRun, err := generateRunOutput(llgoWS, llgoBin, env, sourceFiles, programArgs, opts.Timeout, true, "llgo", goModVersion)
-	metrics.llgoRun += llgoGenRun
+	llgoWS, err := stageRunOutputWorkspace(ws, "llgo-generated", genWS.workDir, []string{goGen})
 	if err != nil {
 		return err
 	}
 
-	goStdout, goStderr, goExit, goRunDur, err := runGeneratedProgram(goWS, goCmd, env, goGen, "go", opts.Timeout)
+	goStdout, goStderr, goExit, goBuildDur, goRunDur, err := runGeneratedProgram(goWS, goCmd, env, goGen, "go", buildTimeout, opts.Timeout)
+	metrics.goBuild += goBuildDur
 	metrics.goRun += goRunDur
 	if err != nil {
 		return err
 	}
-	llgoStdout, llgoStderr, llgoExit, llgoRunDur, err := runGeneratedProgram(llgoWS, llgoBin, env, llgoGen, "llgo", opts.Timeout)
+	llgoStdout, llgoStderr, llgoExit, llgoBuildDur, llgoRunDur, err := runGeneratedProgram(llgoWS, llgoBin, env, goGen, "llgo", buildTimeout, opts.Timeout)
+	metrics.llgoBuild += llgoBuildDur
 	metrics.llgoRun += llgoRunDur
 	if err != nil {
 		return err
@@ -1367,12 +1369,23 @@ func toolchainGoModVersion(goroot string) (string, error) {
 	return parts[0] + "." + parts[1], nil
 }
 
-func runGeneratedProgram(ws caseWorkspace, tool string, env []string, fileName, label string, timeout time.Duration) ([]byte, []byte, int, time.Duration, error) {
-	stdout, stderr, exitCode, runDur, err := runProgram(ws.workDir, tool, env, timeout, "run", fileName)
+func runGeneratedProgram(ws caseWorkspace, tool string, env []string, fileName, label string, buildTimeout, runTimeout time.Duration) ([]byte, []byte, int, time.Duration, time.Duration, error) {
+	out := filepath.Join(ws.rootDir, label+"-generated.out")
+	if runtime.GOOS == "windows" {
+		out += ".exe"
+	}
+	buildStdout, buildStderr, buildExit, buildDur, err := runProgram(ws.workDir, tool, env, buildTimeout, "build", "-o", out, fileName)
+	if err != nil {
+		return nil, nil, 0, buildDur, 0, commandFailure(label+" generated build", buildDur, err, buildStdout, buildStderr, buildExit)
+	}
+	if err := ensureBuiltBinary(out, label+" generated build"); err != nil {
+		return nil, nil, 0, buildDur, 0, err
+	}
+	stdout, stderr, exitCode, runDur, err := runProgram(ws.workDir, out, env, runTimeout)
 	if err != nil {
-		return nil, nil, 0, runDur, commandFailure(label+" generated run", runDur, err, stdout, stderr, exitCode)
+		return nil, nil, 0, buildDur, runDur, commandFailure(label+" generated run", runDur, err, stdout, stderr, exitCode)
 	}
-	return stdout, stderr, exitCode, runDur, nil
+	return stdout, stderr, exitCode, buildDur, runDur, nil
 }
 
 func overlayDir(dstRoot, srcRoot string) error {

test/goroot/runner_unit_test.go

@@ -1,6 +1,7 @@
 package goroot
 
 import (
+	"fmt"
 	"os"
 	"path/filepath"
 	"reflect"
@@ -230,12 +231,30 @@ func TestRunProgramTimeout(t *testing.T) {
 func TestRunGeneratedProgramUsesProvidedTimeout(t *testing.T) {
 	dir := t.TempDir()
 	tool := filepath.Join(dir, "fake-tool.sh")
-	script := "#!/bin/sh\nsleep 0.2\n"
+	script := `#!/bin/sh
+set -eu
+out=""
+prev=""
+for arg in "$@"; do
+	if [ "$prev" = "-o" ]; then
+		out="$arg"
+	fi
+	prev="$arg"
+done
+cat > "$out" <<'EOF'
+#!/bin/sh
+sleep 0.2
+EOF
+chmod +x "$out"
+`
 	if err := os.WriteFile(tool, []byte(script), 0o755); err != nil {
 		t.Fatal(err)
 	}
-	ws := caseWorkspace{workDir: dir}
-	_, _, exitCode, elapsed, err := runGeneratedProgram(ws, tool, os.Environ(), "generated.go", "fake", 50*time.Millisecond)
+	if err := os.WriteFile(filepath.Join(dir, "generated.go"), []byte("package main\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	ws := caseWorkspace{rootDir: dir, workDir: dir}
+	_, _, exitCode, _, elapsed, err := runGeneratedProgram(ws, tool, os.Environ(), "generated.go", "fake", time.Second, 50*time.Millisecond)
 	if err == nil {
 		t.Fatal("expected timeout")
 	}
@@ -419,6 +438,122 @@ func TestEnsureModuleWorkspace(t *testing.T) {
 	}
 }
 
+func TestRunOutputCaseGeneratesWithBaselineGoOnly(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("fake tool scripts use /bin/sh")
+	}
+	dir := t.TempDir()
+	repoRoot := filepath.Join(dir, "repo")
+	if err := os.MkdirAll(repoRoot, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	goroot := filepath.Join(dir, "goroot")
+	if err := os.MkdirAll(goroot, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(goroot, "VERSION"), []byte("go1.24.11\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	srcDir := filepath.Join(dir, "test", "fixedbugs")
+	if err := os.MkdirAll(srcDir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(srcDir, "case.go"), []byte("// runoutput\n\npackage main\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+
+	logPath := filepath.Join(dir, "tools.log")
+	goTool := filepath.Join(dir, "fake-go")
+	llgoTool := filepath.Join(dir, "fake-llgo")
+	writeRunOutputFakeTool(t, goTool, logPath, true)
+	writeRunOutputFakeTool(t, llgoTool, logPath, false)
+
+	tc := testCase{
+		RelPath:   "fixedbugs/case.go",
+		Dir:       srcDir,
+		FileName:  "case.go",
+		Directive: "runoutput",
+	}
+	opts := directiveOptions{Timeout: 5 * time.Second}
+	if err := runOutputCase(t, repoRoot, goroot, goTool, llgoTool, tc, opts, 5*time.Second); err != nil {
+		t.Fatal(err)
+	}
+
+	logData, err := os.ReadFile(logPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	log := string(logData)
+	if !strings.Contains(log, goTool+" run case.go") {
+		t.Fatalf("fake go generator was not run; log:\n%s", log)
+	}
+	if strings.Contains(log, llgoTool+" run") {
+		t.Fatalf("fake llgo should not run the runoutput generator; log:\n%s", log)
+	}
+	if !strings.Contains(log, goTool+" build -o") || !strings.Contains(log, llgoTool+" build -o") {
+		t.Fatalf("generated source was not built by both tools; log:\n%s", log)
+	}
+}
+
+func writeRunOutputFakeTool(t *testing.T, path, logPath string, allowRun bool) {
+	t.Helper()
+	allowRunValue := "false"
+	if allowRun {
+		allowRunValue = "true"
+	}
+	script := fmt.Sprintf(`#!/bin/sh
+set -eu
+printf '%%s\n' "$0 $*" >> %[1]q
+case "$1" in
+run)
+	if [ %[2]q != "true" ]; then
+		echo "unexpected runoutput generator invocation" >&2
+		exit 23
+	fi
+	cat <<'EOF'
+package main
+
+func main() {
+	print("ok\n")
+}
+EOF
+	;;
+build)
+	out=""
+	last=""
+	prev=""
+	for arg in "$@"; do
+		if [ "$prev" = "-o" ]; then
+			out="$arg"
+		fi
+		last="$arg"
+		prev="$arg"
+	done
+	if [ -z "$out" ]; then
+		echo "missing -o" >&2
+		exit 24
+	fi
+	if [ ! -s "$last" ]; then
+		echo "empty generated source: $last" >&2
+		exit 25
+	fi
+	cat > "$out" <<'EOF'
+#!/bin/sh
+printf 'ok\n'
+EOF
+	chmod +x "$out"
+	;;
+*)
+	echo "unexpected command: $*" >&2
+	exit 26
+	;;
+esac
+`, logPath, allowRunValue)
+	if err := os.WriteFile(path, []byte(script), 0o755); err != nil {
+		t.Fatal(err)
+	}
+}
+
 func TestToolchainGoModVersion(t *testing.T) {
 	dir := t.TempDir()
 	if err := os.WriteFile(filepath.Join(dir, "VERSION"), []byte("go1.24.11\n"), 0o644); err != nil {

test/goroot/xfail.yaml

@@ -197,24 +197,6 @@ timeouts:
     case: fixedbugs/bug230.go
     timeout: 2m
     reason: bug230 run runs past the default timeout on darwin/arm64
-  - version: go1.24
-    platform: darwin/arm64
-    directive: runoutput
-    case: fixedbugs/bug449.go
-    timeout: 12m
-    reason: bug449 runoutput generation runs past the 4m timeout on darwin/arm64
-  - version: go1.25
-    platform: darwin/arm64
-    directive: runoutput
-    case: fixedbugs/bug449.go
-    timeout: 12m
-    reason: bug449 runoutput generation runs past the 4m timeout on darwin/arm64
-  - version: go1.26
-    platform: darwin/arm64
-    directive: runoutput
-    case: fixedbugs/bug449.go
-    timeout: 12m
-    reason: bug449 runoutput generation runs past the 4m timeout on darwin/arm64
   - version: go1.24
     platform: darwin/arm64
     directive: run
@@ -683,12 +665,6 @@ timeouts:
     case: fixedbugs/bug230.go
     timeout: 2m
     reason: bug230 run runs past the default timeout on darwin/arm64
-  - version: go1.25
-    platform: darwin/arm64
-    directive: runoutput
-    case: fixedbugs/bug449.go
-    timeout: 4m
-    reason: bug449 runoutput generation runs past the 2m timeout on darwin/arm64
   - version: go1.25
     platform: darwin/arm64
     directive: run
@@ -1085,12 +1061,6 @@ timeouts:
     case: closure.go
     timeout: 2m
     reason: closure run runs past the default timeout on darwin/arm64
-  - version: go1.26
-    platform: darwin/arm64
-    directive: runoutput
-    case: fixedbugs/bug449.go
-    timeout: 4m
-    reason: bug449 runoutput generation runs past the 2m timeout on darwin/arm64
   - version: go1.26
     platform: darwin/arm64
     directive: run
@@ -2312,21 +2282,6 @@ xfails:
     directive: runoutput
     case: rangegen.go
     reason: go1.26 goroot ci-mode runoutput failure on linux/amd64
-  - version: go1.24
-    platform: linux/amd64
-    directive: runoutput
-    case: fixedbugs/bug449.go
-    reason: bug449 runoutput emits an empty llgo_tmp__.go on linux/amd64
-  - version: go1.25
-    platform: linux/amd64
-    directive: runoutput
-    case: fixedbugs/bug449.go
-    reason: bug449 runoutput emits an empty llgo_tmp__.go on linux/amd64
-  - version: go1.26
-    platform: linux/amd64
-    directive: runoutput
-    case: fixedbugs/bug449.go
-    reason: bug449 runoutput emits an empty llgo_tmp__.go on linux/amd64
   - platform: darwin/arm64
     directive: run
     case: deferfin.go