From 0ef91bc0a78789a58fdeaa1e4ea64e2cedf55868 Mon Sep 17 00:00:00 2001 From: duarte Date: Fri, 27 Mar 2026 19:15:38 +0000 Subject: [PATCH 001/118] Package level metadata that includes a mapping of build statements to build targets --- src/core/build_statements.go | 17 +++++++++++++++++ src/core/package.go | 15 ++++++++++++++- src/core/state.go | 6 +++--- src/parse/asp/builtins.go | 2 +- src/parse/asp/interpreter.go | 11 +++++++++++ 5 files changed, 46 insertions(+), 5 deletions(-) create mode 100644 src/core/build_statements.go diff --git a/src/core/build_statements.go b/src/core/build_statements.go new file mode 100644 index 0000000000..9b399f8e21 --- /dev/null +++ b/src/core/build_statements.go @@ -0,0 +1,17 @@ +package core + +type BuildStatement struct { + Start, End int +} + +type BuildFileMetadata struct { + StmtToTarget map[BuildStatement][]*BuildTarget + SubincludeStmts []BuildStatement +} + +func (bfm *BuildFileMetadata) RegisterStatementTarget(stmt *BuildStatement, target *BuildTarget) { + if bfm.StmtToTarget == nil { + bfm.StmtToTarget = make(map[BuildStatement][]*BuildTarget) + } + bfm.StmtToTarget[*stmt] = append(bfm.StmtToTarget[*stmt], target) +} diff --git a/src/core/package.go b/src/core/package.go index 377022a482..899ed95343 100644 --- a/src/core/package.go +++ b/src/core/package.go @@ -34,6 +34,8 @@ type Package struct { targets map[string]*BuildTarget // Set of output files from rules. Outputs map[string]*BuildTarget + // Includes metadata from parsing the package BUILD file. + BuildFileMetadata BuildFileMetadata // Protects access to above mutex sync.RWMutex } @@ -71,10 +73,11 @@ func (pkg *Package) TargetOrDie(name string) *BuildTarget { // AddTarget adds a new target to this package with the given name. // It doesn't check for duplicates. -func (pkg *Package) AddTarget(target *BuildTarget) { +func (pkg *Package) AddTarget(target *BuildTarget, stmt *BuildStatement) { pkg.mutex.Lock() defer pkg.mutex.Unlock() pkg.targets[target.Label.Name] = target + pkg.RegisterStatement(stmt, target) } // AllTargets returns the current set of targets in this package. @@ -230,6 +233,16 @@ func (pkg *Package) verifyOutputs() []string { return ret } +// RegisterStatement maps a build statement to target in the package. +func (pkg *Package) RegisterStatement(stmt *BuildStatement, target *BuildTarget) { + if stmt == nil { + log.Infof("Attempted to register empty build statement for package %s and target %s", + pkg.Name, target.String()) + return + } + pkg.BuildFileMetadata.RegisterStatementTarget(stmt, target) +} + // FindOwningPackages returns build labels corresponding to the packages that own each of the given files. func FindOwningPackages(state *BuildState, files []string) []BuildLabel { ret := make([]BuildLabel, len(files)) diff --git a/src/core/state.go b/src/core/state.go index 18a03d70ab..3df3338a01 100644 --- a/src/core/state.go +++ b/src/core/state.go @@ -921,8 +921,8 @@ func (state *BuildState) WaitForBuiltTarget(l, dependent BuildLabel, mode ParseM } // AddTarget adds a new target to the build graph. -func (state *BuildState) AddTarget(pkg *Package, target *BuildTarget) { - pkg.AddTarget(target) +func (state *BuildState) AddTarget(pkg *Package, target *BuildTarget, stmt *BuildStatement) { + pkg.AddTarget(target, stmt) state.Graph.AddTarget(target) if target.IsFilegroup { // At least register these guys as outputs. @@ -1069,7 +1069,7 @@ func exportFile(state *BuildState, pkg *Package, label BuildLabel) { t.Subrepo = pkg.Subrepo t.IsFilegroup = true t.AddSource(NewFileLabel(label.Name, pkg)) - state.AddTarget(pkg, t) + state.AddTarget(pkg, t, nil) } // CheckArchSubrepo checks if a target refers to a cross-compiling subrepo. diff --git a/src/parse/asp/builtins.go b/src/parse/asp/builtins.go index 8be2cbc8e4..a85dd20685 100644 --- a/src/parse/asp/builtins.go +++ b/src/parse/asp/builtins.go @@ -209,7 +209,7 @@ func buildRule(s *scope, args []pyObject) pyObject { target := createTarget(s, args) s.Assert(s.pkg.Target(target.Label.Name) == nil, "Duplicate build target in %s: %s", s.pkg.Name, target.Label.Name) populateTarget(s, target, args) - s.state.AddTarget(s.pkg, target) + s.state.AddTarget(s.pkg, target, s.CurrentBuildStatement()) if s.Callback { target.AddedPostBuild = true } diff --git a/src/parse/asp/interpreter.go b/src/parse/asp/interpreter.go index 93e4ca0a32..d5ace5916f 100644 --- a/src/parse/asp/interpreter.go +++ b/src/parse/asp/interpreter.go @@ -310,6 +310,8 @@ type scope struct { // True if this scope is for a pre- or post-build callback. Callback bool mode core.ParseMode + // points to the statement currently being interpreted + cursor *Statement } // parseAnnotatedLabelInPackage similarly to parseLabelInPackage, parses the label contextualising it to the provided @@ -525,6 +527,7 @@ func (s *scope) interpretStatements(statements []*Statement) pyObject { } }() for _, stmt = range statements { + s.cursor = stmt if stmt.FuncDef != nil { s.Set(stmt.FuncDef.Name, newPyFunc(s, stmt.FuncDef)) } else if stmt.If != nil { @@ -1077,6 +1080,14 @@ func (s *scope) Constant(expr *Expression) pyObject { return nil } +// CurrentBuildStatement creates a new BuildStatement from the statement that is being currently interpreted. +func (s *scope) CurrentBuildStatement() *core.BuildStatement { + return &core.BuildStatement{ + Start: int(s.cursor.Pos), + End: int(s.cursor.EndPos), + } +} + // pkgFilename returns the filename of the current package, or the empty string if there is none. func (s *scope) pkgFilename() string { if s.pkg != nil { From 63f1b0d0c76579da14b4476b0856bdba61b02de1 Mon Sep 17 00:00:00 2001 From: duarte Date: Fri, 27 Mar 2026 19:54:21 +0000 Subject: [PATCH 002/118] export using BuildFileMetadata logic --- src/core/build_statements.go | 18 +++++ src/export/export.go | 153 ++++++++++++++++++++++++----------- 2 files changed, 123 insertions(+), 48 deletions(-) diff --git a/src/core/build_statements.go b/src/core/build_statements.go index 9b399f8e21..caab5e9be2 100644 --- a/src/core/build_statements.go +++ b/src/core/build_statements.go @@ -1,9 +1,18 @@ package core +import ( + "fmt" + "slices" +) + type BuildStatement struct { Start, End int } +func (bs *BuildStatement) Len() int { + return bs.End - bs.Start +} + type BuildFileMetadata struct { StmtToTarget map[BuildStatement][]*BuildTarget SubincludeStmts []BuildStatement @@ -15,3 +24,12 @@ func (bfm *BuildFileMetadata) RegisterStatementTarget(stmt *BuildStatement, targ } bfm.StmtToTarget[*stmt] = append(bfm.StmtToTarget[*stmt], target) } + +func (bfm *BuildFileMetadata) FindStatement(target *BuildTarget) (*BuildStatement, error) { + for stmt, targets := range bfm.StmtToTarget { + if slices.Contains(targets, target) { + return &stmt, nil + } + } + return nil, fmt.Errorf("Target %s not found in statement metadata.", target.String()) +} diff --git a/src/export/export.go b/src/export/export.go index 2863dfef0c..0bb2d5b7db 100644 --- a/src/export/export.go +++ b/src/export/export.go @@ -4,14 +4,15 @@ package export import ( + "fmt" iofs "io/fs" "os" "path/filepath" + "slices" "github.com/thought-machine/please/src/cli/logging" "github.com/thought-machine/please/src/core" "github.com/thought-machine/please/src/fs" - "github.com/thought-machine/please/src/gc" "github.com/thought-machine/please/src/parse" ) @@ -22,19 +23,21 @@ type export struct { targetDir string noTrim bool - exportedTargets map[core.BuildLabel]bool - exportedPackages map[string]bool + exportedTargets map[core.BuildLabel]bool + exportedPackages map[string]bool + selectedStatements map[*core.Package]map[core.BuildStatement]bool } // ToDir exports a set of targets to the given directory. // It dies on any errors. func ToDir(state *core.BuildState, dir string, noTrim bool, targets []core.BuildLabel) { e := &export{ - state: state, - noTrim: noTrim, - targetDir: dir, - exportedPackages: map[string]bool{}, - exportedTargets: map[core.BuildLabel]bool{}, + state: state, + noTrim: noTrim, + targetDir: dir, + exportedPackages: map[string]bool{}, + exportedTargets: map[core.BuildLabel]bool{}, + selectedStatements: map[*core.Package]map[core.BuildStatement]bool{}, } if err := os.MkdirAll(dir, fs.DirPermissions); err != nil { @@ -47,14 +50,13 @@ func ToDir(state *core.BuildState, dir string, noTrim bool, targets []core.Build e.export(state.Graph.TargetOrDie(includeLabel)) } } + + log.Warningf("Exporting selected targets: %v", targets) for _, target := range targets { e.export(state.Graph.TargetOrDie(target)) } - // Now write all the build files - packages := map[*core.Package]bool{} - for target := range e.exportedTargets { - packages[state.Graph.PackageOrDie(target)] = true - } + + e.writeBuildStatements() // Write any preloaded build defs as well; preloaded subincludes should be fine though. for _, preload := range state.Config.Parse.PreloadBuildDefs { @@ -62,33 +64,6 @@ func ToDir(state *core.BuildState, dir string, noTrim bool, targets []core.Build log.Fatalf("Failed to copy preloaded build def %s: %s", preload, err) } } - - if noTrim { - return // We have already exported the whole directory - } - - for pkg := range packages { - if pkg.Name == parse.InternalPackageName { - continue // This isn't a real package to be copied - } - if pkg.Subrepo != nil { - continue // Don't copy subrepo BUILD files... they don't exist in our source tree - } - dest := filepath.Join(dir, pkg.Filename) - if err := fs.CopyFile(pkg.Filename, dest, 0); err != nil { - log.Fatalf("Failed to copy BUILD file %s: %s\n", pkg.Filename, err) - } - // Now rewrite the unused targets out of it - var victims []string - for _, target := range pkg.AllTargets() { - if !e.exportedTargets[target.Label] && !target.HasParent() { - victims = append(victims, target.Label.Name) - } - } - if err := gc.RewriteFile(state, dest, victims); err != nil { - log.Fatalf("Failed to rewrite BUILD file: %s\n", err) - } - } } func (e *export) exportPlzConf() { @@ -99,7 +74,7 @@ func (e *export) exportPlzConf() { for _, file := range profiles { path := filepath.Join(e.targetDir, file) if err := os.RemoveAll(path); err != nil { - log.Fatalf("failed to copy .plzconfig file %s: %v", file, err) + log.Fatalf("failed to remove .plzconfig file %s: %v", file, err) } if err := fs.CopyFile(file, path, 0); err != nil { log.Fatalf("failed to copy .plzconfig file %s: %v", file, err) @@ -116,6 +91,7 @@ func (e *export) exportSources(target *core.BuildTarget) { if err := fs.RecursiveCopy(p, filepath.Join(e.targetDir, p), 0); err != nil { log.Fatalf("Error copying file: %s\n", err) } + log.Warning("Writing source file: %s", p) } } } @@ -129,8 +105,8 @@ var ignoreDirectories = map[string]bool{ ".hg": true, } -// exportPackage exports the package BUILD file containing the given target and all sources -func (e *export) exportPackage(target *core.BuildTarget) { +// exportEntirePackage exports the package BUILD file containing the given target and all sources +func (e *export) exportEntirePackage(target *core.BuildTarget) { pkgName := target.Label.PackageName if pkgName == parse.InternalPackageName { return @@ -169,31 +145,112 @@ func (e *export) exportPackage(target *core.BuildTarget) { } } +// selectBuildStatement exports BUILD statements that generate the build target. +func (e *export) selectBuildStatement(pkg *core.Package, target *core.BuildTarget) { + if target.Label.PackageName == parse.InternalPackageName { + return + } + + if _, ok := e.selectedStatements[pkg]; !ok { + e.selectedStatements[pkg] = map[core.BuildStatement]bool{} + } + + stmt, err := pkg.BuildFileMetadata.FindStatement(target) + if err != nil { + log.Fatalf("Failed to find statement in %s: %w", pkg.Name, err) + } + e.selectedStatements[pkg][*stmt] = true +} + // export implements the logic of ToDir, but prevents repeating targets. func (e *export) export(target *core.BuildTarget) { if e.exportedTargets[target.Label] { return } + log.Warningf("Exporting %v.\n", target.Label) + e.exportedTargets[target.Label] = true + + pkg := e.state.Graph.PackageOrDie(target.Label) + // We want to export the package that made this subrepo available, but we still need to walk the target deps // as it may depend on other subrepos or first party targets if target.Subrepo != nil { + log.Warningf("Subrepo: %v", target.Subrepo.Target) e.export(target.Subrepo.Target) } else if e.noTrim { // Export the whole package, rather than trying to trim the package down to only the targets we need - e.exportPackage(target) + e.exportEntirePackage(target) } else { + e.selectBuildStatement(pkg, target) e.exportSources(target) } - e.exportedTargets[target.Label] = true for _, dep := range target.Dependencies() { e.export(dep) } - for _, subinclude := range e.state.Graph.PackageOrDie(target.Label).AllSubincludes(e.state.Graph) { + for _, subinclude := range pkg.AllSubincludes(e.state.Graph) { e.export(e.state.Graph.TargetOrDie(subinclude)) } - if parent := target.Parent(e.state.Graph); parent != nil && parent != target { - e.export(parent) + + for stmt := range e.selectedStatements[pkg] { + relatedTargets := pkg.BuildFileMetadata.StmtToTarget[stmt] + for _, t := range relatedTargets { + e.export(t) + } + } +} + +// writeBuildStatements writes the BUILD file statements to the export directory. +func (e *export) writeBuildStatements() { + log.Warningf("Selected Statements: %v", e.selectedStatements) + + for pkg, stmtMap := range e.selectedStatements { + stmts := make([]core.BuildStatement, 0, len(stmtMap)) + for stmt := range stmtMap { + stmts = append(stmts, stmt) + } + // Sort statements by position to keep them in order + slices.SortFunc(stmts, func(a, b core.BuildStatement) int { + return a.Start - b.Start + }) + + e.writeBuildFile(pkg, stmts) + } +} + +func (e *export) writeBuildFile(pkg *core.Package, stmts []core.BuildStatement) { + filename := pkg.Filename + log.Warningf("Writing file: %s", filename) + if err := fs.EnsureDir(filepath.Join(e.targetDir, filename)); err != nil { + log.Fatalf("failed to create directory for %s: %v", filename, err) + } + fw, err := os.Create(filepath.Join(e.targetDir, filename)) + if err != nil { + log.Fatalf("failed to create BUILD file %s: %v", filename, err) + } + defer fw.Close() + + fr, err := os.Open(filename) + if err != nil { + // TODO ensure only visiting correct files and move Warn to Fatal + log.Warningf("failed to open file %s: %v", filename, err) + return + } + defer fr.Close() + + for _, s := range stmts { + buff := make([]byte, s.Len()) + _, err := fr.ReadAt(buff, int64(s.Start)) + if err != nil { + log.Fatalf("failed to read BUILD file %s: %v", filename, err) + } + + if _, err := fw.Write(buff); err != nil { + log.Fatalf("failed to write statement to %s: %v", filename, err) + } + if _, err := fmt.Fprintf(fw, "\n#%+v\n\n", s); err != nil { + log.Fatalf("failed to write newline to %s: %v", filename, err) + } } } From 419bd565e98a2743a5699f63dc5b73c68d3cd116 Mon Sep 17 00:00:00 2001 From: duarte Date: Tue, 7 Apr 2026 12:31:28 +0100 Subject: [PATCH 003/118] register caller scope for callstack-like traversal --- src/parse/asp/interpreter.go | 16 ++++++++++++---- src/parse/asp/objects.go | 1 + 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/parse/asp/interpreter.go b/src/parse/asp/interpreter.go index d5ace5916f..7fdc0cead4 100644 --- a/src/parse/asp/interpreter.go +++ b/src/parse/asp/interpreter.go @@ -304,14 +304,14 @@ type scope struct { subincludeLabel *core.BuildLabel // If set, label of the subinclude we're currently interpreting parsingFor *parseTarget parent *scope + callerScope *scope // caller local scope, nil if not in callstack locals pyDict config *pyConfig globber *fs.Globber // True if this scope is for a pre- or post-build callback. Callback bool mode core.ParseMode - // points to the statement currently being interpreted - cursor *Statement + cursor *Statement // points to the statement currently being interpreted } // parseAnnotatedLabelInPackage similarly to parseLabelInPackage, parses the label contextualising it to the provided @@ -428,6 +428,7 @@ func (s *scope) newScope(pkg *core.Package, mode core.ParseMode, filename string config: s.config, Callback: s.Callback, mode: mode, + cursor: s.cursor, } if pkg != nil && pkg.Subrepo != nil && pkg.Subrepo.State != nil { s2.state = pkg.Subrepo.State @@ -1082,9 +1083,16 @@ func (s *scope) Constant(expr *Expression) pyObject { // CurrentBuildStatement creates a new BuildStatement from the statement that is being currently interpreted. func (s *scope) CurrentBuildStatement() *core.BuildStatement { + stmtScope := s + for curr := s; curr != nil; curr = curr.callerScope { + if curr.pkg != nil && curr.filename == s.pkg.Filename { + stmtScope = curr + } + } + s.NAssert(stmtScope.cursor == nil, "Cursor is not pointing to a statement") return &core.BuildStatement{ - Start: int(s.cursor.Pos), - End: int(s.cursor.EndPos), + Start: int(stmtScope.cursor.Pos), + End: int(stmtScope.cursor.EndPos), } } diff --git a/src/parse/asp/objects.go b/src/parse/asp/objects.go index a783b55b5c..a93b63613b 100644 --- a/src/parse/asp/objects.go +++ b/src/parse/asp/objects.go @@ -699,6 +699,7 @@ func (f *pyFunc) Call(s *scope, c *Call) pyObject { return f.callNative(s, c) } s2 := f.scope.newScope(s.pkg, s.mode, f.scope.filename, len(f.args)+1) + s2.callerScope = s // registering previous scope as caller s2.config = s.config s2.Set("CONFIG", s.config) // This needs to be copied across too :( s2.Callback = s.Callback From 8e1bd00ddef0d1f39b220fd52800fe69ccbad544 Mon Sep 17 00:00:00 2001 From: duarte Date: Tue, 7 Apr 2026 13:11:45 +0100 Subject: [PATCH 004/118] bufio writer for build statements --- src/core/build_statements.go | 10 ++- src/export/BUILD | 1 - src/export/export.go | 123 +++++++++++++++++++---------------- 3 files changed, 73 insertions(+), 61 deletions(-) diff --git a/src/core/build_statements.go b/src/core/build_statements.go index caab5e9be2..cb4ea1d203 100644 --- a/src/core/build_statements.go +++ b/src/core/build_statements.go @@ -9,12 +9,16 @@ type BuildStatement struct { Start, End int } -func (bs *BuildStatement) Len() int { - return bs.End - bs.Start +func (bs *BuildStatement) Len() int64 { + return int64(bs.End - bs.Start) +} + +func (bs *BuildStatement) StartPos() int64 { + return int64(bs.Start) } type BuildFileMetadata struct { - StmtToTarget map[BuildStatement][]*BuildTarget + StmtToTarget map[BuildStatement][]*BuildTarget SubincludeStmts []BuildStatement } diff --git a/src/export/BUILD b/src/export/BUILD index 6b44d63d9b..76217a23e0 100644 --- a/src/export/BUILD +++ b/src/export/BUILD @@ -7,7 +7,6 @@ go_library( "//src/cli/logging", "//src/core", "//src/fs", - "//src/gc", "//src/parse", ], ) diff --git a/src/export/export.go b/src/export/export.go index 0bb2d5b7db..559e97fba0 100644 --- a/src/export/export.go +++ b/src/export/export.go @@ -4,7 +4,8 @@ package export import ( - "fmt" + "bufio" + "io" iofs "io/fs" "os" "path/filepath" @@ -66,6 +67,45 @@ func ToDir(state *core.BuildState, dir string, noTrim bool, targets []core.Build } } +// export implements the logic of ToDir, but prevents repeating targets. +func (e *export) export(target *core.BuildTarget) { + if e.exportedTargets[target.Label] { + return + } + log.Warningf("Exporting %v.\n", target.Label) + e.exportedTargets[target.Label] = true + + pkg := e.state.Graph.PackageOrDie(target.Label) + log.Warningf("%+v", pkg.BuildFileMetadata) + + // We want to export the package that made this subrepo available, but we still need to walk the target deps + // as it may depend on other subrepos or first party targets + if target.Subrepo != nil { + log.Warningf("Subrepo: %v", target.Subrepo.Target) + e.export(target.Subrepo.Target) + } else if e.noTrim { + // Export the whole package, rather than trying to trim the package down to only the targets we need + e.exportEntirePackage(target) + } else { + e.selectBuildStatement(pkg, target) + e.exportSources(target) + } + + for _, dep := range target.Dependencies() { + e.export(dep) + } + for _, subinclude := range pkg.AllSubincludes(e.state.Graph) { + e.export(e.state.Graph.TargetOrDie(subinclude)) + } + + for stmt := range e.selectedStatements[pkg] { + relatedTargets := pkg.BuildFileMetadata.StmtToTarget[stmt] + for _, t := range relatedTargets { + e.export(t) + } + } +} + func (e *export) exportPlzConf() { profiles, err := filepath.Glob(".plzconfig*") if err != nil { @@ -162,44 +202,6 @@ func (e *export) selectBuildStatement(pkg *core.Package, target *core.BuildTarge e.selectedStatements[pkg][*stmt] = true } -// export implements the logic of ToDir, but prevents repeating targets. -func (e *export) export(target *core.BuildTarget) { - if e.exportedTargets[target.Label] { - return - } - log.Warningf("Exporting %v.\n", target.Label) - e.exportedTargets[target.Label] = true - - pkg := e.state.Graph.PackageOrDie(target.Label) - - // We want to export the package that made this subrepo available, but we still need to walk the target deps - // as it may depend on other subrepos or first party targets - if target.Subrepo != nil { - log.Warningf("Subrepo: %v", target.Subrepo.Target) - e.export(target.Subrepo.Target) - } else if e.noTrim { - // Export the whole package, rather than trying to trim the package down to only the targets we need - e.exportEntirePackage(target) - } else { - e.selectBuildStatement(pkg, target) - e.exportSources(target) - } - - for _, dep := range target.Dependencies() { - e.export(dep) - } - for _, subinclude := range pkg.AllSubincludes(e.state.Graph) { - e.export(e.state.Graph.TargetOrDie(subinclude)) - } - - for stmt := range e.selectedStatements[pkg] { - relatedTargets := pkg.BuildFileMetadata.StmtToTarget[stmt] - for _, t := range relatedTargets { - e.export(t) - } - } -} - // writeBuildStatements writes the BUILD file statements to the export directory. func (e *export) writeBuildStatements() { log.Warningf("Selected Statements: %v", e.selectedStatements) @@ -220,38 +222,45 @@ func (e *export) writeBuildStatements() { func (e *export) writeBuildFile(pkg *core.Package, stmts []core.BuildStatement) { filename := pkg.Filename + exportedFilename := filepath.Join(e.targetDir, filename) + log.Warningf("Writing file: %s", filename) - if err := fs.EnsureDir(filepath.Join(e.targetDir, filename)); err != nil { - log.Fatalf("failed to create directory for %s: %v", filename, err) - } - fw, err := os.Create(filepath.Join(e.targetDir, filename)) - if err != nil { - log.Fatalf("failed to create BUILD file %s: %v", filename, err) - } - defer fw.Close() fr, err := os.Open(filename) if err != nil { - // TODO ensure only visiting correct files and move Warn to Fatal - log.Warningf("failed to open file %s: %v", filename, err) + log.Fatalf("failed to open file original BUILD file: %v", err) return } defer fr.Close() + frStat, err := fr.Stat() + if err != nil { + log.Fatalf("failed to get original BUILD file status: %v", err) + } + + fw, err := fs.OpenDirFile(exportedFilename, os.O_CREATE|os.O_WRONLY, frStat.Mode()) + if err != nil { + log.Fatalf("failed to create and open exported BUILD file for %s: %v", exportedFilename, err) + } + defer fw.Close() + + writer := bufio.NewWriter(fw) for _, s := range stmts { - buff := make([]byte, s.Len()) - _, err := fr.ReadAt(buff, int64(s.Start)) - if err != nil { - log.Fatalf("failed to read BUILD file %s: %v", filename, err) + if _, err := fr.Seek(s.StartPos(), io.SeekStart); err != nil { + log.Fatalf("failed to seek in BUILD file %s: %v", filename, err) } - if _, err := fw.Write(buff); err != nil { - log.Fatalf("failed to write statement to %s: %v", filename, err) + if _, err := io.CopyN(writer, fr, s.Len()); err != nil { + log.Fatalf("failed to copy statement from %s to %s: %v", filename, exportedFilename, err) } - if _, err := fmt.Fprintf(fw, "\n#%+v\n\n", s); err != nil { - log.Fatalf("failed to write newline to %s: %v", filename, err) + + if _, err := writer.WriteString("\n"); err != nil { + log.Fatalf("failed to add newline to %s: %v", exportedFilename, err) } } + if err := writer.Flush(); err != nil { + log.Fatalf("failed write exported BUILD file %s: %v", exportedFilename, err) + } } // Outputs exports the outputs of a target. From 258b1704c8732b5bf2f48ae95b49ecd8bd1fa6bb Mon Sep 17 00:00:00 2001 From: duarte Date: Mon, 20 Apr 2026 11:44:56 +0100 Subject: [PATCH 005/118] rewrite export logic into explicit flow --- src/BUILD.plz | 3 +- src/core/package.go | 4 + src/export/export.go | 192 ++++++++++++++++++++++--------------------- src/please.go | 2 +- 4 files changed, 105 insertions(+), 96 deletions(-) diff --git a/src/BUILD.plz b/src/BUILD.plz index c0eaec7e7b..4ef5ea6fd3 100644 --- a/src/BUILD.plz +++ b/src/BUILD.plz @@ -10,8 +10,8 @@ go_binary( deps = [ "///third_party/go/github.com_thought-machine_go-flags//:go-flags", "///third_party/go/go.uber.org_automaxprocs//maxprocs", - "//src/audit", "//src/assets", + "//src/audit", "//src/build", "//src/cache", "//src/clean", @@ -29,7 +29,6 @@ go_binary( "//src/help", "//src/metrics", "//src/output", - "//src/parse", "//src/plz", "//src/plzinit", "//src/process", diff --git a/src/core/package.go b/src/core/package.go index 899ed95343..45270a6362 100644 --- a/src/core/package.go +++ b/src/core/package.go @@ -243,6 +243,10 @@ func (pkg *Package) RegisterStatement(stmt *BuildStatement, target *BuildTarget) pkg.BuildFileMetadata.RegisterStatementTarget(stmt, target) } +func (pkg *Package) FindStatement(target *BuildTarget) (*BuildStatement, error) { + return pkg.BuildFileMetadata.FindStatement(target) +} + // FindOwningPackages returns build labels corresponding to the packages that own each of the given files. func FindOwningPackages(state *BuildState, files []string) []BuildLabel { ret := make([]BuildLabel, len(files)) diff --git a/src/export/export.go b/src/export/export.go index 559e97fba0..60c718b187 100644 --- a/src/export/export.go +++ b/src/export/export.go @@ -5,6 +5,7 @@ package export import ( "bufio" + "cmp" "io" iofs "io/fs" "os" @@ -29,10 +30,39 @@ type export struct { selectedStatements map[*core.Package]map[core.BuildStatement]bool } -// ToDir exports a set of targets to the given directory. -// It dies on any errors. -func ToDir(state *core.BuildState, dir string, noTrim bool, targets []core.BuildLabel) { - e := &export{ +func Repo(state *core.BuildState, dir string, noTrim bool, targets []core.BuildLabel) { + e := newExport(state, dir, noTrim) + + // ensure output dir + if err := os.MkdirAll(dir, fs.DirPermissions); err != nil { + log.Fatalf("failed to create export directory %s: %v", dir, err) + } + + e.plzConfig() + e.preloaded() + e.targets(targets) + e.writeBuildStatements() +} + +// Outputs exports the outputs of a target. +func Outputs(state *core.BuildState, dir string, targets []core.BuildLabel) { + for _, label := range targets { + target := state.Graph.TargetOrDie(label) + for _, out := range target.Outputs() { + fullPath := filepath.Join(dir, out) + outDir := filepath.Dir(fullPath) + if err := os.MkdirAll(outDir, core.DirPermissions); err != nil { + log.Fatalf("Failed to create export dir %s: %s", outDir, err) + } + if err := fs.RecursiveCopy(filepath.Join(target.OutDir(), out), fullPath, target.OutMode()|0200); err != nil { + log.Fatalf("Failed to copy export file: %s", err) + } + } + } +} + +func newExport(state *core.BuildState, dir string, noTrim bool) *export { + return &export{ state: state, noTrim: noTrim, targetDir: dir, @@ -40,90 +70,94 @@ func ToDir(state *core.BuildState, dir string, noTrim bool, targets []core.Build exportedTargets: map[core.BuildLabel]bool{}, selectedStatements: map[*core.Package]map[core.BuildStatement]bool{}, } +} - if err := os.MkdirAll(dir, fs.DirPermissions); err != nil { - log.Fatalf("failed to create export directory %s: %v", dir, err) +func (e *export) plzConfig() { + profiles, err := filepath.Glob(".plzconfig*") + if err != nil { + log.Fatalf("failed to glob .plzconfig files: %v", err) } - - e.exportPlzConf() - for _, target := range state.Config.Parse.PreloadSubincludes { - for _, includeLabel := range append(state.Graph.TransitiveSubincludes(target), target) { - e.export(state.Graph.TargetOrDie(includeLabel)) + for _, file := range profiles { + targetPath := filepath.Join(e.targetDir, file) + if err := os.RemoveAll(targetPath); err != nil { + log.Fatalf("failed to remove .plzconfig file %s: %v", file, err) + } + if err := fs.CopyFile(file, targetPath, 0); err != nil { + log.Fatalf("failed to copy .plzconfig file %s: %v", file, err) } } +} - log.Warningf("Exporting selected targets: %v", targets) - for _, target := range targets { - e.export(state.Graph.TargetOrDie(target)) +func (e *export) preloaded() { + // Write any preloaded build defs + for _, preload := range e.state.Config.Parse.PreloadBuildDefs { + if err := fs.RecursiveCopy(preload, filepath.Join(e.targetDir, preload), 0); err != nil { + log.Fatalf("Failed to copy preloaded build def %s: %s", preload, err) + } } - e.writeBuildStatements() + for _, target := range e.state.Config.Parse.PreloadSubincludes { + e.targets(append(e.state.Graph.TransitiveSubincludes(target), target)) + } +} - // Write any preloaded build defs as well; preloaded subincludes should be fine though. - for _, preload := range state.Config.Parse.PreloadBuildDefs { - if err := fs.RecursiveCopy(preload, filepath.Join(dir, preload), 0); err != nil { - log.Fatalf("Failed to copy preloaded build def %s: %s", preload, err) - } +func (e *export) targets(targets []core.BuildLabel) { + for _, label := range targets { + target := e.state.Graph.TargetOrDie(label) + e.target(target) } } -// export implements the logic of ToDir, but prevents repeating targets. -func (e *export) export(target *core.BuildTarget) { +func (e *export) target(target *core.BuildTarget) { if e.exportedTargets[target.Label] { return } - log.Warningf("Exporting %v.\n", target.Label) e.exportedTargets[target.Label] = true - pkg := e.state.Graph.PackageOrDie(target.Label) - log.Warningf("%+v", pkg.BuildFileMetadata) - // We want to export the package that made this subrepo available, but we still need to walk the target deps // as it may depend on other subrepos or first party targets if target.Subrepo != nil { - log.Warningf("Subrepo: %v", target.Subrepo.Target) - e.export(target.Subrepo.Target) - } else if e.noTrim { - // Export the whole package, rather than trying to trim the package down to only the targets we need - e.exportEntirePackage(target) - } else { - e.selectBuildStatement(pkg, target) - e.exportSources(target) + e.target(target.Subrepo.Target) + // TODO do we need dependencies and sources? + return } - for _, dep := range target.Dependencies() { - e.export(dep) - } + pkg := e.state.Graph.PackageOrDie(target.Label) + + // TODO notrim + + e.subincludes(pkg) + e.buildStatements(pkg, target) + e.sources(target) + e.dependencies(target) +} + +func (e *export) subincludes(pkg *core.Package) { + // TODO update to required subincludes for _, subinclude := range pkg.AllSubincludes(e.state.Graph) { - e.export(e.state.Graph.TargetOrDie(subinclude)) + e.target(e.state.Graph.TargetOrDie(subinclude)) } +} - for stmt := range e.selectedStatements[pkg] { - relatedTargets := pkg.BuildFileMetadata.StmtToTarget[stmt] - for _, t := range relatedTargets { - e.export(t) - } +// buildStatements exports BUILD statements that generate the build target. +func (e *export) buildStatements(pkg *core.Package, target *core.BuildTarget) { + if target.Label.PackageName == parse.InternalPackageName { + // TODO validate if we still need this + return } -} -func (e *export) exportPlzConf() { - profiles, err := filepath.Glob(".plzconfig*") - if err != nil { - log.Fatalf("failed to glob .plzconfig files: %v", err) + if _, ok := e.selectedStatements[pkg]; !ok { + e.selectedStatements[pkg] = map[core.BuildStatement]bool{} } - for _, file := range profiles { - path := filepath.Join(e.targetDir, file) - if err := os.RemoveAll(path); err != nil { - log.Fatalf("failed to remove .plzconfig file %s: %v", file, err) - } - if err := fs.CopyFile(file, path, 0); err != nil { - log.Fatalf("failed to copy .plzconfig file %s: %v", file, err) - } + + stmt, err := pkg.FindStatement(target) + if err != nil { + log.Fatalf("Failed to find statement in %s: %w", pkg.Name, err) } + e.selectedStatements[pkg][*stmt] = true } -// exportSources exports any source files (srcs and data) for the rule -func (e *export) exportSources(target *core.BuildTarget) { +func (e *export) sources(target *core.BuildTarget) { for _, src := range append(target.AllSources(), target.AllData()...) { if _, ok := src.Label(); !ok { // We'll handle these dependencies later for _, p := range src.FullPaths(e.state.Graph) { @@ -138,6 +172,12 @@ func (e *export) exportSources(target *core.BuildTarget) { } } +func (e *export) dependencies(target *core.BuildTarget) { + for _, dep := range target.Dependencies() { + e.target(dep) + } +} + var ignoreDirectories = map[string]bool{ "plz-out": true, ".git": true, @@ -185,23 +225,6 @@ func (e *export) exportEntirePackage(target *core.BuildTarget) { } } -// selectBuildStatement exports BUILD statements that generate the build target. -func (e *export) selectBuildStatement(pkg *core.Package, target *core.BuildTarget) { - if target.Label.PackageName == parse.InternalPackageName { - return - } - - if _, ok := e.selectedStatements[pkg]; !ok { - e.selectedStatements[pkg] = map[core.BuildStatement]bool{} - } - - stmt, err := pkg.BuildFileMetadata.FindStatement(target) - if err != nil { - log.Fatalf("Failed to find statement in %s: %w", pkg.Name, err) - } - e.selectedStatements[pkg][*stmt] = true -} - // writeBuildStatements writes the BUILD file statements to the export directory. func (e *export) writeBuildStatements() { log.Warningf("Selected Statements: %v", e.selectedStatements) @@ -213,7 +236,7 @@ func (e *export) writeBuildStatements() { } // Sort statements by position to keep them in order slices.SortFunc(stmts, func(a, b core.BuildStatement) int { - return a.Start - b.Start + return cmp.Compare(a.Start, b.Start) }) e.writeBuildFile(pkg, stmts) @@ -262,20 +285,3 @@ func (e *export) writeBuildFile(pkg *core.Package, stmts []core.BuildStatement) log.Fatalf("failed write exported BUILD file %s: %v", exportedFilename, err) } } - -// Outputs exports the outputs of a target. -func Outputs(state *core.BuildState, dir string, targets []core.BuildLabel) { - for _, label := range targets { - target := state.Graph.TargetOrDie(label) - for _, out := range target.Outputs() { - fullPath := filepath.Join(dir, out) - outDir := filepath.Dir(fullPath) - if err := os.MkdirAll(outDir, core.DirPermissions); err != nil { - log.Fatalf("Failed to create export dir %s: %s", outDir, err) - } - if err := fs.RecursiveCopy(filepath.Join(target.OutDir(), out), fullPath, target.OutMode()|0200); err != nil { - log.Fatalf("Failed to copy export file: %s", err) - } - } - } -} diff --git a/src/please.go b/src/please.go index 5eda69dfed..7297c7e689 100644 --- a/src/please.go +++ b/src/please.go @@ -769,7 +769,7 @@ var buildFunctions = map[string]func() int{ "export": func() int { success, state := runBuild(opts.Export.Args.Targets, false, false, false) if success { - export.ToDir(state, opts.Export.Output, opts.Export.NoTrim, state.ExpandOriginalLabels()) + export.Repo(state, opts.Export.Output, opts.Export.NoTrim, state.ExpandOriginalLabels()) } return toExitCode(success, state) }, From 4110735d86b70b36bd39e539692227a40fdb8a4e Mon Sep 17 00:00:00 2001 From: duarte Date: Mon, 20 Apr 2026 12:10:24 +0100 Subject: [PATCH 006/118] export targets related to build statement --- src/core/build_statements.go | 10 +++++++++- src/core/package.go | 6 ++++++ src/export/export.go | 14 ++++++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/core/build_statements.go b/src/core/build_statements.go index cb4ea1d203..b553062229 100644 --- a/src/core/build_statements.go +++ b/src/core/build_statements.go @@ -19,7 +19,7 @@ func (bs *BuildStatement) StartPos() int64 { type BuildFileMetadata struct { StmtToTarget map[BuildStatement][]*BuildTarget - SubincludeStmts []BuildStatement + SubincludeStmts map[BuildStatement][]*BuildTarget } func (bfm *BuildFileMetadata) RegisterStatementTarget(stmt *BuildStatement, target *BuildTarget) { @@ -37,3 +37,11 @@ func (bfm *BuildFileMetadata) FindStatement(target *BuildTarget) (*BuildStatemen } return nil, fmt.Errorf("Target %s not found in statement metadata.", target.String()) } + +func (bfm *BuildFileMetadata) FindTargets(stmt *BuildStatement) ([]*BuildTarget, error) { + targets, ok := bfm.StmtToTarget[*stmt] + if !ok { + return nil, fmt.Errorf("Targets not found for statement %v.", stmt) + } + return targets, nil +} diff --git a/src/core/package.go b/src/core/package.go index 45270a6362..dc74729965 100644 --- a/src/core/package.go +++ b/src/core/package.go @@ -243,10 +243,16 @@ func (pkg *Package) RegisterStatement(stmt *BuildStatement, target *BuildTarget) pkg.BuildFileMetadata.RegisterStatementTarget(stmt, target) } +// FindStatement finds the build statement that generated the target. func (pkg *Package) FindStatement(target *BuildTarget) (*BuildStatement, error) { return pkg.BuildFileMetadata.FindStatement(target) } +// FindRelatedTargets finds all the targets related to the build statement. +func (pkg *Package) FindRelatedTargets(stmt *BuildStatement) ([]*BuildTarget, error) { + return pkg.BuildFileMetadata.FindTargets(stmt) +} + // FindOwningPackages returns build labels corresponding to the packages that own each of the given files. func FindOwningPackages(state *BuildState, files []string) []BuildLabel { ret := make([]BuildLabel, len(files)) diff --git a/src/export/export.go b/src/export/export.go index 60c718b187..0184f5c212 100644 --- a/src/export/export.go +++ b/src/export/export.go @@ -154,7 +154,21 @@ func (e *export) buildStatements(pkg *core.Package, target *core.BuildTarget) { if err != nil { log.Fatalf("Failed to find statement in %s: %w", pkg.Name, err) } + + // check if visited before + if e.selectedStatements[pkg][*stmt] == true { + return + } e.selectedStatements[pkg][*stmt] = true + + relatedTargets, err := pkg.FindRelatedTargets(stmt) + if err != nil { + log.Fatalf("Failed to lookup related targets for package %s: %w", pkg.Name, err) + } + + for _, target := range relatedTargets { + e.target(target) + } } func (e *export) sources(target *core.BuildTarget) { From a4252c984090949a6708e9221196efa5c97b48a0 Mon Sep 17 00:00:00 2001 From: duarte Date: Mon, 20 Apr 2026 15:06:33 +0100 Subject: [PATCH 007/118] enrich target with subincludes by looping though all scopes --- src/core/build_target.go | 4 +++- src/parse/asp/interpreter.go | 19 +++++++++++++++++++ src/parse/asp/objects.go | 2 +- src/parse/asp/targets.go | 4 ++++ 4 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/core/build_target.go b/src/core/build_target.go index 3268e55f03..17f4199737 100644 --- a/src/core/build_target.go +++ b/src/core/build_target.go @@ -249,7 +249,9 @@ type BuildTarget struct { // Marks the target as a remote_file. IsRemoteFile bool `print:"false"` // Marks the target as a text_file. - IsTextFile bool `print:"false"` + IsTextFile bool `name:"text_file" print:"false"` + // The subincludes that were active when this target was defined. + Subincludes []BuildLabel `name:"subincludes"` // Marks that the target was added in a post-build function. AddedPostBuild bool `print:"false"` // If true, skips generating environment variables for sources; instead files will be generated in diff --git a/src/parse/asp/interpreter.go b/src/parse/asp/interpreter.go index 7fdc0cead4..05fafb3f21 100644 --- a/src/parse/asp/interpreter.go +++ b/src/parse/asp/interpreter.go @@ -1096,6 +1096,25 @@ func (s *scope) CurrentBuildStatement() *core.BuildStatement { } } +// ActiveSubincludes traces the call stack and scopes to find subincludes that provided the +// macros/functions actively executing to define this target. +func (s *scope) ActiveSubincludes() []core.BuildLabel { + var subincludes []core.BuildLabel + seen := map[core.BuildLabel]bool{} + for curr := s; curr != nil; curr = curr.callerScope { + for localScope := curr; localScope != nil; localScope = localScope.parent { + if localScope.subincludeLabel != nil { + label := *localScope.subincludeLabel + if !seen[label] { + seen[label] = true + subincludes = append(subincludes, label) + } + } + } + } + return subincludes +} + // pkgFilename returns the filename of the current package, or the empty string if there is none. func (s *scope) pkgFilename() string { if s.pkg != nil { diff --git a/src/parse/asp/objects.go b/src/parse/asp/objects.go index a93b63613b..0133d413ba 100644 --- a/src/parse/asp/objects.go +++ b/src/parse/asp/objects.go @@ -694,7 +694,7 @@ func (f *pyFunc) String() string { func (f *pyFunc) Call(s *scope, c *Call) pyObject { if f.nativeCode != nil { if f.kwargs { - return f.callNative(s.NewScope("", 0), c) + return f.callNative(s.NewScope("", 0), c) } return f.callNative(s, c) } diff --git a/src/parse/asp/targets.go b/src/parse/asp/targets.go index 48404b7920..3bcee4663a 100644 --- a/src/parse/asp/targets.go +++ b/src/parse/asp/targets.go @@ -187,9 +187,13 @@ func createTarget(s *scope, args []pyObject) *core.BuildTarget { target.Debug = new(core.DebugFields) target.Debug.Command, _ = decodeCommands(s, args[debugCMDBuildRuleArgIdx]) } + + target.Subincludes = s.ActiveSubincludes() + return target } + // validateSandbox ensures that the target isn't opting out of the build/test sandbox when it's not allowed to func validateSandbox(state *core.BuildState, target *core.BuildTarget) error { if target.IsFilegroup || len(state.Config.Sandbox.ExcludeableTargets) == 0 { From 82f117fb7898b3922dd68fcac5d7e1e7ea98546d Mon Sep 17 00:00:00 2001 From: duarte Date: Mon, 20 Apr 2026 16:21:03 +0100 Subject: [PATCH 008/118] separate buildstmt register from adding target --- src/core/package.go | 7 ++++--- src/core/state.go | 6 +++--- src/parse/asp/builtins.go | 3 ++- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/core/package.go b/src/core/package.go index dc74729965..fbb2bf3f62 100644 --- a/src/core/package.go +++ b/src/core/package.go @@ -73,11 +73,10 @@ func (pkg *Package) TargetOrDie(name string) *BuildTarget { // AddTarget adds a new target to this package with the given name. // It doesn't check for duplicates. -func (pkg *Package) AddTarget(target *BuildTarget, stmt *BuildStatement) { +func (pkg *Package) AddTarget(target *BuildTarget) { pkg.mutex.Lock() defer pkg.mutex.Unlock() pkg.targets[target.Label.Name] = target - pkg.RegisterStatement(stmt, target) } // AllTargets returns the current set of targets in this package. @@ -234,12 +233,14 @@ func (pkg *Package) verifyOutputs() []string { } // RegisterStatement maps a build statement to target in the package. -func (pkg *Package) RegisterStatement(stmt *BuildStatement, target *BuildTarget) { +func (pkg *Package) RegisterStatement(target *BuildTarget, stmt *BuildStatement) { if stmt == nil { log.Infof("Attempted to register empty build statement for package %s and target %s", pkg.Name, target.String()) return } + pkg.mutex.Lock() + defer pkg.mutex.Unlock() pkg.BuildFileMetadata.RegisterStatementTarget(stmt, target) } diff --git a/src/core/state.go b/src/core/state.go index 3df3338a01..18a03d70ab 100644 --- a/src/core/state.go +++ b/src/core/state.go @@ -921,8 +921,8 @@ func (state *BuildState) WaitForBuiltTarget(l, dependent BuildLabel, mode ParseM } // AddTarget adds a new target to the build graph. -func (state *BuildState) AddTarget(pkg *Package, target *BuildTarget, stmt *BuildStatement) { - pkg.AddTarget(target, stmt) +func (state *BuildState) AddTarget(pkg *Package, target *BuildTarget) { + pkg.AddTarget(target) state.Graph.AddTarget(target) if target.IsFilegroup { // At least register these guys as outputs. @@ -1069,7 +1069,7 @@ func exportFile(state *BuildState, pkg *Package, label BuildLabel) { t.Subrepo = pkg.Subrepo t.IsFilegroup = true t.AddSource(NewFileLabel(label.Name, pkg)) - state.AddTarget(pkg, t, nil) + state.AddTarget(pkg, t) } // CheckArchSubrepo checks if a target refers to a cross-compiling subrepo. diff --git a/src/parse/asp/builtins.go b/src/parse/asp/builtins.go index a85dd20685..126a40d6d7 100644 --- a/src/parse/asp/builtins.go +++ b/src/parse/asp/builtins.go @@ -209,7 +209,8 @@ func buildRule(s *scope, args []pyObject) pyObject { target := createTarget(s, args) s.Assert(s.pkg.Target(target.Label.Name) == nil, "Duplicate build target in %s: %s", s.pkg.Name, target.Label.Name) populateTarget(s, target, args) - s.state.AddTarget(s.pkg, target, s.CurrentBuildStatement()) + s.state.AddTarget(s.pkg, target) + s.pkg.RegisterStatement(target, s.CurrentBuildStatement()) if s.Callback { target.AddedPostBuild = true } From eb657de6835822adc129f0d01bf8d87175bafbd3 Mon Sep 17 00:00:00 2001 From: duarte Date: Mon, 20 Apr 2026 18:28:09 +0100 Subject: [PATCH 009/118] select and write subincludes setting subincludes at package level instead of at target level --- src/core/build_statements.go | 20 +++++++++++-- src/core/build_target.go | 4 +-- src/core/package.go | 18 ++++++++++++ src/export/BUILD | 11 +++++++ src/export/export.go | 57 +++++++++++++++++++++++++++++++++--- src/export/export_test.go | 32 ++++++++++++++++++++ src/parse/asp/builtins.go | 2 ++ src/parse/asp/targets.go | 2 -- 8 files changed, 135 insertions(+), 11 deletions(-) create mode 100644 src/export/export_test.go diff --git a/src/core/build_statements.go b/src/core/build_statements.go index b553062229..3ea4df238b 100644 --- a/src/core/build_statements.go +++ b/src/core/build_statements.go @@ -18,8 +18,9 @@ func (bs *BuildStatement) StartPos() int64 { } type BuildFileMetadata struct { - StmtToTarget map[BuildStatement][]*BuildTarget - SubincludeStmts map[BuildStatement][]*BuildTarget + StmtToTarget map[BuildStatement][]*BuildTarget + TargetToSubinclude map[*BuildTarget]BuildLabels + // TODO Untracked stmts - will export every time (e.g. package) } func (bfm *BuildFileMetadata) RegisterStatementTarget(stmt *BuildStatement, target *BuildTarget) { @@ -29,6 +30,13 @@ func (bfm *BuildFileMetadata) RegisterStatementTarget(stmt *BuildStatement, targ bfm.StmtToTarget[*stmt] = append(bfm.StmtToTarget[*stmt], target) } +func (bfm *BuildFileMetadata) RegisterSubinclude(target *BuildTarget, subincludes BuildLabels) { + if bfm.TargetToSubinclude == nil { + bfm.TargetToSubinclude = make(map[*BuildTarget]BuildLabels) + } + bfm.TargetToSubinclude[target] = append(bfm.TargetToSubinclude[target], subincludes...) +} + func (bfm *BuildFileMetadata) FindStatement(target *BuildTarget) (*BuildStatement, error) { for stmt, targets := range bfm.StmtToTarget { if slices.Contains(targets, target) { @@ -45,3 +53,11 @@ func (bfm *BuildFileMetadata) FindTargets(stmt *BuildStatement) ([]*BuildTarget, } return targets, nil } + +func (bfm *BuildFileMetadata) FindSubincludes(target *BuildTarget) (BuildLabels, error) { + subincludes, ok := bfm.TargetToSubinclude[target] + if !ok { + return nil, fmt.Errorf("Subincludes not found for target %v.", target) + } + return subincludes, nil +} diff --git a/src/core/build_target.go b/src/core/build_target.go index 17f4199737..3268e55f03 100644 --- a/src/core/build_target.go +++ b/src/core/build_target.go @@ -249,9 +249,7 @@ type BuildTarget struct { // Marks the target as a remote_file. IsRemoteFile bool `print:"false"` // Marks the target as a text_file. - IsTextFile bool `name:"text_file" print:"false"` - // The subincludes that were active when this target was defined. - Subincludes []BuildLabel `name:"subincludes"` + IsTextFile bool `print:"false"` // Marks that the target was added in a post-build function. AddedPostBuild bool `print:"false"` // If true, skips generating environment variables for sources; instead files will be generated in diff --git a/src/core/package.go b/src/core/package.go index fbb2bf3f62..f7f930a497 100644 --- a/src/core/package.go +++ b/src/core/package.go @@ -244,6 +244,19 @@ func (pkg *Package) RegisterStatement(target *BuildTarget, stmt *BuildStatement) pkg.BuildFileMetadata.RegisterStatementTarget(stmt, target) } +// RegisterRequiredSubincludes maps the required subincludes to generate the target. +func (pkg *Package) RegisterRequiredSubincludes(target *BuildTarget, subincludes BuildLabels) { + if len(subincludes) == 0 { + log.Infof("Attempted to register empty subinclude labels for package %s and target %s", + pkg.Name, target.String()) + return + } + + pkg.mutex.Lock() + defer pkg.mutex.Unlock() + pkg.BuildFileMetadata.RegisterSubinclude(target, subincludes) +} + // FindStatement finds the build statement that generated the target. func (pkg *Package) FindStatement(target *BuildTarget) (*BuildStatement, error) { return pkg.BuildFileMetadata.FindStatement(target) @@ -254,6 +267,11 @@ func (pkg *Package) FindRelatedTargets(stmt *BuildStatement) ([]*BuildTarget, er return pkg.BuildFileMetadata.FindTargets(stmt) } +// FindRequiredSubincludes finds the subincludes target labels required by the given target. +func (pkg *Package) FindRequiredSubincludes(target *BuildTarget) (BuildLabels, error) { + return pkg.BuildFileMetadata.FindSubincludes(target) +} + // FindOwningPackages returns build labels corresponding to the packages that own each of the given files. func FindOwningPackages(state *BuildState, files []string) []BuildLabel { ret := make([]BuildLabel, len(files)) diff --git a/src/export/BUILD b/src/export/BUILD index 76217a23e0..1895be6eff 100644 --- a/src/export/BUILD +++ b/src/export/BUILD @@ -4,9 +4,20 @@ go_library( pgo_file = "//:pgo", visibility = ["PUBLIC"], deps = [ + "///third_party/go/github.com_please-build_buildtools//build", "//src/cli/logging", "//src/core", "//src/fs", "//src/parse", ], ) + +go_test( + name = "export_test", + srcs = ["export_test.go"], + deps = [ + ":export", + "///third_party/go/github.com_stretchr_testify//assert", + "//src/core", + ], +) diff --git a/src/export/export.go b/src/export/export.go index 0184f5c212..7e12ed26a0 100644 --- a/src/export/export.go +++ b/src/export/export.go @@ -12,6 +12,8 @@ import ( "path/filepath" "slices" + "github.com/please-build/buildtools/build" + "github.com/thought-machine/please/src/cli/logging" "github.com/thought-machine/please/src/core" "github.com/thought-machine/please/src/fs" @@ -28,6 +30,7 @@ type export struct { exportedTargets map[core.BuildLabel]bool exportedPackages map[string]bool selectedStatements map[*core.Package]map[core.BuildStatement]bool + requiredSubincludes map[*core.Package]map[core.BuildLabel]bool } func Repo(state *core.BuildState, dir string, noTrim bool, targets []core.BuildLabel) { @@ -69,6 +72,7 @@ func newExport(state *core.BuildState, dir string, noTrim bool) *export { exportedPackages: map[string]bool{}, exportedTargets: map[core.BuildLabel]bool{}, selectedStatements: map[*core.Package]map[core.BuildStatement]bool{}, + requiredSubincludes: map[*core.Package]map[core.BuildLabel]bool{}, } } @@ -126,17 +130,29 @@ func (e *export) target(target *core.BuildTarget) { // TODO notrim - e.subincludes(pkg) + e.subincludes(pkg, target) e.buildStatements(pkg, target) e.sources(target) e.dependencies(target) } -func (e *export) subincludes(pkg *core.Package) { - // TODO update to required subincludes - for _, subinclude := range pkg.AllSubincludes(e.state.Graph) { +func (e *export) subincludes(pkg *core.Package, target *core.BuildTarget) { + subincludes, err := pkg.FindRequiredSubincludes(target) + if err != nil { + log.Infof("No subincludes found, assuming non required.: %w", pkg.Name, err) + return + } + + for _, subinclude := range subincludes { + if _, ok := e.requiredSubincludes[pkg]; !ok { + e.requiredSubincludes[pkg] = map[core.BuildLabel]bool{} + } + e.requiredSubincludes[pkg][subinclude] = true + e.target(e.state.Graph.TargetOrDie(subinclude)) } + + log.Warningf("Parse Metadata Subincludes: %v", pkg.BuildFileMetadata.TargetToSubinclude) } // buildStatements exports BUILD statements that generate the build target. @@ -248,6 +264,7 @@ func (e *export) writeBuildStatements() { for stmt := range stmtMap { stmts = append(stmts, stmt) } + // Sort statements by position to keep them in order slices.SortFunc(stmts, func(a, b core.BuildStatement) int { return cmp.Compare(a.Start, b.Start) @@ -282,6 +299,13 @@ func (e *export) writeBuildFile(pkg *core.Package, stmts []core.BuildStatement) defer fw.Close() writer := bufio.NewWriter(fw) + // Subinclude + if subinclude := e.makeSubincludesStatement(pkg); subinclude != "" { + if _, err := writer.WriteString(subinclude + "\n\n"); err != nil { + log.Fatalf("failed to add subincludes to %s: %v", exportedFilename, err) + } + } + // Statements for _, s := range stmts { if _, err := fr.Seek(s.StartPos(), io.SeekStart); err != nil { log.Fatalf("failed to seek in BUILD file %s: %v", filename, err) @@ -299,3 +323,28 @@ func (e *export) writeBuildFile(pkg *core.Package, stmts []core.BuildStatement) log.Fatalf("failed write exported BUILD file %s: %v", exportedFilename, err) } } + +func (e *export) makeSubincludesStatement(pkg *core.Package) string { + subincludesMap, ok := e.requiredSubincludes[pkg] + if !ok || len(subincludesMap) == 0 { + return "" + } + + labels := make(core.BuildLabels, 0, len(subincludesMap)) + for label := range subincludesMap { + labels = append(labels, label) + } + + slices.SortFunc(labels, func(a, b core.BuildLabel) int { + return cmp.Compare(a.String(), b.String()) + }) + + call := &build.CallExpr{ + X: &build.Ident{Name: "subinclude"}, + } + for _, label := range labels { + call.List = append(call.List, &build.StringExpr{Value: label.String()}) + } + + return build.FormatString(call) +} diff --git a/src/export/export_test.go b/src/export/export_test.go new file mode 100644 index 0000000000..0b7c7b0003 --- /dev/null +++ b/src/export/export_test.go @@ -0,0 +1,32 @@ +package export + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/thought-machine/please/src/core" +) + +func TestMakeSubincludesStatement(t *testing.T) { + e := &export{ + requiredSubincludes: map[*core.Package]map[core.BuildLabel]bool{}, + } + + pkg := &core.Package{Name: "test"} + + // Test case 1: No subincludes + assert.Equal(t, "", e.makeSubincludesStatement(pkg)) + + // Test case 2: Single subinclude + label1 := core.ParseBuildLabel("//build_defs:test", "") + e.requiredSubincludes[pkg] = map[core.BuildLabel]bool{ + label1: true, + } + assert.Equal(t, `subinclude("//build_defs:test")`, e.makeSubincludesStatement(pkg)) + + // Test case 3: Multiple subincludes (sorted) + label2 := core.ParseBuildLabel("//build_defs:abc", "") + e.requiredSubincludes[pkg][label2] = true + expected := "subinclude(\n \"//build_defs:abc\",\n \"//build_defs:test\",\n)" + assert.Equal(t, expected, e.makeSubincludesStatement(pkg)) +} diff --git a/src/parse/asp/builtins.go b/src/parse/asp/builtins.go index 126a40d6d7..8542d4cadc 100644 --- a/src/parse/asp/builtins.go +++ b/src/parse/asp/builtins.go @@ -211,6 +211,8 @@ func buildRule(s *scope, args []pyObject) pyObject { populateTarget(s, target, args) s.state.AddTarget(s.pkg, target) s.pkg.RegisterStatement(target, s.CurrentBuildStatement()) + s.pkg.RegisterRequiredSubincludes(target, s.ActiveSubincludes()) + if s.Callback { target.AddedPostBuild = true } diff --git a/src/parse/asp/targets.go b/src/parse/asp/targets.go index 3bcee4663a..c1785fbe43 100644 --- a/src/parse/asp/targets.go +++ b/src/parse/asp/targets.go @@ -188,8 +188,6 @@ func createTarget(s *scope, args []pyObject) *core.BuildTarget { target.Debug.Command, _ = decodeCommands(s, args[debugCMDBuildRuleArgIdx]) } - target.Subincludes = s.ActiveSubincludes() - return target } From 65357943d5f12ea8e974e12f78c49c8117b16737 Mon Sep 17 00:00:00 2001 From: duarte Date: Tue, 21 Apr 2026 11:20:26 +0100 Subject: [PATCH 010/118] skip statement for preloaded subincludes --- src/export/export.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/export/export.go b/src/export/export.go index 7e12ed26a0..1b57973a2a 100644 --- a/src/export/export.go +++ b/src/export/export.go @@ -31,6 +31,7 @@ type export struct { exportedPackages map[string]bool selectedStatements map[*core.Package]map[core.BuildStatement]bool requiredSubincludes map[*core.Package]map[core.BuildLabel]bool + preloadedSubincludes map[core.BuildLabel]bool } func Repo(state *core.BuildState, dir string, noTrim bool, targets []core.BuildLabel) { @@ -73,6 +74,7 @@ func newExport(state *core.BuildState, dir string, noTrim bool) *export { exportedTargets: map[core.BuildLabel]bool{}, selectedStatements: map[*core.Package]map[core.BuildStatement]bool{}, requiredSubincludes: map[*core.Package]map[core.BuildLabel]bool{}, + preloadedSubincludes: map[core.BuildLabel]bool{}, } } @@ -101,7 +103,11 @@ func (e *export) preloaded() { } for _, target := range e.state.Config.Parse.PreloadSubincludes { - e.targets(append(e.state.Graph.TransitiveSubincludes(target), target)) + targets := append(e.state.Graph.TransitiveSubincludes(target), target) + for _, t := range targets { + e.preloadedSubincludes[t] = true + } + e.targets(targets) } } @@ -144,6 +150,11 @@ func (e *export) subincludes(pkg *core.Package, target *core.BuildTarget) { } for _, subinclude := range subincludes { + // skip for preloaded subincludes + if e.preloadedSubincludes[subinclude] { + continue + } + if _, ok := e.requiredSubincludes[pkg]; !ok { e.requiredSubincludes[pkg] = map[core.BuildLabel]bool{} } From 8e6c9c0fad4da72a3c2b7c6d5beeeb006aa09ae0 Mon Sep 17 00:00:00 2001 From: duarte Date: Tue, 21 Apr 2026 11:42:31 +0100 Subject: [PATCH 011/118] test: trim subincludes --- test/export/test_subinclude_trimming/BUILD | 7 +++++++ .../expected_repo/.plzconfig | 3 +++ .../expected_repo/BUILD_FILE | 7 +++++++ .../expected_repo/build_defs/BUILD_FILE | 5 +++++ .../expected_repo/build_defs/simple.build_defs | 10 ++++++++++ .../expected_repo/file.txt | 1 + .../source_repo/.plzconfig | 3 +++ .../source_repo/BUILD_FILE | 15 +++++++++++++++ .../source_repo/build_defs/BUILD_FILE | 11 +++++++++++ .../source_repo/build_defs/simple.build_defs | 10 ++++++++++ .../source_repo/build_defs/unused.build_defs | 8 ++++++++ .../test_subinclude_trimming/source_repo/file.txt | 1 + 12 files changed, 81 insertions(+) create mode 100644 test/export/test_subinclude_trimming/BUILD create mode 100644 test/export/test_subinclude_trimming/expected_repo/.plzconfig create mode 100644 test/export/test_subinclude_trimming/expected_repo/BUILD_FILE create mode 100644 test/export/test_subinclude_trimming/expected_repo/build_defs/BUILD_FILE create mode 100644 test/export/test_subinclude_trimming/expected_repo/build_defs/simple.build_defs create mode 100644 test/export/test_subinclude_trimming/expected_repo/file.txt create mode 100644 test/export/test_subinclude_trimming/source_repo/.plzconfig create mode 100644 test/export/test_subinclude_trimming/source_repo/BUILD_FILE create mode 100644 test/export/test_subinclude_trimming/source_repo/build_defs/BUILD_FILE create mode 100644 test/export/test_subinclude_trimming/source_repo/build_defs/simple.build_defs create mode 100644 test/export/test_subinclude_trimming/source_repo/build_defs/unused.build_defs create mode 100644 test/export/test_subinclude_trimming/source_repo/file.txt diff --git a/test/export/test_subinclude_trimming/BUILD b/test/export/test_subinclude_trimming/BUILD new file mode 100644 index 0000000000..b363357c3d --- /dev/null +++ b/test/export/test_subinclude_trimming/BUILD @@ -0,0 +1,7 @@ +subinclude("//test/export:export_e2e_test_build_def") + +# Export a target generated by a custom build def and validate that any unused subincludes are trimmed. +please_export_e2e_test( + name = "export_subinclude_trimming", + export_targets = ["//:simple_custom_target"], +) diff --git a/test/export/test_subinclude_trimming/expected_repo/.plzconfig b/test/export/test_subinclude_trimming/expected_repo/.plzconfig new file mode 100644 index 0000000000..8e1ae5655a --- /dev/null +++ b/test/export/test_subinclude_trimming/expected_repo/.plzconfig @@ -0,0 +1,3 @@ +[Parse] + +BuildFileName = BUILD_FILE diff --git a/test/export/test_subinclude_trimming/expected_repo/BUILD_FILE b/test/export/test_subinclude_trimming/expected_repo/BUILD_FILE new file mode 100644 index 0000000000..261729adc3 --- /dev/null +++ b/test/export/test_subinclude_trimming/expected_repo/BUILD_FILE @@ -0,0 +1,7 @@ +subinclude("//build_defs:simple_build_def") + +simple_custom_target( + name = "simple_custom_target", + srcs = ["file.txt"], + outs = ["file_simple.out"], +) diff --git a/test/export/test_subinclude_trimming/expected_repo/build_defs/BUILD_FILE b/test/export/test_subinclude_trimming/expected_repo/build_defs/BUILD_FILE new file mode 100644 index 0000000000..a556ce6d44 --- /dev/null +++ b/test/export/test_subinclude_trimming/expected_repo/build_defs/BUILD_FILE @@ -0,0 +1,5 @@ +filegroup( + name = "simple_build_def", + srcs = ["simple.build_defs"], + visibility = ["PUBLIC"], +) diff --git a/test/export/test_subinclude_trimming/expected_repo/build_defs/simple.build_defs b/test/export/test_subinclude_trimming/expected_repo/build_defs/simple.build_defs new file mode 100644 index 0000000000..8fe3a6021b --- /dev/null +++ b/test/export/test_subinclude_trimming/expected_repo/build_defs/simple.build_defs @@ -0,0 +1,10 @@ +def simple_custom_target( + name:str, + srcs:list=[], + outs:list=[]): + return genrule( + name = name, + srcs = srcs, + outs = outs, + cmd = "cat $SRCS > $OUT", + ) diff --git a/test/export/test_subinclude_trimming/expected_repo/file.txt b/test/export/test_subinclude_trimming/expected_repo/file.txt new file mode 100644 index 0000000000..9768ee14c2 --- /dev/null +++ b/test/export/test_subinclude_trimming/expected_repo/file.txt @@ -0,0 +1 @@ +Test source file diff --git a/test/export/test_subinclude_trimming/source_repo/.plzconfig b/test/export/test_subinclude_trimming/source_repo/.plzconfig new file mode 100644 index 0000000000..8e1ae5655a --- /dev/null +++ b/test/export/test_subinclude_trimming/source_repo/.plzconfig @@ -0,0 +1,3 @@ +[Parse] + +BuildFileName = BUILD_FILE diff --git a/test/export/test_subinclude_trimming/source_repo/BUILD_FILE b/test/export/test_subinclude_trimming/source_repo/BUILD_FILE new file mode 100644 index 0000000000..b678f4858f --- /dev/null +++ b/test/export/test_subinclude_trimming/source_repo/BUILD_FILE @@ -0,0 +1,15 @@ +subinclude( + "//build_defs:simple_build_def", + "//build_defs:unused_build_def", +) + +simple_custom_target( + name = "simple_custom_target", + srcs = ["file.txt"], + outs = ["file_simple.out"], +) + +unused_target( + name = "dummy", + outs = ["dummy.out"], +) diff --git a/test/export/test_subinclude_trimming/source_repo/build_defs/BUILD_FILE b/test/export/test_subinclude_trimming/source_repo/build_defs/BUILD_FILE new file mode 100644 index 0000000000..562e57a297 --- /dev/null +++ b/test/export/test_subinclude_trimming/source_repo/build_defs/BUILD_FILE @@ -0,0 +1,11 @@ +filegroup( + name = "simple_build_def", + srcs = ["simple.build_defs"], + visibility = ["PUBLIC"], +) + +filegroup( + name = "unused_build_def", + srcs = ["unused.build_defs"], + visibility = ["PUBLIC"], +) diff --git a/test/export/test_subinclude_trimming/source_repo/build_defs/simple.build_defs b/test/export/test_subinclude_trimming/source_repo/build_defs/simple.build_defs new file mode 100644 index 0000000000..8fe3a6021b --- /dev/null +++ b/test/export/test_subinclude_trimming/source_repo/build_defs/simple.build_defs @@ -0,0 +1,10 @@ +def simple_custom_target( + name:str, + srcs:list=[], + outs:list=[]): + return genrule( + name = name, + srcs = srcs, + outs = outs, + cmd = "cat $SRCS > $OUT", + ) diff --git a/test/export/test_subinclude_trimming/source_repo/build_defs/unused.build_defs b/test/export/test_subinclude_trimming/source_repo/build_defs/unused.build_defs new file mode 100644 index 0000000000..ebdff7de00 --- /dev/null +++ b/test/export/test_subinclude_trimming/source_repo/build_defs/unused.build_defs @@ -0,0 +1,8 @@ +def unused_target( + name:str, + outs:list=[]): + return genrule( + name = name, + outs = outs, + cmd = "echo dummy > $OUT", + ) diff --git a/test/export/test_subinclude_trimming/source_repo/file.txt b/test/export/test_subinclude_trimming/source_repo/file.txt new file mode 100644 index 0000000000..9768ee14c2 --- /dev/null +++ b/test/export/test_subinclude_trimming/source_repo/file.txt @@ -0,0 +1 @@ +Test source file From 9f0b4fe26a43fd59e6defa3d8972022ce84b757e Mon Sep 17 00:00:00 2001 From: duarte Date: Tue, 21 Apr 2026 12:11:00 +0100 Subject: [PATCH 012/118] rename file --- src/core/{build_statements.go => package_metada.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/core/{build_statements.go => package_metada.go} (100%) diff --git a/src/core/build_statements.go b/src/core/package_metada.go similarity index 100% rename from src/core/build_statements.go rename to src/core/package_metada.go From 0e597c6231bb87662722c4736dff624ed2840e7c Mon Sep 17 00:00:00 2001 From: duarte Date: Tue, 21 Apr 2026 14:37:39 +0100 Subject: [PATCH 013/118] use slices.collect and sort interface --- src/core/package_metada.go | 7 +++++++ src/export/export.go | 23 ++++++----------------- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/src/core/package_metada.go b/src/core/package_metada.go index 3ea4df238b..8cb4031bfa 100644 --- a/src/core/package_metada.go +++ b/src/core/package_metada.go @@ -17,6 +17,13 @@ func (bs *BuildStatement) StartPos() int64 { return int64(bs.Start) } +// BuildStatements is a slice of BuildStatement that implements sort.Interface. +type BuildStatements []BuildStatement + +func (s BuildStatements) Len() int { return len(s) } +func (s BuildStatements) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s BuildStatements) Less(i, j int) bool { return s[i].Start < s[j].Start } + type BuildFileMetadata struct { StmtToTarget map[BuildStatement][]*BuildTarget TargetToSubinclude map[*BuildTarget]BuildLabels diff --git a/src/export/export.go b/src/export/export.go index 1b57973a2a..f6a37c97dd 100644 --- a/src/export/export.go +++ b/src/export/export.go @@ -5,12 +5,13 @@ package export import ( "bufio" - "cmp" "io" iofs "io/fs" + "maps" "os" "path/filepath" "slices" + "sort" "github.com/please-build/buildtools/build" @@ -271,15 +272,9 @@ func (e *export) writeBuildStatements() { log.Warningf("Selected Statements: %v", e.selectedStatements) for pkg, stmtMap := range e.selectedStatements { - stmts := make([]core.BuildStatement, 0, len(stmtMap)) - for stmt := range stmtMap { - stmts = append(stmts, stmt) - } - + stmts := slices.Collect(maps.Keys(stmtMap)) // Sort statements by position to keep them in order - slices.SortFunc(stmts, func(a, b core.BuildStatement) int { - return cmp.Compare(a.Start, b.Start) - }) + sort.Sort(core.BuildStatements(stmts)) e.writeBuildFile(pkg, stmts) } @@ -341,14 +336,8 @@ func (e *export) makeSubincludesStatement(pkg *core.Package) string { return "" } - labels := make(core.BuildLabels, 0, len(subincludesMap)) - for label := range subincludesMap { - labels = append(labels, label) - } - - slices.SortFunc(labels, func(a, b core.BuildLabel) int { - return cmp.Compare(a.String(), b.String()) - }) + labels := slices.Collect(maps.Keys(subincludesMap)) + sort.Sort(core.BuildLabels(labels)) call := &build.CallExpr{ X: &build.Ident{Name: "subinclude"}, From de76e2cb4960545b9cfbb793b8d144d3baeed87b Mon Sep 17 00:00:00 2001 From: duarte Date: Wed, 22 Apr 2026 11:39:09 +0100 Subject: [PATCH 014/118] rewrite export with 2 concrete interfaces: default and notrim --- src/export/export.go | 347 ++++++++++++++++++++++++++++--------------- 1 file changed, 225 insertions(+), 122 deletions(-) diff --git a/src/export/export.go b/src/export/export.go index f6a37c97dd..d8a1a834b4 100644 --- a/src/export/export.go +++ b/src/export/export.go @@ -23,30 +23,26 @@ import ( var log = logging.Log -type export struct { - state *core.BuildState - targetDir string - noTrim bool - - exportedTargets map[core.BuildLabel]bool - exportedPackages map[string]bool - selectedStatements map[*core.Package]map[core.BuildStatement]bool - requiredSubincludes map[*core.Package]map[core.BuildLabel]bool - preloadedSubincludes map[core.BuildLabel]bool +type Exporter interface { + PlzConfig() + Preloaded() + Targets(core.BuildLabels) + Target(target *core.BuildTarget) + WriteBuildStatements() } func Repo(state *core.BuildState, dir string, noTrim bool, targets []core.BuildLabel) { - e := newExport(state, dir, noTrim) + e := newExporter(state, dir, noTrim) // ensure output dir if err := os.MkdirAll(dir, fs.DirPermissions); err != nil { log.Fatalf("failed to create export directory %s: %v", dir, err) } - e.plzConfig() - e.preloaded() - e.targets(targets) - e.writeBuildStatements() + e.PlzConfig() + e.Preloaded() + e.Targets(targets) + e.WriteBuildStatements() } // Outputs exports the outputs of a target. @@ -66,26 +62,50 @@ func Outputs(state *core.BuildState, dir string, targets []core.BuildLabel) { } } -func newExport(state *core.BuildState, dir string, noTrim bool) *export { - return &export{ - state: state, - noTrim: noTrim, - targetDir: dir, - exportedPackages: map[string]bool{}, - exportedTargets: map[core.BuildLabel]bool{}, - selectedStatements: map[*core.Package]map[core.BuildStatement]bool{}, - requiredSubincludes: map[*core.Package]map[core.BuildLabel]bool{}, - preloadedSubincludes: map[core.BuildLabel]bool{}, +func newExporter(state *core.BuildState, dir string, noTrim bool) Exporter { + base := baseExporter{ + state: state, + targetDir: dir, + exportedTargets: map[core.BuildLabel]bool{}, } + + if noTrim { + exporter := &NoTrimExporter{ + baseExporter: base, + exportedPackages: map[string]bool{}, + } + exporter.impl = exporter + return exporter + } else { + exporter := &DefaultExporter{ + baseExporter: base, + selectedStatements: map[*core.Package]map[core.BuildStatement]bool{}, + requiredSubincludes: map[*core.Package]map[core.BuildLabel]bool{}, + preloadedSubincludes: map[core.BuildLabel]bool{}, + } + exporter.impl = exporter + return exporter + } +} + +// baseExporter provides common fields and methods of other exporters. A reference +// to the concrete exporter implementation is included to be used in the common methods. +type baseExporter struct { + state *core.BuildState + targetDir string + + exportedTargets map[core.BuildLabel]bool + impl Exporter } -func (e *export) plzConfig() { +// PlzConfig exports the repo configuration files. +func (be *baseExporter) PlzConfig() { profiles, err := filepath.Glob(".plzconfig*") if err != nil { log.Fatalf("failed to glob .plzconfig files: %v", err) } for _, file := range profiles { - targetPath := filepath.Join(e.targetDir, file) + targetPath := filepath.Join(be.targetDir, file) if err := os.RemoveAll(targetPath); err != nil { log.Fatalf("failed to remove .plzconfig file %s: %v", file, err) } @@ -95,7 +115,48 @@ func (e *export) plzConfig() { } } -func (e *export) preloaded() { +// Targets exports all targets for the given labels. +func (be *baseExporter) Targets(labels core.BuildLabels) { + for _, l := range labels { + target := be.state.Graph.TargetOrDie(l) + be.impl.Target(target) + } +} + +// Dependencies exports dependencies of a target. +func (be *baseExporter) Dependencies(target *core.BuildTarget) { + for _, dep := range target.Dependencies() { + be.impl.Target(dep) + } +} + +// Sources exports all files required by the target. +func (be *baseExporter) Sources(target *core.BuildTarget) { + for _, src := range append(target.AllSources(), target.AllData()...) { + if _, ok := src.Label(); !ok { // We'll handle these dependencies later + for _, p := range src.Paths(be.state.Graph) { + if !filepath.IsAbs(p) { // Don't copy system file deps. + if err := fs.RecursiveCopy(p, filepath.Join(be.targetDir, p), 0); err != nil { + log.Fatalf("Error copying file: %s\n", err) + } + log.Warning("Writing source file: %s", p) + } + } + } + } +} + +// DefaultExporter implements an exporter that trims packages to reach a minimal exported repo. +type DefaultExporter struct { + baseExporter + selectedStatements map[*core.Package]map[core.BuildStatement]bool + requiredSubincludes map[*core.Package]map[core.BuildLabel]bool + preloadedSubincludes map[core.BuildLabel]bool +} + +// Preloaded exports the preloaded targets, build defs and subincludes. These preloads are usually +// defined in the .plzexport config. +func (e *DefaultExporter) Preloaded() { // Write any preloaded build defs for _, preload := range e.state.Config.Parse.PreloadBuildDefs { if err := fs.RecursiveCopy(preload, filepath.Join(e.targetDir, preload), 0); err != nil { @@ -108,18 +169,13 @@ func (e *export) preloaded() { for _, t := range targets { e.preloadedSubincludes[t] = true } - e.targets(targets) - } -} - -func (e *export) targets(targets []core.BuildLabel) { - for _, label := range targets { - target := e.state.Graph.TargetOrDie(label) - e.target(target) + e.Targets(targets) } } -func (e *export) target(target *core.BuildTarget) { +// Target exports an individual target. This implementation will attempt to export a minimal repo +// with only the required targets and statements. +func (e *DefaultExporter) Target(target *core.BuildTarget) { if e.exportedTargets[target.Label] { return } @@ -128,22 +184,22 @@ func (e *export) target(target *core.BuildTarget) { // We want to export the package that made this subrepo available, but we still need to walk the target deps // as it may depend on other subrepos or first party targets if target.Subrepo != nil { - e.target(target.Subrepo.Target) + e.Target(target.Subrepo.Target) // TODO do we need dependencies and sources? return } pkg := e.state.Graph.PackageOrDie(target.Label) - // TODO notrim - - e.subincludes(pkg, target) - e.buildStatements(pkg, target) - e.sources(target) - e.dependencies(target) + e.Subincludes(pkg, target) + e.BuildStatements(pkg, target) + e.Sources(target) + e.Dependencies(target) } -func (e *export) subincludes(pkg *core.Package, target *core.BuildTarget) { +// Subincludes exports the subincluded targets required to generate the target and selects them to +// later be written to the package as statements. +func (e *DefaultExporter) Subincludes(pkg *core.Package, target *core.BuildTarget) { subincludes, err := pkg.FindRequiredSubincludes(target) if err != nil { log.Infof("No subincludes found, assuming non required.: %w", pkg.Name, err) @@ -161,14 +217,14 @@ func (e *export) subincludes(pkg *core.Package, target *core.BuildTarget) { } e.requiredSubincludes[pkg][subinclude] = true - e.target(e.state.Graph.TargetOrDie(subinclude)) + e.Target(e.state.Graph.TargetOrDie(subinclude)) } log.Warningf("Parse Metadata Subincludes: %v", pkg.BuildFileMetadata.TargetToSubinclude) } -// buildStatements exports BUILD statements that generate the build target. -func (e *export) buildStatements(pkg *core.Package, target *core.BuildTarget) { +// BuildStatements exports BUILD statements that generate the build target. +func (e *DefaultExporter) BuildStatements(pkg *core.Package, target *core.BuildTarget) { if target.Label.PackageName == parse.InternalPackageName { // TODO validate if we still need this return @@ -195,80 +251,12 @@ func (e *export) buildStatements(pkg *core.Package, target *core.BuildTarget) { } for _, target := range relatedTargets { - e.target(target) - } -} - -func (e *export) sources(target *core.BuildTarget) { - for _, src := range append(target.AllSources(), target.AllData()...) { - if _, ok := src.Label(); !ok { // We'll handle these dependencies later - for _, p := range src.FullPaths(e.state.Graph) { - if !filepath.IsAbs(p) { // Don't copy system file deps. - if err := fs.RecursiveCopy(p, filepath.Join(e.targetDir, p), 0); err != nil { - log.Fatalf("Error copying file: %s\n", err) - } - log.Warning("Writing source file: %s", p) - } - } - } - } -} - -func (e *export) dependencies(target *core.BuildTarget) { - for _, dep := range target.Dependencies() { - e.target(dep) - } -} - -var ignoreDirectories = map[string]bool{ - "plz-out": true, - ".git": true, - ".svn": true, - ".hg": true, -} - -// exportEntirePackage exports the package BUILD file containing the given target and all sources -func (e *export) exportEntirePackage(target *core.BuildTarget) { - pkgName := target.Label.PackageName - if pkgName == parse.InternalPackageName { - return - } - if e.exportedPackages[pkgName] { - return - } - e.exportedPackages[pkgName] = true - - pkgDir := filepath.Clean(pkgName) - - err := filepath.WalkDir(pkgDir, func(path string, d iofs.DirEntry, err error) error { - if err != nil { - return err - } - if d.IsDir() { - if path != pkgDir && fs.IsPackage(e.state.Config.Parse.BuildFileName, path) { - return filepath.SkipDir // We want to stop when we find another package in our dir tree - } - if ignoreDirectories[d.Name()] { - return filepath.SkipDir - } - return nil - } - if !d.Type().IsRegular() { - return nil // Ignore symlinks, which are almost certainly generated sources. - } - dest := filepath.Join(e.targetDir, path) - if err := fs.EnsureDir(dest); err != nil { - return err - } - return fs.CopyFile(path, dest, 0) - }) - if err != nil { - log.Fatalf("failed to export package %s for %s: %v", pkgName, target.Label, err) + e.Target(target) } } -// writeBuildStatements writes the BUILD file statements to the export directory. -func (e *export) writeBuildStatements() { +// WriteBuildStatements writes the BUILD file statements to the export directory. +func (e *DefaultExporter) WriteBuildStatements() { log.Warningf("Selected Statements: %v", e.selectedStatements) for pkg, stmtMap := range e.selectedStatements { @@ -276,11 +264,12 @@ func (e *export) writeBuildStatements() { // Sort statements by position to keep them in order sort.Sort(core.BuildStatements(stmts)) - e.writeBuildFile(pkg, stmts) + e.writePackageFile(pkg, stmts) } } -func (e *export) writeBuildFile(pkg *core.Package, stmts []core.BuildStatement) { +// writePackageFile writes the selected statements to the please build file in the exported directory. +func (e *DefaultExporter) writePackageFile(pkg *core.Package, stmts []core.BuildStatement) { filename := pkg.Filename exportedFilename := filepath.Join(e.targetDir, filename) @@ -330,7 +319,9 @@ func (e *export) writeBuildFile(pkg *core.Package, stmts []core.BuildStatement) } } -func (e *export) makeSubincludesStatement(pkg *core.Package) string { +// makeSubincludesStatement generates a single subinclude statement (as string) with the required +// targets for this package. +func (e *DefaultExporter) makeSubincludesStatement(pkg *core.Package) string { subincludesMap, ok := e.requiredSubincludes[pkg] if !ok || len(subincludesMap) == 0 { return "" @@ -348,3 +339,115 @@ func (e *export) makeSubincludesStatement(pkg *core.Package) string { return build.FormatString(call) } + +// NoTrimExporter implements an exporter that avoids trimming any packages by exporting all targets +// and statements in a package. +type NoTrimExporter struct { + baseExporter + exportedPackages map[string]bool +} + +func (nte *NoTrimExporter) Preloaded() { + // Write any preloaded build defs + for _, preload := range nte.state.Config.Parse.PreloadBuildDefs { + if err := fs.RecursiveCopy(preload, filepath.Join(nte.targetDir, preload), 0); err != nil { + log.Fatalf("Failed to copy preloaded build def %s: %s", preload, err) + } + } + + for _, target := range nte.state.Config.Parse.PreloadSubincludes { + targets := append(nte.state.Graph.TransitiveSubincludes(target), target) + nte.Targets(targets) + } +} + +// Target exports an individual target. This implementation won't attempted any trimming, exporting +// all targets and statements defined in the package. +func (nte *NoTrimExporter) Target(target *core.BuildTarget) { + if nte.exportedTargets[target.Label] { + return + } + nte.exportedTargets[target.Label] = true + + // We want to export the package that made this subrepo available, but we still need to walk the target deps + // as it may depend on other subrepos or first party targets + if target.Subrepo != nil { + nte.Target(target.Subrepo.Target) + // TODO do we need dependencies and sources? + return + } + + pkg := nte.state.Graph.PackageOrDie(target.Label) + + nte.Package(target) + nte.Subincludes(pkg, target) + nte.AllTargets(pkg) + nte.Dependencies(target) +} + +var ignoreDirectories = map[string]bool{ + "plz-out": true, + ".git": true, + ".svn": true, + ".hg": true, +} + +// Package exports the package BUILD file containing the given target and all sources. +func (nte *NoTrimExporter) Package(target *core.BuildTarget) { + pkgName := target.Label.PackageName + if pkgName == parse.InternalPackageName { + return + } + if nte.exportedPackages[pkgName] { + return + } + nte.exportedPackages[pkgName] = true + + pkgDir := filepath.Clean(pkgName) + + err := filepath.WalkDir(pkgDir, func(path string, d iofs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + if path != pkgDir && fs.IsPackage(nte.state.Config.Parse.BuildFileName, path) { + return filepath.SkipDir // We want to stop when we find another package in our dir tree + } + if ignoreDirectories[d.Name()] { + return filepath.SkipDir + } + return nil + } + if !d.Type().IsRegular() { + return nil // Ignore symlinks, which are almost certainly generated sources. + } + dest := filepath.Join(nte.targetDir, path) + if err := fs.EnsureDir(dest); err != nil { + return err + } + return fs.CopyFile(path, dest, 0) + }) + if err != nil { + log.Fatalf("failed to export package %s for %s: %v", pkgName, target.Label, err) + } +} + +func (nte *NoTrimExporter) Subincludes(pkg *core.Package, target *core.BuildTarget) { + subincludes := pkg.AllSubincludes(nte.state.Graph) + for _, subinclude := range subincludes { + nte.Target(nte.state.Graph.TargetOrDie(subinclude)) + } +} + +// AllTargets will export all the targets in the provided package. +func (nte *NoTrimExporter) AllTargets(pkg *core.Package) { + for _, target := range pkg.AllTargets() { + nte.Target(target) + } +} + +// WriteBuildStatements in the NoTrimExporter doesn't require an implementation due to total copy +// of BUILD package. +func (nte *NoTrimExporter) WriteBuildStatements() { + return +} From 03ed361f102d971a7ea81efabba0ceaffb1c65b7 Mon Sep 17 00:00:00 2001 From: duarte Date: Wed, 22 Apr 2026 12:33:17 +0100 Subject: [PATCH 015/118] notrim: full copy BUILD file and visit sources --- src/export/export.go | 49 ++++++++++---------------------------------- 1 file changed, 11 insertions(+), 38 deletions(-) diff --git a/src/export/export.go b/src/export/export.go index d8a1a834b4..7f1a8c5d64 100644 --- a/src/export/export.go +++ b/src/export/export.go @@ -6,7 +6,6 @@ package export import ( "bufio" "io" - iofs "io/fs" "maps" "os" "path/filepath" @@ -379,22 +378,16 @@ func (nte *NoTrimExporter) Target(target *core.BuildTarget) { pkg := nte.state.Graph.PackageOrDie(target.Label) - nte.Package(target) + nte.Package(pkg) nte.Subincludes(pkg, target) nte.AllTargets(pkg) + nte.Sources(target) nte.Dependencies(target) } -var ignoreDirectories = map[string]bool{ - "plz-out": true, - ".git": true, - ".svn": true, - ".hg": true, -} - -// Package exports the package BUILD file containing the given target and all sources. -func (nte *NoTrimExporter) Package(target *core.BuildTarget) { - pkgName := target.Label.PackageName +// Package exports the package BUILD file. +func (nte *NoTrimExporter) Package(pkg *core.Package) { + pkgName := pkg.Name if pkgName == parse.InternalPackageName { return } @@ -403,35 +396,15 @@ func (nte *NoTrimExporter) Package(target *core.BuildTarget) { } nte.exportedPackages[pkgName] = true - pkgDir := filepath.Clean(pkgName) + pkgFilename := pkg.Filename + exportedFilename := filepath.Join(nte.targetDir, pkgFilename) - err := filepath.WalkDir(pkgDir, func(path string, d iofs.DirEntry, err error) error { - if err != nil { - return err - } - if d.IsDir() { - if path != pkgDir && fs.IsPackage(nte.state.Config.Parse.BuildFileName, path) { - return filepath.SkipDir // We want to stop when we find another package in our dir tree - } - if ignoreDirectories[d.Name()] { - return filepath.SkipDir - } - return nil - } - if !d.Type().IsRegular() { - return nil // Ignore symlinks, which are almost certainly generated sources. - } - dest := filepath.Join(nte.targetDir, path) - if err := fs.EnsureDir(dest); err != nil { - return err - } - return fs.CopyFile(path, dest, 0) - }) - if err != nil { - log.Fatalf("failed to export package %s for %s: %v", pkgName, target.Label, err) + if err := fs.CopyFile(pkgFilename, exportedFilename, 0); err != nil { + log.Fatalf("failed to export package %s: %v", pkgName, err) } } +// Subincludes exports the subincluded targets. func (nte *NoTrimExporter) Subincludes(pkg *core.Package, target *core.BuildTarget) { subincludes := pkg.AllSubincludes(nte.state.Graph) for _, subinclude := range subincludes { @@ -447,7 +420,7 @@ func (nte *NoTrimExporter) AllTargets(pkg *core.Package) { } // WriteBuildStatements in the NoTrimExporter doesn't require an implementation due to total copy -// of BUILD package. +// of BUILD file. func (nte *NoTrimExporter) WriteBuildStatements() { return } From 717aba12727b8cded2d0ddf9527bb37613ce6ce3 Mon Sep 17 00:00:00 2001 From: duarte Date: Wed, 22 Apr 2026 13:17:34 +0100 Subject: [PATCH 016/118] double new lines between targets --- src/export/export.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/export/export.go b/src/export/export.go index 7f1a8c5d64..31e8155f9f 100644 --- a/src/export/export.go +++ b/src/export/export.go @@ -300,7 +300,7 @@ func (e *DefaultExporter) writePackageFile(pkg *core.Package, stmts []core.Build } } // Statements - for _, s := range stmts { + for i, s := range stmts { if _, err := fr.Seek(s.StartPos(), io.SeekStart); err != nil { log.Fatalf("failed to seek in BUILD file %s: %v", filename, err) } @@ -312,6 +312,12 @@ func (e *DefaultExporter) writePackageFile(pkg *core.Package, stmts []core.Build if _, err := writer.WriteString("\n"); err != nil { log.Fatalf("failed to add newline to %s: %v", exportedFilename, err) } + + if i < len(stmts) - 1 { // skip for last stmt + if _, err := writer.WriteString("\n"); err != nil { + log.Fatalf("failed to add extra newline to %s: %v", exportedFilename, err) + } + } } if err := writer.Flush(); err != nil { log.Fatalf("failed write exported BUILD file %s: %v", exportedFilename, err) From 94160bfc269bc22c9742e4596c84ef76cd3339a0 Mon Sep 17 00:00:00 2001 From: duarte Date: Fri, 24 Apr 2026 14:24:19 +0100 Subject: [PATCH 017/118] suppress diff output when enforcing repo differences --- test/export/please_export_e2e_test.build_defs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/export/please_export_e2e_test.build_defs b/test/export/please_export_e2e_test.build_defs index 0f240b5d4b..05f5a113ff 100644 --- a/test/export/please_export_e2e_test.build_defs +++ b/test/export/please_export_e2e_test.build_defs @@ -40,7 +40,7 @@ def please_export_e2e_test( if enforce_different_repos: test_cmd += [ # Enforce source_repo and expected_repo differences - f'(diff -rq "$DATA_SOURCE_REPO" "$DATA_EXPECTED_REPO" && \ + f'(diff -rq "$DATA_SOURCE_REPO" "$DATA_EXPECTED_REPO" > /dev/null && \ echo "Source and Expected repos must differ" && exit 1 || true)', ] From 582319e5f4ce7f7c97814367cf77be51f86952c7 Mon Sep 17 00:00:00 2001 From: duarte Date: Fri, 24 Apr 2026 14:38:49 +0100 Subject: [PATCH 018/118] export by filtering the original BUILD file - register subinclude statements in the package metadata - filter subincludes label - export all non build_target related statements --- src/core/package.go | 28 ++- ...{package_metada.go => package_metadata.go} | 50 +++-- src/export/BUILD | 1 + src/export/export.go | 185 ++++++++++-------- src/parse/asp/builtins.go | 1 + src/parse/asp/interpreter.go | 13 +- src/parse/asp/parser.go | 6 +- 7 files changed, 177 insertions(+), 107 deletions(-) rename src/core/{package_metada.go => package_metadata.go} (51%) diff --git a/src/core/package.go b/src/core/package.go index f7f930a497..eca70d4bee 100644 --- a/src/core/package.go +++ b/src/core/package.go @@ -48,10 +48,11 @@ func NewPackage(name string) *Package { // NewPackageSubrepo constructs a new package with the given name and subrepo. func NewPackageSubrepo(name, subrepo string) *Package { return &Package{ - Name: name, - SubrepoName: subrepo, - targets: map[string]*BuildTarget{}, - Outputs: map[string]*BuildTarget{}, + Name: name, + SubrepoName: subrepo, + targets: map[string]*BuildTarget{}, + Outputs: map[string]*BuildTarget{}, + BuildFileMetadata: *newBuildFileMetadata(), } } @@ -254,7 +255,14 @@ func (pkg *Package) RegisterRequiredSubincludes(target *BuildTarget, subincludes pkg.mutex.Lock() defer pkg.mutex.Unlock() - pkg.BuildFileMetadata.RegisterSubinclude(target, subincludes) + pkg.BuildFileMetadata.RegisterRequiredSubinclude(target, subincludes) +} + +// RegisterSubincludeStmt maps the subincludes build statement to the included targets. +func (pkg *Package) RegisterSubincludeStmt(label BuildLabel, stmt *BuildStatement) { + pkg.mutex.Lock() + defer pkg.mutex.Unlock() + pkg.BuildFileMetadata.RegisterSubincludeStmt(label, stmt) } // FindStatement finds the build statement that generated the target. @@ -267,9 +275,15 @@ func (pkg *Package) FindRelatedTargets(stmt *BuildStatement) ([]*BuildTarget, er return pkg.BuildFileMetadata.FindTargets(stmt) } -// FindRequiredSubincludes finds the subincludes target labels required by the given target. +// FindRequiredSubincludes finds the subincluded target labels required by the given target. func (pkg *Package) FindRequiredSubincludes(target *BuildTarget) (BuildLabels, error) { - return pkg.BuildFileMetadata.FindSubincludes(target) + return pkg.BuildFileMetadata.FindRequiredSubincludes(target) +} + +// GetSubincludedLabels returns the labels subincluded by the given statement and true if it +// is a subinclude. +func (pkg *Package) GetSubincludedLabels(stmt *BuildStatement) (BuildLabels, bool) { + return pkg.BuildFileMetadata.GetSubincludedLabels(stmt) } // FindOwningPackages returns build labels corresponding to the packages that own each of the given files. diff --git a/src/core/package_metada.go b/src/core/package_metadata.go similarity index 51% rename from src/core/package_metada.go rename to src/core/package_metadata.go index 8cb4031bfa..57a30b4c64 100644 --- a/src/core/package_metada.go +++ b/src/core/package_metadata.go @@ -2,6 +2,8 @@ package core import ( "fmt" + "io" + "os" "slices" ) @@ -17,33 +19,52 @@ func (bs *BuildStatement) StartPos() int64 { return int64(bs.Start) } -// BuildStatements is a slice of BuildStatement that implements sort.Interface. +func (bs *BuildStatement) Write(from *os.File, to io.Writer) error { + if _, err := from.Seek(bs.StartPos(), io.SeekStart); err != nil { + return err + } + if _, err := io.CopyN(to, from, bs.Len()); err != nil { + return err + } + return nil +} + +// BuildStatements is a slice of StatementWriter that implements sort.Interface. type BuildStatements []BuildStatement func (s BuildStatements) Len() int { return len(s) } func (s BuildStatements) Swap(i, j int) { s[i], s[j] = s[j], s[i] } -func (s BuildStatements) Less(i, j int) bool { return s[i].Start < s[j].Start } +func (s BuildStatements) Less(i, j int) bool { return s[i].StartPos() < s[j].StartPos() } type BuildFileMetadata struct { - StmtToTarget map[BuildStatement][]*BuildTarget + // a list of targets generated from each built statement + StmtToTarget map[BuildStatement][]*BuildTarget + // the subincluded label dependencies per target TargetToSubinclude map[*BuildTarget]BuildLabels - // TODO Untracked stmts - will export every time (e.g. package) + // all the labels included for each subincludes statement + LabelsPerSubincludeStmt map[BuildStatement]BuildLabels } -func (bfm *BuildFileMetadata) RegisterStatementTarget(stmt *BuildStatement, target *BuildTarget) { - if bfm.StmtToTarget == nil { - bfm.StmtToTarget = make(map[BuildStatement][]*BuildTarget) +func newBuildFileMetadata() *BuildFileMetadata { + return &BuildFileMetadata{ + StmtToTarget: map[BuildStatement][]*BuildTarget{}, + TargetToSubinclude: map[*BuildTarget]BuildLabels{}, + LabelsPerSubincludeStmt: map[BuildStatement]BuildLabels{}, } +} + +func (bfm *BuildFileMetadata) RegisterStatementTarget(stmt *BuildStatement, target *BuildTarget) { bfm.StmtToTarget[*stmt] = append(bfm.StmtToTarget[*stmt], target) } -func (bfm *BuildFileMetadata) RegisterSubinclude(target *BuildTarget, subincludes BuildLabels) { - if bfm.TargetToSubinclude == nil { - bfm.TargetToSubinclude = make(map[*BuildTarget]BuildLabels) - } +func (bfm *BuildFileMetadata) RegisterRequiredSubinclude(target *BuildTarget, subincludes BuildLabels) { bfm.TargetToSubinclude[target] = append(bfm.TargetToSubinclude[target], subincludes...) } +func (bfm *BuildFileMetadata) RegisterSubincludeStmt(label BuildLabel, stmt *BuildStatement) { + bfm.LabelsPerSubincludeStmt[*stmt] = append(bfm.LabelsPerSubincludeStmt[*stmt], label) +} + func (bfm *BuildFileMetadata) FindStatement(target *BuildTarget) (*BuildStatement, error) { for stmt, targets := range bfm.StmtToTarget { if slices.Contains(targets, target) { @@ -61,10 +82,15 @@ func (bfm *BuildFileMetadata) FindTargets(stmt *BuildStatement) ([]*BuildTarget, return targets, nil } -func (bfm *BuildFileMetadata) FindSubincludes(target *BuildTarget) (BuildLabels, error) { +func (bfm *BuildFileMetadata) FindRequiredSubincludes(target *BuildTarget) (BuildLabels, error) { subincludes, ok := bfm.TargetToSubinclude[target] if !ok { return nil, fmt.Errorf("Subincludes not found for target %v.", target) } return subincludes, nil } + +func (bfm *BuildFileMetadata) GetSubincludedLabels(stmt *BuildStatement) (BuildLabels, bool) { + v, ok := bfm.LabelsPerSubincludeStmt[*stmt] + return v, ok +} diff --git a/src/export/BUILD b/src/export/BUILD index 1895be6eff..bf46b3a36c 100644 --- a/src/export/BUILD +++ b/src/export/BUILD @@ -9,6 +9,7 @@ go_library( "//src/core", "//src/fs", "//src/parse", + "//src/parse/asp", ], ) diff --git a/src/export/export.go b/src/export/export.go index 31e8155f9f..ab3251fcea 100644 --- a/src/export/export.go +++ b/src/export/export.go @@ -4,8 +4,7 @@ package export import ( - "bufio" - "io" + "bytes" "maps" "os" "path/filepath" @@ -18,6 +17,7 @@ import ( "github.com/thought-machine/please/src/core" "github.com/thought-machine/please/src/fs" "github.com/thought-machine/please/src/parse" + "github.com/thought-machine/please/src/parse/asp" ) var log = logging.Log @@ -27,7 +27,7 @@ type Exporter interface { Preloaded() Targets(core.BuildLabels) Target(target *core.BuildTarget) - WriteBuildStatements() + WritePackageFiles() } func Repo(state *core.BuildState, dir string, noTrim bool, targets []core.BuildLabel) { @@ -41,7 +41,7 @@ func Repo(state *core.BuildState, dir string, noTrim bool, targets []core.BuildL e.PlzConfig() e.Preloaded() e.Targets(targets) - e.WriteBuildStatements() + e.WritePackageFiles() } // Outputs exports the outputs of a target. @@ -65,7 +65,7 @@ func newExporter(state *core.BuildState, dir string, noTrim bool) Exporter { base := baseExporter{ state: state, targetDir: dir, - exportedTargets: map[core.BuildLabel]bool{}, + exportedTargets: map[*core.Package]map[core.BuildLabel]bool{}, } if noTrim { @@ -78,7 +78,6 @@ func newExporter(state *core.BuildState, dir string, noTrim bool) Exporter { } else { exporter := &DefaultExporter{ baseExporter: base, - selectedStatements: map[*core.Package]map[core.BuildStatement]bool{}, requiredSubincludes: map[*core.Package]map[core.BuildLabel]bool{}, preloadedSubincludes: map[core.BuildLabel]bool{}, } @@ -93,7 +92,7 @@ type baseExporter struct { state *core.BuildState targetDir string - exportedTargets map[core.BuildLabel]bool + exportedTargets map[*core.Package]map[core.BuildLabel]bool impl Exporter } @@ -145,10 +144,20 @@ func (be *baseExporter) Sources(target *core.BuildTarget) { } } +func (be *baseExporter) checkFirstExport(pkg *core.Package, target *core.BuildTarget) bool { + if _, ok := be.exportedTargets[pkg]; !ok { + be.exportedTargets[pkg] = map[core.BuildLabel]bool{} + } + if be.exportedTargets[pkg][target.Label] { + return false + } + be.exportedTargets[pkg][target.Label] = true + return true +} + // DefaultExporter implements an exporter that trims packages to reach a minimal exported repo. type DefaultExporter struct { baseExporter - selectedStatements map[*core.Package]map[core.BuildStatement]bool requiredSubincludes map[*core.Package]map[core.BuildLabel]bool preloadedSubincludes map[core.BuildLabel]bool } @@ -175,10 +184,10 @@ func (e *DefaultExporter) Preloaded() { // Target exports an individual target. This implementation will attempt to export a minimal repo // with only the required targets and statements. func (e *DefaultExporter) Target(target *core.BuildTarget) { - if e.exportedTargets[target.Label] { + pkg := e.state.Graph.PackageOrDie(target.Label) + if e.checkFirstExport(pkg, target) == false { return } - e.exportedTargets[target.Label] = true // We want to export the package that made this subrepo available, but we still need to walk the target deps // as it may depend on other subrepos or first party targets @@ -188,8 +197,6 @@ func (e *DefaultExporter) Target(target *core.BuildTarget) { return } - pkg := e.state.Graph.PackageOrDie(target.Label) - e.Subincludes(pkg, target) e.BuildStatements(pkg, target) e.Sources(target) @@ -229,21 +236,11 @@ func (e *DefaultExporter) BuildStatements(pkg *core.Package, target *core.BuildT return } - if _, ok := e.selectedStatements[pkg]; !ok { - e.selectedStatements[pkg] = map[core.BuildStatement]bool{} - } - stmt, err := pkg.FindStatement(target) if err != nil { log.Fatalf("Failed to find statement in %s: %w", pkg.Name, err) } - // check if visited before - if e.selectedStatements[pkg][*stmt] == true { - return - } - e.selectedStatements[pkg][*stmt] = true - relatedTargets, err := pkg.FindRelatedTargets(stmt) if err != nil { log.Fatalf("Failed to lookup related targets for package %s: %w", pkg.Name, err) @@ -254,91 +251,119 @@ func (e *DefaultExporter) BuildStatements(pkg *core.Package, target *core.BuildT } } -// WriteBuildStatements writes the BUILD file statements to the export directory. -func (e *DefaultExporter) WriteBuildStatements() { - log.Warningf("Selected Statements: %v", e.selectedStatements) +// WritePackageFiles writes the trimmed BUILD files to the export directory. +func (e *DefaultExporter) WritePackageFiles() { + for pkg, labels := range e.exportedTargets { + log.Warningf("On package %v Selected targets: %v", pkg, slices.Collect(maps.Keys(labels))) - for pkg, stmtMap := range e.selectedStatements { - stmts := slices.Collect(maps.Keys(stmtMap)) - // Sort statements by position to keep them in order - sort.Sort(core.BuildStatements(stmts)) + // filter + filteredBytes, err := e.FilterPackageFile(pkg) + if err != nil { + log.Fatalf("Failed to filter the build statements of package %s: %v", pkg.Label(), err) + } - e.writePackageFile(pkg, stmts) + // format + buildParser, err := build.ParseBuild(pkg.Filename, filteredBytes) + formatedBytes := build.Format(buildParser) + + // write + file := e.OpenExportedPackageFile(pkg) + defer file.Close() + if _, err := file.Write(formatedBytes); err != nil { + log.Fatalf("Failed to write to exported BUILD file %s: %v", file.Name(), err) + } } } -// writePackageFile writes the selected statements to the please build file in the exported directory. -func (e *DefaultExporter) writePackageFile(pkg *core.Package, stmts []core.BuildStatement) { +func (e *DefaultExporter) OpenExportedPackageFile(pkg *core.Package) *os.File { filename := pkg.Filename exportedFilename := filepath.Join(e.targetDir, filename) - - log.Warningf("Writing file: %s", filename) - - fr, err := os.Open(filename) + f, err := fs.OpenDirFile(exportedFilename, os.O_CREATE|os.O_WRONLY, 0664) if err != nil { - log.Fatalf("failed to open file original BUILD file: %v", err) - return + log.Fatalf("failed to create and open exported BUILD file for %s: %v", exportedFilename, err) } - defer fr.Close() + return f +} - frStat, err := fr.Stat() +// FilterPackageFile filters the statements to be written to the exported BUILD file. +func (e *DefaultExporter) FilterPackageFile(pkg *core.Package) ([]byte, error) { + p := asp.NewParserOnly() + parsedStmts, err := p.ParseFileOnly(pkg.Filename) if err != nil { - log.Fatalf("failed to get original BUILD file status: %v", err) + log.Fatalf("failed to parse original BUILD file: %v", err) } - fw, err := fs.OpenDirFile(exportedFilename, os.O_CREATE|os.O_WRONLY, frStat.Mode()) + original, err := os.ReadFile(pkg.Filename) if err != nil { - log.Fatalf("failed to create and open exported BUILD file for %s: %v", exportedFilename, err) + log.Fatalf("failed to open original BUILD file: %v", err) } - defer fw.Close() - writer := bufio.NewWriter(fw) - // Subinclude - if subinclude := e.makeSubincludesStatement(pkg); subinclude != "" { - if _, err := writer.WriteString(subinclude + "\n\n"); err != nil { - log.Fatalf("failed to add subincludes to %s: %v", exportedFilename, err) - } - } - // Statements - for i, s := range stmts { - if _, err := fr.Seek(s.StartPos(), io.SeekStart); err != nil { - log.Fatalf("failed to seek in BUILD file %s: %v", filename, err) - } + cursor := 0 + var buffer bytes.Buffer + for _, stmt := range parsedStmts { + bStmt := asp.NewBuildStatement(stmt) - if _, err := io.CopyN(writer, fr, s.Len()); err != nil { - log.Fatalf("failed to copy statement from %s to %s: %v", filename, exportedFilename, err) + if cursor < bStmt.Start { + if _, err := buffer.Write(original[cursor:bStmt.Start]); err != nil { + return nil, err + } + cursor = bStmt.Start } - if _, err := writer.WriteString("\n"); err != nil { - log.Fatalf("failed to add newline to %s: %v", exportedFilename, err) + // Write filtered subincludes + if stmtLabels, ok := pkg.GetSubincludedLabels(bStmt); ok { + + subStmt := e.makeSubincludeStatement(stmtLabels, e.requiredSubincludes[pkg]) + buffer.Write([]byte(subStmt)) + cursor = bStmt.End + continue } - if i < len(stmts) - 1 { // skip for last stmt - if _, err := writer.WriteString("\n"); err != nil { - log.Fatalf("failed to add extra newline to %s: %v", exportedFilename, err) + // Don't write statements that generate targets we are not interested about + if targets, err := pkg.FindRelatedTargets(bStmt); err == nil { + needed := false + for _, target := range targets { + if e.exportedTargets[pkg][target.Label] { + needed = true + } + } + if needed == false { + // don't write + cursor = bStmt.End + continue } } + + // Write every other statement + if buffer.Write(original[bStmt.Start:bStmt.End]); err != nil { + return nil, err + } + cursor = bStmt.End } - if err := writer.Flush(); err != nil { - log.Fatalf("failed write exported BUILD file %s: %v", exportedFilename, err) + + // Write the rest of the original file (non build targets) + if buffer.Write(original[cursor:]); err != nil { + return nil, err } + + return buffer.Bytes(), nil } -// makeSubincludesStatement generates a single subinclude statement (as string) with the required -// targets for this package. -func (e *DefaultExporter) makeSubincludesStatement(pkg *core.Package) string { - subincludesMap, ok := e.requiredSubincludes[pkg] - if !ok || len(subincludesMap) == 0 { - return "" +// makeSubincludeStatement generates a single subinclude statement (as string) with the argument labels. +func (e *DefaultExporter) makeSubincludeStatement(available core.BuildLabels, required map[core.BuildLabel]bool) string { + // make subincludes that contains a subset of the labels defined in the statement + var filteredLabels core.BuildLabels + for _, label := range available { + if required[label] { + filteredLabels = append(filteredLabels, label) + } } - - labels := slices.Collect(maps.Keys(subincludesMap)) - sort.Sort(core.BuildLabels(labels)) + sort.Sort(filteredLabels) call := &build.CallExpr{ X: &build.Ident{Name: "subinclude"}, } - for _, label := range labels { + for _, label := range filteredLabels { call.List = append(call.List, &build.StringExpr{Value: label.String()}) } @@ -369,10 +394,10 @@ func (nte *NoTrimExporter) Preloaded() { // Target exports an individual target. This implementation won't attempted any trimming, exporting // all targets and statements defined in the package. func (nte *NoTrimExporter) Target(target *core.BuildTarget) { - if nte.exportedTargets[target.Label] { + pkg := nte.state.Graph.PackageOrDie(target.Label) + if nte.checkFirstExport(pkg, target) == false { return } - nte.exportedTargets[target.Label] = true // We want to export the package that made this subrepo available, but we still need to walk the target deps // as it may depend on other subrepos or first party targets @@ -382,8 +407,6 @@ func (nte *NoTrimExporter) Target(target *core.BuildTarget) { return } - pkg := nte.state.Graph.PackageOrDie(target.Label) - nte.Package(pkg) nte.Subincludes(pkg, target) nte.AllTargets(pkg) @@ -425,8 +448,8 @@ func (nte *NoTrimExporter) AllTargets(pkg *core.Package) { } } -// WriteBuildStatements in the NoTrimExporter doesn't require an implementation due to total copy +// WritePackageFiles in the NoTrimExporter doesn't require an implementation due to total copy // of BUILD file. -func (nte *NoTrimExporter) WriteBuildStatements() { +func (nte *NoTrimExporter) WritePackageFiles() { return } diff --git a/src/parse/asp/builtins.go b/src/parse/asp/builtins.go index 8542d4cadc..c097a87cf7 100644 --- a/src/parse/asp/builtins.go +++ b/src/parse/asp/builtins.go @@ -419,6 +419,7 @@ func subincludeTarget(s *scope, l core.BuildLabel) *core.BuildTarget { t = s.WaitForSubincludedTarget(l, pkgLabel) if s.pkg != nil { s.pkg.RegisterSubinclude(l) + s.pkg.RegisterSubincludeStmt(l, s.CurrentBuildStatement()) } else if s.subincludeLabel != nil { // If this is nil, that indicates a preloadedSubinclude s.state.Graph.RegisterTransitiveSubinclude(*s.subincludeLabel, l) } diff --git a/src/parse/asp/interpreter.go b/src/parse/asp/interpreter.go index 05fafb3f21..b2f8f18832 100644 --- a/src/parse/asp/interpreter.go +++ b/src/parse/asp/interpreter.go @@ -1090,10 +1090,7 @@ func (s *scope) CurrentBuildStatement() *core.BuildStatement { } } s.NAssert(stmtScope.cursor == nil, "Cursor is not pointing to a statement") - return &core.BuildStatement{ - Start: int(stmtScope.cursor.Pos), - End: int(stmtScope.cursor.EndPos), - } + return NewBuildStatement(stmtScope.cursor) } // ActiveSubincludes traces the call stack and scopes to find subincludes that provided the @@ -1122,3 +1119,11 @@ func (s *scope) pkgFilename() string { } return "" } + +// NewBuildStatement creates a new core.BuildStatment from an asp.statment. +func NewBuildStatement(stmt *Statement) *core.BuildStatement { + return &core.BuildStatement{ + Start: int(stmt.Pos), + End: int(stmt.EndPos), + } +} diff --git a/src/parse/asp/parser.go b/src/parse/asp/parser.go index f67a8605ae..74a96f0475 100644 --- a/src/parse/asp/parser.go +++ b/src/parse/asp/parser.go @@ -35,14 +35,14 @@ type Parser struct { // NewParser creates a new parser instance. One is normally sufficient for a process lifetime. func NewParser(state *core.BuildState) *Parser { - p := newParser() + p := NewParserOnly() p.interpreter = newInterpreter(state, p) p.limiter = p.interpreter.limiter return p } -// newParser creates just the parser with no interpreter. -func newParser() *Parser { +// NewParserOnly creates just the parser with no interpreter. +func NewParserOnly() *Parser { return &Parser{ builtins: map[string][]byte{}, limiter: make(semaphore, 10), From e6a529368ee76874216bdf192862d7a5dafdaaf4 Mon Sep 17 00:00:00 2001 From: duarte Date: Fri, 24 Apr 2026 16:11:56 +0100 Subject: [PATCH 019/118] subincludes use label short string --- src/export/export.go | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/src/export/export.go b/src/export/export.go index ab3251fcea..337c7f2c4a 100644 --- a/src/export/export.go +++ b/src/export/export.go @@ -303,6 +303,7 @@ func (e *DefaultExporter) FilterPackageFile(pkg *core.Package) ([]byte, error) { for _, stmt := range parsedStmts { bStmt := asp.NewBuildStatement(stmt) + // Write content that's between stmts (e.g. comments) if cursor < bStmt.Start { if _, err := buffer.Write(original[cursor:bStmt.Start]); err != nil { return nil, err @@ -312,8 +313,7 @@ func (e *DefaultExporter) FilterPackageFile(pkg *core.Package) ([]byte, error) { // Write filtered subincludes if stmtLabels, ok := pkg.GetSubincludedLabels(bStmt); ok { - - subStmt := e.makeSubincludeStatement(stmtLabels, e.requiredSubincludes[pkg]) + subStmt := e.minimalSubincludeStatement(pkg, stmtLabels) buffer.Write([]byte(subStmt)) cursor = bStmt.End continue @@ -321,14 +321,10 @@ func (e *DefaultExporter) FilterPackageFile(pkg *core.Package) ([]byte, error) { // Don't write statements that generate targets we are not interested about if targets, err := pkg.FindRelatedTargets(bStmt); err == nil { - needed := false - for _, target := range targets { - if e.exportedTargets[pkg][target.Label] { - needed = true - } - } - if needed == false { - // don't write + needed := slices.ContainsFunc(targets, func(target *core.BuildTarget) bool { + return e.exportedTargets[pkg][target.Label] + }) + if !needed { cursor = bStmt.End continue } @@ -349,9 +345,9 @@ func (e *DefaultExporter) FilterPackageFile(pkg *core.Package) ([]byte, error) { return buffer.Bytes(), nil } -// makeSubincludeStatement generates a single subinclude statement (as string) with the argument labels. -func (e *DefaultExporter) makeSubincludeStatement(available core.BuildLabels, required map[core.BuildLabel]bool) string { - // make subincludes that contains a subset of the labels defined in the statement +// minimalSubincludeStatement generates a subinclude statement containing only the required labels. +func (e *DefaultExporter) minimalSubincludeStatement(pkg *core.Package, available core.BuildLabels) string { + required := e.requiredSubincludes[pkg] var filteredLabels core.BuildLabels for _, label := range available { if required[label] { @@ -364,7 +360,7 @@ func (e *DefaultExporter) makeSubincludeStatement(available core.BuildLabels, re X: &build.Ident{Name: "subinclude"}, } for _, label := range filteredLabels { - call.List = append(call.List, &build.StringExpr{Value: label.String()}) + call.List = append(call.List, &build.StringExpr{Value: label.ShortString(pkg.Label())}) } return build.FormatString(call) From ea329c32bcdfe7e092d07512a25c83f5743c53a0 Mon Sep 17 00:00:00 2001 From: duarte Date: Fri, 24 Apr 2026 17:50:04 +0100 Subject: [PATCH 020/118] Simplify package filtering method into a more explicit "switch" case --- src/export/export.go | 48 ++++++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/src/export/export.go b/src/export/export.go index 337c7f2c4a..a06841149e 100644 --- a/src/export/export.go +++ b/src/export/export.go @@ -5,7 +5,6 @@ package export import ( "bytes" - "maps" "os" "path/filepath" "slices" @@ -225,8 +224,6 @@ func (e *DefaultExporter) Subincludes(pkg *core.Package, target *core.BuildTarge e.Target(e.state.Graph.TargetOrDie(subinclude)) } - - log.Warningf("Parse Metadata Subincludes: %v", pkg.BuildFileMetadata.TargetToSubinclude) } // BuildStatements exports BUILD statements that generate the build target. @@ -253,8 +250,7 @@ func (e *DefaultExporter) BuildStatements(pkg *core.Package, target *core.BuildT // WritePackageFiles writes the trimmed BUILD files to the export directory. func (e *DefaultExporter) WritePackageFiles() { - for pkg, labels := range e.exportedTargets { - log.Warningf("On package %v Selected targets: %v", pkg, slices.Collect(maps.Keys(labels))) + for pkg := range e.exportedTargets { // filter filteredBytes, err := e.FilterPackageFile(pkg) @@ -264,12 +260,12 @@ func (e *DefaultExporter) WritePackageFiles() { // format buildParser, err := build.ParseBuild(pkg.Filename, filteredBytes) - formatedBytes := build.Format(buildParser) + formattedBytes := build.Format(buildParser) // write file := e.OpenExportedPackageFile(pkg) defer file.Close() - if _, err := file.Write(formatedBytes); err != nil { + if _, err := file.Write(formattedBytes); err != nil { log.Fatalf("Failed to write to exported BUILD file %s: %v", file.Name(), err) } } @@ -311,29 +307,20 @@ func (e *DefaultExporter) FilterPackageFile(pkg *core.Package) ([]byte, error) { cursor = bStmt.Start } - // Write filtered subincludes if stmtLabels, ok := pkg.GetSubincludedLabels(bStmt); ok { + // Write filtered subincludes subStmt := e.minimalSubincludeStatement(pkg, stmtLabels) buffer.Write([]byte(subStmt)) - cursor = bStmt.End - continue - } - - // Don't write statements that generate targets we are not interested about - if targets, err := pkg.FindRelatedTargets(bStmt); err == nil { - needed := slices.ContainsFunc(targets, func(target *core.BuildTarget) bool { - return e.exportedTargets[pkg][target.Label] - }) - if !needed { - cursor = bStmt.End - continue + } else if !e.isRequiredStatement(pkg, bStmt) { + // Don't write statements that generate targets we are not interested about + // skip + } else { + // Write every other statement + if buffer.Write(original[bStmt.Start:bStmt.End]); err != nil { + return nil, err } } - // Write every other statement - if buffer.Write(original[bStmt.Start:bStmt.End]); err != nil { - return nil, err - } cursor = bStmt.End } @@ -345,6 +332,19 @@ func (e *DefaultExporter) FilterPackageFile(pkg *core.Package) ([]byte, error) { return buffer.Bytes(), nil } +// isRequiredStatement evaluates if the current build statement is required by the export. +func (e *DefaultExporter) isRequiredStatement(pkg *core.Package, stmt *core.BuildStatement) bool { + targets, err := pkg.FindRelatedTargets(stmt) + if err != nil { + return false + } + + required := slices.ContainsFunc(targets, func(target *core.BuildTarget) bool { + return e.exportedTargets[pkg][target.Label] + }) + return required +} + // minimalSubincludeStatement generates a subinclude statement containing only the required labels. func (e *DefaultExporter) minimalSubincludeStatement(pkg *core.Package, available core.BuildLabels) string { required := e.requiredSubincludes[pkg] From 1a47c228b6e3a602625af9b7d6a662527395a3ea Mon Sep 17 00:00:00 2001 From: duarte Date: Fri, 24 Apr 2026 18:13:24 +0100 Subject: [PATCH 021/118] skip subrepos and internal packages when writing package file --- src/export/export.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/export/export.go b/src/export/export.go index a06841149e..2907e73820 100644 --- a/src/export/export.go +++ b/src/export/export.go @@ -251,6 +251,9 @@ func (e *DefaultExporter) BuildStatements(pkg *core.Package, target *core.BuildT // WritePackageFiles writes the trimmed BUILD files to the export directory. func (e *DefaultExporter) WritePackageFiles() { for pkg := range e.exportedTargets { + if pkg.Subrepo != nil || pkg.Name == parse.InternalPackageName { + continue // Skip subrepos and internal packages + } // filter filteredBytes, err := e.FilterPackageFile(pkg) From a35119a912ec9b49ab8f09b7f5331350879abbef Mon Sep 17 00:00:00 2001 From: duarte Date: Fri, 24 Apr 2026 20:12:10 +0100 Subject: [PATCH 022/118] fix reusing err var resulted in failed filtering --- src/export/export.go | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/export/export.go b/src/export/export.go index 2907e73820..682fd285d1 100644 --- a/src/export/export.go +++ b/src/export/export.go @@ -279,7 +279,7 @@ func (e *DefaultExporter) OpenExportedPackageFile(pkg *core.Package) *os.File { exportedFilename := filepath.Join(e.targetDir, filename) f, err := fs.OpenDirFile(exportedFilename, os.O_CREATE|os.O_WRONLY, 0664) if err != nil { - log.Fatalf("failed to create and open exported BUILD file for %s: %v", exportedFilename, err) + log.Fatalf("Failed to create and open exported BUILD file for %s: %v", exportedFilename, err) } return f } @@ -289,12 +289,12 @@ func (e *DefaultExporter) FilterPackageFile(pkg *core.Package) ([]byte, error) { p := asp.NewParserOnly() parsedStmts, err := p.ParseFileOnly(pkg.Filename) if err != nil { - log.Fatalf("failed to parse original BUILD file: %v", err) + log.Fatalf("Failed to parse original BUILD file: %v", err) } original, err := os.ReadFile(pkg.Filename) if err != nil { - log.Fatalf("failed to open original BUILD file: %v", err) + log.Fatalf("Failed to open original BUILD file: %v", err) } cursor := 0 @@ -302,6 +302,7 @@ func (e *DefaultExporter) FilterPackageFile(pkg *core.Package) ([]byte, error) { for _, stmt := range parsedStmts { bStmt := asp.NewBuildStatement(stmt) + log.Debugf("Evaluating statement %s", original[bStmt.Start:bStmt.End]) // Write content that's between stmts (e.g. comments) if cursor < bStmt.Start { if _, err := buffer.Write(original[cursor:bStmt.Start]); err != nil { @@ -314,14 +315,17 @@ func (e *DefaultExporter) FilterPackageFile(pkg *core.Package) ([]byte, error) { // Write filtered subincludes subStmt := e.minimalSubincludeStatement(pkg, stmtLabels) buffer.Write([]byte(subStmt)) - } else if !e.isRequiredStatement(pkg, bStmt) { + log.Debugf("Decision: %s", subStmt) + } else if required, err := e.isRequiredStatement(pkg, bStmt); err == nil && !required { // Don't write statements that generate targets we are not interested about + log.Debugf("Decision: ") // skip } else { // Write every other statement - if buffer.Write(original[bStmt.Start:bStmt.End]); err != nil { + if _, err := buffer.Write(original[bStmt.Start:bStmt.End]); err != nil { return nil, err } + log.Debugf("Decision: ") } cursor = bStmt.End @@ -336,16 +340,16 @@ func (e *DefaultExporter) FilterPackageFile(pkg *core.Package) ([]byte, error) { } // isRequiredStatement evaluates if the current build statement is required by the export. -func (e *DefaultExporter) isRequiredStatement(pkg *core.Package, stmt *core.BuildStatement) bool { +func (e *DefaultExporter) isRequiredStatement(pkg *core.Package, stmt *core.BuildStatement) (bool, error) { targets, err := pkg.FindRelatedTargets(stmt) if err != nil { - return false + return false, err } required := slices.ContainsFunc(targets, func(target *core.BuildTarget) bool { return e.exportedTargets[pkg][target.Label] }) - return required + return required, nil } // minimalSubincludeStatement generates a subinclude statement containing only the required labels. From 9c1522c42937f69449b3710e520b44d39845d39d Mon Sep 17 00:00:00 2001 From: duarte Date: Sun, 26 Apr 2026 07:59:20 +0100 Subject: [PATCH 023/118] doc strings for package metadata --- src/core/package_metadata.go | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/src/core/package_metadata.go b/src/core/package_metadata.go index 57a30b4c64..3c2361406b 100644 --- a/src/core/package_metadata.go +++ b/src/core/package_metadata.go @@ -2,40 +2,32 @@ package core import ( "fmt" - "io" - "os" "slices" ) +// BuildStatement represents the start and end byte positions of a parsed statement in a BUILD file. type BuildStatement struct { Start, End int } +// Len returns the byte length of the build statement. func (bs *BuildStatement) Len() int64 { return int64(bs.End - bs.Start) } +// StartPos returns the starting byte position of the build statement. func (bs *BuildStatement) StartPos() int64 { return int64(bs.Start) } -func (bs *BuildStatement) Write(from *os.File, to io.Writer) error { - if _, err := from.Seek(bs.StartPos(), io.SeekStart); err != nil { - return err - } - if _, err := io.CopyN(to, from, bs.Len()); err != nil { - return err - } - return nil -} - -// BuildStatements is a slice of StatementWriter that implements sort.Interface. +// BuildStatements is a slice of BuildStatement that implements sort.Interface. type BuildStatements []BuildStatement func (s BuildStatements) Len() int { return len(s) } func (s BuildStatements) Swap(i, j int) { s[i], s[j] = s[j], s[i] } func (s BuildStatements) Less(i, j int) bool { return s[i].StartPos() < s[j].StartPos() } +// BuildFileMetadata stores metadata about parsed BUILD files, mapping statements and subincludes to their respective targets. type BuildFileMetadata struct { // a list of targets generated from each built statement StmtToTarget map[BuildStatement][]*BuildTarget @@ -53,18 +45,22 @@ func newBuildFileMetadata() *BuildFileMetadata { } } +// RegisterStatementTarget maps a build statement to a target it generated. func (bfm *BuildFileMetadata) RegisterStatementTarget(stmt *BuildStatement, target *BuildTarget) { bfm.StmtToTarget[*stmt] = append(bfm.StmtToTarget[*stmt], target) } +// RegisterRequiredSubinclude maps a target to the subincludes required to build it. func (bfm *BuildFileMetadata) RegisterRequiredSubinclude(target *BuildTarget, subincludes BuildLabels) { bfm.TargetToSubinclude[target] = append(bfm.TargetToSubinclude[target], subincludes...) } +// RegisterSubincludeStmt maps a subinclude statement to a label it includes. func (bfm *BuildFileMetadata) RegisterSubincludeStmt(label BuildLabel, stmt *BuildStatement) { bfm.LabelsPerSubincludeStmt[*stmt] = append(bfm.LabelsPerSubincludeStmt[*stmt], label) } +// FindStatement returns the build statement that generated the given target. func (bfm *BuildFileMetadata) FindStatement(target *BuildTarget) (*BuildStatement, error) { for stmt, targets := range bfm.StmtToTarget { if slices.Contains(targets, target) { @@ -74,6 +70,7 @@ func (bfm *BuildFileMetadata) FindStatement(target *BuildTarget) (*BuildStatemen return nil, fmt.Errorf("Target %s not found in statement metadata.", target.String()) } +// FindTargets returns all targets generated by the given build statement. func (bfm *BuildFileMetadata) FindTargets(stmt *BuildStatement) ([]*BuildTarget, error) { targets, ok := bfm.StmtToTarget[*stmt] if !ok { @@ -82,6 +79,7 @@ func (bfm *BuildFileMetadata) FindTargets(stmt *BuildStatement) ([]*BuildTarget, return targets, nil } +// FindRequiredSubincludes returns all subinclude labels required by the given target. func (bfm *BuildFileMetadata) FindRequiredSubincludes(target *BuildTarget) (BuildLabels, error) { subincludes, ok := bfm.TargetToSubinclude[target] if !ok { @@ -90,6 +88,7 @@ func (bfm *BuildFileMetadata) FindRequiredSubincludes(target *BuildTarget) (Buil return subincludes, nil } +// GetSubincludedLabels returns the labels included by a given subinclude statement. func (bfm *BuildFileMetadata) GetSubincludedLabels(stmt *BuildStatement) (BuildLabels, bool) { v, ok := bfm.LabelsPerSubincludeStmt[*stmt] return v, ok From 9395a591eebe72bfcc2fb32a91c81575603fd718 Mon Sep 17 00:00:00 2001 From: duarte Date: Sun, 26 Apr 2026 08:23:17 +0100 Subject: [PATCH 024/118] export missing doc strings --- src/export/export.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/export/export.go b/src/export/export.go index 682fd285d1..73019f7233 100644 --- a/src/export/export.go +++ b/src/export/export.go @@ -29,8 +29,9 @@ type Exporter interface { WritePackageFiles() } +// Repo export a new please repo including the targets and dependencies requested. func Repo(state *core.BuildState, dir string, noTrim bool, targets []core.BuildLabel) { - e := newExporter(state, dir, noTrim) + e := NewExporter(state, dir, noTrim) // ensure output dir if err := os.MkdirAll(dir, fs.DirPermissions); err != nil { @@ -60,7 +61,8 @@ func Outputs(state *core.BuildState, dir string, targets []core.BuildLabel) { } } -func newExporter(state *core.BuildState, dir string, noTrim bool) Exporter { +// NewExporter creates a new exporter of a specific type based on the arguments. +func NewExporter(state *core.BuildState, dir string, noTrim bool) Exporter { base := baseExporter{ state: state, targetDir: dir, @@ -143,6 +145,7 @@ func (be *baseExporter) Sources(target *core.BuildTarget) { } } +// checkFirstExport is a helper to ensure we only visit the same target once. func (be *baseExporter) checkFirstExport(pkg *core.Package, target *core.BuildTarget) bool { if _, ok := be.exportedTargets[pkg]; !ok { be.exportedTargets[pkg] = map[core.BuildLabel]bool{} @@ -274,6 +277,7 @@ func (e *DefaultExporter) WritePackageFiles() { } } +// OpenExportedPackageFile creates a new package (BUILD) file in the exported dir. func (e *DefaultExporter) OpenExportedPackageFile(pkg *core.Package) *os.File { filename := pkg.Filename exportedFilename := filepath.Join(e.targetDir, filename) From 3c3d3d0ef3f1a270b5e2097f137f511c6b0221f0 Mon Sep 17 00:00:00 2001 From: duarte Date: Wed, 29 Apr 2026 13:28:01 +0100 Subject: [PATCH 025/118] test: named go_repo and change testify for slimmer UUID --- .../test_go_bin/expected_repo/BUILD_FILE | 2 +- .../expected_repo/third_party/go/BUILD_FILE | 2 ++ .../export/test_go_bin/source_repo/BUILD_FILE | 4 ++-- test/export/test_go_bin/source_repo/dummy.go | 2 +- .../source_repo/third_party/go/BUILD_FILE | 20 +++++-------------- 5 files changed, 11 insertions(+), 19 deletions(-) diff --git a/test/export/test_go_bin/expected_repo/BUILD_FILE b/test/export/test_go_bin/expected_repo/BUILD_FILE index d0d1a145c0..015bd671a5 100644 --- a/test/export/test_go_bin/expected_repo/BUILD_FILE +++ b/test/export/test_go_bin/expected_repo/BUILD_FILE @@ -4,6 +4,6 @@ go_binary( name = "bin_go_dep", srcs = ["main.go"], deps = [ - "///third_party/go/github.com_google_go-cmp//cmp", + "//third_party/go:cmp", ], ) diff --git a/test/export/test_go_bin/expected_repo/third_party/go/BUILD_FILE b/test/export/test_go_bin/expected_repo/third_party/go/BUILD_FILE index e436be33ec..77ef426dd5 100644 --- a/test/export/test_go_bin/expected_repo/third_party/go/BUILD_FILE +++ b/test/export/test_go_bin/expected_repo/third_party/go/BUILD_FILE @@ -20,6 +20,8 @@ go_stdlib( ) go_repo( + name = "cmp", + install = ["..."], module = "github.com/google/go-cmp", version = "v0.5.6", ) diff --git a/test/export/test_go_bin/source_repo/BUILD_FILE b/test/export/test_go_bin/source_repo/BUILD_FILE index d37abc25b4..79cce0e24e 100644 --- a/test/export/test_go_bin/source_repo/BUILD_FILE +++ b/test/export/test_go_bin/source_repo/BUILD_FILE @@ -4,7 +4,7 @@ go_binary( name = "bin_go_dep", srcs = ["main.go"], deps = [ - "///third_party/go/github.com_google_go-cmp//cmp", + "//third_party/go:cmp", ], ) @@ -12,6 +12,6 @@ go_binary( name = "dummy", srcs = ["dummy.go"], deps = [ - "///third_party/go/github.com_stretchr_testify//:testify", + "//third_party/go:uuid", ], ) diff --git a/test/export/test_go_bin/source_repo/dummy.go b/test/export/test_go_bin/source_repo/dummy.go index 3c49d61525..403ada14aa 100644 --- a/test/export/test_go_bin/source_repo/dummy.go +++ b/test/export/test_go_bin/source_repo/dummy.go @@ -3,7 +3,7 @@ package dummy import ( "fmt" - "github.com/stretchr/testify/" + "github.com/google/uuid" ) func main() { diff --git a/test/export/test_go_bin/source_repo/third_party/go/BUILD_FILE b/test/export/test_go_bin/source_repo/third_party/go/BUILD_FILE index 20d92fd1ec..bc30a8e53a 100644 --- a/test/export/test_go_bin/source_repo/third_party/go/BUILD_FILE +++ b/test/export/test_go_bin/source_repo/third_party/go/BUILD_FILE @@ -20,25 +20,15 @@ go_stdlib( ) go_repo( + name = "cmp", + install = ["..."], module = "github.com/google/go-cmp", version = "v0.5.6", ) go_repo( # Dummy, unused target - name = "testify", - install = ["..."], - licences = ["MIT"], - module = "github.com/stretchr/testify", - version = "v1.7.0", - deps = [":yaml.v3"], # test we can depend on go_modules -) - -go_module( - # Dummy, unused target - name = "yaml.v3", - licences = ["MIT"], - module = "gopkg.in/yaml.v3", - version = "v3.0.0-20210107192922-496545a6307b", - visibility = ["PUBLIC"], + name = "uuid", + module = "github.com/google/uuid", + version = "v1.6.0", ) From 13e95cb15c19b8cd7b49be86f5ce535646abab2e Mon Sep 17 00:00:00 2001 From: duarte Date: Wed, 29 Apr 2026 13:29:06 +0100 Subject: [PATCH 026/118] export dependencies of subrepos - 13/14 tests passing --- src/export/export.go | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/src/export/export.go b/src/export/export.go index 73019f7233..d8a7d37604 100644 --- a/src/export/export.go +++ b/src/export/export.go @@ -125,6 +125,7 @@ func (be *baseExporter) Targets(labels core.BuildLabels) { // Dependencies exports dependencies of a target. func (be *baseExporter) Dependencies(target *core.BuildTarget) { for _, dep := range target.Dependencies() { + log.Infof("Dependency of (%v): %v", target.Label, dep.Label) be.impl.Target(dep) } } @@ -132,14 +133,15 @@ func (be *baseExporter) Dependencies(target *core.BuildTarget) { // Sources exports all files required by the target. func (be *baseExporter) Sources(target *core.BuildTarget) { for _, src := range append(target.AllSources(), target.AllData()...) { - if _, ok := src.Label(); !ok { // We'll handle these dependencies later - for _, p := range src.Paths(be.state.Graph) { - if !filepath.IsAbs(p) { // Don't copy system file deps. - if err := fs.RecursiveCopy(p, filepath.Join(be.targetDir, p), 0); err != nil { - log.Fatalf("Error copying file: %s\n", err) - } - log.Warning("Writing source file: %s", p) + if _, ok := src.Label(); ok { + continue // These will be handled as dependencies later + } + for _, p := range src.Paths(be.state.Graph) { + if !filepath.IsAbs(p) { // Don't copy system file deps. + if err := fs.RecursiveCopy(p, filepath.Join(be.targetDir, p), 0); err != nil { + log.Fatalf("Error copying file: %s\n", err) } + log.Warning("Writing source file: %s", p) } } } @@ -191,11 +193,12 @@ func (e *DefaultExporter) Target(target *core.BuildTarget) { return } - // We want to export the package that made this subrepo available, but we still need to walk the target deps - // as it may depend on other subrepos or first party targets + // We want to export the package that made this subrepo available, but we still need to walk the + // target deps as it may depend on other subrepos or first party targets if target.Subrepo != nil { e.Target(target.Subrepo.Target) - // TODO do we need dependencies and sources? + e.Dependencies(target) + // TODO do we need to walk build statements or subincludes? return } @@ -410,7 +413,8 @@ func (nte *NoTrimExporter) Target(target *core.BuildTarget) { // as it may depend on other subrepos or first party targets if target.Subrepo != nil { nte.Target(target.Subrepo.Target) - // TODO do we need dependencies and sources? + nte.Dependencies(target) + // TODO do we need to walk build statements or subincludes? return } From 28b5441bfcb65d5dba87561d263f8faa7b1cc872 Mon Sep 17 00:00:00 2001 From: duarte Date: Wed, 29 Apr 2026 17:38:41 +0100 Subject: [PATCH 027/118] test: internal repo test using temp directory to avoid stale data --- test/export/BUILD | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/export/BUILD b/test/export/BUILD index 545dc35c80..6223161562 100644 --- a/test/export/BUILD +++ b/test/export/BUILD @@ -9,5 +9,6 @@ filegroup( # Generic catch-all test on internal repo. plz_e2e_test( name = "export_src_please_test", - cmd = "plz export --output plz-out/plzexport //src/core && plz --repo_root=$(plz query reporoot)/plz-out/plzexport build //src/core", + cmd = f'plz export --output "$PLZ_EXPORT_DIR" //src/core:core && plz --repo_root="$PLZ_EXPORT_DIR" build //src/core:core', + pre_cmd = f'export PLZ_EXPORT_DIR="$(mktemp -d)"', ) From 781af3a09a48ca2040e2c5bbd99a1ddf14b75cf8 Mon Sep 17 00:00:00 2001 From: duarte Date: Wed, 29 Apr 2026 17:39:12 +0100 Subject: [PATCH 028/118] fix: 0 label subinclude --- src/export/export.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/export/export.go b/src/export/export.go index d8a7d37604..899fefb896 100644 --- a/src/export/export.go +++ b/src/export/export.go @@ -368,6 +368,11 @@ func (e *DefaultExporter) minimalSubincludeStatement(pkg *core.Package, availabl filteredLabels = append(filteredLabels, label) } } + + if len(filteredLabels) == 0 { + return "" + } + sort.Sort(filteredLabels) call := &build.CallExpr{ From 2f8355cfd739f96c6dc780a17342aae5dd160174 Mon Sep 17 00:00:00 2001 From: duarte Date: Wed, 29 Apr 2026 19:14:48 +0100 Subject: [PATCH 029/118] adjusting dependency lookup and adjacent target test to include secondary build def with sources the updated test uses non-standard child naming to validate new trimming logic --- src/export/export.go | 10 +++++----- .../expected_repo/build_defs/BUILD_FILE | 5 ----- .../source_repo/build_defs/BUILD_FILE | 5 ----- .../BUILD | 2 +- .../expected_repo/.plzconfig | 0 .../expected_repo/BUILD_FILE | 0 .../expected_repo/build_defs/BUILD_FILE | 17 +++++++++++++++++ .../expected_repo/build_defs/custom.build_defs | 8 ++++---- .../build_defs/secondary.build_defs | 9 +++++++++ .../expected_repo/build_defs/secondary.file | 1 + .../expected_repo/file.txt | 0 .../source_repo/.plzconfig | 0 .../source_repo/BUILD_FILE | 0 .../source_repo/build_defs/BUILD_FILE | 17 +++++++++++++++++ .../source_repo/build_defs/custom.build_defs | 8 ++++---- .../source_repo/build_defs/secondary.build_defs | 9 +++++++++ .../source_repo/build_defs/secondary.file | 1 + .../source_repo/file.txt | 0 18 files changed, 68 insertions(+), 24 deletions(-) delete mode 100644 test/export/test_custom_def_children/expected_repo/build_defs/BUILD_FILE delete mode 100644 test/export/test_custom_def_children/source_repo/build_defs/BUILD_FILE rename test/export/{test_custom_def_children => test_custom_def_multiple_targets}/BUILD (87%) rename test/export/{test_custom_def_children => test_custom_def_multiple_targets}/expected_repo/.plzconfig (100%) rename test/export/{test_custom_def_children => test_custom_def_multiple_targets}/expected_repo/BUILD_FILE (100%) create mode 100644 test/export/test_custom_def_multiple_targets/expected_repo/build_defs/BUILD_FILE rename test/export/{test_custom_def_children => test_custom_def_multiple_targets}/expected_repo/build_defs/custom.build_defs (69%) create mode 100644 test/export/test_custom_def_multiple_targets/expected_repo/build_defs/secondary.build_defs create mode 100644 test/export/test_custom_def_multiple_targets/expected_repo/build_defs/secondary.file rename test/export/{test_custom_def_children => test_custom_def_multiple_targets}/expected_repo/file.txt (100%) rename test/export/{test_custom_def_children => test_custom_def_multiple_targets}/source_repo/.plzconfig (100%) rename test/export/{test_custom_def_children => test_custom_def_multiple_targets}/source_repo/BUILD_FILE (100%) create mode 100644 test/export/test_custom_def_multiple_targets/source_repo/build_defs/BUILD_FILE rename test/export/{test_custom_def_children => test_custom_def_multiple_targets}/source_repo/build_defs/custom.build_defs (69%) create mode 100644 test/export/test_custom_def_multiple_targets/source_repo/build_defs/secondary.build_defs create mode 100644 test/export/test_custom_def_multiple_targets/source_repo/build_defs/secondary.file rename test/export/{test_custom_def_children => test_custom_def_multiple_targets}/source_repo/file.txt (100%) diff --git a/src/export/export.go b/src/export/export.go index 899fefb896..07f6f53744 100644 --- a/src/export/export.go +++ b/src/export/export.go @@ -124,10 +124,9 @@ func (be *baseExporter) Targets(labels core.BuildLabels) { // Dependencies exports dependencies of a target. func (be *baseExporter) Dependencies(target *core.BuildTarget) { - for _, dep := range target.Dependencies() { - log.Infof("Dependency of (%v): %v", target.Label, dep.Label) - be.impl.Target(dep) - } + deps := target.DeclaredDependencies() + log.Infof("Exporting dependencies of (%v): %v", target.Label, deps) + be.Targets(deps) } // Sources exports all files required by the target. @@ -141,7 +140,7 @@ func (be *baseExporter) Sources(target *core.BuildTarget) { if err := fs.RecursiveCopy(p, filepath.Join(be.targetDir, p), 0); err != nil { log.Fatalf("Error copying file: %s\n", err) } - log.Warning("Writing source file: %s", p) + log.Infof("Writing exported source file: %s", p) } } } @@ -248,6 +247,7 @@ func (e *DefaultExporter) BuildStatements(pkg *core.Package, target *core.BuildT if err != nil { log.Fatalf("Failed to lookup related targets for package %s: %w", pkg.Name, err) } + log.Infof("Exporting related targets to (%v): %v", target.Label, relatedTargets) for _, target := range relatedTargets { e.Target(target) diff --git a/test/export/test_custom_def_children/expected_repo/build_defs/BUILD_FILE b/test/export/test_custom_def_children/expected_repo/build_defs/BUILD_FILE deleted file mode 100644 index e9b1bc8a20..0000000000 --- a/test/export/test_custom_def_children/expected_repo/build_defs/BUILD_FILE +++ /dev/null @@ -1,5 +0,0 @@ -filegroup( - name = "custom_build_def", - srcs = ["custom.build_defs"], - visibility = ["PUBLIC"], -) diff --git a/test/export/test_custom_def_children/source_repo/build_defs/BUILD_FILE b/test/export/test_custom_def_children/source_repo/build_defs/BUILD_FILE deleted file mode 100644 index e9b1bc8a20..0000000000 --- a/test/export/test_custom_def_children/source_repo/build_defs/BUILD_FILE +++ /dev/null @@ -1,5 +0,0 @@ -filegroup( - name = "custom_build_def", - srcs = ["custom.build_defs"], - visibility = ["PUBLIC"], -) diff --git a/test/export/test_custom_def_children/BUILD b/test/export/test_custom_def_multiple_targets/BUILD similarity index 87% rename from test/export/test_custom_def_children/BUILD rename to test/export/test_custom_def_multiple_targets/BUILD index af8ef5b106..abd0536fd1 100644 --- a/test/export/test_custom_def_children/BUILD +++ b/test/export/test_custom_def_multiple_targets/BUILD @@ -5,7 +5,7 @@ please_export_e2e_test( name = "export_custom_with_adjacent_target", cmd_on_export = [ # Adjacent target of build def - "build //:custom_target#adjacent", + "build //:custom_target_adjacent", ], export_targets = ["//:custom_target"], ) diff --git a/test/export/test_custom_def_children/expected_repo/.plzconfig b/test/export/test_custom_def_multiple_targets/expected_repo/.plzconfig similarity index 100% rename from test/export/test_custom_def_children/expected_repo/.plzconfig rename to test/export/test_custom_def_multiple_targets/expected_repo/.plzconfig diff --git a/test/export/test_custom_def_children/expected_repo/BUILD_FILE b/test/export/test_custom_def_multiple_targets/expected_repo/BUILD_FILE similarity index 100% rename from test/export/test_custom_def_children/expected_repo/BUILD_FILE rename to test/export/test_custom_def_multiple_targets/expected_repo/BUILD_FILE diff --git a/test/export/test_custom_def_multiple_targets/expected_repo/build_defs/BUILD_FILE b/test/export/test_custom_def_multiple_targets/expected_repo/build_defs/BUILD_FILE new file mode 100644 index 0000000000..6e20067074 --- /dev/null +++ b/test/export/test_custom_def_multiple_targets/expected_repo/build_defs/BUILD_FILE @@ -0,0 +1,17 @@ +filegroup( + name = "custom_build_def", + srcs = ["custom.build_defs"], + visibility = ["PUBLIC"], +) + +filegroup( + name = "secondary_build_def", + srcs = ["secondary.build_defs"], + visibility = ["PUBLIC"], +) + +export_file( + name = "secondary_file", + src = "secondary.file", + visibility = ["PUBLIC"], +) diff --git a/test/export/test_custom_def_children/expected_repo/build_defs/custom.build_defs b/test/export/test_custom_def_multiple_targets/expected_repo/build_defs/custom.build_defs similarity index 69% rename from test/export/test_custom_def_children/expected_repo/build_defs/custom.build_defs rename to test/export/test_custom_def_multiple_targets/expected_repo/build_defs/custom.build_defs index cc0e256e86..00bf12383e 100644 --- a/test/export/test_custom_def_children/expected_repo/build_defs/custom.build_defs +++ b/test/export/test_custom_def_multiple_targets/expected_repo/build_defs/custom.build_defs @@ -1,13 +1,13 @@ +subinclude("//build_defs:secondary_build_def") + def custom_target( name:str, srcs:list=[], outs:list=[], outs_adjacent:list=[]): - genrule( - name = f"{name}#adjacent", - srcs = srcs, + secondary_custom_target( + name = f"{name}_adjacent", outs = outs_adjacent, - cmd = "echo 'adjacent' > $OUT && cat $SRCS >> $OUT ", ) return genrule( name = name, diff --git a/test/export/test_custom_def_multiple_targets/expected_repo/build_defs/secondary.build_defs b/test/export/test_custom_def_multiple_targets/expected_repo/build_defs/secondary.build_defs new file mode 100644 index 0000000000..85b6470267 --- /dev/null +++ b/test/export/test_custom_def_multiple_targets/expected_repo/build_defs/secondary.build_defs @@ -0,0 +1,9 @@ +def secondary_custom_target( + name:str, + outs:list=[]): + return genrule( + name = name, + srcs = ["//build_defs:secondary_file"], + outs = outs, + cmd = "echo 'secondary' > $OUT && cat $SRCS >> $OUT ", + ) diff --git a/test/export/test_custom_def_multiple_targets/expected_repo/build_defs/secondary.file b/test/export/test_custom_def_multiple_targets/expected_repo/build_defs/secondary.file new file mode 100644 index 0000000000..7a0bf8eb90 --- /dev/null +++ b/test/export/test_custom_def_multiple_targets/expected_repo/build_defs/secondary.file @@ -0,0 +1 @@ +File for testing diff --git a/test/export/test_custom_def_children/expected_repo/file.txt b/test/export/test_custom_def_multiple_targets/expected_repo/file.txt similarity index 100% rename from test/export/test_custom_def_children/expected_repo/file.txt rename to test/export/test_custom_def_multiple_targets/expected_repo/file.txt diff --git a/test/export/test_custom_def_children/source_repo/.plzconfig b/test/export/test_custom_def_multiple_targets/source_repo/.plzconfig similarity index 100% rename from test/export/test_custom_def_children/source_repo/.plzconfig rename to test/export/test_custom_def_multiple_targets/source_repo/.plzconfig diff --git a/test/export/test_custom_def_children/source_repo/BUILD_FILE b/test/export/test_custom_def_multiple_targets/source_repo/BUILD_FILE similarity index 100% rename from test/export/test_custom_def_children/source_repo/BUILD_FILE rename to test/export/test_custom_def_multiple_targets/source_repo/BUILD_FILE diff --git a/test/export/test_custom_def_multiple_targets/source_repo/build_defs/BUILD_FILE b/test/export/test_custom_def_multiple_targets/source_repo/build_defs/BUILD_FILE new file mode 100644 index 0000000000..6e20067074 --- /dev/null +++ b/test/export/test_custom_def_multiple_targets/source_repo/build_defs/BUILD_FILE @@ -0,0 +1,17 @@ +filegroup( + name = "custom_build_def", + srcs = ["custom.build_defs"], + visibility = ["PUBLIC"], +) + +filegroup( + name = "secondary_build_def", + srcs = ["secondary.build_defs"], + visibility = ["PUBLIC"], +) + +export_file( + name = "secondary_file", + src = "secondary.file", + visibility = ["PUBLIC"], +) diff --git a/test/export/test_custom_def_children/source_repo/build_defs/custom.build_defs b/test/export/test_custom_def_multiple_targets/source_repo/build_defs/custom.build_defs similarity index 69% rename from test/export/test_custom_def_children/source_repo/build_defs/custom.build_defs rename to test/export/test_custom_def_multiple_targets/source_repo/build_defs/custom.build_defs index cc0e256e86..00bf12383e 100644 --- a/test/export/test_custom_def_children/source_repo/build_defs/custom.build_defs +++ b/test/export/test_custom_def_multiple_targets/source_repo/build_defs/custom.build_defs @@ -1,13 +1,13 @@ +subinclude("//build_defs:secondary_build_def") + def custom_target( name:str, srcs:list=[], outs:list=[], outs_adjacent:list=[]): - genrule( - name = f"{name}#adjacent", - srcs = srcs, + secondary_custom_target( + name = f"{name}_adjacent", outs = outs_adjacent, - cmd = "echo 'adjacent' > $OUT && cat $SRCS >> $OUT ", ) return genrule( name = name, diff --git a/test/export/test_custom_def_multiple_targets/source_repo/build_defs/secondary.build_defs b/test/export/test_custom_def_multiple_targets/source_repo/build_defs/secondary.build_defs new file mode 100644 index 0000000000..85b6470267 --- /dev/null +++ b/test/export/test_custom_def_multiple_targets/source_repo/build_defs/secondary.build_defs @@ -0,0 +1,9 @@ +def secondary_custom_target( + name:str, + outs:list=[]): + return genrule( + name = name, + srcs = ["//build_defs:secondary_file"], + outs = outs, + cmd = "echo 'secondary' > $OUT && cat $SRCS >> $OUT ", + ) diff --git a/test/export/test_custom_def_multiple_targets/source_repo/build_defs/secondary.file b/test/export/test_custom_def_multiple_targets/source_repo/build_defs/secondary.file new file mode 100644 index 0000000000..7a0bf8eb90 --- /dev/null +++ b/test/export/test_custom_def_multiple_targets/source_repo/build_defs/secondary.file @@ -0,0 +1 @@ +File for testing diff --git a/test/export/test_custom_def_children/source_repo/file.txt b/test/export/test_custom_def_multiple_targets/source_repo/file.txt similarity index 100% rename from test/export/test_custom_def_children/source_repo/file.txt rename to test/export/test_custom_def_multiple_targets/source_repo/file.txt From 3e89c85af5a4969001a1375b24f2ed699ef7222e Mon Sep 17 00:00:00 2001 From: duarte Date: Thu, 30 Apr 2026 10:47:10 +0100 Subject: [PATCH 030/118] test: add custom tool to native test --- test/export/test_builtins/expected_repo/BUILD_FILE | 3 ++- test/export/test_builtins/expected_repo/tools/BUILD_FILE | 6 ++++++ test/export/test_builtins/expected_repo/tools/tool.sh | 2 ++ test/export/test_builtins/source_repo/BUILD_FILE | 3 ++- test/export/test_builtins/source_repo/tools/BUILD_FILE | 6 ++++++ test/export/test_builtins/source_repo/tools/tool.sh | 2 ++ 6 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 test/export/test_builtins/expected_repo/tools/BUILD_FILE create mode 100644 test/export/test_builtins/expected_repo/tools/tool.sh create mode 100644 test/export/test_builtins/source_repo/tools/BUILD_FILE create mode 100644 test/export/test_builtins/source_repo/tools/tool.sh diff --git a/test/export/test_builtins/expected_repo/BUILD_FILE b/test/export/test_builtins/expected_repo/BUILD_FILE index a62a98a8d6..b840c2d9f1 100644 --- a/test/export/test_builtins/expected_repo/BUILD_FILE +++ b/test/export/test_builtins/expected_repo/BUILD_FILE @@ -2,5 +2,6 @@ genrule( name = "native_genrule", srcs = ["file.txt"], outs = ["file.wordcount"], - cmd = "wc $SRCS > $OUT", + cmd = "$TOOLS $SRCS > $OUT", + tools = ["//tools:tool"], ) diff --git a/test/export/test_builtins/expected_repo/tools/BUILD_FILE b/test/export/test_builtins/expected_repo/tools/BUILD_FILE new file mode 100644 index 0000000000..71d3129e2c --- /dev/null +++ b/test/export/test_builtins/expected_repo/tools/BUILD_FILE @@ -0,0 +1,6 @@ +export_file( + name = "tool", + src = "tool.sh", + binary = True, + visibility = ["PUBLIC"], +) diff --git a/test/export/test_builtins/expected_repo/tools/tool.sh b/test/export/test_builtins/expected_repo/tools/tool.sh new file mode 100644 index 0000000000..244a9b4ffd --- /dev/null +++ b/test/export/test_builtins/expected_repo/tools/tool.sh @@ -0,0 +1,2 @@ +#!/bin/bash +wc $@ diff --git a/test/export/test_builtins/source_repo/BUILD_FILE b/test/export/test_builtins/source_repo/BUILD_FILE index db7732c487..7c4e63b45d 100644 --- a/test/export/test_builtins/source_repo/BUILD_FILE +++ b/test/export/test_builtins/source_repo/BUILD_FILE @@ -2,7 +2,8 @@ genrule( name = "native_genrule", srcs = ["file.txt"], outs = ["file.wordcount"], - cmd = "wc $SRCS > $OUT", + cmd = "$TOOLS $SRCS > $OUT", + tools = ["//tools:tool"], ) genrule( diff --git a/test/export/test_builtins/source_repo/tools/BUILD_FILE b/test/export/test_builtins/source_repo/tools/BUILD_FILE new file mode 100644 index 0000000000..71d3129e2c --- /dev/null +++ b/test/export/test_builtins/source_repo/tools/BUILD_FILE @@ -0,0 +1,6 @@ +export_file( + name = "tool", + src = "tool.sh", + binary = True, + visibility = ["PUBLIC"], +) diff --git a/test/export/test_builtins/source_repo/tools/tool.sh b/test/export/test_builtins/source_repo/tools/tool.sh new file mode 100644 index 0000000000..244a9b4ffd --- /dev/null +++ b/test/export/test_builtins/source_repo/tools/tool.sh @@ -0,0 +1,2 @@ +#!/bin/bash +wc $@ From 9c5af3e6f679d2860e06400338826a770d2be163 Mon Sep 17 00:00:00 2001 From: duarte Date: Thu, 30 Apr 2026 13:32:32 +0100 Subject: [PATCH 031/118] test: go_test export with several deps --- test/export/test_go_test/BUILD | 7 ++++ .../test_go_test/expected_repo/.plzconfig | 10 ++++++ .../test_go_test/expected_repo/BUILD_FILE | 11 ++++++ test/export/test_go_test/expected_repo/go.mod | 11 ++++++ .../expected_repo/plugins/BUILD_FILE | 6 ++++ .../export/test_go_test/expected_repo/test.go | 11 ++++++ .../expected_repo/third_party/go/BUILD_FILE | 30 ++++++++++++++++ .../test_go_test/source_repo/.plzconfig | 10 ++++++ .../test_go_test/source_repo/BUILD_FILE | 19 ++++++++++ test/export/test_go_test/source_repo/go.mod | 11 ++++++ test/export/test_go_test/source_repo/main.go | 11 ++++++ .../source_repo/plugins/BUILD_FILE | 6 ++++ test/export/test_go_test/source_repo/test.go | 11 ++++++ .../source_repo/third_party/go/BUILD_FILE | 36 +++++++++++++++++++ 14 files changed, 190 insertions(+) create mode 100644 test/export/test_go_test/BUILD create mode 100644 test/export/test_go_test/expected_repo/.plzconfig create mode 100644 test/export/test_go_test/expected_repo/BUILD_FILE create mode 100644 test/export/test_go_test/expected_repo/go.mod create mode 100644 test/export/test_go_test/expected_repo/plugins/BUILD_FILE create mode 100644 test/export/test_go_test/expected_repo/test.go create mode 100644 test/export/test_go_test/expected_repo/third_party/go/BUILD_FILE create mode 100644 test/export/test_go_test/source_repo/.plzconfig create mode 100644 test/export/test_go_test/source_repo/BUILD_FILE create mode 100644 test/export/test_go_test/source_repo/go.mod create mode 100644 test/export/test_go_test/source_repo/main.go create mode 100644 test/export/test_go_test/source_repo/plugins/BUILD_FILE create mode 100644 test/export/test_go_test/source_repo/test.go create mode 100644 test/export/test_go_test/source_repo/third_party/go/BUILD_FILE diff --git a/test/export/test_go_test/BUILD b/test/export/test_go_test/BUILD new file mode 100644 index 0000000000..e8c41a89e9 --- /dev/null +++ b/test/export/test_go_test/BUILD @@ -0,0 +1,7 @@ +subinclude("//test/export:export_e2e_test_build_def") + +# Test go_test target with a go third_party dependency that implicitly requires other packages. +please_export_e2e_test( + name = "export_go_test", + export_targets = ["//:test"], +) diff --git a/test/export/test_go_test/expected_repo/.plzconfig b/test/export/test_go_test/expected_repo/.plzconfig new file mode 100644 index 0000000000..7fb433a97e --- /dev/null +++ b/test/export/test_go_test/expected_repo/.plzconfig @@ -0,0 +1,10 @@ +[Parse] +BuildFileName = BUILD # required by subrepos +BuildFileName = BUILD_FILE +preloadsubincludes = ///go//build_defs:go + +[Plugin "go"] +Target = //plugins:go +GoTool = //third_party/go:toolchain|go +STDLib = //third_party/go:std +ModFile = //:go_mod diff --git a/test/export/test_go_test/expected_repo/BUILD_FILE b/test/export/test_go_test/expected_repo/BUILD_FILE new file mode 100644 index 0000000000..516ef528ca --- /dev/null +++ b/test/export/test_go_test/expected_repo/BUILD_FILE @@ -0,0 +1,11 @@ +filegroup( + name = "go_mod", + srcs = ["go.mod"], + visibility = ["PUBLIC"], +) + +go_test( + name = "test", + srcs = ["test.go"], + deps = ["///third_party/go/github.com_stretchr_testify//assert"], +) diff --git a/test/export/test_go_test/expected_repo/go.mod b/test/export/test_go_test/expected_repo/go.mod new file mode 100644 index 0000000000..7fcf309969 --- /dev/null +++ b/test/export/test_go_test/expected_repo/go.mod @@ -0,0 +1,11 @@ +module github.com/thought-machine/please/test_repo + +go 1.23.0 + +require github.com/stretchr/testify v1.9.0 + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/test/export/test_go_test/expected_repo/plugins/BUILD_FILE b/test/export/test_go_test/expected_repo/plugins/BUILD_FILE new file mode 100644 index 0000000000..55e81b8e1d --- /dev/null +++ b/test/export/test_go_test/expected_repo/plugins/BUILD_FILE @@ -0,0 +1,6 @@ +plugin_repo( + name = "go", + owner = "please-build", + plugin = "go-rules", + revision = "v1.30.0", +) diff --git a/test/export/test_go_test/expected_repo/test.go b/test/export/test_go_test/expected_repo/test.go new file mode 100644 index 0000000000..aaf3c0c44b --- /dev/null +++ b/test/export/test_go_test/expected_repo/test.go @@ -0,0 +1,11 @@ +package test + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSomething(t *testing.T) { + assert.NotEqual(t, 0, 1) +} diff --git a/test/export/test_go_test/expected_repo/third_party/go/BUILD_FILE b/test/export/test_go_test/expected_repo/third_party/go/BUILD_FILE new file mode 100644 index 0000000000..42a26f4f9c --- /dev/null +++ b/test/export/test_go_test/expected_repo/third_party/go/BUILD_FILE @@ -0,0 +1,30 @@ +go_toolchain( + name = "toolchain", + version = "1.26.2", +) + +go_stdlib(name = "std") + +go_repo( + licences = ["MIT"], + module = "github.com/stretchr/testify", + version = "v1.9.0", +) + +go_repo( + licences = ["ISC"], + module = "github.com/davecgh/go-spew", + version = "v1.1.1", +) + +go_repo( + licences = ["BSD-3-Clause"], + module = "github.com/pmezard/go-difflib", + version = "v1.0.0", +) + +go_repo( + licences = ["MIT"], + module = "gopkg.in/yaml.v3", + version = "v3.0.1", +) diff --git a/test/export/test_go_test/source_repo/.plzconfig b/test/export/test_go_test/source_repo/.plzconfig new file mode 100644 index 0000000000..7fb433a97e --- /dev/null +++ b/test/export/test_go_test/source_repo/.plzconfig @@ -0,0 +1,10 @@ +[Parse] +BuildFileName = BUILD # required by subrepos +BuildFileName = BUILD_FILE +preloadsubincludes = ///go//build_defs:go + +[Plugin "go"] +Target = //plugins:go +GoTool = //third_party/go:toolchain|go +STDLib = //third_party/go:std +ModFile = //:go_mod diff --git a/test/export/test_go_test/source_repo/BUILD_FILE b/test/export/test_go_test/source_repo/BUILD_FILE new file mode 100644 index 0000000000..c916f9d154 --- /dev/null +++ b/test/export/test_go_test/source_repo/BUILD_FILE @@ -0,0 +1,19 @@ +filegroup( + name = "go_mod", + srcs = ["go.mod"], + visibility = ["PUBLIC"], +) + +go_test( + name = "test", + srcs = ["test.go"], + deps = ["///third_party/go/github.com_stretchr_testify//assert"], +) + +go_binary( + name = "unused", + srcs = ["main.go"], + deps = [ + "//third_party/go:cmp", + ], +) diff --git a/test/export/test_go_test/source_repo/go.mod b/test/export/test_go_test/source_repo/go.mod new file mode 100644 index 0000000000..7fcf309969 --- /dev/null +++ b/test/export/test_go_test/source_repo/go.mod @@ -0,0 +1,11 @@ +module github.com/thought-machine/please/test_repo + +go 1.23.0 + +require github.com/stretchr/testify v1.9.0 + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/test/export/test_go_test/source_repo/main.go b/test/export/test_go_test/source_repo/main.go new file mode 100644 index 0000000000..0853120a28 --- /dev/null +++ b/test/export/test_go_test/source_repo/main.go @@ -0,0 +1,11 @@ +package main + +import ( + "fmt" + + "github.com/google/go-cmp/cmp" +) + +func main() { + fmt.Print(cmp.Equal(1, 1)) +} diff --git a/test/export/test_go_test/source_repo/plugins/BUILD_FILE b/test/export/test_go_test/source_repo/plugins/BUILD_FILE new file mode 100644 index 0000000000..55e81b8e1d --- /dev/null +++ b/test/export/test_go_test/source_repo/plugins/BUILD_FILE @@ -0,0 +1,6 @@ +plugin_repo( + name = "go", + owner = "please-build", + plugin = "go-rules", + revision = "v1.30.0", +) diff --git a/test/export/test_go_test/source_repo/test.go b/test/export/test_go_test/source_repo/test.go new file mode 100644 index 0000000000..aaf3c0c44b --- /dev/null +++ b/test/export/test_go_test/source_repo/test.go @@ -0,0 +1,11 @@ +package test + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSomething(t *testing.T) { + assert.NotEqual(t, 0, 1) +} diff --git a/test/export/test_go_test/source_repo/third_party/go/BUILD_FILE b/test/export/test_go_test/source_repo/third_party/go/BUILD_FILE new file mode 100644 index 0000000000..88c12fe2a0 --- /dev/null +++ b/test/export/test_go_test/source_repo/third_party/go/BUILD_FILE @@ -0,0 +1,36 @@ +go_toolchain( + name = "toolchain", + version = "1.26.2", +) + +go_stdlib(name = "std") + +go_repo( + licences = ["BSD-3-Clause"], + module = "github.com/google/go-cmp", + version = "v0.6.0", +) + +go_repo( + licences = ["MIT"], + module = "github.com/stretchr/testify", + version = "v1.9.0", +) + +go_repo( + licences = ["ISC"], + module = "github.com/davecgh/go-spew", + version = "v1.1.1", +) + +go_repo( + licences = ["BSD-3-Clause"], + module = "github.com/pmezard/go-difflib", + version = "v1.0.0", +) + +go_repo( + licences = ["MIT"], + module = "gopkg.in/yaml.v3", + version = "v3.0.1", +) From 922da6fda7d9683280a0d3176c04b3dc8dfbfb06 Mon Sep 17 00:00:00 2001 From: duarte Date: Fri, 1 May 2026 13:44:33 +0100 Subject: [PATCH 032/118] optional metadata parsing --- src/core/package.go | 37 +++++++++--------- src/core/package_metadata.go | 76 ++++++++++++++++++++++++++++++------ src/core/state.go | 2 + src/parse/asp/interpreter.go | 53 +++++++++++++------------ src/parse/parse_step.go | 2 +- src/please.go | 63 +++++++++++++++++------------- 6 files changed, 150 insertions(+), 83 deletions(-) diff --git a/src/core/package.go b/src/core/package.go index eca70d4bee..d9a9edea98 100644 --- a/src/core/package.go +++ b/src/core/package.go @@ -35,7 +35,7 @@ type Package struct { // Set of output files from rules. Outputs map[string]*BuildTarget // Includes metadata from parsing the package BUILD file. - BuildFileMetadata BuildFileMetadata + BuildFileMetadata PackageMetadata // Protects access to above mutex sync.RWMutex } @@ -45,6 +45,16 @@ func NewPackage(name string) *Package { return NewPackageSubrepo(name, "") } +// NewPackageWithOpts constructs a new package with the given name, and enables additional features +// given the flags enabled in the build state. +func NewPackageWithOpts(state *BuildState, name string) *Package { + pkg := NewPackage(name) + if state.ParseMetadata { + pkg.BuildFileMetadata = newBuildFileMetadata() + } + return pkg +} + // NewPackageSubrepo constructs a new package with the given name and subrepo. func NewPackageSubrepo(name, subrepo string) *Package { return &Package{ @@ -52,7 +62,7 @@ func NewPackageSubrepo(name, subrepo string) *Package { SubrepoName: subrepo, targets: map[string]*BuildTarget{}, Outputs: map[string]*BuildTarget{}, - BuildFileMetadata: *newBuildFileMetadata(), + BuildFileMetadata: newNoopPackageMetadata(), } } @@ -234,35 +244,24 @@ func (pkg *Package) verifyOutputs() []string { } // RegisterStatement maps a build statement to target in the package. -func (pkg *Package) RegisterStatement(target *BuildTarget, stmt *BuildStatement) { - if stmt == nil { - log.Infof("Attempted to register empty build statement for package %s and target %s", - pkg.Name, target.String()) - return - } +func (pkg *Package) RegisterStatement(target *BuildTarget, stmtProvider BuildStatementProvider) { pkg.mutex.Lock() defer pkg.mutex.Unlock() - pkg.BuildFileMetadata.RegisterStatementTarget(stmt, target) + pkg.BuildFileMetadata.RegisterStatementTarget(target, stmtProvider) } // RegisterRequiredSubincludes maps the required subincludes to generate the target. -func (pkg *Package) RegisterRequiredSubincludes(target *BuildTarget, subincludes BuildLabels) { - if len(subincludes) == 0 { - log.Infof("Attempted to register empty subinclude labels for package %s and target %s", - pkg.Name, target.String()) - return - } - +func (pkg *Package) RegisterRequiredSubincludes(target *BuildTarget, labelProvider SubincludesLabelProvider) { pkg.mutex.Lock() defer pkg.mutex.Unlock() - pkg.BuildFileMetadata.RegisterRequiredSubinclude(target, subincludes) + pkg.BuildFileMetadata.RegisterRequiredSubinclude(target, labelProvider) } // RegisterSubincludeStmt maps the subincludes build statement to the included targets. -func (pkg *Package) RegisterSubincludeStmt(label BuildLabel, stmt *BuildStatement) { +func (pkg *Package) RegisterSubincludeStmt(label BuildLabel, stmtProvider BuildStatementProvider) { pkg.mutex.Lock() defer pkg.mutex.Unlock() - pkg.BuildFileMetadata.RegisterSubincludeStmt(label, stmt) + pkg.BuildFileMetadata.RegisterSubincludeStmt(label, stmtProvider) } // FindStatement finds the build statement that generated the target. diff --git a/src/core/package_metadata.go b/src/core/package_metadata.go index 3c2361406b..c5181cf32b 100644 --- a/src/core/package_metadata.go +++ b/src/core/package_metadata.go @@ -27,8 +27,27 @@ func (s BuildStatements) Len() int { return len(s) } func (s BuildStatements) Swap(i, j int) { s[i], s[j] = s[j], s[i] } func (s BuildStatements) Less(i, j int) bool { return s[i].StartPos() < s[j].StartPos() } -// BuildFileMetadata stores metadata about parsed BUILD files, mapping statements and subincludes to their respective targets. -type BuildFileMetadata struct { +// BuildStatementProvider is a type for methods that generate new build statements. +type BuildStatementProvider func() *BuildStatement + +// SubincludesLabelProvider is a type for methods that generate labels from a subincludes. +type SubincludesLabelProvider func() BuildLabels + +// PackageMetadata stores metadata about parsed BUILD files, mapping statements and subincludes +// to their respective targets. +type PackageMetadata interface { + RegisterStatementTarget(target *BuildTarget, stmtProvider BuildStatementProvider) + RegisterRequiredSubinclude(target *BuildTarget, labelProvider SubincludesLabelProvider) + RegisterSubincludeStmt(label BuildLabel, stmtProvider BuildStatementProvider) + FindStatement(target *BuildTarget) (*BuildStatement, error) + FindTargets(stmt *BuildStatement) ([]*BuildTarget, error) + FindRequiredSubincludes(target *BuildTarget) (BuildLabels, error) + GetSubincludedLabels(stmt *BuildStatement) (BuildLabels, bool) +} + +// packageMetadataImpl stores metadata about parsed BUILD files, mapping statements and subincludes +// to their respective targets. +type packageMetadataImpl struct { // a list of targets generated from each built statement StmtToTarget map[BuildStatement][]*BuildTarget // the subincluded label dependencies per target @@ -37,8 +56,8 @@ type BuildFileMetadata struct { LabelsPerSubincludeStmt map[BuildStatement]BuildLabels } -func newBuildFileMetadata() *BuildFileMetadata { - return &BuildFileMetadata{ +func newBuildFileMetadata() PackageMetadata { + return &packageMetadataImpl{ StmtToTarget: map[BuildStatement][]*BuildTarget{}, TargetToSubinclude: map[*BuildTarget]BuildLabels{}, LabelsPerSubincludeStmt: map[BuildStatement]BuildLabels{}, @@ -46,22 +65,30 @@ func newBuildFileMetadata() *BuildFileMetadata { } // RegisterStatementTarget maps a build statement to a target it generated. -func (bfm *BuildFileMetadata) RegisterStatementTarget(stmt *BuildStatement, target *BuildTarget) { +func (bfm *packageMetadataImpl) RegisterStatementTarget(target *BuildTarget, stmtProvider BuildStatementProvider) { + stmt := stmtProvider() bfm.StmtToTarget[*stmt] = append(bfm.StmtToTarget[*stmt], target) } // RegisterRequiredSubinclude maps a target to the subincludes required to build it. -func (bfm *BuildFileMetadata) RegisterRequiredSubinclude(target *BuildTarget, subincludes BuildLabels) { - bfm.TargetToSubinclude[target] = append(bfm.TargetToSubinclude[target], subincludes...) +func (bfm *packageMetadataImpl) RegisterRequiredSubinclude(target *BuildTarget, labelProvider SubincludesLabelProvider) { + labels := labelProvider() + if len(labels) == 0 { + log.Infof("Attempted to register empty subinclude labels for target %s", target.String()) + return + } + + bfm.TargetToSubinclude[target] = append(bfm.TargetToSubinclude[target], labels...) } // RegisterSubincludeStmt maps a subinclude statement to a label it includes. -func (bfm *BuildFileMetadata) RegisterSubincludeStmt(label BuildLabel, stmt *BuildStatement) { +func (bfm *packageMetadataImpl) RegisterSubincludeStmt(label BuildLabel, stmtProvider BuildStatementProvider) { + stmt := stmtProvider() bfm.LabelsPerSubincludeStmt[*stmt] = append(bfm.LabelsPerSubincludeStmt[*stmt], label) } // FindStatement returns the build statement that generated the given target. -func (bfm *BuildFileMetadata) FindStatement(target *BuildTarget) (*BuildStatement, error) { +func (bfm *packageMetadataImpl) FindStatement(target *BuildTarget) (*BuildStatement, error) { for stmt, targets := range bfm.StmtToTarget { if slices.Contains(targets, target) { return &stmt, nil @@ -71,7 +98,7 @@ func (bfm *BuildFileMetadata) FindStatement(target *BuildTarget) (*BuildStatemen } // FindTargets returns all targets generated by the given build statement. -func (bfm *BuildFileMetadata) FindTargets(stmt *BuildStatement) ([]*BuildTarget, error) { +func (bfm *packageMetadataImpl) FindTargets(stmt *BuildStatement) ([]*BuildTarget, error) { targets, ok := bfm.StmtToTarget[*stmt] if !ok { return nil, fmt.Errorf("Targets not found for statement %v.", stmt) @@ -80,7 +107,7 @@ func (bfm *BuildFileMetadata) FindTargets(stmt *BuildStatement) ([]*BuildTarget, } // FindRequiredSubincludes returns all subinclude labels required by the given target. -func (bfm *BuildFileMetadata) FindRequiredSubincludes(target *BuildTarget) (BuildLabels, error) { +func (bfm *packageMetadataImpl) FindRequiredSubincludes(target *BuildTarget) (BuildLabels, error) { subincludes, ok := bfm.TargetToSubinclude[target] if !ok { return nil, fmt.Errorf("Subincludes not found for target %v.", target) @@ -89,7 +116,32 @@ func (bfm *BuildFileMetadata) FindRequiredSubincludes(target *BuildTarget) (Buil } // GetSubincludedLabels returns the labels included by a given subinclude statement. -func (bfm *BuildFileMetadata) GetSubincludedLabels(stmt *BuildStatement) (BuildLabels, bool) { +func (bfm *packageMetadataImpl) GetSubincludedLabels(stmt *BuildStatement) (BuildLabels, bool) { v, ok := bfm.LabelsPerSubincludeStmt[*stmt] return v, ok } + +type noopPackageMetadata struct{} + +func newNoopPackageMetadata() PackageMetadata { + return &noopPackageMetadata{} +} + +func (n *noopPackageMetadata) RegisterStatementTarget(target *BuildTarget, stmtProvider BuildStatementProvider) { +} +func (n *noopPackageMetadata) RegisterRequiredSubinclude(target *BuildTarget, labelProvider SubincludesLabelProvider) { +} +func (n *noopPackageMetadata) RegisterSubincludeStmt(label BuildLabel, stmtProvider BuildStatementProvider) { +} +func (n *noopPackageMetadata) FindStatement(target *BuildTarget) (*BuildStatement, error) { + return nil, fmt.Errorf("metadata not tracked") +} +func (n *noopPackageMetadata) FindTargets(stmt *BuildStatement) ([]*BuildTarget, error) { + return nil, fmt.Errorf("metadata not tracked") +} +func (n *noopPackageMetadata) FindRequiredSubincludes(target *BuildTarget) (BuildLabels, error) { + return nil, fmt.Errorf("metadata not tracked") +} +func (n *noopPackageMetadata) GetSubincludedLabels(stmt *BuildStatement) (BuildLabels, bool) { + return nil, false +} diff --git a/src/core/state.go b/src/core/state.go index 18a03d70ab..013d5a59fe 100644 --- a/src/core/state.go +++ b/src/core/state.go @@ -240,6 +240,8 @@ type BuildState struct { // NeedDebugDeps is true if we're doing a `plz debug` and we need to build the debug tools and // data NeedDebugDeps bool + // ParseMetadata is true if we want to store build file metadata + ParseMetadata bool // initOnce is used to control loading the subrepo .plzconfig initOnce *sync.Once diff --git a/src/parse/asp/interpreter.go b/src/parse/asp/interpreter.go index b2f8f18832..330ddde73a 100644 --- a/src/parse/asp/interpreter.go +++ b/src/parse/asp/interpreter.go @@ -1081,35 +1081,40 @@ func (s *scope) Constant(expr *Expression) pyObject { return nil } -// CurrentBuildStatement creates a new BuildStatement from the statement that is being currently interpreted. -func (s *scope) CurrentBuildStatement() *core.BuildStatement { - stmtScope := s - for curr := s; curr != nil; curr = curr.callerScope { - if curr.pkg != nil && curr.filename == s.pkg.Filename { - stmtScope = curr +// CurrentBuildStatement creates a provider for creating a new BuildStatement from the statement +// that is being currently interpreted. +func (s *scope) CurrentBuildStatement() core.BuildStatementProvider { + return func() *core.BuildStatement { + stmtScope := s + for curr := s; curr != nil; curr = curr.callerScope { + if curr.pkg != nil && curr.filename == s.pkg.Filename { + stmtScope = curr + } } - } - s.NAssert(stmtScope.cursor == nil, "Cursor is not pointing to a statement") - return NewBuildStatement(stmtScope.cursor) -} - -// ActiveSubincludes traces the call stack and scopes to find subincludes that provided the -// macros/functions actively executing to define this target. -func (s *scope) ActiveSubincludes() []core.BuildLabel { - var subincludes []core.BuildLabel - seen := map[core.BuildLabel]bool{} - for curr := s; curr != nil; curr = curr.callerScope { - for localScope := curr; localScope != nil; localScope = localScope.parent { - if localScope.subincludeLabel != nil { - label := *localScope.subincludeLabel - if !seen[label] { - seen[label] = true - subincludes = append(subincludes, label) + s.NAssert(stmtScope.cursor == nil, "Cursor is not pointing to a statement") + return NewBuildStatement(stmtScope.cursor) + } +} + +// ActiveSubincludes creates a provider to trace the call stack and scopes to find subincludes that +// provided the macros/functions actively executing to define this target. +func (s *scope) ActiveSubincludes() core.SubincludesLabelProvider { + return func() core.BuildLabels { + var subincludes []core.BuildLabel + seen := map[core.BuildLabel]bool{} + for curr := s; curr != nil; curr = curr.callerScope { + for localScope := curr; localScope != nil; localScope = localScope.parent { + if localScope.subincludeLabel != nil { + label := *localScope.subincludeLabel + if !seen[label] { + seen[label] = true + subincludes = append(subincludes, label) + } } } } + return subincludes } - return subincludes } // pkgFilename returns the filename of the current package, or the empty string if there is none. diff --git a/src/parse/parse_step.go b/src/parse/parse_step.go index 2c9689feab..fb1b775f94 100644 --- a/src/parse/parse_step.go +++ b/src/parse/parse_step.go @@ -182,7 +182,7 @@ func maybeParseSubrepoPackage(state *core.BuildState, subrepoPkg, subrepoSubrepo // parsePackage parses a BUILD file and adds the package to the build graph func parsePackage(state *core.BuildState, label, dependent core.BuildLabel, subrepo *core.Subrepo, mode core.ParseMode) (*core.Package, error) { packageName := label.PackageName - pkg := core.NewPackage(packageName) + pkg := core.NewPackageWithOpts(state, packageName) pkg.Subrepo = subrepo var fileSystem iofs.FS = fs.HostFS if subrepo != nil { diff --git a/src/please.go b/src/please.go index 7297c7e689..a096e2f24b 100644 --- a/src/please.go +++ b/src/please.go @@ -475,7 +475,7 @@ var opts struct { // Functions are called after args are parsed and return a POSIX exit code (0 means success). var buildFunctions = map[string]func() int{ "build": func() int { - success, state := runBuild(opts.Build.Args.Targets, true, false, false) + success, state := runBuild(opts.Build.Args.Targets, buildOpts{Build: true}) if !success || opts.Build.OutDir == "" { return toExitCode(success, state) } @@ -499,7 +499,7 @@ var buildFunctions = map[string]func() int{ if opts.Hash.Update { opts.BehaviorFlags.NoHashVerification = true } - success, state := runBuild(opts.Hash.Args.Targets, true, false, false) + success, state := runBuild(opts.Hash.Args.Targets, buildOpts{Build: true}) if success { if opts.Hash.Detailed { for _, target := range state.ExpandOriginalLabels() { @@ -556,14 +556,14 @@ var buildFunctions = map[string]func() int{ return toExitCode(success, state) }, "debug": func() int { - success, state := runBuild([]core.BuildLabel{opts.Debug.Args.Target}, true, false, false) + success, state := runBuild([]core.BuildLabel{opts.Debug.Args.Target}, buildOpts{Build: true}) if !success { return toExitCode(success, state) } return debug.Debug(state, opts.Debug.Args.Target, opts.Debug.Args.Args, exec.ConvertEnv(opts.Debug.Env), opts.Debug.Share.Network, opts.Debug.Share.Mount) }, "exec": func() int { - success, state := runBuild([]core.BuildLabel{opts.Exec.Args.Target.BuildLabel}, true, false, false) + success, state := runBuild([]core.BuildLabel{opts.Exec.Args.Target.BuildLabel}, buildOpts{Build: true}) if !success { return toExitCode(success, state) } @@ -594,7 +594,7 @@ var buildFunctions = map[string]func() int{ if len(unannotated) == 0 { return 0 } - success, state := runBuild(unannotated, true, false, false) + success, state := runBuild(unannotated, buildOpts{Build: true}) if !success { return toExitCode(success, state) } @@ -608,7 +608,7 @@ var buildFunctions = map[string]func() int{ if len(unannotated) == 0 { return 0 } - success, state := runBuild(unannotated, true, false, false) + success, state := runBuild(unannotated, buildOpts{Build: true}) if !success { return toExitCode(success, state) } @@ -618,7 +618,7 @@ var buildFunctions = map[string]func() int{ return 0 }, "run": func() int { - if success, state := runBuild([]core.BuildLabel{opts.Run.Args.Target.BuildLabel}, true, false, false); success { + if success, state := runBuild([]core.BuildLabel{opts.Run.Args.Target.BuildLabel}, buildOpts{Build: true}); success { var dir string if opts.Run.WD != "" { dir = getAbsolutePath(opts.Run.WD, originalWorkingDirectory) @@ -642,7 +642,7 @@ var buildFunctions = map[string]func() int{ if len(unannotated) == 0 { return 0 } - if success, state := runBuild(unannotated, true, false, false); success { + if success, state := runBuild(unannotated, buildOpts{Build: true}); success { var dir string if opts.Run.WD != "" { dir = getAbsolutePath(opts.Run.WD, originalWorkingDirectory) @@ -659,7 +659,7 @@ var buildFunctions = map[string]func() int{ if len(unannotated) == 0 { return 0 } - if success, state := runBuild(unannotated, true, false, false); success { + if success, state := runBuild(unannotated, buildOpts{Build: true}); success { var dir string if opts.Run.WD != "" { dir = getAbsolutePath(opts.Run.WD, originalWorkingDirectory) @@ -686,7 +686,7 @@ var buildFunctions = map[string]func() int{ } opts.Clean.Args.Targets = core.WholeGraph } - if success, state := runBuild(opts.Clean.Args.Targets, false, false, false); success { + if success, state := runBuild(opts.Clean.Args.Targets, buildOpts{}); success { clean.Targets(state, state.ExpandOriginalLabels()) return 0 } @@ -708,7 +708,7 @@ var buildFunctions = map[string]func() int{ return 1 }, "gc": func() int { - success, state := runBuild(core.WholeGraph, false, false, true) + success, state := runBuild(core.WholeGraph, buildOpts{IsQuery: true}) if success { gc.GarbageCollect(state, opts.Gc.Args.Targets, state.ExpandLabels(state.Config.Gc.Keep), state.Config.Gc.Keep, state.Config.Gc.KeepLabel, opts.Gc.Conservative, opts.Gc.TargetsOnly, opts.Gc.SrcsOnly, opts.Gc.NoPrompt, opts.Gc.DryRun, opts.Gc.Git) @@ -767,14 +767,14 @@ var buildFunctions = map[string]func() int{ return 0 }, "export": func() int { - success, state := runBuild(opts.Export.Args.Targets, false, false, false) + success, state := runBuild(opts.Export.Args.Targets, buildOpts{ParseMetadata: true}) if success { export.Repo(state, opts.Export.Output, opts.Export.NoTrim, state.ExpandOriginalLabels()) } return toExitCode(success, state) }, "export.outputs": func() int { - success, state := runBuild(opts.Export.Outputs.Args.Targets, true, false, true) + success, state := runBuild(opts.Export.Outputs.Args.Targets, buildOpts{Build: true, IsQuery: true, ParseMetadata: true}) if success { export.Outputs(state, opts.Export.Output, state.ExpandOriginalLabels()) } @@ -962,14 +962,14 @@ var buildFunctions = map[string]func() int{ log.Fatalf("%s", err) } readConfig() - _, before := runBuild(core.WholeGraph, false, false, false) + _, before := runBuild(core.WholeGraph, buildOpts{}) // N.B. Ignore failure here; if we can't parse the graph before then it will suffice to // assume that anything we don't know about has changed. if err := scm.Checkout(original); err != nil { log.Fatalf("%s", err) } readConfig() - success, after := runBuild(core.WholeGraph, false, false, false) + success, after := runBuild(core.WholeGraph, buildOpts{}) if !success { return 1 } @@ -1001,7 +1001,7 @@ var buildFunctions = map[string]func() int{ "watch": func() int { targets, args := testTargets(opts.Watch.Args.Target, opts.Watch.Args.Args, false, "") // Don't ask it to test now since we don't know if any of them are tests yet. - success, state := runBuild(targets, true, false, false) + success, state := runBuild(targets, buildOpts{Build: true}) state.NeedRun = opts.Watch.Run watch.Watch(state, state.ExpandOriginalLabels(), args, opts.Watch.NoTest, runPlease) return toExitCode(success, state) @@ -1026,7 +1026,7 @@ var buildFunctions = map[string]func() int{ opts.Generate.Args.Targets = []core.BuildLabel{target} } - if success, state := runBuild(opts.Generate.Args.Targets, true, false, true); success { + if success, state := runBuild(opts.Generate.Args.Targets, buildOpts{Build: true, IsQuery: true}); success { if opts.Generate.Gitignore != "" { err := generate.UpdateGitignore(state.Graph, state.ExpandOriginalLabels(), opts.Generate.Gitignore) if err != nil { @@ -1068,7 +1068,7 @@ func runTool(_tool tool.Tool) int { // We skip loading the repo config in init for `plz tool` to allow this command to work outside of a repo root. If // the tool looks like a build label, we need to set the repo root now. config = mustReadConfigAndSetRoot(false) - if success, state := runBuild(label, true, false, false); success { + if success, state := runBuild(label, buildOpts{Build: true}); success { annotatedOutputLabels := core.AnnotateLabels(label) run.Run(state, annotatedOutputLabels[0], opts.Tool.Args.Args.AsStrings(), false, false, false, "", "") } @@ -1100,7 +1100,7 @@ func runQuery(needFullParse bool, labels []core.BuildLabel, onSuccess func(state if len(labels) == 0 { labels = core.WholeGraph } - if success, state := runBuild(labels, false, false, true); success { + if success, state := runBuild(labels, buildOpts{IsQuery: true}); success { onSuccess(state) return 0 } @@ -1112,7 +1112,7 @@ func doTest(targets []core.BuildLabel, args []string, surefireDir cli.Filepath, fs.RemoveAll(string(resultsFile)) os.MkdirAll(string(surefireDir), core.DirPermissions) opts.Test.StateArgs = args - success, state := runBuild(targets, true, true, false) + success, state := runBuild(targets, buildOpts{Build: true, Test: true}) test.CopySurefireXMLFilesToDir(state, string(surefireDir)) test.WriteResultsToFileOrDie(state.Graph, string(resultsFile), state.Config.Test.StoreTestOutputOnSuccess) return success, state @@ -1127,7 +1127,7 @@ func prettyOutput(interactiveOutput bool, plainOutput bool, verbosity cli.Verbos } // Please starts & runs the main build process through to its completion. -func Please(targets []core.BuildLabel, config *core.Configuration, shouldBuild, shouldTest bool) (bool, *core.BuildState) { +func Please(targets []core.BuildLabel, config *core.Configuration, buildOpts buildOpts) (bool, *core.BuildState) { if opts.BuildFlags.NumThreads > 0 { config.Please.NumThreads = opts.BuildFlags.NumThreads config.Parse.NumThreads = opts.BuildFlags.NumThreads @@ -1150,8 +1150,6 @@ func Please(targets []core.BuildLabel, config *core.Configuration, shouldBuild, state.TestSequentially = opts.Test.Sequentially || opts.Cover.Sequentially // Similarly here. state.TestArgs = opts.Test.StateArgs state.NeedCoverage = opts.Cover.active || config.Build.Config == "cover" - state.NeedBuild = shouldBuild - state.NeedTests = shouldTest state.NeedRun = !opts.Run.Args.Target.IsEmpty() || len(opts.Run.Parallel.PositionalArgs.Targets) > 0 || len(opts.Run.Sequential.PositionalArgs.Targets) > 0 || !opts.Exec.Args.Target.IsEmpty() || len(opts.Exec.Sequential.Args.Targets) > 0 || len(opts.Exec.Parallel.Args.Targets) > 0 || opts.Tool.Args.Tool != "" || debug state.NeedHashesOnly = len(opts.Hash.Args.Targets) > 0 state.PrepareOnly = opts.Build.Shell != "" || opts.Test.Shell != "" || opts.Cover.Shell != "" @@ -1165,6 +1163,9 @@ func Please(targets []core.BuildLabel, config *core.Configuration, shouldBuild, state.ShowAllOutput = opts.OutputFlags.ShowAllOutput state.ParsePackageOnly = opts.ParsePackageOnly state.EnableBreakpoints = opts.BehaviorFlags.Debug + state.NeedBuild = buildOpts.Build + state.NeedTests = buildOpts.Test + state.ParseMetadata = buildOpts.ParseMetadata state.NeedDebugDeps = debug // What outputs get downloaded in remote execution. @@ -1317,10 +1318,18 @@ func readConfig() *core.Configuration { return cfg } +// buildOpts specifies parameter for the core.runBuild method. +type buildOpts struct { + Build bool + Test bool + IsQuery bool + ParseMetadata bool +} + // Runs the actual build // Which phases get run are controlled by shouldBuild and shouldTest. -func runBuild(targets []core.BuildLabel, shouldBuild, shouldTest, isQuery bool) (bool, *core.BuildState) { - if !isQuery { +func runBuild(targets []core.BuildLabel, buildOpts buildOpts) (bool, *core.BuildState) { + if !buildOpts.IsQuery { opts.BuildFlags.Exclude = append(opts.BuildFlags.Exclude, "manual", "manual:"+core.OsArch) } if stat, _ := os.Stdin.Stat(); (stat.Mode()&os.ModeCharDevice) == 0 && !plz.ReadingStdin(targets) { @@ -1332,7 +1341,7 @@ func runBuild(targets []core.BuildLabel, shouldBuild, shouldTest, isQuery bool) if len(targets) == 0 { targets = core.InitialPackage() } - return Please(targets, config, shouldBuild, shouldTest) + return Please(targets, config, buildOpts) } var originalWorkingDirectory string @@ -1430,7 +1439,7 @@ func getCompletions(qry string) (*query.CompletionPackages, []string) { if completions.PackageToParse != "" || completions.IsRoot { labelsToParse := []core.BuildLabel{{PackageName: completions.PackageToParse, Name: "all"}} - if success, state := Please(labelsToParse, config, false, false); success { + if success, state := Please(labelsToParse, config, buildOpts{}); success { return completions, query.Completions(state.Graph, completions, binary, isTest, completions.Hidden) } } From 5aeb36077a67a59d9634ac6eec5e9740d932fae0 Mon Sep 17 00:00:00 2001 From: duarte Date: Fri, 1 May 2026 14:13:42 +0100 Subject: [PATCH 033/118] test: minimal subinclude statement --- src/export/export_test.go | 63 ++++++++++++++++++++++++++++----------- 1 file changed, 46 insertions(+), 17 deletions(-) diff --git a/src/export/export_test.go b/src/export/export_test.go index 0b7c7b0003..58bad1c822 100644 --- a/src/export/export_test.go +++ b/src/export/export_test.go @@ -7,26 +7,55 @@ import ( "github.com/thought-machine/please/src/core" ) -func TestMakeSubincludesStatement(t *testing.T) { - e := &export{ - requiredSubincludes: map[*core.Package]map[core.BuildLabel]bool{}, +func TestMinimalSubincludeStatement(t *testing.T) { + var subincludesTests = []struct { + name string + availableLabels []core.BuildLabel + requiredLabels []core.BuildLabel + out string + }{ + { + "Successful no pruning subinclude", + core.ParseBuildLabels([]string{"//build_defs:test"}), + core.ParseBuildLabels([]string{"//build_defs:test"}), + `subinclude("//build_defs:test")`, + }, + { + "No subincludes", + nil, + nil, + "", + }, + { + "Single subinclude (not required)", + core.ParseBuildLabels([]string{"//build_defs:other"}), + nil, + "", + }, + { + "Multiple subincludes (sorted and filtered)", + core.ParseBuildLabels([]string{"//build_defs:test", "//build_defs:abc", "//build_defs:other"}), + core.ParseBuildLabels([]string{"//build_defs:test", "//build_defs:abc"}), + "subinclude(\n" + + " \"//build_defs:abc\",\n" + + " \"//build_defs:test\",\n" + + ")", + }, } - pkg := &core.Package{Name: "test"} + for _, tt := range subincludesTests { + t.Run(tt.name, func(t *testing.T) { + e := &DefaultExporter{ + requiredSubincludes: map[*core.Package]map[core.BuildLabel]bool{}, + } - // Test case 1: No subincludes - assert.Equal(t, "", e.makeSubincludesStatement(pkg)) + pkg := &core.Package{Name: "test"} + e.requiredSubincludes[pkg] = map[core.BuildLabel]bool{} + for _, labels := range tt.requiredLabels { + e.requiredSubincludes[pkg][labels] = true + } - // Test case 2: Single subinclude - label1 := core.ParseBuildLabel("//build_defs:test", "") - e.requiredSubincludes[pkg] = map[core.BuildLabel]bool{ - label1: true, + assert.Equal(t, tt.out, e.minimalSubincludeStatement(pkg, tt.availableLabels)) + }) } - assert.Equal(t, `subinclude("//build_defs:test")`, e.makeSubincludesStatement(pkg)) - - // Test case 3: Multiple subincludes (sorted) - label2 := core.ParseBuildLabel("//build_defs:abc", "") - e.requiredSubincludes[pkg][label2] = true - expected := "subinclude(\n \"//build_defs:abc\",\n \"//build_defs:test\",\n)" - assert.Equal(t, expected, e.makeSubincludesStatement(pkg)) } From 517486a87f059469c7764e5fd2f49ccc84af02d4 Mon Sep 17 00:00:00 2001 From: duarte Date: Fri, 1 May 2026 15:02:28 +0100 Subject: [PATCH 034/118] collect map keys on active subinclude labels --- src/parse/asp/interpreter.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/parse/asp/interpreter.go b/src/parse/asp/interpreter.go index 330ddde73a..c998e8b50b 100644 --- a/src/parse/asp/interpreter.go +++ b/src/parse/asp/interpreter.go @@ -4,11 +4,13 @@ import ( "context" "fmt" "iter" + "maps" "path/filepath" "reflect" "regexp" "runtime/debug" "runtime/pprof" + "slices" "strings" "sync" @@ -1100,20 +1102,16 @@ func (s *scope) CurrentBuildStatement() core.BuildStatementProvider { // provided the macros/functions actively executing to define this target. func (s *scope) ActiveSubincludes() core.SubincludesLabelProvider { return func() core.BuildLabels { - var subincludes []core.BuildLabel seen := map[core.BuildLabel]bool{} for curr := s; curr != nil; curr = curr.callerScope { for localScope := curr; localScope != nil; localScope = localScope.parent { if localScope.subincludeLabel != nil { label := *localScope.subincludeLabel - if !seen[label] { - seen[label] = true - subincludes = append(subincludes, label) - } + seen[label] = true } } } - return subincludes + return slices.Collect(maps.Keys(seen)) } } From f8f8e3f35fdb38a608afbec69b8e630189cac837 Mon Sep 17 00:00:00 2001 From: duarte Date: Fri, 1 May 2026 15:03:37 +0100 Subject: [PATCH 035/118] non-fatal warning for missing source while exporting --- src/export/export.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/export/export.go b/src/export/export.go index 07f6f53744..f0e1f507cd 100644 --- a/src/export/export.go +++ b/src/export/export.go @@ -138,7 +138,7 @@ func (be *baseExporter) Sources(target *core.BuildTarget) { for _, p := range src.Paths(be.state.Graph) { if !filepath.IsAbs(p) { // Don't copy system file deps. if err := fs.RecursiveCopy(p, filepath.Join(be.targetDir, p), 0); err != nil { - log.Fatalf("Error copying file: %s\n", err) + log.Warningf("Error copying file, skipping...: %s", err) } log.Infof("Writing exported source file: %s", p) } @@ -192,6 +192,8 @@ func (e *DefaultExporter) Target(target *core.BuildTarget) { return } + log.Infof("Exporting target: %v", target.Label) + // We want to export the package that made this subrepo available, but we still need to walk the // target deps as it may depend on other subrepos or first party targets if target.Subrepo != nil { From 8aa98e3d5d63266bcaca429bf5dea6eaf56446b1 Mon Sep 17 00:00:00 2001 From: duarte Date: Fri, 1 May 2026 15:07:48 +0100 Subject: [PATCH 036/118] skip internal package export unclear if this is necessary --- src/export/export.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/export/export.go b/src/export/export.go index f0e1f507cd..4664cf3ecb 100644 --- a/src/export/export.go +++ b/src/export/export.go @@ -194,6 +194,10 @@ func (e *DefaultExporter) Target(target *core.BuildTarget) { log.Infof("Exporting target: %v", target.Label) + // Skip export for internal packages + if target.Label.PackageName == parse.InternalPackageName { + return + } // We want to export the package that made this subrepo available, but we still need to walk the // target deps as it may depend on other subrepos or first party targets if target.Subrepo != nil { @@ -235,11 +239,6 @@ func (e *DefaultExporter) Subincludes(pkg *core.Package, target *core.BuildTarge // BuildStatements exports BUILD statements that generate the build target. func (e *DefaultExporter) BuildStatements(pkg *core.Package, target *core.BuildTarget) { - if target.Label.PackageName == parse.InternalPackageName { - // TODO validate if we still need this - return - } - stmt, err := pkg.FindStatement(target) if err != nil { log.Fatalf("Failed to find statement in %s: %w", pkg.Name, err) From 5fa64f4bbb46b82425ebf68f37d6524f399b806d Mon Sep 17 00:00:00 2001 From: duarte Date: Fri, 1 May 2026 15:20:26 +0100 Subject: [PATCH 037/118] move some fatal to error and continue --- src/export/export.go | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/src/export/export.go b/src/export/export.go index 4664cf3ecb..af0f0045d9 100644 --- a/src/export/export.go +++ b/src/export/export.go @@ -5,6 +5,7 @@ package export import ( "bytes" + "fmt" "os" "path/filepath" "slices" @@ -203,7 +204,6 @@ func (e *DefaultExporter) Target(target *core.BuildTarget) { if target.Subrepo != nil { e.Target(target.Subrepo.Target) e.Dependencies(target) - // TODO do we need to walk build statements or subincludes? return } @@ -241,15 +241,17 @@ func (e *DefaultExporter) Subincludes(pkg *core.Package, target *core.BuildTarge func (e *DefaultExporter) BuildStatements(pkg *core.Package, target *core.BuildTarget) { stmt, err := pkg.FindStatement(target) if err != nil { - log.Fatalf("Failed to find statement in %s: %w", pkg.Name, err) + log.Errorf("Failed to find statement in %s: %w", pkg.Name, err) + return } relatedTargets, err := pkg.FindRelatedTargets(stmt) if err != nil { - log.Fatalf("Failed to lookup related targets for package %s: %w", pkg.Name, err) + log.Errorf("Failed to lookup related targets for package %s: %w", pkg.Name, err) + return } - log.Infof("Exporting related targets to (%v): %v", target.Label, relatedTargets) + log.Infof("Exporting related targets to (%v): %v", target.Label, relatedTargets) for _, target := range relatedTargets { e.Target(target) } @@ -265,7 +267,8 @@ func (e *DefaultExporter) WritePackageFiles() { // filter filteredBytes, err := e.FilterPackageFile(pkg) if err != nil { - log.Fatalf("Failed to filter the build statements of package %s: %v", pkg.Label(), err) + log.Errorf("Failed to filter the build statements of package %s: %v", pkg.Label(), err) + continue } // format @@ -276,7 +279,8 @@ func (e *DefaultExporter) WritePackageFiles() { file := e.OpenExportedPackageFile(pkg) defer file.Close() if _, err := file.Write(formattedBytes); err != nil { - log.Fatalf("Failed to write to exported BUILD file %s: %v", file.Name(), err) + log.Errorf("Failed to write to exported BUILD file %s: %v", file.Name(), err) + continue } } } @@ -297,12 +301,12 @@ func (e *DefaultExporter) FilterPackageFile(pkg *core.Package) ([]byte, error) { p := asp.NewParserOnly() parsedStmts, err := p.ParseFileOnly(pkg.Filename) if err != nil { - log.Fatalf("Failed to parse original BUILD file: %v", err) + return nil, fmt.Errorf("Parsing original BUILD file: %v", err) } original, err := os.ReadFile(pkg.Filename) if err != nil { - log.Fatalf("Failed to open original BUILD file: %v", err) + return nil, fmt.Errorf("Opening original BUILD file: %v", err) } cursor := 0 @@ -340,7 +344,7 @@ func (e *DefaultExporter) FilterPackageFile(pkg *core.Package) ([]byte, error) { } // Write the rest of the original file (non build targets) - if buffer.Write(original[cursor:]); err != nil { + if _, err := buffer.Write(original[cursor:]); err != nil { return nil, err } @@ -397,7 +401,7 @@ func (nte *NoTrimExporter) Preloaded() { // Write any preloaded build defs for _, preload := range nte.state.Config.Parse.PreloadBuildDefs { if err := fs.RecursiveCopy(preload, filepath.Join(nte.targetDir, preload), 0); err != nil { - log.Fatalf("Failed to copy preloaded build def %s: %s", preload, err) + log.Errorf("Failed to copy preloaded build def %s: %s", preload, err) } } @@ -420,7 +424,6 @@ func (nte *NoTrimExporter) Target(target *core.BuildTarget) { if target.Subrepo != nil { nte.Target(target.Subrepo.Target) nte.Dependencies(target) - // TODO do we need to walk build statements or subincludes? return } @@ -446,7 +449,7 @@ func (nte *NoTrimExporter) Package(pkg *core.Package) { exportedFilename := filepath.Join(nte.targetDir, pkgFilename) if err := fs.CopyFile(pkgFilename, exportedFilename, 0); err != nil { - log.Fatalf("failed to export package %s: %v", pkgName, err) + log.Errorf("failed to export package %s: %v", pkgName, err) } } From e60dbb66bab4011f182796d1bff757ee5d3d4307 Mon Sep 17 00:00:00 2001 From: duarte Date: Fri, 1 May 2026 15:22:29 +0100 Subject: [PATCH 038/118] missing docstrings --- src/core/package_metadata.go | 7 +++++++ src/export/export.go | 11 ++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/core/package_metadata.go b/src/core/package_metadata.go index c5181cf32b..c9984500c5 100644 --- a/src/core/package_metadata.go +++ b/src/core/package_metadata.go @@ -36,12 +36,19 @@ type SubincludesLabelProvider func() BuildLabels // PackageMetadata stores metadata about parsed BUILD files, mapping statements and subincludes // to their respective targets. type PackageMetadata interface { + // RegisterStatementTarget maps a build statement to a target it generated. RegisterStatementTarget(target *BuildTarget, stmtProvider BuildStatementProvider) + // RegisterRequiredSubinclude maps a target to the subincludes required to build it. RegisterRequiredSubinclude(target *BuildTarget, labelProvider SubincludesLabelProvider) + // RegisterSubincludeStmt maps a subinclude statement to a label it includes. RegisterSubincludeStmt(label BuildLabel, stmtProvider BuildStatementProvider) + // FindStatement returns the build statement that generated the given target. FindStatement(target *BuildTarget) (*BuildStatement, error) + // FindTargets returns all targets generated by the given build statement. FindTargets(stmt *BuildStatement) ([]*BuildTarget, error) + // FindRequiredSubincludes returns all subinclude labels required by the given target. FindRequiredSubincludes(target *BuildTarget) (BuildLabels, error) + // GetSubincludedLabels returns the labels included by a given subinclude statement. GetSubincludedLabels(stmt *BuildStatement) (BuildLabels, bool) } diff --git a/src/export/export.go b/src/export/export.go index af0f0045d9..bc360c76fb 100644 --- a/src/export/export.go +++ b/src/export/export.go @@ -23,10 +23,15 @@ import ( var log = logging.Log type Exporter interface { + // PlzConfig exports the repo configuration files. PlzConfig() + // Preloaded exports the preloaded targets, build defs and subincludes. Preloaded() + // Targets exports all targets for the given labels. Targets(core.BuildLabels) + // Target exports an individual target and its dependencies. Target(target *core.BuildTarget) + // WritePackageFiles writes the processed BUILD files to the export directory. WritePackageFiles() } @@ -162,7 +167,9 @@ func (be *baseExporter) checkFirstExport(pkg *core.Package, target *core.BuildTa // DefaultExporter implements an exporter that trims packages to reach a minimal exported repo. type DefaultExporter struct { baseExporter - requiredSubincludes map[*core.Package]map[core.BuildLabel]bool + // requiredSubincludes maps packages to the subinclude labels they require. + requiredSubincludes map[*core.Package]map[core.BuildLabel]bool + // preloadedSubincludes tracks subincludes that are preloaded and don't need explicit export. preloadedSubincludes map[core.BuildLabel]bool } @@ -394,9 +401,11 @@ func (e *DefaultExporter) minimalSubincludeStatement(pkg *core.Package, availabl // and statements in a package. type NoTrimExporter struct { baseExporter + // exportedPackages tracks which packages have already had their BUILD files exported. exportedPackages map[string]bool } +// Preloaded exports the preloaded targets, build defs and subincludes. func (nte *NoTrimExporter) Preloaded() { // Write any preloaded build defs for _, preload := range nte.state.Config.Parse.PreloadBuildDefs { From c7409ab82f80fbb22ad29df5b925192758c22521 Mon Sep 17 00:00:00 2001 From: duarte Date: Fri, 1 May 2026 15:27:54 +0100 Subject: [PATCH 039/118] run go fmt and plz fmt --- src/parse/asp/targets.go | 1 - 1 file changed, 1 deletion(-) diff --git a/src/parse/asp/targets.go b/src/parse/asp/targets.go index c1785fbe43..03bf3a3537 100644 --- a/src/parse/asp/targets.go +++ b/src/parse/asp/targets.go @@ -191,7 +191,6 @@ func createTarget(s *scope, args []pyObject) *core.BuildTarget { return target } - // validateSandbox ensures that the target isn't opting out of the build/test sandbox when it's not allowed to func validateSandbox(state *core.BuildState, target *core.BuildTarget) error { if target.IsFilegroup || len(state.Config.Sandbox.ExcludeableTargets) == 0 { From be870206c6614322aa1ad1a5cf12940b0fa2b752 Mon Sep 17 00:00:00 2001 From: duarte Date: Fri, 1 May 2026 16:08:32 +0100 Subject: [PATCH 040/118] move to error and continue for target lookup --- src/export/export.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/export/export.go b/src/export/export.go index bc360c76fb..8b427bdf90 100644 --- a/src/export/export.go +++ b/src/export/export.go @@ -123,7 +123,11 @@ func (be *baseExporter) PlzConfig() { // Targets exports all targets for the given labels. func (be *baseExporter) Targets(labels core.BuildLabels) { for _, l := range labels { - target := be.state.Graph.TargetOrDie(l) + target := be.state.Graph.Target(l) + if target == nil { + log.Errorf("Unable to lookup target %s", l) + continue + } be.impl.Target(target) } } @@ -240,7 +244,12 @@ func (e *DefaultExporter) Subincludes(pkg *core.Package, target *core.BuildTarge } e.requiredSubincludes[pkg][subinclude] = true - e.Target(e.state.Graph.TargetOrDie(subinclude)) + target := e.state.Graph.Target(subinclude) + if target == nil { + log.Errorf("Unable to lookup target %s", subinclude) + continue + } + e.Target(target) } } From 6f73f7824adddb6c1959e729ac05b7e90ee17f1a Mon Sep 17 00:00:00 2001 From: duarte Date: Fri, 1 May 2026 16:10:49 +0100 Subject: [PATCH 041/118] rename new parser method on test files --- src/parse/asp/parser_test.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/parse/asp/parser_test.go b/src/parse/asp/parser_test.go index fd534b31bc..1089bcc012 100644 --- a/src/parse/asp/parser_test.go +++ b/src/parse/asp/parser_test.go @@ -10,7 +10,7 @@ import ( // TODO(peterebden): Might get rid of this, we may want to expose a similar thing on Parser. func parseFileOnly(filename string) (*File, []*Statement, error) { - stmts, err := newParser().ParseFileOnly(filename) + stmts, err := NewParserOnly().ParseFileOnly(filename) return newFile(filename), stmts, err } @@ -448,7 +448,7 @@ func TestAssert(t *testing.T) { } func TestOptimise(t *testing.T) { - p := newParser() + p := NewParserOnly() statements, err := p.parse(nil, "src/parse/asp/test_data/optimise.build") f := newFile("src/parse/asp/test_data/optimise.build") assert.NoError(t, err) @@ -486,7 +486,7 @@ func TestOptimise(t *testing.T) { } func TestOptimiseJoin(t *testing.T) { - p := newParser() + p := NewParserOnly() statements, err := p.parse(nil, "src/parse/asp/test_data/optimise_join.build") assert.NoError(t, err) assert.Equal(t, 1, len(statements)) @@ -658,12 +658,12 @@ func TestMissingNewlines(t *testing.T) { } func TestRepeatedArguments(t *testing.T) { - _, err := newParser().parse(nil, "src/parse/asp/test_data/repeated_arguments.build") + _, err := NewParserOnly().parse(nil, "src/parse/asp/test_data/repeated_arguments.build") assert.Error(t, err) } func TestConstantAssignments(t *testing.T) { - _, err := newParser().parse(nil, "src/parse/asp/test_data/constant_assign.build") + _, err := NewParserOnly().parse(nil, "src/parse/asp/test_data/constant_assign.build") assert.Error(t, err) } @@ -815,7 +815,7 @@ func TestFStringConcat(t *testing.T) { func TestFStringImplicitStringConcat(t *testing.T) { str := "str('testing that we can carry these ' f'over {multiple} lines' r' \\n')" - prog, err := newParser().parseAndHandleErrors(strings.NewReader(strings.ReplaceAll(str, "\t", ""))) + prog, err := NewParserOnly().parseAndHandleErrors(strings.NewReader(strings.ReplaceAll(str, "\t", ""))) require.NoError(t, err) fString := prog[0].Ident.Action.Call.Arguments[0].Value.Val.FString @@ -827,21 +827,21 @@ func TestFStringImplicitStringConcat(t *testing.T) { // F strings should report a sensible error when the {} aren't complete func TestFStringIncompleteError(t *testing.T) { str := "s = f'some {' '.join([])}'" - _, err := newParser().parseAndHandleErrors(strings.NewReader(str)) + _, err := NewParserOnly().parseAndHandleErrors(strings.NewReader(str)) require.Error(t, err) assert.Contains(t, err.Error(), "Unterminated brace in fstring") } // Continue shouldn't be allowed outside a loop func TestContinueOutsideLoop(t *testing.T) { - _, err := newParser().parseAndHandleErrors(strings.NewReader("continue")) + _, err := NewParserOnly().parseAndHandleErrors(strings.NewReader("continue")) require.Error(t, err) assert.Contains(t, err.Error(), "'continue' outside loop") } // Break shouldn't be allowed outside a loop func TestBreakOutsideLoop(t *testing.T) { - _, err := newParser().parseAndHandleErrors(strings.NewReader("break")) + _, err := NewParserOnly().parseAndHandleErrors(strings.NewReader("break")) require.Error(t, err) assert.Contains(t, err.Error(), "'break' outside loop") } @@ -853,7 +853,7 @@ for i in [1,2,3]: def foo(): break ` - _, err := newParser().parseAndHandleErrors(strings.NewReader(code)) + _, err := NewParserOnly().parseAndHandleErrors(strings.NewReader(code)) require.Error(t, err) assert.Contains(t, err.Error(), "'break' outside loop") } From 762c016ad35a4b4d869e74ac8878d7eedecf889a Mon Sep 17 00:00:00 2001 From: DuBento Date: Mon, 11 May 2026 16:03:06 +0100 Subject: [PATCH 042/118] use pkg.Metadata directly and remove intermediate pkg methods --- src/core/package.go | 56 +++++------------------------------- src/core/package_metadata.go | 2 +- src/export/export.go | 10 +++---- src/parse/asp/builtins.go | 6 ++-- 4 files changed, 16 insertions(+), 58 deletions(-) diff --git a/src/core/package.go b/src/core/package.go index d9a9edea98..9cc8a85023 100644 --- a/src/core/package.go +++ b/src/core/package.go @@ -35,7 +35,7 @@ type Package struct { // Set of output files from rules. Outputs map[string]*BuildTarget // Includes metadata from parsing the package BUILD file. - BuildFileMetadata PackageMetadata + Metadata PackageMetadata // Protects access to above mutex sync.RWMutex } @@ -50,7 +50,7 @@ func NewPackage(name string) *Package { func NewPackageWithOpts(state *BuildState, name string) *Package { pkg := NewPackage(name) if state.ParseMetadata { - pkg.BuildFileMetadata = newBuildFileMetadata() + pkg.Metadata = newPackageMetadata() } return pkg } @@ -58,11 +58,11 @@ func NewPackageWithOpts(state *BuildState, name string) *Package { // NewPackageSubrepo constructs a new package with the given name and subrepo. func NewPackageSubrepo(name, subrepo string) *Package { return &Package{ - Name: name, - SubrepoName: subrepo, - targets: map[string]*BuildTarget{}, - Outputs: map[string]*BuildTarget{}, - BuildFileMetadata: newNoopPackageMetadata(), + Name: name, + SubrepoName: subrepo, + targets: map[string]*BuildTarget{}, + Outputs: map[string]*BuildTarget{}, + Metadata: newNoopPackageMetadata(), } } @@ -243,48 +243,6 @@ func (pkg *Package) verifyOutputs() []string { return ret } -// RegisterStatement maps a build statement to target in the package. -func (pkg *Package) RegisterStatement(target *BuildTarget, stmtProvider BuildStatementProvider) { - pkg.mutex.Lock() - defer pkg.mutex.Unlock() - pkg.BuildFileMetadata.RegisterStatementTarget(target, stmtProvider) -} - -// RegisterRequiredSubincludes maps the required subincludes to generate the target. -func (pkg *Package) RegisterRequiredSubincludes(target *BuildTarget, labelProvider SubincludesLabelProvider) { - pkg.mutex.Lock() - defer pkg.mutex.Unlock() - pkg.BuildFileMetadata.RegisterRequiredSubinclude(target, labelProvider) -} - -// RegisterSubincludeStmt maps the subincludes build statement to the included targets. -func (pkg *Package) RegisterSubincludeStmt(label BuildLabel, stmtProvider BuildStatementProvider) { - pkg.mutex.Lock() - defer pkg.mutex.Unlock() - pkg.BuildFileMetadata.RegisterSubincludeStmt(label, stmtProvider) -} - -// FindStatement finds the build statement that generated the target. -func (pkg *Package) FindStatement(target *BuildTarget) (*BuildStatement, error) { - return pkg.BuildFileMetadata.FindStatement(target) -} - -// FindRelatedTargets finds all the targets related to the build statement. -func (pkg *Package) FindRelatedTargets(stmt *BuildStatement) ([]*BuildTarget, error) { - return pkg.BuildFileMetadata.FindTargets(stmt) -} - -// FindRequiredSubincludes finds the subincluded target labels required by the given target. -func (pkg *Package) FindRequiredSubincludes(target *BuildTarget) (BuildLabels, error) { - return pkg.BuildFileMetadata.FindRequiredSubincludes(target) -} - -// GetSubincludedLabels returns the labels subincluded by the given statement and true if it -// is a subinclude. -func (pkg *Package) GetSubincludedLabels(stmt *BuildStatement) (BuildLabels, bool) { - return pkg.BuildFileMetadata.GetSubincludedLabels(stmt) -} - // FindOwningPackages returns build labels corresponding to the packages that own each of the given files. func FindOwningPackages(state *BuildState, files []string) []BuildLabel { ret := make([]BuildLabel, len(files)) diff --git a/src/core/package_metadata.go b/src/core/package_metadata.go index c9984500c5..cad5508b9a 100644 --- a/src/core/package_metadata.go +++ b/src/core/package_metadata.go @@ -63,7 +63,7 @@ type packageMetadataImpl struct { LabelsPerSubincludeStmt map[BuildStatement]BuildLabels } -func newBuildFileMetadata() PackageMetadata { +func newPackageMetadata() PackageMetadata { return &packageMetadataImpl{ StmtToTarget: map[BuildStatement][]*BuildTarget{}, TargetToSubinclude: map[*BuildTarget]BuildLabels{}, diff --git a/src/export/export.go b/src/export/export.go index 8b427bdf90..e0c925344f 100644 --- a/src/export/export.go +++ b/src/export/export.go @@ -227,7 +227,7 @@ func (e *DefaultExporter) Target(target *core.BuildTarget) { // Subincludes exports the subincluded targets required to generate the target and selects them to // later be written to the package as statements. func (e *DefaultExporter) Subincludes(pkg *core.Package, target *core.BuildTarget) { - subincludes, err := pkg.FindRequiredSubincludes(target) + subincludes, err := pkg.Metadata.FindRequiredSubincludes(target) if err != nil { log.Infof("No subincludes found, assuming non required.: %w", pkg.Name, err) return @@ -255,13 +255,13 @@ func (e *DefaultExporter) Subincludes(pkg *core.Package, target *core.BuildTarge // BuildStatements exports BUILD statements that generate the build target. func (e *DefaultExporter) BuildStatements(pkg *core.Package, target *core.BuildTarget) { - stmt, err := pkg.FindStatement(target) + stmt, err := pkg.Metadata.FindStatement(target) if err != nil { log.Errorf("Failed to find statement in %s: %w", pkg.Name, err) return } - relatedTargets, err := pkg.FindRelatedTargets(stmt) + relatedTargets, err := pkg.Metadata.FindTargets(stmt) if err != nil { log.Errorf("Failed to lookup related targets for package %s: %w", pkg.Name, err) return @@ -339,7 +339,7 @@ func (e *DefaultExporter) FilterPackageFile(pkg *core.Package) ([]byte, error) { cursor = bStmt.Start } - if stmtLabels, ok := pkg.GetSubincludedLabels(bStmt); ok { + if stmtLabels, ok := pkg.Metadata.GetSubincludedLabels(bStmt); ok { // Write filtered subincludes subStmt := e.minimalSubincludeStatement(pkg, stmtLabels) buffer.Write([]byte(subStmt)) @@ -369,7 +369,7 @@ func (e *DefaultExporter) FilterPackageFile(pkg *core.Package) ([]byte, error) { // isRequiredStatement evaluates if the current build statement is required by the export. func (e *DefaultExporter) isRequiredStatement(pkg *core.Package, stmt *core.BuildStatement) (bool, error) { - targets, err := pkg.FindRelatedTargets(stmt) + targets, err := pkg.Metadata.FindTargets(stmt) if err != nil { return false, err } diff --git a/src/parse/asp/builtins.go b/src/parse/asp/builtins.go index c097a87cf7..30c7aa706d 100644 --- a/src/parse/asp/builtins.go +++ b/src/parse/asp/builtins.go @@ -210,8 +210,8 @@ func buildRule(s *scope, args []pyObject) pyObject { s.Assert(s.pkg.Target(target.Label.Name) == nil, "Duplicate build target in %s: %s", s.pkg.Name, target.Label.Name) populateTarget(s, target, args) s.state.AddTarget(s.pkg, target) - s.pkg.RegisterStatement(target, s.CurrentBuildStatement()) - s.pkg.RegisterRequiredSubincludes(target, s.ActiveSubincludes()) + s.pkg.Metadata.RegisterStatementTarget(target, s.CurrentBuildStatement()) + s.pkg.Metadata.RegisterRequiredSubinclude(target, s.ActiveSubincludes()) if s.Callback { target.AddedPostBuild = true @@ -419,7 +419,7 @@ func subincludeTarget(s *scope, l core.BuildLabel) *core.BuildTarget { t = s.WaitForSubincludedTarget(l, pkgLabel) if s.pkg != nil { s.pkg.RegisterSubinclude(l) - s.pkg.RegisterSubincludeStmt(l, s.CurrentBuildStatement()) + s.pkg.Metadata.RegisterSubincludeStmt(l, s.CurrentBuildStatement()) } else if s.subincludeLabel != nil { // If this is nil, that indicates a preloadedSubinclude s.state.Graph.RegisterTransitiveSubinclude(*s.subincludeLabel, l) } From 406f95a908c294e8afe73c80e8ccc14132e100e6 Mon Sep 17 00:00:00 2001 From: DuBento Date: Tue, 12 May 2026 11:04:50 +0100 Subject: [PATCH 043/118] mutex in packagemetadata --- src/core/package_metadata.go | 52 ++++++++++++++++++++++++++---------- 1 file changed, 38 insertions(+), 14 deletions(-) diff --git a/src/core/package_metadata.go b/src/core/package_metadata.go index cad5508b9a..9ff5cfcdea 100644 --- a/src/core/package_metadata.go +++ b/src/core/package_metadata.go @@ -3,6 +3,7 @@ package core import ( "fmt" "slices" + "sync" ) // BuildStatement represents the start and end byte positions of a parsed statement in a BUILD file. @@ -61,6 +62,9 @@ type packageMetadataImpl struct { TargetToSubinclude map[*BuildTarget]BuildLabels // all the labels included for each subincludes statement LabelsPerSubincludeStmt map[BuildStatement]BuildLabels + + // Protects access to the above + mutex sync.RWMutex } func newPackageMetadata() PackageMetadata { @@ -72,31 +76,42 @@ func newPackageMetadata() PackageMetadata { } // RegisterStatementTarget maps a build statement to a target it generated. -func (bfm *packageMetadataImpl) RegisterStatementTarget(target *BuildTarget, stmtProvider BuildStatementProvider) { +func (m *packageMetadataImpl) RegisterStatementTarget(target *BuildTarget, stmtProvider BuildStatementProvider) { stmt := stmtProvider() - bfm.StmtToTarget[*stmt] = append(bfm.StmtToTarget[*stmt], target) + + m.mutex.Lock() + defer m.mutex.Unlock() + m.StmtToTarget[*stmt] = append(m.StmtToTarget[*stmt], target) } // RegisterRequiredSubinclude maps a target to the subincludes required to build it. -func (bfm *packageMetadataImpl) RegisterRequiredSubinclude(target *BuildTarget, labelProvider SubincludesLabelProvider) { +func (m *packageMetadataImpl) RegisterRequiredSubinclude(target *BuildTarget, labelProvider SubincludesLabelProvider) { labels := labelProvider() if len(labels) == 0 { log.Infof("Attempted to register empty subinclude labels for target %s", target.String()) return } - bfm.TargetToSubinclude[target] = append(bfm.TargetToSubinclude[target], labels...) + m.mutex.Lock() + defer m.mutex.Unlock() + m.TargetToSubinclude[target] = append(m.TargetToSubinclude[target], labels...) } // RegisterSubincludeStmt maps a subinclude statement to a label it includes. -func (bfm *packageMetadataImpl) RegisterSubincludeStmt(label BuildLabel, stmtProvider BuildStatementProvider) { +func (m *packageMetadataImpl) RegisterSubincludeStmt(label BuildLabel, stmtProvider BuildStatementProvider) { stmt := stmtProvider() - bfm.LabelsPerSubincludeStmt[*stmt] = append(bfm.LabelsPerSubincludeStmt[*stmt], label) + + m.mutex.Lock() + defer m.mutex.Unlock() + m.LabelsPerSubincludeStmt[*stmt] = append(m.LabelsPerSubincludeStmt[*stmt], label) } // FindStatement returns the build statement that generated the given target. -func (bfm *packageMetadataImpl) FindStatement(target *BuildTarget) (*BuildStatement, error) { - for stmt, targets := range bfm.StmtToTarget { +func (m *packageMetadataImpl) FindStatement(target *BuildTarget) (*BuildStatement, error) { + m.mutex.RLock() + defer m.mutex.RUnlock() + + for stmt, targets := range m.StmtToTarget { if slices.Contains(targets, target) { return &stmt, nil } @@ -105,8 +120,11 @@ func (bfm *packageMetadataImpl) FindStatement(target *BuildTarget) (*BuildStatem } // FindTargets returns all targets generated by the given build statement. -func (bfm *packageMetadataImpl) FindTargets(stmt *BuildStatement) ([]*BuildTarget, error) { - targets, ok := bfm.StmtToTarget[*stmt] +func (m *packageMetadataImpl) FindTargets(stmt *BuildStatement) ([]*BuildTarget, error) { + m.mutex.RLock() + defer m.mutex.RUnlock() + + targets, ok := m.StmtToTarget[*stmt] if !ok { return nil, fmt.Errorf("Targets not found for statement %v.", stmt) } @@ -114,8 +132,11 @@ func (bfm *packageMetadataImpl) FindTargets(stmt *BuildStatement) ([]*BuildTarge } // FindRequiredSubincludes returns all subinclude labels required by the given target. -func (bfm *packageMetadataImpl) FindRequiredSubincludes(target *BuildTarget) (BuildLabels, error) { - subincludes, ok := bfm.TargetToSubinclude[target] +func (m *packageMetadataImpl) FindRequiredSubincludes(target *BuildTarget) (BuildLabels, error) { + m.mutex.RLock() + defer m.mutex.RUnlock() + + subincludes, ok := m.TargetToSubinclude[target] if !ok { return nil, fmt.Errorf("Subincludes not found for target %v.", target) } @@ -123,8 +144,11 @@ func (bfm *packageMetadataImpl) FindRequiredSubincludes(target *BuildTarget) (Bu } // GetSubincludedLabels returns the labels included by a given subinclude statement. -func (bfm *packageMetadataImpl) GetSubincludedLabels(stmt *BuildStatement) (BuildLabels, bool) { - v, ok := bfm.LabelsPerSubincludeStmt[*stmt] +func (m *packageMetadataImpl) GetSubincludedLabels(stmt *BuildStatement) (BuildLabels, bool) { + m.mutex.RLock() + defer m.mutex.RUnlock() + + v, ok := m.LabelsPerSubincludeStmt[*stmt] return v, ok } From b86ff9fd6f08b8a2429be93e66a277b179d42e14 Mon Sep 17 00:00:00 2001 From: DuBento Date: Tue, 12 May 2026 11:36:56 +0100 Subject: [PATCH 044/118] update stmt provider to avoid dereference --- src/core/package_metadata.go | 6 +++--- src/export/export.go | 4 ++-- src/parse/asp/interpreter.go | 10 +++++----- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/core/package_metadata.go b/src/core/package_metadata.go index 9ff5cfcdea..f776c504cb 100644 --- a/src/core/package_metadata.go +++ b/src/core/package_metadata.go @@ -29,7 +29,7 @@ func (s BuildStatements) Swap(i, j int) { s[i], s[j] = s[j], s[i] } func (s BuildStatements) Less(i, j int) bool { return s[i].StartPos() < s[j].StartPos() } // BuildStatementProvider is a type for methods that generate new build statements. -type BuildStatementProvider func() *BuildStatement +type BuildStatementProvider func() BuildStatement // SubincludesLabelProvider is a type for methods that generate labels from a subincludes. type SubincludesLabelProvider func() BuildLabels @@ -81,7 +81,7 @@ func (m *packageMetadataImpl) RegisterStatementTarget(target *BuildTarget, stmtP m.mutex.Lock() defer m.mutex.Unlock() - m.StmtToTarget[*stmt] = append(m.StmtToTarget[*stmt], target) + m.StmtToTarget[stmt] = append(m.StmtToTarget[stmt], target) } // RegisterRequiredSubinclude maps a target to the subincludes required to build it. @@ -103,7 +103,7 @@ func (m *packageMetadataImpl) RegisterSubincludeStmt(label BuildLabel, stmtProvi m.mutex.Lock() defer m.mutex.Unlock() - m.LabelsPerSubincludeStmt[*stmt] = append(m.LabelsPerSubincludeStmt[*stmt], label) + m.LabelsPerSubincludeStmt[stmt] = append(m.LabelsPerSubincludeStmt[stmt], label) } // FindStatement returns the build statement that generated the given target. diff --git a/src/export/export.go b/src/export/export.go index e0c925344f..f2d35e5ad0 100644 --- a/src/export/export.go +++ b/src/export/export.go @@ -339,12 +339,12 @@ func (e *DefaultExporter) FilterPackageFile(pkg *core.Package) ([]byte, error) { cursor = bStmt.Start } - if stmtLabels, ok := pkg.Metadata.GetSubincludedLabels(bStmt); ok { + if stmtLabels, ok := pkg.Metadata.GetSubincludedLabels(&bStmt); ok { // Write filtered subincludes subStmt := e.minimalSubincludeStatement(pkg, stmtLabels) buffer.Write([]byte(subStmt)) log.Debugf("Decision: %s", subStmt) - } else if required, err := e.isRequiredStatement(pkg, bStmt); err == nil && !required { + } else if required, err := e.isRequiredStatement(pkg, &bStmt); err == nil && !required { // Don't write statements that generate targets we are not interested about log.Debugf("Decision: ") // skip diff --git a/src/parse/asp/interpreter.go b/src/parse/asp/interpreter.go index c998e8b50b..e627ea7707 100644 --- a/src/parse/asp/interpreter.go +++ b/src/parse/asp/interpreter.go @@ -1086,7 +1086,7 @@ func (s *scope) Constant(expr *Expression) pyObject { // CurrentBuildStatement creates a provider for creating a new BuildStatement from the statement // that is being currently interpreted. func (s *scope) CurrentBuildStatement() core.BuildStatementProvider { - return func() *core.BuildStatement { + return func() core.BuildStatement { stmtScope := s for curr := s; curr != nil; curr = curr.callerScope { if curr.pkg != nil && curr.filename == s.pkg.Filename { @@ -1099,7 +1099,7 @@ func (s *scope) CurrentBuildStatement() core.BuildStatementProvider { } // ActiveSubincludes creates a provider to trace the call stack and scopes to find subincludes that -// provided the macros/functions actively executing to define this target. +// provided the macros/functions actively executing to define this target.1 func (s *scope) ActiveSubincludes() core.SubincludesLabelProvider { return func() core.BuildLabels { seen := map[core.BuildLabel]bool{} @@ -1123,9 +1123,9 @@ func (s *scope) pkgFilename() string { return "" } -// NewBuildStatement creates a new core.BuildStatment from an asp.statment. -func NewBuildStatement(stmt *Statement) *core.BuildStatement { - return &core.BuildStatement{ +// NewBuildStatement creates a new core.BuildStatement from an asp.statement. +func NewBuildStatement(stmt *Statement) core.BuildStatement { + return core.BuildStatement{ Start: int(stmt.Pos), End: int(stmt.EndPos), } From c7cd118ce5b7e6cbbe79bd3ebe6a67099c7c839b Mon Sep 17 00:00:00 2001 From: DuBento Date: Tue, 12 May 2026 14:15:47 +0100 Subject: [PATCH 045/118] NewPackage with variadic optional functions --- src/core/package.go | 46 ++++++++++++++++++----------- src/parse/asp/interpreter.go | 2 +- src/parse/asp/label_context_test.go | 2 +- src/parse/parse_step.go | 6 +++- src/query/changes_test.go | 2 +- 5 files changed, 36 insertions(+), 22 deletions(-) diff --git a/src/core/package.go b/src/core/package.go index 9cc8a85023..9a510750ed 100644 --- a/src/core/package.go +++ b/src/core/package.go @@ -40,30 +40,40 @@ type Package struct { mutex sync.RWMutex } -// NewPackage constructs a new package with the given name. -func NewPackage(name string) *Package { - return NewPackageSubrepo(name, "") +// PackageOptions is a functional option type for configuring a new Package. +type PackageOptions func(*Package) + +// WithPackageSubrepo returns a PackageOptions that sets the subrepo name for a new Package. +func WithPackageSubrepo(name string) PackageOptions { + return func(p *Package) { + p.SubrepoName = name + } } -// NewPackageWithOpts constructs a new package with the given name, and enables additional features -// given the flags enabled in the build state. -func NewPackageWithOpts(state *BuildState, name string) *Package { - pkg := NewPackage(name) - if state.ParseMetadata { - pkg.Metadata = newPackageMetadata() +// WithPackageMetadata returns a PackageOptions that enables tracking of +// metadata (like statement positions and subinclude mappings) for the Package. +// This is required for features like 'plz export'. +func WithPackageMetadata() PackageOptions { + return func(p *Package) { + p.Metadata = newPackageMetadata() } - return pkg } -// NewPackageSubrepo constructs a new package with the given name and subrepo. -func NewPackageSubrepo(name, subrepo string) *Package { - return &Package{ - Name: name, - SubrepoName: subrepo, - targets: map[string]*BuildTarget{}, - Outputs: map[string]*BuildTarget{}, - Metadata: newNoopPackageMetadata(), +// NewPackage constructs a new package with the given name, and enables additional features +// given the PackageOptions provided. +func NewPackage(name string, options ...PackageOptions) *Package { + pkg := &Package{ + Name: name, + targets: map[string]*BuildTarget{}, + Outputs: map[string]*BuildTarget{}, + // Defaults to noop to avoid storing metadata for most operations + Metadata: newNoopPackageMetadata(), } + + for _, option := range options { + option(pkg) + } + return pkg } // Target returns the target with the given name, or nil if this package doesn't have one. diff --git a/src/parse/asp/interpreter.go b/src/parse/asp/interpreter.go index e627ea7707..c13d0d218b 100644 --- a/src/parse/asp/interpreter.go +++ b/src/parse/asp/interpreter.go @@ -401,7 +401,7 @@ func (s *scope) subincludePackage() *core.Package { return pkg } // We're probably doing a local subinclude so the package isn't ready yet - return core.NewPackageSubrepo(s.subincludeLabel.PackageName, s.subincludeLabel.Subrepo) + return core.NewPackage(s.subincludeLabel.PackageName, core.WithPackageSubrepo(s.subincludeLabel.Subrepo)) } return nil } diff --git a/src/parse/asp/label_context_test.go b/src/parse/asp/label_context_test.go index 9b99625bfc..023cb00906 100644 --- a/src/parse/asp/label_context_test.go +++ b/src/parse/asp/label_context_test.go @@ -12,7 +12,7 @@ import ( func newScope(pkgName, subrepo, plugin string) *scope { s := &scope{ - pkg: core.NewPackageSubrepo(pkgName, subrepo), + pkg: core.NewPackage(pkgName, core.WithPackageSubrepo(subrepo)), state: core.NewBuildState(core.DefaultConfiguration()), } if plugin != "" { diff --git a/src/parse/parse_step.go b/src/parse/parse_step.go index fb1b775f94..22ddccd61a 100644 --- a/src/parse/parse_step.go +++ b/src/parse/parse_step.go @@ -182,7 +182,11 @@ func maybeParseSubrepoPackage(state *core.BuildState, subrepoPkg, subrepoSubrepo // parsePackage parses a BUILD file and adds the package to the build graph func parsePackage(state *core.BuildState, label, dependent core.BuildLabel, subrepo *core.Subrepo, mode core.ParseMode) (*core.Package, error) { packageName := label.PackageName - pkg := core.NewPackageWithOpts(state, packageName) + var opts []core.PackageOptions + if state.ParseMetadata { + opts = append(opts, core.WithPackageMetadata()) + } + pkg := core.NewPackage(packageName, opts...) pkg.Subrepo = subrepo var fileSystem iofs.FS = fs.HostFS if subrepo != nil { diff --git a/src/query/changes_test.go b/src/query/changes_test.go index ed4124b684..d3806a4527 100644 --- a/src/query/changes_test.go +++ b/src/query/changes_test.go @@ -160,7 +160,7 @@ func addTarget(state *core.BuildState, label string, dep *core.BuildTarget, sour } pkg := state.Graph.PackageByLabel(t.Label) if pkg == nil { - pkg = core.NewPackageSubrepo(t.Label.PackageName, t.Label.Subrepo) + pkg = core.NewPackage(t.Label.PackageName, core.WithPackageSubrepo(t.Label.Subrepo)) state.Graph.AddPackage(pkg) } pkg.AddTarget(t) From faf8c330db180fe11046469ddd7268fe38c3415a Mon Sep 17 00:00:00 2001 From: DuBento Date: Tue, 12 May 2026 14:43:03 +0100 Subject: [PATCH 046/118] infof to debugf --- src/core/package_metadata.go | 2 +- src/export/export.go | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/core/package_metadata.go b/src/core/package_metadata.go index f776c504cb..c19def4839 100644 --- a/src/core/package_metadata.go +++ b/src/core/package_metadata.go @@ -88,7 +88,7 @@ func (m *packageMetadataImpl) RegisterStatementTarget(target *BuildTarget, stmtP func (m *packageMetadataImpl) RegisterRequiredSubinclude(target *BuildTarget, labelProvider SubincludesLabelProvider) { labels := labelProvider() if len(labels) == 0 { - log.Infof("Attempted to register empty subinclude labels for target %s", target.String()) + log.Debugf("Attempted to register empty subinclude labels for target %s", target.String()) return } diff --git a/src/export/export.go b/src/export/export.go index f2d35e5ad0..1a19d1e272 100644 --- a/src/export/export.go +++ b/src/export/export.go @@ -135,7 +135,7 @@ func (be *baseExporter) Targets(labels core.BuildLabels) { // Dependencies exports dependencies of a target. func (be *baseExporter) Dependencies(target *core.BuildTarget) { deps := target.DeclaredDependencies() - log.Infof("Exporting dependencies of (%v): %v", target.Label, deps) + log.Debugf("Exporting dependencies of (%v): %v", target.Label, deps) be.Targets(deps) } @@ -150,7 +150,7 @@ func (be *baseExporter) Sources(target *core.BuildTarget) { if err := fs.RecursiveCopy(p, filepath.Join(be.targetDir, p), 0); err != nil { log.Warningf("Error copying file, skipping...: %s", err) } - log.Infof("Writing exported source file: %s", p) + log.Debugf("Writing exported source file: %s", p) } } } @@ -204,7 +204,7 @@ func (e *DefaultExporter) Target(target *core.BuildTarget) { return } - log.Infof("Exporting target: %v", target.Label) + log.Debugf("Exporting target: %v", target.Label) // Skip export for internal packages if target.Label.PackageName == parse.InternalPackageName { @@ -229,7 +229,7 @@ func (e *DefaultExporter) Target(target *core.BuildTarget) { func (e *DefaultExporter) Subincludes(pkg *core.Package, target *core.BuildTarget) { subincludes, err := pkg.Metadata.FindRequiredSubincludes(target) if err != nil { - log.Infof("No subincludes found, assuming non required.: %w", pkg.Name, err) + log.Debugf("No subincludes found, assuming non required.: %w", pkg.Name, err) return } @@ -267,7 +267,7 @@ func (e *DefaultExporter) BuildStatements(pkg *core.Package, target *core.BuildT return } - log.Infof("Exporting related targets to (%v): %v", target.Label, relatedTargets) + log.Debugf("Exporting related targets to (%v): %v", target.Label, relatedTargets) for _, target := range relatedTargets { e.Target(target) } From dd8cfca93460caf572512824599aa1dab2489de6 Mon Sep 17 00:00:00 2001 From: DuBento Date: Tue, 12 May 2026 15:18:36 +0100 Subject: [PATCH 047/118] package metadata doc comments improvements --- src/core/package_metadata.go | 67 ++++++++++++++++++++++-------------- src/parse/asp/builtins.go | 2 +- 2 files changed, 42 insertions(+), 27 deletions(-) diff --git a/src/core/package_metadata.go b/src/core/package_metadata.go index c19def4839..0e5a15cbea 100644 --- a/src/core/package_metadata.go +++ b/src/core/package_metadata.go @@ -28,42 +28,62 @@ func (s BuildStatements) Len() int { return len(s) } func (s BuildStatements) Swap(i, j int) { s[i], s[j] = s[j], s[i] } func (s BuildStatements) Less(i, j int) bool { return s[i].StartPos() < s[j].StartPos() } -// BuildStatementProvider is a type for methods that generate new build statements. +// BuildStatementProvider defines a closure that generates new build statements. +// It is used as an argument in PackageMetadata methods to defer evaluation, avoiding +// unnecessary computation when using the no-op implementation. type BuildStatementProvider func() BuildStatement -// SubincludesLabelProvider is a type for methods that generate labels from a subincludes. +// SubincludesLabelProvider defines a closure that generates labels for a subinclude statement. +// It is used as an argument in PackageMetadata methods to defer evaluation, avoiding +// unnecessary computation when using the no-op implementation. type SubincludesLabelProvider func() BuildLabels // PackageMetadata stores metadata about parsed BUILD files, mapping statements and subincludes -// to their respective targets. +// to their respective targets. This supports additional logic for operations such as `plz export` +// but should be disabled for most operations by using the no-op implementation to avoid the overhead. type PackageMetadata interface { - // RegisterStatementTarget maps a build statement to a target it generated. + // RegisterStatementTarget records that the given build target was created as a result of the + // given statement being executed. This should only be called for statements in BUILD files. RegisterStatementTarget(target *BuildTarget, stmtProvider BuildStatementProvider) - // RegisterRequiredSubinclude maps a target to the subincludes required to build it. + // RegisterRequiredSubinclude records that the given build target requires the given subinclude + // labels to be built. This is used to track which subinclude statements contributed to a target's + // definition. This should only be called for statements in BUILD files. RegisterRequiredSubinclude(target *BuildTarget, labelProvider SubincludesLabelProvider) - // RegisterSubincludeStmt maps a subinclude statement to a label it includes. - RegisterSubincludeStmt(label BuildLabel, stmtProvider BuildStatementProvider) - // FindStatement returns the build statement that generated the given target. + // RegisterSubincludeStatement records that the given subinclude statement (provided by stmtProvider) + // includes the given build label. This should only be called for statements in BUILD files. + RegisterSubincludeStatement(label BuildLabel, stmtProvider BuildStatementProvider) + // FindStatement returns the build statement that was responsible for generating the given target. + // Returns an error if the target was not found in the recorded metadata. FindStatement(target *BuildTarget) (*BuildStatement, error) - // FindTargets returns all targets generated by the given build statement. + // FindTargets returns all build targets that were generated by the given build statement. + // Returns an error if no targets were found for the given statement. FindTargets(stmt *BuildStatement) ([]*BuildTarget, error) - // FindRequiredSubincludes returns all subinclude labels required by the given target. + // FindRequiredSubincludes returns all subinclude labels that were required by the given target. + // Returns an error if no subinclude information was found for the target. FindRequiredSubincludes(target *BuildTarget) (BuildLabels, error) - // GetSubincludedLabels returns the labels included by a given subinclude statement. + // GetSubincludedLabels returns all build labels that were included by the given subinclude statement. + // Returns the labels and true if the statement was found, or nil and false otherwise. GetSubincludedLabels(stmt *BuildStatement) (BuildLabels, bool) } -// packageMetadataImpl stores metadata about parsed BUILD files, mapping statements and subincludes -// to their respective targets. +// packageMetadataImpl is the default implementation of the PackageMetadata interface. +// It uses in-memory maps to track the relationships between BUILD file statements, +// subincludes, and the build targets they define. type packageMetadataImpl struct { - // a list of targets generated from each built statement + // StmtToTarget maps each build statement (identified by its byte range in a BUILD file) + // to the targets it produced. Since a single statement (like a custom target or loop) + // can produce multiple targets, this is a one-to-many mapping. StmtToTarget map[BuildStatement][]*BuildTarget - // the subincluded label dependencies per target + // TargetToSubinclude tracks the subinclude labels that were required for a target's + // definition. This allows mapping a target back to the subincluded labels required for building + // the target. TargetToSubinclude map[*BuildTarget]BuildLabels - // all the labels included for each subincludes statement + // LabelsPerSubincludeStmt maps a subinclude statement (identified by its position + // in the BUILD file) to the labels it explicitly subincludes. LabelsPerSubincludeStmt map[BuildStatement]BuildLabels - // Protects access to the above + // mutex protects concurrent access to the metadata maps during the parallel + // parsing of BUILD files. mutex sync.RWMutex } @@ -75,7 +95,6 @@ func newPackageMetadata() PackageMetadata { } } -// RegisterStatementTarget maps a build statement to a target it generated. func (m *packageMetadataImpl) RegisterStatementTarget(target *BuildTarget, stmtProvider BuildStatementProvider) { stmt := stmtProvider() @@ -84,7 +103,6 @@ func (m *packageMetadataImpl) RegisterStatementTarget(target *BuildTarget, stmtP m.StmtToTarget[stmt] = append(m.StmtToTarget[stmt], target) } -// RegisterRequiredSubinclude maps a target to the subincludes required to build it. func (m *packageMetadataImpl) RegisterRequiredSubinclude(target *BuildTarget, labelProvider SubincludesLabelProvider) { labels := labelProvider() if len(labels) == 0 { @@ -97,8 +115,7 @@ func (m *packageMetadataImpl) RegisterRequiredSubinclude(target *BuildTarget, la m.TargetToSubinclude[target] = append(m.TargetToSubinclude[target], labels...) } -// RegisterSubincludeStmt maps a subinclude statement to a label it includes. -func (m *packageMetadataImpl) RegisterSubincludeStmt(label BuildLabel, stmtProvider BuildStatementProvider) { +func (m *packageMetadataImpl) RegisterSubincludeStatement(label BuildLabel, stmtProvider BuildStatementProvider) { stmt := stmtProvider() m.mutex.Lock() @@ -106,7 +123,6 @@ func (m *packageMetadataImpl) RegisterSubincludeStmt(label BuildLabel, stmtProvi m.LabelsPerSubincludeStmt[stmt] = append(m.LabelsPerSubincludeStmt[stmt], label) } -// FindStatement returns the build statement that generated the given target. func (m *packageMetadataImpl) FindStatement(target *BuildTarget) (*BuildStatement, error) { m.mutex.RLock() defer m.mutex.RUnlock() @@ -119,7 +135,6 @@ func (m *packageMetadataImpl) FindStatement(target *BuildTarget) (*BuildStatemen return nil, fmt.Errorf("Target %s not found in statement metadata.", target.String()) } -// FindTargets returns all targets generated by the given build statement. func (m *packageMetadataImpl) FindTargets(stmt *BuildStatement) ([]*BuildTarget, error) { m.mutex.RLock() defer m.mutex.RUnlock() @@ -131,7 +146,6 @@ func (m *packageMetadataImpl) FindTargets(stmt *BuildStatement) ([]*BuildTarget, return targets, nil } -// FindRequiredSubincludes returns all subinclude labels required by the given target. func (m *packageMetadataImpl) FindRequiredSubincludes(target *BuildTarget) (BuildLabels, error) { m.mutex.RLock() defer m.mutex.RUnlock() @@ -143,7 +157,6 @@ func (m *packageMetadataImpl) FindRequiredSubincludes(target *BuildTarget) (Buil return subincludes, nil } -// GetSubincludedLabels returns the labels included by a given subinclude statement. func (m *packageMetadataImpl) GetSubincludedLabels(stmt *BuildStatement) (BuildLabels, bool) { m.mutex.RLock() defer m.mutex.RUnlock() @@ -152,6 +165,8 @@ func (m *packageMetadataImpl) GetSubincludedLabels(stmt *BuildStatement) (BuildL return v, ok } +// noopPackageMetadata implements the PackageMetadata interface with no-op methods. This is used to +// avoid the overhead of parsing metadata for operations that don't depend on it. type noopPackageMetadata struct{} func newNoopPackageMetadata() PackageMetadata { @@ -162,7 +177,7 @@ func (n *noopPackageMetadata) RegisterStatementTarget(target *BuildTarget, stmtP } func (n *noopPackageMetadata) RegisterRequiredSubinclude(target *BuildTarget, labelProvider SubincludesLabelProvider) { } -func (n *noopPackageMetadata) RegisterSubincludeStmt(label BuildLabel, stmtProvider BuildStatementProvider) { +func (n *noopPackageMetadata) RegisterSubincludeStatement(label BuildLabel, stmtProvider BuildStatementProvider) { } func (n *noopPackageMetadata) FindStatement(target *BuildTarget) (*BuildStatement, error) { return nil, fmt.Errorf("metadata not tracked") diff --git a/src/parse/asp/builtins.go b/src/parse/asp/builtins.go index 30c7aa706d..63f28b8a13 100644 --- a/src/parse/asp/builtins.go +++ b/src/parse/asp/builtins.go @@ -419,7 +419,7 @@ func subincludeTarget(s *scope, l core.BuildLabel) *core.BuildTarget { t = s.WaitForSubincludedTarget(l, pkgLabel) if s.pkg != nil { s.pkg.RegisterSubinclude(l) - s.pkg.Metadata.RegisterSubincludeStmt(l, s.CurrentBuildStatement()) + s.pkg.Metadata.RegisterSubincludeStatement(l, s.CurrentBuildStatement()) } else if s.subincludeLabel != nil { // If this is nil, that indicates a preloadedSubinclude s.state.Graph.RegisterTransitiveSubinclude(*s.subincludeLabel, l) } From 084f233a5e296fbc27f7dc0a17169df32ba07bf5 Mon Sep 17 00:00:00 2001 From: DuBento Date: Tue, 12 May 2026 17:42:25 +0100 Subject: [PATCH 048/118] improve doc comments and adjust method visibility for export.go --- src/export/export.go | 254 +++++++++++++++++++------------------- src/export/export_test.go | 2 +- 2 files changed, 130 insertions(+), 126 deletions(-) diff --git a/src/export/export.go b/src/export/export.go index 1a19d1e272..e48cb089f7 100644 --- a/src/export/export.go +++ b/src/export/export.go @@ -22,35 +22,50 @@ import ( var log = logging.Log +// Exporter defines the interface for exporting parts of a Please repository to a new directory. +// It handles the copying of configuration files, preloaded build definitions, and selected +// targets along with their necessary source files and dependencies. type Exporter interface { - // PlzConfig exports the repo configuration files. - PlzConfig() - // Preloaded exports the preloaded targets, build defs and subincludes. - Preloaded() - // Targets exports all targets for the given labels. - Targets(core.BuildLabels) - // Target exports an individual target and its dependencies. - Target(target *core.BuildTarget) - // WritePackageFiles writes the processed BUILD files to the export directory. + // ExportPlzConfig exports the repository's configuration files (e.g., .plzconfig and its + // platform-specific variants) to the target export directory. + ExportPlzConfig() + // ExportPreloaded exports all globally preloaded build definitions and subincluded targets. + // These are usually defined in the repository's configuration file. + ExportPreloaded() + // ExportTargets exports the set of targets identified by the given build labels. + // Each target recursively exports all their source files and required build statements, but also + // targets in their transitive dependencies. + ExportTargets(core.BuildLabels) + // ExportTarget exports an individual build target. + // Each target recursively exports all their source files and required build statements, but also + // targets in their transitive dependencies. + ExportTarget(target *core.BuildTarget) + // WritePackageFiles writes the processed BUILD files for all exported targets to the + // export directory. These BUILD files may be modified (e.g., trimmed) depending on + // the exporter's implementation. WritePackageFiles() } -// Repo export a new please repo including the targets and dependencies requested. +// Repo export a new please repo including the targets and dependencies requested. Depending on the +// noTrim flag, the export will attempt to trim the resulting repository, exporting only the required +// targets and build statements in their packages. If noTrim is set, all targets of a package will be +// exported and not build statement trimming will be attempted, the BUILD file is copied in its entirety. func Repo(state *core.BuildState, dir string, noTrim bool, targets []core.BuildLabel) { - e := NewExporter(state, dir, noTrim) + e := newExporter(state, dir, noTrim) // ensure output dir if err := os.MkdirAll(dir, fs.DirPermissions); err != nil { log.Fatalf("failed to create export directory %s: %v", dir, err) } - e.PlzConfig() - e.Preloaded() - e.Targets(targets) + e.ExportPlzConfig() + e.ExportPreloaded() + e.ExportTargets(targets) e.WritePackageFiles() } -// Outputs exports the outputs of a target. +// Outputs exports the build artifacts (output files) produced by building the specified +// targets to the given output directory. func Outputs(state *core.BuildState, dir string, targets []core.BuildLabel) { for _, label := range targets { target := state.Graph.TargetOrDie(label) @@ -67,8 +82,8 @@ func Outputs(state *core.BuildState, dir string, targets []core.BuildLabel) { } } -// NewExporter creates a new exporter of a specific type based on the arguments. -func NewExporter(state *core.BuildState, dir string, noTrim bool) Exporter { +// newExporter creates a new exporter of a specific type based on the arguments. +func newExporter(state *core.BuildState, dir string, noTrim bool) Exporter { base := baseExporter{ state: state, targetDir: dir, @@ -83,7 +98,7 @@ func NewExporter(state *core.BuildState, dir string, noTrim bool) Exporter { exporter.impl = exporter return exporter } else { - exporter := &DefaultExporter{ + exporter := &defaultExporter{ baseExporter: base, requiredSubincludes: map[*core.Package]map[core.BuildLabel]bool{}, preloadedSubincludes: map[core.BuildLabel]bool{}, @@ -93,18 +108,19 @@ func NewExporter(state *core.BuildState, dir string, noTrim bool) Exporter { } } -// baseExporter provides common fields and methods of other exporters. A reference -// to the concrete exporter implementation is included to be used in the common methods. +// baseExporter provides common fields and methods of other exporters. type baseExporter struct { state *core.BuildState targetDir string + // exportedTargets maintains a record of the targets that have been exported so far. exportedTargets map[*core.Package]map[core.BuildLabel]bool - impl Exporter + // impl is a reference to the concrete exporter implementation. It's included for calling the + // specific exporter implementation from the common methods. + impl Exporter } -// PlzConfig exports the repo configuration files. -func (be *baseExporter) PlzConfig() { +func (be *baseExporter) ExportPlzConfig() { profiles, err := filepath.Glob(".plzconfig*") if err != nil { log.Fatalf("failed to glob .plzconfig files: %v", err) @@ -120,27 +136,26 @@ func (be *baseExporter) PlzConfig() { } } -// Targets exports all targets for the given labels. -func (be *baseExporter) Targets(labels core.BuildLabels) { +func (be *baseExporter) ExportTargets(labels core.BuildLabels) { for _, l := range labels { target := be.state.Graph.Target(l) if target == nil { log.Errorf("Unable to lookup target %s", l) continue } - be.impl.Target(target) + be.impl.ExportTarget(target) } } -// Dependencies exports dependencies of a target. -func (be *baseExporter) Dependencies(target *core.BuildTarget) { +// exportDependencies exports exportDependencies of a target. +func (be *baseExporter) exportDependencies(target *core.BuildTarget) { deps := target.DeclaredDependencies() log.Debugf("Exporting dependencies of (%v): %v", target.Label, deps) - be.Targets(deps) + be.ExportTargets(deps) } -// Sources exports all files required by the target. -func (be *baseExporter) Sources(target *core.BuildTarget) { +// exportSources exports all files required by the target. +func (be *baseExporter) exportSources(target *core.BuildTarget) { for _, src := range append(target.AllSources(), target.AllData()...) { if _, ok := src.Label(); ok { continue // These will be handled as dependencies later @@ -168,8 +183,8 @@ func (be *baseExporter) checkFirstExport(pkg *core.Package, target *core.BuildTa return true } -// DefaultExporter implements an exporter that trims packages to reach a minimal exported repo. -type DefaultExporter struct { +// defaultExporter implements an exporter that trims packages to reach a minimal exported repo. +type defaultExporter struct { baseExporter // requiredSubincludes maps packages to the subinclude labels they require. requiredSubincludes map[*core.Package]map[core.BuildLabel]bool @@ -177,9 +192,7 @@ type DefaultExporter struct { preloadedSubincludes map[core.BuildLabel]bool } -// Preloaded exports the preloaded targets, build defs and subincludes. These preloads are usually -// defined in the .plzexport config. -func (e *DefaultExporter) Preloaded() { +func (e *defaultExporter) ExportPreloaded() { // Write any preloaded build defs for _, preload := range e.state.Config.Parse.PreloadBuildDefs { if err := fs.RecursiveCopy(preload, filepath.Join(e.targetDir, preload), 0); err != nil { @@ -192,15 +205,13 @@ func (e *DefaultExporter) Preloaded() { for _, t := range targets { e.preloadedSubincludes[t] = true } - e.Targets(targets) + e.ExportTargets(targets) } } -// Target exports an individual target. This implementation will attempt to export a minimal repo -// with only the required targets and statements. -func (e *DefaultExporter) Target(target *core.BuildTarget) { +func (e *defaultExporter) ExportTarget(target *core.BuildTarget) { pkg := e.state.Graph.PackageOrDie(target.Label) - if e.checkFirstExport(pkg, target) == false { + if !e.checkFirstExport(pkg, target) { return } @@ -213,20 +224,46 @@ func (e *DefaultExporter) Target(target *core.BuildTarget) { // We want to export the package that made this subrepo available, but we still need to walk the // target deps as it may depend on other subrepos or first party targets if target.Subrepo != nil { - e.Target(target.Subrepo.Target) - e.Dependencies(target) + e.ExportTarget(target.Subrepo.Target) + e.exportDependencies(target) return } - e.Subincludes(pkg, target) - e.BuildStatements(pkg, target) - e.Sources(target) - e.Dependencies(target) + e.exportSubincludes(pkg, target) + e.exportBuildStatements(pkg, target) + e.exportSources(target) + e.exportDependencies(target) } -// Subincludes exports the subincluded targets required to generate the target and selects them to +func (e *defaultExporter) WritePackageFiles() { + for pkg := range e.exportedTargets { + // Skip subrepos and internal packages. These will be generated by build statements in the exported + // repo or included in please internally. + if pkg.Subrepo != nil || pkg.Name == parse.InternalPackageName { + continue + } + + filteredBytes, err := e.filterPackageFile(pkg) + if err != nil { + log.Errorf("Failed to filter the build statements of package %s: %v", pkg.Label(), err) + continue + } + + buildParser, err := build.ParseBuild(pkg.Filename, filteredBytes) + formattedBytes := build.Format(buildParser) + + file := e.openExportedPackageFile(pkg) + defer file.Close() + if _, err := file.Write(formattedBytes); err != nil { + log.Errorf("Failed to write to exported BUILD file %s: %v", file.Name(), err) + continue + } + } +} + +// exportSubincludes exports the subincluded targets required to generate the target and selects them to // later be written to the package as statements. -func (e *DefaultExporter) Subincludes(pkg *core.Package, target *core.BuildTarget) { +func (e *defaultExporter) exportSubincludes(pkg *core.Package, target *core.BuildTarget) { subincludes, err := pkg.Metadata.FindRequiredSubincludes(target) if err != nil { log.Debugf("No subincludes found, assuming non required.: %w", pkg.Name, err) @@ -234,7 +271,8 @@ func (e *DefaultExporter) Subincludes(pkg *core.Package, target *core.BuildTarge } for _, subinclude := range subincludes { - // skip for preloaded subincludes + // skip for preloaded subincludes, these are handled separately at the start to ensure they are + // they are exported even if not directly used by an exported target. if e.preloadedSubincludes[subinclude] { continue } @@ -249,12 +287,12 @@ func (e *DefaultExporter) Subincludes(pkg *core.Package, target *core.BuildTarge log.Errorf("Unable to lookup target %s", subinclude) continue } - e.Target(target) + e.ExportTarget(target) } } -// BuildStatements exports BUILD statements that generate the build target. -func (e *DefaultExporter) BuildStatements(pkg *core.Package, target *core.BuildTarget) { +// exportBuildStatements exports BUILD statements that generate the build target. +func (e *defaultExporter) exportBuildStatements(pkg *core.Package, target *core.BuildTarget) { stmt, err := pkg.Metadata.FindStatement(target) if err != nil { log.Errorf("Failed to find statement in %s: %w", pkg.Name, err) @@ -269,40 +307,12 @@ func (e *DefaultExporter) BuildStatements(pkg *core.Package, target *core.BuildT log.Debugf("Exporting related targets to (%v): %v", target.Label, relatedTargets) for _, target := range relatedTargets { - e.Target(target) + e.ExportTarget(target) } } -// WritePackageFiles writes the trimmed BUILD files to the export directory. -func (e *DefaultExporter) WritePackageFiles() { - for pkg := range e.exportedTargets { - if pkg.Subrepo != nil || pkg.Name == parse.InternalPackageName { - continue // Skip subrepos and internal packages - } - - // filter - filteredBytes, err := e.FilterPackageFile(pkg) - if err != nil { - log.Errorf("Failed to filter the build statements of package %s: %v", pkg.Label(), err) - continue - } - - // format - buildParser, err := build.ParseBuild(pkg.Filename, filteredBytes) - formattedBytes := build.Format(buildParser) - - // write - file := e.OpenExportedPackageFile(pkg) - defer file.Close() - if _, err := file.Write(formattedBytes); err != nil { - log.Errorf("Failed to write to exported BUILD file %s: %v", file.Name(), err) - continue - } - } -} - -// OpenExportedPackageFile creates a new package (BUILD) file in the exported dir. -func (e *DefaultExporter) OpenExportedPackageFile(pkg *core.Package) *os.File { +// openExportedPackageFile creates a new package (BUILD) file in the exported dir. +func (e *defaultExporter) openExportedPackageFile(pkg *core.Package) *os.File { filename := pkg.Filename exportedFilename := filepath.Join(e.targetDir, filename) f, err := fs.OpenDirFile(exportedFilename, os.O_CREATE|os.O_WRONLY, 0664) @@ -312,8 +322,8 @@ func (e *DefaultExporter) OpenExportedPackageFile(pkg *core.Package) *os.File { return f } -// FilterPackageFile filters the statements to be written to the exported BUILD file. -func (e *DefaultExporter) FilterPackageFile(pkg *core.Package) ([]byte, error) { +// filterPackageFile filters the statements to be written to the exported BUILD file. +func (e *defaultExporter) filterPackageFile(pkg *core.Package) ([]byte, error) { p := asp.NewParserOnly() parsedStmts, err := p.ParseFileOnly(pkg.Filename) if err != nil { @@ -368,7 +378,7 @@ func (e *DefaultExporter) FilterPackageFile(pkg *core.Package) ([]byte, error) { } // isRequiredStatement evaluates if the current build statement is required by the export. -func (e *DefaultExporter) isRequiredStatement(pkg *core.Package, stmt *core.BuildStatement) (bool, error) { +func (e *defaultExporter) isRequiredStatement(pkg *core.Package, stmt *core.BuildStatement) (bool, error) { targets, err := pkg.Metadata.FindTargets(stmt) if err != nil { return false, err @@ -381,7 +391,7 @@ func (e *DefaultExporter) isRequiredStatement(pkg *core.Package, stmt *core.Buil } // minimalSubincludeStatement generates a subinclude statement containing only the required labels. -func (e *DefaultExporter) minimalSubincludeStatement(pkg *core.Package, available core.BuildLabels) string { +func (e *defaultExporter) minimalSubincludeStatement(pkg *core.Package, available core.BuildLabels) string { required := e.requiredSubincludes[pkg] var filteredLabels core.BuildLabels for _, label := range available { @@ -414,8 +424,7 @@ type NoTrimExporter struct { exportedPackages map[string]bool } -// Preloaded exports the preloaded targets, build defs and subincludes. -func (nte *NoTrimExporter) Preloaded() { +func (nte *NoTrimExporter) ExportPreloaded() { // Write any preloaded build defs for _, preload := range nte.state.Config.Parse.PreloadBuildDefs { if err := fs.RecursiveCopy(preload, filepath.Join(nte.targetDir, preload), 0); err != nil { @@ -425,69 +434,64 @@ func (nte *NoTrimExporter) Preloaded() { for _, target := range nte.state.Config.Parse.PreloadSubincludes { targets := append(nte.state.Graph.TransitiveSubincludes(target), target) - nte.Targets(targets) + nte.ExportTargets(targets) } } -// Target exports an individual target. This implementation won't attempted any trimming, exporting -// all targets and statements defined in the package. -func (nte *NoTrimExporter) Target(target *core.BuildTarget) { +func (nte *NoTrimExporter) ExportTarget(target *core.BuildTarget) { pkg := nte.state.Graph.PackageOrDie(target.Label) - if nte.checkFirstExport(pkg, target) == false { + if !nte.checkFirstExport(pkg, target) { return } // We want to export the package that made this subrepo available, but we still need to walk the target deps // as it may depend on other subrepos or first party targets if target.Subrepo != nil { - nte.Target(target.Subrepo.Target) - nte.Dependencies(target) + nte.ExportTarget(target.Subrepo.Target) + nte.exportDependencies(target) return } - nte.Package(pkg) - nte.Subincludes(pkg, target) - nte.AllTargets(pkg) - nte.Sources(target) - nte.Dependencies(target) + nte.exportPackage(pkg) + nte.exportSubincludes(pkg) + nte.exportAllTargets(pkg) + nte.exportSources(target) + nte.exportDependencies(target) +} + +func (nte *NoTrimExporter) WritePackageFiles() { } -// Package exports the package BUILD file. -func (nte *NoTrimExporter) Package(pkg *core.Package) { - pkgName := pkg.Name - if pkgName == parse.InternalPackageName { +// exportPackage exports the package BUILD file. +func (nte *NoTrimExporter) exportPackage(pkg *core.Package) { + // Skip subrepos and internal packages. These will be generated by build statements in the exported + // repo or included in please internally. + if pkg.Subrepo != nil || pkg.Name == parse.InternalPackageName { return } - if nte.exportedPackages[pkgName] { + + if nte.exportedPackages[pkg.Name] { return } - nte.exportedPackages[pkgName] = true - - pkgFilename := pkg.Filename - exportedFilename := filepath.Join(nte.targetDir, pkgFilename) + nte.exportedPackages[pkg.Name] = true - if err := fs.CopyFile(pkgFilename, exportedFilename, 0); err != nil { - log.Errorf("failed to export package %s: %v", pkgName, err) + exportedFilename := filepath.Join(nte.targetDir, pkg.Filename) + if err := fs.CopyFile(pkg.Filename, exportedFilename, 0); err != nil { + log.Errorf("failed to export package %s: %v", pkg.Name, err) } } -// Subincludes exports the subincluded targets. -func (nte *NoTrimExporter) Subincludes(pkg *core.Package, target *core.BuildTarget) { +// exportSubincludes exports the subincluded targets. +func (nte *NoTrimExporter) exportSubincludes(pkg *core.Package) { subincludes := pkg.AllSubincludes(nte.state.Graph) for _, subinclude := range subincludes { - nte.Target(nte.state.Graph.TargetOrDie(subinclude)) + nte.ExportTarget(nte.state.Graph.TargetOrDie(subinclude)) } } -// AllTargets will export all the targets in the provided package. -func (nte *NoTrimExporter) AllTargets(pkg *core.Package) { +// exportAllTargets will export all the targets in the provided package. +func (nte *NoTrimExporter) exportAllTargets(pkg *core.Package) { for _, target := range pkg.AllTargets() { - nte.Target(target) + nte.ExportTarget(target) } } - -// WritePackageFiles in the NoTrimExporter doesn't require an implementation due to total copy -// of BUILD file. -func (nte *NoTrimExporter) WritePackageFiles() { - return -} diff --git a/src/export/export_test.go b/src/export/export_test.go index 58bad1c822..b53094f7ef 100644 --- a/src/export/export_test.go +++ b/src/export/export_test.go @@ -45,7 +45,7 @@ func TestMinimalSubincludeStatement(t *testing.T) { for _, tt := range subincludesTests { t.Run(tt.name, func(t *testing.T) { - e := &DefaultExporter{ + e := &defaultExporter{ requiredSubincludes: map[*core.Package]map[core.BuildLabel]bool{}, } From bf0f734de5e595d9ea4a72221e98e211cc37d1a4 Mon Sep 17 00:00:00 2001 From: DuBento Date: Tue, 12 May 2026 17:42:43 +0100 Subject: [PATCH 049/118] hide build output when testing export --- test/export/please_export_e2e_test.build_defs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/export/please_export_e2e_test.build_defs b/test/export/please_export_e2e_test.build_defs index 05f5a113ff..05e6bde2e5 100644 --- a/test/export/please_export_e2e_test.build_defs +++ b/test/export/please_export_e2e_test.build_defs @@ -50,7 +50,7 @@ def please_export_e2e_test( # Golden-Master validation with expected repo. Done before any building to avoid plz-out f'diff -ru "{exported_repo}" "$DATA_EXPECTED_REPO"', # Tests building the exported repo which in turn ensures the sources are included - f'plz --repo_root="{exported_repo}" build //...', + f'plz --repo_root="{exported_repo}" build //... > /dev/null', ] + [f'plz --repo_root="{exported_repo}" {cmd}' for cmd in cmd_on_export] test_cmd = [cmd.replace("plz ", "$TOOLS_PLEASE ") for cmd in test_cmd] From 33d2bc27feb0cc4d5e09f83baf4211a5a89a0514 Mon Sep 17 00:00:00 2001 From: DuBento Date: Tue, 12 May 2026 17:47:14 +0100 Subject: [PATCH 050/118] open and write of exported package file merged into the same method --- src/export/export.go | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/export/export.go b/src/export/export.go index e48cb089f7..7083c29425 100644 --- a/src/export/export.go +++ b/src/export/export.go @@ -252,12 +252,7 @@ func (e *defaultExporter) WritePackageFiles() { buildParser, err := build.ParseBuild(pkg.Filename, filteredBytes) formattedBytes := build.Format(buildParser) - file := e.openExportedPackageFile(pkg) - defer file.Close() - if _, err := file.Write(formattedBytes); err != nil { - log.Errorf("Failed to write to exported BUILD file %s: %v", file.Name(), err) - continue - } + e.WriteExportedPackageFile(pkg, formattedBytes) } } @@ -311,15 +306,19 @@ func (e *defaultExporter) exportBuildStatements(pkg *core.Package, target *core. } } -// openExportedPackageFile creates a new package (BUILD) file in the exported dir. -func (e *defaultExporter) openExportedPackageFile(pkg *core.Package) *os.File { +// WriteExportedPackageFile creates a new package (BUILD) file in the exported dir and writes to it. +func (e *defaultExporter) WriteExportedPackageFile(pkg *core.Package, content []byte) { filename := pkg.Filename exportedFilename := filepath.Join(e.targetDir, filename) f, err := fs.OpenDirFile(exportedFilename, os.O_CREATE|os.O_WRONLY, 0664) if err != nil { log.Fatalf("Failed to create and open exported BUILD file for %s: %v", exportedFilename, err) } - return f + defer f.Close() + + if _, err := f.Write(content); err != nil { + log.Errorf("Failed to write to exported BUILD file %s: %v", f.Name(), err) + } } // filterPackageFile filters the statements to be written to the exported BUILD file. From 54f577c32dea14e88493537de01a6d9c108c4da7 Mon Sep 17 00:00:00 2001 From: DuBento Date: Tue, 12 May 2026 17:53:01 +0100 Subject: [PATCH 051/118] update export_test.go with suggestions --- src/export/export_test.go | 44 +++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/src/export/export_test.go b/src/export/export_test.go index b53094f7ef..70aa31bd42 100644 --- a/src/export/export_test.go +++ b/src/export/export_test.go @@ -8,54 +8,54 @@ import ( ) func TestMinimalSubincludeStatement(t *testing.T) { - var subincludesTests = []struct { + testCases := []struct { name string availableLabels []core.BuildLabel requiredLabels []core.BuildLabel out string }{ { - "Successful no pruning subinclude", - core.ParseBuildLabels([]string{"//build_defs:test"}), - core.ParseBuildLabels([]string{"//build_defs:test"}), - `subinclude("//build_defs:test")`, + name: "Successful no pruning subinclude", + availableLabels: core.ParseBuildLabels([]string{"//build_defs:test"}), + requiredLabels: core.ParseBuildLabels([]string{"//build_defs:test"}), + out: `subinclude("//build_defs:test")`, }, { - "No subincludes", - nil, - nil, - "", + name: "No subincludes", + availableLabels: nil, + requiredLabels: nil, + out: "", }, { - "Single subinclude (not required)", - core.ParseBuildLabels([]string{"//build_defs:other"}), - nil, - "", + name: "Single subinclude (not required)", + availableLabels: core.ParseBuildLabels([]string{"//build_defs:other"}), + requiredLabels: nil, + out: "", }, { - "Multiple subincludes (sorted and filtered)", - core.ParseBuildLabels([]string{"//build_defs:test", "//build_defs:abc", "//build_defs:other"}), - core.ParseBuildLabels([]string{"//build_defs:test", "//build_defs:abc"}), - "subinclude(\n" + + name: "Multiple subincludes (sorted and filtered)", + availableLabels: core.ParseBuildLabels([]string{"//build_defs:test", "//build_defs:abc", "//build_defs:other"}), + requiredLabels: core.ParseBuildLabels([]string{"//build_defs:test", "//build_defs:abc"}), + out: "subinclude(\n" + " \"//build_defs:abc\",\n" + " \"//build_defs:test\",\n" + ")", }, } - for _, tt := range subincludesTests { - t.Run(tt.name, func(t *testing.T) { + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { e := &defaultExporter{ requiredSubincludes: map[*core.Package]map[core.BuildLabel]bool{}, } pkg := &core.Package{Name: "test"} e.requiredSubincludes[pkg] = map[core.BuildLabel]bool{} - for _, labels := range tt.requiredLabels { - e.requiredSubincludes[pkg][labels] = true + for _, label := range test.requiredLabels { + e.requiredSubincludes[pkg][label] = true } - assert.Equal(t, tt.out, e.minimalSubincludeStatement(pkg, tt.availableLabels)) + assert.Equal(t, test.out, e.minimalSubincludeStatement(pkg, test.availableLabels)) }) } } From ae5f520f91f77e2afa48f074e98f2a0d071a010b Mon Sep 17 00:00:00 2001 From: DuBento Date: Wed, 13 May 2026 20:17:32 +0100 Subject: [PATCH 052/118] apply review suggestions to export.go --- src/export/export.go | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/src/export/export.go b/src/export/export.go index 7083c29425..9ee87dadba 100644 --- a/src/export/export.go +++ b/src/export/export.go @@ -161,25 +161,30 @@ func (be *baseExporter) exportSources(target *core.BuildTarget) { continue // These will be handled as dependencies later } for _, p := range src.Paths(be.state.Graph) { - if !filepath.IsAbs(p) { // Don't copy system file deps. - if err := fs.RecursiveCopy(p, filepath.Join(be.targetDir, p), 0); err != nil { - log.Warningf("Error copying file, skipping...: %s", err) - } - log.Debugf("Writing exported source file: %s", p) + if filepath.IsAbs(p) { // Don't copy system file deps. + log.Infof("System dependency detected, skipping...: %s", p) + continue } + if err := fs.RecursiveCopy(p, filepath.Join(be.targetDir, p), 0); err != nil { + log.Warningf("Error copying file, skipping...: %s", err) + } + log.Debugf("Writing exported source file: %s", p) } } } // checkFirstExport is a helper to ensure we only visit the same target once. +// It returns true if this is the first time the target is being exported. func (be *baseExporter) checkFirstExport(pkg *core.Package, target *core.BuildTarget) bool { - if _, ok := be.exportedTargets[pkg]; !ok { - be.exportedTargets[pkg] = map[core.BuildLabel]bool{} + targets, ok := be.exportedTargets[pkg] + if !ok { + targets = make(map[core.BuildLabel]bool) + be.exportedTargets[pkg] = targets } - if be.exportedTargets[pkg][target.Label] { + if targets[target.Label] { return false } - be.exportedTargets[pkg][target.Label] = true + targets[target.Label] = true return true } @@ -230,7 +235,7 @@ func (e *defaultExporter) ExportTarget(target *core.BuildTarget) { } e.exportSubincludes(pkg, target) - e.exportBuildStatements(pkg, target) + e.exportRelatedTargets(pkg, target) e.exportSources(target) e.exportDependencies(target) } @@ -249,8 +254,11 @@ func (e *defaultExporter) WritePackageFiles() { continue } - buildParser, err := build.ParseBuild(pkg.Filename, filteredBytes) - formattedBytes := build.Format(buildParser) + parsedBuild, err := build.ParseBuild(pkg.Filename, filteredBytes) + if err != nil { + log.Fatalf("Failed to parse bytes for formatting: %v", err) + } + formattedBytes := build.Format(parsedBuild) e.WriteExportedPackageFile(pkg, formattedBytes) } @@ -286,8 +294,8 @@ func (e *defaultExporter) exportSubincludes(pkg *core.Package, target *core.Buil } } -// exportBuildStatements exports BUILD statements that generate the build target. -func (e *defaultExporter) exportBuildStatements(pkg *core.Package, target *core.BuildTarget) { +// exportRelatedTargets exports build targets that are related to the build statement that generated. +func (e *defaultExporter) exportRelatedTargets(pkg *core.Package, target *core.BuildTarget) { stmt, err := pkg.Metadata.FindStatement(target) if err != nil { log.Errorf("Failed to find statement in %s: %w", pkg.Name, err) @@ -340,7 +348,8 @@ func (e *defaultExporter) filterPackageFile(pkg *core.Package) ([]byte, error) { bStmt := asp.NewBuildStatement(stmt) log.Debugf("Evaluating statement %s", original[bStmt.Start:bStmt.End]) - // Write content that's between stmts (e.g. comments) + // Write content that's between stmts (e.g. comments). We skip these while parsing so it won't + // be included in "parsedStmts" but we want the resulting BUILD file to include these. if cursor < bStmt.Start { if _, err := buffer.Write(original[cursor:bStmt.Start]); err != nil { return nil, err @@ -356,7 +365,6 @@ func (e *defaultExporter) filterPackageFile(pkg *core.Package) ([]byte, error) { } else if required, err := e.isRequiredStatement(pkg, &bStmt); err == nil && !required { // Don't write statements that generate targets we are not interested about log.Debugf("Decision: ") - // skip } else { // Write every other statement if _, err := buffer.Write(original[bStmt.Start:bStmt.End]); err != nil { From 5f0e549b86e89bd80b8af20910a833ff26be709a Mon Sep 17 00:00:00 2001 From: DuBento Date: Wed, 13 May 2026 20:32:01 +0100 Subject: [PATCH 053/118] rename and doc fields for scope --- src/parse/asp/interpreter.go | 27 ++++++++++++++++----------- src/parse/asp/objects.go | 2 +- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/src/parse/asp/interpreter.go b/src/parse/asp/interpreter.go index c13d0d218b..0de403e44f 100644 --- a/src/parse/asp/interpreter.go +++ b/src/parse/asp/interpreter.go @@ -305,15 +305,20 @@ type scope struct { pkg *core.Package subincludeLabel *core.BuildLabel // If set, label of the subinclude we're currently interpreting parsingFor *parseTarget - parent *scope - callerScope *scope // caller local scope, nil if not in callstack - locals pyDict - config *pyConfig - globber *fs.Globber + // parent points to the lexical parent of this scope. It is used for variable resolution + // and is nil for the root scope. + parent *scope + // caller points to the scope that initiated the call which created this scope. + // It is used to trace the call stack and is nil if not in a call stack. + caller *scope + locals pyDict + config *pyConfig + globber *fs.Globber // True if this scope is for a pre- or post-build callback. Callback bool mode core.ParseMode - cursor *Statement // points to the statement currently being interpreted + // cursor points to the statement currently being interpreted + cursor *Statement } // parseAnnotatedLabelInPackage similarly to parseLabelInPackage, parses the label contextualising it to the provided @@ -1088,7 +1093,7 @@ func (s *scope) Constant(expr *Expression) pyObject { func (s *scope) CurrentBuildStatement() core.BuildStatementProvider { return func() core.BuildStatement { stmtScope := s - for curr := s; curr != nil; curr = curr.callerScope { + for curr := s; curr != nil; curr = curr.caller { if curr.pkg != nil && curr.filename == s.pkg.Filename { stmtScope = curr } @@ -1103,10 +1108,10 @@ func (s *scope) CurrentBuildStatement() core.BuildStatementProvider { func (s *scope) ActiveSubincludes() core.SubincludesLabelProvider { return func() core.BuildLabels { seen := map[core.BuildLabel]bool{} - for curr := s; curr != nil; curr = curr.callerScope { - for localScope := curr; localScope != nil; localScope = localScope.parent { - if localScope.subincludeLabel != nil { - label := *localScope.subincludeLabel + for callScope := s; callScope != nil; callScope = callScope.caller { + for lexicalScope := callScope; lexicalScope != nil; lexicalScope = lexicalScope.parent { + if lexicalScope.subincludeLabel != nil { + label := *lexicalScope.subincludeLabel seen[label] = true } } diff --git a/src/parse/asp/objects.go b/src/parse/asp/objects.go index 0133d413ba..52c1045e51 100644 --- a/src/parse/asp/objects.go +++ b/src/parse/asp/objects.go @@ -699,7 +699,7 @@ func (f *pyFunc) Call(s *scope, c *Call) pyObject { return f.callNative(s, c) } s2 := f.scope.newScope(s.pkg, s.mode, f.scope.filename, len(f.args)+1) - s2.callerScope = s // registering previous scope as caller + s2.caller = s // registering previous scope as caller s2.config = s.config s2.Set("CONFIG", s.config) // This needs to be copied across too :( s2.Callback = s.Callback From e59ab5b7147d01df578001c50fc41940c39a7e44 Mon Sep 17 00:00:00 2001 From: DuBento Date: Thu, 14 May 2026 10:02:18 +0100 Subject: [PATCH 054/118] revert new line change of target --- src/parse/asp/targets.go | 1 - 1 file changed, 1 deletion(-) diff --git a/src/parse/asp/targets.go b/src/parse/asp/targets.go index 03bf3a3537..48404b7920 100644 --- a/src/parse/asp/targets.go +++ b/src/parse/asp/targets.go @@ -187,7 +187,6 @@ func createTarget(s *scope, args []pyObject) *core.BuildTarget { target.Debug = new(core.DebugFields) target.Debug.Command, _ = decodeCommands(s, args[debugCMDBuildRuleArgIdx]) } - return target } From 917fe6183854a86bd3f18c839e4c3c44fb415a77 Mon Sep 17 00:00:00 2001 From: DuBento Date: Thu, 14 May 2026 10:16:39 +0100 Subject: [PATCH 055/118] set builtin code filename for scope for debugging --- src/parse/asp/objects.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/parse/asp/objects.go b/src/parse/asp/objects.go index 52c1045e51..9b62a65817 100644 --- a/src/parse/asp/objects.go +++ b/src/parse/asp/objects.go @@ -694,7 +694,7 @@ func (f *pyFunc) String() string { func (f *pyFunc) Call(s *scope, c *Call) pyObject { if f.nativeCode != nil { if f.kwargs { - return f.callNative(s.NewScope("", 0), c) + return f.callNative(s.NewScope("", 0), c) } return f.callNative(s, c) } From b012ab8f94f76cc7a1ab9ffc2c5846da19f161fb Mon Sep 17 00:00:00 2001 From: DuBento Date: Thu, 14 May 2026 11:19:12 +0100 Subject: [PATCH 056/118] Rework packagemetadata to return empty slices or nil instead of errors. Add fatal to no-op implementation --- src/core/package_metadata.go | 64 ++++++++++++++++-------------------- src/export/export.go | 43 ++++++++++-------------- 2 files changed, 47 insertions(+), 60 deletions(-) diff --git a/src/core/package_metadata.go b/src/core/package_metadata.go index 0e5a15cbea..5fe0ffac57 100644 --- a/src/core/package_metadata.go +++ b/src/core/package_metadata.go @@ -1,7 +1,6 @@ package core import ( - "fmt" "slices" "sync" ) @@ -53,17 +52,17 @@ type PackageMetadata interface { // includes the given build label. This should only be called for statements in BUILD files. RegisterSubincludeStatement(label BuildLabel, stmtProvider BuildStatementProvider) // FindStatement returns the build statement that was responsible for generating the given target. - // Returns an error if the target was not found in the recorded metadata. - FindStatement(target *BuildTarget) (*BuildStatement, error) + // Returns nil if the target was not found in the recorded metadata. + FindStatement(target *BuildTarget) *BuildStatement // FindTargets returns all build targets that were generated by the given build statement. - // Returns an error if no targets were found for the given statement. - FindTargets(stmt *BuildStatement) ([]*BuildTarget, error) + // Returns an empty slice if no targets were found for the given statement. + FindTargets(stmt *BuildStatement) []*BuildTarget // FindRequiredSubincludes returns all subinclude labels that were required by the given target. - // Returns an error if no subinclude information was found for the target. - FindRequiredSubincludes(target *BuildTarget) (BuildLabels, error) + // The return value is empty if no subinclude information was found for the target. + FindRequiredSubincludes(target *BuildTarget) BuildLabels // GetSubincludedLabels returns all build labels that were included by the given subinclude statement. - // Returns the labels and true if the statement was found, or nil and false otherwise. - GetSubincludedLabels(stmt *BuildStatement) (BuildLabels, bool) + // Returns the labels or an empty slice if the statement wasn't found. + GetSubincludedLabels(stmt *BuildStatement) BuildLabels } // packageMetadataImpl is the default implementation of the PackageMetadata interface. @@ -123,46 +122,37 @@ func (m *packageMetadataImpl) RegisterSubincludeStatement(label BuildLabel, stmt m.LabelsPerSubincludeStmt[stmt] = append(m.LabelsPerSubincludeStmt[stmt], label) } -func (m *packageMetadataImpl) FindStatement(target *BuildTarget) (*BuildStatement, error) { +func (m *packageMetadataImpl) FindStatement(target *BuildTarget) *BuildStatement { m.mutex.RLock() defer m.mutex.RUnlock() for stmt, targets := range m.StmtToTarget { if slices.Contains(targets, target) { - return &stmt, nil + return &stmt } } - return nil, fmt.Errorf("Target %s not found in statement metadata.", target.String()) + return nil } -func (m *packageMetadataImpl) FindTargets(stmt *BuildStatement) ([]*BuildTarget, error) { +func (m *packageMetadataImpl) FindTargets(stmt *BuildStatement) []*BuildTarget { m.mutex.RLock() defer m.mutex.RUnlock() - targets, ok := m.StmtToTarget[*stmt] - if !ok { - return nil, fmt.Errorf("Targets not found for statement %v.", stmt) - } - return targets, nil + return m.StmtToTarget[*stmt] } -func (m *packageMetadataImpl) FindRequiredSubincludes(target *BuildTarget) (BuildLabels, error) { +func (m *packageMetadataImpl) FindRequiredSubincludes(target *BuildTarget) BuildLabels { m.mutex.RLock() defer m.mutex.RUnlock() - subincludes, ok := m.TargetToSubinclude[target] - if !ok { - return nil, fmt.Errorf("Subincludes not found for target %v.", target) - } - return subincludes, nil + return m.TargetToSubinclude[target] } -func (m *packageMetadataImpl) GetSubincludedLabels(stmt *BuildStatement) (BuildLabels, bool) { +func (m *packageMetadataImpl) GetSubincludedLabels(stmt *BuildStatement) BuildLabels { m.mutex.RLock() defer m.mutex.RUnlock() - v, ok := m.LabelsPerSubincludeStmt[*stmt] - return v, ok + return m.LabelsPerSubincludeStmt[*stmt] } // noopPackageMetadata implements the PackageMetadata interface with no-op methods. This is used to @@ -179,15 +169,19 @@ func (n *noopPackageMetadata) RegisterRequiredSubinclude(target *BuildTarget, la } func (n *noopPackageMetadata) RegisterSubincludeStatement(label BuildLabel, stmtProvider BuildStatementProvider) { } -func (n *noopPackageMetadata) FindStatement(target *BuildTarget) (*BuildStatement, error) { - return nil, fmt.Errorf("metadata not tracked") +func (n *noopPackageMetadata) FindStatement(target *BuildTarget) *BuildStatement { + log.Fatalf("Metadata not tracked, using no-op implementation.") + return nil } -func (n *noopPackageMetadata) FindTargets(stmt *BuildStatement) ([]*BuildTarget, error) { - return nil, fmt.Errorf("metadata not tracked") +func (n *noopPackageMetadata) FindTargets(stmt *BuildStatement) []*BuildTarget { + log.Fatalf("Metadata not tracked, using no-op implementation.") + return nil } -func (n *noopPackageMetadata) FindRequiredSubincludes(target *BuildTarget) (BuildLabels, error) { - return nil, fmt.Errorf("metadata not tracked") +func (n *noopPackageMetadata) FindRequiredSubincludes(target *BuildTarget) BuildLabels { + log.Fatalf("Metadata not tracked, using no-op implementation.") + return nil } -func (n *noopPackageMetadata) GetSubincludedLabels(stmt *BuildStatement) (BuildLabels, bool) { - return nil, false +func (n *noopPackageMetadata) GetSubincludedLabels(stmt *BuildStatement) BuildLabels { + log.Fatalf("Metadata not tracked, using no-op implementation.") + return nil } diff --git a/src/export/export.go b/src/export/export.go index 9ee87dadba..fa0d3d925e 100644 --- a/src/export/export.go +++ b/src/export/export.go @@ -267,13 +267,7 @@ func (e *defaultExporter) WritePackageFiles() { // exportSubincludes exports the subincluded targets required to generate the target and selects them to // later be written to the package as statements. func (e *defaultExporter) exportSubincludes(pkg *core.Package, target *core.BuildTarget) { - subincludes, err := pkg.Metadata.FindRequiredSubincludes(target) - if err != nil { - log.Debugf("No subincludes found, assuming non required.: %w", pkg.Name, err) - return - } - - for _, subinclude := range subincludes { + for _, subinclude := range pkg.Metadata.FindRequiredSubincludes(target) { // skip for preloaded subincludes, these are handled separately at the start to ensure they are // they are exported even if not directly used by an exported target. if e.preloadedSubincludes[subinclude] { @@ -296,19 +290,14 @@ func (e *defaultExporter) exportSubincludes(pkg *core.Package, target *core.Buil // exportRelatedTargets exports build targets that are related to the build statement that generated. func (e *defaultExporter) exportRelatedTargets(pkg *core.Package, target *core.BuildTarget) { - stmt, err := pkg.Metadata.FindStatement(target) - if err != nil { - log.Errorf("Failed to find statement in %s: %w", pkg.Name, err) + stmt := pkg.Metadata.FindStatement(target) + if stmt == nil { + log.Errorf("Failed to find statement for target %s in %s", target, pkg.Name) return } - relatedTargets, err := pkg.Metadata.FindTargets(stmt) - if err != nil { - log.Errorf("Failed to lookup related targets for package %s: %w", pkg.Name, err) - return - } - - log.Debugf("Exporting related targets to (%v): %v", target.Label, relatedTargets) + relatedTargets := pkg.Metadata.FindTargets(stmt) + log.Debugf("Exporting targets related to %s: %v", target, relatedTargets) for _, target := range relatedTargets { e.ExportTarget(target) } @@ -357,12 +346,12 @@ func (e *defaultExporter) filterPackageFile(pkg *core.Package) ([]byte, error) { cursor = bStmt.Start } - if stmtLabels, ok := pkg.Metadata.GetSubincludedLabels(&bStmt); ok { + if stmtLabels := pkg.Metadata.GetSubincludedLabels(&bStmt); len(stmtLabels) > 0 { // Write filtered subincludes subStmt := e.minimalSubincludeStatement(pkg, stmtLabels) buffer.Write([]byte(subStmt)) log.Debugf("Decision: %s", subStmt) - } else if required, err := e.isRequiredStatement(pkg, &bStmt); err == nil && !required { + } else if e.shouldSkipStatement(pkg, &bStmt) { // Don't write statements that generate targets we are not interested about log.Debugf("Decision: ") } else { @@ -384,17 +373,21 @@ func (e *defaultExporter) filterPackageFile(pkg *core.Package) ([]byte, error) { return buffer.Bytes(), nil } -// isRequiredStatement evaluates if the current build statement is required by the export. -func (e *defaultExporter) isRequiredStatement(pkg *core.Package, stmt *core.BuildStatement) (bool, error) { - targets, err := pkg.Metadata.FindTargets(stmt) - if err != nil { - return false, err +// shouldSkipStatement evaluates if the current build statement should be skipped during export. +// We skip statements that generated build targets, but none of those targets are required by the export. +func (e *defaultExporter) shouldSkipStatement(pkg *core.Package, stmt *core.BuildStatement) bool { + targets := pkg.Metadata.FindTargets(stmt) + if len(targets) == 0 { + // If the statement didn't generate any targets (e.g. variable assignments, package() calls), + // we keep it to ensure the BUILD file remains valid. + return false } required := slices.ContainsFunc(targets, func(target *core.BuildTarget) bool { return e.exportedTargets[pkg][target.Label] }) - return required, nil + // Skip if it generated targets, but none of them are required. + return !required } // minimalSubincludeStatement generates a subinclude statement containing only the required labels. From 2edfee05e98b898a8bd7b26152cf054c23e4a54b Mon Sep 17 00:00:00 2001 From: DuBento Date: Thu, 14 May 2026 11:26:39 +0100 Subject: [PATCH 057/118] linter fix imports --- src/export/export_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/src/export/export_test.go b/src/export/export_test.go index 70aa31bd42..96f082e596 100644 --- a/src/export/export_test.go +++ b/src/export/export_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/thought-machine/please/src/core" ) From f59b5d0691510fddc72be5df6b00daaf5f676ab8 Mon Sep 17 00:00:00 2001 From: DuBento Date: Thu, 14 May 2026 12:09:00 +0100 Subject: [PATCH 058/118] ensure subrepo target exists. Failing to export //... on this repo --- src/export/export.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/export/export.go b/src/export/export.go index fa0d3d925e..b816520d51 100644 --- a/src/export/export.go +++ b/src/export/export.go @@ -228,7 +228,7 @@ func (e *defaultExporter) ExportTarget(target *core.BuildTarget) { } // We want to export the package that made this subrepo available, but we still need to walk the // target deps as it may depend on other subrepos or first party targets - if target.Subrepo != nil { + if target.Subrepo != nil && target.Subrepo.Target != nil { e.ExportTarget(target.Subrepo.Target) e.exportDependencies(target) return From 790a437e1e0a8e9c815f56faaa2d7b444fb2a736 Mon Sep 17 00:00:00 2001 From: DuBento Date: Fri, 15 May 2026 11:03:29 +0100 Subject: [PATCH 059/118] go test for filtered package --- src/export/BUILD | 2 + src/export/export.go | 19 ++-- src/export/export_test.go | 87 +++++++++++++++++-- src/export/test_data/filter.build | 17 ++++ src/export/test_data/filter_expected_a.build | 14 +++ src/export/test_data/filter_expected_b.build | 12 +++ .../test_data/filter_expected_none.build | 9 ++ 7 files changed, 144 insertions(+), 16 deletions(-) create mode 100644 src/export/test_data/filter.build create mode 100644 src/export/test_data/filter_expected_a.build create mode 100644 src/export/test_data/filter_expected_b.build create mode 100644 src/export/test_data/filter_expected_none.build diff --git a/src/export/BUILD b/src/export/BUILD index bf46b3a36c..bb625249a3 100644 --- a/src/export/BUILD +++ b/src/export/BUILD @@ -16,9 +16,11 @@ go_library( go_test( name = "export_test", srcs = ["export_test.go"], + data = ["test_data"], deps = [ ":export", "///third_party/go/github.com_stretchr_testify//assert", "//src/core", + "//src/parse/asp", ], ) diff --git a/src/export/export.go b/src/export/export.go index b816520d51..1ade9bc75f 100644 --- a/src/export/export.go +++ b/src/export/export.go @@ -97,15 +97,15 @@ func newExporter(state *core.BuildState, dir string, noTrim bool) Exporter { } exporter.impl = exporter return exporter - } else { - exporter := &defaultExporter{ - baseExporter: base, - requiredSubincludes: map[*core.Package]map[core.BuildLabel]bool{}, - preloadedSubincludes: map[core.BuildLabel]bool{}, - } - exporter.impl = exporter - return exporter } + + exporter := &defaultExporter{ + baseExporter: base, + requiredSubincludes: map[*core.Package]map[core.BuildLabel]bool{}, + preloadedSubincludes: map[core.BuildLabel]bool{}, + } + exporter.impl = exporter + return exporter } // baseExporter provides common fields and methods of other exporters. @@ -343,7 +343,6 @@ func (e *defaultExporter) filterPackageFile(pkg *core.Package) ([]byte, error) { if _, err := buffer.Write(original[cursor:bStmt.Start]); err != nil { return nil, err } - cursor = bStmt.Start } if stmtLabels := pkg.Metadata.GetSubincludedLabels(&bStmt); len(stmtLabels) > 0 { @@ -362,6 +361,8 @@ func (e *defaultExporter) filterPackageFile(pkg *core.Package) ([]byte, error) { log.Debugf("Decision: ") } + // Move the cursor to the end of the processed statement. The cursor will enable writing of lines + // that are not considered statements by the parser (e.g. comments, new lines). cursor = bStmt.End } diff --git a/src/export/export_test.go b/src/export/export_test.go index 96f082e596..eff7a6f0ee 100644 --- a/src/export/export_test.go +++ b/src/export/export_test.go @@ -1,11 +1,13 @@ package export import ( + "os" "testing" "github.com/stretchr/testify/assert" "github.com/thought-machine/please/src/core" + "github.com/thought-machine/please/src/parse/asp" ) func TestMinimalSubincludeStatement(t *testing.T) { @@ -44,19 +46,90 @@ func TestMinimalSubincludeStatement(t *testing.T) { }, } - for _, test := range testCases { - t.Run(test.name, func(t *testing.T) { - e := &defaultExporter{ - requiredSubincludes: map[*core.Package]map[core.BuildLabel]bool{}, - } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + e := newExporter(nil, "", false).(*defaultExporter) pkg := &core.Package{Name: "test"} e.requiredSubincludes[pkg] = map[core.BuildLabel]bool{} - for _, label := range test.requiredLabels { + for _, label := range tc.requiredLabels { e.requiredSubincludes[pkg][label] = true } - assert.Equal(t, test.out, e.minimalSubincludeStatement(pkg, test.availableLabels)) + assert.Equal(t, tc.out, e.minimalSubincludeStatement(pkg, tc.availableLabels)) + }) + } +} + +func TestFilterPackageFile(t *testing.T) { + testCases := []struct { + name string + required []string + expected string + }{ + { + name: "Keep only A", + required: []string{"a"}, + expected: "src/export/test_data/filter_expected_a.build", + }, + { + name: "Keep only B", + required: []string{"b"}, + expected: "src/export/test_data/filter_expected_b.build", + }, + { + name: "Keep both", + required: []string{"a", "b"}, + expected: "src/export/test_data/filter.build", + }, + { + name: "Keep none", + required: []string{}, + expected: "src/export/test_data/filter_expected_none.build", + }, + } + + contentPath := "src/export/test_data/filter.build" + + p := asp.NewParserOnly() + statements, err := p.ParseFileOnly(contentPath) + assert.NoError(t, err) + + pkg := core.NewPackage("test", core.WithPackageMetadata()) + pkg.Filename = contentPath + + // stmtIndices maps target names to their statement index in filter.build + stmtIndices := map[string]int{ + "a": 2, + "b": 3, + } + + targetLabels := map[string]core.BuildLabel{} + for name, index := range stmtIndices { + label := core.NewBuildLabel("test", name) + targetLabels[name] = label + target := &core.BuildTarget{Label: label} + pkg.Metadata.RegisterStatementTarget(target, func() core.BuildStatement { + return asp.NewBuildStatement(statements[index]) + }) + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + caseTargets := map[core.BuildLabel]bool{} + for _, name := range tc.required { + caseTargets[targetLabels[name]] = true + } + + e := newExporter(nil, "", false).(*defaultExporter) + e.exportedTargets[pkg] = caseTargets + + got, err := e.filterPackageFile(pkg) + assert.NoError(t, err) + + expected, err := os.ReadFile(tc.expected) + assert.NoError(t, err) + assert.Equal(t, string(expected), string(got)) }) } } diff --git a/src/export/test_data/filter.build b/src/export/test_data/filter.build new file mode 100644 index 0000000000..35f90c2265 --- /dev/null +++ b/src/export/test_data/filter.build @@ -0,0 +1,17 @@ +# This is a header comment +package(default_visibility = ["PUBLIC"]) + +VERSION = "1.2.3" + +target( + name = "a", + # This is a comment inside a target + srcs = ["a.py"], + version = VERSION, +) + +# This is a middle comment +target( + name = "b", + srcs = ["b.py"], +) diff --git a/src/export/test_data/filter_expected_a.build b/src/export/test_data/filter_expected_a.build new file mode 100644 index 0000000000..af682d5dfb --- /dev/null +++ b/src/export/test_data/filter_expected_a.build @@ -0,0 +1,14 @@ +# This is a header comment +package(default_visibility = ["PUBLIC"]) + +VERSION = "1.2.3" + +target( + name = "a", + # This is a comment inside a target + srcs = ["a.py"], + version = VERSION, +) + +# This is a middle comment + diff --git a/src/export/test_data/filter_expected_b.build b/src/export/test_data/filter_expected_b.build new file mode 100644 index 0000000000..7e6f18ca21 --- /dev/null +++ b/src/export/test_data/filter_expected_b.build @@ -0,0 +1,12 @@ +# This is a header comment +package(default_visibility = ["PUBLIC"]) + +VERSION = "1.2.3" + + + +# This is a middle comment +target( + name = "b", + srcs = ["b.py"], +) diff --git a/src/export/test_data/filter_expected_none.build b/src/export/test_data/filter_expected_none.build new file mode 100644 index 0000000000..9ee152a241 --- /dev/null +++ b/src/export/test_data/filter_expected_none.build @@ -0,0 +1,9 @@ +# This is a header comment +package(default_visibility = ["PUBLIC"]) + +VERSION = "1.2.3" + + + +# This is a middle comment + From f5899982e48d3f2df784f651771bf553074f6cb4 Mon Sep 17 00:00:00 2001 From: DuBento Date: Fri, 15 May 2026 18:43:51 +0100 Subject: [PATCH 060/118] Add doc comments for providers in asp.interpreter --- src/parse/asp/interpreter.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/parse/asp/interpreter.go b/src/parse/asp/interpreter.go index 0de403e44f..1a613deabd 100644 --- a/src/parse/asp/interpreter.go +++ b/src/parse/asp/interpreter.go @@ -1088,10 +1088,14 @@ func (s *scope) Constant(expr *Expression) pyObject { return nil } -// CurrentBuildStatement creates a provider for creating a new BuildStatement from the statement -// that is being currently interpreted. +// CurrentBuildStatement creates a provider for getting a BuildStatement from the statement +// that is being currently interpreted. A closure is used to avoid unnecessary computation when the +// metadata is not being tracked. func (s *scope) CurrentBuildStatement() core.BuildStatementProvider { return func() core.BuildStatement { + // We walk back on the callstack until we find the highest-level function call in the package file. + // This statement should be the root method call, from a possibly long callstack, at the original + // package level that generated the current build target. stmtScope := s for curr := s; curr != nil; curr = curr.caller { if curr.pkg != nil && curr.filename == s.pkg.Filename { @@ -1103,10 +1107,14 @@ func (s *scope) CurrentBuildStatement() core.BuildStatementProvider { } } -// ActiveSubincludes creates a provider to trace the call stack and scopes to find subincludes that -// provided the macros/functions actively executing to define this target.1 +// ActiveSubincludes creates a provider that reports the active/required subincluded targets for a +// certain scope. This gives the explicitly subincluded targets that generate the methods we used +// in the current callstack, actively executing to define this target. func (s *scope) ActiveSubincludes() core.SubincludesLabelProvider { return func() core.BuildLabels { + // We walk back on the callstack. For each scope of a method call we walk back at the local/lexical + // scopes in that method's context to find the original/root scope. If that scope includes a "subincludeLabel" + // it means this scope was generated by a subinclude statement and we'll register that label as required. seen := map[core.BuildLabel]bool{} for callScope := s; callScope != nil; callScope = callScope.caller { for lexicalScope := callScope; lexicalScope != nil; lexicalScope = lexicalScope.parent { From 75521e2f7599b355765b43076e85286e7f5fbc0f Mon Sep 17 00:00:00 2001 From: DuBento Date: Fri, 15 May 2026 18:44:16 +0100 Subject: [PATCH 061/118] test: function def in package file --- .../BUILD | 2 +- .../expected_repo/.plzconfig | 0 .../expected_repo/BUILD_FILE | 0 .../expected_repo/file.txt | 0 .../expected_repo/simple.build_defs | 0 .../source_repo/.plzconfig | 0 .../source_repo/BUILD_FILE | 0 .../source_repo/dummy.in | 0 .../source_repo/file.txt | 0 .../source_repo/simple.build_defs | 0 test/export/test_in_file_func_def/BUILD | 8 +++++++ .../expected_repo/.plzconfig | 3 +++ .../expected_repo/BUILD_FILE | 16 ++++++++++++++ .../expected_repo/file.txt | 1 + .../source_repo/.plzconfig | 3 +++ .../source_repo/BUILD_FILE | 22 +++++++++++++++++++ .../source_repo/dummy.in | 0 .../source_repo/file.txt | 1 + 18 files changed, 55 insertions(+), 1 deletion(-) rename test/export/{test_custom_in_file_def => test_in_file_build_def}/BUILD (83%) rename test/export/{test_custom_in_file_def => test_in_file_build_def}/expected_repo/.plzconfig (100%) rename test/export/{test_custom_in_file_def => test_in_file_build_def}/expected_repo/BUILD_FILE (100%) rename test/export/{test_custom_in_file_def => test_in_file_build_def}/expected_repo/file.txt (100%) rename test/export/{test_custom_in_file_def => test_in_file_build_def}/expected_repo/simple.build_defs (100%) rename test/export/{test_custom_in_file_def => test_in_file_build_def}/source_repo/.plzconfig (100%) rename test/export/{test_custom_in_file_def => test_in_file_build_def}/source_repo/BUILD_FILE (100%) rename test/export/{test_custom_in_file_def => test_in_file_build_def}/source_repo/dummy.in (100%) rename test/export/{test_custom_in_file_def => test_in_file_build_def}/source_repo/file.txt (100%) rename test/export/{test_custom_in_file_def => test_in_file_build_def}/source_repo/simple.build_defs (100%) create mode 100644 test/export/test_in_file_func_def/BUILD create mode 100644 test/export/test_in_file_func_def/expected_repo/.plzconfig create mode 100644 test/export/test_in_file_func_def/expected_repo/BUILD_FILE create mode 100644 test/export/test_in_file_func_def/expected_repo/file.txt create mode 100644 test/export/test_in_file_func_def/source_repo/.plzconfig create mode 100644 test/export/test_in_file_func_def/source_repo/BUILD_FILE create mode 100644 test/export/test_in_file_func_def/source_repo/dummy.in create mode 100644 test/export/test_in_file_func_def/source_repo/file.txt diff --git a/test/export/test_custom_in_file_def/BUILD b/test/export/test_in_file_build_def/BUILD similarity index 83% rename from test/export/test_custom_in_file_def/BUILD rename to test/export/test_in_file_build_def/BUILD index 5f361a61dd..a9670e332f 100644 --- a/test/export/test_custom_in_file_def/BUILD +++ b/test/export/test_in_file_build_def/BUILD @@ -3,6 +3,6 @@ subinclude("//test/export:export_e2e_test_build_def") # Export a target generated by a custom build def defined in the same # BUILD file. please_export_e2e_test( - name = "export_custom_target_in_file", + name = "export_in_file_build_def", export_targets = ["//:simple_custom_target"], ) diff --git a/test/export/test_custom_in_file_def/expected_repo/.plzconfig b/test/export/test_in_file_build_def/expected_repo/.plzconfig similarity index 100% rename from test/export/test_custom_in_file_def/expected_repo/.plzconfig rename to test/export/test_in_file_build_def/expected_repo/.plzconfig diff --git a/test/export/test_custom_in_file_def/expected_repo/BUILD_FILE b/test/export/test_in_file_build_def/expected_repo/BUILD_FILE similarity index 100% rename from test/export/test_custom_in_file_def/expected_repo/BUILD_FILE rename to test/export/test_in_file_build_def/expected_repo/BUILD_FILE diff --git a/test/export/test_custom_in_file_def/expected_repo/file.txt b/test/export/test_in_file_build_def/expected_repo/file.txt similarity index 100% rename from test/export/test_custom_in_file_def/expected_repo/file.txt rename to test/export/test_in_file_build_def/expected_repo/file.txt diff --git a/test/export/test_custom_in_file_def/expected_repo/simple.build_defs b/test/export/test_in_file_build_def/expected_repo/simple.build_defs similarity index 100% rename from test/export/test_custom_in_file_def/expected_repo/simple.build_defs rename to test/export/test_in_file_build_def/expected_repo/simple.build_defs diff --git a/test/export/test_custom_in_file_def/source_repo/.plzconfig b/test/export/test_in_file_build_def/source_repo/.plzconfig similarity index 100% rename from test/export/test_custom_in_file_def/source_repo/.plzconfig rename to test/export/test_in_file_build_def/source_repo/.plzconfig diff --git a/test/export/test_custom_in_file_def/source_repo/BUILD_FILE b/test/export/test_in_file_build_def/source_repo/BUILD_FILE similarity index 100% rename from test/export/test_custom_in_file_def/source_repo/BUILD_FILE rename to test/export/test_in_file_build_def/source_repo/BUILD_FILE diff --git a/test/export/test_custom_in_file_def/source_repo/dummy.in b/test/export/test_in_file_build_def/source_repo/dummy.in similarity index 100% rename from test/export/test_custom_in_file_def/source_repo/dummy.in rename to test/export/test_in_file_build_def/source_repo/dummy.in diff --git a/test/export/test_custom_in_file_def/source_repo/file.txt b/test/export/test_in_file_build_def/source_repo/file.txt similarity index 100% rename from test/export/test_custom_in_file_def/source_repo/file.txt rename to test/export/test_in_file_build_def/source_repo/file.txt diff --git a/test/export/test_custom_in_file_def/source_repo/simple.build_defs b/test/export/test_in_file_build_def/source_repo/simple.build_defs similarity index 100% rename from test/export/test_custom_in_file_def/source_repo/simple.build_defs rename to test/export/test_in_file_build_def/source_repo/simple.build_defs diff --git a/test/export/test_in_file_func_def/BUILD b/test/export/test_in_file_func_def/BUILD new file mode 100644 index 0000000000..cfb1eabd83 --- /dev/null +++ b/test/export/test_in_file_func_def/BUILD @@ -0,0 +1,8 @@ +subinclude("//test/export:export_e2e_test_build_def") + +# Export a target generated by a custom function def defined in the same +# BUILD file. +please_export_e2e_test( + name = "export_in_file_function", + export_targets = ["//:custom_target"], +) diff --git a/test/export/test_in_file_func_def/expected_repo/.plzconfig b/test/export/test_in_file_func_def/expected_repo/.plzconfig new file mode 100644 index 0000000000..8e1ae5655a --- /dev/null +++ b/test/export/test_in_file_func_def/expected_repo/.plzconfig @@ -0,0 +1,3 @@ +[Parse] + +BuildFileName = BUILD_FILE diff --git a/test/export/test_in_file_func_def/expected_repo/BUILD_FILE b/test/export/test_in_file_func_def/expected_repo/BUILD_FILE new file mode 100644 index 0000000000..4638bb33ab --- /dev/null +++ b/test/export/test_in_file_func_def/expected_repo/BUILD_FILE @@ -0,0 +1,16 @@ +def custom( + name:str, + srcs:list=[], + outs:list=[]): + return genrule( + name = name, + srcs = srcs, + outs = outs, + cmd = "cat $SRCS > $OUT", + ) + +custom( + name = "custom_target", + srcs = ["file.txt"], + outs = ["file_simple.out"], +) diff --git a/test/export/test_in_file_func_def/expected_repo/file.txt b/test/export/test_in_file_func_def/expected_repo/file.txt new file mode 100644 index 0000000000..9768ee14c2 --- /dev/null +++ b/test/export/test_in_file_func_def/expected_repo/file.txt @@ -0,0 +1 @@ +Test source file diff --git a/test/export/test_in_file_func_def/source_repo/.plzconfig b/test/export/test_in_file_func_def/source_repo/.plzconfig new file mode 100644 index 0000000000..8e1ae5655a --- /dev/null +++ b/test/export/test_in_file_func_def/source_repo/.plzconfig @@ -0,0 +1,3 @@ +[Parse] + +BuildFileName = BUILD_FILE diff --git a/test/export/test_in_file_func_def/source_repo/BUILD_FILE b/test/export/test_in_file_func_def/source_repo/BUILD_FILE new file mode 100644 index 0000000000..93fc806076 --- /dev/null +++ b/test/export/test_in_file_func_def/source_repo/BUILD_FILE @@ -0,0 +1,22 @@ +def custom( + name:str, + srcs:list=[], + outs:list=[]): + return genrule( + name = name, + srcs = srcs, + outs = outs, + cmd = "cat $SRCS > $OUT", + ) + +custom( + name = "custom_target", + srcs = ["file.txt"], + outs = ["file_simple.out"], +) + +custom( + name = "dummy", + srcs = ["dummy.in"], + outs = ["dummy.out"], +) diff --git a/test/export/test_in_file_func_def/source_repo/dummy.in b/test/export/test_in_file_func_def/source_repo/dummy.in new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/export/test_in_file_func_def/source_repo/file.txt b/test/export/test_in_file_func_def/source_repo/file.txt new file mode 100644 index 0000000000..9768ee14c2 --- /dev/null +++ b/test/export/test_in_file_func_def/source_repo/file.txt @@ -0,0 +1 @@ +Test source file From 2e2edde2683f825568eee61e90172964a7718c3f Mon Sep 17 00:00:00 2001 From: DuBento Date: Fri, 15 May 2026 19:31:08 +0100 Subject: [PATCH 062/118] unit testing current statement and active subincludes --- src/parse/asp/interpreter_test.go | 122 ++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) diff --git a/src/parse/asp/interpreter_test.go b/src/parse/asp/interpreter_test.go index 511f00b053..a58aee9154 100644 --- a/src/parse/asp/interpreter_test.go +++ b/src/parse/asp/interpreter_test.go @@ -770,3 +770,125 @@ func TestStrRjust(t *testing.T) { _, err = parseFile("src/parse/asp/test_data/interpreter/str/rjust_multiple_fillchars.build") assert.Error(t, err, "fillchar must be exactly one character long") } + +func TestCurrentBuildStatement(t *testing.T) { + pkg := core.NewPackage("test/package") + pkg.Filename = "test/package/BUILD" + + // Root statement in the BUILD file (e.g. a macro call) + rootStmt := &Statement{Pos: 10, EndPos: 20} + rootScope := &scope{ + pkg: pkg, + filename: pkg.Filename, + cursor: rootStmt, + } + + // A nested call inside the same BUILD file (e.g. function def) + NestedStmt := &Statement{Pos: 30, EndPos: 40} + NestedScope := &scope{ + pkg: pkg, + filename: pkg.Filename, + cursor: NestedStmt, + caller: rootScope, + } + + // A call from a different file (e.g. a function inside a subincluded .build_defs file) + defsRootStmt := &Statement{Pos: 50, EndPos: 60} + defsRootScope := &scope{ + pkg: pkg, + filename: "other/file.build_defs", + cursor: defsRootStmt, + caller: NestedScope, + } + + // Another call deep in the other file + defsNestedStmt := &Statement{Pos: 70, EndPos: 80} + defsNestedScope := &scope{ + pkg: pkg, + filename: "other/file.build_defs", + cursor: defsNestedStmt, + caller: defsRootScope, + } + + t.Run("FindsRootStatementFromBUILD", func(t *testing.T) { + // Calling it from buildNestedScope should walk back to buildRootScope + stmt := NestedScope.CurrentBuildStatement()() + assert.Equal(t, NewBuildStatement(rootStmt), stmt) + }) + + t.Run("FindsRootStatementFromOtherFile", func(t *testing.T) { + // Calling it from defsNestedScope should still find the root statement in the BUILD file + stmt := defsNestedScope.CurrentBuildStatement()() + assert.Equal(t, NewBuildStatement(rootStmt), stmt) + }) + + t.Run("HandlesNoPackageFileInStack", func(t *testing.T) { + // A scope that has no pkg/filename context + standaloneScope := &scope{cursor: rootStmt} + stmt := standaloneScope.CurrentBuildStatement()() + assert.Equal(t, NewBuildStatement(rootStmt), stmt) + }) +} + +func TestActiveSubincludes(t *testing.T) { + labelA := core.ParseBuildLabel("//pkg:labelA", "") + labelB := core.ParseBuildLabel("//pkg:labelB", "") + + t.Run("NoSubincludes", func(t *testing.T) { + // BUILD scope + scopeBUILD := &scope{} + // Function execution + scopeFuncExec := &scope{ + caller: scopeBUILD, + } + labels := scopeFuncExec.ActiveSubincludes()() + assert.Empty(t, labels) + }) + + t.Run("SingleSubinclude", func(t *testing.T) { + // File A scope + scopeA := &scope{ + subincludeLabel: &labelA, + } + // Function defined in File A + scopeFuncDef := &scope{ + parent: scopeA, + } + // BUILD scope + scopeBUILD := &scope{} + // Function execution + scopeFuncExec := &scope{ + parent: scopeFuncDef, + caller: scopeBUILD, + } + + labels := scopeFuncExec.ActiveSubincludes()() + assert.Equal(t, core.BuildLabels{labelA}, labels) + }) + + t.Run("NestedSubincludes", func(t *testing.T) { + // File A scope + scopeA := &scope{ + subincludeLabel: &labelA, + } + // File B scope (subincluded by A) + scopeB := &scope{ + subincludeLabel: &labelB, + parent: scopeA, + } + // Function defined in File B + scopeFuncDef := &scope{ + parent: scopeB, + } + // BUILD scope + scopeBUILD := &scope{} + // Function execution + scopeFuncExec := &scope{ + parent: scopeFuncDef, + caller: scopeBUILD, + } + + labels := scopeFuncExec.ActiveSubincludes()() + assert.ElementsMatch(t, core.BuildLabels{labelA, labelB}, labels) + }) +} From 0e74e530a298a81789675acb45a432e7f95d26a0 Mon Sep 17 00:00:00 2001 From: DuBento Date: Mon, 18 May 2026 14:21:10 +0100 Subject: [PATCH 063/118] remove nested maps and pointers in favor of using build labels --- src/core/package_metadata.go | 4 --- src/export/export.go | 65 ++++++++++++++++-------------------- src/export/export_test.go | 13 +++----- 3 files changed, 33 insertions(+), 49 deletions(-) diff --git a/src/core/package_metadata.go b/src/core/package_metadata.go index 5fe0ffac57..03d7b314e6 100644 --- a/src/core/package_metadata.go +++ b/src/core/package_metadata.go @@ -104,10 +104,6 @@ func (m *packageMetadataImpl) RegisterStatementTarget(target *BuildTarget, stmtP func (m *packageMetadataImpl) RegisterRequiredSubinclude(target *BuildTarget, labelProvider SubincludesLabelProvider) { labels := labelProvider() - if len(labels) == 0 { - log.Debugf("Attempted to register empty subinclude labels for target %s", target.String()) - return - } m.mutex.Lock() defer m.mutex.Unlock() diff --git a/src/export/export.go b/src/export/export.go index 1ade9bc75f..9bed8f2073 100644 --- a/src/export/export.go +++ b/src/export/export.go @@ -87,7 +87,7 @@ func newExporter(state *core.BuildState, dir string, noTrim bool) Exporter { base := baseExporter{ state: state, targetDir: dir, - exportedTargets: map[*core.Package]map[core.BuildLabel]bool{}, + exportedTargets: map[core.BuildLabel]bool{}, } if noTrim { @@ -101,7 +101,8 @@ func newExporter(state *core.BuildState, dir string, noTrim bool) Exporter { exporter := &defaultExporter{ baseExporter: base, - requiredSubincludes: map[*core.Package]map[core.BuildLabel]bool{}, + visitedPackages: map[core.BuildLabel]bool{}, + requiredSubincludes: map[core.BuildLabel]core.BuildLabels{}, preloadedSubincludes: map[core.BuildLabel]bool{}, } exporter.impl = exporter @@ -114,7 +115,7 @@ type baseExporter struct { targetDir string // exportedTargets maintains a record of the targets that have been exported so far. - exportedTargets map[*core.Package]map[core.BuildLabel]bool + exportedTargets map[core.BuildLabel]bool // impl is a reference to the concrete exporter implementation. It's included for calling the // specific exporter implementation from the common methods. impl Exporter @@ -173,26 +174,21 @@ func (be *baseExporter) exportSources(target *core.BuildTarget) { } } -// checkFirstExport is a helper to ensure we only visit the same target once. +// checkAndSetVisited is a helper to ensure we only visit the same target once. // It returns true if this is the first time the target is being exported. -func (be *baseExporter) checkFirstExport(pkg *core.Package, target *core.BuildTarget) bool { - targets, ok := be.exportedTargets[pkg] - if !ok { - targets = make(map[core.BuildLabel]bool) - be.exportedTargets[pkg] = targets - } - if targets[target.Label] { - return false - } - targets[target.Label] = true - return true +func (be *baseExporter) checkAndSetVisited(target *core.BuildTarget) bool { + visited := be.exportedTargets[target.Label] + be.exportedTargets[target.Label] = true + return !visited } // defaultExporter implements an exporter that trims packages to reach a minimal exported repo. type defaultExporter struct { baseExporter + // visitedPackages maintains a record of the packages visited during the export process. + visitedPackages map[core.BuildLabel]bool // requiredSubincludes maps packages to the subinclude labels they require. - requiredSubincludes map[*core.Package]map[core.BuildLabel]bool + requiredSubincludes map[core.BuildLabel]core.BuildLabels // preloadedSubincludes tracks subincludes that are preloaded and don't need explicit export. preloadedSubincludes map[core.BuildLabel]bool } @@ -215,8 +211,7 @@ func (e *defaultExporter) ExportPreloaded() { } func (e *defaultExporter) ExportTarget(target *core.BuildTarget) { - pkg := e.state.Graph.PackageOrDie(target.Label) - if !e.checkFirstExport(pkg, target) { + if !e.checkAndSetVisited(target) { return } @@ -234,20 +229,18 @@ func (e *defaultExporter) ExportTarget(target *core.BuildTarget) { return } - e.exportSubincludes(pkg, target) - e.exportRelatedTargets(pkg, target) e.exportSources(target) e.exportDependencies(target) + + pkg := e.state.Graph.PackageOrDie(target.Label) + e.exportSubincludes(pkg, target) + e.exportRelatedTargets(pkg, target) + e.visitedPackages[pkg.Label()] = true } func (e *defaultExporter) WritePackageFiles() { - for pkg := range e.exportedTargets { - // Skip subrepos and internal packages. These will be generated by build statements in the exported - // repo or included in please internally. - if pkg.Subrepo != nil || pkg.Name == parse.InternalPackageName { - continue - } - + for pkgLabel := range e.visitedPackages { + pkg := e.state.Graph.PackageOrDie(pkgLabel) filteredBytes, err := e.filterPackageFile(pkg) if err != nil { log.Errorf("Failed to filter the build statements of package %s: %v", pkg.Label(), err) @@ -274,10 +267,11 @@ func (e *defaultExporter) exportSubincludes(pkg *core.Package, target *core.Buil continue } - if _, ok := e.requiredSubincludes[pkg]; !ok { - e.requiredSubincludes[pkg] = map[core.BuildLabel]bool{} + required := e.requiredSubincludes[pkg.Label()] + if !slices.Contains(required, subinclude) { + required = append(required, subinclude) } - e.requiredSubincludes[pkg][subinclude] = true + e.requiredSubincludes[pkg.Label()] = required target := e.state.Graph.Target(subinclude) if target == nil { @@ -385,7 +379,7 @@ func (e *defaultExporter) shouldSkipStatement(pkg *core.Package, stmt *core.Buil } required := slices.ContainsFunc(targets, func(target *core.BuildTarget) bool { - return e.exportedTargets[pkg][target.Label] + return e.exportedTargets[target.Label] }) // Skip if it generated targets, but none of them are required. return !required @@ -393,11 +387,10 @@ func (e *defaultExporter) shouldSkipStatement(pkg *core.Package, stmt *core.Buil // minimalSubincludeStatement generates a subinclude statement containing only the required labels. func (e *defaultExporter) minimalSubincludeStatement(pkg *core.Package, available core.BuildLabels) string { - required := e.requiredSubincludes[pkg] var filteredLabels core.BuildLabels - for _, label := range available { - if required[label] { - filteredLabels = append(filteredLabels, label) + for _, required := range e.requiredSubincludes[pkg.Label()] { + if slices.Contains(available, required) { + filteredLabels = append(filteredLabels, required) } } @@ -441,7 +434,7 @@ func (nte *NoTrimExporter) ExportPreloaded() { func (nte *NoTrimExporter) ExportTarget(target *core.BuildTarget) { pkg := nte.state.Graph.PackageOrDie(target.Label) - if !nte.checkFirstExport(pkg, target) { + if !nte.checkAndSetVisited(target) { return } diff --git a/src/export/export_test.go b/src/export/export_test.go index eff7a6f0ee..35af3db9e5 100644 --- a/src/export/export_test.go +++ b/src/export/export_test.go @@ -51,10 +51,7 @@ func TestMinimalSubincludeStatement(t *testing.T) { e := newExporter(nil, "", false).(*defaultExporter) pkg := &core.Package{Name: "test"} - e.requiredSubincludes[pkg] = map[core.BuildLabel]bool{} - for _, label := range tc.requiredLabels { - e.requiredSubincludes[pkg][label] = true - } + e.requiredSubincludes[pkg.Label()] = tc.requiredLabels assert.Equal(t, tc.out, e.minimalSubincludeStatement(pkg, tc.availableLabels)) }) @@ -116,13 +113,11 @@ func TestFilterPackageFile(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - caseTargets := map[core.BuildLabel]bool{} + e := newExporter(nil, "", false).(*defaultExporter) for _, name := range tc.required { - caseTargets[targetLabels[name]] = true + e.exportedTargets[targetLabels[name]] = true } - - e := newExporter(nil, "", false).(*defaultExporter) - e.exportedTargets[pkg] = caseTargets + e.visitedPackages[pkg.Label()] = true got, err := e.filterPackageFile(pkg) assert.NoError(t, err) From ef76bca8c8b0edb4b5b208da934a43a8f451032f Mon Sep 17 00:00:00 2001 From: DuBento Date: Mon, 18 May 2026 14:22:24 +0100 Subject: [PATCH 064/118] unexport noTrim implementation --- src/export/export.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/export/export.go b/src/export/export.go index 9bed8f2073..78b5197962 100644 --- a/src/export/export.go +++ b/src/export/export.go @@ -91,7 +91,7 @@ func newExporter(state *core.BuildState, dir string, noTrim bool) Exporter { } if noTrim { - exporter := &NoTrimExporter{ + exporter := &noTrimExporter{ baseExporter: base, exportedPackages: map[string]bool{}, } @@ -410,15 +410,15 @@ func (e *defaultExporter) minimalSubincludeStatement(pkg *core.Package, availabl return build.FormatString(call) } -// NoTrimExporter implements an exporter that avoids trimming any packages by exporting all targets +// noTrimExporter implements an exporter that avoids trimming any packages by exporting all targets // and statements in a package. -type NoTrimExporter struct { +type noTrimExporter struct { baseExporter // exportedPackages tracks which packages have already had their BUILD files exported. exportedPackages map[string]bool } -func (nte *NoTrimExporter) ExportPreloaded() { +func (nte *noTrimExporter) ExportPreloaded() { // Write any preloaded build defs for _, preload := range nte.state.Config.Parse.PreloadBuildDefs { if err := fs.RecursiveCopy(preload, filepath.Join(nte.targetDir, preload), 0); err != nil { @@ -432,7 +432,7 @@ func (nte *NoTrimExporter) ExportPreloaded() { } } -func (nte *NoTrimExporter) ExportTarget(target *core.BuildTarget) { +func (nte *noTrimExporter) ExportTarget(target *core.BuildTarget) { pkg := nte.state.Graph.PackageOrDie(target.Label) if !nte.checkAndSetVisited(target) { return @@ -453,11 +453,11 @@ func (nte *NoTrimExporter) ExportTarget(target *core.BuildTarget) { nte.exportDependencies(target) } -func (nte *NoTrimExporter) WritePackageFiles() { +func (nte *noTrimExporter) WritePackageFiles() { } // exportPackage exports the package BUILD file. -func (nte *NoTrimExporter) exportPackage(pkg *core.Package) { +func (nte *noTrimExporter) exportPackage(pkg *core.Package) { // Skip subrepos and internal packages. These will be generated by build statements in the exported // repo or included in please internally. if pkg.Subrepo != nil || pkg.Name == parse.InternalPackageName { @@ -476,7 +476,7 @@ func (nte *NoTrimExporter) exportPackage(pkg *core.Package) { } // exportSubincludes exports the subincluded targets. -func (nte *NoTrimExporter) exportSubincludes(pkg *core.Package) { +func (nte *noTrimExporter) exportSubincludes(pkg *core.Package) { subincludes := pkg.AllSubincludes(nte.state.Graph) for _, subinclude := range subincludes { nte.ExportTarget(nte.state.Graph.TargetOrDie(subinclude)) @@ -484,7 +484,7 @@ func (nte *NoTrimExporter) exportSubincludes(pkg *core.Package) { } // exportAllTargets will export all the targets in the provided package. -func (nte *NoTrimExporter) exportAllTargets(pkg *core.Package) { +func (nte *noTrimExporter) exportAllTargets(pkg *core.Package) { for _, target := range pkg.AllTargets() { nte.ExportTarget(target) } From 8607a7bb8f4ead3b03135261afdd849deb01b115 Mon Sep 17 00:00:00 2001 From: DuBento Date: Thu, 21 May 2026 17:00:05 +0100 Subject: [PATCH 065/118] AST based trimming (if-else and for): - export and trim for stmts: implementation and tests - remove unused indent level - trim if-else stmt by keeping clauses but writing a pass those not required - rework export to trim per stmt and include extra parsing info for if-else stmt --- src/export/BUILD | 5 +- src/export/export.go | 103 ++------ src/export/export_test.go | 173 +++++++++++-- src/export/test_data/if_trim.build | 5 + src/export/trimmer.go | 235 ++++++++++++++++++ src/parse/asp/grammar.go | 18 +- src/parse/asp/grammar_parse.go | 14 +- test/export/test_for/BUILD | 6 + test/export/test_for/expected_repo/.plzconfig | 2 + test/export/test_for/expected_repo/BUILD_FILE | 11 + test/export/test_for/expected_repo/a.src | 1 + test/export/test_for/expected_repo/b.src | 1 + test/export/test_for/expected_repo/c.src | 1 + test/export/test_for/source_repo/.plzconfig | 2 + test/export/test_for/source_repo/BUILD_FILE | 17 ++ test/export/test_for/source_repo/a.src | 1 + test/export/test_for/source_repo/b.src | 1 + test/export/test_for/source_repo/c.src | 1 + test/export/test_for_if/BUILD | 6 + .../test_for_if/expected_repo/.plzconfig | 2 + .../test_for_if/expected_repo/BUILD_FILE | 13 + test/export/test_for_if/expected_repo/a.src | 1 + .../export/test_for_if/source_repo/.plzconfig | 2 + .../export/test_for_if/source_repo/BUILD_FILE | 15 ++ test/export/test_for_if/source_repo/a.src | 1 + test/export/test_for_if/source_repo/b.src | 1 + test/export/test_for_if_both/BUILD | 10 + .../test_for_if_both/expected_repo/.plzconfig | 2 + .../test_for_if_both/expected_repo/BUILD_FILE | 20 ++ .../test_for_if_both/expected_repo/a.src | 1 + .../test_for_if_both/expected_repo/b.src | 1 + .../test_for_if_both/source_repo/.plzconfig | 2 + .../test_for_if_both/source_repo/BUILD_FILE | 25 ++ .../export/test_for_if_both/source_repo/a.src | 1 + .../export/test_for_if_both/source_repo/b.src | 1 + .../export/test_for_if_both/source_repo/c.src | 1 + test/export/test_if/BUILD | 7 + test/export/test_if/expected_repo/.plzconfig | 2 + test/export/test_if/expected_repo/BUILD_FILE | 8 + test/export/test_if/source_repo/.plzconfig | 2 + test/export/test_if/source_repo/BUILD_FILE | 12 + 41 files changed, 613 insertions(+), 120 deletions(-) create mode 100644 src/export/test_data/if_trim.build create mode 100644 src/export/trimmer.go create mode 100644 test/export/test_for/BUILD create mode 100644 test/export/test_for/expected_repo/.plzconfig create mode 100644 test/export/test_for/expected_repo/BUILD_FILE create mode 100644 test/export/test_for/expected_repo/a.src create mode 100644 test/export/test_for/expected_repo/b.src create mode 100644 test/export/test_for/expected_repo/c.src create mode 100644 test/export/test_for/source_repo/.plzconfig create mode 100644 test/export/test_for/source_repo/BUILD_FILE create mode 100644 test/export/test_for/source_repo/a.src create mode 100644 test/export/test_for/source_repo/b.src create mode 100644 test/export/test_for/source_repo/c.src create mode 100644 test/export/test_for_if/BUILD create mode 100644 test/export/test_for_if/expected_repo/.plzconfig create mode 100644 test/export/test_for_if/expected_repo/BUILD_FILE create mode 100644 test/export/test_for_if/expected_repo/a.src create mode 100644 test/export/test_for_if/source_repo/.plzconfig create mode 100644 test/export/test_for_if/source_repo/BUILD_FILE create mode 100644 test/export/test_for_if/source_repo/a.src create mode 100644 test/export/test_for_if/source_repo/b.src create mode 100644 test/export/test_for_if_both/BUILD create mode 100644 test/export/test_for_if_both/expected_repo/.plzconfig create mode 100644 test/export/test_for_if_both/expected_repo/BUILD_FILE create mode 100644 test/export/test_for_if_both/expected_repo/a.src create mode 100644 test/export/test_for_if_both/expected_repo/b.src create mode 100644 test/export/test_for_if_both/source_repo/.plzconfig create mode 100644 test/export/test_for_if_both/source_repo/BUILD_FILE create mode 100644 test/export/test_for_if_both/source_repo/a.src create mode 100644 test/export/test_for_if_both/source_repo/b.src create mode 100644 test/export/test_for_if_both/source_repo/c.src create mode 100644 test/export/test_if/BUILD create mode 100644 test/export/test_if/expected_repo/.plzconfig create mode 100644 test/export/test_if/expected_repo/BUILD_FILE create mode 100644 test/export/test_if/source_repo/.plzconfig create mode 100644 test/export/test_if/source_repo/BUILD_FILE diff --git a/src/export/BUILD b/src/export/BUILD index bb625249a3..658245afcc 100644 --- a/src/export/BUILD +++ b/src/export/BUILD @@ -1,6 +1,9 @@ go_library( name = "export", - srcs = ["export.go"], + srcs = [ + "export.go", + "trimmer.go", + ], pgo_file = "//:pgo", visibility = ["PUBLIC"], deps = [ diff --git a/src/export/export.go b/src/export/export.go index 78b5197962..191749fb37 100644 --- a/src/export/export.go +++ b/src/export/export.go @@ -4,12 +4,10 @@ package export import ( - "bytes" "fmt" "os" "path/filepath" "slices" - "sort" "github.com/please-build/buildtools/build" @@ -241,7 +239,7 @@ func (e *defaultExporter) ExportTarget(target *core.BuildTarget) { func (e *defaultExporter) WritePackageFiles() { for pkgLabel := range e.visitedPackages { pkg := e.state.Graph.PackageOrDie(pkgLabel) - filteredBytes, err := e.filterPackageFile(pkg) + filteredBytes, err := e.trimPackage(pkg) if err != nil { log.Errorf("Failed to filter the build statements of package %s: %v", pkg.Label(), err) continue @@ -249,7 +247,7 @@ func (e *defaultExporter) WritePackageFiles() { parsedBuild, err := build.ParseBuild(pkg.Filename, filteredBytes) if err != nil { - log.Fatalf("Failed to parse bytes for formatting: %v", err) + log.Fatalf("Failed to parse bytes for formatting: %v\nData:\n%s", err, filteredBytes) } formattedBytes := build.Format(parsedBuild) @@ -312,102 +310,29 @@ func (e *defaultExporter) WriteExportedPackageFile(pkg *core.Package, content [] } } -// filterPackageFile filters the statements to be written to the exported BUILD file. -func (e *defaultExporter) filterPackageFile(pkg *core.Package) ([]byte, error) { +// trimPackage filters the statements to be written to the exported BUILD file. +func (e *defaultExporter) trimPackage(pkg *core.Package) ([]byte, error) { p := asp.NewParserOnly() - parsedStmts, err := p.ParseFileOnly(pkg.Filename) + parsed, err := p.ParseFileOnly(pkg.Filename) if err != nil { return nil, fmt.Errorf("Parsing original BUILD file: %v", err) } - original, err := os.ReadFile(pkg.Filename) + content, err := os.ReadFile(pkg.Filename) if err != nil { return nil, fmt.Errorf("Opening original BUILD file: %v", err) } - cursor := 0 - var buffer bytes.Buffer - for _, stmt := range parsedStmts { - bStmt := asp.NewBuildStatement(stmt) - - log.Debugf("Evaluating statement %s", original[bStmt.Start:bStmt.End]) - // Write content that's between stmts (e.g. comments). We skip these while parsing so it won't - // be included in "parsedStmts" but we want the resulting BUILD file to include these. - if cursor < bStmt.Start { - if _, err := buffer.Write(original[cursor:bStmt.Start]); err != nil { - return nil, err - } - } - - if stmtLabels := pkg.Metadata.GetSubincludedLabels(&bStmt); len(stmtLabels) > 0 { - // Write filtered subincludes - subStmt := e.minimalSubincludeStatement(pkg, stmtLabels) - buffer.Write([]byte(subStmt)) - log.Debugf("Decision: %s", subStmt) - } else if e.shouldSkipStatement(pkg, &bStmt) { - // Don't write statements that generate targets we are not interested about - log.Debugf("Decision: ") - } else { - // Write every other statement - if _, err := buffer.Write(original[bStmt.Start:bStmt.End]); err != nil { - return nil, err - } - log.Debugf("Decision: ") - } - - // Move the cursor to the end of the processed statement. The cursor will enable writing of lines - // that are not considered statements by the parser (e.g. comments, new lines). - cursor = bStmt.End - } - - // Write the rest of the original file (non build targets) - if _, err := buffer.Write(original[cursor:]); err != nil { - return nil, err - } - - return buffer.Bytes(), nil -} - -// shouldSkipStatement evaluates if the current build statement should be skipped during export. -// We skip statements that generated build targets, but none of those targets are required by the export. -func (e *defaultExporter) shouldSkipStatement(pkg *core.Package, stmt *core.BuildStatement) bool { - targets := pkg.Metadata.FindTargets(stmt) - if len(targets) == 0 { - // If the statement didn't generate any targets (e.g. variable assignments, package() calls), - // we keep it to ensure the BUILD file remains valid. - return false - } - - required := slices.ContainsFunc(targets, func(target *core.BuildTarget) bool { - return e.exportedTargets[target.Label] - }) - // Skip if it generated targets, but none of them are required. - return !required -} - -// minimalSubincludeStatement generates a subinclude statement containing only the required labels. -func (e *defaultExporter) minimalSubincludeStatement(pkg *core.Package, available core.BuildLabels) string { - var filteredLabels core.BuildLabels - for _, required := range e.requiredSubincludes[pkg.Label()] { - if slices.Contains(available, required) { - filteredLabels = append(filteredLabels, required) - } - } - - if len(filteredLabels) == 0 { - return "" - } - - sort.Sort(filteredLabels) - - call := &build.CallExpr{ - X: &build.Ident{Name: "subinclude"}, - } - for _, label := range filteredLabels { - call.List = append(call.List, &build.StringExpr{Value: label.ShortString(pkg.Label())}) + trimmer := trimmer{ + origin: content, + pkg: pkg, + exporter: e, + // assuming max len of the original file to avoid reallocations. + bytes: make([]byte, 0, len(content)), } + trimmer.trimBlock(parsed, 0, asp.Position(len(content))) - return build.FormatString(call) + return trimmer.bytes, nil } // noTrimExporter implements an exporter that avoids trimming any packages by exporting all targets diff --git a/src/export/export_test.go b/src/export/export_test.go index 35af3db9e5..630c937e8e 100644 --- a/src/export/export_test.go +++ b/src/export/export_test.go @@ -2,6 +2,8 @@ package export import ( "os" + "slices" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -52,8 +54,12 @@ func TestMinimalSubincludeStatement(t *testing.T) { pkg := &core.Package{Name: "test"} e.requiredSubincludes[pkg.Label()] = tc.requiredLabels + trimmer := trimmer{ + pkg: pkg, + exporter: e, + } - assert.Equal(t, tc.out, e.minimalSubincludeStatement(pkg, tc.availableLabels)) + assert.Equal(t, tc.out, trimmer.minimalSubincludeStatement(tc.availableLabels)) }) } } @@ -94,22 +100,7 @@ func TestFilterPackageFile(t *testing.T) { pkg := core.NewPackage("test", core.WithPackageMetadata()) pkg.Filename = contentPath - - // stmtIndices maps target names to their statement index in filter.build - stmtIndices := map[string]int{ - "a": 2, - "b": 3, - } - - targetLabels := map[string]core.BuildLabel{} - for name, index := range stmtIndices { - label := core.NewBuildLabel("test", name) - targetLabels[name] = label - target := &core.BuildTarget{Label: label} - pkg.Metadata.RegisterStatementTarget(target, func() core.BuildStatement { - return asp.NewBuildStatement(statements[index]) - }) - } + targetLabels := walkASTRegisterTargets(t, statements, pkg, nil) for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { @@ -119,7 +110,7 @@ func TestFilterPackageFile(t *testing.T) { } e.visitedPackages[pkg.Label()] = true - got, err := e.filterPackageFile(pkg) + got, err := e.trimPackage(pkg) assert.NoError(t, err) expected, err := os.ReadFile(tc.expected) @@ -128,3 +119,149 @@ func TestFilterPackageFile(t *testing.T) { }) } } + +func TestStatementTrim(t *testing.T) { + testCases := []struct { + name string + content string + registered []string + required []string + expected string + }{ + { + name: "Keep target in if", + content: ` +if True: + genrule(name = "a", cmd = "echo a > $OUT", outs = ["a"]) +`, + registered: []string{"a"}, + required: []string{"a"}, + expected: ` +if True: + genrule(name = "a", cmd = "echo a > $OUT", outs = ["a"]) +`, + }, + { + name: "Target not required - all statements trimmed", + content: ` +if True: + genrule(name = "a", cmd = "echo a > $OUT", outs = ["a"]) +`, + registered: []string{"a"}, + required: []string{}, + // Empty, all statements pruned. Blank space removal is not performed by trimBlock's implementation so expect the new lines. + expected: ` + +`, + }, + { + name: "Required target in elif", + content: ` +if False: + genrule(name = "a") +elif True: + genrule(name = "b") +else: + genrule(name = "c") +`, + registered: []string{"b"}, + required: []string{"b"}, + expected: ` +if False: + pass #Trimmed during export +elif True: + genrule(name = "b") +else: + pass #Trimmed during export +`}, + { + name: "Required target in for", + content: ` +for i in range(0,2): + genrule(name = "a") +`, + registered: []string{"a"}, + required: []string{"a"}, + expected: ` +for i in range(0,2): + genrule(name = "a") +`}, + { + name: "Required if stmt in for", + content: ` +for i in [ + "a", + "b", +]: + if i == "a": + genrule(name = "a") + elif i == "b": + genrule(name = "b") +`, + registered: []string{"a", "b"}, + required: []string{"a"}, + expected: ` +for i in [ + "a", + "b", +]: + if i == "a": + genrule(name = "a") + elif i == "b": + pass #Trimmed during export +`}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + p := asp.NewParserOnly() + statements, err := p.ParseData([]byte(tc.content), "BUILD") + assert.NoError(t, err) + + pkg := core.NewPackage("test", core.WithPackageMetadata()) + pkg.Filename = "BUILD" + + targetLabels := walkASTRegisterTargets(t, statements, pkg, tc.registered) + e := newExporter(nil, "", false).(*defaultExporter) + for _, name := range tc.required { + e.exportedTargets[targetLabels[name]] = true + } + + trimmer := &trimmer{ + origin: []byte(tc.content), + pkg: pkg, + exporter: e, + } + trimmer.trimBlock(statements, 0, asp.Position(len(tc.content))) + + assert.Equal(t, tc.expected, string(trimmer.bytes)) + }) + } +} + +// walkASTRegisterTargets is a test helper to register simple targets and their build statements. +func walkASTRegisterTargets(t *testing.T, stmts []*asp.Statement, pkg *core.Package, toRegister []string) map[string]core.BuildLabel { + t.Helper() + targetLabels := map[string]core.BuildLabel{} + asp.WalkAST(stmts, func(stmt *asp.Statement) bool { + arg := asp.FindArgument(stmt, "name") + if arg == nil { + return true // Continue + } + + // Not in targets we want to register, continue + name := strings.Trim(arg.Value.Val.String, "\"") + if toRegister != nil && !slices.Contains(toRegister, name) { + return true + } + + label := core.NewBuildLabel(pkg.Name, name) + targetLabels[name] = label + target := &core.BuildTarget{Label: label} + pkg.Metadata.RegisterStatementTarget(target, func() core.BuildStatement { + return asp.NewBuildStatement(stmt) + }) + return true + }) + return targetLabels +} diff --git a/src/export/test_data/if_trim.build b/src/export/test_data/if_trim.build new file mode 100644 index 0000000000..14923efbdf --- /dev/null +++ b/src/export/test_data/if_trim.build @@ -0,0 +1,5 @@ +val = "a" + +if val == "a": + target(name = "a") +elif val == "b": diff --git a/src/export/trimmer.go b/src/export/trimmer.go new file mode 100644 index 0000000000..540e866ca0 --- /dev/null +++ b/src/export/trimmer.go @@ -0,0 +1,235 @@ +package export + +import ( + "slices" + "sort" + + "github.com/please-build/buildtools/build" + "github.com/thought-machine/please/src/core" + "github.com/thought-machine/please/src/parse/asp" +) + +// trimmer implements the filtering logic for statements in package files. +type trimmer struct { + // origin are the bytes from the original package file. + origin []byte + // pkg references the package being trimmed. + pkg *core.Package + // bytes contain the content to be written after the trimming process. + bytes []byte + // exporter is used to lookup target related data from the export process, e.g. which targets are + // required. + exporter *defaultExporter +} + +// statementConsumer defines the type for methods used to visit each statement during a +// file walk. The method accepts the currently interpreted statement. +type statementConsumer func(*asp.Statement) + +// walkFile will walk the file from the specified start to end, consuming the statements and optionally writing non-statement bytes found along the way (e.g. comments and blank space). +func (t *trimmer) walkFile(stmts []*asp.Statement, start, end asp.Position, consumer statementConsumer) { + // cursor tracks the position in a block that's being interpreted. + cursor := start + for _, stmt := range stmts { + + log.Debugf("Evaluating statement %s", t.origin[stmt.Pos:stmt.EndPos]) + // Write content that's between stmts (e.g. comments). We skip these while parsing so it won't + // be included in "parsedStmts" but we want the resulting BUILD file to include these. + if cursor < stmt.Pos { + t.copy(cursor, stmt.Pos) + } + + // Consume stmt with specified method + consumer(stmt) + + // Move the cursor to the end of the processed statement. The cursor will enable writing of lines + // that are not considered statements by the parser (e.g. comments, new lines). + cursor = stmt.EndPos + } + + // Write the rest of the original file (non build targets) + t.copy(cursor, end) +} + +// trimBlock visits all the statements in a block and trims undesired statements. +func (t *trimmer) trimBlock(stmts []*asp.Statement, blockStart, blockEnd asp.Position) { + t.walkFile(stmts, blockStart, blockEnd, func(stmt *asp.Statement) { + if stmt.If != nil { + t.trimIf(stmt) + } else if stmt.For != nil { + t.trimFor(stmt) + } else if stmt.Ident != nil && stmt.Ident.Name == "subinclude" { + t.trimSubinclude(stmt) + } else if relatives := t.relatedTargets(stmt); len(relatives) > 0 { + // Meaning it is a build statement that generated/builds build targets. + if t.anyExported(relatives) { + t.copy(stmt.Pos, stmt.EndPos) + } + } else { + // Write every other statement. + // If the statement didn't generate any targets (e.g. variable assignments, package() calls), + // we keep it to ensure the BUILD file remains valid. + t.copy(stmt.Pos, stmt.EndPos) + } + }) +} + +// trimIf will trim an if-else statement by exporting only the required targets, but keeping the +// if-else primitive -- a implementation decision to help understand the changes caused by an export +// when using a source-control management (SCM) system. +func (t *trimmer) trimIf(stmt *asp.Statement) bool { + type clause struct { + hStart, hEnd asp.Position + stmts []*asp.Statement + } + + clauses := []clause{ + {hStart: stmt.If.HeaderPos, hEnd: stmt.If.HeaderEndPos, stmts: stmt.If.Statements}, + } + for _, elif := range stmt.If.Elif { + clauses = append(clauses, + clause{hStart: elif.HeaderPos, hEnd: elif.HeaderEndPos, stmts: elif.Statements}) + } + if len(stmt.If.ElseStatements) > 0 { + clauses = append(clauses, + clause{hStart: stmt.If.ElseHeaderPos, hEnd: stmt.If.ElseHeaderEndPos, stmts: stmt.If.ElseStatements}) + } + + // In an if-else statement only the interpreted/evaluated block will generate targets, meaning + // that normally only one of the clauses is interpreted, however an if stmt could be inside of + // a loop where the clause condition depends on the iteration meaning more than one clause + // could end up being interpreted. Because of that we lookup all the required clauses before + // writing the statement. + var requiredClauses = make([]bool, len(clauses)) + for i, c := range clauses { + required := t.isRequiredStatements(c.stmts) + requiredClauses[i] = required + } + // No clause is required, skip the if-else stmt entirely + if !slices.Contains(requiredClauses, true) { + return false + } + + for i, c := range clauses { + // Write clause header + t.copy(c.hStart, c.hEnd) + + // Visit statements in block + end := stmt.EndPos + if i+1 < len(clauses) { + end = clauses[i+1].hStart + } + if requiredClauses[i] { + t.trimBlock(c.stmts, c.hEnd, end) + } else { + t.passBlock(c.stmts, c.hEnd, end) + } + } + return true +} + +func (t *trimmer) trimFor(stmt *asp.Statement) { + if len(stmt.For.Statements) == 0 || !t.isRequiredStatement(stmt) { + return + } + + hStart, hEnd := stmt.Pos, stmt.For.Statements[0].Pos + t.copy(hStart, hEnd) + + t.trimBlock(stmt.For.Statements, hEnd, stmt.EndPos) +} + +func (t *trimmer) trimSubinclude(stmt *asp.Statement) { + bStmt := asp.NewBuildStatement(stmt) + stmtLabels := t.pkg.Metadata.GetSubincludedLabels(&bStmt) + subStmt := t.minimalSubincludeStatement(stmtLabels) + t.write([]byte(subStmt)) +} + +// passBlock skips all ASP statements (keeping comments and blank space) and writes a single "pass" +// primitive. +func (t *trimmer) passBlock(stmts []*asp.Statement, blockStart, blockEnd asp.Position) { + var passWritten bool + t.walkFile(stmts, blockStart, blockEnd, func(s *asp.Statement) { + // When the trim results in an empty block, i.e. no statement are written, we write + // the "pass" primitive. This is useful when parsing inner blocks (e.g. if-else stmts). + if !passWritten { + passWritten = true + t.write([]byte("pass #Trimmed during export")) + } + }) + +} + +func (t *trimmer) isRequiredStatements(stmts []*asp.Statement) bool { + return slices.ContainsFunc(stmts, t.isRequiredStatement) +} + +func (t *trimmer) isRequiredStatement(stmt *asp.Statement) bool { + if stmt.If != nil { + // If + if t.isRequiredStatements(stmt.If.Statements) { + return true + } + // Elif + if len(stmt.If.Elif) > 0 { + for _, elif := range stmt.If.Elif { + if t.isRequiredStatements(elif.Statements) { + return true + } + } + } + // Else + if len(stmt.If.ElseStatements) > 0 && t.isRequiredStatements(stmt.If.ElseStatements) { + return true + } + } else if stmt.For != nil { + return t.isRequiredStatements(stmt.For.Statements) + } + return t.anyExported(t.relatedTargets(stmt)) +} + +func (t *trimmer) relatedTargets(stmt *asp.Statement) []*core.BuildTarget { + bStmt := asp.NewBuildStatement(stmt) + return t.pkg.Metadata.FindTargets(&bStmt) +} + +func (t *trimmer) anyExported(targets []*core.BuildTarget) bool { + required := slices.ContainsFunc(targets, func(target *core.BuildTarget) bool { + return t.exporter.exportedTargets[target.Label] + }) + return required +} + +// minimalSubincludeStatement generates a subinclude statement containing only the required labels. +func (t *trimmer) minimalSubincludeStatement(available core.BuildLabels) string { + var filteredLabels core.BuildLabels + for _, required := range t.exporter.requiredSubincludes[t.pkg.Label()] { + if slices.Contains(available, required) { + filteredLabels = append(filteredLabels, required) + } + } + + if len(filteredLabels) == 0 { + return "" + } + + sort.Sort(filteredLabels) + + call := &build.CallExpr{ + X: &build.Ident{Name: "subinclude"}, + } + for _, label := range filteredLabels { + call.List = append(call.List, &build.StringExpr{Value: label.ShortString(t.pkg.Label())}) + } + + return build.FormatString(call) +} + +func (t *trimmer) copy(start, end asp.Position) { + t.bytes = append(t.bytes, t.origin[start:end]...) +} + +func (t *trimmer) write(bytes []byte) { + t.bytes = append(t.bytes, bytes...) +} diff --git a/src/parse/asp/grammar.go b/src/parse/asp/grammar.go index 18084cabc0..1181b71deb 100644 --- a/src/parse/asp/grammar.go +++ b/src/parse/asp/grammar.go @@ -70,16 +70,22 @@ type ForStatement struct { // An IfStatement implements the if-elif-else statement. type IfStatement struct { - Condition Expression - Statements []*Statement - Elif []IfStatementElif - ElseStatements []*Statement + HeaderPos Position + HeaderEndPos Position + Condition Expression + Statements []*Statement + Elif []IfStatementElif + ElseHeaderPos Position + ElseHeaderEndPos Position + ElseStatements []*Statement } // An IfStatementElif holds an elif clause in the if-elif-else statement. type IfStatementElif struct { - Condition Expression - Statements []*Statement + HeaderPos Position + HeaderEndPos Position + Condition Expression + Statements []*Statement } // An Argument represents an argument to a function definition. diff --git a/src/parse/asp/grammar_parse.go b/src/parse/asp/grammar_parse.go index d5bc049bd3..9f582f5c22 100644 --- a/src/parse/asp/grammar_parse.go +++ b/src/parse/asp/grammar_parse.go @@ -312,23 +312,25 @@ func (p *parser) parseArgument() Argument { } func (p *parser) parseIf() *IfStatement { - p.nextv("if") i := &IfStatement{} + i.HeaderPos = p.nextv("if").Pos p.parseExpressionInPlace(&i.Condition) - p.next(':') + i.HeaderEndPos = p.next(':').EndPos() p.next(EOL) i.Statements = p.parseStatements() - for p.optionalv("elif") { + for p.l.Peek().Value == "elif" { elif := IfStatementElif{} + elif.HeaderPos = p.nextv("elif").Pos p.parseExpressionInPlace(&elif.Condition) - p.next(':') + elif.HeaderEndPos = p.next(':').EndPos() p.next(EOL) elif.Statements = p.parseStatements() i.Elif = append(i.Elif, elif) } - if p.optionalv("else") { - p.next(':') + if p.l.Peek().Value == "else" { + i.ElseHeaderPos = p.nextv("else").Pos + i.ElseHeaderEndPos = p.next(':').EndPos() p.next(EOL) i.ElseStatements = p.parseStatements() } diff --git a/test/export/test_for/BUILD b/test/export/test_for/BUILD new file mode 100644 index 0000000000..1b4bb42859 --- /dev/null +++ b/test/export/test_for/BUILD @@ -0,0 +1,6 @@ +subinclude("//test/export:export_e2e_test_build_def") + +please_export_e2e_test( + name = "export_for", + export_targets = ["//:a"], +) diff --git a/test/export/test_for/expected_repo/.plzconfig b/test/export/test_for/expected_repo/.plzconfig new file mode 100644 index 0000000000..f8ba31854d --- /dev/null +++ b/test/export/test_for/expected_repo/.plzconfig @@ -0,0 +1,2 @@ +[Parse] +BuildFileName = BUILD_FILE diff --git a/test/export/test_for/expected_repo/BUILD_FILE b/test/export/test_for/expected_repo/BUILD_FILE new file mode 100644 index 0000000000..5754be2e47 --- /dev/null +++ b/test/export/test_for/expected_repo/BUILD_FILE @@ -0,0 +1,11 @@ +for i in [ + "a", + "b", + "c", +]: + genrule( + name = i, + srcs = [f"{i}.src"], + outs = [f"{i}.txt"], + cmd = f"cat $SRCS > $OUT", + ) diff --git a/test/export/test_for/expected_repo/a.src b/test/export/test_for/expected_repo/a.src new file mode 100644 index 0000000000..7898192261 --- /dev/null +++ b/test/export/test_for/expected_repo/a.src @@ -0,0 +1 @@ +a diff --git a/test/export/test_for/expected_repo/b.src b/test/export/test_for/expected_repo/b.src new file mode 100644 index 0000000000..6178079822 --- /dev/null +++ b/test/export/test_for/expected_repo/b.src @@ -0,0 +1 @@ +b diff --git a/test/export/test_for/expected_repo/c.src b/test/export/test_for/expected_repo/c.src new file mode 100644 index 0000000000..f2ad6c76f0 --- /dev/null +++ b/test/export/test_for/expected_repo/c.src @@ -0,0 +1 @@ +c diff --git a/test/export/test_for/source_repo/.plzconfig b/test/export/test_for/source_repo/.plzconfig new file mode 100644 index 0000000000..f8ba31854d --- /dev/null +++ b/test/export/test_for/source_repo/.plzconfig @@ -0,0 +1,2 @@ +[Parse] +BuildFileName = BUILD_FILE diff --git a/test/export/test_for/source_repo/BUILD_FILE b/test/export/test_for/source_repo/BUILD_FILE new file mode 100644 index 0000000000..50d3a3ca34 --- /dev/null +++ b/test/export/test_for/source_repo/BUILD_FILE @@ -0,0 +1,17 @@ +for i in [ + "a", + "b", + "c", +]: + genrule( + name = i, + srcs = [f"{i}.src"], + outs = [f"{i}.txt"], + cmd = f"cat $SRCS > $OUT", + ) + +genrule( + name = "d", + outs = ["d.txt"], + cmd = "echo d > $OUT", +) diff --git a/test/export/test_for/source_repo/a.src b/test/export/test_for/source_repo/a.src new file mode 100644 index 0000000000..7898192261 --- /dev/null +++ b/test/export/test_for/source_repo/a.src @@ -0,0 +1 @@ +a diff --git a/test/export/test_for/source_repo/b.src b/test/export/test_for/source_repo/b.src new file mode 100644 index 0000000000..6178079822 --- /dev/null +++ b/test/export/test_for/source_repo/b.src @@ -0,0 +1 @@ +b diff --git a/test/export/test_for/source_repo/c.src b/test/export/test_for/source_repo/c.src new file mode 100644 index 0000000000..f2ad6c76f0 --- /dev/null +++ b/test/export/test_for/source_repo/c.src @@ -0,0 +1 @@ +c diff --git a/test/export/test_for_if/BUILD b/test/export/test_for_if/BUILD new file mode 100644 index 0000000000..75b06a3508 --- /dev/null +++ b/test/export/test_for_if/BUILD @@ -0,0 +1,6 @@ +subinclude("//test/export:export_e2e_test_build_def") + +please_export_e2e_test( + name = "export_for_if", + export_targets = ["//:a"], +) diff --git a/test/export/test_for_if/expected_repo/.plzconfig b/test/export/test_for_if/expected_repo/.plzconfig new file mode 100644 index 0000000000..f8ba31854d --- /dev/null +++ b/test/export/test_for_if/expected_repo/.plzconfig @@ -0,0 +1,2 @@ +[Parse] +BuildFileName = BUILD_FILE diff --git a/test/export/test_for_if/expected_repo/BUILD_FILE b/test/export/test_for_if/expected_repo/BUILD_FILE new file mode 100644 index 0000000000..80123416ed --- /dev/null +++ b/test/export/test_for_if/expected_repo/BUILD_FILE @@ -0,0 +1,13 @@ +for i in [ + "a", + "b", +]: + if i == "a": + genrule( + name = "a", + srcs = ["a.src"], + outs = ["a.txt"], + cmd = "cat $SRCS > $OUT", + ) + elif i == "b": + pass #Trimmed during export diff --git a/test/export/test_for_if/expected_repo/a.src b/test/export/test_for_if/expected_repo/a.src new file mode 100644 index 0000000000..7898192261 --- /dev/null +++ b/test/export/test_for_if/expected_repo/a.src @@ -0,0 +1 @@ +a diff --git a/test/export/test_for_if/source_repo/.plzconfig b/test/export/test_for_if/source_repo/.plzconfig new file mode 100644 index 0000000000..f8ba31854d --- /dev/null +++ b/test/export/test_for_if/source_repo/.plzconfig @@ -0,0 +1,2 @@ +[Parse] +BuildFileName = BUILD_FILE diff --git a/test/export/test_for_if/source_repo/BUILD_FILE b/test/export/test_for_if/source_repo/BUILD_FILE new file mode 100644 index 0000000000..7d76b324d1 --- /dev/null +++ b/test/export/test_for_if/source_repo/BUILD_FILE @@ -0,0 +1,15 @@ +for i in ["a", "b"]: + if i == "a": + genrule( + name = "a", + srcs = ["a.src"], + outs = ["a.txt"], + cmd = "cat $SRCS > $OUT", + ) + elif i == "b": + genrule( + name = "b", + srcs = ["b.src"], + outs = ["b.txt"], + cmd = "cat $SRCS > $OUT", + ) diff --git a/test/export/test_for_if/source_repo/a.src b/test/export/test_for_if/source_repo/a.src new file mode 100644 index 0000000000..7898192261 --- /dev/null +++ b/test/export/test_for_if/source_repo/a.src @@ -0,0 +1 @@ +a diff --git a/test/export/test_for_if/source_repo/b.src b/test/export/test_for_if/source_repo/b.src new file mode 100644 index 0000000000..6178079822 --- /dev/null +++ b/test/export/test_for_if/source_repo/b.src @@ -0,0 +1 @@ +b diff --git a/test/export/test_for_if_both/BUILD b/test/export/test_for_if_both/BUILD new file mode 100644 index 0000000000..12734eadef --- /dev/null +++ b/test/export/test_for_if_both/BUILD @@ -0,0 +1,10 @@ +subinclude("//test/export:export_e2e_test_build_def") + +# Export both statements inside the inner if +please_export_e2e_test( + name = "export_for_if_both", + export_targets = [ + "//:a", + "//:b", + ], +) diff --git a/test/export/test_for_if_both/expected_repo/.plzconfig b/test/export/test_for_if_both/expected_repo/.plzconfig new file mode 100644 index 0000000000..f8ba31854d --- /dev/null +++ b/test/export/test_for_if_both/expected_repo/.plzconfig @@ -0,0 +1,2 @@ +[Parse] +BuildFileName = BUILD_FILE diff --git a/test/export/test_for_if_both/expected_repo/BUILD_FILE b/test/export/test_for_if_both/expected_repo/BUILD_FILE new file mode 100644 index 0000000000..7a1337ff98 --- /dev/null +++ b/test/export/test_for_if_both/expected_repo/BUILD_FILE @@ -0,0 +1,20 @@ +for i in [ + "a", + "b", +]: + if i == "a": + genrule( + name = "a", + srcs = ["a.src"], + outs = ["a.txt"], + cmd = "cat $SRCS > $OUT", + ) + elif i == "b": + genrule( + name = "b", + srcs = ["b.src"], + outs = ["b.txt"], + cmd = "cat $SRCS > $OUT", + ) + else: + pass #Trimmed during export diff --git a/test/export/test_for_if_both/expected_repo/a.src b/test/export/test_for_if_both/expected_repo/a.src new file mode 100644 index 0000000000..7898192261 --- /dev/null +++ b/test/export/test_for_if_both/expected_repo/a.src @@ -0,0 +1 @@ +a diff --git a/test/export/test_for_if_both/expected_repo/b.src b/test/export/test_for_if_both/expected_repo/b.src new file mode 100644 index 0000000000..6178079822 --- /dev/null +++ b/test/export/test_for_if_both/expected_repo/b.src @@ -0,0 +1 @@ +b diff --git a/test/export/test_for_if_both/source_repo/.plzconfig b/test/export/test_for_if_both/source_repo/.plzconfig new file mode 100644 index 0000000000..f8ba31854d --- /dev/null +++ b/test/export/test_for_if_both/source_repo/.plzconfig @@ -0,0 +1,2 @@ +[Parse] +BuildFileName = BUILD_FILE diff --git a/test/export/test_for_if_both/source_repo/BUILD_FILE b/test/export/test_for_if_both/source_repo/BUILD_FILE new file mode 100644 index 0000000000..66abd06cb7 --- /dev/null +++ b/test/export/test_for_if_both/source_repo/BUILD_FILE @@ -0,0 +1,25 @@ +for i in [ + "a", + "b", +]: + if i == "a": + genrule( + name = "a", + srcs = ["a.src"], + outs = ["a.txt"], + cmd = "cat $SRCS > $OUT", + ) + elif i == "b": + genrule( + name = "b", + srcs = ["b.src"], + outs = ["b.txt"], + cmd = "cat $SRCS > $OUT", + ) + else: + genrule( + name = "c", + srcs = ["c.src"], + outs = ["c.txt"], + cmd = "cat $SRCS > $OUT", + ) diff --git a/test/export/test_for_if_both/source_repo/a.src b/test/export/test_for_if_both/source_repo/a.src new file mode 100644 index 0000000000..7898192261 --- /dev/null +++ b/test/export/test_for_if_both/source_repo/a.src @@ -0,0 +1 @@ +a diff --git a/test/export/test_for_if_both/source_repo/b.src b/test/export/test_for_if_both/source_repo/b.src new file mode 100644 index 0000000000..6178079822 --- /dev/null +++ b/test/export/test_for_if_both/source_repo/b.src @@ -0,0 +1 @@ +b diff --git a/test/export/test_for_if_both/source_repo/c.src b/test/export/test_for_if_both/source_repo/c.src new file mode 100644 index 0000000000..f2ad6c76f0 --- /dev/null +++ b/test/export/test_for_if_both/source_repo/c.src @@ -0,0 +1 @@ +c diff --git a/test/export/test_if/BUILD b/test/export/test_if/BUILD new file mode 100644 index 0000000000..4af4d89e89 --- /dev/null +++ b/test/export/test_if/BUILD @@ -0,0 +1,7 @@ +subinclude("//test/export:export_e2e_test_build_def") + +# Export a target inside an if block and validate that the else block is correctly trimmed (and replaced with pass). +please_export_e2e_test( + name = "export_if", + export_targets = ["//:a"], +) diff --git a/test/export/test_if/expected_repo/.plzconfig b/test/export/test_if/expected_repo/.plzconfig new file mode 100644 index 0000000000..f8ba31854d --- /dev/null +++ b/test/export/test_if/expected_repo/.plzconfig @@ -0,0 +1,2 @@ +[Parse] +BuildFileName = BUILD_FILE diff --git a/test/export/test_if/expected_repo/BUILD_FILE b/test/export/test_if/expected_repo/BUILD_FILE new file mode 100644 index 0000000000..18e20bb15f --- /dev/null +++ b/test/export/test_if/expected_repo/BUILD_FILE @@ -0,0 +1,8 @@ +if True: + genrule( + name = "a", + outs = ["a.txt"], + cmd = "echo a > $OUT", + ) +else: + pass #Trimmed during export diff --git a/test/export/test_if/source_repo/.plzconfig b/test/export/test_if/source_repo/.plzconfig new file mode 100644 index 0000000000..f8ba31854d --- /dev/null +++ b/test/export/test_if/source_repo/.plzconfig @@ -0,0 +1,2 @@ +[Parse] +BuildFileName = BUILD_FILE diff --git a/test/export/test_if/source_repo/BUILD_FILE b/test/export/test_if/source_repo/BUILD_FILE new file mode 100644 index 0000000000..ef5650c499 --- /dev/null +++ b/test/export/test_if/source_repo/BUILD_FILE @@ -0,0 +1,12 @@ +if True: + genrule( + name = "a", + outs = ["a.txt"], + cmd = "echo a > $OUT", + ) +else: + genrule( + name = "b", + outs = ["b.txt"], + cmd = "echo b > $OUT", + ) From b5bca70774aee63443be30a57a51872171f9e252 Mon Sep 17 00:00:00 2001 From: DuBento Date: Fri, 29 May 2026 17:00:28 +0100 Subject: [PATCH 066/118] handle subinclude variables by tracking origin and used. reworked activeSubincludes compute by relying on new logic --- src/parse/asp/builtins.go | 4 +- src/parse/asp/interpreter.go | 80 ++++++++++++------- test/export/test_subinclude_variable/BUILD | 6 ++ .../expected_repo/.plzconfig | 2 + .../expected_repo/BUILD_FILE | 13 +++ .../expected_repo/defs.build_defs | 1 + .../source_repo/.plzconfig | 2 + .../source_repo/BUILD_FILE | 19 +++++ .../source_repo/defs.build_defs | 1 + 9 files changed, 98 insertions(+), 30 deletions(-) create mode 100644 test/export/test_subinclude_variable/BUILD create mode 100644 test/export/test_subinclude_variable/expected_repo/.plzconfig create mode 100644 test/export/test_subinclude_variable/expected_repo/BUILD_FILE create mode 100644 test/export/test_subinclude_variable/expected_repo/defs.build_defs create mode 100644 test/export/test_subinclude_variable/source_repo/.plzconfig create mode 100644 test/export/test_subinclude_variable/source_repo/BUILD_FILE create mode 100644 test/export/test_subinclude_variable/source_repo/defs.build_defs diff --git a/src/parse/asp/builtins.go b/src/parse/asp/builtins.go index 63f28b8a13..6a57dbce99 100644 --- a/src/parse/asp/builtins.go +++ b/src/parse/asp/builtins.go @@ -310,7 +310,7 @@ func bazelLoad(s *scope, args []pyObject) pyObject { } filename = subrepo.Dir(filename) } - s.SetAll(s.interpreter.Subinclude(s, filename, l, false), false) + s.SetAllWithOrigin(s.interpreter.Subinclude(s, filename, l, false), false, &l) return None } @@ -369,7 +369,7 @@ func subinclude(s *scope, args []pyObject) pyObject { outs = t.Outputs() } for _, out := range outs { - s.SetAll(s.interpreter.Subinclude(s, filepath.Join(t.OutDir(), out), t.Label, false), false) + s.SetAllWithOrigin(s.interpreter.Subinclude(s, filepath.Join(t.OutDir(), out), t.Label, false), false, &t.Label) } } return None diff --git a/src/parse/asp/interpreter.go b/src/parse/asp/interpreter.go index 1a613deabd..21be24d381 100644 --- a/src/parse/asp/interpreter.go +++ b/src/parse/asp/interpreter.go @@ -42,9 +42,11 @@ type interpreter struct { // It loads all the builtin rules at this point. func newInterpreter(state *core.BuildState, p *Parser) *interpreter { s := &scope{ - ctx: context.Background(), - state: state, - locals: map[string]pyObject{}, + ctx: context.Background(), + state: state, + locals: map[string]pyObject{}, + objectOrigins: map[string]core.BuildLabel{}, + requiredOrigins: map[core.BuildLabel]bool{}, } i := &interpreter{ scope: s, @@ -157,7 +159,7 @@ func (i *interpreter) preloadSubinclude(s *scope, label core.BuildLabel) (err er s.interpreter.loadPluginConfig(s, includeState) for _, out := range t.FullOutputs() { - s.SetAll(s.interpreter.Subinclude(s, out, t.Label, true), false) + s.SetAllWithOrigin(s.interpreter.Subinclude(s, out, t.Label, true), false, &t.Label) } return nil } @@ -319,6 +321,10 @@ type scope struct { mode core.ParseMode // cursor points to the statement currently being interpreted cursor *Statement + // objectOrigins tracks the subinclude label that each variable was originally defined in. + objectOrigins map[string]core.BuildLabel + // requiredOrigins tracks which subinclude labels have been requiredOrigins by looking up variables. + requiredOrigins map[core.BuildLabel]bool } // parseAnnotatedLabelInPackage similarly to parseLabelInPackage, parses the label contextualising it to the provided @@ -424,18 +430,20 @@ func (s *scope) NewPackagedScope(pkg *core.Package, mode core.ParseMode, hint in func (s *scope) newScope(pkg *core.Package, mode core.ParseMode, filename string, hint int) *scope { s2 := &scope{ - ctx: s.ctx, - filename: filename, - interpreter: s.interpreter, - state: s.state, - pkg: pkg, - parsingFor: s.parsingFor, - parent: s, - locals: make(pyDict, hint), - config: s.config, - Callback: s.Callback, - mode: mode, - cursor: s.cursor, + ctx: s.ctx, + filename: filename, + interpreter: s.interpreter, + state: s.state, + pkg: pkg, + parsingFor: s.parsingFor, + parent: s, + locals: make(pyDict, hint), + config: s.config, + Callback: s.Callback, + mode: mode, + cursor: s.cursor, + objectOrigins: map[string]core.BuildLabel{}, + requiredOrigins: map[core.BuildLabel]bool{}, } if pkg != nil && pkg.Subrepo != nil && pkg.Subrepo.State != nil { s2.state = pkg.Subrepo.State @@ -466,12 +474,24 @@ func (s *scope) NAssert(condition bool, msg string, args ...interface{}) { // Lookup looks up a variable name in this scope, walking back up its ancestor scopes as needed. // It panics if the variable is not defined. func (s *scope) Lookup(name string) pyObject { + obj, origin := s.lookupWithOrigin(name) + if origin != nil && s.state.ParseMetadata { + s.requiredOrigins[*origin] = true + } + return obj +} + +// lookupWithOrigin is like Lookup but returns the origin label of the variable as well. +func (s *scope) lookupWithOrigin(name string) (pyObject, *core.BuildLabel) { if obj, present := s.locals[name]; present { - return obj + if label, ok := s.objectOrigins[name]; ok { + return obj, &label + } + return obj, nil } else if s.parent != nil { - return s.parent.Lookup(name) + return s.parent.lookupWithOrigin(name) } - return s.Error("name '%s' is not defined", name) + return s.Error("name '%s' is not defined", name), nil } // LocalLookup looks up a variable name in the current scope. @@ -490,6 +510,11 @@ func (s *scope) Set(name string, value pyObject) { // SetAll sets all contents of the given dict in this scope. // Optionally it can filter to just public objects (i.e. those not prefixed with an underscore) func (s *scope) SetAll(d pyDict, publicOnly bool) { + s.SetAllWithOrigin(d, publicOnly, nil) +} + +// SetAllWithOrigin is like SetAll but also records the origin label for all variables. +func (s *scope) SetAllWithOrigin(d pyDict, publicOnly bool, origin *core.BuildLabel) { for k, v := range d { if k == "CONFIG" { // Special case; need to merge config entries rather than overwriting the entire object. @@ -498,6 +523,11 @@ func (s *scope) SetAll(d pyDict, publicOnly bool) { s.config.Merge(c) } else if !publicOnly || k[0] != '_' { s.locals[k] = v + if origin != nil { + s.objectOrigins[k] = *origin + } else if s.subincludeLabel != nil { + s.objectOrigins[k] = *s.subincludeLabel + } } } } @@ -1112,17 +1142,11 @@ func (s *scope) CurrentBuildStatement() core.BuildStatementProvider { // in the current callstack, actively executing to define this target. func (s *scope) ActiveSubincludes() core.SubincludesLabelProvider { return func() core.BuildLabels { - // We walk back on the callstack. For each scope of a method call we walk back at the local/lexical - // scopes in that method's context to find the original/root scope. If that scope includes a "subincludeLabel" - // it means this scope was generated by a subinclude statement and we'll register that label as required. + // We walk back on the callstack. For each scope of a method call we lookup the + // subinclude labels marked as used, meaning values from those subincluded labels were used to generate this target, be it func defs or variables. seen := map[core.BuildLabel]bool{} for callScope := s; callScope != nil; callScope = callScope.caller { - for lexicalScope := callScope; lexicalScope != nil; lexicalScope = lexicalScope.parent { - if lexicalScope.subincludeLabel != nil { - label := *lexicalScope.subincludeLabel - seen[label] = true - } - } + maps.Copy(seen, callScope.requiredOrigins) } return slices.Collect(maps.Keys(seen)) } diff --git a/test/export/test_subinclude_variable/BUILD b/test/export/test_subinclude_variable/BUILD new file mode 100644 index 0000000000..2fcedfa352 --- /dev/null +++ b/test/export/test_subinclude_variable/BUILD @@ -0,0 +1,6 @@ +subinclude("//test/export:export_e2e_test_build_def") + +please_export_e2e_test( + name = "subinclude_variable", + export_targets = ["//:test"], +) diff --git a/test/export/test_subinclude_variable/expected_repo/.plzconfig b/test/export/test_subinclude_variable/expected_repo/.plzconfig new file mode 100644 index 0000000000..f8ba31854d --- /dev/null +++ b/test/export/test_subinclude_variable/expected_repo/.plzconfig @@ -0,0 +1,2 @@ +[Parse] +BuildFileName = BUILD_FILE diff --git a/test/export/test_subinclude_variable/expected_repo/BUILD_FILE b/test/export/test_subinclude_variable/expected_repo/BUILD_FILE new file mode 100644 index 0000000000..646febc216 --- /dev/null +++ b/test/export/test_subinclude_variable/expected_repo/BUILD_FILE @@ -0,0 +1,13 @@ +filegroup( + name = "defs", + srcs = ["defs.build_defs"], + visibility = ["PUBLIC"], +) + +subinclude(":defs") + +genrule( + name = "test", + outs = ["test.txt"], + cmd = f"echo {VAR} > $OUT", +) diff --git a/test/export/test_subinclude_variable/expected_repo/defs.build_defs b/test/export/test_subinclude_variable/expected_repo/defs.build_defs new file mode 100644 index 0000000000..cdd62f88ae --- /dev/null +++ b/test/export/test_subinclude_variable/expected_repo/defs.build_defs @@ -0,0 +1 @@ +VAR = "hello" diff --git a/test/export/test_subinclude_variable/source_repo/.plzconfig b/test/export/test_subinclude_variable/source_repo/.plzconfig new file mode 100644 index 0000000000..f8ba31854d --- /dev/null +++ b/test/export/test_subinclude_variable/source_repo/.plzconfig @@ -0,0 +1,2 @@ +[Parse] +BuildFileName = BUILD_FILE diff --git a/test/export/test_subinclude_variable/source_repo/BUILD_FILE b/test/export/test_subinclude_variable/source_repo/BUILD_FILE new file mode 100644 index 0000000000..c63a0e733b --- /dev/null +++ b/test/export/test_subinclude_variable/source_repo/BUILD_FILE @@ -0,0 +1,19 @@ +filegroup( + name = "defs", + srcs = ["defs.build_defs"], + visibility = ["PUBLIC"], +) + +subinclude(":defs") + +genrule( + name = "test", + outs = ["test.txt"], + cmd = f"echo {VAR} > $OUT", +) + +genrule( + name = "unused", + outs = ["unused.txt"], + cmd = "touch $OUT", +) diff --git a/test/export/test_subinclude_variable/source_repo/defs.build_defs b/test/export/test_subinclude_variable/source_repo/defs.build_defs new file mode 100644 index 0000000000..cdd62f88ae --- /dev/null +++ b/test/export/test_subinclude_variable/source_repo/defs.build_defs @@ -0,0 +1 @@ +VAR = "hello" From bf9851d6966614fd79fb8e1174f7c6ea76054613 Mon Sep 17 00:00:00 2001 From: DuBento Date: Tue, 2 Jun 2026 17:15:57 +0100 Subject: [PATCH 067/118] scope metadata interface for optional metadata processing --- src/parse/asp/interpreter.go | 154 +++++++++++++++++++++++++---------- 1 file changed, 112 insertions(+), 42 deletions(-) diff --git a/src/parse/asp/interpreter.go b/src/parse/asp/interpreter.go index 21be24d381..85c2a02327 100644 --- a/src/parse/asp/interpreter.go +++ b/src/parse/asp/interpreter.go @@ -42,12 +42,15 @@ type interpreter struct { // It loads all the builtin rules at this point. func newInterpreter(state *core.BuildState, p *Parser) *interpreter { s := &scope{ - ctx: context.Background(), - state: state, - locals: map[string]pyObject{}, - objectOrigins: map[string]core.BuildLabel{}, - requiredOrigins: map[core.BuildLabel]bool{}, + ctx: context.Background(), + state: state, + locals: map[string]pyObject{}, + metadata: &noopScopeMetadata{}, + } + if state.ParseMetadata { + s.metadata = &scopeMetadata{} } + i := &interpreter{ scope: s, parser: p, @@ -319,12 +322,7 @@ type scope struct { // True if this scope is for a pre- or post-build callback. Callback bool mode core.ParseMode - // cursor points to the statement currently being interpreted - cursor *Statement - // objectOrigins tracks the subinclude label that each variable was originally defined in. - objectOrigins map[string]core.BuildLabel - // requiredOrigins tracks which subinclude labels have been requiredOrigins by looking up variables. - requiredOrigins map[core.BuildLabel]bool + metadata ScopeMetadata } // parseAnnotatedLabelInPackage similarly to parseLabelInPackage, parses the label contextualising it to the provided @@ -430,20 +428,18 @@ func (s *scope) NewPackagedScope(pkg *core.Package, mode core.ParseMode, hint in func (s *scope) newScope(pkg *core.Package, mode core.ParseMode, filename string, hint int) *scope { s2 := &scope{ - ctx: s.ctx, - filename: filename, - interpreter: s.interpreter, - state: s.state, - pkg: pkg, - parsingFor: s.parsingFor, - parent: s, - locals: make(pyDict, hint), - config: s.config, - Callback: s.Callback, - mode: mode, - cursor: s.cursor, - objectOrigins: map[string]core.BuildLabel{}, - requiredOrigins: map[core.BuildLabel]bool{}, + ctx: s.ctx, + filename: filename, + interpreter: s.interpreter, + state: s.state, + pkg: pkg, + parsingFor: s.parsingFor, + parent: s, + locals: make(pyDict, hint), + config: s.config, + Callback: s.Callback, + mode: mode, + metadata: s.metadata.NewMetadata(), } if pkg != nil && pkg.Subrepo != nil && pkg.Subrepo.State != nil { s2.state = pkg.Subrepo.State @@ -475,19 +471,14 @@ func (s *scope) NAssert(condition bool, msg string, args ...interface{}) { // It panics if the variable is not defined. func (s *scope) Lookup(name string) pyObject { obj, origin := s.lookupWithOrigin(name) - if origin != nil && s.state.ParseMetadata { - s.requiredOrigins[*origin] = true - } + s.metadata.SetRequiredOrigin(origin) return obj } // lookupWithOrigin is like Lookup but returns the origin label of the variable as well. func (s *scope) lookupWithOrigin(name string) (pyObject, *core.BuildLabel) { if obj, present := s.locals[name]; present { - if label, ok := s.objectOrigins[name]; ok { - return obj, &label - } - return obj, nil + return obj, s.metadata.Origin(name) } else if s.parent != nil { return s.parent.lookupWithOrigin(name) } @@ -513,7 +504,7 @@ func (s *scope) SetAll(d pyDict, publicOnly bool) { s.SetAllWithOrigin(d, publicOnly, nil) } -// SetAllWithOrigin is like SetAll but also records the origin label for all variables. +// SetAllWithOrigin is like SetAll but also records the origin labe l for all variables. func (s *scope) SetAllWithOrigin(d pyDict, publicOnly bool, origin *core.BuildLabel) { for k, v := range d { if k == "CONFIG" { @@ -524,9 +515,7 @@ func (s *scope) SetAllWithOrigin(d pyDict, publicOnly bool, origin *core.BuildLa } else if !publicOnly || k[0] != '_' { s.locals[k] = v if origin != nil { - s.objectOrigins[k] = *origin - } else if s.subincludeLabel != nil { - s.objectOrigins[k] = *s.subincludeLabel + s.metadata.SetObjectOrigin(k, *origin) } } } @@ -565,7 +554,7 @@ func (s *scope) interpretStatements(statements []*Statement) pyObject { } }() for _, stmt = range statements { - s.cursor = stmt + s.metadata.SetCursor(stmt) if stmt.FuncDef != nil { s.Set(stmt.FuncDef.Name, newPyFunc(s, stmt.FuncDef)) } else if stmt.If != nil { @@ -1132,8 +1121,8 @@ func (s *scope) CurrentBuildStatement() core.BuildStatementProvider { stmtScope = curr } } - s.NAssert(stmtScope.cursor == nil, "Cursor is not pointing to a statement") - return NewBuildStatement(stmtScope.cursor) + s.NAssert(stmtScope.metadata.Cursor() == nil, "Cursor is not pointing to a statement") + return NewBuildStatement(stmtScope.metadata.Cursor()) } } @@ -1143,10 +1132,10 @@ func (s *scope) CurrentBuildStatement() core.BuildStatementProvider { func (s *scope) ActiveSubincludes() core.SubincludesLabelProvider { return func() core.BuildLabels { // We walk back on the callstack. For each scope of a method call we lookup the - // subinclude labels marked as used, meaning values from those subincluded labels were used to generate this target, be it func defs or variables. - seen := map[core.BuildLabel]bool{} + // subinclude labels marked as used, meaning values from those subincluded labels were used to generate this target, be it function defs or variables. + seen := map[core.BuildLabel]struct{}{} for callScope := s; callScope != nil; callScope = callScope.caller { - maps.Copy(seen, callScope.requiredOrigins) + maps.Copy(seen, callScope.metadata.RequiredOrigins()) } return slices.Collect(maps.Keys(seen)) } @@ -1160,6 +1149,87 @@ func (s *scope) pkgFilename() string { return "" } +// scopeMetadata maintains additional information generated during the interpretation phase. +// This is optionally used for operations (e.g. export) that require more details on the relation +// between targets and statements. The no-op implementation should be used for most operations to +// avoid any computational overhead. +type ScopeMetadata interface { + // NewMetadata creates a new instance of the same ScopeMetadata implementation type. + NewMetadata() ScopeMetadata + // Cursor returns the statement being currently interpreted. + Cursor() *Statement + // Origin gets the origin of the object by name. Should return nil if not found or unimplemented. + Origin(name string) *core.BuildLabel + // RequiredOrigins returns a set of all the origins (subincluded labels) required by the current + // scope. + RequiredOrigins() map[core.BuildLabel]struct{} + // SetCursor register the statement currently being interpreted. + SetCursor(stmt *Statement) + // SetObjectOrigin registers the origin for the given named object. The origin is the label of a + // subincluded target. + SetObjectOrigin(name string, origin core.BuildLabel) + // SetRequiredOrigin marks the named object as required by the current scope. + SetRequiredOrigin(origin *core.BuildLabel) +} + +type scopeMetadata struct { + // cursor points to the statement currently being interpreted + cursor *Statement + // objectOrigins tracks the subinclude label that each variable was originally defined in. + objectOrigins map[string]core.BuildLabel + // requiredOrigins tracks which subinclude labels have been used by looking up objects. + requiredOrigins map[core.BuildLabel]struct{} +} + +func (m *scopeMetadata) NewMetadata() ScopeMetadata { + return &scopeMetadata{ + objectOrigins: map[string]core.BuildLabel{}, + requiredOrigins: map[core.BuildLabel]struct{}{}, + } +} + +func (m *scopeMetadata) Cursor() *Statement { + return m.cursor +} + +func (m *scopeMetadata) Origin(name string) *core.BuildLabel { + if label, ok := m.objectOrigins[name]; ok { + return &label + } + return nil +} + +func (m *scopeMetadata) RequiredOrigins() map[core.BuildLabel]struct{} { + return m.requiredOrigins +} + +func (m *scopeMetadata) SetCursor(stmt *Statement) { + m.cursor = stmt +} + +func (m *scopeMetadata) SetObjectOrigin(name string, origin core.BuildLabel) { + m.objectOrigins[name] = origin +} + +func (m *scopeMetadata) SetRequiredOrigin(origin *core.BuildLabel) { + if origin == nil { + return + } + m.requiredOrigins[*origin] = struct{}{} +} + +// noopScopeMetadata implements the ScopeMetadata interface with no-op methods. This is used to +// avoid the overhead of storing metadata for operations that don't depend on it. +type noopScopeMetadata struct{} + +func (nm *noopScopeMetadata) NewMetadata() ScopeMetadata { return &noopScopeMetadata{} } +func (nm *noopScopeMetadata) Cursor() *Statement { return nil } +func (nm *noopScopeMetadata) Origin(name string) *core.BuildLabel { return nil } +func (nm *noopScopeMetadata) RequiredOrigins() map[core.BuildLabel]struct{} { return nil } +func (nm *noopScopeMetadata) SetCursor(stmt *Statement) {} +func (nm *noopScopeMetadata) SetObjectOrigin(name string, origin core.BuildLabel) {} +func (nm *noopScopeMetadata) SetRequiredOrigin(origin *core.BuildLabel) {} + // NewBuildStatement creates a new core.BuildStatement from an asp.statement. func NewBuildStatement(stmt *Statement) core.BuildStatement { return core.BuildStatement{ From 34f75f8f0bb4fe385bce84e908f0d07a5a41cb2b Mon Sep 17 00:00:00 2001 From: DuBento Date: Wed, 3 Jun 2026 14:25:47 +0100 Subject: [PATCH 068/118] test: adjust interpreter tests with scope metadata and origin tracking changes --- src/parse/asp/interpreter_test.go | 80 +++++++++++++++++++++++-------- 1 file changed, 61 insertions(+), 19 deletions(-) diff --git a/src/parse/asp/interpreter_test.go b/src/parse/asp/interpreter_test.go index a58aee9154..2f6267ef8f 100644 --- a/src/parse/asp/interpreter_test.go +++ b/src/parse/asp/interpreter_test.go @@ -780,39 +780,43 @@ func TestCurrentBuildStatement(t *testing.T) { rootScope := &scope{ pkg: pkg, filename: pkg.Filename, - cursor: rootStmt, + metadata: newScopeMetadata(), } + rootScope.metadata.SetCursor(rootStmt) // A nested call inside the same BUILD file (e.g. function def) - NestedStmt := &Statement{Pos: 30, EndPos: 40} - NestedScope := &scope{ + nestedStmt := &Statement{Pos: 30, EndPos: 40} + nestedScope := &scope{ pkg: pkg, filename: pkg.Filename, - cursor: NestedStmt, caller: rootScope, + metadata: newScopeMetadata(), } + nestedScope.metadata.SetCursor(nestedStmt) // A call from a different file (e.g. a function inside a subincluded .build_defs file) defsRootStmt := &Statement{Pos: 50, EndPos: 60} defsRootScope := &scope{ pkg: pkg, filename: "other/file.build_defs", - cursor: defsRootStmt, - caller: NestedScope, + caller: nestedScope, + metadata: newScopeMetadata(), } + defsRootScope.metadata.SetCursor(defsRootStmt) // Another call deep in the other file defsNestedStmt := &Statement{Pos: 70, EndPos: 80} defsNestedScope := &scope{ pkg: pkg, filename: "other/file.build_defs", - cursor: defsNestedStmt, caller: defsRootScope, + metadata: newScopeMetadata(), } + defsNestedScope.metadata.SetCursor(defsNestedStmt) t.Run("FindsRootStatementFromBUILD", func(t *testing.T) { // Calling it from buildNestedScope should walk back to buildRootScope - stmt := NestedScope.CurrentBuildStatement()() + stmt := nestedScope.CurrentBuildStatement()() assert.Equal(t, NewBuildStatement(rootStmt), stmt) }) @@ -824,7 +828,8 @@ func TestCurrentBuildStatement(t *testing.T) { t.Run("HandlesNoPackageFileInStack", func(t *testing.T) { // A scope that has no pkg/filename context - standaloneScope := &scope{cursor: rootStmt} + standaloneScope := &scope{metadata: newScopeMetadata()} + standaloneScope.metadata.SetCursor(rootStmt) stmt := standaloneScope.CurrentBuildStatement()() assert.Equal(t, NewBuildStatement(rootStmt), stmt) }) @@ -836,10 +841,13 @@ func TestActiveSubincludes(t *testing.T) { t.Run("NoSubincludes", func(t *testing.T) { // BUILD scope - scopeBUILD := &scope{} + scopeBUILD := &scope{ + metadata: newScopeMetadata(), + } // Function execution scopeFuncExec := &scope{ - caller: scopeBUILD, + caller: scopeBUILD, + metadata: newScopeMetadata(), } labels := scopeFuncExec.ActiveSubincludes()() assert.Empty(t, labels) @@ -849,19 +857,30 @@ func TestActiveSubincludes(t *testing.T) { // File A scope scopeA := &scope{ subincludeLabel: &labelA, + locals: make(pyDict), + metadata: newScopeMetadata(), } + scopeA.SetAllWithOrigin(pyDict{"foo": pyString("val")}, false, &labelA) + // Function defined in File A scopeFuncDef := &scope{ - parent: scopeA, + parent: scopeA, + metadata: newScopeMetadata(), } // BUILD scope - scopeBUILD := &scope{} + scopeBUILD := &scope{ + metadata: newScopeMetadata(), + } // Function execution scopeFuncExec := &scope{ - parent: scopeFuncDef, - caller: scopeBUILD, + parent: scopeFuncDef, + caller: scopeBUILD, + metadata: newScopeMetadata(), } + // Lookup triggers tracking of required subincludes + scopeFuncExec.Lookup("foo") + labels := scopeFuncExec.ActiveSubincludes()() assert.Equal(t, core.BuildLabels{labelA}, labels) }) @@ -870,25 +889,48 @@ func TestActiveSubincludes(t *testing.T) { // File A scope scopeA := &scope{ subincludeLabel: &labelA, + locals: make(pyDict), + metadata: newScopeMetadata(), } + scopeA.SetAllWithOrigin(pyDict{"varA": pyString("valA")}, false, &labelA) + // File B scope (subincluded by A) scopeB := &scope{ subincludeLabel: &labelB, parent: scopeA, + locals: make(pyDict), + metadata: newScopeMetadata(), } + scopeB.SetAllWithOrigin(pyDict{"varB": pyString("valB")}, false, &labelB) + // Function defined in File B scopeFuncDef := &scope{ - parent: scopeB, + parent: scopeB, + metadata: newScopeMetadata(), } // BUILD scope - scopeBUILD := &scope{} + scopeBUILD := &scope{ + metadata: newScopeMetadata(), + } // Function execution scopeFuncExec := &scope{ - parent: scopeFuncDef, - caller: scopeBUILD, + parent: scopeFuncDef, + caller: scopeBUILD, + metadata: newScopeMetadata(), } + // Lookups trigger tracking of required subincludes + scopeFuncExec.Lookup("varA") + scopeFuncExec.Lookup("varB") + labels := scopeFuncExec.ActiveSubincludes()() assert.ElementsMatch(t, core.BuildLabels{labelA, labelB}, labels) }) } + +func newScopeMetadata() ScopeMetadata { + return &scopeMetadata{ + objectOrigins: map[string]core.BuildLabel{}, + requiredOrigins: map[core.BuildLabel]struct{}{}, + } +} From f0d26ca68d810c8639832c582201559b616b46e4 Mon Sep 17 00:00:00 2001 From: DuBento Date: Wed, 3 Jun 2026 14:36:43 +0100 Subject: [PATCH 069/118] golint trimmer --- src/export/trimmer.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/export/trimmer.go b/src/export/trimmer.go index 540e866ca0..cd249d87d0 100644 --- a/src/export/trimmer.go +++ b/src/export/trimmer.go @@ -5,6 +5,7 @@ import ( "sort" "github.com/please-build/buildtools/build" + "github.com/thought-machine/please/src/core" "github.com/thought-machine/please/src/parse/asp" ) @@ -31,7 +32,6 @@ func (t *trimmer) walkFile(stmts []*asp.Statement, start, end asp.Position, cons // cursor tracks the position in a block that's being interpreted. cursor := start for _, stmt := range stmts { - log.Debugf("Evaluating statement %s", t.origin[stmt.Pos:stmt.EndPos]) // Write content that's between stmts (e.g. comments). We skip these while parsing so it won't // be included in "parsedStmts" but we want the resulting BUILD file to include these. @@ -158,7 +158,6 @@ func (t *trimmer) passBlock(stmts []*asp.Statement, blockStart, blockEnd asp.Pos t.write([]byte("pass #Trimmed during export")) } }) - } func (t *trimmer) isRequiredStatements(stmts []*asp.Statement) bool { From f9b4b8019941e97055516d2b149e406a18d4aa6e Mon Sep 17 00:00:00 2001 From: DuBento Date: Thu, 4 Jun 2026 19:59:45 +0100 Subject: [PATCH 070/118] fix missing targets by keeping parser running and parsing inline during export added a test case --- src/core/state.go | 34 ++++++++++++++----- src/export/export.go | 18 +++++++--- src/please.go | 15 +++++--- src/plz/plz.go | 23 +++++++++++++ test/export/test_nested_subinclude/BUILD | 6 ++++ .../expected_repo/.plzconfig | 2 ++ .../expected_repo/BUILD_FILE | 5 +++ .../expected_repo/build_defs/BUILD_FILE | 11 ++++++ .../build_defs/primary.build_defs | 10 ++++++ .../build_defs/secondary.build_defs | 7 ++++ .../expected_repo/tools/BUILD_FILE | 5 +++ .../expected_repo/tools/tools.sh | 2 ++ .../source_repo/.plzconfig | 2 ++ .../source_repo/BUILD_FILE | 13 +++++++ .../source_repo/build_defs/BUILD_FILE | 17 ++++++++++ .../source_repo/build_defs/primary.build_defs | 10 ++++++ .../build_defs/secondary.build_defs | 7 ++++ .../source_repo/build_defs/unused.build_defs | 2 ++ .../source_repo/file.txt | 1 + .../source_repo/tools/BUILD_FILE | 5 +++ .../source_repo/tools/tools.sh | 2 ++ 21 files changed, 179 insertions(+), 18 deletions(-) create mode 100644 test/export/test_nested_subinclude/BUILD create mode 100644 test/export/test_nested_subinclude/expected_repo/.plzconfig create mode 100644 test/export/test_nested_subinclude/expected_repo/BUILD_FILE create mode 100644 test/export/test_nested_subinclude/expected_repo/build_defs/BUILD_FILE create mode 100644 test/export/test_nested_subinclude/expected_repo/build_defs/primary.build_defs create mode 100644 test/export/test_nested_subinclude/expected_repo/build_defs/secondary.build_defs create mode 100644 test/export/test_nested_subinclude/expected_repo/tools/BUILD_FILE create mode 100644 test/export/test_nested_subinclude/expected_repo/tools/tools.sh create mode 100644 test/export/test_nested_subinclude/source_repo/.plzconfig create mode 100644 test/export/test_nested_subinclude/source_repo/BUILD_FILE create mode 100644 test/export/test_nested_subinclude/source_repo/build_defs/BUILD_FILE create mode 100644 test/export/test_nested_subinclude/source_repo/build_defs/primary.build_defs create mode 100644 test/export/test_nested_subinclude/source_repo/build_defs/secondary.build_defs create mode 100644 test/export/test_nested_subinclude/source_repo/build_defs/unused.build_defs create mode 100644 test/export/test_nested_subinclude/source_repo/file.txt create mode 100644 test/export/test_nested_subinclude/source_repo/tools/BUILD_FILE create mode 100644 test/export/test_nested_subinclude/source_repo/tools/tools.sh diff --git a/src/core/state.go b/src/core/state.go index 013d5a59fe..04deb2ddcf 100644 --- a/src/core/state.go +++ b/src/core/state.go @@ -242,6 +242,9 @@ type BuildState struct { NeedDebugDeps bool // ParseMetadata is true if we want to store build file metadata ParseMetadata bool + // KeepParserRunning prevents closing task worker (parse and build) channels to support later + // calls to parser. + KeepParserRunning bool // initOnce is used to control loading the subrepo .plzconfig initOnce *sync.Once @@ -283,13 +286,14 @@ func (state *BuildState) Initialise(subrepo *Subrepo) (err error) { // This is split out from above so we can share it between multiple instances. type stateProgress struct { // Used to count the number of currently active/pending targets - numActive int64 - numPending int64 - numDone int64 - numParses atomic.Int64 - mutex sync.Mutex - closeOnce sync.Once - resultOnce sync.Once + numActive int64 + numPending int64 + numDone int64 + numParses atomic.Int64 + mutex sync.Mutex + closeOnce sync.Once + resultOnce sync.Once + buildDoneOnce sync.Once // Used to track subinclude() calls that block until targets are built. Keyed by their label. pendingTargets *cmap.Map[BuildLabel, chan struct{}] // Used to track general package parsing requests. Keyed by a packageKey struct. @@ -312,6 +316,8 @@ type stateProgress struct { internalResults chan *BuildResult // The cycle checker itself. cycleDetector cycleDetector + // buildDone is closed when numPending drops to 0 or less + buildDone chan struct{} } // SystemStats stores information about the system. @@ -411,7 +417,13 @@ func (state *BuildState) taskDone(wasSynthetic bool) { atomic.AddInt64(&state.progress.numDone, 1) } if atomic.AddInt64(&state.progress.numPending, -1) <= 0 { - state.Stop() + state.progress.buildDoneOnce.Do(func() { + close(state.progress.buildDone) + }) + + if !state.KeepParserRunning { + state.Stop() + } } } @@ -922,6 +934,11 @@ func (state *BuildState) WaitForBuiltTarget(l, dependent BuildLabel, mode ParseM return state.WaitForBuiltTarget(l, dependent, mode) } +// WaitForBuildToComplete blocks until all pending tasks are finished. +func (state *BuildState) WaitForBuildToComplete() { + <-state.progress.buildDone +} + // AddTarget adds a new target to the build graph. func (state *BuildState) AddTarget(pkg *Package, target *BuildTarget) { pkg.AddTarget(target) @@ -1488,6 +1505,7 @@ func NewBuildState(config *Configuration) *BuildState { internalResults: make(chan *BuildResult, 1000), cycleDetector: cycleDetector{graph: graph}, originalTargets: NewTargetSet(), + buildDone: make(chan struct{}), }, initOnce: new(sync.Once), preloadDownloadOnce: new(sync.Once), diff --git a/src/export/export.go b/src/export/export.go index 191749fb37..b7340d6795 100644 --- a/src/export/export.go +++ b/src/export/export.go @@ -137,7 +137,7 @@ func (be *baseExporter) ExportPlzConfig() { func (be *baseExporter) ExportTargets(labels core.BuildLabels) { for _, l := range labels { - target := be.state.Graph.Target(l) + target := be.getOrParseTarget(l) if target == nil { log.Errorf("Unable to lookup target %s", l) continue @@ -146,6 +146,16 @@ func (be *baseExporter) ExportTargets(labels core.BuildLabels) { } } +func (be *baseExporter) getOrParseTarget(label core.BuildLabel) *core.BuildTarget { + target := be.state.Graph.Target(label) + if target == nil { + log.Debugf("Target %v not found in graph. Attempting to parse...", label) + be.state.WaitForBuiltTarget(label, core.OriginalTarget, core.ParseModeNormal) + target = be.state.Graph.Target(label) + } + return target +} + // exportDependencies exports exportDependencies of a target. func (be *baseExporter) exportDependencies(target *core.BuildTarget) { deps := target.DeclaredDependencies() @@ -271,7 +281,7 @@ func (e *defaultExporter) exportSubincludes(pkg *core.Package, target *core.Buil } e.requiredSubincludes[pkg.Label()] = required - target := e.state.Graph.Target(subinclude) + target := e.getOrParseTarget(subinclude) if target == nil { log.Errorf("Unable to lookup target %s", subinclude) continue @@ -403,9 +413,7 @@ func (nte *noTrimExporter) exportPackage(pkg *core.Package) { // exportSubincludes exports the subincluded targets. func (nte *noTrimExporter) exportSubincludes(pkg *core.Package) { subincludes := pkg.AllSubincludes(nte.state.Graph) - for _, subinclude := range subincludes { - nte.ExportTarget(nte.state.Graph.TargetOrDie(subinclude)) - } + nte.ExportTargets(subincludes) } // exportAllTargets will export all the targets in the provided package. diff --git a/src/please.go b/src/please.go index a096e2f24b..9f1829f271 100644 --- a/src/please.go +++ b/src/please.go @@ -767,14 +767,18 @@ var buildFunctions = map[string]func() int{ return 0 }, "export": func() int { - success, state := runBuild(opts.Export.Args.Targets, buildOpts{ParseMetadata: true}) + success, state := runBuild(opts.Export.Args.Targets, buildOpts{ParseMetadata: true, KeepParserRunning: true}) + // Required cleanup due to running parser in background + defer plz.CleanUp(state) + if success { export.Repo(state, opts.Export.Output, opts.Export.NoTrim, state.ExpandOriginalLabels()) } + return toExitCode(success, state) }, "export.outputs": func() int { - success, state := runBuild(opts.Export.Outputs.Args.Targets, buildOpts{Build: true, IsQuery: true, ParseMetadata: true}) + success, state := runBuild(opts.Export.Outputs.Args.Targets, buildOpts{Build: true, IsQuery: true}) if success { export.Outputs(state, opts.Export.Output, state.ExpandOriginalLabels()) } @@ -1166,6 +1170,7 @@ func Please(targets []core.BuildLabel, config *core.Configuration, buildOpts bui state.NeedBuild = buildOpts.Build state.NeedTests = buildOpts.Test state.ParseMetadata = buildOpts.ParseMetadata + state.KeepParserRunning = buildOpts.KeepParserRunning state.NeedDebugDeps = debug // What outputs get downloaded in remote execution. @@ -1190,9 +1195,6 @@ func Please(targets []core.BuildLabel, config *core.Configuration, buildOpts bui } runPlease(state, targets) - if state.RemoteClient != nil && !opts.Run.Remote { - defer state.RemoteClient.Disconnect() - } failures, _, _ := state.Failures() return !failures, state } @@ -1324,6 +1326,9 @@ type buildOpts struct { Test bool IsQuery bool ParseMetadata bool + // Keep the workers running in the background for inline parsing during specific ops (e.g. export). + // Note: when running background workers we need to explicit call plz.Cleanup() at the end of the CLI op. + KeepParserRunning bool } // Runs the actual build diff --git a/src/plz/plz.go b/src/plz/plz.go index 7c893a8165..e2ee4a7245 100644 --- a/src/plz/plz.go +++ b/src/plz/plz.go @@ -111,10 +111,33 @@ func Run(targets, preTargets []core.BuildLabel, state *core.BuildState, config * wg.Done() }() // Wait until they've all exited, which they'll do once they have no tasks left. + if state.KeepParserRunning { + // Even though we keep the workers running, we wait for the initial build to finish before + // proceeding with the specific op for the graph to be complete. + state.WaitForBuildToComplete() + reportResults(state, config) + // Cleanup() needs to be called at the end of the CLI run when KeepParserRunning is enabled. + return + } + // Wait for all worker to finish. This should happen as soon as we no longer have pending tasks. + // The last task should call state.Close() and close the queue channels. wg.Wait() + reportResults(state, config) + CleanUp(state) +} + +// CleanUp cleans up and shuts down the build state. +func CleanUp(state *core.BuildState) { if state.Cache != nil { state.Cache.Shutdown() } + if state.RemoteClient != nil { + state.RemoteClient.Disconnect() + } +} + +// reportResults reports on metrics and results at the end of the build. +func reportResults(state *core.BuildState, config *core.Configuration) { if state.RemoteClient != nil { _, _, in, out := state.RemoteClient.DataRate() log.Info("Total remote RPC data in: %d out: %d", in, out) diff --git a/test/export/test_nested_subinclude/BUILD b/test/export/test_nested_subinclude/BUILD new file mode 100644 index 0000000000..acce896754 --- /dev/null +++ b/test/export/test_nested_subinclude/BUILD @@ -0,0 +1,6 @@ +subinclude("//test/export:export_e2e_test_build_def") + +please_export_e2e_test( + name = "nested_subinclude", + export_targets = ["//:target"], +) diff --git a/test/export/test_nested_subinclude/expected_repo/.plzconfig b/test/export/test_nested_subinclude/expected_repo/.plzconfig new file mode 100644 index 0000000000..f8ba31854d --- /dev/null +++ b/test/export/test_nested_subinclude/expected_repo/.plzconfig @@ -0,0 +1,2 @@ +[Parse] +BuildFileName = BUILD_FILE diff --git a/test/export/test_nested_subinclude/expected_repo/BUILD_FILE b/test/export/test_nested_subinclude/expected_repo/BUILD_FILE new file mode 100644 index 0000000000..0ef1d0178a --- /dev/null +++ b/test/export/test_nested_subinclude/expected_repo/BUILD_FILE @@ -0,0 +1,5 @@ +subinclude("//build_defs:primary_build_def") + +helper_rule_primary( + name = "target", +) diff --git a/test/export/test_nested_subinclude/expected_repo/build_defs/BUILD_FILE b/test/export/test_nested_subinclude/expected_repo/build_defs/BUILD_FILE new file mode 100644 index 0000000000..7e567b845a --- /dev/null +++ b/test/export/test_nested_subinclude/expected_repo/build_defs/BUILD_FILE @@ -0,0 +1,11 @@ +filegroup( + name = "primary_build_def", + srcs = ["primary.build_defs"], + visibility = ["PUBLIC"], +) + +filegroup( + name = "secondary_build_def", + srcs = ["secondary.build_defs"], + visibility = ["PUBLIC"], +) diff --git a/test/export/test_nested_subinclude/expected_repo/build_defs/primary.build_defs b/test/export/test_nested_subinclude/expected_repo/build_defs/primary.build_defs new file mode 100644 index 0000000000..333d80bced --- /dev/null +++ b/test/export/test_nested_subinclude/expected_repo/build_defs/primary.build_defs @@ -0,0 +1,10 @@ +subinclude("//build_defs:secondary_build_def") + +def helper_rule_primary(name: str): + # Generates the requested target (which has no unparsed dependencies) + filegroup( + name = name, + srcs = [], + ) + # Generates an adjacent target (which calls helper_rule and has an unparsed dependency) + helper_rule(name + "_adjacent") diff --git a/test/export/test_nested_subinclude/expected_repo/build_defs/secondary.build_defs b/test/export/test_nested_subinclude/expected_repo/build_defs/secondary.build_defs new file mode 100644 index 0000000000..10e3974137 --- /dev/null +++ b/test/export/test_nested_subinclude/expected_repo/build_defs/secondary.build_defs @@ -0,0 +1,7 @@ +def helper_rule(name: str): + build_rule( + name = name, + cmd = "sh $TOOL > $OUT", + tools = ["//tools:tools"], + outs = [name + ".out"], + ) diff --git a/test/export/test_nested_subinclude/expected_repo/tools/BUILD_FILE b/test/export/test_nested_subinclude/expected_repo/tools/BUILD_FILE new file mode 100644 index 0000000000..215281ac57 --- /dev/null +++ b/test/export/test_nested_subinclude/expected_repo/tools/BUILD_FILE @@ -0,0 +1,5 @@ +filegroup( + name = "tools", + srcs = ["tools.sh"], + visibility = ["PUBLIC"], +) diff --git a/test/export/test_nested_subinclude/expected_repo/tools/tools.sh b/test/export/test_nested_subinclude/expected_repo/tools/tools.sh new file mode 100644 index 0000000000..b77678ef4f --- /dev/null +++ b/test/export/test_nested_subinclude/expected_repo/tools/tools.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo "nested dependency success" diff --git a/test/export/test_nested_subinclude/source_repo/.plzconfig b/test/export/test_nested_subinclude/source_repo/.plzconfig new file mode 100644 index 0000000000..f8ba31854d --- /dev/null +++ b/test/export/test_nested_subinclude/source_repo/.plzconfig @@ -0,0 +1,2 @@ +[Parse] +BuildFileName = BUILD_FILE diff --git a/test/export/test_nested_subinclude/source_repo/BUILD_FILE b/test/export/test_nested_subinclude/source_repo/BUILD_FILE new file mode 100644 index 0000000000..f96a5a0d1a --- /dev/null +++ b/test/export/test_nested_subinclude/source_repo/BUILD_FILE @@ -0,0 +1,13 @@ +subinclude( + "//build_defs:primary_build_def", + "//build_defs:unused_build_def", +) + +helper_rule_primary( + name = "target", +) + +filegroup( + name = "unused_target", + srcs = ["file.txt"], +) diff --git a/test/export/test_nested_subinclude/source_repo/build_defs/BUILD_FILE b/test/export/test_nested_subinclude/source_repo/build_defs/BUILD_FILE new file mode 100644 index 0000000000..b65694fcb9 --- /dev/null +++ b/test/export/test_nested_subinclude/source_repo/build_defs/BUILD_FILE @@ -0,0 +1,17 @@ +filegroup( + name = "primary_build_def", + srcs = ["primary.build_defs"], + visibility = ["PUBLIC"], +) + +filegroup( + name = "secondary_build_def", + srcs = ["secondary.build_defs"], + visibility = ["PUBLIC"], +) + +filegroup( + name = "unused_build_def", + srcs = ["unused.build_defs"], + visibility = ["PUBLIC"], +) diff --git a/test/export/test_nested_subinclude/source_repo/build_defs/primary.build_defs b/test/export/test_nested_subinclude/source_repo/build_defs/primary.build_defs new file mode 100644 index 0000000000..333d80bced --- /dev/null +++ b/test/export/test_nested_subinclude/source_repo/build_defs/primary.build_defs @@ -0,0 +1,10 @@ +subinclude("//build_defs:secondary_build_def") + +def helper_rule_primary(name: str): + # Generates the requested target (which has no unparsed dependencies) + filegroup( + name = name, + srcs = [], + ) + # Generates an adjacent target (which calls helper_rule and has an unparsed dependency) + helper_rule(name + "_adjacent") diff --git a/test/export/test_nested_subinclude/source_repo/build_defs/secondary.build_defs b/test/export/test_nested_subinclude/source_repo/build_defs/secondary.build_defs new file mode 100644 index 0000000000..10e3974137 --- /dev/null +++ b/test/export/test_nested_subinclude/source_repo/build_defs/secondary.build_defs @@ -0,0 +1,7 @@ +def helper_rule(name: str): + build_rule( + name = name, + cmd = "sh $TOOL > $OUT", + tools = ["//tools:tools"], + outs = [name + ".out"], + ) diff --git a/test/export/test_nested_subinclude/source_repo/build_defs/unused.build_defs b/test/export/test_nested_subinclude/source_repo/build_defs/unused.build_defs new file mode 100644 index 0000000000..072217364b --- /dev/null +++ b/test/export/test_nested_subinclude/source_repo/build_defs/unused.build_defs @@ -0,0 +1,2 @@ +def unused_func(): + pass diff --git a/test/export/test_nested_subinclude/source_repo/file.txt b/test/export/test_nested_subinclude/source_repo/file.txt new file mode 100644 index 0000000000..d95f3ad14d --- /dev/null +++ b/test/export/test_nested_subinclude/source_repo/file.txt @@ -0,0 +1 @@ +content diff --git a/test/export/test_nested_subinclude/source_repo/tools/BUILD_FILE b/test/export/test_nested_subinclude/source_repo/tools/BUILD_FILE new file mode 100644 index 0000000000..215281ac57 --- /dev/null +++ b/test/export/test_nested_subinclude/source_repo/tools/BUILD_FILE @@ -0,0 +1,5 @@ +filegroup( + name = "tools", + srcs = ["tools.sh"], + visibility = ["PUBLIC"], +) diff --git a/test/export/test_nested_subinclude/source_repo/tools/tools.sh b/test/export/test_nested_subinclude/source_repo/tools/tools.sh new file mode 100644 index 0000000000..b77678ef4f --- /dev/null +++ b/test/export/test_nested_subinclude/source_repo/tools/tools.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo "nested dependency success" From 06eccf0ac9e002252720d5bc34dadb44338d3582 Mon Sep 17 00:00:00 2001 From: DuBento Date: Fri, 5 Jun 2026 12:53:40 +0100 Subject: [PATCH 071/118] split export implementations into different files --- src/export/BUILD | 10 +- src/export/export.go | 241 +---------------------------------- src/export/export_notrim.go | 87 +++++++++++++ src/export/export_trimmed.go | 169 ++++++++++++++++++++++++ 4 files changed, 262 insertions(+), 245 deletions(-) create mode 100644 src/export/export_notrim.go create mode 100644 src/export/export_trimmed.go diff --git a/src/export/BUILD b/src/export/BUILD index 658245afcc..ae3c9f13c6 100644 --- a/src/export/BUILD +++ b/src/export/BUILD @@ -1,9 +1,9 @@ go_library( name = "export", - srcs = [ - "export.go", - "trimmer.go", - ], + srcs = glob( + ["*.go"], + exclude = ["*_test.go"], + ), pgo_file = "//:pgo", visibility = ["PUBLIC"], deps = [ @@ -18,7 +18,7 @@ go_library( go_test( name = "export_test", - srcs = ["export_test.go"], + srcs = glob("*_test.go"), data = ["test_data"], deps = [ ":export", diff --git a/src/export/export.go b/src/export/export.go index b7340d6795..e538b014ef 100644 --- a/src/export/export.go +++ b/src/export/export.go @@ -4,18 +4,12 @@ package export import ( - "fmt" "os" "path/filepath" - "slices" - - "github.com/please-build/buildtools/build" "github.com/thought-machine/please/src/cli/logging" "github.com/thought-machine/please/src/core" "github.com/thought-machine/please/src/fs" - "github.com/thought-machine/please/src/parse" - "github.com/thought-machine/please/src/parse/asp" ) var log = logging.Log @@ -37,7 +31,7 @@ type Exporter interface { // ExportTarget exports an individual build target. // Each target recursively exports all their source files and required build statements, but also // targets in their transitive dependencies. - ExportTarget(target *core.BuildTarget) + ExportTarget(*core.BuildTarget) // WritePackageFiles writes the processed BUILD files for all exported targets to the // export directory. These BUILD files may be modified (e.g., trimmed) depending on // the exporter's implementation. @@ -189,236 +183,3 @@ func (be *baseExporter) checkAndSetVisited(target *core.BuildTarget) bool { be.exportedTargets[target.Label] = true return !visited } - -// defaultExporter implements an exporter that trims packages to reach a minimal exported repo. -type defaultExporter struct { - baseExporter - // visitedPackages maintains a record of the packages visited during the export process. - visitedPackages map[core.BuildLabel]bool - // requiredSubincludes maps packages to the subinclude labels they require. - requiredSubincludes map[core.BuildLabel]core.BuildLabels - // preloadedSubincludes tracks subincludes that are preloaded and don't need explicit export. - preloadedSubincludes map[core.BuildLabel]bool -} - -func (e *defaultExporter) ExportPreloaded() { - // Write any preloaded build defs - for _, preload := range e.state.Config.Parse.PreloadBuildDefs { - if err := fs.RecursiveCopy(preload, filepath.Join(e.targetDir, preload), 0); err != nil { - log.Fatalf("Failed to copy preloaded build def %s: %s", preload, err) - } - } - - for _, target := range e.state.Config.Parse.PreloadSubincludes { - targets := append(e.state.Graph.TransitiveSubincludes(target), target) - for _, t := range targets { - e.preloadedSubincludes[t] = true - } - e.ExportTargets(targets) - } -} - -func (e *defaultExporter) ExportTarget(target *core.BuildTarget) { - if !e.checkAndSetVisited(target) { - return - } - - log.Debugf("Exporting target: %v", target.Label) - - // Skip export for internal packages - if target.Label.PackageName == parse.InternalPackageName { - return - } - // We want to export the package that made this subrepo available, but we still need to walk the - // target deps as it may depend on other subrepos or first party targets - if target.Subrepo != nil && target.Subrepo.Target != nil { - e.ExportTarget(target.Subrepo.Target) - e.exportDependencies(target) - return - } - - e.exportSources(target) - e.exportDependencies(target) - - pkg := e.state.Graph.PackageOrDie(target.Label) - e.exportSubincludes(pkg, target) - e.exportRelatedTargets(pkg, target) - e.visitedPackages[pkg.Label()] = true -} - -func (e *defaultExporter) WritePackageFiles() { - for pkgLabel := range e.visitedPackages { - pkg := e.state.Graph.PackageOrDie(pkgLabel) - filteredBytes, err := e.trimPackage(pkg) - if err != nil { - log.Errorf("Failed to filter the build statements of package %s: %v", pkg.Label(), err) - continue - } - - parsedBuild, err := build.ParseBuild(pkg.Filename, filteredBytes) - if err != nil { - log.Fatalf("Failed to parse bytes for formatting: %v\nData:\n%s", err, filteredBytes) - } - formattedBytes := build.Format(parsedBuild) - - e.WriteExportedPackageFile(pkg, formattedBytes) - } -} - -// exportSubincludes exports the subincluded targets required to generate the target and selects them to -// later be written to the package as statements. -func (e *defaultExporter) exportSubincludes(pkg *core.Package, target *core.BuildTarget) { - for _, subinclude := range pkg.Metadata.FindRequiredSubincludes(target) { - // skip for preloaded subincludes, these are handled separately at the start to ensure they are - // they are exported even if not directly used by an exported target. - if e.preloadedSubincludes[subinclude] { - continue - } - - required := e.requiredSubincludes[pkg.Label()] - if !slices.Contains(required, subinclude) { - required = append(required, subinclude) - } - e.requiredSubincludes[pkg.Label()] = required - - target := e.getOrParseTarget(subinclude) - if target == nil { - log.Errorf("Unable to lookup target %s", subinclude) - continue - } - e.ExportTarget(target) - } -} - -// exportRelatedTargets exports build targets that are related to the build statement that generated. -func (e *defaultExporter) exportRelatedTargets(pkg *core.Package, target *core.BuildTarget) { - stmt := pkg.Metadata.FindStatement(target) - if stmt == nil { - log.Errorf("Failed to find statement for target %s in %s", target, pkg.Name) - return - } - - relatedTargets := pkg.Metadata.FindTargets(stmt) - log.Debugf("Exporting targets related to %s: %v", target, relatedTargets) - for _, target := range relatedTargets { - e.ExportTarget(target) - } -} - -// WriteExportedPackageFile creates a new package (BUILD) file in the exported dir and writes to it. -func (e *defaultExporter) WriteExportedPackageFile(pkg *core.Package, content []byte) { - filename := pkg.Filename - exportedFilename := filepath.Join(e.targetDir, filename) - f, err := fs.OpenDirFile(exportedFilename, os.O_CREATE|os.O_WRONLY, 0664) - if err != nil { - log.Fatalf("Failed to create and open exported BUILD file for %s: %v", exportedFilename, err) - } - defer f.Close() - - if _, err := f.Write(content); err != nil { - log.Errorf("Failed to write to exported BUILD file %s: %v", f.Name(), err) - } -} - -// trimPackage filters the statements to be written to the exported BUILD file. -func (e *defaultExporter) trimPackage(pkg *core.Package) ([]byte, error) { - p := asp.NewParserOnly() - parsed, err := p.ParseFileOnly(pkg.Filename) - if err != nil { - return nil, fmt.Errorf("Parsing original BUILD file: %v", err) - } - - content, err := os.ReadFile(pkg.Filename) - if err != nil { - return nil, fmt.Errorf("Opening original BUILD file: %v", err) - } - - trimmer := trimmer{ - origin: content, - pkg: pkg, - exporter: e, - // assuming max len of the original file to avoid reallocations. - bytes: make([]byte, 0, len(content)), - } - trimmer.trimBlock(parsed, 0, asp.Position(len(content))) - - return trimmer.bytes, nil -} - -// noTrimExporter implements an exporter that avoids trimming any packages by exporting all targets -// and statements in a package. -type noTrimExporter struct { - baseExporter - // exportedPackages tracks which packages have already had their BUILD files exported. - exportedPackages map[string]bool -} - -func (nte *noTrimExporter) ExportPreloaded() { - // Write any preloaded build defs - for _, preload := range nte.state.Config.Parse.PreloadBuildDefs { - if err := fs.RecursiveCopy(preload, filepath.Join(nte.targetDir, preload), 0); err != nil { - log.Errorf("Failed to copy preloaded build def %s: %s", preload, err) - } - } - - for _, target := range nte.state.Config.Parse.PreloadSubincludes { - targets := append(nte.state.Graph.TransitiveSubincludes(target), target) - nte.ExportTargets(targets) - } -} - -func (nte *noTrimExporter) ExportTarget(target *core.BuildTarget) { - pkg := nte.state.Graph.PackageOrDie(target.Label) - if !nte.checkAndSetVisited(target) { - return - } - - // We want to export the package that made this subrepo available, but we still need to walk the target deps - // as it may depend on other subrepos or first party targets - if target.Subrepo != nil { - nte.ExportTarget(target.Subrepo.Target) - nte.exportDependencies(target) - return - } - - nte.exportPackage(pkg) - nte.exportSubincludes(pkg) - nte.exportAllTargets(pkg) - nte.exportSources(target) - nte.exportDependencies(target) -} - -func (nte *noTrimExporter) WritePackageFiles() { -} - -// exportPackage exports the package BUILD file. -func (nte *noTrimExporter) exportPackage(pkg *core.Package) { - // Skip subrepos and internal packages. These will be generated by build statements in the exported - // repo or included in please internally. - if pkg.Subrepo != nil || pkg.Name == parse.InternalPackageName { - return - } - - if nte.exportedPackages[pkg.Name] { - return - } - nte.exportedPackages[pkg.Name] = true - - exportedFilename := filepath.Join(nte.targetDir, pkg.Filename) - if err := fs.CopyFile(pkg.Filename, exportedFilename, 0); err != nil { - log.Errorf("failed to export package %s: %v", pkg.Name, err) - } -} - -// exportSubincludes exports the subincluded targets. -func (nte *noTrimExporter) exportSubincludes(pkg *core.Package) { - subincludes := pkg.AllSubincludes(nte.state.Graph) - nte.ExportTargets(subincludes) -} - -// exportAllTargets will export all the targets in the provided package. -func (nte *noTrimExporter) exportAllTargets(pkg *core.Package) { - for _, target := range pkg.AllTargets() { - nte.ExportTarget(target) - } -} diff --git a/src/export/export_notrim.go b/src/export/export_notrim.go new file mode 100644 index 0000000000..0304d56f44 --- /dev/null +++ b/src/export/export_notrim.go @@ -0,0 +1,87 @@ +package export + +import ( + "path/filepath" + + "github.com/thought-machine/please/src/core" + "github.com/thought-machine/please/src/fs" + "github.com/thought-machine/please/src/parse" +) + +// noTrimExporter implements an exporter that avoids trimming any packages by exporting all targets +// and statements in a package. +type noTrimExporter struct { + baseExporter + // exportedPackages tracks which packages have already had their BUILD files exported. + exportedPackages map[string]bool +} + +func (nte *noTrimExporter) ExportPreloaded() { + // Write any preloaded build defs + for _, preload := range nte.state.Config.Parse.PreloadBuildDefs { + if err := fs.RecursiveCopy(preload, filepath.Join(nte.targetDir, preload), 0); err != nil { + log.Errorf("Failed to copy preloaded build def %s: %s", preload, err) + } + } + + for _, target := range nte.state.Config.Parse.PreloadSubincludes { + targets := append(nte.state.Graph.TransitiveSubincludes(target), target) + nte.ExportTargets(targets) + } +} + +func (nte *noTrimExporter) ExportTarget(target *core.BuildTarget) { + pkg := nte.state.Graph.PackageOrDie(target.Label) + if !nte.checkAndSetVisited(target) { + return + } + + // We want to export the package that made this subrepo available, but we still need to walk the target deps + // as it may depend on other subrepos or first party targets + if target.Subrepo != nil { + nte.ExportTarget(target.Subrepo.Target) + nte.exportDependencies(target) + return + } + + nte.exportPackage(pkg) + nte.exportSubincludes(pkg) + nte.exportAllTargets(pkg) + nte.exportSources(target) + nte.exportDependencies(target) +} + +func (nte *noTrimExporter) WritePackageFiles() { +} + +// exportPackage exports the package BUILD file. +func (nte *noTrimExporter) exportPackage(pkg *core.Package) { + // Skip subrepos and internal packages. These will be generated by build statements in the exported + // repo or included in please internally. + if pkg.Subrepo != nil || pkg.Name == parse.InternalPackageName { + return + } + + if nte.exportedPackages[pkg.Name] { + return + } + nte.exportedPackages[pkg.Name] = true + + exportedFilename := filepath.Join(nte.targetDir, pkg.Filename) + if err := fs.CopyFile(pkg.Filename, exportedFilename, 0); err != nil { + log.Errorf("failed to export package %s: %v", pkg.Name, err) + } +} + +// exportSubincludes exports the subincluded targets. +func (nte *noTrimExporter) exportSubincludes(pkg *core.Package) { + subincludes := pkg.AllSubincludes(nte.state.Graph) + nte.ExportTargets(subincludes) +} + +// exportAllTargets will export all the targets in the provided package. +func (nte *noTrimExporter) exportAllTargets(pkg *core.Package) { + for _, target := range pkg.AllTargets() { + nte.ExportTarget(target) + } +} diff --git a/src/export/export_trimmed.go b/src/export/export_trimmed.go new file mode 100644 index 0000000000..b2ba6fd69c --- /dev/null +++ b/src/export/export_trimmed.go @@ -0,0 +1,169 @@ +package export + +import ( + "fmt" + "os" + "path/filepath" + "slices" + + "github.com/please-build/buildtools/build" + "github.com/thought-machine/please/src/core" + "github.com/thought-machine/please/src/fs" + "github.com/thought-machine/please/src/parse" + "github.com/thought-machine/please/src/parse/asp" +) + +// defaultExporter implements an exporter that trims packages to reach a minimal exported repo. +type defaultExporter struct { + baseExporter + // visitedPackages maintains a record of the packages visited during the export process. + visitedPackages map[core.BuildLabel]bool + // requiredSubincludes maps packages to the subinclude labels they require. + requiredSubincludes map[core.BuildLabel]core.BuildLabels + // preloadedSubincludes tracks subincludes that are preloaded and don't need explicit export. + preloadedSubincludes map[core.BuildLabel]bool +} + +func (e *defaultExporter) ExportPreloaded() { + // Write any preloaded build defs + for _, preload := range e.state.Config.Parse.PreloadBuildDefs { + if err := fs.RecursiveCopy(preload, filepath.Join(e.targetDir, preload), 0); err != nil { + log.Fatalf("Failed to copy preloaded build def %s: %s", preload, err) + } + } + + for _, target := range e.state.Config.Parse.PreloadSubincludes { + targets := append(e.state.Graph.TransitiveSubincludes(target), target) + for _, t := range targets { + e.preloadedSubincludes[t] = true + } + e.ExportTargets(targets) + } +} + +func (e *defaultExporter) ExportTarget(target *core.BuildTarget) { + if !e.checkAndSetVisited(target) { + return + } + + log.Debugf("Exporting target: %v", target.Label) + + // Skip export for internal packages + if target.Label.PackageName == parse.InternalPackageName { + return + } + // We want to export the package that made this subrepo available, but we still need to walk the + // target deps as it may depend on other subrepos or first party targets + if target.Subrepo != nil && target.Subrepo.Target != nil { + e.ExportTarget(target.Subrepo.Target) + e.exportDependencies(target) + return + } + + e.exportSources(target) + e.exportDependencies(target) + + pkg := e.state.Graph.PackageOrDie(target.Label) + e.exportSubincludes(pkg, target) + e.exportRelatedTargets(pkg, target) + e.visitedPackages[pkg.Label()] = true +} + +func (e *defaultExporter) WritePackageFiles() { + for pkgLabel := range e.visitedPackages { + pkg := e.state.Graph.PackageOrDie(pkgLabel) + filteredBytes, err := e.trimPackage(pkg) + if err != nil { + log.Errorf("Failed to filter the build statements of package %s: %v", pkg.Label(), err) + continue + } + + parsedBuild, err := build.ParseBuild(pkg.Filename, filteredBytes) + if err != nil { + log.Fatalf("Failed to parse bytes for formatting: %v\nData:\n%s", err, filteredBytes) + } + formattedBytes := build.Format(parsedBuild) + + e.WriteExportedPackageFile(pkg, formattedBytes) + } +} + +// exportSubincludes exports the subincluded targets required to generate the target and selects them to +// later be written to the package as statements. +func (e *defaultExporter) exportSubincludes(pkg *core.Package, target *core.BuildTarget) { + for _, subinclude := range pkg.Metadata.FindRequiredSubincludes(target) { + // skip for preloaded subincludes, these are handled separately at the start to ensure they are + // they are exported even if not directly used by an exported target. + if e.preloadedSubincludes[subinclude] { + continue + } + + required := e.requiredSubincludes[pkg.Label()] + if !slices.Contains(required, subinclude) { + required = append(required, subinclude) + } + e.requiredSubincludes[pkg.Label()] = required + + target := e.getOrParseTarget(subinclude) + if target == nil { + log.Errorf("Unable to lookup target %s", subinclude) + continue + } + e.ExportTarget(target) + } +} + +// exportRelatedTargets exports build targets that are related to the build statement that generated. +func (e *defaultExporter) exportRelatedTargets(pkg *core.Package, target *core.BuildTarget) { + stmt := pkg.Metadata.FindStatement(target) + if stmt == nil { + log.Errorf("Failed to find statement for target %s in %s", target, pkg.Name) + return + } + + relatedTargets := pkg.Metadata.FindTargets(stmt) + log.Debugf("Exporting targets related to %s: %v", target, relatedTargets) + for _, target := range relatedTargets { + e.ExportTarget(target) + } +} + +// WriteExportedPackageFile creates a new package (BUILD) file in the exported dir and writes to it. +func (e *defaultExporter) WriteExportedPackageFile(pkg *core.Package, content []byte) { + filename := pkg.Filename + exportedFilename := filepath.Join(e.targetDir, filename) + f, err := fs.OpenDirFile(exportedFilename, os.O_CREATE|os.O_WRONLY, 0664) + if err != nil { + log.Fatalf("Failed to create and open exported BUILD file for %s: %v", exportedFilename, err) + } + defer f.Close() + + if _, err := f.Write(content); err != nil { + log.Errorf("Failed to write to exported BUILD file %s: %v", f.Name(), err) + } +} + +// trimPackage filters the statements to be written to the exported BUILD file. +func (e *defaultExporter) trimPackage(pkg *core.Package) ([]byte, error) { + p := asp.NewParserOnly() + parsed, err := p.ParseFileOnly(pkg.Filename) + if err != nil { + return nil, fmt.Errorf("Parsing original BUILD file: %v", err) + } + + content, err := os.ReadFile(pkg.Filename) + if err != nil { + return nil, fmt.Errorf("Opening original BUILD file: %v", err) + } + + trimmer := trimmer{ + origin: content, + pkg: pkg, + exporter: e, + // assuming max len of the original file to avoid reallocations. + bytes: make([]byte, 0, len(content)), + } + trimmer.trimBlock(parsed, 0, asp.Position(len(content))) + + return trimmer.bytes, nil +} From 66eef4255898f4c79d2ba80a81f30d7690fd742c Mon Sep 17 00:00:00 2001 From: DuBento Date: Fri, 5 Jun 2026 15:47:00 +0100 Subject: [PATCH 072/118] fix: propagate cursor --- src/parse/asp/interpreter.go | 1 + 1 file changed, 1 insertion(+) diff --git a/src/parse/asp/interpreter.go b/src/parse/asp/interpreter.go index 85c2a02327..7a5a3d160a 100644 --- a/src/parse/asp/interpreter.go +++ b/src/parse/asp/interpreter.go @@ -1183,6 +1183,7 @@ type scopeMetadata struct { func (m *scopeMetadata) NewMetadata() ScopeMetadata { return &scopeMetadata{ + cursor: m.cursor, objectOrigins: map[string]core.BuildLabel{}, requiredOrigins: map[core.BuildLabel]struct{}{}, } From 45e6cd8926585b6fef4e3f25ea8f0bdbd022c263 Mon Sep 17 00:00:00 2001 From: DuBento Date: Fri, 5 Jun 2026 15:53:01 +0100 Subject: [PATCH 073/118] simplify calls to getOrParseTarget --- src/export/export_trimmed.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/export/export_trimmed.go b/src/export/export_trimmed.go index b2ba6fd69c..d353e8a034 100644 --- a/src/export/export_trimmed.go +++ b/src/export/export_trimmed.go @@ -91,7 +91,8 @@ func (e *defaultExporter) WritePackageFiles() { // exportSubincludes exports the subincluded targets required to generate the target and selects them to // later be written to the package as statements. func (e *defaultExporter) exportSubincludes(pkg *core.Package, target *core.BuildTarget) { - for _, subinclude := range pkg.Metadata.FindRequiredSubincludes(target) { + subincludes := pkg.Metadata.FindRequiredSubincludes(target) + for _, subinclude := range subincludes { // skip for preloaded subincludes, these are handled separately at the start to ensure they are // they are exported even if not directly used by an exported target. if e.preloadedSubincludes[subinclude] { @@ -103,14 +104,8 @@ func (e *defaultExporter) exportSubincludes(pkg *core.Package, target *core.Buil required = append(required, subinclude) } e.requiredSubincludes[pkg.Label()] = required - - target := e.getOrParseTarget(subinclude) - if target == nil { - log.Errorf("Unable to lookup target %s", subinclude) - continue - } - e.ExportTarget(target) } + e.ExportTargets(subincludes) } // exportRelatedTargets exports build targets that are related to the build statement that generated. From 6bf00511506c100f29c23dfdd90e63cc31806d8b Mon Sep 17 00:00:00 2001 From: DuBento Date: Fri, 5 Jun 2026 16:54:01 +0100 Subject: [PATCH 074/118] remove use of formatter and trim newlines --- src/export/export_trimmed.go | 37 ++++++++++++++----- .../test_for_if/expected_repo/BUILD_FILE | 5 +-- 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/src/export/export_trimmed.go b/src/export/export_trimmed.go index d353e8a034..6f2596a947 100644 --- a/src/export/export_trimmed.go +++ b/src/export/export_trimmed.go @@ -1,12 +1,12 @@ package export import ( + "bytes" "fmt" "os" "path/filepath" "slices" - "github.com/please-build/buildtools/build" "github.com/thought-machine/please/src/core" "github.com/thought-machine/please/src/fs" "github.com/thought-machine/please/src/parse" @@ -78,13 +78,8 @@ func (e *defaultExporter) WritePackageFiles() { continue } - parsedBuild, err := build.ParseBuild(pkg.Filename, filteredBytes) - if err != nil { - log.Fatalf("Failed to parse bytes for formatting: %v\nData:\n%s", err, filteredBytes) - } - formattedBytes := build.Format(parsedBuild) - - e.WriteExportedPackageFile(pkg, formattedBytes) + filteredBytes = trimNewlines(filteredBytes) + e.writeExportedPackageFile(pkg, filteredBytes) } } @@ -124,7 +119,7 @@ func (e *defaultExporter) exportRelatedTargets(pkg *core.Package, target *core.B } // WriteExportedPackageFile creates a new package (BUILD) file in the exported dir and writes to it. -func (e *defaultExporter) WriteExportedPackageFile(pkg *core.Package, content []byte) { +func (e *defaultExporter) writeExportedPackageFile(pkg *core.Package, content []byte) { filename := pkg.Filename exportedFilename := filepath.Join(e.targetDir, filename) f, err := fs.OpenDirFile(exportedFilename, os.O_CREATE|os.O_WRONLY, 0664) @@ -162,3 +157,27 @@ func (e *defaultExporter) trimPackage(pkg *core.Package) ([]byte, error) { return trimmer.bytes, nil } + +// trimNewlines trims leading and trailing whitespace and collapses 3+ consecutive newlines to 2. +func trimNewlines(b []byte) []byte { + trimmed := bytes.TrimSpace(b) + var pointer, newlines int + for _, val := range trimmed { + if val == '\n' { + newlines++ + if newlines > 2 { + continue // Skip third (or more) consecutive newline + } + } else { + newlines = 0 + } + trimmed[pointer] = val + pointer++ + } + trimmed = trimmed[:pointer] + + if len(trimmed) > 0 { + trimmed = append(trimmed, '\n') // Trailing newline + } + return trimmed +} diff --git a/test/export/test_for_if/expected_repo/BUILD_FILE b/test/export/test_for_if/expected_repo/BUILD_FILE index 80123416ed..868064f63c 100644 --- a/test/export/test_for_if/expected_repo/BUILD_FILE +++ b/test/export/test_for_if/expected_repo/BUILD_FILE @@ -1,7 +1,4 @@ -for i in [ - "a", - "b", -]: +for i in ["a", "b"]: if i == "a": genrule( name = "a", From 3f36ad4de788413b7ebc113dc0a6b0d72cf48de2 Mon Sep 17 00:00:00 2001 From: DuBento Date: Fri, 5 Jun 2026 17:00:15 +0100 Subject: [PATCH 075/118] reuse parser between build files --- src/export/export_test.go | 3 ++- src/export/export_trimmed.go | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/export/export_test.go b/src/export/export_test.go index 630c937e8e..bc893b6c42 100644 --- a/src/export/export_test.go +++ b/src/export/export_test.go @@ -110,7 +110,8 @@ func TestFilterPackageFile(t *testing.T) { } e.visitedPackages[pkg.Label()] = true - got, err := e.trimPackage(pkg) + p := asp.NewParserOnly() + got, err := e.trimPackage(p, pkg) assert.NoError(t, err) expected, err := os.ReadFile(tc.expected) diff --git a/src/export/export_trimmed.go b/src/export/export_trimmed.go index 6f2596a947..f4c784d20e 100644 --- a/src/export/export_trimmed.go +++ b/src/export/export_trimmed.go @@ -70,9 +70,10 @@ func (e *defaultExporter) ExportTarget(target *core.BuildTarget) { } func (e *defaultExporter) WritePackageFiles() { + p := asp.NewParserOnly() for pkgLabel := range e.visitedPackages { pkg := e.state.Graph.PackageOrDie(pkgLabel) - filteredBytes, err := e.trimPackage(pkg) + filteredBytes, err := e.trimPackage(p, pkg) if err != nil { log.Errorf("Failed to filter the build statements of package %s: %v", pkg.Label(), err) continue @@ -134,8 +135,7 @@ func (e *defaultExporter) writeExportedPackageFile(pkg *core.Package, content [] } // trimPackage filters the statements to be written to the exported BUILD file. -func (e *defaultExporter) trimPackage(pkg *core.Package) ([]byte, error) { - p := asp.NewParserOnly() +func (e *defaultExporter) trimPackage(p *asp.Parser, pkg *core.Package) ([]byte, error) { parsed, err := p.ParseFileOnly(pkg.Filename) if err != nil { return nil, fmt.Errorf("Parsing original BUILD file: %v", err) From b76c2e2fab9458876415cf3da45f3fa708641f82 Mon Sep 17 00:00:00 2001 From: DuBento Date: Mon, 8 Jun 2026 12:20:01 +0100 Subject: [PATCH 076/118] use of baseExporter directly and interface cleanup --- src/export/export.go | 117 +++++++++++++++++++---------------- src/export/export_notrim.go | 6 +- src/export/export_trimmed.go | 6 +- 3 files changed, 69 insertions(+), 60 deletions(-) diff --git a/src/export/export.go b/src/export/export.go index e538b014ef..0a81fd718d 100644 --- a/src/export/export.go +++ b/src/export/export.go @@ -14,30 +14,6 @@ import ( var log = logging.Log -// Exporter defines the interface for exporting parts of a Please repository to a new directory. -// It handles the copying of configuration files, preloaded build definitions, and selected -// targets along with their necessary source files and dependencies. -type Exporter interface { - // ExportPlzConfig exports the repository's configuration files (e.g., .plzconfig and its - // platform-specific variants) to the target export directory. - ExportPlzConfig() - // ExportPreloaded exports all globally preloaded build definitions and subincluded targets. - // These are usually defined in the repository's configuration file. - ExportPreloaded() - // ExportTargets exports the set of targets identified by the given build labels. - // Each target recursively exports all their source files and required build statements, but also - // targets in their transitive dependencies. - ExportTargets(core.BuildLabels) - // ExportTarget exports an individual build target. - // Each target recursively exports all their source files and required build statements, but also - // targets in their transitive dependencies. - ExportTarget(*core.BuildTarget) - // WritePackageFiles writes the processed BUILD files for all exported targets to the - // export directory. These BUILD files may be modified (e.g., trimmed) depending on - // the exporter's implementation. - WritePackageFiles() -} - // Repo export a new please repo including the targets and dependencies requested. Depending on the // noTrim flag, the export will attempt to trim the resulting repository, exporting only the required // targets and build statements in their packages. If noTrim is set, all targets of a package will be @@ -50,10 +26,7 @@ func Repo(state *core.BuildState, dir string, noTrim bool, targets []core.BuildL log.Fatalf("failed to create export directory %s: %v", dir, err) } - e.ExportPlzConfig() - e.ExportPreloaded() - e.ExportTargets(targets) - e.WritePackageFiles() + e.run(targets) } // Outputs exports the build artifacts (output files) produced by building the specified @@ -74,31 +47,48 @@ func Outputs(state *core.BuildState, dir string, targets []core.BuildLabel) { } } +// Exporter defines the interface for exporting parts of a Please repository to a new directory. +// It handles the copying of configuration files, preloaded build definitions, and selected +// targets along with their necessary source files and dependencies. +type Exporter interface { + // ExportPreloaded exports all globally preloaded build definitions and subincluded targets. + // These are usually defined in the repository's configuration file. + ExportPreloaded() + // ExportTarget exports an individual build target. + // Each target recursively exports all their source files and required build statements, but also + // targets in their transitive dependencies. + ExportTarget(*core.BuildTarget) + // WritePackageFiles writes the processed BUILD files for all exported targets to the + // export directory. These BUILD files may be modified (e.g., trimmed) depending on + // the exporter's implementation. + WritePackageFiles() +} + // newExporter creates a new exporter of a specific type based on the arguments. -func newExporter(state *core.BuildState, dir string, noTrim bool) Exporter { - base := baseExporter{ +func newExporter(state *core.BuildState, dir string, noTrim bool) *baseExporter { + base := &baseExporter{ state: state, targetDir: dir, exportedTargets: map[core.BuildLabel]bool{}, } + var exporter Exporter if noTrim { - exporter := &noTrimExporter{ + exporter = &noTrimExporter{ baseExporter: base, exportedPackages: map[string]bool{}, } - exporter.impl = exporter - return exporter + } else { + exporter = &defaultExporter{ + baseExporter: base, + visitedPackages: map[core.BuildLabel]bool{}, + requiredSubincludes: map[core.BuildLabel]core.BuildLabels{}, + preloadedSubincludes: map[core.BuildLabel]bool{}, + } } - exporter := &defaultExporter{ - baseExporter: base, - visitedPackages: map[core.BuildLabel]bool{}, - requiredSubincludes: map[core.BuildLabel]core.BuildLabels{}, - preloadedSubincludes: map[core.BuildLabel]bool{}, - } - exporter.impl = exporter - return exporter + base.impl = exporter + return base } // baseExporter provides common fields and methods of other exporters. @@ -113,7 +103,17 @@ type baseExporter struct { impl Exporter } -func (be *baseExporter) ExportPlzConfig() { +// run specifies the main steps when running an export. +func (be *baseExporter) run(targets core.BuildLabels) { + be.exportPlzConfig() + be.impl.ExportPreloaded() + be.exportTargets(targets) + be.impl.WritePackageFiles() +} + +// exportPlzConfig exports the repository's configuration files (e.g., .plzconfig and its +// platform-specific variants) to the target export directory. +func (be *baseExporter) exportPlzConfig() { profiles, err := filepath.Glob(".plzconfig*") if err != nil { log.Fatalf("failed to glob .plzconfig files: %v", err) @@ -129,7 +129,8 @@ func (be *baseExporter) ExportPlzConfig() { } } -func (be *baseExporter) ExportTargets(labels core.BuildLabels) { +// exportTargets exports the set of targets identified by the given build labels. +func (be *baseExporter) exportTargets(labels core.BuildLabels) { for _, l := range labels { target := be.getOrParseTarget(l) if target == nil { @@ -140,21 +141,11 @@ func (be *baseExporter) ExportTargets(labels core.BuildLabels) { } } -func (be *baseExporter) getOrParseTarget(label core.BuildLabel) *core.BuildTarget { - target := be.state.Graph.Target(label) - if target == nil { - log.Debugf("Target %v not found in graph. Attempting to parse...", label) - be.state.WaitForBuiltTarget(label, core.OriginalTarget, core.ParseModeNormal) - target = be.state.Graph.Target(label) - } - return target -} - // exportDependencies exports exportDependencies of a target. func (be *baseExporter) exportDependencies(target *core.BuildTarget) { deps := target.DeclaredDependencies() log.Debugf("Exporting dependencies of (%v): %v", target.Label, deps) - be.ExportTargets(deps) + be.exportTargets(deps) } // exportSources exports all files required by the target. @@ -176,6 +167,24 @@ func (be *baseExporter) exportSources(target *core.BuildTarget) { } } +// getOrParseTarget attempts to look up a target in the build graph. If the target has not +// been parsed yet, it dynamically requests the package be parsed and blocks until the target is resolved. +// +// This occurs in trimmed-mode exports when walking dependencies of adjacent targets which were not +// explicitly activated or resolved during the initial build/parse phase. +// +// This requires the background parser worker threads to be kept alive as daemons (controlled by the +// "KeepParserRunning" build state option). +func (be *baseExporter) getOrParseTarget(label core.BuildLabel) *core.BuildTarget { + target := be.state.Graph.Target(label) + if target == nil { + log.Debugf("Target %v not found in graph. Attempting to parse...", label) + be.state.WaitForBuiltTarget(label, core.OriginalTarget, core.ParseModeNormal) + target = be.state.Graph.Target(label) + } + return target +} + // checkAndSetVisited is a helper to ensure we only visit the same target once. // It returns true if this is the first time the target is being exported. func (be *baseExporter) checkAndSetVisited(target *core.BuildTarget) bool { diff --git a/src/export/export_notrim.go b/src/export/export_notrim.go index 0304d56f44..2ee7478ca1 100644 --- a/src/export/export_notrim.go +++ b/src/export/export_notrim.go @@ -11,7 +11,7 @@ import ( // noTrimExporter implements an exporter that avoids trimming any packages by exporting all targets // and statements in a package. type noTrimExporter struct { - baseExporter + *baseExporter // exportedPackages tracks which packages have already had their BUILD files exported. exportedPackages map[string]bool } @@ -26,7 +26,7 @@ func (nte *noTrimExporter) ExportPreloaded() { for _, target := range nte.state.Config.Parse.PreloadSubincludes { targets := append(nte.state.Graph.TransitiveSubincludes(target), target) - nte.ExportTargets(targets) + nte.exportTargets(targets) } } @@ -76,7 +76,7 @@ func (nte *noTrimExporter) exportPackage(pkg *core.Package) { // exportSubincludes exports the subincluded targets. func (nte *noTrimExporter) exportSubincludes(pkg *core.Package) { subincludes := pkg.AllSubincludes(nte.state.Graph) - nte.ExportTargets(subincludes) + nte.exportTargets(subincludes) } // exportAllTargets will export all the targets in the provided package. diff --git a/src/export/export_trimmed.go b/src/export/export_trimmed.go index f4c784d20e..16c280577c 100644 --- a/src/export/export_trimmed.go +++ b/src/export/export_trimmed.go @@ -15,7 +15,7 @@ import ( // defaultExporter implements an exporter that trims packages to reach a minimal exported repo. type defaultExporter struct { - baseExporter + *baseExporter // visitedPackages maintains a record of the packages visited during the export process. visitedPackages map[core.BuildLabel]bool // requiredSubincludes maps packages to the subinclude labels they require. @@ -37,7 +37,7 @@ func (e *defaultExporter) ExportPreloaded() { for _, t := range targets { e.preloadedSubincludes[t] = true } - e.ExportTargets(targets) + e.exportTargets(targets) } } @@ -101,7 +101,7 @@ func (e *defaultExporter) exportSubincludes(pkg *core.Package, target *core.Buil } e.requiredSubincludes[pkg.Label()] = required } - e.ExportTargets(subincludes) + e.exportTargets(subincludes) } // exportRelatedTargets exports build targets that are related to the build statement that generated. From 51814bcac40c257bd65f31891c1d673e9c66e1c5 Mon Sep 17 00:00:00 2001 From: DuBento Date: Mon, 8 Jun 2026 17:21:58 +0100 Subject: [PATCH 077/118] interpreter preloads info prevents secondary loading/subincludes successful export of go_binary in complex repo --- src/core/package_metadata.go | 36 ++++++++++++++++++++++++++++++++++++ src/export/export.go | 2 +- src/export/export_test.go | 6 +++--- src/export/export_trimmed.go | 19 ++++++++----------- src/export/trimmer.go | 1 - src/parse/asp/interpreter.go | 32 ++++++++++++++++++++++++++------ 6 files changed, 74 insertions(+), 22 deletions(-) diff --git a/src/core/package_metadata.go b/src/core/package_metadata.go index 03d7b314e6..444f2feeee 100644 --- a/src/core/package_metadata.go +++ b/src/core/package_metadata.go @@ -60,6 +60,10 @@ type PackageMetadata interface { // FindRequiredSubincludes returns all subinclude labels that were required by the given target. // The return value is empty if no subinclude information was found for the target. FindRequiredSubincludes(target *BuildTarget) BuildLabels + // FindRelatedTargets finds all the targets that are related to the argument. In this context, + // target relationship is determined by looking for targets generated by the same build statement. + // The result excludes the target in the argument. + FindRelatedTargets(target *BuildTarget) BuildLabels // GetSubincludedLabels returns all build labels that were included by the given subinclude statement. // Returns the labels or an empty slice if the statement wasn't found. GetSubincludedLabels(stmt *BuildStatement) BuildLabels @@ -119,6 +123,10 @@ func (m *packageMetadataImpl) RegisterSubincludeStatement(label BuildLabel, stmt } func (m *packageMetadataImpl) FindStatement(target *BuildTarget) *BuildStatement { + if target == nil { + return nil + } + m.mutex.RLock() defer m.mutex.RUnlock() @@ -127,10 +135,15 @@ func (m *packageMetadataImpl) FindStatement(target *BuildTarget) *BuildStatement return &stmt } } + log.Debugf("Failed to find statement for target %s", target) return nil } func (m *packageMetadataImpl) FindTargets(stmt *BuildStatement) []*BuildTarget { + if stmt == nil { + return nil + } + m.mutex.RLock() defer m.mutex.RUnlock() @@ -138,13 +151,33 @@ func (m *packageMetadataImpl) FindTargets(stmt *BuildStatement) []*BuildTarget { } func (m *packageMetadataImpl) FindRequiredSubincludes(target *BuildTarget) BuildLabels { + if target == nil { + return nil + } + m.mutex.RLock() defer m.mutex.RUnlock() return m.TargetToSubinclude[target] } +func (m *packageMetadataImpl) FindRelatedTargets(target *BuildTarget) BuildLabels { + stmt := m.FindStatement(target) + relatedTargets := m.FindTargets(stmt) + labels := make(BuildLabels, 0, len(relatedTargets)) + for _, t := range relatedTargets { + if t.Label != target.Label { + labels = append(labels, t.Label) + } + } + return labels +} + func (m *packageMetadataImpl) GetSubincludedLabels(stmt *BuildStatement) BuildLabels { + if stmt == nil { + return nil + } + m.mutex.RLock() defer m.mutex.RUnlock() @@ -177,6 +210,9 @@ func (n *noopPackageMetadata) FindRequiredSubincludes(target *BuildTarget) Build log.Fatalf("Metadata not tracked, using no-op implementation.") return nil } +func (m *noopPackageMetadata) FindRelatedTargets(target *BuildTarget) BuildLabels { + return nil +} func (n *noopPackageMetadata) GetSubincludedLabels(stmt *BuildStatement) BuildLabels { log.Fatalf("Metadata not tracked, using no-op implementation.") return nil diff --git a/src/export/export.go b/src/export/export.go index 0a81fd718d..0ceee2d207 100644 --- a/src/export/export.go +++ b/src/export/export.go @@ -178,7 +178,7 @@ func (be *baseExporter) exportSources(target *core.BuildTarget) { func (be *baseExporter) getOrParseTarget(label core.BuildLabel) *core.BuildTarget { target := be.state.Graph.Target(label) if target == nil { - log.Debugf("Target %v not found in graph. Attempting to parse...", label) + log.Infof("Target %v not found in graph. Attempting to parse...", label) be.state.WaitForBuiltTarget(label, core.OriginalTarget, core.ParseModeNormal) target = be.state.Graph.Target(label) } diff --git a/src/export/export_test.go b/src/export/export_test.go index bc893b6c42..3edede2cc3 100644 --- a/src/export/export_test.go +++ b/src/export/export_test.go @@ -50,7 +50,7 @@ func TestMinimalSubincludeStatement(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - e := newExporter(nil, "", false).(*defaultExporter) + e := newExporter(nil, "", false).impl.(*defaultExporter) pkg := &core.Package{Name: "test"} e.requiredSubincludes[pkg.Label()] = tc.requiredLabels @@ -104,7 +104,7 @@ func TestFilterPackageFile(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - e := newExporter(nil, "", false).(*defaultExporter) + e := newExporter(nil, "", false).impl.(*defaultExporter) for _, name := range tc.required { e.exportedTargets[targetLabels[name]] = true } @@ -223,7 +223,7 @@ for i in [ pkg.Filename = "BUILD" targetLabels := walkASTRegisterTargets(t, statements, pkg, tc.registered) - e := newExporter(nil, "", false).(*defaultExporter) + e := newExporter(nil, "", false).impl.(*defaultExporter) for _, name := range tc.required { e.exportedTargets[targetLabels[name]] = true } diff --git a/src/export/export_trimmed.go b/src/export/export_trimmed.go index 16c280577c..cc88e9d257 100644 --- a/src/export/export_trimmed.go +++ b/src/export/export_trimmed.go @@ -32,7 +32,7 @@ func (e *defaultExporter) ExportPreloaded() { } } - for _, target := range e.state.Config.Parse.PreloadSubincludes { + for _, target := range e.state.GetPreloadedSubincludes() { targets := append(e.state.Graph.TransitiveSubincludes(target), target) for _, t := range targets { e.preloadedSubincludes[t] = true @@ -88,6 +88,11 @@ func (e *defaultExporter) WritePackageFiles() { // later be written to the package as statements. func (e *defaultExporter) exportSubincludes(pkg *core.Package, target *core.BuildTarget) { subincludes := pkg.Metadata.FindRequiredSubincludes(target) + if len(subincludes) == 0 { + return + } + + log.Debugf("Subincludes required for %s: %v", target, subincludes) for _, subinclude := range subincludes { // skip for preloaded subincludes, these are handled separately at the start to ensure they are // they are exported even if not directly used by an exported target. @@ -106,17 +111,9 @@ func (e *defaultExporter) exportSubincludes(pkg *core.Package, target *core.Buil // exportRelatedTargets exports build targets that are related to the build statement that generated. func (e *defaultExporter) exportRelatedTargets(pkg *core.Package, target *core.BuildTarget) { - stmt := pkg.Metadata.FindStatement(target) - if stmt == nil { - log.Errorf("Failed to find statement for target %s in %s", target, pkg.Name) - return - } - - relatedTargets := pkg.Metadata.FindTargets(stmt) + relatedTargets := pkg.Metadata.FindRelatedTargets(target) log.Debugf("Exporting targets related to %s: %v", target, relatedTargets) - for _, target := range relatedTargets { - e.ExportTarget(target) - } + e.exportTargets(relatedTargets) } // WriteExportedPackageFile creates a new package (BUILD) file in the exported dir and writes to it. diff --git a/src/export/trimmer.go b/src/export/trimmer.go index cd249d87d0..c6e22e7ab0 100644 --- a/src/export/trimmer.go +++ b/src/export/trimmer.go @@ -32,7 +32,6 @@ func (t *trimmer) walkFile(stmts []*asp.Statement, start, end asp.Position, cons // cursor tracks the position in a block that's being interpreted. cursor := start for _, stmt := range stmts { - log.Debugf("Evaluating statement %s", t.origin[stmt.Pos:stmt.EndPos]) // Write content that's between stmts (e.g. comments). We skip these while parsing so it won't // be included in "parsedStmts" but we want the resulting BUILD file to include these. if cursor < stmt.Pos { diff --git a/src/parse/asp/interpreter.go b/src/parse/asp/interpreter.go index 7a5a3d160a..e74f093322 100644 --- a/src/parse/asp/interpreter.go +++ b/src/parse/asp/interpreter.go @@ -26,6 +26,8 @@ type interpreter struct { parser *Parser subincludes *cmap.ErrMap[string, pyDict] asts *cmap.ErrMap[string, []*Statement] + // preloaded is a set to register all preloaded objects. + preloaded *cmap.Map[string, struct{}] configs map[*core.BuildState]*pyConfig configsMutex sync.RWMutex @@ -54,6 +56,7 @@ func newInterpreter(state *core.BuildState, p *Parser) *interpreter { i := &interpreter{ scope: s, parser: p, + preloaded: cmap.New[string, struct{}](cmap.SmallShardCount, cmap.XXHash), configs: map[*core.BuildState]*pyConfig{}, limiter: make(semaphore, state.Config.Parse.NumThreads), regexCache: cmap.New[string, *regexp.Regexp](cmap.SmallShardCount, cmap.XXHash), @@ -162,11 +165,25 @@ func (i *interpreter) preloadSubinclude(s *scope, label core.BuildLabel) (err er s.interpreter.loadPluginConfig(s, includeState) for _, out := range t.FullOutputs() { - s.SetAllWithOrigin(s.interpreter.Subinclude(s, out, t.Label, true), false, &t.Label) + globals := s.interpreter.Subinclude(s, out, t.Label, true) + s.interpreter.registerPreloaded(globals) + s.SetAllWithOrigin(globals, false, &t.Label) } return nil } +// registerPreloaded marks objects as preloaded for later reference. +func (i *interpreter) registerPreloaded(d pyDict) { + for k := range d { + if k == "CONFIG" { + // Config will be set for each scope instance from global config. Skipping since every preload + // will override this value and will never use it. + continue + } + i.preloaded.Add(k, struct{}{}) + } +} + // interpretAll runs a series of statements in the scope of the given package. // The first return value is for testing only. func (i *interpreter) interpretAll(pkg *core.Package, forLabel, dependent *core.BuildLabel, mode core.ParseMode, statements []*Statement) (*scope, error) { @@ -478,7 +495,7 @@ func (s *scope) Lookup(name string) pyObject { // lookupWithOrigin is like Lookup but returns the origin label of the variable as well. func (s *scope) lookupWithOrigin(name string) (pyObject, *core.BuildLabel) { if obj, present := s.locals[name]; present { - return obj, s.metadata.Origin(name) + return obj, s.metadata.Origin(s, name) } else if s.parent != nil { return s.parent.lookupWithOrigin(name) } @@ -504,7 +521,7 @@ func (s *scope) SetAll(d pyDict, publicOnly bool) { s.SetAllWithOrigin(d, publicOnly, nil) } -// SetAllWithOrigin is like SetAll but also records the origin labe l for all variables. +// SetAllWithOrigin is like SetAll but also records the origin label for all variables. func (s *scope) SetAllWithOrigin(d pyDict, publicOnly bool, origin *core.BuildLabel) { for k, v := range d { if k == "CONFIG" { @@ -1159,7 +1176,7 @@ type ScopeMetadata interface { // Cursor returns the statement being currently interpreted. Cursor() *Statement // Origin gets the origin of the object by name. Should return nil if not found or unimplemented. - Origin(name string) *core.BuildLabel + Origin(scope *scope, name string) *core.BuildLabel // RequiredOrigins returns a set of all the origins (subincluded labels) required by the current // scope. RequiredOrigins() map[core.BuildLabel]struct{} @@ -1193,7 +1210,10 @@ func (m *scopeMetadata) Cursor() *Statement { return m.cursor } -func (m *scopeMetadata) Origin(name string) *core.BuildLabel { +func (m *scopeMetadata) Origin(scope *scope, name string) *core.BuildLabel { + if scope.interpreter != nil && scope.interpreter.preloaded.Contains(name) { + return nil + } if label, ok := m.objectOrigins[name]; ok { return &label } @@ -1225,7 +1245,7 @@ type noopScopeMetadata struct{} func (nm *noopScopeMetadata) NewMetadata() ScopeMetadata { return &noopScopeMetadata{} } func (nm *noopScopeMetadata) Cursor() *Statement { return nil } -func (nm *noopScopeMetadata) Origin(name string) *core.BuildLabel { return nil } +func (nm *noopScopeMetadata) Origin(scope *scope, name string) *core.BuildLabel { return nil } func (nm *noopScopeMetadata) RequiredOrigins() map[core.BuildLabel]struct{} { return nil } func (nm *noopScopeMetadata) SetCursor(stmt *Statement) {} func (nm *noopScopeMetadata) SetObjectOrigin(name string, origin core.BuildLabel) {} From 58711e9f0de8b282f98d2180a47575d08b4ed88c Mon Sep 17 00:00:00 2001 From: DuBento Date: Tue, 9 Jun 2026 11:55:20 +0100 Subject: [PATCH 078/118] fix: deadlocking on closed results thread --- src/core/state.go | 34 +++++++++++++++++++++++++++++----- src/export/export.go | 3 ++- src/please.go | 6 +++--- src/plz/plz.go | 15 ++------------- 4 files changed, 36 insertions(+), 22 deletions(-) diff --git a/src/core/state.go b/src/core/state.go index 04deb2ddcf..2b30cbf1ef 100644 --- a/src/core/state.go +++ b/src/core/state.go @@ -245,6 +245,8 @@ type BuildState struct { // KeepParserRunning prevents closing task worker (parse and build) channels to support later // calls to parser. KeepParserRunning bool + // WaitForDisplay is a function that blocks until the display thread has finished. + WaitForDisplay func() // initOnce is used to control loading the subrepo .plzconfig initOnce *sync.Once @@ -447,6 +449,22 @@ func (state *BuildState) CloseResults() { } } +// CleanUp cleans up and shuts down the build state. +func (state *BuildState) CleanUp() { + state.CloseResults() + + if state.WaitForDisplay != nil { + state.WaitForDisplay() + } + + if state.Cache != nil { + state.Cache.Shutdown() + } + if state.RemoteClient != nil { + state.RemoteClient.Disconnect() + } +} + // IsOriginalTarget returns true if a target is an original target, ie. one specified on the command line. func (state *BuildState) IsOriginalTarget(target *BuildTarget) bool { return state.isOriginalTarget(target, false) @@ -679,11 +697,17 @@ func (state *BuildState) forwardResults() { delete(activeTargets, target) } } - state.progress.mutex.Lock() - if state.progress.results != nil { - state.progress.results <- result - } - state.progress.mutex.Unlock() + state.sendResult(result) + } +} + +// sendResult sends a unique result to the channel. A simple method that is mostly useful for a +// deferring the mutex close and avoid deadlocks even when we attempt to write to a closed channel. +func (state *BuildState) sendResult(result *BuildResult) { + state.progress.mutex.Lock() + defer state.progress.mutex.Unlock() + if state.progress.results != nil { + state.progress.results <- result } } diff --git a/src/export/export.go b/src/export/export.go index 0ceee2d207..178d242f54 100644 --- a/src/export/export.go +++ b/src/export/export.go @@ -10,6 +10,7 @@ import ( "github.com/thought-machine/please/src/cli/logging" "github.com/thought-machine/please/src/core" "github.com/thought-machine/please/src/fs" + "github.com/thought-machine/please/src/parse" ) var log = logging.Log @@ -179,7 +180,7 @@ func (be *baseExporter) getOrParseTarget(label core.BuildLabel) *core.BuildTarge target := be.state.Graph.Target(label) if target == nil { log.Infof("Target %v not found in graph. Attempting to parse...", label) - be.state.WaitForBuiltTarget(label, core.OriginalTarget, core.ParseModeNormal) + parse.Parse(be.state, label, core.OriginalTarget, core.ParseModeNormal) target = be.state.Graph.Target(label) } return target diff --git a/src/please.go b/src/please.go index 9f1829f271..50e09558d3 100644 --- a/src/please.go +++ b/src/please.go @@ -769,7 +769,7 @@ var buildFunctions = map[string]func() int{ "export": func() int { success, state := runBuild(opts.Export.Args.Targets, buildOpts{ParseMetadata: true, KeepParserRunning: true}) // Required cleanup due to running parser in background - defer plz.CleanUp(state) + defer state.CleanUp() if success { export.Repo(state, opts.Export.Output, opts.Export.NoTrim, state.ExpandOriginalLabels()) @@ -1227,8 +1227,8 @@ func runPlease(state *core.BuildState, targets []core.BuildLabel) { output.MonitorState(state, !pretty, detailedTests, streamTests, shell, shellRun, string(opts.OutputFlags.TraceFile)) wg.Done() }() + state.WaitForDisplay = wg.Wait plz.Run(targets, opts.BuildFlags.PreTargets, state, config, state.TargetArch) - wg.Wait() } // testTargets handles test targets which can be given in two formats; a list of targets or a single @@ -1327,7 +1327,7 @@ type buildOpts struct { IsQuery bool ParseMetadata bool // Keep the workers running in the background for inline parsing during specific ops (e.g. export). - // Note: when running background workers we need to explicit call plz.Cleanup() at the end of the CLI op. + // Note: when running background workers we need to explicit call CleanUp at the end of the CLI op. KeepParserRunning bool } diff --git a/src/plz/plz.go b/src/plz/plz.go index e2ee4a7245..a17a27fe2d 100644 --- a/src/plz/plz.go +++ b/src/plz/plz.go @@ -116,24 +116,14 @@ func Run(targets, preTargets []core.BuildLabel, state *core.BuildState, config * // proceeding with the specific op for the graph to be complete. state.WaitForBuildToComplete() reportResults(state, config) - // Cleanup() needs to be called at the end of the CLI run when KeepParserRunning is enabled. + // state.CleanUp() needs to be called at the end of the CLI run when KeepParserRunning is enabled. return } // Wait for all worker to finish. This should happen as soon as we no longer have pending tasks. // The last task should call state.Close() and close the queue channels. wg.Wait() reportResults(state, config) - CleanUp(state) -} - -// CleanUp cleans up and shuts down the build state. -func CleanUp(state *core.BuildState) { - if state.Cache != nil { - state.Cache.Shutdown() - } - if state.RemoteClient != nil { - state.RemoteClient.Disconnect() - } + state.CleanUp() } // reportResults reports on metrics and results at the end of the build. @@ -142,7 +132,6 @@ func reportResults(state *core.BuildState, config *core.Configuration) { _, _, in, out := state.RemoteClient.DataRate() log.Info("Total remote RPC data in: %d out: %d", in, out) } - state.CloseResults() metrics.Push(config.Metrics, config.IsRemoteExecution()) } From 0e05f0f655a6bb8027bbbd5b0fb163337443d029 Mon Sep 17 00:00:00 2001 From: DuBento Date: Tue, 9 Jun 2026 14:58:45 +0100 Subject: [PATCH 079/118] test: use system cache for speeding up e2e tests --- test/export/BUILD | 8 ++++++-- test/export/please_export_e2e_test.build_defs | 4 +++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/test/export/BUILD b/test/export/BUILD index 6223161562..9b6c7f3125 100644 --- a/test/export/BUILD +++ b/test/export/BUILD @@ -9,6 +9,10 @@ filegroup( # Generic catch-all test on internal repo. plz_e2e_test( name = "export_src_please_test", - cmd = f'plz export --output "$PLZ_EXPORT_DIR" //src/core:core && plz --repo_root="$PLZ_EXPORT_DIR" build //src/core:core', - pre_cmd = f'export PLZ_EXPORT_DIR="$(mktemp -d)"', + cmd = " && ".join([ + 'CACHE_CONF="--override=cache.dir:$(plz query reporoot)/plz-out/test-cache"', + 'plz "$CACHE_CONF" export --output "$PLZ_EXPORT_DIR" //src/core:core', + 'plz "$CACHE_CONF" --repo_root="$PLZ_EXPORT_DIR" build //src/core:core', + ]), + pre_cmd = 'PLZ_EXPORT_DIR="$(mktemp -d)"', ) diff --git a/test/export/please_export_e2e_test.build_defs b/test/export/please_export_e2e_test.build_defs index 05e6bde2e5..87768c82ad 100644 --- a/test/export/please_export_e2e_test.build_defs +++ b/test/export/please_export_e2e_test.build_defs @@ -53,7 +53,9 @@ def please_export_e2e_test( f'plz --repo_root="{exported_repo}" build //... > /dev/null', ] + [f'plz --repo_root="{exported_repo}" {cmd}' for cmd in cmd_on_export] - test_cmd = [cmd.replace("plz ", "$TOOLS_PLEASE ") for cmd in test_cmd] + cache_args = '--override=cache.dir:"$($TOOLS_PLEASE query reporoot)"/plz-out/test-cache' + + test_cmd = [cmd.replace("plz ", f"$TOOLS_PLEASE {cache_args} ") for cmd in test_cmd] test_cmd = " && ".join(test_cmd) data["SOURCE_REPO"] = [source_repo] From 8de64e3cf1b55baeb76215ce8bab19ba4e09d69c Mon Sep 17 00:00:00 2001 From: DuBento Date: Tue, 9 Jun 2026 16:02:06 +0100 Subject: [PATCH 080/118] add explicit targetToStmt mapping to improve CPU performance on FindStatement and RelatedTargets methods --- src/core/package_metadata.go | 54 +++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/src/core/package_metadata.go b/src/core/package_metadata.go index 444f2feeee..f01227e55f 100644 --- a/src/core/package_metadata.go +++ b/src/core/package_metadata.go @@ -1,7 +1,6 @@ package core import ( - "slices" "sync" ) @@ -44,9 +43,9 @@ type PackageMetadata interface { // RegisterStatementTarget records that the given build target was created as a result of the // given statement being executed. This should only be called for statements in BUILD files. RegisterStatementTarget(target *BuildTarget, stmtProvider BuildStatementProvider) - // RegisterRequiredSubinclude records that the given build target requires the given subinclude + // RegisterRequiredSubinclude records that the given build target requires the given subincluded // labels to be built. This is used to track which subinclude statements contributed to a target's - // definition. This should only be called for statements in BUILD files. + // definition. RegisterRequiredSubinclude(target *BuildTarget, labelProvider SubincludesLabelProvider) // RegisterSubincludeStatement records that the given subinclude statement (provided by stmtProvider) // includes the given build label. This should only be called for statements in BUILD files. @@ -69,21 +68,25 @@ type PackageMetadata interface { GetSubincludedLabels(stmt *BuildStatement) BuildLabels } -// packageMetadataImpl is the default implementation of the PackageMetadata interface. +// packageMetadataImpl is the canonical implementation of the PackageMetadata interface. // It uses in-memory maps to track the relationships between BUILD file statements, // subincludes, and the build targets they define. type packageMetadataImpl struct { - // StmtToTarget maps each build statement (identified by its byte range in a BUILD file) + // stmtToTarget maps each build statement (identified by its byte range in a BUILD file) // to the targets it produced. Since a single statement (like a custom target or loop) // can produce multiple targets, this is a one-to-many mapping. - StmtToTarget map[BuildStatement][]*BuildTarget - // TargetToSubinclude tracks the subinclude labels that were required for a target's + stmtToTarget map[BuildStatement][]*BuildTarget + // targetToStmt serves as a reverse-lookup map, linking each generated *BuildTarget + // back to the specific BuildStatement that declared it. This is useful for tracing back a target + // to its statement and to find sibling targets generated by the same statement block. + targetToStmt map[*BuildTarget]BuildStatement + // targetToRequiredSubincludes tracks the subinclude labels that were required for a target's // definition. This allows mapping a target back to the subincluded labels required for building // the target. - TargetToSubinclude map[*BuildTarget]BuildLabels - // LabelsPerSubincludeStmt maps a subinclude statement (identified by its position + targetToRequiredSubincludes map[*BuildTarget]BuildLabels + // labelsPerSubincludeStmt maps a subinclude statement (identified by its position // in the BUILD file) to the labels it explicitly subincludes. - LabelsPerSubincludeStmt map[BuildStatement]BuildLabels + labelsPerSubincludeStmt map[BuildStatement]BuildLabels // mutex protects concurrent access to the metadata maps during the parallel // parsing of BUILD files. @@ -92,9 +95,10 @@ type packageMetadataImpl struct { func newPackageMetadata() PackageMetadata { return &packageMetadataImpl{ - StmtToTarget: map[BuildStatement][]*BuildTarget{}, - TargetToSubinclude: map[*BuildTarget]BuildLabels{}, - LabelsPerSubincludeStmt: map[BuildStatement]BuildLabels{}, + stmtToTarget: map[BuildStatement][]*BuildTarget{}, + targetToStmt: map[*BuildTarget]BuildStatement{}, + targetToRequiredSubincludes: map[*BuildTarget]BuildLabels{}, + labelsPerSubincludeStmt: map[BuildStatement]BuildLabels{}, } } @@ -103,7 +107,8 @@ func (m *packageMetadataImpl) RegisterStatementTarget(target *BuildTarget, stmtP m.mutex.Lock() defer m.mutex.Unlock() - m.StmtToTarget[stmt] = append(m.StmtToTarget[stmt], target) + m.stmtToTarget[stmt] = append(m.stmtToTarget[stmt], target) + m.targetToStmt[target] = stmt } func (m *packageMetadataImpl) RegisterRequiredSubinclude(target *BuildTarget, labelProvider SubincludesLabelProvider) { @@ -111,7 +116,7 @@ func (m *packageMetadataImpl) RegisterRequiredSubinclude(target *BuildTarget, la m.mutex.Lock() defer m.mutex.Unlock() - m.TargetToSubinclude[target] = append(m.TargetToSubinclude[target], labels...) + m.targetToRequiredSubincludes[target] = append(m.targetToRequiredSubincludes[target], labels...) } func (m *packageMetadataImpl) RegisterSubincludeStatement(label BuildLabel, stmtProvider BuildStatementProvider) { @@ -119,7 +124,7 @@ func (m *packageMetadataImpl) RegisterSubincludeStatement(label BuildLabel, stmt m.mutex.Lock() defer m.mutex.Unlock() - m.LabelsPerSubincludeStmt[stmt] = append(m.LabelsPerSubincludeStmt[stmt], label) + m.labelsPerSubincludeStmt[stmt] = append(m.labelsPerSubincludeStmt[stmt], label) } func (m *packageMetadataImpl) FindStatement(target *BuildTarget) *BuildStatement { @@ -130,10 +135,8 @@ func (m *packageMetadataImpl) FindStatement(target *BuildTarget) *BuildStatement m.mutex.RLock() defer m.mutex.RUnlock() - for stmt, targets := range m.StmtToTarget { - if slices.Contains(targets, target) { - return &stmt - } + if stmt, present := m.targetToStmt[target]; present { + return &stmt } log.Debugf("Failed to find statement for target %s", target) return nil @@ -147,7 +150,7 @@ func (m *packageMetadataImpl) FindTargets(stmt *BuildStatement) []*BuildTarget { m.mutex.RLock() defer m.mutex.RUnlock() - return m.StmtToTarget[*stmt] + return m.stmtToTarget[*stmt] } func (m *packageMetadataImpl) FindRequiredSubincludes(target *BuildTarget) BuildLabels { @@ -158,7 +161,7 @@ func (m *packageMetadataImpl) FindRequiredSubincludes(target *BuildTarget) Build m.mutex.RLock() defer m.mutex.RUnlock() - return m.TargetToSubinclude[target] + return m.targetToRequiredSubincludes[target] } func (m *packageMetadataImpl) FindRelatedTargets(target *BuildTarget) BuildLabels { @@ -181,11 +184,12 @@ func (m *packageMetadataImpl) GetSubincludedLabels(stmt *BuildStatement) BuildLa m.mutex.RLock() defer m.mutex.RUnlock() - return m.LabelsPerSubincludeStmt[*stmt] + return m.labelsPerSubincludeStmt[*stmt] } -// noopPackageMetadata implements the PackageMetadata interface with no-op methods. This is used to -// avoid the overhead of parsing metadata for operations that don't depend on it. +// noopPackageMetadata implements the PackageMetadata interface with no-op methods. This is the +// default implementation and is used to avoid the overhead of parsing metadata for operations that +// don't depend on it. type noopPackageMetadata struct{} func newNoopPackageMetadata() PackageMetadata { From 5ca4401d58a5f2dbe01ef5d2fbd2b3d1232d0120 Mon Sep 17 00:00:00 2001 From: DuBento Date: Tue, 9 Jun 2026 16:32:07 +0100 Subject: [PATCH 081/118] use cmap in package metadata, removing explicit usage of a shared mutex for all maps --- src/core/build_target.go | 5 +++ src/core/package_metadata.go | 78 ++++++++++++++---------------------- 2 files changed, 36 insertions(+), 47 deletions(-) diff --git a/src/core/build_target.go b/src/core/build_target.go index 3268e55f03..440a807754 100644 --- a/src/core/build_target.go +++ b/src/core/build_target.go @@ -416,6 +416,11 @@ func (target *BuildTarget) String() string { return target.Label.String() } +// hashBuildTarget returns a mostly unique hash of a BuildTarget by relying on the BuildLabel hash. +func hashBuildTarget(target *BuildTarget) uint64 { + return hashBuildLabel(target.Label) +} + // TmpDir returns the temporary working directory for this target, eg. // //mickey/donald:goofy -> plz-out/tmp/mickey/donald/goofy._build // Note the extra subdirectory to keep rules separate from one another, and the .build suffix diff --git a/src/core/package_metadata.go b/src/core/package_metadata.go index f01227e55f..06f60c8323 100644 --- a/src/core/package_metadata.go +++ b/src/core/package_metadata.go @@ -1,7 +1,7 @@ package core import ( - "sync" + "github.com/thought-machine/please/src/cmap" ) // BuildStatement represents the start and end byte positions of a parsed statement in a BUILD file. @@ -19,6 +19,11 @@ func (bs *BuildStatement) StartPos() int64 { return int64(bs.Start) } +// hashBuildStatement mixes the Start and End byte coordinates to produce a unique 64-bit hash. +func hashBuildStatement(stmt BuildStatement) uint64 { + return uint64(stmt.Start)<<32 | uint64(stmt.End) +} + // BuildStatements is a slice of BuildStatement that implements sort.Interface. type BuildStatements []BuildStatement @@ -55,7 +60,7 @@ type PackageMetadata interface { FindStatement(target *BuildTarget) *BuildStatement // FindTargets returns all build targets that were generated by the given build statement. // Returns an empty slice if no targets were found for the given statement. - FindTargets(stmt *BuildStatement) []*BuildTarget + FindTargets(stmt *BuildStatement) BuildTargets // FindRequiredSubincludes returns all subinclude labels that were required by the given target. // The return value is empty if no subinclude information was found for the target. FindRequiredSubincludes(target *BuildTarget) BuildLabels @@ -69,62 +74,52 @@ type PackageMetadata interface { } // packageMetadataImpl is the canonical implementation of the PackageMetadata interface. -// It uses in-memory maps to track the relationships between BUILD file statements, -// subincludes, and the build targets they define. +// It uses sharded concurrent maps (cmap.Map) to track the relationships between BUILD file statements, +// subincludes, and the build targets they define without the contention of a global read-write lock. type packageMetadataImpl struct { // stmtToTarget maps each build statement (identified by its byte range in a BUILD file) // to the targets it produced. Since a single statement (like a custom target or loop) // can produce multiple targets, this is a one-to-many mapping. - stmtToTarget map[BuildStatement][]*BuildTarget + stmtToTarget *cmap.Map[BuildStatement, BuildTargets] // targetToStmt serves as a reverse-lookup map, linking each generated *BuildTarget // back to the specific BuildStatement that declared it. This is useful for tracing back a target // to its statement and to find sibling targets generated by the same statement block. - targetToStmt map[*BuildTarget]BuildStatement + targetToStmt *cmap.Map[*BuildTarget, BuildStatement] // targetToRequiredSubincludes tracks the subinclude labels that were required for a target's // definition. This allows mapping a target back to the subincluded labels required for building // the target. - targetToRequiredSubincludes map[*BuildTarget]BuildLabels + targetToRequiredSubincludes *cmap.Map[*BuildTarget, BuildLabels] // labelsPerSubincludeStmt maps a subinclude statement (identified by its position // in the BUILD file) to the labels it explicitly subincludes. - labelsPerSubincludeStmt map[BuildStatement]BuildLabels - - // mutex protects concurrent access to the metadata maps during the parallel - // parsing of BUILD files. - mutex sync.RWMutex + labelsPerSubincludeStmt *cmap.Map[BuildStatement, BuildLabels] } func newPackageMetadata() PackageMetadata { return &packageMetadataImpl{ - stmtToTarget: map[BuildStatement][]*BuildTarget{}, - targetToStmt: map[*BuildTarget]BuildStatement{}, - targetToRequiredSubincludes: map[*BuildTarget]BuildLabels{}, - labelsPerSubincludeStmt: map[BuildStatement]BuildLabels{}, + stmtToTarget: cmap.New[BuildStatement, BuildTargets](cmap.SmallShardCount, hashBuildStatement), + targetToStmt: cmap.New[*BuildTarget, BuildStatement](cmap.SmallShardCount, hashBuildTarget), + targetToRequiredSubincludes: cmap.New[*BuildTarget, BuildLabels](cmap.SmallShardCount, hashBuildTarget), + labelsPerSubincludeStmt: cmap.New[BuildStatement, BuildLabels](cmap.SmallShardCount, hashBuildStatement), } } func (m *packageMetadataImpl) RegisterStatementTarget(target *BuildTarget, stmtProvider BuildStatementProvider) { stmt := stmtProvider() - - m.mutex.Lock() - defer m.mutex.Unlock() - m.stmtToTarget[stmt] = append(m.stmtToTarget[stmt], target) - m.targetToStmt[target] = stmt + targets := m.stmtToTarget.Get(stmt) + m.stmtToTarget.Set(stmt, append(targets, target)) + m.targetToStmt.Set(target, stmt) } func (m *packageMetadataImpl) RegisterRequiredSubinclude(target *BuildTarget, labelProvider SubincludesLabelProvider) { labels := labelProvider() - - m.mutex.Lock() - defer m.mutex.Unlock() - m.targetToRequiredSubincludes[target] = append(m.targetToRequiredSubincludes[target], labels...) + existing := m.targetToRequiredSubincludes.Get(target) + m.targetToRequiredSubincludes.Set(target, append(existing, labels...)) } func (m *packageMetadataImpl) RegisterSubincludeStatement(label BuildLabel, stmtProvider BuildStatementProvider) { stmt := stmtProvider() - - m.mutex.Lock() - defer m.mutex.Unlock() - m.labelsPerSubincludeStmt[stmt] = append(m.labelsPerSubincludeStmt[stmt], label) + existing := m.labelsPerSubincludeStmt.Get(stmt) + m.labelsPerSubincludeStmt.Set(stmt, append(existing, label)) } func (m *packageMetadataImpl) FindStatement(target *BuildTarget) *BuildStatement { @@ -132,25 +127,20 @@ func (m *packageMetadataImpl) FindStatement(target *BuildTarget) *BuildStatement return nil } - m.mutex.RLock() - defer m.mutex.RUnlock() - - if stmt, present := m.targetToStmt[target]; present { + if m.targetToStmt.Contains(target) { + stmt := m.targetToStmt.Get(target) return &stmt } log.Debugf("Failed to find statement for target %s", target) return nil } -func (m *packageMetadataImpl) FindTargets(stmt *BuildStatement) []*BuildTarget { +func (m *packageMetadataImpl) FindTargets(stmt *BuildStatement) BuildTargets { if stmt == nil { return nil } - m.mutex.RLock() - defer m.mutex.RUnlock() - - return m.stmtToTarget[*stmt] + return m.stmtToTarget.Get(*stmt) } func (m *packageMetadataImpl) FindRequiredSubincludes(target *BuildTarget) BuildLabels { @@ -158,10 +148,7 @@ func (m *packageMetadataImpl) FindRequiredSubincludes(target *BuildTarget) Build return nil } - m.mutex.RLock() - defer m.mutex.RUnlock() - - return m.targetToRequiredSubincludes[target] + return m.targetToRequiredSubincludes.Get(target) } func (m *packageMetadataImpl) FindRelatedTargets(target *BuildTarget) BuildLabels { @@ -181,10 +168,7 @@ func (m *packageMetadataImpl) GetSubincludedLabels(stmt *BuildStatement) BuildLa return nil } - m.mutex.RLock() - defer m.mutex.RUnlock() - - return m.labelsPerSubincludeStmt[*stmt] + return m.labelsPerSubincludeStmt.Get(*stmt) } // noopPackageMetadata implements the PackageMetadata interface with no-op methods. This is the @@ -206,7 +190,7 @@ func (n *noopPackageMetadata) FindStatement(target *BuildTarget) *BuildStatement log.Fatalf("Metadata not tracked, using no-op implementation.") return nil } -func (n *noopPackageMetadata) FindTargets(stmt *BuildStatement) []*BuildTarget { +func (n *noopPackageMetadata) FindTargets(stmt *BuildStatement) BuildTargets { log.Fatalf("Metadata not tracked, using no-op implementation.") return nil } From 6b9f0c463943df780cb37be629d04bde9154e75d Mon Sep 17 00:00:00 2001 From: DuBento Date: Tue, 9 Jun 2026 16:44:12 +0100 Subject: [PATCH 082/118] docs: interface implementation comment --- src/core/package_metadata.go | 25 ++++++++++++++++++++++++- src/export/export_notrim.go | 3 +++ src/export/export_trimmed.go | 3 +++ src/parse/asp/interpreter.go | 32 ++++++++++++++++++++++++++------ 4 files changed, 56 insertions(+), 7 deletions(-) diff --git a/src/core/package_metadata.go b/src/core/package_metadata.go index 06f60c8323..a1a1e1039c 100644 --- a/src/core/package_metadata.go +++ b/src/core/package_metadata.go @@ -103,6 +103,7 @@ func newPackageMetadata() PackageMetadata { } } +// RegisterStatementTarget implements [PackageMetadata]. func (m *packageMetadataImpl) RegisterStatementTarget(target *BuildTarget, stmtProvider BuildStatementProvider) { stmt := stmtProvider() targets := m.stmtToTarget.Get(stmt) @@ -110,18 +111,21 @@ func (m *packageMetadataImpl) RegisterStatementTarget(target *BuildTarget, stmtP m.targetToStmt.Set(target, stmt) } +// RegisterRequiredSubinclude implements [PackageMetadata]. func (m *packageMetadataImpl) RegisterRequiredSubinclude(target *BuildTarget, labelProvider SubincludesLabelProvider) { labels := labelProvider() existing := m.targetToRequiredSubincludes.Get(target) m.targetToRequiredSubincludes.Set(target, append(existing, labels...)) } +// RegisterSubincludeStatement implements [PackageMetadata]. func (m *packageMetadataImpl) RegisterSubincludeStatement(label BuildLabel, stmtProvider BuildStatementProvider) { stmt := stmtProvider() existing := m.labelsPerSubincludeStmt.Get(stmt) m.labelsPerSubincludeStmt.Set(stmt, append(existing, label)) } +// FindStatement implements [PackageMetadata]. func (m *packageMetadataImpl) FindStatement(target *BuildTarget) *BuildStatement { if target == nil { return nil @@ -135,6 +139,7 @@ func (m *packageMetadataImpl) FindStatement(target *BuildTarget) *BuildStatement return nil } +// FindTargets implements [PackageMetadata]. func (m *packageMetadataImpl) FindTargets(stmt *BuildStatement) BuildTargets { if stmt == nil { return nil @@ -143,6 +148,7 @@ func (m *packageMetadataImpl) FindTargets(stmt *BuildStatement) BuildTargets { return m.stmtToTarget.Get(*stmt) } +// FindRequiredSubincludes implements [PackageMetadata]. func (m *packageMetadataImpl) FindRequiredSubincludes(target *BuildTarget) BuildLabels { if target == nil { return nil @@ -151,6 +157,7 @@ func (m *packageMetadataImpl) FindRequiredSubincludes(target *BuildTarget) Build return m.targetToRequiredSubincludes.Get(target) } +// FindRelatedTargets implements [PackageMetadata]. func (m *packageMetadataImpl) FindRelatedTargets(target *BuildTarget) BuildLabels { stmt := m.FindStatement(target) relatedTargets := m.FindTargets(stmt) @@ -163,6 +170,7 @@ func (m *packageMetadataImpl) FindRelatedTargets(target *BuildTarget) BuildLabel return labels } +// GetSubincludedLabels implements [PackageMetadata]. func (m *packageMetadataImpl) GetSubincludedLabels(stmt *BuildStatement) BuildLabels { if stmt == nil { return nil @@ -180,27 +188,42 @@ func newNoopPackageMetadata() PackageMetadata { return &noopPackageMetadata{} } +// RegisterStatementTarget implements [PackageMetadata]. func (n *noopPackageMetadata) RegisterStatementTarget(target *BuildTarget, stmtProvider BuildStatementProvider) { } + +// RegisterRequiredSubinclude implements [PackageMetadata]. func (n *noopPackageMetadata) RegisterRequiredSubinclude(target *BuildTarget, labelProvider SubincludesLabelProvider) { } + +// RegisterSubincludeStatement implements [PackageMetadata]. func (n *noopPackageMetadata) RegisterSubincludeStatement(label BuildLabel, stmtProvider BuildStatementProvider) { } + +// FindStatement implements [PackageMetadata]. func (n *noopPackageMetadata) FindStatement(target *BuildTarget) *BuildStatement { log.Fatalf("Metadata not tracked, using no-op implementation.") return nil } + +// FindTargets implements [PackageMetadata]. func (n *noopPackageMetadata) FindTargets(stmt *BuildStatement) BuildTargets { log.Fatalf("Metadata not tracked, using no-op implementation.") return nil } + +// FindRequiredSubincludes implements [PackageMetadata]. func (n *noopPackageMetadata) FindRequiredSubincludes(target *BuildTarget) BuildLabels { log.Fatalf("Metadata not tracked, using no-op implementation.") return nil } -func (m *noopPackageMetadata) FindRelatedTargets(target *BuildTarget) BuildLabels { + +// FindRelatedTargets implements [PackageMetadata]. +func (n *noopPackageMetadata) FindRelatedTargets(target *BuildTarget) BuildLabels { return nil } + +// GetSubincludedLabels implements [PackageMetadata]. func (n *noopPackageMetadata) GetSubincludedLabels(stmt *BuildStatement) BuildLabels { log.Fatalf("Metadata not tracked, using no-op implementation.") return nil diff --git a/src/export/export_notrim.go b/src/export/export_notrim.go index 2ee7478ca1..453c96222c 100644 --- a/src/export/export_notrim.go +++ b/src/export/export_notrim.go @@ -16,6 +16,7 @@ type noTrimExporter struct { exportedPackages map[string]bool } +// ExportPreloaded implements [Exporter]. func (nte *noTrimExporter) ExportPreloaded() { // Write any preloaded build defs for _, preload := range nte.state.Config.Parse.PreloadBuildDefs { @@ -30,6 +31,7 @@ func (nte *noTrimExporter) ExportPreloaded() { } } +// ExportTarget implements [Exporter]. func (nte *noTrimExporter) ExportTarget(target *core.BuildTarget) { pkg := nte.state.Graph.PackageOrDie(target.Label) if !nte.checkAndSetVisited(target) { @@ -51,6 +53,7 @@ func (nte *noTrimExporter) ExportTarget(target *core.BuildTarget) { nte.exportDependencies(target) } +// WritePackageFiles implements [Exporter]. func (nte *noTrimExporter) WritePackageFiles() { } diff --git a/src/export/export_trimmed.go b/src/export/export_trimmed.go index cc88e9d257..a3979f959c 100644 --- a/src/export/export_trimmed.go +++ b/src/export/export_trimmed.go @@ -24,6 +24,7 @@ type defaultExporter struct { preloadedSubincludes map[core.BuildLabel]bool } +// ExportPreloaded implements [Exporter]. func (e *defaultExporter) ExportPreloaded() { // Write any preloaded build defs for _, preload := range e.state.Config.Parse.PreloadBuildDefs { @@ -41,6 +42,7 @@ func (e *defaultExporter) ExportPreloaded() { } } +// ExportTarget implements [Exporter]. func (e *defaultExporter) ExportTarget(target *core.BuildTarget) { if !e.checkAndSetVisited(target) { return @@ -69,6 +71,7 @@ func (e *defaultExporter) ExportTarget(target *core.BuildTarget) { e.visitedPackages[pkg.Label()] = true } +// WritePackageFiles implements [Exporter]. func (e *defaultExporter) WritePackageFiles() { p := asp.NewParserOnly() for pkgLabel := range e.visitedPackages { diff --git a/src/parse/asp/interpreter.go b/src/parse/asp/interpreter.go index e74f093322..4f783e963a 100644 --- a/src/parse/asp/interpreter.go +++ b/src/parse/asp/interpreter.go @@ -1198,6 +1198,7 @@ type scopeMetadata struct { requiredOrigins map[core.BuildLabel]struct{} } +// NewMetadata implements [ScopeMetadata]. func (m *scopeMetadata) NewMetadata() ScopeMetadata { return &scopeMetadata{ cursor: m.cursor, @@ -1206,10 +1207,12 @@ func (m *scopeMetadata) NewMetadata() ScopeMetadata { } } +// Cursor implements [ScopeMetadata]. func (m *scopeMetadata) Cursor() *Statement { return m.cursor } +// Origin implements [ScopeMetadata]. func (m *scopeMetadata) Origin(scope *scope, name string) *core.BuildLabel { if scope.interpreter != nil && scope.interpreter.preloaded.Contains(name) { return nil @@ -1220,18 +1223,22 @@ func (m *scopeMetadata) Origin(scope *scope, name string) *core.BuildLabel { return nil } +// RequiredOrigins implements [ScopeMetadata]. func (m *scopeMetadata) RequiredOrigins() map[core.BuildLabel]struct{} { return m.requiredOrigins } +// SetCursor implements [ScopeMetadata]. func (m *scopeMetadata) SetCursor(stmt *Statement) { m.cursor = stmt } +// SetObjectOrigin implements [ScopeMetadata]. func (m *scopeMetadata) SetObjectOrigin(name string, origin core.BuildLabel) { m.objectOrigins[name] = origin } +// SetRequiredOrigin implements [ScopeMetadata]. func (m *scopeMetadata) SetRequiredOrigin(origin *core.BuildLabel) { if origin == nil { return @@ -1243,13 +1250,26 @@ func (m *scopeMetadata) SetRequiredOrigin(origin *core.BuildLabel) { // avoid the overhead of storing metadata for operations that don't depend on it. type noopScopeMetadata struct{} -func (nm *noopScopeMetadata) NewMetadata() ScopeMetadata { return &noopScopeMetadata{} } -func (nm *noopScopeMetadata) Cursor() *Statement { return nil } -func (nm *noopScopeMetadata) Origin(scope *scope, name string) *core.BuildLabel { return nil } -func (nm *noopScopeMetadata) RequiredOrigins() map[core.BuildLabel]struct{} { return nil } -func (nm *noopScopeMetadata) SetCursor(stmt *Statement) {} +// NewMetadata implements [ScopeMetadata]. +func (nm *noopScopeMetadata) NewMetadata() ScopeMetadata { return &noopScopeMetadata{} } + +// Cursor implements [ScopeMetadata]. +func (nm *noopScopeMetadata) Cursor() *Statement { return nil } + +// Origin implements [ScopeMetadata]. +func (nm *noopScopeMetadata) Origin(scope *scope, name string) *core.BuildLabel { return nil } + +// RequiredOrigins implements [ScopeMetadata]. +func (nm *noopScopeMetadata) RequiredOrigins() map[core.BuildLabel]struct{} { return nil } + +// SetCursor implements [ScopeMetadata]. +func (nm *noopScopeMetadata) SetCursor(stmt *Statement) {} + +// SetObjectOrigin implements [ScopeMetadata]. func (nm *noopScopeMetadata) SetObjectOrigin(name string, origin core.BuildLabel) {} -func (nm *noopScopeMetadata) SetRequiredOrigin(origin *core.BuildLabel) {} + +// SetRequiredOrigin implements [ScopeMetadata]. +func (nm *noopScopeMetadata) SetRequiredOrigin(origin *core.BuildLabel) {} // NewBuildStatement creates a new core.BuildStatement from an asp.statement. func NewBuildStatement(stmt *Statement) core.BuildStatement { From 299b25900541bf53310a29f4b3a800c525570dd5 Mon Sep 17 00:00:00 2001 From: DuBento Date: Wed, 10 Jun 2026 19:08:16 +0100 Subject: [PATCH 083/118] rename exporter source files --- src/export/{export_notrim.go => notrim_exporter.go} | 0 src/export/{export_trimmed.go => trimmed_exporter.go} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename src/export/{export_notrim.go => notrim_exporter.go} (100%) rename src/export/{export_trimmed.go => trimmed_exporter.go} (100%) diff --git a/src/export/export_notrim.go b/src/export/notrim_exporter.go similarity index 100% rename from src/export/export_notrim.go rename to src/export/notrim_exporter.go diff --git a/src/export/export_trimmed.go b/src/export/trimmed_exporter.go similarity index 100% rename from src/export/export_trimmed.go rename to src/export/trimmed_exporter.go From 21f37754b0e9c9685949a75f29a0530c7ce31731 Mon Sep 17 00:00:00 2001 From: DuBento Date: Wed, 10 Jun 2026 19:16:04 +0100 Subject: [PATCH 084/118] apply feedback from multiple comments - comments - rename structs, files and interfaces - package parsing - notrim testing --- src/BUILD.plz | 1 + src/core/state.go | 6 +- src/export/BUILD | 2 +- src/export/export.go | 90 ++++++++++++------- src/export/export_test.go | 12 +-- src/export/notrim_exporter.go | 62 ++++++++----- src/export/test_data/if_trim.build | 5 -- src/export/trimmed_exporter.go | 49 ++++++---- src/export/trimmer.go | 9 +- test/export/BUILD | 12 +++ .../test_for_if/expected_repo/BUILD_FILE | 2 +- .../test_for_if_both/expected_repo/BUILD_FILE | 2 +- test/export/test_if/expected_repo/BUILD_FILE | 2 +- 13 files changed, 164 insertions(+), 90 deletions(-) delete mode 100644 src/export/test_data/if_trim.build diff --git a/src/BUILD.plz b/src/BUILD.plz index 4ef5ea6fd3..cb6269400e 100644 --- a/src/BUILD.plz +++ b/src/BUILD.plz @@ -7,6 +7,7 @@ go_binary( "github.com/thought-machine/please/src/version.PleaseVersion": VERSION, }, visibility = ["PUBLIC"], + strip = False, deps = [ "///third_party/go/github.com_thought-machine_go-flags//:go-flags", "///third_party/go/go.uber.org_automaxprocs//maxprocs", diff --git a/src/core/state.go b/src/core/state.go index 2b30cbf1ef..1ed5136bcc 100644 --- a/src/core/state.go +++ b/src/core/state.go @@ -242,8 +242,10 @@ type BuildState struct { NeedDebugDeps bool // ParseMetadata is true if we want to store build file metadata ParseMetadata bool - // KeepParserRunning prevents closing task worker (parse and build) channels to support later - // calls to parser. + // KeepParserRunning prevents closing task workers (parse and build channels) to support later + // calls to the parser. This is needed to support the export operation since the export logic will + // attempt to export targets that have not been parsed during the normal build phase. An example + // is when exporting dependencies of targets that are not explicitly used but adjacent/related. KeepParserRunning bool // WaitForDisplay is a function that blocks until the display thread has finished. WaitForDisplay func() diff --git a/src/export/BUILD b/src/export/BUILD index ae3c9f13c6..0070a559b7 100644 --- a/src/export/BUILD +++ b/src/export/BUILD @@ -18,7 +18,7 @@ go_library( go_test( name = "export_test", - srcs = glob("*_test.go"), + srcs = ["export_test.go"], data = ["test_data"], deps = [ ":export", diff --git a/src/export/export.go b/src/export/export.go index 178d242f54..e64fc6b65a 100644 --- a/src/export/export.go +++ b/src/export/export.go @@ -6,6 +6,7 @@ package export import ( "os" "path/filepath" + "time" "github.com/thought-machine/please/src/cli/logging" "github.com/thought-machine/please/src/core" @@ -18,7 +19,7 @@ var log = logging.Log // Repo export a new please repo including the targets and dependencies requested. Depending on the // noTrim flag, the export will attempt to trim the resulting repository, exporting only the required // targets and build statements in their packages. If noTrim is set, all targets of a package will be -// exported and not build statement trimming will be attempted, the BUILD file is copied in its entirety. +// exported and no build statement trimming will be attempted, the BUILD file is copied in its entirety. func Repo(state *core.BuildState, dir string, noTrim bool, targets []core.BuildLabel) { e := newExporter(state, dir, noTrim) @@ -48,21 +49,21 @@ func Outputs(state *core.BuildState, dir string, targets []core.BuildLabel) { } } -// Exporter defines the interface for exporting parts of a Please repository to a new directory. +// exporterImpl defines the interface for exporting parts of a Please repository to a new directory. // It handles the copying of configuration files, preloaded build definitions, and selected // targets along with their necessary source files and dependencies. -type Exporter interface { - // ExportPreloaded exports all globally preloaded build definitions and subincluded targets. +type exporterImpl interface { + // exportPreloaded exports all globally preloaded build definitions and subincluded targets. // These are usually defined in the repository's configuration file. - ExportPreloaded() - // ExportTarget exports an individual build target. + exportPreloaded() + // exportTarget exports an individual build target. // Each target recursively exports all their source files and required build statements, but also // targets in their transitive dependencies. - ExportTarget(*core.BuildTarget) - // WritePackageFiles writes the processed BUILD files for all exported targets to the + exportTarget(*core.BuildTarget) + // writePackageFiles writes the processed BUILD files for all exported targets to the // export directory. These BUILD files may be modified (e.g., trimmed) depending on // the exporter's implementation. - WritePackageFiles() + writePackageFiles() } // newExporter creates a new exporter of a specific type based on the arguments. @@ -73,19 +74,11 @@ func newExporter(state *core.BuildState, dir string, noTrim bool) *baseExporter exportedTargets: map[core.BuildLabel]bool{}, } - var exporter Exporter + var exporter exporterImpl if noTrim { - exporter = &noTrimExporter{ - baseExporter: base, - exportedPackages: map[string]bool{}, - } + exporter = newNoTrimExporter(base) } else { - exporter = &defaultExporter{ - baseExporter: base, - visitedPackages: map[core.BuildLabel]bool{}, - requiredSubincludes: map[core.BuildLabel]core.BuildLabels{}, - preloadedSubincludes: map[core.BuildLabel]bool{}, - } + exporter = newTrimmedExporter(base) } base.impl = exporter @@ -94,22 +87,39 @@ func newExporter(state *core.BuildState, dir string, noTrim bool) *baseExporter // baseExporter provides common fields and methods of other exporters. type baseExporter struct { - state *core.BuildState - targetDir string + state *core.BuildState + targetDir string + targetCounter int // exportedTargets maintains a record of the targets that have been exported so far. exportedTargets map[core.BuildLabel]bool // impl is a reference to the concrete exporter implementation. It's included for calling the // specific exporter implementation from the common methods. - impl Exporter + impl exporterImpl } // run specifies the main steps when running an export. func (be *baseExporter) run(targets core.BuildLabels) { + go be.startMonitor() + go be.startStateMonitor() be.exportPlzConfig() - be.impl.ExportPreloaded() + be.impl.exportPreloaded() be.exportTargets(targets) - be.impl.WritePackageFiles() + be.impl.writePackageFiles() +} + +func (be *baseExporter) startMonitor() { + for { + time.Sleep(10 * time.Second) + log.Warningf("Number of targets exported: %v", be.targetCounter) + } +} + +func (be *baseExporter) startStateMonitor() { + for { + time.Sleep(1 * time.Minute) + log.Warningf("Targets in graph: %v", len(be.state.Graph.AllTargets())) + } } // exportPlzConfig exports the repository's configuration files (e.g., .plzconfig and its @@ -133,16 +143,19 @@ func (be *baseExporter) exportPlzConfig() { // exportTargets exports the set of targets identified by the given build labels. func (be *baseExporter) exportTargets(labels core.BuildLabels) { for _, l := range labels { + if be.exportedTargets[l] { + continue + } target := be.getOrParseTarget(l) if target == nil { log.Errorf("Unable to lookup target %s", l) continue } - be.impl.ExportTarget(target) + be.impl.exportTarget(target) } } -// exportDependencies exports exportDependencies of a target. +// exportDependencies exports dependencies of a target. func (be *baseExporter) exportDependencies(target *core.BuildTarget) { deps := target.DeclaredDependencies() log.Debugf("Exporting dependencies of (%v): %v", target.Label, deps) @@ -157,7 +170,7 @@ func (be *baseExporter) exportSources(target *core.BuildTarget) { } for _, p := range src.Paths(be.state.Graph) { if filepath.IsAbs(p) { // Don't copy system file deps. - log.Infof("System dependency detected, skipping...: %s", p) + log.Debugf("System dependency detected, skipping...: %s", p) continue } if err := fs.RecursiveCopy(p, filepath.Join(be.targetDir, p), 0); err != nil { @@ -168,10 +181,10 @@ func (be *baseExporter) exportSources(target *core.BuildTarget) { } } -// getOrParseTarget attempts to look up a target in the build graph. If the target has not +// getOrParseTarget attempts to lookup a target in the build graph. If the target has not // been parsed yet, it dynamically requests the package be parsed and blocks until the target is resolved. // -// This occurs in trimmed-mode exports when walking dependencies of adjacent targets which were not +// This occurs during the exports when walking dependencies of adjacent targets which were not // explicitly activated or resolved during the initial build/parse phase. // // This requires the background parser worker threads to be kept alive as daemons (controlled by the @@ -186,10 +199,27 @@ func (be *baseExporter) getOrParseTarget(label core.BuildLabel) *core.BuildTarge return target } +// getOrParsePackage attempts to lookup a package in the build graph. If the package has not +// been parsed yet, it dynamically requests the package be parsed and blocks until resolved. +// +// This requires the background parser worker threads to be kept alive as daemons (controlled by the +// "KeepParserRunning" build state option). +func (be *baseExporter) getOrParsePackage(label core.BuildLabel) *core.Package { + label.Name = "all" + pkg := be.state.Graph.PackageByLabel(label) + if pkg == nil { + log.Infof("Package %v not found in graph. Attempting to parse...", label) + parse.Parse(be.state, label, core.OriginalTarget, core.ParseModeNormal) + pkg = be.state.Graph.PackageByLabel(label) + } + return pkg +} + // checkAndSetVisited is a helper to ensure we only visit the same target once. // It returns true if this is the first time the target is being exported. func (be *baseExporter) checkAndSetVisited(target *core.BuildTarget) bool { visited := be.exportedTargets[target.Label] be.exportedTargets[target.Label] = true + be.targetCounter++ return !visited } diff --git a/src/export/export_test.go b/src/export/export_test.go index 3edede2cc3..4595f22490 100644 --- a/src/export/export_test.go +++ b/src/export/export_test.go @@ -50,7 +50,7 @@ func TestMinimalSubincludeStatement(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - e := newExporter(nil, "", false).impl.(*defaultExporter) + e := newExporter(nil, "", false).impl.(*trimmedExporter) pkg := &core.Package{Name: "test"} e.requiredSubincludes[pkg.Label()] = tc.requiredLabels @@ -104,7 +104,7 @@ func TestFilterPackageFile(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - e := newExporter(nil, "", false).impl.(*defaultExporter) + e := newExporter(nil, "", false).impl.(*trimmedExporter) for _, name := range tc.required { e.exportedTargets[targetLabels[name]] = true } @@ -169,11 +169,11 @@ else: required: []string{"b"}, expected: ` if False: - pass #Trimmed during export + pass # Trimmed during export elif True: genrule(name = "b") else: - pass #Trimmed during export + pass # Trimmed during export `}, { name: "Required target in for", @@ -209,7 +209,7 @@ for i in [ if i == "a": genrule(name = "a") elif i == "b": - pass #Trimmed during export + pass # Trimmed during export `}, } @@ -223,7 +223,7 @@ for i in [ pkg.Filename = "BUILD" targetLabels := walkASTRegisterTargets(t, statements, pkg, tc.registered) - e := newExporter(nil, "", false).impl.(*defaultExporter) + e := newExporter(nil, "", false).impl.(*trimmedExporter) for _, name := range tc.required { e.exportedTargets[targetLabels[name]] = true } diff --git a/src/export/notrim_exporter.go b/src/export/notrim_exporter.go index 453c96222c..c7cebe423e 100644 --- a/src/export/notrim_exporter.go +++ b/src/export/notrim_exporter.go @@ -13,11 +13,18 @@ import ( type noTrimExporter struct { *baseExporter // exportedPackages tracks which packages have already had their BUILD files exported. - exportedPackages map[string]bool + exportedPackages map[core.BuildLabel]bool } -// ExportPreloaded implements [Exporter]. -func (nte *noTrimExporter) ExportPreloaded() { +func newNoTrimExporter(base *baseExporter) exporterImpl { + return &noTrimExporter{ + baseExporter: base, + exportedPackages: map[core.BuildLabel]bool{}, + } +} + +// exportPreloaded implements [exporterImpl]. +func (nte *noTrimExporter) exportPreloaded() { // Write any preloaded build defs for _, preload := range nte.state.Config.Parse.PreloadBuildDefs { if err := fs.RecursiveCopy(preload, filepath.Join(nte.targetDir, preload), 0); err != nil { @@ -31,9 +38,14 @@ func (nte *noTrimExporter) ExportPreloaded() { } } -// ExportTarget implements [Exporter]. -func (nte *noTrimExporter) ExportTarget(target *core.BuildTarget) { - pkg := nte.state.Graph.PackageOrDie(target.Label) +// exportTarget implements [exporterImpl]. +func (nte *noTrimExporter) exportTarget(target *core.BuildTarget) { + pkg := nte.getOrParsePackage(target.Label) + if pkg == nil { + log.Errorf("Unable to lookup package %s", target.Label) + return + } + if !nte.checkAndSetVisited(target) { return } @@ -41,20 +53,31 @@ func (nte *noTrimExporter) ExportTarget(target *core.BuildTarget) { // We want to export the package that made this subrepo available, but we still need to walk the target deps // as it may depend on other subrepos or first party targets if target.Subrepo != nil { - nte.ExportTarget(target.Subrepo.Target) + nte.exportTarget(target.Subrepo.Target) nte.exportDependencies(target) return } - nte.exportPackage(pkg) nte.exportSubincludes(pkg) - nte.exportAllTargets(pkg) + nte.exportPackage(pkg) nte.exportSources(target) nte.exportDependencies(target) } -// WritePackageFiles implements [Exporter]. -func (nte *noTrimExporter) WritePackageFiles() { +// writePackageFiles implements [exporterImpl]. +func (nte *noTrimExporter) writePackageFiles() { + for pkgLabel := range nte.exportedPackages { + pkg := nte.getOrParsePackage(pkgLabel) + if pkg == nil { + log.Errorf("Unable to lookup package %s", pkgLabel) + continue + } + + exportedFilename := filepath.Join(nte.targetDir, pkg.Filename) + if err := fs.CopyFile(pkg.Filename, exportedFilename, 0); err != nil { + log.Errorf("failed to export package %s: %v", pkg.Name, err) + } + } } // exportPackage exports the package BUILD file. @@ -65,14 +88,14 @@ func (nte *noTrimExporter) exportPackage(pkg *core.Package) { return } - if nte.exportedPackages[pkg.Name] { + if nte.exportedPackages[pkg.Label()] { return } - nte.exportedPackages[pkg.Name] = true + nte.exportedPackages[pkg.Label()] = true - exportedFilename := filepath.Join(nte.targetDir, pkg.Filename) - if err := fs.CopyFile(pkg.Filename, exportedFilename, 0); err != nil { - log.Errorf("failed to export package %s: %v", pkg.Name, err) + // Export all the targets in the provided package. + for _, target := range pkg.AllTargets() { + nte.exportTarget(target) } } @@ -81,10 +104,3 @@ func (nte *noTrimExporter) exportSubincludes(pkg *core.Package) { subincludes := pkg.AllSubincludes(nte.state.Graph) nte.exportTargets(subincludes) } - -// exportAllTargets will export all the targets in the provided package. -func (nte *noTrimExporter) exportAllTargets(pkg *core.Package) { - for _, target := range pkg.AllTargets() { - nte.ExportTarget(target) - } -} diff --git a/src/export/test_data/if_trim.build b/src/export/test_data/if_trim.build deleted file mode 100644 index 14923efbdf..0000000000 --- a/src/export/test_data/if_trim.build +++ /dev/null @@ -1,5 +0,0 @@ -val = "a" - -if val == "a": - target(name = "a") -elif val == "b": diff --git a/src/export/trimmed_exporter.go b/src/export/trimmed_exporter.go index a3979f959c..72a309a0b0 100644 --- a/src/export/trimmed_exporter.go +++ b/src/export/trimmed_exporter.go @@ -13,8 +13,8 @@ import ( "github.com/thought-machine/please/src/parse/asp" ) -// defaultExporter implements an exporter that trims packages to reach a minimal exported repo. -type defaultExporter struct { +// trimmedExporter implements an exporter that trims packages to reach a minimal exported repo. +type trimmedExporter struct { *baseExporter // visitedPackages maintains a record of the packages visited during the export process. visitedPackages map[core.BuildLabel]bool @@ -24,8 +24,17 @@ type defaultExporter struct { preloadedSubincludes map[core.BuildLabel]bool } -// ExportPreloaded implements [Exporter]. -func (e *defaultExporter) ExportPreloaded() { +func newTrimmedExporter(base *baseExporter) exporterImpl { + return &trimmedExporter{ + baseExporter: base, + visitedPackages: map[core.BuildLabel]bool{}, + requiredSubincludes: map[core.BuildLabel]core.BuildLabels{}, + preloadedSubincludes: map[core.BuildLabel]bool{}, + } +} + +// exportPreloaded implements [exporterImpl]. +func (e *trimmedExporter) exportPreloaded() { // Write any preloaded build defs for _, preload := range e.state.Config.Parse.PreloadBuildDefs { if err := fs.RecursiveCopy(preload, filepath.Join(e.targetDir, preload), 0); err != nil { @@ -42,8 +51,8 @@ func (e *defaultExporter) ExportPreloaded() { } } -// ExportTarget implements [Exporter]. -func (e *defaultExporter) ExportTarget(target *core.BuildTarget) { +// exportTarget implements [exporterImpl]. +func (e *trimmedExporter) exportTarget(target *core.BuildTarget) { if !e.checkAndSetVisited(target) { return } @@ -57,7 +66,7 @@ func (e *defaultExporter) ExportTarget(target *core.BuildTarget) { // We want to export the package that made this subrepo available, but we still need to walk the // target deps as it may depend on other subrepos or first party targets if target.Subrepo != nil && target.Subrepo.Target != nil { - e.ExportTarget(target.Subrepo.Target) + e.exportTarget(target.Subrepo.Target) e.exportDependencies(target) return } @@ -65,14 +74,18 @@ func (e *defaultExporter) ExportTarget(target *core.BuildTarget) { e.exportSources(target) e.exportDependencies(target) - pkg := e.state.Graph.PackageOrDie(target.Label) + pkg := e.getOrParsePackage(target.Label) + if pkg == nil { + log.Errorf("Unable to lookup package %s", target.Label) + return + } e.exportSubincludes(pkg, target) e.exportRelatedTargets(pkg, target) e.visitedPackages[pkg.Label()] = true } -// WritePackageFiles implements [Exporter]. -func (e *defaultExporter) WritePackageFiles() { +// writePackageFiles implements [exporterImpl]. +func (e *trimmedExporter) writePackageFiles() { p := asp.NewParserOnly() for pkgLabel := range e.visitedPackages { pkg := e.state.Graph.PackageOrDie(pkgLabel) @@ -82,14 +95,13 @@ func (e *defaultExporter) WritePackageFiles() { continue } - filteredBytes = trimNewlines(filteredBytes) - e.writeExportedPackageFile(pkg, filteredBytes) + e.writeExportedPackageFile(pkg, trimNewlines(filteredBytes)) } } // exportSubincludes exports the subincluded targets required to generate the target and selects them to // later be written to the package as statements. -func (e *defaultExporter) exportSubincludes(pkg *core.Package, target *core.BuildTarget) { +func (e *trimmedExporter) exportSubincludes(pkg *core.Package, target *core.BuildTarget) { subincludes := pkg.Metadata.FindRequiredSubincludes(target) if len(subincludes) == 0 { return @@ -112,15 +124,18 @@ func (e *defaultExporter) exportSubincludes(pkg *core.Package, target *core.Buil e.exportTargets(subincludes) } -// exportRelatedTargets exports build targets that are related to the build statement that generated. -func (e *defaultExporter) exportRelatedTargets(pkg *core.Package, target *core.BuildTarget) { +// exportRelatedTargets looks up and exports all build targets that were declared within the same +// build statement (e.g., adjacent targets in build def) as the specified target. This ensures that +// all co-defined targets are preserved in the exported BUILD file, preventing unresolved references +// or partial declarations. +func (e *trimmedExporter) exportRelatedTargets(pkg *core.Package, target *core.BuildTarget) { relatedTargets := pkg.Metadata.FindRelatedTargets(target) log.Debugf("Exporting targets related to %s: %v", target, relatedTargets) e.exportTargets(relatedTargets) } // WriteExportedPackageFile creates a new package (BUILD) file in the exported dir and writes to it. -func (e *defaultExporter) writeExportedPackageFile(pkg *core.Package, content []byte) { +func (e *trimmedExporter) writeExportedPackageFile(pkg *core.Package, content []byte) { filename := pkg.Filename exportedFilename := filepath.Join(e.targetDir, filename) f, err := fs.OpenDirFile(exportedFilename, os.O_CREATE|os.O_WRONLY, 0664) @@ -135,7 +150,7 @@ func (e *defaultExporter) writeExportedPackageFile(pkg *core.Package, content [] } // trimPackage filters the statements to be written to the exported BUILD file. -func (e *defaultExporter) trimPackage(p *asp.Parser, pkg *core.Package) ([]byte, error) { +func (e *trimmedExporter) trimPackage(p *asp.Parser, pkg *core.Package) ([]byte, error) { parsed, err := p.ParseFileOnly(pkg.Filename) if err != nil { return nil, fmt.Errorf("Parsing original BUILD file: %v", err) diff --git a/src/export/trimmer.go b/src/export/trimmer.go index c6e22e7ab0..cdb9c55328 100644 --- a/src/export/trimmer.go +++ b/src/export/trimmer.go @@ -20,7 +20,7 @@ type trimmer struct { bytes []byte // exporter is used to lookup target related data from the export process, e.g. which targets are // required. - exporter *defaultExporter + exporter *trimmedExporter } // statementConsumer defines the type for methods used to visit each statement during a @@ -60,7 +60,7 @@ func (t *trimmer) trimBlock(stmts []*asp.Statement, blockStart, blockEnd asp.Pos } else if stmt.Ident != nil && stmt.Ident.Name == "subinclude" { t.trimSubinclude(stmt) } else if relatives := t.relatedTargets(stmt); len(relatives) > 0 { - // Meaning it is a build statement that generated/builds build targets. + // Meaning it is a build statement that creates build targets. if t.anyExported(relatives) { t.copy(stmt.Pos, stmt.EndPos) } @@ -154,7 +154,7 @@ func (t *trimmer) passBlock(stmts []*asp.Statement, blockStart, blockEnd asp.Pos // the "pass" primitive. This is useful when parsing inner blocks (e.g. if-else stmts). if !passWritten { passWritten = true - t.write([]byte("pass #Trimmed during export")) + t.write([]byte("pass # Trimmed during export")) } }) } @@ -225,6 +225,9 @@ func (t *trimmer) minimalSubincludeStatement(available core.BuildLabels) string } func (t *trimmer) copy(start, end asp.Position) { + if start < 0 || start > end || int(end) > len(t.origin) { + return + } t.bytes = append(t.bytes, t.origin[start:end]...) } diff --git a/test/export/BUILD b/test/export/BUILD index 9b6c7f3125..f4813d5d4c 100644 --- a/test/export/BUILD +++ b/test/export/BUILD @@ -16,3 +16,15 @@ plz_e2e_test( ]), pre_cmd = 'PLZ_EXPORT_DIR="$(mktemp -d)"', ) + + +# Generic catch-all test on internal repo - notrim. +plz_e2e_test( + name = "export_src_please_notrim_test", + cmd = " && ".join([ + 'CACHE_CONF="--override=cache.dir:$(plz query reporoot)/plz-out/test-cache"', + 'plz "$CACHE_CONF" export --output "$PLZ_EXPORT_DIR" --notrim //src/core:core', + 'plz "$CACHE_CONF" --repo_root="$PLZ_EXPORT_DIR" build //src/core:core', + ]), + pre_cmd = 'PLZ_EXPORT_DIR="$(mktemp -d)"', +) diff --git a/test/export/test_for_if/expected_repo/BUILD_FILE b/test/export/test_for_if/expected_repo/BUILD_FILE index 868064f63c..80279d2bb5 100644 --- a/test/export/test_for_if/expected_repo/BUILD_FILE +++ b/test/export/test_for_if/expected_repo/BUILD_FILE @@ -7,4 +7,4 @@ for i in ["a", "b"]: cmd = "cat $SRCS > $OUT", ) elif i == "b": - pass #Trimmed during export + pass # Trimmed during export diff --git a/test/export/test_for_if_both/expected_repo/BUILD_FILE b/test/export/test_for_if_both/expected_repo/BUILD_FILE index 7a1337ff98..73218736ba 100644 --- a/test/export/test_for_if_both/expected_repo/BUILD_FILE +++ b/test/export/test_for_if_both/expected_repo/BUILD_FILE @@ -17,4 +17,4 @@ for i in [ cmd = "cat $SRCS > $OUT", ) else: - pass #Trimmed during export + pass # Trimmed during export diff --git a/test/export/test_if/expected_repo/BUILD_FILE b/test/export/test_if/expected_repo/BUILD_FILE index 18e20bb15f..c69bf4c1c0 100644 --- a/test/export/test_if/expected_repo/BUILD_FILE +++ b/test/export/test_if/expected_repo/BUILD_FILE @@ -5,4 +5,4 @@ if True: cmd = "echo a > $OUT", ) else: - pass #Trimmed during export + pass # Trimmed during export From 681a664e3afb391f217f5d92de4fc87f99a3cb26 Mon Sep 17 00:00:00 2001 From: DuBento Date: Thu, 11 Jun 2026 18:58:43 +0100 Subject: [PATCH 085/118] local subrepos: fixes (relative filenames) and test --- src/core/build_target.go | 36 ++++++++++++++----- src/export/export.go | 14 +++----- src/export/notrim_exporter.go | 3 ++ src/export/trimmed_exporter.go | 16 ++++++--- test/export/test_subrepo_subtarget/BUILD | 8 +++++ .../expected_repo/.plzconfig | 2 ++ .../expected_repo/BUILD_FILE | 12 +++++++ .../expected_repo/file.txt | 1 + .../expected_repo/subrepo/BUILD_FILE | 5 +++ .../source_repo/.plzconfig | 2 ++ .../source_repo/BUILD_FILE | 17 +++++++++ .../source_repo/file.txt | 1 + .../source_repo/subrepo/BUILD_FILE | 11 ++++++ .../source_repo/unneeded.txt | 1 + 14 files changed, 107 insertions(+), 22 deletions(-) create mode 100644 test/export/test_subrepo_subtarget/BUILD create mode 100644 test/export/test_subrepo_subtarget/expected_repo/.plzconfig create mode 100644 test/export/test_subrepo_subtarget/expected_repo/BUILD_FILE create mode 100644 test/export/test_subrepo_subtarget/expected_repo/file.txt create mode 100644 test/export/test_subrepo_subtarget/expected_repo/subrepo/BUILD_FILE create mode 100644 test/export/test_subrepo_subtarget/source_repo/.plzconfig create mode 100644 test/export/test_subrepo_subtarget/source_repo/BUILD_FILE create mode 100644 test/export/test_subrepo_subtarget/source_repo/file.txt create mode 100644 test/export/test_subrepo_subtarget/source_repo/subrepo/BUILD_FILE create mode 100644 test/export/test_subrepo_subtarget/source_repo/unneeded.txt diff --git a/src/core/build_target.go b/src/core/build_target.go index 440a807754..82dd99e220 100644 --- a/src/core/build_target.go +++ b/src/core/build_target.go @@ -1476,7 +1476,7 @@ func (target *BuildTarget) AllTestTools() []BuildInput { if target.Test.namedTools == nil { return target.Test.tools } - return target.allBuildInputs(target.Test.tools, target.Test.namedTools) + return target.selectBuildInputs(target.Test.tools, target.Test.namedTools) } // NamedTestTools returns all named test tools @@ -1492,7 +1492,7 @@ func (target *BuildTarget) AllDebugTools() []BuildInput { if target.Debug.namedTools == nil { return target.Debug.tools } - return target.allBuildInputs(target.Debug.tools, target.Debug.namedTools) + return target.selectBuildInputs(target.Debug.tools, target.Debug.namedTools) } // AddDatum adds a new item of data to the target. @@ -1647,21 +1647,39 @@ func (target *BuildTarget) getCommand(state *BuildState, commands map[string]str return highestCommand } +// AllBuildInputs returns all the inputs for this target. +func (target *BuildTarget) AllBuildInputs() []BuildInput { + srcs := target.AllSources() + data := target.AllData() + tools := target.AllTools() + + size := len(srcs) + len(data) + len(tools) + inputs := make([]BuildInput, 0, size) + inputs = append(inputs, srcs...) + inputs = append(inputs, data...) + inputs = append(inputs, tools...) + return inputs +} + // AllSources returns all the sources of this rule. func (target *BuildTarget) AllSources() []BuildInput { if target.NamedSources == nil { return target.Sources } - return target.allBuildInputs(target.Sources, target.NamedSources) + return target.selectBuildInputs(target.Sources, target.NamedSources) } -func (target *BuildTarget) allBuildInputs(unnamed []BuildInput, named map[string][]BuildInput) []BuildInput { - ret := unnamed +func (target *BuildTarget) selectBuildInputs(unnamed []BuildInput, named map[string][]BuildInput) []BuildInput { keys := make([]string, 0, len(named)) - for k := range named { + size := len(unnamed) + for k, vals := range named { keys = append(keys, k) + size += len(vals) } sort.Strings(keys) + + ret := make([]BuildInput, 0, size) + ret = append(ret, unnamed...) for _, k := range keys { ret = append(ret, named[k]...) } @@ -1713,7 +1731,7 @@ func (target *BuildTarget) AllData() []BuildInput { return target.Data } - return target.allBuildInputs(target.Data, target.NamedData) + return target.selectBuildInputs(target.Data, target.NamedData) } // AllDebugData returns all the data for debugging this rule. @@ -1724,7 +1742,7 @@ func (target *BuildTarget) AllDebugData() []BuildInput { if target.Debug.namedData == nil { return target.Debug.data } - return target.allBuildInputs(target.Debug.data, target.Debug.namedData) + return target.selectBuildInputs(target.Debug.data, target.Debug.namedData) } // DebugData returns unnamed data for debugging this rule. @@ -1748,7 +1766,7 @@ func (target *BuildTarget) AllTools() []BuildInput { if target.namedTools == nil { return target.Tools } - return target.allBuildInputs(target.Tools, target.namedTools) + return target.selectBuildInputs(target.Tools, target.namedTools) } // ToolNames returns an ordered list of tool names. diff --git a/src/export/export.go b/src/export/export.go index e64fc6b65a..e6923a9adb 100644 --- a/src/export/export.go +++ b/src/export/export.go @@ -101,7 +101,6 @@ type baseExporter struct { // run specifies the main steps when running an export. func (be *baseExporter) run(targets core.BuildLabels) { go be.startMonitor() - go be.startStateMonitor() be.exportPlzConfig() be.impl.exportPreloaded() be.exportTargets(targets) @@ -115,13 +114,6 @@ func (be *baseExporter) startMonitor() { } } -func (be *baseExporter) startStateMonitor() { - for { - time.Sleep(1 * time.Minute) - log.Warningf("Targets in graph: %v", len(be.state.Graph.AllTargets())) - } -} - // exportPlzConfig exports the repository's configuration files (e.g., .plzconfig and its // platform-specific variants) to the target export directory. func (be *baseExporter) exportPlzConfig() { @@ -173,7 +165,11 @@ func (be *baseExporter) exportSources(target *core.BuildTarget) { log.Debugf("System dependency detected, skipping...: %s", p) continue } - if err := fs.RecursiveCopy(p, filepath.Join(be.targetDir, p), 0); err != nil { + dest := filepath.Join(be.targetDir, p) + if target.Subrepo != nil { // Adjusting fo for local subrepos + dest = filepath.Join(be.targetDir, target.Subrepo.Dir(p)) + } + if err := fs.RecursiveCopy(p, dest, 0); err != nil { log.Warningf("Error copying file, skipping...: %s", err) } log.Debugf("Writing exported source file: %s", p) diff --git a/src/export/notrim_exporter.go b/src/export/notrim_exporter.go index c7cebe423e..f3f4631228 100644 --- a/src/export/notrim_exporter.go +++ b/src/export/notrim_exporter.go @@ -74,6 +74,9 @@ func (nte *noTrimExporter) writePackageFiles() { } exportedFilename := filepath.Join(nte.targetDir, pkg.Filename) + if pkg.Subrepo != nil { + exportedFilename = filepath.Join(nte.targetDir, pkg.Subrepo.Dir(pkg.Filename)) + } if err := fs.CopyFile(pkg.Filename, exportedFilename, 0); err != nil { log.Errorf("failed to export package %s: %v", pkg.Name, err) } diff --git a/src/export/trimmed_exporter.go b/src/export/trimmed_exporter.go index 72a309a0b0..48056e1e2f 100644 --- a/src/export/trimmed_exporter.go +++ b/src/export/trimmed_exporter.go @@ -110,7 +110,7 @@ func (e *trimmedExporter) exportSubincludes(pkg *core.Package, target *core.Buil log.Debugf("Subincludes required for %s: %v", target, subincludes) for _, subinclude := range subincludes { // skip for preloaded subincludes, these are handled separately at the start to ensure they are - // they are exported even if not directly used by an exported target. + // exported even if not directly used by an exported target. if e.preloadedSubincludes[subinclude] { continue } @@ -138,7 +138,10 @@ func (e *trimmedExporter) exportRelatedTargets(pkg *core.Package, target *core.B func (e *trimmedExporter) writeExportedPackageFile(pkg *core.Package, content []byte) { filename := pkg.Filename exportedFilename := filepath.Join(e.targetDir, filename) - f, err := fs.OpenDirFile(exportedFilename, os.O_CREATE|os.O_WRONLY, 0664) + if pkg.Subrepo != nil { // Adjusting fo for local subrepos + exportedFilename = filepath.Join(e.targetDir, pkg.Subrepo.Dir(filename)) + } + f, err := fs.OpenDirFile(exportedFilename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0664) if err != nil { log.Fatalf("Failed to create and open exported BUILD file for %s: %v", exportedFilename, err) } @@ -151,12 +154,17 @@ func (e *trimmedExporter) writeExportedPackageFile(pkg *core.Package, content [] // trimPackage filters the statements to be written to the exported BUILD file. func (e *trimmedExporter) trimPackage(p *asp.Parser, pkg *core.Package) ([]byte, error) { - parsed, err := p.ParseFileOnly(pkg.Filename) + filename := pkg.Filename + if pkg.Subrepo != nil { // Adjusting fo for local subrepos + filename = pkg.Subrepo.Dir(filename) + } + + parsed, err := p.ParseFileOnly(filename) if err != nil { return nil, fmt.Errorf("Parsing original BUILD file: %v", err) } - content, err := os.ReadFile(pkg.Filename) + content, err := os.ReadFile(filename) if err != nil { return nil, fmt.Errorf("Opening original BUILD file: %v", err) } diff --git a/test/export/test_subrepo_subtarget/BUILD b/test/export/test_subrepo_subtarget/BUILD new file mode 100644 index 0000000000..90bbf0c233 --- /dev/null +++ b/test/export/test_subrepo_subtarget/BUILD @@ -0,0 +1,8 @@ +subinclude("//test/export:export_e2e_test_build_def") + +# Verifies that `plz export` correctly performs lookups to resolve internally generated sub-targets +# belonging to packages defined inside subrepos. +please_export_e2e_test( + name = "subrepo_subtarget_export_test", + export_targets = ["//:target"], +) diff --git a/test/export/test_subrepo_subtarget/expected_repo/.plzconfig b/test/export/test_subrepo_subtarget/expected_repo/.plzconfig new file mode 100644 index 0000000000..4d5174001a --- /dev/null +++ b/test/export/test_subrepo_subtarget/expected_repo/.plzconfig @@ -0,0 +1,2 @@ +[parse] +BuildFileName = BUILD_FILE \ No newline at end of file diff --git a/test/export/test_subrepo_subtarget/expected_repo/BUILD_FILE b/test/export/test_subrepo_subtarget/expected_repo/BUILD_FILE new file mode 100644 index 0000000000..fe636b2061 --- /dev/null +++ b/test/export/test_subrepo_subtarget/expected_repo/BUILD_FILE @@ -0,0 +1,12 @@ +subrepo( + name = "my_subrepo", + path = "subrepo", +) + +filegroup( + name = "target", + srcs = [ + "file.txt", + "///my_subrepo//:_foo#sub", + ], +) diff --git a/test/export/test_subrepo_subtarget/expected_repo/file.txt b/test/export/test_subrepo_subtarget/expected_repo/file.txt new file mode 100644 index 0000000000..95d09f2b10 --- /dev/null +++ b/test/export/test_subrepo_subtarget/expected_repo/file.txt @@ -0,0 +1 @@ +hello world \ No newline at end of file diff --git a/test/export/test_subrepo_subtarget/expected_repo/subrepo/BUILD_FILE b/test/export/test_subrepo_subtarget/expected_repo/subrepo/BUILD_FILE new file mode 100644 index 0000000000..7771998a58 --- /dev/null +++ b/test/export/test_subrepo_subtarget/expected_repo/subrepo/BUILD_FILE @@ -0,0 +1,5 @@ +filegroup( + name = "_foo#sub", + srcs = ["BUILD_FILE"], + visibility = ["PUBLIC"], +) diff --git a/test/export/test_subrepo_subtarget/source_repo/.plzconfig b/test/export/test_subrepo_subtarget/source_repo/.plzconfig new file mode 100644 index 0000000000..4d5174001a --- /dev/null +++ b/test/export/test_subrepo_subtarget/source_repo/.plzconfig @@ -0,0 +1,2 @@ +[parse] +BuildFileName = BUILD_FILE \ No newline at end of file diff --git a/test/export/test_subrepo_subtarget/source_repo/BUILD_FILE b/test/export/test_subrepo_subtarget/source_repo/BUILD_FILE new file mode 100644 index 0000000000..cfa1f38a46 --- /dev/null +++ b/test/export/test_subrepo_subtarget/source_repo/BUILD_FILE @@ -0,0 +1,17 @@ +subrepo( + name = "my_subrepo", + path = "subrepo", +) + +filegroup( + name = "target", + srcs = [ + "file.txt", + "///my_subrepo//:_foo#sub", + ], +) + +filegroup( + name = "unneeded", + srcs = ["unneeded.txt"], +) diff --git a/test/export/test_subrepo_subtarget/source_repo/file.txt b/test/export/test_subrepo_subtarget/source_repo/file.txt new file mode 100644 index 0000000000..95d09f2b10 --- /dev/null +++ b/test/export/test_subrepo_subtarget/source_repo/file.txt @@ -0,0 +1 @@ +hello world \ No newline at end of file diff --git a/test/export/test_subrepo_subtarget/source_repo/subrepo/BUILD_FILE b/test/export/test_subrepo_subtarget/source_repo/subrepo/BUILD_FILE new file mode 100644 index 0000000000..0ae27f18c3 --- /dev/null +++ b/test/export/test_subrepo_subtarget/source_repo/subrepo/BUILD_FILE @@ -0,0 +1,11 @@ +filegroup( + name = "package", + srcs = ["BUILD_FILE"], + visibility = ["PUBLIC"], +) + +filegroup( + name = "_foo#sub", + srcs = ["BUILD_FILE"], + visibility = ["PUBLIC"], +) \ No newline at end of file diff --git a/test/export/test_subrepo_subtarget/source_repo/unneeded.txt b/test/export/test_subrepo_subtarget/source_repo/unneeded.txt new file mode 100644 index 0000000000..9c1fe51193 --- /dev/null +++ b/test/export/test_subrepo_subtarget/source_repo/unneeded.txt @@ -0,0 +1 @@ +unneeded \ No newline at end of file From 1bea58f695eb75d2cd17019f52e8dba3bf60952e Mon Sep 17 00:00:00 2001 From: DuBento Date: Fri, 12 Jun 2026 10:47:28 +0100 Subject: [PATCH 086/118] export transitive subincludes, fixing unexported targets subincluded by build_defs + test cases --- src/export/export.go | 2 +- src/export/trimmed_exporter.go | 31 ++++++++++++++----- .../test_builtins/source_repo/BUILD_FILE | 6 ++-- .../test_custom_def/source_repo/BUILD_FILE | 4 +-- .../source_repo/build_defs/BUILD_FILE | 4 +-- .../source_repo/build_defs/dummy.build_defs | 4 +-- .../source_repo/BUILD_FILE | 6 ++-- test/export/test_deps/source_repo/BUILD_FILE | 4 +-- .../source_repo/other_deps/BUILD_FILE | 6 ++-- .../export/test_go_bin/source_repo/BUILD_FILE | 4 +-- test/export/test_go_bin/source_repo/dummy.go | 11 ------- .../source_repo/third_party/go/BUILD_FILE | 2 +- .../test_go_bin/source_repo/unneeded.go | 5 +++ .../source_repo/BUILD_FILE | 6 ++-- .../source_repo/BUILD_FILE | 6 ++-- .../source_repo/BUILD_FILE | 6 ++-- test/export/test_subinclude_preloaded/BUILD | 8 +++++ .../expected_repo/.plzconfig | 3 ++ .../expected_repo/BUILD_FILE | 6 ++++ .../expected_repo/build_defs/BUILD_FILE | 11 +++++++ .../build_defs/preloaded.build_defs | 1 + .../build_defs/subincluded.build_defs | 7 +++++ .../expected_repo/file.txt | 1 + .../source_repo/.plzconfig | 3 ++ .../source_repo/BUILD_FILE | 11 +++++++ .../source_repo/build_defs/BUILD_FILE | 11 +++++++ .../build_defs/preloaded.build_defs | 1 + .../build_defs/subincluded.build_defs | 7 +++++ .../source_repo/file.txt | 1 + .../source_repo/unneeded.txt | 1 + .../source_repo/BUILD_FILE | 4 +-- .../source_repo/build_defs/unused.build_defs | 2 +- test/export/test_subinclude_unused/BUILD | 9 ++++++ .../expected_repo/.plzconfig | 2 ++ .../expected_repo/BUILD_FILE | 6 ++++ .../expected_repo/build_defs/BUILD_FILE | 11 +++++++ .../build_defs/subincluded.build_defs | 7 +++++ .../build_defs/unused.build_defs | 6 ++++ .../expected_repo/file.txt | 1 + .../source_repo/.plzconfig | 2 ++ .../source_repo/BUILD_FILE | 11 +++++++ .../source_repo/build_defs/BUILD_FILE | 11 +++++++ .../build_defs/subincluded.build_defs | 7 +++++ .../source_repo/build_defs/unused.build_defs | 6 ++++ .../source_repo/file.txt | 1 + .../source_repo/unneeded.txt | 1 + 46 files changed, 215 insertions(+), 51 deletions(-) delete mode 100644 test/export/test_go_bin/source_repo/dummy.go create mode 100644 test/export/test_go_bin/source_repo/unneeded.go create mode 100644 test/export/test_subinclude_preloaded/BUILD create mode 100644 test/export/test_subinclude_preloaded/expected_repo/.plzconfig create mode 100644 test/export/test_subinclude_preloaded/expected_repo/BUILD_FILE create mode 100644 test/export/test_subinclude_preloaded/expected_repo/build_defs/BUILD_FILE create mode 100644 test/export/test_subinclude_preloaded/expected_repo/build_defs/preloaded.build_defs create mode 100644 test/export/test_subinclude_preloaded/expected_repo/build_defs/subincluded.build_defs create mode 100644 test/export/test_subinclude_preloaded/expected_repo/file.txt create mode 100644 test/export/test_subinclude_preloaded/source_repo/.plzconfig create mode 100644 test/export/test_subinclude_preloaded/source_repo/BUILD_FILE create mode 100644 test/export/test_subinclude_preloaded/source_repo/build_defs/BUILD_FILE create mode 100644 test/export/test_subinclude_preloaded/source_repo/build_defs/preloaded.build_defs create mode 100644 test/export/test_subinclude_preloaded/source_repo/build_defs/subincluded.build_defs create mode 100644 test/export/test_subinclude_preloaded/source_repo/file.txt create mode 100644 test/export/test_subinclude_preloaded/source_repo/unneeded.txt create mode 100644 test/export/test_subinclude_unused/BUILD create mode 100644 test/export/test_subinclude_unused/expected_repo/.plzconfig create mode 100644 test/export/test_subinclude_unused/expected_repo/BUILD_FILE create mode 100644 test/export/test_subinclude_unused/expected_repo/build_defs/BUILD_FILE create mode 100644 test/export/test_subinclude_unused/expected_repo/build_defs/subincluded.build_defs create mode 100644 test/export/test_subinclude_unused/expected_repo/build_defs/unused.build_defs create mode 100644 test/export/test_subinclude_unused/expected_repo/file.txt create mode 100644 test/export/test_subinclude_unused/source_repo/.plzconfig create mode 100644 test/export/test_subinclude_unused/source_repo/BUILD_FILE create mode 100644 test/export/test_subinclude_unused/source_repo/build_defs/BUILD_FILE create mode 100644 test/export/test_subinclude_unused/source_repo/build_defs/subincluded.build_defs create mode 100644 test/export/test_subinclude_unused/source_repo/build_defs/unused.build_defs create mode 100644 test/export/test_subinclude_unused/source_repo/file.txt create mode 100644 test/export/test_subinclude_unused/source_repo/unneeded.txt diff --git a/src/export/export.go b/src/export/export.go index e6923a9adb..3b1e2faaee 100644 --- a/src/export/export.go +++ b/src/export/export.go @@ -156,7 +156,7 @@ func (be *baseExporter) exportDependencies(target *core.BuildTarget) { // exportSources exports all files required by the target. func (be *baseExporter) exportSources(target *core.BuildTarget) { - for _, src := range append(target.AllSources(), target.AllData()...) { + for _, src := range target.AllBuildInputs() { if _, ok := src.Label(); ok { continue // These will be handled as dependencies later } diff --git a/src/export/trimmed_exporter.go b/src/export/trimmed_exporter.go index 48056e1e2f..d4214ec67b 100644 --- a/src/export/trimmed_exporter.go +++ b/src/export/trimmed_exporter.go @@ -102,12 +102,29 @@ func (e *trimmedExporter) writePackageFiles() { // exportSubincludes exports the subincluded targets required to generate the target and selects them to // later be written to the package as statements. func (e *trimmedExporter) exportSubincludes(pkg *core.Package, target *core.BuildTarget) { - subincludes := pkg.Metadata.FindRequiredSubincludes(target) - if len(subincludes) == 0 { - return + // Get the actively used subincludes of the target and propagate all transitive subincludes required + // by our used subinclude targets. FindRequiredSubincludes will report the required subincludes + // for this target at the package level but we need to propagate the subincluded targets inside + // build definitions since we are not trimming build_defs files. + usedSubincludes := pkg.Metadata.FindRequiredSubincludes(target) + e.setPackageSubincludes(pkg, usedSubincludes) + + allSubincludes := usedSubincludes + for _, sub := range usedSubincludes { + for _, trans := range e.state.Graph.TransitiveSubincludes(sub) { + if !slices.Contains(allSubincludes, trans) { + allSubincludes = append(allSubincludes, trans) + } + } } - log.Debugf("Subincludes required for %s: %v", target, subincludes) + log.Debugf("Subincludes required for %s: %v", target, allSubincludes) + e.exportTargets(allSubincludes) +} + +// setPackageSubincludes marks the package-level required subincludes after the export. This will be +// used for trimming subinclude statements with [trimmer]. +func (e *trimmedExporter) setPackageSubincludes(pkg *core.Package, subincludes core.BuildLabels) { for _, subinclude := range subincludes { // skip for preloaded subincludes, these are handled separately at the start to ensure they are // exported even if not directly used by an exported target. @@ -115,13 +132,13 @@ func (e *trimmedExporter) exportSubincludes(pkg *core.Package, target *core.Buil continue } - required := e.requiredSubincludes[pkg.Label()] + pkgLabel := pkg.Label() + required := e.requiredSubincludes[pkgLabel] if !slices.Contains(required, subinclude) { required = append(required, subinclude) } - e.requiredSubincludes[pkg.Label()] = required + e.requiredSubincludes[pkgLabel] = required } - e.exportTargets(subincludes) } // exportRelatedTargets looks up and exports all build targets that were declared within the same diff --git a/test/export/test_builtins/source_repo/BUILD_FILE b/test/export/test_builtins/source_repo/BUILD_FILE index 7c4e63b45d..0e09af4883 100644 --- a/test/export/test_builtins/source_repo/BUILD_FILE +++ b/test/export/test_builtins/source_repo/BUILD_FILE @@ -7,8 +7,8 @@ genrule( ) genrule( - name = "dummy_target_to_be_trimmed", - srcs = ["dummy.txt"], - outs = ["dummy"], + name = "unneded_target_to_be_trimmed", + srcs = ["unneded.txt"], + outs = ["unneded"], cmd = "cat $SRCS > $OUT", ) diff --git a/test/export/test_custom_def/source_repo/BUILD_FILE b/test/export/test_custom_def/source_repo/BUILD_FILE index b0e288e983..4ca3ee04a5 100644 --- a/test/export/test_custom_def/source_repo/BUILD_FILE +++ b/test/export/test_custom_def/source_repo/BUILD_FILE @@ -7,7 +7,7 @@ simple_custom_target( ) simple_custom_target( - name = "dummy", + name = "unneded", srcs = ["file.txt"], - outs = ["dummy.out"], + outs = ["unneded.out"], ) diff --git a/test/export/test_custom_def/source_repo/build_defs/BUILD_FILE b/test/export/test_custom_def/source_repo/build_defs/BUILD_FILE index 976d0dc22c..2450ba9afd 100644 --- a/test/export/test_custom_def/source_repo/build_defs/BUILD_FILE +++ b/test/export/test_custom_def/source_repo/build_defs/BUILD_FILE @@ -5,7 +5,7 @@ filegroup( ) filegroup( - name = "dummy_build_def", - srcs = ["dummy.build_defs"], + name = "unneded_build_def", + srcs = ["unneded.build_defs"], visibility = ["PUBLIC"], ) diff --git a/test/export/test_custom_def/source_repo/build_defs/dummy.build_defs b/test/export/test_custom_def/source_repo/build_defs/dummy.build_defs index 8cc3b6112a..17b55895be 100644 --- a/test/export/test_custom_def/source_repo/build_defs/dummy.build_defs +++ b/test/export/test_custom_def/source_repo/build_defs/dummy.build_defs @@ -1,8 +1,8 @@ -def dummy_target( +def unneded_target( name:str, outs:list=[]): return genrule( name = name, outs = outs, - cmd = "echo dummy > $OUT", + cmd = "echo unneded > $OUT", ) diff --git a/test/export/test_custom_def_multiple_targets/source_repo/BUILD_FILE b/test/export/test_custom_def_multiple_targets/source_repo/BUILD_FILE index 9651c3d8fe..ba45446f94 100644 --- a/test/export/test_custom_def_multiple_targets/source_repo/BUILD_FILE +++ b/test/export/test_custom_def_multiple_targets/source_repo/BUILD_FILE @@ -8,8 +8,8 @@ custom_target( ) custom_target( - name = "dummy", + name = "unneded", srcs = ["file.txt"], - outs = ["dummy.out"], - outs_adjacent = ["dummy_adjacent.out"], + outs = ["unneded.out"], + outs_adjacent = ["unneded_adjacent.out"], ) diff --git a/test/export/test_deps/source_repo/BUILD_FILE b/test/export/test_deps/source_repo/BUILD_FILE index 3c6766fc9e..56103fc9fc 100644 --- a/test/export/test_deps/source_repo/BUILD_FILE +++ b/test/export/test_deps/source_repo/BUILD_FILE @@ -17,8 +17,8 @@ genrule( ) genrule( - name = "dummy", + name = "unneded", srcs = ["file1.txt"], - outs = ["dummy.out"], + outs = ["unneded.out"], cmd = "cat $SRCS > $OUT", ) diff --git a/test/export/test_deps/source_repo/other_deps/BUILD_FILE b/test/export/test_deps/source_repo/other_deps/BUILD_FILE index 8acaa9b9eb..cd7262c32c 100644 --- a/test/export/test_deps/source_repo/other_deps/BUILD_FILE +++ b/test/export/test_deps/source_repo/other_deps/BUILD_FILE @@ -6,8 +6,8 @@ genrule( ) genrule( - name = "dummy", - outs = ["dummy.out"], - cmd = "echo 'dummy' > $OUT", + name = "unneded", + outs = ["unneded.out"], + cmd = "echo 'unneded' > $OUT", visibility = ["PUBLIC"], ) diff --git a/test/export/test_go_bin/source_repo/BUILD_FILE b/test/export/test_go_bin/source_repo/BUILD_FILE index 79cce0e24e..980cd46364 100644 --- a/test/export/test_go_bin/source_repo/BUILD_FILE +++ b/test/export/test_go_bin/source_repo/BUILD_FILE @@ -9,8 +9,8 @@ go_binary( ) go_binary( - name = "dummy", - srcs = ["dummy.go"], + name = "unneded", + srcs = ["unneded.go"], deps = [ "//third_party/go:uuid", ], diff --git a/test/export/test_go_bin/source_repo/dummy.go b/test/export/test_go_bin/source_repo/dummy.go deleted file mode 100644 index 403ada14aa..0000000000 --- a/test/export/test_go_bin/source_repo/dummy.go +++ /dev/null @@ -1,11 +0,0 @@ -package dummy - -import ( - "fmt" - - "github.com/google/uuid" -) - -func main() { - return -} diff --git a/test/export/test_go_bin/source_repo/third_party/go/BUILD_FILE b/test/export/test_go_bin/source_repo/third_party/go/BUILD_FILE index bc30a8e53a..70ee041a9f 100644 --- a/test/export/test_go_bin/source_repo/third_party/go/BUILD_FILE +++ b/test/export/test_go_bin/source_repo/third_party/go/BUILD_FILE @@ -27,7 +27,7 @@ go_repo( ) go_repo( - # Dummy, unused target + # unneded, unused target name = "uuid", module = "github.com/google/uuid", version = "v1.6.0", diff --git a/test/export/test_go_bin/source_repo/unneeded.go b/test/export/test_go_bin/source_repo/unneeded.go new file mode 100644 index 0000000000..277888f925 --- /dev/null +++ b/test/export/test_go_bin/source_repo/unneeded.go @@ -0,0 +1,5 @@ +package unneded + +func main() { + return +} diff --git a/test/export/test_in_file_build_def/source_repo/BUILD_FILE b/test/export/test_in_file_build_def/source_repo/BUILD_FILE index 6e5251d844..76b62d74b4 100644 --- a/test/export/test_in_file_build_def/source_repo/BUILD_FILE +++ b/test/export/test_in_file_build_def/source_repo/BUILD_FILE @@ -13,7 +13,7 @@ simple_custom_target( ) simple_custom_target( - name = "dummy", - srcs = ["dummy.in"], - outs = ["dummy.out"], + name = "unneded", + srcs = ["unneded.in"], + outs = ["unneded.out"], ) diff --git a/test/export/test_in_file_func_def/source_repo/BUILD_FILE b/test/export/test_in_file_func_def/source_repo/BUILD_FILE index 93fc806076..1cc13d43b0 100644 --- a/test/export/test_in_file_func_def/source_repo/BUILD_FILE +++ b/test/export/test_in_file_func_def/source_repo/BUILD_FILE @@ -16,7 +16,7 @@ custom( ) custom( - name = "dummy", - srcs = ["dummy.in"], - outs = ["dummy.out"], + name = "unneded", + srcs = ["unneded.in"], + outs = ["unneded.out"], ) diff --git a/test/export/test_multiple_targets/source_repo/BUILD_FILE b/test/export/test_multiple_targets/source_repo/BUILD_FILE index 3dc36004b0..856ca1b4bb 100644 --- a/test/export/test_multiple_targets/source_repo/BUILD_FILE +++ b/test/export/test_multiple_targets/source_repo/BUILD_FILE @@ -13,7 +13,7 @@ genrule( ) genrule( - name = "dummy", - outs = ["dummy.out"], - cmd = "echo dummy > $OUT", + name = "unneded", + outs = ["unneded.out"], + cmd = "echo unneded > $OUT", ) diff --git a/test/export/test_subinclude_preloaded/BUILD b/test/export/test_subinclude_preloaded/BUILD new file mode 100644 index 0000000000..70aa25283d --- /dev/null +++ b/test/export/test_subinclude_preloaded/BUILD @@ -0,0 +1,8 @@ +subinclude("//test/export:export_e2e_test_build_def") + +# Verifies that `plz export` correctly processes a subincluded build definition target +# that in turn subincludes a preloaded target. +please_export_e2e_test( + name = "subinclude_preloaded_export_test", + export_targets = ["//:file"], +) \ No newline at end of file diff --git a/test/export/test_subinclude_preloaded/expected_repo/.plzconfig b/test/export/test_subinclude_preloaded/expected_repo/.plzconfig new file mode 100644 index 0000000000..512d2cf400 --- /dev/null +++ b/test/export/test_subinclude_preloaded/expected_repo/.plzconfig @@ -0,0 +1,3 @@ +[parse] +BuildFileName = BUILD_FILE +preloadsubincludes = //build_defs:preloaded_build_def \ No newline at end of file diff --git a/test/export/test_subinclude_preloaded/expected_repo/BUILD_FILE b/test/export/test_subinclude_preloaded/expected_repo/BUILD_FILE new file mode 100644 index 0000000000..0b66bcffbc --- /dev/null +++ b/test/export/test_subinclude_preloaded/expected_repo/BUILD_FILE @@ -0,0 +1,6 @@ +subinclude("//build_defs:subincluded_build_def") + +custom_file( + name = "file", + src = "file.txt", +) diff --git a/test/export/test_subinclude_preloaded/expected_repo/build_defs/BUILD_FILE b/test/export/test_subinclude_preloaded/expected_repo/build_defs/BUILD_FILE new file mode 100644 index 0000000000..847d65de60 --- /dev/null +++ b/test/export/test_subinclude_preloaded/expected_repo/build_defs/BUILD_FILE @@ -0,0 +1,11 @@ +filegroup( + name = "preloaded_build_def", + srcs = ["preloaded.build_defs"], + visibility = ["PUBLIC"], +) + +filegroup( + name = "subincluded_build_def", + srcs = ["subincluded.build_defs"], + visibility = ["PUBLIC"], +) diff --git a/test/export/test_subinclude_preloaded/expected_repo/build_defs/preloaded.build_defs b/test/export/test_subinclude_preloaded/expected_repo/build_defs/preloaded.build_defs new file mode 100644 index 0000000000..b06f59e0a7 --- /dev/null +++ b/test/export/test_subinclude_preloaded/expected_repo/build_defs/preloaded.build_defs @@ -0,0 +1 @@ +# preloaded build def \ No newline at end of file diff --git a/test/export/test_subinclude_preloaded/expected_repo/build_defs/subincluded.build_defs b/test/export/test_subinclude_preloaded/expected_repo/build_defs/subincluded.build_defs new file mode 100644 index 0000000000..4d7e0f05fe --- /dev/null +++ b/test/export/test_subinclude_preloaded/expected_repo/build_defs/subincluded.build_defs @@ -0,0 +1,7 @@ +subinclude("//build_defs:preloaded_build_def") + +def custom_file(name, src): + filegroup( + name = name, + srcs = [src], + ) \ No newline at end of file diff --git a/test/export/test_subinclude_preloaded/expected_repo/file.txt b/test/export/test_subinclude_preloaded/expected_repo/file.txt new file mode 100644 index 0000000000..f73f3093ff --- /dev/null +++ b/test/export/test_subinclude_preloaded/expected_repo/file.txt @@ -0,0 +1 @@ +file diff --git a/test/export/test_subinclude_preloaded/source_repo/.plzconfig b/test/export/test_subinclude_preloaded/source_repo/.plzconfig new file mode 100644 index 0000000000..512d2cf400 --- /dev/null +++ b/test/export/test_subinclude_preloaded/source_repo/.plzconfig @@ -0,0 +1,3 @@ +[parse] +BuildFileName = BUILD_FILE +preloadsubincludes = //build_defs:preloaded_build_def \ No newline at end of file diff --git a/test/export/test_subinclude_preloaded/source_repo/BUILD_FILE b/test/export/test_subinclude_preloaded/source_repo/BUILD_FILE new file mode 100644 index 0000000000..9b8b35f482 --- /dev/null +++ b/test/export/test_subinclude_preloaded/source_repo/BUILD_FILE @@ -0,0 +1,11 @@ +subinclude("//build_defs:subincluded_build_def") + +custom_file( + name = "file", + src = "file.txt", +) + +custom_file( + name = "unneeded", + src = "unneeded.txt", +) \ No newline at end of file diff --git a/test/export/test_subinclude_preloaded/source_repo/build_defs/BUILD_FILE b/test/export/test_subinclude_preloaded/source_repo/build_defs/BUILD_FILE new file mode 100644 index 0000000000..d804dfc31d --- /dev/null +++ b/test/export/test_subinclude_preloaded/source_repo/build_defs/BUILD_FILE @@ -0,0 +1,11 @@ +filegroup( + name = "preloaded_build_def", + srcs = ["preloaded.build_defs"], + visibility = ["PUBLIC"], +) + +filegroup( + name = "subincluded_build_def", + srcs = ["subincluded.build_defs"], + visibility = ["PUBLIC"], +) \ No newline at end of file diff --git a/test/export/test_subinclude_preloaded/source_repo/build_defs/preloaded.build_defs b/test/export/test_subinclude_preloaded/source_repo/build_defs/preloaded.build_defs new file mode 100644 index 0000000000..b06f59e0a7 --- /dev/null +++ b/test/export/test_subinclude_preloaded/source_repo/build_defs/preloaded.build_defs @@ -0,0 +1 @@ +# preloaded build def \ No newline at end of file diff --git a/test/export/test_subinclude_preloaded/source_repo/build_defs/subincluded.build_defs b/test/export/test_subinclude_preloaded/source_repo/build_defs/subincluded.build_defs new file mode 100644 index 0000000000..4d7e0f05fe --- /dev/null +++ b/test/export/test_subinclude_preloaded/source_repo/build_defs/subincluded.build_defs @@ -0,0 +1,7 @@ +subinclude("//build_defs:preloaded_build_def") + +def custom_file(name, src): + filegroup( + name = name, + srcs = [src], + ) \ No newline at end of file diff --git a/test/export/test_subinclude_preloaded/source_repo/file.txt b/test/export/test_subinclude_preloaded/source_repo/file.txt new file mode 100644 index 0000000000..f73f3093ff --- /dev/null +++ b/test/export/test_subinclude_preloaded/source_repo/file.txt @@ -0,0 +1 @@ +file diff --git a/test/export/test_subinclude_preloaded/source_repo/unneeded.txt b/test/export/test_subinclude_preloaded/source_repo/unneeded.txt new file mode 100644 index 0000000000..b7390936bf --- /dev/null +++ b/test/export/test_subinclude_preloaded/source_repo/unneeded.txt @@ -0,0 +1 @@ +this is unneeded \ No newline at end of file diff --git a/test/export/test_subinclude_trimming/source_repo/BUILD_FILE b/test/export/test_subinclude_trimming/source_repo/BUILD_FILE index b678f4858f..44cf732531 100644 --- a/test/export/test_subinclude_trimming/source_repo/BUILD_FILE +++ b/test/export/test_subinclude_trimming/source_repo/BUILD_FILE @@ -10,6 +10,6 @@ simple_custom_target( ) unused_target( - name = "dummy", - outs = ["dummy.out"], + name = "unneded", + outs = ["unneded.out"], ) diff --git a/test/export/test_subinclude_trimming/source_repo/build_defs/unused.build_defs b/test/export/test_subinclude_trimming/source_repo/build_defs/unused.build_defs index ebdff7de00..9337a65ecf 100644 --- a/test/export/test_subinclude_trimming/source_repo/build_defs/unused.build_defs +++ b/test/export/test_subinclude_trimming/source_repo/build_defs/unused.build_defs @@ -4,5 +4,5 @@ def unused_target( return genrule( name = name, outs = outs, - cmd = "echo dummy > $OUT", + cmd = "echo unneded > $OUT", ) diff --git a/test/export/test_subinclude_unused/BUILD b/test/export/test_subinclude_unused/BUILD new file mode 100644 index 0000000000..b796cd33ab --- /dev/null +++ b/test/export/test_subinclude_unused/BUILD @@ -0,0 +1,9 @@ +subinclude("//test/export:export_e2e_test_build_def") + +# Verifies that when a build definition statically subincludes another target but does not +# actively invoke its methods, the subincluded target is still fully exported to ensure +# static subinclude parsing succeeds in the exported repository. +please_export_e2e_test( + name = "subinclude_unused_export_test", + export_targets = ["//:file"], +) diff --git a/test/export/test_subinclude_unused/expected_repo/.plzconfig b/test/export/test_subinclude_unused/expected_repo/.plzconfig new file mode 100644 index 0000000000..4d5174001a --- /dev/null +++ b/test/export/test_subinclude_unused/expected_repo/.plzconfig @@ -0,0 +1,2 @@ +[parse] +BuildFileName = BUILD_FILE \ No newline at end of file diff --git a/test/export/test_subinclude_unused/expected_repo/BUILD_FILE b/test/export/test_subinclude_unused/expected_repo/BUILD_FILE new file mode 100644 index 0000000000..0b66bcffbc --- /dev/null +++ b/test/export/test_subinclude_unused/expected_repo/BUILD_FILE @@ -0,0 +1,6 @@ +subinclude("//build_defs:subincluded_build_def") + +custom_file( + name = "file", + src = "file.txt", +) diff --git a/test/export/test_subinclude_unused/expected_repo/build_defs/BUILD_FILE b/test/export/test_subinclude_unused/expected_repo/build_defs/BUILD_FILE new file mode 100644 index 0000000000..86e5b46e15 --- /dev/null +++ b/test/export/test_subinclude_unused/expected_repo/build_defs/BUILD_FILE @@ -0,0 +1,11 @@ +filegroup( + name = "subincluded_build_def", + srcs = ["subincluded.build_defs"], + visibility = ["PUBLIC"], +) + +filegroup( + name = "unused_build_def", + srcs = ["unused.build_defs"], + visibility = ["PUBLIC"], +) diff --git a/test/export/test_subinclude_unused/expected_repo/build_defs/subincluded.build_defs b/test/export/test_subinclude_unused/expected_repo/build_defs/subincluded.build_defs new file mode 100644 index 0000000000..9c61aa4a65 --- /dev/null +++ b/test/export/test_subinclude_unused/expected_repo/build_defs/subincluded.build_defs @@ -0,0 +1,7 @@ +subinclude("//build_defs:unused_build_def") + +def custom_file(name, src): + filegroup( + name = name, + srcs = [src], + ) \ No newline at end of file diff --git a/test/export/test_subinclude_unused/expected_repo/build_defs/unused.build_defs b/test/export/test_subinclude_unused/expected_repo/build_defs/unused.build_defs new file mode 100644 index 0000000000..b8a3a700f9 --- /dev/null +++ b/test/export/test_subinclude_unused/expected_repo/build_defs/unused.build_defs @@ -0,0 +1,6 @@ +# unused build def content + +def unused_helper_method(name): + return filegroup( + name = name, + ) diff --git a/test/export/test_subinclude_unused/expected_repo/file.txt b/test/export/test_subinclude_unused/expected_repo/file.txt new file mode 100644 index 0000000000..f73f3093ff --- /dev/null +++ b/test/export/test_subinclude_unused/expected_repo/file.txt @@ -0,0 +1 @@ +file diff --git a/test/export/test_subinclude_unused/source_repo/.plzconfig b/test/export/test_subinclude_unused/source_repo/.plzconfig new file mode 100644 index 0000000000..4d5174001a --- /dev/null +++ b/test/export/test_subinclude_unused/source_repo/.plzconfig @@ -0,0 +1,2 @@ +[parse] +BuildFileName = BUILD_FILE \ No newline at end of file diff --git a/test/export/test_subinclude_unused/source_repo/BUILD_FILE b/test/export/test_subinclude_unused/source_repo/BUILD_FILE new file mode 100644 index 0000000000..9b8b35f482 --- /dev/null +++ b/test/export/test_subinclude_unused/source_repo/BUILD_FILE @@ -0,0 +1,11 @@ +subinclude("//build_defs:subincluded_build_def") + +custom_file( + name = "file", + src = "file.txt", +) + +custom_file( + name = "unneeded", + src = "unneeded.txt", +) \ No newline at end of file diff --git a/test/export/test_subinclude_unused/source_repo/build_defs/BUILD_FILE b/test/export/test_subinclude_unused/source_repo/build_defs/BUILD_FILE new file mode 100644 index 0000000000..549d28ce9a --- /dev/null +++ b/test/export/test_subinclude_unused/source_repo/build_defs/BUILD_FILE @@ -0,0 +1,11 @@ +filegroup( + name = "subincluded_build_def", + srcs = ["subincluded.build_defs"], + visibility = ["PUBLIC"], +) + +filegroup( + name = "unused_build_def", + srcs = ["unused.build_defs"], + visibility = ["PUBLIC"], +) \ No newline at end of file diff --git a/test/export/test_subinclude_unused/source_repo/build_defs/subincluded.build_defs b/test/export/test_subinclude_unused/source_repo/build_defs/subincluded.build_defs new file mode 100644 index 0000000000..9c61aa4a65 --- /dev/null +++ b/test/export/test_subinclude_unused/source_repo/build_defs/subincluded.build_defs @@ -0,0 +1,7 @@ +subinclude("//build_defs:unused_build_def") + +def custom_file(name, src): + filegroup( + name = name, + srcs = [src], + ) \ No newline at end of file diff --git a/test/export/test_subinclude_unused/source_repo/build_defs/unused.build_defs b/test/export/test_subinclude_unused/source_repo/build_defs/unused.build_defs new file mode 100644 index 0000000000..b8a3a700f9 --- /dev/null +++ b/test/export/test_subinclude_unused/source_repo/build_defs/unused.build_defs @@ -0,0 +1,6 @@ +# unused build def content + +def unused_helper_method(name): + return filegroup( + name = name, + ) diff --git a/test/export/test_subinclude_unused/source_repo/file.txt b/test/export/test_subinclude_unused/source_repo/file.txt new file mode 100644 index 0000000000..f73f3093ff --- /dev/null +++ b/test/export/test_subinclude_unused/source_repo/file.txt @@ -0,0 +1 @@ +file diff --git a/test/export/test_subinclude_unused/source_repo/unneeded.txt b/test/export/test_subinclude_unused/source_repo/unneeded.txt new file mode 100644 index 0000000000..9c1fe51193 --- /dev/null +++ b/test/export/test_subinclude_unused/source_repo/unneeded.txt @@ -0,0 +1 @@ +unneeded \ No newline at end of file From acb81e6ac312444ddfc3936f8b39d52ceb2f8d5a Mon Sep 17 00:00:00 2001 From: DuBento Date: Fri, 12 Jun 2026 15:56:30 +0100 Subject: [PATCH 087/118] export gitignore --- src/export/export.go | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/export/export.go b/src/export/export.go index 3b1e2faaee..474d140609 100644 --- a/src/export/export.go +++ b/src/export/export.go @@ -101,7 +101,7 @@ type baseExporter struct { // run specifies the main steps when running an export. func (be *baseExporter) run(targets core.BuildLabels) { go be.startMonitor() - be.exportPlzConfig() + be.exportRepoConfig() be.impl.exportPreloaded() be.exportTargets(targets) be.impl.writePackageFiles() @@ -110,24 +110,28 @@ func (be *baseExporter) run(targets core.BuildLabels) { func (be *baseExporter) startMonitor() { for { time.Sleep(10 * time.Second) - log.Warningf("Number of targets exported: %v", be.targetCounter) + log.Infof("Number of targets exported: %v", be.targetCounter) } } -// exportPlzConfig exports the repository's configuration files (e.g., .plzconfig and its +// exportRepoConfig exports the repository's configuration files (e.g., .gitignore, .plzconfig and its // platform-specific variants) to the target export directory. -func (be *baseExporter) exportPlzConfig() { - profiles, err := filepath.Glob(".plzconfig*") +func (be *baseExporter) exportRepoConfig() { + files, err := filepath.Glob(".plzconfig*") if err != nil { log.Fatalf("failed to glob .plzconfig files: %v", err) } - for _, file := range profiles { + if info, err := os.Stat(".gitignore"); err == nil { + files = append(files, info.Name()) + } + + for _, file := range files { targetPath := filepath.Join(be.targetDir, file) if err := os.RemoveAll(targetPath); err != nil { - log.Fatalf("failed to remove .plzconfig file %s: %v", file, err) + log.Fatalf("failed to remove file %s: %v", file, err) } if err := fs.CopyFile(file, targetPath, 0); err != nil { - log.Fatalf("failed to copy .plzconfig file %s: %v", file, err) + log.Fatalf("failed to copy file %s: %v", file, err) } } } From f1a0abe65be0b37832a3a155e41c538dd74e2923 Mon Sep 17 00:00:00 2001 From: DuBento Date: Mon, 15 Jun 2026 14:30:15 +0100 Subject: [PATCH 088/118] test: glob for sources --- test/export/test_glob/BUILD | 7 +++++++ test/export/test_glob/expected_repo/.plzconfig | 2 ++ test/export/test_glob/expected_repo/BUILD_FILE | 6 ++++++ test/export/test_glob/expected_repo/file1.txt | 1 + test/export/test_glob/expected_repo/file2.txt | 1 + test/export/test_glob/source_repo/.plzconfig | 2 ++ test/export/test_glob/source_repo/BUILD_FILE | 13 +++++++++++++ test/export/test_glob/source_repo/file1.txt | 1 + test/export/test_glob/source_repo/file2.txt | 1 + test/export/test_glob/source_repo/ignored.txt | 1 + test/export/test_glob/source_repo/unused.dat | 1 + 11 files changed, 36 insertions(+) create mode 100644 test/export/test_glob/BUILD create mode 100644 test/export/test_glob/expected_repo/.plzconfig create mode 100644 test/export/test_glob/expected_repo/BUILD_FILE create mode 100644 test/export/test_glob/expected_repo/file1.txt create mode 100644 test/export/test_glob/expected_repo/file2.txt create mode 100644 test/export/test_glob/source_repo/.plzconfig create mode 100644 test/export/test_glob/source_repo/BUILD_FILE create mode 100644 test/export/test_glob/source_repo/file1.txt create mode 100644 test/export/test_glob/source_repo/file2.txt create mode 100644 test/export/test_glob/source_repo/ignored.txt create mode 100644 test/export/test_glob/source_repo/unused.dat diff --git a/test/export/test_glob/BUILD b/test/export/test_glob/BUILD new file mode 100644 index 0000000000..5706c6b860 --- /dev/null +++ b/test/export/test_glob/BUILD @@ -0,0 +1,7 @@ +subinclude("//test/export:export_e2e_test_build_def") + +# Export a target that uses glob to select inputs. +please_export_e2e_test( + name = "export_glob", + export_targets = ["//:glob_target"], +) diff --git a/test/export/test_glob/expected_repo/.plzconfig b/test/export/test_glob/expected_repo/.plzconfig new file mode 100644 index 0000000000..f8ba31854d --- /dev/null +++ b/test/export/test_glob/expected_repo/.plzconfig @@ -0,0 +1,2 @@ +[Parse] +BuildFileName = BUILD_FILE diff --git a/test/export/test_glob/expected_repo/BUILD_FILE b/test/export/test_glob/expected_repo/BUILD_FILE new file mode 100644 index 0000000000..b286492ee9 --- /dev/null +++ b/test/export/test_glob/expected_repo/BUILD_FILE @@ -0,0 +1,6 @@ +genrule( + name = "glob_target", + srcs = glob(["*.txt"], exclude = ["ignored.txt"]), + outs = ["out.txt"], + cmd = "cat $SRCS > $OUT", +) diff --git a/test/export/test_glob/expected_repo/file1.txt b/test/export/test_glob/expected_repo/file1.txt new file mode 100644 index 0000000000..b5cea514f6 --- /dev/null +++ b/test/export/test_glob/expected_repo/file1.txt @@ -0,0 +1 @@ +hello from file 1 diff --git a/test/export/test_glob/expected_repo/file2.txt b/test/export/test_glob/expected_repo/file2.txt new file mode 100644 index 0000000000..7331ba8736 --- /dev/null +++ b/test/export/test_glob/expected_repo/file2.txt @@ -0,0 +1 @@ +hello from file 2 diff --git a/test/export/test_glob/source_repo/.plzconfig b/test/export/test_glob/source_repo/.plzconfig new file mode 100644 index 0000000000..f8ba31854d --- /dev/null +++ b/test/export/test_glob/source_repo/.plzconfig @@ -0,0 +1,2 @@ +[Parse] +BuildFileName = BUILD_FILE diff --git a/test/export/test_glob/source_repo/BUILD_FILE b/test/export/test_glob/source_repo/BUILD_FILE new file mode 100644 index 0000000000..0a25600d17 --- /dev/null +++ b/test/export/test_glob/source_repo/BUILD_FILE @@ -0,0 +1,13 @@ +genrule( + name = "glob_target", + srcs = glob(["*.txt"], exclude = ["ignored.txt"]), + outs = ["out.txt"], + cmd = "cat $SRCS > $OUT", +) + +genrule( + name = "unneeded", + srcs = ["unused.dat"], + outs = ["unneeded.out"], + cmd = "cat $SRCS > $OUT", +) diff --git a/test/export/test_glob/source_repo/file1.txt b/test/export/test_glob/source_repo/file1.txt new file mode 100644 index 0000000000..b5cea514f6 --- /dev/null +++ b/test/export/test_glob/source_repo/file1.txt @@ -0,0 +1 @@ +hello from file 1 diff --git a/test/export/test_glob/source_repo/file2.txt b/test/export/test_glob/source_repo/file2.txt new file mode 100644 index 0000000000..7331ba8736 --- /dev/null +++ b/test/export/test_glob/source_repo/file2.txt @@ -0,0 +1 @@ +hello from file 2 diff --git a/test/export/test_glob/source_repo/ignored.txt b/test/export/test_glob/source_repo/ignored.txt new file mode 100644 index 0000000000..44d282adba --- /dev/null +++ b/test/export/test_glob/source_repo/ignored.txt @@ -0,0 +1 @@ +hello from ignored file diff --git a/test/export/test_glob/source_repo/unused.dat b/test/export/test_glob/source_repo/unused.dat new file mode 100644 index 0000000000..e8f53910e2 --- /dev/null +++ b/test/export/test_glob/source_repo/unused.dat @@ -0,0 +1 @@ +hello from unused file From 5d3073b6e356aee255cab61e0819cd78c916a5c9 Mon Sep 17 00:00:00 2001 From: DuBento Date: Mon, 15 Jun 2026 17:18:48 +0100 Subject: [PATCH 089/118] Package parsing via WaitForPackage --- src/export/export.go | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/export/export.go b/src/export/export.go index 474d140609..c548939979 100644 --- a/src/export/export.go +++ b/src/export/export.go @@ -205,14 +205,7 @@ func (be *baseExporter) getOrParseTarget(label core.BuildLabel) *core.BuildTarge // This requires the background parser worker threads to be kept alive as daemons (controlled by the // "KeepParserRunning" build state option). func (be *baseExporter) getOrParsePackage(label core.BuildLabel) *core.Package { - label.Name = "all" - pkg := be.state.Graph.PackageByLabel(label) - if pkg == nil { - log.Infof("Package %v not found in graph. Attempting to parse...", label) - parse.Parse(be.state, label, core.OriginalTarget, core.ParseModeNormal) - pkg = be.state.Graph.PackageByLabel(label) - } - return pkg + return be.state.WaitForPackage(label, core.OriginalTarget, core.ParseModeNormal) } // checkAndSetVisited is a helper to ensure we only visit the same target once. From c38eba69cbad956931c502607b59132237389fa0 Mon Sep 17 00:00:00 2001 From: DuBento Date: Mon, 15 Jun 2026 18:11:32 +0100 Subject: [PATCH 090/118] test: move trim test files into data --- src/export/export_test.go | 125 ++++++------------ src/export/test_data/trim_elif.build | 6 + .../test_data/trim_elif_expected_b.build | 6 + src/export/test_data/trim_for.build | 2 + .../test_data/trim_for_expected_a.build | 2 + src/export/test_data/trim_for_if.build | 8 ++ .../test_data/trim_for_if_expected_a.build | 8 ++ src/export/test_data/trim_if.build | 2 + src/export/test_data/trim_if_expected_a.build | 2 + .../test_data/trim_if_expected_none.build | 1 + 10 files changed, 76 insertions(+), 86 deletions(-) create mode 100644 src/export/test_data/trim_elif.build create mode 100644 src/export/test_data/trim_elif_expected_b.build create mode 100644 src/export/test_data/trim_for.build create mode 100644 src/export/test_data/trim_for_expected_a.build create mode 100644 src/export/test_data/trim_for_if.build create mode 100644 src/export/test_data/trim_for_if_expected_a.build create mode 100644 src/export/test_data/trim_if.build create mode 100644 src/export/test_data/trim_if_expected_a.build create mode 100644 src/export/test_data/trim_if_expected_none.build diff --git a/src/export/export_test.go b/src/export/export_test.go index 4595f22490..7d7b828308 100644 --- a/src/export/export_test.go +++ b/src/export/export_test.go @@ -123,119 +123,72 @@ func TestFilterPackageFile(t *testing.T) { func TestStatementTrim(t *testing.T) { testCases := []struct { - name string - content string - registered []string - required []string - expected string + name string + content string + required []string + expected string }{ { - name: "Keep target in if", - content: ` -if True: - genrule(name = "a", cmd = "echo a > $OUT", outs = ["a"]) -`, - registered: []string{"a"}, - required: []string{"a"}, - expected: ` -if True: - genrule(name = "a", cmd = "echo a > $OUT", outs = ["a"]) -`, + name: "Keep target in if", + content: "src/export/test_data/trim_if.build", + required: []string{"a"}, + expected: "src/export/test_data/trim_if_expected_a.build", }, { - name: "Target not required - all statements trimmed", - content: ` -if True: - genrule(name = "a", cmd = "echo a > $OUT", outs = ["a"]) -`, - registered: []string{"a"}, - required: []string{}, - // Empty, all statements pruned. Blank space removal is not performed by trimBlock's implementation so expect the new lines. - expected: ` - -`, + name: "Target not required - all statements trimmed", + content: "src/export/test_data/trim_if.build", + required: []string{}, + expected: "src/export/test_data/trim_if_expected_none.build", }, { - name: "Required target in elif", - content: ` -if False: - genrule(name = "a") -elif True: - genrule(name = "b") -else: - genrule(name = "c") -`, - registered: []string{"b"}, - required: []string{"b"}, - expected: ` -if False: - pass # Trimmed during export -elif True: - genrule(name = "b") -else: - pass # Trimmed during export -`}, + name: "Required target in elif", + content: "src/export/test_data/trim_elif.build", + required: []string{"b"}, + expected: "src/export/test_data/trim_elif_expected_b.build", + }, { - name: "Required target in for", - content: ` -for i in range(0,2): - genrule(name = "a") -`, - registered: []string{"a"}, - required: []string{"a"}, - expected: ` -for i in range(0,2): - genrule(name = "a") -`}, + name: "Required target in for", + content: "src/export/test_data/trim_for.build", + required: []string{"a"}, + expected: "src/export/test_data/trim_for_expected_a.build", + }, { - name: "Required if stmt in for", - content: ` -for i in [ - "a", - "b", -]: - if i == "a": - genrule(name = "a") - elif i == "b": - genrule(name = "b") -`, - registered: []string{"a", "b"}, - required: []string{"a"}, - expected: ` -for i in [ - "a", - "b", -]: - if i == "a": - genrule(name = "a") - elif i == "b": - pass # Trimmed during export -`}, + name: "Required if stmt in for", + content: "src/export/test_data/trim_for_if.build", + required: []string{"a"}, + expected: "src/export/test_data/trim_for_if_expected_a.build", + }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { p := asp.NewParserOnly() - statements, err := p.ParseData([]byte(tc.content), "BUILD") + statements, err := p.ParseFileOnly(tc.content) assert.NoError(t, err) pkg := core.NewPackage("test", core.WithPackageMetadata()) - pkg.Filename = "BUILD" + pkg.Filename = tc.content + targetLabels := walkASTRegisterTargets(t, statements, pkg, nil) - targetLabels := walkASTRegisterTargets(t, statements, pkg, tc.registered) e := newExporter(nil, "", false).impl.(*trimmedExporter) for _, name := range tc.required { e.exportedTargets[targetLabels[name]] = true } + contentBytes, err := os.ReadFile(tc.content) + assert.NoError(t, err) + trimmer := &trimmer{ - origin: []byte(tc.content), + origin: contentBytes, pkg: pkg, exporter: e, } - trimmer.trimBlock(statements, 0, asp.Position(len(tc.content))) + trimmer.trimBlock(statements, 0, asp.Position(len(contentBytes))) + + expectedBytes, err := os.ReadFile(tc.expected) + assert.NoError(t, err) - assert.Equal(t, tc.expected, string(trimmer.bytes)) + assert.Equal(t, string(expectedBytes), string(trimmer.bytes)) }) } } diff --git a/src/export/test_data/trim_elif.build b/src/export/test_data/trim_elif.build new file mode 100644 index 0000000000..796b473b2e --- /dev/null +++ b/src/export/test_data/trim_elif.build @@ -0,0 +1,6 @@ +if False: + genrule(name = "a") +elif True: + genrule(name = "b") +else: + genrule(name = "c") diff --git a/src/export/test_data/trim_elif_expected_b.build b/src/export/test_data/trim_elif_expected_b.build new file mode 100644 index 0000000000..c72c6e6131 --- /dev/null +++ b/src/export/test_data/trim_elif_expected_b.build @@ -0,0 +1,6 @@ +if False: + pass # Trimmed during export +elif True: + genrule(name = "b") +else: + pass # Trimmed during export diff --git a/src/export/test_data/trim_for.build b/src/export/test_data/trim_for.build new file mode 100644 index 0000000000..998e4a22aa --- /dev/null +++ b/src/export/test_data/trim_for.build @@ -0,0 +1,2 @@ +for i in range(0,2): + genrule(name = "a") diff --git a/src/export/test_data/trim_for_expected_a.build b/src/export/test_data/trim_for_expected_a.build new file mode 100644 index 0000000000..998e4a22aa --- /dev/null +++ b/src/export/test_data/trim_for_expected_a.build @@ -0,0 +1,2 @@ +for i in range(0,2): + genrule(name = "a") diff --git a/src/export/test_data/trim_for_if.build b/src/export/test_data/trim_for_if.build new file mode 100644 index 0000000000..d13cfd7592 --- /dev/null +++ b/src/export/test_data/trim_for_if.build @@ -0,0 +1,8 @@ +for i in [ + "a", + "b", +]: + if i == "a": + genrule(name = "a") + elif i == "b": + genrule(name = "b") diff --git a/src/export/test_data/trim_for_if_expected_a.build b/src/export/test_data/trim_for_if_expected_a.build new file mode 100644 index 0000000000..e8a8bba074 --- /dev/null +++ b/src/export/test_data/trim_for_if_expected_a.build @@ -0,0 +1,8 @@ +for i in [ + "a", + "b", +]: + if i == "a": + genrule(name = "a") + elif i == "b": + pass # Trimmed during export diff --git a/src/export/test_data/trim_if.build b/src/export/test_data/trim_if.build new file mode 100644 index 0000000000..1f41589c70 --- /dev/null +++ b/src/export/test_data/trim_if.build @@ -0,0 +1,2 @@ +if True: + genrule(name = "a", cmd = "echo a > $OUT", outs = ["a"]) diff --git a/src/export/test_data/trim_if_expected_a.build b/src/export/test_data/trim_if_expected_a.build new file mode 100644 index 0000000000..1f41589c70 --- /dev/null +++ b/src/export/test_data/trim_if_expected_a.build @@ -0,0 +1,2 @@ +if True: + genrule(name = "a", cmd = "echo a > $OUT", outs = ["a"]) diff --git a/src/export/test_data/trim_if_expected_none.build b/src/export/test_data/trim_if_expected_none.build new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/src/export/test_data/trim_if_expected_none.build @@ -0,0 +1 @@ + From c387e50c742dcfe94dc71b2ceb90b45613e36d22 Mon Sep 17 00:00:00 2001 From: DuBento Date: Tue, 16 Jun 2026 13:19:12 +0100 Subject: [PATCH 091/118] test: trim subinclude in third_party/go test --- .../source_repo/third_party/go/BUILD_FILE | 2 +- .../source_repo/third_party/common/BUILD_FILE | 7 +++++++ .../third_party/common/version.build_defs | 1 + .../source_repo/third_party/go/BUILD_FILE | 12 +++++++++++- 4 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 test/export/test_native_target_with_go_dep/source_repo/third_party/common/BUILD_FILE create mode 100644 test/export/test_native_target_with_go_dep/source_repo/third_party/common/version.build_defs diff --git a/test/export/test_go_bin/source_repo/third_party/go/BUILD_FILE b/test/export/test_go_bin/source_repo/third_party/go/BUILD_FILE index 70ee041a9f..052c347845 100644 --- a/test/export/test_go_bin/source_repo/third_party/go/BUILD_FILE +++ b/test/export/test_go_bin/source_repo/third_party/go/BUILD_FILE @@ -27,7 +27,7 @@ go_repo( ) go_repo( - # unneded, unused target + # unneeded, unused target name = "uuid", module = "github.com/google/uuid", version = "v1.6.0", diff --git a/test/export/test_native_target_with_go_dep/source_repo/third_party/common/BUILD_FILE b/test/export/test_native_target_with_go_dep/source_repo/third_party/common/BUILD_FILE new file mode 100644 index 0000000000..4a67400211 --- /dev/null +++ b/test/export/test_native_target_with_go_dep/source_repo/third_party/common/BUILD_FILE @@ -0,0 +1,7 @@ +export_file( + name = "version", + src = "version.build_defs", + visibility = [ + "//third_party/...", + ], +) diff --git a/test/export/test_native_target_with_go_dep/source_repo/third_party/common/version.build_defs b/test/export/test_native_target_with_go_dep/source_repo/third_party/common/version.build_defs new file mode 100644 index 0000000000..3449cc1b8f --- /dev/null +++ b/test/export/test_native_target_with_go_dep/source_repo/third_party/common/version.build_defs @@ -0,0 +1 @@ +LIB_VERSION="v1.6.0" diff --git a/test/export/test_native_target_with_go_dep/source_repo/third_party/go/BUILD_FILE b/test/export/test_native_target_with_go_dep/source_repo/third_party/go/BUILD_FILE index e436be33ec..503411e7c9 100644 --- a/test/export/test_native_target_with_go_dep/source_repo/third_party/go/BUILD_FILE +++ b/test/export/test_native_target_with_go_dep/source_repo/third_party/go/BUILD_FILE @@ -1,4 +1,7 @@ -subinclude("///go//build_defs:go") +subinclude( + "///go//build_defs:go", + "//third_party/common:version", +) package(default_visibility = ["PUBLIC"]) @@ -23,3 +26,10 @@ go_repo( module = "github.com/google/go-cmp", version = "v0.5.6", ) + +go_repo( + # unneeded, unused target + name = "uuid", + module = "github.com/google/uuid", + version = f"{LIB_VERSION}", +) From 2446dad329cba03af974f1b7da7da40853a01487 Mon Sep 17 00:00:00 2001 From: DuBento Date: Tue, 16 Jun 2026 18:37:58 +0100 Subject: [PATCH 092/118] export label set --- src/core/build_label.go | 12 ++++++++ src/core/graph.go | 29 ++++++------------- src/core/package.go | 4 +-- .../source_repo/third_party/go/BUILD_FILE | 10 +++---- 4 files changed, 28 insertions(+), 27 deletions(-) diff --git a/src/core/build_label.go b/src/core/build_label.go index 779e7302e0..26423571f9 100644 --- a/src/core/build_label.go +++ b/src/core/build_label.go @@ -622,3 +622,15 @@ func (slice BuildLabels) String() string { } return strings.Join(s, ", ") } + +// LabelSet defines a set of labels implemented using a map. +type LabelSet map[BuildLabel]struct{} + +func (ls LabelSet) Add(l BuildLabel) { + ls[l] = struct{}{} +} + +func (ls LabelSet) Contains(l BuildLabel) bool { + _, ok := ls[l] + return ok +} diff --git a/src/core/graph.go b/src/core/graph.go index f3f9705e9e..c73961a0e1 100644 --- a/src/core/graph.go +++ b/src/core/graph.go @@ -13,17 +13,6 @@ import ( "github.com/thought-machine/please/src/cmap" ) -type labelSet map[BuildLabel]struct{} - -func (ls labelSet) add(l BuildLabel) { - ls[l] = struct{}{} -} - -func (ls labelSet) contains(l BuildLabel) bool { - _, ok := ls[l] - return ok -} - // A BuildGraph contains all the loaded targets and packages and maintains their // relationships, especially reverse dependencies which are calculated here. type BuildGraph struct { @@ -34,8 +23,8 @@ type BuildGraph struct { // Registered subrepos, as a map of their name to their root. subrepos *cmap.Map[string, *Subrepo] // Subincludes that are subincluded by other subincludes - subincludeSubincludes map[BuildLabel]labelSet - // Use a mutex as a labelSet isn't atomic. We need to guard against inserting as well as mutating the value. + subincludeSubincludes map[BuildLabel]LabelSet + // Use a mutex as a LabelSet isn't atomic. We need to guard against inserting as well as mutating the value. subincMux sync.Mutex } @@ -168,7 +157,7 @@ func NewGraph() *BuildGraph { targets: cmap.New[BuildLabel, *BuildTarget](cmap.DefaultShardCount, hashBuildLabel), packages: cmap.New[packageKey, *Package](cmap.DefaultShardCount, hashPackageKey), subrepos: cmap.New[string, *Subrepo](cmap.SmallShardCount, cmap.XXHash), - subincludeSubincludes: map[BuildLabel]labelSet{}, + subincludeSubincludes: map[BuildLabel]LabelSet{}, } return g } @@ -188,7 +177,7 @@ func (graph *BuildGraph) TransitiveSubincludes(l BuildLabel) []BuildLabel { graph.subincMux.Lock() defer graph.subincMux.Unlock() - incs := labelSet{} + incs := LabelSet{} graph.findTransitiveSubincludes(l, incs) ls := slices.Collect(maps.Keys(incs)) @@ -196,11 +185,11 @@ func (graph *BuildGraph) TransitiveSubincludes(l BuildLabel) []BuildLabel { return ls } -func (graph *BuildGraph) findTransitiveSubincludes(label BuildLabel, includes labelSet) { - if includes.contains(label) { +func (graph *BuildGraph) findTransitiveSubincludes(label BuildLabel, includes LabelSet) { + if includes.Contains(label) { return } - includes.add(label) + includes.Add(label) for l := range graph.subincludeSubincludes[label] { graph.findTransitiveSubincludes(l, includes) } @@ -212,8 +201,8 @@ func (graph *BuildGraph) RegisterTransitiveSubinclude(from, to BuildLabel) { incs, ok := graph.subincludeSubincludes[from] if !ok { - incs = labelSet{} + incs = LabelSet{} graph.subincludeSubincludes[from] = incs } - incs.add(to) + incs.Add(to) } diff --git a/src/core/package.go b/src/core/package.go index 9a510750ed..c7720b7add 100644 --- a/src/core/package.go +++ b/src/core/package.go @@ -128,11 +128,11 @@ func (pkg *Package) RegisterSubinclude(label BuildLabel) { // AllSubincludes returns the full set of subincludes needed for this package, including transitive subincludes func (pkg *Package) AllSubincludes(graph *BuildGraph) []BuildLabel { - includes := make(labelSet, len(pkg.Subincludes)) + includes := make(LabelSet, len(pkg.Subincludes)) for _, s := range pkg.Subincludes { for _, inc := range append(graph.TransitiveSubincludes(s), s) { - includes.add(inc) + includes.Add(inc) } } diff --git a/test/export/test_native_target_with_go_dep/source_repo/third_party/go/BUILD_FILE b/test/export/test_native_target_with_go_dep/source_repo/third_party/go/BUILD_FILE index 503411e7c9..54a9c020b5 100644 --- a/test/export/test_native_target_with_go_dep/source_repo/third_party/go/BUILD_FILE +++ b/test/export/test_native_target_with_go_dep/source_repo/third_party/go/BUILD_FILE @@ -22,14 +22,14 @@ go_stdlib( name = "std", ) -go_repo( - module = "github.com/google/go-cmp", - version = "v0.5.6", -) - go_repo( # unneeded, unused target name = "uuid", module = "github.com/google/uuid", version = f"{LIB_VERSION}", ) + +go_repo( + module = "github.com/google/go-cmp", + version = "v0.5.6", +) From 715210956722dc04e60d7a87b57e360da70d2022 Mon Sep 17 00:00:00 2001 From: DuBento Date: Tue, 16 Jun 2026 18:59:19 +0100 Subject: [PATCH 093/118] track used objects as a stack previous implementation was append only meaning that it would mark a subinclude as required if it was visited before even if not exported. --- src/parse/asp/interpreter.go | 98 +++++++++++++------ src/parse/asp/interpreter_test.go | 11 ++- src/parse/asp/objects.go | 23 ++--- .../test_custom_def/source_repo/BUILD_FILE | 18 +++- .../source_repo/build_defs/BUILD_FILE | 4 +- .../{dummy.build_defs => unneeded.build_defs} | 4 +- 6 files changed, 103 insertions(+), 55 deletions(-) rename test/export/test_custom_def/source_repo/build_defs/{dummy.build_defs => unneeded.build_defs} (64%) diff --git a/src/parse/asp/interpreter.go b/src/parse/asp/interpreter.go index 4f783e963a..28e6809210 100644 --- a/src/parse/asp/interpreter.go +++ b/src/parse/asp/interpreter.go @@ -487,19 +487,19 @@ func (s *scope) NAssert(condition bool, msg string, args ...interface{}) { // Lookup looks up a variable name in this scope, walking back up its ancestor scopes as needed. // It panics if the variable is not defined. func (s *scope) Lookup(name string) pyObject { - obj, origin := s.lookupWithOrigin(name) - s.metadata.SetRequiredOrigin(origin) + obj := s.lookup(name) + s.metadata.PushObjectStack(name) return obj } -// lookupWithOrigin is like Lookup but returns the origin label of the variable as well. -func (s *scope) lookupWithOrigin(name string) (pyObject, *core.BuildLabel) { +// lookup implements the recursive lookup over parent scopes. +func (s *scope) lookup(name string) pyObject { if obj, present := s.locals[name]; present { - return obj, s.metadata.Origin(s, name) + return obj } else if s.parent != nil { - return s.parent.lookupWithOrigin(name) + return s.parent.lookup(name) } - return s.Error("name '%s' is not defined", name), nil + return s.Error("name '%s' is not defined", name) } // LocalLookup looks up a variable name in the current scope. @@ -1091,6 +1091,9 @@ func (s *scope) callObject(name string, obj pyObject, c *Call) pyObject { if !ok { s.Error("Non-callable object '%s' (is a %s)", name, obj.Type()) } + sizeBeforeCall := s.metadata.SizeObjectStack() + // Pop the arguments visited during the call + the function object implicitly added by [Lookup] (-1). + defer s.metadata.ResizeObjectStack(sizeBeforeCall - 1) return f.Call(s, c) } @@ -1150,11 +1153,11 @@ func (s *scope) ActiveSubincludes() core.SubincludesLabelProvider { return func() core.BuildLabels { // We walk back on the callstack. For each scope of a method call we lookup the // subinclude labels marked as used, meaning values from those subincluded labels were used to generate this target, be it function defs or variables. - seen := map[core.BuildLabel]struct{}{} + collector := core.LabelSet{} for callScope := s; callScope != nil; callScope = callScope.caller { - maps.Copy(seen, callScope.metadata.RequiredOrigins()) + callScope.metadata.RequiredOrigins(callScope, &collector) } - return slices.Collect(maps.Keys(seen)) + return slices.Collect(maps.Keys(collector)) } } @@ -1177,33 +1180,38 @@ type ScopeMetadata interface { Cursor() *Statement // Origin gets the origin of the object by name. Should return nil if not found or unimplemented. Origin(scope *scope, name string) *core.BuildLabel - // RequiredOrigins returns a set of all the origins (subincluded labels) required by the current - // scope. - RequiredOrigins() map[core.BuildLabel]struct{} + // RequiredOrigins adds all the origins (subincluded labels) required by the current + // scope into the collector set. + RequiredOrigins(scope *scope, collector *core.LabelSet) // SetCursor register the statement currently being interpreted. SetCursor(stmt *Statement) // SetObjectOrigin registers the origin for the given named object. The origin is the label of a // subincluded target. SetObjectOrigin(name string, origin core.BuildLabel) - // SetRequiredOrigin marks the named object as required by the current scope. - SetRequiredOrigin(origin *core.BuildLabel) + // PushObjectStack adds a named object to the stacks of used objects. + PushObjectStack(name string) + // ResizeObjectStack shrinks the stack to size n, removing the last objects from the stack. + ResizeObjectStack(n int) + // SizeObjectStack returns the size of the object stack, meaning the number of objects registered. + SizeObjectStack() int } type scopeMetadata struct { - // cursor points to the statement currently being interpreted + // cursor points to the statement currently being interpreted. cursor *Statement // objectOrigins tracks the subinclude label that each variable was originally defined in. objectOrigins map[string]core.BuildLabel - // requiredOrigins tracks which subinclude labels have been used by looking up objects. - requiredOrigins map[core.BuildLabel]struct{} + // objectStack tracks which objects are in use for this scope. Objects are added to the stack + // after lookup and can be removed for example to cleanup arguments after a function call. + objectStack []string } // NewMetadata implements [ScopeMetadata]. func (m *scopeMetadata) NewMetadata() ScopeMetadata { return &scopeMetadata{ - cursor: m.cursor, - objectOrigins: map[string]core.BuildLabel{}, - requiredOrigins: map[core.BuildLabel]struct{}{}, + cursor: m.cursor, + objectOrigins: map[string]core.BuildLabel{}, + objectStack: []string{}, } } @@ -1215,17 +1223,31 @@ func (m *scopeMetadata) Cursor() *Statement { // Origin implements [ScopeMetadata]. func (m *scopeMetadata) Origin(scope *scope, name string) *core.BuildLabel { if scope.interpreter != nil && scope.interpreter.preloaded.Contains(name) { + // Preloaded object. return nil } if label, ok := m.objectOrigins[name]; ok { + // Object subincluded into current scope. return &label } + if scope != nil && scope.parent != nil { + // Recursive lookup in parent scopes (previously subincluded). + return scope.parent.metadata.Origin(scope.parent, name) + } + + // The origin for a local object is set to nil return nil } // RequiredOrigins implements [ScopeMetadata]. -func (m *scopeMetadata) RequiredOrigins() map[core.BuildLabel]struct{} { - return m.requiredOrigins +func (m *scopeMetadata) RequiredOrigins(scope *scope, collector *core.LabelSet) { + for _, object := range m.objectStack { + orig := m.Origin(scope, object) + if orig == nil { + continue + } + collector.Add(*orig) + } } // SetCursor implements [ScopeMetadata]. @@ -1238,12 +1260,22 @@ func (m *scopeMetadata) SetObjectOrigin(name string, origin core.BuildLabel) { m.objectOrigins[name] = origin } -// SetRequiredOrigin implements [ScopeMetadata]. -func (m *scopeMetadata) SetRequiredOrigin(origin *core.BuildLabel) { - if origin == nil { +// PushObjectStack implements [ScopeMetadata]. +func (m *scopeMetadata) PushObjectStack(name string) { + m.objectStack = append(m.objectStack, name) +} + +// ResizeObjectStack implements [ScopeMetadata]. +func (m *scopeMetadata) ResizeObjectStack(size int) { + if size < 0 || size > len(m.objectOrigins) { return } - m.requiredOrigins[*origin] = struct{}{} + m.objectStack = m.objectStack[:size] +} + +// SizeObjectStack implements [ScopeMetadata]. +func (m *scopeMetadata) SizeObjectStack() int { + return len(m.objectStack) } // noopScopeMetadata implements the ScopeMetadata interface with no-op methods. This is used to @@ -1260,7 +1292,7 @@ func (nm *noopScopeMetadata) Cursor() *Statement { return nil } func (nm *noopScopeMetadata) Origin(scope *scope, name string) *core.BuildLabel { return nil } // RequiredOrigins implements [ScopeMetadata]. -func (nm *noopScopeMetadata) RequiredOrigins() map[core.BuildLabel]struct{} { return nil } +func (nm *noopScopeMetadata) RequiredOrigins(scope *scope, collector *core.LabelSet) {} // SetCursor implements [ScopeMetadata]. func (nm *noopScopeMetadata) SetCursor(stmt *Statement) {} @@ -1268,8 +1300,14 @@ func (nm *noopScopeMetadata) SetCursor(stmt *Statement) {} // SetObjectOrigin implements [ScopeMetadata]. func (nm *noopScopeMetadata) SetObjectOrigin(name string, origin core.BuildLabel) {} -// SetRequiredOrigin implements [ScopeMetadata]. -func (nm *noopScopeMetadata) SetRequiredOrigin(origin *core.BuildLabel) {} +// PushObjectStack implements [ScopeMetadata]. +func (nm *noopScopeMetadata) PushObjectStack(name string) {} + +// ResizeObjectStack implements [ScopeMetadata]. +func (m *noopScopeMetadata) ResizeObjectStack(n int) {} + +// SizeObjectStack implements [ScopeMetadata]. +func (nm *noopScopeMetadata) SizeObjectStack() int { return 0 } // NewBuildStatement creates a new core.BuildStatement from an asp.statement. func NewBuildStatement(stmt *Statement) core.BuildStatement { diff --git a/src/parse/asp/interpreter_test.go b/src/parse/asp/interpreter_test.go index 2f6267ef8f..4b8fbb89cf 100644 --- a/src/parse/asp/interpreter_test.go +++ b/src/parse/asp/interpreter_test.go @@ -814,6 +814,10 @@ func TestCurrentBuildStatement(t *testing.T) { } defsNestedScope.metadata.SetCursor(defsNestedStmt) + // A scope that has no pkg/filename context + standaloneScope := &scope{metadata: newScopeMetadata()} + standaloneScope.metadata.SetCursor(rootStmt) + t.Run("FindsRootStatementFromBUILD", func(t *testing.T) { // Calling it from buildNestedScope should walk back to buildRootScope stmt := nestedScope.CurrentBuildStatement()() @@ -827,9 +831,6 @@ func TestCurrentBuildStatement(t *testing.T) { }) t.Run("HandlesNoPackageFileInStack", func(t *testing.T) { - // A scope that has no pkg/filename context - standaloneScope := &scope{metadata: newScopeMetadata()} - standaloneScope.metadata.SetCursor(rootStmt) stmt := standaloneScope.CurrentBuildStatement()() assert.Equal(t, NewBuildStatement(rootStmt), stmt) }) @@ -930,7 +931,7 @@ func TestActiveSubincludes(t *testing.T) { func newScopeMetadata() ScopeMetadata { return &scopeMetadata{ - objectOrigins: map[string]core.BuildLabel{}, - requiredOrigins: map[core.BuildLabel]struct{}{}, + objectOrigins: map[string]core.BuildLabel{}, + objectStack: []string{}, } } diff --git a/src/parse/asp/objects.go b/src/parse/asp/objects.go index 9b62a65817..68b4ee82a0 100644 --- a/src/parse/asp/objects.go +++ b/src/parse/asp/objects.go @@ -698,12 +698,13 @@ func (f *pyFunc) Call(s *scope, c *Call) pyObject { } return f.callNative(s, c) } - s2 := f.scope.newScope(s.pkg, s.mode, f.scope.filename, len(f.args)+1) - s2.caller = s // registering previous scope as caller - s2.config = s.config - s2.Set("CONFIG", s.config) // This needs to be copied across too :( - s2.Callback = s.Callback - s2.parsingFor = s.parsingFor + + callScope := f.scope.newScope(s.pkg, s.mode, f.scope.filename, len(f.args)+1) + callScope.caller = s // registering previous scope as caller + callScope.config = s.config + callScope.Set("CONFIG", s.config) // This needs to be copied across too :( + callScope.Callback = s.Callback + callScope.parsingFor = s.parsingFor // Handle implicit 'self' parameter for bound functions. args := c.Arguments if f.self != nil { @@ -721,23 +722,23 @@ func (f *pyFunc) Call(s *scope, c *Call) pyObject { if present { name = f.args[idx] } - s2.Set(name, f.validateType(s, idx, &a.Value)) + callScope.Set(name, f.validateType(s, idx, &a.Value)) } else { if i >= len(f.args) { s.Error("Too many arguments to %s", f.name) } else if f.kwargsonly { s.Error("Function %s can only be called with keyword arguments", f.name) } - s2.Set(f.args[i], f.validateType(s, i, &a.Value)) + callScope.Set(f.args[i], f.validateType(s, i, &a.Value)) } } // Now make sure any arguments with defaults are set, and check any others have been passed. for i, a := range f.args { - if s2.LocalLookup(a) == nil { - s2.Set(a, f.defaultArg(s, i, a)) + if callScope.LocalLookup(a) == nil { + callScope.Set(a, f.defaultArg(s, i, a)) } } - ret := s2.interpretStatements(f.code) + ret := callScope.interpretStatements(f.code) if ret == nil { return None // Implicit 'return None' in any function that didn't do that itself. } diff --git a/test/export/test_custom_def/source_repo/BUILD_FILE b/test/export/test_custom_def/source_repo/BUILD_FILE index 4ca3ee04a5..f74b40a8e3 100644 --- a/test/export/test_custom_def/source_repo/BUILD_FILE +++ b/test/export/test_custom_def/source_repo/BUILD_FILE @@ -1,13 +1,21 @@ -subinclude("//build_defs:simple_build_def") +subinclude( + "//build_defs:simple_build_def", + "//build_defs:unneeded_build_def", +) simple_custom_target( - name = "simple_custom_target", + name = "unneeded", srcs = ["file.txt"], - outs = ["file_simple.out"], + outs = ["unneeded.out"], +) + +unneeded_target( + name = "unneeded2", + outs = ["unneeded2.out"], ) simple_custom_target( - name = "unneded", + name = "simple_custom_target", srcs = ["file.txt"], - outs = ["unneded.out"], + outs = ["file_simple.out"], ) diff --git a/test/export/test_custom_def/source_repo/build_defs/BUILD_FILE b/test/export/test_custom_def/source_repo/build_defs/BUILD_FILE index 2450ba9afd..2aeb84543f 100644 --- a/test/export/test_custom_def/source_repo/build_defs/BUILD_FILE +++ b/test/export/test_custom_def/source_repo/build_defs/BUILD_FILE @@ -5,7 +5,7 @@ filegroup( ) filegroup( - name = "unneded_build_def", - srcs = ["unneded.build_defs"], + name = "unneeded_build_def", + srcs = ["unneeded.build_defs"], visibility = ["PUBLIC"], ) diff --git a/test/export/test_custom_def/source_repo/build_defs/dummy.build_defs b/test/export/test_custom_def/source_repo/build_defs/unneeded.build_defs similarity index 64% rename from test/export/test_custom_def/source_repo/build_defs/dummy.build_defs rename to test/export/test_custom_def/source_repo/build_defs/unneeded.build_defs index 17b55895be..5117901a85 100644 --- a/test/export/test_custom_def/source_repo/build_defs/dummy.build_defs +++ b/test/export/test_custom_def/source_repo/build_defs/unneeded.build_defs @@ -1,8 +1,8 @@ -def unneded_target( +def unneeded_target( name:str, outs:list=[]): return genrule( name = name, outs = outs, - cmd = "echo unneded > $OUT", + cmd = "echo unneeded > $OUT", ) From 50a82406003be0846f5ac7526a55fe1ce82ddd37 Mon Sep 17 00:00:00 2001 From: DuBento Date: Wed, 17 Jun 2026 18:27:16 +0100 Subject: [PATCH 094/118] scopeMetadata: private, rename and comments --- src/parse/asp/interpreter.go | 258 +++++++++++++++++------------- src/parse/asp/interpreter_test.go | 18 +-- 2 files changed, 157 insertions(+), 119 deletions(-) diff --git a/src/parse/asp/interpreter.go b/src/parse/asp/interpreter.go index 28e6809210..89a85cc50c 100644 --- a/src/parse/asp/interpreter.go +++ b/src/parse/asp/interpreter.go @@ -50,7 +50,7 @@ func newInterpreter(state *core.BuildState, p *Parser) *interpreter { metadata: &noopScopeMetadata{}, } if state.ParseMetadata { - s.metadata = &scopeMetadata{} + s.metadata = newTrackingScopeMetadata() } i := &interpreter{ @@ -339,7 +339,7 @@ type scope struct { // True if this scope is for a pre- or post-build callback. Callback bool mode core.ParseMode - metadata ScopeMetadata + metadata scopeMetadata } // parseAnnotatedLabelInPackage similarly to parseLabelInPackage, parses the label contextualising it to the provided @@ -456,7 +456,7 @@ func (s *scope) newScope(pkg *core.Package, mode core.ParseMode, filename string config: s.config, Callback: s.Callback, mode: mode, - metadata: s.metadata.NewMetadata(), + metadata: s.metadata.newMetadata(), } if pkg != nil && pkg.Subrepo != nil && pkg.Subrepo.State != nil { s2.state = pkg.Subrepo.State @@ -487,19 +487,19 @@ func (s *scope) NAssert(condition bool, msg string, args ...interface{}) { // Lookup looks up a variable name in this scope, walking back up its ancestor scopes as needed. // It panics if the variable is not defined. func (s *scope) Lookup(name string) pyObject { - obj := s.lookup(name) - s.metadata.PushObjectStack(name) + obj, orig := s.lookupWithOrigin(name) + s.metadata.pushSymbol(name, orig) return obj } // lookup implements the recursive lookup over parent scopes. -func (s *scope) lookup(name string) pyObject { +func (s *scope) lookupWithOrigin(name string) (pyObject, *core.BuildLabel) { if obj, present := s.locals[name]; present { - return obj + return obj, s.metadata.origin(s, name) } else if s.parent != nil { - return s.parent.lookup(name) + return s.parent.lookupWithOrigin(name) } - return s.Error("name '%s' is not defined", name) + return s.Error("name '%s' is not defined", name), nil } // LocalLookup looks up a variable name in the current scope. @@ -532,7 +532,7 @@ func (s *scope) SetAllWithOrigin(d pyDict, publicOnly bool, origin *core.BuildLa } else if !publicOnly || k[0] != '_' { s.locals[k] = v if origin != nil { - s.metadata.SetObjectOrigin(k, *origin) + s.metadata.setSymbolOrigin(k, *origin) } } } @@ -571,7 +571,7 @@ func (s *scope) interpretStatements(statements []*Statement) pyObject { } }() for _, stmt = range statements { - s.metadata.SetCursor(stmt) + s.metadata.setCursor(stmt) if stmt.FuncDef != nil { s.Set(stmt.FuncDef.Name, newPyFunc(s, stmt.FuncDef)) } else if stmt.If != nil { @@ -1091,9 +1091,15 @@ func (s *scope) callObject(name string, obj pyObject, c *Call) pyObject { if !ok { s.Error("Non-callable object '%s' (is a %s)", name, obj.Type()) } - sizeBeforeCall := s.metadata.SizeObjectStack() - // Pop the arguments visited during the call + the function object implicitly added by [Lookup] (-1). - defer s.metadata.ResizeObjectStack(sizeBeforeCall - 1) + + // Restore the pre-call checkpoint first, then pop the function object itself to ensure explicit sequential cleanup. + // Remove the arguments visited during the call + the function object implicitly added by [Lookup]. + checkpoint := s.metadata.checkpointSymbolStack() + defer func() { + s.metadata.restoreCheckpoint(checkpoint) + s.metadata.popSymbol(name) + }() + return f.Call(s, c) } @@ -1141,8 +1147,8 @@ func (s *scope) CurrentBuildStatement() core.BuildStatementProvider { stmtScope = curr } } - s.NAssert(stmtScope.metadata.Cursor() == nil, "Cursor is not pointing to a statement") - return NewBuildStatement(stmtScope.metadata.Cursor()) + s.NAssert(stmtScope.metadata.cursor() == nil, "Cursor is not pointing to a statement") + return NewBuildStatement(stmtScope.metadata.cursor()) } } @@ -1155,7 +1161,7 @@ func (s *scope) ActiveSubincludes() core.SubincludesLabelProvider { // subinclude labels marked as used, meaning values from those subincluded labels were used to generate this target, be it function defs or variables. collector := core.LabelSet{} for callScope := s; callScope != nil; callScope = callScope.caller { - callScope.metadata.RequiredOrigins(callScope, &collector) + callScope.metadata.requiredOrigins(callScope, &collector) } return slices.Collect(maps.Keys(collector)) } @@ -1169,145 +1175,177 @@ func (s *scope) pkgFilename() string { return "" } -// scopeMetadata maintains additional information generated during the interpretation phase. +// scopeMetadata defines an interface for tracking evaluation metadata (such as AST cursor position +// and symbol subinclude origins) across interpreter scopes. // This is optionally used for operations (e.g. export) that require more details on the relation // between targets and statements. The no-op implementation should be used for most operations to // avoid any computational overhead. -type ScopeMetadata interface { - // NewMetadata creates a new instance of the same ScopeMetadata implementation type. - NewMetadata() ScopeMetadata - // Cursor returns the statement being currently interpreted. - Cursor() *Statement - // Origin gets the origin of the object by name. Should return nil if not found or unimplemented. - Origin(scope *scope, name string) *core.BuildLabel - // RequiredOrigins adds all the origins (subincluded labels) required by the current - // scope into the collector set. - RequiredOrigins(scope *scope, collector *core.LabelSet) - // SetCursor register the statement currently being interpreted. - SetCursor(stmt *Statement) - // SetObjectOrigin registers the origin for the given named object. The origin is the label of a - // subincluded target. - SetObjectOrigin(name string, origin core.BuildLabel) - // PushObjectStack adds a named object to the stacks of used objects. - PushObjectStack(name string) - // ResizeObjectStack shrinks the stack to size n, removing the last objects from the stack. - ResizeObjectStack(n int) - // SizeObjectStack returns the size of the object stack, meaning the number of objects registered. - SizeObjectStack() int -} - -type scopeMetadata struct { +type scopeMetadata interface { + // newMetadata creates a new instance of the same scopeMetadata implementation type. + newMetadata() scopeMetadata + // cursor returns the statement currently being interpreted. + cursor() *Statement + // origin returns the subinclude origin label of a tracked symbol by name. Returns nil if the + // symbol is local (defined in the package) or has no origin/not tracked. + origin(scope *scope, name string) *core.BuildLabel + // requiredOrigins aggregates all subinclude origins currently in the active symbol stack + // into the provided label set. + requiredOrigins(scope *scope, collector *core.LabelSet) + // setCursor registers the statement currently being interpreted. + setCursor(stmt *Statement) + // setSymbolOrigin registers the subinclude origin label for a defined symbol. + setSymbolOrigin(name string, origin core.BuildLabel) + // pushSymbol pushes a symbol name and its subinclude origin onto the active tracking stack. + pushSymbol(name string, origin *core.BuildLabel) + // popSymbol pops the specified symbol from the top of the tracking stack if it matches the name. + popSymbol(name string) + // checkpointSymbolStack checkpoints the current size of the symbol tracking stack. + checkpointSymbolStack() int + // restoreCheckpoint restores the symbol tracking stack back to the given checkpoint size, + // discarding any symbols pushed after the checkpoint was taken. + restoreCheckpoint(checkpoint int) +} + +// trackingScopeMetadata implements the interface [scopeMetadata]. +type trackingScopeMetadata struct { // cursor points to the statement currently being interpreted. - cursor *Statement - // objectOrigins tracks the subinclude label that each variable was originally defined in. - objectOrigins map[string]core.BuildLabel - // objectStack tracks which objects are in use for this scope. Objects are added to the stack - // after lookup and can be removed for example to cleanup arguments after a function call. - objectStack []string + cursorField *Statement + // symbolOrigins tracks the subinclude label that each symbol was originally defined in. + symbolOrigins map[string]core.BuildLabel + // symbolStack tracks which symbols are actively in use during evaluation. + // Symbols are pushed onto the stack during lookups and popped or truncated (restored) after + // function calls. + symbolStack []trackedSymbol } -// NewMetadata implements [ScopeMetadata]. -func (m *scopeMetadata) NewMetadata() ScopeMetadata { - return &scopeMetadata{ - cursor: m.cursor, - objectOrigins: map[string]core.BuildLabel{}, - objectStack: []string{}, - } +type trackedSymbol struct { + name string + origin *core.BuildLabel } -// Cursor implements [ScopeMetadata]. -func (m *scopeMetadata) Cursor() *Statement { - return m.cursor +// newMetadata implements [scopeMetadata]. +func (m *trackingScopeMetadata) newMetadata() scopeMetadata { + meta := newTrackingScopeMetadata() + meta.cursorField = m.cursorField + return meta } -// Origin implements [ScopeMetadata]. -func (m *scopeMetadata) Origin(scope *scope, name string) *core.BuildLabel { +// cursor implements [scopeMetadata]. +func (m *trackingScopeMetadata) cursor() *Statement { + return m.cursorField +} + +// origin implements [scopeMetadata]. +func (m *trackingScopeMetadata) origin(scope *scope, name string) *core.BuildLabel { if scope.interpreter != nil && scope.interpreter.preloaded.Contains(name) { - // Preloaded object. + // Preloaded symbols are treated as local (returning nil origin) because they are implicitly + // available across all package scopes in the repository. + // + // This also prevents erroneous subinclude propagation: since symbol resolution recursively + // traverses the parent scope chain from bottom to top, where the preoloaded symbols are + // defined at the top, if a target subincludes a preloaded target again it will be prefered + // over the preloaded and will potetntially include unwanted symbols so we enforce a + // preference for the preloaded symbols. This could cause issues if our repo relies on + // redefining preloaded symbols. return nil } - if label, ok := m.objectOrigins[name]; ok { + if label, ok := m.symbolOrigins[name]; ok { // Object subincluded into current scope. return &label } - if scope != nil && scope.parent != nil { - // Recursive lookup in parent scopes (previously subincluded). - return scope.parent.metadata.Origin(scope.parent, name) - } - // The origin for a local object is set to nil return nil } -// RequiredOrigins implements [ScopeMetadata]. -func (m *scopeMetadata) RequiredOrigins(scope *scope, collector *core.LabelSet) { - for _, object := range m.objectStack { - orig := m.Origin(scope, object) - if orig == nil { - continue - } - collector.Add(*orig) +// requiredOrigins implements [scopeMetadata]. +func (m *trackingScopeMetadata) requiredOrigins(scope *scope, collector *core.LabelSet) { + for _, v := range m.symbolStack { + collector.Add(*v.origin) } } -// SetCursor implements [ScopeMetadata]. -func (m *scopeMetadata) SetCursor(stmt *Statement) { - m.cursor = stmt +// setCursor implements [scopeMetadata]. +func (m *trackingScopeMetadata) setCursor(stmt *Statement) { + m.cursorField = stmt } -// SetObjectOrigin implements [ScopeMetadata]. -func (m *scopeMetadata) SetObjectOrigin(name string, origin core.BuildLabel) { - m.objectOrigins[name] = origin +// setSymbolOrigin implements [scopeMetadata]. +func (m *trackingScopeMetadata) setSymbolOrigin(name string, origin core.BuildLabel) { + m.symbolOrigins[name] = origin } -// PushObjectStack implements [ScopeMetadata]. -func (m *scopeMetadata) PushObjectStack(name string) { - m.objectStack = append(m.objectStack, name) +// pushSymbol implements [scopeMetadata]. +func (m *trackingScopeMetadata) pushSymbol(name string, origin *core.BuildLabel) { + if origin == nil { + return + } + m.symbolStack = append(m.symbolStack, trackedSymbol{name: name, origin: origin}) } -// ResizeObjectStack implements [ScopeMetadata]. -func (m *scopeMetadata) ResizeObjectStack(size int) { - if size < 0 || size > len(m.objectOrigins) { +// popSymbol implements [scopeMetadata]. +func (m *trackingScopeMetadata) popSymbol(name string) { + if name == "" || len(m.symbolStack) == 0 { return } - m.objectStack = m.objectStack[:size] + if m.symbolStack[len(m.symbolStack)-1].name == name { + m.symbolStack = m.symbolStack[:len(m.symbolStack)-1] + } } -// SizeObjectStack implements [ScopeMetadata]. -func (m *scopeMetadata) SizeObjectStack() int { - return len(m.objectStack) +// checkpointSymbolStack implements [scopeMetadata]. +func (m *trackingScopeMetadata) checkpointSymbolStack() int { + return len(m.symbolStack) } -// noopScopeMetadata implements the ScopeMetadata interface with no-op methods. This is used to +// restoreCheckpoint implements [scopeMetadata]. +func (m *trackingScopeMetadata) restoreCheckpoint(checkpoint int) { + if checkpoint < 0 || checkpoint > len(m.symbolStack) { + return + } + m.symbolStack = m.symbolStack[:checkpoint] +} + +// noopScopeMetadata implements the scopeMetadata interface with no-op methods. This is used to // avoid the overhead of storing metadata for operations that don't depend on it. type noopScopeMetadata struct{} -// NewMetadata implements [ScopeMetadata]. -func (nm *noopScopeMetadata) NewMetadata() ScopeMetadata { return &noopScopeMetadata{} } +// restoreCheckpoint implements [scopeMetadata]. +func (nm *noopScopeMetadata) restoreCheckpoint(checkpoint int) {} + +// checkpointSymbolStack implements [scopeMetadata]. +func (nm *noopScopeMetadata) checkpointSymbolStack() int { return 0 } + +// newMetadata implements [scopeMetadata]. +func (nm *noopScopeMetadata) newMetadata() scopeMetadata { return &noopScopeMetadata{} } -// Cursor implements [ScopeMetadata]. -func (nm *noopScopeMetadata) Cursor() *Statement { return nil } +// cursor implements [scopeMetadata]. +func (nm *noopScopeMetadata) cursor() *Statement { return nil } -// Origin implements [ScopeMetadata]. -func (nm *noopScopeMetadata) Origin(scope *scope, name string) *core.BuildLabel { return nil } +// origin implements [scopeMetadata]. +func (nm *noopScopeMetadata) origin(scope *scope, name string) *core.BuildLabel { return nil } -// RequiredOrigins implements [ScopeMetadata]. -func (nm *noopScopeMetadata) RequiredOrigins(scope *scope, collector *core.LabelSet) {} +// requiredOrigins implements [scopeMetadata]. +func (nm *noopScopeMetadata) requiredOrigins(scope *scope, collector *core.LabelSet) {} -// SetCursor implements [ScopeMetadata]. -func (nm *noopScopeMetadata) SetCursor(stmt *Statement) {} +// setCursor implements [scopeMetadata]. +func (nm *noopScopeMetadata) setCursor(stmt *Statement) {} -// SetObjectOrigin implements [ScopeMetadata]. -func (nm *noopScopeMetadata) SetObjectOrigin(name string, origin core.BuildLabel) {} +// setSymbolOrigin implements [scopeMetadata]. +func (nm *noopScopeMetadata) setSymbolOrigin(name string, origin core.BuildLabel) {} -// PushObjectStack implements [ScopeMetadata]. -func (nm *noopScopeMetadata) PushObjectStack(name string) {} +// pushSymbol implements [scopeMetadata]. +func (nm *noopScopeMetadata) pushSymbol(name string, origin *core.BuildLabel) {} -// ResizeObjectStack implements [ScopeMetadata]. -func (m *noopScopeMetadata) ResizeObjectStack(n int) {} +// popSymbol implements [scopeMetadata]. +func (nm *noopScopeMetadata) popSymbol(name string) {} -// SizeObjectStack implements [ScopeMetadata]. -func (nm *noopScopeMetadata) SizeObjectStack() int { return 0 } +// newTrackingScopeMetadata creates and returns a fully initialized trackingScopeMetadata instance. +func newTrackingScopeMetadata() *trackingScopeMetadata { + return &trackingScopeMetadata{ + symbolOrigins: map[string]core.BuildLabel{}, + symbolStack: []trackedSymbol{}, + } +} // NewBuildStatement creates a new core.BuildStatement from an asp.statement. func NewBuildStatement(stmt *Statement) core.BuildStatement { diff --git a/src/parse/asp/interpreter_test.go b/src/parse/asp/interpreter_test.go index 4b8fbb89cf..f496becfd4 100644 --- a/src/parse/asp/interpreter_test.go +++ b/src/parse/asp/interpreter_test.go @@ -782,7 +782,7 @@ func TestCurrentBuildStatement(t *testing.T) { filename: pkg.Filename, metadata: newScopeMetadata(), } - rootScope.metadata.SetCursor(rootStmt) + rootScope.metadata.setCursor(rootStmt) // A nested call inside the same BUILD file (e.g. function def) nestedStmt := &Statement{Pos: 30, EndPos: 40} @@ -792,7 +792,7 @@ func TestCurrentBuildStatement(t *testing.T) { caller: rootScope, metadata: newScopeMetadata(), } - nestedScope.metadata.SetCursor(nestedStmt) + nestedScope.metadata.setCursor(nestedStmt) // A call from a different file (e.g. a function inside a subincluded .build_defs file) defsRootStmt := &Statement{Pos: 50, EndPos: 60} @@ -802,7 +802,7 @@ func TestCurrentBuildStatement(t *testing.T) { caller: nestedScope, metadata: newScopeMetadata(), } - defsRootScope.metadata.SetCursor(defsRootStmt) + defsRootScope.metadata.setCursor(defsRootStmt) // Another call deep in the other file defsNestedStmt := &Statement{Pos: 70, EndPos: 80} @@ -812,11 +812,11 @@ func TestCurrentBuildStatement(t *testing.T) { caller: defsRootScope, metadata: newScopeMetadata(), } - defsNestedScope.metadata.SetCursor(defsNestedStmt) + defsNestedScope.metadata.setCursor(defsNestedStmt) // A scope that has no pkg/filename context standaloneScope := &scope{metadata: newScopeMetadata()} - standaloneScope.metadata.SetCursor(rootStmt) + standaloneScope.metadata.setCursor(rootStmt) t.Run("FindsRootStatementFromBUILD", func(t *testing.T) { // Calling it from buildNestedScope should walk back to buildRootScope @@ -929,9 +929,9 @@ func TestActiveSubincludes(t *testing.T) { }) } -func newScopeMetadata() ScopeMetadata { - return &scopeMetadata{ - objectOrigins: map[string]core.BuildLabel{}, - objectStack: []string{}, +func newScopeMetadata() scopeMetadata { + return &trackingScopeMetadata{ + symbolOrigins: map[string]core.BuildLabel{}, + symbolStack: []trackedSymbol{}, } } From 51cbb35a2e28b41ee631760cdeba177cff3f3762 Mon Sep 17 00:00:00 2001 From: DuBento Date: Wed, 17 Jun 2026 18:28:40 +0100 Subject: [PATCH 095/118] rename active to required subincludes --- src/parse/asp/builtins.go | 2 +- src/parse/asp/interpreter.go | 4 ++-- src/parse/asp/interpreter_test.go | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/parse/asp/builtins.go b/src/parse/asp/builtins.go index 6a57dbce99..e1b5a2911e 100644 --- a/src/parse/asp/builtins.go +++ b/src/parse/asp/builtins.go @@ -211,7 +211,7 @@ func buildRule(s *scope, args []pyObject) pyObject { populateTarget(s, target, args) s.state.AddTarget(s.pkg, target) s.pkg.Metadata.RegisterStatementTarget(target, s.CurrentBuildStatement()) - s.pkg.Metadata.RegisterRequiredSubinclude(target, s.ActiveSubincludes()) + s.pkg.Metadata.RegisterRequiredSubinclude(target, s.RequiredSubincludes()) if s.Callback { target.AddedPostBuild = true diff --git a/src/parse/asp/interpreter.go b/src/parse/asp/interpreter.go index 89a85cc50c..2b3ae883b0 100644 --- a/src/parse/asp/interpreter.go +++ b/src/parse/asp/interpreter.go @@ -1152,10 +1152,10 @@ func (s *scope) CurrentBuildStatement() core.BuildStatementProvider { } } -// ActiveSubincludes creates a provider that reports the active/required subincluded targets for a +// RequiredSubincludes creates a provider that reports the active/required subincluded targets for a // certain scope. This gives the explicitly subincluded targets that generate the methods we used // in the current callstack, actively executing to define this target. -func (s *scope) ActiveSubincludes() core.SubincludesLabelProvider { +func (s *scope) RequiredSubincludes() core.SubincludesLabelProvider { return func() core.BuildLabels { // We walk back on the callstack. For each scope of a method call we lookup the // subinclude labels marked as used, meaning values from those subincluded labels were used to generate this target, be it function defs or variables. diff --git a/src/parse/asp/interpreter_test.go b/src/parse/asp/interpreter_test.go index f496becfd4..8690959029 100644 --- a/src/parse/asp/interpreter_test.go +++ b/src/parse/asp/interpreter_test.go @@ -850,7 +850,7 @@ func TestActiveSubincludes(t *testing.T) { caller: scopeBUILD, metadata: newScopeMetadata(), } - labels := scopeFuncExec.ActiveSubincludes()() + labels := scopeFuncExec.RequiredSubincludes()() assert.Empty(t, labels) }) @@ -882,7 +882,7 @@ func TestActiveSubincludes(t *testing.T) { // Lookup triggers tracking of required subincludes scopeFuncExec.Lookup("foo") - labels := scopeFuncExec.ActiveSubincludes()() + labels := scopeFuncExec.RequiredSubincludes()() assert.Equal(t, core.BuildLabels{labelA}, labels) }) @@ -924,7 +924,7 @@ func TestActiveSubincludes(t *testing.T) { scopeFuncExec.Lookup("varA") scopeFuncExec.Lookup("varB") - labels := scopeFuncExec.ActiveSubincludes()() + labels := scopeFuncExec.RequiredSubincludes()() assert.ElementsMatch(t, core.BuildLabels{labelA, labelB}, labels) }) } From d491c8428ca3dab22c854f1a63db18ea3156325d Mon Sep 17 00:00:00 2001 From: DuBento Date: Thu, 18 Jun 2026 18:11:36 +0100 Subject: [PATCH 096/118] imp: skip scope and package metadata tracking for external subrepos --- src/core/subrepo.go | 5 +++++ src/export/trimmed_exporter.go | 2 +- src/gc/gc.go | 4 ++-- src/parse/asp/interpreter.go | 10 ++++++++-- src/parse/parse_step.go | 5 +++-- src/query/reverse_deps.go | 2 +- src/query/somepath.go | 2 +- 7 files changed, 21 insertions(+), 9 deletions(-) diff --git a/src/core/subrepo.go b/src/core/subrepo.go index 5244226c08..100d4e0ab8 100644 --- a/src/core/subrepo.go +++ b/src/core/subrepo.go @@ -68,6 +68,11 @@ func (s *Subrepo) IsRemoteSubrepo() bool { return s.Root != "" && s.Target != nil && !s.Target.Local && s.State.RemoteClient != nil } +// IsExternal returns true if this subrepo is external, meaning its sources are fetched by a build target. +func (s *Subrepo) IsExternal() bool { + return s != nil && s.Target != nil +} + // Equal returns true if this subrepo is equivalent to another, or false if it is not. func (s *Subrepo) Equal(other *Subrepo) bool { return s.Name == other.Name && s.Root == other.Root && s.PackageRoot == other.PackageRoot && diff --git a/src/export/trimmed_exporter.go b/src/export/trimmed_exporter.go index d4214ec67b..3073d0706c 100644 --- a/src/export/trimmed_exporter.go +++ b/src/export/trimmed_exporter.go @@ -65,7 +65,7 @@ func (e *trimmedExporter) exportTarget(target *core.BuildTarget) { } // We want to export the package that made this subrepo available, but we still need to walk the // target deps as it may depend on other subrepos or first party targets - if target.Subrepo != nil && target.Subrepo.Target != nil { + if target.Subrepo.IsExternal() { e.exportTarget(target.Subrepo.Target) e.exportDependencies(target) return diff --git a/src/gc/gc.go b/src/gc/gc.go index 1bdc7463df..95eccfd475 100644 --- a/src/gc/gc.go +++ b/src/gc/gc.go @@ -173,7 +173,7 @@ func addTarget(graph *core.BuildGraph, m targetMap, target *core.BuildTarget) { for _, dep := range target.Dependencies() { addTarget(graph, m, dep) } - if target.Subrepo != nil && target.Subrepo.Target != nil { + if target.Subrepo.IsExternal() { addTarget(graph, m, target.Subrepo.Target) } } @@ -208,7 +208,7 @@ func publicDependencies(graph *core.BuildGraph, target *core.BuildTarget) []*cor } } } - if target.Subrepo != nil && target.Subrepo.Target != nil { + if target.Subrepo.IsExternal() { ret = append(ret, target.Subrepo.Target) } return ret diff --git a/src/parse/asp/interpreter.go b/src/parse/asp/interpreter.go index 2b3ae883b0..b649a44962 100644 --- a/src/parse/asp/interpreter.go +++ b/src/parse/asp/interpreter.go @@ -456,11 +456,17 @@ func (s *scope) newScope(pkg *core.Package, mode core.ParseMode, filename string config: s.config, Callback: s.Callback, mode: mode, - metadata: s.metadata.newMetadata(), } if pkg != nil && pkg.Subrepo != nil && pkg.Subrepo.State != nil { s2.state = pkg.Subrepo.State } + if pkg != nil && pkg.Subrepo.IsExternal() { + // Skip metadata tracking for external/remote subrepos. We never trim these, so avoiding + // tracking saves CPU and memory. + s2.metadata = &noopScopeMetadata{} + } else { + s2.metadata = s.metadata.newMetadata() + } return s2 } @@ -1143,7 +1149,7 @@ func (s *scope) CurrentBuildStatement() core.BuildStatementProvider { // package level that generated the current build target. stmtScope := s for curr := s; curr != nil; curr = curr.caller { - if curr.pkg != nil && curr.filename == s.pkg.Filename { + if curr.pkg != nil && curr.pkg.Filename == s.pkg.Filename { stmtScope = curr } } diff --git a/src/parse/parse_step.go b/src/parse/parse_step.go index 22ddccd61a..e88a8228ef 100644 --- a/src/parse/parse_step.go +++ b/src/parse/parse_step.go @@ -68,7 +68,7 @@ func parse(state *core.BuildState, label, dependent core.BuildLabel, mode core.P // If we get here then it falls to us to parse this package. state.LogParseResult(label, core.PackageParsing, "Parsing...") - if subrepo != nil && subrepo.Target != nil { + if subrepo.IsExternal() { // We have got the definition of the subrepo, but it depends on something, make sure that has been built. state.WaitForBuiltTarget(subrepo.Target.Label, label, mode|core.ParseModeForSubinclude) if !subrepo.Target.State().IsBuilt() { @@ -183,7 +183,8 @@ func maybeParseSubrepoPackage(state *core.BuildState, subrepoPkg, subrepoSubrepo func parsePackage(state *core.BuildState, label, dependent core.BuildLabel, subrepo *core.Subrepo, mode core.ParseMode) (*core.Package, error) { packageName := label.PackageName var opts []core.PackageOptions - if state.ParseMetadata { + if state.ParseMetadata && !subrepo.IsExternal() { + // Skip metadata tracking for external subrepos since these are always used as is and never trimmed. opts = append(opts, core.WithPackageMetadata()) } pkg := core.NewPackage(packageName, opts...) diff --git a/src/query/reverse_deps.go b/src/query/reverse_deps.go index cca1ea2f4c..f9ba93e3c3 100644 --- a/src/query/reverse_deps.go +++ b/src/query/reverse_deps.go @@ -123,7 +123,7 @@ func buildRevdeps(graph *core.BuildGraph, includeSubrepos bool) map[core.BuildLa // Targets in a subrepo don't express an explicit dependency on their subrepo's target. // However this is often useful for query commands where you expect to see this kind of // relationship. Hence, if requested, we add the extra 'dependency' here. - if includeSubrepos && t.Subrepo != nil && t.Subrepo.Target != nil { + if includeSubrepos && t.Subrepo.IsExternal() { revdeps[t.Subrepo.Target.Label] = append(revdeps[t.Subrepo.Target.Label], t) } } diff --git a/src/query/somepath.go b/src/query/somepath.go index 75b91bdc10..adc3b9a8bf 100644 --- a/src/query/somepath.go +++ b/src/query/somepath.go @@ -104,7 +104,7 @@ func somePath(graph *core.BuildGraph, target1, target2 *core.BuildTarget, seen, } } } - if target1.Subrepo != nil && target1.Subrepo.Target != nil { + if target1.Subrepo.IsExternal() { if path := somePath(graph, target1.Subrepo.Target, target2, seen, except); len(path) != 0 { return append([]core.BuildLabel{target1.Label}, path...) } From 8a1a702b5ece06aa3d79c6c7a4375a3b6d6db94b Mon Sep 17 00:00:00 2001 From: DuBento Date: Mon, 22 Jun 2026 11:47:47 +0100 Subject: [PATCH 097/118] remove doc reference to required target naming for successful export --- docs/commands.html | 9 --------- 1 file changed, 9 deletions(-) diff --git a/docs/commands.html b/docs/commands.html index b6b2fcfa73..df9590218c 100644 --- a/docs/commands.html +++ b/docs/commands.html @@ -1035,15 +1035,6 @@

Disables trimming unnecessary targets from exported packages. Normally targets in exported packages that aren't dependencies of the originally exported targets are removed.

-

- This trimming syntax based, so doesn't always work depending on how the build definition is authored. Passing - this flag will disable this feature, avoiding cases where these rules will be erroneously trimmed. -

-

- To make sure a rule works without this flag, the rule must follow the naming convention, whereby children of - :name follow the format :_name#{some-tag}. This is the - format tag(name, tag) would produce. -

From 131b4891afa4a7d1d802d6e2b3befdc251960557 Mon Sep 17 00:00:00 2001 From: DuBento Date: Mon, 22 Jun 2026 19:00:40 +0100 Subject: [PATCH 098/118] Only pop symbols from stack in package scope + rework newScopeMetadata to also avoid tracking inside subincludes --- src/parse/asp/interpreter.go | 87 +++++++++++++++++++----------------- 1 file changed, 45 insertions(+), 42 deletions(-) diff --git a/src/parse/asp/interpreter.go b/src/parse/asp/interpreter.go index b649a44962..54cc8a06e7 100644 --- a/src/parse/asp/interpreter.go +++ b/src/parse/asp/interpreter.go @@ -44,14 +44,11 @@ type interpreter struct { // It loads all the builtin rules at this point. func newInterpreter(state *core.BuildState, p *Parser) *interpreter { s := &scope{ - ctx: context.Background(), - state: state, - locals: map[string]pyObject{}, - metadata: &noopScopeMetadata{}, - } - if state.ParseMetadata { - s.metadata = newTrackingScopeMetadata() + ctx: context.Background(), + state: state, + locals: map[string]pyObject{}, } + s.metadata = s.newScopeMetadata() i := &interpreter{ scope: s, @@ -460,13 +457,8 @@ func (s *scope) newScope(pkg *core.Package, mode core.ParseMode, filename string if pkg != nil && pkg.Subrepo != nil && pkg.Subrepo.State != nil { s2.state = pkg.Subrepo.State } - if pkg != nil && pkg.Subrepo.IsExternal() { - // Skip metadata tracking for external/remote subrepos. We never trim these, so avoiding - // tracking saves CPU and memory. - s2.metadata = &noopScopeMetadata{} - } else { - s2.metadata = s.metadata.newMetadata() - } + s2.metadata = s2.newScopeMetadata() + s2.metadata.setCursor(s.metadata.cursor()) return s2 } @@ -1098,13 +1090,17 @@ func (s *scope) callObject(name string, obj pyObject, c *Call) pyObject { s.Error("Non-callable object '%s' (is a %s)", name, obj.Type()) } - // Restore the pre-call checkpoint first, then pop the function object itself to ensure explicit sequential cleanup. - // Remove the arguments visited during the call + the function object implicitly added by [Lookup]. - checkpoint := s.metadata.checkpointSymbolStack() - defer func() { - s.metadata.restoreCheckpoint(checkpoint) - s.metadata.popSymbol(name) - }() + // Restore the pre-call checkpoint first, then pop the function object itself to ensure explicit + // sequential cleanup. Remove the arguments visited during the call + the function object + // implicitly added by [Lookup]. We only pop from the symbol stack when interpreting Package level + // calls to keep track of required symbols a few calls deep. + if s.IsPackageScope() { + checkpoint := s.metadata.checkpointSymbolStack() + defer func() { + s.metadata.restoreCheckpoint(checkpoint) + s.metadata.popSymbol(name) + }() + } return f.Call(s, c) } @@ -1149,7 +1145,7 @@ func (s *scope) CurrentBuildStatement() core.BuildStatementProvider { // package level that generated the current build target. stmtScope := s for curr := s; curr != nil; curr = curr.caller { - if curr.pkg != nil && curr.pkg.Filename == s.pkg.Filename { + if curr.pkg != nil && curr.pkg == s.pkg { stmtScope = curr } } @@ -1181,14 +1177,39 @@ func (s *scope) pkgFilename() string { return "" } +// IsPackageScope returns true if this scope is the package scope +// (i.e. we are interpreting the build file directly and not inside any call). +func (s *scope) IsPackageScope() bool { + return s.caller == nil && s.subincludeLabel == nil && s.pkg != nil +} + +// newScopeMetadata creates and returns a initialized scopeMetadata instance. It will return +// a no-op implementation if state.ParseMetadata is not set or if we simply want to skip the +// tracking for a certain scope. +func (s *scope) newScopeMetadata() scopeMetadata { + if !s.state.ParseMetadata || + s.pkg == nil || + s.pkg.Subrepo.IsExternal() { + // Skip metadata tracking if: + // 1. ParseMetadata flag is disabled; + // 2. Not interpreting a package (e.g. in subincluded targets) + // 3. Any external/remote subrepos. + // For 2 and 3, we never trim these, so avoiding tracking saves CPU and memory. + return &noopScopeMetadata{} + } + + return &trackingScopeMetadata{ + symbolOrigins: map[string]core.BuildLabel{}, + symbolStack: []trackedSymbol{}, + } +} + // scopeMetadata defines an interface for tracking evaluation metadata (such as AST cursor position // and symbol subinclude origins) across interpreter scopes. // This is optionally used for operations (e.g. export) that require more details on the relation // between targets and statements. The no-op implementation should be used for most operations to // avoid any computational overhead. type scopeMetadata interface { - // newMetadata creates a new instance of the same scopeMetadata implementation type. - newMetadata() scopeMetadata // cursor returns the statement currently being interpreted. cursor() *Statement // origin returns the subinclude origin label of a tracked symbol by name. Returns nil if the @@ -1229,13 +1250,6 @@ type trackedSymbol struct { origin *core.BuildLabel } -// newMetadata implements [scopeMetadata]. -func (m *trackingScopeMetadata) newMetadata() scopeMetadata { - meta := newTrackingScopeMetadata() - meta.cursorField = m.cursorField - return meta -} - // cursor implements [scopeMetadata]. func (m *trackingScopeMetadata) cursor() *Statement { return m.cursorField @@ -1321,9 +1335,6 @@ func (nm *noopScopeMetadata) restoreCheckpoint(checkpoint int) {} // checkpointSymbolStack implements [scopeMetadata]. func (nm *noopScopeMetadata) checkpointSymbolStack() int { return 0 } -// newMetadata implements [scopeMetadata]. -func (nm *noopScopeMetadata) newMetadata() scopeMetadata { return &noopScopeMetadata{} } - // cursor implements [scopeMetadata]. func (nm *noopScopeMetadata) cursor() *Statement { return nil } @@ -1345,14 +1356,6 @@ func (nm *noopScopeMetadata) pushSymbol(name string, origin *core.BuildLabel) {} // popSymbol implements [scopeMetadata]. func (nm *noopScopeMetadata) popSymbol(name string) {} -// newTrackingScopeMetadata creates and returns a fully initialized trackingScopeMetadata instance. -func newTrackingScopeMetadata() *trackingScopeMetadata { - return &trackingScopeMetadata{ - symbolOrigins: map[string]core.BuildLabel{}, - symbolStack: []trackedSymbol{}, - } -} - // NewBuildStatement creates a new core.BuildStatement from an asp.statement. func NewBuildStatement(stmt *Statement) core.BuildStatement { return core.BuildStatement{ From f437f8f08d8037479e249aa9f4f1feca030a65fb Mon Sep 17 00:00:00 2001 From: DuBento Date: Tue, 23 Jun 2026 12:33:11 +0100 Subject: [PATCH 099/118] lazy initialize symbol origins --- src/parse/asp/interpreter.go | 16 +++++++++++----- src/parse/asp/objects.go | 3 ++- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/parse/asp/interpreter.go b/src/parse/asp/interpreter.go index 54cc8a06e7..2bab2d2294 100644 --- a/src/parse/asp/interpreter.go +++ b/src/parse/asp/interpreter.go @@ -1199,8 +1199,8 @@ func (s *scope) newScopeMetadata() scopeMetadata { } return &trackingScopeMetadata{ - symbolOrigins: map[string]core.BuildLabel{}, - symbolStack: []trackedSymbol{}, + // symbolOrigins is lazy initialized in [setSymbolOrigin] + symbolStack: []trackedSymbol{}, } } @@ -1262,13 +1262,14 @@ func (m *trackingScopeMetadata) origin(scope *scope, name string) *core.BuildLab // available across all package scopes in the repository. // // This also prevents erroneous subinclude propagation: since symbol resolution recursively - // traverses the parent scope chain from bottom to top, where the preoloaded symbols are - // defined at the top, if a target subincludes a preloaded target again it will be prefered - // over the preloaded and will potetntially include unwanted symbols so we enforce a + // traverses the parent scope chain from bottom to top, where the preloaded symbols are + // defined at the top, if a target subincludes a preloaded target again it will be preferred + // over the preloaded and will potentially include unwanted symbols so we enforce a // preference for the preloaded symbols. This could cause issues if our repo relies on // redefining preloaded symbols. return nil } + if label, ok := m.symbolOrigins[name]; ok { // Object subincluded into current scope. return &label @@ -1291,6 +1292,11 @@ func (m *trackingScopeMetadata) setCursor(stmt *Statement) { // setSymbolOrigin implements [scopeMetadata]. func (m *trackingScopeMetadata) setSymbolOrigin(name string, origin core.BuildLabel) { + if m.symbolOrigins == nil { + // lazy initialization to avoid unnecessary allocation of a map in smaller scopes (no subinclude). + m.symbolOrigins = map[string]core.BuildLabel{} + } + m.symbolOrigins[name] = origin } diff --git a/src/parse/asp/objects.go b/src/parse/asp/objects.go index 68b4ee82a0..c96e7ce832 100644 --- a/src/parse/asp/objects.go +++ b/src/parse/asp/objects.go @@ -834,7 +834,8 @@ func (f *pyFunc) Member(obj pyObject) pyObject { } } -// validateType validates that this argument matches the given type +// validateType validates that this argument matches the given type. It interprets the expression and +// returns its value. func (f *pyFunc) validateType(s *scope, i int, expr *Expression) pyObject { val := s.interpretExpression(expr) if i >= len(f.types) && (f.varargs || f.kwargs) { From b0f200e453cb4a8752b13ff5dc740cf28f7121d6 Mon Sep 17 00:00:00 2001 From: DuBento Date: Wed, 24 Jun 2026 12:36:09 +0100 Subject: [PATCH 100/118] Close package wait channels on error. Introduced a deadlock when KeepParser is set. This wasn't an issue before since we would force close all the channels and fail during the first parse, but now we will attempt to wait for parsed package during an export. --- src/core/state.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/core/state.go b/src/core/state.go index 1ed5136bcc..202990857f 100644 --- a/src/core/state.go +++ b/src/core/state.go @@ -635,6 +635,23 @@ func (state *BuildState) LogTestResult(target *BuildTarget, run int, status Buil // LogBuildError logs a failure for a target to parse, build or test. func (state *BuildState) LogBuildError(label BuildLabel, status BuildResultStatus, err error, format string, args ...interface{}) { + if status == ParseFailed { + // Force close package wait channels to avoid deadlocks when calling waitForPackage() after + // the initial parse, for example when KeepParserRunning is set. + key := packageKey{Name: label.PackageName, Subrepo: label.Subrepo} + if ch := state.progress.pendingPackages.Get(key); ch != nil { + func() { + defer func() { recover() }() // recover if attempted to close a closed channel. + close(ch) // This signals to anyone waiting that it's done (failed, but completed). + }() + } + if ch := state.progress.packageWaits.Get(key); ch != nil { + func() { + defer func() { recover() }() // recover if attempted to close a closed channel. + close(ch) // This signals to anyone waiting that it's done (failed, but completed). + }() + } + } state.logResult(&BuildResult{ Label: label, Status: status, From 9c6cf879fac0824ec14d5a17d3c25360f6fba2c0 Mon Sep 17 00:00:00 2001 From: DuBento Date: Wed, 24 Jun 2026 13:39:26 +0100 Subject: [PATCH 101/118] Support method calls as arguments by popping the symbol stack only at package level --- src/parse/asp/interpreter.go | 83 +++++++++++++++++++++++++----------- 1 file changed, 58 insertions(+), 25 deletions(-) diff --git a/src/parse/asp/interpreter.go b/src/parse/asp/interpreter.go index 2bab2d2294..79d91646fb 100644 --- a/src/parse/asp/interpreter.go +++ b/src/parse/asp/interpreter.go @@ -1090,14 +1090,17 @@ func (s *scope) callObject(name string, obj pyObject, c *Call) pyObject { s.Error("Non-callable object '%s' (is a %s)", name, obj.Type()) } - // Restore the pre-call checkpoint first, then pop the function object itself to ensure explicit - // sequential cleanup. Remove the arguments visited during the call + the function object - // implicitly added by [Lookup]. We only pop from the symbol stack when interpreting Package level - // calls to keep track of required symbols a few calls deep. - if s.IsPackageScope() { - checkpoint := s.metadata.checkpointSymbolStack() + // Ensure explicit sequential cleanup of the symbol stack. We only pop from the symbol stack when + // interpreting Package level calls to keep track of required symbols a few calls deep, for + // example argument lookup. The function object implicitly added by [Lookup] is also removed from + // the stack. We only pop from the symbol stack when interpreting Package level calls to keep + // track of required symbols a few calls deep, for example argument lookup. + s.metadata.incrementCallDepth() + defer s.metadata.decrementCallDepth() + if s.IsPackageScope() && s.metadata.isTopLevelCall() { + checkpoint := s.metadata.getSymbolStackCheckpoint() defer func() { - s.metadata.restoreCheckpoint(checkpoint) + s.metadata.restoreSymbolStack(checkpoint) s.metadata.popSymbol(name) }() } @@ -1226,11 +1229,17 @@ type scopeMetadata interface { pushSymbol(name string, origin *core.BuildLabel) // popSymbol pops the specified symbol from the top of the tracking stack if it matches the name. popSymbol(name string) - // checkpointSymbolStack checkpoints the current size of the symbol tracking stack. - checkpointSymbolStack() int - // restoreCheckpoint restores the symbol tracking stack back to the given checkpoint size, - // discarding any symbols pushed after the checkpoint was taken. - restoreCheckpoint(checkpoint int) + // isTopLevelCall returns true if the interpreter is currently executing at the top level + // of the package scope (not inside any function calls). + isTopLevelCall() bool + // incrementCallDepth increments the current function call depth. + incrementCallDepth() + // decrementCallDepth decrements the current function call depth. + decrementCallDepth() + // getSymbolStackCheckpoint returns the current size of the symbol tracking stack. + getSymbolStackCheckpoint() int + // restoreSymbolStack restores the symbol tracking stack back to the given checkpoint size. + restoreSymbolStack(checkpoint int) } // trackingScopeMetadata implements the interface [scopeMetadata]. @@ -1243,6 +1252,7 @@ type trackingScopeMetadata struct { // Symbols are pushed onto the stack during lookups and popped or truncated (restored) after // function calls. symbolStack []trackedSymbol + callDepth int } type trackedSymbol struct { @@ -1318,29 +1328,37 @@ func (m *trackingScopeMetadata) popSymbol(name string) { } } -// checkpointSymbolStack implements [scopeMetadata]. -func (m *trackingScopeMetadata) checkpointSymbolStack() int { +// isTopLevelCall implements [scopeMetadata]. +func (m *trackingScopeMetadata) isTopLevelCall() bool { + return m.callDepth == 1 +} + +// incrementCallDepth implements [scopeMetadata]. +func (m *trackingScopeMetadata) incrementCallDepth() { + m.callDepth++ +} + +// decrementCallDepth implements [scopeMetadata]. +func (m *trackingScopeMetadata) decrementCallDepth() { + m.callDepth-- +} + +// getSymbolStackCheckpoint implements [scopeMetadata]. +func (m *trackingScopeMetadata) getSymbolStackCheckpoint() int { return len(m.symbolStack) } -// restoreCheckpoint implements [scopeMetadata]. -func (m *trackingScopeMetadata) restoreCheckpoint(checkpoint int) { - if checkpoint < 0 || checkpoint > len(m.symbolStack) { - return +// restoreSymbolStack implements [scopeMetadata]. +func (m *trackingScopeMetadata) restoreSymbolStack(checkpoint int) { + if checkpoint >= 0 && checkpoint <= len(m.symbolStack) { + m.symbolStack = m.symbolStack[:checkpoint] } - m.symbolStack = m.symbolStack[:checkpoint] } // noopScopeMetadata implements the scopeMetadata interface with no-op methods. This is used to // avoid the overhead of storing metadata for operations that don't depend on it. type noopScopeMetadata struct{} -// restoreCheckpoint implements [scopeMetadata]. -func (nm *noopScopeMetadata) restoreCheckpoint(checkpoint int) {} - -// checkpointSymbolStack implements [scopeMetadata]. -func (nm *noopScopeMetadata) checkpointSymbolStack() int { return 0 } - // cursor implements [scopeMetadata]. func (nm *noopScopeMetadata) cursor() *Statement { return nil } @@ -1362,6 +1380,21 @@ func (nm *noopScopeMetadata) pushSymbol(name string, origin *core.BuildLabel) {} // popSymbol implements [scopeMetadata]. func (nm *noopScopeMetadata) popSymbol(name string) {} +// isTopLevelCall implements [scopeMetadata]. +func (nm *noopScopeMetadata) isTopLevelCall() bool { return false } + +// incrementCallDepth implements [scopeMetadata]. +func (nm *noopScopeMetadata) incrementCallDepth() {} + +// decrementCallDepth implements [scopeMetadata]. +func (nm *noopScopeMetadata) decrementCallDepth() {} + +// getSymbolStackCheckpoint implements [scopeMetadata]. +func (nm *noopScopeMetadata) getSymbolStackCheckpoint() int { return 0 } + +// restoreSymbolStack implements [scopeMetadata]. +func (nm *noopScopeMetadata) restoreSymbolStack(checkpoint int) {} + // NewBuildStatement creates a new core.BuildStatement from an asp.statement. func NewBuildStatement(stmt *Statement) core.BuildStatement { return core.BuildStatement{ From 349e3f82a24acf205f0839d7a756a233b3a3d4f6 Mon Sep 17 00:00:00 2001 From: DuBento Date: Wed, 24 Jun 2026 18:32:18 +0100 Subject: [PATCH 102/118] refactor: record all statements and respective origins for all statements (including non-target generators). The reason for pivoting is that the previous implementation failed to identify necessary subincludes for variable declarations or other builtin methods (string join) --- src/core/package_metadata.go | 97 +++++++--- src/export/trimmed_exporter.go | 16 +- src/parse/asp/builtins.go | 1 - src/parse/asp/interpreter.go | 170 +++++------------- src/parse/asp/objects.go | 4 +- test/export/test_subinclude_trimming/BUILD | 5 +- .../expected_repo/BUILD_FILE | 20 +++ .../expected_repo/build_defs/BUILD_FILE | 18 ++ .../expected_repo/build_defs/var1.build_defs | 1 + .../expected_repo/build_defs/var2.build_defs | 1 + .../expected_repo/build_defs/var3.build_defs | 1 + .../source_repo/BUILD_FILE | 35 ++++ .../source_repo/build_defs/BUILD_FILE | 18 ++ .../source_repo/build_defs/unused.build_defs | 4 +- .../source_repo/build_defs/var1.build_defs | 1 + .../source_repo/build_defs/var2.build_defs | 1 + .../source_repo/build_defs/var3.build_defs | 1 + 17 files changed, 233 insertions(+), 161 deletions(-) create mode 100644 test/export/test_subinclude_trimming/expected_repo/build_defs/var1.build_defs create mode 100644 test/export/test_subinclude_trimming/expected_repo/build_defs/var2.build_defs create mode 100644 test/export/test_subinclude_trimming/expected_repo/build_defs/var3.build_defs create mode 100644 test/export/test_subinclude_trimming/source_repo/build_defs/var1.build_defs create mode 100644 test/export/test_subinclude_trimming/source_repo/build_defs/var2.build_defs create mode 100644 test/export/test_subinclude_trimming/source_repo/build_defs/var3.build_defs diff --git a/src/core/package_metadata.go b/src/core/package_metadata.go index a1a1e1039c..2dc9a333f4 100644 --- a/src/core/package_metadata.go +++ b/src/core/package_metadata.go @@ -1,6 +1,9 @@ package core import ( + "maps" + "slices" + "github.com/thought-machine/please/src/cmap" ) @@ -45,13 +48,14 @@ type SubincludesLabelProvider func() BuildLabels // to their respective targets. This supports additional logic for operations such as `plz export` // but should be disabled for most operations by using the no-op implementation to avoid the overhead. type PackageMetadata interface { + // RegisterStatement records a statement of an interpreted BUILD file and its + // dependencies. Dependencies identify the subincluded targets required for a successful + // interpretation of the statement, i.e. targets that provide the required symbols (variables or + // methods). + RegisterStatement(stmt BuildStatement, deps BuildLabels) // RegisterStatementTarget records that the given build target was created as a result of the // given statement being executed. This should only be called for statements in BUILD files. RegisterStatementTarget(target *BuildTarget, stmtProvider BuildStatementProvider) - // RegisterRequiredSubinclude records that the given build target requires the given subincluded - // labels to be built. This is used to track which subinclude statements contributed to a target's - // definition. - RegisterRequiredSubinclude(target *BuildTarget, labelProvider SubincludesLabelProvider) // RegisterSubincludeStatement records that the given subinclude statement (provided by stmtProvider) // includes the given build label. This should only be called for statements in BUILD files. RegisterSubincludeStatement(label BuildLabel, stmtProvider BuildStatementProvider) @@ -62,19 +66,25 @@ type PackageMetadata interface { // Returns an empty slice if no targets were found for the given statement. FindTargets(stmt *BuildStatement) BuildTargets // FindRequiredSubincludes returns all subinclude labels that were required by the given target. + // This method will only report the package level (direct) subincludes, make use of + // [BuildGraph.TransitiveSubincludes] if you want all the required subincludes for one target. // The return value is empty if no subinclude information was found for the target. FindRequiredSubincludes(target *BuildTarget) BuildLabels // FindRelatedTargets finds all the targets that are related to the argument. In this context, // target relationship is determined by looking for targets generated by the same build statement. // The result excludes the target in the argument. FindRelatedTargets(target *BuildTarget) BuildLabels + // FindPackageRequiredSubincludes finds all the subincluded labels required by the package that + // are not associated with BuildTarget generation. An example could be a variable declaration + // that depends on a subincluded value. + FindPackageRequiredSubincludes() BuildLabels // GetSubincludedLabels returns all build labels that were included by the given subinclude statement. // Returns the labels or an empty slice if the statement wasn't found. GetSubincludedLabels(stmt *BuildStatement) BuildLabels } // packageMetadataImpl is the canonical implementation of the PackageMetadata interface. -// It uses sharded concurrent maps (cmap.Map) to track the relationships between BUILD file statements, +// It uses sharded concurrent maps [cmap.Map] to track the relationships between BUILD file statements, // subincludes, and the build targets they define without the contention of a global read-write lock. type packageMetadataImpl struct { // stmtToTarget maps each build statement (identified by its byte range in a BUILD file) @@ -85,10 +95,11 @@ type packageMetadataImpl struct { // back to the specific BuildStatement that declared it. This is useful for tracing back a target // to its statement and to find sibling targets generated by the same statement block. targetToStmt *cmap.Map[*BuildTarget, BuildStatement] - // targetToRequiredSubincludes tracks the subinclude labels that were required for a target's - // definition. This allows mapping a target back to the subincluded labels required for building - // the target. - targetToRequiredSubincludes *cmap.Map[*BuildTarget, BuildLabels] + // stmtToRequiredSubincludes tracks the subinclude labels that were required for the current + // interpretation of the build statement. This, in addition to the statement to target map, + // allows mapping a statement back to the subincluded labels required for building the target. + // One direct, package level, subincludes are included. + stmtToRequiredSubincludes *cmap.Map[BuildStatement, BuildLabels] // labelsPerSubincludeStmt maps a subinclude statement (identified by its position // in the BUILD file) to the labels it explicitly subincludes. labelsPerSubincludeStmt *cmap.Map[BuildStatement, BuildLabels] @@ -96,13 +107,18 @@ type packageMetadataImpl struct { func newPackageMetadata() PackageMetadata { return &packageMetadataImpl{ - stmtToTarget: cmap.New[BuildStatement, BuildTargets](cmap.SmallShardCount, hashBuildStatement), - targetToStmt: cmap.New[*BuildTarget, BuildStatement](cmap.SmallShardCount, hashBuildTarget), - targetToRequiredSubincludes: cmap.New[*BuildTarget, BuildLabels](cmap.SmallShardCount, hashBuildTarget), - labelsPerSubincludeStmt: cmap.New[BuildStatement, BuildLabels](cmap.SmallShardCount, hashBuildStatement), + stmtToTarget: cmap.New[BuildStatement, BuildTargets](cmap.SmallShardCount, hashBuildStatement), + targetToStmt: cmap.New[*BuildTarget, BuildStatement](cmap.SmallShardCount, hashBuildTarget), + stmtToRequiredSubincludes: cmap.New[BuildStatement, BuildLabels](cmap.SmallShardCount, hashBuildStatement), + labelsPerSubincludeStmt: cmap.New[BuildStatement, BuildLabels](cmap.SmallShardCount, hashBuildStatement), } } +// RegisterStatement implements [PackageMetadata]. +func (m *packageMetadataImpl) RegisterStatement(stmt BuildStatement, deps BuildLabels) { + m.stmtToRequiredSubincludes.Set(stmt, deps) +} + // RegisterStatementTarget implements [PackageMetadata]. func (m *packageMetadataImpl) RegisterStatementTarget(target *BuildTarget, stmtProvider BuildStatementProvider) { stmt := stmtProvider() @@ -111,13 +127,6 @@ func (m *packageMetadataImpl) RegisterStatementTarget(target *BuildTarget, stmtP m.targetToStmt.Set(target, stmt) } -// RegisterRequiredSubinclude implements [PackageMetadata]. -func (m *packageMetadataImpl) RegisterRequiredSubinclude(target *BuildTarget, labelProvider SubincludesLabelProvider) { - labels := labelProvider() - existing := m.targetToRequiredSubincludes.Get(target) - m.targetToRequiredSubincludes.Set(target, append(existing, labels...)) -} - // RegisterSubincludeStatement implements [PackageMetadata]. func (m *packageMetadataImpl) RegisterSubincludeStatement(label BuildLabel, stmtProvider BuildStatementProvider) { stmt := stmtProvider() @@ -131,12 +140,12 @@ func (m *packageMetadataImpl) FindStatement(target *BuildTarget) *BuildStatement return nil } - if m.targetToStmt.Contains(target) { - stmt := m.targetToStmt.Get(target) - return &stmt + stmt := m.targetToStmt.Get(target) + if stmt == (BuildStatement{}) { + log.Fatalf("Failed to find statement for target %s", target) + return nil } - log.Debugf("Failed to find statement for target %s", target) - return nil + return &stmt } // FindTargets implements [PackageMetadata]. @@ -154,7 +163,18 @@ func (m *packageMetadataImpl) FindRequiredSubincludes(target *BuildTarget) Build return nil } - return m.targetToRequiredSubincludes.Get(target) + stmt := m.FindStatement(target) + if stmt == nil { + return nil + } + + directSubincludes := m.stmtToRequiredSubincludes.Get(*stmt) + if directSubincludes == nil { + log.Debugf("Failed to find direct subincludes for the build statement.") + return nil + } + + return directSubincludes } // FindRelatedTargets implements [PackageMetadata]. @@ -170,6 +190,21 @@ func (m *packageMetadataImpl) FindRelatedTargets(target *BuildTarget) BuildLabel return labels } +// FindPackageRequiredSubincludes implements [PackageMetadata]. +func (m *packageMetadataImpl) FindPackageRequiredSubincludes() BuildLabels { + collector := LabelSet{} + m.stmtToRequiredSubincludes.Range(func(stmt BuildStatement, labels BuildLabels) { + // Look for build statements that are not registered in the statement to targets mapping, + // and so are unrelated to targets. + if !m.stmtToTarget.Contains(stmt) { + for _, label := range labels { + collector.Add(label) + } + } + }) + return slices.Collect(maps.Keys(collector)) +} + // GetSubincludedLabels implements [PackageMetadata]. func (m *packageMetadataImpl) GetSubincludedLabels(stmt *BuildStatement) BuildLabels { if stmt == nil { @@ -188,6 +223,9 @@ func newNoopPackageMetadata() PackageMetadata { return &noopPackageMetadata{} } +// RegisterStatement implements [PackageMetadata]. +func (n *noopPackageMetadata) RegisterStatement(stmt BuildStatement, deps BuildLabels) {} + // RegisterStatementTarget implements [PackageMetadata]. func (n *noopPackageMetadata) RegisterStatementTarget(target *BuildTarget, stmtProvider BuildStatementProvider) { } @@ -220,6 +258,13 @@ func (n *noopPackageMetadata) FindRequiredSubincludes(target *BuildTarget) Build // FindRelatedTargets implements [PackageMetadata]. func (n *noopPackageMetadata) FindRelatedTargets(target *BuildTarget) BuildLabels { + log.Fatalf("Metadata not tracked, using no-op implementation.") + return nil +} + +// FindPackageRequiredSubincludes implements [PackageMetadata]. +func (n *noopPackageMetadata) FindPackageRequiredSubincludes() BuildLabels { + log.Fatalf("Metadata not tracked, using no-op implementation.") return nil } diff --git a/src/export/trimmed_exporter.go b/src/export/trimmed_exporter.go index 3073d0706c..bfb8a305b6 100644 --- a/src/export/trimmed_exporter.go +++ b/src/export/trimmed_exporter.go @@ -81,7 +81,13 @@ func (e *trimmedExporter) exportTarget(target *core.BuildTarget) { } e.exportSubincludes(pkg, target) e.exportRelatedTargets(pkg, target) - e.visitedPackages[pkg.Label()] = true + + if !e.visitedPackages[pkg.Label()] { + // Export subincluded targets required for other package statements, e.g. variable + // declaration, during the first visit of a package. + e.exportPackageSubincludes(pkg) + e.visitedPackages[pkg.Label()] = true + } } // writePackageFiles implements [exporterImpl]. @@ -122,6 +128,14 @@ func (e *trimmedExporter) exportSubincludes(pkg *core.Package, target *core.Buil e.exportTargets(allSubincludes) } +// exportPackageSubincludes exports the subincluded targets that are required by package but are not +// linked to any [core.BuildTarget]. +func (e *trimmedExporter) exportPackageSubincludes(pkg *core.Package) { + subincludes := pkg.Metadata.FindPackageRequiredSubincludes() + e.setPackageSubincludes(pkg, subincludes) + e.exportTargets(subincludes) +} + // setPackageSubincludes marks the package-level required subincludes after the export. This will be // used for trimming subinclude statements with [trimmer]. func (e *trimmedExporter) setPackageSubincludes(pkg *core.Package, subincludes core.BuildLabels) { diff --git a/src/parse/asp/builtins.go b/src/parse/asp/builtins.go index e1b5a2911e..b592b27d96 100644 --- a/src/parse/asp/builtins.go +++ b/src/parse/asp/builtins.go @@ -211,7 +211,6 @@ func buildRule(s *scope, args []pyObject) pyObject { populateTarget(s, target, args) s.state.AddTarget(s.pkg, target) s.pkg.Metadata.RegisterStatementTarget(target, s.CurrentBuildStatement()) - s.pkg.Metadata.RegisterRequiredSubinclude(target, s.RequiredSubincludes()) if s.Callback { target.AddedPostBuild = true diff --git a/src/parse/asp/interpreter.go b/src/parse/asp/interpreter.go index 79d91646fb..32ee355f79 100644 --- a/src/parse/asp/interpreter.go +++ b/src/parse/asp/interpreter.go @@ -485,19 +485,14 @@ func (s *scope) NAssert(condition bool, msg string, args ...interface{}) { // Lookup looks up a variable name in this scope, walking back up its ancestor scopes as needed. // It panics if the variable is not defined. func (s *scope) Lookup(name string) pyObject { - obj, orig := s.lookupWithOrigin(name) - s.metadata.pushSymbol(name, orig) - return obj -} - -// lookup implements the recursive lookup over parent scopes. -func (s *scope) lookupWithOrigin(name string) (pyObject, *core.BuildLabel) { if obj, present := s.locals[name]; present { - return obj, s.metadata.origin(s, name) + orig := s.metadata.origin(s, name) + s.metadata.pushSymbol(name, orig) + return obj } else if s.parent != nil { - return s.parent.lookupWithOrigin(name) + return s.parent.Lookup(name) } - return s.Error("name '%s' is not defined", name), nil + return s.Error("name '%s' is not defined", name) } // LocalLookup looks up a variable name in the current scope. @@ -613,6 +608,8 @@ func (s *scope) interpretStatements(statements []*Statement) pyObject { } else { s.Error("Unknown statement") // Shouldn't happen, amirite? } + s.metadata.registerBuildStatement(s.pkg) + s.metadata.resetSymbolStack() } return nil } @@ -1089,22 +1086,6 @@ func (s *scope) callObject(name string, obj pyObject, c *Call) pyObject { if !ok { s.Error("Non-callable object '%s' (is a %s)", name, obj.Type()) } - - // Ensure explicit sequential cleanup of the symbol stack. We only pop from the symbol stack when - // interpreting Package level calls to keep track of required symbols a few calls deep, for - // example argument lookup. The function object implicitly added by [Lookup] is also removed from - // the stack. We only pop from the symbol stack when interpreting Package level calls to keep - // track of required symbols a few calls deep, for example argument lookup. - s.metadata.incrementCallDepth() - defer s.metadata.decrementCallDepth() - if s.IsPackageScope() && s.metadata.isTopLevelCall() { - checkpoint := s.metadata.getSymbolStackCheckpoint() - defer func() { - s.metadata.restoreSymbolStack(checkpoint) - s.metadata.popSymbol(name) - }() - } - return f.Call(s, c) } @@ -1157,21 +1138,6 @@ func (s *scope) CurrentBuildStatement() core.BuildStatementProvider { } } -// RequiredSubincludes creates a provider that reports the active/required subincluded targets for a -// certain scope. This gives the explicitly subincluded targets that generate the methods we used -// in the current callstack, actively executing to define this target. -func (s *scope) RequiredSubincludes() core.SubincludesLabelProvider { - return func() core.BuildLabels { - // We walk back on the callstack. For each scope of a method call we lookup the - // subinclude labels marked as used, meaning values from those subincluded labels were used to generate this target, be it function defs or variables. - collector := core.LabelSet{} - for callScope := s; callScope != nil; callScope = callScope.caller { - callScope.metadata.requiredOrigins(callScope, &collector) - } - return slices.Collect(maps.Keys(collector)) - } -} - // pkgFilename returns the filename of the current package, or the empty string if there is none. func (s *scope) pkgFilename() string { if s.pkg != nil { @@ -1180,12 +1146,6 @@ func (s *scope) pkgFilename() string { return "" } -// IsPackageScope returns true if this scope is the package scope -// (i.e. we are interpreting the build file directly and not inside any call). -func (s *scope) IsPackageScope() bool { - return s.caller == nil && s.subincludeLabel == nil && s.pkg != nil -} - // newScopeMetadata creates and returns a initialized scopeMetadata instance. It will return // a no-op implementation if state.ParseMetadata is not set or if we simply want to skip the // tracking for a certain scope. @@ -1218,34 +1178,24 @@ type scopeMetadata interface { // origin returns the subinclude origin label of a tracked symbol by name. Returns nil if the // symbol is local (defined in the package) or has no origin/not tracked. origin(scope *scope, name string) *core.BuildLabel - // requiredOrigins aggregates all subinclude origins currently in the active symbol stack - // into the provided label set. - requiredOrigins(scope *scope, collector *core.LabelSet) // setCursor registers the statement currently being interpreted. setCursor(stmt *Statement) + // registerBuildStatement registers a new Build Statement in the given package. It will also + // register the required dependencies for interpreting that statement by looking up the required + // origins in the symbol stack. + registerBuildStatement(pkg *core.Package) // setSymbolOrigin registers the subinclude origin label for a defined symbol. setSymbolOrigin(name string, origin core.BuildLabel) + // resetSymbolStack cleans the symbol stack into an empty state. + resetSymbolStack() // pushSymbol pushes a symbol name and its subinclude origin onto the active tracking stack. pushSymbol(name string, origin *core.BuildLabel) - // popSymbol pops the specified symbol from the top of the tracking stack if it matches the name. - popSymbol(name string) - // isTopLevelCall returns true if the interpreter is currently executing at the top level - // of the package scope (not inside any function calls). - isTopLevelCall() bool - // incrementCallDepth increments the current function call depth. - incrementCallDepth() - // decrementCallDepth decrements the current function call depth. - decrementCallDepth() - // getSymbolStackCheckpoint returns the current size of the symbol tracking stack. - getSymbolStackCheckpoint() int - // restoreSymbolStack restores the symbol tracking stack back to the given checkpoint size. - restoreSymbolStack(checkpoint int) } // trackingScopeMetadata implements the interface [scopeMetadata]. type trackingScopeMetadata struct { - // cursor points to the statement currently being interpreted. - cursorField *Statement + // stmtCursor points to the statement currently being interpreted. + stmtCursor *Statement // symbolOrigins tracks the subinclude label that each symbol was originally defined in. symbolOrigins map[string]core.BuildLabel // symbolStack tracks which symbols are actively in use during evaluation. @@ -1257,12 +1207,12 @@ type trackingScopeMetadata struct { type trackedSymbol struct { name string - origin *core.BuildLabel + origin core.BuildLabel } // cursor implements [scopeMetadata]. func (m *trackingScopeMetadata) cursor() *Statement { - return m.cursorField + return m.stmtCursor } // origin implements [scopeMetadata]. @@ -1288,22 +1238,34 @@ func (m *trackingScopeMetadata) origin(scope *scope, name string) *core.BuildLab return nil } -// requiredOrigins implements [scopeMetadata]. -func (m *trackingScopeMetadata) requiredOrigins(scope *scope, collector *core.LabelSet) { +// registerBuildStatement implements [scopeMetadata]. +func (m *trackingScopeMetadata) registerBuildStatement(pkg *core.Package) { + if pkg == nil || m.stmtCursor == nil { + return + } + + set := core.LabelSet{} for _, v := range m.symbolStack { - collector.Add(*v.origin) + set.Add(v.origin) } + deps := slices.Collect(maps.Keys(set)) + pkg.Metadata.RegisterStatement(NewBuildStatement(m.stmtCursor), deps) +} + +// resetSymbolStack implements [scopeMetadata]. +func (m *trackingScopeMetadata) resetSymbolStack() { + m.symbolStack = m.symbolStack[:0] } // setCursor implements [scopeMetadata]. func (m *trackingScopeMetadata) setCursor(stmt *Statement) { - m.cursorField = stmt + m.stmtCursor = stmt } // setSymbolOrigin implements [scopeMetadata]. func (m *trackingScopeMetadata) setSymbolOrigin(name string, origin core.BuildLabel) { if m.symbolOrigins == nil { - // lazy initialization to avoid unnecessary allocation of a map in smaller scopes (no subinclude). + // Lazy initialization to avoid unnecessary allocation of a map in smaller scopes (no subinclude). m.symbolOrigins = map[string]core.BuildLabel{} } @@ -1315,44 +1277,7 @@ func (m *trackingScopeMetadata) pushSymbol(name string, origin *core.BuildLabel) if origin == nil { return } - m.symbolStack = append(m.symbolStack, trackedSymbol{name: name, origin: origin}) -} - -// popSymbol implements [scopeMetadata]. -func (m *trackingScopeMetadata) popSymbol(name string) { - if name == "" || len(m.symbolStack) == 0 { - return - } - if m.symbolStack[len(m.symbolStack)-1].name == name { - m.symbolStack = m.symbolStack[:len(m.symbolStack)-1] - } -} - -// isTopLevelCall implements [scopeMetadata]. -func (m *trackingScopeMetadata) isTopLevelCall() bool { - return m.callDepth == 1 -} - -// incrementCallDepth implements [scopeMetadata]. -func (m *trackingScopeMetadata) incrementCallDepth() { - m.callDepth++ -} - -// decrementCallDepth implements [scopeMetadata]. -func (m *trackingScopeMetadata) decrementCallDepth() { - m.callDepth-- -} - -// getSymbolStackCheckpoint implements [scopeMetadata]. -func (m *trackingScopeMetadata) getSymbolStackCheckpoint() int { - return len(m.symbolStack) -} - -// restoreSymbolStack implements [scopeMetadata]. -func (m *trackingScopeMetadata) restoreSymbolStack(checkpoint int) { - if checkpoint >= 0 && checkpoint <= len(m.symbolStack) { - m.symbolStack = m.symbolStack[:checkpoint] - } + m.symbolStack = append(m.symbolStack, trackedSymbol{name: name, origin: *origin}) } // noopScopeMetadata implements the scopeMetadata interface with no-op methods. This is used to @@ -1365,8 +1290,11 @@ func (nm *noopScopeMetadata) cursor() *Statement { return nil } // origin implements [scopeMetadata]. func (nm *noopScopeMetadata) origin(scope *scope, name string) *core.BuildLabel { return nil } -// requiredOrigins implements [scopeMetadata]. -func (nm *noopScopeMetadata) requiredOrigins(scope *scope, collector *core.LabelSet) {} +// registerBuildStatement implements [scopeMetadata]. +func (nm *noopScopeMetadata) registerBuildStatement(pkg *core.Package) {} + +// resetSymbolStack implements [scopeMetadata]. +func (nm *noopScopeMetadata) resetSymbolStack() {} // setCursor implements [scopeMetadata]. func (nm *noopScopeMetadata) setCursor(stmt *Statement) {} @@ -1377,24 +1305,6 @@ func (nm *noopScopeMetadata) setSymbolOrigin(name string, origin core.BuildLabel // pushSymbol implements [scopeMetadata]. func (nm *noopScopeMetadata) pushSymbol(name string, origin *core.BuildLabel) {} -// popSymbol implements [scopeMetadata]. -func (nm *noopScopeMetadata) popSymbol(name string) {} - -// isTopLevelCall implements [scopeMetadata]. -func (nm *noopScopeMetadata) isTopLevelCall() bool { return false } - -// incrementCallDepth implements [scopeMetadata]. -func (nm *noopScopeMetadata) incrementCallDepth() {} - -// decrementCallDepth implements [scopeMetadata]. -func (nm *noopScopeMetadata) decrementCallDepth() {} - -// getSymbolStackCheckpoint implements [scopeMetadata]. -func (nm *noopScopeMetadata) getSymbolStackCheckpoint() int { return 0 } - -// restoreSymbolStack implements [scopeMetadata]. -func (nm *noopScopeMetadata) restoreSymbolStack(checkpoint int) {} - // NewBuildStatement creates a new core.BuildStatement from an asp.statement. func NewBuildStatement(stmt *Statement) core.BuildStatement { return core.BuildStatement{ diff --git a/src/parse/asp/objects.go b/src/parse/asp/objects.go index c96e7ce832..e70be1d052 100644 --- a/src/parse/asp/objects.go +++ b/src/parse/asp/objects.go @@ -694,7 +694,9 @@ func (f *pyFunc) String() string { func (f *pyFunc) Call(s *scope, c *Call) pyObject { if f.nativeCode != nil { if f.kwargs { - return f.callNative(s.NewScope("", 0), c) + callScope := s.NewScope("", 0) + callScope.caller = s + return f.callNative(callScope, c) } return f.callNative(s, c) } diff --git a/test/export/test_subinclude_trimming/BUILD b/test/export/test_subinclude_trimming/BUILD index b363357c3d..ae31a718eb 100644 --- a/test/export/test_subinclude_trimming/BUILD +++ b/test/export/test_subinclude_trimming/BUILD @@ -3,5 +3,8 @@ subinclude("//test/export:export_e2e_test_build_def") # Export a target generated by a custom build def and validate that any unused subincludes are trimmed. please_export_e2e_test( name = "export_subinclude_trimming", - export_targets = ["//:simple_custom_target"], + export_targets = [ + "//:simple_custom_target", + "//:variables_target", + ], ) diff --git a/test/export/test_subinclude_trimming/expected_repo/BUILD_FILE b/test/export/test_subinclude_trimming/expected_repo/BUILD_FILE index 261729adc3..b0cf935099 100644 --- a/test/export/test_subinclude_trimming/expected_repo/BUILD_FILE +++ b/test/export/test_subinclude_trimming/expected_repo/BUILD_FILE @@ -5,3 +5,23 @@ simple_custom_target( srcs = ["file.txt"], outs = ["file_simple.out"], ) + +subinclude( + "//build_defs:var1_build_def", + "//build_defs:var2_build_def", +) + +comma_separated_vars = ",".join([var1, var2]) + +genrule( + name = "variables_target", + outs = ["variable.out"], + cmd = f'echo "{comma_separated_vars}" > $OUT', +) + +subinclude("//build_defs:var3_build_def") + +message = "Testing {service} on {env}".format( + env = "production", + service = var3, +) diff --git a/test/export/test_subinclude_trimming/expected_repo/build_defs/BUILD_FILE b/test/export/test_subinclude_trimming/expected_repo/build_defs/BUILD_FILE index a556ce6d44..7fe80f2ba4 100644 --- a/test/export/test_subinclude_trimming/expected_repo/build_defs/BUILD_FILE +++ b/test/export/test_subinclude_trimming/expected_repo/build_defs/BUILD_FILE @@ -3,3 +3,21 @@ filegroup( srcs = ["simple.build_defs"], visibility = ["PUBLIC"], ) + +filegroup( + name = "var1_build_def", + srcs = ["var1.build_defs"], + visibility = ["PUBLIC"], +) + +filegroup( + name = "var2_build_def", + srcs = ["var2.build_defs"], + visibility = ["PUBLIC"], +) + +filegroup( + name = "var3_build_def", + srcs = ["var3.build_defs"], + visibility = ["PUBLIC"], +) diff --git a/test/export/test_subinclude_trimming/expected_repo/build_defs/var1.build_defs b/test/export/test_subinclude_trimming/expected_repo/build_defs/var1.build_defs new file mode 100644 index 0000000000..07814f1e90 --- /dev/null +++ b/test/export/test_subinclude_trimming/expected_repo/build_defs/var1.build_defs @@ -0,0 +1 @@ +var1 = "Variable 1" diff --git a/test/export/test_subinclude_trimming/expected_repo/build_defs/var2.build_defs b/test/export/test_subinclude_trimming/expected_repo/build_defs/var2.build_defs new file mode 100644 index 0000000000..b17ada62b4 --- /dev/null +++ b/test/export/test_subinclude_trimming/expected_repo/build_defs/var2.build_defs @@ -0,0 +1 @@ +var2 = "Variable 2" diff --git a/test/export/test_subinclude_trimming/expected_repo/build_defs/var3.build_defs b/test/export/test_subinclude_trimming/expected_repo/build_defs/var3.build_defs new file mode 100644 index 0000000000..c5cfb6d90c --- /dev/null +++ b/test/export/test_subinclude_trimming/expected_repo/build_defs/var3.build_defs @@ -0,0 +1 @@ +var3 = "Variable 3" diff --git a/test/export/test_subinclude_trimming/source_repo/BUILD_FILE b/test/export/test_subinclude_trimming/source_repo/BUILD_FILE index 44cf732531..22158a73ea 100644 --- a/test/export/test_subinclude_trimming/source_repo/BUILD_FILE +++ b/test/export/test_subinclude_trimming/source_repo/BUILD_FILE @@ -13,3 +13,38 @@ unused_target( name = "unneded", outs = ["unneded.out"], ) + +subinclude( + "//build_defs:var1_build_def", + "//build_defs:var2_build_def", +) + +comma_separated_vars = ",".join([var1, var2]) + +genrule( + name = "variables_target", + outs = ["variable.out"], + cmd = f'echo "{comma_separated_vars}" > $OUT', +) + + +genrule( + name = "unused_variables_target", + outs = ["unused_variable.out"], + cmd = f'echo "{unused_var}" > $OUT', +) + +subinclude( + "//build_defs:var3_build_def", +) + +message = "Testing {service} on {env}".format( + env = "production", + service = var3, +) + +genrule( + name = "format_target", + outs = ["format.out"], + cmd = f'echo "{message}" > $OUT', +) diff --git a/test/export/test_subinclude_trimming/source_repo/build_defs/BUILD_FILE b/test/export/test_subinclude_trimming/source_repo/build_defs/BUILD_FILE index 562e57a297..7a460f35b5 100644 --- a/test/export/test_subinclude_trimming/source_repo/build_defs/BUILD_FILE +++ b/test/export/test_subinclude_trimming/source_repo/build_defs/BUILD_FILE @@ -4,6 +4,24 @@ filegroup( visibility = ["PUBLIC"], ) +filegroup( + name = "var1_build_def", + srcs = ["var1.build_defs"], + visibility = ["PUBLIC"], +) + +filegroup( + name = "var2_build_def", + srcs = ["var2.build_defs"], + visibility = ["PUBLIC"], +) + +filegroup( + name = "var3_build_def", + srcs = ["var3.build_defs"], + visibility = ["PUBLIC"], +) + filegroup( name = "unused_build_def", srcs = ["unused.build_defs"], diff --git a/test/export/test_subinclude_trimming/source_repo/build_defs/unused.build_defs b/test/export/test_subinclude_trimming/source_repo/build_defs/unused.build_defs index 9337a65ecf..a20573cf39 100644 --- a/test/export/test_subinclude_trimming/source_repo/build_defs/unused.build_defs +++ b/test/export/test_subinclude_trimming/source_repo/build_defs/unused.build_defs @@ -4,5 +4,7 @@ def unused_target( return genrule( name = name, outs = outs, - cmd = "echo unneded > $OUT", + cmd = "echo unneeded > $OUT", ) + +unused_var = "Unused Variable" diff --git a/test/export/test_subinclude_trimming/source_repo/build_defs/var1.build_defs b/test/export/test_subinclude_trimming/source_repo/build_defs/var1.build_defs new file mode 100644 index 0000000000..07814f1e90 --- /dev/null +++ b/test/export/test_subinclude_trimming/source_repo/build_defs/var1.build_defs @@ -0,0 +1 @@ +var1 = "Variable 1" diff --git a/test/export/test_subinclude_trimming/source_repo/build_defs/var2.build_defs b/test/export/test_subinclude_trimming/source_repo/build_defs/var2.build_defs new file mode 100644 index 0000000000..b17ada62b4 --- /dev/null +++ b/test/export/test_subinclude_trimming/source_repo/build_defs/var2.build_defs @@ -0,0 +1 @@ +var2 = "Variable 2" diff --git a/test/export/test_subinclude_trimming/source_repo/build_defs/var3.build_defs b/test/export/test_subinclude_trimming/source_repo/build_defs/var3.build_defs new file mode 100644 index 0000000000..c5cfb6d90c --- /dev/null +++ b/test/export/test_subinclude_trimming/source_repo/build_defs/var3.build_defs @@ -0,0 +1 @@ +var3 = "Variable 3" From 1c67f8bacaa8237fb9e1fe129efa18156a750bd5 Mon Sep 17 00:00:00 2001 From: DuBento Date: Thu, 25 Jun 2026 17:26:44 +0100 Subject: [PATCH 103/118] Package level scope metadata; Tracking done at package level only. --- src/core/package_metadata.go | 100 ++++++++--------- src/export/trimmed_exporter.go | 10 +- src/export/trimmer.go | 4 +- src/parse/asp/interpreter.go | 68 +++++++----- src/parse/asp/interpreter_test.go | 102 ++++++++++-------- src/parse/asp/objects.go | 28 ++--- .../source_repo/BUILD_FILE | 6 +- .../source_repo/unused.in | 0 8 files changed, 181 insertions(+), 137 deletions(-) create mode 100644 test/export/test_in_file_func_def/source_repo/unused.in diff --git a/src/core/package_metadata.go b/src/core/package_metadata.go index 2dc9a333f4..6d0f677965 100644 --- a/src/core/package_metadata.go +++ b/src/core/package_metadata.go @@ -1,6 +1,7 @@ package core import ( + "fmt" "maps" "slices" @@ -60,27 +61,26 @@ type PackageMetadata interface { // includes the given build label. This should only be called for statements in BUILD files. RegisterSubincludeStatement(label BuildLabel, stmtProvider BuildStatementProvider) // FindStatement returns the build statement that was responsible for generating the given target. - // Returns nil if the target was not found in the recorded metadata. - FindStatement(target *BuildTarget) *BuildStatement + // Returns an error if the target was not found in the recorded metadata. + FindStatement(target *BuildTarget) (BuildStatement, error) // FindTargets returns all build targets that were generated by the given build statement. // Returns an empty slice if no targets were found for the given statement. - FindTargets(stmt *BuildStatement) BuildTargets + FindTargets(stmt BuildStatement) BuildTargets // FindRequiredSubincludes returns all subinclude labels that were required by the given target. // This method will only report the package level (direct) subincludes, make use of // [BuildGraph.TransitiveSubincludes] if you want all the required subincludes for one target. - // The return value is empty if no subinclude information was found for the target. - FindRequiredSubincludes(target *BuildTarget) BuildLabels + FindRequiredSubincludes(target *BuildTarget) (BuildLabels, error) // FindRelatedTargets finds all the targets that are related to the argument. In this context, // target relationship is determined by looking for targets generated by the same build statement. // The result excludes the target in the argument. - FindRelatedTargets(target *BuildTarget) BuildLabels + FindRelatedTargets(target *BuildTarget) (BuildLabels, error) // FindPackageRequiredSubincludes finds all the subincluded labels required by the package that // are not associated with BuildTarget generation. An example could be a variable declaration // that depends on a subincluded value. FindPackageRequiredSubincludes() BuildLabels // GetSubincludedLabels returns all build labels that were included by the given subinclude statement. // Returns the labels or an empty slice if the statement wasn't found. - GetSubincludedLabels(stmt *BuildStatement) BuildLabels + GetSubincludedLabels(stmt BuildStatement) BuildLabels } // packageMetadataImpl is the canonical implementation of the PackageMetadata interface. @@ -116,6 +116,11 @@ func newPackageMetadata() PackageMetadata { // RegisterStatement implements [PackageMetadata]. func (m *packageMetadataImpl) RegisterStatement(stmt BuildStatement, deps BuildLabels) { + // fmt.Printf("Registering statement %v deps: %v\n", stmt, deps) + if len(deps) == 0 { + // Skip adding to the map if the statement doesn't require any subincludes. + return + } m.stmtToRequiredSubincludes.Set(stmt, deps) } @@ -135,51 +140,50 @@ func (m *packageMetadataImpl) RegisterSubincludeStatement(label BuildLabel, stmt } // FindStatement implements [PackageMetadata]. -func (m *packageMetadataImpl) FindStatement(target *BuildTarget) *BuildStatement { +func (m *packageMetadataImpl) FindStatement(target *BuildTarget) (BuildStatement, error) { if target == nil { - return nil + return BuildStatement{}, fmt.Errorf("target is nil") } stmt := m.targetToStmt.Get(target) if stmt == (BuildStatement{}) { - log.Fatalf("Failed to find statement for target %s", target) - return nil + return BuildStatement{}, fmt.Errorf("failed to find statement for target %s", target) } - return &stmt + return stmt, nil } // FindTargets implements [PackageMetadata]. -func (m *packageMetadataImpl) FindTargets(stmt *BuildStatement) BuildTargets { - if stmt == nil { - return nil - } - - return m.stmtToTarget.Get(*stmt) +func (m *packageMetadataImpl) FindTargets(stmt BuildStatement) BuildTargets { + return m.stmtToTarget.Get(stmt) } // FindRequiredSubincludes implements [PackageMetadata]. -func (m *packageMetadataImpl) FindRequiredSubincludes(target *BuildTarget) BuildLabels { +func (m *packageMetadataImpl) FindRequiredSubincludes(target *BuildTarget) (BuildLabels, error) { if target == nil { - return nil + return nil, nil } - stmt := m.FindStatement(target) - if stmt == nil { - return nil + stmt, err := m.FindStatement(target) + if err != nil { + return nil, err } - directSubincludes := m.stmtToRequiredSubincludes.Get(*stmt) + directSubincludes := m.stmtToRequiredSubincludes.Get(stmt) if directSubincludes == nil { - log.Debugf("Failed to find direct subincludes for the build statement.") - return nil + // Could be empty if no subincluded label is required + return nil, nil } - return directSubincludes + return directSubincludes, nil } // FindRelatedTargets implements [PackageMetadata]. -func (m *packageMetadataImpl) FindRelatedTargets(target *BuildTarget) BuildLabels { - stmt := m.FindStatement(target) +func (m *packageMetadataImpl) FindRelatedTargets(target *BuildTarget) (BuildLabels, error) { + stmt, err := m.FindStatement(target) + if err != nil { + return nil, err + } + // fmt.Printf("STMT to Target mapping %v\n", m.stmtToTarget.Values()) relatedTargets := m.FindTargets(stmt) labels := make(BuildLabels, 0, len(relatedTargets)) for _, t := range relatedTargets { @@ -187,7 +191,7 @@ func (m *packageMetadataImpl) FindRelatedTargets(target *BuildTarget) BuildLabel labels = append(labels, t.Label) } } - return labels + return labels, nil } // FindPackageRequiredSubincludes implements [PackageMetadata]. @@ -206,12 +210,8 @@ func (m *packageMetadataImpl) FindPackageRequiredSubincludes() BuildLabels { } // GetSubincludedLabels implements [PackageMetadata]. -func (m *packageMetadataImpl) GetSubincludedLabels(stmt *BuildStatement) BuildLabels { - if stmt == nil { - return nil - } - - return m.labelsPerSubincludeStmt.Get(*stmt) +func (m *packageMetadataImpl) GetSubincludedLabels(stmt BuildStatement) BuildLabels { + return m.labelsPerSubincludeStmt.Get(stmt) } // noopPackageMetadata implements the PackageMetadata interface with no-op methods. This is the @@ -239,37 +239,37 @@ func (n *noopPackageMetadata) RegisterSubincludeStatement(label BuildLabel, stmt } // FindStatement implements [PackageMetadata]. -func (n *noopPackageMetadata) FindStatement(target *BuildTarget) *BuildStatement { - log.Fatalf("Metadata not tracked, using no-op implementation.") - return nil +func (n *noopPackageMetadata) FindStatement(target *BuildTarget) (BuildStatement, error) { + log.Fatalf("metadata not tracked, using no-op implementation") + return BuildStatement{}, nil } // FindTargets implements [PackageMetadata]. -func (n *noopPackageMetadata) FindTargets(stmt *BuildStatement) BuildTargets { - log.Fatalf("Metadata not tracked, using no-op implementation.") +func (n *noopPackageMetadata) FindTargets(stmt BuildStatement) BuildTargets { + log.Fatalf("metadata not tracked, using no-op implementation") return nil } // FindRequiredSubincludes implements [PackageMetadata]. -func (n *noopPackageMetadata) FindRequiredSubincludes(target *BuildTarget) BuildLabels { - log.Fatalf("Metadata not tracked, using no-op implementation.") - return nil +func (n *noopPackageMetadata) FindRequiredSubincludes(target *BuildTarget) (BuildLabels, error) { + log.Fatalf("metadata not tracked, using no-op implementation") + return nil, nil } // FindRelatedTargets implements [PackageMetadata]. -func (n *noopPackageMetadata) FindRelatedTargets(target *BuildTarget) BuildLabels { - log.Fatalf("Metadata not tracked, using no-op implementation.") - return nil +func (n *noopPackageMetadata) FindRelatedTargets(target *BuildTarget) (BuildLabels, error) { + log.Fatalf("metadata not tracked, using no-op implementation") + return nil, nil } // FindPackageRequiredSubincludes implements [PackageMetadata]. func (n *noopPackageMetadata) FindPackageRequiredSubincludes() BuildLabels { - log.Fatalf("Metadata not tracked, using no-op implementation.") + log.Fatalf("metadata not tracked, using no-op implementation") return nil } // GetSubincludedLabels implements [PackageMetadata]. -func (n *noopPackageMetadata) GetSubincludedLabels(stmt *BuildStatement) BuildLabels { - log.Fatalf("Metadata not tracked, using no-op implementation.") +func (n *noopPackageMetadata) GetSubincludedLabels(stmt BuildStatement) BuildLabels { + log.Fatalf("metadata not tracked, using no-op implementation") return nil } diff --git a/src/export/trimmed_exporter.go b/src/export/trimmed_exporter.go index bfb8a305b6..026d00636c 100644 --- a/src/export/trimmed_exporter.go +++ b/src/export/trimmed_exporter.go @@ -112,7 +112,10 @@ func (e *trimmedExporter) exportSubincludes(pkg *core.Package, target *core.Buil // by our used subinclude targets. FindRequiredSubincludes will report the required subincludes // for this target at the package level but we need to propagate the subincluded targets inside // build definitions since we are not trimming build_defs files. - usedSubincludes := pkg.Metadata.FindRequiredSubincludes(target) + usedSubincludes, err := pkg.Metadata.FindRequiredSubincludes(target) + if err != nil { + log.Fatalf("failed to find required subincludes for target %s: %s", target, err) + } e.setPackageSubincludes(pkg, usedSubincludes) allSubincludes := usedSubincludes @@ -160,7 +163,10 @@ func (e *trimmedExporter) setPackageSubincludes(pkg *core.Package, subincludes c // all co-defined targets are preserved in the exported BUILD file, preventing unresolved references // or partial declarations. func (e *trimmedExporter) exportRelatedTargets(pkg *core.Package, target *core.BuildTarget) { - relatedTargets := pkg.Metadata.FindRelatedTargets(target) + relatedTargets, err := pkg.Metadata.FindRelatedTargets(target) + if err != nil { + log.Fatalf("failed to find related targets for %s: %s", target, err) + } log.Debugf("Exporting targets related to %s: %v", target, relatedTargets) e.exportTargets(relatedTargets) } diff --git a/src/export/trimmer.go b/src/export/trimmer.go index cdb9c55328..6692740037 100644 --- a/src/export/trimmer.go +++ b/src/export/trimmer.go @@ -140,7 +140,7 @@ func (t *trimmer) trimFor(stmt *asp.Statement) { func (t *trimmer) trimSubinclude(stmt *asp.Statement) { bStmt := asp.NewBuildStatement(stmt) - stmtLabels := t.pkg.Metadata.GetSubincludedLabels(&bStmt) + stmtLabels := t.pkg.Metadata.GetSubincludedLabels(bStmt) subStmt := t.minimalSubincludeStatement(stmtLabels) t.write([]byte(subStmt)) } @@ -189,7 +189,7 @@ func (t *trimmer) isRequiredStatement(stmt *asp.Statement) bool { func (t *trimmer) relatedTargets(stmt *asp.Statement) []*core.BuildTarget { bStmt := asp.NewBuildStatement(stmt) - return t.pkg.Metadata.FindTargets(&bStmt) + return t.pkg.Metadata.FindTargets(bStmt) } func (t *trimmer) anyExported(targets []*core.BuildTarget) bool { diff --git a/src/parse/asp/interpreter.go b/src/parse/asp/interpreter.go index 32ee355f79..54e02f1ab6 100644 --- a/src/parse/asp/interpreter.go +++ b/src/parse/asp/interpreter.go @@ -48,7 +48,7 @@ func newInterpreter(state *core.BuildState, p *Parser) *interpreter { state: state, locals: map[string]pyObject{}, } - s.metadata = s.newScopeMetadata() + s.packageMetadata = s.newScopeMetadata() i := &interpreter{ scope: s, @@ -251,6 +251,8 @@ func (i *interpreter) Subinclude(pkgScope *scope, path string, label core.BuildL mode |= core.ParseModeForPreload } s := i.scope.NewScope(path, mode) + // // Skip metadata tracking for subincludes since these are not trimmed + // s.metadata = &noopScopeMetadata{} s.state = pkgScope.state // Scope needs a local version of CONFIG @@ -334,9 +336,9 @@ type scope struct { config *pyConfig globber *fs.Globber // True if this scope is for a pre- or post-build callback. - Callback bool - mode core.ParseMode - metadata scopeMetadata + Callback bool + mode core.ParseMode + packageMetadata scopeMetadata } // parseAnnotatedLabelInPackage similarly to parseLabelInPackage, parses the label contextualising it to the provided @@ -437,7 +439,19 @@ func (s *scope) NewScope(filename string, mode core.ParseMode) *scope { // NewPackagedScope creates a new child scope of this one pointing to the given package. // hint is a size hint for the new set of locals. func (s *scope) NewPackagedScope(pkg *core.Package, mode core.ParseMode, hint int) *scope { - return s.newScope(pkg, mode, pkg.Filename, hint) + newScope := s.newScope(pkg, mode, pkg.Filename, hint) + newScope.packageMetadata = newScope.newScopeMetadata() + return newScope +} + +func (s *scope) NewScopeWithCaller(filename string, mode core.ParseMode, caller *scope) *scope { + return s.newScopeWithCaller(s.pkg, mode, filename, 0, caller) +} + +func (s *scope) newScopeWithCaller(pkg *core.Package, mode core.ParseMode, filename string, hint int, caller *scope) *scope { + ns := s.newScope(pkg, mode, filename, hint) + ns.caller = caller + return ns } func (s *scope) newScope(pkg *core.Package, mode core.ParseMode, filename string, hint int) *scope { @@ -453,15 +467,22 @@ func (s *scope) newScope(pkg *core.Package, mode core.ParseMode, filename string config: s.config, Callback: s.Callback, mode: mode, + // We only track metadata at the top level scope created with [scope.NewPackagedScope], every + // other child scope or non-packaged (e.g. subincludes) defaults to a noop implementation. + packageMetadata: &noopScopeMetadata{}, } if pkg != nil && pkg.Subrepo != nil && pkg.Subrepo.State != nil { s2.state = pkg.Subrepo.State } - s2.metadata = s2.newScopeMetadata() - s2.metadata.setCursor(s.metadata.cursor()) return s2 } +// IsPackageScope returns true if this scope is the package scope +// (i.e. we are interpreting the build file directly and not inside any call). +func (s *scope) IsPackageScope() bool { + return s.caller == nil && s.subincludeLabel == nil && s.pkg != nil +} + // Error emits an error that stops further interpretation. // For convenience it is declared to return a pyObject but it never actually returns. func (s *scope) Error(msg string, args ...interface{}) pyObject { @@ -486,8 +507,8 @@ func (s *scope) NAssert(condition bool, msg string, args ...interface{}) { // It panics if the variable is not defined. func (s *scope) Lookup(name string) pyObject { if obj, present := s.locals[name]; present { - orig := s.metadata.origin(s, name) - s.metadata.pushSymbol(name, orig) + orig := s.packageMetadata.origin(s, name) + s.packageMetadata.pushSymbol(name, orig) return obj } else if s.parent != nil { return s.parent.Lookup(name) @@ -525,7 +546,7 @@ func (s *scope) SetAllWithOrigin(d pyDict, publicOnly bool, origin *core.BuildLa } else if !publicOnly || k[0] != '_' { s.locals[k] = v if origin != nil { - s.metadata.setSymbolOrigin(k, *origin) + s.packageMetadata.setSymbolOrigin(k, *origin) } } } @@ -564,7 +585,7 @@ func (s *scope) interpretStatements(statements []*Statement) pyObject { } }() for _, stmt = range statements { - s.metadata.setCursor(stmt) + s.packageMetadata.setCursor(stmt) if stmt.FuncDef != nil { s.Set(stmt.FuncDef.Name, newPyFunc(s, stmt.FuncDef)) } else if stmt.If != nil { @@ -608,8 +629,8 @@ func (s *scope) interpretStatements(statements []*Statement) pyObject { } else { s.Error("Unknown statement") // Shouldn't happen, amirite? } - s.metadata.registerBuildStatement(s.pkg) - s.metadata.resetSymbolStack() + s.packageMetadata.registerBuildStatement(s.pkg) + s.packageMetadata.resetSymbolStack() } return nil } @@ -751,7 +772,7 @@ func (s *scope) interpretJoin(base string, list *List) pyObject { } // Has a comprehension. Note that there is only ever one level; by the anecdata, two-level ones // are rare in this context so not worth worrying about here. - cs := s.NewScope(s.filename, s.mode) + cs := s.NewScopeWithCaller(s.filename, s.mode, s) it := s.iterable(list.Comprehension.Expr) first := true cs.evaluateComprehension(it, list.Comprehension, func(li pyObject) { @@ -974,7 +995,7 @@ func (s *scope) interpretList(expr *List) pyList { if expr.Comprehension == nil { return pyList(s.evaluateExpressions(expr.Values)) } - cs := s.NewScope(s.filename, s.mode) + cs := s.NewScopeWithCaller(s.filename, s.mode, s) it, l := s.iterableLen(expr.Comprehension.Expr) ret := make(pyList, 0, l) cs.evaluateComprehension(it, expr.Comprehension, func(li pyObject) { @@ -995,7 +1016,7 @@ func (s *scope) interpretDict(expr *Dict) pyObject { } return d } - cs := s.NewScope(s.filename, s.mode) + cs := s.NewScopeWithCaller(s.filename, s.mode, s) it, l := s.iterableLen(expr.Comprehension.Expr) ret := make(pyDict, l) cs.evaluateComprehension(it, expr.Comprehension, func(li pyObject) { @@ -1128,13 +1149,11 @@ func (s *scope) CurrentBuildStatement() core.BuildStatementProvider { // This statement should be the root method call, from a possibly long callstack, at the original // package level that generated the current build target. stmtScope := s - for curr := s; curr != nil; curr = curr.caller { - if curr.pkg != nil && curr.pkg == s.pkg { - stmtScope = curr - } + for stmtScope.caller != nil { + stmtScope = stmtScope.caller } - s.NAssert(stmtScope.metadata.cursor() == nil, "Cursor is not pointing to a statement") - return NewBuildStatement(stmtScope.metadata.cursor()) + s.NAssert(stmtScope.packageMetadata.cursor() == nil, "Cursor is not pointing to a statement") + return NewBuildStatement(stmtScope.packageMetadata.cursor()) } } @@ -1199,8 +1218,7 @@ type trackingScopeMetadata struct { // symbolOrigins tracks the subinclude label that each symbol was originally defined in. symbolOrigins map[string]core.BuildLabel // symbolStack tracks which symbols are actively in use during evaluation. - // Symbols are pushed onto the stack during lookups and popped or truncated (restored) after - // function calls. + // Symbols are pushed onto the stack during lookups and popped/truncated after each statement. symbolStack []trackedSymbol callDepth int } @@ -1248,6 +1266,8 @@ func (m *trackingScopeMetadata) registerBuildStatement(pkg *core.Package) { for _, v := range m.symbolStack { set.Add(v.origin) } + // fmt.Printf("Symbol stack: %v\nSymbol Origins: %v\n", m.symbolStack, m.symbolOrigins) + deps := slices.Collect(maps.Keys(set)) pkg.Metadata.RegisterStatement(NewBuildStatement(m.stmtCursor), deps) } diff --git a/src/parse/asp/interpreter_test.go b/src/parse/asp/interpreter_test.go index 8690959029..51eaa06d44 100644 --- a/src/parse/asp/interpreter_test.go +++ b/src/parse/asp/interpreter_test.go @@ -778,45 +778,45 @@ func TestCurrentBuildStatement(t *testing.T) { // Root statement in the BUILD file (e.g. a macro call) rootStmt := &Statement{Pos: 10, EndPos: 20} rootScope := &scope{ - pkg: pkg, - filename: pkg.Filename, - metadata: newScopeMetadata(), + pkg: pkg, + filename: pkg.Filename, + packageMetadata: newScopeMetadata(), } - rootScope.metadata.setCursor(rootStmt) + rootScope.packageMetadata.setCursor(rootStmt) // A nested call inside the same BUILD file (e.g. function def) nestedStmt := &Statement{Pos: 30, EndPos: 40} nestedScope := &scope{ - pkg: pkg, - filename: pkg.Filename, - caller: rootScope, - metadata: newScopeMetadata(), + pkg: pkg, + filename: pkg.Filename, + caller: rootScope, + packageMetadata: newScopeMetadata(), } - nestedScope.metadata.setCursor(nestedStmt) + nestedScope.packageMetadata.setCursor(nestedStmt) // A call from a different file (e.g. a function inside a subincluded .build_defs file) defsRootStmt := &Statement{Pos: 50, EndPos: 60} defsRootScope := &scope{ - pkg: pkg, - filename: "other/file.build_defs", - caller: nestedScope, - metadata: newScopeMetadata(), + pkg: pkg, + filename: "other/file.build_defs", + caller: nestedScope, + packageMetadata: newScopeMetadata(), } - defsRootScope.metadata.setCursor(defsRootStmt) + defsRootScope.packageMetadata.setCursor(defsRootStmt) // Another call deep in the other file defsNestedStmt := &Statement{Pos: 70, EndPos: 80} defsNestedScope := &scope{ - pkg: pkg, - filename: "other/file.build_defs", - caller: defsRootScope, - metadata: newScopeMetadata(), + pkg: pkg, + filename: "other/file.build_defs", + caller: defsRootScope, + packageMetadata: newScopeMetadata(), } - defsNestedScope.metadata.setCursor(defsNestedStmt) + defsNestedScope.packageMetadata.setCursor(defsNestedStmt) // A scope that has no pkg/filename context - standaloneScope := &scope{metadata: newScopeMetadata()} - standaloneScope.metadata.setCursor(rootStmt) + standaloneScope := &scope{packageMetadata: newScopeMetadata()} + standaloneScope.packageMetadata.setCursor(rootStmt) t.Run("FindsRootStatementFromBUILD", func(t *testing.T) { // Calling it from buildNestedScope should walk back to buildRootScope @@ -839,59 +839,74 @@ func TestCurrentBuildStatement(t *testing.T) { func TestActiveSubincludes(t *testing.T) { labelA := core.ParseBuildLabel("//pkg:labelA", "") labelB := core.ParseBuildLabel("//pkg:labelB", "") + stmt := &Statement{Pos: 1, EndPos: 10} t.Run("NoSubincludes", func(t *testing.T) { + pkg := core.NewPackage("pkg", core.WithPackageMetadata()) + meta := newScopeMetadata() // BUILD scope scopeBUILD := &scope{ - metadata: newScopeMetadata(), + packageMetadata: meta, } // Function execution scopeFuncExec := &scope{ - caller: scopeBUILD, - metadata: newScopeMetadata(), + caller: scopeBUILD, + packageMetadata: meta, } - labels := scopeFuncExec.RequiredSubincludes()() + scopeFuncExec.packageMetadata.setCursor(stmt) + scopeFuncExec.packageMetadata.registerBuildStatement(pkg) + + labels := pkg.Metadata.FindPackageRequiredSubincludes() assert.Empty(t, labels) }) t.Run("SingleSubinclude", func(t *testing.T) { + pkg := core.NewPackage("pkg", core.WithPackageMetadata()) + meta := newScopeMetadata() + // File A scope scopeA := &scope{ subincludeLabel: &labelA, locals: make(pyDict), - metadata: newScopeMetadata(), + packageMetadata: meta, } scopeA.SetAllWithOrigin(pyDict{"foo": pyString("val")}, false, &labelA) // Function defined in File A scopeFuncDef := &scope{ - parent: scopeA, - metadata: newScopeMetadata(), + parent: scopeA, + packageMetadata: meta, } // BUILD scope scopeBUILD := &scope{ - metadata: newScopeMetadata(), + packageMetadata: meta, } // Function execution scopeFuncExec := &scope{ - parent: scopeFuncDef, - caller: scopeBUILD, - metadata: newScopeMetadata(), + parent: scopeFuncDef, + caller: scopeBUILD, + packageMetadata: meta, } // Lookup triggers tracking of required subincludes scopeFuncExec.Lookup("foo") - labels := scopeFuncExec.RequiredSubincludes()() + scopeFuncExec.packageMetadata.setCursor(stmt) + scopeFuncExec.packageMetadata.registerBuildStatement(pkg) + + labels := pkg.Metadata.FindPackageRequiredSubincludes() assert.Equal(t, core.BuildLabels{labelA}, labels) }) t.Run("NestedSubincludes", func(t *testing.T) { + pkg := core.NewPackage("pkg", core.WithPackageMetadata()) + meta := newScopeMetadata() + // File A scope scopeA := &scope{ subincludeLabel: &labelA, locals: make(pyDict), - metadata: newScopeMetadata(), + packageMetadata: meta, } scopeA.SetAllWithOrigin(pyDict{"varA": pyString("valA")}, false, &labelA) @@ -900,31 +915,34 @@ func TestActiveSubincludes(t *testing.T) { subincludeLabel: &labelB, parent: scopeA, locals: make(pyDict), - metadata: newScopeMetadata(), + packageMetadata: meta, } scopeB.SetAllWithOrigin(pyDict{"varB": pyString("valB")}, false, &labelB) // Function defined in File B scopeFuncDef := &scope{ - parent: scopeB, - metadata: newScopeMetadata(), + parent: scopeB, + packageMetadata: meta, } // BUILD scope scopeBUILD := &scope{ - metadata: newScopeMetadata(), + packageMetadata: meta, } // Function execution scopeFuncExec := &scope{ - parent: scopeFuncDef, - caller: scopeBUILD, - metadata: newScopeMetadata(), + parent: scopeFuncDef, + caller: scopeBUILD, + packageMetadata: meta, } // Lookups trigger tracking of required subincludes scopeFuncExec.Lookup("varA") scopeFuncExec.Lookup("varB") - labels := scopeFuncExec.RequiredSubincludes()() + scopeFuncExec.packageMetadata.setCursor(stmt) + scopeFuncExec.packageMetadata.registerBuildStatement(pkg) + + labels := pkg.Metadata.FindPackageRequiredSubincludes() assert.ElementsMatch(t, core.BuildLabels{labelA, labelB}, labels) }) } diff --git a/src/parse/asp/objects.go b/src/parse/asp/objects.go index e70be1d052..8b098d59fc 100644 --- a/src/parse/asp/objects.go +++ b/src/parse/asp/objects.go @@ -694,19 +694,19 @@ func (f *pyFunc) String() string { func (f *pyFunc) Call(s *scope, c *Call) pyObject { if f.nativeCode != nil { if f.kwargs { - callScope := s.NewScope("", 0) - callScope.caller = s - return f.callNative(callScope, c) + cs := s.NewScope("", 0) + cs.caller = s + return f.callNative(cs, c) } return f.callNative(s, c) } - callScope := f.scope.newScope(s.pkg, s.mode, f.scope.filename, len(f.args)+1) - callScope.caller = s // registering previous scope as caller - callScope.config = s.config - callScope.Set("CONFIG", s.config) // This needs to be copied across too :( - callScope.Callback = s.Callback - callScope.parsingFor = s.parsingFor + cs := f.scope.newScope(s.pkg, s.mode, f.scope.filename, len(f.args)+1) + cs.caller = s // registering previous scope as caller + cs.config = s.config + cs.Set("CONFIG", s.config) // This needs to be copied across too :( + cs.Callback = s.Callback + cs.parsingFor = s.parsingFor // Handle implicit 'self' parameter for bound functions. args := c.Arguments if f.self != nil { @@ -724,23 +724,23 @@ func (f *pyFunc) Call(s *scope, c *Call) pyObject { if present { name = f.args[idx] } - callScope.Set(name, f.validateType(s, idx, &a.Value)) + cs.Set(name, f.validateType(s, idx, &a.Value)) } else { if i >= len(f.args) { s.Error("Too many arguments to %s", f.name) } else if f.kwargsonly { s.Error("Function %s can only be called with keyword arguments", f.name) } - callScope.Set(f.args[i], f.validateType(s, i, &a.Value)) + cs.Set(f.args[i], f.validateType(s, i, &a.Value)) } } // Now make sure any arguments with defaults are set, and check any others have been passed. for i, a := range f.args { - if callScope.LocalLookup(a) == nil { - callScope.Set(a, f.defaultArg(s, i, a)) + if cs.LocalLookup(a) == nil { + cs.Set(a, f.defaultArg(s, i, a)) } } - ret := callScope.interpretStatements(f.code) + ret := cs.interpretStatements(f.code) if ret == nil { return None // Implicit 'return None' in any function that didn't do that itself. } diff --git a/test/export/test_in_file_func_def/source_repo/BUILD_FILE b/test/export/test_in_file_func_def/source_repo/BUILD_FILE index 1cc13d43b0..2f589a44e1 100644 --- a/test/export/test_in_file_func_def/source_repo/BUILD_FILE +++ b/test/export/test_in_file_func_def/source_repo/BUILD_FILE @@ -16,7 +16,7 @@ custom( ) custom( - name = "unneded", - srcs = ["unneded.in"], - outs = ["unneded.out"], + name = "unused", + srcs = ["unused.in"], + outs = ["unused.out"], ) diff --git a/test/export/test_in_file_func_def/source_repo/unused.in b/test/export/test_in_file_func_def/source_repo/unused.in new file mode 100644 index 0000000000..e69de29bb2 From 7d6ba12ec48cebafd359fdee5a3deb163075d4ba Mon Sep 17 00:00:00 2001 From: DuBento Date: Thu, 25 Jun 2026 19:47:01 +0100 Subject: [PATCH 104/118] Package metadata lookup table in interpreter to avoid recursive caller traversal. --- src/core/build_label.go | 3 +- src/core/build_target.go | 2 +- src/core/graph.go | 2 +- src/core/state.go | 2 +- src/parse/asp/interpreter.go | 111 +++++++++++++--------------- src/parse/asp/interpreter_test.go | 119 ++++++++---------------------- src/parse/asp/objects.go | 2 - 7 files changed, 88 insertions(+), 153 deletions(-) diff --git a/src/core/build_label.go b/src/core/build_label.go index 26423571f9..d5f402ba64 100644 --- a/src/core/build_label.go +++ b/src/core/build_label.go @@ -473,7 +473,8 @@ func subrepoLabel(subrepoName, arch string) BuildLabel { return BuildLabel{Name: subrepoName, Subrepo: arch} } -func hashBuildLabel(l BuildLabel) uint64 { +// HashBuildLabel calculates an hash of Build Label, suitable to use for map indexing. +func HashBuildLabel(l BuildLabel) uint64 { return cmap.XXHashes(l.Subrepo, l.PackageName, l.Name) } diff --git a/src/core/build_target.go b/src/core/build_target.go index 82dd99e220..e20aa11d8d 100644 --- a/src/core/build_target.go +++ b/src/core/build_target.go @@ -418,7 +418,7 @@ func (target *BuildTarget) String() string { // hashBuildTarget returns a mostly unique hash of a BuildTarget by relying on the BuildLabel hash. func hashBuildTarget(target *BuildTarget) uint64 { - return hashBuildLabel(target.Label) + return HashBuildLabel(target.Label) } // TmpDir returns the temporary working directory for this target, eg. diff --git a/src/core/graph.go b/src/core/graph.go index c73961a0e1..246d0b01d1 100644 --- a/src/core/graph.go +++ b/src/core/graph.go @@ -154,7 +154,7 @@ func (graph *BuildGraph) PackageMap() map[string]*Package { // NewGraph constructs and returns a new BuildGraph. func NewGraph() *BuildGraph { g := &BuildGraph{ - targets: cmap.New[BuildLabel, *BuildTarget](cmap.DefaultShardCount, hashBuildLabel), + targets: cmap.New[BuildLabel, *BuildTarget](cmap.DefaultShardCount, HashBuildLabel), packages: cmap.New[packageKey, *Package](cmap.DefaultShardCount, hashPackageKey), subrepos: cmap.New[string, *Subrepo](cmap.SmallShardCount, cmap.XXHash), subincludeSubincludes: map[BuildLabel]LabelSet{}, diff --git a/src/core/state.go b/src/core/state.go index 202990857f..a693f854c6 100644 --- a/src/core/state.go +++ b/src/core/state.go @@ -1542,7 +1542,7 @@ func NewBuildState(config *Configuration) *BuildState { progress: &stateProgress{ numActive: 1, // One for the initial target adding on the main thread. numPending: 1, - pendingTargets: cmap.New[BuildLabel, chan struct{}](cmap.DefaultShardCount, hashBuildLabel), + pendingTargets: cmap.New[BuildLabel, chan struct{}](cmap.DefaultShardCount, HashBuildLabel), pendingPackages: cmap.New[packageKey, chan struct{}](cmap.DefaultShardCount, hashPackageKey), packageWaits: cmap.New[packageKey, chan struct{}](cmap.DefaultShardCount, hashPackageKey), internalResults: make(chan *BuildResult, 1000), diff --git a/src/parse/asp/interpreter.go b/src/parse/asp/interpreter.go index 54e02f1ab6..316f03a596 100644 --- a/src/parse/asp/interpreter.go +++ b/src/parse/asp/interpreter.go @@ -38,6 +38,8 @@ type interpreter struct { stringMethods, dictMethods, configMethods map[string]*pyFunc regexCache *cmap.Map[string, *regexp.Regexp] + // packageMetadata store scope level metadata for each package. + packageMetadata *cmap.Map[core.BuildLabel, scopeMetadata] } // newInterpreter creates and returns a new interpreter instance. @@ -48,7 +50,7 @@ func newInterpreter(state *core.BuildState, p *Parser) *interpreter { state: state, locals: map[string]pyObject{}, } - s.packageMetadata = s.newScopeMetadata() + s.metadata = &noopScopeMetadata{} i := &interpreter{ scope: s, @@ -66,6 +68,11 @@ func newInterpreter(state *core.BuildState, p *Parser) *interpreter { i.subincludes = cmap.NewErrMap[string, pyDict](cmap.SmallShardCount, cmap.XXHash, i.limiter) i.asts = cmap.NewErrMap[string, []*Statement](cmap.SmallShardCount, cmap.XXHash, i.limiter) } + + if state.ParseMetadata { + i.packageMetadata = cmap.New[core.BuildLabel, scopeMetadata](cmap.SmallShardCount, core.HashBuildLabel) + } + s.interpreter = i s.LoadSingletons(state) return i @@ -328,17 +335,14 @@ type scope struct { parsingFor *parseTarget // parent points to the lexical parent of this scope. It is used for variable resolution // and is nil for the root scope. - parent *scope - // caller points to the scope that initiated the call which created this scope. - // It is used to trace the call stack and is nil if not in a call stack. - caller *scope + parent *scope locals pyDict config *pyConfig globber *fs.Globber // True if this scope is for a pre- or post-build callback. - Callback bool - mode core.ParseMode - packageMetadata scopeMetadata + Callback bool + mode core.ParseMode + metadata scopeMetadata } // parseAnnotatedLabelInPackage similarly to parseLabelInPackage, parses the label contextualising it to the provided @@ -440,20 +444,10 @@ func (s *scope) NewScope(filename string, mode core.ParseMode) *scope { // hint is a size hint for the new set of locals. func (s *scope) NewPackagedScope(pkg *core.Package, mode core.ParseMode, hint int) *scope { newScope := s.newScope(pkg, mode, pkg.Filename, hint) - newScope.packageMetadata = newScope.newScopeMetadata() + newScope.metadata = newScope.getOrNewMetadata(pkg) return newScope } -func (s *scope) NewScopeWithCaller(filename string, mode core.ParseMode, caller *scope) *scope { - return s.newScopeWithCaller(s.pkg, mode, filename, 0, caller) -} - -func (s *scope) newScopeWithCaller(pkg *core.Package, mode core.ParseMode, filename string, hint int, caller *scope) *scope { - ns := s.newScope(pkg, mode, filename, hint) - ns.caller = caller - return ns -} - func (s *scope) newScope(pkg *core.Package, mode core.ParseMode, filename string, hint int) *scope { s2 := &scope{ ctx: s.ctx, @@ -469,7 +463,7 @@ func (s *scope) newScope(pkg *core.Package, mode core.ParseMode, filename string mode: mode, // We only track metadata at the top level scope created with [scope.NewPackagedScope], every // other child scope or non-packaged (e.g. subincludes) defaults to a noop implementation. - packageMetadata: &noopScopeMetadata{}, + metadata: &noopScopeMetadata{}, } if pkg != nil && pkg.Subrepo != nil && pkg.Subrepo.State != nil { s2.state = pkg.Subrepo.State @@ -477,12 +471,6 @@ func (s *scope) newScope(pkg *core.Package, mode core.ParseMode, filename string return s2 } -// IsPackageScope returns true if this scope is the package scope -// (i.e. we are interpreting the build file directly and not inside any call). -func (s *scope) IsPackageScope() bool { - return s.caller == nil && s.subincludeLabel == nil && s.pkg != nil -} - // Error emits an error that stops further interpretation. // For convenience it is declared to return a pyObject but it never actually returns. func (s *scope) Error(msg string, args ...interface{}) pyObject { @@ -507,8 +495,8 @@ func (s *scope) NAssert(condition bool, msg string, args ...interface{}) { // It panics if the variable is not defined. func (s *scope) Lookup(name string) pyObject { if obj, present := s.locals[name]; present { - orig := s.packageMetadata.origin(s, name) - s.packageMetadata.pushSymbol(name, orig) + orig := s.metadata.origin(s, name) + s.metadata.pushSymbol(name, orig) return obj } else if s.parent != nil { return s.parent.Lookup(name) @@ -546,7 +534,7 @@ func (s *scope) SetAllWithOrigin(d pyDict, publicOnly bool, origin *core.BuildLa } else if !publicOnly || k[0] != '_' { s.locals[k] = v if origin != nil { - s.packageMetadata.setSymbolOrigin(k, *origin) + s.metadata.setSymbolOrigin(k, *origin) } } } @@ -585,7 +573,7 @@ func (s *scope) interpretStatements(statements []*Statement) pyObject { } }() for _, stmt = range statements { - s.packageMetadata.setCursor(stmt) + s.metadata.setCursor(stmt) if stmt.FuncDef != nil { s.Set(stmt.FuncDef.Name, newPyFunc(s, stmt.FuncDef)) } else if stmt.If != nil { @@ -629,8 +617,8 @@ func (s *scope) interpretStatements(statements []*Statement) pyObject { } else { s.Error("Unknown statement") // Shouldn't happen, amirite? } - s.packageMetadata.registerBuildStatement(s.pkg) - s.packageMetadata.resetSymbolStack() + s.metadata.registerBuildStatement(s.pkg) + s.metadata.resetSymbolStack() } return nil } @@ -772,7 +760,7 @@ func (s *scope) interpretJoin(base string, list *List) pyObject { } // Has a comprehension. Note that there is only ever one level; by the anecdata, two-level ones // are rare in this context so not worth worrying about here. - cs := s.NewScopeWithCaller(s.filename, s.mode, s) + cs := s.NewScope(s.filename, s.mode) it := s.iterable(list.Comprehension.Expr) first := true cs.evaluateComprehension(it, list.Comprehension, func(li pyObject) { @@ -995,7 +983,7 @@ func (s *scope) interpretList(expr *List) pyList { if expr.Comprehension == nil { return pyList(s.evaluateExpressions(expr.Values)) } - cs := s.NewScopeWithCaller(s.filename, s.mode, s) + cs := s.NewScope(s.filename, s.mode) it, l := s.iterableLen(expr.Comprehension.Expr) ret := make(pyList, 0, l) cs.evaluateComprehension(it, expr.Comprehension, func(li pyObject) { @@ -1016,7 +1004,7 @@ func (s *scope) interpretDict(expr *Dict) pyObject { } return d } - cs := s.NewScopeWithCaller(s.filename, s.mode, s) + cs := s.NewScope(s.filename, s.mode) it, l := s.iterableLen(expr.Comprehension.Expr) ret := make(pyDict, l) cs.evaluateComprehension(it, expr.Comprehension, func(li pyObject) { @@ -1145,15 +1133,11 @@ func (s *scope) Constant(expr *Expression) pyObject { // metadata is not being tracked. func (s *scope) CurrentBuildStatement() core.BuildStatementProvider { return func() core.BuildStatement { - // We walk back on the callstack until we find the highest-level function call in the package file. - // This statement should be the root method call, from a possibly long callstack, at the original - // package level that generated the current build target. - stmtScope := s - for stmtScope.caller != nil { - stmtScope = stmtScope.caller - } - s.NAssert(stmtScope.packageMetadata.cursor() == nil, "Cursor is not pointing to a statement") - return NewBuildStatement(stmtScope.packageMetadata.cursor()) + // We lookup the package metadata from the interpreter table no matter how deep we are in the + // call stack. We do this to avoid a recursive lookup from leaf scopes to the top level package scope. + meta := s.interpreter.packageMetadata.Get(s.pkg.Label()) + s.NAssert(meta.cursor() == nil, "Cursor is not pointing to a statement") + return NewBuildStatement(meta.cursor()) } } @@ -1165,25 +1149,32 @@ func (s *scope) pkgFilename() string { return "" } -// newScopeMetadata creates and returns a initialized scopeMetadata instance. It will return -// a no-op implementation if state.ParseMetadata is not set or if we simply want to skip the -// tracking for a certain scope. -func (s *scope) newScopeMetadata() scopeMetadata { - if !s.state.ParseMetadata || - s.pkg == nil || - s.pkg.Subrepo.IsExternal() { - // Skip metadata tracking if: - // 1. ParseMetadata flag is disabled; - // 2. Not interpreting a package (e.g. in subincluded targets) - // 3. Any external/remote subrepos. - // For 2 and 3, we never trim these, so avoiding tracking saves CPU and memory. - return &noopScopeMetadata{} +// getOrNewMetadata creates and returns a initialized scopeMetadata instance, or pulls an existing +// metadata instance for that package from the interpreter package metadata table. It will return +// a no-op implementation if we simply want to skip tracking for a certain scope. +func (s *scope) getOrNewMetadata(pkg *core.Package) scopeMetadata { + // Skip metadata tracking if: + // 1. ParseMetadata flag is disabled; + // 2. Not interpreting a package (e.g. in subincluded targets) + // 3. Any external/remote subrepos. + // For 2 and 3, we never trim these, so avoiding tracking saves CPU and memory. + var meta scopeMetadata = &noopScopeMetadata{} + if s.interpreter.packageMetadata == nil { // This will be initialized if ParseMetadata is set. + return meta } - return &trackingScopeMetadata{ - // symbolOrigins is lazy initialized in [setSymbolOrigin] - symbolStack: []trackedSymbol{}, + if pkg == nil || pkg.Subrepo.IsExternal() { + return meta } + + meta, _ = s.interpreter.packageMetadata.AddOrGet(pkg.Label(), func() scopeMetadata { + return &trackingScopeMetadata{ + // symbolOrigins is lazy initialized in [setSymbolOrigin] + symbolStack: []trackedSymbol{}, + } + }) + return meta + } // scopeMetadata defines an interface for tracking evaluation metadata (such as AST cursor position diff --git a/src/parse/asp/interpreter_test.go b/src/parse/asp/interpreter_test.go index 51eaa06d44..a8a644cf63 100644 --- a/src/parse/asp/interpreter_test.go +++ b/src/parse/asp/interpreter_test.go @@ -772,66 +772,23 @@ func TestStrRjust(t *testing.T) { } func TestCurrentBuildStatement(t *testing.T) { - pkg := core.NewPackage("test/package") + pkg := core.NewPackage("test/package", core.WithPackageMetadata()) pkg.Filename = "test/package/BUILD" - // Root statement in the BUILD file (e.g. a macro call) - rootStmt := &Statement{Pos: 10, EndPos: 20} - rootScope := &scope{ - pkg: pkg, - filename: pkg.Filename, - packageMetadata: newScopeMetadata(), - } - rootScope.packageMetadata.setCursor(rootStmt) - - // A nested call inside the same BUILD file (e.g. function def) - nestedStmt := &Statement{Pos: 30, EndPos: 40} - nestedScope := &scope{ - pkg: pkg, - filename: pkg.Filename, - caller: rootScope, - packageMetadata: newScopeMetadata(), - } - nestedScope.packageMetadata.setCursor(nestedStmt) - - // A call from a different file (e.g. a function inside a subincluded .build_defs file) - defsRootStmt := &Statement{Pos: 50, EndPos: 60} - defsRootScope := &scope{ - pkg: pkg, - filename: "other/file.build_defs", - caller: nestedScope, - packageMetadata: newScopeMetadata(), - } - defsRootScope.packageMetadata.setCursor(defsRootStmt) - - // Another call deep in the other file - defsNestedStmt := &Statement{Pos: 70, EndPos: 80} - defsNestedScope := &scope{ - pkg: pkg, - filename: "other/file.build_defs", - caller: defsRootScope, - packageMetadata: newScopeMetadata(), - } - defsNestedScope.packageMetadata.setCursor(defsNestedStmt) + state := core.NewBuildState(core.DefaultConfiguration()) + state.ParseMetadata = true - // A scope that has no pkg/filename context - standaloneScope := &scope{packageMetadata: newScopeMetadata()} - standaloneScope.packageMetadata.setCursor(rootStmt) + parser := &Parser{} + interpreter := newInterpreter(state, parser) - t.Run("FindsRootStatementFromBUILD", func(t *testing.T) { - // Calling it from buildNestedScope should walk back to buildRootScope - stmt := nestedScope.CurrentBuildStatement()() - assert.Equal(t, NewBuildStatement(rootStmt), stmt) - }) + rootScope := interpreter.scope.NewPackagedScope(pkg, 0, 0) - t.Run("FindsRootStatementFromOtherFile", func(t *testing.T) { - // Calling it from defsNestedScope should still find the root statement in the BUILD file - stmt := defsNestedScope.CurrentBuildStatement()() - assert.Equal(t, NewBuildStatement(rootStmt), stmt) - }) + // Root statement in the BUILD file (e.g. a macro call) + rootStmt := &Statement{Pos: 10, EndPos: 20} + rootScope.metadata.setCursor(rootStmt) - t.Run("HandlesNoPackageFileInStack", func(t *testing.T) { - stmt := standaloneScope.CurrentBuildStatement()() + t.Run("FindsRootStatement", func(t *testing.T) { + stmt := rootScope.CurrentBuildStatement()() assert.Equal(t, NewBuildStatement(rootStmt), stmt) }) } @@ -844,17 +801,13 @@ func TestActiveSubincludes(t *testing.T) { t.Run("NoSubincludes", func(t *testing.T) { pkg := core.NewPackage("pkg", core.WithPackageMetadata()) meta := newScopeMetadata() - // BUILD scope - scopeBUILD := &scope{ - packageMetadata: meta, - } + // Function execution scopeFuncExec := &scope{ - caller: scopeBUILD, - packageMetadata: meta, + metadata: meta, } - scopeFuncExec.packageMetadata.setCursor(stmt) - scopeFuncExec.packageMetadata.registerBuildStatement(pkg) + scopeFuncExec.metadata.setCursor(stmt) + scopeFuncExec.metadata.registerBuildStatement(pkg) labels := pkg.Metadata.FindPackageRequiredSubincludes() assert.Empty(t, labels) @@ -868,31 +821,27 @@ func TestActiveSubincludes(t *testing.T) { scopeA := &scope{ subincludeLabel: &labelA, locals: make(pyDict), - packageMetadata: meta, + metadata: meta, } scopeA.SetAllWithOrigin(pyDict{"foo": pyString("val")}, false, &labelA) // Function defined in File A scopeFuncDef := &scope{ - parent: scopeA, - packageMetadata: meta, - } - // BUILD scope - scopeBUILD := &scope{ - packageMetadata: meta, + parent: scopeA, + metadata: meta, } + // Function execution scopeFuncExec := &scope{ - parent: scopeFuncDef, - caller: scopeBUILD, - packageMetadata: meta, + parent: scopeFuncDef, + metadata: meta, } // Lookup triggers tracking of required subincludes scopeFuncExec.Lookup("foo") - scopeFuncExec.packageMetadata.setCursor(stmt) - scopeFuncExec.packageMetadata.registerBuildStatement(pkg) + scopeFuncExec.metadata.setCursor(stmt) + scopeFuncExec.metadata.registerBuildStatement(pkg) labels := pkg.Metadata.FindPackageRequiredSubincludes() assert.Equal(t, core.BuildLabels{labelA}, labels) @@ -906,7 +855,7 @@ func TestActiveSubincludes(t *testing.T) { scopeA := &scope{ subincludeLabel: &labelA, locals: make(pyDict), - packageMetadata: meta, + metadata: meta, } scopeA.SetAllWithOrigin(pyDict{"varA": pyString("valA")}, false, &labelA) @@ -915,32 +864,28 @@ func TestActiveSubincludes(t *testing.T) { subincludeLabel: &labelB, parent: scopeA, locals: make(pyDict), - packageMetadata: meta, + metadata: meta, } scopeB.SetAllWithOrigin(pyDict{"varB": pyString("valB")}, false, &labelB) // Function defined in File B scopeFuncDef := &scope{ - parent: scopeB, - packageMetadata: meta, - } - // BUILD scope - scopeBUILD := &scope{ - packageMetadata: meta, + parent: scopeB, + metadata: meta, } + // Function execution scopeFuncExec := &scope{ - parent: scopeFuncDef, - caller: scopeBUILD, - packageMetadata: meta, + parent: scopeFuncDef, + metadata: meta, } // Lookups trigger tracking of required subincludes scopeFuncExec.Lookup("varA") scopeFuncExec.Lookup("varB") - scopeFuncExec.packageMetadata.setCursor(stmt) - scopeFuncExec.packageMetadata.registerBuildStatement(pkg) + scopeFuncExec.metadata.setCursor(stmt) + scopeFuncExec.metadata.registerBuildStatement(pkg) labels := pkg.Metadata.FindPackageRequiredSubincludes() assert.ElementsMatch(t, core.BuildLabels{labelA, labelB}, labels) diff --git a/src/parse/asp/objects.go b/src/parse/asp/objects.go index 8b098d59fc..9a4b6536f4 100644 --- a/src/parse/asp/objects.go +++ b/src/parse/asp/objects.go @@ -695,14 +695,12 @@ func (f *pyFunc) Call(s *scope, c *Call) pyObject { if f.nativeCode != nil { if f.kwargs { cs := s.NewScope("", 0) - cs.caller = s return f.callNative(cs, c) } return f.callNative(s, c) } cs := f.scope.newScope(s.pkg, s.mode, f.scope.filename, len(f.args)+1) - cs.caller = s // registering previous scope as caller cs.config = s.config cs.Set("CONFIG", s.config) // This needs to be copied across too :( cs.Callback = s.Callback From 0af4d973b81f5f09009d80ad6651d735068bfdc7 Mon Sep 17 00:00:00 2001 From: DuBento Date: Thu, 25 Jun 2026 19:51:43 +0100 Subject: [PATCH 105/118] Trimmer: always keep for expression but trim stmts inside --- src/export/export_test.go | 6 ++++ .../test_data/trim_for_expected_none.build | 2 ++ src/export/trimmer.go | 31 ++++++++++++++----- 3 files changed, 31 insertions(+), 8 deletions(-) create mode 100644 src/export/test_data/trim_for_expected_none.build diff --git a/src/export/export_test.go b/src/export/export_test.go index 7d7b828308..3aa6488eba 100644 --- a/src/export/export_test.go +++ b/src/export/export_test.go @@ -152,6 +152,12 @@ func TestStatementTrim(t *testing.T) { required: []string{"a"}, expected: "src/export/test_data/trim_for_expected_a.build", }, + { + name: "Target not required in for - loop body has a pass", + content: "src/export/test_data/trim_for.build", + required: []string{}, + expected: "src/export/test_data/trim_for_expected_none.build", + }, { name: "Required if stmt in for", content: "src/export/test_data/trim_for_if.build", diff --git a/src/export/test_data/trim_for_expected_none.build b/src/export/test_data/trim_for_expected_none.build new file mode 100644 index 0000000000..c1a7b52be7 --- /dev/null +++ b/src/export/test_data/trim_for_expected_none.build @@ -0,0 +1,2 @@ +for i in range(0,2): + pass # Trimmed during export diff --git a/src/export/trimmer.go b/src/export/trimmer.go index 6692740037..d13a2b7700 100644 --- a/src/export/trimmer.go +++ b/src/export/trimmer.go @@ -10,6 +10,8 @@ import ( "github.com/thought-machine/please/src/parse/asp" ) +var passExpression = []byte("pass # Trimmed during export") + // trimmer implements the filtering logic for statements in package files. type trimmer struct { // origin are the bytes from the original package file. @@ -51,26 +53,35 @@ func (t *trimmer) walkFile(stmts []*asp.Statement, start, end asp.Position, cons } // trimBlock visits all the statements in a block and trims undesired statements. -func (t *trimmer) trimBlock(stmts []*asp.Statement, blockStart, blockEnd asp.Position) { +func (t *trimmer) trimBlock(stmts []*asp.Statement, blockStart, blockEnd asp.Position) bool { + var written bool t.walkFile(stmts, blockStart, blockEnd, func(stmt *asp.Statement) { if stmt.If != nil { - t.trimIf(stmt) + if t.trimIf(stmt) { + written = true + } } else if stmt.For != nil { - t.trimFor(stmt) + if t.trimFor(stmt) { + written = true + } } else if stmt.Ident != nil && stmt.Ident.Name == "subinclude" { t.trimSubinclude(stmt) + written = true } else if relatives := t.relatedTargets(stmt); len(relatives) > 0 { // Meaning it is a build statement that creates build targets. if t.anyExported(relatives) { t.copy(stmt.Pos, stmt.EndPos) + written = true } } else { // Write every other statement. // If the statement didn't generate any targets (e.g. variable assignments, package() calls), // we keep it to ensure the BUILD file remains valid. t.copy(stmt.Pos, stmt.EndPos) + written = true } }) + return written } // trimIf will trim an if-else statement by exporting only the required targets, but keeping the @@ -127,15 +138,19 @@ func (t *trimmer) trimIf(stmt *asp.Statement) bool { return true } -func (t *trimmer) trimFor(stmt *asp.Statement) { - if len(stmt.For.Statements) == 0 || !t.isRequiredStatement(stmt) { - return +func (t *trimmer) trimFor(stmt *asp.Statement) bool { + if len(stmt.For.Statements) == 0 { + return false } hStart, hEnd := stmt.Pos, stmt.For.Statements[0].Pos t.copy(hStart, hEnd) - t.trimBlock(stmt.For.Statements, hEnd, stmt.EndPos) + written := t.trimBlock(stmt.For.Statements, hEnd, stmt.EndPos) + if !written { + t.write(passExpression) + } + return true } func (t *trimmer) trimSubinclude(stmt *asp.Statement) { @@ -154,7 +169,7 @@ func (t *trimmer) passBlock(stmts []*asp.Statement, blockStart, blockEnd asp.Pos // the "pass" primitive. This is useful when parsing inner blocks (e.g. if-else stmts). if !passWritten { passWritten = true - t.write([]byte("pass # Trimmed during export")) + t.write(passExpression) } }) } From 1e090637440e63b0e33f92f203117198ec91f9e8 Mon Sep 17 00:00:00 2001 From: DuBento Date: Fri, 26 Jun 2026 12:19:50 +0100 Subject: [PATCH 106/118] glob statement: register files required by statements --- src/core/package_metadata.go | 54 ++++++++++++------- src/export/export.go | 31 ++++++----- src/export/trimmed_exporter.go | 11 ++-- src/parse/asp/builtins.go | 1 + src/parse/asp/interpreter.go | 35 ++++++++---- src/parse/asp/interpreter_test.go | 6 +-- test/export/test_for_glob/BUILD | 6 +++ .../test_for_glob/expected_repo/.plzconfig | 2 + .../expected_repo/pkg/BUILD_FILE | 17 ++++++ .../test_for_glob/expected_repo/pkg/doc1.in | 1 + .../test_for_glob/expected_repo/pkg/doc2.in | 1 + .../test_for_glob/expected_repo/pkg/file1.in | 1 + .../test_for_glob/expected_repo/pkg/file2.in | 1 + .../test_for_glob/source_repo/.plzconfig | 2 + .../test_for_glob/source_repo/pkg/BUILD_FILE | 29 ++++++++++ .../test_for_glob/source_repo/pkg/doc1.in | 1 + .../test_for_glob/source_repo/pkg/doc2.in | 1 + .../test_for_glob/source_repo/pkg/file1.in | 1 + .../test_for_glob/source_repo/pkg/file2.in | 1 + .../test_for_glob/source_repo/pkg/unneeded.in | 1 + 20 files changed, 154 insertions(+), 49 deletions(-) create mode 100644 test/export/test_for_glob/BUILD create mode 100644 test/export/test_for_glob/expected_repo/.plzconfig create mode 100644 test/export/test_for_glob/expected_repo/pkg/BUILD_FILE create mode 100644 test/export/test_for_glob/expected_repo/pkg/doc1.in create mode 100644 test/export/test_for_glob/expected_repo/pkg/doc2.in create mode 100644 test/export/test_for_glob/expected_repo/pkg/file1.in create mode 100644 test/export/test_for_glob/expected_repo/pkg/file2.in create mode 100644 test/export/test_for_glob/source_repo/.plzconfig create mode 100644 test/export/test_for_glob/source_repo/pkg/BUILD_FILE create mode 100644 test/export/test_for_glob/source_repo/pkg/doc1.in create mode 100644 test/export/test_for_glob/source_repo/pkg/doc2.in create mode 100644 test/export/test_for_glob/source_repo/pkg/file1.in create mode 100644 test/export/test_for_glob/source_repo/pkg/file2.in create mode 100644 test/export/test_for_glob/source_repo/pkg/unneeded.in diff --git a/src/core/package_metadata.go b/src/core/package_metadata.go index 6d0f677965..d6e119cff4 100644 --- a/src/core/package_metadata.go +++ b/src/core/package_metadata.go @@ -52,8 +52,9 @@ type PackageMetadata interface { // RegisterStatement records a statement of an interpreted BUILD file and its // dependencies. Dependencies identify the subincluded targets required for a successful // interpretation of the statement, i.e. targets that provide the required symbols (variables or - // methods). - RegisterStatement(stmt BuildStatement, deps BuildLabels) + // methods). The files argument identify the files required when interpreting that statement, for + // example using glob(), + RegisterStatement(stmt BuildStatement, deps BuildLabels, files []string) // RegisterStatementTarget records that the given build target was created as a result of the // given statement being executed. This should only be called for statements in BUILD files. RegisterStatementTarget(target *BuildTarget, stmtProvider BuildStatementProvider) @@ -74,10 +75,10 @@ type PackageMetadata interface { // target relationship is determined by looking for targets generated by the same build statement. // The result excludes the target in the argument. FindRelatedTargets(target *BuildTarget) (BuildLabels, error) - // FindPackageRequiredSubincludes finds all the subincluded labels required by the package that + // FindPackageFileRequirements finds all the subincluded labels required by the package that // are not associated with BuildTarget generation. An example could be a variable declaration // that depends on a subincluded value. - FindPackageRequiredSubincludes() BuildLabels + FindPackageFileRequirements() (BuildLabels, []string) // GetSubincludedLabels returns all build labels that were included by the given subinclude statement. // Returns the labels or an empty slice if the statement wasn't found. GetSubincludedLabels(stmt BuildStatement) BuildLabels @@ -100,6 +101,9 @@ type packageMetadataImpl struct { // allows mapping a statement back to the subincluded labels required for building the target. // One direct, package level, subincludes are included. stmtToRequiredSubincludes *cmap.Map[BuildStatement, BuildLabels] + // stmtToRequiredFiles tracks the file paths that were required during interpretation of the + // statement (e.g. glob) + stmtToRequiredFiles *cmap.Map[BuildStatement, []string] // labelsPerSubincludeStmt maps a subinclude statement (identified by its position // in the BUILD file) to the labels it explicitly subincludes. labelsPerSubincludeStmt *cmap.Map[BuildStatement, BuildLabels] @@ -110,18 +114,19 @@ func newPackageMetadata() PackageMetadata { stmtToTarget: cmap.New[BuildStatement, BuildTargets](cmap.SmallShardCount, hashBuildStatement), targetToStmt: cmap.New[*BuildTarget, BuildStatement](cmap.SmallShardCount, hashBuildTarget), stmtToRequiredSubincludes: cmap.New[BuildStatement, BuildLabels](cmap.SmallShardCount, hashBuildStatement), + stmtToRequiredFiles: cmap.New[BuildStatement, []string](cmap.SmallShardCount, hashBuildStatement), labelsPerSubincludeStmt: cmap.New[BuildStatement, BuildLabels](cmap.SmallShardCount, hashBuildStatement), } } // RegisterStatement implements [PackageMetadata]. -func (m *packageMetadataImpl) RegisterStatement(stmt BuildStatement, deps BuildLabels) { - // fmt.Printf("Registering statement %v deps: %v\n", stmt, deps) - if len(deps) == 0 { - // Skip adding to the map if the statement doesn't require any subincludes. - return +func (m *packageMetadataImpl) RegisterStatement(stmt BuildStatement, deps BuildLabels, files []string) { + if len(deps) > 0 { + m.stmtToRequiredSubincludes.Set(stmt, deps) + } + if len(files) > 0 { + m.stmtToRequiredFiles.Set(stmt, files) } - m.stmtToRequiredSubincludes.Set(stmt, deps) } // RegisterStatementTarget implements [PackageMetadata]. @@ -194,19 +199,29 @@ func (m *packageMetadataImpl) FindRelatedTargets(target *BuildTarget) (BuildLabe return labels, nil } -// FindPackageRequiredSubincludes implements [PackageMetadata]. -func (m *packageMetadataImpl) FindPackageRequiredSubincludes() BuildLabels { - collector := LabelSet{} +// FindPackageFileRequirements implements [PackageMetadata]. +func (m *packageMetadataImpl) FindPackageFileRequirements() (BuildLabels, []string) { + subincludesSet := LabelSet{} m.stmtToRequiredSubincludes.Range(func(stmt BuildStatement, labels BuildLabels) { // Look for build statements that are not registered in the statement to targets mapping, // and so are unrelated to targets. if !m.stmtToTarget.Contains(stmt) { for _, label := range labels { - collector.Add(label) + subincludesSet.Add(label) } } }) - return slices.Collect(maps.Keys(collector)) + subincludes := slices.Collect(maps.Keys(subincludesSet)) + + filesSet := map[string]struct{}{} + m.stmtToRequiredFiles.Range(func(stmt BuildStatement, files []string) { + // Copy all files. Statements that generate targets are linked to files directly via dependencies. + for _, file := range files { + filesSet[file] = struct{}{} + } + }) + files := slices.Collect(maps.Keys(filesSet)) + return subincludes, files } // GetSubincludedLabels implements [PackageMetadata]. @@ -224,7 +239,8 @@ func newNoopPackageMetadata() PackageMetadata { } // RegisterStatement implements [PackageMetadata]. -func (n *noopPackageMetadata) RegisterStatement(stmt BuildStatement, deps BuildLabels) {} +func (n *noopPackageMetadata) RegisterStatement(stmt BuildStatement, deps BuildLabels, files []string) { +} // RegisterStatementTarget implements [PackageMetadata]. func (n *noopPackageMetadata) RegisterStatementTarget(target *BuildTarget, stmtProvider BuildStatementProvider) { @@ -262,10 +278,10 @@ func (n *noopPackageMetadata) FindRelatedTargets(target *BuildTarget) (BuildLabe return nil, nil } -// FindPackageRequiredSubincludes implements [PackageMetadata]. -func (n *noopPackageMetadata) FindPackageRequiredSubincludes() BuildLabels { +// FindPackageFileRequirements implements [PackageMetadata]. +func (n *noopPackageMetadata) FindPackageFileRequirements() (BuildLabels, []string) { log.Fatalf("metadata not tracked, using no-op implementation") - return nil + return nil, nil } // GetSubincludedLabels implements [PackageMetadata]. diff --git a/src/export/export.go b/src/export/export.go index c548939979..749db0231a 100644 --- a/src/export/export.go +++ b/src/export/export.go @@ -164,20 +164,27 @@ func (be *baseExporter) exportSources(target *core.BuildTarget) { if _, ok := src.Label(); ok { continue // These will be handled as dependencies later } - for _, p := range src.Paths(be.state.Graph) { - if filepath.IsAbs(p) { // Don't copy system file deps. - log.Debugf("System dependency detected, skipping...: %s", p) - continue + paths := src.Paths(be.state.Graph) + if target.Subrepo != nil { // Adjusting fo for local subrepos + for i, p := range paths { + paths[i] = filepath.Join(be.targetDir, target.Subrepo.Dir(p)) } - dest := filepath.Join(be.targetDir, p) - if target.Subrepo != nil { // Adjusting fo for local subrepos - dest = filepath.Join(be.targetDir, target.Subrepo.Dir(p)) - } - if err := fs.RecursiveCopy(p, dest, 0); err != nil { - log.Warningf("Error copying file, skipping...: %s", err) - } - log.Debugf("Writing exported source file: %s", p) } + be.exportFiles(paths) + } +} + +func (be *baseExporter) exportFiles(paths []string) { + for _, p := range paths { + if filepath.IsAbs(p) { // Don't copy system file deps. + log.Debugf("System dependency detected, skipping...: %s", p) + continue + } + dest := filepath.Join(be.targetDir, p) + if err := fs.RecursiveCopy(p, dest, 0); err != nil { + log.Warningf("Error copying file, skipping...: %s", err) + } + log.Debugf("Writing exported source file: %s", p) } } diff --git a/src/export/trimmed_exporter.go b/src/export/trimmed_exporter.go index 026d00636c..1f98e1f0cd 100644 --- a/src/export/trimmed_exporter.go +++ b/src/export/trimmed_exporter.go @@ -85,7 +85,7 @@ func (e *trimmedExporter) exportTarget(target *core.BuildTarget) { if !e.visitedPackages[pkg.Label()] { // Export subincluded targets required for other package statements, e.g. variable // declaration, during the first visit of a package. - e.exportPackageSubincludes(pkg) + e.exportPackageRequirements(pkg) e.visitedPackages[pkg.Label()] = true } } @@ -131,12 +131,13 @@ func (e *trimmedExporter) exportSubincludes(pkg *core.Package, target *core.Buil e.exportTargets(allSubincludes) } -// exportPackageSubincludes exports the subincluded targets that are required by package but are not -// linked to any [core.BuildTarget]. -func (e *trimmedExporter) exportPackageSubincludes(pkg *core.Package) { - subincludes := pkg.Metadata.FindPackageRequiredSubincludes() +// exportPackageRequirements exports any extra package requirements, for example the subincluded +// targets and files that are required by package but are not linked to any [core.BuildTarget]. +func (e *trimmedExporter) exportPackageRequirements(pkg *core.Package) { + subincludes, files := pkg.Metadata.FindPackageFileRequirements() e.setPackageSubincludes(pkg, subincludes) e.exportTargets(subincludes) + e.exportFiles(files) } // setPackageSubincludes marks the package-level required subincludes after the export. This will be diff --git a/src/parse/asp/builtins.go b/src/parse/asp/builtins.go index b592b27d96..96f6ba3d34 100644 --- a/src/parse/asp/builtins.go +++ b/src/parse/asp/builtins.go @@ -732,6 +732,7 @@ func glob(s *scope, args []pyObject) pyObject { exclude = exclude[:len(exclude)-len(s.state.Config.Parse.BuildFileName)] log.Fatalf("glob(include=%s, exclude=%s) in %s returned no files. If this is intended, set allow_empty=True on the glob.", include, exclude, s.pkg.Filename) } + s.metadata.pushFiles(s.pkg.Name, glob) return fromStringList(glob) } diff --git a/src/parse/asp/interpreter.go b/src/parse/asp/interpreter.go index 316f03a596..40f4cee153 100644 --- a/src/parse/asp/interpreter.go +++ b/src/parse/asp/interpreter.go @@ -5,6 +5,7 @@ import ( "fmt" "iter" "maps" + "path" "path/filepath" "reflect" "regexp" @@ -618,7 +619,7 @@ func (s *scope) interpretStatements(statements []*Statement) pyObject { s.Error("Unknown statement") // Shouldn't happen, amirite? } s.metadata.registerBuildStatement(s.pkg) - s.metadata.resetSymbolStack() + s.metadata.resetStacks() } return nil } @@ -1196,10 +1197,12 @@ type scopeMetadata interface { registerBuildStatement(pkg *core.Package) // setSymbolOrigin registers the subinclude origin label for a defined symbol. setSymbolOrigin(name string, origin core.BuildLabel) - // resetSymbolStack cleans the symbol stack into an empty state. - resetSymbolStack() + // resetStacks cleans the symbol stack into an empty state. + resetStacks() // pushSymbol pushes a symbol name and its subinclude origin onto the active tracking stack. pushSymbol(name string, origin *core.BuildLabel) + // pushFiles pushes a slice of filenames onto the active tracking stack. + pushFiles(rootPath string, filenames []string) } // trackingScopeMetadata implements the interface [scopeMetadata]. @@ -1211,7 +1214,9 @@ type trackingScopeMetadata struct { // symbolStack tracks which symbols are actively in use during evaluation. // Symbols are pushed onto the stack during lookups and popped/truncated after each statement. symbolStack []trackedSymbol - callDepth int + // fileStack track which files are required during the evaluation of the current statement. + // These are pushed during native calls such as glob() and popped/truncated after each top level statement. + fileStack []string } type trackedSymbol struct { @@ -1257,15 +1262,15 @@ func (m *trackingScopeMetadata) registerBuildStatement(pkg *core.Package) { for _, v := range m.symbolStack { set.Add(v.origin) } - // fmt.Printf("Symbol stack: %v\nSymbol Origins: %v\n", m.symbolStack, m.symbolOrigins) deps := slices.Collect(maps.Keys(set)) - pkg.Metadata.RegisterStatement(NewBuildStatement(m.stmtCursor), deps) + pkg.Metadata.RegisterStatement(NewBuildStatement(m.stmtCursor), deps, m.fileStack) } -// resetSymbolStack implements [scopeMetadata]. -func (m *trackingScopeMetadata) resetSymbolStack() { +// resetStacks implements [scopeMetadata]. +func (m *trackingScopeMetadata) resetStacks() { m.symbolStack = m.symbolStack[:0] + m.fileStack = m.fileStack[:0] } // setCursor implements [scopeMetadata]. @@ -1291,6 +1296,13 @@ func (m *trackingScopeMetadata) pushSymbol(name string, origin *core.BuildLabel) m.symbolStack = append(m.symbolStack, trackedSymbol{name: name, origin: *origin}) } +// pushFiles implements [scopeMetadata]. +func (m *trackingScopeMetadata) pushFiles(rootPath string, filenames []string) { + for _, filename := range filenames { + m.fileStack = append(m.fileStack, path.Join(rootPath, filename)) + } +} + // noopScopeMetadata implements the scopeMetadata interface with no-op methods. This is used to // avoid the overhead of storing metadata for operations that don't depend on it. type noopScopeMetadata struct{} @@ -1304,8 +1316,8 @@ func (nm *noopScopeMetadata) origin(scope *scope, name string) *core.BuildLabel // registerBuildStatement implements [scopeMetadata]. func (nm *noopScopeMetadata) registerBuildStatement(pkg *core.Package) {} -// resetSymbolStack implements [scopeMetadata]. -func (nm *noopScopeMetadata) resetSymbolStack() {} +// resetStacks implements [scopeMetadata]. +func (nm *noopScopeMetadata) resetStacks() {} // setCursor implements [scopeMetadata]. func (nm *noopScopeMetadata) setCursor(stmt *Statement) {} @@ -1316,6 +1328,9 @@ func (nm *noopScopeMetadata) setSymbolOrigin(name string, origin core.BuildLabel // pushSymbol implements [scopeMetadata]. func (nm *noopScopeMetadata) pushSymbol(name string, origin *core.BuildLabel) {} +// pushFiles implements [scopeMetadata]. +func (nm *noopScopeMetadata) pushFiles(rootPath string, filenames []string) {} + // NewBuildStatement creates a new core.BuildStatement from an asp.statement. func NewBuildStatement(stmt *Statement) core.BuildStatement { return core.BuildStatement{ diff --git a/src/parse/asp/interpreter_test.go b/src/parse/asp/interpreter_test.go index a8a644cf63..63e3bf50d2 100644 --- a/src/parse/asp/interpreter_test.go +++ b/src/parse/asp/interpreter_test.go @@ -809,7 +809,7 @@ func TestActiveSubincludes(t *testing.T) { scopeFuncExec.metadata.setCursor(stmt) scopeFuncExec.metadata.registerBuildStatement(pkg) - labels := pkg.Metadata.FindPackageRequiredSubincludes() + labels, _ := pkg.Metadata.FindPackageFileRequirements() assert.Empty(t, labels) }) @@ -843,7 +843,7 @@ func TestActiveSubincludes(t *testing.T) { scopeFuncExec.metadata.setCursor(stmt) scopeFuncExec.metadata.registerBuildStatement(pkg) - labels := pkg.Metadata.FindPackageRequiredSubincludes() + labels, _ := pkg.Metadata.FindPackageFileRequirements() assert.Equal(t, core.BuildLabels{labelA}, labels) }) @@ -887,7 +887,7 @@ func TestActiveSubincludes(t *testing.T) { scopeFuncExec.metadata.setCursor(stmt) scopeFuncExec.metadata.registerBuildStatement(pkg) - labels := pkg.Metadata.FindPackageRequiredSubincludes() + labels, _ := pkg.Metadata.FindPackageFileRequirements() assert.ElementsMatch(t, core.BuildLabels{labelA, labelB}, labels) }) } diff --git a/test/export/test_for_glob/BUILD b/test/export/test_for_glob/BUILD new file mode 100644 index 0000000000..3733c75062 --- /dev/null +++ b/test/export/test_for_glob/BUILD @@ -0,0 +1,6 @@ +subinclude("//test/export:export_e2e_test_build_def") + +please_export_e2e_test( + name = "export_for_glob", + export_targets = ["//pkg:all_files"], +) diff --git a/test/export/test_for_glob/expected_repo/.plzconfig b/test/export/test_for_glob/expected_repo/.plzconfig new file mode 100644 index 0000000000..f8ba31854d --- /dev/null +++ b/test/export/test_for_glob/expected_repo/.plzconfig @@ -0,0 +1,2 @@ +[Parse] +BuildFileName = BUILD_FILE diff --git a/test/export/test_for_glob/expected_repo/pkg/BUILD_FILE b/test/export/test_for_glob/expected_repo/pkg/BUILD_FILE new file mode 100644 index 0000000000..f4965461be --- /dev/null +++ b/test/export/test_for_glob/expected_repo/pkg/BUILD_FILE @@ -0,0 +1,17 @@ +for file in glob(["file*.in"]): + genrule( + name = "target_" + file.removesuffix(".in"), + srcs = [file], + outs = [file.removesuffix(".in") + ".out"], + cmd = "cp $SRCS $OUT", + ) + +filegroup( + name = "all_files", + srcs = [ + ":target_file1", + ], +) + +for file in glob(["doc*.in"]): + pass # Trimmed during export diff --git a/test/export/test_for_glob/expected_repo/pkg/doc1.in b/test/export/test_for_glob/expected_repo/pkg/doc1.in new file mode 100644 index 0000000000..516682f61d --- /dev/null +++ b/test/export/test_for_glob/expected_repo/pkg/doc1.in @@ -0,0 +1 @@ +doc1 diff --git a/test/export/test_for_glob/expected_repo/pkg/doc2.in b/test/export/test_for_glob/expected_repo/pkg/doc2.in new file mode 100644 index 0000000000..67f7bbeeda --- /dev/null +++ b/test/export/test_for_glob/expected_repo/pkg/doc2.in @@ -0,0 +1 @@ +doc2 diff --git a/test/export/test_for_glob/expected_repo/pkg/file1.in b/test/export/test_for_glob/expected_repo/pkg/file1.in new file mode 100644 index 0000000000..d9039017ab --- /dev/null +++ b/test/export/test_for_glob/expected_repo/pkg/file1.in @@ -0,0 +1 @@ +file1 content diff --git a/test/export/test_for_glob/expected_repo/pkg/file2.in b/test/export/test_for_glob/expected_repo/pkg/file2.in new file mode 100644 index 0000000000..f3c77b12c6 --- /dev/null +++ b/test/export/test_for_glob/expected_repo/pkg/file2.in @@ -0,0 +1 @@ +file2 content diff --git a/test/export/test_for_glob/source_repo/.plzconfig b/test/export/test_for_glob/source_repo/.plzconfig new file mode 100644 index 0000000000..f8ba31854d --- /dev/null +++ b/test/export/test_for_glob/source_repo/.plzconfig @@ -0,0 +1,2 @@ +[Parse] +BuildFileName = BUILD_FILE diff --git a/test/export/test_for_glob/source_repo/pkg/BUILD_FILE b/test/export/test_for_glob/source_repo/pkg/BUILD_FILE new file mode 100644 index 0000000000..07e3702e6a --- /dev/null +++ b/test/export/test_for_glob/source_repo/pkg/BUILD_FILE @@ -0,0 +1,29 @@ +for file in glob(["file*.in"]): + genrule( + name = "target_" + file.removesuffix(".in"), + srcs = [file], + outs = [file.removesuffix(".in") + ".out"], + cmd = "cp $SRCS $OUT", + ) + +genrule( + name = "unneeded", + srcs = glob(["unneeded*.in"]), + outs = ["unneeded.out"], + cmd = "cp $SRCS $OUT", +) + +filegroup( + name = "all_files", + srcs = [ + ":target_file1", + ], +) + +for file in glob(["doc*.in"]): + genrule( + name = "target_" + file.removesuffix(".in"), + srcs = [file], + outs = [file.removesuffix(".in") + ".out"], + cmd = "cp $SRCS $OUT", + ) diff --git a/test/export/test_for_glob/source_repo/pkg/doc1.in b/test/export/test_for_glob/source_repo/pkg/doc1.in new file mode 100644 index 0000000000..516682f61d --- /dev/null +++ b/test/export/test_for_glob/source_repo/pkg/doc1.in @@ -0,0 +1 @@ +doc1 diff --git a/test/export/test_for_glob/source_repo/pkg/doc2.in b/test/export/test_for_glob/source_repo/pkg/doc2.in new file mode 100644 index 0000000000..67f7bbeeda --- /dev/null +++ b/test/export/test_for_glob/source_repo/pkg/doc2.in @@ -0,0 +1 @@ +doc2 diff --git a/test/export/test_for_glob/source_repo/pkg/file1.in b/test/export/test_for_glob/source_repo/pkg/file1.in new file mode 100644 index 0000000000..d9039017ab --- /dev/null +++ b/test/export/test_for_glob/source_repo/pkg/file1.in @@ -0,0 +1 @@ +file1 content diff --git a/test/export/test_for_glob/source_repo/pkg/file2.in b/test/export/test_for_glob/source_repo/pkg/file2.in new file mode 100644 index 0000000000..f3c77b12c6 --- /dev/null +++ b/test/export/test_for_glob/source_repo/pkg/file2.in @@ -0,0 +1 @@ +file2 content diff --git a/test/export/test_for_glob/source_repo/pkg/unneeded.in b/test/export/test_for_glob/source_repo/pkg/unneeded.in new file mode 100644 index 0000000000..a95d9394a0 --- /dev/null +++ b/test/export/test_for_glob/source_repo/pkg/unneeded.in @@ -0,0 +1 @@ +unneeded content From 22041249812f93dde9172f953e53977b599938c5 Mon Sep 17 00:00:00 2001 From: DuBento Date: Fri, 26 Jun 2026 14:59:42 +0100 Subject: [PATCH 107/118] Checkpoint and restore symbol and file stack to support different calls to interpretStatements with the package level scope (e.g. for loop) --- src/parse/asp/interpreter.go | 49 +++++++++++++------ src/parse/asp/interpreter_test.go | 6 +-- .../expected_repo/BUILD_FILE | 5 ++ .../expected_repo/build_defs/BUILD_FILE | 6 +++ .../build_defs/versions.build_defs | 4 ++ .../source_repo/BUILD_FILE | 11 +++++ .../source_repo/build_defs/BUILD_FILE | 6 +++ .../build_defs/versions.build_defs | 4 ++ 8 files changed, 73 insertions(+), 18 deletions(-) create mode 100644 test/export/test_subinclude_trimming/expected_repo/build_defs/versions.build_defs create mode 100644 test/export/test_subinclude_trimming/source_repo/build_defs/versions.build_defs diff --git a/src/parse/asp/interpreter.go b/src/parse/asp/interpreter.go index 40f4cee153..e5b4d796c7 100644 --- a/src/parse/asp/interpreter.go +++ b/src/parse/asp/interpreter.go @@ -575,6 +575,8 @@ func (s *scope) interpretStatements(statements []*Statement) pyObject { }() for _, stmt = range statements { s.metadata.setCursor(stmt) + symbolStackCheckpoint, fileStackCheckpoint := s.metadata.checkpoint() + if stmt.FuncDef != nil { s.Set(stmt.FuncDef.Name, newPyFunc(s, stmt.FuncDef)) } else if stmt.If != nil { @@ -618,8 +620,9 @@ func (s *scope) interpretStatements(statements []*Statement) pyObject { } else { s.Error("Unknown statement") // Shouldn't happen, amirite? } - s.metadata.registerBuildStatement(s.pkg) - s.metadata.resetStacks() + + s.metadata.registerBuildStatement(s.pkg, stmt) + s.metadata.restore(symbolStackCheckpoint, fileStackCheckpoint) } return nil } @@ -1194,11 +1197,13 @@ type scopeMetadata interface { // registerBuildStatement registers a new Build Statement in the given package. It will also // register the required dependencies for interpreting that statement by looking up the required // origins in the symbol stack. - registerBuildStatement(pkg *core.Package) + registerBuildStatement(pkg *core.Package, stmt *Statement) // setSymbolOrigin registers the subinclude origin label for a defined symbol. setSymbolOrigin(name string, origin core.BuildLabel) - // resetStacks cleans the symbol stack into an empty state. - resetStacks() + // checkpoint returns the current lengths of the symbol and file stacks. + checkpoint() (int, int) + // restore truncates the symbol and file stacks to the specified lengths. + restore(symbolLen, fileLen int) // pushSymbol pushes a symbol name and its subinclude origin onto the active tracking stack. pushSymbol(name string, origin *core.BuildLabel) // pushFiles pushes a slice of filenames onto the active tracking stack. @@ -1253,8 +1258,8 @@ func (m *trackingScopeMetadata) origin(scope *scope, name string) *core.BuildLab } // registerBuildStatement implements [scopeMetadata]. -func (m *trackingScopeMetadata) registerBuildStatement(pkg *core.Package) { - if pkg == nil || m.stmtCursor == nil { +func (m *trackingScopeMetadata) registerBuildStatement(pkg *core.Package, stmt *Statement) { + if pkg == nil || stmt == nil { return } @@ -1264,13 +1269,22 @@ func (m *trackingScopeMetadata) registerBuildStatement(pkg *core.Package) { } deps := slices.Collect(maps.Keys(set)) - pkg.Metadata.RegisterStatement(NewBuildStatement(m.stmtCursor), deps, m.fileStack) + pkg.Metadata.RegisterStatement(NewBuildStatement(stmt), deps, m.fileStack) +} + +// checkpoint implements [scopeMetadata]. +func (m *trackingScopeMetadata) checkpoint() (int, int) { + return len(m.symbolStack), len(m.fileStack) } -// resetStacks implements [scopeMetadata]. -func (m *trackingScopeMetadata) resetStacks() { - m.symbolStack = m.symbolStack[:0] - m.fileStack = m.fileStack[:0] +// restore implements [scopeMetadata]. +func (m *trackingScopeMetadata) restore(symbolLen, fileLen int) { + if symbolLen >= 0 && symbolLen <= len(m.symbolStack) { + m.symbolStack = m.symbolStack[:symbolLen] + } + if fileLen >= 0 && fileLen <= len(m.fileStack) { + m.fileStack = m.fileStack[:fileLen] + } } // setCursor implements [scopeMetadata]. @@ -1314,10 +1328,15 @@ func (nm *noopScopeMetadata) cursor() *Statement { return nil } func (nm *noopScopeMetadata) origin(scope *scope, name string) *core.BuildLabel { return nil } // registerBuildStatement implements [scopeMetadata]. -func (nm *noopScopeMetadata) registerBuildStatement(pkg *core.Package) {} +func (nm *noopScopeMetadata) registerBuildStatement(pkg *core.Package, stmt *Statement) {} + +// checkpoint implements [scopeMetadata]. +func (nm *noopScopeMetadata) checkpoint() (int, int) { + return 0, 0 +} -// resetStacks implements [scopeMetadata]. -func (nm *noopScopeMetadata) resetStacks() {} +// restore implements [scopeMetadata]. +func (nm *noopScopeMetadata) restore(symbolLen, fileLen int) {} // setCursor implements [scopeMetadata]. func (nm *noopScopeMetadata) setCursor(stmt *Statement) {} diff --git a/src/parse/asp/interpreter_test.go b/src/parse/asp/interpreter_test.go index 63e3bf50d2..a0fdace269 100644 --- a/src/parse/asp/interpreter_test.go +++ b/src/parse/asp/interpreter_test.go @@ -807,7 +807,7 @@ func TestActiveSubincludes(t *testing.T) { metadata: meta, } scopeFuncExec.metadata.setCursor(stmt) - scopeFuncExec.metadata.registerBuildStatement(pkg) + scopeFuncExec.metadata.registerBuildStatement(pkg, stmt) labels, _ := pkg.Metadata.FindPackageFileRequirements() assert.Empty(t, labels) @@ -841,7 +841,7 @@ func TestActiveSubincludes(t *testing.T) { scopeFuncExec.Lookup("foo") scopeFuncExec.metadata.setCursor(stmt) - scopeFuncExec.metadata.registerBuildStatement(pkg) + scopeFuncExec.metadata.registerBuildStatement(pkg, stmt) labels, _ := pkg.Metadata.FindPackageFileRequirements() assert.Equal(t, core.BuildLabels{labelA}, labels) @@ -885,7 +885,7 @@ func TestActiveSubincludes(t *testing.T) { scopeFuncExec.Lookup("varB") scopeFuncExec.metadata.setCursor(stmt) - scopeFuncExec.metadata.registerBuildStatement(pkg) + scopeFuncExec.metadata.registerBuildStatement(pkg, stmt) labels, _ := pkg.Metadata.FindPackageFileRequirements() assert.ElementsMatch(t, core.BuildLabels{labelA, labelB}, labels) diff --git a/test/export/test_subinclude_trimming/expected_repo/BUILD_FILE b/test/export/test_subinclude_trimming/expected_repo/BUILD_FILE index b0cf935099..3144a15db1 100644 --- a/test/export/test_subinclude_trimming/expected_repo/BUILD_FILE +++ b/test/export/test_subinclude_trimming/expected_repo/BUILD_FILE @@ -25,3 +25,8 @@ message = "Testing {service} on {env}".format( env = "production", service = var3, ) + +subinclude("//build_defs:versions_build_def") + +for version, name in VERSIONS.items(): + pass # Trimmed during export diff --git a/test/export/test_subinclude_trimming/expected_repo/build_defs/BUILD_FILE b/test/export/test_subinclude_trimming/expected_repo/build_defs/BUILD_FILE index 7fe80f2ba4..55ab29141b 100644 --- a/test/export/test_subinclude_trimming/expected_repo/build_defs/BUILD_FILE +++ b/test/export/test_subinclude_trimming/expected_repo/build_defs/BUILD_FILE @@ -21,3 +21,9 @@ filegroup( srcs = ["var3.build_defs"], visibility = ["PUBLIC"], ) + +filegroup( + name = "versions_build_def", + srcs = ["versions.build_defs"], + visibility = ["PUBLIC"], +) diff --git a/test/export/test_subinclude_trimming/expected_repo/build_defs/versions.build_defs b/test/export/test_subinclude_trimming/expected_repo/build_defs/versions.build_defs new file mode 100644 index 0000000000..e565a23f8f --- /dev/null +++ b/test/export/test_subinclude_trimming/expected_repo/build_defs/versions.build_defs @@ -0,0 +1,4 @@ +VERSIONS = { + "1.0": "v1", + "1.1": "v1.1", +} diff --git a/test/export/test_subinclude_trimming/source_repo/BUILD_FILE b/test/export/test_subinclude_trimming/source_repo/BUILD_FILE index 22158a73ea..3de8e6482e 100644 --- a/test/export/test_subinclude_trimming/source_repo/BUILD_FILE +++ b/test/export/test_subinclude_trimming/source_repo/BUILD_FILE @@ -48,3 +48,14 @@ genrule( outs = ["format.out"], cmd = f'echo "{message}" > $OUT', ) + +subinclude( + "//build_defs:versions_build_def", +) + +for version, name in VERSIONS.items(): + genrule( + name = f"version-{version}", + outs = [f"version-{version}.out"], + cmd = f'echo "{name}" > $OUT', + ) diff --git a/test/export/test_subinclude_trimming/source_repo/build_defs/BUILD_FILE b/test/export/test_subinclude_trimming/source_repo/build_defs/BUILD_FILE index 7a460f35b5..b1fdef3b01 100644 --- a/test/export/test_subinclude_trimming/source_repo/build_defs/BUILD_FILE +++ b/test/export/test_subinclude_trimming/source_repo/build_defs/BUILD_FILE @@ -27,3 +27,9 @@ filegroup( srcs = ["unused.build_defs"], visibility = ["PUBLIC"], ) + +filegroup( + name = "versions_build_def", + srcs = ["versions.build_defs"], + visibility = ["PUBLIC"], +) diff --git a/test/export/test_subinclude_trimming/source_repo/build_defs/versions.build_defs b/test/export/test_subinclude_trimming/source_repo/build_defs/versions.build_defs new file mode 100644 index 0000000000..e565a23f8f --- /dev/null +++ b/test/export/test_subinclude_trimming/source_repo/build_defs/versions.build_defs @@ -0,0 +1,4 @@ +VERSIONS = { + "1.0": "v1", + "1.1": "v1.1", +} From e6c8feeb92d6594e1a8d0e2ac7cba3f31110fc7a Mon Sep 17 00:00:00 2001 From: DuBento Date: Fri, 26 Jun 2026 15:55:10 +0100 Subject: [PATCH 108/118] move to BuildLabels in export and metadata to avoid pressure on GC --- src/core/package_metadata.go | 56 +++++++++++++++------------------- src/export/export_test.go | 2 +- src/export/trimmed_exporter.go | 8 ++--- src/export/trimmer.go | 8 ++--- src/parse/asp/builtins.go | 2 +- 5 files changed, 34 insertions(+), 42 deletions(-) diff --git a/src/core/package_metadata.go b/src/core/package_metadata.go index d6e119cff4..a8f14e97e0 100644 --- a/src/core/package_metadata.go +++ b/src/core/package_metadata.go @@ -57,24 +57,24 @@ type PackageMetadata interface { RegisterStatement(stmt BuildStatement, deps BuildLabels, files []string) // RegisterStatementTarget records that the given build target was created as a result of the // given statement being executed. This should only be called for statements in BUILD files. - RegisterStatementTarget(target *BuildTarget, stmtProvider BuildStatementProvider) + RegisterStatementTarget(target BuildLabel, stmtProvider BuildStatementProvider) // RegisterSubincludeStatement records that the given subinclude statement (provided by stmtProvider) // includes the given build label. This should only be called for statements in BUILD files. RegisterSubincludeStatement(label BuildLabel, stmtProvider BuildStatementProvider) // FindStatement returns the build statement that was responsible for generating the given target. // Returns an error if the target was not found in the recorded metadata. - FindStatement(target *BuildTarget) (BuildStatement, error) + FindStatement(target BuildLabel) (BuildStatement, error) // FindTargets returns all build targets that were generated by the given build statement. // Returns an empty slice if no targets were found for the given statement. - FindTargets(stmt BuildStatement) BuildTargets + FindTargets(stmt BuildStatement) BuildLabels // FindRequiredSubincludes returns all subinclude labels that were required by the given target. // This method will only report the package level (direct) subincludes, make use of // [BuildGraph.TransitiveSubincludes] if you want all the required subincludes for one target. - FindRequiredSubincludes(target *BuildTarget) (BuildLabels, error) + FindRequiredSubincludes(target BuildLabel) (BuildLabels, error) // FindRelatedTargets finds all the targets that are related to the argument. In this context, // target relationship is determined by looking for targets generated by the same build statement. // The result excludes the target in the argument. - FindRelatedTargets(target *BuildTarget) (BuildLabels, error) + FindRelatedTargets(target BuildLabel) (BuildLabels, error) // FindPackageFileRequirements finds all the subincluded labels required by the package that // are not associated with BuildTarget generation. An example could be a variable declaration // that depends on a subincluded value. @@ -91,11 +91,11 @@ type packageMetadataImpl struct { // stmtToTarget maps each build statement (identified by its byte range in a BUILD file) // to the targets it produced. Since a single statement (like a custom target or loop) // can produce multiple targets, this is a one-to-many mapping. - stmtToTarget *cmap.Map[BuildStatement, BuildTargets] - // targetToStmt serves as a reverse-lookup map, linking each generated *BuildTarget + stmtToTarget *cmap.Map[BuildStatement, BuildLabels] + // targetToStmt serves as a reverse-lookup map, linking each generated BuildLabel // back to the specific BuildStatement that declared it. This is useful for tracing back a target // to its statement and to find sibling targets generated by the same statement block. - targetToStmt *cmap.Map[*BuildTarget, BuildStatement] + targetToStmt *cmap.Map[BuildLabel, BuildStatement] // stmtToRequiredSubincludes tracks the subinclude labels that were required for the current // interpretation of the build statement. This, in addition to the statement to target map, // allows mapping a statement back to the subincluded labels required for building the target. @@ -111,8 +111,8 @@ type packageMetadataImpl struct { func newPackageMetadata() PackageMetadata { return &packageMetadataImpl{ - stmtToTarget: cmap.New[BuildStatement, BuildTargets](cmap.SmallShardCount, hashBuildStatement), - targetToStmt: cmap.New[*BuildTarget, BuildStatement](cmap.SmallShardCount, hashBuildTarget), + stmtToTarget: cmap.New[BuildStatement, BuildLabels](cmap.SmallShardCount, hashBuildStatement), + targetToStmt: cmap.New[BuildLabel, BuildStatement](cmap.SmallShardCount, HashBuildLabel), stmtToRequiredSubincludes: cmap.New[BuildStatement, BuildLabels](cmap.SmallShardCount, hashBuildStatement), stmtToRequiredFiles: cmap.New[BuildStatement, []string](cmap.SmallShardCount, hashBuildStatement), labelsPerSubincludeStmt: cmap.New[BuildStatement, BuildLabels](cmap.SmallShardCount, hashBuildStatement), @@ -130,7 +130,7 @@ func (m *packageMetadataImpl) RegisterStatement(stmt BuildStatement, deps BuildL } // RegisterStatementTarget implements [PackageMetadata]. -func (m *packageMetadataImpl) RegisterStatementTarget(target *BuildTarget, stmtProvider BuildStatementProvider) { +func (m *packageMetadataImpl) RegisterStatementTarget(target BuildLabel, stmtProvider BuildStatementProvider) { stmt := stmtProvider() targets := m.stmtToTarget.Get(stmt) m.stmtToTarget.Set(stmt, append(targets, target)) @@ -145,11 +145,7 @@ func (m *packageMetadataImpl) RegisterSubincludeStatement(label BuildLabel, stmt } // FindStatement implements [PackageMetadata]. -func (m *packageMetadataImpl) FindStatement(target *BuildTarget) (BuildStatement, error) { - if target == nil { - return BuildStatement{}, fmt.Errorf("target is nil") - } - +func (m *packageMetadataImpl) FindStatement(target BuildLabel) (BuildStatement, error) { stmt := m.targetToStmt.Get(target) if stmt == (BuildStatement{}) { return BuildStatement{}, fmt.Errorf("failed to find statement for target %s", target) @@ -158,16 +154,12 @@ func (m *packageMetadataImpl) FindStatement(target *BuildTarget) (BuildStatement } // FindTargets implements [PackageMetadata]. -func (m *packageMetadataImpl) FindTargets(stmt BuildStatement) BuildTargets { +func (m *packageMetadataImpl) FindTargets(stmt BuildStatement) BuildLabels { return m.stmtToTarget.Get(stmt) } // FindRequiredSubincludes implements [PackageMetadata]. -func (m *packageMetadataImpl) FindRequiredSubincludes(target *BuildTarget) (BuildLabels, error) { - if target == nil { - return nil, nil - } - +func (m *packageMetadataImpl) FindRequiredSubincludes(target BuildLabel) (BuildLabels, error) { stmt, err := m.FindStatement(target) if err != nil { return nil, err @@ -183,7 +175,7 @@ func (m *packageMetadataImpl) FindRequiredSubincludes(target *BuildTarget) (Buil } // FindRelatedTargets implements [PackageMetadata]. -func (m *packageMetadataImpl) FindRelatedTargets(target *BuildTarget) (BuildLabels, error) { +func (m *packageMetadataImpl) FindRelatedTargets(target BuildLabel) (BuildLabels, error) { stmt, err := m.FindStatement(target) if err != nil { return nil, err @@ -191,9 +183,9 @@ func (m *packageMetadataImpl) FindRelatedTargets(target *BuildTarget) (BuildLabe // fmt.Printf("STMT to Target mapping %v\n", m.stmtToTarget.Values()) relatedTargets := m.FindTargets(stmt) labels := make(BuildLabels, 0, len(relatedTargets)) - for _, t := range relatedTargets { - if t.Label != target.Label { - labels = append(labels, t.Label) + for _, l := range relatedTargets { + if l != target { + labels = append(labels, l) } } return labels, nil @@ -243,11 +235,11 @@ func (n *noopPackageMetadata) RegisterStatement(stmt BuildStatement, deps BuildL } // RegisterStatementTarget implements [PackageMetadata]. -func (n *noopPackageMetadata) RegisterStatementTarget(target *BuildTarget, stmtProvider BuildStatementProvider) { +func (n *noopPackageMetadata) RegisterStatementTarget(target BuildLabel, stmtProvider BuildStatementProvider) { } // RegisterRequiredSubinclude implements [PackageMetadata]. -func (n *noopPackageMetadata) RegisterRequiredSubinclude(target *BuildTarget, labelProvider SubincludesLabelProvider) { +func (n *noopPackageMetadata) RegisterRequiredSubinclude(target BuildLabel, labelProvider SubincludesLabelProvider) { } // RegisterSubincludeStatement implements [PackageMetadata]. @@ -255,25 +247,25 @@ func (n *noopPackageMetadata) RegisterSubincludeStatement(label BuildLabel, stmt } // FindStatement implements [PackageMetadata]. -func (n *noopPackageMetadata) FindStatement(target *BuildTarget) (BuildStatement, error) { +func (n *noopPackageMetadata) FindStatement(target BuildLabel) (BuildStatement, error) { log.Fatalf("metadata not tracked, using no-op implementation") return BuildStatement{}, nil } // FindTargets implements [PackageMetadata]. -func (n *noopPackageMetadata) FindTargets(stmt BuildStatement) BuildTargets { +func (n *noopPackageMetadata) FindTargets(stmt BuildStatement) BuildLabels { log.Fatalf("metadata not tracked, using no-op implementation") return nil } // FindRequiredSubincludes implements [PackageMetadata]. -func (n *noopPackageMetadata) FindRequiredSubincludes(target *BuildTarget) (BuildLabels, error) { +func (n *noopPackageMetadata) FindRequiredSubincludes(target BuildLabel) (BuildLabels, error) { log.Fatalf("metadata not tracked, using no-op implementation") return nil, nil } // FindRelatedTargets implements [PackageMetadata]. -func (n *noopPackageMetadata) FindRelatedTargets(target *BuildTarget) (BuildLabels, error) { +func (n *noopPackageMetadata) FindRelatedTargets(target BuildLabel) (BuildLabels, error) { log.Fatalf("metadata not tracked, using no-op implementation") return nil, nil } diff --git a/src/export/export_test.go b/src/export/export_test.go index 3aa6488eba..b3e7a9e20c 100644 --- a/src/export/export_test.go +++ b/src/export/export_test.go @@ -218,7 +218,7 @@ func walkASTRegisterTargets(t *testing.T, stmts []*asp.Statement, pkg *core.Pack label := core.NewBuildLabel(pkg.Name, name) targetLabels[name] = label target := &core.BuildTarget{Label: label} - pkg.Metadata.RegisterStatementTarget(target, func() core.BuildStatement { + pkg.Metadata.RegisterStatementTarget(target.Label, func() core.BuildStatement { return asp.NewBuildStatement(stmt) }) return true diff --git a/src/export/trimmed_exporter.go b/src/export/trimmed_exporter.go index 1f98e1f0cd..f867eba5e2 100644 --- a/src/export/trimmed_exporter.go +++ b/src/export/trimmed_exporter.go @@ -79,8 +79,8 @@ func (e *trimmedExporter) exportTarget(target *core.BuildTarget) { log.Errorf("Unable to lookup package %s", target.Label) return } - e.exportSubincludes(pkg, target) - e.exportRelatedTargets(pkg, target) + e.exportSubincludes(pkg, target.Label) + e.exportRelatedTargets(pkg, target.Label) if !e.visitedPackages[pkg.Label()] { // Export subincluded targets required for other package statements, e.g. variable @@ -107,7 +107,7 @@ func (e *trimmedExporter) writePackageFiles() { // exportSubincludes exports the subincluded targets required to generate the target and selects them to // later be written to the package as statements. -func (e *trimmedExporter) exportSubincludes(pkg *core.Package, target *core.BuildTarget) { +func (e *trimmedExporter) exportSubincludes(pkg *core.Package, target core.BuildLabel) { // Get the actively used subincludes of the target and propagate all transitive subincludes required // by our used subinclude targets. FindRequiredSubincludes will report the required subincludes // for this target at the package level but we need to propagate the subincluded targets inside @@ -163,7 +163,7 @@ func (e *trimmedExporter) setPackageSubincludes(pkg *core.Package, subincludes c // build statement (e.g., adjacent targets in build def) as the specified target. This ensures that // all co-defined targets are preserved in the exported BUILD file, preventing unresolved references // or partial declarations. -func (e *trimmedExporter) exportRelatedTargets(pkg *core.Package, target *core.BuildTarget) { +func (e *trimmedExporter) exportRelatedTargets(pkg *core.Package, target core.BuildLabel) { relatedTargets, err := pkg.Metadata.FindRelatedTargets(target) if err != nil { log.Fatalf("failed to find related targets for %s: %s", target, err) diff --git a/src/export/trimmer.go b/src/export/trimmer.go index d13a2b7700..5d1ad78d7d 100644 --- a/src/export/trimmer.go +++ b/src/export/trimmer.go @@ -202,14 +202,14 @@ func (t *trimmer) isRequiredStatement(stmt *asp.Statement) bool { return t.anyExported(t.relatedTargets(stmt)) } -func (t *trimmer) relatedTargets(stmt *asp.Statement) []*core.BuildTarget { +func (t *trimmer) relatedTargets(stmt *asp.Statement) core.BuildLabels { bStmt := asp.NewBuildStatement(stmt) return t.pkg.Metadata.FindTargets(bStmt) } -func (t *trimmer) anyExported(targets []*core.BuildTarget) bool { - required := slices.ContainsFunc(targets, func(target *core.BuildTarget) bool { - return t.exporter.exportedTargets[target.Label] +func (t *trimmer) anyExported(labels core.BuildLabels) bool { + required := slices.ContainsFunc(labels, func(l core.BuildLabel) bool { + return t.exporter.exportedTargets[l] }) return required } diff --git a/src/parse/asp/builtins.go b/src/parse/asp/builtins.go index 96f6ba3d34..da6d630fc1 100644 --- a/src/parse/asp/builtins.go +++ b/src/parse/asp/builtins.go @@ -210,7 +210,7 @@ func buildRule(s *scope, args []pyObject) pyObject { s.Assert(s.pkg.Target(target.Label.Name) == nil, "Duplicate build target in %s: %s", s.pkg.Name, target.Label.Name) populateTarget(s, target, args) s.state.AddTarget(s.pkg, target) - s.pkg.Metadata.RegisterStatementTarget(target, s.CurrentBuildStatement()) + s.pkg.Metadata.RegisterStatementTarget(target.Label, s.CurrentBuildStatement()) if s.Callback { target.AddedPostBuild = true From e4aa82b72599906d1f73e56f239df82c72f3b14e Mon Sep 17 00:00:00 2001 From: DuBento Date: Fri, 26 Jun 2026 22:18:32 +0100 Subject: [PATCH 109/118] track interpreted statements (mark as 0 targets) to correctly identify if blocks that have interpreted stmts (var redeclaration) --- src/core/package_metadata.go | 14 ++++++++++++-- src/export/export.go | 2 +- src/export/trimmer.go | 29 +++++++++++++++-------------- 3 files changed, 28 insertions(+), 17 deletions(-) diff --git a/src/core/package_metadata.go b/src/core/package_metadata.go index a8f14e97e0..202d278568 100644 --- a/src/core/package_metadata.go +++ b/src/core/package_metadata.go @@ -82,6 +82,9 @@ type PackageMetadata interface { // GetSubincludedLabels returns all build labels that were included by the given subinclude statement. // Returns the labels or an empty slice if the statement wasn't found. GetSubincludedLabels(stmt BuildStatement) BuildLabels + // IsInterpretedStatement returns true if the statement provided matches a registered build + // statement, meaning it was interpreted even if it doesn't generate any targets. + IsInterpretedStatement(stmt BuildStatement) bool } // packageMetadataImpl is the canonical implementation of the PackageMetadata interface. @@ -127,6 +130,9 @@ func (m *packageMetadataImpl) RegisterStatement(stmt BuildStatement, deps BuildL if len(files) > 0 { m.stmtToRequiredFiles.Set(stmt, files) } + // Even if the statement doesn't create any target, it is important to register so we now it was + // interpreted. We'll register the targets separately and use the Add() method to avoid overriding any existing statement to target mapping. + m.stmtToTarget.Add(stmt, BuildLabels{}) } // RegisterStatementTarget implements [PackageMetadata]. @@ -180,7 +186,6 @@ func (m *packageMetadataImpl) FindRelatedTargets(target BuildLabel) (BuildLabels if err != nil { return nil, err } - // fmt.Printf("STMT to Target mapping %v\n", m.stmtToTarget.Values()) relatedTargets := m.FindTargets(stmt) labels := make(BuildLabels, 0, len(relatedTargets)) for _, l := range relatedTargets { @@ -197,7 +202,7 @@ func (m *packageMetadataImpl) FindPackageFileRequirements() (BuildLabels, []stri m.stmtToRequiredSubincludes.Range(func(stmt BuildStatement, labels BuildLabels) { // Look for build statements that are not registered in the statement to targets mapping, // and so are unrelated to targets. - if !m.stmtToTarget.Contains(stmt) { + if len(m.stmtToTarget.Get(stmt)) == 0 { for _, label := range labels { subincludesSet.Add(label) } @@ -221,6 +226,11 @@ func (m *packageMetadataImpl) GetSubincludedLabels(stmt BuildStatement) BuildLab return m.labelsPerSubincludeStmt.Get(stmt) } +// IsInterpretedStatement implements [PackageMetadata]. +func (m *packageMetadataImpl) IsInterpretedStatement(stmt BuildStatement) bool { + return m.stmtToTarget.Contains(stmt) +} + // noopPackageMetadata implements the PackageMetadata interface with no-op methods. This is the // default implementation and is used to avoid the overhead of parsing metadata for operations that // don't depend on it. diff --git a/src/export/export.go b/src/export/export.go index 749db0231a..eaa94794d9 100644 --- a/src/export/export.go +++ b/src/export/export.go @@ -167,7 +167,7 @@ func (be *baseExporter) exportSources(target *core.BuildTarget) { paths := src.Paths(be.state.Graph) if target.Subrepo != nil { // Adjusting fo for local subrepos for i, p := range paths { - paths[i] = filepath.Join(be.targetDir, target.Subrepo.Dir(p)) + paths[i] = target.Subrepo.Dir(p) } } be.exportFiles(paths) diff --git a/src/export/trimmer.go b/src/export/trimmer.go index 5d1ad78d7d..a55c0b00de 100644 --- a/src/export/trimmer.go +++ b/src/export/trimmer.go @@ -67,9 +67,9 @@ func (t *trimmer) trimBlock(stmts []*asp.Statement, blockStart, blockEnd asp.Pos } else if stmt.Ident != nil && stmt.Ident.Name == "subinclude" { t.trimSubinclude(stmt) written = true - } else if relatives := t.relatedTargets(stmt); len(relatives) > 0 { + } else if targets := t.statementTargets(stmt); len(targets) > 0 { // Meaning it is a build statement that creates build targets. - if t.anyExported(relatives) { + if t.anyExported(targets) { t.copy(stmt.Pos, stmt.EndPos) written = true } @@ -81,6 +81,9 @@ func (t *trimmer) trimBlock(stmts []*asp.Statement, blockStart, blockEnd asp.Pos written = true } }) + if !written { + t.write(passExpression) + } return written } @@ -106,19 +109,15 @@ func (t *trimmer) trimIf(stmt *asp.Statement) bool { } // In an if-else statement only the interpreted/evaluated block will generate targets, meaning - // that normally only one of the clauses is interpreted, however an if stmt could be inside of + // that normally only one of the clauses is interpreted, however an if statement could be inside of // a loop where the clause condition depends on the iteration meaning more than one clause // could end up being interpreted. Because of that we lookup all the required clauses before - // writing the statement. + // writing the statement. Any clauses with interpreted statements should be visited. var requiredClauses = make([]bool, len(clauses)) for i, c := range clauses { required := t.isRequiredStatements(c.stmts) requiredClauses[i] = required } - // No clause is required, skip the if-else stmt entirely - if !slices.Contains(requiredClauses, true) { - return false - } for i, c := range clauses { // Write clause header @@ -146,10 +145,7 @@ func (t *trimmer) trimFor(stmt *asp.Statement) bool { hStart, hEnd := stmt.Pos, stmt.For.Statements[0].Pos t.copy(hStart, hEnd) - written := t.trimBlock(stmt.For.Statements, hEnd, stmt.EndPos) - if !written { - t.write(passExpression) - } + t.trimBlock(stmt.For.Statements, hEnd, stmt.EndPos) return true } @@ -174,10 +170,15 @@ func (t *trimmer) passBlock(stmts []*asp.Statement, blockStart, blockEnd asp.Pos }) } +// isRequiredStatement determines if it is necessary to visit any of the statements. Refer to +// [isRequiredStatement] for the decision logic. func (t *trimmer) isRequiredStatements(stmts []*asp.Statement) bool { return slices.ContainsFunc(stmts, t.isRequiredStatement) } +// isRequiredStatement determines if it is necessary to visit a statement. Any statement that was +// interpreted needs to be visited but the trimming logic of the visitor should determine if we need +// to write it or not. func (t *trimmer) isRequiredStatement(stmt *asp.Statement) bool { if stmt.If != nil { // If @@ -199,10 +200,10 @@ func (t *trimmer) isRequiredStatement(stmt *asp.Statement) bool { } else if stmt.For != nil { return t.isRequiredStatements(stmt.For.Statements) } - return t.anyExported(t.relatedTargets(stmt)) + return t.pkg.Metadata.IsInterpretedStatement(asp.NewBuildStatement(stmt)) } -func (t *trimmer) relatedTargets(stmt *asp.Statement) core.BuildLabels { +func (t *trimmer) statementTargets(stmt *asp.Statement) core.BuildLabels { bStmt := asp.NewBuildStatement(stmt) return t.pkg.Metadata.FindTargets(bStmt) } From 8022ad5ab3485cfdf25bc07d9f8b59af09f70c53 Mon Sep 17 00:00:00 2001 From: DuBento Date: Mon, 29 Jun 2026 16:33:27 +0100 Subject: [PATCH 110/118] subinclude tracking for dynamic subincludes --- src/core/package_metadata.go | 54 +++++++++++-- src/parse/asp/builtins.go | 3 + src/parse/asp/interpreter.go | 78 ++++++++++++++----- test/export/test_dynamic_subinclude/BUILD | 11 +++ .../expected_repo/.plzconfig | 3 + .../expected_repo/BUILD_FILE | 7 ++ .../expected_repo/build_defs/BUILD_FILE | 17 ++++ .../expected_repo/build_defs/a.build_defs | 6 ++ .../expected_repo/build_defs/b.build_defs | 6 ++ .../build_defs/dynamic.build_defs | 6 ++ .../source_repo/.plzconfig | 3 + .../source_repo/BUILD_FILE | 7 ++ .../source_repo/build_defs/BUILD_FILE | 23 ++++++ .../source_repo/build_defs/a.build_defs | 6 ++ .../source_repo/build_defs/b.build_defs | 6 ++ .../source_repo/build_defs/dynamic.build_defs | 6 ++ .../build_defs/unneeded.build_defs | 2 + 17 files changed, 218 insertions(+), 26 deletions(-) create mode 100644 test/export/test_dynamic_subinclude/BUILD create mode 100644 test/export/test_dynamic_subinclude/expected_repo/.plzconfig create mode 100644 test/export/test_dynamic_subinclude/expected_repo/BUILD_FILE create mode 100644 test/export/test_dynamic_subinclude/expected_repo/build_defs/BUILD_FILE create mode 100644 test/export/test_dynamic_subinclude/expected_repo/build_defs/a.build_defs create mode 100644 test/export/test_dynamic_subinclude/expected_repo/build_defs/b.build_defs create mode 100644 test/export/test_dynamic_subinclude/expected_repo/build_defs/dynamic.build_defs create mode 100644 test/export/test_dynamic_subinclude/source_repo/.plzconfig create mode 100644 test/export/test_dynamic_subinclude/source_repo/BUILD_FILE create mode 100644 test/export/test_dynamic_subinclude/source_repo/build_defs/BUILD_FILE create mode 100644 test/export/test_dynamic_subinclude/source_repo/build_defs/a.build_defs create mode 100644 test/export/test_dynamic_subinclude/source_repo/build_defs/b.build_defs create mode 100644 test/export/test_dynamic_subinclude/source_repo/build_defs/dynamic.build_defs create mode 100644 test/export/test_dynamic_subinclude/source_repo/build_defs/unneeded.build_defs diff --git a/src/core/package_metadata.go b/src/core/package_metadata.go index 202d278568..ad6dfc3d11 100644 --- a/src/core/package_metadata.go +++ b/src/core/package_metadata.go @@ -85,11 +85,20 @@ type PackageMetadata interface { // IsInterpretedStatement returns true if the statement provided matches a registered build // statement, meaning it was interpreted even if it doesn't generate any targets. IsInterpretedStatement(stmt BuildStatement) bool + // WriteMetadata writes a visualization of the metadata (statements and their required labels/files) to the writer. + // Only statements that generated the specified targets will be displayed. + WriteMetadata(w io.Writer, fileContent []byte, targets BuildLabels, includeSources, includeDeps, includeOutputs bool, resolver func(BuildLabel) *BuildTarget) } -// packageMetadataImpl is the canonical implementation of the PackageMetadata interface. -// It uses sharded concurrent maps [cmap.Map] to track the relationships between BUILD file statements, -// subincludes, and the build targets they define without the contention of a global read-write lock. +// packageMetadataImpl is the canonical implementation of the PackageMetadata interface. It tracks the relationships between BUILD file statements, +// subincludes, and the build targets they define. +// +// Note: this implementation uses sharded concurrent maps [cmap.Map], however writes (interpreter +// phase) are performed on a single-thread per package. Please guarantees that each package's BUILD file +// and its subincludes are interpreted on exactly one parser thread at a time (enforced by +// SyncParsePackage). Because of this guarantee, non-atomic write methods (like the Get-then-Set +// sequence in RegisterStatement) are safe and wont suffer from write-write race conditions. +// Nevertheless we use concurrent maps to support concurrent reads. type packageMetadataImpl struct { // stmtToTarget maps each build statement (identified by its byte range in a BUILD file) // to the targets it produced. Since a single statement (like a custom target or loop) @@ -125,16 +134,41 @@ func newPackageMetadata() PackageMetadata { // RegisterStatement implements [PackageMetadata]. func (m *packageMetadataImpl) RegisterStatement(stmt BuildStatement, deps BuildLabels, files []string) { if len(deps) > 0 { - m.stmtToRequiredSubincludes.Set(stmt, deps) + existingDeps := m.stmtToRequiredSubincludes.Get(stmt) + if len(existingDeps) == 0 { + m.stmtToRequiredSubincludes.Set(stmt, deps) + } else { + mergedDeps := mergeSlices(existingDeps, deps) + m.stmtToRequiredSubincludes.Set(stmt, mergedDeps) + } } if len(files) > 0 { - m.stmtToRequiredFiles.Set(stmt, files) + existingFiles := m.stmtToRequiredFiles.Get(stmt) + if len(existingFiles) == 0 { + m.stmtToRequiredFiles.Set(stmt, files) + } else { + mergedFiles := mergeSlices(existingFiles, files) + m.stmtToRequiredFiles.Set(stmt, mergedFiles) + } } // Even if the statement doesn't create any target, it is important to register so we now it was // interpreted. We'll register the targets separately and use the Add() method to avoid overriding any existing statement to target mapping. m.stmtToTarget.Add(stmt, BuildLabels{}) } +// mergeSlices merges two slices of any comparable type. It de-duplicates elements using +// slices.Contains so it should be used for small slices. A set/map approach should be preferred for +// larger slices. +func mergeSlices[T comparable](existing []T, newElements []T) []T { + merged := append([]T(nil), existing...) + for _, el := range newElements { + if !slices.Contains(merged, el) { + merged = append(merged, el) + } + } + return merged +} + // RegisterStatementTarget implements [PackageMetadata]. func (m *packageMetadataImpl) RegisterStatementTarget(target BuildLabel, stmtProvider BuildStatementProvider) { stmt := stmtProvider() @@ -201,8 +235,8 @@ func (m *packageMetadataImpl) FindPackageFileRequirements() (BuildLabels, []stri subincludesSet := LabelSet{} m.stmtToRequiredSubincludes.Range(func(stmt BuildStatement, labels BuildLabels) { // Look for build statements that are not registered in the statement to targets mapping, - // and so are unrelated to targets. - if len(m.stmtToTarget.Get(stmt)) == 0 { + // and so are unrelated to targets or are subincludes() statements. + if len(m.stmtToTarget.Get(stmt)) == 0 && len(m.labelsPerSubincludeStmt.Get(stmt)) == 0 { for _, label := range labels { subincludesSet.Add(label) } @@ -291,3 +325,9 @@ func (n *noopPackageMetadata) GetSubincludedLabels(stmt BuildStatement) BuildLab log.Fatalf("metadata not tracked, using no-op implementation") return nil } + +// IsInterpretedStatement implements [PackageMetadata]. +func (n *noopPackageMetadata) IsInterpretedStatement(stmt BuildStatement) bool { + log.Fatalf("metadata not tracked, using no-op implementation") + return false +} diff --git a/src/parse/asp/builtins.go b/src/parse/asp/builtins.go index da6d630fc1..a7a4348c79 100644 --- a/src/parse/asp/builtins.go +++ b/src/parse/asp/builtins.go @@ -349,6 +349,7 @@ func subinclude(s *scope, args []pyObject) pyObject { s.Error("cannot subinclude type %s", arg.Type()) } } + var labels core.BuildLabels for _, arg := range si { label, annotation := core.SplitLabelAnnotation(arg) t := subincludeTarget(s, s.parseLabelInContextPkg(label)) @@ -370,7 +371,9 @@ func subinclude(s *scope, args []pyObject) pyObject { for _, out := range outs { s.SetAllWithOrigin(s.interpreter.Subinclude(s, filepath.Join(t.OutDir(), out), t.Label, false), false, &t.Label) } + labels = append(labels, t.Label) } + s.registerSubincludes(labels) return None } diff --git a/src/parse/asp/interpreter.go b/src/parse/asp/interpreter.go index e5b4d796c7..a6871dce49 100644 --- a/src/parse/asp/interpreter.go +++ b/src/parse/asp/interpreter.go @@ -575,7 +575,7 @@ func (s *scope) interpretStatements(statements []*Statement) pyObject { }() for _, stmt = range statements { s.metadata.setCursor(stmt) - symbolStackCheckpoint, fileStackCheckpoint := s.metadata.checkpoint() + checkpoint := s.metadata.checkpoint() if stmt.FuncDef != nil { s.Set(stmt.FuncDef.Name, newPyFunc(s, stmt.FuncDef)) @@ -622,7 +622,7 @@ func (s *scope) interpretStatements(statements []*Statement) pyObject { } s.metadata.registerBuildStatement(s.pkg, stmt) - s.metadata.restore(symbolStackCheckpoint, fileStackCheckpoint) + s.metadata.restore(checkpoint) } return nil } @@ -1178,6 +1178,17 @@ func (s *scope) getOrNewMetadata(pkg *core.Package) scopeMetadata { } }) return meta +} + +func (s *scope) registerSubincludes(labels core.BuildLabels) { + if s.pkg == nil || s.interpreter.packageMetadata == nil { // This will be initialized if ParseMetadata is set. + return + } + meta := s.interpreter.packageMetadata.Get(s.pkg.Label()) + if meta == nil { + return + } + meta.pushSubincludes(labels) } @@ -1200,14 +1211,16 @@ type scopeMetadata interface { registerBuildStatement(pkg *core.Package, stmt *Statement) // setSymbolOrigin registers the subinclude origin label for a defined symbol. setSymbolOrigin(name string, origin core.BuildLabel) - // checkpoint returns the current lengths of the symbol and file stacks. - checkpoint() (int, int) - // restore truncates the symbol and file stacks to the specified lengths. - restore(symbolLen, fileLen int) + // checkpoint returns a checkpoint for the current tracking stacks. + checkpoint() metadataStackCheckpoint + // restore truncates the tracking stacks to the specified lengths from the checkpoint. + restore(cp metadataStackCheckpoint) // pushSymbol pushes a symbol name and its subinclude origin onto the active tracking stack. pushSymbol(name string, origin *core.BuildLabel) // pushFiles pushes a slice of filenames onto the active tracking stack. pushFiles(rootPath string, filenames []string) + // pushSubincludes pushes a label onto the tracking stack for subincludes. + pushSubincludes(labels core.BuildLabels) } // trackingScopeMetadata implements the interface [scopeMetadata]. @@ -1217,11 +1230,16 @@ type trackingScopeMetadata struct { // symbolOrigins tracks the subinclude label that each symbol was originally defined in. symbolOrigins map[string]core.BuildLabel // symbolStack tracks which symbols are actively in use during evaluation. - // Symbols are pushed onto the stack during lookups and popped/truncated after each statement. + // Symbols are pushed onto the stack during lookups and truncated after each statement. symbolStack []trackedSymbol // fileStack track which files are required during the evaluation of the current statement. - // These are pushed during native calls such as glob() and popped/truncated after each top level statement. + // These are pushed during native calls such as glob() and truncated after each top level statement. fileStack []string + // subincludesStack track subinclude calls dynamically made while evaluating a statement, + // this can happen for example when a custom target (function definition) subincludes a target as + // part of the function body. These are pushed during calls to subincludes() and truncated after + // each statement. + subincludesStack core.BuildLabels } type trackedSymbol struct { @@ -1267,23 +1285,37 @@ func (m *trackingScopeMetadata) registerBuildStatement(pkg *core.Package, stmt * for _, v := range m.symbolStack { set.Add(v.origin) } - + for _, l := range m.subincludesStack { + set.Add(l) + } deps := slices.Collect(maps.Keys(set)) + pkg.Metadata.RegisterStatement(NewBuildStatement(stmt), deps, m.fileStack) } +type metadataStackCheckpoint struct { + symbolCheckpoint, fileCheckpoint, subincludesCheckpoint int +} + // checkpoint implements [scopeMetadata]. -func (m *trackingScopeMetadata) checkpoint() (int, int) { - return len(m.symbolStack), len(m.fileStack) +func (m *trackingScopeMetadata) checkpoint() metadataStackCheckpoint { + return metadataStackCheckpoint{ + symbolCheckpoint: len(m.symbolStack), + fileCheckpoint: len(m.fileStack), + subincludesCheckpoint: len(m.subincludesStack), + } } // restore implements [scopeMetadata]. -func (m *trackingScopeMetadata) restore(symbolLen, fileLen int) { - if symbolLen >= 0 && symbolLen <= len(m.symbolStack) { - m.symbolStack = m.symbolStack[:symbolLen] +func (m *trackingScopeMetadata) restore(cp metadataStackCheckpoint) { + if cp.symbolCheckpoint >= 0 && cp.symbolCheckpoint <= len(m.symbolStack) { + m.symbolStack = m.symbolStack[:cp.symbolCheckpoint] } - if fileLen >= 0 && fileLen <= len(m.fileStack) { - m.fileStack = m.fileStack[:fileLen] + if cp.fileCheckpoint >= 0 && cp.fileCheckpoint <= len(m.fileStack) { + m.fileStack = m.fileStack[:cp.fileCheckpoint] + } + if cp.subincludesCheckpoint >= 0 && cp.subincludesCheckpoint <= len(m.subincludesStack) { + m.subincludesStack = m.subincludesStack[:cp.subincludesCheckpoint] } } @@ -1317,6 +1349,11 @@ func (m *trackingScopeMetadata) pushFiles(rootPath string, filenames []string) { } } +// pushSubincludes implements [scopeMetadata]. +func (m *trackingScopeMetadata) pushSubincludes(labels core.BuildLabels) { + m.subincludesStack = append(m.subincludesStack, labels...) +} + // noopScopeMetadata implements the scopeMetadata interface with no-op methods. This is used to // avoid the overhead of storing metadata for operations that don't depend on it. type noopScopeMetadata struct{} @@ -1331,12 +1368,12 @@ func (nm *noopScopeMetadata) origin(scope *scope, name string) *core.BuildLabel func (nm *noopScopeMetadata) registerBuildStatement(pkg *core.Package, stmt *Statement) {} // checkpoint implements [scopeMetadata]. -func (nm *noopScopeMetadata) checkpoint() (int, int) { - return 0, 0 +func (nm *noopScopeMetadata) checkpoint() metadataStackCheckpoint { + return metadataStackCheckpoint{} } // restore implements [scopeMetadata]. -func (nm *noopScopeMetadata) restore(symbolLen, fileLen int) {} +func (nm *noopScopeMetadata) restore(cp metadataStackCheckpoint) {} // setCursor implements [scopeMetadata]. func (nm *noopScopeMetadata) setCursor(stmt *Statement) {} @@ -1350,6 +1387,9 @@ func (nm *noopScopeMetadata) pushSymbol(name string, origin *core.BuildLabel) {} // pushFiles implements [scopeMetadata]. func (nm *noopScopeMetadata) pushFiles(rootPath string, filenames []string) {} +// pushSubincludes implements [scopeMetadata]. +func (nm *noopScopeMetadata) pushSubincludes(labels core.BuildLabels) {} + // NewBuildStatement creates a new core.BuildStatement from an asp.statement. func NewBuildStatement(stmt *Statement) core.BuildStatement { return core.BuildStatement{ diff --git a/test/export/test_dynamic_subinclude/BUILD b/test/export/test_dynamic_subinclude/BUILD new file mode 100644 index 0000000000..b588cde421 --- /dev/null +++ b/test/export/test_dynamic_subinclude/BUILD @@ -0,0 +1,11 @@ +subinclude("//test/export:export_e2e_test_build_def") + +# Export targets generated by a custom build def in a loop with dynamic subincludes, +# and validate that unused subincludes are trimmed. +please_export_e2e_test( + name = "export_dynamic_subinclude", + export_targets = [ + "//:a", + "//:b", + ], +) diff --git a/test/export/test_dynamic_subinclude/expected_repo/.plzconfig b/test/export/test_dynamic_subinclude/expected_repo/.plzconfig new file mode 100644 index 0000000000..8e1ae5655a --- /dev/null +++ b/test/export/test_dynamic_subinclude/expected_repo/.plzconfig @@ -0,0 +1,3 @@ +[Parse] + +BuildFileName = BUILD_FILE diff --git a/test/export/test_dynamic_subinclude/expected_repo/BUILD_FILE b/test/export/test_dynamic_subinclude/expected_repo/BUILD_FILE new file mode 100644 index 0000000000..31cf1e8c06 --- /dev/null +++ b/test/export/test_dynamic_subinclude/expected_repo/BUILD_FILE @@ -0,0 +1,7 @@ +subinclude("//build_defs:dynamic_build_def") + +for i in [ + "a", + "b", +]: + dynamic_target(name = i, def_name = f"{i}_build_def") diff --git a/test/export/test_dynamic_subinclude/expected_repo/build_defs/BUILD_FILE b/test/export/test_dynamic_subinclude/expected_repo/build_defs/BUILD_FILE new file mode 100644 index 0000000000..625da44b75 --- /dev/null +++ b/test/export/test_dynamic_subinclude/expected_repo/build_defs/BUILD_FILE @@ -0,0 +1,17 @@ +filegroup( + name = "dynamic_build_def", + srcs = ["dynamic.build_defs"], + visibility = ["PUBLIC"], +) + +filegroup( + name = "a_build_def", + srcs = ["a.build_defs"], + visibility = ["PUBLIC"], +) + +filegroup( + name = "b_build_def", + srcs = ["b.build_defs"], + visibility = ["PUBLIC"], +) diff --git a/test/export/test_dynamic_subinclude/expected_repo/build_defs/a.build_defs b/test/export/test_dynamic_subinclude/expected_repo/build_defs/a.build_defs new file mode 100644 index 0000000000..e58be5dae4 --- /dev/null +++ b/test/export/test_dynamic_subinclude/expected_repo/build_defs/a.build_defs @@ -0,0 +1,6 @@ +def helper_a(name: str): + genrule( + name = name, + outs = [f"{name}.out"], + cmd = f"echo {name} > $OUT", + ) diff --git a/test/export/test_dynamic_subinclude/expected_repo/build_defs/b.build_defs b/test/export/test_dynamic_subinclude/expected_repo/build_defs/b.build_defs new file mode 100644 index 0000000000..86d44895a6 --- /dev/null +++ b/test/export/test_dynamic_subinclude/expected_repo/build_defs/b.build_defs @@ -0,0 +1,6 @@ +def helper_b(name: str): + genrule( + name = name, + outs = [f"{name}.out"], + cmd = f"echo {name} > $OUT", + ) diff --git a/test/export/test_dynamic_subinclude/expected_repo/build_defs/dynamic.build_defs b/test/export/test_dynamic_subinclude/expected_repo/build_defs/dynamic.build_defs new file mode 100644 index 0000000000..4fd7564e40 --- /dev/null +++ b/test/export/test_dynamic_subinclude/expected_repo/build_defs/dynamic.build_defs @@ -0,0 +1,6 @@ +def dynamic_target(name: str, def_name: str): + subinclude(f"//build_defs:{def_name}") + if name == "a": + helper_a(name = name) + elif name == "b": + helper_b(name = name) diff --git a/test/export/test_dynamic_subinclude/source_repo/.plzconfig b/test/export/test_dynamic_subinclude/source_repo/.plzconfig new file mode 100644 index 0000000000..8e1ae5655a --- /dev/null +++ b/test/export/test_dynamic_subinclude/source_repo/.plzconfig @@ -0,0 +1,3 @@ +[Parse] + +BuildFileName = BUILD_FILE diff --git a/test/export/test_dynamic_subinclude/source_repo/BUILD_FILE b/test/export/test_dynamic_subinclude/source_repo/BUILD_FILE new file mode 100644 index 0000000000..31cf1e8c06 --- /dev/null +++ b/test/export/test_dynamic_subinclude/source_repo/BUILD_FILE @@ -0,0 +1,7 @@ +subinclude("//build_defs:dynamic_build_def") + +for i in [ + "a", + "b", +]: + dynamic_target(name = i, def_name = f"{i}_build_def") diff --git a/test/export/test_dynamic_subinclude/source_repo/build_defs/BUILD_FILE b/test/export/test_dynamic_subinclude/source_repo/build_defs/BUILD_FILE new file mode 100644 index 0000000000..833e53acec --- /dev/null +++ b/test/export/test_dynamic_subinclude/source_repo/build_defs/BUILD_FILE @@ -0,0 +1,23 @@ +filegroup( + name = "dynamic_build_def", + srcs = ["dynamic.build_defs"], + visibility = ["PUBLIC"], +) + +filegroup( + name = "a_build_def", + srcs = ["a.build_defs"], + visibility = ["PUBLIC"], +) + +filegroup( + name = "b_build_def", + srcs = ["b.build_defs"], + visibility = ["PUBLIC"], +) + +filegroup( + name = "unneeded_build_def", + srcs = ["unneeded.build_defs"], + visibility = ["PUBLIC"], +) diff --git a/test/export/test_dynamic_subinclude/source_repo/build_defs/a.build_defs b/test/export/test_dynamic_subinclude/source_repo/build_defs/a.build_defs new file mode 100644 index 0000000000..e58be5dae4 --- /dev/null +++ b/test/export/test_dynamic_subinclude/source_repo/build_defs/a.build_defs @@ -0,0 +1,6 @@ +def helper_a(name: str): + genrule( + name = name, + outs = [f"{name}.out"], + cmd = f"echo {name} > $OUT", + ) diff --git a/test/export/test_dynamic_subinclude/source_repo/build_defs/b.build_defs b/test/export/test_dynamic_subinclude/source_repo/build_defs/b.build_defs new file mode 100644 index 0000000000..86d44895a6 --- /dev/null +++ b/test/export/test_dynamic_subinclude/source_repo/build_defs/b.build_defs @@ -0,0 +1,6 @@ +def helper_b(name: str): + genrule( + name = name, + outs = [f"{name}.out"], + cmd = f"echo {name} > $OUT", + ) diff --git a/test/export/test_dynamic_subinclude/source_repo/build_defs/dynamic.build_defs b/test/export/test_dynamic_subinclude/source_repo/build_defs/dynamic.build_defs new file mode 100644 index 0000000000..4fd7564e40 --- /dev/null +++ b/test/export/test_dynamic_subinclude/source_repo/build_defs/dynamic.build_defs @@ -0,0 +1,6 @@ +def dynamic_target(name: str, def_name: str): + subinclude(f"//build_defs:{def_name}") + if name == "a": + helper_a(name = name) + elif name == "b": + helper_b(name = name) diff --git a/test/export/test_dynamic_subinclude/source_repo/build_defs/unneeded.build_defs b/test/export/test_dynamic_subinclude/source_repo/build_defs/unneeded.build_defs new file mode 100644 index 0000000000..dd2fc99e36 --- /dev/null +++ b/test/export/test_dynamic_subinclude/source_repo/build_defs/unneeded.build_defs @@ -0,0 +1,2 @@ +def unneeded_helper(name: str): + pass From b89bc92ccb8d91f4d6af0a73b73d31e9ebfadb6c Mon Sep 17 00:00:00 2001 From: DuBento Date: Mon, 29 Jun 2026 16:50:08 +0100 Subject: [PATCH 111/118] track inner level globs. --- src/parse/asp/builtins.go | 4 ++-- src/parse/asp/interpreter.go | 20 +++++++++++++++++--- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/parse/asp/builtins.go b/src/parse/asp/builtins.go index a7a4348c79..df58239640 100644 --- a/src/parse/asp/builtins.go +++ b/src/parse/asp/builtins.go @@ -373,7 +373,7 @@ func subinclude(s *scope, args []pyObject) pyObject { } labels = append(labels, t.Label) } - s.registerSubincludes(labels) + s.metadataRegisterSubincludes(labels) return None } @@ -735,7 +735,7 @@ func glob(s *scope, args []pyObject) pyObject { exclude = exclude[:len(exclude)-len(s.state.Config.Parse.BuildFileName)] log.Fatalf("glob(include=%s, exclude=%s) in %s returned no files. If this is intended, set allow_empty=True on the glob.", include, exclude, s.pkg.Filename) } - s.metadata.pushFiles(s.pkg.Name, glob) + s.metadataRegisterFiles(s.pkg.Name, glob) return fromStringList(glob) } diff --git a/src/parse/asp/interpreter.go b/src/parse/asp/interpreter.go index a6871dce49..feed4a9fee 100644 --- a/src/parse/asp/interpreter.go +++ b/src/parse/asp/interpreter.go @@ -1180,16 +1180,30 @@ func (s *scope) getOrNewMetadata(pkg *core.Package) scopeMetadata { return meta } -func (s *scope) registerSubincludes(labels core.BuildLabels) { +// getPackageMetadata returns the metadata for that current scope's package. +func (s *scope) getPackageMetadata() scopeMetadata { if s.pkg == nil || s.interpreter.packageMetadata == nil { // This will be initialized if ParseMetadata is set. + return nil + } + return s.interpreter.packageMetadata.Get(s.pkg.Label()) +} + +// metadataRegisterFiles adds the files to the scope's package metadata. +func (s *scope) metadataRegisterFiles(rootPath string, files []string) { + meta := s.getPackageMetadata() + if meta == nil { return } - meta := s.interpreter.packageMetadata.Get(s.pkg.Label()) + meta.pushFiles(rootPath, files) +} + +// metadataRegisterSubincludes adds the subincluded labels to the scope's package metadata. +func (s *scope) metadataRegisterSubincludes(labels core.BuildLabels) { + meta := s.getPackageMetadata() if meta == nil { return } meta.pushSubincludes(labels) - } // scopeMetadata defines an interface for tracking evaluation metadata (such as AST cursor position From e72ef50701bd08f62d582d1f3d6b7d11f50cc3ce Mon Sep 17 00:00:00 2001 From: DuBento Date: Mon, 29 Jun 2026 18:55:59 +0100 Subject: [PATCH 112/118] feat(cmd): query print metadata - print package metadata --- docs/commands.html | 5 + src/core/package_metadata.go | 61 +++++++++-- src/please.go | 23 ++++ src/query/BUILD | 1 + src/query/metadata.go | 204 +++++++++++++++++++++++++++++++++++ 5 files changed, 287 insertions(+), 7 deletions(-) create mode 100644 src/query/metadata.go diff --git a/docs/commands.html b/docs/commands.html index df9590218c..9ebc62ba54 100644 --- a/docs/commands.html +++ b/docs/commands.html @@ -804,6 +804,11 @@

whatoutputs: Prints out target(s) responsible for outputting provided file(s) +
  • + + metadata: Prints out a structured, tree-like visualization of parsed build statement metadata (required subincludes, files, and generated targets) of a package. + +

    • diff --git a/src/core/package_metadata.go b/src/core/package_metadata.go index ad6dfc3d11..ee4ca36a61 100644 --- a/src/core/package_metadata.go +++ b/src/core/package_metadata.go @@ -1,6 +1,7 @@ package core import ( + "cmp" "fmt" "maps" "slices" @@ -45,6 +46,14 @@ type BuildStatementProvider func() BuildStatement // unnecessary computation when using the no-op implementation. type SubincludesLabelProvider func() BuildLabels +// StatementMetadata represents all parsed metadata associated with a single statement. +type StatementMetadata struct { + Statement BuildStatement + Subincludes BuildLabels + Files []string + Targets BuildLabels +} + // PackageMetadata stores metadata about parsed BUILD files, mapping statements and subincludes // to their respective targets. This supports additional logic for operations such as `plz export` // but should be disabled for most operations by using the no-op implementation to avoid the overhead. @@ -85,9 +94,8 @@ type PackageMetadata interface { // IsInterpretedStatement returns true if the statement provided matches a registered build // statement, meaning it was interpreted even if it doesn't generate any targets. IsInterpretedStatement(stmt BuildStatement) bool - // WriteMetadata writes a visualization of the metadata (statements and their required labels/files) to the writer. - // Only statements that generated the specified targets will be displayed. - WriteMetadata(w io.Writer, fileContent []byte, targets BuildLabels, includeSources, includeDeps, includeOutputs bool, resolver func(BuildLabel) *BuildTarget) + // Statements returns all statement metadata tracked in the package, sorted by BUILD file order. + Statements() []StatementMetadata } // packageMetadataImpl is the canonical implementation of the PackageMetadata interface. It tracks the relationships between BUILD file statements, @@ -265,6 +273,43 @@ func (m *packageMetadataImpl) IsInterpretedStatement(stmt BuildStatement) bool { return m.stmtToTarget.Contains(stmt) } +// Statements implements [PackageMetadata]. +func (m *packageMetadataImpl) Statements() []StatementMetadata { + stmtsMap := map[BuildStatement]*StatementMetadata{} + getOrCreate := func(stmt BuildStatement) *StatementMetadata { + if sm, ok := stmtsMap[stmt]; ok { + return sm + } + sm := &StatementMetadata{Statement: stmt} + stmtsMap[stmt] = sm + return sm + } + + m.stmtToRequiredSubincludes.Range(func(stmt BuildStatement, labels BuildLabels) { + getOrCreate(stmt).Subincludes = labels + }) + + m.stmtToRequiredFiles.Range(func(stmt BuildStatement, files []string) { + getOrCreate(stmt).Files = files + }) + + m.stmtToTarget.Range(func(stmt BuildStatement, labels BuildLabels) { + getOrCreate(stmt).Targets = labels + }) + + res := make([]StatementMetadata, 0, len(stmtsMap)) + for _, sm := range stmtsMap { + res = append(res, *sm) + } + + // Sort the statements to maintain BUILD file order + slices.SortFunc(res, func(i, j StatementMetadata) int { + return cmp.Compare(i.Statement.Start, j.Statement.Start) + }) + + return res +} + // noopPackageMetadata implements the PackageMetadata interface with no-op methods. This is the // default implementation and is used to avoid the overhead of parsing metadata for operations that // don't depend on it. @@ -282,10 +327,6 @@ func (n *noopPackageMetadata) RegisterStatement(stmt BuildStatement, deps BuildL func (n *noopPackageMetadata) RegisterStatementTarget(target BuildLabel, stmtProvider BuildStatementProvider) { } -// RegisterRequiredSubinclude implements [PackageMetadata]. -func (n *noopPackageMetadata) RegisterRequiredSubinclude(target BuildLabel, labelProvider SubincludesLabelProvider) { -} - // RegisterSubincludeStatement implements [PackageMetadata]. func (n *noopPackageMetadata) RegisterSubincludeStatement(label BuildLabel, stmtProvider BuildStatementProvider) { } @@ -331,3 +372,9 @@ func (n *noopPackageMetadata) IsInterpretedStatement(stmt BuildStatement) bool { log.Fatalf("metadata not tracked, using no-op implementation") return false } + +// Statements implements [PackageMetadata]. +func (n *noopPackageMetadata) Statements() []StatementMetadata { + log.Fatalf("metadata not tracked, using no-op implementation") + return nil +} diff --git a/src/please.go b/src/please.go index 50e09558d3..12e8007640 100644 --- a/src/please.go +++ b/src/please.go @@ -462,6 +462,16 @@ var opts struct { Options []string `positional-arg-name:"options" description:"Print specific options."` } `positional-args:"true"` } `command:"config" description:"Prints the configuration settings"` + Metadata struct { + Sources bool `long:"sources" short:"s" description:"Include target sources in visualization"` + Deps bool `long:"deps" short:"d" description:"Include target dependencies in visualization"` + Outputs bool `long:"outputs" short:"o" description:"Include target outputs in visualization"` + All bool `long:"all" short:"a" description:"Include all details (sources, deps, and outputs)"` + AllStatements bool `long:"all_statements" description:"Print all statements in the BUILD file, including those that didn't generate the specified targets"` + Args struct { + Targets []core.BuildLabel `positional-arg-name:"targets" description:"Targets or packages to display metadata for" required:"true"` + } `positional-args:"true" required:"true"` + } `command:"metadata" description:"Prints all metadata (code statements, generated targets and their required subincludes/files) of a package."` } `command:"query" description:"Queries information about the build state"` Generate struct { Gitignore string `long:"update_gitignore" description:"The gitignore file to write the generated sources to"` @@ -836,6 +846,19 @@ var buildFunctions = map[string]func() int{ query.TargetOutputs(state.Graph, state.ExpandOriginalLabels(), opts.Query.Output.JSON) }) }, + "query.metadata": func() int { + if success, state := runBuild(opts.Query.Metadata.Args.Targets, buildOpts{ParseMetadata: true, IsQuery: true}); success { + m := opts.Query.Metadata + query.Metadata(state, state.ExpandOriginalLabels(), query.WriteMetadataOpts{ + IncludeSources: m.Sources || m.All, + IncludeDeps: m.Deps || m.All, + IncludeOutputs: m.Outputs || m.All, + IncludeAllStatements: m.AllStatements, + }) + return 0 + } + return 1 + }, "query.completions": func() int { // Somewhat fiddly because the inputs are not necessarily well-formed at this point. opts.ParsePackageOnly = true diff --git a/src/query/BUILD b/src/query/BUILD index 310bd21794..26a780b6a0 100644 --- a/src/query/BUILD +++ b/src/query/BUILD @@ -10,6 +10,7 @@ go_library( "///third_party/go/github.com_please-build_gcfg//:gcfg", "///third_party/go/golang.org_x_exp//maps", "//src/build", + "//src/cli", "//src/cli/logging", "//src/core", "//src/fs", diff --git a/src/query/metadata.go b/src/query/metadata.go new file mode 100644 index 0000000000..be926176c0 --- /dev/null +++ b/src/query/metadata.go @@ -0,0 +1,204 @@ +package query + +import ( + "fmt" + "os" + "strings" + + "github.com/thought-machine/please/src/cli" + "github.com/thought-machine/please/src/core" +) + +// WriteMetadataOpts contains configuration options for formatting and writing package metadata. +type WriteMetadataOpts struct { + IncludeSources bool + IncludeDeps bool + IncludeOutputs bool + IncludeAllStatements bool +} + +// any reports true if any of the options are set to true. +func (wmo WriteMetadataOpts) any() bool { + return wmo.IncludeDeps || wmo.IncludeOutputs || wmo.IncludeSources +} + +// Metadata prints out a visualization of the parsed build statement metadata for the given targets. +func Metadata(state *core.BuildState, targets []core.BuildLabel, opts WriteMetadataOpts) { + // Group requested targets by their package + packageTargets := map[*core.Package][]core.BuildLabel{} + for _, label := range targets { + pkg := state.Graph.PackageOrDie(label) + packageTargets[pkg] = append(packageTargets[pkg], label) + } + + for pkg, label := range packageTargets { + fmt.Printf("=== Package: %s (File: %s) ===\n", pkg.Label(), pkg.Filename) + + // Read the file content to extract the statement code slices + content, err := os.ReadFile(pkg.Filename) + if err != nil { + fmt.Printf("Error reading BUILD file %s: %v\n\n", pkg.Filename, err) + continue + } + + // Retrieve all raw statement metadata from core + allStatements := pkg.Metadata.Statements() + + // Filter statements if needed + var filterStmts map[core.BuildStatement]struct{} + writeAll := opts.IncludeAllStatements + + if !writeAll && len(label) > 0 { + filterStmts = map[core.BuildStatement]struct{}{} + for _, target := range label { + stmt, err := pkg.Metadata.FindStatement(target) + if err == nil && stmt != (core.BuildStatement{}) { + filterStmts[stmt] = struct{}{} + } + } + } + + // Print statements using tree logic + for _, sm := range allStatements { + if !writeAll { + if _, ok := filterStmts[sm.Statement]; !ok { + continue + } + } + + code := string(content[sm.Statement.Start:sm.Statement.End]) + + cli.Fprintf(os.Stdout, "${BOLD_CYAN}Statement (Offsets: %d-%d):${RESET}\n", sm.Statement.Start, sm.Statement.End) + cli.Fprintf(os.Stdout, " ${CYAN}Code:${RESET}\n") + // Indent the code + for line := range strings.SplitSeq(code, "\n") { + cli.Fprintf(os.Stdout, " %s\n", line) + } + + hasSubincludes := len(sm.Subincludes) > 0 + hasFiles := len(sm.Files) > 0 + hasTargets := len(sm.Targets) > 0 + + // Identify the last section so we use └── instead of ├── + var lastSection string + if hasTargets { + lastSection = "targets" + } else if hasFiles { + lastSection = "files" + } else if hasSubincludes { + lastSection = "subincludes" + } + + // itemDetail holds the text to print and its optional ANSI color formatting string + type itemDetail struct { + text string + color string + } + + // Helper to print a titled section of items using precise tree box-drawing characters + printSection := func(prefix, title string, items []itemDetail, isLast bool) string { + if len(items) == 0 { + return "" + } + branch := "├──" + childPrefix := prefix + "│ " + if isLast { + branch = "└──" + childPrefix = prefix + " " + } + cli.Fprintf(os.Stdout, "%s%s ${CYAN}%s:${RESET}\n", prefix, branch, title) + for idx, item := range items { + itemBranch := "├──" + if idx == len(items)-1 { + itemBranch = "└──" + } + cli.Fprintf(os.Stdout, "%s%s "+item.color+"%s${RESET}\n", childPrefix, itemBranch, item.text) + } + return childPrefix + } + + labelsToItems := func(labels core.BuildLabels, defaultColor string) []itemDetail { + res := make([]itemDetail, len(labels)) + for i, l := range labels { + res[i] = itemDetail{text: l.String(), color: defaultColor} + } + return res + } + + stringsToItems := func(strs []string, defaultColor string) []itemDetail { + res := make([]itemDetail, len(strs)) + for i, s := range strs { + res[i] = itemDetail{text: s, color: defaultColor} + } + return res + } + + inputsToItems := func(inputs []core.BuildInput) []itemDetail { + res := make([]itemDetail, len(inputs)) + for i, inp := range inputs { + color := "" + if _, ok := inp.Label(); ok { + color = "${GREEN}" + } + res[i] = itemDetail{text: inp.String(), color: color} + } + return res + } + + basePrefix := " " + if hasSubincludes { + printSection(basePrefix, "Required Subincludes", labelsToItems(sm.Subincludes, "${YELLOW}"), lastSection == "subincludes") + } + + if hasFiles { + printSection(basePrefix, "Required Files", stringsToItems(sm.Files, ""), lastSection == "files") + } + + if hasTargets { + branch := "├──" + childPrefix := basePrefix + "│ " + if lastSection == "targets" { + branch = "└──" + childPrefix = basePrefix + " " + } + cli.Fprintf(os.Stdout, "%s%s ${CYAN}Generated Targets:${RESET}\n", basePrefix, branch) + + for i, t := range sm.Targets { + targetBranch := "├──" + targetChildPrefix := childPrefix + "│ " + if i == len(sm.Targets)-1 { + targetBranch = "└──" + targetChildPrefix = childPrefix + " " + } + cli.Fprintf(os.Stdout, "%s%s ${BOLD_GREEN}%s${RESET}\n", childPrefix, targetBranch, t) + + // Look up and display optional target details + if opts.any() { + if target := state.Graph.Target(t); target != nil { + type optDetail struct { + title string + items []itemDetail + } + var details []optDetail + if opts.IncludeSources && len(target.AllSources()) > 0 { + details = append(details, optDetail{"Sources", inputsToItems(target.AllSources())}) + } + if opts.IncludeDeps && len(target.DeclaredDependencies()) > 0 { + details = append(details, optDetail{"Dependencies", labelsToItems(target.DeclaredDependencies(), "${GREEN}")}) + } + if opts.IncludeOutputs && len(target.Outputs()) > 0 { + details = append(details, optDetail{"Outputs", stringsToItems(target.Outputs(), "")}) + } + + for idx, det := range details { + isLastDetail := idx == len(details)-1 + printSection(targetChildPrefix, det.title, det.items, isLastDetail) + } + } + } + } + } + fmt.Fprintln(os.Stdout) + } + } +} From 70f47018f0846503e488a2ad3c1c0e04dd87da26 Mon Sep 17 00:00:00 2001 From: DuBento Date: Tue, 30 Jun 2026 16:47:21 +0100 Subject: [PATCH 113/118] track inner statements (e.g. if clauses) and trim clauses based on if they were interpreted oir not --- src/export/trimmer.go | 43 ++++++------------------------------ src/parse/asp/interpreter.go | 26 ++++++++++++---------- 2 files changed, 21 insertions(+), 48 deletions(-) diff --git a/src/export/trimmer.go b/src/export/trimmer.go index a55c0b00de..07836e4a8c 100644 --- a/src/export/trimmer.go +++ b/src/export/trimmer.go @@ -111,11 +111,15 @@ func (t *trimmer) trimIf(stmt *asp.Statement) bool { // In an if-else statement only the interpreted/evaluated block will generate targets, meaning // that normally only one of the clauses is interpreted, however an if statement could be inside of // a loop where the clause condition depends on the iteration meaning more than one clause - // could end up being interpreted. Because of that we lookup all the required clauses before - // writing the statement. Any clauses with interpreted statements should be visited. + // could end up being interpreted. Because of that we'll visit all the clauses, writing only the + // required statements -- any clauses with interpreted statements should be visited. var requiredClauses = make([]bool, len(clauses)) for i, c := range clauses { - required := t.isRequiredStatements(c.stmts) + if len(c.stmts) == 0 { + continue + } + bStmt := asp.NewBuildStatement(c.stmts[0]) + required := t.pkg.Metadata.IsInterpretedStatement(bStmt) requiredClauses[i] = required } @@ -170,39 +174,6 @@ func (t *trimmer) passBlock(stmts []*asp.Statement, blockStart, blockEnd asp.Pos }) } -// isRequiredStatement determines if it is necessary to visit any of the statements. Refer to -// [isRequiredStatement] for the decision logic. -func (t *trimmer) isRequiredStatements(stmts []*asp.Statement) bool { - return slices.ContainsFunc(stmts, t.isRequiredStatement) -} - -// isRequiredStatement determines if it is necessary to visit a statement. Any statement that was -// interpreted needs to be visited but the trimming logic of the visitor should determine if we need -// to write it or not. -func (t *trimmer) isRequiredStatement(stmt *asp.Statement) bool { - if stmt.If != nil { - // If - if t.isRequiredStatements(stmt.If.Statements) { - return true - } - // Elif - if len(stmt.If.Elif) > 0 { - for _, elif := range stmt.If.Elif { - if t.isRequiredStatements(elif.Statements) { - return true - } - } - } - // Else - if len(stmt.If.ElseStatements) > 0 && t.isRequiredStatements(stmt.If.ElseStatements) { - return true - } - } else if stmt.For != nil { - return t.isRequiredStatements(stmt.For.Statements) - } - return t.pkg.Metadata.IsInterpretedStatement(asp.NewBuildStatement(stmt)) -} - func (t *trimmer) statementTargets(stmt *asp.Statement) core.BuildLabels { bStmt := asp.NewBuildStatement(stmt) return t.pkg.Metadata.FindTargets(bStmt) diff --git a/src/parse/asp/interpreter.go b/src/parse/asp/interpreter.go index feed4a9fee..88c00b7e32 100644 --- a/src/parse/asp/interpreter.go +++ b/src/parse/asp/interpreter.go @@ -577,23 +577,21 @@ func (s *scope) interpretStatements(statements []*Statement) pyObject { s.metadata.setCursor(stmt) checkpoint := s.metadata.checkpoint() + var ret pyObject if stmt.FuncDef != nil { s.Set(stmt.FuncDef.Name, newPyFunc(s, stmt.FuncDef)) } else if stmt.If != nil { - if ret := s.interpretIf(stmt.If); ret != nil { - return ret - } + ret = s.interpretIf(stmt.If) } else if stmt.For != nil { - if ret := s.interpretFor(stmt.For); ret != nil { - return ret - } + ret = s.interpretFor(stmt.For) } else if stmt.Return != nil { if len(stmt.Return.Values) == 0 { - return None + ret = None } else if len(stmt.Return.Values) == 1 { - return s.interpretExpression(stmt.Return.Values[0]) + ret = s.interpretExpression(stmt.Return.Values[0]) + } else { + ret = pyList(s.evaluateExpressions(stmt.Return.Values)) } - return pyList(s.evaluateExpressions(stmt.Return.Values)) } else if stmt.Ident != nil { s.interpretIdentStatement(stmt.Ident) } else if stmt.Assert != nil { @@ -611,18 +609,22 @@ func (s *scope) interpretStatements(statements []*Statement) pyObject { s.interpretExpression(stmt.Literal) } else if stmt.Continue { // This is definitely awkward since we need to control a for loop that's happening in a function outside this scope. - return continueIteration + ret = continueIteration } else if stmt.Break { // Similar to above, although CPython does do this the same way... - return stopIteration + ret = stopIteration } else if stmt.Pass { - continue // Nothing to do... + // Nothing to do... } else { s.Error("Unknown statement") // Shouldn't happen, amirite? } s.metadata.registerBuildStatement(s.pkg, stmt) s.metadata.restore(checkpoint) + + if ret != nil { + return ret + } } return nil } From 0e7a3b289cc7b934f6621a96c65d5ebbf20e8e96 Mon Sep 17 00:00:00 2001 From: DuBento Date: Tue, 30 Jun 2026 16:52:37 +0100 Subject: [PATCH 114/118] lint suggestions: remove hashBuildTarget and initialise labels slice with cap --- src/core/build_target.go | 5 ----- src/parse/asp/builtins.go | 2 +- src/please.go | 5 +++-- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/src/core/build_target.go b/src/core/build_target.go index e20aa11d8d..2581da5b50 100644 --- a/src/core/build_target.go +++ b/src/core/build_target.go @@ -416,11 +416,6 @@ func (target *BuildTarget) String() string { return target.Label.String() } -// hashBuildTarget returns a mostly unique hash of a BuildTarget by relying on the BuildLabel hash. -func hashBuildTarget(target *BuildTarget) uint64 { - return HashBuildLabel(target.Label) -} - // TmpDir returns the temporary working directory for this target, eg. // //mickey/donald:goofy -> plz-out/tmp/mickey/donald/goofy._build // Note the extra subdirectory to keep rules separate from one another, and the .build suffix diff --git a/src/parse/asp/builtins.go b/src/parse/asp/builtins.go index df58239640..ae3f5c9d62 100644 --- a/src/parse/asp/builtins.go +++ b/src/parse/asp/builtins.go @@ -349,7 +349,7 @@ func subinclude(s *scope, args []pyObject) pyObject { s.Error("cannot subinclude type %s", arg.Type()) } } - var labels core.BuildLabels + var labels = make(core.BuildLabels, 0, len(si)) for _, arg := range si { label, annotation := core.SplitLabelAnnotation(arg) t := subincludeTarget(s, s.parseLabelInContextPkg(label)) diff --git a/src/please.go b/src/please.go index 12e8007640..226cad274c 100644 --- a/src/please.go +++ b/src/please.go @@ -1349,8 +1349,9 @@ type buildOpts struct { Test bool IsQuery bool ParseMetadata bool - // Keep the workers running in the background for inline parsing during specific ops (e.g. export). - // Note: when running background workers we need to explicit call CleanUp at the end of the CLI op. + // Keep the workers running in the background for inline parsing during specific operations (e.g. + // export). Note: when running background workers we need to explicit call CleanUp at the end of + // the CLI operation. KeepParserRunning bool } From d3f27bfffd5d6fa2fcd6ef104c14567a45507495 Mon Sep 17 00:00:00 2001 From: DuBento Date: Tue, 30 Jun 2026 17:45:15 +0100 Subject: [PATCH 115/118] feat(cmd): metadata json formatting --- src/please.go | 2 + src/query/metadata.go | 278 ++++++++++++++++++++++++++++++------------ 2 files changed, 205 insertions(+), 75 deletions(-) diff --git a/src/please.go b/src/please.go index 226cad274c..9e805f726f 100644 --- a/src/please.go +++ b/src/please.go @@ -468,6 +468,7 @@ var opts struct { Outputs bool `long:"outputs" short:"o" description:"Include target outputs in visualization"` All bool `long:"all" short:"a" description:"Include all details (sources, deps, and outputs)"` AllStatements bool `long:"all_statements" description:"Print all statements in the BUILD file, including those that didn't generate the specified targets"` + JSON bool `long:"json" short:"j" description:"Output metadata as JSON"` Args struct { Targets []core.BuildLabel `positional-arg-name:"targets" description:"Targets or packages to display metadata for" required:"true"` } `positional-args:"true" required:"true"` @@ -854,6 +855,7 @@ var buildFunctions = map[string]func() int{ IncludeDeps: m.Deps || m.All, IncludeOutputs: m.Outputs || m.All, IncludeAllStatements: m.AllStatements, + FormatJSON: m.JSON, }) return 0 } diff --git a/src/query/metadata.go b/src/query/metadata.go index be926176c0..8f06f631bd 100644 --- a/src/query/metadata.go +++ b/src/query/metadata.go @@ -1,6 +1,7 @@ package query import ( + "encoding/json" "fmt" "os" "strings" @@ -15,6 +16,7 @@ type WriteMetadataOpts struct { IncludeDeps bool IncludeOutputs bool IncludeAllStatements bool + FormatJSON bool } // any reports true if any of the options are set to true. @@ -24,6 +26,10 @@ func (wmo WriteMetadataOpts) any() bool { // Metadata prints out a visualization of the parsed build statement metadata for the given targets. func Metadata(state *core.BuildState, targets []core.BuildLabel, opts WriteMetadataOpts) { + if !cli.ShowColouredOutput || !cli.IsATerminal(os.Stdout) { + cli.ShowColouredOutput = false + } + // Group requested targets by their package packageTargets := map[*core.Package][]core.BuildLabel{} for _, label := range targets { @@ -31,34 +37,83 @@ func Metadata(state *core.BuildState, targets []core.BuildLabel, opts WriteMetad packageTargets[pkg] = append(packageTargets[pkg], label) } + if opts.FormatJSON { + printJSON(state, packageTargets, opts) + } else { + printTerminal(state, packageTargets, opts) + } +} + +// printTerminal formats and draws the metadata as a beautiful colorized terminal tree-box layout. +func printTerminal(state *core.BuildState, packageTargets map[*core.Package][]core.BuildLabel, opts WriteMetadataOpts) { + // itemDetail holds the text to print and its optional ANSI color formatting string + type itemDetail struct { + text string + color string + } + + // Helper to print a titled section of items using precise tree box-drawing characters + printSection := func(prefix, title string, items []itemDetail, isLast bool) string { + if len(items) == 0 { + return "" + } + branch := "├──" + childPrefix := prefix + "│ " + if isLast { + branch = "└──" + childPrefix = prefix + " " + } + cli.Fprintf(os.Stdout, "%s%s ${CYAN}%s:${RESET}\n", prefix, branch, title) + for idx, item := range items { + itemBranch := "├──" + if idx == len(items)-1 { + itemBranch = "└──" + } + cli.Fprintf(os.Stdout, "%s%s "+item.color+"%s${RESET}\n", childPrefix, itemBranch, item.text) + } + return childPrefix + } + + labelsToItems := func(labels core.BuildLabels, defaultColor string) []itemDetail { + res := make([]itemDetail, len(labels)) + for i, l := range labels { + res[i] = itemDetail{text: l.String(), color: defaultColor} + } + return res + } + + stringsToItems := func(strs []string, defaultColor string) []itemDetail { + res := make([]itemDetail, len(strs)) + for i, s := range strs { + res[i] = itemDetail{text: s, color: defaultColor} + } + return res + } + + inputsToItems := func(inputs []core.BuildInput) []itemDetail { + res := make([]itemDetail, len(inputs)) + for i, inp := range inputs { + color := "" + if _, ok := inp.Label(); ok { + color = "${GREEN}" + } + res[i] = itemDetail{text: inp.String(), color: color} + } + return res + } + for pkg, label := range packageTargets { fmt.Printf("=== Package: %s (File: %s) ===\n", pkg.Label(), pkg.Filename) - // Read the file content to extract the statement code slices content, err := os.ReadFile(pkg.Filename) if err != nil { fmt.Printf("Error reading BUILD file %s: %v\n\n", pkg.Filename, err) continue } - // Retrieve all raw statement metadata from core allStatements := pkg.Metadata.Statements() + writeAll, filterStmts := filterStatements(pkg, label, opts.IncludeAllStatements) - // Filter statements if needed - var filterStmts map[core.BuildStatement]struct{} - writeAll := opts.IncludeAllStatements - - if !writeAll && len(label) > 0 { - filterStmts = map[core.BuildStatement]struct{}{} - for _, target := range label { - stmt, err := pkg.Metadata.FindStatement(target) - if err == nil && stmt != (core.BuildStatement{}) { - filterStmts[stmt] = struct{}{} - } - } - } - - // Print statements using tree logic for _, sm := range allStatements { if !writeAll { if _, ok := filterStmts[sm.Statement]; !ok { @@ -79,7 +134,6 @@ func Metadata(state *core.BuildState, targets []core.BuildLabel, opts WriteMetad hasFiles := len(sm.Files) > 0 hasTargets := len(sm.Targets) > 0 - // Identify the last section so we use └── instead of ├── var lastSection string if hasTargets { lastSection = "targets" @@ -89,62 +143,6 @@ func Metadata(state *core.BuildState, targets []core.BuildLabel, opts WriteMetad lastSection = "subincludes" } - // itemDetail holds the text to print and its optional ANSI color formatting string - type itemDetail struct { - text string - color string - } - - // Helper to print a titled section of items using precise tree box-drawing characters - printSection := func(prefix, title string, items []itemDetail, isLast bool) string { - if len(items) == 0 { - return "" - } - branch := "├──" - childPrefix := prefix + "│ " - if isLast { - branch = "└──" - childPrefix = prefix + " " - } - cli.Fprintf(os.Stdout, "%s%s ${CYAN}%s:${RESET}\n", prefix, branch, title) - for idx, item := range items { - itemBranch := "├──" - if idx == len(items)-1 { - itemBranch = "└──" - } - cli.Fprintf(os.Stdout, "%s%s "+item.color+"%s${RESET}\n", childPrefix, itemBranch, item.text) - } - return childPrefix - } - - labelsToItems := func(labels core.BuildLabels, defaultColor string) []itemDetail { - res := make([]itemDetail, len(labels)) - for i, l := range labels { - res[i] = itemDetail{text: l.String(), color: defaultColor} - } - return res - } - - stringsToItems := func(strs []string, defaultColor string) []itemDetail { - res := make([]itemDetail, len(strs)) - for i, s := range strs { - res[i] = itemDetail{text: s, color: defaultColor} - } - return res - } - - inputsToItems := func(inputs []core.BuildInput) []itemDetail { - res := make([]itemDetail, len(inputs)) - for i, inp := range inputs { - color := "" - if _, ok := inp.Label(); ok { - color = "${GREEN}" - } - res[i] = itemDetail{text: inp.String(), color: color} - } - return res - } - basePrefix := " " if hasSubincludes { printSection(basePrefix, "Required Subincludes", labelsToItems(sm.Subincludes, "${YELLOW}"), lastSection == "subincludes") @@ -172,7 +170,6 @@ func Metadata(state *core.BuildState, targets []core.BuildLabel, opts WriteMetad } cli.Fprintf(os.Stdout, "%s%s ${BOLD_GREEN}%s${RESET}\n", childPrefix, targetBranch, t) - // Look up and display optional target details if opts.any() { if target := state.Graph.Target(t); target != nil { type optDetail struct { @@ -202,3 +199,134 @@ func Metadata(state *core.BuildState, targets []core.BuildLabel, opts WriteMetad } } } + +type jsonTargetMetadata struct { + Name string `json:"name"` + Sources []string `json:"sources,omitempty"` + Dependencies []string `json:"dependencies,omitempty"` + Outputs []string `json:"outputs,omitempty"` +} + +type jsonStatementMetadata struct { + Start int `json:"start"` + End int `json:"end"` + Code string `json:"code"` + Subincludes []string `json:"subincludes,omitempty"` + Files []string `json:"files,omitempty"` + Targets []jsonTargetMetadata `json:"targets,omitempty"` +} + +type jsonPackageMetadata struct { + Package string `json:"package"` + BuildFile string `json:"build_file"` + Statements []jsonStatementMetadata `json:"statements"` +} + +// printJSON formats and serializes the metadata to stdout as indented JSON. +func printJSON(state *core.BuildState, packageTargets map[*core.Package][]core.BuildLabel, opts WriteMetadataOpts) { + var jsonOutput []jsonPackageMetadata + + for pkg, label := range packageTargets { + content, err := os.ReadFile(pkg.Filename) + if err != nil { + continue + } + + allStatements := pkg.Metadata.Statements() + writeAll, filterStmts := filterStatements(pkg, label, opts.IncludeAllStatements) + + var packageMeta jsonPackageMetadata + packageMeta.Package = pkg.Label().String() + packageMeta.BuildFile = pkg.Filename + + for _, sm := range allStatements { + if !writeAll { + if _, ok := filterStmts[sm.Statement]; !ok { + continue + } + } + + code := string(content[sm.Statement.Start:sm.Statement.End]) + var stmtMeta jsonStatementMetadata + stmtMeta.Start = sm.Statement.Start + stmtMeta.End = sm.Statement.End + stmtMeta.Code = code + + if len(sm.Subincludes) > 0 { + stmtMeta.Subincludes = make([]string, len(sm.Subincludes)) + for idx, sub := range sm.Subincludes { + stmtMeta.Subincludes[idx] = sub.String() + } + } + + if len(sm.Files) > 0 { + stmtMeta.Files = sm.Files + } + + if len(sm.Targets) > 0 { + stmtMeta.Targets = make([]jsonTargetMetadata, len(sm.Targets)) + for idx, t := range sm.Targets { + var tMeta jsonTargetMetadata + tMeta.Name = t.String() + + if opts.any() { + if target := state.Graph.Target(t); target != nil { + if opts.IncludeSources && len(target.AllSources()) > 0 { + tMeta.Sources = make([]string, len(target.AllSources())) + for i, src := range target.AllSources() { + tMeta.Sources[i] = src.String() + } + } + if opts.IncludeDeps && len(target.DeclaredDependencies()) > 0 { + tMeta.Dependencies = make([]string, len(target.DeclaredDependencies())) + for i, dep := range target.DeclaredDependencies() { + tMeta.Dependencies[i] = dep.String() + } + } + if opts.IncludeOutputs && len(target.Outputs()) > 0 { + tMeta.Outputs = target.Outputs() + } + } + } + stmtMeta.Targets[idx] = tMeta + } + } + + packageMeta.Statements = append(packageMeta.Statements, stmtMeta) + } + jsonOutput = append(jsonOutput, packageMeta) + } + + enc := json.NewEncoder(os.Stdout) + enc.SetEscapeHTML(false) + enc.SetIndent("", " ") + if err := enc.Encode(jsonOutput); err != nil { + panic(err) + } +} + +// filterStatements calculates if all statements should be written, and computes the targeted filterStmts list. +func filterStatements(pkg *core.Package, labels []core.BuildLabel, includeAll bool) (bool, map[core.BuildStatement]struct{}) { + writeAll := includeAll + if !writeAll { + for _, l := range labels { + if l.IsAllTargets() { + writeAll = true + break + } + } + } + + var filterStmts map[core.BuildStatement]struct{} + if !writeAll && len(labels) > 0 { + filterStmts = map[core.BuildStatement]struct{}{} + for _, target := range labels { + stmt, err := pkg.Metadata.FindStatement(target) + if err == nil && stmt != (core.BuildStatement{}) { + filterStmts[stmt] = struct{}{} + } + } + } + + return writeAll, filterStmts +} From 4959029f5e72e82b527b842c664640d1c672a81a Mon Sep 17 00:00:00 2001 From: DuBento Date: Wed, 1 Jul 2026 18:29:12 +0100 Subject: [PATCH 116/118] progress monitor for export --- src/cli/logging.go | 15 ++++++--- src/export/BUILD | 1 + src/export/export.go | 54 ++++++++++++++++++++++++++----- src/export/trimmed_exporter.go | 4 --- src/output/interactive_display.go | 9 ++++-- 5 files changed, 65 insertions(+), 18 deletions(-) diff --git a/src/cli/logging.go b/src/cli/logging.go index 673ba92f0a..41547e0b22 100644 --- a/src/cli/logging.go +++ b/src/cli/logging.go @@ -13,7 +13,7 @@ import ( "sync" cli "github.com/peterebden/go-cli-init/v5/logging" - "github.com/peterebden/go-deferred-regex" + deferredregex "github.com/peterebden/go-deferred-regex" "golang.org/x/term" "gopkg.in/op/go-logging.v1" @@ -184,9 +184,6 @@ func (backend *LogBackend) calcOutput() []string { ret = append(ret, new...) } } - if len(ret) > 0 { - ret = append(ret, "Messages:") - } return reverse(ret) } @@ -248,6 +245,16 @@ func (backend *LogBackend) Output() []string { return backend.output[:] } +// FlushOutput returns the current set of preformatted log messages and clears them. +func (backend *LogBackend) FlushOutput() []string { + backend.mutex.Lock() + defer backend.mutex.Unlock() + ret := backend.output[:] + backend.output = nil + backend.logMessages.Init() + return ret +} + // Wraps a string across multiple lines. Returned slice is reversed. func (backend *LogBackend) lineWrap(msg string) []string { lines := strings.Split(msg, "\n") diff --git a/src/export/BUILD b/src/export/BUILD index 0070a559b7..5069f9dc85 100644 --- a/src/export/BUILD +++ b/src/export/BUILD @@ -8,6 +8,7 @@ go_library( visibility = ["PUBLIC"], deps = [ "///third_party/go/github.com_please-build_buildtools//build", + "//src/cli", "//src/cli/logging", "//src/core", "//src/fs", diff --git a/src/export/export.go b/src/export/export.go index eaa94794d9..a8780d42a2 100644 --- a/src/export/export.go +++ b/src/export/export.go @@ -4,10 +4,12 @@ package export import ( + "fmt" "os" "path/filepath" "time" + "github.com/thought-machine/please/src/cli" "github.com/thought-machine/please/src/cli/logging" "github.com/thought-machine/please/src/core" "github.com/thought-machine/please/src/fs" @@ -100,22 +102,60 @@ type baseExporter struct { // run specifies the main steps when running an export. func (be *baseExporter) run(targets core.BuildLabels) { - go be.startMonitor() + stop := make(chan struct{}) + go be.startMonitor(stop) + defer close(stop) + be.exportRepoConfig() be.impl.exportPreloaded() be.exportTargets(targets) be.impl.writePackageFiles() } -func (be *baseExporter) startMonitor() { +func (be *baseExporter) startMonitor(stop chan struct{}) { + // this total is meant to provide some general idea of the progress but in reality we might + // visit more than the targets currently in the build graph (those will have to be parsed + // adhoc). + total := len(be.state.Graph.AllTargets()) + startTime := time.Now() + isTerminal := cli.StdErrIsATerminal + + if isTerminal { + cli.CurrentBackend.SetPassthrough(false, 1, false) + defer cli.CurrentBackend.SetPassthrough(true, 1, false) + } + + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + for { - time.Sleep(10 * time.Second) - log.Infof("Number of targets exported: %v", be.targetCounter) + select { + case <-stop: + log.Infof("Exported %d targets (%.1fs)...", be.targetCounter, time.Since(startTime).Seconds()) + return + case <-ticker.C: + done := be.targetCounter + elapsed := time.Since(startTime).Seconds() + stats := be.state.SystemStats() + + if isTerminal { + logs := cli.CurrentBackend.FlushOutput() + if len(logs) > 0 { + // Remove footer line before writing logs. + cli.Fprintf(os.Stderr, "${RESETLN}") + for _, line := range logs { + fmt.Fprintln(os.Stderr, line) + } + } + cli.Fprintf(os.Stderr, "${RESETLN}${BOLD_WHITE}Exporting${RESET} [${BOLD_GREEN}%d${RESET}/${BOLD_GREEN}%d${RESET}, ${BOLD_YELLOW}%.1fs${RESET}]:: CPU: %5.1f%% Mem: %5.1f%% IO: %5.1f%%${RESET}", + done, total, elapsed, stats.CPU.Used, stats.Memory.UsedPercent, stats.CPU.IOWait) + } + } } } -// exportRepoConfig exports the repository's configuration files (e.g., .gitignore, .plzconfig and its -// platform-specific variants) to the target export directory. +// exportRepoConfig exports the repository's configuration files (e.g., .gitignore, .plzconfig and +// its platform-specific variants) to the target export directory. func (be *baseExporter) exportRepoConfig() { files, err := filepath.Glob(".plzconfig*") if err != nil { @@ -154,7 +194,6 @@ func (be *baseExporter) exportTargets(labels core.BuildLabels) { // exportDependencies exports dependencies of a target. func (be *baseExporter) exportDependencies(target *core.BuildTarget) { deps := target.DeclaredDependencies() - log.Debugf("Exporting dependencies of (%v): %v", target.Label, deps) be.exportTargets(deps) } @@ -184,7 +223,6 @@ func (be *baseExporter) exportFiles(paths []string) { if err := fs.RecursiveCopy(p, dest, 0); err != nil { log.Warningf("Error copying file, skipping...: %s", err) } - log.Debugf("Writing exported source file: %s", p) } } diff --git a/src/export/trimmed_exporter.go b/src/export/trimmed_exporter.go index f867eba5e2..bfd23209ef 100644 --- a/src/export/trimmed_exporter.go +++ b/src/export/trimmed_exporter.go @@ -57,8 +57,6 @@ func (e *trimmedExporter) exportTarget(target *core.BuildTarget) { return } - log.Debugf("Exporting target: %v", target.Label) - // Skip export for internal packages if target.Label.PackageName == parse.InternalPackageName { return @@ -127,7 +125,6 @@ func (e *trimmedExporter) exportSubincludes(pkg *core.Package, target core.Build } } - log.Debugf("Subincludes required for %s: %v", target, allSubincludes) e.exportTargets(allSubincludes) } @@ -168,7 +165,6 @@ func (e *trimmedExporter) exportRelatedTargets(pkg *core.Package, target core.Bu if err != nil { log.Fatalf("failed to find related targets for %s: %s", target, err) } - log.Debugf("Exporting targets related to %s: %v", target, relatedTargets) e.exportTargets(relatedTargets) } diff --git a/src/output/interactive_display.go b/src/output/interactive_display.go index 1940c9e590..9224ac44dd 100644 --- a/src/output/interactive_display.go +++ b/src/output/interactive_display.go @@ -100,9 +100,14 @@ func (d *interactiveDisplay) Update(targets []buildingTarget) { d.maxRows, d.maxCols = cli.CurrentBackend.MaxDimensions() d.moveToFirstLine() d.printLines(targets) - for _, line := range cli.CurrentBackend.Output() { - d.printf("${ERASE_AFTER}%s\n", line) + logs := cli.CurrentBackend.Output() + if len(logs) > 0 { + d.printf("${ERASE_AFTER}Messages:\n") d.lines++ + for _, line := range logs { + d.printf("${ERASE_AFTER}%s\n", line) + d.lines++ + } } // Clean out any lines that were visible last time but are not now. if d.lines < d.lastLines { From cae0433bd7355e1720e659ada347c65c0e49a983 Mon Sep 17 00:00:00 2001 From: DuBento Date: Fri, 3 Jul 2026 19:40:22 +0100 Subject: [PATCH 117/118] metadata: drop json printing and optional hidden targets --- src/please.go | 4 +- src/query/metadata.go | 177 ++++++++---------------------------------- 2 files changed, 35 insertions(+), 146 deletions(-) diff --git a/src/please.go b/src/please.go index 9e805f726f..608d1b776e 100644 --- a/src/please.go +++ b/src/please.go @@ -468,7 +468,7 @@ var opts struct { Outputs bool `long:"outputs" short:"o" description:"Include target outputs in visualization"` All bool `long:"all" short:"a" description:"Include all details (sources, deps, and outputs)"` AllStatements bool `long:"all_statements" description:"Print all statements in the BUILD file, including those that didn't generate the specified targets"` - JSON bool `long:"json" short:"j" description:"Output metadata as JSON"` + Hidden bool `long:"hidden" description:"Show hidden targets"` Args struct { Targets []core.BuildLabel `positional-arg-name:"targets" description:"Targets or packages to display metadata for" required:"true"` } `positional-args:"true" required:"true"` @@ -855,7 +855,7 @@ var buildFunctions = map[string]func() int{ IncludeDeps: m.Deps || m.All, IncludeOutputs: m.Outputs || m.All, IncludeAllStatements: m.AllStatements, - FormatJSON: m.JSON, + ShowHidden: m.Hidden, }) return 0 } diff --git a/src/query/metadata.go b/src/query/metadata.go index 8f06f631bd..beb6d9e1d1 100644 --- a/src/query/metadata.go +++ b/src/query/metadata.go @@ -1,7 +1,6 @@ package query import ( - "encoding/json" "fmt" "os" "strings" @@ -16,7 +15,7 @@ type WriteMetadataOpts struct { IncludeDeps bool IncludeOutputs bool IncludeAllStatements bool - FormatJSON bool + ShowHidden bool } // any reports true if any of the options are set to true. @@ -25,28 +24,24 @@ func (wmo WriteMetadataOpts) any() bool { } // Metadata prints out a visualization of the parsed build statement metadata for the given targets. -func Metadata(state *core.BuildState, targets []core.BuildLabel, opts WriteMetadataOpts) { +func Metadata(state *core.BuildState, targets core.BuildLabels, opts WriteMetadataOpts) { if !cli.ShowColouredOutput || !cli.IsATerminal(os.Stdout) { cli.ShowColouredOutput = false } // Group requested targets by their package - packageTargets := map[*core.Package][]core.BuildLabel{} + packageTargets := map[*core.Package]core.BuildLabels{} for _, label := range targets { pkg := state.Graph.PackageOrDie(label) packageTargets[pkg] = append(packageTargets[pkg], label) } - if opts.FormatJSON { - printJSON(state, packageTargets, opts) - } else { - printTerminal(state, packageTargets, opts) - } + printTerminal(state, packageTargets, opts) } // printTerminal formats and draws the metadata as a beautiful colorized terminal tree-box layout. -func printTerminal(state *core.BuildState, packageTargets map[*core.Package][]core.BuildLabel, opts WriteMetadataOpts) { - // itemDetail holds the text to print and its optional ANSI color formatting string +func printTerminal(state *core.BuildState, packageTargets map[*core.Package]core.BuildLabels, opts WriteMetadataOpts) { + // itemDetail holds the text to print and its optional color formatting string type itemDetail struct { text string color string @@ -102,7 +97,7 @@ func printTerminal(state *core.BuildState, packageTargets map[*core.Package][]co return res } - for pkg, label := range packageTargets { + for pkg, labels := range packageTargets { fmt.Printf("=== Package: %s (File: %s) ===\n", pkg.Label(), pkg.Filename) content, err := os.ReadFile(pkg.Filename) @@ -112,10 +107,13 @@ func printTerminal(state *core.BuildState, packageTargets map[*core.Package][]co } allStatements := pkg.Metadata.Statements() - writeAll, filterStmts := filterStatements(pkg, label, opts.IncludeAllStatements) + var filterStmts map[core.BuildStatement]struct{} + if !opts.IncludeAllStatements { + filterStmts = filterStatements(pkg, labels) + } for _, sm := range allStatements { - if !writeAll { + if filterStmts != nil { if _, ok := filterStmts[sm.Statement]; !ok { continue } @@ -130,9 +128,19 @@ func printTerminal(state *core.BuildState, packageTargets map[*core.Package][]co cli.Fprintf(os.Stdout, " %s\n", line) } + targets := sm.Targets + if !opts.ShowHidden { + targets = make(core.BuildLabels, 0, len(sm.Targets)) + for _, t := range sm.Targets { + if !t.IsHidden() { + targets = append(targets, t) + } + } + } + hasSubincludes := len(sm.Subincludes) > 0 hasFiles := len(sm.Files) > 0 - hasTargets := len(sm.Targets) > 0 + hasTargets := len(targets) > 0 var lastSection string if hasTargets { @@ -161,10 +169,10 @@ func printTerminal(state *core.BuildState, packageTargets map[*core.Package][]co } cli.Fprintf(os.Stdout, "%s%s ${CYAN}Generated Targets:${RESET}\n", basePrefix, branch) - for i, t := range sm.Targets { + for i, t := range targets { targetBranch := "├──" targetChildPrefix := childPrefix + "│ " - if i == len(sm.Targets)-1 { + if i == len(targets)-1 { targetBranch = "└──" targetChildPrefix = childPrefix + " " } @@ -200,133 +208,14 @@ func printTerminal(state *core.BuildState, packageTargets map[*core.Package][]co } } -type jsonTargetMetadata struct { - Name string `json:"name"` - Sources []string `json:"sources,omitempty"` - Dependencies []string `json:"dependencies,omitempty"` - Outputs []string `json:"outputs,omitempty"` -} - -type jsonStatementMetadata struct { - Start int `json:"start"` - End int `json:"end"` - Code string `json:"code"` - Subincludes []string `json:"subincludes,omitempty"` - Files []string `json:"files,omitempty"` - Targets []jsonTargetMetadata `json:"targets,omitempty"` -} - -type jsonPackageMetadata struct { - Package string `json:"package"` - BuildFile string `json:"build_file"` - Statements []jsonStatementMetadata `json:"statements"` -} - -// printJSON formats and serializes the metadata to stdout as indented JSON. -func printJSON(state *core.BuildState, packageTargets map[*core.Package][]core.BuildLabel, opts WriteMetadataOpts) { - var jsonOutput []jsonPackageMetadata - - for pkg, label := range packageTargets { - content, err := os.ReadFile(pkg.Filename) - if err != nil { - continue - } - - allStatements := pkg.Metadata.Statements() - writeAll, filterStmts := filterStatements(pkg, label, opts.IncludeAllStatements) - - var packageMeta jsonPackageMetadata - packageMeta.Package = pkg.Label().String() - packageMeta.BuildFile = pkg.Filename - - for _, sm := range allStatements { - if !writeAll { - if _, ok := filterStmts[sm.Statement]; !ok { - continue - } - } - - code := string(content[sm.Statement.Start:sm.Statement.End]) - var stmtMeta jsonStatementMetadata - stmtMeta.Start = sm.Statement.Start - stmtMeta.End = sm.Statement.End - stmtMeta.Code = code - - if len(sm.Subincludes) > 0 { - stmtMeta.Subincludes = make([]string, len(sm.Subincludes)) - for idx, sub := range sm.Subincludes { - stmtMeta.Subincludes[idx] = sub.String() - } - } - - if len(sm.Files) > 0 { - stmtMeta.Files = sm.Files - } - - if len(sm.Targets) > 0 { - stmtMeta.Targets = make([]jsonTargetMetadata, len(sm.Targets)) - for idx, t := range sm.Targets { - var tMeta jsonTargetMetadata - tMeta.Name = t.String() - - if opts.any() { - if target := state.Graph.Target(t); target != nil { - if opts.IncludeSources && len(target.AllSources()) > 0 { - tMeta.Sources = make([]string, len(target.AllSources())) - for i, src := range target.AllSources() { - tMeta.Sources[i] = src.String() - } - } - if opts.IncludeDeps && len(target.DeclaredDependencies()) > 0 { - tMeta.Dependencies = make([]string, len(target.DeclaredDependencies())) - for i, dep := range target.DeclaredDependencies() { - tMeta.Dependencies[i] = dep.String() - } - } - if opts.IncludeOutputs && len(target.Outputs()) > 0 { - tMeta.Outputs = target.Outputs() - } - } - } - stmtMeta.Targets[idx] = tMeta - } - } - - packageMeta.Statements = append(packageMeta.Statements, stmtMeta) - } - jsonOutput = append(jsonOutput, packageMeta) - } - - enc := json.NewEncoder(os.Stdout) - enc.SetEscapeHTML(false) - enc.SetIndent("", " ") - if err := enc.Encode(jsonOutput); err != nil { - panic(err) - } -} - -// filterStatements calculates if all statements should be written, and computes the targeted filterStmts list. -func filterStatements(pkg *core.Package, labels []core.BuildLabel, includeAll bool) (bool, map[core.BuildStatement]struct{}) { - writeAll := includeAll - if !writeAll { - for _, l := range labels { - if l.IsAllTargets() { - writeAll = true - break - } +// filterStatements calculates if all statements should be written. +func filterStatements(pkg *core.Package, labels core.BuildLabels) map[core.BuildStatement]struct{} { + filterStmts := map[core.BuildStatement]struct{}{} + for _, target := range labels { + stmt, err := pkg.Metadata.FindStatement(target) + if err == nil && stmt != (core.BuildStatement{}) { + filterStmts[stmt] = struct{}{} } } - - var filterStmts map[core.BuildStatement]struct{} - if !writeAll && len(labels) > 0 { - filterStmts = map[core.BuildStatement]struct{}{} - for _, target := range labels { - stmt, err := pkg.Metadata.FindStatement(target) - if err == nil && stmt != (core.BuildStatement{}) { - filterStmts[stmt] = struct{}{} - } - } - } - - return writeAll, filterStmts + return filterStmts } From 75390e8461b30a077199f00af6dc77b90f58847a Mon Sep 17 00:00:00 2001 From: DuBento Date: Fri, 3 Jul 2026 22:12:41 +0100 Subject: [PATCH 118/118] maintain subinclude label order --- src/core/package_metadata.go | 40 ++++++++------- src/core/state.go | 4 +- src/export/trimmer.go | 18 ++++--- src/parse/asp/builtins.go | 49 ++++++++++++------- src/parse/asp/interpreter.go | 23 +++++---- src/please.go | 2 +- src/plz/plz.go | 2 +- .../source_repo/BUILD_FILE | 8 +-- 8 files changed, 83 insertions(+), 63 deletions(-) diff --git a/src/core/package_metadata.go b/src/core/package_metadata.go index ee4ca36a61..a0f9c8584b 100644 --- a/src/core/package_metadata.go +++ b/src/core/package_metadata.go @@ -67,9 +67,9 @@ type PackageMetadata interface { // RegisterStatementTarget records that the given build target was created as a result of the // given statement being executed. This should only be called for statements in BUILD files. RegisterStatementTarget(target BuildLabel, stmtProvider BuildStatementProvider) - // RegisterSubincludeStatement records that the given subinclude statement (provided by stmtProvider) - // includes the given build label. This should only be called for statements in BUILD files. - RegisterSubincludeStatement(label BuildLabel, stmtProvider BuildStatementProvider) + // RegisterSubincludeStatement records the given build statement (provided by stmtProvider) + // as being a subincluded statement. This should only be called for statements in BUILD files. + RegisterSubincludeStatement(stmtProvider BuildStatementProvider) // FindStatement returns the build statement that was responsible for generating the given target. // Returns an error if the target was not found in the recorded metadata. FindStatement(target BuildLabel) (BuildStatement, error) @@ -124,9 +124,9 @@ type packageMetadataImpl struct { // stmtToRequiredFiles tracks the file paths that were required during interpretation of the // statement (e.g. glob) stmtToRequiredFiles *cmap.Map[BuildStatement, []string] - // labelsPerSubincludeStmt maps a subinclude statement (identified by its position - // in the BUILD file) to the labels it explicitly subincludes. - labelsPerSubincludeStmt *cmap.Map[BuildStatement, BuildLabels] + // subincludeStmts tracks which build statements (identified by its position in the BUILD file) + // are subinclude calls. + subincludeStmts *cmap.Map[BuildStatement, struct{}] } func newPackageMetadata() PackageMetadata { @@ -135,7 +135,7 @@ func newPackageMetadata() PackageMetadata { targetToStmt: cmap.New[BuildLabel, BuildStatement](cmap.SmallShardCount, HashBuildLabel), stmtToRequiredSubincludes: cmap.New[BuildStatement, BuildLabels](cmap.SmallShardCount, hashBuildStatement), stmtToRequiredFiles: cmap.New[BuildStatement, []string](cmap.SmallShardCount, hashBuildStatement), - labelsPerSubincludeStmt: cmap.New[BuildStatement, BuildLabels](cmap.SmallShardCount, hashBuildStatement), + subincludeStmts: cmap.New[BuildStatement, struct{}](cmap.SmallShardCount, hashBuildStatement), } } @@ -186,10 +186,9 @@ func (m *packageMetadataImpl) RegisterStatementTarget(target BuildLabel, stmtPro } // RegisterSubincludeStatement implements [PackageMetadata]. -func (m *packageMetadataImpl) RegisterSubincludeStatement(label BuildLabel, stmtProvider BuildStatementProvider) { +func (m *packageMetadataImpl) RegisterSubincludeStatement(stmtProvider BuildStatementProvider) { stmt := stmtProvider() - existing := m.labelsPerSubincludeStmt.Get(stmt) - m.labelsPerSubincludeStmt.Set(stmt, append(existing, label)) + m.subincludeStmts.Set(stmt, struct{}{}) } // FindStatement implements [PackageMetadata]. @@ -240,17 +239,17 @@ func (m *packageMetadataImpl) FindRelatedTargets(target BuildLabel) (BuildLabels // FindPackageFileRequirements implements [PackageMetadata]. func (m *packageMetadataImpl) FindPackageFileRequirements() (BuildLabels, []string) { - subincludesSet := LabelSet{} + requiredSet := LabelSet{} m.stmtToRequiredSubincludes.Range(func(stmt BuildStatement, labels BuildLabels) { // Look for build statements that are not registered in the statement to targets mapping, - // and so are unrelated to targets or are subincludes() statements. - if len(m.stmtToTarget.Get(stmt)) == 0 && len(m.labelsPerSubincludeStmt.Get(stmt)) == 0 { + // and so are unrelated to targets and are not subinclude statements. + if len(m.stmtToTarget.Get(stmt)) == 0 && !m.subincludeStmts.Contains(stmt) { for _, label := range labels { - subincludesSet.Add(label) + requiredSet.Add(label) } } }) - subincludes := slices.Collect(maps.Keys(subincludesSet)) + origins := slices.Collect(maps.Keys(requiredSet)) filesSet := map[string]struct{}{} m.stmtToRequiredFiles.Range(func(stmt BuildStatement, files []string) { @@ -260,12 +259,17 @@ func (m *packageMetadataImpl) FindPackageFileRequirements() (BuildLabels, []stri } }) files := slices.Collect(maps.Keys(filesSet)) - return subincludes, files + return origins, files } // GetSubincludedLabels implements [PackageMetadata]. func (m *packageMetadataImpl) GetSubincludedLabels(stmt BuildStatement) BuildLabels { - return m.labelsPerSubincludeStmt.Get(stmt) + if !m.subincludeStmts.Contains(stmt) { + return nil + } + // After determining that this is a subincludes statement we can return the required subincludes + // registered in the general statement mapping. + return m.stmtToRequiredSubincludes.Get(stmt) } // IsInterpretedStatement implements [PackageMetadata]. @@ -328,7 +332,7 @@ func (n *noopPackageMetadata) RegisterStatementTarget(target BuildLabel, stmtPro } // RegisterSubincludeStatement implements [PackageMetadata]. -func (n *noopPackageMetadata) RegisterSubincludeStatement(label BuildLabel, stmtProvider BuildStatementProvider) { +func (n *noopPackageMetadata) RegisterSubincludeStatement(stmtProvider BuildStatementProvider) { } // FindStatement implements [PackageMetadata]. diff --git a/src/core/state.go b/src/core/state.go index a693f854c6..f0c12aa228 100644 --- a/src/core/state.go +++ b/src/core/state.go @@ -451,8 +451,8 @@ func (state *BuildState) CloseResults() { } } -// CleanUp cleans up and shuts down the build state. -func (state *BuildState) CleanUp() { +// Cleanup cleans up and shuts down the build state. +func (state *BuildState) Cleanup() { state.CloseResults() if state.WaitForDisplay != nil { diff --git a/src/export/trimmer.go b/src/export/trimmer.go index 07836e4a8c..1890748c4f 100644 --- a/src/export/trimmer.go +++ b/src/export/trimmer.go @@ -2,7 +2,6 @@ package export import ( "slices" - "sort" "github.com/please-build/buildtools/build" @@ -156,7 +155,7 @@ func (t *trimmer) trimFor(stmt *asp.Statement) bool { func (t *trimmer) trimSubinclude(stmt *asp.Statement) { bStmt := asp.NewBuildStatement(stmt) stmtLabels := t.pkg.Metadata.GetSubincludedLabels(bStmt) - subStmt := t.minimalSubincludeStatement(stmtLabels) + subStmt := t.minimalSubincludeStatement(stmt, stmtLabels) t.write([]byte(subStmt)) } @@ -187,19 +186,22 @@ func (t *trimmer) anyExported(labels core.BuildLabels) bool { } // minimalSubincludeStatement generates a subinclude statement containing only the required labels. -func (t *trimmer) minimalSubincludeStatement(available core.BuildLabels) string { +func (t *trimmer) minimalSubincludeStatement(stmt *asp.Statement, available core.BuildLabels) string { var filteredLabels core.BuildLabels - for _, required := range t.exporter.requiredSubincludes[t.pkg.Label()] { - if slices.Contains(available, required) { - filteredLabels = append(filteredLabels, required) + required := t.exporter.requiredSubincludes[t.pkg.Label()] + for _, label := range available { + if slices.Contains(required, label) { + filteredLabels = append(filteredLabels, label) } } if len(filteredLabels) == 0 { return "" } - - sort.Sort(filteredLabels) + if len(available) == len(filteredLabels) { + // If all labels are required just keep the statement as is. + return string(t.origin[stmt.Pos:stmt.EndPos]) + } call := &build.CallExpr{ X: &build.Ident{Name: "subinclude"}, diff --git a/src/parse/asp/builtins.go b/src/parse/asp/builtins.go index ae3f5c9d62..7476ddd154 100644 --- a/src/parse/asp/builtins.go +++ b/src/parse/asp/builtins.go @@ -333,24 +333,13 @@ func subinclude(s *scope, args []pyObject) pyObject { if s.contextPackage() == nil { s.Error("cannot subinclude from this scope") } - var si []string - for _, arg := range args { - if l, ok := arg.(pyList); ok { - for _, e := range l { - if l, ok := e.(pyString); ok { - si = append(si, string(l)) - } else { - s.Error("cannot subinclude type %s", e.Type()) - } - } - } else if l, ok := arg.(pyString); ok { - si = append(si, string(l)) - } else { - s.Error("cannot subinclude type %s", arg.Type()) - } + stringArgs, err := subincludeArgs(args) + if err != nil { + s.Error("%s", err.Error()) } - var labels = make(core.BuildLabels, 0, len(si)) - for _, arg := range si { + + var labels = make(core.BuildLabels, 0, len(stringArgs)) + for _, arg := range stringArgs { label, annotation := core.SplitLabelAnnotation(arg) t := subincludeTarget(s, s.parseLabelInContextPkg(label)) s.Assert(s.contextPackage().Label().CanSee(s.state, t), "Target %s isn't visible to be subincluded into %s", t.Label, s.contextPackage().Label()) @@ -374,9 +363,34 @@ func subinclude(s *scope, args []pyObject) pyObject { labels = append(labels, t.Label) } s.metadataRegisterSubincludes(labels) + if s.pkg != nil { + s.pkg.Metadata.RegisterSubincludeStatement(s.CurrentBuildStatement()) + } return None } +// subincludeArgs extracts and validates the arguments from a subinclude() call. Acceptable values +// are strings and lists of strings. +func subincludeArgs(args []pyObject) ([]string, error) { + var si []string + for _, arg := range args { + if l, ok := arg.(pyList); ok { + for _, e := range l { + if l, ok := e.(pyString); ok { + si = append(si, string(l)) + } else { + return nil, fmt.Errorf("cannot subinclude type %s", e.Type()) + } + } + } else if l, ok := arg.(pyString); ok { + si = append(si, string(l)) + } else { + return nil, fmt.Errorf("cannot subinclude type %s", arg.Type()) + } + } + return si, nil +} + // subincludeTarget returns the target for a subinclude() call to a label. // It blocks until the target exists and is built. func subincludeTarget(s *scope, l core.BuildLabel) *core.BuildTarget { @@ -421,7 +435,6 @@ func subincludeTarget(s *scope, l core.BuildLabel) *core.BuildTarget { t = s.WaitForSubincludedTarget(l, pkgLabel) if s.pkg != nil { s.pkg.RegisterSubinclude(l) - s.pkg.Metadata.RegisterSubincludeStatement(l, s.CurrentBuildStatement()) } else if s.subincludeLabel != nil { // If this is nil, that indicates a preloadedSubinclude s.state.Graph.RegisterTransitiveSubinclude(*s.subincludeLabel, l) } diff --git a/src/parse/asp/interpreter.go b/src/parse/asp/interpreter.go index 88c00b7e32..8919149942 100644 --- a/src/parse/asp/interpreter.go +++ b/src/parse/asp/interpreter.go @@ -4,14 +4,12 @@ import ( "context" "fmt" "iter" - "maps" "path" "path/filepath" "reflect" "regexp" "runtime/debug" "runtime/pprof" - "slices" "strings" "sync" @@ -27,7 +25,7 @@ type interpreter struct { parser *Parser subincludes *cmap.ErrMap[string, pyDict] asts *cmap.ErrMap[string, []*Statement] - // preloaded is a set to register all preloaded objects. + // preloaded is a set to register all preloaded symbols (variables and function names). preloaded *cmap.Map[string, struct{}] configs map[*core.BuildState]*pyConfig @@ -1297,14 +1295,21 @@ func (m *trackingScopeMetadata) registerBuildStatement(pkg *core.Package, stmt * return } - set := core.LabelSet{} - for _, v := range m.symbolStack { - set.Add(v.origin) - } + var deps core.BuildLabels + seen := make(map[core.BuildLabel]struct{}) for _, l := range m.subincludesStack { - set.Add(l) + if _, ok := seen[l]; !ok { + deps = append(deps, l) + seen[l] = struct{}{} + } + } + for _, v := range m.symbolStack { + l := v.origin + if _, ok := seen[l]; !ok { + deps = append(deps, l) + seen[l] = struct{}{} + } } - deps := slices.Collect(maps.Keys(set)) pkg.Metadata.RegisterStatement(NewBuildStatement(stmt), deps, m.fileStack) } diff --git a/src/please.go b/src/please.go index 608d1b776e..d741104c7c 100644 --- a/src/please.go +++ b/src/please.go @@ -780,7 +780,7 @@ var buildFunctions = map[string]func() int{ "export": func() int { success, state := runBuild(opts.Export.Args.Targets, buildOpts{ParseMetadata: true, KeepParserRunning: true}) // Required cleanup due to running parser in background - defer state.CleanUp() + defer state.Cleanup() if success { export.Repo(state, opts.Export.Output, opts.Export.NoTrim, state.ExpandOriginalLabels()) diff --git a/src/plz/plz.go b/src/plz/plz.go index a17a27fe2d..6114ec1c56 100644 --- a/src/plz/plz.go +++ b/src/plz/plz.go @@ -123,7 +123,7 @@ func Run(targets, preTargets []core.BuildLabel, state *core.BuildState, config * // The last task should call state.Close() and close the queue channels. wg.Wait() reportResults(state, config) - state.CleanUp() + state.Cleanup() } // reportResults reports on metrics and results at the end of the build. diff --git a/test/export/test_subinclude_trimming/source_repo/BUILD_FILE b/test/export/test_subinclude_trimming/source_repo/BUILD_FILE index 3de8e6482e..2632f80601 100644 --- a/test/export/test_subinclude_trimming/source_repo/BUILD_FILE +++ b/test/export/test_subinclude_trimming/source_repo/BUILD_FILE @@ -34,9 +34,7 @@ genrule( cmd = f'echo "{unused_var}" > $OUT', ) -subinclude( - "//build_defs:var3_build_def", -) +subinclude("//build_defs:var3_build_def") message = "Testing {service} on {env}".format( env = "production", @@ -49,9 +47,7 @@ genrule( cmd = f'echo "{message}" > $OUT', ) -subinclude( - "//build_defs:versions_build_def", -) +subinclude("//build_defs:versions_build_def") for version, name in VERSIONS.items(): genrule(