diff --git a/docs/commands.html b/docs/commands.html
index b6b2fcfa73..9ebc62ba54 100644
--- a/docs/commands.html
+++ b/docs/commands.html
@@ -804,6 +804,11 @@
whatoutputs: Prints out target(s) responsible for outputting provided file(s)
+
+
+ metadata: Prints out a structured, tree-like visualization of parsed build statement metadata (required subincludes, files, and generated targets) of a package.
+
+
@@ -1035,15 +1040,6 @@
Disables trimming unnecessary targets from exported packages. Normally targets in exported packages that
aren't dependencies of the originally exported targets are removed.
-
- This trimming syntax based, so doesn't always work depending on how the build definition is authored. Passing
- this flag will disable this feature, avoiding cases where these rules will be erroneously trimmed.
-
-
- To make sure a rule works without this flag, the rule must follow the naming convention, whereby children of
- :name follow the format :_name#{some-tag}. This is the
- format tag(name, tag) would produce.
-
diff --git a/src/BUILD.plz b/src/BUILD.plz
index c0eaec7e7b..cb6269400e 100644
--- a/src/BUILD.plz
+++ b/src/BUILD.plz
@@ -7,11 +7,12 @@ go_binary(
"github.com/thought-machine/please/src/version.PleaseVersion": VERSION,
},
visibility = ["PUBLIC"],
+ strip = False,
deps = [
"///third_party/go/github.com_thought-machine_go-flags//:go-flags",
"///third_party/go/go.uber.org_automaxprocs//maxprocs",
- "//src/audit",
"//src/assets",
+ "//src/audit",
"//src/build",
"//src/cache",
"//src/clean",
@@ -29,7 +30,6 @@ go_binary(
"//src/help",
"//src/metrics",
"//src/output",
- "//src/parse",
"//src/plz",
"//src/plzinit",
"//src/process",
diff --git a/src/cli/logging.go b/src/cli/logging.go
index 673ba92f0a..41547e0b22 100644
--- a/src/cli/logging.go
+++ b/src/cli/logging.go
@@ -13,7 +13,7 @@ import (
"sync"
cli "github.com/peterebden/go-cli-init/v5/logging"
- "github.com/peterebden/go-deferred-regex"
+ deferredregex "github.com/peterebden/go-deferred-regex"
"golang.org/x/term"
"gopkg.in/op/go-logging.v1"
@@ -184,9 +184,6 @@ func (backend *LogBackend) calcOutput() []string {
ret = append(ret, new...)
}
}
- if len(ret) > 0 {
- ret = append(ret, "Messages:")
- }
return reverse(ret)
}
@@ -248,6 +245,16 @@ func (backend *LogBackend) Output() []string {
return backend.output[:]
}
+// FlushOutput returns the current set of preformatted log messages and clears them.
+func (backend *LogBackend) FlushOutput() []string {
+ backend.mutex.Lock()
+ defer backend.mutex.Unlock()
+ ret := backend.output[:]
+ backend.output = nil
+ backend.logMessages.Init()
+ return ret
+}
+
// Wraps a string across multiple lines. Returned slice is reversed.
func (backend *LogBackend) lineWrap(msg string) []string {
lines := strings.Split(msg, "\n")
diff --git a/src/core/build_label.go b/src/core/build_label.go
index 779e7302e0..d5f402ba64 100644
--- a/src/core/build_label.go
+++ b/src/core/build_label.go
@@ -473,7 +473,8 @@ func subrepoLabel(subrepoName, arch string) BuildLabel {
return BuildLabel{Name: subrepoName, Subrepo: arch}
}
-func hashBuildLabel(l BuildLabel) uint64 {
+// HashBuildLabel calculates an hash of Build Label, suitable to use for map indexing.
+func HashBuildLabel(l BuildLabel) uint64 {
return cmap.XXHashes(l.Subrepo, l.PackageName, l.Name)
}
@@ -622,3 +623,15 @@ func (slice BuildLabels) String() string {
}
return strings.Join(s, ", ")
}
+
+// LabelSet defines a set of labels implemented using a map.
+type LabelSet map[BuildLabel]struct{}
+
+func (ls LabelSet) Add(l BuildLabel) {
+ ls[l] = struct{}{}
+}
+
+func (ls LabelSet) Contains(l BuildLabel) bool {
+ _, ok := ls[l]
+ return ok
+}
diff --git a/src/core/build_target.go b/src/core/build_target.go
index 3268e55f03..2581da5b50 100644
--- a/src/core/build_target.go
+++ b/src/core/build_target.go
@@ -1471,7 +1471,7 @@ func (target *BuildTarget) AllTestTools() []BuildInput {
if target.Test.namedTools == nil {
return target.Test.tools
}
- return target.allBuildInputs(target.Test.tools, target.Test.namedTools)
+ return target.selectBuildInputs(target.Test.tools, target.Test.namedTools)
}
// NamedTestTools returns all named test tools
@@ -1487,7 +1487,7 @@ func (target *BuildTarget) AllDebugTools() []BuildInput {
if target.Debug.namedTools == nil {
return target.Debug.tools
}
- return target.allBuildInputs(target.Debug.tools, target.Debug.namedTools)
+ return target.selectBuildInputs(target.Debug.tools, target.Debug.namedTools)
}
// AddDatum adds a new item of data to the target.
@@ -1642,21 +1642,39 @@ func (target *BuildTarget) getCommand(state *BuildState, commands map[string]str
return highestCommand
}
+// AllBuildInputs returns all the inputs for this target.
+func (target *BuildTarget) AllBuildInputs() []BuildInput {
+ srcs := target.AllSources()
+ data := target.AllData()
+ tools := target.AllTools()
+
+ size := len(srcs) + len(data) + len(tools)
+ inputs := make([]BuildInput, 0, size)
+ inputs = append(inputs, srcs...)
+ inputs = append(inputs, data...)
+ inputs = append(inputs, tools...)
+ return inputs
+}
+
// AllSources returns all the sources of this rule.
func (target *BuildTarget) AllSources() []BuildInput {
if target.NamedSources == nil {
return target.Sources
}
- return target.allBuildInputs(target.Sources, target.NamedSources)
+ return target.selectBuildInputs(target.Sources, target.NamedSources)
}
-func (target *BuildTarget) allBuildInputs(unnamed []BuildInput, named map[string][]BuildInput) []BuildInput {
- ret := unnamed
+func (target *BuildTarget) selectBuildInputs(unnamed []BuildInput, named map[string][]BuildInput) []BuildInput {
keys := make([]string, 0, len(named))
- for k := range named {
+ size := len(unnamed)
+ for k, vals := range named {
keys = append(keys, k)
+ size += len(vals)
}
sort.Strings(keys)
+
+ ret := make([]BuildInput, 0, size)
+ ret = append(ret, unnamed...)
for _, k := range keys {
ret = append(ret, named[k]...)
}
@@ -1708,7 +1726,7 @@ func (target *BuildTarget) AllData() []BuildInput {
return target.Data
}
- return target.allBuildInputs(target.Data, target.NamedData)
+ return target.selectBuildInputs(target.Data, target.NamedData)
}
// AllDebugData returns all the data for debugging this rule.
@@ -1719,7 +1737,7 @@ func (target *BuildTarget) AllDebugData() []BuildInput {
if target.Debug.namedData == nil {
return target.Debug.data
}
- return target.allBuildInputs(target.Debug.data, target.Debug.namedData)
+ return target.selectBuildInputs(target.Debug.data, target.Debug.namedData)
}
// DebugData returns unnamed data for debugging this rule.
@@ -1743,7 +1761,7 @@ func (target *BuildTarget) AllTools() []BuildInput {
if target.namedTools == nil {
return target.Tools
}
- return target.allBuildInputs(target.Tools, target.namedTools)
+ return target.selectBuildInputs(target.Tools, target.namedTools)
}
// ToolNames returns an ordered list of tool names.
diff --git a/src/core/graph.go b/src/core/graph.go
index f3f9705e9e..246d0b01d1 100644
--- a/src/core/graph.go
+++ b/src/core/graph.go
@@ -13,17 +13,6 @@ import (
"github.com/thought-machine/please/src/cmap"
)
-type labelSet map[BuildLabel]struct{}
-
-func (ls labelSet) add(l BuildLabel) {
- ls[l] = struct{}{}
-}
-
-func (ls labelSet) contains(l BuildLabel) bool {
- _, ok := ls[l]
- return ok
-}
-
// A BuildGraph contains all the loaded targets and packages and maintains their
// relationships, especially reverse dependencies which are calculated here.
type BuildGraph struct {
@@ -34,8 +23,8 @@ type BuildGraph struct {
// Registered subrepos, as a map of their name to their root.
subrepos *cmap.Map[string, *Subrepo]
// Subincludes that are subincluded by other subincludes
- subincludeSubincludes map[BuildLabel]labelSet
- // Use a mutex as a labelSet isn't atomic. We need to guard against inserting as well as mutating the value.
+ subincludeSubincludes map[BuildLabel]LabelSet
+ // Use a mutex as a LabelSet isn't atomic. We need to guard against inserting as well as mutating the value.
subincMux sync.Mutex
}
@@ -165,10 +154,10 @@ func (graph *BuildGraph) PackageMap() map[string]*Package {
// NewGraph constructs and returns a new BuildGraph.
func NewGraph() *BuildGraph {
g := &BuildGraph{
- targets: cmap.New[BuildLabel, *BuildTarget](cmap.DefaultShardCount, hashBuildLabel),
+ targets: cmap.New[BuildLabel, *BuildTarget](cmap.DefaultShardCount, HashBuildLabel),
packages: cmap.New[packageKey, *Package](cmap.DefaultShardCount, hashPackageKey),
subrepos: cmap.New[string, *Subrepo](cmap.SmallShardCount, cmap.XXHash),
- subincludeSubincludes: map[BuildLabel]labelSet{},
+ subincludeSubincludes: map[BuildLabel]LabelSet{},
}
return g
}
@@ -188,7 +177,7 @@ func (graph *BuildGraph) TransitiveSubincludes(l BuildLabel) []BuildLabel {
graph.subincMux.Lock()
defer graph.subincMux.Unlock()
- incs := labelSet{}
+ incs := LabelSet{}
graph.findTransitiveSubincludes(l, incs)
ls := slices.Collect(maps.Keys(incs))
@@ -196,11 +185,11 @@ func (graph *BuildGraph) TransitiveSubincludes(l BuildLabel) []BuildLabel {
return ls
}
-func (graph *BuildGraph) findTransitiveSubincludes(label BuildLabel, includes labelSet) {
- if includes.contains(label) {
+func (graph *BuildGraph) findTransitiveSubincludes(label BuildLabel, includes LabelSet) {
+ if includes.Contains(label) {
return
}
- includes.add(label)
+ includes.Add(label)
for l := range graph.subincludeSubincludes[label] {
graph.findTransitiveSubincludes(l, includes)
}
@@ -212,8 +201,8 @@ func (graph *BuildGraph) RegisterTransitiveSubinclude(from, to BuildLabel) {
incs, ok := graph.subincludeSubincludes[from]
if !ok {
- incs = labelSet{}
+ incs = LabelSet{}
graph.subincludeSubincludes[from] = incs
}
- incs.add(to)
+ incs.Add(to)
}
diff --git a/src/core/package.go b/src/core/package.go
index 377022a482..c7720b7add 100644
--- a/src/core/package.go
+++ b/src/core/package.go
@@ -34,23 +34,46 @@ type Package struct {
targets map[string]*BuildTarget
// Set of output files from rules.
Outputs map[string]*BuildTarget
+ // Includes metadata from parsing the package BUILD file.
+ Metadata PackageMetadata
// Protects access to above
mutex sync.RWMutex
}
-// NewPackage constructs a new package with the given name.
-func NewPackage(name string) *Package {
- return NewPackageSubrepo(name, "")
+// PackageOptions is a functional option type for configuring a new Package.
+type PackageOptions func(*Package)
+
+// WithPackageSubrepo returns a PackageOptions that sets the subrepo name for a new Package.
+func WithPackageSubrepo(name string) PackageOptions {
+ return func(p *Package) {
+ p.SubrepoName = name
+ }
}
-// NewPackageSubrepo constructs a new package with the given name and subrepo.
-func NewPackageSubrepo(name, subrepo string) *Package {
- return &Package{
- Name: name,
- SubrepoName: subrepo,
- targets: map[string]*BuildTarget{},
- Outputs: map[string]*BuildTarget{},
+// WithPackageMetadata returns a PackageOptions that enables tracking of
+// metadata (like statement positions and subinclude mappings) for the Package.
+// This is required for features like 'plz export'.
+func WithPackageMetadata() PackageOptions {
+ return func(p *Package) {
+ p.Metadata = newPackageMetadata()
+ }
+}
+
+// NewPackage constructs a new package with the given name, and enables additional features
+// given the PackageOptions provided.
+func NewPackage(name string, options ...PackageOptions) *Package {
+ pkg := &Package{
+ Name: name,
+ targets: map[string]*BuildTarget{},
+ Outputs: map[string]*BuildTarget{},
+ // Defaults to noop to avoid storing metadata for most operations
+ Metadata: newNoopPackageMetadata(),
+ }
+
+ for _, option := range options {
+ option(pkg)
}
+ return pkg
}
// Target returns the target with the given name, or nil if this package doesn't have one.
@@ -105,11 +128,11 @@ func (pkg *Package) RegisterSubinclude(label BuildLabel) {
// AllSubincludes returns the full set of subincludes needed for this package, including transitive subincludes
func (pkg *Package) AllSubincludes(graph *BuildGraph) []BuildLabel {
- includes := make(labelSet, len(pkg.Subincludes))
+ includes := make(LabelSet, len(pkg.Subincludes))
for _, s := range pkg.Subincludes {
for _, inc := range append(graph.TransitiveSubincludes(s), s) {
- includes.add(inc)
+ includes.Add(inc)
}
}
diff --git a/src/core/package_metadata.go b/src/core/package_metadata.go
new file mode 100644
index 0000000000..a0f9c8584b
--- /dev/null
+++ b/src/core/package_metadata.go
@@ -0,0 +1,384 @@
+package core
+
+import (
+ "cmp"
+ "fmt"
+ "maps"
+ "slices"
+
+ "github.com/thought-machine/please/src/cmap"
+)
+
+// BuildStatement represents the start and end byte positions of a parsed statement in a BUILD file.
+type BuildStatement struct {
+ Start, End int
+}
+
+// Len returns the byte length of the build statement.
+func (bs *BuildStatement) Len() int64 {
+ return int64(bs.End - bs.Start)
+}
+
+// StartPos returns the starting byte position of the build statement.
+func (bs *BuildStatement) StartPos() int64 {
+ return int64(bs.Start)
+}
+
+// hashBuildStatement mixes the Start and End byte coordinates to produce a unique 64-bit hash.
+func hashBuildStatement(stmt BuildStatement) uint64 {
+ return uint64(stmt.Start)<<32 | uint64(stmt.End)
+}
+
+// BuildStatements is a slice of BuildStatement that implements sort.Interface.
+type BuildStatements []BuildStatement
+
+func (s BuildStatements) Len() int { return len(s) }
+func (s BuildStatements) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
+func (s BuildStatements) Less(i, j int) bool { return s[i].StartPos() < s[j].StartPos() }
+
+// BuildStatementProvider defines a closure that generates new build statements.
+// It is used as an argument in PackageMetadata methods to defer evaluation, avoiding
+// unnecessary computation when using the no-op implementation.
+type BuildStatementProvider func() BuildStatement
+
+// SubincludesLabelProvider defines a closure that generates labels for a subinclude statement.
+// It is used as an argument in PackageMetadata methods to defer evaluation, avoiding
+// unnecessary computation when using the no-op implementation.
+type SubincludesLabelProvider func() BuildLabels
+
+// StatementMetadata represents all parsed metadata associated with a single statement.
+type StatementMetadata struct {
+ Statement BuildStatement
+ Subincludes BuildLabels
+ Files []string
+ Targets BuildLabels
+}
+
+// PackageMetadata stores metadata about parsed BUILD files, mapping statements and subincludes
+// to their respective targets. This supports additional logic for operations such as `plz export`
+// but should be disabled for most operations by using the no-op implementation to avoid the overhead.
+type PackageMetadata interface {
+ // RegisterStatement records a statement of an interpreted BUILD file and its
+ // dependencies. Dependencies identify the subincluded targets required for a successful
+ // interpretation of the statement, i.e. targets that provide the required symbols (variables or
+ // methods). The files argument identify the files required when interpreting that statement, for
+ // example using glob(),
+ RegisterStatement(stmt BuildStatement, deps BuildLabels, files []string)
+ // RegisterStatementTarget records that the given build target was created as a result of the
+ // given statement being executed. This should only be called for statements in BUILD files.
+ RegisterStatementTarget(target BuildLabel, stmtProvider BuildStatementProvider)
+ // RegisterSubincludeStatement records the given build statement (provided by stmtProvider)
+ // as being a subincluded statement. This should only be called for statements in BUILD files.
+ RegisterSubincludeStatement(stmtProvider BuildStatementProvider)
+ // FindStatement returns the build statement that was responsible for generating the given target.
+ // Returns an error if the target was not found in the recorded metadata.
+ FindStatement(target BuildLabel) (BuildStatement, error)
+ // FindTargets returns all build targets that were generated by the given build statement.
+ // Returns an empty slice if no targets were found for the given statement.
+ FindTargets(stmt BuildStatement) BuildLabels
+ // FindRequiredSubincludes returns all subinclude labels that were required by the given target.
+ // This method will only report the package level (direct) subincludes, make use of
+ // [BuildGraph.TransitiveSubincludes] if you want all the required subincludes for one target.
+ FindRequiredSubincludes(target BuildLabel) (BuildLabels, error)
+ // FindRelatedTargets finds all the targets that are related to the argument. In this context,
+ // target relationship is determined by looking for targets generated by the same build statement.
+ // The result excludes the target in the argument.
+ FindRelatedTargets(target BuildLabel) (BuildLabels, error)
+ // FindPackageFileRequirements finds all the subincluded labels required by the package that
+ // are not associated with BuildTarget generation. An example could be a variable declaration
+ // that depends on a subincluded value.
+ FindPackageFileRequirements() (BuildLabels, []string)
+ // GetSubincludedLabels returns all build labels that were included by the given subinclude statement.
+ // Returns the labels or an empty slice if the statement wasn't found.
+ GetSubincludedLabels(stmt BuildStatement) BuildLabels
+ // IsInterpretedStatement returns true if the statement provided matches a registered build
+ // statement, meaning it was interpreted even if it doesn't generate any targets.
+ IsInterpretedStatement(stmt BuildStatement) bool
+ // Statements returns all statement metadata tracked in the package, sorted by BUILD file order.
+ Statements() []StatementMetadata
+}
+
+// packageMetadataImpl is the canonical implementation of the PackageMetadata interface. It tracks the relationships between BUILD file statements,
+// subincludes, and the build targets they define.
+//
+// Note: this implementation uses sharded concurrent maps [cmap.Map], however writes (interpreter
+// phase) are performed on a single-thread per package. Please guarantees that each package's BUILD file
+// and its subincludes are interpreted on exactly one parser thread at a time (enforced by
+// SyncParsePackage). Because of this guarantee, non-atomic write methods (like the Get-then-Set
+// sequence in RegisterStatement) are safe and wont suffer from write-write race conditions.
+// Nevertheless we use concurrent maps to support concurrent reads.
+type packageMetadataImpl struct {
+ // stmtToTarget maps each build statement (identified by its byte range in a BUILD file)
+ // to the targets it produced. Since a single statement (like a custom target or loop)
+ // can produce multiple targets, this is a one-to-many mapping.
+ stmtToTarget *cmap.Map[BuildStatement, BuildLabels]
+ // targetToStmt serves as a reverse-lookup map, linking each generated BuildLabel
+ // back to the specific BuildStatement that declared it. This is useful for tracing back a target
+ // to its statement and to find sibling targets generated by the same statement block.
+ targetToStmt *cmap.Map[BuildLabel, BuildStatement]
+ // stmtToRequiredSubincludes tracks the subinclude labels that were required for the current
+ // interpretation of the build statement. This, in addition to the statement to target map,
+ // allows mapping a statement back to the subincluded labels required for building the target.
+ // One direct, package level, subincludes are included.
+ stmtToRequiredSubincludes *cmap.Map[BuildStatement, BuildLabels]
+ // stmtToRequiredFiles tracks the file paths that were required during interpretation of the
+ // statement (e.g. glob)
+ stmtToRequiredFiles *cmap.Map[BuildStatement, []string]
+ // subincludeStmts tracks which build statements (identified by its position in the BUILD file)
+ // are subinclude calls.
+ subincludeStmts *cmap.Map[BuildStatement, struct{}]
+}
+
+func newPackageMetadata() PackageMetadata {
+ return &packageMetadataImpl{
+ stmtToTarget: cmap.New[BuildStatement, BuildLabels](cmap.SmallShardCount, hashBuildStatement),
+ targetToStmt: cmap.New[BuildLabel, BuildStatement](cmap.SmallShardCount, HashBuildLabel),
+ stmtToRequiredSubincludes: cmap.New[BuildStatement, BuildLabels](cmap.SmallShardCount, hashBuildStatement),
+ stmtToRequiredFiles: cmap.New[BuildStatement, []string](cmap.SmallShardCount, hashBuildStatement),
+ subincludeStmts: cmap.New[BuildStatement, struct{}](cmap.SmallShardCount, hashBuildStatement),
+ }
+}
+
+// RegisterStatement implements [PackageMetadata].
+func (m *packageMetadataImpl) RegisterStatement(stmt BuildStatement, deps BuildLabels, files []string) {
+ if len(deps) > 0 {
+ existingDeps := m.stmtToRequiredSubincludes.Get(stmt)
+ if len(existingDeps) == 0 {
+ m.stmtToRequiredSubincludes.Set(stmt, deps)
+ } else {
+ mergedDeps := mergeSlices(existingDeps, deps)
+ m.stmtToRequiredSubincludes.Set(stmt, mergedDeps)
+ }
+ }
+ if len(files) > 0 {
+ existingFiles := m.stmtToRequiredFiles.Get(stmt)
+ if len(existingFiles) == 0 {
+ m.stmtToRequiredFiles.Set(stmt, files)
+ } else {
+ mergedFiles := mergeSlices(existingFiles, files)
+ m.stmtToRequiredFiles.Set(stmt, mergedFiles)
+ }
+ }
+ // Even if the statement doesn't create any target, it is important to register so we now it was
+ // interpreted. We'll register the targets separately and use the Add() method to avoid overriding any existing statement to target mapping.
+ m.stmtToTarget.Add(stmt, BuildLabels{})
+}
+
+// mergeSlices merges two slices of any comparable type. It de-duplicates elements using
+// slices.Contains so it should be used for small slices. A set/map approach should be preferred for
+// larger slices.
+func mergeSlices[T comparable](existing []T, newElements []T) []T {
+ merged := append([]T(nil), existing...)
+ for _, el := range newElements {
+ if !slices.Contains(merged, el) {
+ merged = append(merged, el)
+ }
+ }
+ return merged
+}
+
+// RegisterStatementTarget implements [PackageMetadata].
+func (m *packageMetadataImpl) RegisterStatementTarget(target BuildLabel, stmtProvider BuildStatementProvider) {
+ stmt := stmtProvider()
+ targets := m.stmtToTarget.Get(stmt)
+ m.stmtToTarget.Set(stmt, append(targets, target))
+ m.targetToStmt.Set(target, stmt)
+}
+
+// RegisterSubincludeStatement implements [PackageMetadata].
+func (m *packageMetadataImpl) RegisterSubincludeStatement(stmtProvider BuildStatementProvider) {
+ stmt := stmtProvider()
+ m.subincludeStmts.Set(stmt, struct{}{})
+}
+
+// FindStatement implements [PackageMetadata].
+func (m *packageMetadataImpl) FindStatement(target BuildLabel) (BuildStatement, error) {
+ stmt := m.targetToStmt.Get(target)
+ if stmt == (BuildStatement{}) {
+ return BuildStatement{}, fmt.Errorf("failed to find statement for target %s", target)
+ }
+ return stmt, nil
+}
+
+// FindTargets implements [PackageMetadata].
+func (m *packageMetadataImpl) FindTargets(stmt BuildStatement) BuildLabels {
+ return m.stmtToTarget.Get(stmt)
+}
+
+// FindRequiredSubincludes implements [PackageMetadata].
+func (m *packageMetadataImpl) FindRequiredSubincludes(target BuildLabel) (BuildLabels, error) {
+ stmt, err := m.FindStatement(target)
+ if err != nil {
+ return nil, err
+ }
+
+ directSubincludes := m.stmtToRequiredSubincludes.Get(stmt)
+ if directSubincludes == nil {
+ // Could be empty if no subincluded label is required
+ return nil, nil
+ }
+
+ return directSubincludes, nil
+}
+
+// FindRelatedTargets implements [PackageMetadata].
+func (m *packageMetadataImpl) FindRelatedTargets(target BuildLabel) (BuildLabels, error) {
+ stmt, err := m.FindStatement(target)
+ if err != nil {
+ return nil, err
+ }
+ relatedTargets := m.FindTargets(stmt)
+ labels := make(BuildLabels, 0, len(relatedTargets))
+ for _, l := range relatedTargets {
+ if l != target {
+ labels = append(labels, l)
+ }
+ }
+ return labels, nil
+}
+
+// FindPackageFileRequirements implements [PackageMetadata].
+func (m *packageMetadataImpl) FindPackageFileRequirements() (BuildLabels, []string) {
+ requiredSet := LabelSet{}
+ m.stmtToRequiredSubincludes.Range(func(stmt BuildStatement, labels BuildLabels) {
+ // Look for build statements that are not registered in the statement to targets mapping,
+ // and so are unrelated to targets and are not subinclude statements.
+ if len(m.stmtToTarget.Get(stmt)) == 0 && !m.subincludeStmts.Contains(stmt) {
+ for _, label := range labels {
+ requiredSet.Add(label)
+ }
+ }
+ })
+ origins := slices.Collect(maps.Keys(requiredSet))
+
+ filesSet := map[string]struct{}{}
+ m.stmtToRequiredFiles.Range(func(stmt BuildStatement, files []string) {
+ // Copy all files. Statements that generate targets are linked to files directly via dependencies.
+ for _, file := range files {
+ filesSet[file] = struct{}{}
+ }
+ })
+ files := slices.Collect(maps.Keys(filesSet))
+ return origins, files
+}
+
+// GetSubincludedLabels implements [PackageMetadata].
+func (m *packageMetadataImpl) GetSubincludedLabels(stmt BuildStatement) BuildLabels {
+ if !m.subincludeStmts.Contains(stmt) {
+ return nil
+ }
+ // After determining that this is a subincludes statement we can return the required subincludes
+ // registered in the general statement mapping.
+ return m.stmtToRequiredSubincludes.Get(stmt)
+}
+
+// IsInterpretedStatement implements [PackageMetadata].
+func (m *packageMetadataImpl) IsInterpretedStatement(stmt BuildStatement) bool {
+ return m.stmtToTarget.Contains(stmt)
+}
+
+// Statements implements [PackageMetadata].
+func (m *packageMetadataImpl) Statements() []StatementMetadata {
+ stmtsMap := map[BuildStatement]*StatementMetadata{}
+ getOrCreate := func(stmt BuildStatement) *StatementMetadata {
+ if sm, ok := stmtsMap[stmt]; ok {
+ return sm
+ }
+ sm := &StatementMetadata{Statement: stmt}
+ stmtsMap[stmt] = sm
+ return sm
+ }
+
+ m.stmtToRequiredSubincludes.Range(func(stmt BuildStatement, labels BuildLabels) {
+ getOrCreate(stmt).Subincludes = labels
+ })
+
+ m.stmtToRequiredFiles.Range(func(stmt BuildStatement, files []string) {
+ getOrCreate(stmt).Files = files
+ })
+
+ m.stmtToTarget.Range(func(stmt BuildStatement, labels BuildLabels) {
+ getOrCreate(stmt).Targets = labels
+ })
+
+ res := make([]StatementMetadata, 0, len(stmtsMap))
+ for _, sm := range stmtsMap {
+ res = append(res, *sm)
+ }
+
+ // Sort the statements to maintain BUILD file order
+ slices.SortFunc(res, func(i, j StatementMetadata) int {
+ return cmp.Compare(i.Statement.Start, j.Statement.Start)
+ })
+
+ return res
+}
+
+// noopPackageMetadata implements the PackageMetadata interface with no-op methods. This is the
+// default implementation and is used to avoid the overhead of parsing metadata for operations that
+// don't depend on it.
+type noopPackageMetadata struct{}
+
+func newNoopPackageMetadata() PackageMetadata {
+ return &noopPackageMetadata{}
+}
+
+// RegisterStatement implements [PackageMetadata].
+func (n *noopPackageMetadata) RegisterStatement(stmt BuildStatement, deps BuildLabels, files []string) {
+}
+
+// RegisterStatementTarget implements [PackageMetadata].
+func (n *noopPackageMetadata) RegisterStatementTarget(target BuildLabel, stmtProvider BuildStatementProvider) {
+}
+
+// RegisterSubincludeStatement implements [PackageMetadata].
+func (n *noopPackageMetadata) RegisterSubincludeStatement(stmtProvider BuildStatementProvider) {
+}
+
+// FindStatement implements [PackageMetadata].
+func (n *noopPackageMetadata) FindStatement(target BuildLabel) (BuildStatement, error) {
+ log.Fatalf("metadata not tracked, using no-op implementation")
+ return BuildStatement{}, nil
+}
+
+// FindTargets implements [PackageMetadata].
+func (n *noopPackageMetadata) FindTargets(stmt BuildStatement) BuildLabels {
+ log.Fatalf("metadata not tracked, using no-op implementation")
+ return nil
+}
+
+// FindRequiredSubincludes implements [PackageMetadata].
+func (n *noopPackageMetadata) FindRequiredSubincludes(target BuildLabel) (BuildLabels, error) {
+ log.Fatalf("metadata not tracked, using no-op implementation")
+ return nil, nil
+}
+
+// FindRelatedTargets implements [PackageMetadata].
+func (n *noopPackageMetadata) FindRelatedTargets(target BuildLabel) (BuildLabels, error) {
+ log.Fatalf("metadata not tracked, using no-op implementation")
+ return nil, nil
+}
+
+// FindPackageFileRequirements implements [PackageMetadata].
+func (n *noopPackageMetadata) FindPackageFileRequirements() (BuildLabels, []string) {
+ log.Fatalf("metadata not tracked, using no-op implementation")
+ return nil, nil
+}
+
+// GetSubincludedLabels implements [PackageMetadata].
+func (n *noopPackageMetadata) GetSubincludedLabels(stmt BuildStatement) BuildLabels {
+ log.Fatalf("metadata not tracked, using no-op implementation")
+ return nil
+}
+
+// IsInterpretedStatement implements [PackageMetadata].
+func (n *noopPackageMetadata) IsInterpretedStatement(stmt BuildStatement) bool {
+ log.Fatalf("metadata not tracked, using no-op implementation")
+ return false
+}
+
+// Statements implements [PackageMetadata].
+func (n *noopPackageMetadata) Statements() []StatementMetadata {
+ log.Fatalf("metadata not tracked, using no-op implementation")
+ return nil
+}
diff --git a/src/core/state.go b/src/core/state.go
index 18a03d70ab..f0c12aa228 100644
--- a/src/core/state.go
+++ b/src/core/state.go
@@ -240,6 +240,15 @@ type BuildState struct {
// NeedDebugDeps is true if we're doing a `plz debug` and we need to build the debug tools and
// data
NeedDebugDeps bool
+ // ParseMetadata is true if we want to store build file metadata
+ ParseMetadata bool
+ // KeepParserRunning prevents closing task workers (parse and build channels) to support later
+ // calls to the parser. This is needed to support the export operation since the export logic will
+ // attempt to export targets that have not been parsed during the normal build phase. An example
+ // is when exporting dependencies of targets that are not explicitly used but adjacent/related.
+ KeepParserRunning bool
+ // WaitForDisplay is a function that blocks until the display thread has finished.
+ WaitForDisplay func()
// initOnce is used to control loading the subrepo .plzconfig
initOnce *sync.Once
@@ -281,13 +290,14 @@ func (state *BuildState) Initialise(subrepo *Subrepo) (err error) {
// This is split out from above so we can share it between multiple instances.
type stateProgress struct {
// Used to count the number of currently active/pending targets
- numActive int64
- numPending int64
- numDone int64
- numParses atomic.Int64
- mutex sync.Mutex
- closeOnce sync.Once
- resultOnce sync.Once
+ numActive int64
+ numPending int64
+ numDone int64
+ numParses atomic.Int64
+ mutex sync.Mutex
+ closeOnce sync.Once
+ resultOnce sync.Once
+ buildDoneOnce sync.Once
// Used to track subinclude() calls that block until targets are built. Keyed by their label.
pendingTargets *cmap.Map[BuildLabel, chan struct{}]
// Used to track general package parsing requests. Keyed by a packageKey struct.
@@ -310,6 +320,8 @@ type stateProgress struct {
internalResults chan *BuildResult
// The cycle checker itself.
cycleDetector cycleDetector
+ // buildDone is closed when numPending drops to 0 or less
+ buildDone chan struct{}
}
// SystemStats stores information about the system.
@@ -409,7 +421,13 @@ func (state *BuildState) taskDone(wasSynthetic bool) {
atomic.AddInt64(&state.progress.numDone, 1)
}
if atomic.AddInt64(&state.progress.numPending, -1) <= 0 {
- state.Stop()
+ state.progress.buildDoneOnce.Do(func() {
+ close(state.progress.buildDone)
+ })
+
+ if !state.KeepParserRunning {
+ state.Stop()
+ }
}
}
@@ -433,6 +451,22 @@ func (state *BuildState) CloseResults() {
}
}
+// Cleanup cleans up and shuts down the build state.
+func (state *BuildState) Cleanup() {
+ state.CloseResults()
+
+ if state.WaitForDisplay != nil {
+ state.WaitForDisplay()
+ }
+
+ if state.Cache != nil {
+ state.Cache.Shutdown()
+ }
+ if state.RemoteClient != nil {
+ state.RemoteClient.Disconnect()
+ }
+}
+
// IsOriginalTarget returns true if a target is an original target, ie. one specified on the command line.
func (state *BuildState) IsOriginalTarget(target *BuildTarget) bool {
return state.isOriginalTarget(target, false)
@@ -601,6 +635,23 @@ func (state *BuildState) LogTestResult(target *BuildTarget, run int, status Buil
// LogBuildError logs a failure for a target to parse, build or test.
func (state *BuildState) LogBuildError(label BuildLabel, status BuildResultStatus, err error, format string, args ...interface{}) {
+ if status == ParseFailed {
+ // Force close package wait channels to avoid deadlocks when calling waitForPackage() after
+ // the initial parse, for example when KeepParserRunning is set.
+ key := packageKey{Name: label.PackageName, Subrepo: label.Subrepo}
+ if ch := state.progress.pendingPackages.Get(key); ch != nil {
+ func() {
+ defer func() { recover() }() // recover if attempted to close a closed channel.
+ close(ch) // This signals to anyone waiting that it's done (failed, but completed).
+ }()
+ }
+ if ch := state.progress.packageWaits.Get(key); ch != nil {
+ func() {
+ defer func() { recover() }() // recover if attempted to close a closed channel.
+ close(ch) // This signals to anyone waiting that it's done (failed, but completed).
+ }()
+ }
+ }
state.logResult(&BuildResult{
Label: label,
Status: status,
@@ -665,11 +716,17 @@ func (state *BuildState) forwardResults() {
delete(activeTargets, target)
}
}
- state.progress.mutex.Lock()
- if state.progress.results != nil {
- state.progress.results <- result
- }
- state.progress.mutex.Unlock()
+ state.sendResult(result)
+ }
+}
+
+// sendResult sends a unique result to the channel. A simple method that is mostly useful for a
+// deferring the mutex close and avoid deadlocks even when we attempt to write to a closed channel.
+func (state *BuildState) sendResult(result *BuildResult) {
+ state.progress.mutex.Lock()
+ defer state.progress.mutex.Unlock()
+ if state.progress.results != nil {
+ state.progress.results <- result
}
}
@@ -920,6 +977,11 @@ func (state *BuildState) WaitForBuiltTarget(l, dependent BuildLabel, mode ParseM
return state.WaitForBuiltTarget(l, dependent, mode)
}
+// WaitForBuildToComplete blocks until all pending tasks are finished.
+func (state *BuildState) WaitForBuildToComplete() {
+ <-state.progress.buildDone
+}
+
// AddTarget adds a new target to the build graph.
func (state *BuildState) AddTarget(pkg *Package, target *BuildTarget) {
pkg.AddTarget(target)
@@ -1480,12 +1542,13 @@ func NewBuildState(config *Configuration) *BuildState {
progress: &stateProgress{
numActive: 1, // One for the initial target adding on the main thread.
numPending: 1,
- pendingTargets: cmap.New[BuildLabel, chan struct{}](cmap.DefaultShardCount, hashBuildLabel),
+ pendingTargets: cmap.New[BuildLabel, chan struct{}](cmap.DefaultShardCount, HashBuildLabel),
pendingPackages: cmap.New[packageKey, chan struct{}](cmap.DefaultShardCount, hashPackageKey),
packageWaits: cmap.New[packageKey, chan struct{}](cmap.DefaultShardCount, hashPackageKey),
internalResults: make(chan *BuildResult, 1000),
cycleDetector: cycleDetector{graph: graph},
originalTargets: NewTargetSet(),
+ buildDone: make(chan struct{}),
},
initOnce: new(sync.Once),
preloadDownloadOnce: new(sync.Once),
diff --git a/src/core/subrepo.go b/src/core/subrepo.go
index 5244226c08..100d4e0ab8 100644
--- a/src/core/subrepo.go
+++ b/src/core/subrepo.go
@@ -68,6 +68,11 @@ func (s *Subrepo) IsRemoteSubrepo() bool {
return s.Root != "" && s.Target != nil && !s.Target.Local && s.State.RemoteClient != nil
}
+// IsExternal returns true if this subrepo is external, meaning its sources are fetched by a build target.
+func (s *Subrepo) IsExternal() bool {
+ return s != nil && s.Target != nil
+}
+
// Equal returns true if this subrepo is equivalent to another, or false if it is not.
func (s *Subrepo) Equal(other *Subrepo) bool {
return s.Name == other.Name && s.Root == other.Root && s.PackageRoot == other.PackageRoot &&
diff --git a/src/export/BUILD b/src/export/BUILD
index 6b44d63d9b..5069f9dc85 100644
--- a/src/export/BUILD
+++ b/src/export/BUILD
@@ -1,13 +1,30 @@
go_library(
name = "export",
- srcs = ["export.go"],
+ srcs = glob(
+ ["*.go"],
+ exclude = ["*_test.go"],
+ ),
pgo_file = "//:pgo",
visibility = ["PUBLIC"],
deps = [
+ "///third_party/go/github.com_please-build_buildtools//build",
+ "//src/cli",
"//src/cli/logging",
"//src/core",
"//src/fs",
- "//src/gc",
"//src/parse",
+ "//src/parse/asp",
+ ],
+)
+
+go_test(
+ name = "export_test",
+ srcs = ["export_test.go"],
+ data = ["test_data"],
+ deps = [
+ ":export",
+ "///third_party/go/github.com_stretchr_testify//assert",
+ "//src/core",
+ "//src/parse/asp",
],
)
diff --git a/src/export/export.go b/src/export/export.go
index 2863dfef0c..a8780d42a2 100644
--- a/src/export/export.go
+++ b/src/export/export.go
@@ -4,212 +4,260 @@
package export
import (
- iofs "io/fs"
+ "fmt"
"os"
"path/filepath"
+ "time"
+ "github.com/thought-machine/please/src/cli"
"github.com/thought-machine/please/src/cli/logging"
"github.com/thought-machine/please/src/core"
"github.com/thought-machine/please/src/fs"
- "github.com/thought-machine/please/src/gc"
"github.com/thought-machine/please/src/parse"
)
var log = logging.Log
-type export struct {
- state *core.BuildState
- targetDir string
- noTrim bool
-
- exportedTargets map[core.BuildLabel]bool
- exportedPackages map[string]bool
-}
-
-// ToDir exports a set of targets to the given directory.
-// It dies on any errors.
-func ToDir(state *core.BuildState, dir string, noTrim bool, targets []core.BuildLabel) {
- e := &export{
- state: state,
- noTrim: noTrim,
- targetDir: dir,
- exportedPackages: map[string]bool{},
- exportedTargets: map[core.BuildLabel]bool{},
- }
+// Repo export a new please repo including the targets and dependencies requested. Depending on the
+// noTrim flag, the export will attempt to trim the resulting repository, exporting only the required
+// targets and build statements in their packages. If noTrim is set, all targets of a package will be
+// exported and no build statement trimming will be attempted, the BUILD file is copied in its entirety.
+func Repo(state *core.BuildState, dir string, noTrim bool, targets []core.BuildLabel) {
+ e := newExporter(state, dir, noTrim)
+ // ensure output dir
if err := os.MkdirAll(dir, fs.DirPermissions); err != nil {
log.Fatalf("failed to create export directory %s: %v", dir, err)
}
- e.exportPlzConf()
- for _, target := range state.Config.Parse.PreloadSubincludes {
- for _, includeLabel := range append(state.Graph.TransitiveSubincludes(target), target) {
- e.export(state.Graph.TargetOrDie(includeLabel))
+ e.run(targets)
+}
+
+// Outputs exports the build artifacts (output files) produced by building the specified
+// targets to the given output directory.
+func Outputs(state *core.BuildState, dir string, targets []core.BuildLabel) {
+ for _, label := range targets {
+ target := state.Graph.TargetOrDie(label)
+ for _, out := range target.Outputs() {
+ fullPath := filepath.Join(dir, out)
+ outDir := filepath.Dir(fullPath)
+ if err := os.MkdirAll(outDir, core.DirPermissions); err != nil {
+ log.Fatalf("Failed to create export dir %s: %s", outDir, err)
+ }
+ if err := fs.RecursiveCopy(filepath.Join(target.OutDir(), out), fullPath, target.OutMode()|0200); err != nil {
+ log.Fatalf("Failed to copy export file: %s", err)
+ }
}
}
- for _, target := range targets {
- e.export(state.Graph.TargetOrDie(target))
- }
- // Now write all the build files
- packages := map[*core.Package]bool{}
- for target := range e.exportedTargets {
- packages[state.Graph.PackageOrDie(target)] = true
- }
+}
- // Write any preloaded build defs as well; preloaded subincludes should be fine though.
- for _, preload := range state.Config.Parse.PreloadBuildDefs {
- if err := fs.RecursiveCopy(preload, filepath.Join(dir, preload), 0); err != nil {
- log.Fatalf("Failed to copy preloaded build def %s: %s", preload, err)
- }
+// exporterImpl defines the interface for exporting parts of a Please repository to a new directory.
+// It handles the copying of configuration files, preloaded build definitions, and selected
+// targets along with their necessary source files and dependencies.
+type exporterImpl interface {
+ // exportPreloaded exports all globally preloaded build definitions and subincluded targets.
+ // These are usually defined in the repository's configuration file.
+ exportPreloaded()
+ // exportTarget exports an individual build target.
+ // Each target recursively exports all their source files and required build statements, but also
+ // targets in their transitive dependencies.
+ exportTarget(*core.BuildTarget)
+ // writePackageFiles writes the processed BUILD files for all exported targets to the
+ // export directory. These BUILD files may be modified (e.g., trimmed) depending on
+ // the exporter's implementation.
+ writePackageFiles()
+}
+
+// newExporter creates a new exporter of a specific type based on the arguments.
+func newExporter(state *core.BuildState, dir string, noTrim bool) *baseExporter {
+ base := &baseExporter{
+ state: state,
+ targetDir: dir,
+ exportedTargets: map[core.BuildLabel]bool{},
}
+ var exporter exporterImpl
if noTrim {
- return // We have already exported the whole directory
+ exporter = newNoTrimExporter(base)
+ } else {
+ exporter = newTrimmedExporter(base)
}
- for pkg := range packages {
- if pkg.Name == parse.InternalPackageName {
- continue // This isn't a real package to be copied
- }
- if pkg.Subrepo != nil {
- continue // Don't copy subrepo BUILD files... they don't exist in our source tree
- }
- dest := filepath.Join(dir, pkg.Filename)
- if err := fs.CopyFile(pkg.Filename, dest, 0); err != nil {
- log.Fatalf("Failed to copy BUILD file %s: %s\n", pkg.Filename, err)
- }
- // Now rewrite the unused targets out of it
- var victims []string
- for _, target := range pkg.AllTargets() {
- if !e.exportedTargets[target.Label] && !target.HasParent() {
- victims = append(victims, target.Label.Name)
+ base.impl = exporter
+ return base
+}
+
+// baseExporter provides common fields and methods of other exporters.
+type baseExporter struct {
+ state *core.BuildState
+ targetDir string
+ targetCounter int
+
+ // exportedTargets maintains a record of the targets that have been exported so far.
+ exportedTargets map[core.BuildLabel]bool
+ // impl is a reference to the concrete exporter implementation. It's included for calling the
+ // specific exporter implementation from the common methods.
+ impl exporterImpl
+}
+
+// run specifies the main steps when running an export.
+func (be *baseExporter) run(targets core.BuildLabels) {
+ stop := make(chan struct{})
+ go be.startMonitor(stop)
+ defer close(stop)
+
+ be.exportRepoConfig()
+ be.impl.exportPreloaded()
+ be.exportTargets(targets)
+ be.impl.writePackageFiles()
+}
+
+func (be *baseExporter) startMonitor(stop chan struct{}) {
+ // this total is meant to provide some general idea of the progress but in reality we might
+ // visit more than the targets currently in the build graph (those will have to be parsed
+ // adhoc).
+ total := len(be.state.Graph.AllTargets())
+ startTime := time.Now()
+ isTerminal := cli.StdErrIsATerminal
+
+ if isTerminal {
+ cli.CurrentBackend.SetPassthrough(false, 1, false)
+ defer cli.CurrentBackend.SetPassthrough(true, 1, false)
+ }
+
+ ticker := time.NewTicker(100 * time.Millisecond)
+ defer ticker.Stop()
+
+ for {
+ select {
+ case <-stop:
+ log.Infof("Exported %d targets (%.1fs)...", be.targetCounter, time.Since(startTime).Seconds())
+ return
+ case <-ticker.C:
+ done := be.targetCounter
+ elapsed := time.Since(startTime).Seconds()
+ stats := be.state.SystemStats()
+
+ if isTerminal {
+ logs := cli.CurrentBackend.FlushOutput()
+ if len(logs) > 0 {
+ // Remove footer line before writing logs.
+ cli.Fprintf(os.Stderr, "${RESETLN}")
+ for _, line := range logs {
+ fmt.Fprintln(os.Stderr, line)
+ }
+ }
+ cli.Fprintf(os.Stderr, "${RESETLN}${BOLD_WHITE}Exporting${RESET} [${BOLD_GREEN}%d${RESET}/${BOLD_GREEN}%d${RESET}, ${BOLD_YELLOW}%.1fs${RESET}]:: CPU: %5.1f%% Mem: %5.1f%% IO: %5.1f%%${RESET}",
+ done, total, elapsed, stats.CPU.Used, stats.Memory.UsedPercent, stats.CPU.IOWait)
}
}
- if err := gc.RewriteFile(state, dest, victims); err != nil {
- log.Fatalf("Failed to rewrite BUILD file: %s\n", err)
- }
}
}
-func (e *export) exportPlzConf() {
- profiles, err := filepath.Glob(".plzconfig*")
+// exportRepoConfig exports the repository's configuration files (e.g., .gitignore, .plzconfig and
+// its platform-specific variants) to the target export directory.
+func (be *baseExporter) exportRepoConfig() {
+ files, err := filepath.Glob(".plzconfig*")
if err != nil {
log.Fatalf("failed to glob .plzconfig files: %v", err)
}
- for _, file := range profiles {
- path := filepath.Join(e.targetDir, file)
- if err := os.RemoveAll(path); err != nil {
- log.Fatalf("failed to copy .plzconfig file %s: %v", file, err)
+ if info, err := os.Stat(".gitignore"); err == nil {
+ files = append(files, info.Name())
+ }
+
+ for _, file := range files {
+ targetPath := filepath.Join(be.targetDir, file)
+ if err := os.RemoveAll(targetPath); err != nil {
+ log.Fatalf("failed to remove file %s: %v", file, err)
}
- if err := fs.CopyFile(file, path, 0); err != nil {
- log.Fatalf("failed to copy .plzconfig file %s: %v", file, err)
+ if err := fs.CopyFile(file, targetPath, 0); err != nil {
+ log.Fatalf("failed to copy file %s: %v", file, err)
}
}
}
-// exportSources exports any source files (srcs and data) for the rule
-func (e *export) exportSources(target *core.BuildTarget) {
- for _, src := range append(target.AllSources(), target.AllData()...) {
- if _, ok := src.Label(); !ok { // We'll handle these dependencies later
- for _, p := range src.FullPaths(e.state.Graph) {
- if !filepath.IsAbs(p) { // Don't copy system file deps.
- if err := fs.RecursiveCopy(p, filepath.Join(e.targetDir, p), 0); err != nil {
- log.Fatalf("Error copying file: %s\n", err)
- }
- }
- }
+// exportTargets exports the set of targets identified by the given build labels.
+func (be *baseExporter) exportTargets(labels core.BuildLabels) {
+ for _, l := range labels {
+ if be.exportedTargets[l] {
+ continue
+ }
+ target := be.getOrParseTarget(l)
+ if target == nil {
+ log.Errorf("Unable to lookup target %s", l)
+ continue
}
+ be.impl.exportTarget(target)
}
}
-var ignoreDirectories = map[string]bool{
- "plz-out": true,
- ".git": true,
- ".svn": true,
- ".hg": true,
+// exportDependencies exports dependencies of a target.
+func (be *baseExporter) exportDependencies(target *core.BuildTarget) {
+ deps := target.DeclaredDependencies()
+ be.exportTargets(deps)
}
-// exportPackage exports the package BUILD file containing the given target and all sources
-func (e *export) exportPackage(target *core.BuildTarget) {
- pkgName := target.Label.PackageName
- if pkgName == parse.InternalPackageName {
- return
- }
- if e.exportedPackages[pkgName] {
- return
- }
- e.exportedPackages[pkgName] = true
-
- pkgDir := filepath.Clean(pkgName)
-
- err := filepath.WalkDir(pkgDir, func(path string, d iofs.DirEntry, err error) error {
- if err != nil {
- return err
+// exportSources exports all files required by the target.
+func (be *baseExporter) exportSources(target *core.BuildTarget) {
+ for _, src := range target.AllBuildInputs() {
+ if _, ok := src.Label(); ok {
+ continue // These will be handled as dependencies later
}
- if d.IsDir() {
- if path != pkgDir && fs.IsPackage(e.state.Config.Parse.BuildFileName, path) {
- return filepath.SkipDir // We want to stop when we find another package in our dir tree
- }
- if ignoreDirectories[d.Name()] {
- return filepath.SkipDir
+ paths := src.Paths(be.state.Graph)
+ if target.Subrepo != nil { // Adjusting fo for local subrepos
+ for i, p := range paths {
+ paths[i] = target.Subrepo.Dir(p)
}
- return nil
}
- if !d.Type().IsRegular() {
- return nil // Ignore symlinks, which are almost certainly generated sources.
+ be.exportFiles(paths)
+ }
+}
+
+func (be *baseExporter) exportFiles(paths []string) {
+ for _, p := range paths {
+ if filepath.IsAbs(p) { // Don't copy system file deps.
+ log.Debugf("System dependency detected, skipping...: %s", p)
+ continue
}
- dest := filepath.Join(e.targetDir, path)
- if err := fs.EnsureDir(dest); err != nil {
- return err
+ dest := filepath.Join(be.targetDir, p)
+ if err := fs.RecursiveCopy(p, dest, 0); err != nil {
+ log.Warningf("Error copying file, skipping...: %s", err)
}
- return fs.CopyFile(path, dest, 0)
- })
- if err != nil {
- log.Fatalf("failed to export package %s for %s: %v", pkgName, target.Label, err)
}
}
-// export implements the logic of ToDir, but prevents repeating targets.
-func (e *export) export(target *core.BuildTarget) {
- if e.exportedTargets[target.Label] {
- return
- }
- // We want to export the package that made this subrepo available, but we still need to walk the target deps
- // as it may depend on other subrepos or first party targets
- if target.Subrepo != nil {
- e.export(target.Subrepo.Target)
- } else if e.noTrim {
- // Export the whole package, rather than trying to trim the package down to only the targets we need
- e.exportPackage(target)
- } else {
- e.exportSources(target)
+// getOrParseTarget attempts to lookup a target in the build graph. If the target has not
+// been parsed yet, it dynamically requests the package be parsed and blocks until the target is resolved.
+//
+// This occurs during the exports when walking dependencies of adjacent targets which were not
+// explicitly activated or resolved during the initial build/parse phase.
+//
+// This requires the background parser worker threads to be kept alive as daemons (controlled by the
+// "KeepParserRunning" build state option).
+func (be *baseExporter) getOrParseTarget(label core.BuildLabel) *core.BuildTarget {
+ target := be.state.Graph.Target(label)
+ if target == nil {
+ log.Infof("Target %v not found in graph. Attempting to parse...", label)
+ parse.Parse(be.state, label, core.OriginalTarget, core.ParseModeNormal)
+ target = be.state.Graph.Target(label)
}
+ return target
+}
- e.exportedTargets[target.Label] = true
- for _, dep := range target.Dependencies() {
- e.export(dep)
- }
- for _, subinclude := range e.state.Graph.PackageOrDie(target.Label).AllSubincludes(e.state.Graph) {
- e.export(e.state.Graph.TargetOrDie(subinclude))
- }
- if parent := target.Parent(e.state.Graph); parent != nil && parent != target {
- e.export(parent)
- }
+// getOrParsePackage attempts to lookup a package in the build graph. If the package has not
+// been parsed yet, it dynamically requests the package be parsed and blocks until resolved.
+//
+// This requires the background parser worker threads to be kept alive as daemons (controlled by the
+// "KeepParserRunning" build state option).
+func (be *baseExporter) getOrParsePackage(label core.BuildLabel) *core.Package {
+ return be.state.WaitForPackage(label, core.OriginalTarget, core.ParseModeNormal)
}
-// Outputs exports the outputs of a target.
-func Outputs(state *core.BuildState, dir string, targets []core.BuildLabel) {
- for _, label := range targets {
- target := state.Graph.TargetOrDie(label)
- for _, out := range target.Outputs() {
- fullPath := filepath.Join(dir, out)
- outDir := filepath.Dir(fullPath)
- if err := os.MkdirAll(outDir, core.DirPermissions); err != nil {
- log.Fatalf("Failed to create export dir %s: %s", outDir, err)
- }
- if err := fs.RecursiveCopy(filepath.Join(target.OutDir(), out), fullPath, target.OutMode()|0200); err != nil {
- log.Fatalf("Failed to copy export file: %s", err)
- }
- }
- }
+// checkAndSetVisited is a helper to ensure we only visit the same target once.
+// It returns true if this is the first time the target is being exported.
+func (be *baseExporter) checkAndSetVisited(target *core.BuildTarget) bool {
+ visited := be.exportedTargets[target.Label]
+ be.exportedTargets[target.Label] = true
+ be.targetCounter++
+ return !visited
}
diff --git a/src/export/export_test.go b/src/export/export_test.go
new file mode 100644
index 0000000000..b3e7a9e20c
--- /dev/null
+++ b/src/export/export_test.go
@@ -0,0 +1,227 @@
+package export
+
+import (
+ "os"
+ "slices"
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+
+ "github.com/thought-machine/please/src/core"
+ "github.com/thought-machine/please/src/parse/asp"
+)
+
+func TestMinimalSubincludeStatement(t *testing.T) {
+ testCases := []struct {
+ name string
+ availableLabels []core.BuildLabel
+ requiredLabels []core.BuildLabel
+ out string
+ }{
+ {
+ name: "Successful no pruning subinclude",
+ availableLabels: core.ParseBuildLabels([]string{"//build_defs:test"}),
+ requiredLabels: core.ParseBuildLabels([]string{"//build_defs:test"}),
+ out: `subinclude("//build_defs:test")`,
+ },
+ {
+ name: "No subincludes",
+ availableLabels: nil,
+ requiredLabels: nil,
+ out: "",
+ },
+ {
+ name: "Single subinclude (not required)",
+ availableLabels: core.ParseBuildLabels([]string{"//build_defs:other"}),
+ requiredLabels: nil,
+ out: "",
+ },
+ {
+ name: "Multiple subincludes (sorted and filtered)",
+ availableLabels: core.ParseBuildLabels([]string{"//build_defs:test", "//build_defs:abc", "//build_defs:other"}),
+ requiredLabels: core.ParseBuildLabels([]string{"//build_defs:test", "//build_defs:abc"}),
+ out: "subinclude(\n" +
+ " \"//build_defs:abc\",\n" +
+ " \"//build_defs:test\",\n" +
+ ")",
+ },
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ e := newExporter(nil, "", false).impl.(*trimmedExporter)
+
+ pkg := &core.Package{Name: "test"}
+ e.requiredSubincludes[pkg.Label()] = tc.requiredLabels
+ trimmer := trimmer{
+ pkg: pkg,
+ exporter: e,
+ }
+
+ assert.Equal(t, tc.out, trimmer.minimalSubincludeStatement(tc.availableLabels))
+ })
+ }
+}
+
+func TestFilterPackageFile(t *testing.T) {
+ testCases := []struct {
+ name string
+ required []string
+ expected string
+ }{
+ {
+ name: "Keep only A",
+ required: []string{"a"},
+ expected: "src/export/test_data/filter_expected_a.build",
+ },
+ {
+ name: "Keep only B",
+ required: []string{"b"},
+ expected: "src/export/test_data/filter_expected_b.build",
+ },
+ {
+ name: "Keep both",
+ required: []string{"a", "b"},
+ expected: "src/export/test_data/filter.build",
+ },
+ {
+ name: "Keep none",
+ required: []string{},
+ expected: "src/export/test_data/filter_expected_none.build",
+ },
+ }
+
+ contentPath := "src/export/test_data/filter.build"
+
+ p := asp.NewParserOnly()
+ statements, err := p.ParseFileOnly(contentPath)
+ assert.NoError(t, err)
+
+ pkg := core.NewPackage("test", core.WithPackageMetadata())
+ pkg.Filename = contentPath
+ targetLabels := walkASTRegisterTargets(t, statements, pkg, nil)
+
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ e := newExporter(nil, "", false).impl.(*trimmedExporter)
+ for _, name := range tc.required {
+ e.exportedTargets[targetLabels[name]] = true
+ }
+ e.visitedPackages[pkg.Label()] = true
+
+ p := asp.NewParserOnly()
+ got, err := e.trimPackage(p, pkg)
+ assert.NoError(t, err)
+
+ expected, err := os.ReadFile(tc.expected)
+ assert.NoError(t, err)
+ assert.Equal(t, string(expected), string(got))
+ })
+ }
+}
+
+func TestStatementTrim(t *testing.T) {
+ testCases := []struct {
+ name string
+ content string
+ required []string
+ expected string
+ }{
+ {
+ name: "Keep target in if",
+ content: "src/export/test_data/trim_if.build",
+ required: []string{"a"},
+ expected: "src/export/test_data/trim_if_expected_a.build",
+ },
+ {
+ name: "Target not required - all statements trimmed",
+ content: "src/export/test_data/trim_if.build",
+ required: []string{},
+ expected: "src/export/test_data/trim_if_expected_none.build",
+ },
+ {
+ name: "Required target in elif",
+ content: "src/export/test_data/trim_elif.build",
+ required: []string{"b"},
+ expected: "src/export/test_data/trim_elif_expected_b.build",
+ },
+ {
+ name: "Required target in for",
+ content: "src/export/test_data/trim_for.build",
+ required: []string{"a"},
+ expected: "src/export/test_data/trim_for_expected_a.build",
+ },
+ {
+ name: "Target not required in for - loop body has a pass",
+ content: "src/export/test_data/trim_for.build",
+ required: []string{},
+ expected: "src/export/test_data/trim_for_expected_none.build",
+ },
+ {
+ name: "Required if stmt in for",
+ content: "src/export/test_data/trim_for_if.build",
+ required: []string{"a"},
+ expected: "src/export/test_data/trim_for_if_expected_a.build",
+ },
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ p := asp.NewParserOnly()
+ statements, err := p.ParseFileOnly(tc.content)
+ assert.NoError(t, err)
+
+ pkg := core.NewPackage("test", core.WithPackageMetadata())
+ pkg.Filename = tc.content
+ targetLabels := walkASTRegisterTargets(t, statements, pkg, nil)
+
+ e := newExporter(nil, "", false).impl.(*trimmedExporter)
+ for _, name := range tc.required {
+ e.exportedTargets[targetLabels[name]] = true
+ }
+
+ contentBytes, err := os.ReadFile(tc.content)
+ assert.NoError(t, err)
+
+ trimmer := &trimmer{
+ origin: contentBytes,
+ pkg: pkg,
+ exporter: e,
+ }
+ trimmer.trimBlock(statements, 0, asp.Position(len(contentBytes)))
+
+ expectedBytes, err := os.ReadFile(tc.expected)
+ assert.NoError(t, err)
+
+ assert.Equal(t, string(expectedBytes), string(trimmer.bytes))
+ })
+ }
+}
+
+// walkASTRegisterTargets is a test helper to register simple targets and their build statements.
+func walkASTRegisterTargets(t *testing.T, stmts []*asp.Statement, pkg *core.Package, toRegister []string) map[string]core.BuildLabel {
+ t.Helper()
+ targetLabels := map[string]core.BuildLabel{}
+ asp.WalkAST(stmts, func(stmt *asp.Statement) bool {
+ arg := asp.FindArgument(stmt, "name")
+ if arg == nil {
+ return true // Continue
+ }
+
+ // Not in targets we want to register, continue
+ name := strings.Trim(arg.Value.Val.String, "\"")
+ if toRegister != nil && !slices.Contains(toRegister, name) {
+ return true
+ }
+
+ label := core.NewBuildLabel(pkg.Name, name)
+ targetLabels[name] = label
+ target := &core.BuildTarget{Label: label}
+ pkg.Metadata.RegisterStatementTarget(target.Label, func() core.BuildStatement {
+ return asp.NewBuildStatement(stmt)
+ })
+ return true
+ })
+ return targetLabels
+}
diff --git a/src/export/notrim_exporter.go b/src/export/notrim_exporter.go
new file mode 100644
index 0000000000..f3f4631228
--- /dev/null
+++ b/src/export/notrim_exporter.go
@@ -0,0 +1,109 @@
+package export
+
+import (
+ "path/filepath"
+
+ "github.com/thought-machine/please/src/core"
+ "github.com/thought-machine/please/src/fs"
+ "github.com/thought-machine/please/src/parse"
+)
+
+// noTrimExporter implements an exporter that avoids trimming any packages by exporting all targets
+// and statements in a package.
+type noTrimExporter struct {
+ *baseExporter
+ // exportedPackages tracks which packages have already had their BUILD files exported.
+ exportedPackages map[core.BuildLabel]bool
+}
+
+func newNoTrimExporter(base *baseExporter) exporterImpl {
+ return &noTrimExporter{
+ baseExporter: base,
+ exportedPackages: map[core.BuildLabel]bool{},
+ }
+}
+
+// exportPreloaded implements [exporterImpl].
+func (nte *noTrimExporter) exportPreloaded() {
+ // Write any preloaded build defs
+ for _, preload := range nte.state.Config.Parse.PreloadBuildDefs {
+ if err := fs.RecursiveCopy(preload, filepath.Join(nte.targetDir, preload), 0); err != nil {
+ log.Errorf("Failed to copy preloaded build def %s: %s", preload, err)
+ }
+ }
+
+ for _, target := range nte.state.Config.Parse.PreloadSubincludes {
+ targets := append(nte.state.Graph.TransitiveSubincludes(target), target)
+ nte.exportTargets(targets)
+ }
+}
+
+// exportTarget implements [exporterImpl].
+func (nte *noTrimExporter) exportTarget(target *core.BuildTarget) {
+ pkg := nte.getOrParsePackage(target.Label)
+ if pkg == nil {
+ log.Errorf("Unable to lookup package %s", target.Label)
+ return
+ }
+
+ if !nte.checkAndSetVisited(target) {
+ return
+ }
+
+ // We want to export the package that made this subrepo available, but we still need to walk the target deps
+ // as it may depend on other subrepos or first party targets
+ if target.Subrepo != nil {
+ nte.exportTarget(target.Subrepo.Target)
+ nte.exportDependencies(target)
+ return
+ }
+
+ nte.exportSubincludes(pkg)
+ nte.exportPackage(pkg)
+ nte.exportSources(target)
+ nte.exportDependencies(target)
+}
+
+// writePackageFiles implements [exporterImpl].
+func (nte *noTrimExporter) writePackageFiles() {
+ for pkgLabel := range nte.exportedPackages {
+ pkg := nte.getOrParsePackage(pkgLabel)
+ if pkg == nil {
+ log.Errorf("Unable to lookup package %s", pkgLabel)
+ continue
+ }
+
+ exportedFilename := filepath.Join(nte.targetDir, pkg.Filename)
+ if pkg.Subrepo != nil {
+ exportedFilename = filepath.Join(nte.targetDir, pkg.Subrepo.Dir(pkg.Filename))
+ }
+ if err := fs.CopyFile(pkg.Filename, exportedFilename, 0); err != nil {
+ log.Errorf("failed to export package %s: %v", pkg.Name, err)
+ }
+ }
+}
+
+// exportPackage exports the package BUILD file.
+func (nte *noTrimExporter) exportPackage(pkg *core.Package) {
+ // Skip subrepos and internal packages. These will be generated by build statements in the exported
+ // repo or included in please internally.
+ if pkg.Subrepo != nil || pkg.Name == parse.InternalPackageName {
+ return
+ }
+
+ if nte.exportedPackages[pkg.Label()] {
+ return
+ }
+ nte.exportedPackages[pkg.Label()] = true
+
+ // Export all the targets in the provided package.
+ for _, target := range pkg.AllTargets() {
+ nte.exportTarget(target)
+ }
+}
+
+// exportSubincludes exports the subincluded targets.
+func (nte *noTrimExporter) exportSubincludes(pkg *core.Package) {
+ subincludes := pkg.AllSubincludes(nte.state.Graph)
+ nte.exportTargets(subincludes)
+}
diff --git a/src/export/test_data/filter.build b/src/export/test_data/filter.build
new file mode 100644
index 0000000000..35f90c2265
--- /dev/null
+++ b/src/export/test_data/filter.build
@@ -0,0 +1,17 @@
+# This is a header comment
+package(default_visibility = ["PUBLIC"])
+
+VERSION = "1.2.3"
+
+target(
+ name = "a",
+ # This is a comment inside a target
+ srcs = ["a.py"],
+ version = VERSION,
+)
+
+# This is a middle comment
+target(
+ name = "b",
+ srcs = ["b.py"],
+)
diff --git a/src/export/test_data/filter_expected_a.build b/src/export/test_data/filter_expected_a.build
new file mode 100644
index 0000000000..af682d5dfb
--- /dev/null
+++ b/src/export/test_data/filter_expected_a.build
@@ -0,0 +1,14 @@
+# This is a header comment
+package(default_visibility = ["PUBLIC"])
+
+VERSION = "1.2.3"
+
+target(
+ name = "a",
+ # This is a comment inside a target
+ srcs = ["a.py"],
+ version = VERSION,
+)
+
+# This is a middle comment
+
diff --git a/src/export/test_data/filter_expected_b.build b/src/export/test_data/filter_expected_b.build
new file mode 100644
index 0000000000..7e6f18ca21
--- /dev/null
+++ b/src/export/test_data/filter_expected_b.build
@@ -0,0 +1,12 @@
+# This is a header comment
+package(default_visibility = ["PUBLIC"])
+
+VERSION = "1.2.3"
+
+
+
+# This is a middle comment
+target(
+ name = "b",
+ srcs = ["b.py"],
+)
diff --git a/src/export/test_data/filter_expected_none.build b/src/export/test_data/filter_expected_none.build
new file mode 100644
index 0000000000..9ee152a241
--- /dev/null
+++ b/src/export/test_data/filter_expected_none.build
@@ -0,0 +1,9 @@
+# This is a header comment
+package(default_visibility = ["PUBLIC"])
+
+VERSION = "1.2.3"
+
+
+
+# This is a middle comment
+
diff --git a/src/export/test_data/trim_elif.build b/src/export/test_data/trim_elif.build
new file mode 100644
index 0000000000..796b473b2e
--- /dev/null
+++ b/src/export/test_data/trim_elif.build
@@ -0,0 +1,6 @@
+if False:
+ genrule(name = "a")
+elif True:
+ genrule(name = "b")
+else:
+ genrule(name = "c")
diff --git a/src/export/test_data/trim_elif_expected_b.build b/src/export/test_data/trim_elif_expected_b.build
new file mode 100644
index 0000000000..c72c6e6131
--- /dev/null
+++ b/src/export/test_data/trim_elif_expected_b.build
@@ -0,0 +1,6 @@
+if False:
+ pass # Trimmed during export
+elif True:
+ genrule(name = "b")
+else:
+ pass # Trimmed during export
diff --git a/src/export/test_data/trim_for.build b/src/export/test_data/trim_for.build
new file mode 100644
index 0000000000..998e4a22aa
--- /dev/null
+++ b/src/export/test_data/trim_for.build
@@ -0,0 +1,2 @@
+for i in range(0,2):
+ genrule(name = "a")
diff --git a/src/export/test_data/trim_for_expected_a.build b/src/export/test_data/trim_for_expected_a.build
new file mode 100644
index 0000000000..998e4a22aa
--- /dev/null
+++ b/src/export/test_data/trim_for_expected_a.build
@@ -0,0 +1,2 @@
+for i in range(0,2):
+ genrule(name = "a")
diff --git a/src/export/test_data/trim_for_expected_none.build b/src/export/test_data/trim_for_expected_none.build
new file mode 100644
index 0000000000..c1a7b52be7
--- /dev/null
+++ b/src/export/test_data/trim_for_expected_none.build
@@ -0,0 +1,2 @@
+for i in range(0,2):
+ pass # Trimmed during export
diff --git a/src/export/test_data/trim_for_if.build b/src/export/test_data/trim_for_if.build
new file mode 100644
index 0000000000..d13cfd7592
--- /dev/null
+++ b/src/export/test_data/trim_for_if.build
@@ -0,0 +1,8 @@
+for i in [
+ "a",
+ "b",
+]:
+ if i == "a":
+ genrule(name = "a")
+ elif i == "b":
+ genrule(name = "b")
diff --git a/src/export/test_data/trim_for_if_expected_a.build b/src/export/test_data/trim_for_if_expected_a.build
new file mode 100644
index 0000000000..e8a8bba074
--- /dev/null
+++ b/src/export/test_data/trim_for_if_expected_a.build
@@ -0,0 +1,8 @@
+for i in [
+ "a",
+ "b",
+]:
+ if i == "a":
+ genrule(name = "a")
+ elif i == "b":
+ pass # Trimmed during export
diff --git a/src/export/test_data/trim_if.build b/src/export/test_data/trim_if.build
new file mode 100644
index 0000000000..1f41589c70
--- /dev/null
+++ b/src/export/test_data/trim_if.build
@@ -0,0 +1,2 @@
+if True:
+ genrule(name = "a", cmd = "echo a > $OUT", outs = ["a"])
diff --git a/src/export/test_data/trim_if_expected_a.build b/src/export/test_data/trim_if_expected_a.build
new file mode 100644
index 0000000000..1f41589c70
--- /dev/null
+++ b/src/export/test_data/trim_if_expected_a.build
@@ -0,0 +1,2 @@
+if True:
+ genrule(name = "a", cmd = "echo a > $OUT", outs = ["a"])
diff --git a/src/export/test_data/trim_if_expected_none.build b/src/export/test_data/trim_if_expected_none.build
new file mode 100644
index 0000000000..8b13789179
--- /dev/null
+++ b/src/export/test_data/trim_if_expected_none.build
@@ -0,0 +1 @@
+
diff --git a/src/export/trimmed_exporter.go b/src/export/trimmed_exporter.go
new file mode 100644
index 0000000000..bfd23209ef
--- /dev/null
+++ b/src/export/trimmed_exporter.go
@@ -0,0 +1,240 @@
+package export
+
+import (
+ "bytes"
+ "fmt"
+ "os"
+ "path/filepath"
+ "slices"
+
+ "github.com/thought-machine/please/src/core"
+ "github.com/thought-machine/please/src/fs"
+ "github.com/thought-machine/please/src/parse"
+ "github.com/thought-machine/please/src/parse/asp"
+)
+
+// trimmedExporter implements an exporter that trims packages to reach a minimal exported repo.
+type trimmedExporter struct {
+ *baseExporter
+ // visitedPackages maintains a record of the packages visited during the export process.
+ visitedPackages map[core.BuildLabel]bool
+ // requiredSubincludes maps packages to the subinclude labels they require.
+ requiredSubincludes map[core.BuildLabel]core.BuildLabels
+ // preloadedSubincludes tracks subincludes that are preloaded and don't need explicit export.
+ preloadedSubincludes map[core.BuildLabel]bool
+}
+
+func newTrimmedExporter(base *baseExporter) exporterImpl {
+ return &trimmedExporter{
+ baseExporter: base,
+ visitedPackages: map[core.BuildLabel]bool{},
+ requiredSubincludes: map[core.BuildLabel]core.BuildLabels{},
+ preloadedSubincludes: map[core.BuildLabel]bool{},
+ }
+}
+
+// exportPreloaded implements [exporterImpl].
+func (e *trimmedExporter) exportPreloaded() {
+ // Write any preloaded build defs
+ for _, preload := range e.state.Config.Parse.PreloadBuildDefs {
+ if err := fs.RecursiveCopy(preload, filepath.Join(e.targetDir, preload), 0); err != nil {
+ log.Fatalf("Failed to copy preloaded build def %s: %s", preload, err)
+ }
+ }
+
+ for _, target := range e.state.GetPreloadedSubincludes() {
+ targets := append(e.state.Graph.TransitiveSubincludes(target), target)
+ for _, t := range targets {
+ e.preloadedSubincludes[t] = true
+ }
+ e.exportTargets(targets)
+ }
+}
+
+// exportTarget implements [exporterImpl].
+func (e *trimmedExporter) exportTarget(target *core.BuildTarget) {
+ if !e.checkAndSetVisited(target) {
+ return
+ }
+
+ // Skip export for internal packages
+ if target.Label.PackageName == parse.InternalPackageName {
+ return
+ }
+ // We want to export the package that made this subrepo available, but we still need to walk the
+ // target deps as it may depend on other subrepos or first party targets
+ if target.Subrepo.IsExternal() {
+ e.exportTarget(target.Subrepo.Target)
+ e.exportDependencies(target)
+ return
+ }
+
+ e.exportSources(target)
+ e.exportDependencies(target)
+
+ pkg := e.getOrParsePackage(target.Label)
+ if pkg == nil {
+ log.Errorf("Unable to lookup package %s", target.Label)
+ return
+ }
+ e.exportSubincludes(pkg, target.Label)
+ e.exportRelatedTargets(pkg, target.Label)
+
+ if !e.visitedPackages[pkg.Label()] {
+ // Export subincluded targets required for other package statements, e.g. variable
+ // declaration, during the first visit of a package.
+ e.exportPackageRequirements(pkg)
+ e.visitedPackages[pkg.Label()] = true
+ }
+}
+
+// writePackageFiles implements [exporterImpl].
+func (e *trimmedExporter) writePackageFiles() {
+ p := asp.NewParserOnly()
+ for pkgLabel := range e.visitedPackages {
+ pkg := e.state.Graph.PackageOrDie(pkgLabel)
+ filteredBytes, err := e.trimPackage(p, pkg)
+ if err != nil {
+ log.Errorf("Failed to filter the build statements of package %s: %v", pkg.Label(), err)
+ continue
+ }
+
+ e.writeExportedPackageFile(pkg, trimNewlines(filteredBytes))
+ }
+}
+
+// exportSubincludes exports the subincluded targets required to generate the target and selects them to
+// later be written to the package as statements.
+func (e *trimmedExporter) exportSubincludes(pkg *core.Package, target core.BuildLabel) {
+ // Get the actively used subincludes of the target and propagate all transitive subincludes required
+ // by our used subinclude targets. FindRequiredSubincludes will report the required subincludes
+ // for this target at the package level but we need to propagate the subincluded targets inside
+ // build definitions since we are not trimming build_defs files.
+ usedSubincludes, err := pkg.Metadata.FindRequiredSubincludes(target)
+ if err != nil {
+ log.Fatalf("failed to find required subincludes for target %s: %s", target, err)
+ }
+ e.setPackageSubincludes(pkg, usedSubincludes)
+
+ allSubincludes := usedSubincludes
+ for _, sub := range usedSubincludes {
+ for _, trans := range e.state.Graph.TransitiveSubincludes(sub) {
+ if !slices.Contains(allSubincludes, trans) {
+ allSubincludes = append(allSubincludes, trans)
+ }
+ }
+ }
+
+ e.exportTargets(allSubincludes)
+}
+
+// exportPackageRequirements exports any extra package requirements, for example the subincluded
+// targets and files that are required by package but are not linked to any [core.BuildTarget].
+func (e *trimmedExporter) exportPackageRequirements(pkg *core.Package) {
+ subincludes, files := pkg.Metadata.FindPackageFileRequirements()
+ e.setPackageSubincludes(pkg, subincludes)
+ e.exportTargets(subincludes)
+ e.exportFiles(files)
+}
+
+// setPackageSubincludes marks the package-level required subincludes after the export. This will be
+// used for trimming subinclude statements with [trimmer].
+func (e *trimmedExporter) setPackageSubincludes(pkg *core.Package, subincludes core.BuildLabels) {
+ for _, subinclude := range subincludes {
+ // skip for preloaded subincludes, these are handled separately at the start to ensure they are
+ // exported even if not directly used by an exported target.
+ if e.preloadedSubincludes[subinclude] {
+ continue
+ }
+
+ pkgLabel := pkg.Label()
+ required := e.requiredSubincludes[pkgLabel]
+ if !slices.Contains(required, subinclude) {
+ required = append(required, subinclude)
+ }
+ e.requiredSubincludes[pkgLabel] = required
+ }
+}
+
+// exportRelatedTargets looks up and exports all build targets that were declared within the same
+// build statement (e.g., adjacent targets in build def) as the specified target. This ensures that
+// all co-defined targets are preserved in the exported BUILD file, preventing unresolved references
+// or partial declarations.
+func (e *trimmedExporter) exportRelatedTargets(pkg *core.Package, target core.BuildLabel) {
+ relatedTargets, err := pkg.Metadata.FindRelatedTargets(target)
+ if err != nil {
+ log.Fatalf("failed to find related targets for %s: %s", target, err)
+ }
+ e.exportTargets(relatedTargets)
+}
+
+// WriteExportedPackageFile creates a new package (BUILD) file in the exported dir and writes to it.
+func (e *trimmedExporter) writeExportedPackageFile(pkg *core.Package, content []byte) {
+ filename := pkg.Filename
+ exportedFilename := filepath.Join(e.targetDir, filename)
+ if pkg.Subrepo != nil { // Adjusting fo for local subrepos
+ exportedFilename = filepath.Join(e.targetDir, pkg.Subrepo.Dir(filename))
+ }
+ f, err := fs.OpenDirFile(exportedFilename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0664)
+ if err != nil {
+ log.Fatalf("Failed to create and open exported BUILD file for %s: %v", exportedFilename, err)
+ }
+ defer f.Close()
+
+ if _, err := f.Write(content); err != nil {
+ log.Errorf("Failed to write to exported BUILD file %s: %v", f.Name(), err)
+ }
+}
+
+// trimPackage filters the statements to be written to the exported BUILD file.
+func (e *trimmedExporter) trimPackage(p *asp.Parser, pkg *core.Package) ([]byte, error) {
+ filename := pkg.Filename
+ if pkg.Subrepo != nil { // Adjusting fo for local subrepos
+ filename = pkg.Subrepo.Dir(filename)
+ }
+
+ parsed, err := p.ParseFileOnly(filename)
+ if err != nil {
+ return nil, fmt.Errorf("Parsing original BUILD file: %v", err)
+ }
+
+ content, err := os.ReadFile(filename)
+ if err != nil {
+ return nil, fmt.Errorf("Opening original BUILD file: %v", err)
+ }
+
+ trimmer := trimmer{
+ origin: content,
+ pkg: pkg,
+ exporter: e,
+ // assuming max len of the original file to avoid reallocations.
+ bytes: make([]byte, 0, len(content)),
+ }
+ trimmer.trimBlock(parsed, 0, asp.Position(len(content)))
+
+ return trimmer.bytes, nil
+}
+
+// trimNewlines trims leading and trailing whitespace and collapses 3+ consecutive newlines to 2.
+func trimNewlines(b []byte) []byte {
+ trimmed := bytes.TrimSpace(b)
+ var pointer, newlines int
+ for _, val := range trimmed {
+ if val == '\n' {
+ newlines++
+ if newlines > 2 {
+ continue // Skip third (or more) consecutive newline
+ }
+ } else {
+ newlines = 0
+ }
+ trimmed[pointer] = val
+ pointer++
+ }
+ trimmed = trimmed[:pointer]
+
+ if len(trimmed) > 0 {
+ trimmed = append(trimmed, '\n') // Trailing newline
+ }
+ return trimmed
+}
diff --git a/src/export/trimmer.go b/src/export/trimmer.go
new file mode 100644
index 0000000000..1890748c4f
--- /dev/null
+++ b/src/export/trimmer.go
@@ -0,0 +1,225 @@
+package export
+
+import (
+ "slices"
+
+ "github.com/please-build/buildtools/build"
+
+ "github.com/thought-machine/please/src/core"
+ "github.com/thought-machine/please/src/parse/asp"
+)
+
+var passExpression = []byte("pass # Trimmed during export")
+
+// trimmer implements the filtering logic for statements in package files.
+type trimmer struct {
+ // origin are the bytes from the original package file.
+ origin []byte
+ // pkg references the package being trimmed.
+ pkg *core.Package
+ // bytes contain the content to be written after the trimming process.
+ bytes []byte
+ // exporter is used to lookup target related data from the export process, e.g. which targets are
+ // required.
+ exporter *trimmedExporter
+}
+
+// statementConsumer defines the type for methods used to visit each statement during a
+// file walk. The method accepts the currently interpreted statement.
+type statementConsumer func(*asp.Statement)
+
+// walkFile will walk the file from the specified start to end, consuming the statements and optionally writing non-statement bytes found along the way (e.g. comments and blank space).
+func (t *trimmer) walkFile(stmts []*asp.Statement, start, end asp.Position, consumer statementConsumer) {
+ // cursor tracks the position in a block that's being interpreted.
+ cursor := start
+ for _, stmt := range stmts {
+ // Write content that's between stmts (e.g. comments). We skip these while parsing so it won't
+ // be included in "parsedStmts" but we want the resulting BUILD file to include these.
+ if cursor < stmt.Pos {
+ t.copy(cursor, stmt.Pos)
+ }
+
+ // Consume stmt with specified method
+ consumer(stmt)
+
+ // Move the cursor to the end of the processed statement. The cursor will enable writing of lines
+ // that are not considered statements by the parser (e.g. comments, new lines).
+ cursor = stmt.EndPos
+ }
+
+ // Write the rest of the original file (non build targets)
+ t.copy(cursor, end)
+}
+
+// trimBlock visits all the statements in a block and trims undesired statements.
+func (t *trimmer) trimBlock(stmts []*asp.Statement, blockStart, blockEnd asp.Position) bool {
+ var written bool
+ t.walkFile(stmts, blockStart, blockEnd, func(stmt *asp.Statement) {
+ if stmt.If != nil {
+ if t.trimIf(stmt) {
+ written = true
+ }
+ } else if stmt.For != nil {
+ if t.trimFor(stmt) {
+ written = true
+ }
+ } else if stmt.Ident != nil && stmt.Ident.Name == "subinclude" {
+ t.trimSubinclude(stmt)
+ written = true
+ } else if targets := t.statementTargets(stmt); len(targets) > 0 {
+ // Meaning it is a build statement that creates build targets.
+ if t.anyExported(targets) {
+ t.copy(stmt.Pos, stmt.EndPos)
+ written = true
+ }
+ } else {
+ // Write every other statement.
+ // If the statement didn't generate any targets (e.g. variable assignments, package() calls),
+ // we keep it to ensure the BUILD file remains valid.
+ t.copy(stmt.Pos, stmt.EndPos)
+ written = true
+ }
+ })
+ if !written {
+ t.write(passExpression)
+ }
+ return written
+}
+
+// trimIf will trim an if-else statement by exporting only the required targets, but keeping the
+// if-else primitive -- a implementation decision to help understand the changes caused by an export
+// when using a source-control management (SCM) system.
+func (t *trimmer) trimIf(stmt *asp.Statement) bool {
+ type clause struct {
+ hStart, hEnd asp.Position
+ stmts []*asp.Statement
+ }
+
+ clauses := []clause{
+ {hStart: stmt.If.HeaderPos, hEnd: stmt.If.HeaderEndPos, stmts: stmt.If.Statements},
+ }
+ for _, elif := range stmt.If.Elif {
+ clauses = append(clauses,
+ clause{hStart: elif.HeaderPos, hEnd: elif.HeaderEndPos, stmts: elif.Statements})
+ }
+ if len(stmt.If.ElseStatements) > 0 {
+ clauses = append(clauses,
+ clause{hStart: stmt.If.ElseHeaderPos, hEnd: stmt.If.ElseHeaderEndPos, stmts: stmt.If.ElseStatements})
+ }
+
+ // In an if-else statement only the interpreted/evaluated block will generate targets, meaning
+ // that normally only one of the clauses is interpreted, however an if statement could be inside of
+ // a loop where the clause condition depends on the iteration meaning more than one clause
+ // could end up being interpreted. Because of that we'll visit all the clauses, writing only the
+ // required statements -- any clauses with interpreted statements should be visited.
+ var requiredClauses = make([]bool, len(clauses))
+ for i, c := range clauses {
+ if len(c.stmts) == 0 {
+ continue
+ }
+ bStmt := asp.NewBuildStatement(c.stmts[0])
+ required := t.pkg.Metadata.IsInterpretedStatement(bStmt)
+ requiredClauses[i] = required
+ }
+
+ for i, c := range clauses {
+ // Write clause header
+ t.copy(c.hStart, c.hEnd)
+
+ // Visit statements in block
+ end := stmt.EndPos
+ if i+1 < len(clauses) {
+ end = clauses[i+1].hStart
+ }
+ if requiredClauses[i] {
+ t.trimBlock(c.stmts, c.hEnd, end)
+ } else {
+ t.passBlock(c.stmts, c.hEnd, end)
+ }
+ }
+ return true
+}
+
+func (t *trimmer) trimFor(stmt *asp.Statement) bool {
+ if len(stmt.For.Statements) == 0 {
+ return false
+ }
+
+ hStart, hEnd := stmt.Pos, stmt.For.Statements[0].Pos
+ t.copy(hStart, hEnd)
+
+ t.trimBlock(stmt.For.Statements, hEnd, stmt.EndPos)
+ return true
+}
+
+func (t *trimmer) trimSubinclude(stmt *asp.Statement) {
+ bStmt := asp.NewBuildStatement(stmt)
+ stmtLabels := t.pkg.Metadata.GetSubincludedLabels(bStmt)
+ subStmt := t.minimalSubincludeStatement(stmt, stmtLabels)
+ t.write([]byte(subStmt))
+}
+
+// passBlock skips all ASP statements (keeping comments and blank space) and writes a single "pass"
+// primitive.
+func (t *trimmer) passBlock(stmts []*asp.Statement, blockStart, blockEnd asp.Position) {
+ var passWritten bool
+ t.walkFile(stmts, blockStart, blockEnd, func(s *asp.Statement) {
+ // When the trim results in an empty block, i.e. no statement are written, we write
+ // the "pass" primitive. This is useful when parsing inner blocks (e.g. if-else stmts).
+ if !passWritten {
+ passWritten = true
+ t.write(passExpression)
+ }
+ })
+}
+
+func (t *trimmer) statementTargets(stmt *asp.Statement) core.BuildLabels {
+ bStmt := asp.NewBuildStatement(stmt)
+ return t.pkg.Metadata.FindTargets(bStmt)
+}
+
+func (t *trimmer) anyExported(labels core.BuildLabels) bool {
+ required := slices.ContainsFunc(labels, func(l core.BuildLabel) bool {
+ return t.exporter.exportedTargets[l]
+ })
+ return required
+}
+
+// minimalSubincludeStatement generates a subinclude statement containing only the required labels.
+func (t *trimmer) minimalSubincludeStatement(stmt *asp.Statement, available core.BuildLabels) string {
+ var filteredLabels core.BuildLabels
+ required := t.exporter.requiredSubincludes[t.pkg.Label()]
+ for _, label := range available {
+ if slices.Contains(required, label) {
+ filteredLabels = append(filteredLabels, label)
+ }
+ }
+
+ if len(filteredLabels) == 0 {
+ return ""
+ }
+ if len(available) == len(filteredLabels) {
+ // If all labels are required just keep the statement as is.
+ return string(t.origin[stmt.Pos:stmt.EndPos])
+ }
+
+ call := &build.CallExpr{
+ X: &build.Ident{Name: "subinclude"},
+ }
+ for _, label := range filteredLabels {
+ call.List = append(call.List, &build.StringExpr{Value: label.ShortString(t.pkg.Label())})
+ }
+
+ return build.FormatString(call)
+}
+
+func (t *trimmer) copy(start, end asp.Position) {
+ if start < 0 || start > end || int(end) > len(t.origin) {
+ return
+ }
+ t.bytes = append(t.bytes, t.origin[start:end]...)
+}
+
+func (t *trimmer) write(bytes []byte) {
+ t.bytes = append(t.bytes, bytes...)
+}
diff --git a/src/gc/gc.go b/src/gc/gc.go
index 1bdc7463df..95eccfd475 100644
--- a/src/gc/gc.go
+++ b/src/gc/gc.go
@@ -173,7 +173,7 @@ func addTarget(graph *core.BuildGraph, m targetMap, target *core.BuildTarget) {
for _, dep := range target.Dependencies() {
addTarget(graph, m, dep)
}
- if target.Subrepo != nil && target.Subrepo.Target != nil {
+ if target.Subrepo.IsExternal() {
addTarget(graph, m, target.Subrepo.Target)
}
}
@@ -208,7 +208,7 @@ func publicDependencies(graph *core.BuildGraph, target *core.BuildTarget) []*cor
}
}
}
- if target.Subrepo != nil && target.Subrepo.Target != nil {
+ if target.Subrepo.IsExternal() {
ret = append(ret, target.Subrepo.Target)
}
return ret
diff --git a/src/output/interactive_display.go b/src/output/interactive_display.go
index 1940c9e590..9224ac44dd 100644
--- a/src/output/interactive_display.go
+++ b/src/output/interactive_display.go
@@ -100,9 +100,14 @@ func (d *interactiveDisplay) Update(targets []buildingTarget) {
d.maxRows, d.maxCols = cli.CurrentBackend.MaxDimensions()
d.moveToFirstLine()
d.printLines(targets)
- for _, line := range cli.CurrentBackend.Output() {
- d.printf("${ERASE_AFTER}%s\n", line)
+ logs := cli.CurrentBackend.Output()
+ if len(logs) > 0 {
+ d.printf("${ERASE_AFTER}Messages:\n")
d.lines++
+ for _, line := range logs {
+ d.printf("${ERASE_AFTER}%s\n", line)
+ d.lines++
+ }
}
// Clean out any lines that were visible last time but are not now.
if d.lines < d.lastLines {
diff --git a/src/parse/asp/builtins.go b/src/parse/asp/builtins.go
index 8be2cbc8e4..7476ddd154 100644
--- a/src/parse/asp/builtins.go
+++ b/src/parse/asp/builtins.go
@@ -210,6 +210,8 @@ func buildRule(s *scope, args []pyObject) pyObject {
s.Assert(s.pkg.Target(target.Label.Name) == nil, "Duplicate build target in %s: %s", s.pkg.Name, target.Label.Name)
populateTarget(s, target, args)
s.state.AddTarget(s.pkg, target)
+ s.pkg.Metadata.RegisterStatementTarget(target.Label, s.CurrentBuildStatement())
+
if s.Callback {
target.AddedPostBuild = true
}
@@ -307,7 +309,7 @@ func bazelLoad(s *scope, args []pyObject) pyObject {
}
filename = subrepo.Dir(filename)
}
- s.SetAll(s.interpreter.Subinclude(s, filename, l, false), false)
+ s.SetAllWithOrigin(s.interpreter.Subinclude(s, filename, l, false), false, &l)
return None
}
@@ -331,23 +333,13 @@ func subinclude(s *scope, args []pyObject) pyObject {
if s.contextPackage() == nil {
s.Error("cannot subinclude from this scope")
}
- var si []string
- for _, arg := range args {
- if l, ok := arg.(pyList); ok {
- for _, e := range l {
- if l, ok := e.(pyString); ok {
- si = append(si, string(l))
- } else {
- s.Error("cannot subinclude type %s", e.Type())
- }
- }
- } else if l, ok := arg.(pyString); ok {
- si = append(si, string(l))
- } else {
- s.Error("cannot subinclude type %s", arg.Type())
- }
+ stringArgs, err := subincludeArgs(args)
+ if err != nil {
+ s.Error("%s", err.Error())
}
- for _, arg := range si {
+
+ var labels = make(core.BuildLabels, 0, len(stringArgs))
+ for _, arg := range stringArgs {
label, annotation := core.SplitLabelAnnotation(arg)
t := subincludeTarget(s, s.parseLabelInContextPkg(label))
s.Assert(s.contextPackage().Label().CanSee(s.state, t), "Target %s isn't visible to be subincluded into %s", t.Label, s.contextPackage().Label())
@@ -366,12 +358,39 @@ func subinclude(s *scope, args []pyObject) pyObject {
outs = t.Outputs()
}
for _, out := range outs {
- s.SetAll(s.interpreter.Subinclude(s, filepath.Join(t.OutDir(), out), t.Label, false), false)
+ s.SetAllWithOrigin(s.interpreter.Subinclude(s, filepath.Join(t.OutDir(), out), t.Label, false), false, &t.Label)
}
+ labels = append(labels, t.Label)
+ }
+ s.metadataRegisterSubincludes(labels)
+ if s.pkg != nil {
+ s.pkg.Metadata.RegisterSubincludeStatement(s.CurrentBuildStatement())
}
return None
}
+// subincludeArgs extracts and validates the arguments from a subinclude() call. Acceptable values
+// are strings and lists of strings.
+func subincludeArgs(args []pyObject) ([]string, error) {
+ var si []string
+ for _, arg := range args {
+ if l, ok := arg.(pyList); ok {
+ for _, e := range l {
+ if l, ok := e.(pyString); ok {
+ si = append(si, string(l))
+ } else {
+ return nil, fmt.Errorf("cannot subinclude type %s", e.Type())
+ }
+ }
+ } else if l, ok := arg.(pyString); ok {
+ si = append(si, string(l))
+ } else {
+ return nil, fmt.Errorf("cannot subinclude type %s", arg.Type())
+ }
+ }
+ return si, nil
+}
+
// subincludeTarget returns the target for a subinclude() call to a label.
// It blocks until the target exists and is built.
func subincludeTarget(s *scope, l core.BuildLabel) *core.BuildTarget {
@@ -729,6 +748,7 @@ func glob(s *scope, args []pyObject) pyObject {
exclude = exclude[:len(exclude)-len(s.state.Config.Parse.BuildFileName)]
log.Fatalf("glob(include=%s, exclude=%s) in %s returned no files. If this is intended, set allow_empty=True on the glob.", include, exclude, s.pkg.Filename)
}
+ s.metadataRegisterFiles(s.pkg.Name, glob)
return fromStringList(glob)
}
diff --git a/src/parse/asp/grammar.go b/src/parse/asp/grammar.go
index 18084cabc0..1181b71deb 100644
--- a/src/parse/asp/grammar.go
+++ b/src/parse/asp/grammar.go
@@ -70,16 +70,22 @@ type ForStatement struct {
// An IfStatement implements the if-elif-else statement.
type IfStatement struct {
- Condition Expression
- Statements []*Statement
- Elif []IfStatementElif
- ElseStatements []*Statement
+ HeaderPos Position
+ HeaderEndPos Position
+ Condition Expression
+ Statements []*Statement
+ Elif []IfStatementElif
+ ElseHeaderPos Position
+ ElseHeaderEndPos Position
+ ElseStatements []*Statement
}
// An IfStatementElif holds an elif clause in the if-elif-else statement.
type IfStatementElif struct {
- Condition Expression
- Statements []*Statement
+ HeaderPos Position
+ HeaderEndPos Position
+ Condition Expression
+ Statements []*Statement
}
// An Argument represents an argument to a function definition.
diff --git a/src/parse/asp/grammar_parse.go b/src/parse/asp/grammar_parse.go
index d5bc049bd3..9f582f5c22 100644
--- a/src/parse/asp/grammar_parse.go
+++ b/src/parse/asp/grammar_parse.go
@@ -312,23 +312,25 @@ func (p *parser) parseArgument() Argument {
}
func (p *parser) parseIf() *IfStatement {
- p.nextv("if")
i := &IfStatement{}
+ i.HeaderPos = p.nextv("if").Pos
p.parseExpressionInPlace(&i.Condition)
- p.next(':')
+ i.HeaderEndPos = p.next(':').EndPos()
p.next(EOL)
i.Statements = p.parseStatements()
- for p.optionalv("elif") {
+ for p.l.Peek().Value == "elif" {
elif := IfStatementElif{}
+ elif.HeaderPos = p.nextv("elif").Pos
p.parseExpressionInPlace(&elif.Condition)
- p.next(':')
+ elif.HeaderEndPos = p.next(':').EndPos()
p.next(EOL)
elif.Statements = p.parseStatements()
i.Elif = append(i.Elif, elif)
}
- if p.optionalv("else") {
- p.next(':')
+ if p.l.Peek().Value == "else" {
+ i.ElseHeaderPos = p.nextv("else").Pos
+ i.ElseHeaderEndPos = p.next(':').EndPos()
p.next(EOL)
i.ElseStatements = p.parseStatements()
}
diff --git a/src/parse/asp/interpreter.go b/src/parse/asp/interpreter.go
index 93e4ca0a32..8919149942 100644
--- a/src/parse/asp/interpreter.go
+++ b/src/parse/asp/interpreter.go
@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"iter"
+ "path"
"path/filepath"
"reflect"
"regexp"
@@ -24,6 +25,8 @@ type interpreter struct {
parser *Parser
subincludes *cmap.ErrMap[string, pyDict]
asts *cmap.ErrMap[string, []*Statement]
+ // preloaded is a set to register all preloaded symbols (variables and function names).
+ preloaded *cmap.Map[string, struct{}]
configs map[*core.BuildState]*pyConfig
configsMutex sync.RWMutex
@@ -34,6 +37,8 @@ type interpreter struct {
stringMethods, dictMethods, configMethods map[string]*pyFunc
regexCache *cmap.Map[string, *regexp.Regexp]
+ // packageMetadata store scope level metadata for each package.
+ packageMetadata *cmap.Map[core.BuildLabel, scopeMetadata]
}
// newInterpreter creates and returns a new interpreter instance.
@@ -44,9 +49,12 @@ func newInterpreter(state *core.BuildState, p *Parser) *interpreter {
state: state,
locals: map[string]pyObject{},
}
+ s.metadata = &noopScopeMetadata{}
+
i := &interpreter{
scope: s,
parser: p,
+ preloaded: cmap.New[string, struct{}](cmap.SmallShardCount, cmap.XXHash),
configs: map[*core.BuildState]*pyConfig{},
limiter: make(semaphore, state.Config.Parse.NumThreads),
regexCache: cmap.New[string, *regexp.Regexp](cmap.SmallShardCount, cmap.XXHash),
@@ -59,6 +67,11 @@ func newInterpreter(state *core.BuildState, p *Parser) *interpreter {
i.subincludes = cmap.NewErrMap[string, pyDict](cmap.SmallShardCount, cmap.XXHash, i.limiter)
i.asts = cmap.NewErrMap[string, []*Statement](cmap.SmallShardCount, cmap.XXHash, i.limiter)
}
+
+ if state.ParseMetadata {
+ i.packageMetadata = cmap.New[core.BuildLabel, scopeMetadata](cmap.SmallShardCount, core.HashBuildLabel)
+ }
+
s.interpreter = i
s.LoadSingletons(state)
return i
@@ -155,11 +168,25 @@ func (i *interpreter) preloadSubinclude(s *scope, label core.BuildLabel) (err er
s.interpreter.loadPluginConfig(s, includeState)
for _, out := range t.FullOutputs() {
- s.SetAll(s.interpreter.Subinclude(s, out, t.Label, true), false)
+ globals := s.interpreter.Subinclude(s, out, t.Label, true)
+ s.interpreter.registerPreloaded(globals)
+ s.SetAllWithOrigin(globals, false, &t.Label)
}
return nil
}
+// registerPreloaded marks objects as preloaded for later reference.
+func (i *interpreter) registerPreloaded(d pyDict) {
+ for k := range d {
+ if k == "CONFIG" {
+ // Config will be set for each scope instance from global config. Skipping since every preload
+ // will override this value and will never use it.
+ continue
+ }
+ i.preloaded.Add(k, struct{}{})
+ }
+}
+
// interpretAll runs a series of statements in the scope of the given package.
// The first return value is for testing only.
func (i *interpreter) interpretAll(pkg *core.Package, forLabel, dependent *core.BuildLabel, mode core.ParseMode, statements []*Statement) (*scope, error) {
@@ -230,6 +257,8 @@ func (i *interpreter) Subinclude(pkgScope *scope, path string, label core.BuildL
mode |= core.ParseModeForPreload
}
s := i.scope.NewScope(path, mode)
+ // // Skip metadata tracking for subincludes since these are not trimmed
+ // s.metadata = &noopScopeMetadata{}
s.state = pkgScope.state
// Scope needs a local version of CONFIG
@@ -303,13 +332,16 @@ type scope struct {
pkg *core.Package
subincludeLabel *core.BuildLabel // If set, label of the subinclude we're currently interpreting
parsingFor *parseTarget
- parent *scope
- locals pyDict
- config *pyConfig
- globber *fs.Globber
+ // parent points to the lexical parent of this scope. It is used for variable resolution
+ // and is nil for the root scope.
+ parent *scope
+ locals pyDict
+ config *pyConfig
+ globber *fs.Globber
// True if this scope is for a pre- or post-build callback.
Callback bool
mode core.ParseMode
+ metadata scopeMetadata
}
// parseAnnotatedLabelInPackage similarly to parseLabelInPackage, parses the label contextualising it to the provided
@@ -397,7 +429,7 @@ func (s *scope) subincludePackage() *core.Package {
return pkg
}
// We're probably doing a local subinclude so the package isn't ready yet
- return core.NewPackageSubrepo(s.subincludeLabel.PackageName, s.subincludeLabel.Subrepo)
+ return core.NewPackage(s.subincludeLabel.PackageName, core.WithPackageSubrepo(s.subincludeLabel.Subrepo))
}
return nil
}
@@ -410,7 +442,9 @@ func (s *scope) NewScope(filename string, mode core.ParseMode) *scope {
// NewPackagedScope creates a new child scope of this one pointing to the given package.
// hint is a size hint for the new set of locals.
func (s *scope) NewPackagedScope(pkg *core.Package, mode core.ParseMode, hint int) *scope {
- return s.newScope(pkg, mode, pkg.Filename, hint)
+ newScope := s.newScope(pkg, mode, pkg.Filename, hint)
+ newScope.metadata = newScope.getOrNewMetadata(pkg)
+ return newScope
}
func (s *scope) newScope(pkg *core.Package, mode core.ParseMode, filename string, hint int) *scope {
@@ -426,6 +460,9 @@ func (s *scope) newScope(pkg *core.Package, mode core.ParseMode, filename string
config: s.config,
Callback: s.Callback,
mode: mode,
+ // We only track metadata at the top level scope created with [scope.NewPackagedScope], every
+ // other child scope or non-packaged (e.g. subincludes) defaults to a noop implementation.
+ metadata: &noopScopeMetadata{},
}
if pkg != nil && pkg.Subrepo != nil && pkg.Subrepo.State != nil {
s2.state = pkg.Subrepo.State
@@ -457,6 +494,8 @@ func (s *scope) NAssert(condition bool, msg string, args ...interface{}) {
// It panics if the variable is not defined.
func (s *scope) Lookup(name string) pyObject {
if obj, present := s.locals[name]; present {
+ orig := s.metadata.origin(s, name)
+ s.metadata.pushSymbol(name, orig)
return obj
} else if s.parent != nil {
return s.parent.Lookup(name)
@@ -480,6 +519,11 @@ func (s *scope) Set(name string, value pyObject) {
// SetAll sets all contents of the given dict in this scope.
// Optionally it can filter to just public objects (i.e. those not prefixed with an underscore)
func (s *scope) SetAll(d pyDict, publicOnly bool) {
+ s.SetAllWithOrigin(d, publicOnly, nil)
+}
+
+// SetAllWithOrigin is like SetAll but also records the origin label for all variables.
+func (s *scope) SetAllWithOrigin(d pyDict, publicOnly bool, origin *core.BuildLabel) {
for k, v := range d {
if k == "CONFIG" {
// Special case; need to merge config entries rather than overwriting the entire object.
@@ -488,6 +532,9 @@ func (s *scope) SetAll(d pyDict, publicOnly bool) {
s.config.Merge(c)
} else if !publicOnly || k[0] != '_' {
s.locals[k] = v
+ if origin != nil {
+ s.metadata.setSymbolOrigin(k, *origin)
+ }
}
}
}
@@ -525,23 +572,24 @@ func (s *scope) interpretStatements(statements []*Statement) pyObject {
}
}()
for _, stmt = range statements {
+ s.metadata.setCursor(stmt)
+ checkpoint := s.metadata.checkpoint()
+
+ var ret pyObject
if stmt.FuncDef != nil {
s.Set(stmt.FuncDef.Name, newPyFunc(s, stmt.FuncDef))
} else if stmt.If != nil {
- if ret := s.interpretIf(stmt.If); ret != nil {
- return ret
- }
+ ret = s.interpretIf(stmt.If)
} else if stmt.For != nil {
- if ret := s.interpretFor(stmt.For); ret != nil {
- return ret
- }
+ ret = s.interpretFor(stmt.For)
} else if stmt.Return != nil {
if len(stmt.Return.Values) == 0 {
- return None
+ ret = None
} else if len(stmt.Return.Values) == 1 {
- return s.interpretExpression(stmt.Return.Values[0])
+ ret = s.interpretExpression(stmt.Return.Values[0])
+ } else {
+ ret = pyList(s.evaluateExpressions(stmt.Return.Values))
}
- return pyList(s.evaluateExpressions(stmt.Return.Values))
} else if stmt.Ident != nil {
s.interpretIdentStatement(stmt.Ident)
} else if stmt.Assert != nil {
@@ -559,15 +607,22 @@ func (s *scope) interpretStatements(statements []*Statement) pyObject {
s.interpretExpression(stmt.Literal)
} else if stmt.Continue {
// This is definitely awkward since we need to control a for loop that's happening in a function outside this scope.
- return continueIteration
+ ret = continueIteration
} else if stmt.Break {
// Similar to above, although CPython does do this the same way...
- return stopIteration
+ ret = stopIteration
} else if stmt.Pass {
- continue // Nothing to do...
+ // Nothing to do...
} else {
s.Error("Unknown statement") // Shouldn't happen, amirite?
}
+
+ s.metadata.registerBuildStatement(s.pkg, stmt)
+ s.metadata.restore(checkpoint)
+
+ if ret != nil {
+ return ret
+ }
}
return nil
}
@@ -1077,6 +1132,19 @@ func (s *scope) Constant(expr *Expression) pyObject {
return nil
}
+// CurrentBuildStatement creates a provider for getting a BuildStatement from the statement
+// that is being currently interpreted. A closure is used to avoid unnecessary computation when the
+// metadata is not being tracked.
+func (s *scope) CurrentBuildStatement() core.BuildStatementProvider {
+ return func() core.BuildStatement {
+ // We lookup the package metadata from the interpreter table no matter how deep we are in the
+ // call stack. We do this to avoid a recursive lookup from leaf scopes to the top level package scope.
+ meta := s.interpreter.packageMetadata.Get(s.pkg.Label())
+ s.NAssert(meta.cursor() == nil, "Cursor is not pointing to a statement")
+ return NewBuildStatement(meta.cursor())
+ }
+}
+
// pkgFilename returns the filename of the current package, or the empty string if there is none.
func (s *scope) pkgFilename() string {
if s.pkg != nil {
@@ -1084,3 +1152,269 @@ func (s *scope) pkgFilename() string {
}
return ""
}
+
+// getOrNewMetadata creates and returns a initialized scopeMetadata instance, or pulls an existing
+// metadata instance for that package from the interpreter package metadata table. It will return
+// a no-op implementation if we simply want to skip tracking for a certain scope.
+func (s *scope) getOrNewMetadata(pkg *core.Package) scopeMetadata {
+ // Skip metadata tracking if:
+ // 1. ParseMetadata flag is disabled;
+ // 2. Not interpreting a package (e.g. in subincluded targets)
+ // 3. Any external/remote subrepos.
+ // For 2 and 3, we never trim these, so avoiding tracking saves CPU and memory.
+ var meta scopeMetadata = &noopScopeMetadata{}
+ if s.interpreter.packageMetadata == nil { // This will be initialized if ParseMetadata is set.
+ return meta
+ }
+
+ if pkg == nil || pkg.Subrepo.IsExternal() {
+ return meta
+ }
+
+ meta, _ = s.interpreter.packageMetadata.AddOrGet(pkg.Label(), func() scopeMetadata {
+ return &trackingScopeMetadata{
+ // symbolOrigins is lazy initialized in [setSymbolOrigin]
+ symbolStack: []trackedSymbol{},
+ }
+ })
+ return meta
+}
+
+// getPackageMetadata returns the metadata for that current scope's package.
+func (s *scope) getPackageMetadata() scopeMetadata {
+ if s.pkg == nil || s.interpreter.packageMetadata == nil { // This will be initialized if ParseMetadata is set.
+ return nil
+ }
+ return s.interpreter.packageMetadata.Get(s.pkg.Label())
+}
+
+// metadataRegisterFiles adds the files to the scope's package metadata.
+func (s *scope) metadataRegisterFiles(rootPath string, files []string) {
+ meta := s.getPackageMetadata()
+ if meta == nil {
+ return
+ }
+ meta.pushFiles(rootPath, files)
+}
+
+// metadataRegisterSubincludes adds the subincluded labels to the scope's package metadata.
+func (s *scope) metadataRegisterSubincludes(labels core.BuildLabels) {
+ meta := s.getPackageMetadata()
+ if meta == nil {
+ return
+ }
+ meta.pushSubincludes(labels)
+}
+
+// scopeMetadata defines an interface for tracking evaluation metadata (such as AST cursor position
+// and symbol subinclude origins) across interpreter scopes.
+// This is optionally used for operations (e.g. export) that require more details on the relation
+// between targets and statements. The no-op implementation should be used for most operations to
+// avoid any computational overhead.
+type scopeMetadata interface {
+ // cursor returns the statement currently being interpreted.
+ cursor() *Statement
+ // origin returns the subinclude origin label of a tracked symbol by name. Returns nil if the
+ // symbol is local (defined in the package) or has no origin/not tracked.
+ origin(scope *scope, name string) *core.BuildLabel
+ // setCursor registers the statement currently being interpreted.
+ setCursor(stmt *Statement)
+ // registerBuildStatement registers a new Build Statement in the given package. It will also
+ // register the required dependencies for interpreting that statement by looking up the required
+ // origins in the symbol stack.
+ registerBuildStatement(pkg *core.Package, stmt *Statement)
+ // setSymbolOrigin registers the subinclude origin label for a defined symbol.
+ setSymbolOrigin(name string, origin core.BuildLabel)
+ // checkpoint returns a checkpoint for the current tracking stacks.
+ checkpoint() metadataStackCheckpoint
+ // restore truncates the tracking stacks to the specified lengths from the checkpoint.
+ restore(cp metadataStackCheckpoint)
+ // pushSymbol pushes a symbol name and its subinclude origin onto the active tracking stack.
+ pushSymbol(name string, origin *core.BuildLabel)
+ // pushFiles pushes a slice of filenames onto the active tracking stack.
+ pushFiles(rootPath string, filenames []string)
+ // pushSubincludes pushes a label onto the tracking stack for subincludes.
+ pushSubincludes(labels core.BuildLabels)
+}
+
+// trackingScopeMetadata implements the interface [scopeMetadata].
+type trackingScopeMetadata struct {
+ // stmtCursor points to the statement currently being interpreted.
+ stmtCursor *Statement
+ // symbolOrigins tracks the subinclude label that each symbol was originally defined in.
+ symbolOrigins map[string]core.BuildLabel
+ // symbolStack tracks which symbols are actively in use during evaluation.
+ // Symbols are pushed onto the stack during lookups and truncated after each statement.
+ symbolStack []trackedSymbol
+ // fileStack track which files are required during the evaluation of the current statement.
+ // These are pushed during native calls such as glob() and truncated after each top level statement.
+ fileStack []string
+ // subincludesStack track subinclude calls dynamically made while evaluating a statement,
+ // this can happen for example when a custom target (function definition) subincludes a target as
+ // part of the function body. These are pushed during calls to subincludes() and truncated after
+ // each statement.
+ subincludesStack core.BuildLabels
+}
+
+type trackedSymbol struct {
+ name string
+ origin core.BuildLabel
+}
+
+// cursor implements [scopeMetadata].
+func (m *trackingScopeMetadata) cursor() *Statement {
+ return m.stmtCursor
+}
+
+// origin implements [scopeMetadata].
+func (m *trackingScopeMetadata) origin(scope *scope, name string) *core.BuildLabel {
+ if scope.interpreter != nil && scope.interpreter.preloaded.Contains(name) {
+ // Preloaded symbols are treated as local (returning nil origin) because they are implicitly
+ // available across all package scopes in the repository.
+ //
+ // This also prevents erroneous subinclude propagation: since symbol resolution recursively
+ // traverses the parent scope chain from bottom to top, where the preloaded symbols are
+ // defined at the top, if a target subincludes a preloaded target again it will be preferred
+ // over the preloaded and will potentially include unwanted symbols so we enforce a
+ // preference for the preloaded symbols. This could cause issues if our repo relies on
+ // redefining preloaded symbols.
+ return nil
+ }
+
+ if label, ok := m.symbolOrigins[name]; ok {
+ // Object subincluded into current scope.
+ return &label
+ }
+ // The origin for a local object is set to nil
+ return nil
+}
+
+// registerBuildStatement implements [scopeMetadata].
+func (m *trackingScopeMetadata) registerBuildStatement(pkg *core.Package, stmt *Statement) {
+ if pkg == nil || stmt == nil {
+ return
+ }
+
+ var deps core.BuildLabels
+ seen := make(map[core.BuildLabel]struct{})
+ for _, l := range m.subincludesStack {
+ if _, ok := seen[l]; !ok {
+ deps = append(deps, l)
+ seen[l] = struct{}{}
+ }
+ }
+ for _, v := range m.symbolStack {
+ l := v.origin
+ if _, ok := seen[l]; !ok {
+ deps = append(deps, l)
+ seen[l] = struct{}{}
+ }
+ }
+
+ pkg.Metadata.RegisterStatement(NewBuildStatement(stmt), deps, m.fileStack)
+}
+
+type metadataStackCheckpoint struct {
+ symbolCheckpoint, fileCheckpoint, subincludesCheckpoint int
+}
+
+// checkpoint implements [scopeMetadata].
+func (m *trackingScopeMetadata) checkpoint() metadataStackCheckpoint {
+ return metadataStackCheckpoint{
+ symbolCheckpoint: len(m.symbolStack),
+ fileCheckpoint: len(m.fileStack),
+ subincludesCheckpoint: len(m.subincludesStack),
+ }
+}
+
+// restore implements [scopeMetadata].
+func (m *trackingScopeMetadata) restore(cp metadataStackCheckpoint) {
+ if cp.symbolCheckpoint >= 0 && cp.symbolCheckpoint <= len(m.symbolStack) {
+ m.symbolStack = m.symbolStack[:cp.symbolCheckpoint]
+ }
+ if cp.fileCheckpoint >= 0 && cp.fileCheckpoint <= len(m.fileStack) {
+ m.fileStack = m.fileStack[:cp.fileCheckpoint]
+ }
+ if cp.subincludesCheckpoint >= 0 && cp.subincludesCheckpoint <= len(m.subincludesStack) {
+ m.subincludesStack = m.subincludesStack[:cp.subincludesCheckpoint]
+ }
+}
+
+// setCursor implements [scopeMetadata].
+func (m *trackingScopeMetadata) setCursor(stmt *Statement) {
+ m.stmtCursor = stmt
+}
+
+// setSymbolOrigin implements [scopeMetadata].
+func (m *trackingScopeMetadata) setSymbolOrigin(name string, origin core.BuildLabel) {
+ if m.symbolOrigins == nil {
+ // Lazy initialization to avoid unnecessary allocation of a map in smaller scopes (no subinclude).
+ m.symbolOrigins = map[string]core.BuildLabel{}
+ }
+
+ m.symbolOrigins[name] = origin
+}
+
+// pushSymbol implements [scopeMetadata].
+func (m *trackingScopeMetadata) pushSymbol(name string, origin *core.BuildLabel) {
+ if origin == nil {
+ return
+ }
+ m.symbolStack = append(m.symbolStack, trackedSymbol{name: name, origin: *origin})
+}
+
+// pushFiles implements [scopeMetadata].
+func (m *trackingScopeMetadata) pushFiles(rootPath string, filenames []string) {
+ for _, filename := range filenames {
+ m.fileStack = append(m.fileStack, path.Join(rootPath, filename))
+ }
+}
+
+// pushSubincludes implements [scopeMetadata].
+func (m *trackingScopeMetadata) pushSubincludes(labels core.BuildLabels) {
+ m.subincludesStack = append(m.subincludesStack, labels...)
+}
+
+// noopScopeMetadata implements the scopeMetadata interface with no-op methods. This is used to
+// avoid the overhead of storing metadata for operations that don't depend on it.
+type noopScopeMetadata struct{}
+
+// cursor implements [scopeMetadata].
+func (nm *noopScopeMetadata) cursor() *Statement { return nil }
+
+// origin implements [scopeMetadata].
+func (nm *noopScopeMetadata) origin(scope *scope, name string) *core.BuildLabel { return nil }
+
+// registerBuildStatement implements [scopeMetadata].
+func (nm *noopScopeMetadata) registerBuildStatement(pkg *core.Package, stmt *Statement) {}
+
+// checkpoint implements [scopeMetadata].
+func (nm *noopScopeMetadata) checkpoint() metadataStackCheckpoint {
+ return metadataStackCheckpoint{}
+}
+
+// restore implements [scopeMetadata].
+func (nm *noopScopeMetadata) restore(cp metadataStackCheckpoint) {}
+
+// setCursor implements [scopeMetadata].
+func (nm *noopScopeMetadata) setCursor(stmt *Statement) {}
+
+// setSymbolOrigin implements [scopeMetadata].
+func (nm *noopScopeMetadata) setSymbolOrigin(name string, origin core.BuildLabel) {}
+
+// pushSymbol implements [scopeMetadata].
+func (nm *noopScopeMetadata) pushSymbol(name string, origin *core.BuildLabel) {}
+
+// pushFiles implements [scopeMetadata].
+func (nm *noopScopeMetadata) pushFiles(rootPath string, filenames []string) {}
+
+// pushSubincludes implements [scopeMetadata].
+func (nm *noopScopeMetadata) pushSubincludes(labels core.BuildLabels) {}
+
+// NewBuildStatement creates a new core.BuildStatement from an asp.statement.
+func NewBuildStatement(stmt *Statement) core.BuildStatement {
+ return core.BuildStatement{
+ Start: int(stmt.Pos),
+ End: int(stmt.EndPos),
+ }
+}
diff --git a/src/parse/asp/interpreter_test.go b/src/parse/asp/interpreter_test.go
index 511f00b053..a0fdace269 100644
--- a/src/parse/asp/interpreter_test.go
+++ b/src/parse/asp/interpreter_test.go
@@ -770,3 +770,131 @@ func TestStrRjust(t *testing.T) {
_, err = parseFile("src/parse/asp/test_data/interpreter/str/rjust_multiple_fillchars.build")
assert.Error(t, err, "fillchar must be exactly one character long")
}
+
+func TestCurrentBuildStatement(t *testing.T) {
+ pkg := core.NewPackage("test/package", core.WithPackageMetadata())
+ pkg.Filename = "test/package/BUILD"
+
+ state := core.NewBuildState(core.DefaultConfiguration())
+ state.ParseMetadata = true
+
+ parser := &Parser{}
+ interpreter := newInterpreter(state, parser)
+
+ rootScope := interpreter.scope.NewPackagedScope(pkg, 0, 0)
+
+ // Root statement in the BUILD file (e.g. a macro call)
+ rootStmt := &Statement{Pos: 10, EndPos: 20}
+ rootScope.metadata.setCursor(rootStmt)
+
+ t.Run("FindsRootStatement", func(t *testing.T) {
+ stmt := rootScope.CurrentBuildStatement()()
+ assert.Equal(t, NewBuildStatement(rootStmt), stmt)
+ })
+}
+
+func TestActiveSubincludes(t *testing.T) {
+ labelA := core.ParseBuildLabel("//pkg:labelA", "")
+ labelB := core.ParseBuildLabel("//pkg:labelB", "")
+ stmt := &Statement{Pos: 1, EndPos: 10}
+
+ t.Run("NoSubincludes", func(t *testing.T) {
+ pkg := core.NewPackage("pkg", core.WithPackageMetadata())
+ meta := newScopeMetadata()
+
+ // Function execution
+ scopeFuncExec := &scope{
+ metadata: meta,
+ }
+ scopeFuncExec.metadata.setCursor(stmt)
+ scopeFuncExec.metadata.registerBuildStatement(pkg, stmt)
+
+ labels, _ := pkg.Metadata.FindPackageFileRequirements()
+ assert.Empty(t, labels)
+ })
+
+ t.Run("SingleSubinclude", func(t *testing.T) {
+ pkg := core.NewPackage("pkg", core.WithPackageMetadata())
+ meta := newScopeMetadata()
+
+ // File A scope
+ scopeA := &scope{
+ subincludeLabel: &labelA,
+ locals: make(pyDict),
+ metadata: meta,
+ }
+ scopeA.SetAllWithOrigin(pyDict{"foo": pyString("val")}, false, &labelA)
+
+ // Function defined in File A
+ scopeFuncDef := &scope{
+ parent: scopeA,
+ metadata: meta,
+ }
+
+ // Function execution
+ scopeFuncExec := &scope{
+ parent: scopeFuncDef,
+ metadata: meta,
+ }
+
+ // Lookup triggers tracking of required subincludes
+ scopeFuncExec.Lookup("foo")
+
+ scopeFuncExec.metadata.setCursor(stmt)
+ scopeFuncExec.metadata.registerBuildStatement(pkg, stmt)
+
+ labels, _ := pkg.Metadata.FindPackageFileRequirements()
+ assert.Equal(t, core.BuildLabels{labelA}, labels)
+ })
+
+ t.Run("NestedSubincludes", func(t *testing.T) {
+ pkg := core.NewPackage("pkg", core.WithPackageMetadata())
+ meta := newScopeMetadata()
+
+ // File A scope
+ scopeA := &scope{
+ subincludeLabel: &labelA,
+ locals: make(pyDict),
+ metadata: meta,
+ }
+ scopeA.SetAllWithOrigin(pyDict{"varA": pyString("valA")}, false, &labelA)
+
+ // File B scope (subincluded by A)
+ scopeB := &scope{
+ subincludeLabel: &labelB,
+ parent: scopeA,
+ locals: make(pyDict),
+ metadata: meta,
+ }
+ scopeB.SetAllWithOrigin(pyDict{"varB": pyString("valB")}, false, &labelB)
+
+ // Function defined in File B
+ scopeFuncDef := &scope{
+ parent: scopeB,
+ metadata: meta,
+ }
+
+ // Function execution
+ scopeFuncExec := &scope{
+ parent: scopeFuncDef,
+ metadata: meta,
+ }
+
+ // Lookups trigger tracking of required subincludes
+ scopeFuncExec.Lookup("varA")
+ scopeFuncExec.Lookup("varB")
+
+ scopeFuncExec.metadata.setCursor(stmt)
+ scopeFuncExec.metadata.registerBuildStatement(pkg, stmt)
+
+ labels, _ := pkg.Metadata.FindPackageFileRequirements()
+ assert.ElementsMatch(t, core.BuildLabels{labelA, labelB}, labels)
+ })
+}
+
+func newScopeMetadata() scopeMetadata {
+ return &trackingScopeMetadata{
+ symbolOrigins: map[string]core.BuildLabel{},
+ symbolStack: []trackedSymbol{},
+ }
+}
diff --git a/src/parse/asp/label_context_test.go b/src/parse/asp/label_context_test.go
index 9b99625bfc..023cb00906 100644
--- a/src/parse/asp/label_context_test.go
+++ b/src/parse/asp/label_context_test.go
@@ -12,7 +12,7 @@ import (
func newScope(pkgName, subrepo, plugin string) *scope {
s := &scope{
- pkg: core.NewPackageSubrepo(pkgName, subrepo),
+ pkg: core.NewPackage(pkgName, core.WithPackageSubrepo(subrepo)),
state: core.NewBuildState(core.DefaultConfiguration()),
}
if plugin != "" {
diff --git a/src/parse/asp/objects.go b/src/parse/asp/objects.go
index a783b55b5c..9a4b6536f4 100644
--- a/src/parse/asp/objects.go
+++ b/src/parse/asp/objects.go
@@ -694,15 +694,17 @@ func (f *pyFunc) String() string {
func (f *pyFunc) Call(s *scope, c *Call) pyObject {
if f.nativeCode != nil {
if f.kwargs {
- return f.callNative(s.NewScope("", 0), c)
+ cs := s.NewScope("", 0)
+ return f.callNative(cs, c)
}
return f.callNative(s, c)
}
- s2 := f.scope.newScope(s.pkg, s.mode, f.scope.filename, len(f.args)+1)
- s2.config = s.config
- s2.Set("CONFIG", s.config) // This needs to be copied across too :(
- s2.Callback = s.Callback
- s2.parsingFor = s.parsingFor
+
+ cs := f.scope.newScope(s.pkg, s.mode, f.scope.filename, len(f.args)+1)
+ cs.config = s.config
+ cs.Set("CONFIG", s.config) // This needs to be copied across too :(
+ cs.Callback = s.Callback
+ cs.parsingFor = s.parsingFor
// Handle implicit 'self' parameter for bound functions.
args := c.Arguments
if f.self != nil {
@@ -720,23 +722,23 @@ func (f *pyFunc) Call(s *scope, c *Call) pyObject {
if present {
name = f.args[idx]
}
- s2.Set(name, f.validateType(s, idx, &a.Value))
+ cs.Set(name, f.validateType(s, idx, &a.Value))
} else {
if i >= len(f.args) {
s.Error("Too many arguments to %s", f.name)
} else if f.kwargsonly {
s.Error("Function %s can only be called with keyword arguments", f.name)
}
- s2.Set(f.args[i], f.validateType(s, i, &a.Value))
+ cs.Set(f.args[i], f.validateType(s, i, &a.Value))
}
}
// Now make sure any arguments with defaults are set, and check any others have been passed.
for i, a := range f.args {
- if s2.LocalLookup(a) == nil {
- s2.Set(a, f.defaultArg(s, i, a))
+ if cs.LocalLookup(a) == nil {
+ cs.Set(a, f.defaultArg(s, i, a))
}
}
- ret := s2.interpretStatements(f.code)
+ ret := cs.interpretStatements(f.code)
if ret == nil {
return None // Implicit 'return None' in any function that didn't do that itself.
}
@@ -832,7 +834,8 @@ func (f *pyFunc) Member(obj pyObject) pyObject {
}
}
-// validateType validates that this argument matches the given type
+// validateType validates that this argument matches the given type. It interprets the expression and
+// returns its value.
func (f *pyFunc) validateType(s *scope, i int, expr *Expression) pyObject {
val := s.interpretExpression(expr)
if i >= len(f.types) && (f.varargs || f.kwargs) {
diff --git a/src/parse/asp/parser.go b/src/parse/asp/parser.go
index f67a8605ae..74a96f0475 100644
--- a/src/parse/asp/parser.go
+++ b/src/parse/asp/parser.go
@@ -35,14 +35,14 @@ type Parser struct {
// NewParser creates a new parser instance. One is normally sufficient for a process lifetime.
func NewParser(state *core.BuildState) *Parser {
- p := newParser()
+ p := NewParserOnly()
p.interpreter = newInterpreter(state, p)
p.limiter = p.interpreter.limiter
return p
}
-// newParser creates just the parser with no interpreter.
-func newParser() *Parser {
+// NewParserOnly creates just the parser with no interpreter.
+func NewParserOnly() *Parser {
return &Parser{
builtins: map[string][]byte{},
limiter: make(semaphore, 10),
diff --git a/src/parse/asp/parser_test.go b/src/parse/asp/parser_test.go
index fd534b31bc..1089bcc012 100644
--- a/src/parse/asp/parser_test.go
+++ b/src/parse/asp/parser_test.go
@@ -10,7 +10,7 @@ import (
// TODO(peterebden): Might get rid of this, we may want to expose a similar thing on Parser.
func parseFileOnly(filename string) (*File, []*Statement, error) {
- stmts, err := newParser().ParseFileOnly(filename)
+ stmts, err := NewParserOnly().ParseFileOnly(filename)
return newFile(filename), stmts, err
}
@@ -448,7 +448,7 @@ func TestAssert(t *testing.T) {
}
func TestOptimise(t *testing.T) {
- p := newParser()
+ p := NewParserOnly()
statements, err := p.parse(nil, "src/parse/asp/test_data/optimise.build")
f := newFile("src/parse/asp/test_data/optimise.build")
assert.NoError(t, err)
@@ -486,7 +486,7 @@ func TestOptimise(t *testing.T) {
}
func TestOptimiseJoin(t *testing.T) {
- p := newParser()
+ p := NewParserOnly()
statements, err := p.parse(nil, "src/parse/asp/test_data/optimise_join.build")
assert.NoError(t, err)
assert.Equal(t, 1, len(statements))
@@ -658,12 +658,12 @@ func TestMissingNewlines(t *testing.T) {
}
func TestRepeatedArguments(t *testing.T) {
- _, err := newParser().parse(nil, "src/parse/asp/test_data/repeated_arguments.build")
+ _, err := NewParserOnly().parse(nil, "src/parse/asp/test_data/repeated_arguments.build")
assert.Error(t, err)
}
func TestConstantAssignments(t *testing.T) {
- _, err := newParser().parse(nil, "src/parse/asp/test_data/constant_assign.build")
+ _, err := NewParserOnly().parse(nil, "src/parse/asp/test_data/constant_assign.build")
assert.Error(t, err)
}
@@ -815,7 +815,7 @@ func TestFStringConcat(t *testing.T) {
func TestFStringImplicitStringConcat(t *testing.T) {
str := "str('testing that we can carry these ' f'over {multiple} lines' r' \\n')"
- prog, err := newParser().parseAndHandleErrors(strings.NewReader(strings.ReplaceAll(str, "\t", "")))
+ prog, err := NewParserOnly().parseAndHandleErrors(strings.NewReader(strings.ReplaceAll(str, "\t", "")))
require.NoError(t, err)
fString := prog[0].Ident.Action.Call.Arguments[0].Value.Val.FString
@@ -827,21 +827,21 @@ func TestFStringImplicitStringConcat(t *testing.T) {
// F strings should report a sensible error when the {} aren't complete
func TestFStringIncompleteError(t *testing.T) {
str := "s = f'some {' '.join([])}'"
- _, err := newParser().parseAndHandleErrors(strings.NewReader(str))
+ _, err := NewParserOnly().parseAndHandleErrors(strings.NewReader(str))
require.Error(t, err)
assert.Contains(t, err.Error(), "Unterminated brace in fstring")
}
// Continue shouldn't be allowed outside a loop
func TestContinueOutsideLoop(t *testing.T) {
- _, err := newParser().parseAndHandleErrors(strings.NewReader("continue"))
+ _, err := NewParserOnly().parseAndHandleErrors(strings.NewReader("continue"))
require.Error(t, err)
assert.Contains(t, err.Error(), "'continue' outside loop")
}
// Break shouldn't be allowed outside a loop
func TestBreakOutsideLoop(t *testing.T) {
- _, err := newParser().parseAndHandleErrors(strings.NewReader("break"))
+ _, err := NewParserOnly().parseAndHandleErrors(strings.NewReader("break"))
require.Error(t, err)
assert.Contains(t, err.Error(), "'break' outside loop")
}
@@ -853,7 +853,7 @@ for i in [1,2,3]:
def foo():
break
`
- _, err := newParser().parseAndHandleErrors(strings.NewReader(code))
+ _, err := NewParserOnly().parseAndHandleErrors(strings.NewReader(code))
require.Error(t, err)
assert.Contains(t, err.Error(), "'break' outside loop")
}
diff --git a/src/parse/parse_step.go b/src/parse/parse_step.go
index 2c9689feab..e88a8228ef 100644
--- a/src/parse/parse_step.go
+++ b/src/parse/parse_step.go
@@ -68,7 +68,7 @@ func parse(state *core.BuildState, label, dependent core.BuildLabel, mode core.P
// If we get here then it falls to us to parse this package.
state.LogParseResult(label, core.PackageParsing, "Parsing...")
- if subrepo != nil && subrepo.Target != nil {
+ if subrepo.IsExternal() {
// We have got the definition of the subrepo, but it depends on something, make sure that has been built.
state.WaitForBuiltTarget(subrepo.Target.Label, label, mode|core.ParseModeForSubinclude)
if !subrepo.Target.State().IsBuilt() {
@@ -182,7 +182,12 @@ func maybeParseSubrepoPackage(state *core.BuildState, subrepoPkg, subrepoSubrepo
// parsePackage parses a BUILD file and adds the package to the build graph
func parsePackage(state *core.BuildState, label, dependent core.BuildLabel, subrepo *core.Subrepo, mode core.ParseMode) (*core.Package, error) {
packageName := label.PackageName
- pkg := core.NewPackage(packageName)
+ var opts []core.PackageOptions
+ if state.ParseMetadata && !subrepo.IsExternal() {
+ // Skip metadata tracking for external subrepos since these are always used as is and never trimmed.
+ opts = append(opts, core.WithPackageMetadata())
+ }
+ pkg := core.NewPackage(packageName, opts...)
pkg.Subrepo = subrepo
var fileSystem iofs.FS = fs.HostFS
if subrepo != nil {
diff --git a/src/please.go b/src/please.go
index 5eda69dfed..d741104c7c 100644
--- a/src/please.go
+++ b/src/please.go
@@ -462,6 +462,17 @@ var opts struct {
Options []string `positional-arg-name:"options" description:"Print specific options."`
} `positional-args:"true"`
} `command:"config" description:"Prints the configuration settings"`
+ Metadata struct {
+ Sources bool `long:"sources" short:"s" description:"Include target sources in visualization"`
+ Deps bool `long:"deps" short:"d" description:"Include target dependencies in visualization"`
+ Outputs bool `long:"outputs" short:"o" description:"Include target outputs in visualization"`
+ All bool `long:"all" short:"a" description:"Include all details (sources, deps, and outputs)"`
+ AllStatements bool `long:"all_statements" description:"Print all statements in the BUILD file, including those that didn't generate the specified targets"`
+ Hidden bool `long:"hidden" description:"Show hidden targets"`
+ Args struct {
+ Targets []core.BuildLabel `positional-arg-name:"targets" description:"Targets or packages to display metadata for" required:"true"`
+ } `positional-args:"true" required:"true"`
+ } `command:"metadata" description:"Prints all metadata (code statements, generated targets and their required subincludes/files) of a package."`
} `command:"query" description:"Queries information about the build state"`
Generate struct {
Gitignore string `long:"update_gitignore" description:"The gitignore file to write the generated sources to"`
@@ -475,7 +486,7 @@ var opts struct {
// Functions are called after args are parsed and return a POSIX exit code (0 means success).
var buildFunctions = map[string]func() int{
"build": func() int {
- success, state := runBuild(opts.Build.Args.Targets, true, false, false)
+ success, state := runBuild(opts.Build.Args.Targets, buildOpts{Build: true})
if !success || opts.Build.OutDir == "" {
return toExitCode(success, state)
}
@@ -499,7 +510,7 @@ var buildFunctions = map[string]func() int{
if opts.Hash.Update {
opts.BehaviorFlags.NoHashVerification = true
}
- success, state := runBuild(opts.Hash.Args.Targets, true, false, false)
+ success, state := runBuild(opts.Hash.Args.Targets, buildOpts{Build: true})
if success {
if opts.Hash.Detailed {
for _, target := range state.ExpandOriginalLabels() {
@@ -556,14 +567,14 @@ var buildFunctions = map[string]func() int{
return toExitCode(success, state)
},
"debug": func() int {
- success, state := runBuild([]core.BuildLabel{opts.Debug.Args.Target}, true, false, false)
+ success, state := runBuild([]core.BuildLabel{opts.Debug.Args.Target}, buildOpts{Build: true})
if !success {
return toExitCode(success, state)
}
return debug.Debug(state, opts.Debug.Args.Target, opts.Debug.Args.Args, exec.ConvertEnv(opts.Debug.Env), opts.Debug.Share.Network, opts.Debug.Share.Mount)
},
"exec": func() int {
- success, state := runBuild([]core.BuildLabel{opts.Exec.Args.Target.BuildLabel}, true, false, false)
+ success, state := runBuild([]core.BuildLabel{opts.Exec.Args.Target.BuildLabel}, buildOpts{Build: true})
if !success {
return toExitCode(success, state)
}
@@ -594,7 +605,7 @@ var buildFunctions = map[string]func() int{
if len(unannotated) == 0 {
return 0
}
- success, state := runBuild(unannotated, true, false, false)
+ success, state := runBuild(unannotated, buildOpts{Build: true})
if !success {
return toExitCode(success, state)
}
@@ -608,7 +619,7 @@ var buildFunctions = map[string]func() int{
if len(unannotated) == 0 {
return 0
}
- success, state := runBuild(unannotated, true, false, false)
+ success, state := runBuild(unannotated, buildOpts{Build: true})
if !success {
return toExitCode(success, state)
}
@@ -618,7 +629,7 @@ var buildFunctions = map[string]func() int{
return 0
},
"run": func() int {
- if success, state := runBuild([]core.BuildLabel{opts.Run.Args.Target.BuildLabel}, true, false, false); success {
+ if success, state := runBuild([]core.BuildLabel{opts.Run.Args.Target.BuildLabel}, buildOpts{Build: true}); success {
var dir string
if opts.Run.WD != "" {
dir = getAbsolutePath(opts.Run.WD, originalWorkingDirectory)
@@ -642,7 +653,7 @@ var buildFunctions = map[string]func() int{
if len(unannotated) == 0 {
return 0
}
- if success, state := runBuild(unannotated, true, false, false); success {
+ if success, state := runBuild(unannotated, buildOpts{Build: true}); success {
var dir string
if opts.Run.WD != "" {
dir = getAbsolutePath(opts.Run.WD, originalWorkingDirectory)
@@ -659,7 +670,7 @@ var buildFunctions = map[string]func() int{
if len(unannotated) == 0 {
return 0
}
- if success, state := runBuild(unannotated, true, false, false); success {
+ if success, state := runBuild(unannotated, buildOpts{Build: true}); success {
var dir string
if opts.Run.WD != "" {
dir = getAbsolutePath(opts.Run.WD, originalWorkingDirectory)
@@ -686,7 +697,7 @@ var buildFunctions = map[string]func() int{
}
opts.Clean.Args.Targets = core.WholeGraph
}
- if success, state := runBuild(opts.Clean.Args.Targets, false, false, false); success {
+ if success, state := runBuild(opts.Clean.Args.Targets, buildOpts{}); success {
clean.Targets(state, state.ExpandOriginalLabels())
return 0
}
@@ -708,7 +719,7 @@ var buildFunctions = map[string]func() int{
return 1
},
"gc": func() int {
- success, state := runBuild(core.WholeGraph, false, false, true)
+ success, state := runBuild(core.WholeGraph, buildOpts{IsQuery: true})
if success {
gc.GarbageCollect(state, opts.Gc.Args.Targets, state.ExpandLabels(state.Config.Gc.Keep), state.Config.Gc.Keep, state.Config.Gc.KeepLabel,
opts.Gc.Conservative, opts.Gc.TargetsOnly, opts.Gc.SrcsOnly, opts.Gc.NoPrompt, opts.Gc.DryRun, opts.Gc.Git)
@@ -767,14 +778,18 @@ var buildFunctions = map[string]func() int{
return 0
},
"export": func() int {
- success, state := runBuild(opts.Export.Args.Targets, false, false, false)
+ success, state := runBuild(opts.Export.Args.Targets, buildOpts{ParseMetadata: true, KeepParserRunning: true})
+ // Required cleanup due to running parser in background
+ defer state.Cleanup()
+
if success {
- export.ToDir(state, opts.Export.Output, opts.Export.NoTrim, state.ExpandOriginalLabels())
+ export.Repo(state, opts.Export.Output, opts.Export.NoTrim, state.ExpandOriginalLabels())
}
+
return toExitCode(success, state)
},
"export.outputs": func() int {
- success, state := runBuild(opts.Export.Outputs.Args.Targets, true, false, true)
+ success, state := runBuild(opts.Export.Outputs.Args.Targets, buildOpts{Build: true, IsQuery: true})
if success {
export.Outputs(state, opts.Export.Output, state.ExpandOriginalLabels())
}
@@ -832,6 +847,20 @@ var buildFunctions = map[string]func() int{
query.TargetOutputs(state.Graph, state.ExpandOriginalLabels(), opts.Query.Output.JSON)
})
},
+ "query.metadata": func() int {
+ if success, state := runBuild(opts.Query.Metadata.Args.Targets, buildOpts{ParseMetadata: true, IsQuery: true}); success {
+ m := opts.Query.Metadata
+ query.Metadata(state, state.ExpandOriginalLabels(), query.WriteMetadataOpts{
+ IncludeSources: m.Sources || m.All,
+ IncludeDeps: m.Deps || m.All,
+ IncludeOutputs: m.Outputs || m.All,
+ IncludeAllStatements: m.AllStatements,
+ ShowHidden: m.Hidden,
+ })
+ return 0
+ }
+ return 1
+ },
"query.completions": func() int {
// Somewhat fiddly because the inputs are not necessarily well-formed at this point.
opts.ParsePackageOnly = true
@@ -962,14 +991,14 @@ var buildFunctions = map[string]func() int{
log.Fatalf("%s", err)
}
readConfig()
- _, before := runBuild(core.WholeGraph, false, false, false)
+ _, before := runBuild(core.WholeGraph, buildOpts{})
// N.B. Ignore failure here; if we can't parse the graph before then it will suffice to
// assume that anything we don't know about has changed.
if err := scm.Checkout(original); err != nil {
log.Fatalf("%s", err)
}
readConfig()
- success, after := runBuild(core.WholeGraph, false, false, false)
+ success, after := runBuild(core.WholeGraph, buildOpts{})
if !success {
return 1
}
@@ -1001,7 +1030,7 @@ var buildFunctions = map[string]func() int{
"watch": func() int {
targets, args := testTargets(opts.Watch.Args.Target, opts.Watch.Args.Args, false, "")
// Don't ask it to test now since we don't know if any of them are tests yet.
- success, state := runBuild(targets, true, false, false)
+ success, state := runBuild(targets, buildOpts{Build: true})
state.NeedRun = opts.Watch.Run
watch.Watch(state, state.ExpandOriginalLabels(), args, opts.Watch.NoTest, runPlease)
return toExitCode(success, state)
@@ -1026,7 +1055,7 @@ var buildFunctions = map[string]func() int{
opts.Generate.Args.Targets = []core.BuildLabel{target}
}
- if success, state := runBuild(opts.Generate.Args.Targets, true, false, true); success {
+ if success, state := runBuild(opts.Generate.Args.Targets, buildOpts{Build: true, IsQuery: true}); success {
if opts.Generate.Gitignore != "" {
err := generate.UpdateGitignore(state.Graph, state.ExpandOriginalLabels(), opts.Generate.Gitignore)
if err != nil {
@@ -1068,7 +1097,7 @@ func runTool(_tool tool.Tool) int {
// We skip loading the repo config in init for `plz tool` to allow this command to work outside of a repo root. If
// the tool looks like a build label, we need to set the repo root now.
config = mustReadConfigAndSetRoot(false)
- if success, state := runBuild(label, true, false, false); success {
+ if success, state := runBuild(label, buildOpts{Build: true}); success {
annotatedOutputLabels := core.AnnotateLabels(label)
run.Run(state, annotatedOutputLabels[0], opts.Tool.Args.Args.AsStrings(), false, false, false, "", "")
}
@@ -1100,7 +1129,7 @@ func runQuery(needFullParse bool, labels []core.BuildLabel, onSuccess func(state
if len(labels) == 0 {
labels = core.WholeGraph
}
- if success, state := runBuild(labels, false, false, true); success {
+ if success, state := runBuild(labels, buildOpts{IsQuery: true}); success {
onSuccess(state)
return 0
}
@@ -1112,7 +1141,7 @@ func doTest(targets []core.BuildLabel, args []string, surefireDir cli.Filepath,
fs.RemoveAll(string(resultsFile))
os.MkdirAll(string(surefireDir), core.DirPermissions)
opts.Test.StateArgs = args
- success, state := runBuild(targets, true, true, false)
+ success, state := runBuild(targets, buildOpts{Build: true, Test: true})
test.CopySurefireXMLFilesToDir(state, string(surefireDir))
test.WriteResultsToFileOrDie(state.Graph, string(resultsFile), state.Config.Test.StoreTestOutputOnSuccess)
return success, state
@@ -1127,7 +1156,7 @@ func prettyOutput(interactiveOutput bool, plainOutput bool, verbosity cli.Verbos
}
// Please starts & runs the main build process through to its completion.
-func Please(targets []core.BuildLabel, config *core.Configuration, shouldBuild, shouldTest bool) (bool, *core.BuildState) {
+func Please(targets []core.BuildLabel, config *core.Configuration, buildOpts buildOpts) (bool, *core.BuildState) {
if opts.BuildFlags.NumThreads > 0 {
config.Please.NumThreads = opts.BuildFlags.NumThreads
config.Parse.NumThreads = opts.BuildFlags.NumThreads
@@ -1150,8 +1179,6 @@ func Please(targets []core.BuildLabel, config *core.Configuration, shouldBuild,
state.TestSequentially = opts.Test.Sequentially || opts.Cover.Sequentially // Similarly here.
state.TestArgs = opts.Test.StateArgs
state.NeedCoverage = opts.Cover.active || config.Build.Config == "cover"
- state.NeedBuild = shouldBuild
- state.NeedTests = shouldTest
state.NeedRun = !opts.Run.Args.Target.IsEmpty() || len(opts.Run.Parallel.PositionalArgs.Targets) > 0 || len(opts.Run.Sequential.PositionalArgs.Targets) > 0 || !opts.Exec.Args.Target.IsEmpty() || len(opts.Exec.Sequential.Args.Targets) > 0 || len(opts.Exec.Parallel.Args.Targets) > 0 || opts.Tool.Args.Tool != "" || debug
state.NeedHashesOnly = len(opts.Hash.Args.Targets) > 0
state.PrepareOnly = opts.Build.Shell != "" || opts.Test.Shell != "" || opts.Cover.Shell != ""
@@ -1165,6 +1192,10 @@ func Please(targets []core.BuildLabel, config *core.Configuration, shouldBuild,
state.ShowAllOutput = opts.OutputFlags.ShowAllOutput
state.ParsePackageOnly = opts.ParsePackageOnly
state.EnableBreakpoints = opts.BehaviorFlags.Debug
+ state.NeedBuild = buildOpts.Build
+ state.NeedTests = buildOpts.Test
+ state.ParseMetadata = buildOpts.ParseMetadata
+ state.KeepParserRunning = buildOpts.KeepParserRunning
state.NeedDebugDeps = debug
// What outputs get downloaded in remote execution.
@@ -1189,9 +1220,6 @@ func Please(targets []core.BuildLabel, config *core.Configuration, shouldBuild,
}
runPlease(state, targets)
- if state.RemoteClient != nil && !opts.Run.Remote {
- defer state.RemoteClient.Disconnect()
- }
failures, _, _ := state.Failures()
return !failures, state
}
@@ -1224,8 +1252,8 @@ func runPlease(state *core.BuildState, targets []core.BuildLabel) {
output.MonitorState(state, !pretty, detailedTests, streamTests, shell, shellRun, string(opts.OutputFlags.TraceFile))
wg.Done()
}()
+ state.WaitForDisplay = wg.Wait
plz.Run(targets, opts.BuildFlags.PreTargets, state, config, state.TargetArch)
- wg.Wait()
}
// testTargets handles test targets which can be given in two formats; a list of targets or a single
@@ -1317,10 +1345,22 @@ func readConfig() *core.Configuration {
return cfg
}
+// buildOpts specifies parameter for the core.runBuild method.
+type buildOpts struct {
+ Build bool
+ Test bool
+ IsQuery bool
+ ParseMetadata bool
+ // Keep the workers running in the background for inline parsing during specific operations (e.g.
+ // export). Note: when running background workers we need to explicit call CleanUp at the end of
+ // the CLI operation.
+ KeepParserRunning bool
+}
+
// Runs the actual build
// Which phases get run are controlled by shouldBuild and shouldTest.
-func runBuild(targets []core.BuildLabel, shouldBuild, shouldTest, isQuery bool) (bool, *core.BuildState) {
- if !isQuery {
+func runBuild(targets []core.BuildLabel, buildOpts buildOpts) (bool, *core.BuildState) {
+ if !buildOpts.IsQuery {
opts.BuildFlags.Exclude = append(opts.BuildFlags.Exclude, "manual", "manual:"+core.OsArch)
}
if stat, _ := os.Stdin.Stat(); (stat.Mode()&os.ModeCharDevice) == 0 && !plz.ReadingStdin(targets) {
@@ -1332,7 +1372,7 @@ func runBuild(targets []core.BuildLabel, shouldBuild, shouldTest, isQuery bool)
if len(targets) == 0 {
targets = core.InitialPackage()
}
- return Please(targets, config, shouldBuild, shouldTest)
+ return Please(targets, config, buildOpts)
}
var originalWorkingDirectory string
@@ -1430,7 +1470,7 @@ func getCompletions(qry string) (*query.CompletionPackages, []string) {
if completions.PackageToParse != "" || completions.IsRoot {
labelsToParse := []core.BuildLabel{{PackageName: completions.PackageToParse, Name: "all"}}
- if success, state := Please(labelsToParse, config, false, false); success {
+ if success, state := Please(labelsToParse, config, buildOpts{}); success {
return completions, query.Completions(state.Graph, completions, binary, isTest, completions.Hidden)
}
}
diff --git a/src/plz/plz.go b/src/plz/plz.go
index 7c893a8165..6114ec1c56 100644
--- a/src/plz/plz.go
+++ b/src/plz/plz.go
@@ -111,15 +111,27 @@ func Run(targets, preTargets []core.BuildLabel, state *core.BuildState, config *
wg.Done()
}()
// Wait until they've all exited, which they'll do once they have no tasks left.
- wg.Wait()
- if state.Cache != nil {
- state.Cache.Shutdown()
+ if state.KeepParserRunning {
+ // Even though we keep the workers running, we wait for the initial build to finish before
+ // proceeding with the specific op for the graph to be complete.
+ state.WaitForBuildToComplete()
+ reportResults(state, config)
+ // state.CleanUp() needs to be called at the end of the CLI run when KeepParserRunning is enabled.
+ return
}
+ // Wait for all worker to finish. This should happen as soon as we no longer have pending tasks.
+ // The last task should call state.Close() and close the queue channels.
+ wg.Wait()
+ reportResults(state, config)
+ state.Cleanup()
+}
+
+// reportResults reports on metrics and results at the end of the build.
+func reportResults(state *core.BuildState, config *core.Configuration) {
if state.RemoteClient != nil {
_, _, in, out := state.RemoteClient.DataRate()
log.Info("Total remote RPC data in: %d out: %d", in, out)
}
- state.CloseResults()
metrics.Push(config.Metrics, config.IsRemoteExecution())
}
diff --git a/src/query/BUILD b/src/query/BUILD
index 310bd21794..26a780b6a0 100644
--- a/src/query/BUILD
+++ b/src/query/BUILD
@@ -10,6 +10,7 @@ go_library(
"///third_party/go/github.com_please-build_gcfg//:gcfg",
"///third_party/go/golang.org_x_exp//maps",
"//src/build",
+ "//src/cli",
"//src/cli/logging",
"//src/core",
"//src/fs",
diff --git a/src/query/changes_test.go b/src/query/changes_test.go
index ed4124b684..d3806a4527 100644
--- a/src/query/changes_test.go
+++ b/src/query/changes_test.go
@@ -160,7 +160,7 @@ func addTarget(state *core.BuildState, label string, dep *core.BuildTarget, sour
}
pkg := state.Graph.PackageByLabel(t.Label)
if pkg == nil {
- pkg = core.NewPackageSubrepo(t.Label.PackageName, t.Label.Subrepo)
+ pkg = core.NewPackage(t.Label.PackageName, core.WithPackageSubrepo(t.Label.Subrepo))
state.Graph.AddPackage(pkg)
}
pkg.AddTarget(t)
diff --git a/src/query/metadata.go b/src/query/metadata.go
new file mode 100644
index 0000000000..beb6d9e1d1
--- /dev/null
+++ b/src/query/metadata.go
@@ -0,0 +1,221 @@
+package query
+
+import (
+ "fmt"
+ "os"
+ "strings"
+
+ "github.com/thought-machine/please/src/cli"
+ "github.com/thought-machine/please/src/core"
+)
+
+// WriteMetadataOpts contains configuration options for formatting and writing package metadata.
+type WriteMetadataOpts struct {
+ IncludeSources bool
+ IncludeDeps bool
+ IncludeOutputs bool
+ IncludeAllStatements bool
+ ShowHidden bool
+}
+
+// any reports true if any of the options are set to true.
+func (wmo WriteMetadataOpts) any() bool {
+ return wmo.IncludeDeps || wmo.IncludeOutputs || wmo.IncludeSources
+}
+
+// Metadata prints out a visualization of the parsed build statement metadata for the given targets.
+func Metadata(state *core.BuildState, targets core.BuildLabels, opts WriteMetadataOpts) {
+ if !cli.ShowColouredOutput || !cli.IsATerminal(os.Stdout) {
+ cli.ShowColouredOutput = false
+ }
+
+ // Group requested targets by their package
+ packageTargets := map[*core.Package]core.BuildLabels{}
+ for _, label := range targets {
+ pkg := state.Graph.PackageOrDie(label)
+ packageTargets[pkg] = append(packageTargets[pkg], label)
+ }
+
+ printTerminal(state, packageTargets, opts)
+}
+
+// printTerminal formats and draws the metadata as a beautiful colorized terminal tree-box layout.
+func printTerminal(state *core.BuildState, packageTargets map[*core.Package]core.BuildLabels, opts WriteMetadataOpts) {
+ // itemDetail holds the text to print and its optional color formatting string
+ type itemDetail struct {
+ text string
+ color string
+ }
+
+ // Helper to print a titled section of items using precise tree box-drawing characters
+ printSection := func(prefix, title string, items []itemDetail, isLast bool) string {
+ if len(items) == 0 {
+ return ""
+ }
+ branch := "├──"
+ childPrefix := prefix + "│ "
+ if isLast {
+ branch = "└──"
+ childPrefix = prefix + " "
+ }
+ cli.Fprintf(os.Stdout, "%s%s ${CYAN}%s:${RESET}\n", prefix, branch, title)
+ for idx, item := range items {
+ itemBranch := "├──"
+ if idx == len(items)-1 {
+ itemBranch = "└──"
+ }
+ cli.Fprintf(os.Stdout, "%s%s "+item.color+"%s${RESET}\n", childPrefix, itemBranch, item.text)
+ }
+ return childPrefix
+ }
+
+ labelsToItems := func(labels core.BuildLabels, defaultColor string) []itemDetail {
+ res := make([]itemDetail, len(labels))
+ for i, l := range labels {
+ res[i] = itemDetail{text: l.String(), color: defaultColor}
+ }
+ return res
+ }
+
+ stringsToItems := func(strs []string, defaultColor string) []itemDetail {
+ res := make([]itemDetail, len(strs))
+ for i, s := range strs {
+ res[i] = itemDetail{text: s, color: defaultColor}
+ }
+ return res
+ }
+
+ inputsToItems := func(inputs []core.BuildInput) []itemDetail {
+ res := make([]itemDetail, len(inputs))
+ for i, inp := range inputs {
+ color := ""
+ if _, ok := inp.Label(); ok {
+ color = "${GREEN}"
+ }
+ res[i] = itemDetail{text: inp.String(), color: color}
+ }
+ return res
+ }
+
+ for pkg, labels := range packageTargets {
+ fmt.Printf("=== Package: %s (File: %s) ===\n", pkg.Label(), pkg.Filename)
+
+ content, err := os.ReadFile(pkg.Filename)
+ if err != nil {
+ fmt.Printf("Error reading BUILD file %s: %v\n\n", pkg.Filename, err)
+ continue
+ }
+
+ allStatements := pkg.Metadata.Statements()
+ var filterStmts map[core.BuildStatement]struct{}
+ if !opts.IncludeAllStatements {
+ filterStmts = filterStatements(pkg, labels)
+ }
+
+ for _, sm := range allStatements {
+ if filterStmts != nil {
+ if _, ok := filterStmts[sm.Statement]; !ok {
+ continue
+ }
+ }
+
+ code := string(content[sm.Statement.Start:sm.Statement.End])
+
+ cli.Fprintf(os.Stdout, "${BOLD_CYAN}Statement (Offsets: %d-%d):${RESET}\n", sm.Statement.Start, sm.Statement.End)
+ cli.Fprintf(os.Stdout, " ${CYAN}Code:${RESET}\n")
+ // Indent the code
+ for line := range strings.SplitSeq(code, "\n") {
+ cli.Fprintf(os.Stdout, " %s\n", line)
+ }
+
+ targets := sm.Targets
+ if !opts.ShowHidden {
+ targets = make(core.BuildLabels, 0, len(sm.Targets))
+ for _, t := range sm.Targets {
+ if !t.IsHidden() {
+ targets = append(targets, t)
+ }
+ }
+ }
+
+ hasSubincludes := len(sm.Subincludes) > 0
+ hasFiles := len(sm.Files) > 0
+ hasTargets := len(targets) > 0
+
+ var lastSection string
+ if hasTargets {
+ lastSection = "targets"
+ } else if hasFiles {
+ lastSection = "files"
+ } else if hasSubincludes {
+ lastSection = "subincludes"
+ }
+
+ basePrefix := " "
+ if hasSubincludes {
+ printSection(basePrefix, "Required Subincludes", labelsToItems(sm.Subincludes, "${YELLOW}"), lastSection == "subincludes")
+ }
+
+ if hasFiles {
+ printSection(basePrefix, "Required Files", stringsToItems(sm.Files, ""), lastSection == "files")
+ }
+
+ if hasTargets {
+ branch := "├──"
+ childPrefix := basePrefix + "│ "
+ if lastSection == "targets" {
+ branch = "└──"
+ childPrefix = basePrefix + " "
+ }
+ cli.Fprintf(os.Stdout, "%s%s ${CYAN}Generated Targets:${RESET}\n", basePrefix, branch)
+
+ for i, t := range targets {
+ targetBranch := "├──"
+ targetChildPrefix := childPrefix + "│ "
+ if i == len(targets)-1 {
+ targetBranch = "└──"
+ targetChildPrefix = childPrefix + " "
+ }
+ cli.Fprintf(os.Stdout, "%s%s ${BOLD_GREEN}%s${RESET}\n", childPrefix, targetBranch, t)
+
+ if opts.any() {
+ if target := state.Graph.Target(t); target != nil {
+ type optDetail struct {
+ title string
+ items []itemDetail
+ }
+ var details []optDetail
+ if opts.IncludeSources && len(target.AllSources()) > 0 {
+ details = append(details, optDetail{"Sources", inputsToItems(target.AllSources())})
+ }
+ if opts.IncludeDeps && len(target.DeclaredDependencies()) > 0 {
+ details = append(details, optDetail{"Dependencies", labelsToItems(target.DeclaredDependencies(), "${GREEN}")})
+ }
+ if opts.IncludeOutputs && len(target.Outputs()) > 0 {
+ details = append(details, optDetail{"Outputs", stringsToItems(target.Outputs(), "")})
+ }
+
+ for idx, det := range details {
+ isLastDetail := idx == len(details)-1
+ printSection(targetChildPrefix, det.title, det.items, isLastDetail)
+ }
+ }
+ }
+ }
+ }
+ fmt.Fprintln(os.Stdout)
+ }
+ }
+}
+
+// filterStatements calculates if all statements should be written.
+func filterStatements(pkg *core.Package, labels core.BuildLabels) map[core.BuildStatement]struct{} {
+ filterStmts := map[core.BuildStatement]struct{}{}
+ for _, target := range labels {
+ stmt, err := pkg.Metadata.FindStatement(target)
+ if err == nil && stmt != (core.BuildStatement{}) {
+ filterStmts[stmt] = struct{}{}
+ }
+ }
+ return filterStmts
+}
diff --git a/src/query/reverse_deps.go b/src/query/reverse_deps.go
index cca1ea2f4c..f9ba93e3c3 100644
--- a/src/query/reverse_deps.go
+++ b/src/query/reverse_deps.go
@@ -123,7 +123,7 @@ func buildRevdeps(graph *core.BuildGraph, includeSubrepos bool) map[core.BuildLa
// Targets in a subrepo don't express an explicit dependency on their subrepo's target.
// However this is often useful for query commands where you expect to see this kind of
// relationship. Hence, if requested, we add the extra 'dependency' here.
- if includeSubrepos && t.Subrepo != nil && t.Subrepo.Target != nil {
+ if includeSubrepos && t.Subrepo.IsExternal() {
revdeps[t.Subrepo.Target.Label] = append(revdeps[t.Subrepo.Target.Label], t)
}
}
diff --git a/src/query/somepath.go b/src/query/somepath.go
index 75b91bdc10..adc3b9a8bf 100644
--- a/src/query/somepath.go
+++ b/src/query/somepath.go
@@ -104,7 +104,7 @@ func somePath(graph *core.BuildGraph, target1, target2 *core.BuildTarget, seen,
}
}
}
- if target1.Subrepo != nil && target1.Subrepo.Target != nil {
+ if target1.Subrepo.IsExternal() {
if path := somePath(graph, target1.Subrepo.Target, target2, seen, except); len(path) != 0 {
return append([]core.BuildLabel{target1.Label}, path...)
}
diff --git a/test/export/BUILD b/test/export/BUILD
index 545dc35c80..f4813d5d4c 100644
--- a/test/export/BUILD
+++ b/test/export/BUILD
@@ -9,5 +9,22 @@ filegroup(
# Generic catch-all test on internal repo.
plz_e2e_test(
name = "export_src_please_test",
- cmd = "plz export --output plz-out/plzexport //src/core && plz --repo_root=$(plz query reporoot)/plz-out/plzexport build //src/core",
+ cmd = " && ".join([
+ 'CACHE_CONF="--override=cache.dir:$(plz query reporoot)/plz-out/test-cache"',
+ 'plz "$CACHE_CONF" export --output "$PLZ_EXPORT_DIR" //src/core:core',
+ 'plz "$CACHE_CONF" --repo_root="$PLZ_EXPORT_DIR" build //src/core:core',
+ ]),
+ pre_cmd = 'PLZ_EXPORT_DIR="$(mktemp -d)"',
+)
+
+
+# Generic catch-all test on internal repo - notrim.
+plz_e2e_test(
+ name = "export_src_please_notrim_test",
+ cmd = " && ".join([
+ 'CACHE_CONF="--override=cache.dir:$(plz query reporoot)/plz-out/test-cache"',
+ 'plz "$CACHE_CONF" export --output "$PLZ_EXPORT_DIR" --notrim //src/core:core',
+ 'plz "$CACHE_CONF" --repo_root="$PLZ_EXPORT_DIR" build //src/core:core',
+ ]),
+ pre_cmd = 'PLZ_EXPORT_DIR="$(mktemp -d)"',
)
diff --git a/test/export/please_export_e2e_test.build_defs b/test/export/please_export_e2e_test.build_defs
index 0f240b5d4b..87768c82ad 100644
--- a/test/export/please_export_e2e_test.build_defs
+++ b/test/export/please_export_e2e_test.build_defs
@@ -40,7 +40,7 @@ def please_export_e2e_test(
if enforce_different_repos:
test_cmd += [
# Enforce source_repo and expected_repo differences
- f'(diff -rq "$DATA_SOURCE_REPO" "$DATA_EXPECTED_REPO" && \
+ f'(diff -rq "$DATA_SOURCE_REPO" "$DATA_EXPECTED_REPO" > /dev/null && \
echo "Source and Expected repos must differ" && exit 1 || true)',
]
@@ -50,10 +50,12 @@ def please_export_e2e_test(
# Golden-Master validation with expected repo. Done before any building to avoid plz-out
f'diff -ru "{exported_repo}" "$DATA_EXPECTED_REPO"',
# Tests building the exported repo which in turn ensures the sources are included
- f'plz --repo_root="{exported_repo}" build //...',
+ f'plz --repo_root="{exported_repo}" build //... > /dev/null',
] + [f'plz --repo_root="{exported_repo}" {cmd}' for cmd in cmd_on_export]
- test_cmd = [cmd.replace("plz ", "$TOOLS_PLEASE ") for cmd in test_cmd]
+ cache_args = '--override=cache.dir:"$($TOOLS_PLEASE query reporoot)"/plz-out/test-cache'
+
+ test_cmd = [cmd.replace("plz ", f"$TOOLS_PLEASE {cache_args} ") for cmd in test_cmd]
test_cmd = " && ".join(test_cmd)
data["SOURCE_REPO"] = [source_repo]
diff --git a/test/export/test_builtins/expected_repo/BUILD_FILE b/test/export/test_builtins/expected_repo/BUILD_FILE
index a62a98a8d6..b840c2d9f1 100644
--- a/test/export/test_builtins/expected_repo/BUILD_FILE
+++ b/test/export/test_builtins/expected_repo/BUILD_FILE
@@ -2,5 +2,6 @@ genrule(
name = "native_genrule",
srcs = ["file.txt"],
outs = ["file.wordcount"],
- cmd = "wc $SRCS > $OUT",
+ cmd = "$TOOLS $SRCS > $OUT",
+ tools = ["//tools:tool"],
)
diff --git a/test/export/test_builtins/expected_repo/tools/BUILD_FILE b/test/export/test_builtins/expected_repo/tools/BUILD_FILE
new file mode 100644
index 0000000000..71d3129e2c
--- /dev/null
+++ b/test/export/test_builtins/expected_repo/tools/BUILD_FILE
@@ -0,0 +1,6 @@
+export_file(
+ name = "tool",
+ src = "tool.sh",
+ binary = True,
+ visibility = ["PUBLIC"],
+)
diff --git a/test/export/test_builtins/expected_repo/tools/tool.sh b/test/export/test_builtins/expected_repo/tools/tool.sh
new file mode 100644
index 0000000000..244a9b4ffd
--- /dev/null
+++ b/test/export/test_builtins/expected_repo/tools/tool.sh
@@ -0,0 +1,2 @@
+#!/bin/bash
+wc $@
diff --git a/test/export/test_builtins/source_repo/BUILD_FILE b/test/export/test_builtins/source_repo/BUILD_FILE
index db7732c487..0e09af4883 100644
--- a/test/export/test_builtins/source_repo/BUILD_FILE
+++ b/test/export/test_builtins/source_repo/BUILD_FILE
@@ -2,12 +2,13 @@ genrule(
name = "native_genrule",
srcs = ["file.txt"],
outs = ["file.wordcount"],
- cmd = "wc $SRCS > $OUT",
+ cmd = "$TOOLS $SRCS > $OUT",
+ tools = ["//tools:tool"],
)
genrule(
- name = "dummy_target_to_be_trimmed",
- srcs = ["dummy.txt"],
- outs = ["dummy"],
+ name = "unneded_target_to_be_trimmed",
+ srcs = ["unneded.txt"],
+ outs = ["unneded"],
cmd = "cat $SRCS > $OUT",
)
diff --git a/test/export/test_builtins/source_repo/tools/BUILD_FILE b/test/export/test_builtins/source_repo/tools/BUILD_FILE
new file mode 100644
index 0000000000..71d3129e2c
--- /dev/null
+++ b/test/export/test_builtins/source_repo/tools/BUILD_FILE
@@ -0,0 +1,6 @@
+export_file(
+ name = "tool",
+ src = "tool.sh",
+ binary = True,
+ visibility = ["PUBLIC"],
+)
diff --git a/test/export/test_builtins/source_repo/tools/tool.sh b/test/export/test_builtins/source_repo/tools/tool.sh
new file mode 100644
index 0000000000..244a9b4ffd
--- /dev/null
+++ b/test/export/test_builtins/source_repo/tools/tool.sh
@@ -0,0 +1,2 @@
+#!/bin/bash
+wc $@
diff --git a/test/export/test_custom_def/source_repo/BUILD_FILE b/test/export/test_custom_def/source_repo/BUILD_FILE
index b0e288e983..f74b40a8e3 100644
--- a/test/export/test_custom_def/source_repo/BUILD_FILE
+++ b/test/export/test_custom_def/source_repo/BUILD_FILE
@@ -1,13 +1,21 @@
-subinclude("//build_defs:simple_build_def")
+subinclude(
+ "//build_defs:simple_build_def",
+ "//build_defs:unneeded_build_def",
+)
simple_custom_target(
- name = "simple_custom_target",
+ name = "unneeded",
srcs = ["file.txt"],
- outs = ["file_simple.out"],
+ outs = ["unneeded.out"],
+)
+
+unneeded_target(
+ name = "unneeded2",
+ outs = ["unneeded2.out"],
)
simple_custom_target(
- name = "dummy",
+ name = "simple_custom_target",
srcs = ["file.txt"],
- outs = ["dummy.out"],
+ outs = ["file_simple.out"],
)
diff --git a/test/export/test_custom_def/source_repo/build_defs/BUILD_FILE b/test/export/test_custom_def/source_repo/build_defs/BUILD_FILE
index 976d0dc22c..2aeb84543f 100644
--- a/test/export/test_custom_def/source_repo/build_defs/BUILD_FILE
+++ b/test/export/test_custom_def/source_repo/build_defs/BUILD_FILE
@@ -5,7 +5,7 @@ filegroup(
)
filegroup(
- name = "dummy_build_def",
- srcs = ["dummy.build_defs"],
+ name = "unneeded_build_def",
+ srcs = ["unneeded.build_defs"],
visibility = ["PUBLIC"],
)
diff --git a/test/export/test_custom_def/source_repo/build_defs/dummy.build_defs b/test/export/test_custom_def/source_repo/build_defs/unneeded.build_defs
similarity index 64%
rename from test/export/test_custom_def/source_repo/build_defs/dummy.build_defs
rename to test/export/test_custom_def/source_repo/build_defs/unneeded.build_defs
index 8cc3b6112a..5117901a85 100644
--- a/test/export/test_custom_def/source_repo/build_defs/dummy.build_defs
+++ b/test/export/test_custom_def/source_repo/build_defs/unneeded.build_defs
@@ -1,8 +1,8 @@
-def dummy_target(
+def unneeded_target(
name:str,
outs:list=[]):
return genrule(
name = name,
outs = outs,
- cmd = "echo dummy > $OUT",
+ cmd = "echo unneeded > $OUT",
)
diff --git a/test/export/test_custom_def_children/expected_repo/build_defs/BUILD_FILE b/test/export/test_custom_def_children/expected_repo/build_defs/BUILD_FILE
deleted file mode 100644
index e9b1bc8a20..0000000000
--- a/test/export/test_custom_def_children/expected_repo/build_defs/BUILD_FILE
+++ /dev/null
@@ -1,5 +0,0 @@
-filegroup(
- name = "custom_build_def",
- srcs = ["custom.build_defs"],
- visibility = ["PUBLIC"],
-)
diff --git a/test/export/test_custom_def_children/source_repo/build_defs/BUILD_FILE b/test/export/test_custom_def_children/source_repo/build_defs/BUILD_FILE
deleted file mode 100644
index e9b1bc8a20..0000000000
--- a/test/export/test_custom_def_children/source_repo/build_defs/BUILD_FILE
+++ /dev/null
@@ -1,5 +0,0 @@
-filegroup(
- name = "custom_build_def",
- srcs = ["custom.build_defs"],
- visibility = ["PUBLIC"],
-)
diff --git a/test/export/test_custom_def_children/BUILD b/test/export/test_custom_def_multiple_targets/BUILD
similarity index 87%
rename from test/export/test_custom_def_children/BUILD
rename to test/export/test_custom_def_multiple_targets/BUILD
index af8ef5b106..abd0536fd1 100644
--- a/test/export/test_custom_def_children/BUILD
+++ b/test/export/test_custom_def_multiple_targets/BUILD
@@ -5,7 +5,7 @@ please_export_e2e_test(
name = "export_custom_with_adjacent_target",
cmd_on_export = [
# Adjacent target of build def
- "build //:custom_target#adjacent",
+ "build //:custom_target_adjacent",
],
export_targets = ["//:custom_target"],
)
diff --git a/test/export/test_custom_def_children/expected_repo/.plzconfig b/test/export/test_custom_def_multiple_targets/expected_repo/.plzconfig
similarity index 100%
rename from test/export/test_custom_def_children/expected_repo/.plzconfig
rename to test/export/test_custom_def_multiple_targets/expected_repo/.plzconfig
diff --git a/test/export/test_custom_def_children/expected_repo/BUILD_FILE b/test/export/test_custom_def_multiple_targets/expected_repo/BUILD_FILE
similarity index 100%
rename from test/export/test_custom_def_children/expected_repo/BUILD_FILE
rename to test/export/test_custom_def_multiple_targets/expected_repo/BUILD_FILE
diff --git a/test/export/test_custom_def_multiple_targets/expected_repo/build_defs/BUILD_FILE b/test/export/test_custom_def_multiple_targets/expected_repo/build_defs/BUILD_FILE
new file mode 100644
index 0000000000..6e20067074
--- /dev/null
+++ b/test/export/test_custom_def_multiple_targets/expected_repo/build_defs/BUILD_FILE
@@ -0,0 +1,17 @@
+filegroup(
+ name = "custom_build_def",
+ srcs = ["custom.build_defs"],
+ visibility = ["PUBLIC"],
+)
+
+filegroup(
+ name = "secondary_build_def",
+ srcs = ["secondary.build_defs"],
+ visibility = ["PUBLIC"],
+)
+
+export_file(
+ name = "secondary_file",
+ src = "secondary.file",
+ visibility = ["PUBLIC"],
+)
diff --git a/test/export/test_custom_def_children/source_repo/build_defs/custom.build_defs b/test/export/test_custom_def_multiple_targets/expected_repo/build_defs/custom.build_defs
similarity index 69%
rename from test/export/test_custom_def_children/source_repo/build_defs/custom.build_defs
rename to test/export/test_custom_def_multiple_targets/expected_repo/build_defs/custom.build_defs
index cc0e256e86..00bf12383e 100644
--- a/test/export/test_custom_def_children/source_repo/build_defs/custom.build_defs
+++ b/test/export/test_custom_def_multiple_targets/expected_repo/build_defs/custom.build_defs
@@ -1,13 +1,13 @@
+subinclude("//build_defs:secondary_build_def")
+
def custom_target(
name:str,
srcs:list=[],
outs:list=[],
outs_adjacent:list=[]):
- genrule(
- name = f"{name}#adjacent",
- srcs = srcs,
+ secondary_custom_target(
+ name = f"{name}_adjacent",
outs = outs_adjacent,
- cmd = "echo 'adjacent' > $OUT && cat $SRCS >> $OUT ",
)
return genrule(
name = name,
diff --git a/test/export/test_custom_def_multiple_targets/expected_repo/build_defs/secondary.build_defs b/test/export/test_custom_def_multiple_targets/expected_repo/build_defs/secondary.build_defs
new file mode 100644
index 0000000000..85b6470267
--- /dev/null
+++ b/test/export/test_custom_def_multiple_targets/expected_repo/build_defs/secondary.build_defs
@@ -0,0 +1,9 @@
+def secondary_custom_target(
+ name:str,
+ outs:list=[]):
+ return genrule(
+ name = name,
+ srcs = ["//build_defs:secondary_file"],
+ outs = outs,
+ cmd = "echo 'secondary' > $OUT && cat $SRCS >> $OUT ",
+ )
diff --git a/test/export/test_custom_def_multiple_targets/expected_repo/build_defs/secondary.file b/test/export/test_custom_def_multiple_targets/expected_repo/build_defs/secondary.file
new file mode 100644
index 0000000000..7a0bf8eb90
--- /dev/null
+++ b/test/export/test_custom_def_multiple_targets/expected_repo/build_defs/secondary.file
@@ -0,0 +1 @@
+File for testing
diff --git a/test/export/test_custom_def_children/expected_repo/file.txt b/test/export/test_custom_def_multiple_targets/expected_repo/file.txt
similarity index 100%
rename from test/export/test_custom_def_children/expected_repo/file.txt
rename to test/export/test_custom_def_multiple_targets/expected_repo/file.txt
diff --git a/test/export/test_custom_def_children/source_repo/.plzconfig b/test/export/test_custom_def_multiple_targets/source_repo/.plzconfig
similarity index 100%
rename from test/export/test_custom_def_children/source_repo/.plzconfig
rename to test/export/test_custom_def_multiple_targets/source_repo/.plzconfig
diff --git a/test/export/test_custom_def_children/source_repo/BUILD_FILE b/test/export/test_custom_def_multiple_targets/source_repo/BUILD_FILE
similarity index 70%
rename from test/export/test_custom_def_children/source_repo/BUILD_FILE
rename to test/export/test_custom_def_multiple_targets/source_repo/BUILD_FILE
index 9651c3d8fe..ba45446f94 100644
--- a/test/export/test_custom_def_children/source_repo/BUILD_FILE
+++ b/test/export/test_custom_def_multiple_targets/source_repo/BUILD_FILE
@@ -8,8 +8,8 @@ custom_target(
)
custom_target(
- name = "dummy",
+ name = "unneded",
srcs = ["file.txt"],
- outs = ["dummy.out"],
- outs_adjacent = ["dummy_adjacent.out"],
+ outs = ["unneded.out"],
+ outs_adjacent = ["unneded_adjacent.out"],
)
diff --git a/test/export/test_custom_def_multiple_targets/source_repo/build_defs/BUILD_FILE b/test/export/test_custom_def_multiple_targets/source_repo/build_defs/BUILD_FILE
new file mode 100644
index 0000000000..6e20067074
--- /dev/null
+++ b/test/export/test_custom_def_multiple_targets/source_repo/build_defs/BUILD_FILE
@@ -0,0 +1,17 @@
+filegroup(
+ name = "custom_build_def",
+ srcs = ["custom.build_defs"],
+ visibility = ["PUBLIC"],
+)
+
+filegroup(
+ name = "secondary_build_def",
+ srcs = ["secondary.build_defs"],
+ visibility = ["PUBLIC"],
+)
+
+export_file(
+ name = "secondary_file",
+ src = "secondary.file",
+ visibility = ["PUBLIC"],
+)
diff --git a/test/export/test_custom_def_children/expected_repo/build_defs/custom.build_defs b/test/export/test_custom_def_multiple_targets/source_repo/build_defs/custom.build_defs
similarity index 69%
rename from test/export/test_custom_def_children/expected_repo/build_defs/custom.build_defs
rename to test/export/test_custom_def_multiple_targets/source_repo/build_defs/custom.build_defs
index cc0e256e86..00bf12383e 100644
--- a/test/export/test_custom_def_children/expected_repo/build_defs/custom.build_defs
+++ b/test/export/test_custom_def_multiple_targets/source_repo/build_defs/custom.build_defs
@@ -1,13 +1,13 @@
+subinclude("//build_defs:secondary_build_def")
+
def custom_target(
name:str,
srcs:list=[],
outs:list=[],
outs_adjacent:list=[]):
- genrule(
- name = f"{name}#adjacent",
- srcs = srcs,
+ secondary_custom_target(
+ name = f"{name}_adjacent",
outs = outs_adjacent,
- cmd = "echo 'adjacent' > $OUT && cat $SRCS >> $OUT ",
)
return genrule(
name = name,
diff --git a/test/export/test_custom_def_multiple_targets/source_repo/build_defs/secondary.build_defs b/test/export/test_custom_def_multiple_targets/source_repo/build_defs/secondary.build_defs
new file mode 100644
index 0000000000..85b6470267
--- /dev/null
+++ b/test/export/test_custom_def_multiple_targets/source_repo/build_defs/secondary.build_defs
@@ -0,0 +1,9 @@
+def secondary_custom_target(
+ name:str,
+ outs:list=[]):
+ return genrule(
+ name = name,
+ srcs = ["//build_defs:secondary_file"],
+ outs = outs,
+ cmd = "echo 'secondary' > $OUT && cat $SRCS >> $OUT ",
+ )
diff --git a/test/export/test_custom_def_multiple_targets/source_repo/build_defs/secondary.file b/test/export/test_custom_def_multiple_targets/source_repo/build_defs/secondary.file
new file mode 100644
index 0000000000..7a0bf8eb90
--- /dev/null
+++ b/test/export/test_custom_def_multiple_targets/source_repo/build_defs/secondary.file
@@ -0,0 +1 @@
+File for testing
diff --git a/test/export/test_custom_def_children/source_repo/file.txt b/test/export/test_custom_def_multiple_targets/source_repo/file.txt
similarity index 100%
rename from test/export/test_custom_def_children/source_repo/file.txt
rename to test/export/test_custom_def_multiple_targets/source_repo/file.txt
diff --git a/test/export/test_deps/source_repo/BUILD_FILE b/test/export/test_deps/source_repo/BUILD_FILE
index 3c6766fc9e..56103fc9fc 100644
--- a/test/export/test_deps/source_repo/BUILD_FILE
+++ b/test/export/test_deps/source_repo/BUILD_FILE
@@ -17,8 +17,8 @@ genrule(
)
genrule(
- name = "dummy",
+ name = "unneded",
srcs = ["file1.txt"],
- outs = ["dummy.out"],
+ outs = ["unneded.out"],
cmd = "cat $SRCS > $OUT",
)
diff --git a/test/export/test_deps/source_repo/other_deps/BUILD_FILE b/test/export/test_deps/source_repo/other_deps/BUILD_FILE
index 8acaa9b9eb..cd7262c32c 100644
--- a/test/export/test_deps/source_repo/other_deps/BUILD_FILE
+++ b/test/export/test_deps/source_repo/other_deps/BUILD_FILE
@@ -6,8 +6,8 @@ genrule(
)
genrule(
- name = "dummy",
- outs = ["dummy.out"],
- cmd = "echo 'dummy' > $OUT",
+ name = "unneded",
+ outs = ["unneded.out"],
+ cmd = "echo 'unneded' > $OUT",
visibility = ["PUBLIC"],
)
diff --git a/test/export/test_dynamic_subinclude/BUILD b/test/export/test_dynamic_subinclude/BUILD
new file mode 100644
index 0000000000..b588cde421
--- /dev/null
+++ b/test/export/test_dynamic_subinclude/BUILD
@@ -0,0 +1,11 @@
+subinclude("//test/export:export_e2e_test_build_def")
+
+# Export targets generated by a custom build def in a loop with dynamic subincludes,
+# and validate that unused subincludes are trimmed.
+please_export_e2e_test(
+ name = "export_dynamic_subinclude",
+ export_targets = [
+ "//:a",
+ "//:b",
+ ],
+)
diff --git a/test/export/test_custom_in_file_def/expected_repo/.plzconfig b/test/export/test_dynamic_subinclude/expected_repo/.plzconfig
similarity index 100%
rename from test/export/test_custom_in_file_def/expected_repo/.plzconfig
rename to test/export/test_dynamic_subinclude/expected_repo/.plzconfig
diff --git a/test/export/test_dynamic_subinclude/expected_repo/BUILD_FILE b/test/export/test_dynamic_subinclude/expected_repo/BUILD_FILE
new file mode 100644
index 0000000000..31cf1e8c06
--- /dev/null
+++ b/test/export/test_dynamic_subinclude/expected_repo/BUILD_FILE
@@ -0,0 +1,7 @@
+subinclude("//build_defs:dynamic_build_def")
+
+for i in [
+ "a",
+ "b",
+]:
+ dynamic_target(name = i, def_name = f"{i}_build_def")
diff --git a/test/export/test_dynamic_subinclude/expected_repo/build_defs/BUILD_FILE b/test/export/test_dynamic_subinclude/expected_repo/build_defs/BUILD_FILE
new file mode 100644
index 0000000000..625da44b75
--- /dev/null
+++ b/test/export/test_dynamic_subinclude/expected_repo/build_defs/BUILD_FILE
@@ -0,0 +1,17 @@
+filegroup(
+ name = "dynamic_build_def",
+ srcs = ["dynamic.build_defs"],
+ visibility = ["PUBLIC"],
+)
+
+filegroup(
+ name = "a_build_def",
+ srcs = ["a.build_defs"],
+ visibility = ["PUBLIC"],
+)
+
+filegroup(
+ name = "b_build_def",
+ srcs = ["b.build_defs"],
+ visibility = ["PUBLIC"],
+)
diff --git a/test/export/test_dynamic_subinclude/expected_repo/build_defs/a.build_defs b/test/export/test_dynamic_subinclude/expected_repo/build_defs/a.build_defs
new file mode 100644
index 0000000000..e58be5dae4
--- /dev/null
+++ b/test/export/test_dynamic_subinclude/expected_repo/build_defs/a.build_defs
@@ -0,0 +1,6 @@
+def helper_a(name: str):
+ genrule(
+ name = name,
+ outs = [f"{name}.out"],
+ cmd = f"echo {name} > $OUT",
+ )
diff --git a/test/export/test_dynamic_subinclude/expected_repo/build_defs/b.build_defs b/test/export/test_dynamic_subinclude/expected_repo/build_defs/b.build_defs
new file mode 100644
index 0000000000..86d44895a6
--- /dev/null
+++ b/test/export/test_dynamic_subinclude/expected_repo/build_defs/b.build_defs
@@ -0,0 +1,6 @@
+def helper_b(name: str):
+ genrule(
+ name = name,
+ outs = [f"{name}.out"],
+ cmd = f"echo {name} > $OUT",
+ )
diff --git a/test/export/test_dynamic_subinclude/expected_repo/build_defs/dynamic.build_defs b/test/export/test_dynamic_subinclude/expected_repo/build_defs/dynamic.build_defs
new file mode 100644
index 0000000000..4fd7564e40
--- /dev/null
+++ b/test/export/test_dynamic_subinclude/expected_repo/build_defs/dynamic.build_defs
@@ -0,0 +1,6 @@
+def dynamic_target(name: str, def_name: str):
+ subinclude(f"//build_defs:{def_name}")
+ if name == "a":
+ helper_a(name = name)
+ elif name == "b":
+ helper_b(name = name)
diff --git a/test/export/test_custom_in_file_def/source_repo/.plzconfig b/test/export/test_dynamic_subinclude/source_repo/.plzconfig
similarity index 100%
rename from test/export/test_custom_in_file_def/source_repo/.plzconfig
rename to test/export/test_dynamic_subinclude/source_repo/.plzconfig
diff --git a/test/export/test_dynamic_subinclude/source_repo/BUILD_FILE b/test/export/test_dynamic_subinclude/source_repo/BUILD_FILE
new file mode 100644
index 0000000000..31cf1e8c06
--- /dev/null
+++ b/test/export/test_dynamic_subinclude/source_repo/BUILD_FILE
@@ -0,0 +1,7 @@
+subinclude("//build_defs:dynamic_build_def")
+
+for i in [
+ "a",
+ "b",
+]:
+ dynamic_target(name = i, def_name = f"{i}_build_def")
diff --git a/test/export/test_dynamic_subinclude/source_repo/build_defs/BUILD_FILE b/test/export/test_dynamic_subinclude/source_repo/build_defs/BUILD_FILE
new file mode 100644
index 0000000000..833e53acec
--- /dev/null
+++ b/test/export/test_dynamic_subinclude/source_repo/build_defs/BUILD_FILE
@@ -0,0 +1,23 @@
+filegroup(
+ name = "dynamic_build_def",
+ srcs = ["dynamic.build_defs"],
+ visibility = ["PUBLIC"],
+)
+
+filegroup(
+ name = "a_build_def",
+ srcs = ["a.build_defs"],
+ visibility = ["PUBLIC"],
+)
+
+filegroup(
+ name = "b_build_def",
+ srcs = ["b.build_defs"],
+ visibility = ["PUBLIC"],
+)
+
+filegroup(
+ name = "unneeded_build_def",
+ srcs = ["unneeded.build_defs"],
+ visibility = ["PUBLIC"],
+)
diff --git a/test/export/test_dynamic_subinclude/source_repo/build_defs/a.build_defs b/test/export/test_dynamic_subinclude/source_repo/build_defs/a.build_defs
new file mode 100644
index 0000000000..e58be5dae4
--- /dev/null
+++ b/test/export/test_dynamic_subinclude/source_repo/build_defs/a.build_defs
@@ -0,0 +1,6 @@
+def helper_a(name: str):
+ genrule(
+ name = name,
+ outs = [f"{name}.out"],
+ cmd = f"echo {name} > $OUT",
+ )
diff --git a/test/export/test_dynamic_subinclude/source_repo/build_defs/b.build_defs b/test/export/test_dynamic_subinclude/source_repo/build_defs/b.build_defs
new file mode 100644
index 0000000000..86d44895a6
--- /dev/null
+++ b/test/export/test_dynamic_subinclude/source_repo/build_defs/b.build_defs
@@ -0,0 +1,6 @@
+def helper_b(name: str):
+ genrule(
+ name = name,
+ outs = [f"{name}.out"],
+ cmd = f"echo {name} > $OUT",
+ )
diff --git a/test/export/test_dynamic_subinclude/source_repo/build_defs/dynamic.build_defs b/test/export/test_dynamic_subinclude/source_repo/build_defs/dynamic.build_defs
new file mode 100644
index 0000000000..4fd7564e40
--- /dev/null
+++ b/test/export/test_dynamic_subinclude/source_repo/build_defs/dynamic.build_defs
@@ -0,0 +1,6 @@
+def dynamic_target(name: str, def_name: str):
+ subinclude(f"//build_defs:{def_name}")
+ if name == "a":
+ helper_a(name = name)
+ elif name == "b":
+ helper_b(name = name)
diff --git a/test/export/test_dynamic_subinclude/source_repo/build_defs/unneeded.build_defs b/test/export/test_dynamic_subinclude/source_repo/build_defs/unneeded.build_defs
new file mode 100644
index 0000000000..dd2fc99e36
--- /dev/null
+++ b/test/export/test_dynamic_subinclude/source_repo/build_defs/unneeded.build_defs
@@ -0,0 +1,2 @@
+def unneeded_helper(name: str):
+ pass
diff --git a/test/export/test_for/BUILD b/test/export/test_for/BUILD
new file mode 100644
index 0000000000..1b4bb42859
--- /dev/null
+++ b/test/export/test_for/BUILD
@@ -0,0 +1,6 @@
+subinclude("//test/export:export_e2e_test_build_def")
+
+please_export_e2e_test(
+ name = "export_for",
+ export_targets = ["//:a"],
+)
diff --git a/test/export/test_for/expected_repo/.plzconfig b/test/export/test_for/expected_repo/.plzconfig
new file mode 100644
index 0000000000..f8ba31854d
--- /dev/null
+++ b/test/export/test_for/expected_repo/.plzconfig
@@ -0,0 +1,2 @@
+[Parse]
+BuildFileName = BUILD_FILE
diff --git a/test/export/test_for/expected_repo/BUILD_FILE b/test/export/test_for/expected_repo/BUILD_FILE
new file mode 100644
index 0000000000..5754be2e47
--- /dev/null
+++ b/test/export/test_for/expected_repo/BUILD_FILE
@@ -0,0 +1,11 @@
+for i in [
+ "a",
+ "b",
+ "c",
+]:
+ genrule(
+ name = i,
+ srcs = [f"{i}.src"],
+ outs = [f"{i}.txt"],
+ cmd = f"cat $SRCS > $OUT",
+ )
diff --git a/test/export/test_for/expected_repo/a.src b/test/export/test_for/expected_repo/a.src
new file mode 100644
index 0000000000..7898192261
--- /dev/null
+++ b/test/export/test_for/expected_repo/a.src
@@ -0,0 +1 @@
+a
diff --git a/test/export/test_for/expected_repo/b.src b/test/export/test_for/expected_repo/b.src
new file mode 100644
index 0000000000..6178079822
--- /dev/null
+++ b/test/export/test_for/expected_repo/b.src
@@ -0,0 +1 @@
+b
diff --git a/test/export/test_for/expected_repo/c.src b/test/export/test_for/expected_repo/c.src
new file mode 100644
index 0000000000..f2ad6c76f0
--- /dev/null
+++ b/test/export/test_for/expected_repo/c.src
@@ -0,0 +1 @@
+c
diff --git a/test/export/test_for/source_repo/.plzconfig b/test/export/test_for/source_repo/.plzconfig
new file mode 100644
index 0000000000..f8ba31854d
--- /dev/null
+++ b/test/export/test_for/source_repo/.plzconfig
@@ -0,0 +1,2 @@
+[Parse]
+BuildFileName = BUILD_FILE
diff --git a/test/export/test_for/source_repo/BUILD_FILE b/test/export/test_for/source_repo/BUILD_FILE
new file mode 100644
index 0000000000..50d3a3ca34
--- /dev/null
+++ b/test/export/test_for/source_repo/BUILD_FILE
@@ -0,0 +1,17 @@
+for i in [
+ "a",
+ "b",
+ "c",
+]:
+ genrule(
+ name = i,
+ srcs = [f"{i}.src"],
+ outs = [f"{i}.txt"],
+ cmd = f"cat $SRCS > $OUT",
+ )
+
+genrule(
+ name = "d",
+ outs = ["d.txt"],
+ cmd = "echo d > $OUT",
+)
diff --git a/test/export/test_for/source_repo/a.src b/test/export/test_for/source_repo/a.src
new file mode 100644
index 0000000000..7898192261
--- /dev/null
+++ b/test/export/test_for/source_repo/a.src
@@ -0,0 +1 @@
+a
diff --git a/test/export/test_for/source_repo/b.src b/test/export/test_for/source_repo/b.src
new file mode 100644
index 0000000000..6178079822
--- /dev/null
+++ b/test/export/test_for/source_repo/b.src
@@ -0,0 +1 @@
+b
diff --git a/test/export/test_for/source_repo/c.src b/test/export/test_for/source_repo/c.src
new file mode 100644
index 0000000000..f2ad6c76f0
--- /dev/null
+++ b/test/export/test_for/source_repo/c.src
@@ -0,0 +1 @@
+c
diff --git a/test/export/test_for_glob/BUILD b/test/export/test_for_glob/BUILD
new file mode 100644
index 0000000000..3733c75062
--- /dev/null
+++ b/test/export/test_for_glob/BUILD
@@ -0,0 +1,6 @@
+subinclude("//test/export:export_e2e_test_build_def")
+
+please_export_e2e_test(
+ name = "export_for_glob",
+ export_targets = ["//pkg:all_files"],
+)
diff --git a/test/export/test_for_glob/expected_repo/.plzconfig b/test/export/test_for_glob/expected_repo/.plzconfig
new file mode 100644
index 0000000000..f8ba31854d
--- /dev/null
+++ b/test/export/test_for_glob/expected_repo/.plzconfig
@@ -0,0 +1,2 @@
+[Parse]
+BuildFileName = BUILD_FILE
diff --git a/test/export/test_for_glob/expected_repo/pkg/BUILD_FILE b/test/export/test_for_glob/expected_repo/pkg/BUILD_FILE
new file mode 100644
index 0000000000..f4965461be
--- /dev/null
+++ b/test/export/test_for_glob/expected_repo/pkg/BUILD_FILE
@@ -0,0 +1,17 @@
+for file in glob(["file*.in"]):
+ genrule(
+ name = "target_" + file.removesuffix(".in"),
+ srcs = [file],
+ outs = [file.removesuffix(".in") + ".out"],
+ cmd = "cp $SRCS $OUT",
+ )
+
+filegroup(
+ name = "all_files",
+ srcs = [
+ ":target_file1",
+ ],
+)
+
+for file in glob(["doc*.in"]):
+ pass # Trimmed during export
diff --git a/test/export/test_for_glob/expected_repo/pkg/doc1.in b/test/export/test_for_glob/expected_repo/pkg/doc1.in
new file mode 100644
index 0000000000..516682f61d
--- /dev/null
+++ b/test/export/test_for_glob/expected_repo/pkg/doc1.in
@@ -0,0 +1 @@
+doc1
diff --git a/test/export/test_for_glob/expected_repo/pkg/doc2.in b/test/export/test_for_glob/expected_repo/pkg/doc2.in
new file mode 100644
index 0000000000..67f7bbeeda
--- /dev/null
+++ b/test/export/test_for_glob/expected_repo/pkg/doc2.in
@@ -0,0 +1 @@
+doc2
diff --git a/test/export/test_for_glob/expected_repo/pkg/file1.in b/test/export/test_for_glob/expected_repo/pkg/file1.in
new file mode 100644
index 0000000000..d9039017ab
--- /dev/null
+++ b/test/export/test_for_glob/expected_repo/pkg/file1.in
@@ -0,0 +1 @@
+file1 content
diff --git a/test/export/test_for_glob/expected_repo/pkg/file2.in b/test/export/test_for_glob/expected_repo/pkg/file2.in
new file mode 100644
index 0000000000..f3c77b12c6
--- /dev/null
+++ b/test/export/test_for_glob/expected_repo/pkg/file2.in
@@ -0,0 +1 @@
+file2 content
diff --git a/test/export/test_for_glob/source_repo/.plzconfig b/test/export/test_for_glob/source_repo/.plzconfig
new file mode 100644
index 0000000000..f8ba31854d
--- /dev/null
+++ b/test/export/test_for_glob/source_repo/.plzconfig
@@ -0,0 +1,2 @@
+[Parse]
+BuildFileName = BUILD_FILE
diff --git a/test/export/test_for_glob/source_repo/pkg/BUILD_FILE b/test/export/test_for_glob/source_repo/pkg/BUILD_FILE
new file mode 100644
index 0000000000..07e3702e6a
--- /dev/null
+++ b/test/export/test_for_glob/source_repo/pkg/BUILD_FILE
@@ -0,0 +1,29 @@
+for file in glob(["file*.in"]):
+ genrule(
+ name = "target_" + file.removesuffix(".in"),
+ srcs = [file],
+ outs = [file.removesuffix(".in") + ".out"],
+ cmd = "cp $SRCS $OUT",
+ )
+
+genrule(
+ name = "unneeded",
+ srcs = glob(["unneeded*.in"]),
+ outs = ["unneeded.out"],
+ cmd = "cp $SRCS $OUT",
+)
+
+filegroup(
+ name = "all_files",
+ srcs = [
+ ":target_file1",
+ ],
+)
+
+for file in glob(["doc*.in"]):
+ genrule(
+ name = "target_" + file.removesuffix(".in"),
+ srcs = [file],
+ outs = [file.removesuffix(".in") + ".out"],
+ cmd = "cp $SRCS $OUT",
+ )
diff --git a/test/export/test_for_glob/source_repo/pkg/doc1.in b/test/export/test_for_glob/source_repo/pkg/doc1.in
new file mode 100644
index 0000000000..516682f61d
--- /dev/null
+++ b/test/export/test_for_glob/source_repo/pkg/doc1.in
@@ -0,0 +1 @@
+doc1
diff --git a/test/export/test_for_glob/source_repo/pkg/doc2.in b/test/export/test_for_glob/source_repo/pkg/doc2.in
new file mode 100644
index 0000000000..67f7bbeeda
--- /dev/null
+++ b/test/export/test_for_glob/source_repo/pkg/doc2.in
@@ -0,0 +1 @@
+doc2
diff --git a/test/export/test_for_glob/source_repo/pkg/file1.in b/test/export/test_for_glob/source_repo/pkg/file1.in
new file mode 100644
index 0000000000..d9039017ab
--- /dev/null
+++ b/test/export/test_for_glob/source_repo/pkg/file1.in
@@ -0,0 +1 @@
+file1 content
diff --git a/test/export/test_for_glob/source_repo/pkg/file2.in b/test/export/test_for_glob/source_repo/pkg/file2.in
new file mode 100644
index 0000000000..f3c77b12c6
--- /dev/null
+++ b/test/export/test_for_glob/source_repo/pkg/file2.in
@@ -0,0 +1 @@
+file2 content
diff --git a/test/export/test_for_glob/source_repo/pkg/unneeded.in b/test/export/test_for_glob/source_repo/pkg/unneeded.in
new file mode 100644
index 0000000000..a95d9394a0
--- /dev/null
+++ b/test/export/test_for_glob/source_repo/pkg/unneeded.in
@@ -0,0 +1 @@
+unneeded content
diff --git a/test/export/test_for_if/BUILD b/test/export/test_for_if/BUILD
new file mode 100644
index 0000000000..75b06a3508
--- /dev/null
+++ b/test/export/test_for_if/BUILD
@@ -0,0 +1,6 @@
+subinclude("//test/export:export_e2e_test_build_def")
+
+please_export_e2e_test(
+ name = "export_for_if",
+ export_targets = ["//:a"],
+)
diff --git a/test/export/test_for_if/expected_repo/.plzconfig b/test/export/test_for_if/expected_repo/.plzconfig
new file mode 100644
index 0000000000..f8ba31854d
--- /dev/null
+++ b/test/export/test_for_if/expected_repo/.plzconfig
@@ -0,0 +1,2 @@
+[Parse]
+BuildFileName = BUILD_FILE
diff --git a/test/export/test_for_if/expected_repo/BUILD_FILE b/test/export/test_for_if/expected_repo/BUILD_FILE
new file mode 100644
index 0000000000..80279d2bb5
--- /dev/null
+++ b/test/export/test_for_if/expected_repo/BUILD_FILE
@@ -0,0 +1,10 @@
+for i in ["a", "b"]:
+ if i == "a":
+ genrule(
+ name = "a",
+ srcs = ["a.src"],
+ outs = ["a.txt"],
+ cmd = "cat $SRCS > $OUT",
+ )
+ elif i == "b":
+ pass # Trimmed during export
diff --git a/test/export/test_for_if/expected_repo/a.src b/test/export/test_for_if/expected_repo/a.src
new file mode 100644
index 0000000000..7898192261
--- /dev/null
+++ b/test/export/test_for_if/expected_repo/a.src
@@ -0,0 +1 @@
+a
diff --git a/test/export/test_for_if/source_repo/.plzconfig b/test/export/test_for_if/source_repo/.plzconfig
new file mode 100644
index 0000000000..f8ba31854d
--- /dev/null
+++ b/test/export/test_for_if/source_repo/.plzconfig
@@ -0,0 +1,2 @@
+[Parse]
+BuildFileName = BUILD_FILE
diff --git a/test/export/test_for_if/source_repo/BUILD_FILE b/test/export/test_for_if/source_repo/BUILD_FILE
new file mode 100644
index 0000000000..7d76b324d1
--- /dev/null
+++ b/test/export/test_for_if/source_repo/BUILD_FILE
@@ -0,0 +1,15 @@
+for i in ["a", "b"]:
+ if i == "a":
+ genrule(
+ name = "a",
+ srcs = ["a.src"],
+ outs = ["a.txt"],
+ cmd = "cat $SRCS > $OUT",
+ )
+ elif i == "b":
+ genrule(
+ name = "b",
+ srcs = ["b.src"],
+ outs = ["b.txt"],
+ cmd = "cat $SRCS > $OUT",
+ )
diff --git a/test/export/test_for_if/source_repo/a.src b/test/export/test_for_if/source_repo/a.src
new file mode 100644
index 0000000000..7898192261
--- /dev/null
+++ b/test/export/test_for_if/source_repo/a.src
@@ -0,0 +1 @@
+a
diff --git a/test/export/test_for_if/source_repo/b.src b/test/export/test_for_if/source_repo/b.src
new file mode 100644
index 0000000000..6178079822
--- /dev/null
+++ b/test/export/test_for_if/source_repo/b.src
@@ -0,0 +1 @@
+b
diff --git a/test/export/test_for_if_both/BUILD b/test/export/test_for_if_both/BUILD
new file mode 100644
index 0000000000..12734eadef
--- /dev/null
+++ b/test/export/test_for_if_both/BUILD
@@ -0,0 +1,10 @@
+subinclude("//test/export:export_e2e_test_build_def")
+
+# Export both statements inside the inner if
+please_export_e2e_test(
+ name = "export_for_if_both",
+ export_targets = [
+ "//:a",
+ "//:b",
+ ],
+)
diff --git a/test/export/test_for_if_both/expected_repo/.plzconfig b/test/export/test_for_if_both/expected_repo/.plzconfig
new file mode 100644
index 0000000000..f8ba31854d
--- /dev/null
+++ b/test/export/test_for_if_both/expected_repo/.plzconfig
@@ -0,0 +1,2 @@
+[Parse]
+BuildFileName = BUILD_FILE
diff --git a/test/export/test_for_if_both/expected_repo/BUILD_FILE b/test/export/test_for_if_both/expected_repo/BUILD_FILE
new file mode 100644
index 0000000000..73218736ba
--- /dev/null
+++ b/test/export/test_for_if_both/expected_repo/BUILD_FILE
@@ -0,0 +1,20 @@
+for i in [
+ "a",
+ "b",
+]:
+ if i == "a":
+ genrule(
+ name = "a",
+ srcs = ["a.src"],
+ outs = ["a.txt"],
+ cmd = "cat $SRCS > $OUT",
+ )
+ elif i == "b":
+ genrule(
+ name = "b",
+ srcs = ["b.src"],
+ outs = ["b.txt"],
+ cmd = "cat $SRCS > $OUT",
+ )
+ else:
+ pass # Trimmed during export
diff --git a/test/export/test_for_if_both/expected_repo/a.src b/test/export/test_for_if_both/expected_repo/a.src
new file mode 100644
index 0000000000..7898192261
--- /dev/null
+++ b/test/export/test_for_if_both/expected_repo/a.src
@@ -0,0 +1 @@
+a
diff --git a/test/export/test_for_if_both/expected_repo/b.src b/test/export/test_for_if_both/expected_repo/b.src
new file mode 100644
index 0000000000..6178079822
--- /dev/null
+++ b/test/export/test_for_if_both/expected_repo/b.src
@@ -0,0 +1 @@
+b
diff --git a/test/export/test_for_if_both/source_repo/.plzconfig b/test/export/test_for_if_both/source_repo/.plzconfig
new file mode 100644
index 0000000000..f8ba31854d
--- /dev/null
+++ b/test/export/test_for_if_both/source_repo/.plzconfig
@@ -0,0 +1,2 @@
+[Parse]
+BuildFileName = BUILD_FILE
diff --git a/test/export/test_for_if_both/source_repo/BUILD_FILE b/test/export/test_for_if_both/source_repo/BUILD_FILE
new file mode 100644
index 0000000000..66abd06cb7
--- /dev/null
+++ b/test/export/test_for_if_both/source_repo/BUILD_FILE
@@ -0,0 +1,25 @@
+for i in [
+ "a",
+ "b",
+]:
+ if i == "a":
+ genrule(
+ name = "a",
+ srcs = ["a.src"],
+ outs = ["a.txt"],
+ cmd = "cat $SRCS > $OUT",
+ )
+ elif i == "b":
+ genrule(
+ name = "b",
+ srcs = ["b.src"],
+ outs = ["b.txt"],
+ cmd = "cat $SRCS > $OUT",
+ )
+ else:
+ genrule(
+ name = "c",
+ srcs = ["c.src"],
+ outs = ["c.txt"],
+ cmd = "cat $SRCS > $OUT",
+ )
diff --git a/test/export/test_for_if_both/source_repo/a.src b/test/export/test_for_if_both/source_repo/a.src
new file mode 100644
index 0000000000..7898192261
--- /dev/null
+++ b/test/export/test_for_if_both/source_repo/a.src
@@ -0,0 +1 @@
+a
diff --git a/test/export/test_for_if_both/source_repo/b.src b/test/export/test_for_if_both/source_repo/b.src
new file mode 100644
index 0000000000..6178079822
--- /dev/null
+++ b/test/export/test_for_if_both/source_repo/b.src
@@ -0,0 +1 @@
+b
diff --git a/test/export/test_for_if_both/source_repo/c.src b/test/export/test_for_if_both/source_repo/c.src
new file mode 100644
index 0000000000..f2ad6c76f0
--- /dev/null
+++ b/test/export/test_for_if_both/source_repo/c.src
@@ -0,0 +1 @@
+c
diff --git a/test/export/test_glob/BUILD b/test/export/test_glob/BUILD
new file mode 100644
index 0000000000..5706c6b860
--- /dev/null
+++ b/test/export/test_glob/BUILD
@@ -0,0 +1,7 @@
+subinclude("//test/export:export_e2e_test_build_def")
+
+# Export a target that uses glob to select inputs.
+please_export_e2e_test(
+ name = "export_glob",
+ export_targets = ["//:glob_target"],
+)
diff --git a/test/export/test_glob/expected_repo/.plzconfig b/test/export/test_glob/expected_repo/.plzconfig
new file mode 100644
index 0000000000..f8ba31854d
--- /dev/null
+++ b/test/export/test_glob/expected_repo/.plzconfig
@@ -0,0 +1,2 @@
+[Parse]
+BuildFileName = BUILD_FILE
diff --git a/test/export/test_glob/expected_repo/BUILD_FILE b/test/export/test_glob/expected_repo/BUILD_FILE
new file mode 100644
index 0000000000..b286492ee9
--- /dev/null
+++ b/test/export/test_glob/expected_repo/BUILD_FILE
@@ -0,0 +1,6 @@
+genrule(
+ name = "glob_target",
+ srcs = glob(["*.txt"], exclude = ["ignored.txt"]),
+ outs = ["out.txt"],
+ cmd = "cat $SRCS > $OUT",
+)
diff --git a/test/export/test_glob/expected_repo/file1.txt b/test/export/test_glob/expected_repo/file1.txt
new file mode 100644
index 0000000000..b5cea514f6
--- /dev/null
+++ b/test/export/test_glob/expected_repo/file1.txt
@@ -0,0 +1 @@
+hello from file 1
diff --git a/test/export/test_glob/expected_repo/file2.txt b/test/export/test_glob/expected_repo/file2.txt
new file mode 100644
index 0000000000..7331ba8736
--- /dev/null
+++ b/test/export/test_glob/expected_repo/file2.txt
@@ -0,0 +1 @@
+hello from file 2
diff --git a/test/export/test_glob/source_repo/.plzconfig b/test/export/test_glob/source_repo/.plzconfig
new file mode 100644
index 0000000000..f8ba31854d
--- /dev/null
+++ b/test/export/test_glob/source_repo/.plzconfig
@@ -0,0 +1,2 @@
+[Parse]
+BuildFileName = BUILD_FILE
diff --git a/test/export/test_glob/source_repo/BUILD_FILE b/test/export/test_glob/source_repo/BUILD_FILE
new file mode 100644
index 0000000000..0a25600d17
--- /dev/null
+++ b/test/export/test_glob/source_repo/BUILD_FILE
@@ -0,0 +1,13 @@
+genrule(
+ name = "glob_target",
+ srcs = glob(["*.txt"], exclude = ["ignored.txt"]),
+ outs = ["out.txt"],
+ cmd = "cat $SRCS > $OUT",
+)
+
+genrule(
+ name = "unneeded",
+ srcs = ["unused.dat"],
+ outs = ["unneeded.out"],
+ cmd = "cat $SRCS > $OUT",
+)
diff --git a/test/export/test_glob/source_repo/file1.txt b/test/export/test_glob/source_repo/file1.txt
new file mode 100644
index 0000000000..b5cea514f6
--- /dev/null
+++ b/test/export/test_glob/source_repo/file1.txt
@@ -0,0 +1 @@
+hello from file 1
diff --git a/test/export/test_glob/source_repo/file2.txt b/test/export/test_glob/source_repo/file2.txt
new file mode 100644
index 0000000000..7331ba8736
--- /dev/null
+++ b/test/export/test_glob/source_repo/file2.txt
@@ -0,0 +1 @@
+hello from file 2
diff --git a/test/export/test_glob/source_repo/ignored.txt b/test/export/test_glob/source_repo/ignored.txt
new file mode 100644
index 0000000000..44d282adba
--- /dev/null
+++ b/test/export/test_glob/source_repo/ignored.txt
@@ -0,0 +1 @@
+hello from ignored file
diff --git a/test/export/test_glob/source_repo/unused.dat b/test/export/test_glob/source_repo/unused.dat
new file mode 100644
index 0000000000..e8f53910e2
--- /dev/null
+++ b/test/export/test_glob/source_repo/unused.dat
@@ -0,0 +1 @@
+hello from unused file
diff --git a/test/export/test_go_bin/expected_repo/BUILD_FILE b/test/export/test_go_bin/expected_repo/BUILD_FILE
index d0d1a145c0..015bd671a5 100644
--- a/test/export/test_go_bin/expected_repo/BUILD_FILE
+++ b/test/export/test_go_bin/expected_repo/BUILD_FILE
@@ -4,6 +4,6 @@ go_binary(
name = "bin_go_dep",
srcs = ["main.go"],
deps = [
- "///third_party/go/github.com_google_go-cmp//cmp",
+ "//third_party/go:cmp",
],
)
diff --git a/test/export/test_go_bin/expected_repo/third_party/go/BUILD_FILE b/test/export/test_go_bin/expected_repo/third_party/go/BUILD_FILE
index e436be33ec..77ef426dd5 100644
--- a/test/export/test_go_bin/expected_repo/third_party/go/BUILD_FILE
+++ b/test/export/test_go_bin/expected_repo/third_party/go/BUILD_FILE
@@ -20,6 +20,8 @@ go_stdlib(
)
go_repo(
+ name = "cmp",
+ install = ["..."],
module = "github.com/google/go-cmp",
version = "v0.5.6",
)
diff --git a/test/export/test_go_bin/source_repo/BUILD_FILE b/test/export/test_go_bin/source_repo/BUILD_FILE
index d37abc25b4..980cd46364 100644
--- a/test/export/test_go_bin/source_repo/BUILD_FILE
+++ b/test/export/test_go_bin/source_repo/BUILD_FILE
@@ -4,14 +4,14 @@ go_binary(
name = "bin_go_dep",
srcs = ["main.go"],
deps = [
- "///third_party/go/github.com_google_go-cmp//cmp",
+ "//third_party/go:cmp",
],
)
go_binary(
- name = "dummy",
- srcs = ["dummy.go"],
+ name = "unneded",
+ srcs = ["unneded.go"],
deps = [
- "///third_party/go/github.com_stretchr_testify//:testify",
+ "//third_party/go:uuid",
],
)
diff --git a/test/export/test_go_bin/source_repo/dummy.go b/test/export/test_go_bin/source_repo/dummy.go
deleted file mode 100644
index 3c49d61525..0000000000
--- a/test/export/test_go_bin/source_repo/dummy.go
+++ /dev/null
@@ -1,11 +0,0 @@
-package dummy
-
-import (
- "fmt"
-
- "github.com/stretchr/testify/"
-)
-
-func main() {
- return
-}
diff --git a/test/export/test_go_bin/source_repo/third_party/go/BUILD_FILE b/test/export/test_go_bin/source_repo/third_party/go/BUILD_FILE
index 20d92fd1ec..052c347845 100644
--- a/test/export/test_go_bin/source_repo/third_party/go/BUILD_FILE
+++ b/test/export/test_go_bin/source_repo/third_party/go/BUILD_FILE
@@ -20,25 +20,15 @@ go_stdlib(
)
go_repo(
+ name = "cmp",
+ install = ["..."],
module = "github.com/google/go-cmp",
version = "v0.5.6",
)
go_repo(
- # Dummy, unused target
- name = "testify",
- install = ["..."],
- licences = ["MIT"],
- module = "github.com/stretchr/testify",
- version = "v1.7.0",
- deps = [":yaml.v3"], # test we can depend on go_modules
-)
-
-go_module(
- # Dummy, unused target
- name = "yaml.v3",
- licences = ["MIT"],
- module = "gopkg.in/yaml.v3",
- version = "v3.0.0-20210107192922-496545a6307b",
- visibility = ["PUBLIC"],
+ # unneeded, unused target
+ name = "uuid",
+ module = "github.com/google/uuid",
+ version = "v1.6.0",
)
diff --git a/test/export/test_go_bin/source_repo/unneeded.go b/test/export/test_go_bin/source_repo/unneeded.go
new file mode 100644
index 0000000000..277888f925
--- /dev/null
+++ b/test/export/test_go_bin/source_repo/unneeded.go
@@ -0,0 +1,5 @@
+package unneded
+
+func main() {
+ return
+}
diff --git a/test/export/test_go_test/BUILD b/test/export/test_go_test/BUILD
new file mode 100644
index 0000000000..e8c41a89e9
--- /dev/null
+++ b/test/export/test_go_test/BUILD
@@ -0,0 +1,7 @@
+subinclude("//test/export:export_e2e_test_build_def")
+
+# Test go_test target with a go third_party dependency that implicitly requires other packages.
+please_export_e2e_test(
+ name = "export_go_test",
+ export_targets = ["//:test"],
+)
diff --git a/test/export/test_go_test/expected_repo/.plzconfig b/test/export/test_go_test/expected_repo/.plzconfig
new file mode 100644
index 0000000000..7fb433a97e
--- /dev/null
+++ b/test/export/test_go_test/expected_repo/.plzconfig
@@ -0,0 +1,10 @@
+[Parse]
+BuildFileName = BUILD # required by subrepos
+BuildFileName = BUILD_FILE
+preloadsubincludes = ///go//build_defs:go
+
+[Plugin "go"]
+Target = //plugins:go
+GoTool = //third_party/go:toolchain|go
+STDLib = //third_party/go:std
+ModFile = //:go_mod
diff --git a/test/export/test_go_test/expected_repo/BUILD_FILE b/test/export/test_go_test/expected_repo/BUILD_FILE
new file mode 100644
index 0000000000..516ef528ca
--- /dev/null
+++ b/test/export/test_go_test/expected_repo/BUILD_FILE
@@ -0,0 +1,11 @@
+filegroup(
+ name = "go_mod",
+ srcs = ["go.mod"],
+ visibility = ["PUBLIC"],
+)
+
+go_test(
+ name = "test",
+ srcs = ["test.go"],
+ deps = ["///third_party/go/github.com_stretchr_testify//assert"],
+)
diff --git a/test/export/test_go_test/expected_repo/go.mod b/test/export/test_go_test/expected_repo/go.mod
new file mode 100644
index 0000000000..7fcf309969
--- /dev/null
+++ b/test/export/test_go_test/expected_repo/go.mod
@@ -0,0 +1,11 @@
+module github.com/thought-machine/please/test_repo
+
+go 1.23.0
+
+require github.com/stretchr/testify v1.9.0
+
+require (
+ github.com/davecgh/go-spew v1.1.1 // indirect
+ github.com/pmezard/go-difflib v1.0.0 // indirect
+ gopkg.in/yaml.v3 v3.0.1 // indirect
+)
diff --git a/test/export/test_go_test/expected_repo/plugins/BUILD_FILE b/test/export/test_go_test/expected_repo/plugins/BUILD_FILE
new file mode 100644
index 0000000000..55e81b8e1d
--- /dev/null
+++ b/test/export/test_go_test/expected_repo/plugins/BUILD_FILE
@@ -0,0 +1,6 @@
+plugin_repo(
+ name = "go",
+ owner = "please-build",
+ plugin = "go-rules",
+ revision = "v1.30.0",
+)
diff --git a/test/export/test_go_test/expected_repo/test.go b/test/export/test_go_test/expected_repo/test.go
new file mode 100644
index 0000000000..aaf3c0c44b
--- /dev/null
+++ b/test/export/test_go_test/expected_repo/test.go
@@ -0,0 +1,11 @@
+package test
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestSomething(t *testing.T) {
+ assert.NotEqual(t, 0, 1)
+}
diff --git a/test/export/test_go_test/expected_repo/third_party/go/BUILD_FILE b/test/export/test_go_test/expected_repo/third_party/go/BUILD_FILE
new file mode 100644
index 0000000000..42a26f4f9c
--- /dev/null
+++ b/test/export/test_go_test/expected_repo/third_party/go/BUILD_FILE
@@ -0,0 +1,30 @@
+go_toolchain(
+ name = "toolchain",
+ version = "1.26.2",
+)
+
+go_stdlib(name = "std")
+
+go_repo(
+ licences = ["MIT"],
+ module = "github.com/stretchr/testify",
+ version = "v1.9.0",
+)
+
+go_repo(
+ licences = ["ISC"],
+ module = "github.com/davecgh/go-spew",
+ version = "v1.1.1",
+)
+
+go_repo(
+ licences = ["BSD-3-Clause"],
+ module = "github.com/pmezard/go-difflib",
+ version = "v1.0.0",
+)
+
+go_repo(
+ licences = ["MIT"],
+ module = "gopkg.in/yaml.v3",
+ version = "v3.0.1",
+)
diff --git a/test/export/test_go_test/source_repo/.plzconfig b/test/export/test_go_test/source_repo/.plzconfig
new file mode 100644
index 0000000000..7fb433a97e
--- /dev/null
+++ b/test/export/test_go_test/source_repo/.plzconfig
@@ -0,0 +1,10 @@
+[Parse]
+BuildFileName = BUILD # required by subrepos
+BuildFileName = BUILD_FILE
+preloadsubincludes = ///go//build_defs:go
+
+[Plugin "go"]
+Target = //plugins:go
+GoTool = //third_party/go:toolchain|go
+STDLib = //third_party/go:std
+ModFile = //:go_mod
diff --git a/test/export/test_go_test/source_repo/BUILD_FILE b/test/export/test_go_test/source_repo/BUILD_FILE
new file mode 100644
index 0000000000..c916f9d154
--- /dev/null
+++ b/test/export/test_go_test/source_repo/BUILD_FILE
@@ -0,0 +1,19 @@
+filegroup(
+ name = "go_mod",
+ srcs = ["go.mod"],
+ visibility = ["PUBLIC"],
+)
+
+go_test(
+ name = "test",
+ srcs = ["test.go"],
+ deps = ["///third_party/go/github.com_stretchr_testify//assert"],
+)
+
+go_binary(
+ name = "unused",
+ srcs = ["main.go"],
+ deps = [
+ "//third_party/go:cmp",
+ ],
+)
diff --git a/test/export/test_go_test/source_repo/go.mod b/test/export/test_go_test/source_repo/go.mod
new file mode 100644
index 0000000000..7fcf309969
--- /dev/null
+++ b/test/export/test_go_test/source_repo/go.mod
@@ -0,0 +1,11 @@
+module github.com/thought-machine/please/test_repo
+
+go 1.23.0
+
+require github.com/stretchr/testify v1.9.0
+
+require (
+ github.com/davecgh/go-spew v1.1.1 // indirect
+ github.com/pmezard/go-difflib v1.0.0 // indirect
+ gopkg.in/yaml.v3 v3.0.1 // indirect
+)
diff --git a/test/export/test_go_test/source_repo/main.go b/test/export/test_go_test/source_repo/main.go
new file mode 100644
index 0000000000..0853120a28
--- /dev/null
+++ b/test/export/test_go_test/source_repo/main.go
@@ -0,0 +1,11 @@
+package main
+
+import (
+ "fmt"
+
+ "github.com/google/go-cmp/cmp"
+)
+
+func main() {
+ fmt.Print(cmp.Equal(1, 1))
+}
diff --git a/test/export/test_go_test/source_repo/plugins/BUILD_FILE b/test/export/test_go_test/source_repo/plugins/BUILD_FILE
new file mode 100644
index 0000000000..55e81b8e1d
--- /dev/null
+++ b/test/export/test_go_test/source_repo/plugins/BUILD_FILE
@@ -0,0 +1,6 @@
+plugin_repo(
+ name = "go",
+ owner = "please-build",
+ plugin = "go-rules",
+ revision = "v1.30.0",
+)
diff --git a/test/export/test_go_test/source_repo/test.go b/test/export/test_go_test/source_repo/test.go
new file mode 100644
index 0000000000..aaf3c0c44b
--- /dev/null
+++ b/test/export/test_go_test/source_repo/test.go
@@ -0,0 +1,11 @@
+package test
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestSomething(t *testing.T) {
+ assert.NotEqual(t, 0, 1)
+}
diff --git a/test/export/test_go_test/source_repo/third_party/go/BUILD_FILE b/test/export/test_go_test/source_repo/third_party/go/BUILD_FILE
new file mode 100644
index 0000000000..88c12fe2a0
--- /dev/null
+++ b/test/export/test_go_test/source_repo/third_party/go/BUILD_FILE
@@ -0,0 +1,36 @@
+go_toolchain(
+ name = "toolchain",
+ version = "1.26.2",
+)
+
+go_stdlib(name = "std")
+
+go_repo(
+ licences = ["BSD-3-Clause"],
+ module = "github.com/google/go-cmp",
+ version = "v0.6.0",
+)
+
+go_repo(
+ licences = ["MIT"],
+ module = "github.com/stretchr/testify",
+ version = "v1.9.0",
+)
+
+go_repo(
+ licences = ["ISC"],
+ module = "github.com/davecgh/go-spew",
+ version = "v1.1.1",
+)
+
+go_repo(
+ licences = ["BSD-3-Clause"],
+ module = "github.com/pmezard/go-difflib",
+ version = "v1.0.0",
+)
+
+go_repo(
+ licences = ["MIT"],
+ module = "gopkg.in/yaml.v3",
+ version = "v3.0.1",
+)
diff --git a/test/export/test_if/BUILD b/test/export/test_if/BUILD
new file mode 100644
index 0000000000..4af4d89e89
--- /dev/null
+++ b/test/export/test_if/BUILD
@@ -0,0 +1,7 @@
+subinclude("//test/export:export_e2e_test_build_def")
+
+# Export a target inside an if block and validate that the else block is correctly trimmed (and replaced with pass).
+please_export_e2e_test(
+ name = "export_if",
+ export_targets = ["//:a"],
+)
diff --git a/test/export/test_if/expected_repo/.plzconfig b/test/export/test_if/expected_repo/.plzconfig
new file mode 100644
index 0000000000..f8ba31854d
--- /dev/null
+++ b/test/export/test_if/expected_repo/.plzconfig
@@ -0,0 +1,2 @@
+[Parse]
+BuildFileName = BUILD_FILE
diff --git a/test/export/test_if/expected_repo/BUILD_FILE b/test/export/test_if/expected_repo/BUILD_FILE
new file mode 100644
index 0000000000..c69bf4c1c0
--- /dev/null
+++ b/test/export/test_if/expected_repo/BUILD_FILE
@@ -0,0 +1,8 @@
+if True:
+ genrule(
+ name = "a",
+ outs = ["a.txt"],
+ cmd = "echo a > $OUT",
+ )
+else:
+ pass # Trimmed during export
diff --git a/test/export/test_if/source_repo/.plzconfig b/test/export/test_if/source_repo/.plzconfig
new file mode 100644
index 0000000000..f8ba31854d
--- /dev/null
+++ b/test/export/test_if/source_repo/.plzconfig
@@ -0,0 +1,2 @@
+[Parse]
+BuildFileName = BUILD_FILE
diff --git a/test/export/test_if/source_repo/BUILD_FILE b/test/export/test_if/source_repo/BUILD_FILE
new file mode 100644
index 0000000000..ef5650c499
--- /dev/null
+++ b/test/export/test_if/source_repo/BUILD_FILE
@@ -0,0 +1,12 @@
+if True:
+ genrule(
+ name = "a",
+ outs = ["a.txt"],
+ cmd = "echo a > $OUT",
+ )
+else:
+ genrule(
+ name = "b",
+ outs = ["b.txt"],
+ cmd = "echo b > $OUT",
+ )
diff --git a/test/export/test_custom_in_file_def/BUILD b/test/export/test_in_file_build_def/BUILD
similarity index 83%
rename from test/export/test_custom_in_file_def/BUILD
rename to test/export/test_in_file_build_def/BUILD
index 5f361a61dd..a9670e332f 100644
--- a/test/export/test_custom_in_file_def/BUILD
+++ b/test/export/test_in_file_build_def/BUILD
@@ -3,6 +3,6 @@ subinclude("//test/export:export_e2e_test_build_def")
# Export a target generated by a custom build def defined in the same
# BUILD file.
please_export_e2e_test(
- name = "export_custom_target_in_file",
+ name = "export_in_file_build_def",
export_targets = ["//:simple_custom_target"],
)
diff --git a/test/export/test_in_file_build_def/expected_repo/.plzconfig b/test/export/test_in_file_build_def/expected_repo/.plzconfig
new file mode 100644
index 0000000000..8e1ae5655a
--- /dev/null
+++ b/test/export/test_in_file_build_def/expected_repo/.plzconfig
@@ -0,0 +1,3 @@
+[Parse]
+
+BuildFileName = BUILD_FILE
diff --git a/test/export/test_custom_in_file_def/expected_repo/BUILD_FILE b/test/export/test_in_file_build_def/expected_repo/BUILD_FILE
similarity index 100%
rename from test/export/test_custom_in_file_def/expected_repo/BUILD_FILE
rename to test/export/test_in_file_build_def/expected_repo/BUILD_FILE
diff --git a/test/export/test_custom_in_file_def/expected_repo/file.txt b/test/export/test_in_file_build_def/expected_repo/file.txt
similarity index 100%
rename from test/export/test_custom_in_file_def/expected_repo/file.txt
rename to test/export/test_in_file_build_def/expected_repo/file.txt
diff --git a/test/export/test_custom_in_file_def/expected_repo/simple.build_defs b/test/export/test_in_file_build_def/expected_repo/simple.build_defs
similarity index 100%
rename from test/export/test_custom_in_file_def/expected_repo/simple.build_defs
rename to test/export/test_in_file_build_def/expected_repo/simple.build_defs
diff --git a/test/export/test_in_file_build_def/source_repo/.plzconfig b/test/export/test_in_file_build_def/source_repo/.plzconfig
new file mode 100644
index 0000000000..8e1ae5655a
--- /dev/null
+++ b/test/export/test_in_file_build_def/source_repo/.plzconfig
@@ -0,0 +1,3 @@
+[Parse]
+
+BuildFileName = BUILD_FILE
diff --git a/test/export/test_custom_in_file_def/source_repo/BUILD_FILE b/test/export/test_in_file_build_def/source_repo/BUILD_FILE
similarity index 79%
rename from test/export/test_custom_in_file_def/source_repo/BUILD_FILE
rename to test/export/test_in_file_build_def/source_repo/BUILD_FILE
index 6e5251d844..76b62d74b4 100644
--- a/test/export/test_custom_in_file_def/source_repo/BUILD_FILE
+++ b/test/export/test_in_file_build_def/source_repo/BUILD_FILE
@@ -13,7 +13,7 @@ simple_custom_target(
)
simple_custom_target(
- name = "dummy",
- srcs = ["dummy.in"],
- outs = ["dummy.out"],
+ name = "unneded",
+ srcs = ["unneded.in"],
+ outs = ["unneded.out"],
)
diff --git a/test/export/test_custom_in_file_def/source_repo/dummy.in b/test/export/test_in_file_build_def/source_repo/dummy.in
similarity index 100%
rename from test/export/test_custom_in_file_def/source_repo/dummy.in
rename to test/export/test_in_file_build_def/source_repo/dummy.in
diff --git a/test/export/test_custom_in_file_def/source_repo/file.txt b/test/export/test_in_file_build_def/source_repo/file.txt
similarity index 100%
rename from test/export/test_custom_in_file_def/source_repo/file.txt
rename to test/export/test_in_file_build_def/source_repo/file.txt
diff --git a/test/export/test_custom_in_file_def/source_repo/simple.build_defs b/test/export/test_in_file_build_def/source_repo/simple.build_defs
similarity index 100%
rename from test/export/test_custom_in_file_def/source_repo/simple.build_defs
rename to test/export/test_in_file_build_def/source_repo/simple.build_defs
diff --git a/test/export/test_in_file_func_def/BUILD b/test/export/test_in_file_func_def/BUILD
new file mode 100644
index 0000000000..cfb1eabd83
--- /dev/null
+++ b/test/export/test_in_file_func_def/BUILD
@@ -0,0 +1,8 @@
+subinclude("//test/export:export_e2e_test_build_def")
+
+# Export a target generated by a custom function def defined in the same
+# BUILD file.
+please_export_e2e_test(
+ name = "export_in_file_function",
+ export_targets = ["//:custom_target"],
+)
diff --git a/test/export/test_in_file_func_def/expected_repo/.plzconfig b/test/export/test_in_file_func_def/expected_repo/.plzconfig
new file mode 100644
index 0000000000..8e1ae5655a
--- /dev/null
+++ b/test/export/test_in_file_func_def/expected_repo/.plzconfig
@@ -0,0 +1,3 @@
+[Parse]
+
+BuildFileName = BUILD_FILE
diff --git a/test/export/test_in_file_func_def/expected_repo/BUILD_FILE b/test/export/test_in_file_func_def/expected_repo/BUILD_FILE
new file mode 100644
index 0000000000..4638bb33ab
--- /dev/null
+++ b/test/export/test_in_file_func_def/expected_repo/BUILD_FILE
@@ -0,0 +1,16 @@
+def custom(
+ name:str,
+ srcs:list=[],
+ outs:list=[]):
+ return genrule(
+ name = name,
+ srcs = srcs,
+ outs = outs,
+ cmd = "cat $SRCS > $OUT",
+ )
+
+custom(
+ name = "custom_target",
+ srcs = ["file.txt"],
+ outs = ["file_simple.out"],
+)
diff --git a/test/export/test_in_file_func_def/expected_repo/file.txt b/test/export/test_in_file_func_def/expected_repo/file.txt
new file mode 100644
index 0000000000..9768ee14c2
--- /dev/null
+++ b/test/export/test_in_file_func_def/expected_repo/file.txt
@@ -0,0 +1 @@
+Test source file
diff --git a/test/export/test_in_file_func_def/source_repo/.plzconfig b/test/export/test_in_file_func_def/source_repo/.plzconfig
new file mode 100644
index 0000000000..8e1ae5655a
--- /dev/null
+++ b/test/export/test_in_file_func_def/source_repo/.plzconfig
@@ -0,0 +1,3 @@
+[Parse]
+
+BuildFileName = BUILD_FILE
diff --git a/test/export/test_in_file_func_def/source_repo/BUILD_FILE b/test/export/test_in_file_func_def/source_repo/BUILD_FILE
new file mode 100644
index 0000000000..2f589a44e1
--- /dev/null
+++ b/test/export/test_in_file_func_def/source_repo/BUILD_FILE
@@ -0,0 +1,22 @@
+def custom(
+ name:str,
+ srcs:list=[],
+ outs:list=[]):
+ return genrule(
+ name = name,
+ srcs = srcs,
+ outs = outs,
+ cmd = "cat $SRCS > $OUT",
+ )
+
+custom(
+ name = "custom_target",
+ srcs = ["file.txt"],
+ outs = ["file_simple.out"],
+)
+
+custom(
+ name = "unused",
+ srcs = ["unused.in"],
+ outs = ["unused.out"],
+)
diff --git a/test/export/test_in_file_func_def/source_repo/dummy.in b/test/export/test_in_file_func_def/source_repo/dummy.in
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/test/export/test_in_file_func_def/source_repo/file.txt b/test/export/test_in_file_func_def/source_repo/file.txt
new file mode 100644
index 0000000000..9768ee14c2
--- /dev/null
+++ b/test/export/test_in_file_func_def/source_repo/file.txt
@@ -0,0 +1 @@
+Test source file
diff --git a/test/export/test_in_file_func_def/source_repo/unused.in b/test/export/test_in_file_func_def/source_repo/unused.in
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/test/export/test_multiple_targets/source_repo/BUILD_FILE b/test/export/test_multiple_targets/source_repo/BUILD_FILE
index 3dc36004b0..856ca1b4bb 100644
--- a/test/export/test_multiple_targets/source_repo/BUILD_FILE
+++ b/test/export/test_multiple_targets/source_repo/BUILD_FILE
@@ -13,7 +13,7 @@ genrule(
)
genrule(
- name = "dummy",
- outs = ["dummy.out"],
- cmd = "echo dummy > $OUT",
+ name = "unneded",
+ outs = ["unneded.out"],
+ cmd = "echo unneded > $OUT",
)
diff --git a/test/export/test_native_target_with_go_dep/source_repo/third_party/common/BUILD_FILE b/test/export/test_native_target_with_go_dep/source_repo/third_party/common/BUILD_FILE
new file mode 100644
index 0000000000..4a67400211
--- /dev/null
+++ b/test/export/test_native_target_with_go_dep/source_repo/third_party/common/BUILD_FILE
@@ -0,0 +1,7 @@
+export_file(
+ name = "version",
+ src = "version.build_defs",
+ visibility = [
+ "//third_party/...",
+ ],
+)
diff --git a/test/export/test_native_target_with_go_dep/source_repo/third_party/common/version.build_defs b/test/export/test_native_target_with_go_dep/source_repo/third_party/common/version.build_defs
new file mode 100644
index 0000000000..3449cc1b8f
--- /dev/null
+++ b/test/export/test_native_target_with_go_dep/source_repo/third_party/common/version.build_defs
@@ -0,0 +1 @@
+LIB_VERSION="v1.6.0"
diff --git a/test/export/test_native_target_with_go_dep/source_repo/third_party/go/BUILD_FILE b/test/export/test_native_target_with_go_dep/source_repo/third_party/go/BUILD_FILE
index e436be33ec..54a9c020b5 100644
--- a/test/export/test_native_target_with_go_dep/source_repo/third_party/go/BUILD_FILE
+++ b/test/export/test_native_target_with_go_dep/source_repo/third_party/go/BUILD_FILE
@@ -1,4 +1,7 @@
-subinclude("///go//build_defs:go")
+subinclude(
+ "///go//build_defs:go",
+ "//third_party/common:version",
+)
package(default_visibility = ["PUBLIC"])
@@ -19,6 +22,13 @@ go_stdlib(
name = "std",
)
+go_repo(
+ # unneeded, unused target
+ name = "uuid",
+ module = "github.com/google/uuid",
+ version = f"{LIB_VERSION}",
+)
+
go_repo(
module = "github.com/google/go-cmp",
version = "v0.5.6",
diff --git a/test/export/test_nested_subinclude/BUILD b/test/export/test_nested_subinclude/BUILD
new file mode 100644
index 0000000000..acce896754
--- /dev/null
+++ b/test/export/test_nested_subinclude/BUILD
@@ -0,0 +1,6 @@
+subinclude("//test/export:export_e2e_test_build_def")
+
+please_export_e2e_test(
+ name = "nested_subinclude",
+ export_targets = ["//:target"],
+)
diff --git a/test/export/test_nested_subinclude/expected_repo/.plzconfig b/test/export/test_nested_subinclude/expected_repo/.plzconfig
new file mode 100644
index 0000000000..f8ba31854d
--- /dev/null
+++ b/test/export/test_nested_subinclude/expected_repo/.plzconfig
@@ -0,0 +1,2 @@
+[Parse]
+BuildFileName = BUILD_FILE
diff --git a/test/export/test_nested_subinclude/expected_repo/BUILD_FILE b/test/export/test_nested_subinclude/expected_repo/BUILD_FILE
new file mode 100644
index 0000000000..0ef1d0178a
--- /dev/null
+++ b/test/export/test_nested_subinclude/expected_repo/BUILD_FILE
@@ -0,0 +1,5 @@
+subinclude("//build_defs:primary_build_def")
+
+helper_rule_primary(
+ name = "target",
+)
diff --git a/test/export/test_nested_subinclude/expected_repo/build_defs/BUILD_FILE b/test/export/test_nested_subinclude/expected_repo/build_defs/BUILD_FILE
new file mode 100644
index 0000000000..7e567b845a
--- /dev/null
+++ b/test/export/test_nested_subinclude/expected_repo/build_defs/BUILD_FILE
@@ -0,0 +1,11 @@
+filegroup(
+ name = "primary_build_def",
+ srcs = ["primary.build_defs"],
+ visibility = ["PUBLIC"],
+)
+
+filegroup(
+ name = "secondary_build_def",
+ srcs = ["secondary.build_defs"],
+ visibility = ["PUBLIC"],
+)
diff --git a/test/export/test_nested_subinclude/expected_repo/build_defs/primary.build_defs b/test/export/test_nested_subinclude/expected_repo/build_defs/primary.build_defs
new file mode 100644
index 0000000000..333d80bced
--- /dev/null
+++ b/test/export/test_nested_subinclude/expected_repo/build_defs/primary.build_defs
@@ -0,0 +1,10 @@
+subinclude("//build_defs:secondary_build_def")
+
+def helper_rule_primary(name: str):
+ # Generates the requested target (which has no unparsed dependencies)
+ filegroup(
+ name = name,
+ srcs = [],
+ )
+ # Generates an adjacent target (which calls helper_rule and has an unparsed dependency)
+ helper_rule(name + "_adjacent")
diff --git a/test/export/test_nested_subinclude/expected_repo/build_defs/secondary.build_defs b/test/export/test_nested_subinclude/expected_repo/build_defs/secondary.build_defs
new file mode 100644
index 0000000000..10e3974137
--- /dev/null
+++ b/test/export/test_nested_subinclude/expected_repo/build_defs/secondary.build_defs
@@ -0,0 +1,7 @@
+def helper_rule(name: str):
+ build_rule(
+ name = name,
+ cmd = "sh $TOOL > $OUT",
+ tools = ["//tools:tools"],
+ outs = [name + ".out"],
+ )
diff --git a/test/export/test_nested_subinclude/expected_repo/tools/BUILD_FILE b/test/export/test_nested_subinclude/expected_repo/tools/BUILD_FILE
new file mode 100644
index 0000000000..215281ac57
--- /dev/null
+++ b/test/export/test_nested_subinclude/expected_repo/tools/BUILD_FILE
@@ -0,0 +1,5 @@
+filegroup(
+ name = "tools",
+ srcs = ["tools.sh"],
+ visibility = ["PUBLIC"],
+)
diff --git a/test/export/test_nested_subinclude/expected_repo/tools/tools.sh b/test/export/test_nested_subinclude/expected_repo/tools/tools.sh
new file mode 100644
index 0000000000..b77678ef4f
--- /dev/null
+++ b/test/export/test_nested_subinclude/expected_repo/tools/tools.sh
@@ -0,0 +1,2 @@
+#!/bin/sh
+echo "nested dependency success"
diff --git a/test/export/test_nested_subinclude/source_repo/.plzconfig b/test/export/test_nested_subinclude/source_repo/.plzconfig
new file mode 100644
index 0000000000..f8ba31854d
--- /dev/null
+++ b/test/export/test_nested_subinclude/source_repo/.plzconfig
@@ -0,0 +1,2 @@
+[Parse]
+BuildFileName = BUILD_FILE
diff --git a/test/export/test_nested_subinclude/source_repo/BUILD_FILE b/test/export/test_nested_subinclude/source_repo/BUILD_FILE
new file mode 100644
index 0000000000..f96a5a0d1a
--- /dev/null
+++ b/test/export/test_nested_subinclude/source_repo/BUILD_FILE
@@ -0,0 +1,13 @@
+subinclude(
+ "//build_defs:primary_build_def",
+ "//build_defs:unused_build_def",
+)
+
+helper_rule_primary(
+ name = "target",
+)
+
+filegroup(
+ name = "unused_target",
+ srcs = ["file.txt"],
+)
diff --git a/test/export/test_nested_subinclude/source_repo/build_defs/BUILD_FILE b/test/export/test_nested_subinclude/source_repo/build_defs/BUILD_FILE
new file mode 100644
index 0000000000..b65694fcb9
--- /dev/null
+++ b/test/export/test_nested_subinclude/source_repo/build_defs/BUILD_FILE
@@ -0,0 +1,17 @@
+filegroup(
+ name = "primary_build_def",
+ srcs = ["primary.build_defs"],
+ visibility = ["PUBLIC"],
+)
+
+filegroup(
+ name = "secondary_build_def",
+ srcs = ["secondary.build_defs"],
+ visibility = ["PUBLIC"],
+)
+
+filegroup(
+ name = "unused_build_def",
+ srcs = ["unused.build_defs"],
+ visibility = ["PUBLIC"],
+)
diff --git a/test/export/test_nested_subinclude/source_repo/build_defs/primary.build_defs b/test/export/test_nested_subinclude/source_repo/build_defs/primary.build_defs
new file mode 100644
index 0000000000..333d80bced
--- /dev/null
+++ b/test/export/test_nested_subinclude/source_repo/build_defs/primary.build_defs
@@ -0,0 +1,10 @@
+subinclude("//build_defs:secondary_build_def")
+
+def helper_rule_primary(name: str):
+ # Generates the requested target (which has no unparsed dependencies)
+ filegroup(
+ name = name,
+ srcs = [],
+ )
+ # Generates an adjacent target (which calls helper_rule and has an unparsed dependency)
+ helper_rule(name + "_adjacent")
diff --git a/test/export/test_nested_subinclude/source_repo/build_defs/secondary.build_defs b/test/export/test_nested_subinclude/source_repo/build_defs/secondary.build_defs
new file mode 100644
index 0000000000..10e3974137
--- /dev/null
+++ b/test/export/test_nested_subinclude/source_repo/build_defs/secondary.build_defs
@@ -0,0 +1,7 @@
+def helper_rule(name: str):
+ build_rule(
+ name = name,
+ cmd = "sh $TOOL > $OUT",
+ tools = ["//tools:tools"],
+ outs = [name + ".out"],
+ )
diff --git a/test/export/test_nested_subinclude/source_repo/build_defs/unused.build_defs b/test/export/test_nested_subinclude/source_repo/build_defs/unused.build_defs
new file mode 100644
index 0000000000..072217364b
--- /dev/null
+++ b/test/export/test_nested_subinclude/source_repo/build_defs/unused.build_defs
@@ -0,0 +1,2 @@
+def unused_func():
+ pass
diff --git a/test/export/test_nested_subinclude/source_repo/file.txt b/test/export/test_nested_subinclude/source_repo/file.txt
new file mode 100644
index 0000000000..d95f3ad14d
--- /dev/null
+++ b/test/export/test_nested_subinclude/source_repo/file.txt
@@ -0,0 +1 @@
+content
diff --git a/test/export/test_nested_subinclude/source_repo/tools/BUILD_FILE b/test/export/test_nested_subinclude/source_repo/tools/BUILD_FILE
new file mode 100644
index 0000000000..215281ac57
--- /dev/null
+++ b/test/export/test_nested_subinclude/source_repo/tools/BUILD_FILE
@@ -0,0 +1,5 @@
+filegroup(
+ name = "tools",
+ srcs = ["tools.sh"],
+ visibility = ["PUBLIC"],
+)
diff --git a/test/export/test_nested_subinclude/source_repo/tools/tools.sh b/test/export/test_nested_subinclude/source_repo/tools/tools.sh
new file mode 100644
index 0000000000..b77678ef4f
--- /dev/null
+++ b/test/export/test_nested_subinclude/source_repo/tools/tools.sh
@@ -0,0 +1,2 @@
+#!/bin/sh
+echo "nested dependency success"
diff --git a/test/export/test_subinclude_preloaded/BUILD b/test/export/test_subinclude_preloaded/BUILD
new file mode 100644
index 0000000000..70aa25283d
--- /dev/null
+++ b/test/export/test_subinclude_preloaded/BUILD
@@ -0,0 +1,8 @@
+subinclude("//test/export:export_e2e_test_build_def")
+
+# Verifies that `plz export` correctly processes a subincluded build definition target
+# that in turn subincludes a preloaded target.
+please_export_e2e_test(
+ name = "subinclude_preloaded_export_test",
+ export_targets = ["//:file"],
+)
\ No newline at end of file
diff --git a/test/export/test_subinclude_preloaded/expected_repo/.plzconfig b/test/export/test_subinclude_preloaded/expected_repo/.plzconfig
new file mode 100644
index 0000000000..512d2cf400
--- /dev/null
+++ b/test/export/test_subinclude_preloaded/expected_repo/.plzconfig
@@ -0,0 +1,3 @@
+[parse]
+BuildFileName = BUILD_FILE
+preloadsubincludes = //build_defs:preloaded_build_def
\ No newline at end of file
diff --git a/test/export/test_subinclude_preloaded/expected_repo/BUILD_FILE b/test/export/test_subinclude_preloaded/expected_repo/BUILD_FILE
new file mode 100644
index 0000000000..0b66bcffbc
--- /dev/null
+++ b/test/export/test_subinclude_preloaded/expected_repo/BUILD_FILE
@@ -0,0 +1,6 @@
+subinclude("//build_defs:subincluded_build_def")
+
+custom_file(
+ name = "file",
+ src = "file.txt",
+)
diff --git a/test/export/test_subinclude_preloaded/expected_repo/build_defs/BUILD_FILE b/test/export/test_subinclude_preloaded/expected_repo/build_defs/BUILD_FILE
new file mode 100644
index 0000000000..847d65de60
--- /dev/null
+++ b/test/export/test_subinclude_preloaded/expected_repo/build_defs/BUILD_FILE
@@ -0,0 +1,11 @@
+filegroup(
+ name = "preloaded_build_def",
+ srcs = ["preloaded.build_defs"],
+ visibility = ["PUBLIC"],
+)
+
+filegroup(
+ name = "subincluded_build_def",
+ srcs = ["subincluded.build_defs"],
+ visibility = ["PUBLIC"],
+)
diff --git a/test/export/test_subinclude_preloaded/expected_repo/build_defs/preloaded.build_defs b/test/export/test_subinclude_preloaded/expected_repo/build_defs/preloaded.build_defs
new file mode 100644
index 0000000000..b06f59e0a7
--- /dev/null
+++ b/test/export/test_subinclude_preloaded/expected_repo/build_defs/preloaded.build_defs
@@ -0,0 +1 @@
+# preloaded build def
\ No newline at end of file
diff --git a/test/export/test_subinclude_preloaded/expected_repo/build_defs/subincluded.build_defs b/test/export/test_subinclude_preloaded/expected_repo/build_defs/subincluded.build_defs
new file mode 100644
index 0000000000..4d7e0f05fe
--- /dev/null
+++ b/test/export/test_subinclude_preloaded/expected_repo/build_defs/subincluded.build_defs
@@ -0,0 +1,7 @@
+subinclude("//build_defs:preloaded_build_def")
+
+def custom_file(name, src):
+ filegroup(
+ name = name,
+ srcs = [src],
+ )
\ No newline at end of file
diff --git a/test/export/test_subinclude_preloaded/expected_repo/file.txt b/test/export/test_subinclude_preloaded/expected_repo/file.txt
new file mode 100644
index 0000000000..f73f3093ff
--- /dev/null
+++ b/test/export/test_subinclude_preloaded/expected_repo/file.txt
@@ -0,0 +1 @@
+file
diff --git a/test/export/test_subinclude_preloaded/source_repo/.plzconfig b/test/export/test_subinclude_preloaded/source_repo/.plzconfig
new file mode 100644
index 0000000000..512d2cf400
--- /dev/null
+++ b/test/export/test_subinclude_preloaded/source_repo/.plzconfig
@@ -0,0 +1,3 @@
+[parse]
+BuildFileName = BUILD_FILE
+preloadsubincludes = //build_defs:preloaded_build_def
\ No newline at end of file
diff --git a/test/export/test_subinclude_preloaded/source_repo/BUILD_FILE b/test/export/test_subinclude_preloaded/source_repo/BUILD_FILE
new file mode 100644
index 0000000000..9b8b35f482
--- /dev/null
+++ b/test/export/test_subinclude_preloaded/source_repo/BUILD_FILE
@@ -0,0 +1,11 @@
+subinclude("//build_defs:subincluded_build_def")
+
+custom_file(
+ name = "file",
+ src = "file.txt",
+)
+
+custom_file(
+ name = "unneeded",
+ src = "unneeded.txt",
+)
\ No newline at end of file
diff --git a/test/export/test_subinclude_preloaded/source_repo/build_defs/BUILD_FILE b/test/export/test_subinclude_preloaded/source_repo/build_defs/BUILD_FILE
new file mode 100644
index 0000000000..d804dfc31d
--- /dev/null
+++ b/test/export/test_subinclude_preloaded/source_repo/build_defs/BUILD_FILE
@@ -0,0 +1,11 @@
+filegroup(
+ name = "preloaded_build_def",
+ srcs = ["preloaded.build_defs"],
+ visibility = ["PUBLIC"],
+)
+
+filegroup(
+ name = "subincluded_build_def",
+ srcs = ["subincluded.build_defs"],
+ visibility = ["PUBLIC"],
+)
\ No newline at end of file
diff --git a/test/export/test_subinclude_preloaded/source_repo/build_defs/preloaded.build_defs b/test/export/test_subinclude_preloaded/source_repo/build_defs/preloaded.build_defs
new file mode 100644
index 0000000000..b06f59e0a7
--- /dev/null
+++ b/test/export/test_subinclude_preloaded/source_repo/build_defs/preloaded.build_defs
@@ -0,0 +1 @@
+# preloaded build def
\ No newline at end of file
diff --git a/test/export/test_subinclude_preloaded/source_repo/build_defs/subincluded.build_defs b/test/export/test_subinclude_preloaded/source_repo/build_defs/subincluded.build_defs
new file mode 100644
index 0000000000..4d7e0f05fe
--- /dev/null
+++ b/test/export/test_subinclude_preloaded/source_repo/build_defs/subincluded.build_defs
@@ -0,0 +1,7 @@
+subinclude("//build_defs:preloaded_build_def")
+
+def custom_file(name, src):
+ filegroup(
+ name = name,
+ srcs = [src],
+ )
\ No newline at end of file
diff --git a/test/export/test_subinclude_preloaded/source_repo/file.txt b/test/export/test_subinclude_preloaded/source_repo/file.txt
new file mode 100644
index 0000000000..f73f3093ff
--- /dev/null
+++ b/test/export/test_subinclude_preloaded/source_repo/file.txt
@@ -0,0 +1 @@
+file
diff --git a/test/export/test_subinclude_preloaded/source_repo/unneeded.txt b/test/export/test_subinclude_preloaded/source_repo/unneeded.txt
new file mode 100644
index 0000000000..b7390936bf
--- /dev/null
+++ b/test/export/test_subinclude_preloaded/source_repo/unneeded.txt
@@ -0,0 +1 @@
+this is unneeded
\ No newline at end of file
diff --git a/test/export/test_subinclude_trimming/BUILD b/test/export/test_subinclude_trimming/BUILD
new file mode 100644
index 0000000000..ae31a718eb
--- /dev/null
+++ b/test/export/test_subinclude_trimming/BUILD
@@ -0,0 +1,10 @@
+subinclude("//test/export:export_e2e_test_build_def")
+
+# Export a target generated by a custom build def and validate that any unused subincludes are trimmed.
+please_export_e2e_test(
+ name = "export_subinclude_trimming",
+ export_targets = [
+ "//:simple_custom_target",
+ "//:variables_target",
+ ],
+)
diff --git a/test/export/test_subinclude_trimming/expected_repo/.plzconfig b/test/export/test_subinclude_trimming/expected_repo/.plzconfig
new file mode 100644
index 0000000000..8e1ae5655a
--- /dev/null
+++ b/test/export/test_subinclude_trimming/expected_repo/.plzconfig
@@ -0,0 +1,3 @@
+[Parse]
+
+BuildFileName = BUILD_FILE
diff --git a/test/export/test_subinclude_trimming/expected_repo/BUILD_FILE b/test/export/test_subinclude_trimming/expected_repo/BUILD_FILE
new file mode 100644
index 0000000000..3144a15db1
--- /dev/null
+++ b/test/export/test_subinclude_trimming/expected_repo/BUILD_FILE
@@ -0,0 +1,32 @@
+subinclude("//build_defs:simple_build_def")
+
+simple_custom_target(
+ name = "simple_custom_target",
+ srcs = ["file.txt"],
+ outs = ["file_simple.out"],
+)
+
+subinclude(
+ "//build_defs:var1_build_def",
+ "//build_defs:var2_build_def",
+)
+
+comma_separated_vars = ",".join([var1, var2])
+
+genrule(
+ name = "variables_target",
+ outs = ["variable.out"],
+ cmd = f'echo "{comma_separated_vars}" > $OUT',
+)
+
+subinclude("//build_defs:var3_build_def")
+
+message = "Testing {service} on {env}".format(
+ env = "production",
+ service = var3,
+)
+
+subinclude("//build_defs:versions_build_def")
+
+for version, name in VERSIONS.items():
+ pass # Trimmed during export
diff --git a/test/export/test_subinclude_trimming/expected_repo/build_defs/BUILD_FILE b/test/export/test_subinclude_trimming/expected_repo/build_defs/BUILD_FILE
new file mode 100644
index 0000000000..55ab29141b
--- /dev/null
+++ b/test/export/test_subinclude_trimming/expected_repo/build_defs/BUILD_FILE
@@ -0,0 +1,29 @@
+filegroup(
+ name = "simple_build_def",
+ srcs = ["simple.build_defs"],
+ visibility = ["PUBLIC"],
+)
+
+filegroup(
+ name = "var1_build_def",
+ srcs = ["var1.build_defs"],
+ visibility = ["PUBLIC"],
+)
+
+filegroup(
+ name = "var2_build_def",
+ srcs = ["var2.build_defs"],
+ visibility = ["PUBLIC"],
+)
+
+filegroup(
+ name = "var3_build_def",
+ srcs = ["var3.build_defs"],
+ visibility = ["PUBLIC"],
+)
+
+filegroup(
+ name = "versions_build_def",
+ srcs = ["versions.build_defs"],
+ visibility = ["PUBLIC"],
+)
diff --git a/test/export/test_subinclude_trimming/expected_repo/build_defs/simple.build_defs b/test/export/test_subinclude_trimming/expected_repo/build_defs/simple.build_defs
new file mode 100644
index 0000000000..8fe3a6021b
--- /dev/null
+++ b/test/export/test_subinclude_trimming/expected_repo/build_defs/simple.build_defs
@@ -0,0 +1,10 @@
+def simple_custom_target(
+ name:str,
+ srcs:list=[],
+ outs:list=[]):
+ return genrule(
+ name = name,
+ srcs = srcs,
+ outs = outs,
+ cmd = "cat $SRCS > $OUT",
+ )
diff --git a/test/export/test_subinclude_trimming/expected_repo/build_defs/var1.build_defs b/test/export/test_subinclude_trimming/expected_repo/build_defs/var1.build_defs
new file mode 100644
index 0000000000..07814f1e90
--- /dev/null
+++ b/test/export/test_subinclude_trimming/expected_repo/build_defs/var1.build_defs
@@ -0,0 +1 @@
+var1 = "Variable 1"
diff --git a/test/export/test_subinclude_trimming/expected_repo/build_defs/var2.build_defs b/test/export/test_subinclude_trimming/expected_repo/build_defs/var2.build_defs
new file mode 100644
index 0000000000..b17ada62b4
--- /dev/null
+++ b/test/export/test_subinclude_trimming/expected_repo/build_defs/var2.build_defs
@@ -0,0 +1 @@
+var2 = "Variable 2"
diff --git a/test/export/test_subinclude_trimming/expected_repo/build_defs/var3.build_defs b/test/export/test_subinclude_trimming/expected_repo/build_defs/var3.build_defs
new file mode 100644
index 0000000000..c5cfb6d90c
--- /dev/null
+++ b/test/export/test_subinclude_trimming/expected_repo/build_defs/var3.build_defs
@@ -0,0 +1 @@
+var3 = "Variable 3"
diff --git a/test/export/test_subinclude_trimming/expected_repo/build_defs/versions.build_defs b/test/export/test_subinclude_trimming/expected_repo/build_defs/versions.build_defs
new file mode 100644
index 0000000000..e565a23f8f
--- /dev/null
+++ b/test/export/test_subinclude_trimming/expected_repo/build_defs/versions.build_defs
@@ -0,0 +1,4 @@
+VERSIONS = {
+ "1.0": "v1",
+ "1.1": "v1.1",
+}
diff --git a/test/export/test_subinclude_trimming/expected_repo/file.txt b/test/export/test_subinclude_trimming/expected_repo/file.txt
new file mode 100644
index 0000000000..9768ee14c2
--- /dev/null
+++ b/test/export/test_subinclude_trimming/expected_repo/file.txt
@@ -0,0 +1 @@
+Test source file
diff --git a/test/export/test_subinclude_trimming/source_repo/.plzconfig b/test/export/test_subinclude_trimming/source_repo/.plzconfig
new file mode 100644
index 0000000000..8e1ae5655a
--- /dev/null
+++ b/test/export/test_subinclude_trimming/source_repo/.plzconfig
@@ -0,0 +1,3 @@
+[Parse]
+
+BuildFileName = BUILD_FILE
diff --git a/test/export/test_subinclude_trimming/source_repo/BUILD_FILE b/test/export/test_subinclude_trimming/source_repo/BUILD_FILE
new file mode 100644
index 0000000000..2632f80601
--- /dev/null
+++ b/test/export/test_subinclude_trimming/source_repo/BUILD_FILE
@@ -0,0 +1,57 @@
+subinclude(
+ "//build_defs:simple_build_def",
+ "//build_defs:unused_build_def",
+)
+
+simple_custom_target(
+ name = "simple_custom_target",
+ srcs = ["file.txt"],
+ outs = ["file_simple.out"],
+)
+
+unused_target(
+ name = "unneded",
+ outs = ["unneded.out"],
+)
+
+subinclude(
+ "//build_defs:var1_build_def",
+ "//build_defs:var2_build_def",
+)
+
+comma_separated_vars = ",".join([var1, var2])
+
+genrule(
+ name = "variables_target",
+ outs = ["variable.out"],
+ cmd = f'echo "{comma_separated_vars}" > $OUT',
+)
+
+
+genrule(
+ name = "unused_variables_target",
+ outs = ["unused_variable.out"],
+ cmd = f'echo "{unused_var}" > $OUT',
+)
+
+subinclude("//build_defs:var3_build_def")
+
+message = "Testing {service} on {env}".format(
+ env = "production",
+ service = var3,
+)
+
+genrule(
+ name = "format_target",
+ outs = ["format.out"],
+ cmd = f'echo "{message}" > $OUT',
+)
+
+subinclude("//build_defs:versions_build_def")
+
+for version, name in VERSIONS.items():
+ genrule(
+ name = f"version-{version}",
+ outs = [f"version-{version}.out"],
+ cmd = f'echo "{name}" > $OUT',
+ )
diff --git a/test/export/test_subinclude_trimming/source_repo/build_defs/BUILD_FILE b/test/export/test_subinclude_trimming/source_repo/build_defs/BUILD_FILE
new file mode 100644
index 0000000000..b1fdef3b01
--- /dev/null
+++ b/test/export/test_subinclude_trimming/source_repo/build_defs/BUILD_FILE
@@ -0,0 +1,35 @@
+filegroup(
+ name = "simple_build_def",
+ srcs = ["simple.build_defs"],
+ visibility = ["PUBLIC"],
+)
+
+filegroup(
+ name = "var1_build_def",
+ srcs = ["var1.build_defs"],
+ visibility = ["PUBLIC"],
+)
+
+filegroup(
+ name = "var2_build_def",
+ srcs = ["var2.build_defs"],
+ visibility = ["PUBLIC"],
+)
+
+filegroup(
+ name = "var3_build_def",
+ srcs = ["var3.build_defs"],
+ visibility = ["PUBLIC"],
+)
+
+filegroup(
+ name = "unused_build_def",
+ srcs = ["unused.build_defs"],
+ visibility = ["PUBLIC"],
+)
+
+filegroup(
+ name = "versions_build_def",
+ srcs = ["versions.build_defs"],
+ visibility = ["PUBLIC"],
+)
diff --git a/test/export/test_subinclude_trimming/source_repo/build_defs/simple.build_defs b/test/export/test_subinclude_trimming/source_repo/build_defs/simple.build_defs
new file mode 100644
index 0000000000..8fe3a6021b
--- /dev/null
+++ b/test/export/test_subinclude_trimming/source_repo/build_defs/simple.build_defs
@@ -0,0 +1,10 @@
+def simple_custom_target(
+ name:str,
+ srcs:list=[],
+ outs:list=[]):
+ return genrule(
+ name = name,
+ srcs = srcs,
+ outs = outs,
+ cmd = "cat $SRCS > $OUT",
+ )
diff --git a/test/export/test_subinclude_trimming/source_repo/build_defs/unused.build_defs b/test/export/test_subinclude_trimming/source_repo/build_defs/unused.build_defs
new file mode 100644
index 0000000000..a20573cf39
--- /dev/null
+++ b/test/export/test_subinclude_trimming/source_repo/build_defs/unused.build_defs
@@ -0,0 +1,10 @@
+def unused_target(
+ name:str,
+ outs:list=[]):
+ return genrule(
+ name = name,
+ outs = outs,
+ cmd = "echo unneeded > $OUT",
+ )
+
+unused_var = "Unused Variable"
diff --git a/test/export/test_subinclude_trimming/source_repo/build_defs/var1.build_defs b/test/export/test_subinclude_trimming/source_repo/build_defs/var1.build_defs
new file mode 100644
index 0000000000..07814f1e90
--- /dev/null
+++ b/test/export/test_subinclude_trimming/source_repo/build_defs/var1.build_defs
@@ -0,0 +1 @@
+var1 = "Variable 1"
diff --git a/test/export/test_subinclude_trimming/source_repo/build_defs/var2.build_defs b/test/export/test_subinclude_trimming/source_repo/build_defs/var2.build_defs
new file mode 100644
index 0000000000..b17ada62b4
--- /dev/null
+++ b/test/export/test_subinclude_trimming/source_repo/build_defs/var2.build_defs
@@ -0,0 +1 @@
+var2 = "Variable 2"
diff --git a/test/export/test_subinclude_trimming/source_repo/build_defs/var3.build_defs b/test/export/test_subinclude_trimming/source_repo/build_defs/var3.build_defs
new file mode 100644
index 0000000000..c5cfb6d90c
--- /dev/null
+++ b/test/export/test_subinclude_trimming/source_repo/build_defs/var3.build_defs
@@ -0,0 +1 @@
+var3 = "Variable 3"
diff --git a/test/export/test_subinclude_trimming/source_repo/build_defs/versions.build_defs b/test/export/test_subinclude_trimming/source_repo/build_defs/versions.build_defs
new file mode 100644
index 0000000000..e565a23f8f
--- /dev/null
+++ b/test/export/test_subinclude_trimming/source_repo/build_defs/versions.build_defs
@@ -0,0 +1,4 @@
+VERSIONS = {
+ "1.0": "v1",
+ "1.1": "v1.1",
+}
diff --git a/test/export/test_subinclude_trimming/source_repo/file.txt b/test/export/test_subinclude_trimming/source_repo/file.txt
new file mode 100644
index 0000000000..9768ee14c2
--- /dev/null
+++ b/test/export/test_subinclude_trimming/source_repo/file.txt
@@ -0,0 +1 @@
+Test source file
diff --git a/test/export/test_subinclude_unused/BUILD b/test/export/test_subinclude_unused/BUILD
new file mode 100644
index 0000000000..b796cd33ab
--- /dev/null
+++ b/test/export/test_subinclude_unused/BUILD
@@ -0,0 +1,9 @@
+subinclude("//test/export:export_e2e_test_build_def")
+
+# Verifies that when a build definition statically subincludes another target but does not
+# actively invoke its methods, the subincluded target is still fully exported to ensure
+# static subinclude parsing succeeds in the exported repository.
+please_export_e2e_test(
+ name = "subinclude_unused_export_test",
+ export_targets = ["//:file"],
+)
diff --git a/test/export/test_subinclude_unused/expected_repo/.plzconfig b/test/export/test_subinclude_unused/expected_repo/.plzconfig
new file mode 100644
index 0000000000..4d5174001a
--- /dev/null
+++ b/test/export/test_subinclude_unused/expected_repo/.plzconfig
@@ -0,0 +1,2 @@
+[parse]
+BuildFileName = BUILD_FILE
\ No newline at end of file
diff --git a/test/export/test_subinclude_unused/expected_repo/BUILD_FILE b/test/export/test_subinclude_unused/expected_repo/BUILD_FILE
new file mode 100644
index 0000000000..0b66bcffbc
--- /dev/null
+++ b/test/export/test_subinclude_unused/expected_repo/BUILD_FILE
@@ -0,0 +1,6 @@
+subinclude("//build_defs:subincluded_build_def")
+
+custom_file(
+ name = "file",
+ src = "file.txt",
+)
diff --git a/test/export/test_subinclude_unused/expected_repo/build_defs/BUILD_FILE b/test/export/test_subinclude_unused/expected_repo/build_defs/BUILD_FILE
new file mode 100644
index 0000000000..86e5b46e15
--- /dev/null
+++ b/test/export/test_subinclude_unused/expected_repo/build_defs/BUILD_FILE
@@ -0,0 +1,11 @@
+filegroup(
+ name = "subincluded_build_def",
+ srcs = ["subincluded.build_defs"],
+ visibility = ["PUBLIC"],
+)
+
+filegroup(
+ name = "unused_build_def",
+ srcs = ["unused.build_defs"],
+ visibility = ["PUBLIC"],
+)
diff --git a/test/export/test_subinclude_unused/expected_repo/build_defs/subincluded.build_defs b/test/export/test_subinclude_unused/expected_repo/build_defs/subincluded.build_defs
new file mode 100644
index 0000000000..9c61aa4a65
--- /dev/null
+++ b/test/export/test_subinclude_unused/expected_repo/build_defs/subincluded.build_defs
@@ -0,0 +1,7 @@
+subinclude("//build_defs:unused_build_def")
+
+def custom_file(name, src):
+ filegroup(
+ name = name,
+ srcs = [src],
+ )
\ No newline at end of file
diff --git a/test/export/test_subinclude_unused/expected_repo/build_defs/unused.build_defs b/test/export/test_subinclude_unused/expected_repo/build_defs/unused.build_defs
new file mode 100644
index 0000000000..b8a3a700f9
--- /dev/null
+++ b/test/export/test_subinclude_unused/expected_repo/build_defs/unused.build_defs
@@ -0,0 +1,6 @@
+# unused build def content
+
+def unused_helper_method(name):
+ return filegroup(
+ name = name,
+ )
diff --git a/test/export/test_subinclude_unused/expected_repo/file.txt b/test/export/test_subinclude_unused/expected_repo/file.txt
new file mode 100644
index 0000000000..f73f3093ff
--- /dev/null
+++ b/test/export/test_subinclude_unused/expected_repo/file.txt
@@ -0,0 +1 @@
+file
diff --git a/test/export/test_subinclude_unused/source_repo/.plzconfig b/test/export/test_subinclude_unused/source_repo/.plzconfig
new file mode 100644
index 0000000000..4d5174001a
--- /dev/null
+++ b/test/export/test_subinclude_unused/source_repo/.plzconfig
@@ -0,0 +1,2 @@
+[parse]
+BuildFileName = BUILD_FILE
\ No newline at end of file
diff --git a/test/export/test_subinclude_unused/source_repo/BUILD_FILE b/test/export/test_subinclude_unused/source_repo/BUILD_FILE
new file mode 100644
index 0000000000..9b8b35f482
--- /dev/null
+++ b/test/export/test_subinclude_unused/source_repo/BUILD_FILE
@@ -0,0 +1,11 @@
+subinclude("//build_defs:subincluded_build_def")
+
+custom_file(
+ name = "file",
+ src = "file.txt",
+)
+
+custom_file(
+ name = "unneeded",
+ src = "unneeded.txt",
+)
\ No newline at end of file
diff --git a/test/export/test_subinclude_unused/source_repo/build_defs/BUILD_FILE b/test/export/test_subinclude_unused/source_repo/build_defs/BUILD_FILE
new file mode 100644
index 0000000000..549d28ce9a
--- /dev/null
+++ b/test/export/test_subinclude_unused/source_repo/build_defs/BUILD_FILE
@@ -0,0 +1,11 @@
+filegroup(
+ name = "subincluded_build_def",
+ srcs = ["subincluded.build_defs"],
+ visibility = ["PUBLIC"],
+)
+
+filegroup(
+ name = "unused_build_def",
+ srcs = ["unused.build_defs"],
+ visibility = ["PUBLIC"],
+)
\ No newline at end of file
diff --git a/test/export/test_subinclude_unused/source_repo/build_defs/subincluded.build_defs b/test/export/test_subinclude_unused/source_repo/build_defs/subincluded.build_defs
new file mode 100644
index 0000000000..9c61aa4a65
--- /dev/null
+++ b/test/export/test_subinclude_unused/source_repo/build_defs/subincluded.build_defs
@@ -0,0 +1,7 @@
+subinclude("//build_defs:unused_build_def")
+
+def custom_file(name, src):
+ filegroup(
+ name = name,
+ srcs = [src],
+ )
\ No newline at end of file
diff --git a/test/export/test_subinclude_unused/source_repo/build_defs/unused.build_defs b/test/export/test_subinclude_unused/source_repo/build_defs/unused.build_defs
new file mode 100644
index 0000000000..b8a3a700f9
--- /dev/null
+++ b/test/export/test_subinclude_unused/source_repo/build_defs/unused.build_defs
@@ -0,0 +1,6 @@
+# unused build def content
+
+def unused_helper_method(name):
+ return filegroup(
+ name = name,
+ )
diff --git a/test/export/test_subinclude_unused/source_repo/file.txt b/test/export/test_subinclude_unused/source_repo/file.txt
new file mode 100644
index 0000000000..f73f3093ff
--- /dev/null
+++ b/test/export/test_subinclude_unused/source_repo/file.txt
@@ -0,0 +1 @@
+file
diff --git a/test/export/test_subinclude_unused/source_repo/unneeded.txt b/test/export/test_subinclude_unused/source_repo/unneeded.txt
new file mode 100644
index 0000000000..9c1fe51193
--- /dev/null
+++ b/test/export/test_subinclude_unused/source_repo/unneeded.txt
@@ -0,0 +1 @@
+unneeded
\ No newline at end of file
diff --git a/test/export/test_subinclude_variable/BUILD b/test/export/test_subinclude_variable/BUILD
new file mode 100644
index 0000000000..2fcedfa352
--- /dev/null
+++ b/test/export/test_subinclude_variable/BUILD
@@ -0,0 +1,6 @@
+subinclude("//test/export:export_e2e_test_build_def")
+
+please_export_e2e_test(
+ name = "subinclude_variable",
+ export_targets = ["//:test"],
+)
diff --git a/test/export/test_subinclude_variable/expected_repo/.plzconfig b/test/export/test_subinclude_variable/expected_repo/.plzconfig
new file mode 100644
index 0000000000..f8ba31854d
--- /dev/null
+++ b/test/export/test_subinclude_variable/expected_repo/.plzconfig
@@ -0,0 +1,2 @@
+[Parse]
+BuildFileName = BUILD_FILE
diff --git a/test/export/test_subinclude_variable/expected_repo/BUILD_FILE b/test/export/test_subinclude_variable/expected_repo/BUILD_FILE
new file mode 100644
index 0000000000..646febc216
--- /dev/null
+++ b/test/export/test_subinclude_variable/expected_repo/BUILD_FILE
@@ -0,0 +1,13 @@
+filegroup(
+ name = "defs",
+ srcs = ["defs.build_defs"],
+ visibility = ["PUBLIC"],
+)
+
+subinclude(":defs")
+
+genrule(
+ name = "test",
+ outs = ["test.txt"],
+ cmd = f"echo {VAR} > $OUT",
+)
diff --git a/test/export/test_subinclude_variable/expected_repo/defs.build_defs b/test/export/test_subinclude_variable/expected_repo/defs.build_defs
new file mode 100644
index 0000000000..cdd62f88ae
--- /dev/null
+++ b/test/export/test_subinclude_variable/expected_repo/defs.build_defs
@@ -0,0 +1 @@
+VAR = "hello"
diff --git a/test/export/test_subinclude_variable/source_repo/.plzconfig b/test/export/test_subinclude_variable/source_repo/.plzconfig
new file mode 100644
index 0000000000..f8ba31854d
--- /dev/null
+++ b/test/export/test_subinclude_variable/source_repo/.plzconfig
@@ -0,0 +1,2 @@
+[Parse]
+BuildFileName = BUILD_FILE
diff --git a/test/export/test_subinclude_variable/source_repo/BUILD_FILE b/test/export/test_subinclude_variable/source_repo/BUILD_FILE
new file mode 100644
index 0000000000..c63a0e733b
--- /dev/null
+++ b/test/export/test_subinclude_variable/source_repo/BUILD_FILE
@@ -0,0 +1,19 @@
+filegroup(
+ name = "defs",
+ srcs = ["defs.build_defs"],
+ visibility = ["PUBLIC"],
+)
+
+subinclude(":defs")
+
+genrule(
+ name = "test",
+ outs = ["test.txt"],
+ cmd = f"echo {VAR} > $OUT",
+)
+
+genrule(
+ name = "unused",
+ outs = ["unused.txt"],
+ cmd = "touch $OUT",
+)
diff --git a/test/export/test_subinclude_variable/source_repo/defs.build_defs b/test/export/test_subinclude_variable/source_repo/defs.build_defs
new file mode 100644
index 0000000000..cdd62f88ae
--- /dev/null
+++ b/test/export/test_subinclude_variable/source_repo/defs.build_defs
@@ -0,0 +1 @@
+VAR = "hello"
diff --git a/test/export/test_subrepo_subtarget/BUILD b/test/export/test_subrepo_subtarget/BUILD
new file mode 100644
index 0000000000..90bbf0c233
--- /dev/null
+++ b/test/export/test_subrepo_subtarget/BUILD
@@ -0,0 +1,8 @@
+subinclude("//test/export:export_e2e_test_build_def")
+
+# Verifies that `plz export` correctly performs lookups to resolve internally generated sub-targets
+# belonging to packages defined inside subrepos.
+please_export_e2e_test(
+ name = "subrepo_subtarget_export_test",
+ export_targets = ["//:target"],
+)
diff --git a/test/export/test_subrepo_subtarget/expected_repo/.plzconfig b/test/export/test_subrepo_subtarget/expected_repo/.plzconfig
new file mode 100644
index 0000000000..4d5174001a
--- /dev/null
+++ b/test/export/test_subrepo_subtarget/expected_repo/.plzconfig
@@ -0,0 +1,2 @@
+[parse]
+BuildFileName = BUILD_FILE
\ No newline at end of file
diff --git a/test/export/test_subrepo_subtarget/expected_repo/BUILD_FILE b/test/export/test_subrepo_subtarget/expected_repo/BUILD_FILE
new file mode 100644
index 0000000000..fe636b2061
--- /dev/null
+++ b/test/export/test_subrepo_subtarget/expected_repo/BUILD_FILE
@@ -0,0 +1,12 @@
+subrepo(
+ name = "my_subrepo",
+ path = "subrepo",
+)
+
+filegroup(
+ name = "target",
+ srcs = [
+ "file.txt",
+ "///my_subrepo//:_foo#sub",
+ ],
+)
diff --git a/test/export/test_subrepo_subtarget/expected_repo/file.txt b/test/export/test_subrepo_subtarget/expected_repo/file.txt
new file mode 100644
index 0000000000..95d09f2b10
--- /dev/null
+++ b/test/export/test_subrepo_subtarget/expected_repo/file.txt
@@ -0,0 +1 @@
+hello world
\ No newline at end of file
diff --git a/test/export/test_subrepo_subtarget/expected_repo/subrepo/BUILD_FILE b/test/export/test_subrepo_subtarget/expected_repo/subrepo/BUILD_FILE
new file mode 100644
index 0000000000..7771998a58
--- /dev/null
+++ b/test/export/test_subrepo_subtarget/expected_repo/subrepo/BUILD_FILE
@@ -0,0 +1,5 @@
+filegroup(
+ name = "_foo#sub",
+ srcs = ["BUILD_FILE"],
+ visibility = ["PUBLIC"],
+)
diff --git a/test/export/test_subrepo_subtarget/source_repo/.plzconfig b/test/export/test_subrepo_subtarget/source_repo/.plzconfig
new file mode 100644
index 0000000000..4d5174001a
--- /dev/null
+++ b/test/export/test_subrepo_subtarget/source_repo/.plzconfig
@@ -0,0 +1,2 @@
+[parse]
+BuildFileName = BUILD_FILE
\ No newline at end of file
diff --git a/test/export/test_subrepo_subtarget/source_repo/BUILD_FILE b/test/export/test_subrepo_subtarget/source_repo/BUILD_FILE
new file mode 100644
index 0000000000..cfa1f38a46
--- /dev/null
+++ b/test/export/test_subrepo_subtarget/source_repo/BUILD_FILE
@@ -0,0 +1,17 @@
+subrepo(
+ name = "my_subrepo",
+ path = "subrepo",
+)
+
+filegroup(
+ name = "target",
+ srcs = [
+ "file.txt",
+ "///my_subrepo//:_foo#sub",
+ ],
+)
+
+filegroup(
+ name = "unneeded",
+ srcs = ["unneeded.txt"],
+)
diff --git a/test/export/test_subrepo_subtarget/source_repo/file.txt b/test/export/test_subrepo_subtarget/source_repo/file.txt
new file mode 100644
index 0000000000..95d09f2b10
--- /dev/null
+++ b/test/export/test_subrepo_subtarget/source_repo/file.txt
@@ -0,0 +1 @@
+hello world
\ No newline at end of file
diff --git a/test/export/test_subrepo_subtarget/source_repo/subrepo/BUILD_FILE b/test/export/test_subrepo_subtarget/source_repo/subrepo/BUILD_FILE
new file mode 100644
index 0000000000..0ae27f18c3
--- /dev/null
+++ b/test/export/test_subrepo_subtarget/source_repo/subrepo/BUILD_FILE
@@ -0,0 +1,11 @@
+filegroup(
+ name = "package",
+ srcs = ["BUILD_FILE"],
+ visibility = ["PUBLIC"],
+)
+
+filegroup(
+ name = "_foo#sub",
+ srcs = ["BUILD_FILE"],
+ visibility = ["PUBLIC"],
+)
\ No newline at end of file
diff --git a/test/export/test_subrepo_subtarget/source_repo/unneeded.txt b/test/export/test_subrepo_subtarget/source_repo/unneeded.txt
new file mode 100644
index 0000000000..9c1fe51193
--- /dev/null
+++ b/test/export/test_subrepo_subtarget/source_repo/unneeded.txt
@@ -0,0 +1 @@
+unneeded
\ No newline at end of file