From 9fef35c40b686e7da42ce6889466946f0c4b82d7 Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Thu, 12 Mar 2026 09:15:48 +0100 Subject: [PATCH 1/4] e2e: test build records export Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- hack/test-driver | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/hack/test-driver b/hack/test-driver index e70315e31554..889c6acb784f 100755 --- a/hack/test-driver +++ b/hack/test-driver @@ -134,6 +134,8 @@ buildxCmd build ${buildPlatformFlag} \ --metadata-file="${context}/metadata-build.json" \ "${context}" cat "${context}/metadata-build.json" +buildRef=$(awk -F'"' '/"buildx.build.ref"/ {print $4; exit}' "${context}/metadata-build.json") +buildID=${buildRef##*/} # load to docker store if [ "$DRIVER" != "docker" ]; then @@ -143,6 +145,15 @@ if [ "$DRIVER" != "docker" ]; then "${context}" fi +# list build records +buildxCmd --builder="${builderName}" history ls + +# export build records +buildxCmd --builder="${builderName}" history export --finalize "${buildID}" --output "${context}/record.dockerbuild" +file "${context}/record.dockerbuild" +buildxCmd --builder="${builderName}" history export --finalize --all --output "${context}/records.dockerbuild" +file "${context}/records.dockerbuild" + # create bake def cat > "${bakedef}" < Date: Thu, 12 Mar 2026 14:42:09 +0100 Subject: [PATCH 2/4] history: finalize export records on their owning node Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- commands/history/export.go | 50 ++++++++++++++++++++++++-------------- commands/history/trace.go | 2 +- commands/history/utils.go | 30 +++++++++-------------- 3 files changed, 45 insertions(+), 37 deletions(-) diff --git a/commands/history/export.go b/commands/history/export.go index 35114450d2de..4869056c516d 100644 --- a/commands/history/export.go +++ b/commands/history/export.go @@ -4,7 +4,6 @@ import ( "context" "io" "os" - "slices" "github.com/containerd/console" "github.com/containerd/platforms" @@ -53,37 +52,52 @@ func runExport(ctx context.Context, dockerCli command.Cli, opts exportOptions) e return errors.Errorf("no record found for ref %q", ref) } + toExport := recs + if !opts.all && ref == "" { + latestRef := recs[0].Ref + recCount := 0 + for _, rec := range recs { + if rec.Ref != latestRef { + break + } + recCount++ + } + toExport = recs[:recCount] + } if opts.finalize { + seen := make(map[string]struct{}, len(toExport)) var finalized bool - for _, rec := range recs { - if rec.Trace == nil { - finalized = true - if err := finalizeRecord(ctx, rec.Ref, nodes); err != nil { - return err - } + for _, rec := range toExport { + if rec.node == nil || rec.Trace != nil { + continue + } + key := rec.node.Builder + "\x00" + rec.node.Name + "\x00" + rec.Ref + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + finalized = true + if err := finalizeRecord(ctx, rec.Ref, *rec.node); err != nil { + return err } } if finalized { - recs, err = queryRecords(ctx, ref, nodes, &queryOptions{ + queryRef := ref + if !opts.all { + queryRef = toExport[0].Ref + } + recs, err = queryRecords(ctx, queryRef, nodes, &queryOptions{ CompletedOnly: true, }) if err != nil { return err } + toExport = recs } } - - if ref == "" { - slices.SortFunc(recs, func(a, b historyRecord) int { - return b.CreatedAt.AsTime().Compare(a.CreatedAt.AsTime()) - }) - } - + res = append(res, toExport...) if opts.all { - res = append(res, recs...) break - } else { - res = append(res, recs[0]) } } diff --git a/commands/history/trace.go b/commands/history/trace.go index 1b2145dd8b71..dd806b6497ea 100644 --- a/commands/history/trace.go +++ b/commands/history/trace.go @@ -56,7 +56,7 @@ func loadTrace(ctx context.Context, ref string, nodes []builder.Node) (string, [ // build is complete but no trace yet. try to finalize the trace time.Sleep(1 * time.Second) // give some extra time for last parts of trace to be written - err := finalizeRecord(ctx, rec.Ref, []builder.Node{*rec.node}) + err := finalizeRecord(ctx, rec.Ref, *rec.node) if err != nil { return "", nil, err } diff --git a/commands/history/utils.go b/commands/history/utils.go index beab15a22a77..f792602540b6 100644 --- a/commands/history/utils.go +++ b/commands/history/utils.go @@ -257,25 +257,19 @@ func queryRecords(ctx context.Context, ref string, nodes []builder.Node, opts *q return out, nil } -func finalizeRecord(ctx context.Context, ref string, nodes []builder.Node) error { - eg, ctx := errgroup.WithContext(ctx) - for _, node := range nodes { - eg.Go(func() error { - if node.Driver == nil { - return nil - } - c, err := node.Driver.Client(ctx) - if err != nil { - return err - } - _, err = c.ControlClient().UpdateBuildHistory(ctx, &controlapi.UpdateBuildHistoryRequest{ - Ref: ref, - Finalize: true, - }) - return err - }) +func finalizeRecord(ctx context.Context, ref string, node builder.Node) error { + if node.Driver == nil { + return nil } - return eg.Wait() + c, err := node.Driver.Client(ctx) + if err != nil { + return err + } + _, err = c.ControlClient().UpdateBuildHistory(ctx, &controlapi.UpdateBuildHistoryRequest{ + Ref: ref, + Finalize: true, + }) + return err } func formatDuration(d time.Duration) string { From c4e9dfcad9aafeb588df9c213e05ca1d7ca53a9d Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Thu, 12 Mar 2026 09:15:48 +0100 Subject: [PATCH 3/4] bundle: use all node content stores during export Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- util/desktop/bundle/export.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/desktop/bundle/export.go b/util/desktop/bundle/export.go index 3ecb87a3e736..ad716f6fbd8a 100644 --- a/util/desktop/bundle/export.go +++ b/util/desktop/bundle/export.go @@ -38,7 +38,7 @@ func Export(ctx context.Context, c []*client.Client, w io.Writer, records []*Rec s := proxy.NewContentStore(c.ContentClient()) if store == nil { store = s - break + continue } store = &nsFallbackStore{ main: store, From 5245d50a5bc85d5b8aa9bcbe798878c1a511a85d Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Thu, 12 Mar 2026 12:36:11 +0100 Subject: [PATCH 4/4] tests: history finalize on multi-node Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- tests/history.go | 182 +++++++++++++++++++++++++---------------------- 1 file changed, 97 insertions(+), 85 deletions(-) diff --git a/tests/history.go b/tests/history.go index eae26d69355b..6437968cf7e4 100644 --- a/tests/history.go +++ b/tests/history.go @@ -19,6 +19,8 @@ import ( var historyTests = []func(t *testing.T, sb integration.Sandbox){ testHistoryExport, testHistoryExportFinalize, + testHistoryExportFinalizeMultiNodeRef, + testHistoryExportFinalizeMultiNodeAll, testHistoryInspect, testHistoryLs, testHistoryRm, @@ -29,10 +31,11 @@ var historyTests = []func(t *testing.T, sb integration.Sandbox){ func testHistoryExport(t *testing.T, sb integration.Sandbox) { ref := buildTestProject(t, sb) require.NotEmpty(t, ref.Ref) + requireHistoryRef(t, sb, ref.Ref) outFile := path.Join(t.TempDir(), "export.dockerbuild") cmd := buildxCmd(sb, withArgs("history", "export", ref.Ref, "--output", outFile)) - out, err := cmd.Output() + out, err := cmd.CombinedOutput() require.NoError(t, err, string(out)) require.FileExists(t, outFile) } @@ -40,10 +43,43 @@ func testHistoryExport(t *testing.T, sb integration.Sandbox) { func testHistoryExportFinalize(t *testing.T, sb integration.Sandbox) { ref := buildTestProject(t, sb) require.NotEmpty(t, ref.Ref) + requireHistoryRef(t, sb, ref.Ref) outFile := path.Join(t.TempDir(), "export.dockerbuild") cmd := buildxCmd(sb, withArgs("history", "export", ref.Ref, "--finalize", "--output", outFile)) - out, err := cmd.Output() + out, err := cmd.CombinedOutput() + require.NoError(t, err, string(out)) + require.FileExists(t, outFile) +} + +func testHistoryExportFinalizeMultiNodeRef(t *testing.T, sb integration.Sandbox) { + if !isRemoteMultiNodeWorker(sb) { + t.Skip("only testing with remote multi-node worker") + } + + ref := buildTestProject(t, sb, withArgs("--platform=linux/amd64,linux/arm64", "--output=type=cacheonly")) + require.NotEmpty(t, ref.Ref) + requireHistoryRef(t, sb, ref.Ref) + + outFile := path.Join(t.TempDir(), "export.dockerbuild") + cmd := buildxCmd(sb, withArgs("history", "export", ref.Ref, "--finalize", "--output", outFile)) + out, err := cmd.CombinedOutput() + require.NoError(t, err, string(out)) + require.FileExists(t, outFile) +} + +func testHistoryExportFinalizeMultiNodeAll(t *testing.T, sb integration.Sandbox) { + if !isRemoteMultiNodeWorker(sb) { + t.Skip("only testing with remote multi-node worker") + } + + ref := buildTestProject(t, sb, withArgs("--platform=linux/amd64,linux/arm64", "--output=type=cacheonly")) + require.NotEmpty(t, ref.Ref) + requireHistoryRef(t, sb, ref.Ref) + + outFile := path.Join(t.TempDir(), "export.dockerbuild") + cmd := buildxCmd(sb, withArgs("history", "export", "--finalize", "--all", "--output", outFile)) + out, err := cmd.CombinedOutput() require.NoError(t, err, string(out)) require.FileExists(t, outFile) } @@ -53,7 +89,7 @@ func testHistoryInspect(t *testing.T, sb integration.Sandbox) { require.NotEmpty(t, ref.Ref) cmd := buildxCmd(sb, withArgs("history", "inspect", ref.Ref, "--format=json")) - out, err := cmd.Output() + out, err := cmd.CombinedOutput() require.NoError(t, err, string(out)) type recT struct { @@ -77,31 +113,10 @@ func testHistoryInspect(t *testing.T, sb integration.Sandbox) { } func testHistoryLs(t *testing.T, sb integration.Sandbox) { - if isRemoteMultiNodeWorker(sb) { - // FIXME: "history ls" fails on multi nodes - t.Skip("fails with multi nodes") - } - ref := buildTestProject(t, sb) require.NotEmpty(t, ref.Ref) - cmd := buildxCmd(sb, withArgs("history", "ls", "--filter=ref="+ref.Ref, "--format=json")) - out, err := cmd.Output() - require.NoError(t, err, string(out)) - - type recT struct { - Ref string `json:"ref"` - Name string `json:"name"` - Status string `json:"status"` - CreatedAt *time.Time `json:"created_at"` - CompletedAt *time.Time `json:"completed_at"` - TotalSteps int32 `json:"total_steps"` - CompletedSteps int32 `json:"completed_steps"` - CachedSteps int32 `json:"cached_steps"` - } - var rec recT - err = json.Unmarshal(out, &rec) - require.NoError(t, err) + rec := requireHistoryRecord(t, sb, ref.String()) require.Equal(t, ref.String(), rec.Ref) require.NotEmpty(t, rec.Name) } @@ -146,15 +161,10 @@ func testHistoryLsStoppedBuilder(t *testing.T, sb integration.Sandbox) { } func testHistoryBuildName(t *testing.T, sb integration.Sandbox) { - if isRemoteMultiNodeWorker(sb) { - // FIXME: "history ls" fails on multi nodes - t.Skip("fails with multi nodes") - } - t.Run("override", func(t *testing.T) { dir := createTestProject(t) out, err := buildCmd(sb, withArgs("--build-arg=BUILDKIT_BUILD_NAME=foobar", "--metadata-file", filepath.Join(dir, "md.json"), dir)) - require.NoError(t, err, string(out)) + require.NoError(t, err, out) dt, err := os.ReadFile(filepath.Join(dir, "md.json")) require.NoError(t, err) @@ -169,23 +179,7 @@ func testHistoryBuildName(t *testing.T, sb integration.Sandbox) { refParts := strings.Split(md.BuildRef, "/") require.Len(t, refParts, 3) - cmd := buildxCmd(sb, withArgs("history", "ls", "--filter=ref="+refParts[2], "--format=json")) - bout, err := cmd.Output() - require.NoError(t, err, string(bout)) - - type recT struct { - Ref string `json:"ref"` - Name string `json:"name"` - Status string `json:"status"` - CreatedAt *time.Time `json:"created_at"` - CompletedAt *time.Time `json:"completed_at"` - TotalSteps int32 `json:"total_steps"` - CompletedSteps int32 `json:"completed_steps"` - CachedSteps int32 `json:"cached_steps"` - } - var rec recT - err = json.Unmarshal(bout, &rec) - require.NoError(t, err) + rec := requireHistoryRecord(t, sb, md.BuildRef) require.Equal(t, md.BuildRef, rec.Ref) require.Equal(t, "foobar", rec.Name) }) @@ -230,23 +224,7 @@ COPY foo /foo refParts := strings.Split(md.BuildRef, "/") require.Len(t, refParts, 3) - cmd := buildxCmd(sb, withArgs("history", "ls", "--filter=ref="+refParts[2], "--format=json")) - bout, err := cmd.Output() - require.NoError(t, err, string(bout)) - - type recT struct { - Ref string `json:"ref"` - Name string `json:"name"` - Status string `json:"status"` - CreatedAt *time.Time `json:"created_at"` - CompletedAt *time.Time `json:"completed_at"` - TotalSteps int32 `json:"total_steps"` - CompletedSteps int32 `json:"completed_steps"` - CachedSteps int32 `json:"cached_steps"` - } - var rec recT - err = json.Unmarshal(bout, &rec) - require.NoError(t, err) + rec := requireHistoryRecord(t, sb, md.BuildRef) require.Equal(t, md.BuildRef, rec.Ref) require.Equal(t, addr+"#main", rec.Name) }) @@ -296,23 +274,7 @@ EOT refParts := strings.Split(md.Default.BuildRef, "/") require.Len(t, refParts, 3) - cmd := buildxCmd(sb, withArgs("history", "ls", "--filter=ref="+refParts[2], "--format=json")) - bout, err := cmd.Output() - require.NoError(t, err, string(bout)) - - type recT struct { - Ref string `json:"ref"` - Name string `json:"name"` - Status string `json:"status"` - CreatedAt *time.Time `json:"created_at"` - CompletedAt *time.Time `json:"completed_at"` - TotalSteps int32 `json:"total_steps"` - CompletedSteps int32 `json:"completed_steps"` - CachedSteps int32 `json:"cached_steps"` - } - var rec recT - err = json.Unmarshal(bout, &rec) - require.NoError(t, err) + rec := requireHistoryRecord(t, sb, md.Default.BuildRef) require.Equal(t, md.Default.BuildRef, rec.Ref) require.Equal(t, addr, rec.Name) }) @@ -328,10 +290,11 @@ func (b buildRef) String() string { return b.Builder + "/" + b.Node + "/" + b.Ref } -func buildTestProject(t *testing.T, sb integration.Sandbox) buildRef { +func buildTestProject(t *testing.T, sb integration.Sandbox, opts ...cmdOpt) buildRef { dir := createTestProject(t) - out, err := buildCmd(sb, withArgs("--metadata-file", filepath.Join(dir, "md.json"), dir)) - require.NoError(t, err, string(out)) + opts = append(opts, withArgs("--metadata-file", filepath.Join(dir, "md.json"), dir)) + out, err := buildCmd(sb, opts...) + require.NoError(t, err, out) dt, err := os.ReadFile(filepath.Join(dir, "md.json")) require.NoError(t, err) @@ -352,3 +315,52 @@ func buildTestProject(t *testing.T, sb integration.Sandbox) buildRef { Ref: refParts[2], } } + +func requireHistoryRef(t *testing.T, sb integration.Sandbox, ref string) { + cmd := buildxCmd(sb, withArgs("history", "ls", "--format={{.Ref}}")) + out, err := cmd.CombinedOutput() + require.NoError(t, err, string(out)) + var matches int + for line := range strings.SplitSeq(strings.TrimSpace(string(out)), "\n") { + if strings.TrimSpace(line) == ref { + matches++ + } + } + require.GreaterOrEqual(t, matches, 1) +} + +type historyLsRecord struct { + Ref string `json:"ref"` + Name string `json:"name"` + Status string `json:"status"` + CreatedAt *time.Time `json:"created_at"` + CompletedAt *time.Time `json:"completed_at"` + TotalSteps int32 `json:"total_steps"` + CompletedSteps int32 `json:"completed_steps"` + CachedSteps int32 `json:"cached_steps"` +} + +func requireHistoryRecord(t *testing.T, sb integration.Sandbox, ref string, opts ...cmdOpt) historyLsRecord { + cmd := buildxCmd(sb, append(opts, withArgs("history", "ls", "--format=json"))...) + out, err := cmd.CombinedOutput() + require.NoError(t, err, string(out)) + + for line := range strings.SplitSeq(strings.TrimSpace(string(out)), "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + if !strings.HasPrefix(line, "{") { + continue + } + var rec historyLsRecord + err := json.Unmarshal([]byte(line), &rec) + require.NoError(t, err) + if rec.Ref == ref { + return rec + } + } + + require.Failf(t, "history record not found", "ref %q was not found in history ls output:\n%s", ref, string(out)) + return historyLsRecord{} +}