From a7ce8e9304693e1ac66964a85495ba77dd92cb2d Mon Sep 17 00:00:00 2001 From: Robin Burchell Date: Tue, 13 Jan 2026 19:32:29 +0100 Subject: [PATCH 1/2] feat: Add output formats --- README.md | 16 ++++- cmd/multibuild/integration_test.go | 78 +++++++++++++++++++++ cmd/multibuild/main.go | 1 + cmd/multibuild/multibuild.go | 109 +++++++++++++++++++++++++++-- cmd/multibuild/options.go | 59 +++++++++++++++- cmd/multibuild/options_test.go | 85 ++++++++++++++++++++++ 6 files changed, 340 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index e716b52..f727775 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ This configuration will use the same naming, but place all binaries in a `bin/` An `output` configuration must have all three `${TARGET}`, `${GOOS}`, `${GOARCH}` placeholders present, but the ordering can change. -Windows, as a special case, will always have ".exe" appended to the filename. +Windows, as a special case, will always have ".exe" appended to the filename of a raw binary. The `TARGET` placeholder expands to the default build target name that `go build` would produce. The `GOOS` placeholder is expands to the `GOOS` under build. @@ -90,6 +90,20 @@ The `GOARCH` placeholder expands to the `GOARCH` under build. Only a single `output` directive may be found in a package. +## Output formats + +multibuild can produce several types of output. + +`//go:multibuild:format=raw,zip,tar.gz` + +The list of formats is comma separated, and any of the following are supported: + +* `raw` - The default, the raw binary produced by `go build`. +* `zip` - A zip archive of the raw binary. +* `tar.gz` - A tar.gz'd archive of the raw binary. + +Only a single `format` directive may be found in a package. + # Differences to `go build` As multibuild is a wrapper around `go build`, most of the behaviour you will see come from there. diff --git a/cmd/multibuild/integration_test.go b/cmd/multibuild/integration_test.go index f2d8963..252b9ac 100644 --- a/cmd/multibuild/integration_test.go +++ b/cmd/multibuild/integration_test.go @@ -110,6 +110,7 @@ func main() { expectedConfig: `//go:multibuild:include=linux/amd64,linux/arm64 //go:multibuild:exclude=android/*,ios/* //go:multibuild:output=${TARGET}-${GOOS}-${GOARCH} +//go:multibuild:format=raw `, expectedTargets: "linux/amd64\nlinux/arm64\n", }, @@ -124,6 +125,7 @@ func main() { expectedConfig: `//go:multibuild:include=*/arm64 //go:multibuild:exclude=android/arm64,darwin/arm64,freebsd/arm64,ios/arm64,netbsd/arm64,openbsd/arm64,windows/arm64,android/*,ios/* //go:multibuild:output=${TARGET}-${GOOS}-${GOARCH} +//go:multibuild:format=raw `, expectedTargets: "linux/arm64\n", }, @@ -139,6 +141,75 @@ func main() { expectedConfig: `//go:multibuild:include=linux/amd64,linux/arm64 //go:multibuild:exclude=android/*,ios/* //go:multibuild:output=bin/${TARGET}-hello-${GOOS}-world-${GOARCH} +//go:multibuild:format=raw +`, + expectedTargets: "linux/amd64\nlinux/arm64\n", + }, + { + name: "format=raw", + config: `//go:multibuild:include=linux/amd64,linux/arm64 +//go:multibuild:format=raw +`, + expectedBinaries: []string{ + "${TARGET}-linux-amd64", + "${TARGET}-linux-arm64", + }, + expectedConfig: `//go:multibuild:include=linux/amd64,linux/arm64 +//go:multibuild:exclude=android/*,ios/* +//go:multibuild:output=${TARGET}-${GOOS}-${GOARCH} +//go:multibuild:format=raw +`, + expectedTargets: "linux/amd64\nlinux/arm64\n", + }, + { + name: "format=zip", + config: `//go:multibuild:include=linux/amd64,linux/arm64 +//go:multibuild:format=zip +`, + expectedBinaries: []string{ + "${TARGET}-linux-amd64.zip", + "${TARGET}-linux-arm64.zip", + }, + expectedConfig: `//go:multibuild:include=linux/amd64,linux/arm64 +//go:multibuild:exclude=android/*,ios/* +//go:multibuild:output=${TARGET}-${GOOS}-${GOARCH} +//go:multibuild:format=zip +`, + expectedTargets: "linux/amd64\nlinux/arm64\n", + }, + { + name: "format=tar.gz", + config: `//go:multibuild:include=linux/amd64,linux/arm64 +//go:multibuild:format=tar.gz +`, + expectedBinaries: []string{ + "${TARGET}-linux-amd64.tar.gz", + "${TARGET}-linux-arm64.tar.gz", + }, + expectedConfig: `//go:multibuild:include=linux/amd64,linux/arm64 +//go:multibuild:exclude=android/*,ios/* +//go:multibuild:output=${TARGET}-${GOOS}-${GOARCH} +//go:multibuild:format=tar.gz +`, + expectedTargets: "linux/amd64\nlinux/arm64\n", + }, + { + name: "format=raw,zip,tar.gz", + config: `//go:multibuild:include=linux/amd64,linux/arm64 +//go:multibuild:format=raw,zip,tar.gz +`, + expectedBinaries: []string{ + "${TARGET}-linux-amd64", + "${TARGET}-linux-arm64", + "${TARGET}-linux-amd64.zip", + "${TARGET}-linux-arm64.zip", + "${TARGET}-linux-amd64.tar.gz", + "${TARGET}-linux-arm64.tar.gz", + }, + expectedConfig: `//go:multibuild:include=linux/amd64,linux/arm64 +//go:multibuild:exclude=android/*,ios/* +//go:multibuild:output=${TARGET}-${GOOS}-${GOARCH} +//go:multibuild:format=raw,zip,tar.gz `, expectedTargets: "linux/amd64\nlinux/arm64\n", }, @@ -187,7 +258,14 @@ func main() { if err != nil { t.Fatalf("failed to multibuild: %v\nOutput:\n%s", err, out) } + if len(out) != 0 { + t.Fatalf("unexpected output: %s", out) + } + // FIXME: This test has a small oversight. It was written to assert that the expected output is created. + // But ideally it should also be asserting that no *unexpected* output is created. + // + // An example here is that if we request format=zip, we should assert that the 'raw' binaries are removed. for _, want := range test.expectedBinaries { want := strings.ReplaceAll(want, "${TARGET}", filepath.Base(testTmp)) if _, err := os.Stat(filepath.Join(testTmp, want)); err != nil { diff --git a/cmd/multibuild/main.go b/cmd/multibuild/main.go index d4b3264..01aaaa4 100644 --- a/cmd/multibuild/main.go +++ b/cmd/multibuild/main.go @@ -31,6 +31,7 @@ func displayConfigAndExit(opts options) { fmt.Fprintf(os.Stderr, "//go:multibuild:include=%s\n", strings.Join(mapSlice(opts.Include, func(f filter) string { return string(f) }), ",")) fmt.Fprintf(os.Stderr, "//go:multibuild:exclude=%s\n", strings.Join(mapSlice(opts.Exclude, func(f filter) string { return string(f) }), ",")) fmt.Fprintf(os.Stderr, "//go:multibuild:output=%s\n", opts.Output) + fmt.Fprintf(os.Stderr, "//go:multibuild:format=%s\n", strings.Join(mapSlice(opts.Format, func(f format) string { return string(f) }), ",")) os.Exit(0) } diff --git a/cmd/multibuild/multibuild.go b/cmd/multibuild/multibuild.go index f6d4dc9..d6c25c3 100644 --- a/cmd/multibuild/multibuild.go +++ b/cmd/multibuild/multibuild.go @@ -5,14 +5,18 @@ package main import ( + "archive/tar" + "archive/zip" "bufio" "bytes" + "compress/gzip" "encoding/json" "fmt" "io" "os" "os/exec" "path/filepath" + "slices" "strings" "sync" ) @@ -117,30 +121,123 @@ func doMultibuild(args cliArgs) { out := formattedOutput out = strings.ReplaceAll(out, "${GOOS}", goos) out = strings.ReplaceAll(out, "${GOARCH}", goarch) + outBin := out if goos == "windows" { - out += ".exe" + outBin += ".exe" } - buildArgs := []string{"-o", out} + buildArgs := []string{"-o", outBin} buildArgs = append(buildArgs, args.goBuildArgs...) wg.Add(1) // acquire for global - go func(goos, goarch string, buildArgs []string) { + go func(out, outBin, goos, goarch string, buildArgs []string) { if args.verbose { fmt.Fprintf(os.Stderr, "%s/%s: waiting\n", goos, goarch) } sem <- struct{}{} // acquire for job if args.verbose { - fmt.Fprintf(os.Stderr, "%s/%s: building\n", goos, goarch) + fmt.Fprintf(os.Stderr, "%s/%s: build\n", goos, goarch) } runBuild(buildArgs, goos, goarch) if args.verbose { - fmt.Fprintf(os.Stderr, "%s/%s: done\n", goos, goarch) + fmt.Fprintf(os.Stderr, "%s/%s: archive\n", goos, goarch) + } + + for _, format := range opts.Format { + switch format { + case formatRaw: + // already built (obvs).. + case formatZip: + arPath := out + ".zip" + f, err := os.Create(arPath) + defer f.Close() + if err != nil { + fmt.Fprintf(os.Stderr, "%s/%s: failed to create archive %s: %s\n", goos, goarch, arPath, err) + os.Exit(1) + } + + zw := zip.NewWriter(f) + defer zw.Close() + + w, err := zw.Create(outBin) + if err != nil { + fmt.Fprintf(os.Stderr, "%s/%s: failed to create header %s: %s\n", goos, goarch, arPath, err) + os.Exit(1) + } + + st, err := os.Stat(outBin) + if err != nil { + fmt.Fprintf(os.Stderr, "%s/%s: failed to stat raw %s: %s\n", goos, goarch, outBin, err) + os.Exit(1) + } + bin, err := os.Open(outBin) + if err != nil { + fmt.Fprintf(os.Stderr, "%s/%s: failed to open raw %s: %s\n", goos, goarch, outBin, err) + os.Exit(1) + } + defer bin.Close() + sz, err := io.Copy(w, bin) + if err != nil { + fmt.Fprintf(os.Stderr, "%s/%s: failed to copy %s: %s\n", goos, goarch, outBin, err) + os.Exit(1) + } + if sz != st.Size() { + fmt.Fprintf(os.Stderr, "%s/%s: size mismatch in copy of %s: (%d vs %d)\n", goos, goarch, outBin, sz, st.Size()) + os.Exit(1) + } + case formatTgz: + arPath := out + ".tar.gz" + f, err := os.Create(arPath) + if err != nil { + fmt.Fprintf(os.Stderr, "%s/%s: failed to create archive %s: %s\n", goos, goarch, arPath, err) + os.Exit(1) + } + defer f.Close() + + gz := gzip.NewWriter(f) + defer gz.Close() + + tw := tar.NewWriter(gz) + defer tw.Close() + + st, err := os.Stat(outBin) + if err != nil { + fmt.Fprintf(os.Stderr, "%s/%s: failed to stat raw %s: %s\n", goos, goarch, outBin, err) + os.Exit(1) + } + bin, err := os.Open(outBin) + if err != nil { + fmt.Fprintf(os.Stderr, "%s/%s: failed to open raw %s: %s\n", goos, goarch, outBin, err) + os.Exit(1) + } + defer bin.Close() + + hdr := &tar.Header{Name: outBin, Mode: 0755, Size: st.Size()} + tw.WriteHeader(hdr) + sz, err := io.Copy(tw, bin) + if err != nil { + fmt.Fprintf(os.Stderr, "%s/%s: failed to copy %s: %s\n", goos, goarch, outBin, err) + os.Exit(1) + } + if sz != st.Size() { + fmt.Fprintf(os.Stderr, "%s/%s: size mismatch in copy of %s: (%d vs %d)\n", goos, goarch, outBin, sz, st.Size()) + os.Exit(1) + } + } + } + + // If the format list specifically excluded raw, remove the binary. + // I don't know why one would want to do this, but nevertheless... + if !slices.Contains(opts.Format, formatRaw) { + err := os.Remove(outBin) + if err != nil { + fmt.Fprintf(os.Stderr, "%s/%s: failed to remove unwanted raw output %s: %s\n", goos, goarch, outBin, err) + } } <-sem // release for job wg.Done() // release for global - }(goos, goarch, buildArgs) + }(out, outBin, goos, goarch, buildArgs) } wg.Wait() diff --git a/cmd/multibuild/options.go b/cmd/multibuild/options.go index 1b15450..72c68c7 100644 --- a/cmd/multibuild/options.go +++ b/cmd/multibuild/options.go @@ -28,11 +28,23 @@ type target string // e.g. ${TARGET}_${GOOS}_${GOARCH} type outputTemplate string +// raw, tar.gz, ... +type format string + +const ( + formatRaw format = "raw" + formatZip = "zip" + formatTgz = "tar.gz" +) + // All options for multibuild go here.. type options struct { - // Output format + // Output filename format Output outputTemplate + // Output formats to produce + Format []format + // Targets to include Include []filter @@ -171,6 +183,30 @@ func validateTemplate(s string) (outputTemplate, error) { return outputTemplate(s), nil } +// Validates that the 's' is a list of formats. +func validateFormatString(s string) ([]format, error) { + if s == "" { + return nil, fmt.Errorf("empty string is not a valid format") + } + + var allowedFormats = map[format]struct{}{ + formatRaw: {}, + formatZip: {}, + formatTgz: {}, + } + + var formats []format + formatStrs := strings.SplitSeq(s, ",") + for formatStr := range formatStrs { + format := format(formatStr) + if _, ok := allowedFormats[format]; !ok { + return nil, fmt.Errorf("format %q is not valid", formatStr) + } + formats = append(formats, format) + } + return formats, nil +} + func validateFilterString(s string) ([]filter, error) { isAlphaNum := func(b byte) bool { return (b >= 'a' && b <= 'z') || @@ -274,6 +310,19 @@ func scanBuildPath(reader io.Reader, path string) (options, error) { return options{}, fmt.Errorf("%s:%d: go:multibuild:output=%s is invalid: %s", path, i, rest, err) } opts.Output = parsed + } else if strings.HasPrefix(line, "//go:multibuild:format=") { + if dlog { + log.Printf("Found format: %s:%d: %s", path, i, line) + } + rest := strings.TrimPrefix(line, "//go:multibuild:format=") + if len(opts.Format) > 0 { + return options{}, fmt.Errorf("%s:%d: go:multibuild:format was already set to %s, found: %q here", path, i, opts.Format, rest) + } + parsed, err := validateFormatString(rest) + if err != nil { + return options{}, fmt.Errorf("%s:%d: go:multibuild:format=%s is invalid: %s", path, i, rest, err) + } + opts.Format = parsed } else if strings.HasPrefix(line, "//go:multibuild:include=") { if dlog { log.Printf("Found include: %s:%d: %s", path, i, line) @@ -321,6 +370,11 @@ func scanBuildDir(sources []string) (options, error) { } else if len(topts.Output) > 0 { opts.Output = topts.Output } + if len(opts.Format) > 0 && len(topts.Format) > 0 { + return options{}, fmt.Errorf("%s: format= already set elsewhere", path) + } else if len(topts.Format) > 0 { + opts.Format = topts.Format + } opts.Exclude = append(opts.Exclude, topts.Exclude...) opts.Include = append(opts.Include, topts.Include...) } @@ -329,6 +383,9 @@ func scanBuildDir(sources []string) (options, error) { if len(opts.Include) == 0 { opts.Include = []filter{"*/*"} } + if len(opts.Format) == 0 { + opts.Format = []format{formatRaw} + } // These require CGO_ENABLED=1, which I don't want to touch right now. // As I don't have a use for it, let's just disable them. diff --git a/cmd/multibuild/options_test.go b/cmd/multibuild/options_test.go index 82e2e4b..1d535ee 100644 --- a/cmd/multibuild/options_test.go +++ b/cmd/multibuild/options_test.go @@ -511,3 +511,88 @@ func TestValidateFilters_Invalid(t *testing.T) { }) } } + +func TestValidateFormatString(t *testing.T) { + tests := []struct { + name string + input string + wantErr bool + outputs []format + }{ + { + name: "single (raw)", + input: "raw", + wantErr: false, + outputs: []format{formatRaw}, + }, + { + name: "single (zip)", + input: "zip", + wantErr: false, + outputs: []format{formatZip}, + }, + { + name: "single (tgz)", + input: "tar.gz", + wantErr: false, + outputs: []format{formatTgz}, + }, + { + name: "all", + input: "raw,zip,tar.gz", + wantErr: false, + outputs: []format{formatRaw, formatZip, formatTgz}, + }, + { + name: "empty", + input: "", + wantErr: true, + outputs: []format{}, + }, + { + name: "all-unknown", + input: "wat", + wantErr: true, + outputs: []format{}, + }, + { + name: "partly-unknown", + input: "zip,wat", + wantErr: true, + outputs: []format{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + out, err := validateFormatString(tt.input) + + if tt.wantErr { + if err == nil { + t.Fatalf("expected error, got nil (output=%v)", out) + } + return + } + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + equalFormats := func(a, b []format) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true + } + + if !equalFormats(out, tt.outputs) { + t.Fatalf("output mismatch: got %q, want %q", out, tt.input) + } + }) + } +} From 8a2ca9c2e097983aa76604aefcc49c2df802789a Mon Sep 17 00:00:00 2001 From: Robin Burchell Date: Tue, 27 Jan 2026 11:43:50 +0100 Subject: [PATCH 2/2] options_test: Use slices.Equal rather than hand-rolling it --- cmd/multibuild/options_test.go | 27 ++++----------------------- 1 file changed, 4 insertions(+), 23 deletions(-) diff --git a/cmd/multibuild/options_test.go b/cmd/multibuild/options_test.go index 1d535ee..f066d05 100644 --- a/cmd/multibuild/options_test.go +++ b/cmd/multibuild/options_test.go @@ -180,18 +180,11 @@ func TestScanBuildPath(t *testing.T) { } equalOptions := func(a, b options) bool { - if len(a.Include) != len(b.Include) || len(a.Exclude) != len(b.Exclude) { + if !slices.Equal(a.Include, b.Include) { return false } - for i := range a.Include { - if a.Include[i] != b.Include[i] { - return false - } - } - for i := range a.Exclude { - if a.Exclude[i] != b.Exclude[i] { - return false - } + if !slices.Equal(a.Exclude, b.Exclude) { + return false } return true } @@ -578,19 +571,7 @@ func TestValidateFormatString(t *testing.T) { t.Fatalf("unexpected error: %v", err) } - equalFormats := func(a, b []format) bool { - if len(a) != len(b) { - return false - } - for i := range a { - if a[i] != b[i] { - return false - } - } - return true - } - - if !equalFormats(out, tt.outputs) { + if !slices.Equal(out, tt.outputs) { t.Fatalf("output mismatch: got %q, want %q", out, tt.input) } })