diff --git a/.github/workflows/create-draft-release.yml b/.github/workflows/create-draft-release.yml index 3ffde8e..d24a3ea 100644 --- a/.github/workflows/create-draft-release.yml +++ b/.github/workflows/create-draft-release.yml @@ -20,12 +20,12 @@ jobs: name: Unit Tests runs-on: ubuntu-22.04 steps: + - name: Checkout + uses: actions/checkout@v4 - name: Setup Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v5 with: - go-version: 'stable' - - name: Checkout - uses: actions/checkout@v3 + go-version-file: go.mod - name: Setup Docker Multi-Platform Builds run: | docker run --privileged --rm docker/binfmt:a7996909642ee92942dcd6cff44b9b95f08dad64 @@ -37,12 +37,12 @@ jobs: runs-on: ubuntu-22.04 needs: unit steps: + - name: Checkout + uses: actions/checkout@v4 - name: Setup Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v5 with: - go-version: 'stable' - - name: Checkout - uses: actions/checkout@v3 + go-version-file: go.mod - run: git fetch --depth=1 origin +refs/tags/*:refs/tags/* || true - name: Reset Draft Release id: reset diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 24c2800..611d56c 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -15,14 +15,14 @@ jobs: name: lint runs-on: ubuntu-22.04 steps: - - name: Setup Go - uses: actions/setup-go@v3 - with: - go-version: 'stable' - - name: Checkout uses: actions/checkout@v4 + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + - name: golangci-lint uses: golangci/golangci-lint-action@v3 with: diff --git a/.github/workflows/test-pull-request.yml b/.github/workflows/test-pull-request.yml index 957647d..bd291a3 100644 --- a/.github/workflows/test-pull-request.yml +++ b/.github/workflows/test-pull-request.yml @@ -11,14 +11,14 @@ jobs: name: Unit Tests runs-on: ubuntu-22.04 steps: - - name: Setup Go - uses: actions/setup-go@v3 - with: - go-version: 'stable' - - name: Checkout uses: actions/checkout@v4 + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + - name: Setup Docker Multi-Platform Builds run: | docker run --privileged --rm docker/binfmt:a7996909642ee92942dcd6cff44b9b95f08dad64 diff --git a/commands/pack.go b/commands/pack.go index a740c76..8118e51 100644 --- a/commands/pack.go +++ b/commands/pack.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "path/filepath" + "runtime" "strings" "github.com/paketo-buildpacks/jam/v2/internal" @@ -124,8 +125,54 @@ func packRun(flags packFlags) error { return fmt.Errorf("failed to cache dependencies: %s", err) } + // linux/amd64 will be the default target dir when dependencies don't specify os and arch + defaultTargetDir := "linux/amd64" + runtimeTargetDir := filepath.Join(runtime.GOOS, runtime.GOARCH) + for _, dependency := range config.Metadata.Dependencies { - config.Metadata.IncludeFiles = append(config.Metadata.IncludeFiles, strings.TrimPrefix(dependency.URI, "file:///")) + var targetPlatformDir string + shouldMoveToPlatformDir := false + checkTargetDirPaths := []string{} + if dependency.OS != "" && dependency.Arch != "" { + checkTargetDirPaths = append(checkTargetDirPaths, filepath.Join(dependency.OS, dependency.Arch)) + } + checkTargetDirPaths = append(checkTargetDirPaths, runtimeTargetDir, defaultTargetDir) + + for _, dir := range checkTargetDirPaths { + info, err := os.Stat(filepath.Join(tmpDir, dir)) + if err == nil && info.IsDir() && !os.IsNotExist(err) { + shouldMoveToPlatformDir = true + targetPlatformDir = dir + break + } + } + + if shouldMoveToPlatformDir { + // This is a multi-arch buildpack and dependencies need to be moved into the platform-specific directory because + // `pack buildpack package` will be called with `--target /` and files outside the path will not be included + offlinePath := strings.TrimPrefix(dependency.URI, "file:///") + dependenciesDir := filepath.Dir(offlinePath) + offlineFilename := filepath.Base(offlinePath) + + info, err := os.Stat(filepath.Join(tmpDir, dependenciesDir)) + if err != nil || os.IsNotExist(err) || !info.IsDir() { + return fmt.Errorf("expected dependencies directory does not exist: %s", err) + } + + err = os.MkdirAll(filepath.Join(tmpDir, targetPlatformDir, dependenciesDir), os.ModePerm) + if err != nil { + return fmt.Errorf("failed to create platform specific dependencies directory: %s", err) + } + + err = os.Rename(filepath.Join(tmpDir, offlinePath), filepath.Join(tmpDir, targetPlatformDir, dependenciesDir, offlineFilename)) + if err != nil { + return fmt.Errorf("failed to move offline dependency to platform specific directory: %s", err) + } + + config.Metadata.IncludeFiles = append(config.Metadata.IncludeFiles, filepath.Join(targetPlatformDir, dependenciesDir, offlineFilename)) + } else { + config.Metadata.IncludeFiles = append(config.Metadata.IncludeFiles, strings.TrimPrefix(dependency.URI, "file:///")) + } } } diff --git a/integration/pack_test.go b/integration/pack_test.go index 907fdb7..8e84d0c 100644 --- a/integration/pack_test.go +++ b/integration/pack_test.go @@ -1,6 +1,7 @@ package integration_test import ( + "bytes" "fmt" "net/http" "net/http/httptest" @@ -124,46 +125,13 @@ func testPack(t *testing.T, context spec.G, it spec.S) { }) context("when packaging an implementation buildpack", func() { - it.Before(func() { - err := cargo.NewDirectoryDuplicator().Duplicate(filepath.Join("testdata", "example-cnb"), buildpackDir) - Expect(err).NotTo(HaveOccurred()) - }) - - it("creates a packaged buildpack", func() { - command := exec.Command( - path, "pack", - "--buildpack", filepath.Join(buildpackDir, "buildpack.toml"), - "--output", filepath.Join(tmpDir, "output.tgz"), - "--version", "some-version", - ) - session, err := gexec.Start(command, buffer, buffer) - Expect(err).NotTo(HaveOccurred()) - Eventually(session, "5s").Should(gexec.Exit(0), func() string { return buffer.String() }) - - Expect(session.Out).To(gbytes.Say("Packing some-buildpack-name some-version...")) - Expect(session.Out).To(gbytes.Say(" Executing pre-packaging script: ./scripts/build.sh")) - Expect(session.Out).To(gbytes.Say(" hello from the pre-packaging script")) - Expect(session.Out).To(gbytes.Say(fmt.Sprintf(" Building tarball: %s", filepath.Join(tmpDir, "output.tgz")))) - Expect(session.Out).To(gbytes.Say(" bin/build")) - Expect(session.Out).To(gbytes.Say(" bin/detect")) - Expect(session.Out).To(gbytes.Say(" bin/link")) - Expect(session.Out).To(gbytes.Say(" buildpack.toml")) - Expect(session.Out).To(gbytes.Say(" generated-file")) - - file, err := os.Open(filepath.Join(tmpDir, "output.tgz")) - Expect(err).NotTo(HaveOccurred()) - - u, err := user.Current() - Expect(err).NotTo(HaveOccurred()) - userName := u.Username - - group, err := user.LookupGroupId(u.Gid) - Expect(err).NotTo(HaveOccurred()) - groupName := group.Name + context("that is single architecture", func() { + it.Before(func() { + err := cargo.NewDirectoryDuplicator().Duplicate(filepath.Join("testdata", "example-cnb"), buildpackDir) + Expect(err).NotTo(HaveOccurred()) + }) - contents, hdr, err := ExtractFile(file, "buildpack.toml") - Expect(err).NotTo(HaveOccurred()) - Expect(contents).To(MatchTOML(`api = "0.6" + expectedBuildpackToml := `api = "0.6" [buildpack] description = "some-buildpack-description" @@ -223,109 +191,191 @@ func testPack(t *testing.T, context spec.G, it spec.S) { [[targets]] arch = "some-other-arch" - os = "some-other-os"`)) - Expect(hdr.Mode).To(Equal(int64(0644))) - - contents, hdr, err = ExtractFile(file, "bin/build") - Expect(err).NotTo(HaveOccurred()) - Expect(string(contents)).To(Equal("build-contents")) - Expect(hdr.Mode).To(Equal(int64(0755))) - Expect(hdr.Uname).To(Equal(userName)) - Expect(hdr.Gname).To(Equal(groupName)) - - contents, hdr, err = ExtractFile(file, "bin/detect") - Expect(err).NotTo(HaveOccurred()) - Expect(string(contents)).To(Equal("detect-contents")) - Expect(hdr.Mode).To(Equal(int64(0755))) - Expect(hdr.Uname).To(Equal(userName)) - Expect(hdr.Gname).To(Equal(groupName)) - - _, hdr, err = ExtractFile(file, "bin/link") - Expect(err).NotTo(HaveOccurred()) - Expect(hdr.Linkname).To(Equal("build")) - Expect(hdr.Uname).To(Equal(userName)) - Expect(hdr.Gname).To(Equal(groupName)) - - contents, hdr, err = ExtractFile(file, "generated-file") - Expect(err).NotTo(HaveOccurred()) - Expect(string(contents)).To(Equal("hello\n")) - Expect(hdr.Mode).To(Equal(int64(0644))) - Expect(hdr.Uname).To(Equal(userName)) - Expect(hdr.Gname).To(Equal(groupName)) + os = "some-other-os"` - Expect(filepath.Join(buildpackDir, "generated-file")).NotTo(BeARegularFile()) - }) - - context("when the buildpack is built to run offline", func() { - var server *httptest.Server - it.Before(func() { - server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - if req.URL.Path != "/some-dependency.tgz" { - http.NotFound(w, req) - } - - _, _ = fmt.Fprint(w, "dependency-contents") - })) - - config, err := cargo.NewBuildpackParser().Parse(filepath.Join(buildpackDir, "buildpack.toml")) - Expect(err).NotTo(HaveOccurred()) - Expect(config.Metadata.Dependencies).To(HaveLen(2)) - - config.Metadata.Dependencies[0].URI = fmt.Sprintf("%s/some-dependency.tgz", server.URL) - config.Metadata.Dependencies[0].Checksum = "sha256:f058c8bf6b65b829e200ef5c2d22fde0ee65b96c1fbd1b88869be133aafab64a" - - bpTomlWriter, err := os.Create(filepath.Join(buildpackDir, "buildpack.toml")) - Expect(err).NotTo(HaveOccurred()) - - Expect(cargo.EncodeConfig(bpTomlWriter, config)).To(Succeed()) - }) - - it.After(func() { - server.Close() - }) - - it("creates an offline packaged buildpack", func() { + it("creates a packaged buildpack", func() { command := exec.Command( path, "pack", "--buildpack", filepath.Join(buildpackDir, "buildpack.toml"), "--output", filepath.Join(tmpDir, "output.tgz"), "--version", "some-version", - "--offline", - "--stack", "io.buildpacks.stacks.bionic", ) session, err := gexec.Start(command, buffer, buffer) Expect(err).NotTo(HaveOccurred()) - Eventually(session).Should(gexec.Exit(0), func() string { return buffer.String() }) + Eventually(session, "5s").Should(gexec.Exit(0), func() string { return buffer.String() }) Expect(session.Out).To(gbytes.Say("Packing some-buildpack-name some-version...")) Expect(session.Out).To(gbytes.Say(" Executing pre-packaging script: ./scripts/build.sh")) Expect(session.Out).To(gbytes.Say(" hello from the pre-packaging script")) - Expect(session.Out).To(gbytes.Say(" Downloading dependencies...")) - Expect(session.Out).To(gbytes.Say(` some-dependency \(1.2.3\) \[io.buildpacks.stacks.bionic, org.cloudfoundry.stacks.tiny\]`)) - Expect(session.Out).To(gbytes.Say(" ↳ dependencies/f058c8bf6b65b829e200ef5c2d22fde0ee65b96c1fbd1b88869be133aafab64a")) Expect(session.Out).To(gbytes.Say(fmt.Sprintf(" Building tarball: %s", filepath.Join(tmpDir, "output.tgz")))) - Expect(session.Out).To(gbytes.Say(" bin")) Expect(session.Out).To(gbytes.Say(" bin/build")) Expect(session.Out).To(gbytes.Say(" bin/detect")) + Expect(session.Out).To(gbytes.Say(" bin/link")) Expect(session.Out).To(gbytes.Say(" buildpack.toml")) - Expect(session.Out).To(gbytes.Say(" dependencies")) - Expect(session.Out).To(gbytes.Say(" dependencies/f058c8bf6b65b829e200ef5c2d22fde0ee65b96c1fbd1b88869be133aafab64a")) Expect(session.Out).To(gbytes.Say(" generated-file")) - Expect(string(session.Out.Contents())).NotTo(ContainSubstring("other-dependency")) - file, err := os.Open(filepath.Join(tmpDir, "output.tgz")) Expect(err).NotTo(HaveOccurred()) + u, err := user.Current() + Expect(err).NotTo(HaveOccurred()) + userName := u.Username + + group, err := user.LookupGroupId(u.Gid) + Expect(err).NotTo(HaveOccurred()) + groupName := group.Name + contents, hdr, err := ExtractFile(file, "buildpack.toml") Expect(err).NotTo(HaveOccurred()) - Expect(contents).To(MatchTOML(`api = "0.6" + Expect(contents).To(MatchTOML(expectedBuildpackToml)) + Expect(hdr.Mode).To(Equal(int64(0644))) + + contents, hdr, err = ExtractFile(file, "bin/build") + Expect(err).NotTo(HaveOccurred()) + Expect(string(contents)).To(Equal("build-contents")) + Expect(hdr.Mode).To(Equal(int64(0755))) + Expect(hdr.Uname).To(Equal(userName)) + Expect(hdr.Gname).To(Equal(groupName)) + + contents, hdr, err = ExtractFile(file, "bin/detect") + Expect(err).NotTo(HaveOccurred()) + Expect(string(contents)).To(Equal("detect-contents")) + Expect(hdr.Mode).To(Equal(int64(0755))) + Expect(hdr.Uname).To(Equal(userName)) + Expect(hdr.Gname).To(Equal(groupName)) + + _, hdr, err = ExtractFile(file, "bin/link") + Expect(err).NotTo(HaveOccurred()) + Expect(hdr.Linkname).To(Equal("build")) + Expect(hdr.Uname).To(Equal(userName)) + Expect(hdr.Gname).To(Equal(groupName)) + + contents, hdr, err = ExtractFile(file, "generated-file") + Expect(err).NotTo(HaveOccurred()) + Expect(string(contents)).To(Equal("hello\n")) + Expect(hdr.Mode).To(Equal(int64(0644))) + Expect(hdr.Uname).To(Equal(userName)) + Expect(hdr.Gname).To(Equal(groupName)) + + Expect(filepath.Join(buildpackDir, "generated-file")).NotTo(BeARegularFile()) + }) + + context("when the buildpack is built to run offline", func() { + var server *httptest.Server + var config cargo.Config + it.Before(func() { + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if req.URL.Path != "/some-dependency.tgz" { + http.NotFound(w, req) + } + + _, _ = fmt.Fprint(w, "dependency-contents") + })) + + var err error + config, err = cargo.NewBuildpackParser().Parse(filepath.Join(buildpackDir, "buildpack.toml")) + Expect(err).NotTo(HaveOccurred()) + Expect(config.Metadata.Dependencies).To(HaveLen(2)) + + config.Metadata.Dependencies[0].URI = fmt.Sprintf("%s/some-dependency.tgz", server.URL) + config.Metadata.Dependencies[0].Checksum = "sha256:f058c8bf6b65b829e200ef5c2d22fde0ee65b96c1fbd1b88869be133aafab64a" + + bpTomlWriter, err := os.Create(filepath.Join(buildpackDir, "buildpack.toml")) + Expect(err).NotTo(HaveOccurred()) + + Expect(cargo.EncodeConfig(bpTomlWriter, config)).To(Succeed()) + }) + + it.After(func() { + server.Close() + }) + + it("creates an offline packaged buildpack", func() { + command := exec.Command( + path, "pack", + "--buildpack", filepath.Join(buildpackDir, "buildpack.toml"), + "--output", filepath.Join(tmpDir, "output.tgz"), + "--version", "some-version", + "--offline", + "--stack", "io.buildpacks.stacks.bionic", + ) + session, err := gexec.Start(command, buffer, buffer) + Expect(err).NotTo(HaveOccurred()) + Eventually(session).Should(gexec.Exit(0), func() string { return buffer.String() }) + + relativeDependencyPath0 := "dependencies/f058c8bf6b65b829e200ef5c2d22fde0ee65b96c1fbd1b88869be133aafab64a" + + Expect(session.Out).To(gbytes.Say("Packing some-buildpack-name some-version...")) + Expect(session.Out).To(gbytes.Say(" Executing pre-packaging script: ./scripts/build.sh")) + Expect(session.Out).To(gbytes.Say(" hello from the pre-packaging script")) + Expect(session.Out).To(gbytes.Say(" Downloading dependencies...")) + Expect(session.Out).To(gbytes.Say(` some-dependency \(1.2.3\) \[io.buildpacks.stacks.bionic, org.cloudfoundry.stacks.tiny\]`)) + Expect(session.Out).To(gbytes.Say(fmt.Sprintf(" ↳ %s", relativeDependencyPath0))) + Expect(session.Out).To(gbytes.Say(fmt.Sprintf(" Building tarball: %s", filepath.Join(tmpDir, "output.tgz")))) + Expect(session.Out).To(gbytes.Say(" bin")) + Expect(session.Out).To(gbytes.Say(" bin/build")) + Expect(session.Out).To(gbytes.Say(" bin/detect")) + Expect(session.Out).To(gbytes.Say(" buildpack.toml")) + Expect(session.Out).To(gbytes.Say(" dependencies")) + Expect(session.Out).To(gbytes.Say(fmt.Sprintf(" %s", relativeDependencyPath0))) + Expect(session.Out).To(gbytes.Say(" generated-file")) + + Expect(string(session.Out.Contents())).NotTo(ContainSubstring("other-dependency")) + + file, err := os.Open(filepath.Join(tmpDir, "output.tgz")) + Expect(err).NotTo(HaveOccurred()) + + var extractedBuildpackConfig cargo.Config + contents, hdr, err := ExtractFile(file, "buildpack.toml") + Expect(err).NotTo(HaveOccurred()) + Expect(hdr.Mode).To(Equal(int64(0644))) + buff := bytes.NewBuffer(contents) + err = cargo.DecodeConfig(buff, &extractedBuildpackConfig) + Expect(err).NotTo(HaveOccurred()) + + updatedIncludeFiles := config.Metadata.IncludeFiles + updatedIncludeFiles = append(updatedIncludeFiles, relativeDependencyPath0) + + Expect(extractedBuildpackConfig.Metadata.IncludeFiles).To(Equal(updatedIncludeFiles)) + Expect(extractedBuildpackConfig.Metadata.Dependencies[0].URI).To(Equal(fmt.Sprintf(`file:///%s`, relativeDependencyPath0))) + Expect(extractedBuildpackConfig.Metadata.Dependencies[0].Checksum).To(Equal(config.Metadata.Dependencies[0].Checksum)) + + contents, hdr, err = ExtractFile(file, "bin/build") + Expect(err).NotTo(HaveOccurred()) + Expect(string(contents)).To(Equal("build-contents")) + Expect(hdr.Mode).To(Equal(int64(0755))) + + contents, hdr, err = ExtractFile(file, "bin/detect") + Expect(err).NotTo(HaveOccurred()) + Expect(string(contents)).To(Equal("detect-contents")) + Expect(hdr.Mode).To(Equal(int64(0755))) + + contents, hdr, err = ExtractFile(file, "generated-file") + Expect(err).NotTo(HaveOccurred()) + Expect(string(contents)).To(Equal("hello\n")) + Expect(hdr.Mode).To(Equal(int64(0644))) + + contents, hdr, err = ExtractFile(file, "dependencies/f058c8bf6b65b829e200ef5c2d22fde0ee65b96c1fbd1b88869be133aafab64a") + Expect(err).NotTo(HaveOccurred()) + Expect(string(contents)).To(Equal("dependency-contents")) + Expect(hdr.Mode).To(Equal(int64(0644))) + }) + }) + }) + + context("that is multi architecture with os and arch and matching directory layout", func() { + it.Before(func() { + err := cargo.NewDirectoryDuplicator().Duplicate(filepath.Join("testdata", "example-cnb-multi-arch-with-os-arch"), buildpackDir) + Expect(err).NotTo(HaveOccurred()) + }) + + expectedBuildpackToml := `api = "0.6" [buildpack] description = "some-buildpack-description" homepage = "some-homepage-link" id = "some-buildpack-id" - keywords = [ "some-buildpack-keyword" ] + keywords = ["some-buildpack-keyword"] name = "some-buildpack-name" version = "some-version" @@ -334,58 +384,324 @@ func testPack(t *testing.T, context spec.G, it spec.S) { uri = "some-buildpack-license-uri" [metadata] - include-files = ["bin/build", "bin/detect", "bin/link", "buildpack.toml", "generated-file", "dependencies/f058c8bf6b65b829e200ef5c2d22fde0ee65b96c1fbd1b88869be133aafab64a"] + include-files = [ "some-os/some-arch/bin/build", "some-os/some-arch/bin/detect", "some-os/some-arch/bin/link", "some-os/some-arch/generated-file", "some-other-os/some-other-arch/bin/build", "some-other-os/some-other-arch/bin/detect", "some-other-os/some-other-arch/bin/link", "some-other-os/some-other-arch/generated-file", "buildpack.toml" ] pre-package = "./scripts/build.sh" [metadata.default-versions] some-dependency = "some-default-version" [[metadata.dependencies]] arch = "some-arch" - checksum = "sha256:f058c8bf6b65b829e200ef5c2d22fde0ee65b96c1fbd1b88869be133aafab64a" + checksum = "sha256:shasum" deprecation_date = "2019-04-01T00:00:00Z" id = "some-dependency" name = "Some Dependency" os = "some-os" stacks = ["io.buildpacks.stacks.bionic", "org.cloudfoundry.stacks.tiny"] - uri = "file:///dependencies/f058c8bf6b65b829e200ef5c2d22fde0ee65b96c1fbd1b88869be133aafab64a" + uri = "http://some-url" version = "1.2.3" [[metadata.dependencies.distros]] name = "some-distro-name" version = "some-distro-version" + [[metadata.dependencies]] + arch = "some-other-arch" + checksum = "sha256:shasum" + deprecation_date = "2022-04-01T00:00:00Z" + id = "other-dependency" + name = "Other Dependency" + os = "some-other-os" + stacks = ["org.cloudfoundry.stacks.tiny"] + uri = "http://other-url" + version = "4.5.6" + + [[metadata.dependencies.distros]] + name = "some-other-distro-name" + version = "some-other-distro-version" + [[stacks]] id = "some-stack-id" - mixins = ["some-mixin-id"] [[targets]] - arch = "some-arch" os = "some-os" + arch = "some-arch" [[targets]] - arch = "some-other-arch" - os = "some-other-os"`)) - Expect(hdr.Mode).To(Equal(int64(0644))) + os = "some-other-os" + arch = "some-other-arch"` + + platforms := []struct { + os string + arch string + }{ + {"some-os", "some-arch"}, + {"some-other-os", "some-other-arch"}, + } + + it("creates a packaged buildpack", func() { + command := exec.Command( + path, "pack", + "--buildpack", filepath.Join(buildpackDir, "buildpack.toml"), + "--output", filepath.Join(tmpDir, "output.tgz"), + "--version", "some-version", + ) + session, err := gexec.Start(command, buffer, buffer) + Expect(err).NotTo(HaveOccurred()) + Eventually(session, "5s").Should(gexec.Exit(0), func() string { return buffer.String() }) - contents, hdr, err = ExtractFile(file, "bin/build") + Expect(session.Out).To(gbytes.Say("Packing some-buildpack-name some-version...")) + Expect(session.Out).To(gbytes.Say(" Executing pre-packaging script: ./scripts/build.sh")) + Expect(session.Out).To(gbytes.Say(" hello from the pre-packaging script")) + Expect(session.Out).To(gbytes.Say(fmt.Sprintf(" Building tarball: %s", filepath.Join(tmpDir, "output.tgz")))) + Expect(session.Out).To(gbytes.Say(" buildpack.toml")) + Expect(session.Out).To(gbytes.Say(" some-os/some-arch/bin/build")) + Expect(session.Out).To(gbytes.Say(" some-os/some-arch/bin/detect")) + Expect(session.Out).To(gbytes.Say(" some-os/some-arch/bin/link")) + Expect(session.Out).To(gbytes.Say(" some-os/some-arch/generated-file")) + Expect(session.Out).To(gbytes.Say(" some-other-os/some-other-arch/bin/build")) + Expect(session.Out).To(gbytes.Say(" some-other-os/some-other-arch/bin/detect")) + Expect(session.Out).To(gbytes.Say(" some-other-os/some-other-arch/bin/link")) + Expect(session.Out).To(gbytes.Say(" some-other-os/some-other-arch/generated-file")) + + file, err := os.Open(filepath.Join(tmpDir, "output.tgz")) Expect(err).NotTo(HaveOccurred()) - Expect(string(contents)).To(Equal("build-contents")) - Expect(hdr.Mode).To(Equal(int64(0755))) - contents, hdr, err = ExtractFile(file, "bin/detect") + u, err := user.Current() Expect(err).NotTo(HaveOccurred()) - Expect(string(contents)).To(Equal("detect-contents")) - Expect(hdr.Mode).To(Equal(int64(0755))) + userName := u.Username - contents, hdr, err = ExtractFile(file, "generated-file") + group, err := user.LookupGroupId(u.Gid) Expect(err).NotTo(HaveOccurred()) - Expect(string(contents)).To(Equal("hello\n")) - Expect(hdr.Mode).To(Equal(int64(0644))) + groupName := group.Name - contents, hdr, err = ExtractFile(file, "dependencies/f058c8bf6b65b829e200ef5c2d22fde0ee65b96c1fbd1b88869be133aafab64a") + contents, hdr, err := ExtractFile(file, "buildpack.toml") Expect(err).NotTo(HaveOccurred()) - Expect(string(contents)).To(Equal("dependency-contents")) + Expect(contents).To(MatchTOML(expectedBuildpackToml)) Expect(hdr.Mode).To(Equal(int64(0644))) + + for _, platform := range platforms { + targetOs := platform.os + targetArch := platform.arch + contents, hdr, err = ExtractFile(file, fmt.Sprintf("%s/%s/bin/build", targetOs, targetArch)) + Expect(err).NotTo(HaveOccurred()) + Expect(string(contents)).To(Equal(fmt.Sprintf("%s/%s/build-contents", targetOs, targetArch))) + Expect(hdr.Mode).To(Equal(int64(0755))) + Expect(hdr.Uname).To(Equal(userName)) + Expect(hdr.Gname).To(Equal(groupName)) + + contents, hdr, err = ExtractFile(file, fmt.Sprintf("%s/%s/bin/detect", targetOs, targetArch)) + Expect(err).NotTo(HaveOccurred()) + Expect(string(contents)).To(Equal(fmt.Sprintf("%s/%s/detect-contents", targetOs, targetArch))) + Expect(hdr.Mode).To(Equal(int64(0755))) + Expect(hdr.Uname).To(Equal(userName)) + Expect(hdr.Gname).To(Equal(groupName)) + + _, hdr, err = ExtractFile(file, fmt.Sprintf("%s/%s/bin/link", targetOs, targetArch)) + Expect(err).NotTo(HaveOccurred()) + Expect(hdr.Linkname).To(Equal("build")) + Expect(hdr.Uname).To(Equal(userName)) + Expect(hdr.Gname).To(Equal(groupName)) + + contents, hdr, err = ExtractFile(file, fmt.Sprintf("%s/%s/generated-file", targetOs, targetArch)) + Expect(err).NotTo(HaveOccurred()) + Expect(string(contents)).To(Equal(fmt.Sprintf("%s/%s/hello\n", targetOs, targetArch))) + Expect(hdr.Mode).To(Equal(int64(0644))) + Expect(hdr.Uname).To(Equal(userName)) + Expect(hdr.Gname).To(Equal(groupName)) + + Expect(filepath.Join(buildpackDir, fmt.Sprintf("%s/%s/generated-file", targetOs, targetArch))).NotTo(BeARegularFile()) + } + }) + + context("when the buildpack is built to run offline", func() { + var server *httptest.Server + var config cargo.Config + it.Before(func() { + var err error + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + switch req.URL.Path { + case "/some-dependency.tgz": + _, _ = fmt.Fprint(w, "some-dependency-contents") + case "/other-dependency.tgz": + _, _ = fmt.Fprint(w, "other-dependency-contents") + default: + http.NotFound(w, req) + } + })) + + config, err = cargo.NewBuildpackParser().Parse(filepath.Join(buildpackDir, "buildpack.toml")) + Expect(err).NotTo(HaveOccurred()) + Expect(config.Metadata.Dependencies).To(HaveLen(2)) + + config.Metadata.Dependencies[0].URI = fmt.Sprintf("%s/some-dependency.tgz", server.URL) + config.Metadata.Dependencies[0].Checksum = "sha256:1e49f27b5eaafc6c50ebe66d7a2b1d17ced63c2862b5d54ccbcdac816260ecbb" + config.Metadata.Dependencies[1].URI = fmt.Sprintf("%s/other-dependency.tgz", server.URL) + config.Metadata.Dependencies[1].Checksum = "sha256:be06b2ecb65c937562db18c106ff4c15f32a914aa3c6bf7002b6d82390a9bb13" + + bpTomlWriter, err := os.Create(filepath.Join(buildpackDir, "buildpack.toml")) + Expect(err).NotTo(HaveOccurred()) + + Expect(cargo.EncodeConfig(bpTomlWriter, config)).To(Succeed()) + }) + + it.After(func() { + server.Close() + }) + + it("creates an offline packaged buildpack", func() { + command := exec.Command( + path, "pack", + "--buildpack", filepath.Join(buildpackDir, "buildpack.toml"), + "--output", filepath.Join(tmpDir, "output.tgz"), + "--version", "some-version", + "--offline", + ) + session, err := gexec.Start(command, buffer, buffer) + Expect(err).NotTo(HaveOccurred()) + Eventually(session).Should(gexec.Exit(0), func() string { return buffer.String() }) + + relativeDependencyPath0 := "dependencies/1e49f27b5eaafc6c50ebe66d7a2b1d17ced63c2862b5d54ccbcdac816260ecbb" + platformSpecificDependencyPath0 := fmt.Sprintf("some-os/some-arch/%s", relativeDependencyPath0) + relativeDependencyPath1 := "dependencies/be06b2ecb65c937562db18c106ff4c15f32a914aa3c6bf7002b6d82390a9bb13" + platformSpecificDependencyPath1 := fmt.Sprintf("some-other-os/some-other-arch/%s", relativeDependencyPath1) + + Expect(session.Out).To(gbytes.Say("Packing some-buildpack-name some-version...")) + Expect(session.Out).To(gbytes.Say(" Executing pre-packaging script: ./scripts/build.sh")) + Expect(session.Out).To(gbytes.Say(" hello from the pre-packaging script")) + Expect(session.Out).To(gbytes.Say(" Downloading dependencies...")) + Expect(session.Out).To(gbytes.Say(` some-dependency \(1.2.3\) \[io.buildpacks.stacks.bionic, org.cloudfoundry.stacks.tiny\]`)) + Expect(session.Out).To(gbytes.Say(fmt.Sprintf(" ↳ %s", relativeDependencyPath0))) + Expect(session.Out).To(gbytes.Say(fmt.Sprintf(" ↳ %s", relativeDependencyPath1))) + Expect(session.Out).To(gbytes.Say(fmt.Sprintf(" Building tarball: %s", filepath.Join(tmpDir, "output.tgz")))) + Expect(session.Out).To(gbytes.Say(" some-os/some-arch/dependencies")) + Expect(session.Out).To(gbytes.Say(fmt.Sprintf(" %s", platformSpecificDependencyPath0))) + Expect(session.Out).To(gbytes.Say(fmt.Sprintf(" %s", platformSpecificDependencyPath1))) + + file, err := os.Open(filepath.Join(tmpDir, "output.tgz")) + Expect(err).NotTo(HaveOccurred()) + + var extractedBuildpackConfig cargo.Config + contents, hdr, err := ExtractFile(file, "buildpack.toml") + Expect(err).NotTo(HaveOccurred()) + Expect(hdr.Mode).To(Equal(int64(0644))) + buff := bytes.NewBuffer(contents) + err = cargo.DecodeConfig(buff, &extractedBuildpackConfig) + Expect(err).NotTo(HaveOccurred()) + + Expect(config.Metadata.Dependencies).To(HaveLen(2)) + Expect(extractedBuildpackConfig.Metadata.Dependencies).To(HaveLen(2)) + + updatedIncludeFiles := config.Metadata.IncludeFiles + updatedIncludeFiles = append(updatedIncludeFiles, platformSpecificDependencyPath0, platformSpecificDependencyPath1) + + Expect(extractedBuildpackConfig.Metadata.IncludeFiles).To(Equal(updatedIncludeFiles)) + + Expect(extractedBuildpackConfig.Metadata.Dependencies[0].URI).To(Equal(fmt.Sprintf(`file:///%s`, relativeDependencyPath0))) + Expect(extractedBuildpackConfig.Metadata.Dependencies[0].Checksum).To(Equal(config.Metadata.Dependencies[0].Checksum)) + Expect(extractedBuildpackConfig.Metadata.Dependencies[1].URI).To(Equal(fmt.Sprintf(`file:///%s`, relativeDependencyPath1))) + Expect(extractedBuildpackConfig.Metadata.Dependencies[1].Checksum).To(Equal(config.Metadata.Dependencies[1].Checksum)) + + contents, hdr, err = ExtractFile(file, platformSpecificDependencyPath0) + Expect(err).NotTo(HaveOccurred()) + Expect(string(contents)).To(Equal("some-dependency-contents")) + Expect(hdr.Mode).To(Equal(int64(0644))) + + contents, hdr, err = ExtractFile(file, platformSpecificDependencyPath1) + Expect(err).NotTo(HaveOccurred()) + Expect(string(contents)).To(Equal("other-dependency-contents")) + Expect(hdr.Mode).To(Equal(int64(0644))) + }) + }) + }) + + context("that is multi architecture without os and arch but with default linux/amd64 directory layout", func() { + it.Before(func() { + err := cargo.NewDirectoryDuplicator().Duplicate(filepath.Join("testdata", "example-cnb-multi-arch-without-os-arch"), buildpackDir) + Expect(err).NotTo(HaveOccurred()) + }) + + context("when the buildpack is built to run offline", func() { + var server *httptest.Server + var config cargo.Config + it.Before(func() { + var err error + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + switch req.URL.Path { + case "/no-platform-dependency.tgz": + _, _ = fmt.Fprint(w, "no-platform-dependency-contents") + default: + http.NotFound(w, req) + } + })) + + config, err = cargo.NewBuildpackParser().Parse(filepath.Join(buildpackDir, "buildpack.toml")) + Expect(err).NotTo(HaveOccurred()) + Expect(config.Metadata.Dependencies).To(HaveLen(1)) + + config.Metadata.Dependencies[0].URI = fmt.Sprintf("%s/no-platform-dependency.tgz", server.URL) + config.Metadata.Dependencies[0].Checksum = "sha256:1f384c990b7aba4b80f2b12d85dc4a2d8ffaf9097ca542818a504a592d041642" + + bpTomlWriter, err := os.Create(filepath.Join(buildpackDir, "buildpack.toml")) + Expect(err).NotTo(HaveOccurred()) + + Expect(cargo.EncodeConfig(bpTomlWriter, config)).To(Succeed()) + }) + + it.After(func() { + server.Close() + }) + + it("creates an offline packaged buildpack", func() { + command := exec.Command( + path, "pack", + "--buildpack", filepath.Join(buildpackDir, "buildpack.toml"), + "--output", filepath.Join(tmpDir, "output.tgz"), + "--version", "some-version", + "--offline", + ) + session, err := gexec.Start(command, buffer, buffer) + Expect(err).NotTo(HaveOccurred()) + Eventually(session).Should(gexec.Exit(0), func() string { return buffer.String() }) + + relativeDependencyPath0 := "dependencies/1f384c990b7aba4b80f2b12d85dc4a2d8ffaf9097ca542818a504a592d041642" + platformSpecificDependencyPath0 := fmt.Sprintf("linux/amd64/%s", relativeDependencyPath0) + + Expect(session.Out).To(gbytes.Say("Packing some-buildpack-name some-version...")) + Expect(session.Out).To(gbytes.Say(" Executing pre-packaging script: ./scripts/build.sh")) + Expect(session.Out).To(gbytes.Say(" hello from the pre-packaging script")) + Expect(session.Out).To(gbytes.Say(" Downloading dependencies...")) + Expect(session.Out).To(gbytes.Say(` no-platform-dependency \(7.8.9\) \[some-stack-id\]`)) + Expect(session.Out).To(gbytes.Say(fmt.Sprintf(" ↳ %s", relativeDependencyPath0))) + Expect(session.Out).To(gbytes.Say(fmt.Sprintf(" Building tarball: %s", filepath.Join(tmpDir, "output.tgz")))) + Expect(session.Out).To(gbytes.Say(" linux/amd64/dependencies")) + Expect(session.Out).To(gbytes.Say(fmt.Sprintf(" %s", platformSpecificDependencyPath0))) + + file, err := os.Open(filepath.Join(tmpDir, "output.tgz")) + Expect(err).NotTo(HaveOccurred()) + + var extractedBuildpackConfig cargo.Config + contents, hdr, err := ExtractFile(file, "buildpack.toml") + Expect(err).NotTo(HaveOccurred()) + Expect(hdr.Mode).To(Equal(int64(0644))) + buff := bytes.NewBuffer(contents) + err = cargo.DecodeConfig(buff, &extractedBuildpackConfig) + Expect(err).NotTo(HaveOccurred()) + + Expect(config.Metadata.Dependencies).To(HaveLen(1)) + Expect(extractedBuildpackConfig.Metadata.Dependencies).To(HaveLen(1)) + + updatedIncludeFiles := config.Metadata.IncludeFiles + updatedIncludeFiles = append(updatedIncludeFiles, platformSpecificDependencyPath0) + + Expect(extractedBuildpackConfig.Metadata.IncludeFiles).To(Equal(updatedIncludeFiles)) + Expect(extractedBuildpackConfig.Metadata.Dependencies[0].URI).To(Equal(fmt.Sprintf(`file:///%s`, relativeDependencyPath0))) + Expect(extractedBuildpackConfig.Metadata.Dependencies[0].Checksum).To(Equal(config.Metadata.Dependencies[0].Checksum)) + + contents, hdr, err = ExtractFile(file, platformSpecificDependencyPath0) + Expect(err).NotTo(HaveOccurred()) + Expect(string(contents)).To(Equal("no-platform-dependency-contents")) + Expect(hdr.Mode).To(Equal(int64(0644))) + }) }) }) }) diff --git a/integration/testdata/example-cnb-multi-arch-with-os-arch/buildpack.toml b/integration/testdata/example-cnb-multi-arch-with-os-arch/buildpack.toml new file mode 100644 index 0000000..964f3ba --- /dev/null +++ b/integration/testdata/example-cnb-multi-arch-with-os-arch/buildpack.toml @@ -0,0 +1,60 @@ +api = "0.6" + +[buildpack] + id = "some-buildpack-id" + name = "some-buildpack-name" + version = "version-string" + homepage = "some-homepage-link" + description = "some-buildpack-description" + keywords = [ "some-buildpack-keyword" ] + +[[buildpack.licenses]] + type = "some-buildpack-license-type" + uri = "some-buildpack-license-uri" + +[metadata] + include-files = [ "some-os/some-arch/bin/build", "some-os/some-arch/bin/detect", "some-os/some-arch/bin/link", "some-os/some-arch/generated-file", "some-other-os/some-other-arch/bin/build", "some-other-os/some-other-arch/bin/detect", "some-other-os/some-other-arch/bin/link", "some-other-os/some-other-arch/generated-file", "buildpack.toml" ] + pre-package = "./scripts/build.sh" + [metadata.default-versions] + some-dependency = "some-default-version" + + [[metadata.dependencies]] + deprecation_date = 2019-04-01T00:00:00Z + id = "some-dependency" + name = "Some Dependency" + checksum = "sha256:shasum" + stacks = ["io.buildpacks.stacks.bionic", "org.cloudfoundry.stacks.tiny"] + uri = "http://some-url" + version = "1.2.3" + os = "some-os" + arch = "some-arch" + + [[metadata.dependencies.distros]] + name = "some-distro-name" + version = "some-distro-version" + + [[metadata.dependencies]] + deprecation_date = 2022-04-01T00:00:00Z + id = "other-dependency" + name = "Other Dependency" + checksum = "sha256:shasum" + stacks = ["org.cloudfoundry.stacks.tiny"] + uri = "http://other-url" + version = "4.5.6" + os = "some-other-os" + arch = "some-other-arch" + + [[metadata.dependencies.distros]] + name = "some-other-distro-name" + version = "some-other-distro-version" + +[[stacks]] + id = "some-stack-id" + +[[targets]] + os = "some-os" + arch = "some-arch" + +[[targets]] + os = "some-other-os" + arch = "some-other-arch" \ No newline at end of file diff --git a/integration/testdata/example-cnb-multi-arch-with-os-arch/scripts/build.sh b/integration/testdata/example-cnb-multi-arch-with-os-arch/scripts/build.sh new file mode 100755 index 0000000..df6c027 --- /dev/null +++ b/integration/testdata/example-cnb-multi-arch-with-os-arch/scripts/build.sh @@ -0,0 +1,12 @@ +#!/bin/bash +readonly PROGDIR="$(cd "$(dirname "${0}")" && pwd)" + +echo "hello from the pre-packaging script" + +for dir in some-os/some-arch some-other-os/some-other-arch; do + mkdir -p "$PROGDIR/../$dir" + + echo "$dir/hello" > "$PROGDIR/../$dir/generated-file" + + chmod 644 "$PROGDIR/../$dir/generated-file" +done diff --git a/integration/testdata/example-cnb-multi-arch-with-os-arch/some-os/some-arch/bin/build b/integration/testdata/example-cnb-multi-arch-with-os-arch/some-os/some-arch/bin/build new file mode 100755 index 0000000..626203d --- /dev/null +++ b/integration/testdata/example-cnb-multi-arch-with-os-arch/some-os/some-arch/bin/build @@ -0,0 +1 @@ +some-os/some-arch/build-contents \ No newline at end of file diff --git a/integration/testdata/example-cnb-multi-arch-with-os-arch/some-os/some-arch/bin/detect b/integration/testdata/example-cnb-multi-arch-with-os-arch/some-os/some-arch/bin/detect new file mode 100755 index 0000000..5687a7f --- /dev/null +++ b/integration/testdata/example-cnb-multi-arch-with-os-arch/some-os/some-arch/bin/detect @@ -0,0 +1 @@ +some-os/some-arch/detect-contents \ No newline at end of file diff --git a/integration/testdata/example-cnb-multi-arch-with-os-arch/some-os/some-arch/bin/link b/integration/testdata/example-cnb-multi-arch-with-os-arch/some-os/some-arch/bin/link new file mode 120000 index 0000000..9581f1d --- /dev/null +++ b/integration/testdata/example-cnb-multi-arch-with-os-arch/some-os/some-arch/bin/link @@ -0,0 +1 @@ +./build \ No newline at end of file diff --git a/integration/testdata/example-cnb-multi-arch-with-os-arch/some-other-os/some-other-arch/bin/build b/integration/testdata/example-cnb-multi-arch-with-os-arch/some-other-os/some-other-arch/bin/build new file mode 100755 index 0000000..bdb3474 --- /dev/null +++ b/integration/testdata/example-cnb-multi-arch-with-os-arch/some-other-os/some-other-arch/bin/build @@ -0,0 +1 @@ +some-other-os/some-other-arch/build-contents \ No newline at end of file diff --git a/integration/testdata/example-cnb-multi-arch-with-os-arch/some-other-os/some-other-arch/bin/detect b/integration/testdata/example-cnb-multi-arch-with-os-arch/some-other-os/some-other-arch/bin/detect new file mode 100755 index 0000000..0b2474a --- /dev/null +++ b/integration/testdata/example-cnb-multi-arch-with-os-arch/some-other-os/some-other-arch/bin/detect @@ -0,0 +1 @@ +some-other-os/some-other-arch/detect-contents \ No newline at end of file diff --git a/integration/testdata/example-cnb-multi-arch-with-os-arch/some-other-os/some-other-arch/bin/link b/integration/testdata/example-cnb-multi-arch-with-os-arch/some-other-os/some-other-arch/bin/link new file mode 120000 index 0000000..9581f1d --- /dev/null +++ b/integration/testdata/example-cnb-multi-arch-with-os-arch/some-other-os/some-other-arch/bin/link @@ -0,0 +1 @@ +./build \ No newline at end of file diff --git a/integration/testdata/example-cnb-multi-arch-without-os-arch/buildpack.toml b/integration/testdata/example-cnb-multi-arch-without-os-arch/buildpack.toml new file mode 100644 index 0000000..4b4d074 --- /dev/null +++ b/integration/testdata/example-cnb-multi-arch-without-os-arch/buildpack.toml @@ -0,0 +1,35 @@ +api = "0.6" + +[buildpack] + id = "some-buildpack-id" + name = "some-buildpack-name" + version = "version-string" + homepage = "some-homepage-link" + description = "some-buildpack-description" + keywords = [ "some-buildpack-keyword" ] + +[[buildpack.licenses]] + type = "some-buildpack-license-type" + uri = "some-buildpack-license-uri" + +[metadata] + include-files = [ "linux/amd64/bin/build", "linux/amd64/bin/detect", "linux/amd64/bin/link", "linux/amd64/generated-file", "buildpack.toml" ] + pre-package = "./scripts/build.sh" + [metadata.default-versions] + some-dependency = "some-default-version" + + [[metadata.dependencies]] + deprecation_date = 2019-04-01T00:00:00Z + id = "no-platform-dependency" + name = "Some Dependency without os and arch that should save offline dependencies to linux/amd64 path" + checksum = "sha256:shasum" + stacks = ["some-stack-id"] + uri = "http://another-url" + version = "7.8.9" + +[[stacks]] + id = "some-stack-id" + +[[targets]] + os = "linux" + arch = "amd64" \ No newline at end of file diff --git a/integration/testdata/example-cnb-multi-arch-without-os-arch/linux/amd64/bin/build b/integration/testdata/example-cnb-multi-arch-without-os-arch/linux/amd64/bin/build new file mode 100755 index 0000000..8314dc3 --- /dev/null +++ b/integration/testdata/example-cnb-multi-arch-without-os-arch/linux/amd64/bin/build @@ -0,0 +1 @@ +linux/amd64/build-contents \ No newline at end of file diff --git a/integration/testdata/example-cnb-multi-arch-without-os-arch/linux/amd64/bin/detect b/integration/testdata/example-cnb-multi-arch-without-os-arch/linux/amd64/bin/detect new file mode 100755 index 0000000..6109725 --- /dev/null +++ b/integration/testdata/example-cnb-multi-arch-without-os-arch/linux/amd64/bin/detect @@ -0,0 +1 @@ +linux/amd64/detect-contents \ No newline at end of file diff --git a/integration/testdata/example-cnb-multi-arch-without-os-arch/linux/amd64/bin/link b/integration/testdata/example-cnb-multi-arch-without-os-arch/linux/amd64/bin/link new file mode 120000 index 0000000..9581f1d --- /dev/null +++ b/integration/testdata/example-cnb-multi-arch-without-os-arch/linux/amd64/bin/link @@ -0,0 +1 @@ +./build \ No newline at end of file diff --git a/integration/testdata/example-cnb-multi-arch-without-os-arch/scripts/build.sh b/integration/testdata/example-cnb-multi-arch-without-os-arch/scripts/build.sh new file mode 100755 index 0000000..eed456b --- /dev/null +++ b/integration/testdata/example-cnb-multi-arch-without-os-arch/scripts/build.sh @@ -0,0 +1,12 @@ +#!/bin/bash +readonly PROGDIR="$(cd "$(dirname "${0}")" && pwd)" + +echo "hello from the pre-packaging script" + +for dir in linux/amd64; do + mkdir -p "$PROGDIR/../$dir" + + echo "$dir/hello" > "$PROGDIR/../$dir/generated-file" + + chmod 644 "$PROGDIR/../$dir/generated-file" +done