From ea6012d591a20a08f115c0fc4119ee68cce9f65f Mon Sep 17 00:00:00 2001 From: Radmir Khurum Date: Tue, 23 Jun 2026 16:49:45 +0300 Subject: [PATCH 1/2] feat(sbom): enforce network isolation for Stapel stages when SBOM enabled Signed-off-by: Radmir Khurum --- pkg/build/image/stapel.go | 27 ++ pkg/build/stage/base.go | 20 +- pkg/build/stage/package_resolve.go | 109 ++++++++ pkg/build/stage/package_resolve_test.go | 256 ++++++++++++++++++ .../managed_deps_enforcement/werf.yaml | 12 + test/e2e/build/network_test.go | 17 ++ 6 files changed, 438 insertions(+), 3 deletions(-) create mode 100644 pkg/build/stage/package_resolve.go create mode 100644 pkg/build/stage/package_resolve_test.go create mode 100644 test/e2e/build/_fixtures/network/managed_deps_enforcement/werf.yaml diff --git a/pkg/build/image/stapel.go b/pkg/build/image/stapel.go index 9879351dca..fb57ad4f0f 100644 --- a/pkg/build/image/stapel.go +++ b/pkg/build/image/stapel.go @@ -113,6 +113,10 @@ func initStages(ctx context.Context, image *Image, metaConfig *config.Meta, stap stages = append(stages, stage.NewGitArchiveStage(gitArchiveStageOptions, baseStageOptions)) } + for i, pkg := range imageBaseConfig.Packages { + stages = appendIfExist(ctx, stages, stage.GeneratePackageResolveStage(pkg, i, baseStageOptions)) + } + stages = appendIfExist(ctx, stages, stage.GenerateInstallStage(ctx, imageBaseConfig, gitPatchStageOptions, baseStageOptions)) stages = appendIfExist(ctx, stages, stage.GenerateDependenciesAfterInstallStage(imageBaseConfig, baseStageOptions)) stages = appendIfExist(ctx, stages, stage.GenerateBeforeSetupStage(ctx, imageBaseConfig, gitPatchStageOptions, baseStageOptions)) @@ -149,11 +153,34 @@ func initStages(ctx context.Context, image *Image, metaConfig *config.Meta, stap } } + sbomEnabled := metaConfig.Build.Sbom != nil && metaConfig.Build.Sbom.Enable + if sbomEnabled { + logboek.Context(ctx).Warn().LogLn("Network is disabled for shell stages (build.sbom.enable is true). Declare dependencies via 'packages' directive.") + + for _, s := range stages { + if stageHasNetworkAccess(s) { + continue + } + if no, ok := s.(interface{ SetNetworkOverride(string) }); ok { + no.SetNetworkOverride("none") + } + } + } + image.SetStages(stages) return nil } +func stageHasNetworkAccess(s stage.Interface) bool { + switch s.Name() { + case stage.From, stage.GitArchive, stage.GitCache, stage.GitLatestPatch: + return true + } + _, isPackageResolve := s.(*stage.PackageResolveStage) + return isPackageResolve +} + // TODO(v3): make this a hard error instead of a warning. func warnStageDependenciesWithoutInstructions(ctx context.Context, imageBaseConfig *config.StapelImageBase, gitMappings []*stage.GitMapping) { for _, gitMapping := range gitMappings { diff --git a/pkg/build/stage/base.go b/pkg/build/stage/base.go index f45b9a0a3f..6ad0e50e6b 100644 --- a/pkg/build/stage/base.go +++ b/pkg/build/stage/base.go @@ -138,6 +138,7 @@ type BaseStage struct { configMounts []*config.Mount projectName string network string + networkOverride string meta *StageMeta } @@ -151,6 +152,14 @@ func (s *BaseStage) IsBuildable() bool { return true } +func (s *BaseStage) SetNetworkOverride(network string) { + s.networkOverride = network +} + +func (s *BaseStage) NetworkOverrideValue() string { + return s.networkOverride +} + func (s *BaseStage) IsMutable() bool { return false } @@ -344,11 +353,16 @@ func (s *BaseStage) PrepareImage(ctx context.Context, c Conveyor, cb container_b s.addProjectRepoCommitLabel(ctx, c, cb, stageImage) - if s.network != "" { + network := s.network + if s.networkOverride != "" { + network = s.networkOverride + } + + if network != "" { if c.UseLegacyStapelBuilder(cb) { - stageImage.Builder.LegacyStapelStageBuilder().Container().RunOptions().AddNetwork(s.network) + stageImage.Builder.LegacyStapelStageBuilder().Container().RunOptions().AddNetwork(network) } else { - stageImage.Builder.StapelStageBuilder().SetNetwork(s.network) + stageImage.Builder.StapelStageBuilder().SetNetwork(network) } } diff --git a/pkg/build/stage/package_resolve.go b/pkg/build/stage/package_resolve.go new file mode 100644 index 0000000000..2739b21248 --- /dev/null +++ b/pkg/build/stage/package_resolve.go @@ -0,0 +1,109 @@ +package stage + +import ( + "context" + "fmt" + "path" + + "github.com/werf/common-go/pkg/util" + "github.com/werf/werf/v2/pkg/config" + "github.com/werf/werf/v2/pkg/container_backend" +) + +type PackageResolveStage struct { + *BaseStage + directive *config.PackagesDirective + index int +} + +func GeneratePackageResolveStage(directive *config.PackagesDirective, index int, baseStageOptions *BaseStageOptions) *PackageResolveStage { + if directive == nil { + return nil + } + s := newPackageResolveStage(directive, index, baseStageOptions) + if len(s.resolveCommands()) == 0 { + return nil + } + return s +} + +func newPackageResolveStage(directive *config.PackagesDirective, index int, baseStageOptions *BaseStageOptions) *PackageResolveStage { + s := &PackageResolveStage{ + directive: directive, + index: index, + } + s.BaseStage = NewBaseStage(StageName(fmt.Sprintf("packageResolve%d", index)), baseStageOptions) + return s +} + +func (s *PackageResolveStage) Name() StageName { + return StageName(fmt.Sprintf("packageResolve%d", s.index)) +} + +func (s *PackageResolveStage) SetGitMappings(gitMappings []*GitMapping) { + s.BaseStage.SetGitMappings(gitMappings) + + lockfilePath := s.lockfilePath() + if lockfilePath == "" { + return + } + + for _, gm := range gitMappings { + if gm.StagesDependencies == nil { + gm.StagesDependencies = make(map[StageName][]string) + } + gm.StagesDependencies[s.Name()] = append(gm.StagesDependencies[s.Name()], lockfilePath) + } +} + +func (s *PackageResolveStage) IsEmpty(ctx context.Context, c Conveyor, prevBuiltImage *StageImage) (bool, error) { + return false, nil +} + +func (s *PackageResolveStage) GetDependencies(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevImage, prevBuiltImage *StageImage, buildContextArchive container_backend.BuildContextArchiver) (string, error) { + args := []string{string(s.directive.Type), s.lockfilePath()} + + for _, gitMapping := range s.gitMappings { + checksum, err := gitMapping.StageDependenciesChecksum(ctx, c, s.Name()) + if err != nil { + return "", fmt.Errorf("get lockfile checksum: %w", err) + } + if checksum != "" { + args = append(args, checksum) + } + } + + return util.Sha256Hash(args...), nil +} + +func (s *PackageResolveStage) PrepareImage(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage, buildContextArchive container_backend.BuildContextArchiver) error { + if err := s.BaseStage.PrepareImage(ctx, c, cb, prevBuiltImage, stageImage, buildContextArchive); err != nil { + return err + } + + commands := s.resolveCommands() + + if c.UseLegacyStapelBuilder(cb) { + stageImage.Builder.LegacyStapelStageBuilder().Container().AddRunCommands(commands...) + } + + return nil +} + +func (s *PackageResolveStage) resolveCommands() []string { + switch s.directive.Type { + case config.PackagesDirectiveTypeGoMod: + return []string{fmt.Sprintf("cd %s && go mod download", s.directive.GoMod.Workdir)} + default: + return nil + } +} + +func (s *PackageResolveStage) lockfilePath() string { + switch s.directive.Type { + case config.PackagesDirectiveTypeGoMod: + return path.Join(s.directive.GoMod.Workdir, s.directive.GoMod.Lock) + default: + return "" + } +} diff --git a/pkg/build/stage/package_resolve_test.go b/pkg/build/stage/package_resolve_test.go new file mode 100644 index 0000000000..05c3fd1950 --- /dev/null +++ b/pkg/build/stage/package_resolve_test.go @@ -0,0 +1,256 @@ +package stage + +import ( + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/werf/werf/v2/pkg/config" +) + +var _ = Describe("NetworkOverride", func() { + DescribeTable("BaseStage.PrepareImage network resolution", + func(imageNetwork, override, expectedEffective string) { + s := NewBaseStage("test", &BaseStageOptions{Network: imageNetwork}) + if override != "" { + s.SetNetworkOverride(override) + } + + effective := s.network + if s.networkOverride != "" { + effective = s.networkOverride + } + + Expect(effective).To(Equal(expectedEffective)) + }, + + Entry("override=none takes priority over image network=host", + "host", "none", "none"), + Entry("override=none takes priority over image network=default", + "default", "none", "none"), + Entry("empty override uses image network", + "host", "", "host"), + Entry("empty override with empty image network stays empty", + "", "", ""), + Entry("override=none with empty image network uses override", + "", "none", "none"), + ) + + Describe("SetNetworkOverride", func() { + It("sets the networkOverride field without affecting network", func() { + s := NewBaseStage("test", &BaseStageOptions{Network: "host"}) + s.SetNetworkOverride("none") + + Expect(s.networkOverride).To(Equal("none")) + Expect(s.network).To(Equal("host")) + }) + }) +}) + +var _ = Describe("PackageResolveStage", func() { + DescribeTable("resolveCommands", + func(directive *config.PackagesDirective, expectedCommands []string) { + s := newPackageResolveStage(directive, 0, &BaseStageOptions{}) + Expect(s.resolveCommands()).To(Equal(expectedCommands)) + }, + + Entry("go-mod with workdir /app", + &config.PackagesDirective{ + Type: config.PackagesDirectiveTypeGoMod, + GoMod: config.GoModSpec{Workdir: "/app", Spec: "go.mod", Lock: "go.sum"}, + }, + []string{"cd /app && go mod download"}, + ), + Entry("go-mod with workdir /src/backend", + &config.PackagesDirective{ + Type: config.PackagesDirectiveTypeGoMod, + GoMod: config.GoModSpec{Workdir: "/src/backend", Spec: "go.mod", Lock: "go.sum"}, + }, + []string{"cd /src/backend && go mod download"}, + ), + Entry("unsupported type returns nil", + &config.PackagesDirective{ + Type: config.PackagesDirectiveType("unknown"), + }, + ([]string)(nil), + ), + ) + + DescribeTable("Name includes index for uniqueness", + func(index int, expectedName StageName) { + s := newPackageResolveStage(&config.PackagesDirective{ + Type: config.PackagesDirectiveTypeGoMod, + GoMod: config.GoModSpec{Workdir: "/app", Spec: "go.mod", Lock: "go.sum"}, + }, index, &BaseStageOptions{}) + Expect(s.Name()).To(Equal(expectedName)) + }, + + Entry("index 0", 0, StageName("packageResolve0")), + Entry("index 1", 1, StageName("packageResolve1")), + Entry("index 5", 5, StageName("packageResolve5")), + ) + + Describe("IsEmpty", func() { + It("always returns false for a configured directive", func() { + s := newPackageResolveStage(&config.PackagesDirective{ + Type: config.PackagesDirectiveTypeGoMod, + GoMod: config.GoModSpec{Workdir: "/app", Spec: "go.mod", Lock: "go.sum"}, + }, 0, &BaseStageOptions{}) + + empty, err := s.IsEmpty(nil, nil, nil) + Expect(err).NotTo(HaveOccurred()) + Expect(empty).To(BeFalse()) + }) + }) + + Describe("GeneratePackageResolveStage", func() { + It("returns nil when directive is nil", func() { + Expect(GeneratePackageResolveStage(nil, 0, &BaseStageOptions{})).To(BeNil()) + }) + + It("returns a valid stage when directive is provided", func() { + d := &config.PackagesDirective{ + Type: config.PackagesDirectiveTypeGoMod, + GoMod: config.GoModSpec{Workdir: "/app", Spec: "go.mod", Lock: "go.sum"}, + } + s := GeneratePackageResolveStage(d, 0, &BaseStageOptions{}) + Expect(s).NotTo(BeNil()) + Expect(s.directive).To(Equal(d)) + }) + + DescribeTable("multiple entries produce independent stages", + func(directives []*config.PackagesDirective, expectedCount int) { + var stages []*PackageResolveStage + for i, d := range directives { + s := GeneratePackageResolveStage(d, i, &BaseStageOptions{}) + if s != nil { + stages = append(stages, s) + } + } + Expect(stages).To(HaveLen(expectedCount)) + + for i, s := range stages { + Expect(s.Name()).To(Equal(StageName(fmt.Sprintf("packageResolve%d", i)))) + } + }, + + Entry("two go-mod entries", []*config.PackagesDirective{ + {Type: config.PackagesDirectiveTypeGoMod, GoMod: config.GoModSpec{Workdir: "/app", Spec: "go.mod", Lock: "go.sum"}}, + {Type: config.PackagesDirectiveTypeGoMod, GoMod: config.GoModSpec{Workdir: "/lib", Spec: "go.mod", Lock: "go.sum"}}, + }, 2), + Entry("single entry", []*config.PackagesDirective{ + {Type: config.PackagesDirectiveTypeGoMod, GoMod: config.GoModSpec{Workdir: "/app", Spec: "go.mod", Lock: "go.sum"}}, + }, 1), + Entry("nil filtered out", []*config.PackagesDirective{nil}, 0), + ) + }) + + Describe("SetGitMappings", func() { + It("injects lockfile path into StagesDependencies for go-mod", func() { + d := &config.PackagesDirective{ + Type: config.PackagesDirectiveTypeGoMod, + GoMod: config.GoModSpec{Workdir: "/app", Spec: "go.mod", Lock: "go.sum"}, + } + s := newPackageResolveStage(d, 0, &BaseStageOptions{}) + + gm := NewGitMapping() + gm.StagesDependencies = make(map[StageName][]string) + + s.SetGitMappings([]*GitMapping{gm}) + + Expect(gm.StagesDependencies[s.Name()]).To(Equal([]string{"/app/go.sum"})) + }) + + It("injects lockfile path for multiple git mappings", func() { + d := &config.PackagesDirective{ + Type: config.PackagesDirectiveTypeGoMod, + GoMod: config.GoModSpec{Workdir: "/src", Spec: "go.mod", Lock: "go.sum"}, + } + s := newPackageResolveStage(d, 1, &BaseStageOptions{}) + + gm1 := NewGitMapping() + gm1.StagesDependencies = make(map[StageName][]string) + gm2 := NewGitMapping() + gm2.StagesDependencies = make(map[StageName][]string) + + s.SetGitMappings([]*GitMapping{gm1, gm2}) + + Expect(gm1.StagesDependencies[StageName("packageResolve1")]).To(Equal([]string{"/src/go.sum"})) + Expect(gm2.StagesDependencies[StageName("packageResolve1")]).To(Equal([]string{"/src/go.sum"})) + }) + + It("does not inject when lockfilePath is empty (unknown type)", func() { + d := &config.PackagesDirective{ + Type: config.PackagesDirectiveType("unknown"), + } + s := newPackageResolveStage(d, 0, &BaseStageOptions{}) + + gm := NewGitMapping() + gm.StagesDependencies = make(map[StageName][]string) + + s.SetGitMappings([]*GitMapping{gm}) + + Expect(gm.StagesDependencies[s.Name()]).To(BeEmpty()) + }) + + It("initializes StagesDependencies map if nil", func() { + d := &config.PackagesDirective{ + Type: config.PackagesDirectiveTypeGoMod, + GoMod: config.GoModSpec{Workdir: "/app", Spec: "go.mod", Lock: "go.sum"}, + } + s := newPackageResolveStage(d, 0, &BaseStageOptions{}) + + gm := NewGitMapping() + + s.SetGitMappings([]*GitMapping{gm}) + + Expect(gm.StagesDependencies).NotTo(BeNil()) + Expect(gm.StagesDependencies[s.Name()]).To(Equal([]string{"/app/go.sum"})) + }) + }) + + DescribeTable("lockfilePath", + func(directive *config.PackagesDirective, expectedPath string) { + s := newPackageResolveStage(directive, 0, &BaseStageOptions{}) + Expect(s.lockfilePath()).To(Equal(expectedPath)) + }, + + Entry("go-mod with workdir /app", + &config.PackagesDirective{ + Type: config.PackagesDirectiveTypeGoMod, + GoMod: config.GoModSpec{Workdir: "/app", Spec: "go.mod", Lock: "go.sum"}, + }, + "/app/go.sum", + ), + Entry("go-mod with custom lock filename", + &config.PackagesDirective{ + Type: config.PackagesDirectiveTypeGoMod, + GoMod: config.GoModSpec{Workdir: "/src", Spec: "go.mod", Lock: "go.sum.custom"}, + }, + "/src/go.sum.custom", + ), + Entry("unknown type returns empty", + &config.PackagesDirective{ + Type: config.PackagesDirectiveType("pip"), + }, + "", + ), + ) + + Describe("NetworkOverrideValue getter", func() { + It("returns empty by default", func() { + s := newPackageResolveStage(&config.PackagesDirective{ + Type: config.PackagesDirectiveTypeGoMod, + GoMod: config.GoModSpec{Workdir: "/app", Spec: "go.mod", Lock: "go.sum"}, + }, 0, &BaseStageOptions{}) + Expect(s.NetworkOverrideValue()).To(Equal("")) + }) + + It("returns value after SetNetworkOverride", func() { + s := NewBaseStage("test", &BaseStageOptions{}) + s.SetNetworkOverride("none") + Expect(s.NetworkOverrideValue()).To(Equal("none")) + }) + }) +}) diff --git a/test/e2e/build/_fixtures/network/managed_deps_enforcement/werf.yaml b/test/e2e/build/_fixtures/network/managed_deps_enforcement/werf.yaml new file mode 100644 index 0000000000..18a2d1143a --- /dev/null +++ b/test/e2e/build/_fixtures/network/managed_deps_enforcement/werf.yaml @@ -0,0 +1,12 @@ +project: network-managed-deps-test +configVersion: 1 +build: + sbom: + enable: true + standard: "cyclonedx@1.6" +--- +image: stapel +from: registry.werf.io/base/alpine +shell: + setup: + - apk add --no-cache bash diff --git a/test/e2e/build/network_test.go b/test/e2e/build/network_test.go index e2b388cd96..a9fb07b7f5 100644 --- a/test/e2e/build/network_test.go +++ b/test/e2e/build/network_test.go @@ -120,5 +120,22 @@ var _ = Describe("Network isolation build", Label("e2e", "build", "network"), fu FixturePath: "network/stapel_yml_success", NetworkNone: true, // CLI 'none' overrides YAML 'host' }), + + // SBOM enforcement: shell stages get --network=none when build.sbom.enable=true + Entry("Stapel (Vanilla): sbom.enable=true enforces --network=none on shell stages (should fail)", Label("stapel", "managed-deps"), networkTestOptions{ + setupEnvOptions: setupEnvOptions{ContainerBackendMode: "vanilla-docker", WithLocalRepo: false}, + ExpectError: true, + FixturePath: "network/managed_deps_enforcement", + NetworkNone: false, + ExpectNetworkValue: "none", + }), + + // SBOM enforcement, regression: sbom.enable=false does NOT enforce network isolation + Entry("Stapel (Vanilla): sbom.enable=false does not enforce --network=none (should succeed)", Label("stapel", "managed-deps"), networkTestOptions{ + setupEnvOptions: setupEnvOptions{ContainerBackendMode: "vanilla-docker", WithLocalRepo: false}, + ExpectError: false, + FixturePath: "network/stapel", + NetworkNone: false, + }), ) }) From 97c8405240c5e51a00788f55273264f250aa8901 Mon Sep 17 00:00:00 2001 From: Radmir Khurum Date: Wed, 24 Jun 2026 18:43:58 +0300 Subject: [PATCH 2/2] feat(sbom): replace PackageResolveStage with shell-based packages stage Signed-off-by: Radmir Khurum --- pkg/build/build_phase.go | 4 + pkg/build/builder/ansible.go | 8 + pkg/build/builder/builder.go | 3 + pkg/build/builder/shell.go | 9 +- pkg/build/image/image_tree.go | 1 + pkg/build/image/stapel.go | 20 +- pkg/build/stage/base.go | 8 +- pkg/build/stage/package_resolve.go | 109 -------- pkg/build/stage/package_resolve_test.go | 256 ------------------- pkg/build/stage/packages.go | 52 ++++ pkg/build/stage/packages_test.go | 315 ++++++++++++++++++++++++ pkg/config/packages_commands.go | 13 + pkg/config/raw_stage_dependencies.go | 7 + pkg/config/raw_stapel_image.go | 10 + pkg/config/shell.go | 2 + pkg/config/stage_dependencies.go | 3 + 16 files changed, 442 insertions(+), 378 deletions(-) delete mode 100644 pkg/build/stage/package_resolve.go delete mode 100644 pkg/build/stage/package_resolve_test.go create mode 100644 pkg/build/stage/packages.go create mode 100644 pkg/build/stage/packages_test.go create mode 100644 pkg/config/packages_commands.go diff --git a/pkg/build/build_phase.go b/pkg/build/build_phase.go index 0420417dc7..3117700e96 100644 --- a/pkg/build/build_phase.go +++ b/pkg/build/build_phase.go @@ -354,6 +354,10 @@ func (phase *BuildPhase) scanOptionsForImage(img *image.Image) scanner.ScanOptio return scanOpts } + if len(scanOpts.Commands) == 0 { + return scanOpts + } + catalogers := managedinput.ToCatalogers(stapelConfig.ImageBaseConfig().Packages) for i := range scanOpts.Commands { scanOpts.Commands[i].Catalogers = catalogers diff --git a/pkg/build/builder/ansible.go b/pkg/build/builder/ansible.go index 065ba09f3b..fcff422328 100644 --- a/pkg/build/builder/ansible.go +++ b/pkg/build/builder/ansible.go @@ -73,6 +73,14 @@ func (b *Ansible) BeforeSetupChecksum(ctx context.Context) string { } func (b *Ansible) SetupChecksum(ctx context.Context) string { return b.stageChecksum(ctx, "Setup") } +func (b *Ansible) IsPackagesEmpty(_ context.Context) bool { return true } + +func (b *Ansible) Packages(_ context.Context, _ container_backend.ContainerBackend, _ stage_builder.StageBuilderInterface, _ bool) error { + return nil +} + +func (b *Ansible) PackagesChecksum(_ context.Context) string { return "" } + func (b *Ansible) isEmptyStage(ctx context.Context, userStageName string) bool { return b.stageChecksum(ctx, userStageName) == "" } diff --git a/pkg/build/builder/builder.go b/pkg/build/builder/builder.go index 4f1faf4f37..0af9607210 100644 --- a/pkg/build/builder/builder.go +++ b/pkg/build/builder/builder.go @@ -13,14 +13,17 @@ type Builder interface { IsInstallEmpty(ctx context.Context) bool IsBeforeSetupEmpty(ctx context.Context) bool IsSetupEmpty(ctx context.Context) bool + IsPackagesEmpty(ctx context.Context) bool BeforeInstall(ctx context.Context, cr container_backend.ContainerBackend, stageBuilder stage_builder.StageBuilderInterface, useLegacyStapelBuilder bool) error Install(ctx context.Context, cr container_backend.ContainerBackend, stageBuilder stage_builder.StageBuilderInterface, useLegacyStapelBuilder bool) error BeforeSetup(ctx context.Context, cr container_backend.ContainerBackend, stageBuilder stage_builder.StageBuilderInterface, useLegacyStapelBuilder bool) error Setup(ctx context.Context, cr container_backend.ContainerBackend, stageBuilder stage_builder.StageBuilderInterface, useLegacyStapelBuilder bool) error + Packages(ctx context.Context, cr container_backend.ContainerBackend, stageBuilder stage_builder.StageBuilderInterface, useLegacyStapelBuilder bool) error BeforeInstallChecksum(ctx context.Context) string InstallChecksum(ctx context.Context) string BeforeSetupChecksum(ctx context.Context) string SetupChecksum(ctx context.Context) string + PackagesChecksum(ctx context.Context) string } type Container interface { diff --git a/pkg/build/builder/shell.go b/pkg/build/builder/shell.go index 94e69d3bf9..ad23c86be1 100644 --- a/pkg/build/builder/shell.go +++ b/pkg/build/builder/shell.go @@ -40,6 +40,8 @@ func (b *Shell) IsBeforeSetupEmpty(ctx context.Context) bool { } func (b *Shell) IsSetupEmpty(ctx context.Context) bool { return b.isEmptyStage(ctx, "Setup") } +func (b *Shell) IsPackagesEmpty(ctx context.Context) bool { return b.isEmptyStage(ctx, "Packages") } + func (b *Shell) BeforeInstall(_ context.Context, cr container_backend.ContainerBackend, stageBuilder stage_builder.StageBuilderInterface, useLegacyStapelBuilder bool) error { return b.stage(cr, stageBuilder, useLegacyStapelBuilder, "BeforeInstall") } @@ -56,6 +58,10 @@ func (b *Shell) Setup(_ context.Context, cr container_backend.ContainerBackend, return b.stage(cr, stageBuilder, useLegacyStapelBuilder, "Setup") } +func (b *Shell) Packages(_ context.Context, cr container_backend.ContainerBackend, stageBuilder stage_builder.StageBuilderInterface, useLegacyStapelBuilder bool) error { + return b.stage(cr, stageBuilder, useLegacyStapelBuilder, "Packages") +} + func (b *Shell) BeforeInstallChecksum(ctx context.Context) string { return b.stageChecksum(ctx, "BeforeInstall") } @@ -63,7 +69,8 @@ func (b *Shell) InstallChecksum(ctx context.Context) string { return b.stageChec func (b *Shell) BeforeSetupChecksum(ctx context.Context) string { return b.stageChecksum(ctx, "BeforeSetup") } -func (b *Shell) SetupChecksum(ctx context.Context) string { return b.stageChecksum(ctx, "Setup") } +func (b *Shell) SetupChecksum(ctx context.Context) string { return b.stageChecksum(ctx, "Setup") } +func (b *Shell) PackagesChecksum(ctx context.Context) string { return b.stageChecksum(ctx, "Packages") } func (b *Shell) isEmptyStage(ctx context.Context, userStageName string) bool { return b.stageChecksum(ctx, userStageName) == "" diff --git a/pkg/build/image/image_tree.go b/pkg/build/image/image_tree.go index c2810f9bbe..9d280e78ee 100644 --- a/pkg/build/image/image_tree.go +++ b/pkg/build/image/image_tree.go @@ -431,6 +431,7 @@ func stageDependenciesToMap(sd *config.StageDependencies) map[stage.StageName][] stage.Install: sd.Install, stage.BeforeSetup: sd.BeforeSetup, stage.Setup: sd.Setup, + stage.Packages: sd.Packages, } return result diff --git a/pkg/build/image/stapel.go b/pkg/build/image/stapel.go index fb57ad4f0f..4eebe5f585 100644 --- a/pkg/build/image/stapel.go +++ b/pkg/build/image/stapel.go @@ -113,9 +113,7 @@ func initStages(ctx context.Context, image *Image, metaConfig *config.Meta, stap stages = append(stages, stage.NewGitArchiveStage(gitArchiveStageOptions, baseStageOptions)) } - for i, pkg := range imageBaseConfig.Packages { - stages = appendIfExist(ctx, stages, stage.GeneratePackageResolveStage(pkg, i, baseStageOptions)) - } + stages = appendIfExist(ctx, stages, stage.GeneratePackagesStage(ctx, imageBaseConfig, gitPatchStageOptions, baseStageOptions)) stages = appendIfExist(ctx, stages, stage.GenerateInstallStage(ctx, imageBaseConfig, gitPatchStageOptions, baseStageOptions)) stages = appendIfExist(ctx, stages, stage.GenerateDependenciesAfterInstallStage(imageBaseConfig, baseStageOptions)) @@ -155,16 +153,20 @@ func initStages(ctx context.Context, image *Image, metaConfig *config.Meta, stap sbomEnabled := metaConfig.Build.Sbom != nil && metaConfig.Build.Sbom.Enable if sbomEnabled { - logboek.Context(ctx).Warn().LogLn("Network is disabled for shell stages (build.sbom.enable is true). Declare dependencies via 'packages' directive.") - + hasShellStages := false for _, s := range stages { if stageHasNetworkAccess(s) { continue } if no, ok := s.(interface{ SetNetworkOverride(string) }); ok { no.SetNetworkOverride("none") + hasShellStages = true } } + + if hasShellStages { + logboek.Context(ctx).Warn().LogLn("Network is disabled for shell stages (build.sbom.enable is true). Declare dependencies via 'packages' directive.") + } } image.SetStages(stages) @@ -173,12 +175,10 @@ func initStages(ctx context.Context, image *Image, metaConfig *config.Meta, stap } func stageHasNetworkAccess(s stage.Interface) bool { - switch s.Name() { - case stage.From, stage.GitArchive, stage.GitCache, stage.GitLatestPatch: - return true + if nn, ok := s.(interface{ NeedsNetwork() bool }); ok { + return nn.NeedsNetwork() } - _, isPackageResolve := s.(*stage.PackageResolveStage) - return isPackageResolve + return false } // TODO(v3): make this a hard error instead of a warning. diff --git a/pkg/build/stage/base.go b/pkg/build/stage/base.go index 6ad0e50e6b..ce4ab3143e 100644 --- a/pkg/build/stage/base.go +++ b/pkg/build/stage/base.go @@ -33,6 +33,7 @@ const ( DependenciesBeforeSetup StageName = "dependenciesBeforeSetup" Setup StageName = "setup" DependenciesAfterSetup StageName = "dependenciesAfterSetup" + Packages StageName = "packages" GitCache StageName = "gitCache" GitLatestPatch StageName = "gitLatestPatch" DockerInstructions StageName = "dockerInstructions" @@ -88,6 +89,7 @@ type BaseStageOptions struct { ContainerWerfDir string ProjectName string Network string + NeedsNetwork bool } const disableGitCommitAncestryCheckEnv = "WERF_DISABLE_GIT_COMMIT_ANCESTRY_CHECK" @@ -120,6 +122,7 @@ func NewBaseStage(name StageName, options *BaseStageOptions) *BaseStage { s.containerWerfDir = options.ContainerWerfDir s.projectName = options.ProjectName s.network = options.Network + s.needsNetwork = options.NeedsNetwork s.meta = &StageMeta{} return s } @@ -139,6 +142,7 @@ type BaseStage struct { projectName string network string networkOverride string + needsNetwork bool meta *StageMeta } @@ -156,8 +160,8 @@ func (s *BaseStage) SetNetworkOverride(network string) { s.networkOverride = network } -func (s *BaseStage) NetworkOverrideValue() string { - return s.networkOverride +func (s *BaseStage) NeedsNetwork() bool { + return s.needsNetwork } func (s *BaseStage) IsMutable() bool { diff --git a/pkg/build/stage/package_resolve.go b/pkg/build/stage/package_resolve.go deleted file mode 100644 index 2739b21248..0000000000 --- a/pkg/build/stage/package_resolve.go +++ /dev/null @@ -1,109 +0,0 @@ -package stage - -import ( - "context" - "fmt" - "path" - - "github.com/werf/common-go/pkg/util" - "github.com/werf/werf/v2/pkg/config" - "github.com/werf/werf/v2/pkg/container_backend" -) - -type PackageResolveStage struct { - *BaseStage - directive *config.PackagesDirective - index int -} - -func GeneratePackageResolveStage(directive *config.PackagesDirective, index int, baseStageOptions *BaseStageOptions) *PackageResolveStage { - if directive == nil { - return nil - } - s := newPackageResolveStage(directive, index, baseStageOptions) - if len(s.resolveCommands()) == 0 { - return nil - } - return s -} - -func newPackageResolveStage(directive *config.PackagesDirective, index int, baseStageOptions *BaseStageOptions) *PackageResolveStage { - s := &PackageResolveStage{ - directive: directive, - index: index, - } - s.BaseStage = NewBaseStage(StageName(fmt.Sprintf("packageResolve%d", index)), baseStageOptions) - return s -} - -func (s *PackageResolveStage) Name() StageName { - return StageName(fmt.Sprintf("packageResolve%d", s.index)) -} - -func (s *PackageResolveStage) SetGitMappings(gitMappings []*GitMapping) { - s.BaseStage.SetGitMappings(gitMappings) - - lockfilePath := s.lockfilePath() - if lockfilePath == "" { - return - } - - for _, gm := range gitMappings { - if gm.StagesDependencies == nil { - gm.StagesDependencies = make(map[StageName][]string) - } - gm.StagesDependencies[s.Name()] = append(gm.StagesDependencies[s.Name()], lockfilePath) - } -} - -func (s *PackageResolveStage) IsEmpty(ctx context.Context, c Conveyor, prevBuiltImage *StageImage) (bool, error) { - return false, nil -} - -func (s *PackageResolveStage) GetDependencies(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevImage, prevBuiltImage *StageImage, buildContextArchive container_backend.BuildContextArchiver) (string, error) { - args := []string{string(s.directive.Type), s.lockfilePath()} - - for _, gitMapping := range s.gitMappings { - checksum, err := gitMapping.StageDependenciesChecksum(ctx, c, s.Name()) - if err != nil { - return "", fmt.Errorf("get lockfile checksum: %w", err) - } - if checksum != "" { - args = append(args, checksum) - } - } - - return util.Sha256Hash(args...), nil -} - -func (s *PackageResolveStage) PrepareImage(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage, buildContextArchive container_backend.BuildContextArchiver) error { - if err := s.BaseStage.PrepareImage(ctx, c, cb, prevBuiltImage, stageImage, buildContextArchive); err != nil { - return err - } - - commands := s.resolveCommands() - - if c.UseLegacyStapelBuilder(cb) { - stageImage.Builder.LegacyStapelStageBuilder().Container().AddRunCommands(commands...) - } - - return nil -} - -func (s *PackageResolveStage) resolveCommands() []string { - switch s.directive.Type { - case config.PackagesDirectiveTypeGoMod: - return []string{fmt.Sprintf("cd %s && go mod download", s.directive.GoMod.Workdir)} - default: - return nil - } -} - -func (s *PackageResolveStage) lockfilePath() string { - switch s.directive.Type { - case config.PackagesDirectiveTypeGoMod: - return path.Join(s.directive.GoMod.Workdir, s.directive.GoMod.Lock) - default: - return "" - } -} diff --git a/pkg/build/stage/package_resolve_test.go b/pkg/build/stage/package_resolve_test.go deleted file mode 100644 index 05c3fd1950..0000000000 --- a/pkg/build/stage/package_resolve_test.go +++ /dev/null @@ -1,256 +0,0 @@ -package stage - -import ( - "fmt" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/werf/werf/v2/pkg/config" -) - -var _ = Describe("NetworkOverride", func() { - DescribeTable("BaseStage.PrepareImage network resolution", - func(imageNetwork, override, expectedEffective string) { - s := NewBaseStage("test", &BaseStageOptions{Network: imageNetwork}) - if override != "" { - s.SetNetworkOverride(override) - } - - effective := s.network - if s.networkOverride != "" { - effective = s.networkOverride - } - - Expect(effective).To(Equal(expectedEffective)) - }, - - Entry("override=none takes priority over image network=host", - "host", "none", "none"), - Entry("override=none takes priority over image network=default", - "default", "none", "none"), - Entry("empty override uses image network", - "host", "", "host"), - Entry("empty override with empty image network stays empty", - "", "", ""), - Entry("override=none with empty image network uses override", - "", "none", "none"), - ) - - Describe("SetNetworkOverride", func() { - It("sets the networkOverride field without affecting network", func() { - s := NewBaseStage("test", &BaseStageOptions{Network: "host"}) - s.SetNetworkOverride("none") - - Expect(s.networkOverride).To(Equal("none")) - Expect(s.network).To(Equal("host")) - }) - }) -}) - -var _ = Describe("PackageResolveStage", func() { - DescribeTable("resolveCommands", - func(directive *config.PackagesDirective, expectedCommands []string) { - s := newPackageResolveStage(directive, 0, &BaseStageOptions{}) - Expect(s.resolveCommands()).To(Equal(expectedCommands)) - }, - - Entry("go-mod with workdir /app", - &config.PackagesDirective{ - Type: config.PackagesDirectiveTypeGoMod, - GoMod: config.GoModSpec{Workdir: "/app", Spec: "go.mod", Lock: "go.sum"}, - }, - []string{"cd /app && go mod download"}, - ), - Entry("go-mod with workdir /src/backend", - &config.PackagesDirective{ - Type: config.PackagesDirectiveTypeGoMod, - GoMod: config.GoModSpec{Workdir: "/src/backend", Spec: "go.mod", Lock: "go.sum"}, - }, - []string{"cd /src/backend && go mod download"}, - ), - Entry("unsupported type returns nil", - &config.PackagesDirective{ - Type: config.PackagesDirectiveType("unknown"), - }, - ([]string)(nil), - ), - ) - - DescribeTable("Name includes index for uniqueness", - func(index int, expectedName StageName) { - s := newPackageResolveStage(&config.PackagesDirective{ - Type: config.PackagesDirectiveTypeGoMod, - GoMod: config.GoModSpec{Workdir: "/app", Spec: "go.mod", Lock: "go.sum"}, - }, index, &BaseStageOptions{}) - Expect(s.Name()).To(Equal(expectedName)) - }, - - Entry("index 0", 0, StageName("packageResolve0")), - Entry("index 1", 1, StageName("packageResolve1")), - Entry("index 5", 5, StageName("packageResolve5")), - ) - - Describe("IsEmpty", func() { - It("always returns false for a configured directive", func() { - s := newPackageResolveStage(&config.PackagesDirective{ - Type: config.PackagesDirectiveTypeGoMod, - GoMod: config.GoModSpec{Workdir: "/app", Spec: "go.mod", Lock: "go.sum"}, - }, 0, &BaseStageOptions{}) - - empty, err := s.IsEmpty(nil, nil, nil) - Expect(err).NotTo(HaveOccurred()) - Expect(empty).To(BeFalse()) - }) - }) - - Describe("GeneratePackageResolveStage", func() { - It("returns nil when directive is nil", func() { - Expect(GeneratePackageResolveStage(nil, 0, &BaseStageOptions{})).To(BeNil()) - }) - - It("returns a valid stage when directive is provided", func() { - d := &config.PackagesDirective{ - Type: config.PackagesDirectiveTypeGoMod, - GoMod: config.GoModSpec{Workdir: "/app", Spec: "go.mod", Lock: "go.sum"}, - } - s := GeneratePackageResolveStage(d, 0, &BaseStageOptions{}) - Expect(s).NotTo(BeNil()) - Expect(s.directive).To(Equal(d)) - }) - - DescribeTable("multiple entries produce independent stages", - func(directives []*config.PackagesDirective, expectedCount int) { - var stages []*PackageResolveStage - for i, d := range directives { - s := GeneratePackageResolveStage(d, i, &BaseStageOptions{}) - if s != nil { - stages = append(stages, s) - } - } - Expect(stages).To(HaveLen(expectedCount)) - - for i, s := range stages { - Expect(s.Name()).To(Equal(StageName(fmt.Sprintf("packageResolve%d", i)))) - } - }, - - Entry("two go-mod entries", []*config.PackagesDirective{ - {Type: config.PackagesDirectiveTypeGoMod, GoMod: config.GoModSpec{Workdir: "/app", Spec: "go.mod", Lock: "go.sum"}}, - {Type: config.PackagesDirectiveTypeGoMod, GoMod: config.GoModSpec{Workdir: "/lib", Spec: "go.mod", Lock: "go.sum"}}, - }, 2), - Entry("single entry", []*config.PackagesDirective{ - {Type: config.PackagesDirectiveTypeGoMod, GoMod: config.GoModSpec{Workdir: "/app", Spec: "go.mod", Lock: "go.sum"}}, - }, 1), - Entry("nil filtered out", []*config.PackagesDirective{nil}, 0), - ) - }) - - Describe("SetGitMappings", func() { - It("injects lockfile path into StagesDependencies for go-mod", func() { - d := &config.PackagesDirective{ - Type: config.PackagesDirectiveTypeGoMod, - GoMod: config.GoModSpec{Workdir: "/app", Spec: "go.mod", Lock: "go.sum"}, - } - s := newPackageResolveStage(d, 0, &BaseStageOptions{}) - - gm := NewGitMapping() - gm.StagesDependencies = make(map[StageName][]string) - - s.SetGitMappings([]*GitMapping{gm}) - - Expect(gm.StagesDependencies[s.Name()]).To(Equal([]string{"/app/go.sum"})) - }) - - It("injects lockfile path for multiple git mappings", func() { - d := &config.PackagesDirective{ - Type: config.PackagesDirectiveTypeGoMod, - GoMod: config.GoModSpec{Workdir: "/src", Spec: "go.mod", Lock: "go.sum"}, - } - s := newPackageResolveStage(d, 1, &BaseStageOptions{}) - - gm1 := NewGitMapping() - gm1.StagesDependencies = make(map[StageName][]string) - gm2 := NewGitMapping() - gm2.StagesDependencies = make(map[StageName][]string) - - s.SetGitMappings([]*GitMapping{gm1, gm2}) - - Expect(gm1.StagesDependencies[StageName("packageResolve1")]).To(Equal([]string{"/src/go.sum"})) - Expect(gm2.StagesDependencies[StageName("packageResolve1")]).To(Equal([]string{"/src/go.sum"})) - }) - - It("does not inject when lockfilePath is empty (unknown type)", func() { - d := &config.PackagesDirective{ - Type: config.PackagesDirectiveType("unknown"), - } - s := newPackageResolveStage(d, 0, &BaseStageOptions{}) - - gm := NewGitMapping() - gm.StagesDependencies = make(map[StageName][]string) - - s.SetGitMappings([]*GitMapping{gm}) - - Expect(gm.StagesDependencies[s.Name()]).To(BeEmpty()) - }) - - It("initializes StagesDependencies map if nil", func() { - d := &config.PackagesDirective{ - Type: config.PackagesDirectiveTypeGoMod, - GoMod: config.GoModSpec{Workdir: "/app", Spec: "go.mod", Lock: "go.sum"}, - } - s := newPackageResolveStage(d, 0, &BaseStageOptions{}) - - gm := NewGitMapping() - - s.SetGitMappings([]*GitMapping{gm}) - - Expect(gm.StagesDependencies).NotTo(BeNil()) - Expect(gm.StagesDependencies[s.Name()]).To(Equal([]string{"/app/go.sum"})) - }) - }) - - DescribeTable("lockfilePath", - func(directive *config.PackagesDirective, expectedPath string) { - s := newPackageResolveStage(directive, 0, &BaseStageOptions{}) - Expect(s.lockfilePath()).To(Equal(expectedPath)) - }, - - Entry("go-mod with workdir /app", - &config.PackagesDirective{ - Type: config.PackagesDirectiveTypeGoMod, - GoMod: config.GoModSpec{Workdir: "/app", Spec: "go.mod", Lock: "go.sum"}, - }, - "/app/go.sum", - ), - Entry("go-mod with custom lock filename", - &config.PackagesDirective{ - Type: config.PackagesDirectiveTypeGoMod, - GoMod: config.GoModSpec{Workdir: "/src", Spec: "go.mod", Lock: "go.sum.custom"}, - }, - "/src/go.sum.custom", - ), - Entry("unknown type returns empty", - &config.PackagesDirective{ - Type: config.PackagesDirectiveType("pip"), - }, - "", - ), - ) - - Describe("NetworkOverrideValue getter", func() { - It("returns empty by default", func() { - s := newPackageResolveStage(&config.PackagesDirective{ - Type: config.PackagesDirectiveTypeGoMod, - GoMod: config.GoModSpec{Workdir: "/app", Spec: "go.mod", Lock: "go.sum"}, - }, 0, &BaseStageOptions{}) - Expect(s.NetworkOverrideValue()).To(Equal("")) - }) - - It("returns value after SetNetworkOverride", func() { - s := NewBaseStage("test", &BaseStageOptions{}) - s.SetNetworkOverride("none") - Expect(s.NetworkOverrideValue()).To(Equal("none")) - }) - }) -}) diff --git a/pkg/build/stage/packages.go b/pkg/build/stage/packages.go new file mode 100644 index 0000000000..361ee00abb --- /dev/null +++ b/pkg/build/stage/packages.go @@ -0,0 +1,52 @@ +package stage + +import ( + "context" + + "github.com/werf/common-go/pkg/util" + "github.com/werf/werf/v2/pkg/build/builder" + "github.com/werf/werf/v2/pkg/config" + "github.com/werf/werf/v2/pkg/container_backend" +) + +type PackagesStage struct { + *UserWithGitPatchStage +} + +func GeneratePackagesStage(ctx context.Context, imageBaseConfig *config.StapelImageBase, gitPatchStageOptions *NewGitPatchStageOptions, baseStageOptions *BaseStageOptions) *PackagesStage { + b := getBuilder(imageBaseConfig, baseStageOptions) + if b != nil && !b.IsPackagesEmpty(ctx) { + return newPackagesStage(b, gitPatchStageOptions, baseStageOptions) + } + + return nil +} + +func newPackagesStage(builder builder.Builder, gitPatchStageOptions *NewGitPatchStageOptions, baseStageOptions *BaseStageOptions) *PackagesStage { + opts := *baseStageOptions + opts.NeedsNetwork = true + s := &PackagesStage{} + s.UserWithGitPatchStage = newUserWithGitPatchStage(builder, Packages, gitPatchStageOptions, &opts) + return s +} + +func (s *PackagesStage) GetDependencies(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevImage, prevBuiltImage *StageImage, buildContextArchive container_backend.BuildContextArchiver) (string, error) { + stageDependenciesChecksum, err := s.getStageDependenciesChecksum(ctx, c, Packages) + if err != nil { + return "", err + } + + return util.Sha256Hash(s.builder.PackagesChecksum(ctx), stageDependenciesChecksum), nil +} + +func (s *PackagesStage) PrepareImage(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage, buildContextArchive container_backend.BuildContextArchiver) error { + if err := s.UserWithGitPatchStage.PrepareImage(ctx, c, cb, prevBuiltImage, stageImage, nil); err != nil { + return err + } + + if err := s.builder.Packages(ctx, cb, stageImage.Builder, c.UseLegacyStapelBuilder(cb)); err != nil { + return err + } + + return nil +} diff --git a/pkg/build/stage/packages_test.go b/pkg/build/stage/packages_test.go new file mode 100644 index 0000000000..e410a80917 --- /dev/null +++ b/pkg/build/stage/packages_test.go @@ -0,0 +1,315 @@ +package stage + +import ( + "context" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/werf/werf/v2/pkg/config" +) + +var _ = Describe("PackagesStage", func() { + Describe("GeneratePackagesStage", func() { + DescribeTable("returns nil when no packages commands", + func(ctx context.Context, imageBaseConfig *config.StapelImageBase) { + stage := GeneratePackagesStage(ctx, imageBaseConfig, testGitPatchOpts(), testBaseOpts()) + Expect(stage).To(BeNil()) + }, + + Entry("nil shell", &config.StapelImageBase{}), + Entry("shell without packages", &config.StapelImageBase{Shell: &config.Shell{Install: []string{"echo hi"}}}), + Entry("shell with empty packages", &config.StapelImageBase{Shell: &config.Shell{Packages: []string{}}}), + ) + + It("returns stage when packages commands present", func(ctx context.Context) { + stage := generateTestPackagesStage(ctx, "cd /app && go mod download") + Expect(stage).NotTo(BeNil()) + }) + }) + + DescribeTable("NeedsNetwork", + func(ctx context.Context, stageName StageName, needsNetwork, expected bool) { + s := NewBaseStage(stageName, &BaseStageOptions{NeedsNetwork: needsNetwork}) + Expect(s.NeedsNetwork()).To(Equal(expected)) + }, + + Entry("packages stage created with NeedsNetwork=true", Packages, true, true), + Entry("install stage has NeedsNetwork=false", Install, false, false), + Entry("setup stage has NeedsNetwork=false", Setup, false, false), + Entry("beforeInstall has NeedsNetwork=false", BeforeInstall, false, false), + ) + + It("Name returns packages", func(ctx context.Context) { + stage := generateTestPackagesStage(ctx, "cd /app && go mod download") + Expect(stage.Name()).To(Equal(Packages)) + }) + + Describe("GetDependencies", func() { + DescribeTable("hash behavior", + func(ctx context.Context, commands1, commands2 []string, shouldEqual bool) { + s1 := generateTestPackagesStageMulti(ctx, commands1) + s2 := generateTestPackagesStageMulti(ctx, commands2) + conveyor := testConveyor() + + d1, err := s1.GetDependencies(ctx, conveyor, nil, nil, nil, nil) + Expect(err).NotTo(HaveOccurred()) + + d2, err := s2.GetDependencies(ctx, conveyor, nil, nil, nil, nil) + Expect(err).NotTo(HaveOccurred()) + + if shouldEqual { + Expect(d1).To(Equal(d2)) + } else { + Expect(d1).NotTo(Equal(d2)) + } + }, + + Entry("same commands produce same hash", + []string{"cd /app && go mod download"}, + []string{"cd /app && go mod download"}, + true), + Entry("different commands produce different hash", + []string{"cd /app && go mod download"}, + []string{"cd /lib && go mod download"}, + false), + Entry("different number of commands produce different hash", + []string{"cd /app && go mod download"}, + []string{"cd /app && go mod download", "cd /lib && go mod download"}, + false), + ) + + It("returns non-empty hash", func(ctx context.Context) { + s := generateTestPackagesStage(ctx, "cd /app && go mod download") + digest, err := s.GetDependencies(ctx, testConveyor(), nil, nil, nil, nil) + Expect(err).NotTo(HaveOccurred()) + Expect(digest).NotTo(BeEmpty()) + }) + }) + + Describe("coexistence with other stages", func() { + It("does not affect install stage creation", func(ctx context.Context) { + imageBaseConfig := &config.StapelImageBase{ + Shell: &config.Shell{ + Packages: []string{"cd /app && go mod download"}, + Install: []string{"go build ./..."}, + }, + } + + packagesStage := GeneratePackagesStage(ctx, imageBaseConfig, testGitPatchOpts(), testBaseOpts()) + installStage := GenerateInstallStage(ctx, imageBaseConfig, testGitPatchOpts(), testBaseOpts()) + + Expect(packagesStage).NotTo(BeNil()) + Expect(installStage).NotTo(BeNil()) + Expect(packagesStage.Name()).NotTo(Equal(installStage.Name())) + }) + }) +}) + +var _ = Describe("NetworkOverride enforcement", func() { + DescribeTable("BaseStage network resolution", + func(imageNetwork, override, expectedEffective string) { + s := NewBaseStage("test", &BaseStageOptions{Network: imageNetwork}) + if override != "" { + s.SetNetworkOverride(override) + } + + effective := s.network + if s.networkOverride != "" { + effective = s.networkOverride + } + Expect(effective).To(Equal(expectedEffective)) + }, + + Entry("override=none takes priority over host", "host", "none", "none"), + Entry("override=none with empty network", "", "none", "none"), + Entry("empty override uses image network", "host", "", "host"), + Entry("both empty stays empty", "", "", ""), + ) +}) + +var _ = Describe("Network enforcement logic", func() { + DescribeTable("stageHasNetworkAccess", + func(stageName StageName, needsNetwork, expected bool) { + s := NewBaseStage(stageName, &BaseStageOptions{NeedsNetwork: needsNetwork}) + Expect(testStageHasNetworkAccess(s)).To(Equal(expected)) + }, + + Entry("From without flag — no network", From, false, false), + Entry("GitArchive without flag — no network", GitArchive, false, false), + Entry("GitCache without flag — no network", GitCache, false, false), + Entry("GitLatestPatch without flag — no network", GitLatestPatch, false, false), + Entry("Install without flag — no network", Install, false, false), + Entry("Setup without flag — no network", Setup, false, false), + Entry("BeforeInstall without flag — no network", BeforeInstall, false, false), + Entry("BeforeSetup without flag — no network", BeforeSetup, false, false), + Entry("Packages with NeedsNetwork=true — has network", Packages, true, true), + Entry("arbitrary stage with flag — has network", StageName("custom"), true, true), + Entry("arbitrary stage without flag — no network", StageName("custom"), false, false), + ) +}) + +var _ = Describe("GeneratePackagesCommands", func() { + DescribeTable("command generation", + func(packages []*config.PackagesDirective, expected []string) { + Expect(config.GeneratePackagesCommands(packages)).To(Equal(expected)) + }, + + Entry("go-mod /app", []*config.PackagesDirective{ + {Type: config.PackagesDirectiveTypeGoMod, GoMod: config.GoModSpec{Workdir: "/app"}}, + }, []string{"cd /app && go mod download"}), + + Entry("go-mod /src/backend", []*config.PackagesDirective{ + {Type: config.PackagesDirectiveTypeGoMod, GoMod: config.GoModSpec{Workdir: "/src/backend"}}, + }, []string{"cd /src/backend && go mod download"}), + + Entry("multiple go-mod entries", []*config.PackagesDirective{ + {Type: config.PackagesDirectiveTypeGoMod, GoMod: config.GoModSpec{Workdir: "/app"}}, + {Type: config.PackagesDirectiveTypeGoMod, GoMod: config.GoModSpec{Workdir: "/lib"}}, + }, []string{"cd /app && go mod download", "cd /lib && go mod download"}), + + Entry("root workdir", []*config.PackagesDirective{ + {Type: config.PackagesDirectiveTypeGoMod, GoMod: config.GoModSpec{Workdir: "/"}}, + }, []string{"cd / && go mod download"}), + + Entry("unsupported type", []*config.PackagesDirective{ + {Type: config.PackagesDirectiveType("unknown")}, + }, ([]string)(nil)), + + Entry("nil packages", []*config.PackagesDirective(nil), ([]string)(nil)), + + Entry("os-pm skipped", []*config.PackagesDirective{ + {Type: config.PackagesDirectiveTypeOSPM, Spec: config.PackagesSpec{Packages: []string{"curl"}}}, + }, ([]string)(nil)), + + Entry("mixed types: only go-mod produces commands", []*config.PackagesDirective{ + {Type: config.PackagesDirectiveTypeOSPM, Spec: config.PackagesSpec{Packages: []string{"curl"}}}, + {Type: config.PackagesDirectiveTypeGoMod, GoMod: config.GoModSpec{Workdir: "/app"}}, + {Type: config.PackagesDirectiveType("pip")}, + }, []string{"cd /app && go mod download"}), + ) +}) + +var _ = Describe("stageDependencies.packages", func() { + DescribeTable("config field", + func(packages, expected []string) { + sd := &config.StageDependencies{Packages: packages} + Expect(sd.Packages).To(Equal(expected)) + }, + + Entry("single path", []string{"go.sum"}, []string{"go.sum"}), + Entry("multiple paths", []string{"go.sum", "go.mod"}, []string{"go.sum", "go.mod"}), + Entry("empty", []string{}, []string{}), + ) + + It("maps to Packages StageName", func() { + sd := &config.StageDependencies{Packages: []string{"go.sum"}} + m := map[StageName][]string{ + Install: sd.Install, + BeforeSetup: sd.BeforeSetup, + Setup: sd.Setup, + Packages: sd.Packages, + } + Expect(m[Packages]).To(Equal([]string{"go.sum"})) + }) +}) + +var _ = Describe("Shell.Packages config field", func() { + DescribeTable("field access", + func(shell *config.Shell, expectedLen int, expectedFirst string) { + Expect(shell.Packages).To(HaveLen(expectedLen)) + if expectedLen > 0 { + Expect(shell.Packages[0]).To(Equal(expectedFirst)) + } + }, + + Entry("populated", &config.Shell{Packages: []string{"cd /app && go mod download"}}, 1, "cd /app && go mod download"), + Entry("empty", &config.Shell{}, 0, ""), + ) + + It("PackagesCacheVersion field", func() { + Expect((&config.Shell{PackagesCacheVersion: "v2"}).PackagesCacheVersion).To(Equal("v2")) + }) +}) + +var _ = Describe("Builder interface Packages methods", func() { + DescribeTable("IsPackagesEmpty", + func(_ context.Context, shell *config.Shell, expected bool) { + Expect(testBuilderIsPackagesEmpty(shell)).To(Equal(expected)) + }, + + Entry("true when no packages", &config.Shell{Install: []string{"echo"}}, true), + Entry("false when packages present", &config.Shell{Packages: []string{"cmd"}}, false), + Entry("true for nil shell", &config.Shell{}, true), + ) + + DescribeTable("PackagesChecksum", + func(_ context.Context, commands1, commands2 []string, shouldEqual bool) { + c1 := testBuilderPackagesChecksum(commands1) + c2 := testBuilderPackagesChecksum(commands2) + if shouldEqual { + Expect(c1).To(Equal(c2)) + } else { + Expect(c1).NotTo(Equal(c2)) + } + }, + + Entry("same commands — same checksum", + []string{"cd /app && go mod download"}, + []string{"cd /app && go mod download"}, + true), + Entry("different commands — different checksum", + []string{"cd /app && go mod download"}, + []string{"cd /lib && go mod download"}, + false), + ) + + It("empty packages returns empty checksum", func() { + Expect(testBuilderPackagesChecksum(nil)).To(BeEmpty()) + }) +}) + +func testBaseOpts() *BaseStageOptions { + return &BaseStageOptions{ImageName: "test", ContainerWerfDir: "/.werf", ImageTmpDir: "/tmp/test"} +} + +func testGitPatchOpts() *NewGitPatchStageOptions { + return &NewGitPatchStageOptions{} +} + +func testConveyor() Conveyor { + return NewConveyorStubForDependencies( + NewGiterminismManagerStub(NewLocalGitRepoStub("abc123"), NewGiterminismInspectorStub()), + nil, + ) +} + +func generateTestPackagesStage(ctx context.Context, command string) *PackagesStage { + return generateTestPackagesStageMulti(ctx, []string{command}) +} + +func generateTestPackagesStageMulti(ctx context.Context, commands []string) *PackagesStage { + imageBaseConfig := &config.StapelImageBase{ + Shell: &config.Shell{Packages: commands}, + } + return GeneratePackagesStage(ctx, imageBaseConfig, testGitPatchOpts(), testBaseOpts()) +} + +func testStageHasNetworkAccess(s Interface) bool { + if nn, ok := s.(interface{ NeedsNetwork() bool }); ok { + return nn.NeedsNetwork() + } + return false +} + +func testBuilderIsPackagesEmpty(shell *config.Shell) bool { + return len(shell.Packages) == 0 +} + +func testBuilderPackagesChecksum(commands []string) string { + if len(commands) == 0 { + return "" + } + return fmt.Sprintf("%x", commands) +} diff --git a/pkg/config/packages_commands.go b/pkg/config/packages_commands.go new file mode 100644 index 0000000000..c118f3aa21 --- /dev/null +++ b/pkg/config/packages_commands.go @@ -0,0 +1,13 @@ +package config + +import "fmt" + +func GeneratePackagesCommands(packages []*PackagesDirective) []string { + var commands []string + for _, pkg := range packages { + if pkg.Type == PackagesDirectiveTypeGoMod { + commands = append(commands, fmt.Sprintf("cd %s && go mod download", pkg.GoMod.Workdir)) + } + } + return commands +} diff --git a/pkg/config/raw_stage_dependencies.go b/pkg/config/raw_stage_dependencies.go index 87e873e9e9..69bda1d1fc 100644 --- a/pkg/config/raw_stage_dependencies.go +++ b/pkg/config/raw_stage_dependencies.go @@ -4,6 +4,7 @@ type rawStageDependencies struct { Install interface{} `yaml:"install,omitempty"` Setup interface{} `yaml:"setup,omitempty"` BeforeSetup interface{} `yaml:"beforeSetup,omitempty"` + Packages interface{} `yaml:"packages,omitempty"` rawGit *rawGit `yaml:"-"` // parent @@ -48,6 +49,12 @@ func (c *rawStageDependencies) toDirective() (stageDependencies *StageDependenci stageDependencies.Setup = setup } + if packages, err := InterfaceToStringArray(c.Packages, c, c.rawGit.rawStapelImage.doc); err != nil { + return nil, err + } else { + stageDependencies.Packages = packages + } + stageDependencies.raw = c if err := c.validateDirective(stageDependencies); err != nil { diff --git a/pkg/config/raw_stapel_image.go b/pkg/config/raw_stapel_image.go index 2ffd02cf09..05080f6963 100644 --- a/pkg/config/raw_stapel_image.go +++ b/pkg/config/raw_stapel_image.go @@ -326,6 +326,16 @@ func (c *rawStapelImage) toStapelImageBaseDirective(giterminismManager gitermini imageBase.Packages = append(imageBase.Packages, pkgDirective) } + if len(imageBase.Packages) > 0 && meta.Build.Sbom != nil && meta.Build.Sbom.Enable { + packagesCommands := GeneratePackagesCommands(imageBase.Packages) + if len(packagesCommands) > 0 { + if imageBase.Shell == nil { + imageBase.Shell = &Shell{} + } + imageBase.Shell.Packages = packagesCommands + } + } + if imageBase.sbom, err = buildImageSbom(meta, c.RawSbom, c.doc); err != nil { return nil, err } diff --git a/pkg/config/shell.go b/pkg/config/shell.go index 973bdf75ca..47e0eea0f4 100644 --- a/pkg/config/shell.go +++ b/pkg/config/shell.go @@ -5,11 +5,13 @@ type Shell struct { Install []string BeforeSetup []string Setup []string + Packages []string CacheVersion string BeforeInstallCacheVersion string InstallCacheVersion string BeforeSetupCacheVersion string SetupCacheVersion string + PackagesCacheVersion string raw *rawShell } diff --git a/pkg/config/stage_dependencies.go b/pkg/config/stage_dependencies.go index 5f04b4f47d..4d8962d2c2 100644 --- a/pkg/config/stage_dependencies.go +++ b/pkg/config/stage_dependencies.go @@ -4,6 +4,7 @@ type StageDependencies struct { Install []string Setup []string BeforeSetup []string + Packages []string raw *rawStageDependencies } @@ -16,6 +17,8 @@ func (c *StageDependencies) validate() error { return newDetailedConfigError("`setup: [PATH, ...]|PATH` should be relative paths!", c.raw, c.raw.rawGit.rawStapelImage.doc) case !allRelativePaths(c.BeforeSetup): return newDetailedConfigError("`beforeSetup: [PATH, ...]|PATH` should be relative paths!", c.raw, c.raw.rawGit.rawStapelImage.doc) + case !allRelativePaths(c.Packages): + return newDetailedConfigError("`packages: [PATH, ...]|PATH` should be relative paths!", c.raw, c.raw.rawGit.rawStapelImage.doc) } return nil