diff --git a/pkg/build/build.test b/pkg/build/build.test new file mode 100755 index 0000000000..3a9490bf4e Binary files /dev/null and b/pkg/build/build.test differ diff --git a/pkg/build/build_phase.go b/pkg/build/build_phase.go index d227b54598..932c7bd62b 100644 --- a/pkg/build/build_phase.go +++ b/pkg/build/build_phase.go @@ -34,6 +34,7 @@ import ( "github.com/werf/werf/v2/pkg/sbom/externalref" "github.com/werf/werf/v2/pkg/sbom/gomod" sbomImage "github.com/werf/werf/v2/pkg/sbom/image" + osPm "github.com/werf/werf/v2/pkg/sbom/packages/os_pm" "github.com/werf/werf/v2/pkg/sbom/scanner" "github.com/werf/werf/v2/pkg/stapel" "github.com/werf/werf/v2/pkg/storage" @@ -331,9 +332,16 @@ func (phase *BuildPhase) convergeImageSbom(ctx context.Context, name string, ima goModPatcher := gomod.NewBOMPatcher(gitRepo, commit, imageContext) + var hasOsPmPackages bool + if primaryImg.StapelImageConfig != nil && primaryImg.StapelImageConfig.ImageBaseConfig() != nil { + hasOsPmPackages = len(primaryImg.StapelImageConfig.ImageBaseConfig().Packages) > 0 + } + osPmPatcher := osPm.NewBOMPatcher(stageDesc.Info.Name, hasOsPmPackages) + patchers := []BOMPatcherInterface{ externalRefPatcher, goModPatcher, + osPmPatcher, } if err := phase.sbomStep.ConvergeWithMerge(ctx, name, stageDesc, scanner.DefaultSyftScanOptions(), mergeOpts, patchers, primaryImg.TargetPlatform); err != nil { diff --git a/pkg/build/image/stapel.go b/pkg/build/image/stapel.go index 9879351dca..ea197a62ba 100644 --- a/pkg/build/image/stapel.go +++ b/pkg/build/image/stapel.go @@ -113,6 +113,7 @@ func initStages(ctx context.Context, image *Image, metaConfig *config.Meta, stap stages = append(stages, stage.NewGitArchiveStage(gitArchiveStageOptions, baseStageOptions)) } + stages = appendIfExist(ctx, stages, stage.GeneratePackagesInstallStage(ctx, imageBaseConfig, baseStageOptions)) stages = appendIfExist(ctx, stages, stage.GenerateInstallStage(ctx, imageBaseConfig, gitPatchStageOptions, baseStageOptions)) stages = appendIfExist(ctx, stages, stage.GenerateDependenciesAfterInstallStage(imageBaseConfig, baseStageOptions)) stages = appendIfExist(ctx, stages, stage.GenerateBeforeSetupStage(ctx, imageBaseConfig, gitPatchStageOptions, baseStageOptions)) diff --git a/pkg/build/stage/base.go b/pkg/build/stage/base.go index f45b9a0a3f..ef79d9328d 100644 --- a/pkg/build/stage/base.go +++ b/pkg/build/stage/base.go @@ -26,6 +26,7 @@ const ( From StageName = "from" BeforeInstall StageName = "beforeInstall" DependenciesBeforeInstall StageName = "dependenciesBeforeInstall" + PackagesInstall StageName = "packagesInstall" GitArchive StageName = "gitArchive" Install StageName = "install" DependenciesAfterInstall StageName = "dependenciesAfterInstall" @@ -63,6 +64,7 @@ var AllStages = []StageName{ From, BeforeInstall, DependenciesBeforeInstall, + PackagesInstall, GitArchive, Install, DependenciesAfterInstall, diff --git a/pkg/build/stage/packages_install.go b/pkg/build/stage/packages_install.go new file mode 100644 index 0000000000..22202c79a7 --- /dev/null +++ b/pkg/build/stage/packages_install.go @@ -0,0 +1,57 @@ +package stage + +import ( + "context" + "fmt" + "strings" + + "github.com/werf/common-go/pkg/util" + "github.com/werf/werf/v2/pkg/config" + "github.com/werf/werf/v2/pkg/container_backend" +) + +func GeneratePackagesInstallStage(_ context.Context, imageBaseConfig *config.StapelImageBase, baseStageOptions *BaseStageOptions) *PackagesInstallStage { + var resolvedPackages []string + for _, pkg := range imageBaseConfig.Packages { + if pkg.Type != config.PackagesDirectiveTypeOSPM { + continue + } + resolvedPackages = append(resolvedPackages, pkg.Spec.Packages...) + } + + if len(resolvedPackages) == 0 { + return nil + } + + s := &PackagesInstallStage{} + s.resolvedPackages = resolvedPackages + s.BaseStage = NewBaseStage(PackagesInstall, baseStageOptions) + + return s +} + +type PackagesInstallStage struct { + *BaseStage + + resolvedPackages []string +} + +func (s *PackagesInstallStage) GetDependencies(_ context.Context, _ Conveyor, _ container_backend.ContainerBackend, _, _ *StageImage, _ container_backend.BuildContextArchiver) (string, error) { + return util.Sha256Hash(s.resolvedPackages...), nil +} + +func (s *PackagesInstallStage) PrepareImage(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage, _ container_backend.BuildContextArchiver) error { + if err := s.BaseStage.PrepareImage(ctx, c, cb, prevBuiltImage, stageImage, nil); err != nil { + return fmt.Errorf("error preparing base stage: %w", err) + } + + installCmd := "pm install " + strings.Join(s.resolvedPackages, " ") + + if c.UseLegacyStapelBuilder(cb) { + stageImage.Builder.LegacyStapelStageBuilder().BuilderContainer().AddRunCommands(installCmd) + } else { + stageImage.Builder.StapelStageBuilder().AddCommands(installCmd) + } + + return nil +} diff --git a/pkg/build/stage/packages_install_test.go b/pkg/build/stage/packages_install_test.go new file mode 100644 index 0000000000..8aed7d2807 --- /dev/null +++ b/pkg/build/stage/packages_install_test.go @@ -0,0 +1,174 @@ +package stage + +import ( + "context" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/types" + + "github.com/werf/common-go/pkg/util" + "github.com/werf/werf/v2/pkg/config" + "github.com/werf/werf/v2/pkg/container_backend/stage_builder" + imagePkg "github.com/werf/werf/v2/pkg/image" +) + +var _ = Describe("PackagesInstallStage", func() { + DescribeTable("GeneratePackagesInstallStage", testGeneratePackagesInstallStage, + Entry("returns nil when Packages is nil", + &config.StapelImageBase{}, + BeNil(), + ), + + Entry("returns nil when Packages is empty", + &config.StapelImageBase{ + Packages: []*config.PackagesDirective{}, + }, + BeNil(), + ), + + Entry("returns nil when Packages has empty spec", + &config.StapelImageBase{ + Packages: []*config.PackagesDirective{ + { + Type: config.PackagesDirectiveTypeOSPM, + }, + }, + }, + BeNil(), + ), + + Entry("creates stage with single package", + &config.StapelImageBase{ + Packages: []*config.PackagesDirective{ + { + Type: config.PackagesDirectiveTypeOSPM, + Spec: config.PackagesSpec{Packages: []string{"curl"}}, + }, + }, + }, + ConsistOf("curl"), + ), + + Entry("creates stage with multiple packages", + &config.StapelImageBase{ + Packages: []*config.PackagesDirective{ + { + Type: config.PackagesDirectiveTypeOSPM, + Spec: config.PackagesSpec{Packages: []string{"curl", "jq"}}, + }, + }, + }, + ConsistOf("curl", "jq"), + ), + ) + + Describe("GetDependencies", func() { + DescribeTable("should return deterministic hash based on resolved packages", + func(ctx context.Context, packages []string) { + stage := &PackagesInstallStage{ + resolvedPackages: packages, + BaseStage: NewBaseStage(PackagesInstall, &BaseStageOptions{}), + } + digest, err := stage.GetDependencies(ctx, nil, nil, nil, nil, nil) + Expect(err).To(Succeed()) + + expected := util.Sha256Hash(packages...) + Expect(digest).To(Equal(expected)) + }, + + Entry("empty list", []string{}), + Entry("single package", []string{"curl"}), + Entry("multiple packages", []string{"curl", "jq"}), + ) + + It("should be consistent for same packages", func(ctx context.Context) { + stage1 := &PackagesInstallStage{ + resolvedPackages: []string{"curl", "jq"}, + BaseStage: NewBaseStage(PackagesInstall, &BaseStageOptions{}), + } + stage2 := &PackagesInstallStage{ + resolvedPackages: []string{"curl", "jq"}, + BaseStage: NewBaseStage(PackagesInstall, &BaseStageOptions{}), + } + + digest1, err := stage1.GetDependencies(ctx, nil, nil, nil, nil, nil) + Expect(err).To(Succeed()) + + digest2, err := stage2.GetDependencies(ctx, nil, nil, nil, nil, nil) + Expect(err).To(Succeed()) + + Expect(digest1).To(Equal(digest2)) + }) + + It("should produce different hashes for different packages", func(ctx context.Context) { + stage1 := &PackagesInstallStage{ + resolvedPackages: []string{"curl"}, + BaseStage: NewBaseStage(PackagesInstall, &BaseStageOptions{}), + } + stage2 := &PackagesInstallStage{ + resolvedPackages: []string{"jq"}, + BaseStage: NewBaseStage(PackagesInstall, &BaseStageOptions{}), + } + + digest1, err := stage1.GetDependencies(ctx, nil, nil, nil, nil, nil) + Expect(err).To(Succeed()) + + digest2, err := stage2.GetDependencies(ctx, nil, nil, nil, nil, nil) + Expect(err).To(Succeed()) + + Expect(digest1).NotTo(Equal(digest2)) + }) + }) + + Describe("PrepareImage", func() { + const commit = "9d8059842b6fde712c58315ca0ab4713d90761c0" + + It("should prepare image with stapel builder and add commands", func(ctx SpecContext) { + conveyor := &nonLegacyDependenciesConveyorStub{ + ConveyorStub: NewConveyorStubForDependencies( + NewGiterminismManagerStub(NewLocalGitRepoStub(commit), NewGiterminismInspectorStub()), + nil, + ), + } + containerBackend := NewContainerBackendStub() + + stage := &PackagesInstallStage{ + resolvedPackages: []string{"curl", "jq"}, + BaseStage: NewBaseStage(PackagesInstall, &BaseStageOptions{ImageName: "test-image"}), + } + + _, stageBuilder, stageImage := newStageImage(containerBackend) + + err := stage.PrepareImage(ctx, conveyor, containerBackend, nil, stageImage, nil) + Expect(err).To(Succeed()) + + sb := stageBuilder.GetStapelStageBuilderImplementation() + Expect(sb).NotTo(BeNil()) + Expect(sb.Commands).To(ContainElement("pm install curl jq")) + Expect(sb.Labels).To(ContainElement(fmt.Sprintf("%s=%s", imagePkg.WerfProjectRepoCommitLabel, commit))) + }) + }) +}) + +func testGeneratePackagesInstallStage(ctx context.Context, imageBaseConfig *config.StapelImageBase, packagesMatcher types.GomegaMatcher) { + options := &BaseStageOptions{ImageName: "test-image", ProjectName: "test-project"} + stage := GeneratePackagesInstallStage(ctx, imageBaseConfig, options) + + var packages []string + if stage != nil { + packages = stage.resolvedPackages + } + Expect(packages).To(packagesMatcher) +} + +func newStageImage(containerBackend *ContainerBackendStub) (*LegacyImageStub, *stage_builder.StageBuilder, *StageImage) { + img := NewLegacyImageStub() + stageBuilder := stage_builder.NewStageBuilder(containerBackend, "", img) + + return img, stageBuilder, &StageImage{ + Image: img, + Builder: stageBuilder, + } +} diff --git a/pkg/config/packages_directive.go b/pkg/config/packages_directive.go new file mode 100644 index 0000000000..140d234f57 --- /dev/null +++ b/pkg/config/packages_directive.go @@ -0,0 +1,67 @@ +package config + +import ( + "fmt" + "sort" +) + +// PackagesDirectiveType enumerates supported package source types. +type PackagesDirectiveType string + +const ( + PackagesDirectiveTypeOSPM PackagesDirectiveType = "os-pm" +) + +// PackagesSpec stores a list of package names resolved from a packages directive entry. +type PackagesSpec struct { + Packages []string +} + +// PackagesDirective represents a single entry in the image-level packages list. +type PackagesDirective struct { + Type PackagesDirectiveType + Spec PackagesSpec +} + +func (d *PackagesDirective) validate() error { + if d.Type != PackagesDirectiveTypeOSPM { + return fmt.Errorf("unsupported packages type %q", d.Type) + } + + if len(d.Spec.Packages) == 0 { + return fmt.Errorf("packages spec must not be empty for type %q", d.Type) + } + + return nil +} + +// normalizePackages flattens all packages across every directive, deduplicates +// and sorts them, and returns a single directive with the normalized list. +// This is called during config conversion so that the build stage receives +// a ready-to-use package list without needing to re-resolve or deduplicate. +func normalizePackages(packages []*PackagesDirective) []*PackagesDirective { + seen := map[string]bool{} + var all []string + + for _, p := range packages { + for _, name := range p.Spec.Packages { + if !seen[name] { + seen[name] = true + all = append(all, name) + } + } + } + + if len(all) == 0 { + return nil + } + + sort.Strings(all) + + return []*PackagesDirective{ + { + Type: PackagesDirectiveTypeOSPM, + Spec: PackagesSpec{Packages: all}, + }, + } +} diff --git a/pkg/config/raw_packages_directive.go b/pkg/config/raw_packages_directive.go new file mode 100644 index 0000000000..c6ccf355ce --- /dev/null +++ b/pkg/config/raw_packages_directive.go @@ -0,0 +1,76 @@ +package config + +import "fmt" + +type rawPackagesDirective struct { + Type string `yaml:"type,omitempty"` + Spec interface{} `yaml:"spec,omitempty"` + + rawStapelImage *rawStapelImage `yaml:"-"` + + UnsupportedAttributes map[string]interface{} `yaml:",inline"` +} + +func (r *rawPackagesDirective) UnmarshalYAML(unmarshal func(interface{}) error) error { + if parent, ok := parentStack.Peek().(*rawStapelImage); ok { + r.rawStapelImage = parent + } + + parentStack.Push(r) + type plain rawPackagesDirective + err := unmarshal((*plain)(r)) + parentStack.Pop() + if err != nil { + return err + } + + if err := checkOverflow(r.UnsupportedAttributes, nil, r.docForErrors()); err != nil { + return err + } + + if r.Type == "" { + return newDetailedConfigError("the `type` is required for each packages directive entry!", nil, r.docForErrors()) + } + + if r.Spec == nil { + return newDetailedConfigError("the `spec` is required for each packages directive entry!", nil, r.docForErrors()) + } + + return nil +} + +func (r *rawPackagesDirective) docForErrors() *doc { + if r.rawStapelImage != nil { + return r.rawStapelImage.doc + } + return &doc{Content: []byte{}} +} + +func (r *rawPackagesDirective) toDirective() (*PackagesDirective, error) { + d := &PackagesDirective{ + Type: PackagesDirectiveType(r.Type), + } + + switch v := r.Spec.(type) { + case []interface{}: + packages, err := InterfaceToStringArray(v, nil, r.rawStapelImage.doc) + if err != nil { + return nil, err + } + d.Spec.Packages = packages + case string: + return nil, newDetailedConfigError( + "file-based spec is not currently supported; file-based package lists require giterminism support and are tracked as a technical debt", + nil, + r.docForErrors(), + ) + default: + return nil, fmt.Errorf("unsupported packages spec type %T for type %q", r.Spec, r.Type) + } + + if err := d.validate(); err != nil { + return nil, err + } + + return d, nil +} diff --git a/pkg/config/raw_packages_directive_test.go b/pkg/config/raw_packages_directive_test.go new file mode 100644 index 0000000000..b747a74000 --- /dev/null +++ b/pkg/config/raw_packages_directive_test.go @@ -0,0 +1,213 @@ +package config + +import ( + "errors" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gopkg.in/yaml.v2" + + "github.com/werf/common-go/pkg/util" +) + +var _ = Describe("rawPackagesDirective", func() { + var localGitRepo *LocalGitRepoStub + var giterminismManager *GiterminismManagerStub + + BeforeEach(func() { + parentStack = util.NewStack() + localGitRepo = NewLocalGitRepoStub("9d8059842b6fde712c58315ca0ab4713d90761c0") + giterminismManager = NewGiterminismManagerStub(localGitRepo) + }) + + DescribeTable("unmarshal and convert to directive succeed", + func(yamlMap map[string]interface{}, expectedPackages []*PackagesDirective) { + rawYaml, err := yaml.Marshal(yamlMap) + Expect(err).To(Succeed()) + + doc := &doc{Content: rawYaml} + rawStapelImage := &rawStapelImage{doc: doc} + + Expect(yaml.UnmarshalStrict(doc.Content, rawStapelImage)).To(Succeed()) + + meta := &Meta{} + + stapelImage, err := rawStapelImage.toStapelImageDirective(giterminismManager, meta, "image1") + Expect(err).To(Succeed()) + + Expect(stapelImage.Packages).To(HaveLen(len(expectedPackages))) + for i, expected := range expectedPackages { + Expect(stapelImage.Packages[i].Type).To(Equal(expected.Type)) + Expect(stapelImage.Packages[i].Spec.Packages).To(Equal(expected.Spec.Packages)) + } + }, + + Entry("os-pm with inline package list", + map[string]interface{}{ + "image": "image1", + "from": "alpine:latest", + "packages": []map[string]interface{}{ + { + "type": "os-pm", + "spec": []string{"curl", "openssl=3.3.7"}, + }, + }, + }, + []*PackagesDirective{ + { + Type: PackagesDirectiveTypeOSPM, + Spec: PackagesSpec{ + Packages: []string{"curl", "openssl=3.3.7"}, + }, + }, + }, + ), + + Entry("packages section is optional (omitted)", + map[string]interface{}{ + "image": "image1", + "from": "alpine:latest", + }, + []*PackagesDirective{}, + ), + ) + + DescribeTable("unmarshal fails with configError when required fields are missing", + func(yamlMap map[string]interface{}) { + rawYaml, err := yaml.Marshal(yamlMap) + Expect(err).To(Succeed()) + + doc := &doc{Content: rawYaml} + rawStapelImage := &rawStapelImage{doc: doc} + + var errConf *configError + err = yaml.UnmarshalStrict(doc.Content, rawStapelImage) + Expect(errors.As(err, &errConf)).To(BeTrue()) + }, + + Entry("packages entry without type", + map[string]interface{}{ + "image": "image1", + "from": "alpine:latest", + "packages": []map[string]interface{}{ + { + "spec": "packages.txt", + }, + }, + }, + ), + + Entry("packages entry without spec", + map[string]interface{}{ + "image": "image1", + "from": "alpine:latest", + "packages": []map[string]interface{}{ + { + "type": "os-pm", + }, + }, + }, + ), + ) + + DescribeTable("convert to directive fails with configError for invalid content", + func(yamlMap map[string]interface{}) { + rawYaml, err := yaml.Marshal(yamlMap) + Expect(err).To(Succeed()) + + doc := &doc{Content: rawYaml} + rawStapelImage := &rawStapelImage{doc: doc} + + Expect(yaml.UnmarshalStrict(doc.Content, rawStapelImage)).To(Succeed()) + + meta := &Meta{} + + _, err = rawStapelImage.toStapelImageDirective(giterminismManager, meta, "image1") + Expect(err).To(HaveOccurred()) + }, + + Entry("packages entry with file spec", + map[string]interface{}{ + "image": "image1", + "from": "alpine:latest", + "packages": []map[string]interface{}{ + { + "type": "os-pm", + "spec": "packages.txt", + }, + }, + }, + ), + + Entry("packages entry with unsupported type", + map[string]interface{}{ + "image": "image1", + "from": "alpine:latest", + "packages": []map[string]interface{}{ + { + "type": "go-mod", + "spec": "packages.txt", + }, + }, + }, + ), + + Entry("packages entry with empty spec list", + map[string]interface{}{ + "image": "image1", + "from": "alpine:latest", + "packages": []map[string]interface{}{ + { + "type": "os-pm", + "spec": []string{}, + }, + }, + }, + ), + ) +}) + +var _ = Describe("normalizePackages", func() { + It("returns nil for nil input", func() { + result := normalizePackages(nil) + Expect(result).To(BeNil()) + }) + + It("returns nil for empty slice", func() { + result := normalizePackages([]*PackagesDirective{}) + Expect(result).To(BeNil()) + }) + + It("preserves single directive packages", func() { + result := normalizePackages([]*PackagesDirective{ + {Type: PackagesDirectiveTypeOSPM, Spec: PackagesSpec{Packages: []string{"curl", "jq"}}}, + }) + Expect(result).To(HaveLen(1)) + Expect(result[0].Spec.Packages).To(Equal([]string{"curl", "jq"})) + }) + + It("merges multiple directives", func() { + result := normalizePackages([]*PackagesDirective{ + {Type: PackagesDirectiveTypeOSPM, Spec: PackagesSpec{Packages: []string{"curl"}}}, + {Type: PackagesDirectiveTypeOSPM, Spec: PackagesSpec{Packages: []string{"jq"}}}, + }) + Expect(result).To(HaveLen(1)) + Expect(result[0].Spec.Packages).To(ConsistOf("curl", "jq")) + }) + + It("deduplicates across directives", func() { + result := normalizePackages([]*PackagesDirective{ + {Type: PackagesDirectiveTypeOSPM, Spec: PackagesSpec{Packages: []string{"curl", "jq"}}}, + {Type: PackagesDirectiveTypeOSPM, Spec: PackagesSpec{Packages: []string{"curl"}}}, + }) + Expect(result).To(HaveLen(1)) + Expect(result[0].Spec.Packages).To(ConsistOf("curl", "jq")) + }) + + It("sorts packages deterministically", func() { + result := normalizePackages([]*PackagesDirective{ + {Type: PackagesDirectiveTypeOSPM, Spec: PackagesSpec{Packages: []string{"jq", "curl", "brotli"}}}, + }) + Expect(result[0].Spec.Packages).To(Equal([]string{"brotli", "curl", "jq"})) + }) +}) diff --git a/pkg/config/raw_stapel_image.go b/pkg/config/raw_stapel_image.go index 611afdaa1c..c024da4449 100644 --- a/pkg/config/raw_stapel_image.go +++ b/pkg/config/raw_stapel_image.go @@ -8,28 +8,29 @@ import ( ) type rawStapelImage struct { - Images []string `yaml:"-"` - Final *bool `yaml:"final,omitempty"` - Artifact string `yaml:"artifact,omitempty"` - CacheVersion string `yaml:"cacheVersion,omitempty"` - From string `yaml:"from,omitempty"` - FromLatest bool `yaml:"fromLatest,omitempty"` - FromCacheVersion string `yaml:"fromCacheVersion,omitempty"` - FromImage string `yaml:"fromImage,omitempty"` - FromArtifact string `yaml:"fromArtifact,omitempty"` - DisableGitAfterPatch bool `yaml:"disableGitAfterPatch,omitempty"` - RawGit []*rawGit `yaml:"git,omitempty"` - RawShell *rawShell `yaml:"shell,omitempty"` - RawAnsible *rawAnsible `yaml:"ansible,omitempty"` - RawMount []*rawMount `yaml:"mount,omitempty"` - RawDocker *rawDocker `yaml:"docker,omitempty"` - RawImport []*rawImport `yaml:"import,omitempty"` - RawDependencies []*rawDependency `yaml:"dependencies,omitempty"` - Platform []string `yaml:"platform,omitempty"` - Network string `yaml:"network,omitempty"` - RawSbom *rawSbom `yaml:"sbom,omitempty"` - RawSecrets []*rawSecret `yaml:"secrets,omitempty"` - RawImageSpec *rawImageSpec `yaml:"imageSpec,omitempty"` + Images []string `yaml:"-"` + Final *bool `yaml:"final,omitempty"` + Artifact string `yaml:"artifact,omitempty"` + CacheVersion string `yaml:"cacheVersion,omitempty"` + From string `yaml:"from,omitempty"` + FromLatest bool `yaml:"fromLatest,omitempty"` + FromCacheVersion string `yaml:"fromCacheVersion,omitempty"` + FromImage string `yaml:"fromImage,omitempty"` + FromArtifact string `yaml:"fromArtifact,omitempty"` + DisableGitAfterPatch bool `yaml:"disableGitAfterPatch,omitempty"` + RawGit []*rawGit `yaml:"git,omitempty"` + RawShell *rawShell `yaml:"shell,omitempty"` + RawAnsible *rawAnsible `yaml:"ansible,omitempty"` + RawMount []*rawMount `yaml:"mount,omitempty"` + RawDocker *rawDocker `yaml:"docker,omitempty"` + RawImport []*rawImport `yaml:"import,omitempty"` + RawDependencies []*rawDependency `yaml:"dependencies,omitempty"` + Platform []string `yaml:"platform,omitempty"` + Network string `yaml:"network,omitempty"` + RawSbom *rawSbom `yaml:"sbom,omitempty"` + RawSecrets []*rawSecret `yaml:"secrets,omitempty"` + RawImageSpec *rawImageSpec `yaml:"imageSpec,omitempty"` + RawPackages []*rawPackagesDirective `yaml:"packages,omitempty"` doc *doc `yaml:"-"` // parent @@ -316,6 +317,19 @@ func (c *rawStapelImage) toStapelImageBaseDirective(giterminismManager gitermini imageBase.ImageSpec = c.RawImageSpec.toDirective() } + for _, rawPkg := range c.RawPackages { + pkgDirective, err := rawPkg.toDirective() + if err != nil { + return nil, err + } + + imageBase.Packages = append(imageBase.Packages, pkgDirective) + } + + if len(imageBase.Packages) > 0 { + imageBase.Packages = normalizePackages(imageBase.Packages) + } + if imageBase.sbom, err = buildImageSbom(meta, c.RawSbom, c.doc); err != nil { return nil, err } diff --git a/pkg/config/stapel_image_base.go b/pkg/config/stapel_image_base.go index 8498ef5089..c2fcc577fb 100644 --- a/pkg/config/stapel_image_base.go +++ b/pkg/config/stapel_image_base.go @@ -24,6 +24,7 @@ type StapelImageBase struct { Secrets []Secret ImageSpec *ImageSpec Network string + Packages []*PackagesDirective FromExternal bool cacheVersion string diff --git a/pkg/sbom/packages/os_pm/os_pm.go b/pkg/sbom/packages/os_pm/os_pm.go new file mode 100644 index 0000000000..fbcfca1d74 --- /dev/null +++ b/pkg/sbom/packages/os_pm/os_pm.go @@ -0,0 +1,80 @@ +package os_pm + +import ( + "encoding/json" + "fmt" + "sort" + + cdx "github.com/CycloneDX/cyclonedx-go" +) + +type PmPackageInfo struct { + Name string `json:"name"` + Arch []string `json:"arch"` + Default bool `json:"default"` + Description string `json:"description"` + License string `json:"license"` + OriginalRepo string `json:"originalRepo"` + Repo string `json:"repo"` + Type string `json:"type"` + Version string `json:"version"` + Digest string `json:"digest"` + Depends []string `json:"depends,omitempty"` +} + +func ParsePmInstalledJSON(data []byte) (map[string]PmPackageInfo, error) { + var result map[string]PmPackageInfo + if err := json.Unmarshal(data, &result); err != nil { + return nil, fmt.Errorf("parse pm info: %w", err) + } + + if len(result) == 0 { + return nil, fmt.Errorf("parse pm info: empty package list") + } + + return result, nil +} + +func ConvertToCycloneDX(pkgs map[string]PmPackageInfo) *cdx.BOM { + if len(pkgs) == 0 { + return nil + } + + components := make([]cdx.Component, 0, len(pkgs)) + + keys := make([]string, 0, len(pkgs)) + for k := range pkgs { + keys = append(keys, k) + } + sort.Strings(keys) + + for _, key := range keys { + pkg := pkgs[key] + comp := cdx.Component{ + Name: pkg.Name, + Version: pkg.Version, + Type: cdx.ComponentTypeLibrary, + } + + if pkg.License != "" { + license := cdx.LicenseChoice{ + License: &cdx.License{ID: pkg.License}, + } + comp.Licenses = &cdx.Licenses{license} + } + + purl := fmt.Sprintf("pkg:generic/%s@%s", pkg.Name, pkg.Version) + if pkg.Repo != "" { + purl = fmt.Sprintf("pkg:generic/%s@%s?repository_url=%s", pkg.Name, pkg.Version, pkg.Repo) + } + comp.PackageURL = purl + + components = append(components, comp) + } + + return &cdx.BOM{ + BOMFormat: "CycloneDX", + SpecVersion: cdx.SpecVersion1_6, + Components: &components, + } +} diff --git a/pkg/sbom/packages/os_pm/os_pm_test.go b/pkg/sbom/packages/os_pm/os_pm_test.go new file mode 100644 index 0000000000..f4ccf5fccd --- /dev/null +++ b/pkg/sbom/packages/os_pm/os_pm_test.go @@ -0,0 +1,229 @@ +package os_pm + +import ( + cdx "github.com/CycloneDX/cyclonedx-go" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var examplePmInstalledJSON = []byte(`{ + "brotli": { + "name": "brotli", + "arch": ["linux/amd64"], + "default": true, + "description": "Generic lossless compressor", + "license": "MIT", + "originalRepo": "https://github.com/google/brotli", + "repo": "google/brotli", + "type": "runtime", + "version": "1.1.0", + "digest": "sha256:82dcd7127798a506c1ab00993dffaf4ddd2bf576fba97a5d1cc45b931bcd2a0f" + }, + "curl": { + "name": "curl", + "arch": ["linux/amd64"], + "default": true, + "depends": ["brotli", "libpsl"], + "description": "URL retrival utility and library", + "license": "curl", + "originalRepo": "https://github.com/curl/curl", + "repo": "curl/curl", + "type": "runtime", + "version": "8.12.1", + "digest": "sha256:6f2108c511daa7c46ace9879c0d9bbef2573fb5fd88bee5fad745d96ceda081d" + }, + "jq": { + "name": "jq", + "arch": ["linux/amd64"], + "default": true, + "description": "A lightweight and flexible command-line JSON processor", + "license": "MIT", + "originalRepo": "https://jqlang.github.io/jq/", + "repo": "jqlang/jq", + "type": "runtime", + "version": "1.8.1", + "digest": "sha256:4b36dcf53c35b50e0afbc445232713aff15f788a61b832cd720bf9e88fc9fba8" + }, + "libidn2": { + "name": "libidn2", + "arch": ["linux/amd64"], + "default": true, + "depends": ["libunistring"], + "description": "Encode/Decode library for internationalized domain names", + "license": "BSD-3-Clause", + "originalRepo": "https://gitlab.com/libidn/libidn2", + "repo": "libidn/libidn2", + "type": "runtime", + "version": "2.3.8", + "digest": "sha256:71efcb507c12a77b262038c18043695ee65d27f79f2d6dba052bb0a9e59589e5" + }, + "libpsl": { + "name": "libpsl", + "arch": ["linux/amd64"], + "default": true, + "depends": ["libidn2", "libunistring"], + "description": "C library for the Publix Suffix List.", + "license": "MIT", + "originalRepo": "https://github.com/rockdaboot/libpsl", + "repo": "rockdaboot/libpsl", + "type": "runtime", + "version": "0.21.5", + "digest": "sha256:9cec4175f81c57c445b8f4904dfecc3c98e35416e8bbcabd3206537b7055687e" + }, + "libunistring": { + "name": "libunistring", + "arch": ["linux/amd64"], + "default": true, + "description": "Library for manipulating Unicode strings and C strings.", + "license": "LGPL-3.0-or-later", + "originalRepo": "https://git.savannah.gnu.org/git/libunistring.git", + "repo": "git/libunistring", + "type": "runtime", + "version": "1.4.1", + "digest": "sha256:2d2a7d27c1c23f4b169c58bcf0104509a28c3bd73d8293969f067fa4820fb79b" + } +}`) + +var _ = Describe("ParsePmInstalledJSON", func() { + It("should parse valid pm info JSON", func() { + pkgs, err := ParsePmInstalledJSON(examplePmInstalledJSON) + Expect(err).To(Succeed()) + Expect(pkgs).To(HaveLen(6)) + }) + + It("should parse curl package fields correctly", func() { + pkgs, err := ParsePmInstalledJSON(examplePmInstalledJSON) + Expect(err).To(Succeed()) + + curl, ok := pkgs["curl"] + Expect(ok).To(BeTrue()) + Expect(curl.Name).To(Equal("curl")) + Expect(curl.Version).To(Equal("8.12.1")) + Expect(curl.License).To(Equal("curl")) + Expect(curl.Digest).To(Equal("sha256:6f2108c511daa7c46ace9879c0d9bbef2573fb5fd88bee5fad745d96ceda081d")) + Expect(curl.Depends).To(ConsistOf("brotli", "libpsl")) + }) + + It("should parse jq package fields correctly", func() { + pkgs, err := ParsePmInstalledJSON(examplePmInstalledJSON) + Expect(err).To(Succeed()) + + jq, ok := pkgs["jq"] + Expect(ok).To(BeTrue()) + Expect(jq.Name).To(Equal("jq")) + Expect(jq.Version).To(Equal("1.8.1")) + Expect(jq.License).To(Equal("MIT")) + Expect(jq.Digest).To(Equal("sha256:4b36dcf53c35b50e0afbc445232713aff15f788a61b832cd720bf9e88fc9fba8")) + Expect(jq.Depends).To(BeEmpty()) + }) + + It("should parse transitive dependency fields", func() { + pkgs, err := ParsePmInstalledJSON(examplePmInstalledJSON) + Expect(err).To(Succeed()) + + libpsl, ok := pkgs["libpsl"] + Expect(ok).To(BeTrue()) + Expect(libpsl.Version).To(Equal("0.21.5")) + Expect(libpsl.License).To(Equal("MIT")) + Expect(libpsl.Depends).To(ConsistOf("libidn2", "libunistring")) + }) + + It("should parse package without dependencies", func() { + pkgs, err := ParsePmInstalledJSON(examplePmInstalledJSON) + Expect(err).To(Succeed()) + + brotli, ok := pkgs["brotli"] + Expect(ok).To(BeTrue()) + Expect(brotli.Version).To(Equal("1.1.0")) + Expect(brotli.License).To(Equal("MIT")) + Expect(brotli.Depends).To(BeEmpty()) + }) + + It("should return error for invalid JSON", func() { + _, err := ParsePmInstalledJSON([]byte(`{invalid}`)) + Expect(err).To(HaveOccurred()) + }) + + It("should return error for empty JSON", func() { + _, err := ParsePmInstalledJSON([]byte(`{}`)) + Expect(err).To(HaveOccurred()) + }) +}) + +var _ = Describe("ConvertToCycloneDX", func() { + It("should generate valid CycloneDX BOM with correct component count", func() { + pkgs, err := ParsePmInstalledJSON(examplePmInstalledJSON) + Expect(err).To(Succeed()) + + bom := ConvertToCycloneDX(pkgs) + Expect(bom).ToNot(BeNil()) + Expect(*bom.Components).To(HaveLen(6)) + }) + + It("should set component name and version from package info", func() { + pkgs, err := ParsePmInstalledJSON(examplePmInstalledJSON) + Expect(err).To(Succeed()) + + bom := ConvertToCycloneDX(pkgs) + Expect(*bom.Components).To(ContainElement(HaveField("Name", "curl"))) + Expect(*bom.Components).To(ContainElement(HaveField("Version", "8.12.1"))) + }) + + It("should set component type to Library", func() { + pkgs, err := ParsePmInstalledJSON(examplePmInstalledJSON) + Expect(err).To(Succeed()) + + bom := ConvertToCycloneDX(pkgs) + for _, comp := range *bom.Components { + Expect(comp.Type).To(Equal(cdx.ComponentTypeLibrary)) + } + }) + + It("should set licenses from package info", func() { + pkgs, err := ParsePmInstalledJSON(examplePmInstalledJSON) + Expect(err).To(Succeed()) + + bom := ConvertToCycloneDX(pkgs) + + var mitComponents int + for _, comp := range *bom.Components { + if comp.Licenses != nil { + for _, l := range *comp.Licenses { + if l.License != nil && l.License.ID == "MIT" { + mitComponents++ + } + } + } + } + Expect(mitComponents).To(BeNumerically(">=", 1)) + }) + + It("should set PURL for each component", func() { + pkgs, err := ParsePmInstalledJSON(examplePmInstalledJSON) + Expect(err).To(Succeed()) + + bom := ConvertToCycloneDX(pkgs) + for _, comp := range *bom.Components { + Expect(comp.PackageURL).ToNot(BeEmpty(), "component %s should have PURL", comp.Name) + } + }) + + It("should return nil for empty input", func() { + bom := ConvertToCycloneDX(map[string]PmPackageInfo{}) + Expect(bom).To(BeNil()) + }) + + It("should handle packages with SPDX license IDs correctly", func() { + pkgs, err := ParsePmInstalledJSON(examplePmInstalledJSON) + Expect(err).To(Succeed()) + + bom := ConvertToCycloneDX(pkgs) + + for _, comp := range *bom.Components { + if comp.Name == "libunistring" { + Expect(comp.Licenses).ToNot(BeNil()) + Expect((*comp.Licenses)[0].License.ID).To(Equal("LGPL-3.0-or-later")) + } + } + }) +}) diff --git a/pkg/sbom/packages/os_pm/patcher.go b/pkg/sbom/packages/os_pm/patcher.go new file mode 100644 index 0000000000..831a5494c2 --- /dev/null +++ b/pkg/sbom/packages/os_pm/patcher.go @@ -0,0 +1,59 @@ +package os_pm + +import ( + "bytes" + "context" + "fmt" + "strings" + + cdx "github.com/CycloneDX/cyclonedx-go" + + werfExec "github.com/werf/werf/v2/pkg/werf/exec" +) + +type BOMPatcher struct { + imageRef string + hasPackages bool +} + +func NewBOMPatcher(imageRef string, hasPackages bool) *BOMPatcher { + return &BOMPatcher{ + imageRef: imageRef, + hasPackages: hasPackages, + } +} + +func (p *BOMPatcher) Apply(ctx context.Context, bom *cdx.BOM) (*cdx.BOM, error) { + if !p.hasPackages || p.imageRef == "" { + return bom, nil + } + + cmd := werfExec.CommandContextCancellation(ctx, "docker", "run", "--rm", "--entrypoint", "", p.imageRef, "pm", "info", "--installed", "--json") + + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + werfExec.TerminateIfCanceled(ctx) + return nil, fmt.Errorf("run pm info in image %q: %w (stderr: %s)", p.imageRef, err, strings.TrimSpace(stderr.String())) + } + + pkgs, err := ParsePmInstalledJSON(stdout.Bytes()) + if err != nil { + return nil, fmt.Errorf("parse pm info from image %q: %w", p.imageRef, err) + } + + pmBOM := ConvertToCycloneDX(pkgs) + if pmBOM == nil { + return bom, nil + } + + if bom.Components == nil { + bom.Components = pmBOM.Components + } else { + *bom.Components = append(*bom.Components, *pmBOM.Components...) + } + + return bom, nil +} diff --git a/pkg/sbom/packages/os_pm/patcher_test.go b/pkg/sbom/packages/os_pm/patcher_test.go new file mode 100644 index 0000000000..326b5036e9 --- /dev/null +++ b/pkg/sbom/packages/os_pm/patcher_test.go @@ -0,0 +1,36 @@ +package os_pm + +import ( + cdx "github.com/CycloneDX/cyclonedx-go" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("BOMPatcher", func() { + It("should return BOM unchanged when no packages configured", func(ctx SpecContext) { + patcher := NewBOMPatcher("test-image:latest", false) + originalBOM := &cdx.BOM{ + BOMFormat: "CycloneDX", + SpecVersion: cdx.SpecVersion1_6, + Components: &[]cdx.Component{ + {Name: "existing-component", Version: "1.0.0"}, + }, + } + + result, err := patcher.Apply(ctx, originalBOM) + Expect(err).To(Succeed()) + Expect(result).To(Equal(originalBOM)) + }) + + It("should return BOM unchanged when imageRef is empty", func(ctx SpecContext) { + patcher := NewBOMPatcher("", true) + originalBOM := &cdx.BOM{ + BOMFormat: "CycloneDX", + SpecVersion: cdx.SpecVersion1_6, + } + + result, err := patcher.Apply(ctx, originalBOM) + Expect(err).To(Succeed()) + Expect(result).To(Equal(originalBOM)) + }) +}) diff --git a/pkg/sbom/packages/os_pm/suite_test.go b/pkg/sbom/packages/os_pm/suite_test.go new file mode 100644 index 0000000000..95f358a282 --- /dev/null +++ b/pkg/sbom/packages/os_pm/suite_test.go @@ -0,0 +1,13 @@ +package os_pm + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestSbomPackagesOsPm(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Sbom Packages OsPm Suite") +}