From c88b135b2780416f7ad4cf21bc94a4f3529edd45 Mon Sep 17 00:00:00 2001 From: Alex Guidi Date: Wed, 11 Feb 2026 11:20:15 +0100 Subject: [PATCH] copy: add option to strip sparse manifest lists Add the ability to strip non-copied instances from manifest lists when using CopySpecificImages. This implements the functionality originally proposed in containers/image#1707. When copying a subset of images from a multi-architecture manifest list using CopySpecificImages, the current behavior always produces a sparse manifest list - a manifest list that references platforms that weren't actually copied. While this maintains the original digest, many registries reject sparse manifests with "blob unknown to registry" errors. This commit adds a new SparseManifestListAction option that gives users control over this behavior: - KeepSparseManifestList (default): preserves existing behavior - StripSparseManifestList (new): removes non-copied instances The implementation includes: 1. New SparseManifestListAction enum in image/copy 2. New ListOpDelete operation in image/internal/manifest with support for both OCI1Index and Schema2List formats 3. Index-based deletion (not digest-based) to handle platform variants that share the same digest 4. Stripping logic in copyMultipleImages that activates when StripSparseManifestList is set 5. Comprehensive test coverage for deletion operations Based on original work by @bertbaron and @mtrmac in containers/image#1707, adapted for the container-libs monorepo structure. Relates to #227 Signed-off-by: Alex Guidi --- image/copy/copy.go | 20 ++++++++++ image/copy/multiple.go | 35 ++++++++++++++++++ .../internal/manifest/docker_schema2_list.go | 6 +++ image/internal/manifest/list.go | 4 ++ image/internal/manifest/list_test.go | 37 +++++++++++++++++++ image/internal/manifest/oci_index.go | 6 +++ 6 files changed, 108 insertions(+) diff --git a/image/copy/copy.go b/image/copy/copy.go index cc165ff711..11652ff590 100644 --- a/image/copy/copy.go +++ b/image/copy/copy.go @@ -71,6 +71,22 @@ const ( // specific images from the source reference. type ImageListSelection int +const ( + // KeepSparseManifestList is the default value which, when set in + // Options.SparseManifestListAction, indicates that the manifest is kept + // as is even though some images from the list may be missing. Some + // registries may not support this. + KeepSparseManifestList SparseManifestListAction = iota + + // StripSparseManifestList will strip missing images from the manifest + // list. When images are stripped the digest will differ from the original. + StripSparseManifestList +) + +// SparseManifestListAction is one of KeepSparseManifestList or StripSparseManifestList +// to control the behavior when only a subset of images from a manifest list is copied +type SparseManifestListAction int + // Options allows supplying non-default configuration modifying the behavior of CopyImage. type Options struct { RemoveSignatures bool // Remove any pre-existing signatures. Signers and SignBy… will still add a new signature. @@ -129,6 +145,10 @@ type Options struct { // to not indicate "nondistributable". DownloadForeignLayers bool + // When only a subset of images of a list is copied, this action indicates if the manifest should be kept or stripped. + // See CopySpecificImages. + SparseManifestListAction SparseManifestListAction + // Contains slice of OptionCompressionVariant, where copy will ensure that for each platform // in the manifest list, a variant with the requested compression will exist. // Invalid when copying a non-multi-architecture image. That will probably diff --git a/image/copy/multiple.go b/image/copy/multiple.go index 9ab82f9bb0..ab12323fcb 100644 --- a/image/copy/multiple.go +++ b/image/copy/multiple.go @@ -289,6 +289,41 @@ func (c *copier) copyMultipleImages(ctx context.Context) (copiedManifest []byte, return nil, fmt.Errorf("updating manifest list: %w", err) } + // Remove skipped instances from the manifest list if StripSparseManifestList is enabled + if c.options.ImageListSelection == CopySpecificImages && c.options.SparseManifestListAction == StripSparseManifestList { + // Build a set of digests that were copied + copiedDigests := set.New[digest.Digest]() + for _, instance := range instanceCopyList { + copiedDigests.Add(instance.sourceDigest) + } + + // Find which indices were skipped + var indicesToDelete []int + for i, instanceDigest := range instanceDigests { + if !copiedDigests.Contains(instanceDigest) { + indicesToDelete = append(indicesToDelete, i) + } + } + + // Build delete operations for skipped instances + // Delete from highest to lowest index to avoid shifting + var deleteEdits []internalManifest.ListEdit + for i := len(indicesToDelete) - 1; i >= 0; i-- { + deleteEdits = append(deleteEdits, internalManifest.ListEdit{ + ListOperation: internalManifest.ListOpDelete, + DeleteIndex: indicesToDelete[i], + }) + } + + // Remove skipped instances from the manifest list using EditInstances + if len(deleteEdits) > 0 { + logrus.Debugf("Removing %d instances from manifest list", len(deleteEdits)) + if err := updatedList.EditInstances(deleteEdits, false); err != nil { + return nil, fmt.Errorf("stripping sparse manifest list: %w", err) + } + } + } + // Iterate through supported list types, preferred format first. c.Printf("Writing manifest list to image destination\n") var errs []string diff --git a/image/internal/manifest/docker_schema2_list.go b/image/internal/manifest/docker_schema2_list.go index af4cda5bdb..886a248bb6 100644 --- a/image/internal/manifest/docker_schema2_list.go +++ b/image/internal/manifest/docker_schema2_list.go @@ -132,6 +132,12 @@ func (list *Schema2ListPublic) editInstances(editInstances []ListEdit, cannotMod }, schema2PlatformSpecFromOCIPlatform(*editInstance.AddPlatform), }) + case ListOpDelete: + if editInstance.DeleteIndex < 0 || editInstance.DeleteIndex >= len(list.Manifests) { + return fmt.Errorf("Schema2List.EditInstances: invalid delete index %d (list has %d instances)", editInstance.DeleteIndex, len(list.Manifests)) + } + // Remove the element by appending slices before and after the target index + list.Manifests = append(list.Manifests[:editInstance.DeleteIndex], list.Manifests[editInstance.DeleteIndex+1:]...) default: return fmt.Errorf("internal error: invalid operation: %d", editInstance.ListOperation) } diff --git a/image/internal/manifest/list.go b/image/internal/manifest/list.go index a0d535891e..82b237ff21 100644 --- a/image/internal/manifest/list.go +++ b/image/internal/manifest/list.go @@ -82,6 +82,7 @@ const ( listOpInvalid ListOp = iota ListOpAdd ListOpUpdate + ListOpDelete ) // ListEdit includes the fields which a List's EditInstances() method will modify. @@ -105,6 +106,9 @@ type ListEdit struct { AddPlatform *imgspecv1.Platform AddAnnotations map[string]string AddCompressionAlgorithms []compression.Algorithm + + // If Op = ListOpDelete. Must delete from highest index to lowest to avoid index shifting. + DeleteIndex int } // ListPublicFromBlob parses a list of manifests. diff --git a/image/internal/manifest/list_test.go b/image/internal/manifest/list_test.go index fb758369ed..b7cb1fe3d4 100644 --- a/image/internal/manifest/list_test.go +++ b/image/internal/manifest/list_test.go @@ -159,3 +159,40 @@ func TestChooseInstance(t *testing.T) { } } } + +// TestListDeleteInstances tests the ListOpDelete functionality for both OCI and Docker manifest formats. +func TestListDeleteInstances(t *testing.T) { + manifestFiles := []string{ + "ociv1.image.index.json", + "v2list.manifest.json", + } + + for _, manifestFile := range manifestFiles { + t.Run(manifestFile, func(t *testing.T) { + validManifest, err := os.ReadFile(filepath.Join("testdata", manifestFile)) + require.NoError(t, err) + list, err := ListFromBlob(validManifest, GuessMIMEType(validManifest)) + require.NoError(t, err) + + originalInstances := list.Instances() + require.GreaterOrEqual(t, len(originalInstances), 1) + + // Test successful deletion + err = list.EditInstances([]ListEdit{{ + ListOperation: ListOpDelete, + DeleteIndex: 0, + }}, false) + require.NoError(t, err) + assert.Equal(t, len(originalInstances)-1, len(list.Instances())) + assert.Equal(t, originalInstances[1:], list.Instances()) + + // Test error on invalid index + err = list.EditInstances([]ListEdit{{ + ListOperation: ListOpDelete, + DeleteIndex: 999, + }}, false) + assert.Error(t, err) + assert.Contains(t, err.Error(), "invalid delete index") + }) + } +} diff --git a/image/internal/manifest/oci_index.go b/image/internal/manifest/oci_index.go index 4e4060255a..ef234a40f5 100644 --- a/image/internal/manifest/oci_index.go +++ b/image/internal/manifest/oci_index.go @@ -176,6 +176,12 @@ func (index *OCI1IndexPublic) editInstances(editInstances []ListEdit, cannotModi Platform: editInstance.AddPlatform, Annotations: annotations, }) + case ListOpDelete: + if editInstance.DeleteIndex < 0 || editInstance.DeleteIndex >= len(index.Manifests) { + return fmt.Errorf("OCI1Index.EditInstances: invalid delete index %d (list has %d instances)", editInstance.DeleteIndex, len(index.Manifests)) + } + // Remove the element by appending slices before and after the target index + index.Manifests = append(index.Manifests[:editInstance.DeleteIndex], index.Manifests[editInstance.DeleteIndex+1:]...) default: return fmt.Errorf("internal error: invalid operation: %d", editInstance.ListOperation) }