From 21e2b75b0b07d2ae47270ee72733b45f58e76cce Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Fri, 13 Mar 2026 13:18:20 +0100 Subject: [PATCH] buildx(imagetools): make manifest retries configurable Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- __tests__/buildx/imagetools.test.itg.ts | 20 ++-- __tests__/buildx/imagetools.test.ts | 125 +++++++++++++++++++++++- src/buildx/imagetools.ts | 81 +++++++++------ src/sigstore/sigstore.ts | 15 ++- src/types/buildx/imagetools.ts | 12 ++- src/types/sigstore/sigstore.ts | 3 + 6 files changed, 213 insertions(+), 43 deletions(-) diff --git a/__tests__/buildx/imagetools.test.itg.ts b/__tests__/buildx/imagetools.test.itg.ts index 4aaa48da..c2931aee 100644 --- a/__tests__/buildx/imagetools.test.itg.ts +++ b/__tests__/buildx/imagetools.test.itg.ts @@ -30,12 +30,12 @@ const maybe = !process.env.GITHUB_ACTIONS || (process.env.GITHUB_ACTIONS === 'tr maybe('inspectImage', () => { it('inspect single platform', async () => { - const image = await new ImageTools().inspectImage('moby/buildkit:latest@sha256:5769c54b98840147b74128f38fb0b0a049e24b11a75bd81664131edd2854593f'); + const image = await new ImageTools().inspectImage({name: 'moby/buildkit:latest@sha256:5769c54b98840147b74128f38fb0b0a049e24b11a75bd81664131edd2854593f'}); const expectedImage = JSON.parse(fs.readFileSync(path.join(fixturesDir, 'imagetools-01.json'), {encoding: 'utf-8'}).trim()); expect(image).toEqual(expectedImage); }); it('inspect multi platform', async () => { - const image = await new ImageTools().inspectImage('moby/buildkit:latest@sha256:86c0ad9d1137c186e9d455912167df20e530bdf7f7c19de802e892bb8ca16552'); + const image = await new ImageTools().inspectImage({name: 'moby/buildkit:latest@sha256:86c0ad9d1137c186e9d455912167df20e530bdf7f7c19de802e892bb8ca16552'}); const expectedImage = >JSON.parse(fs.readFileSync(path.join(fixturesDir, 'imagetools-02.json'), {encoding: 'utf-8'}).trim()); expect(image).toEqual(expectedImage); }); @@ -43,12 +43,12 @@ maybe('inspectImage', () => { maybe('inspectManifest', () => { it('inspect descriptor', async () => { - const manifest = await new ImageTools().inspectManifest('moby/buildkit:latest@sha256:dccc69dd895968c4f21aa9e43e715f25f0cedfce4b17f1014c88c307928e22fc'); + const manifest = await new ImageTools().inspectManifest({name: 'moby/buildkit:latest@sha256:dccc69dd895968c4f21aa9e43e715f25f0cedfce4b17f1014c88c307928e22fc'}); const expectedManifest = JSON.parse(fs.readFileSync(path.join(fixturesDir, 'imagetools-03.json'), {encoding: 'utf-8'}).trim()); expect(manifest).toEqual(expectedManifest); }); it('inspect index', async () => { - const manifest = await new ImageTools().inspectManifest('moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6'); + const manifest = await new ImageTools().inspectManifest({name: 'moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6'}); const expectedManifest = JSON.parse(fs.readFileSync(path.join(fixturesDir, 'imagetools-04.json'), {encoding: 'utf-8'}).trim()); expect(manifest).toEqual(expectedManifest); }); @@ -56,17 +56,17 @@ maybe('inspectManifest', () => { maybe('attestationDescriptors', () => { it('returns buildkit attestations descriptors', async () => { - const attestations = await new ImageTools().attestationDescriptors('moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6'); + const attestations = await new ImageTools().attestationDescriptors({name: 'moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6'}); const expectedAttestations = >JSON.parse(fs.readFileSync(path.join(fixturesDir, 'imagetools-05.json'), {encoding: 'utf-8'}).trim()); expect(attestations).toEqual(expectedAttestations); }); it('returns buildkit attestations descriptors for linux/amd64', async () => { - const attestations = await new ImageTools().attestationDescriptors('moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6', {os: 'linux', architecture: 'amd64'}); + const attestations = await new ImageTools().attestationDescriptors({name: 'moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6', platform: {os: 'linux', architecture: 'amd64'}}); const expectedAttestations = >JSON.parse(fs.readFileSync(path.join(fixturesDir, 'imagetools-06.json'), {encoding: 'utf-8'}).trim()); expect(attestations).toEqual(expectedAttestations); }); it('returns buildkit attestations descriptors for linux/arm/v7', async () => { - const attestations = await new ImageTools().attestationDescriptors('moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6', {os: 'linux', architecture: 'arm', variant: 'v7'}); + const attestations = await new ImageTools().attestationDescriptors({name: 'moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6', platform: {os: 'linux', architecture: 'arm', variant: 'v7'}}); const expectedAttestations = >JSON.parse(fs.readFileSync(path.join(fixturesDir, 'imagetools-07.json'), {encoding: 'utf-8'}).trim()); expect(attestations).toEqual(expectedAttestations); }); @@ -74,7 +74,7 @@ maybe('attestationDescriptors', () => { maybe('attestationDigests', () => { it('returns buildkit attestations digests', async () => { - const digests = await new ImageTools().attestationDigests('moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6'); + const digests = await new ImageTools().attestationDigests({name: 'moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6'}); // prettier-ignore expect(digests).toEqual([ 'sha256:2ba4ad6eae1efcafee73a971953093c7c32b6938f2f9fd4998c8bf4d0fbe76f2', @@ -86,11 +86,11 @@ maybe('attestationDigests', () => { ]); }); it('returns buildkit attestations digests for linux/amd64', async () => { - const digests = await new ImageTools().attestationDigests('moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6', {os: 'linux', architecture: 'amd64'}); + const digests = await new ImageTools().attestationDigests({name: 'moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6', platform: {os: 'linux', architecture: 'amd64'}}); expect(digests).toEqual(['sha256:2ba4ad6eae1efcafee73a971953093c7c32b6938f2f9fd4998c8bf4d0fbe76f2']); }); it('returns buildkit attestations digests for linux/arm/v7', async () => { - const digests = await new ImageTools().attestationDigests('moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6', {os: 'linux', architecture: 'arm', variant: 'v7'}); + const digests = await new ImageTools().attestationDigests({name: 'moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6', platform: {os: 'linux', architecture: 'arm', variant: 'v7'}}); expect(digests).toEqual(['sha256:0709528fae1747ce17638ad2978ee7936b38a294136eaadaf692e415f64b1e03']); }); }); diff --git a/__tests__/buildx/imagetools.test.ts b/__tests__/buildx/imagetools.test.ts index 9fa08faf..4898e906 100644 --- a/__tests__/buildx/imagetools.test.ts +++ b/__tests__/buildx/imagetools.test.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import {afterEach, describe, expect, it, vi} from 'vitest'; +import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest'; import fs from 'fs'; import os from 'os'; import path from 'path'; @@ -38,10 +38,133 @@ vi.spyOn(Context, 'tmpName').mockImplementation((): string => { }); afterEach(() => { + vi.useRealTimers(); vi.clearAllMocks(); rimraf.sync(tmpDir); }); +beforeEach(() => { + vi.useRealTimers(); + fs.mkdirSync(tmpDir, {recursive: true}); +}); + +describe('inspectManifest', () => { + it('retries transient manifest unknown errors when requested', async () => { + vi.useFakeTimers(); + + const getCommand = vi.fn().mockResolvedValue({ + command: 'docker', + args: ['buildx', 'imagetools', 'inspect'] + }); + const buildx = {getCommand} as unknown as Buildx; + const execSpy = vi + .spyOn(Exec, 'getExecOutput') + .mockResolvedValueOnce({ + exitCode: 1, + stdout: '', + stderr: 'ERROR: MANIFEST_UNKNOWN: manifest unknown' + }) + .mockResolvedValueOnce({ + exitCode: 0, + stdout: JSON.stringify({ + schemaVersion: 2, + mediaType: 'application/vnd.oci.image.index.v1+json', + manifests: [] + }), + stderr: '' + }); + + const inspectPromise = new ImageTools({buildx}).inspectManifest({ + name: 'docker.io/library/alpine:latest', + retryOnManifestUnknown: true, + retryLimit: 2 + }); + + await vi.runAllTimersAsync(); + + expect(await inspectPromise).toEqual({ + schemaVersion: 2, + mediaType: 'application/vnd.oci.image.index.v1+json', + manifests: [] + }); + expect(getCommand).toHaveBeenCalledWith(['imagetools', 'inspect', 'docker.io/library/alpine:latest', '--format', '{{json .Manifest}}']); + expect(execSpy).toHaveBeenCalledTimes(2); + }); + + it('does not retry non-manifest errors', async () => { + const getCommand = vi.fn().mockResolvedValue({ + command: 'docker', + args: ['buildx', 'imagetools', 'inspect'] + }); + const buildx = {getCommand} as unknown as Buildx; + const execSpy = vi.spyOn(Exec, 'getExecOutput').mockResolvedValue({ + exitCode: 1, + stdout: '', + stderr: 'ERROR: unauthorized' + }); + + const result = await new ImageTools({buildx}) + .inspectManifest({ + name: 'docker.io/library/alpine:latest', + retryOnManifestUnknown: true + }) + .then( + value => ({value, error: undefined}), + error => ({value: undefined, error: error as Error}) + ); + + expect(result.value).toBeUndefined(); + expect(result.error).toBeInstanceOf(Error); + expect(result.error?.message).toContain('ERROR: unauthorized'); + + expect(execSpy).toHaveBeenCalledTimes(1); + }); +}); + +describe('inspectImage', () => { + it('retries transient manifest unknown errors when requested', async () => { + vi.useFakeTimers(); + + const getCommand = vi.fn().mockResolvedValue({ + command: 'docker', + args: ['buildx', 'imagetools', 'inspect'] + }); + const buildx = {getCommand} as unknown as Buildx; + const execSpy = vi + .spyOn(Exec, 'getExecOutput') + .mockResolvedValueOnce({ + exitCode: 1, + stdout: '', + stderr: 'ERROR: MANIFEST_UNKNOWN: manifest unknown' + }) + .mockResolvedValueOnce({ + exitCode: 0, + stdout: JSON.stringify({ + config: { + digest: 'sha256:test' + } + }), + stderr: '' + }); + + const inspectPromise = new ImageTools({buildx}).inspectImage({ + name: 'docker.io/library/alpine:latest', + retryOnManifestUnknown: true, + retryLimit: 2 + }); + + await vi.runAllTimersAsync(); + + expect(await inspectPromise).toEqual({ + config: { + digest: 'sha256:test' + } + }); + expect(getCommand).toHaveBeenCalledWith(['imagetools', 'inspect', 'docker.io/library/alpine:latest', '--format', '{{json .Image}}']); + expect(execSpy).toHaveBeenCalledTimes(2); + }); +}); + describe('create', () => { it('parses metadata and supports cwd sources', async () => { const getCommand = vi.fn().mockResolvedValue({ diff --git a/src/buildx/imagetools.ts b/src/buildx/imagetools.ts index 319ca319..ab1d677f 100644 --- a/src/buildx/imagetools.ts +++ b/src/buildx/imagetools.ts @@ -21,9 +21,9 @@ import {Buildx} from './buildx.js'; import {Context} from '../context.js'; import {Exec} from '../exec.js'; -import {CreateOpts, CreateResponse, CreateResult, Manifest as ImageToolsManifest} from '../types/buildx/imagetools.js'; +import {AttestationInspectOpts, CreateOpts, CreateResponse, CreateResult, InspectOpts, Manifest as ImageToolsManifest} from '../types/buildx/imagetools.js'; import {Image} from '../types/oci/config.js'; -import {Descriptor, Platform} from '../types/oci/descriptor.js'; +import {Descriptor} from '../types/oci/descriptor.js'; import {Digest} from '../types/oci/digest.js'; export interface ImageToolsOpts { @@ -49,16 +49,8 @@ export class ImageTools { return await this.getCommand(['create', ...args]); } - public async inspectImage(name: string): Promise | Image> { - const cmd = await this.getInspectCommand([name, '--format', '{{json .Image}}']); - return await Exec.getExecOutput(cmd.command, cmd.args, { - ignoreReturnCode: true, - silent: true - }).then(res => { - if (res.stderr.length > 0 && res.exitCode != 0) { - throw new Error(res.stderr.trim()); - } - const parsedOutput = JSON.parse(res.stdout); + public async inspectImage(opts: InspectOpts): Promise | Image> { + return await this.inspect(opts, '{{json .Image}}', parsedOutput => { if (typeof parsedOutput === 'object' && !Array.isArray(parsedOutput) && parsedOutput !== null) { if (Object.prototype.hasOwnProperty.call(parsedOutput, 'config')) { return parsedOutput; @@ -70,16 +62,8 @@ export class ImageTools { }); } - public async inspectManifest(name: string): Promise { - const cmd = await this.getInspectCommand([name, '--format', '{{json .Manifest}}']); - return await Exec.getExecOutput(cmd.command, cmd.args, { - ignoreReturnCode: true, - silent: true - }).then(res => { - if (res.stderr.length > 0 && res.exitCode != 0) { - throw new Error(res.stderr.trim()); - } - const parsedOutput = JSON.parse(res.stdout); + public async inspectManifest(opts: InspectOpts): Promise { + return await this.inspect(opts, '{{json .Manifest}}', parsedOutput => { if (typeof parsedOutput === 'object' && !Array.isArray(parsedOutput) && parsedOutput !== null) { if (Object.prototype.hasOwnProperty.call(parsedOutput, 'manifests')) { return parsedOutput; @@ -91,17 +75,18 @@ export class ImageTools { }); } - public async attestationDescriptors(name: string, platform?: Platform): Promise> { - const manifest = await this.inspectManifest(name); + public async attestationDescriptors(opts: AttestationInspectOpts): Promise> { + const manifest = await this.inspectManifest(opts); if (typeof manifest !== 'object' || manifest === null || !('manifests' in manifest) || !Array.isArray(manifest.manifests)) { - throw new Error(`No descriptor found for ${name}`); + throw new Error(`No descriptor found for ${opts.name}`); } const attestations = manifest.manifests.filter(m => m.annotations?.['vnd.docker.reference.type'] === 'attestation-manifest'); - if (!platform) { + if (!opts.platform) { return attestations; } + const platform = opts.platform; const manifestByDigest = new Map(); for (const m of manifest.manifests) { @@ -123,8 +108,8 @@ export class ImageTools { }); } - public async attestationDigests(name: string, platform?: Platform): Promise> { - return (await this.attestationDescriptors(name, platform)).map(attestation => attestation.digest); + public async attestationDigests(opts: AttestationInspectOpts): Promise> { + return (await this.attestationDescriptors(opts)).map(attestation => attestation.digest); } public async create(opts: CreateOpts): Promise { @@ -205,4 +190,44 @@ export class ImageTools { } }); } + + private async inspect(opts: InspectOpts, format: string, parser: (parsedOutput: unknown) => T): Promise { + const cmd = await this.getInspectCommand([opts.name, '--format', format]); + if (!opts.retryOnManifestUnknown) { + return await this.execInspect(cmd.command, cmd.args, parser); + } + + const retries = opts.retryLimit ?? 15; + let lastError: Error | undefined; + for (let attempt = 0; attempt < retries; attempt++) { + try { + return await this.execInspect(cmd.command, cmd.args, parser); + } catch (err) { + lastError = err as Error; + if (!ImageTools.isManifestUnknownError(lastError.message) || attempt === retries - 1) { + throw lastError; + } + core.info(`buildx imagetools inspect command failed with MANIFEST_UNKNOWN, retrying attempt ${attempt + 1}/${retries}...\n${lastError.message}`); + await new Promise(res => setTimeout(res, Math.pow(2, attempt) * 100)); + } + } + + throw lastError ?? new Error(`ImageTools inspect command failed for ${opts.name}`); + } + + private async execInspect(command: string, args: Array, parser: (parsedOutput: unknown) => T): Promise { + return await Exec.getExecOutput(command, args, { + ignoreReturnCode: true, + silent: true + }).then(res => { + if (res.stderr.length > 0 && res.exitCode != 0) { + throw new Error(res.stderr.trim()); + } + return parser(JSON.parse(res.stdout)); + }); + } + + private static isManifestUnknownError(message: string): boolean { + return /(MANIFEST_UNKNOWN|manifest unknown|not found: not found)/i.test(message); + } } diff --git a/src/sigstore/sigstore.ts b/src/sigstore/sigstore.ts index 552d82ad..2b69afe1 100644 --- a/src/sigstore/sigstore.ts +++ b/src/sigstore/sigstore.ts @@ -113,7 +113,11 @@ export class Sigstore { } for (const imageName of opts.imageNames) { - const attestationDigests = await this.imageTools.attestationDigests(`${imageName}@${opts.imageDigest}`); + const attestationDigests = await this.imageTools.attestationDigests({ + name: `${imageName}@${opts.imageDigest}`, + retryOnManifestUnknown: opts.retryOnManifestUnknown, + retryLimit: opts.retryLimit + }); for (const attestationDigest of attestationDigests) { const attestationRef = `${imageName}@${attestationDigest}`; await core.group(`Signing attestation manifest ${attestationRef}`, async () => { @@ -183,7 +187,12 @@ export class Sigstore { public async verifyImageAttestations(image: string, opts: VerifySignedManifestsOpts): Promise> { const result: Record = {}; - const attestationDigests = await this.imageTools.attestationDigests(image, opts.platform); + const attestationDigests = await this.imageTools.attestationDigests({ + name: image, + platform: opts.platform, + retryOnManifestUnknown: opts.retryOnManifestUnknown, + retryLimit: opts.retryLimit + }); if (attestationDigests.length === 0) { throw new Error(`No attestation manifests found for ${image}`); } @@ -237,7 +246,7 @@ export class Sigstore { }; } - const retries = 15; + const retries = opts.retryLimit ?? 15; let lastError: Error | undefined; core.info(`[command]cosign ${[...cosignArgs, attestationRef].join(' ')}`); for (let attempt = 0; attempt < retries; attempt++) { diff --git a/src/types/buildx/imagetools.ts b/src/types/buildx/imagetools.ts index af88ab58..1177bb97 100644 --- a/src/types/buildx/imagetools.ts +++ b/src/types/buildx/imagetools.ts @@ -15,9 +15,19 @@ */ import {Versioned} from '../oci/versioned.js'; -import {Descriptor} from '../oci/descriptor.js'; +import {Descriptor, Platform} from '../oci/descriptor.js'; import {Digest} from '../oci/digest.js'; +export interface InspectOpts { + name: string; + retryOnManifestUnknown?: boolean; + retryLimit?: number; +} + +export interface AttestationInspectOpts extends InspectOpts { + platform?: Platform; +} + // https://github.com/docker/buildx/blob/62857022a08552bee5cad0c3044a9a3b185f0b32/util/imagetools/printers.go#L109-L123 export interface Manifest extends Versioned { mediaType?: string; diff --git a/src/types/sigstore/sigstore.ts b/src/types/sigstore/sigstore.ts index e1cd7b87..355b95bd 100644 --- a/src/types/sigstore/sigstore.ts +++ b/src/types/sigstore/sigstore.ts @@ -40,6 +40,8 @@ export interface SignAttestationManifestsOpts { imageNames: Array; imageDigest: string; noTransparencyLog?: boolean; + retryOnManifestUnknown?: boolean; + retryLimit?: number; } export interface SignAttestationManifestsResult extends ParsedBundle { @@ -51,6 +53,7 @@ export interface VerifySignedManifestsOpts { platform?: Platform; noTransparencyLog?: boolean; retryOnManifestUnknown?: boolean; + retryLimit?: number; } export interface VerifySignedManifestsResult {