Skip to content

buildx(imagetools): make manifest retries configurable #1016

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
crazy-max merged 1 commit into from
Mar 13, 2026
Merged

buildx(imagetools): make manifest retries configurable #1016

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 10 additions & 10 deletions __tests__/buildx/imagetools.test.itg.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,51 +30,51 @@ const maybe = !process.env.GITHUB_ACTIONS || (process.env.GITHUB_ACTIONS === 'tr

maybe('inspectImage', () => {
it('inspect single platform', async () => {
const image = await new ImageTools().inspectImage('moby/buildkit:latest@sha256:5769c54b98840147b74128f38fb0b0a049e24b11a75bd81664131edd2854593f');
const image = await new ImageTools().inspectImage({name: 'moby/buildkit:latest@sha256:5769c54b98840147b74128f38fb0b0a049e24b11a75bd81664131edd2854593f'});
const expectedImage = <Image>JSON.parse(fs.readFileSync(path.join(fixturesDir, 'imagetools-01.json'), {encoding: 'utf-8'}).trim());
expect(image).toEqual(expectedImage);
});
it('inspect multi platform', async () => {
const image = await new ImageTools().inspectImage('moby/buildkit:latest@sha256:86c0ad9d1137c186e9d455912167df20e530bdf7f7c19de802e892bb8ca16552');
const image = await new ImageTools().inspectImage({name: 'moby/buildkit:latest@sha256:86c0ad9d1137c186e9d455912167df20e530bdf7f7c19de802e892bb8ca16552'});
const expectedImage = <Record<string, Image>>JSON.parse(fs.readFileSync(path.join(fixturesDir, 'imagetools-02.json'), {encoding: 'utf-8'}).trim());
expect(image).toEqual(expectedImage);
});
});

maybe('inspectManifest', () => {
it('inspect descriptor', async () => {
const manifest = await new ImageTools().inspectManifest('moby/buildkit:latest@sha256:dccc69dd895968c4f21aa9e43e715f25f0cedfce4b17f1014c88c307928e22fc');
const manifest = await new ImageTools().inspectManifest({name: 'moby/buildkit:latest@sha256:dccc69dd895968c4f21aa9e43e715f25f0cedfce4b17f1014c88c307928e22fc'});
const expectedManifest = <Descriptor>JSON.parse(fs.readFileSync(path.join(fixturesDir, 'imagetools-03.json'), {encoding: 'utf-8'}).trim());
expect(manifest).toEqual(expectedManifest);
});
it('inspect index', async () => {
const manifest = await new ImageTools().inspectManifest('moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6');
const manifest = await new ImageTools().inspectManifest({name: 'moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6'});
const expectedManifest = <ImageToolsManifest>JSON.parse(fs.readFileSync(path.join(fixturesDir, 'imagetools-04.json'), {encoding: 'utf-8'}).trim());
expect(manifest).toEqual(expectedManifest);
});
});

maybe('attestationDescriptors', () => {
it('returns buildkit attestations descriptors', async () => {
const attestations = await new ImageTools().attestationDescriptors('moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6');
const attestations = await new ImageTools().attestationDescriptors({name: 'moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6'});
const expectedAttestations = <Array<Descriptor>>JSON.parse(fs.readFileSync(path.join(fixturesDir, 'imagetools-05.json'), {encoding: 'utf-8'}).trim());
expect(attestations).toEqual(expectedAttestations);
});
it('returns buildkit attestations descriptors for linux/amd64', async () => {
const attestations = await new ImageTools().attestationDescriptors('moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6', {os: 'linux', architecture: 'amd64'});
const attestations = await new ImageTools().attestationDescriptors({name: 'moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6', platform: {os: 'linux', architecture: 'amd64'}});
const expectedAttestations = <Array<Descriptor>>JSON.parse(fs.readFileSync(path.join(fixturesDir, 'imagetools-06.json'), {encoding: 'utf-8'}).trim());
expect(attestations).toEqual(expectedAttestations);
});
it('returns buildkit attestations descriptors for linux/arm/v7', async () => {
const attestations = await new ImageTools().attestationDescriptors('moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6', {os: 'linux', architecture: 'arm', variant: 'v7'});
const attestations = await new ImageTools().attestationDescriptors({name: 'moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6', platform: {os: 'linux', architecture: 'arm', variant: 'v7'}});
const expectedAttestations = <Array<Descriptor>>JSON.parse(fs.readFileSync(path.join(fixturesDir, 'imagetools-07.json'), {encoding: 'utf-8'}).trim());
expect(attestations).toEqual(expectedAttestations);
});
});

maybe('attestationDigests', () => {
it('returns buildkit attestations digests', async () => {
const digests = await new ImageTools().attestationDigests('moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6');
const digests = await new ImageTools().attestationDigests({name: 'moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6'});
// prettier-ignore
expect(digests).toEqual([
'sha256:2ba4ad6eae1efcafee73a971953093c7c32b6938f2f9fd4998c8bf4d0fbe76f2',
Expand All @@ -86,11 +86,11 @@ maybe('attestationDigests', () => {
]);
});
it('returns buildkit attestations digests for linux/amd64', async () => {
const digests = await new ImageTools().attestationDigests('moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6', {os: 'linux', architecture: 'amd64'});
const digests = await new ImageTools().attestationDigests({name: 'moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6', platform: {os: 'linux', architecture: 'amd64'}});
expect(digests).toEqual(['sha256:2ba4ad6eae1efcafee73a971953093c7c32b6938f2f9fd4998c8bf4d0fbe76f2']);
});
it('returns buildkit attestations digests for linux/arm/v7', async () => {
const digests = await new ImageTools().attestationDigests('moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6', {os: 'linux', architecture: 'arm', variant: 'v7'});
const digests = await new ImageTools().attestationDigests({name: 'moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6', platform: {os: 'linux', architecture: 'arm', variant: 'v7'}});
expect(digests).toEqual(['sha256:0709528fae1747ce17638ad2978ee7936b38a294136eaadaf692e415f64b1e03']);
});
});
Expand Down
125 changes: 124 additions & 1 deletion __tests__/buildx/imagetools.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
* limitations under the License.
*/

import {afterEach, describe, expect, it, vi} from 'vitest';
import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest';
import fs from 'fs';
import os from 'os';
import path from 'path';
Expand All @@ -38,10 +38,133 @@ vi.spyOn(Context, 'tmpName').mockImplementation((): string => {
});

afterEach(() => {
vi.useRealTimers();
vi.clearAllMocks();
rimraf.sync(tmpDir);
});

beforeEach(() => {
vi.useRealTimers();
fs.mkdirSync(tmpDir, {recursive: true});
});

describe('inspectManifest', () => {
it('retries transient manifest unknown errors when requested', async () => {
vi.useFakeTimers();

const getCommand = vi.fn().mockResolvedValue({
command: 'docker',
args: ['buildx', 'imagetools', 'inspect']
});
const buildx = {getCommand} as unknown as Buildx;
const execSpy = vi
.spyOn(Exec, 'getExecOutput')
.mockResolvedValueOnce({
exitCode: 1,
stdout: '',
stderr: 'ERROR: MANIFEST_UNKNOWN: manifest unknown'
})
.mockResolvedValueOnce({
exitCode: 0,
stdout: JSON.stringify({
schemaVersion: 2,
mediaType: 'application/vnd.oci.image.index.v1+json',
manifests: []
}),
stderr: ''
});

const inspectPromise = new ImageTools({buildx}).inspectManifest({
name: 'docker.io/library/alpine:latest',
retryOnManifestUnknown: true,
retryLimit: 2
});

await vi.runAllTimersAsync();

expect(await inspectPromise).toEqual({
schemaVersion: 2,
mediaType: 'application/vnd.oci.image.index.v1+json',
manifests: []
});
expect(getCommand).toHaveBeenCalledWith(['imagetools', 'inspect', 'docker.io/library/alpine:latest', '--format', '{{json .Manifest}}']);
expect(execSpy).toHaveBeenCalledTimes(2);
});

it('does not retry non-manifest errors', async () => {
const getCommand = vi.fn().mockResolvedValue({
command: 'docker',
args: ['buildx', 'imagetools', 'inspect']
});
const buildx = {getCommand} as unknown as Buildx;
const execSpy = vi.spyOn(Exec, 'getExecOutput').mockResolvedValue({
exitCode: 1,
stdout: '',
stderr: 'ERROR: unauthorized'
});

const result = await new ImageTools({buildx})
.inspectManifest({
name: 'docker.io/library/alpine:latest',
retryOnManifestUnknown: true
})
.then(
value => ({value, error: undefined}),
error => ({value: undefined, error: error as Error})
);

expect(result.value).toBeUndefined();
expect(result.error).toBeInstanceOf(Error);
expect(result.error?.message).toContain('ERROR: unauthorized');

expect(execSpy).toHaveBeenCalledTimes(1);
});
});

describe('inspectImage', () => {
it('retries transient manifest unknown errors when requested', async () => {
vi.useFakeTimers();

const getCommand = vi.fn().mockResolvedValue({
command: 'docker',
args: ['buildx', 'imagetools', 'inspect']
});
const buildx = {getCommand} as unknown as Buildx;
const execSpy = vi
.spyOn(Exec, 'getExecOutput')
.mockResolvedValueOnce({
exitCode: 1,
stdout: '',
stderr: 'ERROR: MANIFEST_UNKNOWN: manifest unknown'
})
.mockResolvedValueOnce({
exitCode: 0,
stdout: JSON.stringify({
config: {
digest: 'sha256:test'
}
}),
stderr: ''
});

const inspectPromise = new ImageTools({buildx}).inspectImage({
name: 'docker.io/library/alpine:latest',
retryOnManifestUnknown: true,
retryLimit: 2
});

await vi.runAllTimersAsync();

expect(await inspectPromise).toEqual({
config: {
digest: 'sha256:test'
}
});
expect(getCommand).toHaveBeenCalledWith(['imagetools', 'inspect', 'docker.io/library/alpine:latest', '--format', '{{json .Image}}']);
expect(execSpy).toHaveBeenCalledTimes(2);
});
});

describe('create', () => {
it('parses metadata and supports cwd sources', async () => {
const getCommand = vi.fn().mockResolvedValue({
Expand Down
81 changes: 53 additions & 28 deletions src/buildx/imagetools.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,9 @@ import {Buildx} from './buildx.js';
import {Context} from '../context.js';
import {Exec} from '../exec.js';

import {CreateOpts, CreateResponse, CreateResult, Manifest as ImageToolsManifest} from '../types/buildx/imagetools.js';
import {AttestationInspectOpts, CreateOpts, CreateResponse, CreateResult, InspectOpts, Manifest as ImageToolsManifest} from '../types/buildx/imagetools.js';
import {Image} from '../types/oci/config.js';
import {Descriptor, Platform} from '../types/oci/descriptor.js';
import {Descriptor} from '../types/oci/descriptor.js';
import {Digest} from '../types/oci/digest.js';

export interface ImageToolsOpts {
Expand All @@ -49,16 +49,8 @@ export class ImageTools {
return await this.getCommand(['create', ...args]);
}

public async inspectImage(name: string): Promise<Record<string, Image> | Image> {
const cmd = await this.getInspectCommand([name, '--format', '{{json .Image}}']);
return await Exec.getExecOutput(cmd.command, cmd.args, {
ignoreReturnCode: true,
silent: true
}).then(res => {
if (res.stderr.length > 0 && res.exitCode != 0) {
throw new Error(res.stderr.trim());
}
const parsedOutput = JSON.parse(res.stdout);
public async inspectImage(opts: InspectOpts): Promise<Record<string, Image> | Image> {
return await this.inspect(opts, '{{json .Image}}', parsedOutput => {
if (typeof parsedOutput === 'object' && !Array.isArray(parsedOutput) && parsedOutput !== null) {
if (Object.prototype.hasOwnProperty.call(parsedOutput, 'config')) {
return <Image>parsedOutput;
Expand All @@ -70,16 +62,8 @@ export class ImageTools {
});
}

public async inspectManifest(name: string): Promise<ImageToolsManifest | Descriptor> {
const cmd = await this.getInspectCommand([name, '--format', '{{json .Manifest}}']);
return await Exec.getExecOutput(cmd.command, cmd.args, {
ignoreReturnCode: true,
silent: true
}).then(res => {
if (res.stderr.length > 0 && res.exitCode != 0) {
throw new Error(res.stderr.trim());
}
const parsedOutput = JSON.parse(res.stdout);
public async inspectManifest(opts: InspectOpts): Promise<ImageToolsManifest | Descriptor> {
return await this.inspect(opts, '{{json .Manifest}}', parsedOutput => {
if (typeof parsedOutput === 'object' && !Array.isArray(parsedOutput) && parsedOutput !== null) {
if (Object.prototype.hasOwnProperty.call(parsedOutput, 'manifests')) {
return <ImageToolsManifest>parsedOutput;
Expand All @@ -91,17 +75,18 @@ export class ImageTools {
});
}

public async attestationDescriptors(name: string, platform?: Platform): Promise<Array<Descriptor>> {
const manifest = await this.inspectManifest(name);
public async attestationDescriptors(opts: AttestationInspectOpts): Promise<Array<Descriptor>> {
const manifest = await this.inspectManifest(opts);

if (typeof manifest !== 'object' || manifest === null || !('manifests' in manifest) || !Array.isArray(manifest.manifests)) {
throw new Error(`No descriptor found for ${name}`);
throw new Error(`No descriptor found for ${opts.name}`);
}

const attestations = manifest.manifests.filter(m => m.annotations?.['vnd.docker.reference.type'] === 'attestation-manifest');
if (!platform) {
if (!opts.platform) {
return attestations;
}
const platform = opts.platform;

const manifestByDigest = new Map<string, Descriptor>();
for (const m of manifest.manifests) {
Expand All @@ -123,8 +108,8 @@ export class ImageTools {
});
}

public async attestationDigests(name: string, platform?: Platform): Promise<Array<Digest>> {
return (await this.attestationDescriptors(name, platform)).map(attestation => attestation.digest);
public async attestationDigests(opts: AttestationInspectOpts): Promise<Array<Digest>> {
return (await this.attestationDescriptors(opts)).map(attestation => attestation.digest);
}

public async create(opts: CreateOpts): Promise<CreateResult | undefined> {
Expand Down Expand Up @@ -205,4 +190,44 @@ export class ImageTools {
}
});
}

private async inspect<T>(opts: InspectOpts, format: string, parser: (parsedOutput: unknown) => T): Promise<T> {
const cmd = await this.getInspectCommand([opts.name, '--format', format]);
if (!opts.retryOnManifestUnknown) {
return await this.execInspect(cmd.command, cmd.args, parser);
}

const retries = opts.retryLimit ?? 15;
let lastError: Error | undefined;
for (let attempt = 0; attempt < retries; attempt++) {
try {
return await this.execInspect(cmd.command, cmd.args, parser);
} catch (err) {
lastError = err as Error;
if (!ImageTools.isManifestUnknownError(lastError.message) || attempt === retries - 1) {
throw lastError;
}
core.info(`buildx imagetools inspect command failed with MANIFEST_UNKNOWN, retrying attempt ${attempt + 1}/${retries}...\n${lastError.message}`);
await new Promise(res => setTimeout(res, Math.pow(2, attempt) * 100));
}
}

throw lastError ?? new Error(`ImageTools inspect command failed for ${opts.name}`);
}

private async execInspect<T>(command: string, args: Array<string>, parser: (parsedOutput: unknown) => T): Promise<T> {
return await Exec.getExecOutput(command, args, {
ignoreReturnCode: true,
silent: true
}).then(res => {
if (res.stderr.length > 0 && res.exitCode != 0) {
throw new Error(res.stderr.trim());
}
return parser(JSON.parse(res.stdout));
});
}

private static isManifestUnknownError(message: string): boolean {
return /(MANIFEST_UNKNOWN|manifest unknown|not found: not found)/i.test(message);
}
}
15 changes: 12 additions & 3 deletions src/sigstore/sigstore.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,11 @@ export class Sigstore {
}

for (const imageName of opts.imageNames) {
const attestationDigests = await this.imageTools.attestationDigests(`${imageName}@${opts.imageDigest}`);
const attestationDigests = await this.imageTools.attestationDigests({
name: `${imageName}@${opts.imageDigest}`,
retryOnManifestUnknown: opts.retryOnManifestUnknown,
retryLimit: opts.retryLimit
});
for (const attestationDigest of attestationDigests) {
const attestationRef = `${imageName}@${attestationDigest}`;
await core.group(`Signing attestation manifest ${attestationRef}`, async () => {
Expand Down Expand Up @@ -183,7 +187,12 @@ export class Sigstore {
public async verifyImageAttestations(image: string, opts: VerifySignedManifestsOpts): Promise<Record<string, VerifySignedManifestsResult>> {
const result: Record<string, VerifySignedManifestsResult> = {};

const attestationDigests = await this.imageTools.attestationDigests(image, opts.platform);
const attestationDigests = await this.imageTools.attestationDigests({
name: image,
platform: opts.platform,
retryOnManifestUnknown: opts.retryOnManifestUnknown,
retryLimit: opts.retryLimit
});
if (attestationDigests.length === 0) {
throw new Error(`No attestation manifests found for ${image}`);
}
Expand Down Expand Up @@ -237,7 +246,7 @@ export class Sigstore {
};
}

const retries = 15;
const retries = opts.retryLimit ?? 15;
let lastError: Error | undefined;
core.info(`[command]cosign ${[...cosignArgs, attestationRef].join(' ')}`);
for (let attempt = 0; attempt < retries; attempt++) {
Expand Down
Loading
Loading