Invoke newt to build devcontainers with cloudbuild

CHANGELOG.md

@@ -4,9 +4,25 @@ Notable changes.
 
 ## November 2022
 
+### [0.25.2]
+
+- Fix Feature/Template publishing issue when a capital letter is in the repo name (https://github.com/devcontainers/cli/pull/303)
+
+### [0.25.1]
+- Fix regression in https://github.com/devcontainers/cli/pull/298
+
+### [0.25.0]
+
+- `features test`: Respect image label metadata. (https://github.com/devcontainers/cli/pull/288)
+- Surface first error (https://github.com/microsoft/vscode-remote-release/issues/7382)
+- `templates publish`: Exit for "Failed to PUT manifest for tag x" error. (https://github.com/devcontainers/cli/pull/296)
+- Respect devcontainer.json when using image without features. (https://github.com/devcontainers/cli/issues/299)
+- Emit response from registry on failed `postUploadSessionId` (https://github.com/devcontainers/cli/pull/298)
+- downcase OCI identifiers and validate input of getRef() (https://github.com/devcontainers/cli/pull/293)
+
 ### [0.24.1]
 
--   `features test`: Respects testing scenarios where 'remoteUser' is non-root (https://github.com/devcontainers/cli/pull/286)
+- `features test`: Respects testing scenarios where 'remoteUser' is non-root (https://github.com/devcontainers/cli/pull/286)
 
 ### [0.24.0]
 

package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@devcontainers/cli",
 	"description": "Dev Containers CLI",
-	"version": "0.24.1",
+	"version": "0.25.2",
 	"bin": {
 		"devcontainer": "devcontainer.js"
 	},

src/spec-configuration/containerCollectionsOCI.ts

@@ -13,7 +13,9 @@ export const DEVCONTAINER_COLLECTION_LAYER_MEDIATYPE = 'application/vnd.devconta
 
 export type HEADERS = { 'authorization'?: string; 'user-agent': string; 'content-type'?: string; 'accept'?: string };
 
-// ghcr.io/devcontainers/features/go:1.0.0
+// Represents the unique OCI identifier for a Feature or Template.
+// eg:  ghcr.io/devcontainers/features/go:1.0.0
+// Constructed by 'getRef()'
 export interface OCIRef {
 	registry: string; 		// 'ghcr.io'
 	owner: string;			// 'devcontainers'
@@ -24,11 +26,14 @@ export interface OCIRef {
 	version?: string;		// '1.0.0'
 }
 
-// ghcr.io/devcontainers/features:latest
+// Represents the unique OCI identifier for a Collection's Metadata artifact.
+// eg:  ghcr.io/devcontainers/features:latest
+// Constructed by 'getCollectionRef()'
 export interface OCICollectionRef {
 	registry: string;		// 'ghcr.io'
 	path: string;			// 'devcontainers/features'
-	version: 'latest';		// 'latest'
+	resource: string;		// 'ghcr.io/devcontainers/features'
+	version: 'latest';		// 'latest' (always)
 }
 
 export interface OCILayer {
@@ -58,13 +63,38 @@ interface OCITagList {
 	tags: string[];
 }
 
-export function getRef(output: Log, resourceAndVersion: string): OCIRef {
-
-	// ex: ghcr.io/codspace/features/ruby:1
-	// ex: ghcr.io/codspace/templates/ruby:1
-	const splitOnColon = resourceAndVersion.split(':');
-	const resource = splitOnColon[0];
-	const version = splitOnColon[1] ? splitOnColon[1] : 'latest';
+// Following Spec:   https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pulling-manifests
+// Alternative Spec: https://docs.docker.com/registry/spec/api/#overview
+//
+// Entire path ('namespace' in spec terminology) for the given repository 
+// (eg: devcontainers/features/go)
+const regexForPath = /^[a-z0-9]+([._-][a-z0-9]+)*(\/[a-z0-9]+([._-][a-z0-9]+)*)*$/;
+// MUST be either (a) the digest of the manifest or (b) a tag
+// MUST be at most 128 characters in length and MUST match the following regular expression:
+const regexForReference = /^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$/;
+
+// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pulling-manifests
+// Attempts to parse the given string into an OCIRef
+export function getRef(output: Log, input: string): OCIRef | undefined {
+	// Normalize input by downcasing entire string
+	input = input.toLowerCase();
+
+	const indexOfLastColon = input.lastIndexOf(':');
+
+	let resource = '';
+	let version = ''; // TODO: Support parsing out manifest digest (...@sha256:...)
+
+	// 'If' condition is true in the following cases:
+	//  1. The final colon is before the first slash (a port) :  eg:   ghcr.io:8081/codspace/features/ruby
+	//  2. There is no version :      				   			 eg:   ghcr.io/codspace/features/ruby
+	// In both cases, assume 'latest' tag.
+	if (indexOfLastColon === -1 || indexOfLastColon < input.indexOf('/')) {
+		resource = input;
+		version = 'latest';
+	} else {
+		resource = input.substring(0, indexOfLastColon);
+		version = input.substring(indexOfLastColon + 1);
+	}
 
 	const splitOnSlash = resource.split('/');
 
@@ -75,13 +105,27 @@ export function getRef(output: Log, resourceAndVersion: string): OCIRef {
 
 	const path = `${namespace}/${id}`;
 
-	output.write(`resource: ${resource}`, LogLevel.Trace);
-	output.write(`id: ${id}`, LogLevel.Trace);
-	output.write(`version: ${version}`, LogLevel.Trace);
-	output.write(`owner: ${owner}`, LogLevel.Trace);
-	output.write(`namespace: ${namespace}`, LogLevel.Trace);
-	output.write(`registry: ${registry}`, LogLevel.Trace);
-	output.write(`path: ${path}`, LogLevel.Trace);
+	output.write(`> input: ${input}`, LogLevel.Trace);
+	output.write(`>`, LogLevel.Trace);
+	output.write(`> resource: ${resource}`, LogLevel.Trace);
+	output.write(`> id: ${id}`, LogLevel.Trace);
+	output.write(`> version: ${version}`, LogLevel.Trace);
+	output.write(`> owner: ${owner}`, LogLevel.Trace);
+	output.write(`> namespace: ${namespace}`, LogLevel.Trace); // TODO: We assume 'namespace' includes at least one slash (eg: 'devcontainers/features')
+	output.write(`> registry: ${registry}`, LogLevel.Trace);
+	output.write(`> path: ${path}`, LogLevel.Trace);
+
+	// Validate results of parse.
+
+	if (!regexForPath.exec(path)) {
+		output.write(`Parsed path '${path}' for input '${input}' failed validation.`, LogLevel.Error);
+		return undefined;
+	}
+
+	if (!regexForReference.test(version)) {
+		output.write(`Parsed version '${version}' for input '${input}' failed validation.`, LogLevel.Error);
+		return undefined;
+	}
 
 	return {
 		id,
@@ -94,6 +138,31 @@ export function getRef(output: Log, resourceAndVersion: string): OCIRef {
 	};
 }
 
+export function getCollectionRef(output: Log, registry: string, namespace: string): OCICollectionRef | undefined {
+	// Normalize input by downcasing entire string
+	registry = registry.toLowerCase();
+	namespace = namespace.toLowerCase();
+
+	const path = namespace;
+	const resource = `${registry}/${path}`;
+
+	output.write(`> Inputs: registry='${registry}' namespace='${namespace}'`, LogLevel.Trace);
+	output.write(`>`, LogLevel.Trace);
+	output.write(`> resource: ${resource}`, LogLevel.Trace);
+
+	if (!regexForPath.exec(path)) {
+		output.write(`Parsed path '${path}' from input failed validation.`, LogLevel.Error);
+		return undefined;
+	}
+
+	return {
+		registry,
+		path,
+		resource,
+		version: 'latest'
+	};
+}
+
 // Validate if a manifest exists and is reachable about the declared feature/template.
 // Specification: https://github.com/opencontainers/distribution-spec/blob/v1.0.1/spec.md#pulling-manifests
 export async function fetchOCIManifestIfExists(output: Log, env: NodeJS.ProcessEnv, ref: OCIRef | OCICollectionRef, manifestDigest?: string, authToken?: string): Promise<OCIManifest | undefined> {
@@ -229,7 +298,7 @@ export async function getPublishedVersions(ref: OCIRef, output: Log, sorted: boo
 		let authToken = await fetchRegistryAuthToken(output, ref.registry, ref.path, process.env, 'pull');
 
 		if (!authToken) {
-			output.write(`(!) ERR: Failed to publish ${collectionType}: ${ref.resource}`, LogLevel.Error);
+			output.write(`(!) ERR: Failed to get published versions for ${collectionType}: ${ref.resource}`, LogLevel.Error);
 			return undefined;
 		}
 

src/spec-configuration/containerCollectionsOCIPush.ts

@@ -161,14 +161,14 @@ async function putManifestWithTags(output: Log, manifestStr: string, ociRef: OCI
 			data: Buffer.from(manifestStr),
 		};
 
-		let { statusCode, resHeaders } = await requestResolveHeaders(options);
+		let { statusCode, resHeaders } = await requestResolveHeaders(options, output);
 
 		// Retry logic: when request fails with HTTP 429: too many requests
 		if (statusCode === 429) {
 			output.write(`Failed to PUT manifest for tag ${tag} due to too many requests. Retrying...`, LogLevel.Warning);
 			await delay(2000);
 
-			let response = await requestResolveHeaders(options);
+			let response = await requestResolveHeaders(options, output);
 			statusCode = response.statusCode;
 			resHeaders = response.resHeaders;
 		}
@@ -212,7 +212,7 @@ async function putBlob(output: Log, pathToBlob: string, blobPutLocationUriPath:
 
 	output.write(`Crafted blob url:  ${url}`, LogLevel.Trace);
 
-	const { statusCode } = await requestResolveHeaders({ type: 'PUT', url, headers, data: await readLocalFile(pathToBlob) });
+	const { statusCode } = await requestResolveHeaders({ type: 'PUT', url, headers, data: await readLocalFile(pathToBlob) }, output);
 	if (statusCode !== 201) {
 		output.write(`${statusCode}: Failed to upload blob '${pathToBlob}' to '${url}'`, LogLevel.Error);
 		return false;
@@ -311,7 +311,7 @@ async function postUploadSessionId(output: Log, ociRef: OCIRef | OCICollectionRe
 
 	const url = `https://${ociRef.registry}/v2/${ociRef.path}/blobs/uploads/`;
 	output.write(`Generating Upload URL -> ${url}`, LogLevel.Trace);
-	const { statusCode, resHeaders } = await requestResolveHeaders({ type: 'POST', url, headers }, output);
+	const { statusCode, resHeaders, resBody } = await requestResolveHeaders({ type: 'POST', url, headers }, output);
 	output.write(`${url}: ${statusCode}`, LogLevel.Trace);
 	if (statusCode === 202) {
 		const locationHeader = resHeaders['location'] || resHeaders['Location'];
@@ -320,8 +320,13 @@ async function postUploadSessionId(output: Log, ociRef: OCIRef | OCICollectionRe
 			return undefined;
 		}
 		return locationHeader;
+	} else {
+		// Any other statusCode besides 202 is unexpected
+		// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#error-codes
+		const displayResBody = resBody ? ` -> ${resBody}` : '';
+		output.write(`${url}: Unexpected status code '${statusCode}'${displayResBody}`, LogLevel.Error);
+		return undefined;
 	}
-	return undefined;
 }
 
 export async function calculateManifestAndContentDigest(output: Log, dataLayer: OCILayer, annotations: { [key: string]: string } | undefined) {

src/spec-configuration/containerFeaturesConfiguration.ts

@@ -12,7 +12,7 @@ import { mkdirpLocal, readLocalFile, rmLocal, writeLocalFile, cpDirectoryLocal,
 import { Log, LogLevel } from '../spec-utils/log';
 import { request } from '../spec-utils/httpRequest';
 import { computeFeatureInstallationOrder } from './containerFeaturesOrder';
-import { fetchOCIFeature, getOCIFeatureSet, fetchOCIFeatureManifestIfExistsFromUserIdentifier } from './containerFeaturesOCI';
+import { fetchOCIFeature, tryGetOCIFeatureSet, fetchOCIFeatureManifestIfExistsFromUserIdentifier } from './containerFeaturesOCI';
 import { OCIManifest, OCIRef } from './containerCollectionsOCI';
 
 // v1
@@ -769,8 +769,7 @@ export async function processFeatureIdentifier(output: Log, configPath: string,
 
 	// (6) Oci Identifier
 	if (type === 'oci' && manifest) {
-		let newFeaturesSet: FeatureSet = getOCIFeatureSet(output, userFeature.id, userFeature.options, manifest, originalUserFeatureId);
-		return newFeaturesSet;
+		return tryGetOCIFeatureSet(output, userFeature.id, userFeature.options, manifest, originalUserFeatureId);
 	}
 
 	output.write(`Github feature.`);

src/spec-configuration/containerFeaturesOCI.ts

@@ -2,9 +2,12 @@ import { Log, LogLevel } from '../spec-utils/log';
 import { Feature, FeatureSet } from './containerFeaturesConfiguration';
 import { fetchOCIManifestIfExists, getBlob, getRef, OCIManifest } from './containerCollectionsOCI';
 
-export function getOCIFeatureSet(output: Log, identifier: string, options: boolean | string | Record<string, boolean | string | undefined>, manifest: OCIManifest, originalUserFeatureId: string): FeatureSet {
-
+export function tryGetOCIFeatureSet(output: Log, identifier: string, options: boolean | string | Record<string, boolean | string | undefined>, manifest: OCIManifest, originalUserFeatureId: string): FeatureSet | undefined {
 	const featureRef = getRef(output, identifier);
+	if (!featureRef) {
+		output.write(`Unable to parse '${identifier}'`, LogLevel.Error);
+		return undefined;
+	}
 
 	const feat: Feature = {
 		id: featureRef.id,
@@ -30,6 +33,9 @@ export function getOCIFeatureSet(output: Log, identifier: string, options: boole
 
 export async function fetchOCIFeatureManifestIfExistsFromUserIdentifier(output: Log, env: NodeJS.ProcessEnv, identifier: string, manifestDigest?: string, authToken?: string): Promise<OCIManifest | undefined> {
 	const featureRef = getRef(output, identifier);
+	if (!featureRef) {
+		return undefined;
+	}
 	return await fetchOCIManifestIfExists(output, env, featureRef, manifestDigest, authToken);
 }
 

src/spec-configuration/containerTemplatesOCI.ts

@@ -122,6 +122,9 @@ export async function fetchTemplate(output: Log, selectedTemplate: SelectedTempl
 
 async function fetchOCITemplateManifestIfExistsFromUserIdentifier(output: Log, env: NodeJS.ProcessEnv, identifier: string, manifestDigest?: string, authToken?: string): Promise<OCIManifest | undefined> {
 	const templateRef = getRef(output, identifier);
+	if (!templateRef) {
+		return undefined;
+	}
 	return await fetchOCIManifestIfExists(output, env, templateRef, manifestDigest, authToken);
 }
 

src/spec-node/collectionCommonUtils/publishCommandImpl.ts

@@ -44,7 +44,7 @@ export async function doPublishCommand(version: string, ociRef: OCIRef, outputDi
 	const publishedVersions = await getPublishedVersions(ociRef, output);
 
 	if (!publishedVersions) {
-		process.exit(1);
+		return false;
 	}
 
 	const semanticVersions: string[] | undefined = getSermanticVersions(version, publishedVersions, output);
@@ -53,11 +53,12 @@ export async function doPublishCommand(version: string, ociRef: OCIRef, outputDi
 		output.write(`Publishing versions: ${semanticVersions.toString()}...`, LogLevel.Info);
 		const pathToTgz = path.join(outputDir, getArchiveName(ociRef.id, collectionType));
 		if (! await pushOCIFeatureOrTemplate(output, ociRef, pathToTgz, semanticVersions, collectionType)) {
-			output.write(`(!) ERR: Failed to publish ${collectionType}: ${ociRef.resource}`, LogLevel.Error);
-			process.exit(1);
+			output.write(`(!) ERR: Failed to publish ${collectionType}: '${ociRef.resource}'`, LogLevel.Error);
+			return false;
 		}
-		output.write(`Published ${collectionType}: ${ociRef.id}...`, LogLevel.Info);
 	}
+	output.write(`Published ${collectionType}: ${ociRef.id}...`, LogLevel.Info);
+	return true;
 }
 
 export async function doPublishMetadata(collectionRef: OCICollectionRef, outputDir: string, output: Log, collectionType: string) {
@@ -67,7 +68,8 @@ export async function doPublishMetadata(collectionRef: OCICollectionRef, outputD
 	const pathToCollectionFile = path.join(outputDir, OCICollectionFileName);
 	if (! await pushCollectionMetadata(output, collectionRef, pathToCollectionFile, collectionType)) {
 		output.write(`(!) ERR: Failed to publish collection metadata: ${OCICollectionFileName}`, LogLevel.Error);
-		process.exit(1);
+		return false;
 	}
 	output.write('Published collection metadata...', LogLevel.Info);
+	return true;
 }

src/spec-node/containerFeatures.ts

@@ -39,7 +39,7 @@ export async function extendImage(params: DockerResolverParameters, config: Subs
 		// no feature extensions - return
 		return {
 			updatedImageName: [imageName],
-			imageMetadata: imageBuildInfo.metadata,
+			imageMetadata: getDevcontainerMetadata(imageBuildInfo.metadata, config, extendImageDetails?.featuresConfig),
 			imageDetails: async () => imageBuildInfo.imageDetails,
 			labels: extendImageDetails?.labels,
 		};

src/spec-node/devContainers.ts

@@ -17,7 +17,6 @@ import { LogLevel, LogDimensions, toErrorText, createCombinedLog, createTerminal
 import { dockerComposeCLIConfig } from './dockerCompose';
 import { Mount } from '../spec-configuration/containerFeaturesConfiguration';
 import { getPackageConfig, PackageConfiguration } from '../spec-utils/product';
-import { dockerBuildKitVersion } from '../spec-shutdown/dockerUtils';
 
 export const experimentalImageMetadataDefault = true;
 
@@ -142,13 +141,7 @@ export async function createDockerParams(options: ProvisionOptions, disposables:
 		env: cliHost.env,
 		output: common.output,
 	}, dockerPath, dockerComposePath);
-	const buildKitVersion = options.useBuildKit === 'never' ? null : (await dockerBuildKitVersion({
-		cliHost,
-		dockerCLI: dockerPath,
-		dockerComposeCLI,
-		env: cliHost.env,
-		output
-	}));
+	const buildKitVersion = options.useBuildKit === 'never' ? null : 'v0.10.0';
 	return {
 		common,
 		parsedAuthority,

src/spec-node/devContainersSpecCLI.ts

@@ -397,15 +397,7 @@ async function doBuild({
 
 			// Build the base image and extend with features etc.
 			let { updatedImageName } = await buildNamedImageAndExtend(params, configWithRaw as SubstitutedConfig<DevContainerFromDockerfileConfig>, additionalFeatures, false, imageNames);
-
-			if (imageNames) {
-				if (!buildxPush && !buildxOutput) {
-					await Promise.all(imageNames.map(imageName => dockerPtyCLI(params, 'tag', updatedImageName[0], imageName)));
-				}
-				imageNameResult = imageNames;
-			} else {
-				imageNameResult = updatedImageName;
-			}
+			imageNameResult = updatedImageName;
 		} else if ('dockerComposeFile' in config) {
 
 			if (buildxPlatform || buildxPush) {

src/spec-node/featuresCLI/info.ts

@@ -40,6 +40,14 @@ async function featuresInfo({
 	}, pkg, new Date(), disposables, true);
 
 	const featureOciRef = getRef(output, featureId);
+	if (!featureOciRef) {
+		if (outputFormat === 'json') {
+			output.raw(JSON.stringify({}), LogLevel.Info);
+		} else {
+			output.raw(`Failed to parse Feature identifier '${featureId}'\n`, LogLevel.Error);
+		}
+		process.exit(1);
+	}
 
 	const publishedVersions = await getPublishedVersions(featureOciRef, output, true);
 	if (!publishedVersions || publishedVersions.length === 0) {