chore: Migrate Skaffold presubmits to new Kokoro instance

Makefile

@@ -33,6 +33,39 @@ GCP_ONLY ?= false
 GCP_PROJECT ?= k8s-skaffold
 GKE_CLUSTER_NAME ?= integration-tests
 GKE_ZONE ?= us-central1-a
+GKE_REGION=us-central1
+AR_REGION ?= us-central1
+
+# Set registry/auth/cluster location based on GCP_PROJECT
+ifeq ($(GCP_PROJECT),skaffold-ci-cd)
+  # Presubmit environment: skaffold-ci-cd project with Artifact Registry
+  IMAGE_REPO_BASE := $(AR_REGION)-docker.pkg.dev/$(GCP_PROJECT)
+
+  # Define full paths including the 4th segment (Image Name) for AR
+  SKAFFOLD_IMAGE         := $(IMAGE_REPO_BASE)/skaffold/skaffold
+  # Artifact registry does not allow _ in the repo name (with GCR this was 
+  # build_deps).
+  SKAFFOLD_DEPS_IMAGE    := $(IMAGE_REPO_BASE)/builddeps/skaffold-deps
+  SKAFFOLD_BUILDER_IMAGE := $(IMAGE_REPO_BASE)/skaffold-builder/skaffold-builder
+
+  # For Integration Tests: Export a 4-segment default repo
+  export SKAFFOLD_DEFAULT_REPO := $(IMAGE_REPO_BASE)/testing
+  GCLOUD_AUTH_CONFIG := $(AR_REGION)-docker.pkg.dev
+  GKE_LOCATION_FLAG := --region $(GKE_REGION)
+  $(info Using Artifact Registry config for project: $(GCP_PROJECT))
+else
+  # k8s-skaffold project with GCR
+  IMAGE_REPO_BASE := gcr.io/$(GCP_PROJECT)
+
+  # Define full paths using the 3 segments GCR expects
+  SKAFFOLD_IMAGE         := $(IMAGE_REPO_BASE)/skaffold
+  SKAFFOLD_DEPS_IMAGE    := $(IMAGE_REPO_BASE)/build_deps
+  SKAFFOLD_BUILDER_IMAGE := $(IMAGE_REPO_BASE)/skaffold-builder
+
+  GCLOUD_AUTH_CONFIG := gcr.io
+  GKE_LOCATION_FLAG := --zone $(GKE_ZONE)
+  $(info Using GCR config for project: $(GCP_PROJECT))
+endif
 
 SUPPORTED_PLATFORMS = linux-amd64 darwin-amd64 windows-amd64.exe linux-arm64 darwin-arm64
 BUILD_PACKAGE = $(REPOPATH)/v2/cmd/skaffold
@@ -142,9 +175,16 @@ integration-tests:
 ifeq ($(GCP_ONLY),true)
 	gcloud container clusters get-credentials \
 		$(GKE_CLUSTER_NAME) \
-		--zone $(GKE_ZONE) \
+		$(GKE_LOCATION_FLAG) \
 		--project $(GCP_PROJECT)
-	gcloud auth configure-docker us-central1-docker.pkg.dev
+
+  # Conditional Docker authentication: ONLY when GCR is used
+  ifneq ($(GCP_PROJECT),skaffold-ci-cd)
+	@echo "Configuring Docker for GCR: $(GCLOUD_AUTH_CONFIG)"
+	gcloud auth configure-docker $(GCLOUD_AUTH_CONFIG) -q
+  else
+	@echo "Docker auth is handled in the build script for skaffold-ci-cd"
+  endif
 endif
 	@ GCP_ONLY=$(GCP_ONLY) GKE_CLUSTER_NAME=$(GKE_CLUSTER_NAME) ./hack/gotest.sh -v $(REPOPATH)/v2/integration -timeout 50m $(INTEGRATION_TEST_ARGS)
 
@@ -155,74 +195,102 @@ integration: install integration-tests
 release: $(BUILD_DIR)/VERSION
 	docker build \
 		--build-arg VERSION=$(VERSION) \
+		--build-arg BASE_IMAGE=$(SKAFFOLD_DEPS_IMAGE):latest \
 		-f deploy/skaffold/Dockerfile \
 		--target release \
-		-t gcr.io/$(GCP_PROJECT)/skaffold:$(VERSION) \
-                -t gcr.io/$(GCP_PROJECT)/skaffold:latest \
+		-t $(SKAFFOLD_IMAGE):$(VERSION) \
+		-t $(SKAFFOLD_IMAGE):latest \
 		.
 
 .PHONY: release-build
 release-build:
 	docker build \
 		-f deploy/skaffold/Dockerfile \
+		--build-arg BASE_IMAGE=$(SKAFFOLD_DEPS_IMAGE):latest \
 		--target release \
-		-t gcr.io/$(GCP_PROJECT)/skaffold:edge \
-		-t gcr.io/$(GCP_PROJECT)/skaffold:$(COMMIT) \
+		-t $(SKAFFOLD_IMAGE):edge \
+		-t $(SKAFFOLD_IMAGE):$(COMMIT) \
 		.
 
 .PHONY: release-lts
 release-lts: $(BUILD_DIR)/VERSION
 	docker build \
 		--build-arg VERSION=$(VERSION) \
+		--build-arg BASE_IMAGE=$(SKAFFOLD_DEPS_IMAGE):latest \
 		-f deploy/skaffold/Dockerfile.lts \
 		--target release \
-		-t gcr.io/$(GCP_PROJECT)/skaffold:lts \
-		-t gcr.io/$(GCP_PROJECT)/skaffold:$(VERSION)-lts \
-		-t gcr.io/$(GCP_PROJECT)/skaffold:$(SCANNING_MARKER)-lts \
+		-t $(SKAFFOLD_IMAGE):lts \
+		-t $(SKAFFOLD_IMAGE):$(VERSION)-lts \
+		-t $(SKAFFOLD_IMAGE):$(SCANNING_MARKER)-lts \
 		.
 
 .PHONY: release-lts-build
 release-lts-build:
 	docker build \
 		-f deploy/skaffold/Dockerfile.lts \
+		--build-arg BASE_IMAGE=$(SKAFFOLD_DEPS_IMAGE):latest \
 		--target release \
-		-t gcr.io/$(GCP_PROJECT)/skaffold:edge-lts \
-		-t gcr.io/$(GCP_PROJECT)/skaffold:$(COMMIT)-lts \
+		-t $(SKAFFOLD_IMAGE):edge-lts \
+		-t $(SKAFFOLD_IMAGE):$(COMMIT)-lts \
 		.
 
 .PHONY: clean
 clean:
 	rm -rf $(BUILD_DIR) hack/bin $(EMBEDDED_FILES_CHECK) fs/assets/schemas_generated/
 
+# Runs a script to calculate a hash/digest of the build dependencies. Store it
+# in DEPS_DIGEST. Then push the dependency image to GCR/AR.
 .PHONY: build_deps
 build_deps:
 	$(eval DEPS_DIGEST := $(shell ./hack/skaffold-deps-sha1.sh))
 	docker build \
 		-f deploy/skaffold/Dockerfile.deps \
-		-t gcr.io/$(GCP_PROJECT)/build_deps:$(DEPS_DIGEST) \
+		-t $(SKAFFOLD_DEPS_IMAGE):$(DEPS_DIGEST) \
+		-t $(SKAFFOLD_DEPS_IMAGE):latest \
 		deploy/skaffold
-	docker push gcr.io/$(GCP_PROJECT)/build_deps:$(DEPS_DIGEST)
 
+# Prepares the Docker images needed to run integration tests.
+# First part builds the base image containing all build-time dependencies and pushes to AR.
+# Second part builds the actual image used for running the integration tests,
+# using the 'builddeps' image as a base. It build only up to the 'builder' stage
+# in the Dockerfile.
+# 
+# The push flag is needed to tell Buildx to build the image and push it directly to AR.
+# AR supports multi-architecture images and manifest lists so this will pass for hybrid tests.
+# Note: --provenance=false and --sbom=false are required because modern Buildx 
+# attempts to push attestation manifests that are currently rejected by 
+# GCP Artifact Registry with a '400 Bad Request' error.
+# --cache-from $(IMAGE_REPO_BASE)/$(BUILD_DEPS_REPO_NAME) 
 skaffold-builder-ci:
-	docker build \
-		--cache-from gcr.io/$(GCP_PROJECT)/build_deps \
+	docker system prune -a -f
+	docker buildx build \
+		--provenance=false \
+		--sbom=false \
+		--push \
 		-f deploy/skaffold/Dockerfile.deps \
-		-t gcr.io/$(GCP_PROJECT)/build_deps \
+		-t $(SKAFFOLD_DEPS_IMAGE):latest \
 		.
-	time docker build \
+
+	time docker buildx build \
+	    --provenance=false \
+		--sbom=false \
+	    --push \
 		-f deploy/skaffold/Dockerfile \
 		--target builder \
-		-t gcr.io/$(GCP_PROJECT)/skaffold-builder \
+		-t $(SKAFFOLD_BUILDER_IMAGE):latest \
 		.
 
 .PHONY: skaffold-builder
 skaffold-builder:
-	time docker build \
+	time docker buildx build \
+		--provenance=false --sbom=false --push \
 		-f deploy/skaffold/Dockerfile \
+		--build-arg BASE_IMAGE=$(SKAFFOLD_DEPS_IMAGE):latest \
 		--target builder \
-		-t gcr.io/$(GCP_PROJECT)/skaffold-builder \
+		-t $(SKAFFOLD_BUILDER_IMAGE):latest \
 		.
 
+# Run integration tests within a local kind (Kubernetes IN Docker) cluster.
 .PHONY: integration-in-kind
 integration-in-kind: skaffold-builder
 	echo '{}' > /tmp/docker-config
@@ -232,12 +300,11 @@ integration-in-kind: skaffold-builder
 		-v $(HOME)/.gradle:/root/.gradle \
 		-v $(HOME)/.cache:/root/.cache \
 		-v /tmp/docker-config:/root/.docker/config.json \
-		-v $(CURDIR)/hack/maven/settings.xml:/root/.m2/settings.xml \
 		-e KUBECONFIG=/tmp/kind-config \
 		-e INTEGRATION_TEST_ARGS=$(INTEGRATION_TEST_ARGS) \
 		-e IT_PARTITION=$(IT_PARTITION) \
 		--network kind \
-		gcr.io/$(GCP_PROJECT)/skaffold-builder \
+		$(SKAFFOLD_BUILDER_IMAGE) \
 		sh -eu -c ' \
 			if ! kind get clusters | grep -q kind; then \
 			  trap "kind delete cluster" 0 1 2 15; \
@@ -262,7 +329,7 @@ integration-in-k3d: skaffold-builder
 		-v $(CURDIR)/hack/maven/settings.xml:/root/.m2/settings.xml \
 		-e INTEGRATION_TEST_ARGS=$(INTEGRATION_TEST_ARGS) \
 		-e IT_PARTITION=$(IT_PARTITION) \
-		gcr.io/$(GCP_PROJECT)/skaffold-builder \
+		$(SKAFFOLD_BUILDER_IMAGE) \
 		sh -eu -c ' \
 			if ! k3d cluster list | grep -q k3s-default; then \
 			  trap "k3d cluster delete" 0 1 2 15; \
@@ -276,27 +343,54 @@ integration-in-k3d: skaffold-builder
 			make integration \
 		'
 
+# The `gcloud auth configure-docker` below is needed this starts a separate 
+# container (us-central1-docker.pkg.dev/skaffold-ci-cd/skaffold-builder) to run 
+# the tests. This inner container doesn't inherit the Docker configuration from
+# the host.
+# On these new Kokoro instances, the standard Docker driver doesn't support the
+# multi-platform manifest lists that Skaffold builds for its hybrid tests.
 .PHONY: integration-in-docker
 integration-in-docker: skaffold-builder-ci
+	docker run --privileged --rm mirror.gcr.io/tonistiigi/binfmt --install all
 	docker run --rm \
 		-v /var/run/docker.sock:/var/run/docker.sock \
 		-v $(HOME)/.config/gcloud:/root/.config/gcloud \
-		-v $(GOOGLE_APPLICATION_CREDENTIALS):$(GOOGLE_APPLICATION_CREDENTIALS) \
-		-v $(CURDIR)/hack/maven/settings.xml:/root/.m2/settings.xml \
+		-v $(HOME)/.docker:/root/.docker \
 		-e GCP_ONLY=$(GCP_ONLY) \
 		-e GCP_PROJECT=$(GCP_PROJECT) \
+		-e AR_REGION=$(AR_REGION) \
 		-e GKE_CLUSTER_NAME=$(GKE_CLUSTER_NAME) \
 		-e GKE_ZONE=$(GKE_ZONE) \
 		-e DOCKER_CONFIG=/root/.docker \
-		-e GOOGLE_APPLICATION_CREDENTIALS=$(GOOGLE_APPLICATION_CREDENTIALS) \
+		-e DOCKER_BUILDKIT=1 \
 		-e INTEGRATION_TEST_ARGS=$(INTEGRATION_TEST_ARGS) \
 		-e IT_PARTITION=$(IT_PARTITION) \
-		gcr.io/$(GCP_PROJECT)/skaffold-builder \
-		make integration-tests
+		-e MAVEN_OPTS \
+		-e GRADLE_USER_HOME \
+		$(SKAFFOLD_BUILDER_IMAGE) \
+		sh -c "gcloud auth configure-docker us-central1-docker.pkg.dev -q && \
+		if [ \"\$${GKE_CLUSTER_NAME}\" = \"presubmit-hybrid\" ]; then \
+			echo 'Using docker-container driver for hybrid tests'; \
+			docker buildx rm skaffold-builder || true; \
+			docker buildx create --use --name skaffold-builder --driver docker-container --driver-opt network=host --platform linux/amd64,linux/arm64 --bootstrap; \
+			BUILDER=skaffold-builder; \
+		else \
+			echo 'Using default driver for standard tests'; \
+			docker buildx use default; \
+			BUILDER=default; \
+		fi && \
+		export BUILDX_BUILDER=\$$BUILDER && \
+		echo 'DEBUG: Current buildx builder:'; docker buildx ls && \
+		echo \"DEBUG: BUILDX_BUILDER variable is set to: \$$BUILDX_BUILDER\" && \
+		echo 'DEBUG: Active gcloud account:'; gcloud auth list && \
+		BUILDX_BUILDER=\$$BUILDER make integration-tests"
+
+
 
 .PHONY: submit-build-trigger
 submit-build-trigger:
 	gcloud builds submit . \
+		--project=$(GCP_PROJECT) \
 		--config=deploy/cloudbuild.yaml \
 		--substitutions="_RELEASE_BUCKET=$(RELEASE_BUCKET),COMMIT_SHA=$(COMMIT)"
 

deploy/skaffold/Dockerfile

@@ -12,8 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+ARG BASE_IMAGE=gcr.io/k8s-skaffold/build_deps:latest
 # This base image is built using docker from cache every single time as build step.
-FROM gcr.io/k8s-skaffold/build_deps:latest as build
+FROM ${BASE_IMAGE} as build
 WORKDIR /skaffold
 
 FROM build as builder

deploy/skaffold/Dockerfile.deps

@@ -16,7 +16,7 @@ ARG BASE_PREFIX=mirror.gcr.io/library/
 ARG ARCH=amd64
 
 FROM ${BASE_PREFIX}docker:28.1.1 as docker-source
-FROM docker/buildx-bin:0.23.0 as buildx-source
+FROM docker/buildx-bin:0.31.1 as buildx-source
 FROM ${BASE_PREFIX}golang:1.25.5 as golang-source
 
 # Download kubectl

examples/cross-platform-builds/Dockerfile

@@ -1,13 +1,13 @@
 ARG BASE_PREFIX=mirror.gcr.io/library/
-FROM ${BASE_PREFIX}golang:1.18 as builder
+FROM ${BASE_PREFIX}golang:1.23 as builder
 WORKDIR /code
 COPY main.go .
 COPY go.mod .
 # `skaffold debug` sets SKAFFOLD_GO_GCFLAGS to disable compiler optimizations
 ARG SKAFFOLD_GO_GCFLAGS
 RUN go build -gcflags="${SKAFFOLD_GO_GCFLAGS}" -trimpath -o /app main.go
 
-FROM ${BASE_PREFIX}alpine:3
+FROM ${BASE_PREFIX}alpine:3.20
 # Define GOTRACEBACK to mark this container as using the Go language runtime
 # for `skaffold debug` (https://skaffold.dev/docs/workflows/debug/).
 ENV GOTRACEBACK=single

examples/cross-platform-builds/k8s-pod.yaml

@@ -3,6 +3,18 @@ kind: Pod
 metadata:
   name: getting-started
 spec:
+  tolerations:
+  - operator: "Exists"
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+        - matchExpressions:
+          - key: kubernetes.io/arch
+            operator: In
+            values:
+            - amd64
+            - arm64
   containers:
   - name: getting-started
-    image: skaffold-example
+    image: skaffold-example

examples/cross-platform-builds/skaffold.yaml

@@ -11,6 +11,7 @@ build:
     gitCommit: {}
   local:
     concurrency: 1
+    useDockerCLI: true
 manifests:
   rawYaml:
   - k8s-*

examples/helm-deployment/charts/templates/deployment.yaml

@@ -14,6 +14,8 @@ spec:
       labels:
         app: {{ .Chart.Name }}
     spec:
+      tolerations:
+      - operator: "Exists"
       containers:
       - name: {{ .Chart.Name }}
         image: {{ .Values.image }}

examples/nodejs/backend/Dockerfile

@@ -1,5 +1,5 @@
 ARG BASE_PREFIX=mirror.gcr.io/library/
-FROM node:14.9-alpine
+FROM ${BASE_PREFIX}node:14.9-alpine
 
 USER node
 RUN mkdir /home/node/app

hack/kokoro/presubmit.sh

@@ -14,10 +14,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-export DOCKER_NAMESPACE=gcr.io/k8s-skaffold
-source $KOKORO_GFILE_DIR/common.sh
-
+# Changes the current directory to where Kokoro has checked out the GitHub repository.
 pushd $KOKORO_ARTIFACTS_DIR/github/skaffold >/dev/null
-    GCP_ONLY=true make integration-in-docker
+    # Prevent Jib (Maven/Gradle) from crashing on Kokoro.
+    # Kokoro is a "clean" environment and doesn't have a Maven settings file (~/.m2/settings.xml).
+    # When Skaffold tries to sync that non-existent file into a Docker container, Docker 
+    # mistakenly creates a FOLDER named 'settings.xml' instead. Jib then crashes because 
+    # it can't read a folder as a configuration file. 
+    # Pointing home to /tmp avoids this file-vs-folder conflict.
+    export MAVEN_OPTS="-Duser.home=/tmp"
+    export GRADLE_USER_HOME="/tmp/.gradle"
+    GCP_ONLY=true GCP_PROJECT=skaffold-ci-cd AR_REGION=us-central1 GKE_REGION=us-central1 make integration-in-docker
 popd
 

hack/tests/main.go

@@ -44,6 +44,8 @@ func main() {
 }
 
 func goTest(testArgs []string) error {
+	fmt.Printf("DEBUG: hack/tests/main.go - BUILDX_BUILDER=\"%s\"\n", os.Getenv("BUILDX_BUILDER"))
+
 	args := append([]string{"test", "-json"}, testArgs...)
 	verbose := isVerbose(testArgs)