Finished Chapter05

Author: Barroclough01

Chapter05/going_with_the_kubeflow/README.md

@@ -0,0 +1,37 @@
+## Proxy-agent on Minikube
+
+When deploying Kubeflow Pipelines on a local cluster (e.g., Minikube), the `proxy-agent` may crash
+because it attempts to auto-register using the Google Compute metadata service and expects cloud
+credentials or an external proxy/back-end configuration. If you see `proxy-agent` in
+`CrashLoopBackOff` or `Error`, you can either disable it for local use or configure it properly for
+cloud use.
+
+Quick fixes for local Minikube:
+
+- Scale the agent to zero replicas (simple and safe):
+
+```sh
+kubectl -n kubeflow scale deployment proxy-agent --replicas=0
+```
+
+- Or set a minimal `Hostname` so the agent doesn't query the GCE metadata server:
+
+```sh
+kubectl -n kubeflow patch configmap inverse-proxy-config --type=merge \
+	-p '{"data":{"Hostname":"minikube"}}'
+kubectl -n kubeflow rollout restart deployment proxy-agent
+```
+
+For cloud (GCP) usage: populate `inverse-proxy-config` with `Hostname`, `ProxyUrl`, and `BackendId`,
+and ensure the cluster has appropriate Google Application Default Credentials / service account
+access so the agent can register successfully.
+
+Note: The `PIPELINE_VERSION` used by the deploy scripts selects the upstream manifests (e.g., `1.8.5`,
+`2.15.2`) which may reference specific container image tags; if you get `ImagePullBackOff` errors,
+check image tags in the release manifests or try a different `PIPELINE_VERSION`.
+
+Quick helper scripts
+
+- `disable_proxy_agent.ps1` — PowerShell helper to set `inverse-proxy-config` Hostname and scale the
+	`proxy-agent` deployment to `0`. Useful for Minikube/local runs.
+- `disable_proxy_agent.sh` — same as above for users running Bash.

Chapter05/going_with_the_kubeflow/going_with_the_kubeflow/deploy_kubeflow_pipelines.ps1

@@ -0,0 +1,56 @@
+<#
+.SYNOPSIS
+    Deploy Kubeflow Pipelines to a Kubernetes cluster (PowerShell version of deploy_kubeflow_pipelines.zsh)
+
+.DESCRIPTION
+    Sets the PIPELINE_VERSION environment variable and applies the Kubeflow Pipelines kustomize manifests
+    from the official GitHub repo. Intended to be run in an elevated PowerShell session with `kubectl`
+    configured to point at the target cluster.
+
+.EXAMPLE
+    ./deploy_kubeflow_pipelines.ps1
+    ./deploy_kubeflow_pipelines.ps1 -PipelineVersion '1.8.5'
+#>
+
+param(
+    [string]
+    $PipelineVersion = '2.15.2'
+)
+
+Set-StrictMode -Version Latest
+
+function Assert-CommandExists {
+    param([string]$Cmd)
+    if (-not (Get-Command $Cmd -ErrorAction SilentlyContinue)) {
+        Write-Error "Required command '$Cmd' not found in PATH. Install it and try again."
+        exit 2
+    }
+}
+
+Assert-CommandExists -Cmd 'kubectl'
+
+$env:PIPELINE_VERSION = $PipelineVersion
+Write-Host "Using PIPELINE_VERSION=$env:PIPELINE_VERSION"
+
+try {
+    Write-Host 'Applying cluster-scoped resources...'
+    kubectl apply -k "github.com/kubeflow/pipelines/manifests/kustomize/cluster-scoped-resources?ref=$env:PIPELINE_VERSION"
+
+    Write-Host 'Waiting for Applications CRD to be established...'
+    kubectl wait --for condition=established --timeout=60s crd/applications.app.k8s.io
+
+    Write-Host 'Applying environment manifests (dev)...'
+    kubectl apply -k "github.com/kubeflow/pipelines/manifests/kustomize/env/dev?ref=$env:PIPELINE_VERSION"
+
+    Write-Host 'Deployment applied. Recommended verification commands:'
+    Write-Host "  kubectl -n kubeflow get pods"
+    Write-Host "  kubectl -n kubeflow get pods --watch"
+    Write-Host "To access the UI run: kubectl port-forward -n kubeflow svc/ml-pipeline-ui 8080:80"
+    Write-Host "Then open http://localhost:8080 in your browser (Start-Process 'http://localhost:8080')."
+}
+catch {
+    Write-Error "Deployment failed: $_"
+    exit 1
+}
+
+Write-Host 'Done.'

Chapter05/going_with_the_kubeflow/going_with_the_kubeflow/disable_proxy_agent.ps1

@@ -0,0 +1,34 @@
+<#
+.SYNOPSIS
+    Disable the proxy-agent for local Minikube environments
+
+.DESCRIPTION
+    Sets a minimal Hostname in the inverse-proxy-config ConfigMap and scales the proxy-agent
+    deployment to 0 replicas to prevent CrashLoopBackOff when running Kubeflow Pipelines locally.
+
+.EXAMPLE
+    ./disable_proxy_agent.ps1
+#>
+
+Set-StrictMode -Version Latest
+
+if (-not (Get-Command kubectl -ErrorAction SilentlyContinue)) {
+    Write-Error 'kubectl not found in PATH. Install/configure kubectl and try again.'
+    exit 2
+}
+
+Write-Host 'Patching inverse-proxy-config ConfigMap (Hostname=minikube)...'
+kubectl -n kubeflow apply -f - <<'YAML'
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: inverse-proxy-config
+  namespace: kubeflow
+data:
+  Hostname: minikube
+YAML
+
+Write-Host 'Scaling proxy-agent deployment to 0 replicas...'
+kubectl -n kubeflow scale deployment proxy-agent --replicas=0
+
+Write-Host 'Done.'

Chapter05/going_with_the_kubeflow/going_with_the_kubeflow/disable_proxy_agent.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# Disable proxy-agent for local Minikube environments
+set -euo pipefail
+
+command -v kubectl >/dev/null 2>&1 || { echo "kubectl not found in PATH" >&2; exit 2; }
+
+cat <<EOF | kubectl -n kubeflow apply -f -
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: inverse-proxy-config
+  namespace: kubeflow
+data:
+  Hostname: minikube
+EOF
+
+kubectl -n kubeflow scale deployment proxy-agent --replicas=0
+echo "Done. proxy-agent scaled to 0 and Hostname set to minikube."

Chapter05/going_with_the_kubeflow/going_with_the_kubeflow/pipeline_basic.py

@@ -1,15 +1,11 @@
 from typing import List
 
-from kfp import Client
-import kfp.dsl
-from kfp.v2 import dsl
-from kfp.v2.dsl import Dataset
-from kfp.v2.dsl import Input
-from kfp.v2.dsl import Model
-from kfp.v2.dsl import Output
+from kfp import dsl, Client
+from kfp.dsl import Dataset, Input, Model, Output
 
 
-@dsl.component(packages_to_install=['pandas==1.3.5'])
+
+@dsl.component(packages_to_install=['pandas==2.2.3'], base_image='python:3.11')
 def create_dataset(iris_dataset: Output[Dataset]):
     import pandas as pd
 
@@ -24,7 +20,7 @@ def create_dataset(iris_dataset: Output[Dataset]):
         df.to_csv(f)
 
 
-@dsl.component(packages_to_install=['pandas==1.3.5', 'scikit-learn==1.0.2'])
+@dsl.component(packages_to_install=['pandas==2.2.3', 'scikit-learn==1.4.2'], base_image='python:3.11')
 def normalize_dataset(
     input_iris_dataset: Input[Dataset],
     normalized_iris_dataset: Output[Dataset],
@@ -54,7 +50,7 @@ def normalize_dataset(
         df.to_csv(f)
 
 
-@dsl.component(packages_to_install=['pandas==1.3.5', 'scikit-learn==1.0.2'])
+@dsl.component(packages_to_install=['pandas==2.2.3', 'scikit-learn==1.4.2'], base_image='python:3.11')
 def train_model(
     normalized_iris_dataset: Input[Dataset],
     model: Output[Model],
@@ -80,38 +76,46 @@ def train_model(
         pickle.dump(clf, f)
 
 
-@dsl.pipeline(name='iris-training-pipeline')
+@dsl.pipeline(
+    name="iris-training-pipeline",
+    description="Iris pipeline",
+    pipeline_root="",  # optional
+)
 def my_pipeline(
     standard_scaler: bool,
     min_max_scaler: bool,
     neighbors: List[int],
 ):
     create_dataset_task = create_dataset()
+    create_dataset_task.set_caching_options(False)
 
     normalize_dataset_task = normalize_dataset(
-        input_iris_dataset=create_dataset_task.outputs['iris_dataset'],
-        standard_scaler=True,
-        min_max_scaler=False)
+        input_iris_dataset=create_dataset_task.outputs["iris_dataset"],
+        standard_scaler=standard_scaler,
+        min_max_scaler=min_max_scaler,
+    )
+    normalize_dataset_task.set_caching_options(False)
 
     with dsl.ParallelFor(neighbors) as n_neighbors:
-        train_model(
-            normalized_iris_dataset=normalize_dataset_task
-            .outputs['normalized_iris_dataset'],
-            n_neighbors=n_neighbors)
+        t = train_model(
+            normalized_iris_dataset=normalize_dataset_task.outputs["normalized_iris_dataset"],
+            n_neighbors=n_neighbors,
+        )
+        t.set_caching_options(False)
 
 
-endpoint = 'http://localhost:8080' #as a result of port-forwarding.
-# got this from running kubectl cluster-info --context kind-mlewp (this is cluster name)
-#endpoint = 'https://127.0.0.1:50663' 
+endpoint = "http://localhost:8080"
 kfp_client = Client(host=endpoint)
+
 run = kfp_client.create_run_from_pipeline_func(
     my_pipeline,
-    mode=kfp.dsl.PipelineExecutionMode.V2_COMPATIBLE,
     arguments={
-        'min_max_scaler': True,
-        'standard_scaler': False,
-        'neighbors': [3, 6, 9]
+        "min_max_scaler": True,
+        "standard_scaler": False,
+        "neighbors": [3, 6, 9],
     },
+    enable_caching=False,
 )
-url = f'{endpoint}/#/runs/details/{run.run_id}'
-print(url)
+
+url = f"{endpoint}/#/runs/details/{run.run_id}"
+print(url)