[WIP]CNTRLPLANE-752: Add manual verification wait step for NodePool API testing

ci-operator/config/openshift/hypershift/openshift-hypershift-release-4.21.yaml

@@ -213,6 +213,23 @@ tests:
       ENABLE_HYPERSHIFT_CERT_ROTATION_SCALE: "true"
       REQUEST_SERVING_COMPONENT_TEST: "true"
     workflow: hypershift-aws-e2e-external
+- always_run: false
+  as: e2e-aws-manual-verify
+  capabilities:
+  - build-tmpfs
+  optional: true
+  steps:
+    cluster_profile: hypershift
+    dependencies:
+      OCP_IMAGE_N1: release:n1minor
+      OCP_IMAGE_N2: release:n2minor
+      OCP_IMAGE_N3: release:n3minor
+      OCP_IMAGE_N4: release:n4minor
+    env:
+      ENABLE_HYPERSHIFT_CERT_ROTATION_SCALE: "true"
+      REQUEST_SERVING_COMPONENT_TEST: "true"
+      WAIT_DURATION: "7200"
+    workflow: hypershift-aws-e2e-manual-verify
 - always_run: false
   as: e2e-aws-minimal
   optional: true

ci-operator/jobs/openshift/hypershift/openshift-hypershift-release-4.21-presubmits.yaml

@@ -440,6 +440,80 @@ presubmits:
         secret:
           secretName: result-aggregator
     trigger: (?m)^/test( | .* )e2e-aws-autonode,?($|\s.*)
+  - agent: kubernetes
+    always_run: false
+    branches:
+    - ^release-4\.21$
+    - ^release-4\.21-
+    cluster: build01
+    context: ci/prow/e2e-aws-manual-verify
+    decorate: true
+    labels:
+      capability/build-tmpfs: build-tmpfs
+      ci-operator.openshift.io/cloud: hypershift
+      ci-operator.openshift.io/cloud-cluster-profile: hypershift
+      ci.openshift.io/generator: prowgen
+      pj-rehearse.openshift.io/can-be-rehearsed: "true"
+    name: pull-ci-openshift-hypershift-release-4.21-e2e-aws-manual-verify
+    optional: true
+    rerun_command: /test e2e-aws-manual-verify
+    spec:
+      containers:
+      - args:
+        - --gcs-upload-secret=/secrets/gcs/service-account.json
+        - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
+        - --lease-server-credentials-file=/etc/boskos/credentials
+        - --report-credentials-file=/etc/report/credentials
+        - --secret-dir=/secrets/ci-pull-credentials
+        - --target=e2e-aws-manual-verify
+        command:
+        - ci-operator
+        image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest
+        imagePullPolicy: Always
+        name: ""
+        resources:
+          requests:
+            cpu: 10m
+        volumeMounts:
+        - mountPath: /etc/boskos
+          name: boskos
+          readOnly: true
+        - mountPath: /secrets/ci-pull-credentials
+          name: ci-pull-credentials
+          readOnly: true
+        - mountPath: /secrets/gcs
+          name: gcs-credentials
+          readOnly: true
+        - mountPath: /secrets/manifest-tool
+          name: manifest-tool-local-pusher
+          readOnly: true
+        - mountPath: /etc/pull-secret
+          name: pull-secret
+          readOnly: true
+        - mountPath: /etc/report
+          name: result-aggregator
+          readOnly: true
+      serviceAccountName: ci-operator
+      volumes:
+      - name: boskos
+        secret:
+          items:
+          - key: credentials
+            path: credentials
+          secretName: boskos-credentials
+      - name: ci-pull-credentials
+        secret:
+          secretName: ci-pull-credentials
+      - name: manifest-tool-local-pusher
+        secret:
+          secretName: manifest-tool-local-pusher
+      - name: pull-secret
+        secret:
+          secretName: registry-pull-credentials
+      - name: result-aggregator
+        secret:
+          secretName: result-aggregator
+    trigger: (?m)^/test( | .* )e2e-aws-manual-verify,?($|\s.*)
   - agent: kubernetes
     always_run: false
     branches:

ci-operator/step-registry/hypershift/aws/e2e/manual-verify/OWNERS

@@ -0,0 +1,12 @@
+approvers:
+- csrwng
+- enxebre
+- sjenning
+- imain
+- davidvossel
+- LiangquanLi930
+reviewers:
+- csrwng
+- enxebre
+- sjenning
+- imain

ci-operator/step-registry/hypershift/aws/e2e/manual-verify/hypershift-aws-e2e-manual-verify-workflow.metadata.json

@@ -0,0 +1,19 @@
+{
+	"path": "hypershift/aws/e2e/manual-verify/hypershift-aws-e2e-manual-verify-workflow.yaml",
+	"owners": {
+		"approvers": [
+			"csrwng",
+			"enxebre",
+			"sjenning",
+			"imain",
+			"davidvossel",
+			"LiangquanLi930"
+		],
+		"reviewers": [
+			"csrwng",
+			"enxebre",
+			"sjenning",
+			"imain"
+		]
+	}
+}

ci-operator/step-registry/hypershift/aws/e2e/manual-verify/hypershift-aws-e2e-manual-verify-workflow.yaml

@@ -0,0 +1,34 @@
+workflow:
+  as: hypershift-aws-e2e-manual-verify
+  documentation: |-
+    The HyperShift aws e2e manual verification workflow creates a test cluster
+    and then pauses for manual verification before running automated tests.
+
+    This workflow creates a HostedCluster with NodePools during the pre-phase,
+    then pauses to allow manual inspection of actual NodePool instances and
+    API configuration before proceeding with the full e2e test suite.
+
+    Administrative access to the control plane is provided via the `KUBECONFIG`
+    environment variable.
+
+    The HyperShift launch capability is currently supported by the HyperShift
+    team. For now, please direct all questions and comments to:
+
+    - Alberto Lamela (agarcial@redhat.com)
+    - Seth Jennings (sjenning@redhat.com)
+    - Cesar Wong (cewong@redhat.com)
+
+    Learn more about HyperShift here: https://github.com/openshift/hypershift
+
+    Track HyperShift's development here: https://issues.redhat.com/projects/HOSTEDCP
+  steps:
+    pre:
+    - ref: ipi-install-rbac
+    - chain: hypershift-setup-nested-management-cluster
+    - ref: hypershift-install
+    - chain: hypershift-aws-create
+    test:
+    - ref: hypershift-aws-wait-for-manual-verification
+    - ref: hypershift-aws-run-e2e-external
+    post:
+    - chain: hypershift-destroy-nested-management-cluster

ci-operator/step-registry/hypershift/aws/wait-for-manual-verification/OWNERS

@@ -0,0 +1,12 @@
+approvers:
+- csrwng
+- enxebre
+- sjenning
+- imain
+- davidvossel
+- LiangquanLi930
+reviewers:
+- csrwng
+- enxebre
+- sjenning
+- imain

ci-operator/step-registry/hypershift/aws/wait-for-manual-verification/hypershift-aws-wait-for-manual-verification-commands.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+
+set -exuo pipefail
+
+# Ensure that oc commands run against the management cluster
+export KUBECONFIG="${SHARED_DIR}/kubeconfig"
+if [[ -f "${SHARED_DIR}/mgmt_kubeconfig" ]]; then
+    export KUBECONFIG="${SHARED_DIR}/mgmt_kubeconfig"
+fi
+
+CLUSTER_NAME="$(echo -n $PROW_JOB_ID|sha256sum|cut -c-20)"
+
+echo "================================================================================"
+echo "MANUAL VERIFICATION WAIT POINT"
+echo "================================================================================"
+echo ""
+echo "Cluster Name: ${CLUSTER_NAME}"
+echo "Namespace: clusters"
+echo ""
+echo "A test cluster has been created with NodePools. You can now inspect them."
+echo ""
+echo "QUICK COMMANDS:"
+echo "-------------------------------------------------------------------------------"
+echo ""
+echo "  # List all NodePools"
+echo "  oc get nodepool -n clusters"
+echo ""
+echo "  # Get first NodePool name"
+echo "  NP=\$(oc get nodepool -n clusters -o jsonpath='{.items[0].metadata.name}')"
+echo ""
+echo "  # View NodePool YAML"
+echo "  oc get nodepool -n clusters \$NP -o yaml"
+echo ""
+echo "  # Check imageType field (CNTRLPLANE-408)"
+echo "  oc get nodepool -n clusters \$NP -o jsonpath='{.spec.platform.aws.imageType}'"
+echo ""
+echo "  # View all NodePool AWS fields"
+echo "  oc explain nodepool.spec.platform.aws"
+echo ""
+echo "  # List HostedClusters"
+echo "  oc get hostedcluster -n clusters"
+echo ""
+echo "  # View Machines (AWS instances)"
+echo "  oc get machine -n clusters-${CLUSTER_NAME}"
+echo ""
+echo "CURRENT STATE:"
+echo "-------------------------------------------------------------------------------"
+echo ""
+echo "HostedClusters:"
+oc get hostedcluster -n clusters -o wide || true
+echo ""
+echo "NodePools:"
+oc get nodepool -n clusters -o wide || true
+echo ""
+echo "Machines:"
+oc get machine -n clusters-${CLUSTER_NAME} -o wide 2>/dev/null || echo "  (No machines found yet)"
+echo ""
+echo "================================================================================"
+echo "Waiting ${WAIT_DURATION:-3600} seconds for manual verification..."
+echo "To skip this wait, you can kill this step from the Prow UI."
+echo "================================================================================"
+
+sleep ${WAIT_DURATION:-3600}
+
+echo ""
+echo "Manual verification wait period completed."
+echo "Proceeding with automated tests..."

ci-operator/step-registry/hypershift/aws/wait-for-manual-verification/hypershift-aws-wait-for-manual-verification-ref.metadata.json

@@ -0,0 +1,19 @@
+{
+	"path": "hypershift/aws/wait-for-manual-verification/hypershift-aws-wait-for-manual-verification-ref.yaml",
+	"owners": {
+		"approvers": [
+			"csrwng",
+			"enxebre",
+			"sjenning",
+			"imain",
+			"davidvossel",
+			"LiangquanLi930"
+		],
+		"reviewers": [
+			"csrwng",
+			"enxebre",
+			"sjenning",
+			"imain"
+		]
+	}
+}

ci-operator/step-registry/hypershift/aws/wait-for-manual-verification/hypershift-aws-wait-for-manual-verification-ref.yaml

@@ -0,0 +1,26 @@
+ref:
+  as: hypershift-aws-wait-for-manual-verification
+  from: cli
+  cli: latest
+  commands: hypershift-aws-wait-for-manual-verification-commands.sh
+  env:
+  - name: WAIT_DURATION
+    default: "3600"
+    documentation: "Duration in seconds to wait for manual verification (default: 1 hour)"
+  grace_period: 10m0s
+  resources:
+    requests:
+      cpu: 100m
+      memory: 200Mi
+  documentation: |-
+    This step pauses the CI workflow to allow manual verification of NodePool API Schema.
+    It displays helpful commands for inspecting NodePools and waits for a configurable
+    duration before proceeding with automated tests.
+
+    Use this step to:
+    - Verify NodePool API schema fields (e.g., imageType from CNTRLPLANE-408)
+    - Inspect NodePool configurations before running tests
+    - Debug NodePool creation issues
+
+    Set WAIT_DURATION environment variable to control how long to wait (in seconds).
+    Default is 3600 seconds (1 hour).