diff --git a/ci-operator/config/openshift/verification-tests/openshift-verification-tests-main__installation-nightly-4.21.yaml b/ci-operator/config/openshift/verification-tests/openshift-verification-tests-main__installation-nightly-4.21.yaml index 1a6d1d4a010c0..af3d7af2416e3 100644 --- a/ci-operator/config/openshift/verification-tests/openshift-verification-tests-main__installation-nightly-4.21.yaml +++ b/ci-operator/config/openshift/verification-tests/openshift-verification-tests-main__installation-nightly-4.21.yaml @@ -1124,6 +1124,12 @@ tests: dependencies: OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE: release:multi-latest workflow: cucushift-installer-rehearse-gcp-cases-valid-confidential-computing +- as: gcp-ipi-valid-dns-private-zone-f14 + cron: 23 2 13,27 * * + steps: + allow_skip_on_success: true + cluster_profile: gcp-qe + workflow: cucushift-installer-rehearse-gcp-cases-valid-dns-private-zone - as: gcp-ipi-dns-peering-zone-f28 cron: 21 0 17 * * steps: diff --git a/ci-operator/jobs/openshift/verification-tests/openshift-verification-tests-main-periodics.yaml b/ci-operator/jobs/openshift/verification-tests/openshift-verification-tests-main-periodics.yaml index a8472fc0a679c..41383098146c7 100644 --- a/ci-operator/jobs/openshift/verification-tests/openshift-verification-tests-main-periodics.yaml +++ b/ci-operator/jobs/openshift/verification-tests/openshift-verification-tests-main-periodics.yaml @@ -36685,6 +36685,81 @@ periodics: - name: result-aggregator secret: secretName: result-aggregator +- agent: kubernetes + cluster: build02 + cron: 23 2 13,27 * * + decorate: true + decoration_config: + skip_cloning: true + extra_refs: + - base_ref: main + org: openshift + repo: verification-tests + labels: + ci-operator.openshift.io/cloud: gcp + ci-operator.openshift.io/cloud-cluster-profile: gcp-qe + ci-operator.openshift.io/variant: installation-nightly-4.21 + ci.openshift.io/generator: prowgen + job-release: "4.21" + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: periodic-ci-openshift-verification-tests-main-installation-nightly-4.21-gcp-ipi-valid-dns-private-zone-f14 + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --target=gcp-ipi-valid-dns-private-zone-f14 + - --variant=installation-nightly-4.21 + command: + - ci-operator + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator - agent: kubernetes cluster: build02 cron: 22 18 13 * * diff --git a/ci-operator/step-registry/cucushift/installer/rehearse/gcp/cases/valid-dns-private-zone/OWNERS b/ci-operator/step-registry/cucushift/installer/rehearse/gcp/cases/valid-dns-private-zone/OWNERS new file mode 120000 index 0000000000000..ec405d65a79df --- /dev/null +++ b/ci-operator/step-registry/cucushift/installer/rehearse/gcp/cases/valid-dns-private-zone/OWNERS @@ -0,0 +1 @@ +../OWNERS \ No newline at end of file diff --git a/ci-operator/step-registry/cucushift/installer/rehearse/gcp/cases/valid-dns-private-zone/cucushift-installer-rehearse-gcp-cases-valid-dns-private-zone-workflow.metadata.json b/ci-operator/step-registry/cucushift/installer/rehearse/gcp/cases/valid-dns-private-zone/cucushift-installer-rehearse-gcp-cases-valid-dns-private-zone-workflow.metadata.json new file mode 100644 index 0000000000000..63825bc127e8b --- /dev/null +++ b/ci-operator/step-registry/cucushift/installer/rehearse/gcp/cases/valid-dns-private-zone/cucushift-installer-rehearse-gcp-cases-valid-dns-private-zone-workflow.metadata.json @@ -0,0 +1,15 @@ +{ + "path": "cucushift/installer/rehearse/gcp/cases/valid-dns-private-zone/cucushift-installer-rehearse-gcp-cases-valid-dns-private-zone-workflow.yaml", + "owners": { + "approvers": [ + "jianlinliu", + "gpei", + "jianli-wei" + ], + "reviewers": [ + "jianlinliu", + "gpei", + "jianli-wei" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/cucushift/installer/rehearse/gcp/cases/valid-dns-private-zone/cucushift-installer-rehearse-gcp-cases-valid-dns-private-zone-workflow.yaml b/ci-operator/step-registry/cucushift/installer/rehearse/gcp/cases/valid-dns-private-zone/cucushift-installer-rehearse-gcp-cases-valid-dns-private-zone-workflow.yaml new file mode 100644 index 0000000000000..6d27223d66035 --- /dev/null +++ b/ci-operator/step-registry/cucushift/installer/rehearse/gcp/cases/valid-dns-private-zone/cucushift-installer-rehearse-gcp-cases-valid-dns-private-zone-workflow.yaml @@ -0,0 +1,10 @@ +workflow: + as: cucushift-installer-rehearse-gcp-cases-valid-dns-private-zone + steps: + pre: + - chain: cucushift-installer-rehearse-gcp-cases-valid-dns-private-zone-provision + - ref: cucushift-installer-reportportal-marker + post: + - ref: send-results-to-reportportal + documentation: |- + Validate DNS private zone in separate project, of Shared VPC (XPN) installations. diff --git a/ci-operator/step-registry/cucushift/installer/rehearse/gcp/cases/valid-dns-private-zone/provision/OWNERS b/ci-operator/step-registry/cucushift/installer/rehearse/gcp/cases/valid-dns-private-zone/provision/OWNERS new file mode 120000 index 0000000000000..ec405d65a79df --- /dev/null +++ b/ci-operator/step-registry/cucushift/installer/rehearse/gcp/cases/valid-dns-private-zone/provision/OWNERS @@ -0,0 +1 @@ +../OWNERS \ No newline at end of file diff --git a/ci-operator/step-registry/cucushift/installer/rehearse/gcp/cases/valid-dns-private-zone/provision/cucushift-installer-rehearse-gcp-cases-valid-dns-private-zone-provision-chain.metadata.json b/ci-operator/step-registry/cucushift/installer/rehearse/gcp/cases/valid-dns-private-zone/provision/cucushift-installer-rehearse-gcp-cases-valid-dns-private-zone-provision-chain.metadata.json new file mode 100644 index 0000000000000..c24ce5f5aae3c --- /dev/null +++ b/ci-operator/step-registry/cucushift/installer/rehearse/gcp/cases/valid-dns-private-zone/provision/cucushift-installer-rehearse-gcp-cases-valid-dns-private-zone-provision-chain.metadata.json @@ -0,0 +1,15 @@ +{ + "path": "cucushift/installer/rehearse/gcp/cases/valid-dns-private-zone/provision/cucushift-installer-rehearse-gcp-cases-valid-dns-private-zone-provision-chain.yaml", + "owners": { + "approvers": [ + "jianlinliu", + "gpei", + "jianli-wei" + ], + "reviewers": [ + "jianlinliu", + "gpei", + "jianli-wei" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/cucushift/installer/rehearse/gcp/cases/valid-dns-private-zone/provision/cucushift-installer-rehearse-gcp-cases-valid-dns-private-zone-provision-chain.yaml b/ci-operator/step-registry/cucushift/installer/rehearse/gcp/cases/valid-dns-private-zone/provision/cucushift-installer-rehearse-gcp-cases-valid-dns-private-zone-provision-chain.yaml new file mode 100644 index 0000000000000..6ed5f6f272eff --- /dev/null +++ b/ci-operator/step-registry/cucushift/installer/rehearse/gcp/cases/valid-dns-private-zone/provision/cucushift-installer-rehearse-gcp-cases-valid-dns-private-zone-provision-chain.yaml @@ -0,0 +1,8 @@ +chain: + as: cucushift-installer-rehearse-gcp-cases-valid-dns-private-zone-provision + steps: + - ref: ipi-install-rbac + - ref: openshift-cluster-bot-rbac + - ref: cucushift-installer-rehearse-gcp-cases-valid-dns-private-zone-provision + documentation: |- + Validate DNS private zone in separate project, of Shared VPC (XPN) installations. diff --git a/ci-operator/step-registry/cucushift/installer/rehearse/gcp/cases/valid-dns-private-zone/provision/cucushift-installer-rehearse-gcp-cases-valid-dns-private-zone-provision-commands.sh b/ci-operator/step-registry/cucushift/installer/rehearse/gcp/cases/valid-dns-private-zone/provision/cucushift-installer-rehearse-gcp-cases-valid-dns-private-zone-provision-commands.sh new file mode 100644 index 0000000000000..759912ca40897 --- /dev/null +++ b/ci-operator/step-registry/cucushift/installer/rehearse/gcp/cases/valid-dns-private-zone/provision/cucushift-installer-rehearse-gcp-cases-valid-dns-private-zone-provision-commands.sh @@ -0,0 +1,199 @@ +#!/bin/bash + +set -o nounset +set -o errexit +set -o pipefail + +# ----------------------------------------- +# OCP-84040 - [IPI-on-GCP] install a cluster into GCP shared VPC with conflicting DNS private zone in separate project +# ----------------------------------------- + +# save the exit code for junit xml file generated in step gather-must-gather +# pre configuration steps before running installation, exit code 100 if failed, +# save to install-pre-config-status.txt +# post check steps after cluster installation, exit code 101 if failed, +# save to install-post-check-status.txt +EXIT_CODE=101 +trap 'if [[ "$?" == 0 ]]; then EXIT_CODE=0; fi; echo "${EXIT_CODE}" > "${SHARED_DIR}/install-post-check-status.txt"; eval "${cleanup_script}"' EXIT TERM + + +if [[ -f "${CLUSTER_PROFILE_DIR}/openshift_gcp_dns_project" ]]; then + PRIVATE_ZONE_PROJECT="$(< ${CLUSTER_PROFILE_DIR}/openshift_gcp_dns_project)" +else + echo "Failed to find out PRIVATE_ZONE_PROJECT, abort." && exit 1 +fi + +export INSTALLER_BINARY="openshift-install" +${INSTALLER_BINARY} version + +GOOGLE_PROJECT_ID="$(< ${CLUSTER_PROFILE_DIR}/openshift_gcp_project)" +export GOOGLE_CLOUD_KEYFILE_JSON="${CLUSTER_PROFILE_DIR}/gce.json" +sa_email=$(jq -r .client_email ${GOOGLE_CLOUD_KEYFILE_JSON}) +if ! gcloud auth list | grep -E "\*\s+${sa_email}" +then + gcloud auth activate-service-account --key-file="${GOOGLE_CLOUD_KEYFILE_JSON}" + gcloud config set project "${GOOGLE_PROJECT_ID}" +fi + +BASE_DOMAIN="$(< ${CLUSTER_PROFILE_DIR}/public_hosted_zone)" +REGION=${LEASED_RESOURCE} + +SSH_PUB_KEY=$(<"${CLUSTER_PROFILE_DIR}/ssh-publickey") +PULL_SECRET=$(<"${CLUSTER_PROFILE_DIR}/pull-secret") + +CLUSTER_NAME="${NAMESPACE}-${UNIQUE_HASH}" +HOST_PROJECT=$(jq -r '.hostProject' "${CLUSTER_PROFILE_DIR}/xpn_project_setting.json") +HOST_PROJECT_NETWORK=$(jq -r '.clusterNetwork' "${CLUSTER_PROFILE_DIR}/xpn_project_setting.json") +HOST_PROJECT_COMPUTE_SUBNET=$(jq -r '.computeSubnet' "${CLUSTER_PROFILE_DIR}/xpn_project_setting.json") +HOST_PROJECT_CONTROL_SUBNET=$(jq -r '.controlSubnet' "${CLUSTER_PROFILE_DIR}/xpn_project_setting.json") +NETWORK=$(basename ${HOST_PROJECT_NETWORK}) +CONTROL_SUBNET=$(basename ${HOST_PROJECT_CONTROL_SUBNET}) +COMPUTE_SUBNET=$(basename ${HOST_PROJECT_COMPUTE_SUBNET}) + +function save_artifacts() +{ + local -r install_dir="$1" + local -r testing_scenario_num="$2" + local current_time + + set +o errexit + current_time=$(date +%s) + sed ' + s/password: .*/password: REDACTED/; + s/X-Auth-Token.*/X-Auth-Token REDACTED/; + s/UserData:.*,/UserData: REDACTED,/; + ' "${install_dir}/.openshift_install.log" > "${ARTIFACT_DIR}/cluster_${testing_scenario_num}_openshift_install-${current_time}.log" + + set -o errexit +} + +function create_install_config() +{ + local -r cluster_name=$1; shift + local -r dns_private_zone_name=$1; shift + local -r dns_private_zone_project=$1; shift + local -r install_dir=$1 + + cat > ${install_dir}/install-config.yaml << EOF +apiVersion: v1 +baseDomain: ${BASE_DOMAIN} +credentialsMode: Passthrough +compute: +- architecture: amd64 + hyperthreading: Enabled + name: worker + platform: {} + replicas: 3 +controlPlane: + architecture: amd64 + hyperthreading: Enabled + name: master + platform: {} + replicas: 3 +metadata: + creationTimestamp: null + name: ${cluster_name} +networking: + clusterNetwork: + - cidr: 10.128.0.0/14 + hostPrefix: 23 + machineNetwork: + - cidr: 10.0.0.0/16 + serviceNetwork: + - 172.30.0.0/16 +platform: + gcp: + projectID: ${GOOGLE_PROJECT_ID} + region: ${REGION} + networkProjectID: ${HOST_PROJECT} + network: ${NETWORK} + controlPlaneSubnet: ${CONTROL_SUBNET} + computeSubnet: ${COMPUTE_SUBNET} + dns: + privateZone: + name: ${dns_private_zone_name} + projectID: ${dns_private_zone_project} +publish: External +pullSecret: > + ${PULL_SECRET} +sshKey: | + ${SSH_PUB_KEY} +EOF +} + +function run_command() { + local CMD="$1" + echo "Running command: ${CMD}" + eval "${CMD}" +} + +## main +result=0 +cleanup_script=$(mktemp); chmod +x "${cleanup_script}" +tmp_output=$(mktemp) + +echo "$(date -u --rfc-3339=seconds) - Scenario A: a DNS private zone of different name, but with matching dns name and network is present" +cluster_name="${CLUSTER_NAME}-$RANDOM" +install_dir="/tmp/${cluster_name}" +mkdir -p "${install_dir}" 2>/dev/null + +expected_zone_name="test-$RANDOM-priv-zone" +create_install_config "${cluster_name}" "${expected_zone_name}" "${PRIVATE_ZONE_PROJECT}" "${install_dir}" + +wrong_zone_name="test-$RANDOM-private-zone" +echo "$(date -u --rfc-3339=seconds) - Scenario A: expected_zone_name '${expected_zone_name}', wrong_zone_name '${wrong_zone_name}'" +expected_err_msg="failed to create install config: platform.gcp.dns.privateZone.name: Invalid value: \"${expected_zone_name}\": found existing private zone ${wrong_zone_name} in project ${PRIVATE_ZONE_PROJECT} with DNS name ${cluster_name}.${BASE_DOMAIN}" + +cmd="gcloud --project ${PRIVATE_ZONE_PROJECT} dns managed-zones create ${wrong_zone_name} --dns-name ${cluster_name}.${BASE_DOMAIN}. --visibility=private --networks ${HOST_PROJECT_NETWORK} --description \"private zone of OCP cluster '${cluster_name}'\"" +run_command "${cmd}" + +cp "${install_dir}/install-config.yaml" "${install_dir}/install-config.yaml.bak" +echo "$(date -u --rfc-3339=seconds) - INFO '${INSTALLER_BINARY} create manifests --dir ${install_dir}'" +${INSTALLER_BINARY} create manifests --dir ${install_dir} &> ${tmp_output} || true +cat "${tmp_output}" +if ! grep -qF "${expected_err_msg}" "${tmp_output}"; then + echo "$(date -u --rfc-3339=seconds) - Scenario A: FAILED, the expected error messages are not found in '.openshift_install.log'." + result=$((result+1)) +else + echo "$(date -u --rfc-3339=seconds) - Scenario A: PASSED, found the expected error messages in '.openshift_install.log'." +fi +save_artifacts "${install_dir}" "scenario-a" + +cmd="gcloud --project ${PRIVATE_ZONE_PROJECT} dns managed-zones delete -q ${wrong_zone_name}" +#run_command "${cmd}" +echo "${cmd} || true" >> "${cleanup_script}" +rm -fr "${install_dir}" + +echo "$(date -u --rfc-3339=seconds) - Scenario B: a DNS private zone of matching name and network, but different dns name is present" +cluster_name="${CLUSTER_NAME}-$RANDOM" +install_dir="/tmp/${cluster_name}" +mkdir -p "${install_dir}" 2>/dev/null + +expected_zone_name="test-$RANDOM-priv-zone" +create_install_config "${cluster_name}" "${expected_zone_name}" "${PRIVATE_ZONE_PROJECT}" "${install_dir}" + +echo "$(date -u --rfc-3339=seconds) - Scenario B: expected_zone_name '${expected_zone_name}'" +expected_err_msg="failed to create install config: baseDomain: Invalid value: \"${BASE_DOMAIN}\": failed to find matching DNS zone for ${expected_zone_name} with DNS name ${cluster_name}.${BASE_DOMAIN}" + +cmd="gcloud --project ${PRIVATE_ZONE_PROJECT} dns managed-zones create ${expected_zone_name} --dns-name ${cluster_name}.dns.${BASE_DOMAIN}. --visibility=private --networks ${HOST_PROJECT_NETWORK} --description \"private zone of OCP cluster '${cluster_name}'\"" +run_command "${cmd}" + +cp "${install_dir}/install-config.yaml" "${install_dir}/install-config.yaml.bak" +echo "$(date -u --rfc-3339=seconds) - INFO '${INSTALLER_BINARY} create manifests --dir ${install_dir}'" +${INSTALLER_BINARY} create manifests --dir ${install_dir} &> ${tmp_output} || true +cat "${tmp_output}" +if ! grep -qF "${expected_err_msg}" "${tmp_output}"; then + echo "$(date -u --rfc-3339=seconds) - Scenario B: FAILED, the expected error messages are not found in '.openshift_install.log'." + result=$((result+1)) +else + echo "$(date -u --rfc-3339=seconds) - Scenario B: PASSED, found the expected error messages in '.openshift_install.log'." +fi +save_artifacts "${install_dir}" "scenario-b" + +cmd="gcloud --project ${PRIVATE_ZONE_PROJECT} dns managed-zones delete -q ${expected_zone_name}" +#run_command "${cmd}" +echo "${cmd} || true" >> "${cleanup_script}" +rm -fr "${install_dir}" + +echo "Exit code: '$result'" +exit $result diff --git a/ci-operator/step-registry/cucushift/installer/rehearse/gcp/cases/valid-dns-private-zone/provision/cucushift-installer-rehearse-gcp-cases-valid-dns-private-zone-provision-ref.metadata.json b/ci-operator/step-registry/cucushift/installer/rehearse/gcp/cases/valid-dns-private-zone/provision/cucushift-installer-rehearse-gcp-cases-valid-dns-private-zone-provision-ref.metadata.json new file mode 100644 index 0000000000000..1194f30c09cd3 --- /dev/null +++ b/ci-operator/step-registry/cucushift/installer/rehearse/gcp/cases/valid-dns-private-zone/provision/cucushift-installer-rehearse-gcp-cases-valid-dns-private-zone-provision-ref.metadata.json @@ -0,0 +1,15 @@ +{ + "path": "cucushift/installer/rehearse/gcp/cases/valid-dns-private-zone/provision/cucushift-installer-rehearse-gcp-cases-valid-dns-private-zone-provision-ref.yaml", + "owners": { + "approvers": [ + "jianlinliu", + "gpei", + "jianli-wei" + ], + "reviewers": [ + "jianlinliu", + "gpei", + "jianli-wei" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/cucushift/installer/rehearse/gcp/cases/valid-dns-private-zone/provision/cucushift-installer-rehearse-gcp-cases-valid-dns-private-zone-provision-ref.yaml b/ci-operator/step-registry/cucushift/installer/rehearse/gcp/cases/valid-dns-private-zone/provision/cucushift-installer-rehearse-gcp-cases-valid-dns-private-zone-provision-ref.yaml new file mode 100644 index 0000000000000..ee11b58711677 --- /dev/null +++ b/ci-operator/step-registry/cucushift/installer/rehearse/gcp/cases/valid-dns-private-zone/provision/cucushift-installer-rehearse-gcp-cases-valid-dns-private-zone-provision-ref.yaml @@ -0,0 +1,11 @@ +ref: + as: cucushift-installer-rehearse-gcp-cases-valid-dns-private-zone-provision + from: upi-installer + grace_period: 10m + commands: cucushift-installer-rehearse-gcp-cases-valid-dns-private-zone-provision-commands.sh + resources: + requests: + cpu: 10m + memory: 100Mi + documentation: >- + Validate DNS private zone in separate project, of Shared VPC (XPN) installations.