diff --git a/test/resources/c2cc.resource b/test/resources/c2cc.resource index f51aa73da7..a35dbc940f 100644 --- a/test/resources/c2cc.resource +++ b/test/resources/c2cc.resource @@ -414,7 +414,7 @@ DNS Lookup Should Succeed Verify All RemoteClusters Healthy [Documentation] Wait for all RemoteCluster CRs on all clusters to report Healthy. - FOR ${alias} IN cluster-a cluster-b cluster-c + FOR ${alias} IN @{ALL_CLUSTERS} Wait Until Keyword Succeeds 3m 10s ... Verify RemoteCluster State ${alias} Healthy END @@ -572,3 +572,57 @@ Reconnect To Cluster Deregister Remote Cluster ${alias} Wait Until Keyword Succeeds ${timeout} 15s ... Register Remote Cluster ${alias} ${host} ${ssh_port} ${kubeconfig} + +Reboot Clusters Simultaneously + [Documentation] Reboot one or more clusters and wait for all to come back + ... with a new boot ID and greenboot health check passed. + ... Issues reboot on all targets before waiting, so reboots overlap. + [Arguments] @{cluster_aliases} + VAR &{boot_ids}= &{EMPTY} + FOR ${alias} IN @{cluster_aliases} + ${conn_id}= Get From Dictionary ${C2CC_SSH_IDS} ${alias} + SSHLibrary.Switch Connection ${conn_id} + ${bootid}= Get Current Boot Id + Set To Dictionary ${boot_ids} ${alias} ${bootid} + END + FOR ${alias} IN @{cluster_aliases} + ${conn_id}= Get From Dictionary ${C2CC_SSH_IDS} ${alias} + SSHLibrary.Switch Connection ${conn_id} + SSHLibrary.Start Command reboot sudo=True + END + FOR ${alias} IN @{cluster_aliases} + ${old_bootid}= Get From Dictionary ${boot_ids} ${alias} + Wait Until Keyword Succeeds 10m 15s + ... Cluster Should Be Rebooted ${alias} ${old_bootid} + END + +Cluster Should Be Rebooted + [Documentation] Assert that a cluster has rebooted and greenboot health check has passed. + [Arguments] ${alias} ${old_bootid} + Reconnect Cluster ${alias} + ${new_bootid}= Get Current Boot Id + Should Not Be Equal ${new_bootid} ${old_bootid} + ${stdout}= Command On Cluster ${alias} + ... systemctl show -p SubState greenboot-healthcheck.service --value + Should Not Be Equal As Strings ${stdout} failed strip_spaces=True + ... msg=Greenboot healthcheck FAILED on ${alias}. Check greenboot logs. + Should Be Equal As Strings ${stdout} exited strip_spaces=True + +Reconnect Cluster + [Documentation] Re-establish SSH connection to a cluster after reboot. + ... Resolves host/port from the alias and delegates to Reconnect To Cluster. + [Arguments] ${alias} + IF '${alias}' == 'cluster-a' + VAR ${host}= ${USHIFT_HOST} + VAR ${port}= ${SSH_PORT} + ELSE IF '${alias}' == 'cluster-b' + VAR ${host}= ${HOST2_IP} + VAR ${port}= ${HOST2_SSH_PORT} + ELSE IF '${alias}' == 'cluster-c' + VAR ${host}= ${HOST3_IP} + VAR ${port}= ${HOST3_SSH_PORT} + ELSE + Fail Unknown cluster alias: ${alias} + END + ${kubeconfig}= Get From Dictionary ${C2CC_KUBECONFIGS} ${alias} + Reconnect To Cluster ${alias} ${host} ${port} ${kubeconfig} timeout=10m diff --git a/test/scenarios-bootc/c2cc/el102-src@c2cc-disruptive.sh b/test/scenarios-bootc/c2cc/el102-src@c2cc-disruptive.sh index 49220a4c07..35c1c30644 100755 --- a/test/scenarios-bootc/c2cc/el102-src@c2cc-disruptive.sh +++ b/test/scenarios-bootc/c2cc/el102-src@c2cc-disruptive.sh @@ -20,5 +20,5 @@ scenario_remove_vms() { scenario_run_tests() { # shellcheck disable=SC2119 configure_c2cc_hosts - c2cc_run_tests "suites/c2cc/disruptive.robot" "" "" "disruptive" + c2cc_run_tests "suites/c2cc/" "" "" "disruptive" } diff --git a/test/scenarios-bootc/c2cc/el98-src@c2cc-disruptive.sh b/test/scenarios-bootc/c2cc/el98-src@c2cc-disruptive.sh index 37ab603ad2..79cab102e4 100755 --- a/test/scenarios-bootc/c2cc/el98-src@c2cc-disruptive.sh +++ b/test/scenarios-bootc/c2cc/el98-src@c2cc-disruptive.sh @@ -20,5 +20,5 @@ scenario_remove_vms() { scenario_run_tests() { # shellcheck disable=SC2119 configure_c2cc_hosts - c2cc_run_tests "suites/c2cc/disruptive.robot" "" "" "disruptive" + c2cc_run_tests "suites/c2cc/" "" "" "disruptive" } diff --git a/test/suites/c2cc/reboot.robot b/test/suites/c2cc/reboot.robot new file mode 100644 index 0000000000..25df402ad6 --- /dev/null +++ b/test/suites/c2cc/reboot.robot @@ -0,0 +1,173 @@ +*** Settings *** +Documentation Verify C2CC survives VM reboots in escalating scenarios. +... Tests single cluster, two clusters simultaneously, and all three +... clusters simultaneously. After each reboot cycle, full-stack +... verification confirms connectivity, infrastructure, health probes, +... and DNS all recover. + +Resource ../../resources/c2cc.resource + +Suite Setup Setup +Suite Teardown Teardown + +Test Tags disruptive + + +*** Test Cases *** +Reboot Single Cluster + [Documentation] Reboot cluster-a while cluster-b and cluster-c remain up. + ... Verifies that the rebooted cluster re-establishes C2CC connectivity + ... with both peers. + [Setup] Verify All RemoteClusters Healthy + Reboot Clusters Simultaneously cluster-a + Wait For Clusters Ready + Verify Full C2CC Stack + +Reboot Two Clusters Simultaneously + [Documentation] Reboot cluster-b and cluster-c at the same time. + ... The surviving cluster-a must wait for both peers to recover. + ... The two rebooted clusters must also reconnect with each other. + [Setup] Verify All RemoteClusters Healthy + Reboot Clusters Simultaneously cluster-b cluster-c + Wait For Clusters Ready + Verify Full C2CC Stack + +Reboot All Three Clusters Simultaneously + [Documentation] Reboot all three clusters at once. + ... Every cluster starts from scratch simultaneously — no running peer + ... to reference. All must independently reconstruct C2CC state. + [Setup] Verify All RemoteClusters Healthy + Reboot Clusters Simultaneously cluster-a cluster-b cluster-c + Wait For Clusters Ready + Verify Full C2CC Stack + + +*** Keywords *** +Setup + [Documentation] Set up clusters and deploy test workloads on all. + Check Required Env Variables + Login MicroShift Host + Setup Kubeconfig + Logout MicroShift Host + + Register Remote Cluster cluster-a ${USHIFT_HOST} ${SSH_PORT} ${KUBECONFIG} + Register Remote Cluster cluster-b ${HOST2_IP} ${HOST2_SSH_PORT} ${KUBECONFIG_B} + Register Remote Cluster cluster-c ${HOST3_IP} ${HOST3_SSH_PORT} ${KUBECONFIG_C} + Deploy Test Workloads + Verify Full C2CC Stack + +Teardown + [Documentation] Remove test workloads and close connections. + Cleanup Test Workloads + Teardown All Remote Clusters + Remove Kubeconfig + +Wait For Clusters Ready + [Documentation] Wait for test pods and service endpoints to become ready + ... after a reboot cycle. + Wait For Test Pods + Wait For Service Endpoints + +Verify Full C2CC Stack + [Documentation] Comprehensive verification of all C2CC components across all clusters. + Wait Until Keyword Succeeds 10m 10s Verify C2CC Connectivity + Wait Until Keyword Succeeds 10m 10s Verify C2CC Infrastructure + Wait Until Keyword Succeeds 10m 10s Verify C2CC Health Probes + Wait Until Keyword Succeeds 10m 10s Verify C2CC DNS + +Verify C2CC Connectivity + [Documentation] Verify pod-to-pod, pod-to-service connectivity and source IP preservation + ... across all 6 cluster pairs. + FOR ${src} ${dst} IN + ... cluster-a cluster-b + ... cluster-a cluster-c + ... cluster-b cluster-a + ... cluster-b cluster-c + ... cluster-c cluster-a + ... cluster-c cluster-b + Test Connectivity Between Clusters ${src} ${dst} pod + Test Connectivity Between Clusters ${src} ${dst} service + Test Source IP Preserved Between Clusters ${src} ${dst} pod + Test Source IP Preserved Between Clusters ${src} ${dst} service + END + +Verify C2CC Infrastructure + [Documentation] Verify routes, IP rules, nftables, OVN static routes, + ... and node annotations for all cluster-peer combinations. + Verify Infra For Remote Peer + ... cluster-a + ... ${CLUSTER_B_POD_CIDR} + ... ${CLUSTER_B_SVC_CIDR} + ... ${CLUSTER_A_SVC_CIDR} + Verify Infra For Remote Peer + ... cluster-a + ... ${CLUSTER_C_POD_CIDR} + ... ${CLUSTER_C_SVC_CIDR} + ... ${CLUSTER_A_SVC_CIDR} + Verify Infra For Remote Peer + ... cluster-b + ... ${CLUSTER_A_POD_CIDR} + ... ${CLUSTER_A_SVC_CIDR} + ... ${CLUSTER_B_SVC_CIDR} + Verify Infra For Remote Peer + ... cluster-b + ... ${CLUSTER_C_POD_CIDR} + ... ${CLUSTER_C_SVC_CIDR} + ... ${CLUSTER_B_SVC_CIDR} + Verify Infra For Remote Peer + ... cluster-c + ... ${CLUSTER_A_POD_CIDR} + ... ${CLUSTER_A_SVC_CIDR} + ... ${CLUSTER_C_SVC_CIDR} + Verify Infra For Remote Peer + ... cluster-c + ... ${CLUSTER_B_POD_CIDR} + ... ${CLUSTER_B_SVC_CIDR} + ... ${CLUSTER_C_SVC_CIDR} + +Verify Infra For Remote Peer + [Documentation] Verify all infrastructure components on a cluster for one remote peer. + [Arguments] ${alias} ${remote_pod_cidr} ${remote_svc_cidr} ${local_svc_cidr} + Verify Routes In Table 200 ${alias} ${remote_pod_cidr} ${remote_svc_cidr} + Verify IP Rules For Table 200 ${alias} ${remote_pod_cidr} ${remote_svc_cidr} + Verify Routes In Table 201 ${alias} ${local_svc_cidr} + Verify Service IP Rules ${alias} ${remote_pod_cidr} ${remote_svc_cidr} ${local_svc_cidr} + Verify NFTables Bypass Rules ${alias} ${remote_pod_cidr} ${remote_svc_cidr} + Verify OVN Static Routes ${alias} ${remote_pod_cidr} ${remote_svc_cidr} + Verify Node SNAT Annotation ${alias} ${remote_pod_cidr} ${remote_svc_cidr} + Verify C2CC Tracking Annotation ${alias} ${remote_pod_cidr} ${remote_svc_cidr} + +Verify C2CC Health Probes + [Documentation] Verify all RemoteCluster CRs report Healthy with populated timestamps. + FOR ${alias} IN cluster-a cluster-b cluster-c + Verify RemoteCluster State ${alias} Healthy + ${stdout}= Oc On Cluster ${alias} + ... oc get remoteclusters.microshift.io -o jsonpath='{.items[*].status.lastProbeTime}' + Should Not Be Empty ${stdout} + ${stdout}= Oc On Cluster ${alias} + ... oc get remoteclusters.microshift.io -o jsonpath='{.items[*].status.lastSuccessfulProbe}' + Should Not Be Empty ${stdout} + END + +Verify C2CC DNS + [Documentation] Verify CoreDNS Corefile contains C2CC server blocks and + ... cross-cluster DNS resolution works for all pairs. + Verify Corefile Contains C2CC Server Block cluster-a ${CLUSTER_B_DOMAIN} + Verify Corefile Contains C2CC Server Block cluster-a ${CLUSTER_C_DOMAIN} + Verify Corefile Contains C2CC Server Block cluster-b ${CLUSTER_A_DOMAIN} + Verify Corefile Contains C2CC Server Block cluster-b ${CLUSTER_C_DOMAIN} + Verify Corefile Contains C2CC Server Block cluster-c ${CLUSTER_A_DOMAIN} + Verify Corefile Contains C2CC Server Block cluster-c ${CLUSTER_B_DOMAIN} + Verify DNS Connectivity All Pairs + +Verify DNS Connectivity All Pairs + [Documentation] Verify cross-cluster DNS-based service access for all cluster pairs. + FOR ${src} ${dst} IN + ... cluster-a cluster-b + ... cluster-a cluster-c + ... cluster-b cluster-a + ... cluster-b cluster-c + ... cluster-c cluster-a + ... cluster-c cluster-b + Curl Remote Service Via DNS ${src} ${dst} + END