From 0b0da451acd6a3fca0d0e3a2d9332e38366ddb96 Mon Sep 17 00:00:00 2001 From: Vincenzo Mauro Date: Thu, 25 Jun 2026 10:01:30 +0200 Subject: [PATCH 1/7] Add reboot test suite --- test/resources/c2cc.resource | 63 ++++++++++++ test/suites/c2cc/reboot.robot | 179 ++++++++++++++++++++++++++++++++++ 2 files changed, 242 insertions(+) create mode 100644 test/suites/c2cc/reboot.robot diff --git a/test/resources/c2cc.resource b/test/resources/c2cc.resource index f51aa73da7..65cd11c798 100644 --- a/test/resources/c2cc.resource +++ b/test/resources/c2cc.resource @@ -572,3 +572,66 @@ Reconnect To Cluster Deregister Remote Cluster ${alias} Wait Until Keyword Succeeds ${timeout} 15s ... Register Remote Cluster ${alias} ${host} ${ssh_port} ${kubeconfig} + +Verify Cluster Is Healthy + [Documentation] Verify MicroShift and all core workloads are ready. + [Arguments] ${alias} + Command On Cluster ${alias} microshift healthcheck --timeout=300s + +Verify All RemoteClusters Healthy + [Documentation] Wait for all RemoteCluster CRs on all clusters to report Healthy. + FOR ${alias} IN @{ALL_CLUSTERS} + Wait Until Keyword Succeeds 3m 10s + ... Verify RemoteCluster State ${alias} Healthy + END + +Reboot Clusters Simultaneously + [Documentation] Reboot one or more clusters and wait for all to come back + ... with a new boot ID and greenboot health check passed. + [Arguments] @{cluster_aliases} + &{boot_ids}= Create Dictionary + FOR ${alias} IN @{cluster_aliases} + ${conn_id}= Get From Dictionary ${C2CC_SSH_IDS} ${alias} + SSHLibrary.Switch Connection ${conn_id} + ${bootid}= Get Current Boot Id + Set To Dictionary ${boot_ids} ${alias} ${bootid} + END + FOR ${alias} IN @{cluster_aliases} + ${conn_id}= Get From Dictionary ${C2CC_SSH_IDS} ${alias} + SSHLibrary.Switch Connection ${conn_id} + SSHLibrary.Start Command reboot sudo=True + END + FOR ${alias} IN @{cluster_aliases} + ${old_bootid}= Get From Dictionary ${boot_ids} ${alias} + Wait Until Keyword Succeeds 10m 15s + ... Cluster Should Be Rebooted ${alias} ${old_bootid} + END + +Cluster Should Be Rebooted + [Documentation] Assert that a cluster has rebooted and greenboot health check has passed. + [Arguments] ${alias} ${old_bootid} + Reconnect Cluster ${alias} + ${new_bootid}= Get Current Boot Id + Should Not Be Equal ${new_bootid} ${old_bootid} + ${stdout}= Command On Cluster ${alias} + ... systemctl show -p SubState greenboot-healthcheck.service --value + Should Be Equal As Strings ${stdout} exited strip_spaces=True + +Reconnect Cluster + [Documentation] Re-establish SSH connection to a cluster after reboot. + ... Resolves host/port from the alias and delegates to Reconnect To Cluster. + [Arguments] ${alias} + IF '${alias}' == 'cluster-a' + VAR ${host} ${USHIFT_HOST} + VAR ${port} ${SSH_PORT} + ELSE IF '${alias}' == 'cluster-b' + VAR ${host} ${HOST2_IP} + VAR ${port} ${HOST2_SSH_PORT} + ELSE IF '${alias}' == 'cluster-c' + VAR ${host} ${HOST3_IP} + VAR ${port} ${HOST3_SSH_PORT} + ELSE + Fail Unknown cluster alias: ${alias} + END + ${kubeconfig}= Get From Dictionary ${C2CC_KUBECONFIGS} ${alias} + Reconnect To Cluster ${alias} ${host} ${port} ${kubeconfig} timeout=10m diff --git a/test/suites/c2cc/reboot.robot b/test/suites/c2cc/reboot.robot new file mode 100644 index 0000000000..0bdbcf2394 --- /dev/null +++ b/test/suites/c2cc/reboot.robot @@ -0,0 +1,179 @@ +*** Settings *** +Documentation Verify C2CC survives VM reboots in escalating scenarios. +... Tests single cluster, two clusters simultaneously, and all three +... clusters simultaneously. After each reboot cycle, full-stack +... verification confirms connectivity, infrastructure, health probes, +... and DNS all recover. + +Resource ../../resources/microshift-process.resource +Resource ../../resources/kubeconfig.resource +Resource ../../resources/oc.resource +Resource ../../resources/c2cc.resource +Resource ../../resources/ostree-health.resource + +Suite Setup Setup +Suite Teardown Teardown + +Test Tags c2cc + + +*** Test Cases *** +Reboot Single Cluster + [Documentation] Reboot cluster-a while cluster-b and cluster-c remain up. + ... Verifies that the rebooted cluster re-establishes C2CC connectivity + ... with both peers. + [Setup] Ensure All Clusters Healthy + Reboot Clusters Simultaneously cluster-a + Wait For Clusters Ready cluster-a + Verify Full C2CC Stack + +Reboot Two Clusters Simultaneously + [Documentation] Reboot cluster-b and cluster-c at the same time. + ... The surviving cluster-a must wait for both peers to recover. + ... The two rebooted clusters must also reconnect with each other. + [Setup] Ensure All Clusters Healthy + Reboot Clusters Simultaneously cluster-b cluster-c + Wait For Clusters Ready cluster-b cluster-c + Verify Full C2CC Stack + +Reboot All Three Clusters Simultaneously + [Documentation] Reboot all three clusters at once. + ... Every cluster starts from scratch simultaneously — no running peer + ... to reference. All must independently reconstruct C2CC state. + [Setup] Ensure All Clusters Healthy + Reboot Clusters Simultaneously cluster-a cluster-b cluster-c + Wait For Clusters Ready cluster-a cluster-b cluster-c + Verify Full C2CC Stack + + +*** Keywords *** +Setup + [Documentation] Set up clusters and deploy test workloads on all. + Check Required Env Variables + Login MicroShift Host + Setup Kubeconfig + Register Local Cluster cluster-a + Register Remote Cluster cluster-b ${HOST2_IP} ${HOST2_SSH_PORT} ${KUBECONFIG_B} + Register Remote Cluster cluster-c ${HOST3_IP} ${HOST3_SSH_PORT} ${KUBECONFIG_C} + Deploy Test Workloads + Verify Full C2CC Stack + +Teardown + [Documentation] Remove test workloads and close connections. + Cleanup Test Workloads + Teardown All Remote Clusters + Remove Kubeconfig + Logout MicroShift Host + +Wait For Clusters Ready + [Documentation] Wait for rebooted clusters to finish greenboot and for test pods + ... and service endpoints to become ready. + [Arguments] @{cluster_aliases} + Wait Until Keyword Succeeds 10m 15s + ... All Clusters Greenboot Exited @{cluster_aliases} + Wait For Test Pods + Wait For Service Endpoints + +All Clusters Greenboot Exited + [Documentation] Check that greenboot has exited on all given clusters. + ... Fails if any cluster has not finished yet, causing the caller's + ... Wait Until Keyword Succeeds to retry. + [Arguments] @{cluster_aliases} + FOR ${alias} IN @{cluster_aliases} + ${conn_id}= Get From Dictionary ${C2CC_SSH_IDS} ${alias} + SSHLibrary.Switch Connection ${conn_id} + Greenboot Health Check Exited + END + +Verify Full C2CC Stack + [Documentation] Comprehensive verification of all C2CC components across all clusters. + Wait Until Keyword Succeeds 10m 10s Verify C2CC Connectivity + Wait Until Keyword Succeeds 10m 10s Verify C2CC Infrastructure + Wait Until Keyword Succeeds 10m 10s Verify C2CC Health Probes + Wait Until Keyword Succeeds 10m 10s Verify C2CC DNS + +Verify C2CC Connectivity + [Documentation] Verify pod-to-pod, pod-to-service connectivity and source IP preservation + ... across all 6 cluster pairs. + FOR ${src} ${dst} IN + ... cluster-a cluster-b + ... cluster-a cluster-c + ... cluster-b cluster-a + ... cluster-b cluster-c + ... cluster-c cluster-a + ... cluster-c cluster-b + Test Connectivity Between Clusters ${src} ${dst} pod + Test Connectivity Between Clusters ${src} ${dst} service + Test Source IP Preserved Between Clusters ${src} ${dst} pod + Test Source IP Preserved Between Clusters ${src} ${dst} service + END + +Verify C2CC Infrastructure + [Documentation] Verify routes, IP rules, nftables, OVN static routes, + ... and node annotations for all cluster-peer combinations. + Verify Infra For Remote Peer cluster-a ${CLUSTER_B_POD_CIDR} ${CLUSTER_B_SVC_CIDR} ${CLUSTER_A_SVC_CIDR} + Verify Infra For Remote Peer cluster-a ${CLUSTER_C_POD_CIDR} ${CLUSTER_C_SVC_CIDR} ${CLUSTER_A_SVC_CIDR} + Verify Infra For Remote Peer cluster-b ${CLUSTER_A_POD_CIDR} ${CLUSTER_A_SVC_CIDR} ${CLUSTER_B_SVC_CIDR} + Verify Infra For Remote Peer cluster-b ${CLUSTER_C_POD_CIDR} ${CLUSTER_C_SVC_CIDR} ${CLUSTER_B_SVC_CIDR} + Verify Infra For Remote Peer cluster-c ${CLUSTER_A_POD_CIDR} ${CLUSTER_A_SVC_CIDR} ${CLUSTER_C_SVC_CIDR} + Verify Infra For Remote Peer cluster-c ${CLUSTER_B_POD_CIDR} ${CLUSTER_B_SVC_CIDR} ${CLUSTER_C_SVC_CIDR} + +Verify Infra For Remote Peer + [Documentation] Verify all infrastructure components on a cluster for one remote peer. + [Arguments] ${alias} ${remote_pod_cidr} ${remote_svc_cidr} ${local_svc_cidr} + Verify Routes In Table 200 ${alias} ${remote_pod_cidr} ${remote_svc_cidr} + Verify IP Rules For Table 200 ${alias} ${remote_pod_cidr} ${remote_svc_cidr} + Verify Routes In Table 201 ${alias} ${local_svc_cidr} + Verify Service IP Rules ${alias} ${remote_pod_cidr} ${remote_svc_cidr} ${local_svc_cidr} + Verify NFTables Bypass Rules ${alias} ${remote_pod_cidr} ${remote_svc_cidr} + Verify OVN Static Routes ${alias} ${remote_pod_cidr} ${remote_svc_cidr} + Verify Node SNAT Annotation ${alias} ${remote_pod_cidr} ${remote_svc_cidr} + Verify C2CC Tracking Annotation ${alias} ${remote_pod_cidr} ${remote_svc_cidr} + +Verify C2CC Health Probes + [Documentation] Verify all RemoteCluster CRs report Healthy with populated timestamps. + FOR ${alias} IN cluster-a cluster-b cluster-c + Verify RemoteCluster State ${alias} Healthy + ${stdout}= Oc On Cluster ${alias} + ... oc get remoteclusters.microshift.io -o jsonpath='{.items[*].status.lastProbeTime}' + Should Not Be Empty ${stdout} + ${stdout}= Oc On Cluster ${alias} + ... oc get remoteclusters.microshift.io -o jsonpath='{.items[*].status.lastSuccessfulProbe}' + Should Not Be Empty ${stdout} + END + +Verify C2CC DNS + [Documentation] Verify CoreDNS Corefile contains C2CC server blocks and + ... cross-cluster DNS resolution works for all pairs. + Verify Corefile Contains C2CC Server Block cluster-a ${CLUSTER_B_DOMAIN} + Verify Corefile Contains C2CC Server Block cluster-a ${CLUSTER_C_DOMAIN} + Verify Corefile Contains C2CC Server Block cluster-b ${CLUSTER_A_DOMAIN} + Verify Corefile Contains C2CC Server Block cluster-b ${CLUSTER_C_DOMAIN} + Verify Corefile Contains C2CC Server Block cluster-c ${CLUSTER_A_DOMAIN} + Verify Corefile Contains C2CC Server Block cluster-c ${CLUSTER_B_DOMAIN} + Curl Remote Service Via DNS cluster-a cluster-b + Curl Remote Service Via DNS cluster-a cluster-c + Curl Remote Service Via DNS cluster-b cluster-a + Curl Remote Service Via DNS cluster-b cluster-c + Curl Remote Service Via DNS cluster-c cluster-a + Curl Remote Service Via DNS cluster-c cluster-b + +Verify RemoteCluster State + [Documentation] Check that all RemoteCluster CRs on this cluster have the expected state. + [Arguments] ${alias} ${expected_state} + ${stdout}= Oc On Cluster ${alias} + ... oc get remoteclusters.microshift.io -o jsonpath='{.items[*].status.state}' + Should Not Be Empty ${stdout} + @{states}= Split String ${stdout} + ${count}= Get Length ${states} + Should Be Equal As Integers ${count} 2 Expected 2 RemoteCluster states, got ${count} + FOR ${state} IN @{states} + Should Be Equal As Strings ${state} ${expected_state} + END + +Ensure All Clusters Healthy + [Documentation] Pre-condition: all clusters must have Healthy RemoteCluster CRs. + FOR ${alias} IN cluster-a cluster-b cluster-c + Wait Until Keyword Succeeds 3m 10s + ... Verify RemoteCluster State ${alias} Healthy + END From 36af46cc1c95b9dd7b2bf5fded4a1ba03601b496 Mon Sep 17 00:00:00 2001 From: Vincenzo Mauro Date: Thu, 25 Jun 2026 11:23:42 +0200 Subject: [PATCH 2/7] fixed cluster health checks --- test/suites/c2cc/reboot.robot | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/test/suites/c2cc/reboot.robot b/test/suites/c2cc/reboot.robot index 0bdbcf2394..c8238b709f 100644 --- a/test/suites/c2cc/reboot.robot +++ b/test/suites/c2cc/reboot.robot @@ -158,22 +158,6 @@ Verify C2CC DNS Curl Remote Service Via DNS cluster-c cluster-a Curl Remote Service Via DNS cluster-c cluster-b -Verify RemoteCluster State - [Documentation] Check that all RemoteCluster CRs on this cluster have the expected state. - [Arguments] ${alias} ${expected_state} - ${stdout}= Oc On Cluster ${alias} - ... oc get remoteclusters.microshift.io -o jsonpath='{.items[*].status.state}' - Should Not Be Empty ${stdout} - @{states}= Split String ${stdout} - ${count}= Get Length ${states} - Should Be Equal As Integers ${count} 2 Expected 2 RemoteCluster states, got ${count} - FOR ${state} IN @{states} - Should Be Equal As Strings ${state} ${expected_state} - END - Ensure All Clusters Healthy [Documentation] Pre-condition: all clusters must have Healthy RemoteCluster CRs. - FOR ${alias} IN cluster-a cluster-b cluster-c - Wait Until Keyword Succeeds 3m 10s - ... Verify RemoteCluster State ${alias} Healthy - END + Verify All RemoteClusters Healthy From 8609857d841fb052f8efb19f26b0e51ac5e78365 Mon Sep 17 00:00:00 2001 From: Vincenzo Mauro Date: Thu, 25 Jun 2026 15:23:40 +0200 Subject: [PATCH 3/7] register cluster-a as remote and remove unused code --- test/resources/c2cc.resource | 1 + test/suites/c2cc/reboot.robot | 33 ++++++++------------------------- 2 files changed, 9 insertions(+), 25 deletions(-) diff --git a/test/resources/c2cc.resource b/test/resources/c2cc.resource index 65cd11c798..75126b1806 100644 --- a/test/resources/c2cc.resource +++ b/test/resources/c2cc.resource @@ -588,6 +588,7 @@ Verify All RemoteClusters Healthy Reboot Clusters Simultaneously [Documentation] Reboot one or more clusters and wait for all to come back ... with a new boot ID and greenboot health check passed. + ... Issues reboot on all targets before waiting, so reboots overlap. [Arguments] @{cluster_aliases} &{boot_ids}= Create Dictionary FOR ${alias} IN @{cluster_aliases} diff --git a/test/suites/c2cc/reboot.robot b/test/suites/c2cc/reboot.robot index c8238b709f..f309f91b06 100644 --- a/test/suites/c2cc/reboot.robot +++ b/test/suites/c2cc/reboot.robot @@ -5,11 +5,7 @@ Documentation Verify C2CC survives VM reboots in escalating scenarios. ... verification confirms connectivity, infrastructure, health probes, ... and DNS all recover. -Resource ../../resources/microshift-process.resource -Resource ../../resources/kubeconfig.resource -Resource ../../resources/oc.resource Resource ../../resources/c2cc.resource -Resource ../../resources/ostree-health.resource Suite Setup Setup Suite Teardown Teardown @@ -24,7 +20,7 @@ Reboot Single Cluster ... with both peers. [Setup] Ensure All Clusters Healthy Reboot Clusters Simultaneously cluster-a - Wait For Clusters Ready cluster-a + Wait For Clusters Ready Verify Full C2CC Stack Reboot Two Clusters Simultaneously @@ -33,7 +29,7 @@ Reboot Two Clusters Simultaneously ... The two rebooted clusters must also reconnect with each other. [Setup] Ensure All Clusters Healthy Reboot Clusters Simultaneously cluster-b cluster-c - Wait For Clusters Ready cluster-b cluster-c + Wait For Clusters Ready Verify Full C2CC Stack Reboot All Three Clusters Simultaneously @@ -42,7 +38,7 @@ Reboot All Three Clusters Simultaneously ... to reference. All must independently reconstruct C2CC state. [Setup] Ensure All Clusters Healthy Reboot Clusters Simultaneously cluster-a cluster-b cluster-c - Wait For Clusters Ready cluster-a cluster-b cluster-c + Wait For Clusters Ready Verify Full C2CC Stack @@ -52,7 +48,9 @@ Setup Check Required Env Variables Login MicroShift Host Setup Kubeconfig - Register Local Cluster cluster-a + Logout MicroShift Host + + Register Remote Cluster cluster-a ${USHIFT_HOST} ${SSH_PORT} ${KUBECONFIG} Register Remote Cluster cluster-b ${HOST2_IP} ${HOST2_SSH_PORT} ${KUBECONFIG_B} Register Remote Cluster cluster-c ${HOST3_IP} ${HOST3_SSH_PORT} ${KUBECONFIG_C} Deploy Test Workloads @@ -63,28 +61,13 @@ Teardown Cleanup Test Workloads Teardown All Remote Clusters Remove Kubeconfig - Logout MicroShift Host Wait For Clusters Ready - [Documentation] Wait for rebooted clusters to finish greenboot and for test pods - ... and service endpoints to become ready. - [Arguments] @{cluster_aliases} - Wait Until Keyword Succeeds 10m 15s - ... All Clusters Greenboot Exited @{cluster_aliases} + [Documentation] Wait for test pods and service endpoints to become ready + ... after a reboot cycle. Wait For Test Pods Wait For Service Endpoints -All Clusters Greenboot Exited - [Documentation] Check that greenboot has exited on all given clusters. - ... Fails if any cluster has not finished yet, causing the caller's - ... Wait Until Keyword Succeeds to retry. - [Arguments] @{cluster_aliases} - FOR ${alias} IN @{cluster_aliases} - ${conn_id}= Get From Dictionary ${C2CC_SSH_IDS} ${alias} - SSHLibrary.Switch Connection ${conn_id} - Greenboot Health Check Exited - END - Verify Full C2CC Stack [Documentation] Comprehensive verification of all C2CC components across all clusters. Wait Until Keyword Succeeds 10m 10s Verify C2CC Connectivity From 9504deeb018c65730fe4956313375b38a7178d29 Mon Sep 17 00:00:00 2001 From: Vincenzo Mauro Date: Fri, 26 Jun 2026 12:20:01 +0200 Subject: [PATCH 4/7] Add disruptive test tag and addressed PR comments --- test/resources/c2cc.resource | 14 +------------- test/suites/c2cc/reboot.robot | 2 +- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/test/resources/c2cc.resource b/test/resources/c2cc.resource index 75126b1806..069a1d7c97 100644 --- a/test/resources/c2cc.resource +++ b/test/resources/c2cc.resource @@ -414,7 +414,7 @@ DNS Lookup Should Succeed Verify All RemoteClusters Healthy [Documentation] Wait for all RemoteCluster CRs on all clusters to report Healthy. - FOR ${alias} IN cluster-a cluster-b cluster-c + FOR ${alias} IN @{ALL_CLUSTERS} Wait Until Keyword Succeeds 3m 10s ... Verify RemoteCluster State ${alias} Healthy END @@ -573,18 +573,6 @@ Reconnect To Cluster Wait Until Keyword Succeeds ${timeout} 15s ... Register Remote Cluster ${alias} ${host} ${ssh_port} ${kubeconfig} -Verify Cluster Is Healthy - [Documentation] Verify MicroShift and all core workloads are ready. - [Arguments] ${alias} - Command On Cluster ${alias} microshift healthcheck --timeout=300s - -Verify All RemoteClusters Healthy - [Documentation] Wait for all RemoteCluster CRs on all clusters to report Healthy. - FOR ${alias} IN @{ALL_CLUSTERS} - Wait Until Keyword Succeeds 3m 10s - ... Verify RemoteCluster State ${alias} Healthy - END - Reboot Clusters Simultaneously [Documentation] Reboot one or more clusters and wait for all to come back ... with a new boot ID and greenboot health check passed. diff --git a/test/suites/c2cc/reboot.robot b/test/suites/c2cc/reboot.robot index f309f91b06..61c48a9b80 100644 --- a/test/suites/c2cc/reboot.robot +++ b/test/suites/c2cc/reboot.robot @@ -10,7 +10,7 @@ Resource ../../resources/c2cc.resource Suite Setup Setup Suite Teardown Teardown -Test Tags c2cc +Test Tags disruptive *** Test Cases *** From cfece6a649f70c167c8c13e9f0d851868464efd2 Mon Sep 17 00:00:00 2001 From: Vincenzo Mauro Date: Fri, 26 Jun 2026 14:18:39 +0200 Subject: [PATCH 5/7] Fixed linting and addressed PR comments --- test/resources/c2cc.resource | 16 +++++---- test/suites/c2cc/reboot.robot | 65 +++++++++++++++++++++++++---------- 2 files changed, 55 insertions(+), 26 deletions(-) diff --git a/test/resources/c2cc.resource b/test/resources/c2cc.resource index 069a1d7c97..e1340d1055 100644 --- a/test/resources/c2cc.resource +++ b/test/resources/c2cc.resource @@ -578,7 +578,7 @@ Reboot Clusters Simultaneously ... with a new boot ID and greenboot health check passed. ... Issues reboot on all targets before waiting, so reboots overlap. [Arguments] @{cluster_aliases} - &{boot_ids}= Create Dictionary + VAR &{boot_ids}= FOR ${alias} IN @{cluster_aliases} ${conn_id}= Get From Dictionary ${C2CC_SSH_IDS} ${alias} SSHLibrary.Switch Connection ${conn_id} @@ -604,6 +604,8 @@ Cluster Should Be Rebooted Should Not Be Equal ${new_bootid} ${old_bootid} ${stdout}= Command On Cluster ${alias} ... systemctl show -p SubState greenboot-healthcheck.service --value + Should Not Be Equal As Strings ${stdout} failed strip_spaces=True + ... msg=Greenboot healthcheck FAILED on ${alias}. Check greenboot logs. Should Be Equal As Strings ${stdout} exited strip_spaces=True Reconnect Cluster @@ -611,14 +613,14 @@ Reconnect Cluster ... Resolves host/port from the alias and delegates to Reconnect To Cluster. [Arguments] ${alias} IF '${alias}' == 'cluster-a' - VAR ${host} ${USHIFT_HOST} - VAR ${port} ${SSH_PORT} + VAR ${host}= ${USHIFT_HOST} + VAR ${port}= ${SSH_PORT} ELSE IF '${alias}' == 'cluster-b' - VAR ${host} ${HOST2_IP} - VAR ${port} ${HOST2_SSH_PORT} + VAR ${host}= ${HOST2_IP} + VAR ${port}= ${HOST2_SSH_PORT} ELSE IF '${alias}' == 'cluster-c' - VAR ${host} ${HOST3_IP} - VAR ${port} ${HOST3_SSH_PORT} + VAR ${host}= ${HOST3_IP} + VAR ${port}= ${HOST3_SSH_PORT} ELSE Fail Unknown cluster alias: ${alias} END diff --git a/test/suites/c2cc/reboot.robot b/test/suites/c2cc/reboot.robot index 61c48a9b80..25df402ad6 100644 --- a/test/suites/c2cc/reboot.robot +++ b/test/suites/c2cc/reboot.robot @@ -18,7 +18,7 @@ Reboot Single Cluster [Documentation] Reboot cluster-a while cluster-b and cluster-c remain up. ... Verifies that the rebooted cluster re-establishes C2CC connectivity ... with both peers. - [Setup] Ensure All Clusters Healthy + [Setup] Verify All RemoteClusters Healthy Reboot Clusters Simultaneously cluster-a Wait For Clusters Ready Verify Full C2CC Stack @@ -27,7 +27,7 @@ Reboot Two Clusters Simultaneously [Documentation] Reboot cluster-b and cluster-c at the same time. ... The surviving cluster-a must wait for both peers to recover. ... The two rebooted clusters must also reconnect with each other. - [Setup] Ensure All Clusters Healthy + [Setup] Verify All RemoteClusters Healthy Reboot Clusters Simultaneously cluster-b cluster-c Wait For Clusters Ready Verify Full C2CC Stack @@ -36,7 +36,7 @@ Reboot All Three Clusters Simultaneously [Documentation] Reboot all three clusters at once. ... Every cluster starts from scratch simultaneously — no running peer ... to reference. All must independently reconstruct C2CC state. - [Setup] Ensure All Clusters Healthy + [Setup] Verify All RemoteClusters Healthy Reboot Clusters Simultaneously cluster-a cluster-b cluster-c Wait For Clusters Ready Verify Full C2CC Stack @@ -94,12 +94,36 @@ Verify C2CC Connectivity Verify C2CC Infrastructure [Documentation] Verify routes, IP rules, nftables, OVN static routes, ... and node annotations for all cluster-peer combinations. - Verify Infra For Remote Peer cluster-a ${CLUSTER_B_POD_CIDR} ${CLUSTER_B_SVC_CIDR} ${CLUSTER_A_SVC_CIDR} - Verify Infra For Remote Peer cluster-a ${CLUSTER_C_POD_CIDR} ${CLUSTER_C_SVC_CIDR} ${CLUSTER_A_SVC_CIDR} - Verify Infra For Remote Peer cluster-b ${CLUSTER_A_POD_CIDR} ${CLUSTER_A_SVC_CIDR} ${CLUSTER_B_SVC_CIDR} - Verify Infra For Remote Peer cluster-b ${CLUSTER_C_POD_CIDR} ${CLUSTER_C_SVC_CIDR} ${CLUSTER_B_SVC_CIDR} - Verify Infra For Remote Peer cluster-c ${CLUSTER_A_POD_CIDR} ${CLUSTER_A_SVC_CIDR} ${CLUSTER_C_SVC_CIDR} - Verify Infra For Remote Peer cluster-c ${CLUSTER_B_POD_CIDR} ${CLUSTER_B_SVC_CIDR} ${CLUSTER_C_SVC_CIDR} + Verify Infra For Remote Peer + ... cluster-a + ... ${CLUSTER_B_POD_CIDR} + ... ${CLUSTER_B_SVC_CIDR} + ... ${CLUSTER_A_SVC_CIDR} + Verify Infra For Remote Peer + ... cluster-a + ... ${CLUSTER_C_POD_CIDR} + ... ${CLUSTER_C_SVC_CIDR} + ... ${CLUSTER_A_SVC_CIDR} + Verify Infra For Remote Peer + ... cluster-b + ... ${CLUSTER_A_POD_CIDR} + ... ${CLUSTER_A_SVC_CIDR} + ... ${CLUSTER_B_SVC_CIDR} + Verify Infra For Remote Peer + ... cluster-b + ... ${CLUSTER_C_POD_CIDR} + ... ${CLUSTER_C_SVC_CIDR} + ... ${CLUSTER_B_SVC_CIDR} + Verify Infra For Remote Peer + ... cluster-c + ... ${CLUSTER_A_POD_CIDR} + ... ${CLUSTER_A_SVC_CIDR} + ... ${CLUSTER_C_SVC_CIDR} + Verify Infra For Remote Peer + ... cluster-c + ... ${CLUSTER_B_POD_CIDR} + ... ${CLUSTER_B_SVC_CIDR} + ... ${CLUSTER_C_SVC_CIDR} Verify Infra For Remote Peer [Documentation] Verify all infrastructure components on a cluster for one remote peer. @@ -134,13 +158,16 @@ Verify C2CC DNS Verify Corefile Contains C2CC Server Block cluster-b ${CLUSTER_C_DOMAIN} Verify Corefile Contains C2CC Server Block cluster-c ${CLUSTER_A_DOMAIN} Verify Corefile Contains C2CC Server Block cluster-c ${CLUSTER_B_DOMAIN} - Curl Remote Service Via DNS cluster-a cluster-b - Curl Remote Service Via DNS cluster-a cluster-c - Curl Remote Service Via DNS cluster-b cluster-a - Curl Remote Service Via DNS cluster-b cluster-c - Curl Remote Service Via DNS cluster-c cluster-a - Curl Remote Service Via DNS cluster-c cluster-b - -Ensure All Clusters Healthy - [Documentation] Pre-condition: all clusters must have Healthy RemoteCluster CRs. - Verify All RemoteClusters Healthy + Verify DNS Connectivity All Pairs + +Verify DNS Connectivity All Pairs + [Documentation] Verify cross-cluster DNS-based service access for all cluster pairs. + FOR ${src} ${dst} IN + ... cluster-a cluster-b + ... cluster-a cluster-c + ... cluster-b cluster-a + ... cluster-b cluster-c + ... cluster-c cluster-a + ... cluster-c cluster-b + Curl Remote Service Via DNS ${src} ${dst} + END From 15cc38a5b5f1bf9624e0ab439f5d26c7c651636e Mon Sep 17 00:00:00 2001 From: Vincenzo Mauro Date: Fri, 26 Jun 2026 15:08:22 +0200 Subject: [PATCH 6/7] small verification fix --- test/resources/c2cc.resource | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/resources/c2cc.resource b/test/resources/c2cc.resource index e1340d1055..a35dbc940f 100644 --- a/test/resources/c2cc.resource +++ b/test/resources/c2cc.resource @@ -578,7 +578,7 @@ Reboot Clusters Simultaneously ... with a new boot ID and greenboot health check passed. ... Issues reboot on all targets before waiting, so reboots overlap. [Arguments] @{cluster_aliases} - VAR &{boot_ids}= + VAR &{boot_ids}= &{EMPTY} FOR ${alias} IN @{cluster_aliases} ${conn_id}= Get From Dictionary ${C2CC_SSH_IDS} ${alias} SSHLibrary.Switch Connection ${conn_id} From 7d833ebef6054263306076137043c45088f887c4 Mon Sep 17 00:00:00 2001 From: Vincenzo Mauro Date: Sat, 27 Jun 2026 11:56:41 +0200 Subject: [PATCH 7/7] removed disruptive.robot filtering --- test/scenarios-bootc/c2cc/el102-src@c2cc-disruptive.sh | 2 +- test/scenarios-bootc/c2cc/el98-src@c2cc-disruptive.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/scenarios-bootc/c2cc/el102-src@c2cc-disruptive.sh b/test/scenarios-bootc/c2cc/el102-src@c2cc-disruptive.sh index 49220a4c07..35c1c30644 100755 --- a/test/scenarios-bootc/c2cc/el102-src@c2cc-disruptive.sh +++ b/test/scenarios-bootc/c2cc/el102-src@c2cc-disruptive.sh @@ -20,5 +20,5 @@ scenario_remove_vms() { scenario_run_tests() { # shellcheck disable=SC2119 configure_c2cc_hosts - c2cc_run_tests "suites/c2cc/disruptive.robot" "" "" "disruptive" + c2cc_run_tests "suites/c2cc/" "" "" "disruptive" } diff --git a/test/scenarios-bootc/c2cc/el98-src@c2cc-disruptive.sh b/test/scenarios-bootc/c2cc/el98-src@c2cc-disruptive.sh index 37ab603ad2..79cab102e4 100755 --- a/test/scenarios-bootc/c2cc/el98-src@c2cc-disruptive.sh +++ b/test/scenarios-bootc/c2cc/el98-src@c2cc-disruptive.sh @@ -20,5 +20,5 @@ scenario_remove_vms() { scenario_run_tests() { # shellcheck disable=SC2119 configure_c2cc_hosts - c2cc_run_tests "suites/c2cc/disruptive.robot" "" "" "disruptive" + c2cc_run_tests "suites/c2cc/" "" "" "disruptive" }