Allow unstable error messages in tests

.github/regression.sh

@@ -34,13 +34,6 @@ mkdir -p "$WORKDIR"
 export TMPDIR="$WORKDIR/tmp"
 mkdir -p "$TMPDIR"
 
-if [ "${CI_TOPOLOGY:-}" = "legacy" ]; then
-  export ENABLE_LEGACY=1
-elif [ "${CI_TOPOLOGY:-}" = "mixed" ]; then
-  export MIXED_P2P=1
-  export NUM_POOLS="${NUM_POOLS:-4}"
-fi
-
 export ARTIFACTS_DIR="${ARTIFACTS_DIR:-".artifacts"}"
 rm -rf "${ARTIFACTS_DIR:?}"
 
@@ -93,6 +86,10 @@ else
   export TESTNET_VARIANT="${TESTNET_VARIANT:-"${CLUSTER_ERA}_fast"}"
 fi
 
+if [ "${ALLOW_UNSTABLE_ERROR_MESSAGES:-"false"}" == "false" ]; then
+  unset ALLOW_UNSTABLE_ERROR_MESSAGES
+fi
+
 export CARDANO_NODE_SOCKET_PATH_CI="$WORKDIR/state-cluster0/bft1.socket"
 
 # assume we run tests on testnet when `BOOTSTRAP_DIR` is set

.github/workflows/regression-dbsync.yaml

@@ -36,14 +36,6 @@ on:
         - dbsync and plutus
         - dbsync and not long
         default: dbsync
-      topology:
-        type: choice
-        description: "Network topology"
-        options:
-        - p2p
-        - legacy
-        - mixed
-        default: p2p
       utxo_backend:
         type: choice
         description: "UTxO backend"
@@ -53,6 +45,10 @@ on:
         - disk
         - disklmdb
         default: ""
+      allow_unstable_error_msgs:
+        type: boolean
+        default: false
+        description: "Allow unstable error messages"
       byron_cluster:
         type: boolean
         default: false
@@ -77,8 +73,8 @@ jobs:
       dbsync_rev: ${{ inputs.dbsync_rev }}
       cluster_era: ${{ inputs.cluster_era }}
       markexpr: ${{ inputs.markexpr }}
-      topology: ${{ inputs.topology }}
       utxo_backend: ${{ inputs.utxo_backend }}
+      allow_unstable_error_msgs: ${{ inputs.allow_unstable_error_msgs }}
       byron_cluster: ${{ inputs.byron_cluster }}
       testrun_name: ${{ inputs.testrun_name }}
       skip_passed: ${{ inputs.skip_passed }}

.github/workflows/regression.yaml

@@ -28,14 +28,6 @@ on:
         - not long
         - conway only
         default: all
-      topology:
-        type: choice
-        description: "Network topology"
-        options:
-        - p2p
-        - legacy
-        - mixed
-        default: p2p
       utxo_backend:
         type: choice
         description: "UTxO backend"
@@ -45,6 +37,10 @@ on:
         - disk
         - disklmdb
         default: ""
+      allow_unstable_error_msgs:
+        type: boolean
+        default: false
+        description: "Allow unstable error messages"
       byron_cluster:
         type: boolean
         default: false
@@ -68,8 +64,8 @@ jobs:
       cli_rev: ${{ inputs.cli_rev }}
       cluster_era: ${{ inputs.cluster_era }}
       markexpr: ${{ inputs.markexpr }}
-      topology: ${{ inputs.topology }}
       utxo_backend: ${{ inputs.utxo_backend }}
+      allow_unstable_error_msgs: ${{ inputs.allow_unstable_error_msgs }}
       byron_cluster: ${{ inputs.byron_cluster }}
       testrun_name: ${{ inputs.testrun_name }}
       skip_passed: ${{ inputs.skip_passed }}

.github/workflows/regression_reusable.yaml

@@ -23,22 +23,18 @@ on:
         required: false
         type: string
         default: ""
-      tx_era:
-        required: false
-        type: string
-        default: ""
       markexpr:
         required: false
         type: string
         default: ""
-      topology:
-        required: false
-        type: string
-        default: ""
       utxo_backend:
         required: false
         type: string
         default: ""
+      allow_unstable_error_msgs:
+        required: false
+        type: boolean
+        default: false
       byron_cluster:
         required: false
         type: boolean
@@ -92,10 +88,9 @@ jobs:
           echo "CARDANO_CLI_REV=${{ inputs.cli_rev }}" >> .github_ci_env
           echo "DBSYNC_REV=${{ inputs.dbsync_rev }}" >> .github_ci_env
           echo "CLUSTER_ERA=${{ inputs.cluster_era }}" >> .github_ci_env
-          echo "TX_ERA=${{ inputs.tx_era }}" >> .github_ci_env
           echo "MARKEXPR=${{ inputs.markexpr }}" >> .github_ci_env
-          echo "CI_TOPOLOGY=${{ inputs.topology }}" >> .github_ci_env
           echo "UTXO_BACKEND=${{ inputs.utxo_backend }}" >> .github_ci_env
+          echo "ALLOW_UNSTABLE_ERROR_MESSAGES=${{ inputs.allow_unstable_error_msgs }}" >> .github_ci_env
           echo "CI_BYRON_CLUSTER=${{ inputs.byron_cluster }}" >> .github_ci_env
           echo "CI_TESTRUN_NAME=${{ inputs.testrun_name }}" >> .github_ci_env
           echo "CI_SKIP_PASSED=${{ inputs.skip_passed }}" >> .github_ci_env

cardano_node_tests/tests/common.py

@@ -1,3 +1,4 @@
+import contextlib
 import logging
 import string
 import time
@@ -603,3 +604,19 @@ def is_fee_in_interval(fee: float, expected_fee: float, frac: float = 0.1) -> bo
     if cluster_nodes.get_cluster_type().type == cluster_nodes.ClusterType.TESTNET:
         return True
     return helpers.is_in_interval(fee, expected_fee, frac=frac)
+
+
+@contextlib.contextmanager
+def allow_unstable_error_messages() -> tp.Iterator[None]:
+    """Catch AssertionError and either log it or raise it.
+
+    Used in tests where error messages can vary between node/CLI versions.
+    """
+    if not configuration.ALLOW_UNSTABLE_ERROR_MESSAGES:
+        yield
+        return
+
+    try:
+        yield
+    except AssertionError:
+        LOGGER.exception("AssertionError suppressed")

cardano_node_tests/tests/test_addr_registration.py

@@ -536,8 +536,9 @@ def test_registration_cert_with_wrong_key(
                 deposit_amt=common.get_conway_address_deposit(cluster_obj=cluster),
                 stake_vkey_file=pool_users[0].payment.vkey_file,
             )
-        err_msg = str(excinfo.value)
-        assert "Expected: StakeVerificationKeyShelley" in err_msg, err_msg
+        exc_value = str(excinfo.value)
+        with common.allow_unstable_error_messages():
+            assert "Expected: StakeVerificationKeyShelley" in exc_value, exc_value
 
     @allure.link(helpers.get_vcs_link())
     @pytest.mark.smoke
@@ -574,8 +575,9 @@ def test_register_addr_with_wrong_key(
             cluster.g_transaction.send_tx(
                 src_address=user_payment.address, tx_name=temp_template, tx_files=tx_files
             )
-        err_msg = str(excinfo.value)
-        assert "MissingVKeyWitnessesUTXOW" in err_msg, err_msg
+        exc_value = str(excinfo.value)
+        with common.allow_unstable_error_messages():
+            assert "MissingVKeyWitnessesUTXOW" in exc_value, exc_value
 
     @allure.link(helpers.get_vcs_link())
     @common.PARAM_BUILD_METHOD_NO_EST
@@ -638,8 +640,9 @@ def _build_dereg() -> clusterlib.TxRawOutput:
                     witness_count_add=len(tx_files.signing_key_files),
                 )
 
-        err_msg = str(excinfo.value)
-        assert "StakeKeyNotRegisteredDELEG" in err_msg, err_msg
+        exc_value = str(excinfo.value)
+        with common.allow_unstable_error_messages():
+            assert "StakeKeyNotRegisteredDELEG" in exc_value, exc_value
 
     @allure.link(helpers.get_vcs_link())
     @common.PARAM_BUILD_METHOD_NO_EST
@@ -768,8 +771,9 @@ def _submit_tx(
                     tx_files=tx_files_script,
                     signing_key_files=signing_key_files,
                 )
-            err_msg = str(excinfo.value)
-            assert "MissingScriptWitnessesUTXOW" in err_msg, err_msg
+            exc_value = str(excinfo.value)
+            with common.allow_unstable_error_messages():
+                assert "MissingScriptWitnessesUTXOW" in exc_value, exc_value
 
         # Scenario2: One skey is missing when witnesing the Tx
         elif issue == "missing_skey":
@@ -783,8 +787,9 @@ def _submit_tx(
                     complex_certs=[reg_cert_script],
                     signing_key_files=signing_key_files[:-1],
                 )
-            err_msg = str(excinfo.value)
-            assert "ScriptWitnessNotValidatingUTXOW" in err_msg, err_msg
+            exc_value = str(excinfo.value)
+            with common.allow_unstable_error_messages():
+                assert "ScriptWitnessNotValidatingUTXOW" in exc_value, exc_value
 
         else:
             err = f"Invalid issue: {issue}"

cardano_node_tests/tests/test_blocks.py

@@ -233,12 +233,13 @@ def test_unstable_stake_distribution(
                 cold_vkey_file=pool_rec["cold_key_pair"].vkey_file,
                 for_next=True,
             )
-        err_str = str(excinfo.value)
+        exc_value = str(excinfo.value)
 
-        if "PastHorizon" in err_str:
+        if "PastHorizon" in exc_value:
             issues.node_4002.finish_test()
 
-        assert "current stake distribution is currently unstable" in err_str, err_str
+        with common.allow_unstable_error_messages():
+            assert "current stake distribution is currently unstable" in exc_value, exc_value
 
 
 @pytest.mark.skipif(

cardano_node_tests/tests/test_cli.py

@@ -404,10 +404,9 @@ def test_address_info_with_invalid_address(self, cluster: clusterlib.ClusterLib,
 
         with pytest.raises(clusterlib.CLIError) as excinfo:
             cluster.g_address.get_address_info(address=address)
-
-        err_str = str(excinfo.value)
-
-        assert "Invalid address" in err_str, err_str
+        exc_value = str(excinfo.value)
+        with common.allow_unstable_error_messages():
+            assert "Invalid address" in exc_value, exc_value
 
 
 class TestAddressBuild:
@@ -535,10 +534,9 @@ def test_invalid_payment_info(
                 payment_vkey_file=vkey_file,
                 payment_script_file=script_file,
             )
-
-        err_str = str(excinfo.value)
-
-        assert "Invalid key" in err_str or "Syntax error in script" in err_str, err_str
+        exc_value = str(excinfo.value)
+        with common.allow_unstable_error_messages():
+            assert "Invalid key" in exc_value or "Syntax error in script" in exc_value, exc_value
 
     @allure.link(helpers.get_vcs_link())
     @pytest.mark.parametrize(
@@ -597,14 +595,13 @@ def test_invalid_stake_info(
                 stake_script_file=script_file,
                 stake_address=key if option == "address" else None,
             )
-
-        err_str = str(excinfo.value)
-
-        assert (
-            "Invalid key" in err_str
-            or "Syntax error in script" in err_str
-            or "invalid address" in err_str
-        ), err_str
+        exc_value = str(excinfo.value)
+        with common.allow_unstable_error_messages():
+            assert (
+                "Invalid key" in exc_value
+                or "Syntax error in script" in exc_value
+                or "invalid address" in exc_value
+            ), exc_value
 
 
 class TestAddressKeyHash:
@@ -665,10 +662,9 @@ def test_invalid_verification_key(self, cluster: clusterlib.ClusterLib, option:
                 payment_vkey=vkey if option == "vkey" else None,
                 payment_vkey_file=vkey_file,
             )
-
-        err_str = str(excinfo.value)
-
-        assert "Invalid key" in err_str, err_str
+        exc_value = str(excinfo.value)
+        with common.allow_unstable_error_messages():
+            assert "Invalid key" in exc_value, exc_value
 
 
 class TestKey:
@@ -737,12 +733,12 @@ def test_non_extended_key_error(self, cluster: clusterlib.ClusterLib):
             cluster.g_key.gen_non_extended_verification_key(
                 key_name=temp_template, extended_verification_key_file=payment_keys.skey_file
             )
-
-        err_str = str(excinfo.value)
-        assert (
-            "Error: Invalid key." in err_str
-            or "TextEnvelope type error:  Expected one of:" in err_str
-        ), err_str
+        exc_value = str(excinfo.value)
+        with common.allow_unstable_error_messages():
+            assert (
+                "Error: Invalid key." in exc_value
+                or "TextEnvelope type error:  Expected one of:" in exc_value
+            ), exc_value
 
 
 class TestQueryUTxO:
@@ -873,16 +869,15 @@ def test_tx_in_invalid_data(
                     *cluster.magic_args,
                 ]
             )
-
-        err_str = str(excinfo.value)
-
-        if invalid_param == "tx_hash":
-            assert (
-                "expecting hexadecimal digit" in err_str
-                or "expecting transaction id (hexadecimal)" in err_str
-            ), err_str
-        else:
-            assert "expecting digit" in err_str, err_str
+        exc_value = str(excinfo.value)
+        with common.allow_unstable_error_messages():
+            if invalid_param == "tx_hash":
+                assert (
+                    "expecting hexadecimal digit" in exc_value
+                    or "expecting transaction id (hexadecimal)" in exc_value
+                ), exc_value
+            else:
+                assert "expecting digit" in exc_value, exc_value
 
     @allure.link(helpers.get_vcs_link())
     @hypothesis.given(filter_str=st.text(alphabet=common.ADDR_ALPHABET, min_size=1))
@@ -905,9 +900,9 @@ def test_address_invalid_data(self, cluster: clusterlib.ClusterLib, filter_str:
                     *cluster.magic_args,
                 ]
             )
-
-        err_str = str(excinfo.value)
-        assert "invalid address" in err_str, err_str
+        exc_value = str(excinfo.value)
+        with common.allow_unstable_error_messages():
+            assert "invalid address" in exc_value, exc_value
 
 
 class TestStakeAddressKeyHash:
@@ -968,10 +963,9 @@ def test_invalid_verification_key(self, cluster: clusterlib.ClusterLib, option:
                 stake_vkey=vkey if option == "vkey" else None,
                 stake_vkey_file=vkey_file,
             )
-
-        err_str = str(excinfo.value)
-
-        assert "Invalid key" in err_str, err_str
+        exc_value = str(excinfo.value)
+        with common.allow_unstable_error_messages():
+            assert "Invalid key" in exc_value, exc_value
 
 
 class TestAdvancedQueries:
@@ -1427,9 +1421,9 @@ def test_slot_number_invalid_format(
 
         with pytest.raises(clusterlib.CLIError) as excinfo:
             cluster.g_query.query_cli(["slot-number", timestamp_str])
-        err_str = str(excinfo.value)
-
-        assert "parseTimeOrError" in err_str, err_str
+        exc_value = str(excinfo.value)
+        with common.allow_unstable_error_messages():
+            assert "parseTimeOrError" in exc_value, exc_value
 
     @allure.link(helpers.get_vcs_link())
     @pytest.mark.parametrize("time_val", ("above", "bellow"))
@@ -1451,6 +1445,6 @@ def test_slot_number_out_of_range(
 
         with pytest.raises(clusterlib.CLIError) as excinfo:
             cluster.g_query.get_slot_number(timestamp=timestamp)
-        err_str = str(excinfo.value)
-
-        assert "PastHorizon" in err_str, err_str
+        exc_value = str(excinfo.value)
+        with common.allow_unstable_error_messages():
+            assert "PastHorizon" in exc_value, exc_value