refactor(tests): centralize unstable error handling

Introduce `allow_unstable_error_messages` context manager in `common.py` to
handle assertion errors for unstable error messages across different node/CLI
versions. Refactor all test files to use this context manager, replacing
direct assertions on error messages.
This allows for temporarily ignoring specific error message checks when testing new
features.

Author: mkoura

cardano_node_tests/tests/common.py

@@ -1,3 +1,4 @@
+import contextlib
 import logging
 import string
 import time
@@ -603,3 +604,19 @@ def is_fee_in_interval(fee: float, expected_fee: float, frac: float = 0.1) -> bo
     if cluster_nodes.get_cluster_type().type == cluster_nodes.ClusterType.TESTNET:
         return True
     return helpers.is_in_interval(fee, expected_fee, frac=frac)
+
+
+@contextlib.contextmanager
+def allow_unstable_error_messages() -> tp.Iterator[None]:
+    """Catch AssertionError and either log it or raise it.
+
+    Used in tests where error messages can vary between node/CLI versions.
+    """
+    if not configuration.ALLOW_UNSTABLE_ERROR_MESSAGES:
+        yield
+        return
+
+    try:
+        yield
+    except AssertionError:
+        LOGGER.exception("AssertionError suppressed")

cardano_node_tests/tests/test_addr_registration.py

@@ -536,8 +536,9 @@ def test_registration_cert_with_wrong_key(
                 deposit_amt=common.get_conway_address_deposit(cluster_obj=cluster),
                 stake_vkey_file=pool_users[0].payment.vkey_file,
             )
-        err_msg = str(excinfo.value)
-        assert "Expected: StakeVerificationKeyShelley" in err_msg, err_msg
+        exc_value = str(excinfo.value)
+        with common.allow_unstable_error_messages():
+            assert "Expected: StakeVerificationKeyShelley" in exc_value, exc_value
 
     @allure.link(helpers.get_vcs_link())
     @pytest.mark.smoke
@@ -574,8 +575,9 @@ def test_register_addr_with_wrong_key(
             cluster.g_transaction.send_tx(
                 src_address=user_payment.address, tx_name=temp_template, tx_files=tx_files
             )
-        err_msg = str(excinfo.value)
-        assert "MissingVKeyWitnessesUTXOW" in err_msg, err_msg
+        exc_value = str(excinfo.value)
+        with common.allow_unstable_error_messages():
+            assert "MissingVKeyWitnessesUTXOW" in exc_value, exc_value
 
     @allure.link(helpers.get_vcs_link())
     @common.PARAM_BUILD_METHOD_NO_EST
@@ -638,8 +640,9 @@ def _build_dereg() -> clusterlib.TxRawOutput:
                     witness_count_add=len(tx_files.signing_key_files),
                 )
 
-        err_msg = str(excinfo.value)
-        assert "StakeKeyNotRegisteredDELEG" in err_msg, err_msg
+        exc_value = str(excinfo.value)
+        with common.allow_unstable_error_messages():
+            assert "StakeKeyNotRegisteredDELEG" in exc_value, exc_value
 
     @allure.link(helpers.get_vcs_link())
     @common.PARAM_BUILD_METHOD_NO_EST
@@ -768,8 +771,9 @@ def _submit_tx(
                     tx_files=tx_files_script,
                     signing_key_files=signing_key_files,
                 )
-            err_msg = str(excinfo.value)
-            assert "MissingScriptWitnessesUTXOW" in err_msg, err_msg
+            exc_value = str(excinfo.value)
+            with common.allow_unstable_error_messages():
+                assert "MissingScriptWitnessesUTXOW" in exc_value, exc_value
 
         # Scenario2: One skey is missing when witnesing the Tx
         elif issue == "missing_skey":
@@ -783,8 +787,9 @@ def _submit_tx(
                     complex_certs=[reg_cert_script],
                     signing_key_files=signing_key_files[:-1],
                 )
-            err_msg = str(excinfo.value)
-            assert "ScriptWitnessNotValidatingUTXOW" in err_msg, err_msg
+            exc_value = str(excinfo.value)
+            with common.allow_unstable_error_messages():
+                assert "ScriptWitnessNotValidatingUTXOW" in exc_value, exc_value
 
         else:
             err = f"Invalid issue: {issue}"

cardano_node_tests/tests/test_blocks.py

@@ -233,12 +233,13 @@ def test_unstable_stake_distribution(
                 cold_vkey_file=pool_rec["cold_key_pair"].vkey_file,
                 for_next=True,
             )
-        err_str = str(excinfo.value)
+        exc_value = str(excinfo.value)
 
-        if "PastHorizon" in err_str:
+        if "PastHorizon" in exc_value:
             issues.node_4002.finish_test()
 
-        assert "current stake distribution is currently unstable" in err_str, err_str
+        with common.allow_unstable_error_messages():
+            assert "current stake distribution is currently unstable" in exc_value, exc_value
 
 
 @pytest.mark.skipif(

cardano_node_tests/tests/test_cli.py

@@ -404,10 +404,9 @@ def test_address_info_with_invalid_address(self, cluster: clusterlib.ClusterLib,
 
         with pytest.raises(clusterlib.CLIError) as excinfo:
             cluster.g_address.get_address_info(address=address)
-
-        err_str = str(excinfo.value)
-
-        assert "Invalid address" in err_str, err_str
+        exc_value = str(excinfo.value)
+        with common.allow_unstable_error_messages():
+            assert "Invalid address" in exc_value, exc_value
 
 
 class TestAddressBuild:
@@ -535,10 +534,9 @@ def test_invalid_payment_info(
                 payment_vkey_file=vkey_file,
                 payment_script_file=script_file,
             )
-
-        err_str = str(excinfo.value)
-
-        assert "Invalid key" in err_str or "Syntax error in script" in err_str, err_str
+        exc_value = str(excinfo.value)
+        with common.allow_unstable_error_messages():
+            assert "Invalid key" in exc_value or "Syntax error in script" in exc_value, exc_value
 
     @allure.link(helpers.get_vcs_link())
     @pytest.mark.parametrize(
@@ -597,14 +595,13 @@ def test_invalid_stake_info(
                 stake_script_file=script_file,
                 stake_address=key if option == "address" else None,
             )
-
-        err_str = str(excinfo.value)
-
-        assert (
-            "Invalid key" in err_str
-            or "Syntax error in script" in err_str
-            or "invalid address" in err_str
-        ), err_str
+        exc_value = str(excinfo.value)
+        with common.allow_unstable_error_messages():
+            assert (
+                "Invalid key" in exc_value
+                or "Syntax error in script" in exc_value
+                or "invalid address" in exc_value
+            ), exc_value
 
 
 class TestAddressKeyHash:
@@ -665,10 +662,9 @@ def test_invalid_verification_key(self, cluster: clusterlib.ClusterLib, option:
                 payment_vkey=vkey if option == "vkey" else None,
                 payment_vkey_file=vkey_file,
             )
-
-        err_str = str(excinfo.value)
-
-        assert "Invalid key" in err_str, err_str
+        exc_value = str(excinfo.value)
+        with common.allow_unstable_error_messages():
+            assert "Invalid key" in exc_value, exc_value
 
 
 class TestKey:
@@ -737,12 +733,12 @@ def test_non_extended_key_error(self, cluster: clusterlib.ClusterLib):
             cluster.g_key.gen_non_extended_verification_key(
                 key_name=temp_template, extended_verification_key_file=payment_keys.skey_file
             )
-
-        err_str = str(excinfo.value)
-        assert (
-            "Error: Invalid key." in err_str
-            or "TextEnvelope type error:  Expected one of:" in err_str
-        ), err_str
+        exc_value = str(excinfo.value)
+        with common.allow_unstable_error_messages():
+            assert (
+                "Error: Invalid key." in exc_value
+                or "TextEnvelope type error:  Expected one of:" in exc_value
+            ), exc_value
 
 
 class TestQueryUTxO:
@@ -873,16 +869,15 @@ def test_tx_in_invalid_data(
                     *cluster.magic_args,
                 ]
             )
-
-        err_str = str(excinfo.value)
-
-        if invalid_param == "tx_hash":
-            assert (
-                "expecting hexadecimal digit" in err_str
-                or "expecting transaction id (hexadecimal)" in err_str
-            ), err_str
-        else:
-            assert "expecting digit" in err_str, err_str
+        exc_value = str(excinfo.value)
+        with common.allow_unstable_error_messages():
+            if invalid_param == "tx_hash":
+                assert (
+                    "expecting hexadecimal digit" in exc_value
+                    or "expecting transaction id (hexadecimal)" in exc_value
+                ), exc_value
+            else:
+                assert "expecting digit" in exc_value, exc_value
 
     @allure.link(helpers.get_vcs_link())
     @hypothesis.given(filter_str=st.text(alphabet=common.ADDR_ALPHABET, min_size=1))
@@ -905,9 +900,9 @@ def test_address_invalid_data(self, cluster: clusterlib.ClusterLib, filter_str:
                     *cluster.magic_args,
                 ]
             )
-
-        err_str = str(excinfo.value)
-        assert "invalid address" in err_str, err_str
+        exc_value = str(excinfo.value)
+        with common.allow_unstable_error_messages():
+            assert "invalid address" in exc_value, exc_value
 
 
 class TestStakeAddressKeyHash:
@@ -968,10 +963,9 @@ def test_invalid_verification_key(self, cluster: clusterlib.ClusterLib, option:
                 stake_vkey=vkey if option == "vkey" else None,
                 stake_vkey_file=vkey_file,
             )
-
-        err_str = str(excinfo.value)
-
-        assert "Invalid key" in err_str, err_str
+        exc_value = str(excinfo.value)
+        with common.allow_unstable_error_messages():
+            assert "Invalid key" in exc_value, exc_value
 
 
 class TestAdvancedQueries:
@@ -1427,9 +1421,9 @@ def test_slot_number_invalid_format(
 
         with pytest.raises(clusterlib.CLIError) as excinfo:
             cluster.g_query.query_cli(["slot-number", timestamp_str])
-        err_str = str(excinfo.value)
-
-        assert "parseTimeOrError" in err_str, err_str
+        exc_value = str(excinfo.value)
+        with common.allow_unstable_error_messages():
+            assert "parseTimeOrError" in exc_value, exc_value
 
     @allure.link(helpers.get_vcs_link())
     @pytest.mark.parametrize("time_val", ("above", "bellow"))
@@ -1451,6 +1445,6 @@ def test_slot_number_out_of_range(
 
         with pytest.raises(clusterlib.CLIError) as excinfo:
             cluster.g_query.get_slot_number(timestamp=timestamp)
-        err_str = str(excinfo.value)
-
-        assert "PastHorizon" in err_str, err_str
+        exc_value = str(excinfo.value)
+        with common.allow_unstable_error_messages():
+            assert "PastHorizon" in exc_value, exc_value

cardano_node_tests/tests/test_delegation.py

@@ -493,11 +493,12 @@ def test_deregister_delegated(
                 tx_name=f"{temp_template}_dereg_fail",
                 tx_files=tx_files_deregister,
             )
-        err_msg = str(excinfo.value)
-        assert (
-            "StakeKeyNonZeroAccountBalanceDELEG" in err_msg
-            or "StakeKeyHasNonZeroRewardAccountBalanceDELEG" in err_msg
-        ), err_msg
+        exc_value = str(excinfo.value)
+        with common.allow_unstable_error_messages():
+            assert (
+                "StakeKeyNonZeroAccountBalanceDELEG" in exc_value
+                or "StakeKeyHasNonZeroRewardAccountBalanceDELEG" in exc_value
+            ), exc_value
 
         src_payment_balance = cluster.g_query.get_address_balance(src_address)
         reward_balance = cluster.g_query.get_stake_addr_info(
@@ -664,11 +665,12 @@ def test_delegate_multisig(
                 tx_files=tx_files_deregister,
                 complex_certs=[dereg_cert_script],
             )
-        err_msg = str(excinfo.value)
-        assert (
-            "StakeKeyNonZeroAccountBalanceDELEG" in err_msg
-            or "StakeKeyHasNonZeroRewardAccountBalanceDELEG" in err_msg
-        ), err_msg
+        exc_value = str(excinfo.value)
+        with common.allow_unstable_error_messages():
+            assert (
+                "StakeKeyNonZeroAccountBalanceDELEG" in exc_value
+                or "StakeKeyHasNonZeroRewardAccountBalanceDELEG" in exc_value
+            ), exc_value
 
         src_payment_balance = cluster.g_query.get_address_balance(src_address)
         reward_balance = cluster.g_query.get_stake_addr_info(
@@ -1020,11 +1022,12 @@ def test_delegation_cert_with_wrong_key(
                 stake_pool_id=pool_id,
                 always_abstain=True,
             )
-        err_msg = str(excinfo.value)
-        assert (
-            "Expected: StakeVerificationKeyShelley" in err_msg
-            or "MissingVKeyWitnessesUTXOW" in err_msg
-        ), err_msg
+        exc_value = str(excinfo.value)
+        with common.allow_unstable_error_messages():
+            assert (
+                "Expected: StakeVerificationKeyShelley" in exc_value
+                or "MissingVKeyWitnessesUTXOW" in exc_value
+            ), exc_value
 
     @allure.link(helpers.get_vcs_link())
     @pytest.mark.smoke
@@ -1073,8 +1076,9 @@ def test_delegate_addr_with_wrong_key(
                 tx_name=f"{temp_template}_deleg",
                 tx_files=tx_files,
             )
-        err_msg = str(excinfo.value)
-        assert "MissingVKeyWitnessesUTXOW" in err_msg, err_msg
+        exc_value = str(excinfo.value)
+        with common.allow_unstable_error_messages():
+            assert "MissingVKeyWitnessesUTXOW" in exc_value, exc_value
 
     @allure.link(helpers.get_vcs_link())
     @common.PARAM_BUILD_METHOD_NO_EST
@@ -1121,10 +1125,12 @@ def test_delegate_unknown_addr(
                 witness_override=len(tx_files.signing_key_files),
             )
 
-        err_msg = str(excinfo.value)
-        assert (
-            "StakeDelegationImpossibleDELEG" in err_msg or "StakeKeyNotRegisteredDELEG" in err_msg
-        ), err_msg
+        exc_value = str(excinfo.value)
+        with common.allow_unstable_error_messages():
+            assert (
+                "StakeDelegationImpossibleDELEG" in exc_value
+                or "StakeKeyNotRegisteredDELEG" in exc_value
+            ), exc_value
 
     @allure.link(helpers.get_vcs_link())
     @common.PARAM_BUILD_METHOD_NO_EST
@@ -1190,10 +1196,12 @@ def test_delegate_deregistered_addr(
                 witness_override=len(tx_files.signing_key_files),
             )
 
-        err_msg = str(excinfo.value)
-        assert (
-            "StakeDelegationImpossibleDELEG" in err_msg or "StakeKeyNotRegisteredDELEG" in err_msg
-        ), err_msg
+        exc_value = str(excinfo.value)
+        with common.allow_unstable_error_messages():
+            assert (
+                "StakeDelegationImpossibleDELEG" in exc_value
+                or "StakeKeyNotRegisteredDELEG" in exc_value
+            ), exc_value
 
     @allure.link(helpers.get_vcs_link())
     @pytest.mark.smoke
@@ -1244,8 +1252,9 @@ def test_delegatee_not_registered(
                 tx_name=f"{temp_template}_deleg",
                 tx_files=tx_files,
             )
-        err_msg = str(excinfo.value)
-        assert (
-            "DelegateeNotRegisteredDELEG" in err_msg  # Before cardano-node 10.0.0
-            or "DelegateeStakePoolNotRegisteredDELEG" in err_msg
-        ), err_msg
+        exc_value = str(excinfo.value)
+        with common.allow_unstable_error_messages():
+            assert (
+                "DelegateeNotRegisteredDELEG" in exc_value  # Before cardano-node 10.0.0
+                or "DelegateeStakePoolNotRegisteredDELEG" in exc_value
+            ), exc_value

cardano_node_tests/tests/test_kes.py

@@ -805,7 +805,10 @@ def test_no_kes_period_arg(
                     str(out_file),
                 ]
             )
-        assert "Missing: --kes-period NATURAL" in str(excinfo.value)
+        exc_value = str(excinfo.value)
+        with common.allow_unstable_error_messages():
+            assert "Missing: --kes-period NATURAL" in exc_value, exc_value
+
         assert not out_file.exists(), "New operational certificate was generated"
 
     @allure.link(helpers.get_vcs_link())

cardano_node_tests/tests/test_mir_certs.py

@@ -1071,5 +1071,6 @@ def test_exceed_pay_stake_addr_from(
                 tx_name=temp_template,
                 tx_files=tx_files,
             )
-        err_str = str(excinfo.value)
-        assert "InsufficientForInstantaneousRewardsDELEG" in err_str, err_str
+        exc_value = str(excinfo.value)
+        with common.allow_unstable_error_messages():
+            assert "InsufficientForInstantaneousRewardsDELEG" in exc_value, exc_value