Enable HF testing in protocol version 11

.github/env_nightly

@@ -1,2 +1,3 @@
 CLUSTER_ERA=conway
 COMMAND_ERA=conway
+ENABLE_EXPERIMENTAL=true

.github/env_nightly_cli

@@ -1,3 +1,4 @@
 CLUSTER_ERA=conway
 COMMAND_ERA=conway
 CARDANO_CLI_REV=master
+ENABLE_EXPERIMENTAL=true

.github/env_nightly_dbsync

@@ -5,3 +5,4 @@ DBSYNC_REV=13.6.0.5
 DBSYNC_TAR_URL=https://github.com/IntersectMBO/cardano-db-sync/releases/download/13.6.0.5/cardano-db-sync-13.6.0.5-linux.tar.gz
 SMASH=true
 DBSYNC_SKIP_INDEXES=true
+ENABLE_EXPERIMENTAL=true

.github/env_regression

@@ -0,0 +1 @@
+ENABLE_EXPERIMENTAL=true

.github/env_regression_dbsync

@@ -1 +1,2 @@
 DBSYNC_SKIP_INDEXES=true
+ENABLE_EXPERIMENTAL=true

.github/node_upgrade_pytest.sh

@@ -6,16 +6,15 @@ trap 'echo "Error at line $LINENO"' ERR
 retval=0
 
 export CARDANO_NODE_SOCKET_PATH="$CARDANO_NODE_SOCKET_PATH_CI"
+STATE_CLUSTER="${CARDANO_NODE_SOCKET_PATH_CI%/*}"
 
+# default era to use, can be overridden in each step if needed
 export CLUSTER_ERA="${CLUSTER_ERA:-"conway"}"
-export COMMAND_ERA="$CLUSTER_ERA"
-
+export COMMAND_ERA="${COMMAND_ERA:-"$CLUSTER_ERA"}"
 CLUSTER_SCRIPTS_DIR="$WORKDIR/cluster0_${CLUSTER_ERA}"
-STATE_CLUSTER="${CARDANO_NODE_SOCKET_PATH_CI%/*}"
 
 # init dir for step1 binaries
 STEP1_BIN="$WORKDIR/step1-bin"
-mkdir -p "$STEP1_BIN"
 
 # init reports dir before each step
 export REPORTS_DIR="${REPORTS_DIR:-".reports"}"
@@ -71,6 +70,7 @@ if [ "$1" = "step1" ]; then
   "$CLUSTER_SCRIPTS_DIR/start-cluster" || exit 6
 
   # backup the original cardano binaries
+  mkdir -p "$STEP1_BIN"
   ln -s "$(command -v cardano-node)" "$STEP1_BIN/cardano-node-step1"
   ln -s "$(command -v cardano-cli)" "$STEP1_BIN/cardano-cli-step1"
 
@@ -119,7 +119,7 @@ elif [ "$1" = "step2" ]; then
 
   # re-generate config and topology files
   CARDANO_NODE_SOCKET_PATH="$WORKDIR/dry_config_step2/state-cluster0/bft1.socket" \
-    DRY_RUN=1 \
+    DRY_RUN=true \
     "$CLUSTER_SCRIPTS_DIR/start-cluster"
 
   # copy newly generated topology files to the cluster state dir
@@ -137,7 +137,7 @@ elif [ "$1" = "step2" ]; then
       "$WORKDIR/dry_config_step2/state-cluster0/shelley/genesis.conway.json" > "$STATE_CLUSTER/shelley/genesis.conway.json"
   fi
 
-  # use the original Shelley and Byron genesis files
+  # use the original Byron, Shelley and Dijkstra genesis files
   BYRON_GENESIS_HASH="$(jq -r ".ByronGenesisHash" "$STATE_CLUSTER/config-bft1.json")"
   SHELLEY_GENESIS_HASH="$(jq -r ".ShelleyGenesisHash" "$STATE_CLUSTER/config-bft1.json")"
   # hashes of the original alonzo and conway genesis files
@@ -274,7 +274,8 @@ elif [ "$1" = "step3" ]; then
 
   # re-generate config and topology files
   CARDANO_NODE_SOCKET_PATH="$WORKDIR/dry_config_step3/state-cluster0/bft1.socket" \
-    DRY_RUN=1 \
+    ENABLE_EXPERIMENTAL=true \
+    DRY_RUN=true \
     "$CLUSTER_SCRIPTS_DIR/start-cluster"
 
   # copy newly generated topology files to the cluster state dir
@@ -285,17 +286,25 @@ elif [ "$1" = "step3" ]; then
   SHELLEY_GENESIS_HASH="$(jq -r ".ShelleyGenesisHash" "$STATE_CLUSTER/config-bft1.json")"
   ALONZO_GENESIS_HASH="$(jq -r ".AlonzoGenesisHash" "$STATE_CLUSTER/config-bft1.json")"
   CONWAY_GENESIS_HASH="$(jq -r ".ConwayGenesisHash" "$STATE_CLUSTER/config-bft1.json")"
+  DIJKSTRA_GENESIS_HASH="$(jq -r ".DijkstraGenesisHash" "$STATE_CLUSTER/config-bft1.json")"
+  if [ -z "$DIJKSTRA_GENESIS_HASH" ] || [ "$DIJKSTRA_GENESIS_HASH" = "null" ]; then
+    cp -f "$WORKDIR/dry_config_step3/state-cluster0/shelley/genesis.dijkstra.json" "$STATE_CLUSTER/shelley"
+    DIJKSTRA_GENESIS_HASH="$(cardano-cli latest genesis hash --genesis \
+      "$STATE_CLUSTER/shelley/genesis.dijkstra.json")"
+  fi
   for conf in "$WORKDIR"/dry_config_step3/state-cluster0/config-*.json; do
     fname="${conf##*/}"
     jq \
       --arg byron_hash "$BYRON_GENESIS_HASH" \
       --arg shelley_hash "$SHELLEY_GENESIS_HASH" \
       --arg alonzo_hash "$ALONZO_GENESIS_HASH" \
       --arg conway_hash "$CONWAY_GENESIS_HASH" \
+      --arg dijkstra_hash "$DIJKSTRA_GENESIS_HASH" \
       '.ByronGenesisHash = $byron_hash
       | .ShelleyGenesisHash = $shelley_hash
       | .AlonzoGenesisHash = $alonzo_hash
-      | .ConwayGenesisHash = $conway_hash' \
+      | .ConwayGenesisHash = $conway_hash
+      | .DijkstraGenesisHash = $dijkstra_hash' \
       "$conf" > "$STATE_CLUSTER/$fname"
   done
 

cardano_node_tests/tests/plutus_common.py

@@ -875,7 +875,7 @@ def check_return_collateral(cluster_obj: clusterlib.ClusterLib, tx_output: clust
 
 def xfail_on_secp_error(cluster_obj: clusterlib.ClusterLib, algorithm: str, err_msg: str):
     """Xfail a test based on error message when using SECP functions."""
-    before_pv8 = cluster_obj.g_query.get_protocol_params()["protocolVersion"]["major"] < 8
+    before_pv8 = clusterlib_utils.get_protocol_version(cluster_obj=cluster_obj) < 8
 
     # The SECP256k1 functions should work from PV8.
     # Before PV8 the SECP256k1 is blocked or limited by high cost model

cardano_node_tests/tests/test_node_upgrade.py

@@ -1,5 +1,6 @@
 """Tests for node upgrade."""
 
+import json
 import logging
 import os
 import pathlib as pl
@@ -13,6 +14,7 @@
 from cardano_node_tests.cluster_management import cluster_management
 from cardano_node_tests.tests import common
 from cardano_node_tests.tests.tests_conway import conway_common
+from cardano_node_tests.utils import cluster_nodes
 from cardano_node_tests.utils import clusterlib_utils
 from cardano_node_tests.utils import governance_setup
 from cardano_node_tests.utils import governance_utils
@@ -105,7 +107,6 @@ def test_ignore_log_errors(
             )
 
     @allure.link(helpers.get_vcs_link())
-    @pytest.mark.disabled(reason="The test is not needed when we are already in PV10 on mainnet")
     @pytest.mark.skipif(UPGRADE_TESTS_STEP != 2, reason="runs only on step 2 of upgrade testing")
     def test_update_cost_models(
         self,
@@ -130,7 +131,6 @@ def test_update_cost_models(
         )
 
     @allure.link(helpers.get_vcs_link())
-    @pytest.mark.disabled(reason="The test is not needed when we are already in PV10 on mainnet")
     @pytest.mark.skipif(UPGRADE_TESTS_STEP != 3, reason="runs only on step 3 of upgrade testing")
     def test_hardfork(
         self,
@@ -142,6 +142,22 @@ def test_hardfork(
         cluster = cluster_singleton
         temp_template = common.get_test_id(cluster)
 
+        prot_ver_init = clusterlib_utils.get_protocol_version(cluster_obj=cluster)
+        prot_ver_target = prot_ver_init + 1
+
+        if prot_ver_init >= VERSIONS.LAST_KNOWN_PROTOCOL_VERSION:
+            pytest.skip(
+                "The major protocol version needs to be at most "
+                f"{VERSIONS.LAST_KNOWN_PROTOCOL_VERSION - 1}."
+            )
+
+        with open(
+            cluster_nodes.get_cluster_env().state_dir / "config-pool1.json", encoding="utf-8"
+        ) as in_json:
+            is_experimental_enabled = bool(json.load(in_json).get("ExperimentalHardForksEnabled"))
+        if not is_experimental_enabled:
+            pytest.skip("Experimental hard-forks are not enabled on the cluster.")
+
         governance_data = governance_setup.get_default_governance(
             cluster_manager=cluster_manager, cluster_obj=cluster
         )
@@ -160,7 +176,7 @@ def test_hardfork(
             deposit_amt=deposit_amt,
             anchor_url=anchor_data.url,
             anchor_data_hash=anchor_data.hash,
-            protocol_major_version=10,
+            protocol_major_version=prot_ver_target,
             protocol_minor_version=0,
             prev_action_txid=prev_action_rec.txid,
             prev_action_ix=prev_action_rec.ix,
@@ -214,6 +230,7 @@ def test_hardfork(
             action_txid=action_txid,
             action_ix=action_ix,
             approve_cc=True,
+            approve_drep=True if prot_ver_init > 9 else None,
             approve_spo=True,
         )
 
@@ -232,7 +249,7 @@ def test_hardfork(
         )
         assert rat_action, "Action not found in ratified actions"
 
-        assert rat_gov_state["currentPParams"]["protocolVersion"]["major"] == 9, (
+        assert rat_gov_state["currentPParams"]["protocolVersion"]["major"] == prot_ver_init, (
             "Incorrect major version"
         )
 
@@ -242,7 +259,7 @@ def test_hardfork(
         conway_common.save_gov_state(
             gov_state=enact_gov_state, name_template=f"{temp_template}_enact_{enact_epoch}"
         )
-        assert enact_gov_state["currentPParams"]["protocolVersion"]["major"] == 10, (
+        assert enact_gov_state["currentPParams"]["protocolVersion"]["major"] == prot_ver_target, (
             "Incorrect major version"
         )
 

cardano_node_tests/tests/tests_conway/conway_common.py

@@ -31,8 +31,7 @@ def is_in_bootstrap(
     cluster_obj: clusterlib.ClusterLib,
 ) -> bool:
     """Check if the cluster is in bootstrap period."""
-    pv = cluster_obj.g_query.get_protocol_params()["protocolVersion"]["major"]
-    return bool(pv == 9)
+    return bool(clusterlib_utils.get_protocol_version(cluster_obj=cluster_obj) == 9)
 
 
 def get_committee_val(data: dict[str, tp.Any]) -> dict[str, tp.Any]:

cardano_node_tests/tests/tests_conway/test_hardfork.py

@@ -1,5 +1,6 @@
 """Tests for Conway hard-fork."""
 
+import json
 import logging
 
 import allure
@@ -10,6 +11,7 @@
 from cardano_node_tests.tests import common
 from cardano_node_tests.tests import reqs_conway as reqc
 from cardano_node_tests.tests.tests_conway import conway_common
+from cardano_node_tests.utils import cluster_nodes
 from cardano_node_tests.utils import clusterlib_utils
 from cardano_node_tests.utils import governance_utils
 from cardano_node_tests.utils import helpers
@@ -55,7 +57,6 @@ def test_hardfork(
         """Test hardfork action.
 
         * create a "hardfork" action
-        * check that DReps cannot vote during the bootstrap period
         * vote to disapprove the action
         * vote to approve the action
         * check that the action is ratified
@@ -66,8 +67,21 @@ def test_hardfork(
         cluster, governance_data = cluster_lock_governance
         temp_template = common.get_test_id(cluster)
 
-        if not conway_common.is_in_bootstrap(cluster_obj=cluster):
-            pytest.skip("The major protocol version needs to be 9.")
+        prot_ver_init = clusterlib_utils.get_protocol_version(cluster_obj=cluster)
+        prot_ver_target = prot_ver_init + 1
+
+        if prot_ver_init >= VERSIONS.LAST_KNOWN_PROTOCOL_VERSION:
+            pytest.skip(
+                "The major protocol version needs to be at most "
+                f"{VERSIONS.LAST_KNOWN_PROTOCOL_VERSION - 1}."
+            )
+
+        with open(
+            cluster_nodes.get_cluster_env().state_dir / "config-pool1.json", encoding="utf-8"
+        ) as in_json:
+            is_experimental_enabled = bool(json.load(in_json).get("ExperimentalHardForksEnabled"))
+        if not is_experimental_enabled:
+            pytest.skip("Experimental hard-forks are not enabled on the cluster.")
 
         init_return_account_balance = cluster.g_query.get_stake_addr_info(
             pool_user_lg.stake.address
@@ -92,7 +106,7 @@ def test_hardfork(
             deposit_amt=deposit_amt,
             anchor_url=anchor_data.url,
             anchor_data_hash=anchor_data.hash,
-            protocol_major_version=10,
+            protocol_major_version=prot_ver_target,
             protocol_minor_version=0,
             prev_action_txid=prev_action_rec.txid,
             prev_action_ix=prev_action_rec.ix,
@@ -149,23 +163,24 @@ def test_hardfork(
         action_ix = prop_action["actionId"]["govActionIx"]
 
         # Check that DReps cannot vote
-        reqc.cip026_04.start(url=helpers.get_vcs_link())
-        with pytest.raises(clusterlib.CLIError) as excinfo:
-            conway_common.cast_vote(
-                cluster_obj=cluster,
-                governance_data=governance_data,
-                name_template=f"{temp_template}_no",
-                payment_addr=pool_user_lg.payment,
-                action_txid=action_txid,
-                action_ix=action_ix,
-                approve_cc=False,
-                approve_drep=False,
-                approve_spo=False,
-            )
-        exc_value = str(excinfo.value)
-        with common.allow_unstable_error_messages():
-            assert "DisallowedVotesDuringBootstrap ((DRepVoter" in exc_value, exc_value
-        reqc.cip026_04.success()
+        if prot_ver_init == 9:
+            reqc.cip026_04.start(url=helpers.get_vcs_link())
+            with pytest.raises(clusterlib.CLIError) as excinfo:
+                conway_common.cast_vote(
+                    cluster_obj=cluster,
+                    governance_data=governance_data,
+                    name_template=f"{temp_template}_no",
+                    payment_addr=pool_user_lg.payment,
+                    action_txid=action_txid,
+                    action_ix=action_ix,
+                    approve_cc=False,
+                    approve_drep=False,
+                    approve_spo=False,
+                )
+            exc_value = str(excinfo.value)
+            with common.allow_unstable_error_messages():
+                assert "DisallowedVotesDuringBootstrap ((DRepVoter" in exc_value, exc_value
+            reqc.cip026_04.success()
 
         # Vote & disapprove the action
         reqc.cip043_01.start(url=helpers.get_vcs_link())
@@ -177,6 +192,7 @@ def test_hardfork(
             action_txid=action_txid,
             action_ix=action_ix,
             approve_cc=False,
+            approve_drep=False if prot_ver_init > 9 else None,
             approve_spo=False,
         )
         reqc.cli019.success()
@@ -190,6 +206,7 @@ def test_hardfork(
             action_txid=action_txid,
             action_ix=action_ix,
             approve_cc=True,
+            approve_drep=True if prot_ver_init > 9 else None,
             approve_spo=True,
         )
 
@@ -221,26 +238,27 @@ def test_hardfork(
             action_txid=action_txid,
             action_ix=action_ix,
             approve_cc=False,
+            approve_drep=False if prot_ver_init > 9 else None,
             approve_spo=False,
         )
 
         assert rat_gov_state["nextRatifyState"]["ratificationDelayed"], "Ratification not delayed"
         reqc.cip038_07.success()
 
-        assert rat_gov_state["currentPParams"]["protocolVersion"]["major"] == 9, (
+        assert rat_gov_state["currentPParams"]["protocolVersion"]["major"] == prot_ver_init, (
             "Incorrect major version"
         )
 
         # Check enactment
-        expected_msgs = [("pool1.stdout", r"ProtVer \{pvMajor = Version 10")]
+        expected_msgs = [("pool1.stdout", rf"ProtVer \{{pvMajor = Version {prot_ver_target}")]
         with logfiles.expect_messages(expected_msgs):
             enact_epoch = cluster.wait_for_epoch(epoch_no=init_epoch + 2, padding_seconds=15)
 
         enact_gov_state = cluster.g_query.get_gov_state()
         conway_common.save_gov_state(
             gov_state=enact_gov_state, name_template=f"{temp_template}_enact_{enact_epoch}"
         )
-        assert enact_gov_state["currentPParams"]["protocolVersion"]["major"] == 10, (
+        assert enact_gov_state["currentPParams"]["protocolVersion"]["major"] == prot_ver_target, (
             "Incorrect major version"
         )
 
@@ -268,7 +286,8 @@ def test_hardfork(
                 payment_addr=pool_user_lg.payment,
                 action_txid=action_txid,
                 action_ix=action_ix,
-                approve_drep=False,
+                approve_cc=False,
+                approve_drep=False if prot_ver_init > 9 else None,
                 approve_spo=False,
             )
         exc_value = str(excinfo.value)

cardano_node_tests/tests/tests_plutus_v2/test_mint_secp256k1_build.py

@@ -12,6 +12,7 @@
 from cardano_node_tests.tests import common
 from cardano_node_tests.tests import plutus_common
 from cardano_node_tests.tests.tests_plutus_v2 import mint_build
+from cardano_node_tests.utils import clusterlib_utils
 from cardano_node_tests.utils import helpers
 
 LOGGER = logging.getLogger(__name__)
@@ -167,7 +168,7 @@ def test_use_secp_builtin_functions(
                 redeemer_file=redeemer_file,
             )
         except clusterlib.CLIError as err:
-            before_pv8 = cluster.g_query.get_protocol_params()["protocolVersion"]["major"] < 8
+            before_pv8 = clusterlib_utils.get_protocol_version(cluster_obj=cluster) < 8
 
             # The SECP256k1 functions should work from protocol version 8
             if not before_pv8:
@@ -233,7 +234,7 @@ def test_negative_secp_builtin_functions(
 
         redeemer_file = redeemer_dir / f"{test_vector}.redeemer"
 
-        before_pv8 = cluster.g_query.get_protocol_params()["protocolVersion"]["major"] < 8
+        before_pv8 = clusterlib_utils.get_protocol_version(cluster_obj=cluster) < 8
 
         with pytest.raises(clusterlib.CLIError) as excinfo:
             self._fund_issuer_mint_token(