diff --git a/cardano-node-chairman/test/Spec/Chairman/Cardano.hs b/cardano-node-chairman/test/Spec/Chairman/Cardano.hs index e443ea2bd07..ce64c4e0a5a 100644 --- a/cardano-node-chairman/test/Spec/Chairman/Cardano.hs +++ b/cardano-node-chairman/test/Spec/Chairman/Cardano.hs @@ -19,7 +19,7 @@ hprop_chairman :: H.Property hprop_chairman = integrationRetryWorkspace 2 "cardano-chairman" $ \tempAbsPath' -> H.runWithDefaultWatchdog_ $ do conf <- mkConf tempAbsPath' - let creationOptions = def{ creationNodes = cardanoDefaultTestnetNodeOptions } + let creationOptions = def{ creationNodes = cardanoDefaultTestnetNodesWithOptions } allNodes <- testnetNodes <$> createAndRunTestnet creationOptions def conf chairmanOver 120 50 conf allNodes diff --git a/cardano-node/src/Cardano/Node/Testnet/Paths.hs b/cardano-node/src/Cardano/Node/Testnet/Paths.hs index 9c125ffcc05..82feb6f35eb 100644 --- a/cardano-node/src/Cardano/Node/Testnet/Paths.hs +++ b/cardano-node/src/Cardano/Node/Testnet/Paths.hs @@ -15,6 +15,7 @@ module Cardano.Node.Testnet.Paths , defaultSocketPath , defaultConfigFile , defaultPortFile + , defaultNodeEnvFile ) where import System.FilePath (()) @@ -62,3 +63,7 @@ defaultConfigFile = "configuration.yaml" -- | Relative path to a node's port file: @defaultNodeDataDir n "port"@ defaultPortFile :: Int -> FilePath defaultPortFile n = defaultNodeDataDir n "port" + +-- | Relative path to a node's env file: @defaultNodeDataDir n "env"@ +defaultNodeEnvFile :: Int -> FilePath +defaultNodeEnvFile n = defaultNodeDataDir n "env" diff --git a/cardano-testnet/cardano-testnet.cabal b/cardano-testnet/cardano-testnet.cabal index 43b7a3cea20..395ba15ec78 100644 --- a/cardano-testnet/cardano-testnet.cabal +++ b/cardano-testnet/cardano-testnet.cabal @@ -84,6 +84,7 @@ library , network , network-mux , optparse-applicative-fork + , parsec , ouroboros-network:{api, framework, ouroboros-network} ^>= 1.1 , cardano-diffusion:{api, cardano-diffusion} ^>= 1.0 , prettyprinter diff --git a/cardano-testnet/changelog.d/20260506_035740_palas_testnet_specify_node_bin_per_node.md b/cardano-testnet/changelog.d/20260506_035740_palas_testnet_specify_node_bin_per_node.md new file mode 100644 index 00000000000..2ece341692e --- /dev/null +++ b/cardano-testnet/changelog.d/20260506_035740_palas_testnet_specify_node_bin_per_node.md @@ -0,0 +1,4 @@ +### Added + +- Added `--nodes` flag to specify node roles (SPO/relay) and custom `cardano-node` binaries per node. + Example: `--nodes spo,spo:node-bin=/path/to/bin,relay,relay`. diff --git a/cardano-testnet/src/Cardano/Testnet.hs b/cardano-testnet/src/Cardano/Testnet.hs index b965252d776..0c0059789a7 100644 --- a/cardano-testnet/src/Cardano/Testnet.hs +++ b/cardano-testnet/src/Cardano/Testnet.hs @@ -14,9 +14,9 @@ module Cardano.Testnet ( TestnetRuntimeOptions(..), TestnetEnvOptions(..), RpcSupport(..), - TestnetNodeOptions(..), - NodeOptions(..), - cardanoDefaultTestnetNodeOptions, + TestnetNodesWithOptions(..), + NodeWithOptions(..), + cardanoDefaultTestnetNodesWithOptions, getDefaultAlonzoGenesis, getDefaultShelleyGenesis, diff --git a/cardano-testnet/src/Parsers/Cardano.hs b/cardano-testnet/src/Parsers/Cardano.hs index d93ec846982..7ba916f8020 100644 --- a/cardano-testnet/src/Parsers/Cardano.hs +++ b/cardano-testnet/src/Parsers/Cardano.hs @@ -3,6 +3,7 @@ module Parsers.Cardano ( cmdCardano , cmdCreateEnv + , parseNodeSpecs ) where import Cardano.Api (AnyShelleyBasedEra (..)) @@ -13,6 +14,7 @@ import Cardano.Prelude (readMaybe) import Prelude import Control.Applicative (optional, (<|>)) +import Control.Monad (unless) import Data.Default.Class (def) import qualified Data.List as L import Data.List.NonEmpty (NonEmpty ((:|))) @@ -21,6 +23,9 @@ import Data.Word (Word64) import Options.Applicative (CommandFields, Mod, Parser) import qualified Options.Applicative as OA import Options.Applicative.Types (readerAsk) +import Text.Parsec (char, many1, noneOf, + sepBy1, string, try, (), parse, eof, notFollowedBy) +import qualified Text.Parsec as Parsec import Testnet.Defaults (defaultEra) import Testnet.Start.Cardano @@ -55,7 +60,7 @@ pFromEnv = TestnetEnvOptions pCreationOptions :: Parser TestnetCreationOptions pCreationOptions = TestnetCreationOptions - <$> pTestnetNodeOptions + <$> pTestnetNodesWithOptions <*> pure (AnyShelleyBasedEra defaultEra) <*> pMaxLovelaceSupply <*> pNumDReps @@ -105,21 +110,19 @@ pKesSource = OA.flag UseKesKeyFile UseKesSocket <> OA.showDefault ) -pTestnetNodeOptions :: Parser TestnetNodeOptions -pTestnetNodeOptions = - fmap (maybe cardanoDefaultTestnetNodeOptions mkPoolNodes) <$> - optional $ OA.option ensureAtLeastOne - ( OA.long "num-pool-nodes" - <> OA.help "Number of pool nodes. Note this uses a default node configuration for all nodes." - <> OA.metavar "COUNT" - ) +pTestnetNodesWithOptions :: Parser TestnetNodesWithOptions +pTestnetNodesWithOptions = + pNodes <|> pNumPoolNodes <|> pure cardanoDefaultTestnetNodesWithOptions where - defaultSpoOption = NodeOptions [] - - mkPoolNodes num = TestnetNodeOptions - { optSpoNodes = defaultSpoOption :| L.replicate (num - 1) defaultSpoOption - , optRelayNodes = [] - } + pNumPoolNodes :: Parser TestnetNodesWithOptions + pNumPoolNodes = + (\num -> TestnetNodesWithOptions { optSpoNodes = defaultSpoOption :| L.replicate (num - 1) defaultSpoOption, optRelayNodes = [] }) <$> + OA.option ensureAtLeastOne + ( OA.long "num-pool-nodes" + <> OA.help "Number of pool nodes. Note this uses a default node configuration for all nodes." + <> OA.metavar "COUNT" + ) + defaultSpoOption = NodeWithOptions Nothing [] ensureAtLeastOne :: OA.ReadM Int ensureAtLeastOne = readerAsk >>= \arg -> @@ -127,6 +130,65 @@ pTestnetNodeOptions = Just n | n >= 1 -> pure n _ -> fail "Need at least one SPO node to produce blocks, but got none." + pNodes :: Parser TestnetNodesWithOptions + pNodes = OA.option readNodeSpecs + ( OA.long "nodes" + <> OA.help "Comma-separated node specifications. SPO nodes must come before relay nodes. \ + \Each spec is a role (spo or relay) optionally followed by :node-bin=. \ + \If the path contains commas, colons, double quotes, or backslashes, wrap it \ + \in double quotes and escape any literal double quotes as \\\" and backslashes \ + \as \\\\ within. To prevent bash from consuming the double quotes, enclose the \ + \whole argument in single quotes. \ + \Examples: --nodes spo,spo:node-bin=/path/to/bin,relay,relay | \ + \--nodes 'spo:node-bin=\"/path,with:commas\",relay'" + <> OA.metavar "SPEC[,SPEC...]" + ) + + readNodeSpecs :: OA.ReadM TestnetNodesWithOptions + readNodeSpecs = readerAsk >>= either (fail . show) pure . parseNodeSpecs + +-- | Parse a @--nodes@ argument string into 'TestnetNodesWithOptions'. +parseNodeSpecs :: String -> Either Parsec.ParseError TestnetNodesWithOptions +parseNodeSpecs = parse (nodeSpecsParser <* eof) "Error parsing node specifications" + where + nodeSpecsParser :: Parsec.Parsec String () TestnetNodesWithOptions + nodeSpecsParser = do + specs <- nodeSpec `sepBy1` char ',' + let (spos, relays) = span (\(role, _) -> role == Spo) specs + unless (all (\(role, _) -> role == Relay) relays) $ + fail "SPO nodes must come before relay nodes. Example: --nodes spo,spo,relay,relay" + case map snd spos of + [] -> fail "Need at least one SPO node to produce blocks." + (s:ss) -> pure $ TestnetNodesWithOptions + { optSpoNodes = s :| ss + , optRelayNodes = map snd relays + } + + nodeSpec :: Parsec.Parsec String () (NodeRole, NodeWithOptions) + nodeSpec = do + role <- nodeRole + bin <- optional $ char ':' *> nodeBinKV + pure (role, NodeWithOptions bin []) + + nodeRole :: Parsec.Parsec String () NodeRole + nodeRole = + Spo <$ try (string "spo" <* notFollowedBy (noneOf ",:\"\\")) + <|> Relay <$ try (string "relay" <* notFollowedBy (noneOf ",:\"\\")) + "node role (\"spo\" or \"relay\")" + + nodeBinKV :: Parsec.Parsec String () FilePath + nodeBinKV = string "node-bin=" *> (quotedPath <|> unquotedPath) "\"node-bin=\", where is the path to the node binary, optionally quoted if it contains special characters" + + quotedPath :: Parsec.Parsec String () FilePath + quotedPath = char '"' *> Parsec.many quotedChar <* char '"' + where + quotedChar = try (char '\\' *> (char '"' <|> char '\\')) <|> noneOf "\"" + + unquotedPath :: Parsec.Parsec String () FilePath + unquotedPath = many1 (noneOf ",:\"\\") + +data NodeRole = Spo | Relay deriving Eq + pOnChainParams :: Parser TestnetOnChainParams pOnChainParams = fmap (fromMaybe DefaultParams) <$> optional $ pCustomParamsFile <|> pMainnetParams diff --git a/cardano-testnet/src/Parsers/Run.hs b/cardano-testnet/src/Parsers/Run.hs index 145427db762..1d5971dadac 100644 --- a/cardano-testnet/src/Parsers/Run.hs +++ b/cardano-testnet/src/Parsers/Run.hs @@ -91,7 +91,7 @@ runCardanoOptions = \case let dirName = envPath fromEnvOptions unlessM (doesDirectoryExist dirName) $ error $ "The provided path does not exist or is not a directory: " <> dirName conf <- mkConfigAbs dirName - nodes <- readNodeOptionsFromEnv (unTmpAbsPath (tempAbsPath conf)) + nodes <- readNodesWithOptionsFromEnv (unTmpAbsPath (tempAbsPath conf)) runSimpleApp . runResourceT $ do logInfo $ "Starting testnet in environment: " <> display (tempAbsPath conf) void $ cardanoTestnet nodes fromEnvRuntimeOptions diff --git a/cardano-testnet/src/Testnet/Process/RunIO.hs b/cardano-testnet/src/Testnet/Process/RunIO.hs index 719e2eeb832..fd664db0a64 100644 --- a/cardano-testnet/src/Testnet/Process/RunIO.hs +++ b/cardano-testnet/src/Testnet/Process/RunIO.hs @@ -12,6 +12,7 @@ module Testnet.Process.RunIO , procNode , procKesAgent , execKesAgentControl_ + , procCustom , procFlex , liftIOAnnotated ) where @@ -144,8 +145,7 @@ execFlexAny' execConfig pkgBin envBin arguments = GHC.withFrozenCallStack $ do cp <- procFlex' execConfig pkgBin envBin arguments liftIOAnnotated $ IO.readCreateProcessWithExitCode cp "" - - +-- | Like 'procFlex', but takes an explicit 'ExecConfig' instead of using 'defaultExecConfig'. procFlex' :: HasCallStack => MonadIO m @@ -160,7 +160,20 @@ procFlex' -- ^ Captured stdout procFlex' execConfig pkg binaryEnv arguments = GHC.withFrozenCallStack $ do bin <- binFlex pkg binaryEnv - return (IO.proc bin arguments) + procCustom' execConfig bin arguments + +-- | Build a 'CreateProcess' from an already-resolved binary path, arguments, and 'ExecConfig'. +procCustom' + :: (HasCallStack) + => MonadIO m + => ExecConfig + -> FilePath + -- ^ Path to the binary + -> [String] + -- ^ Arguments to the CLI command + -> m CreateProcess +procCustom' execConfig bin arguments = GHC.withFrozenCallStack $ + pure (IO.proc bin arguments) { IO.env = getLast $ execConfigEnv execConfig , IO.cwd = getLast $ execConfigCwd execConfig -- this allows sending signals to the created processes, without killing the test-suite process @@ -311,6 +324,17 @@ procFlex -- ^ Captured stdout procFlex = procFlex' defaultExecConfig +-- | Like 'procFlex', but takes an explicit binary path instead of resolving +-- via package name and environment variable. +procCustom + :: (HasCallStack) + => FilePath + -- ^ Path to the binary + -> [String] + -- ^ Arguments to the CLI command + -> RIO env CreateProcess +procCustom = procCustom' defaultExecConfig + -- This will also catch async exceptions as well. liftIOAnnotated :: (HasCallStack, MonadIO m) => IO a -> m a liftIOAnnotated action = GHC.withFrozenCallStack $ diff --git a/cardano-testnet/src/Testnet/Runtime.hs b/cardano-testnet/src/Testnet/Runtime.hs index bd5dbe1527c..e271f264d42 100644 --- a/cardano-testnet/src/Testnet/Runtime.hs +++ b/cardano-testnet/src/Testnet/Runtime.hs @@ -57,7 +57,7 @@ import Cardano.Node.Testnet.Paths (defaultSocketName) import qualified Testnet.Ping as Ping import Testnet.Process.Run (ProcessError (..), initiateProcess) import Testnet.Process.RunIO (execCli_, execKesAgentControl_, liftIOAnnotated, - procKesAgent, procNode) + procCustom, procKesAgent, procNode) import Testnet.Types (TestnetKesAgent (..), TestnetNode (..), TestnetRuntime (configurationFile), showIpv4Address, testnetSprockets) @@ -121,11 +121,13 @@ startNode -- ^ Node port -> Int -- ^ Testnet magic + -> Maybe FilePath + -- ^ Optional custom node binary. 'Nothing' uses the default resolution. -> [String] -- ^ The command to execute to start the node. -- @--socket-path@, @--port@, and @--host-addr@ gets added automatically. -> ExceptT NodeStartFailure m TestnetNode -startNode tp node ipv4 port _testnetMagic nodeCmd = GHC.withFrozenCallStack $ do +startNode tp node ipv4 port _testnetMagic mNodeBin nodeCmd = GHC.withFrozenCallStack $ do let tempBaseAbsPath = makeTmpBaseAbsPath tp socketDir = makeSocketDir tp logDir = makeLogDir tp @@ -156,7 +158,10 @@ startNode tp node ipv4 port _testnetMagic nodeCmd = GHC.withFrozenCallStack $ do , "--port", show port , "--host-addr", showIpv4Address ipv4 ] - nodeProcess <- newExceptT . fmap (first ExecutableRelatedFailure) . try $ runRIO () $ procNode completeNodeCmd + nodeProcess <- newExceptT . fmap (first ExecutableRelatedFailure) . try $ runRIO () $ + case mNodeBin of + Nothing -> procNode completeNodeCmd + Just bin -> procCustom bin completeNodeCmd -- The port number if it is obtained using 'H.randomPort', it is firstly bound to and then closed. The closing -- and release in the operating system is done asynchronously and can be slow. Here we wait until the port diff --git a/cardano-testnet/src/Testnet/Start/Cardano.hs b/cardano-testnet/src/Testnet/Start/Cardano.hs index 0395bd1ba91..e5c89af4e5c 100644 --- a/cardano-testnet/src/Testnet/Start/Cardano.hs +++ b/cardano-testnet/src/Testnet/Start/Cardano.hs @@ -17,9 +17,9 @@ module Testnet.Start.Cardano , TestnetCreationOptions(..) , TestnetRuntimeOptions(..) , TestnetEnvOptions(..) - , TestnetNodeOptions(..) - , NodeOptions(..) - , cardanoDefaultTestnetNodeOptions + , TestnetNodesWithOptions(..) + , NodeWithOptions(..) + , cardanoDefaultTestnetNodesWithOptions , TestnetRuntime (..) @@ -28,7 +28,7 @@ module Testnet.Start.Cardano , createTestnetEnv , getDefaultAlonzoGenesis , getDefaultShelleyGenesis - , readNodeOptionsFromEnv + , readNodesWithOptionsFromEnv , retryOnAddressInUseError , liftToIntegration @@ -47,11 +47,14 @@ import Ouroboros.Network.PeerSelection.RelayAccessPoint (RelayAccessPo import Prelude hiding (lines) import Control.Concurrent (threadDelay) -import Control.Monad (forM, forM_, unless, when) +import Control.Monad (forM, forM_, guard, unless, when) +import Control.Monad.Trans.Maybe (runMaybeT) +import Control.Exception (IOException) import Control.Monad.Catch import Control.Monad.Trans.Resource (MonadResource, getInternalState) import Data.Aeson import qualified Data.Aeson.Encode.Pretty as A +import qualified Data.Yaml as Yaml import qualified Data.ByteString.Lazy as LBS import Data.Default.Class () import Data.Either @@ -67,12 +70,13 @@ import Data.Time.Clock (NominalDiffTime) import qualified Data.Time.Clock as DTC import GHC.Stack import qualified System.Directory as IO +import qualified System.Process as Process import System.FilePath (()) import Testnet.Components.Configuration import qualified Testnet.Defaults as Defaults -import Cardano.Node.Testnet.Paths (defaultConfigFile, defaultPortFile, - defaultUtxoAddrPath) +import Cardano.Node.Testnet.Paths (defaultConfigFile, defaultNodeEnvFile, + defaultPortFile, defaultUtxoAddrPath) import Testnet.Filepath import Testnet.Handlers (interruptNodesOnSigINT) import Testnet.Orphans () @@ -93,6 +97,7 @@ import RIO.State (put) import UnliftIO.Async import UnliftIO.Exception (stringException) + liftToIntegration :: HasCallStack => RIO ResourceMap a -> H.Integration a liftToIntegration r = do rMap <- lift $ lift getInternalState @@ -109,7 +114,7 @@ createTestnetEnv :: () createTestnetEnv creationOptions@TestnetCreationOptions { creationEra=asbe - , creationNodes=TestnetNodeOptions{optSpoNodes, optRelayNodes} + , creationNodes=TestnetNodesWithOptions{optSpoNodes, optRelayNodes} } Conf { genesisHashesPolicy @@ -139,8 +144,8 @@ createTestnetEnv let portNumbersMap = Map.fromList portNumbers - -- Create network topology and write port files - forM_ nodeIds $ \i -> do + -- Create network topology, write port files, and write env files for custom binaries + forM_ numberedNodes $ \(i, nodeOption) -> do let nodeDataDir = tmpAbsPath Defaults.defaultNodeDataDir i liftIOAnnotated $ IO.createDirectoryIfMissing True nodeDataDir @@ -153,6 +158,14 @@ createTestnetEnv let topology = Defaults.defaultP2PTopology producers liftIOAnnotated . LBS.writeFile (nodeDataDir "topology.json") $ A.encodePretty topology + -- Write env file for nodes with custom binaries + forM_ (nodeBin nodeOption) $ \bin -> do + absBin <- liftIOAnnotated $ IO.makeAbsolute bin + version <- getNodeVersion absBin + let envFile = tmpAbsPath defaultNodeEnvFile i + nodeEnv = NodeEnv { nodeBinary = absBin, nodeVersion = version } + liftIOAnnotated $ Yaml.encodeFile envFile nodeEnv + -- | Starts a number of nodes, as given by the first argument. You can either: -- -- 1. Pass a value 'UserProvidedNodeOptions filepath' to specify your own node configuration file. @@ -224,12 +237,12 @@ cardanoTestnet => MonadResource m => MonadCatch m => MonadFail m - => TestnetNodeOptions -- ^ The nodes to start + => TestnetNodesWithOptions -- ^ The nodes to start -> TestnetRuntimeOptions -- ^ Runtime options -> Conf -- ^ Path to the test sandbox -> m TestnetRuntime cardanoTestnet - TestnetNodeOptions{optSpoNodes=cardanoSpoNodes, optRelayNodes=cardanoRelayNodes} + TestnetNodesWithOptions{optSpoNodes=cardanoSpoNodes, optRelayNodes=cardanoRelayNodes} TestnetRuntimeOptions { runtimeEnableNewEpochStateLogging=enableNewEpochStateLogging , runtimeEnableRpc=cardanoEnableRpc @@ -310,7 +323,7 @@ cardanoTestnet let portNumbersMap = Map.fromList portNumbers - eTestnetNodes <- forConcurrently (zip [1..] allNodes) $ \(i, (isSpo, nodeOptions)) -> do + eTestnetNodes <- forConcurrently (zip [1..] allNodes) $ \(i, (isSpo, nodeWithOptions)) -> do port <- case Map.lookup i portNumbersMap of Just p -> pure p Nothing -> throwString $ "Port not found for node " <> show i @@ -318,48 +331,48 @@ cardanoTestnet nodeDataDir = tmpAbsPath Defaults.defaultNodeDataDir i nodePoolKeysDir = tmpAbsPath Defaults.defaultSpoKeysDir i (mKeys, spoNodeCliArgs) <- if not isSpo then pure (Nothing, []) else do - -- depending on testnet configuration, either start a 'kes-agent' or use a key from disk - kesSourceCliArg <- - case cardanoKESSource of - UseKesKeyFile -> pure ["--shelley-kes-key", nodePoolKeysDir "kes.skey"] - UseKesSocket -> do - -- wait startTimeOffsetSeconds so that the startTime from shelly-genesis.json is not in the future, - -- as otherwise we will trigger an underflow in kes-agent with a negative time difference. - liftIOAnnotated $ threadDelay (startTimeOffsetSeconds * 1_000_000) - kesAgent <- runExceptT $ - initAndStartKesAgent (TmpAbsolutePath tmpAbsPath) nodeName - TestnetKesAgentArgs{ tkaaShelleyGenesisFile = shelleyGenesisFile - , tkaaColdVKeyFile = nodePoolKeysDir "cold.vkey" - , tkaaColdSKeyFile = nodePoolKeysDir "cold.skey" - , tkaaKesVKeyFile = nodePoolKeysDir "kes.vkey" - , tkaaOpcertCounterFile = nodePoolKeysDir "opcert.counter" - , tkaaOpcertFile = nodePoolKeysDir "opcert.cert" - } - case kesAgent of - Left e -> do - -- TODO: fail if could not start KES agent - liftIOAnnotated . putStrLn $ "Could not start KES agent: " <> show e - pure ["--shelley-kes-key", nodePoolKeysDir "kes.skey"] - Right (TestnetKesAgent{kesAgentServiceSprocket}) -> - pure ["--shelley-kes-agent-socket", sprocketSystemName kesAgentServiceSprocket] - let shelleyCliArgs = [ "--shelley-vrf-key", unFile $ signingKey poolNodeKeysVrf - , "--shelley-operational-certificate", nodePoolKeysDir "opcert.cert" - ] - byronCliArgs = [ "--byron-delegation-certificate", nodePoolKeysDir "byron-delegation.cert" - , "--byron-signing-key", nodePoolKeysDir "byron-delegate.key" - ] - keys@SpoNodeKeys{poolNodeKeysVrf} = mkTestnetNodeKeyPaths i - pure (Just keys, kesSourceCliArg <> shelleyCliArgs <> byronCliArgs) + -- depending on testnet configuration, either start a 'kes-agent' or use a key from disk + kesSourceCliArg <- + case cardanoKESSource of + UseKesKeyFile -> pure ["--shelley-kes-key", nodePoolKeysDir "kes.skey"] + UseKesSocket -> do + -- wait startTimeOffsetSeconds so that the startTime from shelly-genesis.json is not in the future, + -- as otherwise we will trigger an underflow in kes-agent with a negative time difference. + liftIOAnnotated $ threadDelay (startTimeOffsetSeconds * 1_000_000) + kesAgent <- runExceptT $ + initAndStartKesAgent (TmpAbsolutePath tmpAbsPath) nodeName + TestnetKesAgentArgs{ tkaaShelleyGenesisFile = shelleyGenesisFile + , tkaaColdVKeyFile = nodePoolKeysDir "cold.vkey" + , tkaaColdSKeyFile = nodePoolKeysDir "cold.skey" + , tkaaKesVKeyFile = nodePoolKeysDir "kes.vkey" + , tkaaOpcertCounterFile = nodePoolKeysDir "opcert.counter" + , tkaaOpcertFile = nodePoolKeysDir "opcert.cert" + } + case kesAgent of + Left e -> do + -- TODO: fail if could not start KES agent + liftIOAnnotated . putStrLn $ "Could not start KES agent: " <> show e + pure ["--shelley-kes-key", nodePoolKeysDir "kes.skey"] + Right (TestnetKesAgent{kesAgentServiceSprocket}) -> + pure ["--shelley-kes-agent-socket", sprocketSystemName kesAgentServiceSprocket] + let shelleyCliArgs = [ "--shelley-vrf-key", unFile $ signingKey poolNodeKeysVrf + , "--shelley-operational-certificate", nodePoolKeysDir "opcert.cert" + ] + byronCliArgs = [ "--byron-delegation-certificate", nodePoolKeysDir "byron-delegation.cert" + , "--byron-signing-key", nodePoolKeysDir "byron-delegate.key" + ] + keys@SpoNodeKeys{poolNodeKeysVrf} = mkTestnetNodeKeyPaths i + pure (Just keys, kesSourceCliArg <> shelleyCliArgs <> byronCliArgs) eRuntime <- runExceptT . retryOnAddressInUseError $ - startNode (TmpAbsolutePath tmpAbsPath) nodeName testnetDefaultIpv4Address port testnetMagic $ + startNode (TmpAbsolutePath tmpAbsPath) nodeName testnetDefaultIpv4Address port testnetMagic (nodeBin nodeWithOptions) $ [ "run" , "--config", nodeConfigFile , "--topology", nodeDataDir "topology.json" , "--database-path", nodeDataDir "db" ] <> spoNodeCliArgs - <> nodeExtraCliArgs nodeOptions + <> nodeExtraCliArgs nodeWithOptions <> ["--grpc-enable" | RpcEnabled <- [cardanoEnableRpc]] pure $ eRuntime <&> \rt -> rt{poolKeys=mKeys} @@ -501,8 +514,8 @@ retryOnAddressInUseError act = withFrozenCallStack $ go maximumTimeout retryTime -- and checks @pools-keys/@ to classify each as SPO or relay. -- Validates that nodes are consecutively numbered starting from 1, -- and that all SPO nodes come before relay nodes. -readNodeOptionsFromEnv :: HasCallStack => MonadIO m => FilePath -> m TestnetNodeOptions -readNodeOptionsFromEnv envDir = do +readNodesWithOptionsFromEnv :: HasCallStack => MonadIO m => FilePath -> m TestnetNodesWithOptions +readNodesWithOptionsFromEnv envDir = do entries <- liftIO $ IO.listDirectory (envDir "node-data") let nodeNums = sort $ mapMaybe parseNodeNum entries when (null nodeNums) $ @@ -517,12 +530,48 @@ readNodeOptionsFromEnv envDir = do when (null spoFlags) $ throwString "No SPO node directories found in environment" let nSpos = length spoFlags - let spoOpts = map (const (NodeOptions [])) [1 .. nSpos] - relayOpts = map (const (NodeOptions [])) [nSpos + 1 .. length nodeNums] + spoOpts <- mapM readNodeOpt [1 .. nSpos] + relayOpts <- mapM readNodeOpt [nSpos + 1 .. length nodeNums] case spoOpts of - (s:ss) -> pure $ TestnetNodeOptions { optSpoNodes = s :| ss, optRelayNodes = relayOpts } + (s:ss) -> pure $ TestnetNodesWithOptions { optSpoNodes = s :| ss, optRelayNodes = relayOpts } [] -> throwString "No SPO node directories found in environment" where parseNodeNum s = do rest <- stripPrefix "node" s readMaybe rest :: Maybe Int + readNodeOpt i = do + bin <- readNodeBinFromEnvFile (envDir defaultNodeEnvFile i) + pure $ NodeWithOptions bin [] + +data NodeEnv = NodeEnv + { nodeBinary :: FilePath + , nodeVersion :: String + } deriving (Eq, Show) + +instance FromJSON NodeEnv where + parseJSON = withObject "NodeEnv" $ \o -> + NodeEnv <$> o .: "node_binary" + <*> o .: "node_version" + +instance ToJSON NodeEnv where + toJSON NodeEnv{nodeBinary, nodeVersion} = + object [ "node_binary" .= nodeBinary + , "node_version" .= nodeVersion + ] + +readNodeBinFromEnvFile :: (HasCallStack, MonadIO m) => FilePath -> m (Maybe FilePath) +readNodeBinFromEnvFile envFile = runMaybeT $ do + guard =<< liftIOAnnotated (IO.doesFileExist envFile) + NodeEnv{nodeBinary} <- either failParse pure =<< liftIOAnnotated (Yaml.decodeFileEither envFile) + pure nodeBinary + where + failParse err = throwString $ "Failed to parse node env file " <> envFile <> ": " <> show err + +getNodeVersion :: HasCallStack => MonadIO m => FilePath -> m String +getNodeVersion bin = liftIOAnnotated $ do + output <- Process.readProcess bin ["--version"] "" + `catch` \(e :: IOException) -> + throwString $ "Failed to run " <> bin <> " --version: " <> displayException e + case words output of + ("cardano-node":version:_) -> pure version + _ -> throwString $ "Unexpected output from " <> bin <> " --version (expected 'cardano-node ...'): " <> output diff --git a/cardano-testnet/src/Testnet/Start/Types.hs b/cardano-testnet/src/Testnet/Start/Types.hs index c221f451480..c53ce344d9c 100644 --- a/cardano-testnet/src/Testnet/Start/Types.hs +++ b/cardano-testnet/src/Testnet/Start/Types.hs @@ -31,9 +31,9 @@ module Testnet.Start.Types , UpdateTimestamps(..) , TestnetOnChainParams(..) , mainnetParamsRequest - , TestnetNodeOptions(..) - , NodeOptions(..) - , cardanoDefaultTestnetNodeOptions + , TestnetNodesWithOptions(..) + , NodeWithOptions(..) + , cardanoDefaultTestnetNodesWithOptions , GenesisOptions(..) , UserProvidedData(..) , UserProvidedGeneses(..) @@ -176,7 +176,7 @@ data RpcSupport -- 'Testnet.Start.Cardano.createAndRunTestnet' in tests. data TestnetCreationOptions = TestnetCreationOptions { -- | Options controlling how many nodes to create and of which type. - creationNodes :: TestnetNodeOptions + creationNodes :: TestnetNodesWithOptions , creationEra :: AnyShelleyBasedEra -- ^ The era to start at , creationMaxSupply :: Word64 -- ^ The amount of Lovelace you are starting your testnet with (forwarded to shelley genesis) -- TODO move me to GenesisOptions when https://github.com/IntersectMBO/cardano-cli/pull/874 makes it to cardano-node @@ -187,7 +187,7 @@ data TestnetCreationOptions = TestnetCreationOptions instance Default TestnetCreationOptions where def = TestnetCreationOptions - { creationNodes = cardanoDefaultTestnetNodeOptions + { creationNodes = cardanoDefaultTestnetNodesWithOptions , creationEra = AnyShelleyBasedEra ShelleyBasedEraConway , creationMaxSupply = 100_000_020_000_000 , creationNumDReps = 3 @@ -259,15 +259,16 @@ instance Default GenesisOptions where } -- | Configuration specific to each node -newtype NodeOptions = NodeOptions - { nodeExtraCliArgs :: [String] -- ^ Extra CLI arguments passed to @cardano-node run@ +data NodeWithOptions = NodeWithOptions + { nodeBin :: Maybe FilePath -- ^ Path to the @cardano-node@ binary to use for running this node. 'Nothing' uses the default resolution mechanism. + , nodeExtraCliArgs :: [String] -- ^ Extra CLI arguments passed to @cardano-node run@ } deriving (Eq, Show) -- | Specifies the nodes to create for the testnet, split by role (SPO and relay). -- SPO nodes participate in block production. Relay nodes only forward blocks. -data TestnetNodeOptions = TestnetNodeOptions - { optSpoNodes :: NonEmpty NodeOptions -- ^ SPO (stake pool operator) nodes. Must have at least one. - , optRelayNodes :: [NodeOptions] -- ^ Relay (non-producing) nodes +data TestnetNodesWithOptions = TestnetNodesWithOptions + { optSpoNodes :: NonEmpty NodeWithOptions -- ^ SPO (stake pool operator) nodes. Must have at least one. + , optRelayNodes :: [NodeWithOptions] -- ^ Relay (non-producing) nodes } deriving (Eq, Show) -- | Type used to track whether the user is providing its data (node configuration file path, genesis file, etc.) @@ -280,11 +281,11 @@ data UserProvidedData a = instance Default (UserProvidedData a) where def = NoUserProvidedData -cardanoDefaultTestnetNodeOptions :: TestnetNodeOptions -cardanoDefaultTestnetNodeOptions = TestnetNodeOptions - { optSpoNodes = NodeOptions [] :| [] - , optRelayNodes = [ NodeOptions [] - , NodeOptions [] +cardanoDefaultTestnetNodesWithOptions :: TestnetNodesWithOptions +cardanoDefaultTestnetNodesWithOptions = TestnetNodesWithOptions + { optSpoNodes = NodeWithOptions Nothing [] :| [] + , optRelayNodes = [ NodeWithOptions Nothing [] + , NodeWithOptions Nothing [] ] } diff --git a/cardano-testnet/test/cardano-testnet-golden/files/golden/help.cli b/cardano-testnet/test/cardano-testnet-golden/files/golden/help.cli index 3338b742378..727d84e5bb3 100644 --- a/cardano-testnet/test/cardano-testnet-golden/files/golden/help.cli +++ b/cardano-testnet/test/cardano-testnet-golden/files/golden/help.cli @@ -2,7 +2,7 @@ Usage: cardano-testnet (cardano | create-env | version | help) Usage: cardano-testnet cardano [ --node-env FILEPATH [--preserve-timestamps] - | [--num-pool-nodes COUNT] + | [--nodes SPEC[,SPEC...] | --num-pool-nodes COUNT] [--max-lovelace-supply WORD64] [--num-dreps NUMBER] [--testnet-magic INT] @@ -18,7 +18,8 @@ Usage: cardano-testnet cardano Start a testnet and keep it running until stopped -Usage: cardano-testnet create-env [--num-pool-nodes COUNT] +Usage: cardano-testnet create-env + [--nodes SPEC[,SPEC...] | --num-pool-nodes COUNT] [--max-lovelace-supply WORD64] [--num-dreps NUMBER] [--testnet-magic INT] diff --git a/cardano-testnet/test/cardano-testnet-golden/files/golden/help/cardano.cli b/cardano-testnet/test/cardano-testnet-golden/files/golden/help/cardano.cli index af2f574cdec..cad989c099a 100644 --- a/cardano-testnet/test/cardano-testnet-golden/files/golden/help/cardano.cli +++ b/cardano-testnet/test/cardano-testnet-golden/files/golden/help/cardano.cli @@ -1,6 +1,6 @@ Usage: cardano-testnet cardano [ --node-env FILEPATH [--preserve-timestamps] - | [--num-pool-nodes COUNT] + | [--nodes SPEC[,SPEC...] | --num-pool-nodes COUNT] [--max-lovelace-supply WORD64] [--num-dreps NUMBER] [--testnet-magic INT] @@ -23,6 +23,17 @@ Available options: pass it with this argument. --preserve-timestamps Do not update the time stamps in genesis files to current date. + --nodes SPEC[,SPEC...] Comma-separated node specifications. SPO nodes must + come before relay nodes. Each spec is a role (spo or + relay) optionally followed by :node-bin=. If + the path contains commas, colons, double quotes, or + backslashes, wrap it in double quotes and escape any + literal double quotes as \" and backslashes as \\ + within. To prevent bash from consuming the double + quotes, enclose the whole argument in single quotes. + Examples: --nodes + spo,spo:node-bin=/path/to/bin,relay,relay | --nodes + 'spo:node-bin="/path,with:commas",relay' --num-pool-nodes COUNT Number of pool nodes. Note this uses a default node configuration for all nodes. --max-lovelace-supply WORD64 diff --git a/cardano-testnet/test/cardano-testnet-golden/files/golden/help/create-env.cli b/cardano-testnet/test/cardano-testnet-golden/files/golden/help/create-env.cli index 20d7b4405c6..27776f085e5 100644 --- a/cardano-testnet/test/cardano-testnet-golden/files/golden/help/create-env.cli +++ b/cardano-testnet/test/cardano-testnet-golden/files/golden/help/create-env.cli @@ -1,4 +1,5 @@ -Usage: cardano-testnet create-env [--num-pool-nodes COUNT] +Usage: cardano-testnet create-env + [--nodes SPEC[,SPEC...] | --num-pool-nodes COUNT] [--max-lovelace-supply WORD64] [--num-dreps NUMBER] [--testnet-magic INT] @@ -11,6 +12,17 @@ Usage: cardano-testnet create-env [--num-pool-nodes COUNT] Create a sandbox for Cardano testnet Available options: + --nodes SPEC[,SPEC...] Comma-separated node specifications. SPO nodes must + come before relay nodes. Each spec is a role (spo or + relay) optionally followed by :node-bin=. If + the path contains commas, colons, double quotes, or + backslashes, wrap it in double quotes and escape any + literal double quotes as \" and backslashes as \\ + within. To prevent bash from consuming the double + quotes, enclose the whole argument in single quotes. + Examples: --nodes + spo,spo:node-bin=/path/to/bin,relay,relay | --nodes + 'spo:node-bin="/path,with:commas",relay' --num-pool-nodes COUNT Number of pool nodes. Note this uses a default node configuration for all nodes. --max-lovelace-supply WORD64 diff --git a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Cli/KesPeriodInfo.hs b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Cli/KesPeriodInfo.hs index bb8f062593b..fb3db527b5d 100644 --- a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Cli/KesPeriodInfo.hs +++ b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Cli/KesPeriodInfo.hs @@ -262,7 +262,7 @@ hprop_kes_period_info = integrationRetryWorkspace 2 "kes-period-info" $ \tempAbs H.lbsWriteFile (unFile configurationFile) jsonBS newNodePortNumber <- H.randomPort testnetDefaultIpv4Address eRuntime <- runExceptT . retryOnAddressInUseError $ - startNode tempAbsPath "test-spo" testnetDefaultIpv4Address newNodePortNumber testnetMagic + startNode tempAbsPath "test-spo" testnetDefaultIpv4Address newNodePortNumber testnetMagic Nothing [ "run" , "--config", unFile configurationFile , "--topology", topologyFile diff --git a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Cli/LeadershipSchedule.hs b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Cli/LeadershipSchedule.hs index 91e98ad35ea..176c464ca71 100644 --- a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Cli/LeadershipSchedule.hs +++ b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Cli/LeadershipSchedule.hs @@ -71,10 +71,10 @@ hprop_leadershipSchedule = integrationRetryWorkspace 2 "leadership-schedule" $ \ cTestnetOptions = def { creationEra = asbe , creationNodes = - TestnetNodeOptions - { optSpoNodes = NodeOptions [] :| - [ NodeOptions [] - , NodeOptions [] + TestnetNodesWithOptions + { optSpoNodes = NodeWithOptions Nothing [] :| + [ NodeWithOptions Nothing [] + , NodeWithOptions Nothing [] ] , optRelayNodes = [] } @@ -267,7 +267,7 @@ hprop_leadershipSchedule = integrationRetryWorkspace 2 "leadership-schedule" $ \ H.lbsWriteFile (unFile configurationFile) jsonBS newNodePort <- H.randomPort testnetDefaultIpv4Address eRuntime <- runExceptT . retryOnAddressInUseError $ - startNode (TmpAbsolutePath work) "test-spo" testnetDefaultIpv4Address newNodePort testnetMagic + startNode (TmpAbsolutePath work) "test-spo" testnetDefaultIpv4Address newNodePort testnetMagic Nothing [ "run" , "--config", unFile configurationFile , "--topology", topologyFile diff --git a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Gov/ProposeNewConstitutionSPO.hs b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Gov/ProposeNewConstitutionSPO.hs index 5e810cb287a..7add69da333 100644 --- a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Gov/ProposeNewConstitutionSPO.hs +++ b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Gov/ProposeNewConstitutionSPO.hs @@ -59,10 +59,10 @@ hprop_ledger_events_propose_new_constitution_spo = integrationRetryWorkspace 2 " creationOptions = def { creationEra = AnyShelleyBasedEra sbe , creationNodes = - TestnetNodeOptions - { optSpoNodes = NodeOptions [] :| - [ NodeOptions [] - , NodeOptions [] + TestnetNodesWithOptions + { optSpoNodes = NodeWithOptions Nothing [] :| + [ NodeWithOptions Nothing [] + , NodeWithOptions Nothing [] ] , optRelayNodes = [] } diff --git a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Node/Shutdown.hs b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Node/Shutdown.hs index 00408cfcdeb..4e19f302e6e 100644 --- a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Node/Shutdown.hs +++ b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Node/Shutdown.hs @@ -208,8 +208,8 @@ hprop_shutdownOnSlotSynced = integrationRetryWorkspace 2 "shutdown-on-slot-synce slotLen = 0.1 let creationOptions = def { creationNodes = - TestnetNodeOptions - { optSpoNodes = NodeOptions ["--shutdown-on-slot-synced", show maxSlot] :| [] + TestnetNodesWithOptions + { optSpoNodes = NodeWithOptions Nothing ["--shutdown-on-slot-synced", show maxSlot] :| [] , optRelayNodes = [] } , creationGenesisOptions = def