From 52d6a4f60bbc7dcfc48946b07986962b65fe9f3b Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Tue, 27 Jun 2023 13:42:51 +0200 Subject: [PATCH 01/47] frame::system: code removed --- frame/system/src/lib.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/frame/system/src/lib.rs b/frame/system/src/lib.rs index 9fcd7c12fec4e..5dd0db9bf84aa 100644 --- a/frame/system/src/lib.rs +++ b/frame/system/src/lib.rs @@ -673,8 +673,6 @@ pub mod pallet { #[derive(frame_support::DefaultNoBound)] #[pallet::genesis_config] pub struct GenesisConfig { - #[serde(with = "sp_core::bytes")] - pub code: Vec, #[serde(skip)] pub _config: sp_std::marker::PhantomData, } @@ -688,7 +686,6 @@ pub mod pallet { >::put(true); >::put(true); - sp_io::storage::set(well_known_keys::CODE, &self.code); sp_io::storage::set(well_known_keys::EXTRINSIC_INDEX, &0u32.encode()); } } From e333e1601c46c31f5b75d28c56b27f71f977a31e Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Sat, 1 Jul 2023 13:27:12 +0200 Subject: [PATCH 02/47] chain-spec: mod genesis -> genesis_block --- client/chain-spec/src/{genesis.rs => genesis_block.rs} | 0 client/chain-spec/src/genesis_config_builder.rs | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename client/chain-spec/src/{genesis.rs => genesis_block.rs} (100%) create mode 100644 client/chain-spec/src/genesis_config_builder.rs diff --git a/client/chain-spec/src/genesis.rs b/client/chain-spec/src/genesis_block.rs similarity index 100% rename from client/chain-spec/src/genesis.rs rename to client/chain-spec/src/genesis_block.rs diff --git a/client/chain-spec/src/genesis_config_builder.rs b/client/chain-spec/src/genesis_config_builder.rs new file mode 100644 index 0000000000000..e69de29bb2d1d From ea2e2529b8a4e66d4724627606b755b450814328 Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Thu, 6 Jul 2023 14:25:20 +0200 Subject: [PATCH 03/47] chain-spec: genesis_config_builder (wasm based) --- .../chain-spec/src/genesis_config_builder.rs | 86 +++++++++++++++++++ client/chain-spec/src/lib.rs | 5 +- 2 files changed, 89 insertions(+), 2 deletions(-) diff --git a/client/chain-spec/src/genesis_config_builder.rs b/client/chain-spec/src/genesis_config_builder.rs index e69de29bb2d1d..5138e32aa8c2c 100644 --- a/client/chain-spec/src/genesis_config_builder.rs +++ b/client/chain-spec/src/genesis_config_builder.rs @@ -0,0 +1,86 @@ +use codec::{Decode, Encode}; +use sc_executor::{error::Result, WasmExecutor}; +use sc_executor_common::runtime_blob::RuntimeBlob; +use serde_json::{from_slice, Value}; +use sp_core::{storage::Storage, traits::Externalities}; +use sp_genesis_builder::Result as BuildResult; +use sp_state_machine::BasicExternalities; + +// todo: +// - tests: focus on errors +// - reuse executor +// - pub api + internal function + +fn executor_call( + ext: &mut dyn Externalities, + code: &[u8], + method: &str, + data: &[u8], +) -> Result> { + WasmExecutor::::builder().build().uncached_call( + RuntimeBlob::uncompress_if_needed(code.into()).unwrap(), + ext, + true, + method, + data, + ) +} + +pub fn get_storage_for_config(code: &[u8], config: Value) -> core::result::Result { + log::info!("get_storage_for_config: {:?}", config); + let mut ext = BasicExternalities::new_empty(); + + let call_result = + executor_call(&mut ext, code, "GenesisBuilder_build_config", &config.to_string().encode()) + .map_err(|e| format!("wasm call error {e}"))?; + + log::info!("get_storage_for_config: call_restul {:?}", call_result); + log::info!("get_storage_for_config: ext {:?}", ext); + let build_result = BuildResult::decode(&mut &call_result[..]) + .map_err(|e| format!("scale codec error: {e}"))?; + log::info!("get_storage_for_config: br {:?}", build_result); + + Ok(build_result.map(|_| ext.into_storages())?) +} + +pub fn get_storage_for_patch(code: &[u8], patch: Value) -> core::result::Result { + log::info!("get_storage_for_patch: {:?}", patch); + let mut t = BasicExternalities::new_empty(); + let call_result = executor_call(&mut t, code, "GenesisBuilder_create_default_config", &vec![]) + .map_err(|e| format!("wasm call error {e}"))?; + + let default_config = + Vec::::decode(&mut &call_result[..]).map_err(|e| format!("scale codec error: {e}"))?; + + let mut config: Value = from_slice(&default_config[..]).expect("returned value is json. qed."); + + json_patch::merge(&mut config, &patch); + get_storage_for_config(code, config) +} + +// pub fn get_default_config(code: &[u8]) -> core::result::Result { +// let mut t = BasicExternalities::new_empty(); +// let call_result = executor_call(&mut t, code, "GenesisBuilder_create_default_config", &vec![]) +// .map_err(|e| format!("wasm call error {e}"))?; + +// let default_config = +// Vec::::decode(&mut &call_result[..]).map_err(|e| format!("scale codec error: {e}"))?; + +// let mut config: Value = from_slice(&default_config[..]).expect("returned value is json. qed."); + +// json_patch::merge(&mut config, &patch); +// get_storage_for_config(code, config) +// } + +// #[test] +// fn build_config_from_json_works() { +// sp_tracing::try_init_simple(); +// let j = include_str!("../../../test-utils/runtime/src/test_json/default_genesis_config.json"); + +// let mut t = BasicExternalities::new_empty(); +// let r = executor_call(&mut t, "GenesisBuilder_build_config", &j.encode()).unwrap(); +// let r = BuildResult::decode(&mut &r[..]); +// assert!(r.is_ok()); + +// // let mut _keys = t.into_storages().top.keys().cloned().map(hex).collect::>(); +// } diff --git a/client/chain-spec/src/lib.rs b/client/chain-spec/src/lib.rs index 6239eb7326b78..c33c9e960c1bb 100644 --- a/client/chain-spec/src/lib.rs +++ b/client/chain-spec/src/lib.rs @@ -177,12 +177,13 @@ mod chain_spec; mod extension; -mod genesis; +mod genesis_block; +mod genesis_config_builder; pub use self::{ chain_spec::{ChainSpec as GenericChainSpec, NoExtension}, extension::{get_extension, get_extension_mut, Extension, Fork, Forks, GetExtension, Group}, - genesis::{ + genesis_block::{ construct_genesis_block, resolve_state_version_from_wasm, BuildGenesisBlock, GenesisBlockBuilder, }, From b4f526723f0191ab974415fc3f4d086a12bc80ad Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Fri, 7 Jul 2023 14:20:20 +0200 Subject: [PATCH 04/47] chain-spec: support for code and config patching --- client/chain-spec/Cargo.toml | 14 + .../substrate_test_runtime_from_config.json | 126 ++++++ ...bstrate_test_runtime_from_config__raw.json | 53 +++ .../substrate_test_runtime_from_patch.json | 30 ++ ...ubstrate_test_runtime_from_patch__raw.json | 32 ++ client/chain-spec/src/chain_spec.rs | 420 ++++++++++++++++-- client/chain-spec/src/lib.rs | 2 +- 7 files changed, 644 insertions(+), 33 deletions(-) create mode 100644 client/chain-spec/res/substrate_test_runtime_from_config.json create mode 100644 client/chain-spec/res/substrate_test_runtime_from_config__raw.json create mode 100644 client/chain-spec/res/substrate_test_runtime_from_patch.json create mode 100644 client/chain-spec/res/substrate_test_runtime_from_patch__raw.json diff --git a/client/chain-spec/Cargo.toml b/client/chain-spec/Cargo.toml index 1b4ce6e44605a..ea9962889b9a4 100644 --- a/client/chain-spec/Cargo.toml +++ b/client/chain-spec/Cargo.toml @@ -13,15 +13,29 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] +codec = { package = "parity-scale-codec", version = "3.6.1", default-features = false, features = ["derive"] } +json-patch = { version = "1.0.0", default-features = false } memmap2 = "0.5.0" serde = { version = "1.0.163", features = ["derive"] } serde_json = "1.0.85" sc-client-api = { version = "4.0.0-dev", path = "../api" } sc-chain-spec-derive = { version = "4.0.0-dev", path = "./derive" } sc-executor = { version = "0.10.0-dev", path = "../executor" } +sc-executor-common = { version = "0.10.0-dev", path = "../../client/executor/common" } +sp-io = { version = "23.0.0", default-features = false, path = "../../primitives/io" } sc-network = { version = "0.10.0-dev", path = "../network" } sc-telemetry = { version = "4.0.0-dev", path = "../telemetry" } sp-blockchain = { version = "4.0.0-dev", path = "../../primitives/blockchain" } sp-core = { version = "21.0.0", path = "../../primitives/core" } +sp-genesis-builder = { version = "0.1.0-dev", path = "../../primitives/genesis-builder" } sp-runtime = { version = "24.0.0", path = "../../primitives/runtime" } sp-state-machine = { version = "0.28.0", path = "../../primitives/state-machine" } +log = { version = "0.4.17", default-features = false } +array-bytes = { version = "6.1" } #debug + +[dev-dependencies] +sp-tracing = { version = "10.0.0", path = "../../primitives/tracing" } +kitchensink-runtime = { path = "../../bin/node/runtime" } +substrate-test-runtime = { path = "../../test-utils/runtime" } +sp-keyring = { version = "24.0.0", path = "../../primitives/keyring" } +sp-application-crypto = { version = "23.0.0", default-features = false, path = "../../primitives/application-crypto", features = ["serde"] } diff --git a/client/chain-spec/res/substrate_test_runtime_from_config.json b/client/chain-spec/res/substrate_test_runtime_from_config.json new file mode 100644 index 0000000000000..81f950796f20a --- /dev/null +++ b/client/chain-spec/res/substrate_test_runtime_from_config.json @@ -0,0 +1,126 @@ +{ + "name": "TestName", + "id": "test_id", + "chainType": "Local", + "bootNodes": [], + "telemetryEndpoints": null, + "protocolId": null, + "properties": null, + "codeSubstitutes": {}, + "genesis": { + "jsonFull": { + "babe": { + "authorities": [ + [ + "5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY", + 1 + ], + [ + "5FHneW46xGXgs5mUiveU4sbTyGBzmstUspZC92UhjJM694ty", + 1 + ], + [ + "5FLSigC9HGRKVhB9FiEo4Y3koPsNmBmLJbpXg2mp1hXcS59Y", + 1 + ] + ], + "epochConfig": { + "allowed_slots": "PrimaryAndSecondaryPlainSlots", + "c": [ + 3, + 10 + ] + } + }, + "balances": { + "balances": [ + [ + "5D34dL5prEUaGNQtPPZ3yN5Y6BnkfXunKXXz6fo7ZJbLwRRH", + 100000000000000000 + ], + [ + "5GBNeWRhZc2jXu7D55rBimKYDk8PGk8itRYFTPfC8RJLKG5o", + 100000000000000000 + ], + [ + "5Dfis6XL8J2P6JHUnUtArnFWndn62SydeP8ee8sG2ky9nfm9", + 100000000000000000 + ], + [ + "5F4H97f7nQovyrbiq4ZetaaviNwThSVcFobcA5aGab6167dK", + 100000000000000000 + ], + [ + "5DiDShBWa1fQx6gLzpf3SFBhMinCoyvHM1BWjPNsmXS8hkrW", + 100000000000000000 + ], + [ + "5EFb84yH9tpcFuiKUcsmdoF7xeeY3ajG1ZLQimxQoFt9HMKR", + 100000000000000000 + ], + [ + "5DZLHESsfGrJ5YzT3HuRPXsSNb589xQ4Unubh1mYLodzKdVY", + 100000000000000000 + ], + [ + "5GHJzqvG6tXnngCpG7B12qjUvbo5e4e9z8Xjidk3CQZHxTPZ", + 100000000000000000 + ], + [ + "5CUnSsgAyLND3bxxnfNhgWXSe9Wn676JzLpGLgyJv858qhoX", + 100000000000000000 + ], + [ + "5CVKn7HAZW1Ky4r7Vkgsr7VEW88C2sHgUNDiwHY9Ct2hjU8q", + 100000000000000000 + ], + [ + "5H673aukQ4PeDe1U2nuv1bi32xDEziimh3PZz7hDdYUB7TNz", + 100000000000000000 + ], + [ + "5HTe9L15LJryjUAt1jZXZCBPnzbbGnpvFwbjE3NwCWaAqovf", + 100000000000000000 + ], + [ + "5D7LFzGpMwHPyDBavkRbWSKWTtJhCaPPZ379wWLT23bJwXJz", + 100000000000000000 + ], + [ + "5CLepMARnEgtVR1EkUuJVUvKh97gzergpSxUU3yKGx1v6EwC", + 100000000000000000 + ], + [ + "5Chb2UhfvZpmjjEziHbFbotM4quX32ZscRV6QJBt1rUKzz51", + 100000000000000000 + ], + [ + "5HmRp3i3ZZk7xsAvbi8hyXVP6whSMnBJGebVC4FsiZVhx52e", + 100000000000000000 + ], + [ + "5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY", + 100000000000000000 + ], + [ + "5FHneW46xGXgs5mUiveU4sbTyGBzmstUspZC92UhjJM694ty", + 100000000000000000 + ], + [ + "5FLSigC9HGRKVhB9FiEo4Y3koPsNmBmLJbpXg2mp1hXcS59Y", + 100000000000000000 + ] + ] + }, + "substrateTest": { + "authorities": [ + "5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY", + "5FHneW46xGXgs5mUiveU4sbTyGBzmstUspZC92UhjJM694ty", + "5FLSigC9HGRKVhB9FiEo4Y3koPsNmBmLJbpXg2mp1hXcS59Y" + ] + }, + "system": {} + } + }, + "code": "0x" +} diff --git a/client/chain-spec/res/substrate_test_runtime_from_config__raw.json b/client/chain-spec/res/substrate_test_runtime_from_config__raw.json new file mode 100644 index 0000000000000..c667b22371d58 --- /dev/null +++ b/client/chain-spec/res/substrate_test_runtime_from_config__raw.json @@ -0,0 +1,53 @@ +{ + "name": "TestName", + "id": "test_id", + "chainType": "Local", + "bootNodes": [], + "telemetryEndpoints": null, + "protocolId": null, + "properties": null, + "codeSubstitutes": {}, + "genesis": { + "raw": { + "top": { + "0x00771836bebdd29870ff246d305c578c4e7b9012096b41c4eb3aaf947f6ea429": "0x0000", + "0x00771836bebdd29870ff246d305c578c5e0621c4869aa60c02be9adcc98a0d1d": "0x0cd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d8eaf04151687736326c9fea17e25fc5287613693c912909cb226aa4794f26a4890b5ab205c6974c9ea841be688864633dc9ca8a357843eeacf2314649965fe22", + "0x1cb6f36e027abb2091cfb5110ab5087f4e7b9012096b41c4eb3aaf947f6ea429": "0x0000", + "0x1cb6f36e027abb2091cfb5110ab5087f5e0621c4869aa60c02be9adcc98a0d1d": "0x0cd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d01000000000000008eaf04151687736326c9fea17e25fc5287613693c912909cb226aa4794f26a48010000000000000090b5ab205c6974c9ea841be688864633dc9ca8a357843eeacf2314649965fe220100000000000000", + "0x1cb6f36e027abb2091cfb5110ab5087f66e8f035c8adbe7f1547b43c51e6f8a4": "0x00000000", + "0x1cb6f36e027abb2091cfb5110ab5087faacf00b9b41fda7a9268821c2a2b3e4c": "0x0cd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d01000000000000008eaf04151687736326c9fea17e25fc5287613693c912909cb226aa4794f26a48010000000000000090b5ab205c6974c9ea841be688864633dc9ca8a357843eeacf2314649965fe220100000000000000", + "0x1cb6f36e027abb2091cfb5110ab5087fdc6b171b77304263c292cc3ea5ed31ef": "0x03000000000000000a0000000000000001", + "0x26aa394eea5630e07c48ae0c9558cef74e7b9012096b41c4eb3aaf947f6ea429": "0x0000", + "0x26aa394eea5630e07c48ae0c9558cef75684a022a34dd8bfa2baaf44f172b710": "0x01", + "0x26aa394eea5630e07c48ae0c9558cef78a42f33323cb5ced3b44dd825fda9fcc": "0x4545454545454545454545454545454545454545454545454545454545454545", + "0x26aa394eea5630e07c48ae0c9558cef7a44704b568d21667356a5a050c118746bb1bdbcacd6ac9340000000000000000": "0x4545454545454545454545454545454545454545454545454545454545454545", + "0x26aa394eea5630e07c48ae0c9558cef7a7fd6c28836b9a28522dc924110cf439": "0x01", + "0x26aa394eea5630e07c48ae0c9558cef7b99d880ec681799c0cf30e8886371da901cae4e3edfbb32c91ed3f01ab964f4eeeab50338d8e5176d3141802d7b010a55dadcd5f23cf8aaafa724627e967e90e": "0x000000000000000000000000010000000000000000008a5d784563010000000000000000000000000000000000000000000000000000000000000080", + "0x26aa394eea5630e07c48ae0c9558cef7b99d880ec681799c0cf30e8886371da91b614bd4a126f2d5d294e9a8af9da25248d7e931307afb4b68d8d565d4c66e00d856c6d65f5fed6bb82dcfb60e936c67": "0x000000000000000000000000010000000000000000008a5d784563010000000000000000000000000000000000000000000000000000000000000080", + "0x26aa394eea5630e07c48ae0c9558cef7b99d880ec681799c0cf30e8886371da94b21aff9fe1e8b2fc4b0775b8cbeff28ba8e2c7594dd74730f3ca835e95455d199261897edc9735d602ea29615e2b10b": "0x000000000000000000000000010000000000000000008a5d784563010000000000000000000000000000000000000000000000000000000000000080", + "0x26aa394eea5630e07c48ae0c9558cef7b99d880ec681799c0cf30e8886371da94f9aea1afa791265fae359272badc1cf8eaf04151687736326c9fea17e25fc5287613693c912909cb226aa4794f26a48": "0x000000000000000000000000010000000000000000008a5d784563010000000000000000000000000000000000000000000000000000000000000080", + "0x26aa394eea5630e07c48ae0c9558cef7b99d880ec681799c0cf30e8886371da95786a2916fcb81e1bd5dcd81e0d2452884617f575372edb5a36d85c04cdf2e4699f96fe33eb5f94a28c041b88e398d0c": "0x000000000000000000000000010000000000000000008a5d784563010000000000000000000000000000000000000000000000000000000000000080", + "0x26aa394eea5630e07c48ae0c9558cef7b99d880ec681799c0cf30e8886371da95b8542d9672c7b7e779cc7c1e6b605691c2115d06120ea2bee32dd601d02f36367564e7ddf84ae2717ca3f097459652e": "0x000000000000000000000000010000000000000000008a5d784563010000000000000000000000000000000000000000000000000000000000000080", + "0x26aa394eea5630e07c48ae0c9558cef7b99d880ec681799c0cf30e8886371da996c30bdbfab640838e6b6d3c33ab4adb4211b79e34ee8072eab506edd4b93a7b85a14c9a05e5cdd056d98e7dbca87730": "0x000000000000000000000000010000000000000000008a5d784563010000000000000000000000000000000000000000000000000000000000000080", + "0x26aa394eea5630e07c48ae0c9558cef7b99d880ec681799c0cf30e8886371da99dc65b1339ec388fbf2ca0cdef51253512c6cfd663203ea16968594f24690338befd906856c4d2f4ef32dad578dba20c": "0x000000000000000000000000010000000000000000008a5d784563010000000000000000000000000000000000000000000000000000000000000080", + "0x26aa394eea5630e07c48ae0c9558cef7b99d880ec681799c0cf30e8886371da99e6eb5abd62f5fd54793da91a47e6af6125d57171ff9241f07acaa1bb6a6103517965cf2cd00e643b27e7599ebccba70": "0x000000000000000000000000010000000000000000008a5d784563010000000000000000000000000000000000000000000000000000000000000080", + "0x26aa394eea5630e07c48ae0c9558cef7b99d880ec681799c0cf30e8886371da9b0edae20838083f2cde1c4080db8cf8090b5ab205c6974c9ea841be688864633dc9ca8a357843eeacf2314649965fe22": "0x000000000000000000000000010000000000000000008a5d784563010000000000000000000000000000000000000000000000000000000000000080", + "0x26aa394eea5630e07c48ae0c9558cef7b99d880ec681799c0cf30e8886371da9d0052993b6f3bd0544fd1f5e4125b9fbde3e789ecd53431fe5c06c12b72137153496dace35c695b5f4d7b41f7ed5763b": "0x000000000000000000000000010000000000000000008a5d784563010000000000000000000000000000000000000000000000000000000000000080", + "0x26aa394eea5630e07c48ae0c9558cef7b99d880ec681799c0cf30e8886371da9d6b7e9a5f12bc571053265dade10d3b4b606fc73f57f03cdb4c932d475ab426043e429cecc2ffff0d2672b0df8398c48": "0x000000000000000000000000010000000000000000008a5d784563010000000000000000000000000000000000000000000000000000000000000080", + "0x26aa394eea5630e07c48ae0c9558cef7b99d880ec681799c0cf30e8886371da9de1e86a9a8c739864cf3cc5ec2bea59fd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d": "0x000000000000000000000000010000000000000000008a5d784563010000000000000000000000000000000000000000000000000000000000000080", + "0x26aa394eea5630e07c48ae0c9558cef7b99d880ec681799c0cf30e8886371da9e1a35f56ee295d39287cbffcfc60c4b346f136b564e1fad55031404dd84e5cd3fa76bfe7cc7599b39d38fd06663bbc0a": "0x000000000000000000000000010000000000000000008a5d784563010000000000000000000000000000000000000000000000000000000000000080", + "0x26aa394eea5630e07c48ae0c9558cef7b99d880ec681799c0cf30e8886371da9e2c1dc507e2035edbbd8776c440d870460c57f0008067cc01c5ff9eb2e2f9b3a94299a915a91198bd1021a6c55596f57": "0x000000000000000000000000010000000000000000008a5d784563010000000000000000000000000000000000000000000000000000000000000080", + "0x26aa394eea5630e07c48ae0c9558cef7b99d880ec681799c0cf30e8886371da9eca0e653a94f4080f6311b4e7b6934eb2afba9278e30ccf6a6ceb3a8b6e336b70068f045c666f2e7f4f9cc5f47db8972": "0x000000000000000000000000010000000000000000008a5d784563010000000000000000000000000000000000000000000000000000000000000080", + "0x26aa394eea5630e07c48ae0c9558cef7b99d880ec681799c0cf30e8886371da9ee8bf7ef90fc56a8aa3b90b344c599550c29b161e27ff8ba45bf6bad4711f326fc506a8803453a4d7e3158e993495f10": "0x000000000000000000000000010000000000000000008a5d784563010000000000000000000000000000000000000000000000000000000000000080", + "0x26aa394eea5630e07c48ae0c9558cef7b99d880ec681799c0cf30e8886371da9f5d6f1c082fe63eec7a71fcad00f4a892e3d43b7b0d04e776e69e7be35247cecdac65504c579195731eaf64b7940966e": "0x000000000000000000000000010000000000000000008a5d784563010000000000000000000000000000000000000000000000000000000000000080", + "0x26aa394eea5630e07c48ae0c9558cef7b99d880ec681799c0cf30e8886371da9fbf0818841edf110e05228a6379763c4fc3c37459d9bdc61f58a5ebc01e9e2305a19d390c0543dc733861ec3cf1de01f": "0x000000000000000000000000010000000000000000008a5d784563010000000000000000000000000000000000000000000000000000000000000080", + "0x26aa394eea5630e07c48ae0c9558cef7f9cce9c888469bb1a0dceaa129672ef8": "0x0000", + "0x3a65787472696e7369635f696e646578": "0x00000000", + "0xc2261276cc9d1f8598ea4b6a74b15c2f4e7b9012096b41c4eb3aaf947f6ea429": "0x0100", + "0xc2261276cc9d1f8598ea4b6a74b15c2f57c875e4cff74148e4628f264b974c80": "0x00003ef1ee275e1a" + }, + "childrenDefault": {} + } + }, + "code": "0x" +} diff --git a/client/chain-spec/res/substrate_test_runtime_from_patch.json b/client/chain-spec/res/substrate_test_runtime_from_patch.json new file mode 100644 index 0000000000000..f1ad312f2e6b0 --- /dev/null +++ b/client/chain-spec/res/substrate_test_runtime_from_patch.json @@ -0,0 +1,30 @@ +{ + "name": "TestName", + "id": "test_id", + "chainType": "Local", + "bootNodes": [], + "telemetryEndpoints": null, + "protocolId": null, + "properties": null, + "codeSubstitutes": {}, + "genesis": { + "jsonPatch": { + "babe": { + "epochConfig": { + "allowed_slots": "PrimaryAndSecondaryPlainSlots", + "c": [ + 7, + 10 + ] + } + }, + "substrateTest": { + "authorities": [ + "5CiPPseXPECbkjWCa6MnjNokrgYjMqmKndv2rSnekmSK2DjL", + "5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY" + ] + } + } + }, + "code": "0x" +} diff --git a/client/chain-spec/res/substrate_test_runtime_from_patch__raw.json b/client/chain-spec/res/substrate_test_runtime_from_patch__raw.json new file mode 100644 index 0000000000000..80668f20a36bd --- /dev/null +++ b/client/chain-spec/res/substrate_test_runtime_from_patch__raw.json @@ -0,0 +1,32 @@ +{ + "name": "TestName", + "id": "test_id", + "chainType": "Local", + "bootNodes": [], + "telemetryEndpoints": null, + "protocolId": null, + "properties": null, + "codeSubstitutes": {}, + "genesis": { + "raw": { + "top": { + "0x00771836bebdd29870ff246d305c578c4e7b9012096b41c4eb3aaf947f6ea429": "0x0000", + "0x00771836bebdd29870ff246d305c578c5e0621c4869aa60c02be9adcc98a0d1d": "0x081cbd2d43530a44705ad088af313e18f80b53ef16b36177cd4b77b846f2a5f07cd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d", + "0x1cb6f36e027abb2091cfb5110ab5087f4e7b9012096b41c4eb3aaf947f6ea429": "0x0000", + "0x1cb6f36e027abb2091cfb5110ab5087f66e8f035c8adbe7f1547b43c51e6f8a4": "0x00000000", + "0x1cb6f36e027abb2091cfb5110ab5087fdc6b171b77304263c292cc3ea5ed31ef": "0x07000000000000000a0000000000000001", + "0x26aa394eea5630e07c48ae0c9558cef74e7b9012096b41c4eb3aaf947f6ea429": "0x0000", + "0x26aa394eea5630e07c48ae0c9558cef75684a022a34dd8bfa2baaf44f172b710": "0x01", + "0x26aa394eea5630e07c48ae0c9558cef78a42f33323cb5ced3b44dd825fda9fcc": "0x4545454545454545454545454545454545454545454545454545454545454545", + "0x26aa394eea5630e07c48ae0c9558cef7a44704b568d21667356a5a050c118746bb1bdbcacd6ac9340000000000000000": "0x4545454545454545454545454545454545454545454545454545454545454545", + "0x26aa394eea5630e07c48ae0c9558cef7a7fd6c28836b9a28522dc924110cf439": "0x01", + "0x26aa394eea5630e07c48ae0c9558cef7f9cce9c888469bb1a0dceaa129672ef8": "0x0000", + "0x3a65787472696e7369635f696e646578": "0x00000000", + "0xc2261276cc9d1f8598ea4b6a74b15c2f4e7b9012096b41c4eb3aaf947f6ea429": "0x0100", + "0xc2261276cc9d1f8598ea4b6a74b15c2f57c875e4cff74148e4628f264b974c80": "0x0000000000000000" + }, + "childrenDefault": {} + } + }, + "code": "0x" +} diff --git a/client/chain-spec/src/chain_spec.rs b/client/chain-spec/src/chain_spec.rs index 96e36d8399ed5..c527f4a137625 100644 --- a/client/chain-spec/src/chain_spec.rs +++ b/client/chain-spec/src/chain_spec.rs @@ -29,13 +29,27 @@ use sp_core::{ Bytes, }; use sp_runtime::BuildStorage; -use std::{borrow::Cow, collections::BTreeMap, fs::File, path::PathBuf, sync::Arc}; +use std::{ + borrow::Cow, collections::BTreeMap, fs::File, marker::PhantomData, path::PathBuf, sync::Arc, +}; + +#[derive(Serialize, Deserialize, Clone)] +#[serde(rename_all = "camelCase")] +pub enum GenesisBuildAction { + #[serde(alias = "RuntimePatch")] + Patch(serde_json::Value), + #[serde(alias = "Runtime")] + Full(serde_json::Value), +} enum GenesisSource { File(PathBuf), Binary(Cow<'static, [u8]>), + #[deprecated(note = "todo: add note")] Factory(Arc G + Send + Sync>), Storage(Storage), + //maybe a factory for providing json of GenesisBuildAction?? + GenesisBuilderApi(GenesisBuildAction), } impl Clone for GenesisSource { @@ -45,6 +59,7 @@ impl Clone for GenesisSource { Self::Binary(ref d) => Self::Binary(d.clone()), Self::Factory(ref f) => Self::Factory(f.clone()), Self::Storage(ref s) => Self::Storage(s.clone()), + Self::GenesisBuilderApi(ref s) => Self::GenesisBuilderApi(s.clone()), } } } @@ -72,12 +87,12 @@ impl GenesisSource { let genesis: GenesisContainer = json::from_slice(&bytes) .map_err(|e| format!("Error parsing spec file: {}", e))?; - Ok(genesis.genesis) + Ok(genesis.genesis) //GenesisRuntime (typically) }, Self::Binary(buf) => { let genesis: GenesisContainer = json::from_reader(buf.as_ref()) .map_err(|e| format!("Error parsing embedded file: {}", e))?; - Ok(genesis.genesis) + Ok(genesis.genesis) //GenesisRuntime (typically) }, Self::Factory(f) => Ok(Genesis::Runtime(f())), Self::Storage(storage) => { @@ -104,12 +119,21 @@ impl GenesisSource { Ok(Genesis::Raw(RawGenesis { top, children_default })) }, + // maybe a Factory for getting GenesisBuilderApi command? + Self::GenesisBuilderApi(GenesisBuildAction::Full(config)) => + Ok(Genesis::JsonFull(config.clone())), + Self::GenesisBuilderApi(GenesisBuildAction::Patch(patch)) => + Ok(Genesis::JsonPatch(patch.clone())), } } } impl BuildStorage for ChainSpec { fn assimilate_storage(&self, storage: &mut Storage) -> Result<(), String> { + storage + .top + .insert(sp_core::storage::well_known_keys::CODE.to_vec(), self.code.clone()); + match self.genesis.resolve()? { Genesis::Runtime(gc) => gc.assimilate_storage(storage), Genesis::Raw(RawGenesis { top: map, children_default: children_map }) => { @@ -129,7 +153,15 @@ impl BuildStorage for ChainSpec { // it, but Substrate itself isn't capable of loading chain specs with just a hash at the // moment. Genesis::StateRootHash(_) => Err("Genesis storage in hash format not supported".into()), - } + Genesis::JsonFull(config) => + crate::genesis_config_builder::get_storage_for_config(&self.code[..], config)? + .assimilate_storage(storage), + Genesis::JsonPatch(patch) => + crate::genesis_config_builder::get_storage_for_patch(&self.code[..], patch)? + .assimilate_storage(storage), + }?; + + Ok(()) } } @@ -144,20 +176,50 @@ pub struct RawGenesis { pub children_default: BTreeMap, } +impl From for RawGenesis { + fn from(value: sp_core::storage::Storage) -> Self { + Self { + top: value.top.into_iter().map(|(k, v)| (StorageKey(k), StorageData(v))).collect(), + children_default: value + .children_default + .into_iter() + .map(|(sk, child)| { + ( + StorageKey(sk), + child + .data + .into_iter() + .map(|(k, v)| (StorageKey(k), StorageData(v))) + .collect(), + ) + }) + .collect(), + } + } +} + #[derive(Serialize, Deserialize)] #[serde(rename_all = "camelCase")] #[serde(deny_unknown_fields)] enum Genesis { - Runtime(G), + //deprecated + // #[serde(alias = "RuntimeDeprecated")] + #[deprecated(note = "todo: add note")] + Runtime(G), //G is RuntimeGenesisConfig Raw(RawGenesis), /// State root hash of the genesis storage. StateRootHash(StorageData), + + // CodeWithPatch(Option), (PatchDefault?) + #[serde(alias = "Runtime")] + JsonFull(serde_json::Value), + #[serde(alias = "Patch")] + JsonPatch(serde_json::Value), } /// A configuration of a client. Does not include runtime storage initialization. #[derive(Serialize, Deserialize, Clone, Debug)] #[serde(rename_all = "camelCase")] -#[serde(deny_unknown_fields)] struct ClientSpec { name: String, id: String, @@ -181,6 +243,9 @@ struct ClientSpec { #[serde(skip_serializing)] #[allow(unused)] genesis: serde::de::IgnoredAny, + #[serde(skip)] + #[allow(unused)] + code: serde::de::IgnoredAny, /// Mapping from `block_number` to `wasm_code`. /// /// The given `wasm_code` will be used to substitute the on-chain wasm code starting with the @@ -194,15 +259,143 @@ struct ClientSpec { /// We use `Option` here since `()` is not flattenable by serde. pub type NoExtension = Option<()>; +#[allow(missing_docs)] +pub struct ChainSpecBuilder { + name: Option, + id: Option, + chain_type: Option, + boot_nodes: Option>, + telemetry_endpoints: Option, + protocol_id: Option, + fork_id: Option, + properties: Option, + extensions: Option, + code: Option>, + genesis_build_action: Option, + _genesis: PhantomData, +} + +#[allow(missing_docs)] //todo: add doc! +impl ChainSpecBuilder { + pub fn new() -> Self { + Self { + name: None, + id: None, + chain_type: None, + boot_nodes: None, + telemetry_endpoints: None, + protocol_id: None, + fork_id: None, + properties: None, + extensions: None, + code: None, + genesis_build_action: None, + _genesis: Default::default(), + } + } + + pub fn with_name(mut self, name: &str) -> Self { + self.name = Some(name.into()); + self + } + + pub fn with_id(mut self, id: &str) -> Self { + self.id = Some(id.into()); + self + } + + pub fn with_chain_type(mut self, chain_type: ChainType) -> Self { + self.chain_type = Some(chain_type); + self + } + + pub fn with_boot_nodes(mut self, boot_nodes: Vec) -> Self { + self.boot_nodes = Some(boot_nodes); + self + } + + pub fn with_telemetry_endpoints(mut self, telemetry_endpoints: TelemetryEndpoints) -> Self { + self.telemetry_endpoints = Some(telemetry_endpoints); + self + } + + pub fn with_protocol_id(mut self, protocol_id: &str) -> Self { + self.protocol_id = Some(protocol_id.into()); + self + } + + pub fn with_fork_id(mut self, fork_id: &str) -> Self { + self.fork_id = Some(fork_id.into()); + self + } + + pub fn with_properties(mut self, properties: Properties) -> Self { + self.properties = Some(properties); + self + } + + pub fn with_extensions(mut self, extensions: E) -> Self { + self.extensions = Some(extensions); + self + } + + pub fn with_code(mut self, code: &[u8]) -> Self { + self.code = Some(code.into()); + self + } + + pub fn with_genesis_patch(mut self, patch: serde_json::Value) -> Self { + self.genesis_build_action = Some(GenesisBuildAction::Patch(patch)); + self + } + + pub fn with_no_genesis_defaults(mut self, config: serde_json::Value) -> Self { + self.genesis_build_action = Some(GenesisBuildAction::Full(config)); + self + } + + pub fn build(self) -> ChainSpec { + let client_spec = ClientSpec { + name: self.name.expect("with_name must be called."), + id: self.id.expect("with_id must be called."), + chain_type: self.chain_type.expect("with_chain_type must be called."), + boot_nodes: self.boot_nodes.unwrap_or(Default::default()), + telemetry_endpoints: self.telemetry_endpoints, + protocol_id: self.protocol_id, + fork_id: self.fork_id, + properties: self.properties, + extensions: self.extensions.expect("with_extensions must be called"), + consensus_engine: (), + genesis: Default::default(), + code: Default::default(), + code_substitutes: BTreeMap::new(), + }; + + ChainSpec { + client_spec, + genesis: GenesisSource::GenesisBuilderApi( + self.genesis_build_action + .expect("with_genesis_patch or with_no_genesis_defaults must be called."), + ), + code: self.code.expect("with code must be called.").into(), + } + } +} + /// A configuration of a chain. Can be used to build a genesis block. pub struct ChainSpec { client_spec: ClientSpec, genesis: GenesisSource, + code: Vec, } impl Clone for ChainSpec { fn clone(&self) -> Self { - ChainSpec { client_spec: self.client_spec.clone(), genesis: self.genesis.clone() } + ChainSpec { + client_spec: self.client_spec.clone(), + genesis: self.genesis.clone(), + code: self.code.clone(), + } } } @@ -260,6 +453,7 @@ impl ChainSpec { } /// Create hardcoded spec. + #[deprecated(note = "todo: add note")] pub fn from_genesis G + 'static + Send + Sync>( name: &str, id: &str, @@ -271,6 +465,7 @@ impl ChainSpec { fork_id: Option<&str>, properties: Option, extensions: E, + code: &[u8], ) -> Self { let client_spec = ClientSpec { name: name.to_owned(), @@ -284,25 +479,46 @@ impl ChainSpec { extensions, consensus_engine: (), genesis: Default::default(), + code: Default::default(), code_substitutes: BTreeMap::new(), }; - ChainSpec { client_spec, genesis: GenesisSource::Factory(Arc::new(constructor)) } + ChainSpec { + client_spec, + #[allow(deprecated)] + genesis: GenesisSource::Factory(Arc::new(constructor)), + code: code.into(), + } } /// Type of the chain. fn chain_type(&self) -> ChainType { self.client_spec.chain_type.clone() } + + /// Provides a `ChainSpec` builder. + pub fn builder() -> ChainSpecBuilder { + ChainSpecBuilder::new() + } } -impl ChainSpec { +#[derive(Serialize, Deserialize)] +struct CodeContainer { + #[serde(default, with = "sp_core::bytes")] + code: Vec, +} + +impl ChainSpec { /// Parse json content into a `ChainSpec` pub fn from_json_bytes(json: impl Into>) -> Result { let json = json.into(); let client_spec = json::from_slice(json.as_ref()) .map_err(|e| format!("Error parsing spec file: {}", e))?; - Ok(ChainSpec { client_spec, genesis: GenesisSource::Binary(json) }) + + let code: CodeContainer = json::from_slice(json.as_ref()) + .map_err(|e| format!("Error parsing spec file: {}", e))?; + + Ok(ChainSpec { client_spec, genesis: GenesisSource::Binary(json), code: code.code }) } /// Parse json file into a `ChainSpec` @@ -321,44 +537,44 @@ impl ChainSpec { let client_spec = json::from_slice(&bytes).map_err(|e| format!("Error parsing spec file: {}", e))?; - Ok(ChainSpec { client_spec, genesis: GenesisSource::File(path) }) + + let code: CodeContainer = + json::from_slice(&bytes).map_err(|e| format!("Error parsing spec file: {}", e))?; + + Ok(ChainSpec { client_spec, genesis: GenesisSource::File(path), code: code.code }) } } #[derive(Serialize, Deserialize)] -struct JsonContainer { +#[serde(deny_unknown_fields)] +struct ChainSpecJsonContainer { #[serde(flatten)] client_spec: ClientSpec, genesis: Genesis, + #[serde(with = "sp_core::bytes")] + code: Vec, } impl ChainSpec { - fn json_container(&self, raw: bool) -> Result, String> { + fn json_container(&self, raw: bool) -> Result, String> { let genesis = match (raw, self.genesis.resolve()?) { + (true, Genesis::JsonPatch(patch)) => Genesis::Raw(RawGenesis::from( + crate::genesis_config_builder::get_storage_for_patch(&self.code[..], patch)?, + )), + (true, Genesis::JsonFull(config)) => Genesis::Raw(RawGenesis::from( + crate::genesis_config_builder::get_storage_for_config(&self.code[..], config)?, + )), (true, Genesis::Runtime(g)) => { let storage = g.build_storage()?; - let top = - storage.top.into_iter().map(|(k, v)| (StorageKey(k), StorageData(v))).collect(); - let children_default = storage - .children_default - .into_iter() - .map(|(sk, child)| { - ( - StorageKey(sk), - child - .data - .into_iter() - .map(|(k, v)| (StorageKey(k), StorageData(v))) - .collect(), - ) - }) - .collect(); - - Genesis::Raw(RawGenesis { top, children_default }) + Genesis::Raw(RawGenesis::from(storage)) }, (_, genesis) => genesis, }; - Ok(JsonContainer { client_spec: self.client_spec.clone(), genesis }) + Ok(ChainSpecJsonContainer { + client_spec: self.client_spec.clone(), + genesis, + code: self.code.clone(), + }) } /// Dump to json string. @@ -445,6 +661,9 @@ where #[cfg(test)] mod tests { use super::*; + use serde_json::json; + use sp_application_crypto::Ss58Codec; + use sp_keyring::AccountKeyring; #[derive(Debug, Serialize, Deserialize)] struct Genesis(BTreeMap); @@ -536,4 +755,141 @@ mod tests { ); } } + + fn remove_code_key_from_json(json: &str) -> serde_json::Value { + serde_json::from_str::(json) + .map(|mut c| { + c.as_object_mut().map(|j| { + j.remove_entry("code").unwrap(); + // .expect("code field should be in blob returned by ChainSpec.as_json"); + }); + c + }) + .unwrap() + } + + #[test] + fn generate_chain_spec_with_patch_works() { + sp_tracing::try_init_simple(); + let output: ChainSpec<()> = ChainSpecBuilder::new() + .with_name("TestName") + .with_id("test_id") + .with_extensions(Default::default()) + .with_chain_type(ChainType::Local) + .with_code(substrate_test_runtime::wasm_binary_unwrap().into()) + .with_genesis_patch(json!({ + "babe": { + "epochConfig": { + "c": [ + 7, + 10 + ], + "allowed_slots": "PrimaryAndSecondaryPlainSlots" + } + }, + "substrateTest": { + "authorities": [ + AccountKeyring::Ferdie.public().to_ss58check(), + AccountKeyring::Alice.public().to_ss58check() + ], + } + })) + .build(); + + // std::fs::write("/tmp/patch.json", output.as_json(false).unwrap()); + // std::fs::write("/tmp/patch_raw.json", output.as_json(true).unwrap()); + let acutal = output.as_json(false).unwrap(); + let acutal_raw = output.as_json(true).unwrap(); + + let expected = include_str!("../res/substrate_test_runtime_from_patch.json"); + let expected_raw = include_str!("../res/substrate_test_runtime_from_patch__raw.json"); + + //'code' may chang so let's remove it. Only ensure it is there: + let actual = remove_code_key_from_json(acutal.as_str()); + let actual_raw = remove_code_key_from_json(acutal_raw.as_str()); + + // Removing key changes order of keys, so modify expected too: + let expected = remove_code_key_from_json(expected); + let expected_raw = remove_code_key_from_json(expected_raw); + + assert_eq!(actual, expected); + assert_eq!(actual_raw, expected_raw); + } + + #[test] + fn generate_chain_spec_with_full_config_works() { + sp_tracing::try_init_simple(); + let j = + include_str!("../../../test-utils/runtime/src/test_json/default_genesis_config.json"); + let output: ChainSpec<()> = ChainSpecBuilder::new() + .with_name("TestName") + .with_id("test_id") + .with_extensions(Default::default()) + .with_chain_type(ChainType::Local) + .with_code(substrate_test_runtime::wasm_binary_unwrap().into()) + .with_no_genesis_defaults(serde_json::from_str(j).unwrap()) + .build(); + + // std::fs::write("/tmp/config.json", output.as_json(false).unwrap()); + // std::fs::write("/tmp/config_raw.json", output.as_json(true).unwrap()); + + let acutal = output.as_json(false).unwrap(); + let acutal_raw = output.as_json(true).unwrap(); + + let expected = include_str!("../res/substrate_test_runtime_from_config.json"); + let expected_raw = include_str!("../res/substrate_test_runtime_from_config__raw.json"); + + //'code' may chang so let's remove it. Only ensure it is there: + let actual = remove_code_key_from_json(acutal.as_str()); + let actual_raw = remove_code_key_from_json(acutal_raw.as_str()); + + // Removing key changes order of keys, so modify expected too: + let expected = remove_code_key_from_json(expected); + let expected_raw = remove_code_key_from_json(expected_raw); + + assert_eq!(actual, expected); + assert_eq!(actual_raw, expected_raw); + } + + #[test] + fn chain_spec_as_json_fails_with_invalid_config() { + sp_tracing::try_init_simple(); + let j = include_str!( + // "../../../test-utils/runtime/src/test_json/default_genesis_config_invalid_2.json" + "../../../test-utils/runtime/src/test_json/default_genesis_config_invalid_2.json" + ); + let output: ChainSpec<()> = ChainSpecBuilder::new() + .with_name("TestName") + .with_id("test_id") + .with_extensions(Default::default()) + .with_chain_type(ChainType::Local) + .with_code(substrate_test_runtime::wasm_binary_unwrap().into()) + .with_no_genesis_defaults(serde_json::from_str(j).unwrap()) + .build(); + + assert_eq!( output.as_json(true), Err("Invalid JSON blob: unknown field `babex`, expected one of `system`, `babe`, `substrateTest`, `balances` at line 1 column 8".to_string()) ); + } + + #[test] + fn chain_spec_as_json_fails_with_invalid_patch() { + sp_tracing::try_init_simple(); + let output: ChainSpec<()> = ChainSpecBuilder::new() + .with_name("TestName") + .with_id("test_id") + .with_extensions(Default::default()) + .with_chain_type(ChainType::Local) + .with_code(substrate_test_runtime::wasm_binary_unwrap().into()) + .with_genesis_patch(json!({ + "invalid_pallet": {}, + "substrateTest": { + "authorities": [ + AccountKeyring::Ferdie.public().to_ss58check(), + AccountKeyring::Alice.public().to_ss58check() + ], + } + })) + .build(); + + assert!(output.as_json(true).unwrap_err().contains("Invalid JSON blob: unknown field `invalid_pallet`, expected one of `system`, `babe`, `substrateTest`, `balances`")); + } } diff --git a/client/chain-spec/src/lib.rs b/client/chain-spec/src/lib.rs index c33c9e960c1bb..c18cfb868b49b 100644 --- a/client/chain-spec/src/lib.rs +++ b/client/chain-spec/src/lib.rs @@ -181,7 +181,7 @@ mod genesis_block; mod genesis_config_builder; pub use self::{ - chain_spec::{ChainSpec as GenericChainSpec, NoExtension}, + chain_spec::{ChainSpec as GenericChainSpec, ChainSpecBuilder, NoExtension}, extension::{get_extension, get_extension_mut, Extension, Fork, Forks, GetExtension, Group}, genesis_block::{ construct_genesis_block, resolve_state_version_from_wasm, BuildGenesisBlock, From baacd30584c5759fa03139ea5f4cbb84daa32455 Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Sun, 9 Jul 2023 08:04:58 +0200 Subject: [PATCH 05/47] chain-spec: runtime caller added --- client/chain-spec/Cargo.toml | 1 + client/chain-spec/src/chain_spec.rs | 23 ++- .../chain-spec/src/genesis_config_builder.rs | 187 +++++++++++------- client/chain-spec/src/lib.rs | 1 + 4 files changed, 134 insertions(+), 78 deletions(-) diff --git a/client/chain-spec/Cargo.toml b/client/chain-spec/Cargo.toml index ea9962889b9a4..73c531aa82955 100644 --- a/client/chain-spec/Cargo.toml +++ b/client/chain-spec/Cargo.toml @@ -39,3 +39,4 @@ kitchensink-runtime = { path = "../../bin/node/runtime" } substrate-test-runtime = { path = "../../test-utils/runtime" } sp-keyring = { version = "24.0.0", path = "../../primitives/keyring" } sp-application-crypto = { version = "23.0.0", default-features = false, path = "../../primitives/application-crypto", features = ["serde"] } +sp-consensus-babe = { version = "0.10.0-dev", default-features = false, path = "../../primitives/consensus/babe", features = ["serde"] } diff --git a/client/chain-spec/src/chain_spec.rs b/client/chain-spec/src/chain_spec.rs index c527f4a137625..383c5fcfe71e8 100644 --- a/client/chain-spec/src/chain_spec.rs +++ b/client/chain-spec/src/chain_spec.rs @@ -19,7 +19,10 @@ //! Substrate chain configurations. #![warn(missing_docs)] -use crate::{extension::GetExtension, ChainType, Properties, RuntimeGenesis}; +use crate::{ + extension::GetExtension, ChainType, GenesisConfigBuilderRuntimeCaller as RuntimeCaller, + Properties, RuntimeGenesis, +}; use sc_network::config::MultiaddrWithPeerId; use sc_telemetry::TelemetryEndpoints; use serde::{Deserialize, Serialize}; @@ -153,12 +156,12 @@ impl BuildStorage for ChainSpec { // it, but Substrate itself isn't capable of loading chain specs with just a hash at the // moment. Genesis::StateRootHash(_) => Err("Genesis storage in hash format not supported".into()), - Genesis::JsonFull(config) => - crate::genesis_config_builder::get_storage_for_config(&self.code[..], config)? - .assimilate_storage(storage), - Genesis::JsonPatch(patch) => - crate::genesis_config_builder::get_storage_for_patch(&self.code[..], patch)? - .assimilate_storage(storage), + Genesis::JsonFull(config) => RuntimeCaller::new(&self.code[..]) + .get_storage_for_config(config)? + .assimilate_storage(storage), + Genesis::JsonPatch(patch) => RuntimeCaller::new(&self.code[..]) + .get_storage_for_patch(patch)? + .assimilate_storage(storage), }?; Ok(()) @@ -559,10 +562,10 @@ impl ChainSpec { fn json_container(&self, raw: bool) -> Result, String> { let genesis = match (raw, self.genesis.resolve()?) { (true, Genesis::JsonPatch(patch)) => Genesis::Raw(RawGenesis::from( - crate::genesis_config_builder::get_storage_for_patch(&self.code[..], patch)?, + RuntimeCaller::new(&self.code[..]).get_storage_for_patch(patch)?, )), (true, Genesis::JsonFull(config)) => Genesis::Raw(RawGenesis::from( - crate::genesis_config_builder::get_storage_for_config(&self.code[..], config)?, + RuntimeCaller::new(&self.code[..]).get_storage_for_config(config)?, )), (true, Genesis::Runtime(g)) => { let storage = g.build_storage()?; @@ -890,6 +893,6 @@ mod tests { })) .build(); - assert!(output.as_json(true).unwrap_err().contains("Invalid JSON blob: unknown field `invalid_pallet`, expected one of `system`, `babe`, `substrateTest`, `balances`")); + assert!(output.as_json(true).unwrap_err().contains("Invalid JSON blob: unknown field `invalid_pallet`, expected one of `system`, `babe`, `substrateTest`, `balances`")); } } diff --git a/client/chain-spec/src/genesis_config_builder.rs b/client/chain-spec/src/genesis_config_builder.rs index 5138e32aa8c2c..dd5160bb40aea 100644 --- a/client/chain-spec/src/genesis_config_builder.rs +++ b/client/chain-spec/src/genesis_config_builder.rs @@ -1,86 +1,137 @@ +use std::{borrow::Cow, str::FromStr}; + use codec::{Decode, Encode}; use sc_executor::{error::Result, WasmExecutor}; -use sc_executor_common::runtime_blob::RuntimeBlob; +// use sc_executor_common::runtime_blob::RuntimeBlob; use serde_json::{from_slice, Value}; -use sp_core::{storage::Storage, traits::Externalities}; +use sp_core::{ + storage::Storage, + traits::{CallContext, CodeExecutor, Externalities, FetchRuntimeCode, RuntimeCode}, +}; use sp_genesis_builder::Result as BuildResult; use sp_state_machine::BasicExternalities; -// todo: -// - tests: focus on errors -// - reuse executor -// - pub api + internal function - -fn executor_call( - ext: &mut dyn Externalities, - code: &[u8], - method: &str, - data: &[u8], -) -> Result> { - WasmExecutor::::builder().build().uncached_call( - RuntimeBlob::uncompress_if_needed(code.into()).unwrap(), - ext, - true, - method, - data, - ) +pub struct GenesisConfigBuilderRuntimeCaller<'a> { + code: Cow<'a, [u8]>, + code_hash: Vec, + executor: WasmExecutor, } -pub fn get_storage_for_config(code: &[u8], config: Value) -> core::result::Result { - log::info!("get_storage_for_config: {:?}", config); - let mut ext = BasicExternalities::new_empty(); - - let call_result = - executor_call(&mut ext, code, "GenesisBuilder_build_config", &config.to_string().encode()) - .map_err(|e| format!("wasm call error {e}"))?; - - log::info!("get_storage_for_config: call_restul {:?}", call_result); - log::info!("get_storage_for_config: ext {:?}", ext); - let build_result = BuildResult::decode(&mut &call_result[..]) - .map_err(|e| format!("scale codec error: {e}"))?; - log::info!("get_storage_for_config: br {:?}", build_result); - - Ok(build_result.map(|_| ext.into_storages())?) +impl<'a> FetchRuntimeCode for GenesisConfigBuilderRuntimeCaller<'a> { + fn fetch_runtime_code(&self) -> Option> { + Some(self.code.as_ref().into()) + } } -pub fn get_storage_for_patch(code: &[u8], patch: Value) -> core::result::Result { - log::info!("get_storage_for_patch: {:?}", patch); - let mut t = BasicExternalities::new_empty(); - let call_result = executor_call(&mut t, code, "GenesisBuilder_create_default_config", &vec![]) - .map_err(|e| format!("wasm call error {e}"))?; - - let default_config = - Vec::::decode(&mut &call_result[..]).map_err(|e| format!("scale codec error: {e}"))?; - - let mut config: Value = from_slice(&default_config[..]).expect("returned value is json. qed."); - - json_patch::merge(&mut config, &patch); - get_storage_for_config(code, config) +impl<'a> GenesisConfigBuilderRuntimeCaller<'a> { + pub fn new(code: &'a [u8]) -> Self { + GenesisConfigBuilderRuntimeCaller { + code: code.into(), + code_hash: sp_core::blake2_256(&code).to_vec(), + executor: WasmExecutor::::builder().build(), + } + } + + fn call(&self, ext: &mut dyn Externalities, method: &str, data: &[u8]) -> Result> { + self.executor + .call( + ext, + &RuntimeCode { heap_pages: None, code_fetcher: self, hash: self.code_hash.clone() }, + method, + data, + false, + CallContext::Offchain, + ) + .0 + } } -// pub fn get_default_config(code: &[u8]) -> core::result::Result { -// let mut t = BasicExternalities::new_empty(); -// let call_result = executor_call(&mut t, code, "GenesisBuilder_create_default_config", &vec![]) -// .map_err(|e| format!("wasm call error {e}"))?; +impl<'a> GenesisConfigBuilderRuntimeCaller<'a> { + pub fn get_default_config(&self) -> core::result::Result { + let mut t = BasicExternalities::new_empty(); + let call_result = self + .call(&mut t, "GenesisBuilder_create_default_config", &vec![]) + .map_err(|e| format!("wasm call error {e}"))?; + let default_config = Vec::::decode(&mut &call_result[..]) + .map_err(|e| format!("scale codec error: {e}"))?; + Ok(from_slice(&default_config[..]).expect("returned value is json. qed.")) + } -// let default_config = -// Vec::::decode(&mut &call_result[..]).map_err(|e| format!("scale codec error: {e}"))?; + pub fn get_storage_for_config(&self, config: Value) -> core::result::Result { + let mut ext = BasicExternalities::new_empty(); -// let mut config: Value = from_slice(&default_config[..]).expect("returned value is json. qed."); + let call_result = self + .call(&mut ext, "GenesisBuilder_build_config", &config.to_string().encode()) + .map_err(|e| format!("wasm call error {e}"))?; -// json_patch::merge(&mut config, &patch); -// get_storage_for_config(code, config) -// } + let build_result = BuildResult::decode(&mut &call_result[..]) + .map_err(|e| format!("scale codec error: {e}"))?; -// #[test] -// fn build_config_from_json_works() { -// sp_tracing::try_init_simple(); -// let j = include_str!("../../../test-utils/runtime/src/test_json/default_genesis_config.json"); + Ok(build_result.map(|_| ext.into_storages())?) + } -// let mut t = BasicExternalities::new_empty(); -// let r = executor_call(&mut t, "GenesisBuilder_build_config", &j.encode()).unwrap(); -// let r = BuildResult::decode(&mut &r[..]); -// assert!(r.is_ok()); + pub fn get_storage_for_patch(&self, patch: Value) -> core::result::Result { + let mut config = self.get_default_config()?; + json_patch::merge(&mut config, &patch); + self.get_storage_for_config(config) + } +} -// // let mut _keys = t.into_storages().top.keys().cloned().map(hex).collect::>(); -// } +#[cfg(test)] +mod tests { + use super::*; + pub use sp_consensus_babe::{AllowedSlots, BabeEpochConfiguration, Slot}; + + #[test] + fn get_default_config_works() { + sp_tracing::try_init_simple(); + let config = + GenesisConfigBuilderRuntimeCaller::new(substrate_test_runtime::wasm_binary_unwrap()) + .get_default_config() + .unwrap(); + let expected = r#"{"system":{},"babe":{"authorities":[],"epochConfig":null},"substrateTest":{"authorities":[]},"balances":{"balances":[]}}"#; + assert_eq!(serde_json::from_str::(expected).unwrap(), config); + } + + #[test] + fn get_storage_for_patch_works() { + sp_tracing::try_init_simple(); + + let patch = serde_json::json!({ + "babe": { + "epochConfig": { + "c": [ + 69, + 696 + ], + "allowed_slots": "PrimaryAndSecondaryPlainSlots" + } + }, + }); + + let storage = + GenesisConfigBuilderRuntimeCaller::new(substrate_test_runtime::wasm_binary_unwrap()) + .get_storage_for_patch(patch) + .unwrap(); + + //Babe|Authorities + let value: Vec = storage + .top + .get( + &array_bytes::hex2bytes( + "1cb6f36e027abb2091cfb5110ab5087fdc6b171b77304263c292cc3ea5ed31ef", + ) + .unwrap(), + ) + .unwrap() + .clone(); + + assert_eq!( + BabeEpochConfiguration::decode(&mut &value[..]).unwrap(), + BabeEpochConfiguration { + c: (69, 696), + allowed_slots: AllowedSlots::PrimaryAndSecondaryPlainSlots + } + ); + } +} diff --git a/client/chain-spec/src/lib.rs b/client/chain-spec/src/lib.rs index c18cfb868b49b..cd83e1649ed66 100644 --- a/client/chain-spec/src/lib.rs +++ b/client/chain-spec/src/lib.rs @@ -187,6 +187,7 @@ pub use self::{ construct_genesis_block, resolve_state_version_from_wasm, BuildGenesisBlock, GenesisBlockBuilder, }, + genesis_config_builder::GenesisConfigBuilderRuntimeCaller, }; pub use sc_chain_spec_derive::{ChainSpecExtension, ChainSpecGroup}; From b40193fc427a8a06d290c29039df09b89b8fb7bc Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Mon, 10 Jul 2023 18:07:52 +0200 Subject: [PATCH 06/47] chain-spec: doc + minor improvements --- client/chain-spec/src/chain_spec.rs | 2 +- .../chain-spec/src/genesis_config_builder.rs | 37 ++++++++++++++++++- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/client/chain-spec/src/chain_spec.rs b/client/chain-spec/src/chain_spec.rs index 383c5fcfe71e8..e2e28f4ae5288 100644 --- a/client/chain-spec/src/chain_spec.rs +++ b/client/chain-spec/src/chain_spec.rs @@ -38,7 +38,7 @@ use std::{ #[derive(Serialize, Deserialize, Clone)] #[serde(rename_all = "camelCase")] -pub enum GenesisBuildAction { +enum GenesisBuildAction { #[serde(alias = "RuntimePatch")] Patch(serde_json::Value), #[serde(alias = "Runtime")] diff --git a/client/chain-spec/src/genesis_config_builder.rs b/client/chain-spec/src/genesis_config_builder.rs index dd5160bb40aea..edfbbab17ec72 100644 --- a/client/chain-spec/src/genesis_config_builder.rs +++ b/client/chain-spec/src/genesis_config_builder.rs @@ -11,6 +11,7 @@ use sp_core::{ use sp_genesis_builder::Result as BuildResult; use sp_state_machine::BasicExternalities; +/// Util that allows to call GenesisBuilder API from the runtime wasm code blob. pub struct GenesisConfigBuilderRuntimeCaller<'a> { code: Cow<'a, [u8]>, code_hash: Vec, @@ -24,11 +25,14 @@ impl<'a> FetchRuntimeCode for GenesisConfigBuilderRuntimeCaller<'a> { } impl<'a> GenesisConfigBuilderRuntimeCaller<'a> { + /// Creates new instance using provided code blob. pub fn new(code: &'a [u8]) -> Self { GenesisConfigBuilderRuntimeCaller { code: code.into(), code_hash: sp_core::blake2_256(&code).to_vec(), - executor: WasmExecutor::::builder().build(), + executor: WasmExecutor::::builder() + .with_allow_missing_host_functions(true) + .build(), } } @@ -47,6 +51,11 @@ impl<'a> GenesisConfigBuilderRuntimeCaller<'a> { } impl<'a> GenesisConfigBuilderRuntimeCaller<'a> { + /// Get the default `GenesisConfig` as a JSON blob. + /// + /// This function instantiates the default `GenesisConfig` struct for the runtime and serializes + /// it into a JSON blob. It returns a `Vec` containing the JSON representation of the + /// default `GenesisConfig`. pub fn get_default_config(&self) -> core::result::Result { let mut t = BasicExternalities::new_empty(); let call_result = self @@ -57,6 +66,14 @@ impl<'a> GenesisConfigBuilderRuntimeCaller<'a> { Ok(from_slice(&default_config[..]).expect("returned value is json. qed.")) } + /// Build `GenesisConfig` from a JSON blob not using any defaults and store it in the storage. + /// + /// This function deserializes the full `GenesisConfig` from the given JSON blob and puts it + /// into the storage. If the provided JSON blob is incorrect or incomplete or the + /// deserialization fails, an error is returned. + /// + /// Please note that provided json blob must contain all `GenesisConfig` fields, no defaults + /// will be used. pub fn get_storage_for_config(&self, config: Value) -> core::result::Result { let mut ext = BasicExternalities::new_empty(); @@ -70,6 +87,24 @@ impl<'a> GenesisConfigBuilderRuntimeCaller<'a> { Ok(build_result.map(|_| ext.into_storages())?) } + /// Patch default `GenesisConfig` using given JSON patch and store it in the storage. + /// + /// This function generates the `GenesisConfig` for the runtime by applying a provided JSON + /// patch. The patch modifies the default `GenesisConfig` allowing customization of the specific + /// keys. The resulting `GenesisConfig` is then deserialized from the patched JSON + /// representation and stored in the storage. + /// + /// If the provided JSON patch is incorrect or the deserialization fails the error will be + /// returned. + /// + /// The patching process modifies the default `GenesisConfig` according to the following rules: + /// 1. Existing keys in the default configuration will be overridden by the corresponding values + /// in the patch. 2. If a key exists in the patch but not in the default configuration, it will + /// be added to the resulting `GenesisConfig`. 3. Keys in the default configuration that have + /// null values in the patch will be removed from the resulting `GenesisConfig`. This is + /// helpful for changing enum variant value. + /// + /// Please note that the patch may contain full `GenesisConfig`. pub fn get_storage_for_patch(&self, patch: Value) -> core::result::Result { let mut config = self.get_default_config()?; json_patch::merge(&mut config, &patch); From dc6c927e25e41f267b66b7677e7b8b646281a6e1 Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Tue, 11 Jul 2023 12:19:49 +0200 Subject: [PATCH 07/47] chain-spec: set_code added --- client/chain-spec/src/chain_spec.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/client/chain-spec/src/chain_spec.rs b/client/chain-spec/src/chain_spec.rs index e2e28f4ae5288..6005c331e4189 100644 --- a/client/chain-spec/src/chain_spec.rs +++ b/client/chain-spec/src/chain_spec.rs @@ -499,6 +499,11 @@ impl ChainSpec { self.client_spec.chain_type.clone() } + /// Sets the code. + pub fn set_code(&mut self, code: &[u8]) { + self.code = code.into(); + } + /// Provides a `ChainSpec` builder. pub fn builder() -> ChainSpecBuilder { ChainSpecBuilder::new() From ae9b2a83c6eb96b14b5b04ecc9d3c37586a1b0fc Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Thu, 6 Jul 2023 14:18:23 +0200 Subject: [PATCH 08/47] chain-spec-builder: from_genesis removed --- bin/utils/chain-spec-builder/Cargo.toml | 1 + bin/utils/chain-spec-builder/src/main.rs | 60 ++++++++---------------- 2 files changed, 21 insertions(+), 40 deletions(-) diff --git a/bin/utils/chain-spec-builder/Cargo.toml b/bin/utils/chain-spec-builder/Cargo.toml index 412b41009340c..1d28e2f26a297 100644 --- a/bin/utils/chain-spec-builder/Cargo.toml +++ b/bin/utils/chain-spec-builder/Cargo.toml @@ -22,3 +22,4 @@ sc-chain-spec = { version = "4.0.0-dev", path = "../../../client/chain-spec" } sc-keystore = { version = "4.0.0-dev", path = "../../../client/keystore" } sp-core = { version = "21.0.0", path = "../../../primitives/core" } sp-keystore = { version = "0.27.0", path = "../../../primitives/keystore" } +kitchensink-runtime = { version = "3.0.0-dev", path = "../../node/runtime" } diff --git a/bin/utils/chain-spec-builder/src/main.rs b/bin/utils/chain-spec-builder/src/main.rs index deb191a67cfbe..d951beebf553b 100644 --- a/bin/utils/chain-spec-builder/src/main.rs +++ b/bin/utils/chain-spec-builder/src/main.rs @@ -95,26 +95,6 @@ impl ChainSpecBuilder { } } -fn genesis_constructor( - authority_seeds: &[String], - nominator_accounts: &[AccountId], - endowed_accounts: &[AccountId], - sudo_account: &AccountId, -) -> chain_spec::RuntimeGenesisConfig { - let authorities = authority_seeds - .iter() - .map(AsRef::as_ref) - .map(chain_spec::authority_keys_from_seed) - .collect::>(); - - chain_spec::testnet_genesis( - authorities, - nominator_accounts.to_vec(), - sudo_account.clone(), - Some(endowed_accounts.to_vec()), - ) -} - fn generate_chain_spec( authority_seeds: Vec, nominator_accounts: Vec, @@ -138,27 +118,27 @@ fn generate_chain_spec( let sudo_account = parse_account(sudo_account)?; - let chain_spec = chain_spec::ChainSpec::from_genesis( - "Custom", - "custom", - sc_chain_spec::ChainType::Live, - move || { - genesis_constructor( - &authority_seeds, - &nominator_accounts, - &endowed_accounts, - &sudo_account, - ) - }, - vec![], - None, - None, - None, - None, - Default::default(), - ); + let authorities = authority_seeds + .iter() + .map(AsRef::as_ref) + .map(chain_spec::authority_keys_from_seed) + .collect::>(); - chain_spec.as_json(false) + chain_spec::ChainSpec::builder() + .with_name("Custom") + .with_id("custom") + .with_chain_type(sc_chain_spec::ChainType::Live) + .with_genesis_patch(chain_spec::testnet_genesis( + authorities, + nominator_accounts, + sudo_account, + Some(endowed_accounts), + )) + .with_boot_nodes(Default::default()) + .with_extensions(Default::default()) + .with_code(kitchensink_runtime::wasm_binary_unwrap()) + .build() + .as_json(false) } fn generate_authority_keys_and_store(seeds: &[String], keystore_path: &Path) -> Result<(), String> { From e166fec33e58e3d11861bc16504188c227131be9 Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Thu, 6 Jul 2023 16:19:37 +0200 Subject: [PATCH 09/47] chain-spec-builder: boot_nodes call optional --- bin/utils/chain-spec-builder/src/main.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/utils/chain-spec-builder/src/main.rs b/bin/utils/chain-spec-builder/src/main.rs index d951beebf553b..1f8574763dcd6 100644 --- a/bin/utils/chain-spec-builder/src/main.rs +++ b/bin/utils/chain-spec-builder/src/main.rs @@ -134,7 +134,6 @@ fn generate_chain_spec( sudo_account, Some(endowed_accounts), )) - .with_boot_nodes(Default::default()) .with_extensions(Default::default()) .with_code(kitchensink_runtime::wasm_binary_unwrap()) .build() From c200e2694cbb03b40f26ef60e5958c1647fb5990 Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Tue, 11 Jul 2023 12:19:23 +0200 Subject: [PATCH 10/47] chain-spec-builder: support for runtime wasm blobs --- bin/utils/chain-spec-builder/Cargo.toml | 3 + bin/utils/chain-spec-builder/src/main.rs | 321 ++++++++++++++++++----- 2 files changed, 252 insertions(+), 72 deletions(-) diff --git a/bin/utils/chain-spec-builder/Cargo.toml b/bin/utils/chain-spec-builder/Cargo.toml index 1d28e2f26a297..4469fc66afda8 100644 --- a/bin/utils/chain-spec-builder/Cargo.toml +++ b/bin/utils/chain-spec-builder/Cargo.toml @@ -23,3 +23,6 @@ sc-keystore = { version = "4.0.0-dev", path = "../../../client/keystore" } sp-core = { version = "21.0.0", path = "../../../primitives/core" } sp-keystore = { version = "0.27.0", path = "../../../primitives/keystore" } kitchensink-runtime = { version = "3.0.0-dev", path = "../../node/runtime" } +serde_json = "1.0.100" +sp-tracing = { version = "10.0.0", path = "../../../primitives/tracing" } +log = "0.4.17" diff --git a/bin/utils/chain-spec-builder/src/main.rs b/bin/utils/chain-spec-builder/src/main.rs index 1f8574763dcd6..6cf1e637833a1 100644 --- a/bin/utils/chain-spec-builder/src/main.rs +++ b/bin/utils/chain-spec-builder/src/main.rs @@ -22,11 +22,13 @@ use std::{ }; use ansi_term::Style; -use clap::Parser; +use clap::{Parser, Subcommand}; use rand::{distributions::Alphanumeric, rngs::OsRng, Rng}; +use sc_chain_spec::{GenericChainSpec, GenesisConfigBuilderRuntimeCaller}; use node_cli::chain_spec::{self, AccountId}; use sc_keystore::LocalKeystore; +use serde_json::Value; use sp_core::{ crypto::{ByteArray, Ss58Codec}, sr25519, @@ -35,64 +37,152 @@ use sp_keystore::KeystorePtr; /// A utility to easily create a testnet chain spec definition with a given set /// of authorities and endowed accounts and/or generate random accounts. -#[derive(Parser)] +#[derive(Debug, Parser)] #[command(rename_all = "kebab-case")] -enum ChainSpecBuilder { - /// Create a new chain spec with the given authorities, endowed and sudo - /// accounts. - New { - /// Authority key seed. - #[arg(long, short, required = true)] - authority_seeds: Vec, - /// Active nominators (SS58 format), each backing a random subset of the aforementioned - /// authorities. - #[arg(long, short, default_value = "0")] - nominator_accounts: Vec, - /// Endowed account address (SS58 format). - #[arg(long, short)] - endowed_accounts: Vec, - /// Sudo account address (SS58 format). - #[arg(long, short)] - sudo_account: String, - /// The path where the chain spec should be saved. - #[arg(long, short, default_value = "./chain_spec.json")] - chain_spec_path: PathBuf, - }, - /// Create a new chain spec with the given number of authorities and endowed - /// accounts. Random keys will be generated as required. - Generate { - /// The number of authorities. - #[arg(long, short)] - authorities: usize, - /// The number of nominators backing the aforementioned authorities. - /// - /// Will nominate a random subset of `authorities`. - #[arg(long, short, default_value_t = 0)] - nominators: usize, - /// The number of endowed accounts. - #[arg(long, short, default_value_t = 0)] - endowed: usize, - /// The path where the chain spec should be saved. - #[arg(long, short, default_value = "./chain_spec.json")] - chain_spec_path: PathBuf, - /// Path to use when saving generated keystores for each authority. - /// - /// At this path, a new folder will be created for each authority's - /// keystore named `auth-$i` where `i` is the authority index, i.e. - /// `auth-0`, `auth-1`, etc. - #[arg(long, short)] - keystore_path: Option, - }, +struct ChainSpecBuilder { + #[command(subcommand)] + command: ChainSpecBuilderCmd, + /// The path where the chain spec should be saved. + #[arg(long, short, default_value = "./chain_spec.json")] + chain_spec_path: PathBuf, } -impl ChainSpecBuilder { - /// Returns the path where the chain spec should be saved. - fn chain_spec_path(&self) -> &Path { - match self { - ChainSpecBuilder::New { chain_spec_path, .. } => chain_spec_path.as_path(), - ChainSpecBuilder::Generate { chain_spec_path, .. } => chain_spec_path.as_path(), - } - } +#[derive(Debug, Subcommand)] +#[command(rename_all = "kebab-case")] +enum ChainSpecBuilderCmd { + New(NewCmd), + Generate(GenerateCmd), + Runtime(RuntimeCmd), + Edit(EditCmd), + Verify(VerifyCmd), +} + +/// Create a new chain spec with the given authorities, endowed and sudo +/// accounts. Only works for kitchen-sink runtime +#[derive(Parser, Debug)] +#[command(rename_all = "kebab-case")] +struct NewCmd { + /// Authority key seed. + #[arg(long, short, required = true)] + authority_seeds: Vec, + /// Active nominators (SS58 format), each backing a random subset of the aforementioned + /// authorities. + #[arg(long, short, default_value = "0")] + nominator_accounts: Vec, + /// Endowed account address (SS58 format). + #[arg(long, short)] + endowed_accounts: Vec, + /// Sudo account address (SS58 format). + #[arg(long, short)] + sudo_account: String, +} + +/// Create a new chain spec with the given number of authorities and endowed +/// accounts. Random keys will be generated as required. Only works for kitchen-sink runtime +#[derive(Parser, Debug)] +struct GenerateCmd { + /// The number of authorities. + #[arg(long, short)] + authorities: usize, + /// The number of nominators backing the aforementioned authorities. + /// + /// Will nominate a random subset of `authorities`. + #[arg(long, short, default_value_t = 0)] + nominators: usize, + /// The number of endowed accounts. + #[arg(long, short, default_value_t = 0)] + endowed: usize, + /// Path to use when saving generated keystores for each authority. + /// + /// At this path, a new folder will be created for each authority's + /// keystore named `auth-$i` where `i` is the authority index, i.e. + /// `auth-0`, `auth-1`, etc. + #[arg(long, short)] + keystore_path: Option, +} + +/// Create a new chain spec by interacting with the provided runtime wasm blob. +#[derive(Parser, Debug)] +struct RuntimeCmd { + /// The name of chain + #[arg(long, short = 'n', default_value = "Custom")] + chain_name: String, + /// The chain id + #[arg(long, short = 'i', default_value = "custom")] + chain_id: String, + /// The path to runtime wasm blob + #[arg(long, short)] + runtime_wasm_path: PathBuf, + /// Export chainspec as raw storage + #[arg(long, short = 's')] + raw_storage: bool, + /// Verify the genesis config. This silently generates the raw storage from genesis config. Any + /// errors will be reported. + #[arg(long, short = 'v')] + verify: bool, + #[command(subcommand)] + action: GenesisBuildAction, +} + +#[derive(Subcommand, Debug, Clone)] +enum GenesisBuildAction { + Patch(PatchCmd), + Full(FullCmd), + Default(DefaultCmd), +} + +/// Patches the runtime's default genesis config with provided patch. +#[derive(Parser, Debug, Clone)] +struct PatchCmd { + /// The path to the runtime genesis config patch. + #[arg(long, short)] + patch_path: PathBuf, +} + +/// Build the genesis config for runtime using provided json file. No defaults will be used. +#[derive(Parser, Debug, Clone)] +struct FullCmd { + /// The path to the full runtime genesis config json file. + #[arg(long, short)] + config_path: PathBuf, +} + +/// Gets the default genesis config for the runtime and uses it in ChainSpec. Please note that +/// default genesis config may not be valid. For some runtimes initial values should be added there +/// (e.g. session keys, babe epoch). +#[derive(Parser, Debug, Clone)] +struct DefaultCmd { + #[arg(long, short)] + /// If provided stores the default genesis config json file at given path (in addition to + /// chain-spec). + default_config_path: Option, +} + +/// Edits provided input chain spec. Input can be converted into raw storage chain-spec. The code +/// can be updated with the runtime provided in the command line. +#[derive(Parser, Debug, Clone)] +struct EditCmd { + #[arg(long, short)] + /// Chain spec to be edited + input_chain_spec: PathBuf, + /// The path to new runtime wasm blob to be stored into chain-spec + #[arg(long, short = 'r')] + runtime_wasm_path: Option, + /// Convert genesis spec to raw format + #[arg(long, short = 's')] + convert_to_raw: bool, +} + +/// Verifies provided input chain spec. If the runtime is provided verification is performed against +/// new runtime. +#[derive(Parser, Debug, Clone)] +struct VerifyCmd { + #[arg(long, short)] + /// Chain spec to be edited + input_chain_spec: PathBuf, + /// The path to new runtime wasm blob to be stored into chain-spec + #[arg(long, short = 'r')] + runtime_wasm_path: Option, } fn generate_chain_spec( @@ -207,19 +297,80 @@ fn print_seeds( println!("//{}", sudo_seed); } +fn generate_chain_spec_for_runtime(cmd: &RuntimeCmd) -> Result { + let code = fs::read(cmd.runtime_wasm_path.as_path()).expect("wasm blob file is readable"); + + let builder = chain_spec::ChainSpec::builder() + .with_name(&cmd.chain_name[..]) + .with_id(&cmd.chain_id[..]) + .with_chain_type(sc_chain_spec::ChainType::Live) + .with_extensions(Default::default()) + .with_code(&code[..]); + + let builder = match cmd.action { + GenesisBuildAction::Patch(PatchCmd { ref patch_path }) => { + let patch = fs::read(patch_path.as_path()) + .map_err(|e| format!("patch file {patch_path:?} shall be readable: {e}"))?; + builder.with_genesis_patch(serde_json::from_slice::(&patch[..]).map_err( + |e| format!("patch file {patch_path:?} shall contain a valid json: {e}"), + )?) + }, + GenesisBuildAction::Full(FullCmd { ref config_path }) => { + let config = fs::read(config_path.as_path()) + .map_err(|e| format!("config file {config_path:?} shall be readable: {e}"))?; + builder.with_no_genesis_defaults(serde_json::from_slice::(&config[..]).map_err( + |e| format!("config file {config_path:?} shall contain a valid json: {e}"), + )?) + }, + GenesisBuildAction::Default(DefaultCmd { ref default_config_path }) => { + let caller = GenesisConfigBuilderRuntimeCaller::new(&code[..]); + let default_config = caller + .get_default_config() + .expect("getting default config from runtime should work"); + default_config_path.clone().map(|path| { + fs::write(path.as_path(), serde_json::to_string_pretty(&default_config).unwrap()) + .map_err(|err| err.to_string()) + }); + builder.with_no_genesis_defaults(default_config) + }, + }; + + let chain_spec = builder.build(); + + match (cmd.verify, cmd.raw_storage) { + (_, true) => chain_spec.as_json(true), + (true, false) => { + chain_spec.as_json(true)?; + println!("Genesis config verification: OK"); + chain_spec.as_json(false) + }, + (false, false) => chain_spec.as_json(false), + } +} + fn main() -> Result<(), String> { + sp_tracing::try_init_simple(); + + let builder = ChainSpecBuilder::parse(); #[cfg(build_type = "debug")] - println!( - "The chain spec builder builds a chain specification that includes a Substrate runtime \ + if matches!(builder.command, ChainSpecBuilderCmd::Generate(_) | ChainSpecBuilderCmd::New(_)) { + println!( + "The chain spec builder builds a chain specification that includes a Substrate runtime \ compiled as WASM. To ensure proper functioning of the included runtime compile (or run) \ the chain spec builder binary in `--release` mode.\n", - ); + ); + } - let builder = ChainSpecBuilder::parse(); - let chain_spec_path = builder.chain_spec_path().to_path_buf(); + let chain_spec_path = builder.chain_spec_path.to_path_buf(); + let mut write_chain_spec = true; - let (authority_seeds, nominator_accounts, endowed_accounts, sudo_account) = match builder { - ChainSpecBuilder::Generate { authorities, nominators, endowed, keystore_path, .. } => { + let chain_spec_json = match builder.command { + ChainSpecBuilderCmd::Generate(GenerateCmd { + authorities, + nominators, + endowed, + keystore_path, + }) => { let authorities = authorities.max(1); let rand_str = || -> String { OsRng.sample_iter(&Alphanumeric).take(32).map(char::from).collect() @@ -253,19 +404,45 @@ fn main() -> Result<(), String> { let sudo_account = chain_spec::get_account_id_from_seed::(&sudo_seed).to_ss58check(); - (authority_seeds, nominator_accounts, endowed_accounts, sudo_account) + generate_chain_spec(authority_seeds, nominator_accounts, endowed_accounts, sudo_account) }, - ChainSpecBuilder::New { + ChainSpecBuilderCmd::New(NewCmd { authority_seeds, nominator_accounts, endowed_accounts, sudo_account, - .. - } => (authority_seeds, nominator_accounts, endowed_accounts, sudo_account), - }; - - let json = - generate_chain_spec(authority_seeds, nominator_accounts, endowed_accounts, sudo_account)?; + }) => + generate_chain_spec(authority_seeds, nominator_accounts, endowed_accounts, sudo_account), + ChainSpecBuilderCmd::Runtime(cmd) => generate_chain_spec_for_runtime(&cmd), + ChainSpecBuilderCmd::Edit(EditCmd { + ref input_chain_spec, + ref runtime_wasm_path, + convert_to_raw, + }) => { + let mut chain_spec = GenericChainSpec::<()>::from_json_file(input_chain_spec.clone())?; + runtime_wasm_path.clone().and_then(|path| { + chain_spec + .set_code(&fs::read(path.as_path()).expect("wasm blob file is readable")[..]) + .into() + }); + + chain_spec.as_json(convert_to_raw) + }, + ChainSpecBuilderCmd::Verify(VerifyCmd { ref input_chain_spec, ref runtime_wasm_path }) => { + write_chain_spec = false; + let mut chain_spec = GenericChainSpec::<()>::from_json_file(input_chain_spec.clone())?; + runtime_wasm_path.clone().and_then(|path| { + chain_spec + .set_code(&fs::read(path.as_path()).expect("wasm blob file is readable")[..]) + .into() + }); + chain_spec.as_json(true) + }, + }?; - fs::write(chain_spec_path, json).map_err(|err| err.to_string()) + if write_chain_spec { + fs::write(chain_spec_path, chain_spec_json).map_err(|err| err.to_string()) + } else { + Ok(()) + } } From e64b7d1d1f7e8d8d7561ad90981e88433d2049fd Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Wed, 24 May 2023 14:36:53 +0200 Subject: [PATCH 11/47] bin/node: GenesisBuilder implementation added - json test added (wip) --- bin/node/executor/Cargo.toml | 7 ++++--- bin/node/executor/tests/basic.rs | 16 ++++++++++++++++ bin/node/runtime/Cargo.toml | 26 +++++++++++++++----------- bin/node/runtime/src/lib.rs | 14 ++++++++++++++ 4 files changed, 49 insertions(+), 14 deletions(-) diff --git a/bin/node/executor/Cargo.toml b/bin/node/executor/Cargo.toml index b24414482952c..55b4c42754fb9 100644 --- a/bin/node/executor/Cargo.toml +++ b/bin/node/executor/Cargo.toml @@ -13,18 +13,19 @@ publish = false targets = ["x86_64-unknown-linux-gnu"] [dependencies] +log = { version = "0.4.17", default-features = false } codec = { package = "parity-scale-codec", version = "3.6.1" } -scale-info = { version = "2.5.0", features = ["derive"] } +scale-info = { version = "2.5.0", features = ["derive", "serde"] } frame-benchmarking = { version = "4.0.0-dev", path = "../../../frame/benchmarking" } node-primitives = { version = "2.0.0", path = "../primitives" } kitchensink-runtime = { version = "3.0.0-dev", path = "../runtime" } sc-executor = { version = "0.10.0-dev", path = "../../../client/executor" } -sp-core = { version = "21.0.0", path = "../../../primitives/core" } +sp-core = { version = "21.0.0", path = "../../../primitives/core", features=["serde"] } sp-keystore = { version = "0.27.0", path = "../../../primitives/keystore" } sp-state-machine = { version = "0.28.0", path = "../../../primitives/state-machine" } +sp-statement-store = { version = "4.0.0-dev", path = "../../../primitives/statement-store", features=["serde"] } sp-tracing = { version = "10.0.0", path = "../../../primitives/tracing" } sp-trie = { version = "22.0.0", path = "../../../primitives/trie" } -sp-statement-store = { version = "4.0.0-dev", path = "../../../primitives/statement-store" } [dev-dependencies] criterion = "0.4.0" diff --git a/bin/node/executor/tests/basic.rs b/bin/node/executor/tests/basic.rs index d301aa06f90b0..71d2e8df7a88e 100644 --- a/bin/node/executor/tests/basic.rs +++ b/bin/node/executor/tests/basic.rs @@ -857,3 +857,19 @@ fn should_import_block_with_test_client() { futures::executor::block_on(client.import(BlockOrigin::Own, block)).unwrap(); } + +#[test] +fn default_config_as_json_works() { + sp_tracing::try_init_simple(); + // let mut t = BasicExternalities::new_empty(); + let mut t = new_test_ext(compact_code_unwrap()); + let r = executor_call(&mut t, "GenesisBuilder_default_genesis_config_as_json", &vec![], false) + .0 + .unwrap(); + let r = Vec::::decode(&mut &r[..]).unwrap(); + let json = String::from_utf8(r.into()).expect("returned value is json. qed."); + + let expected = r#"{"system":{"code":"0x"},"babe":{"authorities":[],"epochConfig":null},"indices":{"indices":[]},"balances":{"balances":[]},"transactionPayment":{"multiplier":"1000000000000000000"},"staking":{"validatorCount":0,"minimumValidatorCount":0,"invulnerables":[],"forceEra":"NotForcing","slashRewardFraction":0,"canceledPayout":0,"stakers":[],"minNominatorBond":0,"minValidatorBond":0,"maxValidatorCount":null,"maxNominatorCount":null},"session":{"keys":[]},"democracy":{"phantom":null},"council":{"phantom":null,"members":[]},"technicalCommittee":{"phantom":null,"members":[]},"elections":{"members":[]},"technicalMembership":{"members":[],"phantom":null},"grandpa":{"authorities":[]},"treasury":null,"sudo":{"key":null},"imOnline":{"keys":[]},"authorityDiscovery":{"keys":[]},"society":{"pot":0,"members":[],"maxMembers":0},"vesting":{"vesting":[]},"glutton":{"compute":0,"storage":0,"trashDataCount":0},"assets":{"assets":[],"metadata":[],"accounts":[]},"poolAssets":{"assets":[],"metadata":[],"accounts":[]},"transactionStorage":{"byteFee":10,"entryFee":1000,"storagePeriod":100800},"allianceMotion":{"phantom":null,"members":[]},"alliance":{"fellows":[],"allies":[],"phantom":null},"nominationPools":{"minJoinBond":0,"minCreateBond":0,"maxPools":16,"maxMembersPerPool":32,"maxMembers":512,"globalMaxCommission":null}}"#; + log::info!("---> json: {:#?}", json); + assert_eq!(expected.to_string(), json); +} diff --git a/bin/node/runtime/Cargo.toml b/bin/node/runtime/Cargo.toml index 30f0052c6eaa6..b0d4d62783cbc 100644 --- a/bin/node/runtime/Cargo.toml +++ b/bin/node/runtime/Cargo.toml @@ -20,7 +20,7 @@ codec = { package = "parity-scale-codec", version = "3.6.1", default-features = "derive", "max-encoded-len", ] } -scale-info = { version = "2.5.0", default-features = false, features = ["derive"] } +scale-info = { version = "2.5.0", default-features = false, features = ["derive", "serde"] } static_assertions = "1.1.0" log = { version = "0.4.17", default-features = false } @@ -28,23 +28,25 @@ log = { version = "0.4.17", default-features = false } primitive-types = { version = "0.12.0", default-features = false, features = ["codec", "scale-info", "num-traits"] } # primitives -sp-authority-discovery = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/authority-discovery" } -sp-consensus-babe = { version = "0.10.0-dev", default-features = false, path = "../../../primitives/consensus/babe" } -sp-consensus-grandpa = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/consensus/grandpa" } +sp-authority-discovery = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/authority-discovery", features=["serde"] } +sp-consensus-babe = { version = "0.10.0-dev", default-features = false, path = "../../../primitives/consensus/babe", features=["serde"] } +sp-consensus-grandpa = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/consensus/grandpa", features=["serde"] } sp-block-builder = { path = "../../../primitives/block-builder", default-features = false, version = "4.0.0-dev" } +sp-genesis-builder = { version = "0.1.0-dev", default-features = false, path = "../../../primitives/genesis-builder" } +sc-genesis-builder = { version = "0.1.0-dev", default-features = false, path = "../../../client/genesis-builder" } sp-inherents = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/inherents" } node-primitives = { version = "2.0.0", default-features = false, path = "../primitives" } sp-offchain = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/offchain" } -sp-core = { version = "21.0.0", default-features = false, path = "../../../primitives/core" } -sp-std = { version = "8.0.0", default-features = false, path = "../../../primitives/std" } sp-api = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/api" } -sp-runtime = { version = "24.0.0", default-features = false, path = "../../../primitives/runtime" } -sp-staking = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/staking" } +sp-core = { version = "21.0.0", default-features = false, path = "../../../primitives/core", features=["serde"] } +sp-io = { version = "23.0.0", default-features = false, path = "../../../primitives/io" } +sp-runtime = { version = "24.0.0", default-features = false, path = "../../../primitives/runtime", features=["serde"] } sp-session = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/session" } +sp-staking = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/staking", features=["serde"] } +sp-statement-store = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/statement-store", features=["serde"] } +sp-std = { version = "8.0.0", default-features = false, path = "../../../primitives/std" } sp-transaction-pool = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/transaction-pool" } -sp-statement-store = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/statement-store" } -sp-version = { version = "22.0.0", default-features = false, path = "../../../primitives/version" } -sp-io = { version = "23.0.0", default-features = false, path = "../../../primitives/io" } +sp-version = { version = "22.0.0", default-features = false, path = "../../../primitives/version", features=["serde"] } # frame dependencies frame-executive = { version = "4.0.0-dev", default-features = false, path = "../../../frame/executive" } @@ -127,6 +129,7 @@ pallet-transaction-storage = { version = "4.0.0-dev", default-features = false, pallet-uniques = { version = "4.0.0-dev", default-features = false, path = "../../../frame/uniques" } pallet-vesting = { version = "4.0.0-dev", default-features = false, path = "../../../frame/vesting" } pallet-whitelist = { version = "4.0.0-dev", default-features = false, path = "../../../frame/whitelist" } +serde_json = { version = "1.0.85", default-features = false, features = ["alloc"] } [build-dependencies] substrate-wasm-builder = { version = "5.0.0-dev", path = "../../../utils/wasm-builder", optional = true } @@ -235,6 +238,7 @@ std = [ "pallet-child-bounties/std", "pallet-alliance/std", "substrate-wasm-builder", + "sp-genesis-builder/std", ] runtime-benchmarks = [ "frame-benchmarking/runtime-benchmarks", diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 058f43dad00b1..16ee6d36ed363 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -68,6 +68,8 @@ use sp_api::impl_runtime_apis; use sp_authority_discovery::AuthorityId as AuthorityDiscoveryId; use sp_consensus_grandpa::AuthorityId as GrandpaId; use sp_core::{crypto::KeyTypeId, OpaqueMetadata}; +// #[cfg(feature = "genesis-builder")] +use sc_genesis_builder::GenesisBuilderHelper; use sp_inherents::{CheckInherentsResult, InherentData}; use sp_runtime::{ create_runtime_str, @@ -2075,6 +2077,8 @@ mod benches { ); } +use frame_support::traits::GenesisBuild; + impl_runtime_apis! { impl sp_api::Core for Runtime { fn version() -> RuntimeVersion { @@ -2597,6 +2601,16 @@ impl_runtime_apis! { Ok(batches) } } + + impl sp_genesis_builder::GenesisBuilder for Runtime { + fn get_default_as_json() -> Vec { + GenesisBuilderHelper::::get_default_as_json() + } + + fn build_config(patch_json: Vec) { + GenesisBuilderHelper::::build_config(patch_json); + } + } } #[cfg(test)] From 5d450a4629d2eae867e9c81faf5d7c8f73782806 Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Thu, 25 May 2023 17:15:12 +0200 Subject: [PATCH 12/47] bin/node-template: serde enabled in runtime deps --- bin/node-template/runtime/Cargo.toml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/bin/node-template/runtime/Cargo.toml b/bin/node-template/runtime/Cargo.toml index 52b40989b2f93..8cdf34b95262f 100644 --- a/bin/node-template/runtime/Cargo.toml +++ b/bin/node-template/runtime/Cargo.toml @@ -14,8 +14,7 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] codec = { package = "parity-scale-codec", version = "3.6.1", default-features = false, features = ["derive"] } -scale-info = { version = "2.5.0", default-features = false, features = ["derive"] } - +scale-info = { version = "2.5.0", default-features = false, features = ["derive", "serde"] } pallet-aura = { version = "4.0.0-dev", default-features = false, path = "../../../frame/aura" } pallet-balances = { version = "4.0.0-dev", default-features = false, path = "../../../frame/balances" } frame-support = { version = "4.0.0-dev", default-features = false, path = "../../../frame/support" } @@ -28,16 +27,16 @@ pallet-transaction-payment = { version = "4.0.0-dev", default-features = false, frame-executive = { version = "4.0.0-dev", default-features = false, path = "../../../frame/executive" } sp-api = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/api" } sp-block-builder = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/block-builder"} -sp-consensus-aura = { version = "0.10.0-dev", default-features = false, path = "../../../primitives/consensus/aura" } -sp-consensus-grandpa = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/consensus/grandpa" } -sp-core = { version = "21.0.0", default-features = false, path = "../../../primitives/core" } +sp-consensus-aura = { version = "0.10.0-dev", default-features = false, path = "../../../primitives/consensus/aura", features = ["serde"] } +sp-consensus-grandpa = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/consensus/grandpa", features = ["serde"] } +sp-core = { version = "21.0.0", default-features = false, path = "../../../primitives/core", features=["serde"] } sp-inherents = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/inherents"} sp-offchain = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/offchain" } -sp-runtime = { version = "24.0.0", default-features = false, path = "../../../primitives/runtime" } +sp-runtime = { version = "24.0.0", default-features = false, path = "../../../primitives/runtime", features = ["serde"] } sp-session = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/session" } sp-std = { version = "8.0.0", default-features = false, path = "../../../primitives/std" } sp-transaction-pool = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/transaction-pool" } -sp-version = { version = "22.0.0", default-features = false, path = "../../../primitives/version" } +sp-version = { version = "22.0.0", default-features = false, path = "../../../primitives/version", features=["serde"] } # Used for the node template's RPCs frame-system-rpc-runtime-api = { version = "4.0.0-dev", default-features = false, path = "../../../frame/system/rpc/runtime-api/" } From b945d4da58f17c828ef870e818ef9b3eba1d0f31 Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Tue, 11 Jul 2023 12:20:56 +0200 Subject: [PATCH 13/47] bin/node/runtime: arbitrary_precision feature added to serde_json --- bin/node/runtime/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/node/runtime/Cargo.toml b/bin/node/runtime/Cargo.toml index b0d4d62783cbc..13ee6db6bed10 100644 --- a/bin/node/runtime/Cargo.toml +++ b/bin/node/runtime/Cargo.toml @@ -129,7 +129,7 @@ pallet-transaction-storage = { version = "4.0.0-dev", default-features = false, pallet-uniques = { version = "4.0.0-dev", default-features = false, path = "../../../frame/uniques" } pallet-vesting = { version = "4.0.0-dev", default-features = false, path = "../../../frame/vesting" } pallet-whitelist = { version = "4.0.0-dev", default-features = false, path = "../../../frame/whitelist" } -serde_json = { version = "1.0.85", default-features = false, features = ["alloc"] } +serde_json = { version = "1.0.85", default-features = false, features = ["alloc","arbitrary_precision"] } [build-dependencies] substrate-wasm-builder = { version = "5.0.0-dev", path = "../../../utils/wasm-builder", optional = true } From ae9415faf0893a6a0b74fdf58faea95cbd83d6d2 Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Tue, 20 Jun 2023 15:51:41 +0200 Subject: [PATCH 14/47] bin/node: fixes --- bin/node/executor/tests/basic.rs | 4 ++-- bin/node/runtime/Cargo.toml | 1 - bin/node/runtime/src/lib.rs | 13 ++++++------- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/bin/node/executor/tests/basic.rs b/bin/node/executor/tests/basic.rs index 71d2e8df7a88e..17afa8f6f1b9c 100644 --- a/bin/node/executor/tests/basic.rs +++ b/bin/node/executor/tests/basic.rs @@ -863,13 +863,13 @@ fn default_config_as_json_works() { sp_tracing::try_init_simple(); // let mut t = BasicExternalities::new_empty(); let mut t = new_test_ext(compact_code_unwrap()); - let r = executor_call(&mut t, "GenesisBuilder_default_genesis_config_as_json", &vec![], false) + let r = executor_call(&mut t, "GenesisBuilder_get_default_as_json", &vec![], false) .0 .unwrap(); let r = Vec::::decode(&mut &r[..]).unwrap(); let json = String::from_utf8(r.into()).expect("returned value is json. qed."); - let expected = r#"{"system":{"code":"0x"},"babe":{"authorities":[],"epochConfig":null},"indices":{"indices":[]},"balances":{"balances":[]},"transactionPayment":{"multiplier":"1000000000000000000"},"staking":{"validatorCount":0,"minimumValidatorCount":0,"invulnerables":[],"forceEra":"NotForcing","slashRewardFraction":0,"canceledPayout":0,"stakers":[],"minNominatorBond":0,"minValidatorBond":0,"maxValidatorCount":null,"maxNominatorCount":null},"session":{"keys":[]},"democracy":{"phantom":null},"council":{"phantom":null,"members":[]},"technicalCommittee":{"phantom":null,"members":[]},"elections":{"members":[]},"technicalMembership":{"members":[],"phantom":null},"grandpa":{"authorities":[]},"treasury":null,"sudo":{"key":null},"imOnline":{"keys":[]},"authorityDiscovery":{"keys":[]},"society":{"pot":0,"members":[],"maxMembers":0},"vesting":{"vesting":[]},"glutton":{"compute":0,"storage":0,"trashDataCount":0},"assets":{"assets":[],"metadata":[],"accounts":[]},"poolAssets":{"assets":[],"metadata":[],"accounts":[]},"transactionStorage":{"byteFee":10,"entryFee":1000,"storagePeriod":100800},"allianceMotion":{"phantom":null,"members":[]},"alliance":{"fellows":[],"allies":[],"phantom":null},"nominationPools":{"minJoinBond":0,"minCreateBond":0,"maxPools":16,"maxMembersPerPool":32,"maxMembers":512,"globalMaxCommission":null}}"#; + let expected = r#"{"system":{"code":"0x"},"babe":{"authorities":[],"epochConfig":null},"indices":{"indices":[]},"balances":{"balances":[]},"transactionPayment":{"multiplier":"1000000000000000000"},"staking":{"validatorCount":0,"minimumValidatorCount":0,"invulnerables":[],"forceEra":"NotForcing","slashRewardFraction":0,"canceledPayout":0,"stakers":[],"minNominatorBond":0,"minValidatorBond":0,"maxValidatorCount":null,"maxNominatorCount":null},"session":{"keys":[]},"democracy":{},"council":{"members":[]},"technicalCommittee":{"members":[]},"elections":{"members":[]},"technicalMembership":{"members":[]},"grandpa":{"authorities":[]},"treasury":{},"sudo":{"key":null},"imOnline":{"keys":[]},"authorityDiscovery":{"keys":[]},"society":{"pot":0},"vesting":{"vesting":[]},"glutton":{"compute":"0","storage":"0","trashDataCount":0},"assets":{"assets":[],"metadata":[],"accounts":[]},"poolAssets":{"assets":[],"metadata":[],"accounts":[]},"transactionStorage":{"byteFee":10,"entryFee":1000,"storagePeriod":100800},"allianceMotion":{"members":[]},"alliance":{"fellows":[],"allies":[]},"nominationPools":{"minJoinBond":0,"minCreateBond":0,"maxPools":16,"maxMembersPerPool":32,"maxMembers":512,"globalMaxCommission":null}}"#; log::info!("---> json: {:#?}", json); assert_eq!(expected.to_string(), json); } diff --git a/bin/node/runtime/Cargo.toml b/bin/node/runtime/Cargo.toml index 13ee6db6bed10..066bb595ce7f3 100644 --- a/bin/node/runtime/Cargo.toml +++ b/bin/node/runtime/Cargo.toml @@ -33,7 +33,6 @@ sp-consensus-babe = { version = "0.10.0-dev", default-features = false, path = " sp-consensus-grandpa = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/consensus/grandpa", features=["serde"] } sp-block-builder = { path = "../../../primitives/block-builder", default-features = false, version = "4.0.0-dev" } sp-genesis-builder = { version = "0.1.0-dev", default-features = false, path = "../../../primitives/genesis-builder" } -sc-genesis-builder = { version = "0.1.0-dev", default-features = false, path = "../../../client/genesis-builder" } sp-inherents = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/inherents" } node-primitives = { version = "2.0.0", default-features = false, path = "../primitives" } sp-offchain = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/offchain" } diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 16ee6d36ed363..df86c5c761bd6 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -33,6 +33,7 @@ use frame_support::{ ord_parameter_types, pallet_prelude::Get, parameter_types, + tests::RuntimeGenesisConfig, traits::{ fungible::ItemOf, tokens::{nonfungibles_v2::Inspect, GetSalary, PayFromAccount}, @@ -69,7 +70,7 @@ use sp_authority_discovery::AuthorityId as AuthorityDiscoveryId; use sp_consensus_grandpa::AuthorityId as GrandpaId; use sp_core::{crypto::KeyTypeId, OpaqueMetadata}; // #[cfg(feature = "genesis-builder")] -use sc_genesis_builder::GenesisBuilderHelper; +use frame_support::genesis_builder_helper::GenesisBuilderHelper; use sp_inherents::{CheckInherentsResult, InherentData}; use sp_runtime::{ create_runtime_str, @@ -2077,8 +2078,6 @@ mod benches { ); } -use frame_support::traits::GenesisBuild; - impl_runtime_apis! { impl sp_api::Core for Runtime { fn version() -> RuntimeVersion { @@ -2603,12 +2602,12 @@ impl_runtime_apis! { } impl sp_genesis_builder::GenesisBuilder for Runtime { - fn get_default_as_json() -> Vec { - GenesisBuilderHelper::::get_default_as_json() + fn create_default_config() -> Vec { + GenesisBuilderHelper::::create_default_config() } - fn build_config(patch_json: Vec) { - GenesisBuilderHelper::::build_config(patch_json); + fn build_config(config: Vec) -> sp_genesis_builder::Result { + GenesisBuilderHelper::::build_config(config) } } } From 98a9a58a7cee153d1e3d8051a32f0fd63195ff57 Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Thu, 6 Jul 2023 14:37:40 +0200 Subject: [PATCH 15/47] bin/node: from_genesis moved to ChainSpecBuilder - ChainSpecBuilder used - tests using RuntimeGenesisConfig added to verify patching approach --- bin/node/cli/Cargo.toml | 5 +- bin/node/cli/src/chain_spec.rs | 414 ++++++++++++++++++++++++++------- 2 files changed, 332 insertions(+), 87 deletions(-) diff --git a/bin/node/cli/Cargo.toml b/bin/node/cli/Cargo.toml index 032ba271f3c81..c3546ffda9b0e 100644 --- a/bin/node/cli/Cargo.toml +++ b/bin/node/cli/Cargo.toml @@ -35,10 +35,11 @@ crate-type = ["cdylib", "rlib"] [dependencies] # third-party dependencies -array-bytes = "6.1" +array-bytes = { version = "6.1" } clap = { version = "4.2.5", features = ["derive"], optional = true } codec = { package = "parity-scale-codec", version = "3.6.1" } serde = { version = "1.0.163", features = ["derive"] } +serde_json = { version = "1.0.85", features = ["arbitrary_precision"] } jsonrpsee = { version = "0.16.2", features = ["server"] } futures = "0.3.21" log = "0.4.17" @@ -104,7 +105,6 @@ sc-cli = { version = "0.10.0-dev", optional = true, path = "../../../client/cli" frame-benchmarking-cli = { version = "4.0.0-dev", optional = true, path = "../../../utils/frame/benchmarking-cli" } node-inspect = { version = "0.9.0-dev", optional = true, path = "../inspect" } try-runtime-cli = { version = "0.10.0-dev", optional = true, path = "../../../utils/frame/try-runtime/cli" } -serde_json = "1.0.85" [dev-dependencies] sc-keystore = { version = "4.0.0-dev", path = "../../../client/keystore" } @@ -120,7 +120,6 @@ futures = "0.3.21" tempfile = "3.1.0" assert_cmd = "2.0.2" nix = { version = "0.26.1", features = ["signal"] } -serde_json = "1.0" regex = "1.6.0" platforms = "3.0" soketto = "0.7.1" diff --git a/bin/node/cli/src/chain_spec.rs b/bin/node/cli/src/chain_spec.rs index a56c4e7b4016e..f383591e87ece 100644 --- a/bin/node/cli/src/chain_spec.rs +++ b/bin/node/cli/src/chain_spec.rs @@ -20,10 +20,9 @@ use grandpa_primitives::AuthorityId as GrandpaId; use kitchensink_runtime::{ - constants::currency::*, wasm_binary_unwrap, BabeConfig, BalancesConfig, Block, CouncilConfig, - DemocracyConfig, ElectionsConfig, ImOnlineConfig, IndicesConfig, MaxNominations, - NominationPoolsConfig, SessionConfig, SessionKeys, SocietyConfig, StakerStatus, StakingConfig, - SudoConfig, SystemConfig, TechnicalCommitteeConfig, + constants::currency::*, wasm_binary_unwrap, AssetsConfig, BabeConfig, BalancesConfig, Block, + ElectionsConfig, ImOnlineConfig, MaxNominations, NominationPoolsConfig, SessionConfig, + SessionKeys, SocietyConfig, StakerStatus, StakingConfig, SudoConfig, TechnicalCommitteeConfig, }; use pallet_im_online::sr25519::AuthorityId as ImOnlineId; use sc_chain_spec::ChainSpecExtension; @@ -44,6 +43,8 @@ pub use node_primitives::{AccountId, Balance, Signature}; type AccountPublic = ::Signer; const STAGING_TELEMETRY_URL: &str = "wss://telemetry.polkadot.io/submit/"; +const ENDOWMENT: Balance = 10_000_000 * DOLLARS; +const STASH: Balance = ENDOWMENT / 1000; /// Node `ChainSpec` extensions. /// @@ -76,7 +77,11 @@ fn session_keys( SessionKeys { grandpa, babe, im_online, authority_discovery } } -fn staging_testnet_config_genesis() -> RuntimeGenesisConfig { +fn configure_accounts_for_staging_testnet() -> ( + Vec<(AccountId, AccountId, GrandpaId, BabeId, ImOnlineId, AuthorityDiscoveryId)>, + AccountId, + Vec, +) { #[rustfmt::skip] // stash, controller, session-key // generated with secret: @@ -175,28 +180,31 @@ fn staging_testnet_config_genesis() -> RuntimeGenesisConfig { ); let endowed_accounts: Vec = vec![root_key.clone()]; + (initial_authorities, root_key, endowed_accounts) +} +fn staging_testnet_config_genesis() -> serde_json::Value { + let (initial_authorities, root_key, endowed_accounts) = + configure_accounts_for_staging_testnet(); testnet_genesis(initial_authorities, vec![], root_key, Some(endowed_accounts)) } /// Staging testnet config. pub fn staging_testnet_config() -> ChainSpec { let boot_nodes = vec![]; - ChainSpec::from_genesis( - "Staging Testnet", - "staging_testnet", - ChainType::Live, - staging_testnet_config_genesis, - boot_nodes, - Some( + ChainSpec::builder() + .with_name("Staging Testnet") + .with_id("staging_testnet") + .with_chain_type(ChainType::Live) + .with_genesis_patch(staging_testnet_config_genesis()) + .with_boot_nodes(boot_nodes) + .with_telemetry_endpoints( TelemetryEndpoints::new(vec![(STAGING_TELEMETRY_URL.to_string(), 0)]) .expect("Staging telemetry url is valid; qed"), - ), - None, - None, - None, - Default::default(), - ) + ) + .with_extensions(Default::default()) + .with_code(wasm_binary_unwrap()) + .build() } /// Helper function to generate a crypto pair from seed @@ -228,8 +236,7 @@ pub fn authority_keys_from_seed( ) } -/// Helper function to create RuntimeGenesisConfig for testing -pub fn testnet_genesis( +fn configure_accounts( initial_authorities: Vec<( AccountId, AccountId, @@ -239,9 +246,14 @@ pub fn testnet_genesis( AuthorityDiscoveryId, )>, initial_nominators: Vec, - root_key: AccountId, endowed_accounts: Option>, -) -> RuntimeGenesisConfig { + stash: Balance, +) -> ( + Vec<(AccountId, AccountId, GrandpaId, BabeId, ImOnlineId, AuthorityDiscoveryId)>, + Vec, + usize, + Vec<(AccountId, AccountId, Balance, StakerStatus)>, +) { let mut endowed_accounts: Vec = endowed_accounts.unwrap_or_else(|| { vec![ get_account_id_from_seed::("Alice"), @@ -273,7 +285,7 @@ pub fn testnet_genesis( let mut rng = rand::thread_rng(); let stakers = initial_authorities .iter() - .map(|x| (x.0.clone(), x.0.clone(), STASH, StakerStatus::Validator)) + .map(|x| (x.0.clone(), x.0.clone(), stash, StakerStatus::Validator)) .chain(initial_nominators.iter().map(|x| { use rand::{seq::SliceRandom, Rng}; let limit = (MaxNominations::get() as usize).min(initial_authorities.len()); @@ -284,22 +296,37 @@ pub fn testnet_genesis( .into_iter() .map(|choice| choice.0.clone()) .collect::>(); - (x.clone(), x.clone(), STASH, StakerStatus::Nominator(nominations)) + (x.clone(), x.clone(), stash, StakerStatus::Nominator(nominations)) })) .collect::>(); let num_endowed_accounts = endowed_accounts.len(); - const ENDOWMENT: Balance = 10_000_000 * DOLLARS; - const STASH: Balance = ENDOWMENT / 1000; + (initial_authorities, endowed_accounts, num_endowed_accounts, stakers) +} + +/// Helper function to create RuntimeGenesisConfig json patch for testing +pub fn testnet_genesis( + initial_authorities: Vec<( + AccountId, + AccountId, + GrandpaId, + BabeId, + ImOnlineId, + AuthorityDiscoveryId, + )>, + initial_nominators: Vec, + root_key: AccountId, + endowed_accounts: Option>, +) -> serde_json::Value { + let (initial_authorities, endowed_accounts, num_endowed_accounts, stakers) = + configure_accounts(initial_authorities, initial_nominators, endowed_accounts, STASH); - RuntimeGenesisConfig { - system: SystemConfig { code: wasm_binary_unwrap().to_vec(), ..Default::default() }, - balances: BalancesConfig { + let json = serde_json::json!({ + "balances": BalancesConfig { balances: endowed_accounts.iter().cloned().map(|x| (x, ENDOWMENT)).collect(), }, - indices: IndicesConfig { indices: vec![] }, - session: SessionConfig { + "session": SessionConfig { keys: initial_authorities .iter() .map(|x| { @@ -311,16 +338,15 @@ pub fn testnet_genesis( }) .collect::>(), }, - staking: StakingConfig { + "staking": StakingConfig { validator_count: initial_authorities.len() as u32, minimum_validator_count: initial_authorities.len() as u32, invulnerables: initial_authorities.iter().map(|x| x.0.clone()).collect(), slash_reward_fraction: Perbill::from_percent(10), - stakers, + stakers: stakers.clone(), ..Default::default() }, - democracy: DemocracyConfig::default(), - elections: ElectionsConfig { + "elections": ElectionsConfig { members: endowed_accounts .iter() .take((num_endowed_accounts + 1) / 2) @@ -328,47 +354,37 @@ pub fn testnet_genesis( .map(|member| (member, STASH)) .collect(), }, - council: CouncilConfig::default(), - technical_committee: TechnicalCommitteeConfig { + "technicalCommittee": TechnicalCommitteeConfig { members: endowed_accounts .iter() .take((num_endowed_accounts + 1) / 2) .cloned() .collect(), - phantom: Default::default(), + ..Default::default() }, - sudo: SudoConfig { key: Some(root_key) }, - babe: BabeConfig { + "sudo": SudoConfig { key: Some(root_key.clone()) }, + "babe": BabeConfig { epoch_config: Some(kitchensink_runtime::BABE_GENESIS_EPOCH_CONFIG), ..Default::default() }, - im_online: ImOnlineConfig { keys: vec![] }, - authority_discovery: Default::default(), - grandpa: Default::default(), - technical_membership: Default::default(), - treasury: Default::default(), - society: SocietyConfig { pot: 0 }, - vesting: Default::default(), - assets: pallet_assets::GenesisConfig { + "imOnline": ImOnlineConfig { keys: vec![] }, + "society": SocietyConfig { pot: 0 }, + "assets": AssetsConfig { // This asset is used by the NIS pallet as counterpart currency. assets: vec![(9, get_account_id_from_seed::("Alice"), true, 1)], ..Default::default() }, - pool_assets: Default::default(), - transaction_storage: Default::default(), - transaction_payment: Default::default(), - alliance: Default::default(), - alliance_motion: Default::default(), - nomination_pools: NominationPoolsConfig { + "nominationPools": NominationPoolsConfig { min_create_bond: 10 * DOLLARS, min_join_bond: 1 * DOLLARS, ..Default::default() }, - glutton: Default::default(), - } + }); + log::info!("json -> {}", json); + json } -fn development_config_genesis() -> RuntimeGenesisConfig { +fn development_config_genesis_json() -> serde_json::Value { testnet_genesis( vec![authority_keys_from_seed("Alice")], vec![], @@ -379,21 +395,18 @@ fn development_config_genesis() -> RuntimeGenesisConfig { /// Development config (single validator Alice) pub fn development_config() -> ChainSpec { - ChainSpec::from_genesis( - "Development", - "dev", - ChainType::Development, - development_config_genesis, - vec![], - None, - None, - None, - None, - Default::default(), - ) + ChainSpec::builder() + .with_name("Development") + .with_id("dev") + .with_chain_type(ChainType::Development) + .with_genesis_patch(development_config_genesis_json()) + .with_boot_nodes(vec![]) + .with_extensions(Default::default()) + .with_code(wasm_binary_unwrap()) + .build() } -fn local_testnet_genesis() -> RuntimeGenesisConfig { +fn local_testnet_genesis() -> serde_json::Value { testnet_genesis( vec![authority_keys_from_seed("Alice"), authority_keys_from_seed("Bob")], vec![], @@ -404,18 +417,15 @@ fn local_testnet_genesis() -> RuntimeGenesisConfig { /// Local testnet config (multivalidator Alice + Bob) pub fn local_testnet_config() -> ChainSpec { - ChainSpec::from_genesis( - "Local Testnet", - "local_testnet", - ChainType::Local, - local_testnet_genesis, - vec![], - None, - None, - None, - None, - Default::default(), - ) + ChainSpec::builder() + .with_name("Local Testnet") + .with_id("local_testnet") + .with_chain_type(ChainType::Local) + .with_genesis_patch(local_testnet_genesis()) + .with_boot_nodes(vec![]) + .with_extensions(Default::default()) + .with_code(wasm_binary_unwrap()) + .build() } #[cfg(test)] @@ -426,7 +436,7 @@ pub(crate) mod tests { use sp_runtime::BuildStorage; fn local_testnet_genesis_instant_single() -> RuntimeGenesisConfig { - testnet_genesis( + json_vs_legacy::testnet_genesis_legacy( vec![authority_keys_from_seed("Alice")], vec![], get_account_id_from_seed::("Alice"), @@ -447,6 +457,7 @@ pub(crate) mod tests { None, None, Default::default(), + wasm_binary_unwrap(), ) } @@ -456,13 +467,14 @@ pub(crate) mod tests { "Integration Test", "test", ChainType::Development, - local_testnet_genesis, + json_vs_legacy::local_testnet_genesis_legacy, vec![], None, None, None, None, Default::default(), + wasm_binary_unwrap(), ) } @@ -498,4 +510,238 @@ pub(crate) mod tests { fn test_staging_test_net_chain_spec() { staging_testnet_config().build_storage().unwrap(); } + + mod json_vs_legacy { + use crate::chain_spec::*; + use kitchensink_runtime::{CouncilConfig, DemocracyConfig, IndicesConfig}; + use sp_runtime::BuildStorage; + /// Helper function to create RuntimeGenesisConfig for testing + pub(super) fn testnet_genesis_legacy( + initial_authorities: Vec<( + AccountId, + AccountId, + GrandpaId, + BabeId, + ImOnlineId, + AuthorityDiscoveryId, + )>, + initial_nominators: Vec, + root_key: AccountId, + endowed_accounts: Option>, + ) -> RuntimeGenesisConfig { + let (initial_authorities, endowed_accounts, num_endowed_accounts, stakers) = + configure_accounts( + initial_authorities, + initial_nominators, + endowed_accounts, + STASH, + ); + + RuntimeGenesisConfig { + system: Default::default(), + balances: BalancesConfig { + balances: endowed_accounts.iter().cloned().map(|x| (x, ENDOWMENT)).collect(), + }, + indices: IndicesConfig { indices: vec![] }, + session: SessionConfig { + keys: initial_authorities + .iter() + .map(|x| { + ( + x.0.clone(), + x.0.clone(), + session_keys(x.2.clone(), x.3.clone(), x.4.clone(), x.5.clone()), + ) + }) + .collect::>(), + }, + staking: StakingConfig { + validator_count: initial_authorities.len() as u32, + minimum_validator_count: initial_authorities.len() as u32, + invulnerables: initial_authorities.iter().map(|x| x.0.clone()).collect(), + slash_reward_fraction: Perbill::from_percent(10), + stakers, + ..Default::default() + }, + democracy: DemocracyConfig::default(), + elections: ElectionsConfig { + members: endowed_accounts + .iter() + .take((num_endowed_accounts + 1) / 2) + .cloned() + .map(|member| (member, STASH)) + .collect(), + }, + council: CouncilConfig::default(), + technical_committee: TechnicalCommitteeConfig { + members: endowed_accounts + .iter() + .take((num_endowed_accounts + 1) / 2) + .cloned() + .collect(), + phantom: Default::default(), + }, + sudo: SudoConfig { key: Some(root_key) }, + babe: BabeConfig { + epoch_config: Some(kitchensink_runtime::BABE_GENESIS_EPOCH_CONFIG), + ..Default::default() + }, + im_online: ImOnlineConfig { keys: vec![] }, + authority_discovery: Default::default(), + grandpa: Default::default(), + technical_membership: Default::default(), + treasury: Default::default(), + society: SocietyConfig { pot: 0 }, + vesting: Default::default(), + assets: pallet_assets::GenesisConfig { + // This asset is used by the NIS pallet as counterpart currency. + assets: vec![( + 9, + get_account_id_from_seed::("Alice"), + true, + 1, + )], + ..Default::default() + }, + pool_assets: Default::default(), + transaction_storage: Default::default(), + transaction_payment: Default::default(), + alliance: Default::default(), + alliance_motion: Default::default(), + nomination_pools: NominationPoolsConfig { + min_create_bond: 10 * DOLLARS, + min_join_bond: 1 * DOLLARS, + ..Default::default() + }, + glutton: Default::default(), + } + } + + fn development_config_genesis_legacy() -> RuntimeGenesisConfig { + testnet_genesis_legacy( + vec![authority_keys_from_seed("Alice")], + vec![], + get_account_id_from_seed::("Alice"), + None, + ) + } + + /// Development config (single validator Alice). Legacy building method, for testing. + fn development_config_legacy() -> ChainSpec { + #[allow(deprecated)] + ChainSpec::from_genesis( + "Development", + "dev", + ChainType::Development, + development_config_genesis_legacy, + vec![], + None, + None, + None, + None, + Default::default(), + wasm_binary_unwrap(), + ) + } + + use std::io::prelude::*; + #[test] + fn development_config_building_test() { + sp_tracing::try_init_simple(); + let j1 = development_config().as_json(true).unwrap(); + let j2 = development_config_legacy().as_json(true).unwrap(); + std::fs::write("/tmp/j1.json", j1.clone()).unwrap(); + std::fs::write("/tmp/j2.json", j2.clone()).unwrap(); + assert_eq!(j1, j2); + } + + pub(super) fn local_testnet_genesis_legacy() -> RuntimeGenesisConfig { + testnet_genesis_legacy( + vec![authority_keys_from_seed("Alice"), authority_keys_from_seed("Bob")], + vec![], + get_account_id_from_seed::("Alice"), + None, + ) + } + /// Local testnet config (multivalidator Alice + Bob). Legacy building method, for testing. + fn local_testnet_config_legacy() -> ChainSpec { + #[allow(deprecated)] + ChainSpec::from_genesis( + "Local Testnet", + "local_testnet", + ChainType::Local, + local_testnet_genesis_legacy, + vec![], + None, + None, + None, + None, + Default::default(), + wasm_binary_unwrap(), + ) + } + + #[test] + fn local_testnet_config_building_test() { + sp_tracing::try_init_simple(); + let j1 = local_testnet_config().as_json(true).unwrap(); + let j2 = local_testnet_config_legacy().as_json(true).unwrap(); + std::fs::write("/tmp/i1.json", j1.clone()).unwrap(); + std::fs::write("/tmp/i2.json", j2.clone()).unwrap(); + assert_eq!(j1, j2); + } + + fn staging_testnet_config_genesis_legacy() -> RuntimeGenesisConfig { + let (initial_authorities, root_key, endowed_accounts) = + configure_accounts_for_staging_testnet(); + testnet_genesis_legacy(initial_authorities, vec![], root_key, Some(endowed_accounts)) + } + + /// Staging testnet config. Legacy building method, for testing. + fn staging_testnet_config_legacy() -> ChainSpec { + let boot_nodes = vec![]; + #[allow(deprecated)] + ChainSpec::from_genesis( + "Staging Testnet", + "staging_testnet", + ChainType::Live, + staging_testnet_config_genesis_legacy, + boot_nodes, + Some( + TelemetryEndpoints::new(vec![(STAGING_TELEMETRY_URL.to_string(), 0)]) + .expect("Staging telemetry url is valid; qed"), + ), + None, + None, + None, + Default::default(), + wasm_binary_unwrap(), + ) + } + + fn hex(x: T) -> String + where + T: array_bytes::Hex, + { + x.hex(Default::default()) + } + + #[test] + fn staging_testnet_config_building_test() { + sp_tracing::try_init_simple(); + + let chain_spec = staging_testnet_config(); + std::fs::write("/tmp/k1.json", chain_spec.as_json(false).unwrap()).unwrap(); + let storage = chain_spec.build_storage().unwrap(); + let mut keys = storage.top.keys().cloned().map(hex).collect::>(); + keys.sort(); + + let chain_spec = staging_testnet_config_legacy(); + std::fs::write("/tmp/k2.json", chain_spec.as_json(false).unwrap()).unwrap(); + let storage = chain_spec.build_storage().unwrap(); + let mut keys_expected = storage.top.keys().cloned().map(hex).collect::>(); + keys_expected.sort(); + assert_eq!(keys, keys_expected); + } + } } From f49362e77f6c84422ad32db2c9e8b5bc7ae0962e Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Thu, 6 Jul 2023 16:19:10 +0200 Subject: [PATCH 16/47] bin/node: boot_nodes call optional --- bin/node/cli/src/chain_spec.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/bin/node/cli/src/chain_spec.rs b/bin/node/cli/src/chain_spec.rs index f383591e87ece..eb57699369022 100644 --- a/bin/node/cli/src/chain_spec.rs +++ b/bin/node/cli/src/chain_spec.rs @@ -191,13 +191,11 @@ fn staging_testnet_config_genesis() -> serde_json::Value { /// Staging testnet config. pub fn staging_testnet_config() -> ChainSpec { - let boot_nodes = vec![]; ChainSpec::builder() .with_name("Staging Testnet") .with_id("staging_testnet") .with_chain_type(ChainType::Live) .with_genesis_patch(staging_testnet_config_genesis()) - .with_boot_nodes(boot_nodes) .with_telemetry_endpoints( TelemetryEndpoints::new(vec![(STAGING_TELEMETRY_URL.to_string(), 0)]) .expect("Staging telemetry url is valid; qed"), @@ -400,7 +398,6 @@ pub fn development_config() -> ChainSpec { .with_id("dev") .with_chain_type(ChainType::Development) .with_genesis_patch(development_config_genesis_json()) - .with_boot_nodes(vec![]) .with_extensions(Default::default()) .with_code(wasm_binary_unwrap()) .build() @@ -422,7 +419,6 @@ pub fn local_testnet_config() -> ChainSpec { .with_id("local_testnet") .with_chain_type(ChainType::Local) .with_genesis_patch(local_testnet_genesis()) - .with_boot_nodes(vec![]) .with_extensions(Default::default()) .with_code(wasm_binary_unwrap()) .build() From f62284d7fa3d09999b92614854b79485600f839c Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Thu, 6 Jul 2023 12:39:56 +0200 Subject: [PATCH 17/47] bin/node: GenesisBuilder helper struct removed --- bin/node/runtime/src/lib.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index df86c5c761bd6..98ab5caa64578 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -33,7 +33,6 @@ use frame_support::{ ord_parameter_types, pallet_prelude::Get, parameter_types, - tests::RuntimeGenesisConfig, traits::{ fungible::ItemOf, tokens::{nonfungibles_v2::Inspect, GetSalary, PayFromAccount}, @@ -70,7 +69,7 @@ use sp_authority_discovery::AuthorityId as AuthorityDiscoveryId; use sp_consensus_grandpa::AuthorityId as GrandpaId; use sp_core::{crypto::KeyTypeId, OpaqueMetadata}; // #[cfg(feature = "genesis-builder")] -use frame_support::genesis_builder_helper::GenesisBuilderHelper; +use frame_support::genesis_builder_helper::{build_config, create_default_config}; use sp_inherents::{CheckInherentsResult, InherentData}; use sp_runtime::{ create_runtime_str, @@ -2603,11 +2602,11 @@ impl_runtime_apis! { impl sp_genesis_builder::GenesisBuilder for Runtime { fn create_default_config() -> Vec { - GenesisBuilderHelper::::create_default_config() + create_default_config::() } fn build_config(config: Vec) -> sp_genesis_builder::Result { - GenesisBuilderHelper::::build_config(config) + build_config::(config) } } } From 12dac2cf16b351fba36682fd834c5f8ad0390310 Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Thu, 6 Jul 2023 14:04:53 +0200 Subject: [PATCH 18/47] bin/node: frame_system::code removed --- bin/node/executor/benches/bench.rs | 2 +- bin/node/executor/tests/common.rs | 2 +- bin/node/testing/src/bench.rs | 5 +---- bin/node/testing/src/client.rs | 2 +- bin/node/testing/src/genesis.rs | 11 ++++------- 5 files changed, 8 insertions(+), 14 deletions(-) diff --git a/bin/node/executor/benches/bench.rs b/bin/node/executor/benches/bench.rs index 1c9c002492cf5..75cd041ffed07 100644 --- a/bin/node/executor/benches/bench.rs +++ b/bin/node/executor/benches/bench.rs @@ -189,7 +189,7 @@ fn bench_execute_block(c: &mut Criterion) { for strategy in execution_methods { group.bench_function(format!("{:?}", strategy), |b| { - let genesis_config = node_testing::genesis::config(Some(compact_code_unwrap())); + let genesis_config = node_testing::genesis::config(); let use_native = match strategy { ExecutionMethod::Native => true, ExecutionMethod::Wasm(..) => false, diff --git a/bin/node/executor/tests/common.rs b/bin/node/executor/tests/common.rs index 6ce9ea3a01090..c487f16f2634b 100644 --- a/bin/node/executor/tests/common.rs +++ b/bin/node/executor/tests/common.rs @@ -123,7 +123,7 @@ pub fn executor_call( pub fn new_test_ext(code: &[u8]) -> TestExternalities { let ext = TestExternalities::new_with_code( code, - node_testing::genesis::config(Some(code)).build_storage().unwrap(), + node_testing::genesis::config().build_storage().unwrap(), ); ext } diff --git a/bin/node/testing/src/bench.rs b/bin/node/testing/src/bench.rs index f1ab2212239b1..bbaf1877f6032 100644 --- a/bin/node/testing/src/bench.rs +++ b/bin/node/testing/src/bench.rs @@ -587,10 +587,7 @@ impl BenchKeyring { /// Generate genesis with accounts from this keyring endowed with some balance. pub fn generate_genesis(&self) -> kitchensink_runtime::RuntimeGenesisConfig { - crate::genesis::config_endowed( - Some(kitchensink_runtime::wasm_binary_unwrap()), - self.collect_account_ids(), - ) + crate::genesis::config_endowed(self.collect_account_ids()) } } diff --git a/bin/node/testing/src/client.rs b/bin/node/testing/src/client.rs index 01495525a9c8c..38ddb00dfe32f 100644 --- a/bin/node/testing/src/client.rs +++ b/bin/node/testing/src/client.rs @@ -45,7 +45,7 @@ pub struct GenesisParameters; impl substrate_test_client::GenesisInit for GenesisParameters { fn genesis_storage(&self) -> Storage { - crate::genesis::config(None).build_storage().unwrap() + crate::genesis::config().build_storage().unwrap() } } diff --git a/bin/node/testing/src/genesis.rs b/bin/node/testing/src/genesis.rs index 8063b8ef45708..55a503071e4ba 100644 --- a/bin/node/testing/src/genesis.rs +++ b/bin/node/testing/src/genesis.rs @@ -29,13 +29,13 @@ use sp_keyring::{Ed25519Keyring, Sr25519Keyring}; use sp_runtime::Perbill; /// Create genesis runtime configuration for tests. -pub fn config(code: Option<&[u8]>) -> RuntimeGenesisConfig { - config_endowed(code, Default::default()) +pub fn config() -> RuntimeGenesisConfig { + config_endowed(Default::default()) } /// Create genesis runtime configuration for tests with some extra /// endowed accounts. -pub fn config_endowed(code: Option<&[u8]>, extra_endowed: Vec) -> RuntimeGenesisConfig { +pub fn config_endowed(extra_endowed: Vec) -> RuntimeGenesisConfig { let mut endowed = vec![ (alice(), 111 * DOLLARS), (bob(), 100 * DOLLARS), @@ -48,10 +48,7 @@ pub fn config_endowed(code: Option<&[u8]>, extra_endowed: Vec) -> Run endowed.extend(extra_endowed.into_iter().map(|endowed| (endowed, 100 * DOLLARS))); RuntimeGenesisConfig { - system: SystemConfig { - code: code.map(|x| x.to_vec()).unwrap_or_else(|| wasm_binary_unwrap().to_vec()), - ..Default::default() - }, + system: Default::default(), indices: IndicesConfig { indices: vec![] }, balances: BalancesConfig { balances: endowed }, session: SessionConfig { From ab4ae6bcc5ff7890c88dfccf5df14881ecb86dbc Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Thu, 6 Jul 2023 14:22:04 +0200 Subject: [PATCH 19/47] node-template: frame_system::code removed --- bin/node-template/node/src/chain_spec.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/bin/node-template/node/src/chain_spec.rs b/bin/node-template/node/src/chain_spec.rs index 2cd2d07293026..a13375347fe9d 100644 --- a/bin/node-template/node/src/chain_spec.rs +++ b/bin/node-template/node/src/chain_spec.rs @@ -38,7 +38,6 @@ pub fn authority_keys_from_seed(s: &str) -> (AuraId, GrandpaId) { pub fn development_config() -> Result { let wasm_binary = WASM_BINARY.ok_or_else(|| "Development wasm not available".to_string())?; - Ok(ChainSpec::from_genesis( // Name "Development", @@ -47,7 +46,6 @@ pub fn development_config() -> Result { ChainType::Development, move || { testnet_genesis( - wasm_binary, // Initial PoA authorities vec![authority_keys_from_seed("Alice")], // Sudo account @@ -73,12 +71,12 @@ pub fn development_config() -> Result { None, // Extensions None, + wasm_binary, )) } pub fn local_testnet_config() -> Result { let wasm_binary = WASM_BINARY.ok_or_else(|| "Development wasm not available".to_string())?; - Ok(ChainSpec::from_genesis( // Name "Local Testnet", @@ -87,7 +85,6 @@ pub fn local_testnet_config() -> Result { ChainType::Local, move || { testnet_genesis( - wasm_binary, // Initial PoA authorities vec![authority_keys_from_seed("Alice"), authority_keys_from_seed("Bob")], // Sudo account @@ -121,12 +118,12 @@ pub fn local_testnet_config() -> Result { None, // Extensions None, + wasm_binary, )) } /// Configure initial storage state for FRAME modules. fn testnet_genesis( - wasm_binary: &[u8], initial_authorities: Vec<(AuraId, GrandpaId)>, root_key: AccountId, endowed_accounts: Vec, @@ -135,7 +132,6 @@ fn testnet_genesis( RuntimeGenesisConfig { system: SystemConfig { // Add Wasm runtime to storage. - code: wasm_binary.to_vec(), ..Default::default() }, balances: BalancesConfig { From fc55a8d650a5e16250a6d992f8a039c4714968b8 Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Thu, 6 Jul 2023 14:19:51 +0200 Subject: [PATCH 20/47] client/cli: from_genesis - code arg added --- client/cli/src/commands/insert_key.rs | 1 + client/cli/src/runner.rs | 1 + 2 files changed, 2 insertions(+) diff --git a/client/cli/src/commands/insert_key.rs b/client/cli/src/commands/insert_key.rs index 732d874319a83..7556475742481 100644 --- a/client/cli/src/commands/insert_key.rs +++ b/client/cli/src/commands/insert_key.rs @@ -137,6 +137,7 @@ mod tests { None, None, NoExtension::None, + Default::default(), ))) } } diff --git a/client/cli/src/runner.rs b/client/cli/src/runner.rs index c96f494354f9a..5b46d102ebb17 100644 --- a/client/cli/src/runner.rs +++ b/client/cli/src/runner.rs @@ -262,6 +262,7 @@ mod tests { None, None, NoExtension::None, + Default::default(), )), wasm_method: Default::default(), wasm_runtime_overrides: None, From dbe0c8b484be7561d357ccf6dbd420c60f9caa8f Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Mon, 17 Jul 2023 11:19:16 +0200 Subject: [PATCH 21/47] chain-spec: naming fixed --- bin/node/cli/src/chain_spec.rs | 6 +- bin/utils/chain-spec-builder/src/main.rs | 8 +-- .../substrate_test_runtime_from_config.json | 2 +- .../substrate_test_runtime_from_patch.json | 2 +- client/chain-spec/src/chain_spec.rs | 57 ++++++++++--------- .../chain-spec/src/genesis_config_builder.rs | 21 ++++++- 6 files changed, 60 insertions(+), 36 deletions(-) diff --git a/bin/node/cli/src/chain_spec.rs b/bin/node/cli/src/chain_spec.rs index eb57699369022..11d41971fa8da 100644 --- a/bin/node/cli/src/chain_spec.rs +++ b/bin/node/cli/src/chain_spec.rs @@ -195,7 +195,7 @@ pub fn staging_testnet_config() -> ChainSpec { .with_name("Staging Testnet") .with_id("staging_testnet") .with_chain_type(ChainType::Live) - .with_genesis_patch(staging_testnet_config_genesis()) + .with_genesis_config_patch(staging_testnet_config_genesis()) .with_telemetry_endpoints( TelemetryEndpoints::new(vec![(STAGING_TELEMETRY_URL.to_string(), 0)]) .expect("Staging telemetry url is valid; qed"), @@ -397,7 +397,7 @@ pub fn development_config() -> ChainSpec { .with_name("Development") .with_id("dev") .with_chain_type(ChainType::Development) - .with_genesis_patch(development_config_genesis_json()) + .with_genesis_config_patch(development_config_genesis_json()) .with_extensions(Default::default()) .with_code(wasm_binary_unwrap()) .build() @@ -418,7 +418,7 @@ pub fn local_testnet_config() -> ChainSpec { .with_name("Local Testnet") .with_id("local_testnet") .with_chain_type(ChainType::Local) - .with_genesis_patch(local_testnet_genesis()) + .with_genesis_config_patch(local_testnet_genesis()) .with_extensions(Default::default()) .with_code(wasm_binary_unwrap()) .build() diff --git a/bin/utils/chain-spec-builder/src/main.rs b/bin/utils/chain-spec-builder/src/main.rs index 6cf1e637833a1..095d46bd4bf44 100644 --- a/bin/utils/chain-spec-builder/src/main.rs +++ b/bin/utils/chain-spec-builder/src/main.rs @@ -218,7 +218,7 @@ fn generate_chain_spec( .with_name("Custom") .with_id("custom") .with_chain_type(sc_chain_spec::ChainType::Live) - .with_genesis_patch(chain_spec::testnet_genesis( + .with_genesis_config_patch(chain_spec::testnet_genesis( authorities, nominator_accounts, sudo_account, @@ -311,14 +311,14 @@ fn generate_chain_spec_for_runtime(cmd: &RuntimeCmd) -> Result { GenesisBuildAction::Patch(PatchCmd { ref patch_path }) => { let patch = fs::read(patch_path.as_path()) .map_err(|e| format!("patch file {patch_path:?} shall be readable: {e}"))?; - builder.with_genesis_patch(serde_json::from_slice::(&patch[..]).map_err( + builder.with_genesis_config_patch(serde_json::from_slice::(&patch[..]).map_err( |e| format!("patch file {patch_path:?} shall contain a valid json: {e}"), )?) }, GenesisBuildAction::Full(FullCmd { ref config_path }) => { let config = fs::read(config_path.as_path()) .map_err(|e| format!("config file {config_path:?} shall be readable: {e}"))?; - builder.with_no_genesis_defaults(serde_json::from_slice::(&config[..]).map_err( + builder.with_genesis_config(serde_json::from_slice::(&config[..]).map_err( |e| format!("config file {config_path:?} shall contain a valid json: {e}"), )?) }, @@ -331,7 +331,7 @@ fn generate_chain_spec_for_runtime(cmd: &RuntimeCmd) -> Result { fs::write(path.as_path(), serde_json::to_string_pretty(&default_config).unwrap()) .map_err(|err| err.to_string()) }); - builder.with_no_genesis_defaults(default_config) + builder.with_genesis_config(default_config) }, }; diff --git a/client/chain-spec/res/substrate_test_runtime_from_config.json b/client/chain-spec/res/substrate_test_runtime_from_config.json index 81f950796f20a..ae627cadf4326 100644 --- a/client/chain-spec/res/substrate_test_runtime_from_config.json +++ b/client/chain-spec/res/substrate_test_runtime_from_config.json @@ -8,7 +8,7 @@ "properties": null, "codeSubstitutes": {}, "genesis": { - "jsonFull": { + "runtimeConfig": { "babe": { "authorities": [ [ diff --git a/client/chain-spec/res/substrate_test_runtime_from_patch.json b/client/chain-spec/res/substrate_test_runtime_from_patch.json index f1ad312f2e6b0..c797fdda20799 100644 --- a/client/chain-spec/res/substrate_test_runtime_from_patch.json +++ b/client/chain-spec/res/substrate_test_runtime_from_patch.json @@ -8,7 +8,7 @@ "properties": null, "codeSubstitutes": {}, "genesis": { - "jsonPatch": { + "runtimeConfigPatch": { "babe": { "epochConfig": { "allowed_slots": "PrimaryAndSecondaryPlainSlots", diff --git a/client/chain-spec/src/chain_spec.rs b/client/chain-spec/src/chain_spec.rs index 6005c331e4189..43646a4412c7e 100644 --- a/client/chain-spec/src/chain_spec.rs +++ b/client/chain-spec/src/chain_spec.rs @@ -45,13 +45,15 @@ enum GenesisBuildAction { Full(serde_json::Value), } +#[allow(deprecated)] enum GenesisSource { File(PathBuf), Binary(Cow<'static, [u8]>), - #[deprecated(note = "todo: add note")] + #[deprecated( + note = "Factory is planned to be removed in December 2023. Use `GenesisBuilderApi` instead." + )] Factory(Arc G + Send + Sync>), Storage(Storage), - //maybe a factory for providing json of GenesisBuildAction?? GenesisBuilderApi(GenesisBuildAction), } @@ -60,6 +62,7 @@ impl Clone for GenesisSource { match *self { Self::File(ref path) => Self::File(path.clone()), Self::Binary(ref d) => Self::Binary(d.clone()), + #[allow(deprecated)] Self::Factory(ref f) => Self::Factory(f.clone()), Self::Storage(ref s) => Self::Storage(s.clone()), Self::GenesisBuilderApi(ref s) => Self::GenesisBuilderApi(s.clone()), @@ -97,6 +100,7 @@ impl GenesisSource { .map_err(|e| format!("Error parsing embedded file: {}", e))?; Ok(genesis.genesis) //GenesisRuntime (typically) }, + #[allow(deprecated)] Self::Factory(f) => Ok(Genesis::Runtime(f())), Self::Storage(storage) => { let top = storage @@ -124,9 +128,9 @@ impl GenesisSource { }, // maybe a Factory for getting GenesisBuilderApi command? Self::GenesisBuilderApi(GenesisBuildAction::Full(config)) => - Ok(Genesis::JsonFull(config.clone())), + Ok(Genesis::RuntimeConfig(config.clone())), Self::GenesisBuilderApi(GenesisBuildAction::Patch(patch)) => - Ok(Genesis::JsonPatch(patch.clone())), + Ok(Genesis::RuntimeConfigPatch(patch.clone())), } } } @@ -138,6 +142,7 @@ impl BuildStorage for ChainSpec { .insert(sp_core::storage::well_known_keys::CODE.to_vec(), self.code.clone()); match self.genesis.resolve()? { + #[allow(deprecated)] Genesis::Runtime(gc) => gc.assimilate_storage(storage), Genesis::Raw(RawGenesis { top: map, children_default: children_map }) => { storage.top.extend(map.into_iter().map(|(k, v)| (k.0, v.0))); @@ -156,10 +161,10 @@ impl BuildStorage for ChainSpec { // it, but Substrate itself isn't capable of loading chain specs with just a hash at the // moment. Genesis::StateRootHash(_) => Err("Genesis storage in hash format not supported".into()), - Genesis::JsonFull(config) => RuntimeCaller::new(&self.code[..]) + Genesis::RuntimeConfig(config) => RuntimeCaller::new(&self.code[..]) .get_storage_for_config(config)? .assimilate_storage(storage), - Genesis::JsonPatch(patch) => RuntimeCaller::new(&self.code[..]) + Genesis::RuntimeConfigPatch(patch) => RuntimeCaller::new(&self.code[..]) .get_storage_for_patch(patch)? .assimilate_storage(storage), }?; @@ -205,19 +210,15 @@ impl From for RawGenesis { #[serde(rename_all = "camelCase")] #[serde(deny_unknown_fields)] enum Genesis { - //deprecated - // #[serde(alias = "RuntimeDeprecated")] - #[deprecated(note = "todo: add note")] - Runtime(G), //G is RuntimeGenesisConfig + /// note: this will be removed together with [`ChainSpec::from_genesis`] + Runtime(G), Raw(RawGenesis), /// State root hash of the genesis storage. StateRootHash(StorageData), - - // CodeWithPatch(Option), (PatchDefault?) - #[serde(alias = "Runtime")] - JsonFull(serde_json::Value), - #[serde(alias = "Patch")] - JsonPatch(serde_json::Value), + /// Full runtime genesis config. + RuntimeConfig(serde_json::Value), + /// Patch for default runtime genesis config. + RuntimeConfigPatch(serde_json::Value), } /// A configuration of a client. Does not include runtime storage initialization. @@ -347,12 +348,12 @@ impl ChainSpecBuilder { self } - pub fn with_genesis_patch(mut self, patch: serde_json::Value) -> Self { + pub fn with_genesis_config_patch(mut self, patch: serde_json::Value) -> Self { self.genesis_build_action = Some(GenesisBuildAction::Patch(patch)); self } - pub fn with_no_genesis_defaults(mut self, config: serde_json::Value) -> Self { + pub fn with_genesis_config(mut self, config: serde_json::Value) -> Self { self.genesis_build_action = Some(GenesisBuildAction::Full(config)); self } @@ -456,7 +457,10 @@ impl ChainSpec { } /// Create hardcoded spec. - #[deprecated(note = "todo: add note")] + #[deprecated( + note = "`from_genesis` is planned to be removed in December 2023. Use [`builder()`] instead." + )] + // deprecated note: Genesis::Runtime + GenesisSource::Factory shall also be removed pub fn from_genesis G + 'static + Send + Sync>( name: &str, id: &str, @@ -486,9 +490,9 @@ impl ChainSpec { code_substitutes: BTreeMap::new(), }; + #[allow(deprecated)] ChainSpec { client_spec, - #[allow(deprecated)] genesis: GenesisSource::Factory(Arc::new(constructor)), code: code.into(), } @@ -566,12 +570,13 @@ struct ChainSpecJsonContainer { impl ChainSpec { fn json_container(&self, raw: bool) -> Result, String> { let genesis = match (raw, self.genesis.resolve()?) { - (true, Genesis::JsonPatch(patch)) => Genesis::Raw(RawGenesis::from( + (true, Genesis::RuntimeConfigPatch(patch)) => Genesis::Raw(RawGenesis::from( RuntimeCaller::new(&self.code[..]).get_storage_for_patch(patch)?, )), - (true, Genesis::JsonFull(config)) => Genesis::Raw(RawGenesis::from( + (true, Genesis::RuntimeConfig(config)) => Genesis::Raw(RawGenesis::from( RuntimeCaller::new(&self.code[..]).get_storage_for_config(config)?, )), + #[allow(deprecated)] (true, Genesis::Runtime(g)) => { let storage = g.build_storage()?; Genesis::Raw(RawGenesis::from(storage)) @@ -785,7 +790,7 @@ mod tests { .with_extensions(Default::default()) .with_chain_type(ChainType::Local) .with_code(substrate_test_runtime::wasm_binary_unwrap().into()) - .with_genesis_patch(json!({ + .with_genesis_config_patch(json!({ "babe": { "epochConfig": { "c": [ @@ -835,7 +840,7 @@ mod tests { .with_extensions(Default::default()) .with_chain_type(ChainType::Local) .with_code(substrate_test_runtime::wasm_binary_unwrap().into()) - .with_no_genesis_defaults(serde_json::from_str(j).unwrap()) + .with_genesis_config(serde_json::from_str(j).unwrap()) .build(); // std::fs::write("/tmp/config.json", output.as_json(false).unwrap()); @@ -872,7 +877,7 @@ mod tests { .with_extensions(Default::default()) .with_chain_type(ChainType::Local) .with_code(substrate_test_runtime::wasm_binary_unwrap().into()) - .with_no_genesis_defaults(serde_json::from_str(j).unwrap()) + .with_genesis_config(serde_json::from_str(j).unwrap()) .build(); assert_eq!( output.as_json(true), Err("Invalid JSON blob: unknown field `babex`, expected one of `system`, `babe`, `substrateTest`, `balances` at line 1 column 8".to_string()) ); @@ -887,7 +892,7 @@ mod tests { .with_extensions(Default::default()) .with_chain_type(ChainType::Local) .with_code(substrate_test_runtime::wasm_binary_unwrap().into()) - .with_genesis_patch(json!({ + .with_genesis_config_patch(json!({ "invalid_pallet": {}, "substrateTest": { "authorities": [ diff --git a/client/chain-spec/src/genesis_config_builder.rs b/client/chain-spec/src/genesis_config_builder.rs index edfbbab17ec72..d132cfb74654e 100644 --- a/client/chain-spec/src/genesis_config_builder.rs +++ b/client/chain-spec/src/genesis_config_builder.rs @@ -1,4 +1,23 @@ -use std::{borrow::Cow, str::FromStr}; +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +///! Helper for calling GenesisBuilder API from arbitrary wasm blob. +use std::borrow::Cow; use codec::{Decode, Encode}; use sc_executor::{error::Result, WasmExecutor}; From 01f10b9af3770bc11ac96e23fcfbe7cb01604e3f Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Mon, 17 Jul 2023 14:29:03 +0200 Subject: [PATCH 22/47] node-template: ChainSpecBuilder used --- bin/node-template/node/Cargo.toml | 1 + bin/node-template/node/src/chain_spec.rs | 147 +++++++++-------------- bin/node-template/runtime/Cargo.toml | 1 + 3 files changed, 58 insertions(+), 91 deletions(-) diff --git a/bin/node-template/node/Cargo.toml b/bin/node-template/node/Cargo.toml index ea7989be454fd..689e0773c929e 100644 --- a/bin/node-template/node/Cargo.toml +++ b/bin/node-template/node/Cargo.toml @@ -42,6 +42,7 @@ sp-inherents = { version = "4.0.0-dev", path = "../../../primitives/inherents" } sp-keyring = { version = "24.0.0", path = "../../../primitives/keyring" } frame-system = { version = "4.0.0-dev", path = "../../../frame/system" } pallet-transaction-payment = { version = "4.0.0-dev", default-features = false, path = "../../../frame/transaction-payment" } +serde_json = "1.0.85" # These dependencies are used for the node template's RPCs jsonrpsee = { version = "0.16.2", features = ["server"] } diff --git a/bin/node-template/node/src/chain_spec.rs b/bin/node-template/node/src/chain_spec.rs index a13375347fe9d..c40ae9b197c1a 100644 --- a/bin/node-template/node/src/chain_spec.rs +++ b/bin/node-template/node/src/chain_spec.rs @@ -1,6 +1,6 @@ use node_template_runtime::{ AccountId, AuraConfig, BalancesConfig, GrandpaConfig, RuntimeGenesisConfig, Signature, - SudoConfig, SystemConfig, WASM_BINARY, + SudoConfig, WASM_BINARY, }; use sc_service::ChainType; use sp_consensus_aura::sr25519::AuthorityId as AuraId; @@ -37,89 +37,59 @@ pub fn authority_keys_from_seed(s: &str) -> (AuraId, GrandpaId) { } pub fn development_config() -> Result { - let wasm_binary = WASM_BINARY.ok_or_else(|| "Development wasm not available".to_string())?; - Ok(ChainSpec::from_genesis( - // Name - "Development", - // ID - "dev", - ChainType::Development, - move || { - testnet_genesis( - // Initial PoA authorities - vec![authority_keys_from_seed("Alice")], - // Sudo account + Ok(ChainSpec::builder() + .with_name("Development") + .with_id("dev") + .with_chain_type(ChainType::Development) + .with_extensions(None) + .with_code(WASM_BINARY.ok_or_else(|| "Development wasm not available".to_string())?) + .with_genesis_config_patch(testnet_genesis( + // Initial PoA authorities + vec![authority_keys_from_seed("Alice")], + // Sudo account + get_account_id_from_seed::("Alice"), + // Pre-funded accounts + vec![ get_account_id_from_seed::("Alice"), - // Pre-funded accounts - vec![ - get_account_id_from_seed::("Alice"), - get_account_id_from_seed::("Bob"), - get_account_id_from_seed::("Alice//stash"), - get_account_id_from_seed::("Bob//stash"), - ], - true, - ) - }, - // Bootnodes - vec![], - // Telemetry - None, - // Protocol ID - None, - None, - // Properties - None, - // Extensions - None, - wasm_binary, - )) + get_account_id_from_seed::("Bob"), + get_account_id_from_seed::("Alice//stash"), + get_account_id_from_seed::("Bob//stash"), + ], + true, + )) + .build()) } pub fn local_testnet_config() -> Result { - let wasm_binary = WASM_BINARY.ok_or_else(|| "Development wasm not available".to_string())?; - Ok(ChainSpec::from_genesis( - // Name - "Local Testnet", - // ID - "local_testnet", - ChainType::Local, - move || { - testnet_genesis( - // Initial PoA authorities - vec![authority_keys_from_seed("Alice"), authority_keys_from_seed("Bob")], - // Sudo account + Ok(ChainSpec::builder() + .with_name("Local Testnet") + .with_id("local_testnet") + .with_chain_type(ChainType::Local) + .with_extensions(None) + .with_code(WASM_BINARY.ok_or_else(|| "Development wasm not available".to_string())?) + .with_genesis_config_patch(testnet_genesis( + // Initial PoA authorities + vec![authority_keys_from_seed("Alice"), authority_keys_from_seed("Bob")], + // Sudo account + get_account_id_from_seed::("Alice"), + // Pre-funded accounts + vec![ get_account_id_from_seed::("Alice"), - // Pre-funded accounts - vec![ - get_account_id_from_seed::("Alice"), - get_account_id_from_seed::("Bob"), - get_account_id_from_seed::("Charlie"), - get_account_id_from_seed::("Dave"), - get_account_id_from_seed::("Eve"), - get_account_id_from_seed::("Ferdie"), - get_account_id_from_seed::("Alice//stash"), - get_account_id_from_seed::("Bob//stash"), - get_account_id_from_seed::("Charlie//stash"), - get_account_id_from_seed::("Dave//stash"), - get_account_id_from_seed::("Eve//stash"), - get_account_id_from_seed::("Ferdie//stash"), - ], - true, - ) - }, - // Bootnodes - vec![], - // Telemetry - None, - // Protocol ID - None, - // Properties - None, - None, - // Extensions - None, - wasm_binary, - )) + get_account_id_from_seed::("Bob"), + get_account_id_from_seed::("Charlie"), + get_account_id_from_seed::("Dave"), + get_account_id_from_seed::("Eve"), + get_account_id_from_seed::("Ferdie"), + get_account_id_from_seed::("Alice//stash"), + get_account_id_from_seed::("Bob//stash"), + get_account_id_from_seed::("Charlie//stash"), + get_account_id_from_seed::("Dave//stash"), + get_account_id_from_seed::("Eve//stash"), + get_account_id_from_seed::("Ferdie//stash"), + ], + true, + )) + .build()) } /// Configure initial storage state for FRAME modules. @@ -128,27 +98,22 @@ fn testnet_genesis( root_key: AccountId, endowed_accounts: Vec, _enable_println: bool, -) -> RuntimeGenesisConfig { - RuntimeGenesisConfig { - system: SystemConfig { - // Add Wasm runtime to storage. - ..Default::default() - }, - balances: BalancesConfig { +) -> serde_json::Value { + serde_json::json!({ + "balances": BalancesConfig { // Configure endowed accounts with initial balance of 1 << 60. balances: endowed_accounts.iter().cloned().map(|k| (k, 1 << 60)).collect(), }, - aura: AuraConfig { + "aura": AuraConfig { authorities: initial_authorities.iter().map(|x| (x.0.clone())).collect(), }, - grandpa: GrandpaConfig { + "grandpa": GrandpaConfig { authorities: initial_authorities.iter().map(|x| (x.1.clone(), 1)).collect(), ..Default::default() }, - sudo: SudoConfig { + "sudo": SudoConfig { // Assign network admin rights. key: Some(root_key), }, - transaction_payment: Default::default(), - } + }) } diff --git a/bin/node-template/runtime/Cargo.toml b/bin/node-template/runtime/Cargo.toml index 8cdf34b95262f..84f2966930528 100644 --- a/bin/node-template/runtime/Cargo.toml +++ b/bin/node-template/runtime/Cargo.toml @@ -37,6 +37,7 @@ sp-session = { version = "4.0.0-dev", default-features = false, path = "../../.. sp-std = { version = "8.0.0", default-features = false, path = "../../../primitives/std" } sp-transaction-pool = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/transaction-pool" } sp-version = { version = "22.0.0", default-features = false, path = "../../../primitives/version", features=["serde"] } +serde_json = { version = "1.0.85", default-features = false, features = ["alloc"] } # Used for the node template's RPCs frame-system-rpc-runtime-api = { version = "4.0.0-dev", default-features = false, path = "../../../frame/system/rpc/runtime-api/" } From 70155491a8fcaae73e12f8bb92e0fd7fc97db903 Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Mon, 17 Jul 2023 14:30:23 +0200 Subject: [PATCH 23/47] bin/node: ChainSpecBuilder used (tests) --- bin/node/cli/src/chain_spec.rs | 121 ++++++++++++++++++++++---------- bin/node/testing/src/genesis.rs | 7 +- 2 files changed, 88 insertions(+), 40 deletions(-) diff --git a/bin/node/cli/src/chain_spec.rs b/bin/node/cli/src/chain_spec.rs index 11d41971fa8da..9b01b2c8111e9 100644 --- a/bin/node/cli/src/chain_spec.rs +++ b/bin/node/cli/src/chain_spec.rs @@ -431,47 +431,33 @@ pub(crate) mod tests { use sc_service_test; use sp_runtime::BuildStorage; - fn local_testnet_genesis_instant_single() -> RuntimeGenesisConfig { - json_vs_legacy::testnet_genesis_legacy( - vec![authority_keys_from_seed("Alice")], - vec![], - get_account_id_from_seed::("Alice"), - None, - ) - } - /// Local testnet config (single validator - Alice) pub fn integration_test_config_with_single_authority() -> ChainSpec { - ChainSpec::from_genesis( - "Integration Test", - "test", - ChainType::Development, - local_testnet_genesis_instant_single, - vec![], - None, - None, - None, - None, - Default::default(), - wasm_binary_unwrap(), - ) + ChainSpec::builder() + .with_name("Integration Test") + .with_id("test") + .with_chain_type(ChainType::Development) + .with_genesis_config_patch(testnet_genesis( + vec![authority_keys_from_seed("Alice")], + vec![], + get_account_id_from_seed::("Alice"), + None, + )) + .with_extensions(Default::default()) + .with_code(wasm_binary_unwrap()) + .build() } /// Local testnet config (multivalidator Alice + Bob) pub fn integration_test_config_with_two_authorities() -> ChainSpec { - ChainSpec::from_genesis( - "Integration Test", - "test", - ChainType::Development, - json_vs_legacy::local_testnet_genesis_legacy, - vec![], - None, - None, - None, - None, - Default::default(), - wasm_binary_unwrap(), - ) + ChainSpec::builder() + .with_name("Integration Test") + .with_id("test") + .with_chain_type(ChainType::Development) + .with_genesis_config_patch(local_testnet_genesis()) + .with_extensions(Default::default()) + .with_code(wasm_binary_unwrap()) + .build() } #[test] @@ -507,6 +493,9 @@ pub(crate) mod tests { staging_testnet_config().build_storage().unwrap(); } + // This compares the output of RuntimeGenesisConfig-based ChainSpec building against JSON-based + // ChainSpec building. Once RuntimeGenesisConfig is removed from client-sode, entire mod can be + // removed. mod json_vs_legacy { use crate::chain_spec::*; use kitchensink_runtime::{CouncilConfig, DemocracyConfig, IndicesConfig}; @@ -640,7 +629,6 @@ pub(crate) mod tests { ) } - use std::io::prelude::*; #[test] fn development_config_building_test() { sp_tracing::try_init_simple(); @@ -739,5 +727,66 @@ pub(crate) mod tests { keys_expected.sort(); assert_eq!(keys, keys_expected); } + + fn local_testnet_genesis_instant_single_legacy() -> RuntimeGenesisConfig { + testnet_genesis_legacy( + vec![authority_keys_from_seed("Alice")], + vec![], + get_account_id_from_seed::("Alice"), + None, + ) + } + + /// Local testnet config (single validator - Alice) + pub(super) fn integration_test_config_with_single_authority_legacy() -> ChainSpec { + #[allow(deprecated)] + ChainSpec::from_genesis( + "Integration Test", + "test", + ChainType::Development, + local_testnet_genesis_instant_single_legacy, + vec![], + None, + None, + None, + None, + Default::default(), + wasm_binary_unwrap(), + ) + } + + #[test] + fn integration_test_config_with_single_authority_legacy_compare_test() { + sp_tracing::try_init_simple(); + let j1 = integration_test_config_with_single_authority_legacy().as_json(true).unwrap(); + let j2 = super::integration_test_config_with_single_authority().as_json(true).unwrap(); + assert_eq!(j1, j2); + } + + /// Local testnet config (multivalidator Alice + Bob) + pub fn integration_test_config_with_two_authorities_legacy() -> ChainSpec { + #[allow(deprecated)] + ChainSpec::from_genesis( + "Integration Test", + "test", + ChainType::Development, + local_testnet_genesis_legacy, + vec![], + None, + None, + None, + None, + Default::default(), + wasm_binary_unwrap(), + ) + } + + #[test] + fn integration_test_config_with_with_two_authorities_compare_test() { + sp_tracing::try_init_simple(); + let j1 = integration_test_config_with_two_authorities_legacy().as_json(true).unwrap(); + let j2 = super::integration_test_config_with_two_authorities().as_json(true).unwrap(); + assert_eq!(j1, j2); + } } } diff --git a/bin/node/testing/src/genesis.rs b/bin/node/testing/src/genesis.rs index 55a503071e4ba..92243e352863d 100644 --- a/bin/node/testing/src/genesis.rs +++ b/bin/node/testing/src/genesis.rs @@ -20,10 +20,9 @@ use crate::keyring::*; use kitchensink_runtime::{ - constants::currency::*, wasm_binary_unwrap, AccountId, AssetsConfig, BabeConfig, - BalancesConfig, GluttonConfig, GrandpaConfig, IndicesConfig, RuntimeGenesisConfig, - SessionConfig, SocietyConfig, StakerStatus, StakingConfig, SystemConfig, - BABE_GENESIS_EPOCH_CONFIG, + constants::currency::*, AccountId, AssetsConfig, BabeConfig, BalancesConfig, GluttonConfig, + GrandpaConfig, IndicesConfig, RuntimeGenesisConfig, SessionConfig, SocietyConfig, StakerStatus, + StakingConfig, BABE_GENESIS_EPOCH_CONFIG, }; use sp_keyring::{Ed25519Keyring, Sr25519Keyring}; use sp_runtime::Perbill; From 86ab3d37f1bf7db6af0e967ed05303904f33c180 Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Mon, 17 Jul 2023 14:50:13 +0200 Subject: [PATCH 24/47] cli/cmd/insert_key: ChainSpecBuilder used (tests) --- client/cli/src/commands/insert_key.rs | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/client/cli/src/commands/insert_key.rs b/client/cli/src/commands/insert_key.rs index 7556475742481..3617b52aa6a1a 100644 --- a/client/cli/src/commands/insert_key.rs +++ b/client/cli/src/commands/insert_key.rs @@ -126,19 +126,16 @@ mod tests { } fn load_spec(&self, _: &str) -> std::result::Result, String> { - Ok(Box::new(GenericChainSpec::from_genesis( - "test", - "test_id", - ChainType::Development, - || unimplemented!("Not required in tests"), - Vec::new(), - None, - None, - None, - None, - NoExtension::None, - Default::default(), - ))) + Ok(Box::new( + GenericChainSpec::<()>::builder() + .with_name("test") + .with_id("test_id") + .with_extensions(NoExtension::None) + .with_chain_type(ChainType::Development) + .with_code(Default::default()) + .with_genesis_config_patch(Default::default()) + .build(), + )) } } From 14185580e78cdede4402f0fba14b5c553db4aa10 Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Mon, 17 Jul 2023 14:52:16 +0200 Subject: [PATCH 25/47] cli/runner: ChainSpecBuilder used (tests) --- client/cli/src/runner.rs | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/client/cli/src/runner.rs b/client/cli/src/runner.rs index 5b46d102ebb17..be2b66d6df5fc 100644 --- a/client/cli/src/runner.rs +++ b/client/cli/src/runner.rs @@ -251,19 +251,16 @@ mod tests { trie_cache_maximum_size: None, state_pruning: None, blocks_pruning: sc_client_db::BlocksPruning::KeepAll, - chain_spec: Box::new(GenericChainSpec::from_genesis( - "test", - "test_id", - ChainType::Development, - || unimplemented!("Not required in tests"), - Vec::new(), - None, - None, - None, - None, - NoExtension::None, - Default::default(), - )), + chain_spec: Box::new( + GenericChainSpec::<()>::builder() + .with_name("test") + .with_id("test_id") + .with_extensions(NoExtension::None) + .with_chain_type(ChainType::Development) + .with_code(Default::default()) + .with_genesis_config_patch(Default::default()) + .build(), + ), wasm_method: Default::default(), wasm_runtime_overrides: None, rpc_addr: None, From 2faefc182d715f42ca25b5aff529a788f997a23a Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Mon, 17 Jul 2023 14:56:07 +0200 Subject: [PATCH 26/47] ChainSpec: note corrected --- client/chain-spec/src/chain_spec.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/chain-spec/src/chain_spec.rs b/client/chain-spec/src/chain_spec.rs index 43646a4412c7e..61d582228ac4d 100644 --- a/client/chain-spec/src/chain_spec.rs +++ b/client/chain-spec/src/chain_spec.rs @@ -50,7 +50,7 @@ enum GenesisSource { File(PathBuf), Binary(Cow<'static, [u8]>), #[deprecated( - note = "Factory is planned to be removed in December 2023. Use `GenesisBuilderApi` instead." + note = "Factory and G type parameter are planned to be removed in December 2023. Use `GenesisBuilderApi` instead." )] Factory(Arc G + Send + Sync>), Storage(Storage), From fc1c393591a029beea2d171c281325f95d33de9d Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Thu, 20 Jul 2023 14:17:58 +0200 Subject: [PATCH 27/47] chain-spec-builder: after-merge cleanup --- bin/utils/chain-spec-builder/bin/main.rs | 76 ++++-- bin/utils/chain-spec-builder/src/lib.rs | 314 ++++++++++++++++------- bin/utils/chain-spec-builder/src/main.rs | 298 --------------------- 3 files changed, 276 insertions(+), 412 deletions(-) delete mode 100644 bin/utils/chain-spec-builder/src/main.rs diff --git a/bin/utils/chain-spec-builder/bin/main.rs b/bin/utils/chain-spec-builder/bin/main.rs index 53e11abbf6282..24ad1dc1e925b 100644 --- a/bin/utils/chain-spec-builder/bin/main.rs +++ b/bin/utils/chain-spec-builder/bin/main.rs @@ -17,27 +17,39 @@ // along with this program. If not, see . use chain_spec_builder::{ - generate_authority_keys_and_store, generate_chain_spec, print_seeds, ChainSpecBuilder, + generate_authority_keys_and_store, generate_chain_spec, generate_chain_spec_for_runtime, + print_seeds, ChainSpecBuilder, ChainSpecBuilderCmd, EditCmd, GenerateCmd, NewCmd, VerifyCmd, }; use clap::Parser; use node_cli::chain_spec; use rand::{distributions::Alphanumeric, rngs::OsRng, Rng}; +use sc_chain_spec::GenericChainSpec; use sp_core::{crypto::Ss58Codec, sr25519}; use std::fs; fn main() -> Result<(), String> { + sp_tracing::try_init_simple(); + + let builder = ChainSpecBuilder::parse(); #[cfg(build_type = "debug")] - println!( - "The chain spec builder builds a chain specification that includes a Substrate runtime \ + if matches!(builder.command, ChainSpecBuilderCmd::Generate(_) | ChainSpecBuilderCmd::New(_)) { + println!( + "The chain spec builder builds a chain specification that includes a Substrate runtime \ compiled as WASM. To ensure proper functioning of the included runtime compile (or run) \ the chain spec builder binary in `--release` mode.\n", - ); - - let builder = ChainSpecBuilder::parse(); - let chain_spec_path = builder.chain_spec_path().to_path_buf(); - - let (authority_seeds, nominator_accounts, endowed_accounts, sudo_account) = match builder { - ChainSpecBuilder::Generate { authorities, nominators, endowed, keystore_path, .. } => { + ); + } + + let chain_spec_path = builder.chain_spec_path.to_path_buf(); + let mut write_chain_spec = true; + + let chain_spec_json = match builder.command { + ChainSpecBuilderCmd::Generate(GenerateCmd { + authorities, + nominators, + endowed, + keystore_path, + }) => { let authorities = authorities.max(1); let rand_str = || -> String { OsRng.sample_iter(&Alphanumeric).take(32).map(char::from).collect() @@ -71,19 +83,45 @@ fn main() -> Result<(), String> { let sudo_account = chain_spec::get_account_id_from_seed::(&sudo_seed).to_ss58check(); - (authority_seeds, nominator_accounts, endowed_accounts, sudo_account) + generate_chain_spec(authority_seeds, nominator_accounts, endowed_accounts, sudo_account) }, - ChainSpecBuilder::New { + ChainSpecBuilderCmd::New(NewCmd { authority_seeds, nominator_accounts, endowed_accounts, sudo_account, - .. - } => (authority_seeds, nominator_accounts, endowed_accounts, sudo_account), - }; - - let json = - generate_chain_spec(authority_seeds, nominator_accounts, endowed_accounts, sudo_account)?; + }) => + generate_chain_spec(authority_seeds, nominator_accounts, endowed_accounts, sudo_account), + ChainSpecBuilderCmd::Runtime(cmd) => generate_chain_spec_for_runtime(&cmd), + ChainSpecBuilderCmd::Edit(EditCmd { + ref input_chain_spec, + ref runtime_wasm_path, + convert_to_raw, + }) => { + let mut chain_spec = GenericChainSpec::<()>::from_json_file(input_chain_spec.clone())?; + runtime_wasm_path.clone().and_then(|path| { + chain_spec + .set_code(&fs::read(path.as_path()).expect("wasm blob file is readable")[..]) + .into() + }); + + chain_spec.as_json(convert_to_raw) + }, + ChainSpecBuilderCmd::Verify(VerifyCmd { ref input_chain_spec, ref runtime_wasm_path }) => { + write_chain_spec = false; + let mut chain_spec = GenericChainSpec::<()>::from_json_file(input_chain_spec.clone())?; + runtime_wasm_path.clone().and_then(|path| { + chain_spec + .set_code(&fs::read(path.as_path()).expect("wasm blob file is readable")[..]) + .into() + }); + chain_spec.as_json(true) + }, + }?; - fs::write(chain_spec_path, json).map_err(|err| err.to_string()) + if write_chain_spec { + fs::write(chain_spec_path, chain_spec_json).map_err(|err| err.to_string()) + } else { + Ok(()) + } } diff --git a/bin/utils/chain-spec-builder/src/lib.rs b/bin/utils/chain-spec-builder/src/lib.rs index fa3c6835c799e..27bb3ee0d74d1 100644 --- a/bin/utils/chain-spec-builder/src/lib.rs +++ b/bin/utils/chain-spec-builder/src/lib.rs @@ -22,96 +22,169 @@ //! //! See [`ChainSpecBuilder`] for a list of available commands. -use std::path::{Path, PathBuf}; +use std::{ + fs, + path::{Path, PathBuf}, +}; use ansi_term::Style; -use clap::Parser; +use clap::{Parser, Subcommand}; +use sc_chain_spec::GenesisConfigBuilderRuntimeCaller; use node_cli::chain_spec::{self, AccountId}; use sc_keystore::LocalKeystore; +use serde_json::Value; use sp_core::crypto::{ByteArray, Ss58Codec}; use sp_keystore::KeystorePtr; /// A utility to easily create a testnet chain spec definition with a given set /// of authorities and endowed accounts and/or generate random accounts. -#[derive(Parser)] +#[derive(Debug, Parser)] #[command(rename_all = "kebab-case")] -pub enum ChainSpecBuilder { - /// Create a new chain spec with the given authorities, endowed and sudo - /// accounts. - New { - /// Authority key seed. - #[arg(long, short, required = true)] - authority_seeds: Vec, - /// Active nominators (SS58 format), each backing a random subset of the aforementioned - /// authorities. - #[arg(long, short, default_value = "0")] - nominator_accounts: Vec, - /// Endowed account address (SS58 format). - #[arg(long, short)] - endowed_accounts: Vec, - /// Sudo account address (SS58 format). - #[arg(long, short)] - sudo_account: String, - /// The path where the chain spec should be saved. - #[arg(long, short, default_value = "./chain_spec.json")] - chain_spec_path: PathBuf, - }, - /// Create a new chain spec with the given number of authorities and endowed - /// accounts. Random keys will be generated as required. - Generate { - /// The number of authorities. - #[arg(long, short)] - authorities: usize, - /// The number of nominators backing the aforementioned authorities. - /// - /// Will nominate a random subset of `authorities`. - #[arg(long, short, default_value_t = 0)] - nominators: usize, - /// The number of endowed accounts. - #[arg(long, short, default_value_t = 0)] - endowed: usize, - /// The path where the chain spec should be saved. - #[arg(long, short, default_value = "./chain_spec.json")] - chain_spec_path: PathBuf, - /// Path to use when saving generated keystores for each authority. - /// - /// At this path, a new folder will be created for each authority's - /// keystore named `auth-$i` where `i` is the authority index, i.e. - /// `auth-0`, `auth-1`, etc. - #[arg(long, short)] - keystore_path: Option, - }, +pub struct ChainSpecBuilder { + #[command(subcommand)] + pub command: ChainSpecBuilderCmd, + /// The path where the chain spec should be saved. + #[arg(long, short, default_value = "./chain_spec.json")] + pub chain_spec_path: PathBuf, } -impl ChainSpecBuilder { - /// Returns the path where the chain spec should be saved. - pub fn chain_spec_path(&self) -> &Path { - match self { - ChainSpecBuilder::New { chain_spec_path, .. } => chain_spec_path.as_path(), - ChainSpecBuilder::Generate { chain_spec_path, .. } => chain_spec_path.as_path(), - } - } +#[derive(Debug, Subcommand)] +#[command(rename_all = "kebab-case")] +pub enum ChainSpecBuilderCmd { + New(NewCmd), + Generate(GenerateCmd), + Runtime(RuntimeCmd), + Edit(EditCmd), + Verify(VerifyCmd), } -fn genesis_constructor( - authority_seeds: &[String], - nominator_accounts: &[AccountId], - endowed_accounts: &[AccountId], - sudo_account: &AccountId, -) -> chain_spec::RuntimeGenesisConfig { - let authorities = authority_seeds - .iter() - .map(AsRef::as_ref) - .map(chain_spec::authority_keys_from_seed) - .collect::>(); +/// Create a new chain spec with the given authorities, endowed and sudo +/// accounts. Only works for kitchen-sink runtime +#[derive(Parser, Debug)] +#[command(rename_all = "kebab-case")] +pub struct NewCmd { + /// Authority key seed. + #[arg(long, short, required = true)] + pub authority_seeds: Vec, + /// Active nominators (SS58 format), each backing a random subset of the aforementioned + /// authorities. + #[arg(long, short, default_value = "0")] + pub nominator_accounts: Vec, + /// Endowed account address (SS58 format). + #[arg(long, short)] + pub endowed_accounts: Vec, + /// Sudo account address (SS58 format). + #[arg(long, short)] + pub sudo_account: String, +} + +/// Create a new chain spec with the given number of authorities and endowed +/// accounts. Random keys will be generated as required. +#[derive(Parser, Debug)] +pub struct GenerateCmd { + /// The number of authorities. + #[arg(long, short)] + pub authorities: usize, + /// The number of nominators backing the aforementioned authorities. + /// + /// Will nominate a random subset of `authorities`. + #[arg(long, short, default_value_t = 0)] + pub nominators: usize, + /// The number of endowed accounts. + #[arg(long, short, default_value_t = 0)] + pub endowed: usize, + /// Path to use when saving generated keystores for each authority. + /// + /// At this path, a new folder will be created for each authority's + /// keystore named `auth-$i` where `i` is the authority index, i.e. + /// `auth-0`, `auth-1`, etc. + #[arg(long, short)] + pub keystore_path: Option, +} + +/// Create a new chain spec by interacting with the provided runtime wasm blob. +#[derive(Parser, Debug)] +pub struct RuntimeCmd { + /// The name of chain + #[arg(long, short = 'n', default_value = "Custom")] + chain_name: String, + /// The chain id + #[arg(long, short = 'i', default_value = "custom")] + chain_id: String, + /// The path to runtime wasm blob + #[arg(long, short)] + runtime_wasm_path: PathBuf, + /// Export chainspec as raw storage + #[arg(long, short = 's')] + raw_storage: bool, + /// Verify the genesis config. This silently generates the raw storage from genesis config. Any + /// errors will be reported. + #[arg(long, short = 'v')] + verify: bool, + #[command(subcommand)] + action: GenesisBuildAction, +} + +#[derive(Subcommand, Debug, Clone)] +enum GenesisBuildAction { + Patch(PatchCmd), + Full(FullCmd), + Default(DefaultCmd), +} + +/// Patches the runtime's default genesis config with provided patch. +#[derive(Parser, Debug, Clone)] +struct PatchCmd { + /// The path to the runtime genesis config patch. + #[arg(long, short)] + patch_path: PathBuf, +} + +/// Build the genesis config for runtime using provided json file. No defaults will be used. +#[derive(Parser, Debug, Clone)] +struct FullCmd { + /// The path to the full runtime genesis config json file. + #[arg(long, short)] + config_path: PathBuf, +} - chain_spec::testnet_genesis( - authorities, - nominator_accounts.to_vec(), - sudo_account.clone(), - Some(endowed_accounts.to_vec()), - ) +/// Gets the default genesis config for the runtime and uses it in ChainSpec. Please note that +/// default genesis config may not be valid. For some runtimes initial values should be added there +/// (e.g. session keys, babe epoch). +#[derive(Parser, Debug, Clone)] +struct DefaultCmd { + #[arg(long, short)] + /// If provided stores the default genesis config json file at given path (in addition to + /// chain-spec). + default_config_path: Option, +} + +/// Edits provided input chain spec. Input can be converted into raw storage chain-spec. The code +/// can be updated with the runtime provided in the command line. +#[derive(Parser, Debug, Clone)] +pub struct EditCmd { + #[arg(long, short)] + /// Chain spec to be edited + pub input_chain_spec: PathBuf, + /// The path to new runtime wasm blob to be stored into chain-spec + #[arg(long, short = 'r')] + pub runtime_wasm_path: Option, + /// Convert genesis spec to raw format + #[arg(long, short = 's')] + pub convert_to_raw: bool, +} + +/// Verifies provided input chain spec. If the runtime is provided verification is performed against +/// new runtime. +#[derive(Parser, Debug, Clone)] +pub struct VerifyCmd { + #[arg(long, short)] + /// Chain spec to be edited + pub input_chain_spec: PathBuf, + /// The path to new runtime wasm blob to be stored into chain-spec + #[arg(long, short = 'r')] + pub runtime_wasm_path: Option, } /// Generate the chain spec using the given seeds and accounts. @@ -138,27 +211,26 @@ pub fn generate_chain_spec( let sudo_account = parse_account(sudo_account)?; - let chain_spec = chain_spec::ChainSpec::from_genesis( - "Custom", - "custom", - sc_chain_spec::ChainType::Live, - move || { - genesis_constructor( - &authority_seeds, - &nominator_accounts, - &endowed_accounts, - &sudo_account, - ) - }, - vec![], - None, - None, - None, - None, - Default::default(), - ); - - chain_spec.as_json(false) + let authorities = authority_seeds + .iter() + .map(AsRef::as_ref) + .map(chain_spec::authority_keys_from_seed) + .collect::>(); + + chain_spec::ChainSpec::builder() + .with_name("Custom") + .with_id("custom") + .with_chain_type(sc_chain_spec::ChainType::Live) + .with_genesis_config_patch(chain_spec::testnet_genesis( + authorities, + nominator_accounts, + sudo_account, + Some(endowed_accounts), + )) + .with_extensions(Default::default()) + .with_code(kitchensink_runtime::wasm_binary_unwrap()) + .build() + .as_json(false) } /// Generate the authority keys and store them in the given `keystore_path`. @@ -232,3 +304,55 @@ pub fn print_seeds( println!("{}", header.paint("Sudo seed")); println!("//{}", sudo_seed); } + +/// Processes `RuntimeCmd` and returns JSON version of `ChainSpec` +pub fn generate_chain_spec_for_runtime(cmd: &RuntimeCmd) -> Result { + let code = fs::read(cmd.runtime_wasm_path.as_path()).expect("wasm blob file is readable"); + + let builder = chain_spec::ChainSpec::builder() + .with_name(&cmd.chain_name[..]) + .with_id(&cmd.chain_id[..]) + .with_chain_type(sc_chain_spec::ChainType::Live) + .with_extensions(Default::default()) + .with_code(&code[..]); + + let builder = match cmd.action { + GenesisBuildAction::Patch(PatchCmd { ref patch_path }) => { + let patch = fs::read(patch_path.as_path()) + .map_err(|e| format!("patch file {patch_path:?} shall be readable: {e}"))?; + builder.with_genesis_config_patch(serde_json::from_slice::(&patch[..]).map_err( + |e| format!("patch file {patch_path:?} shall contain a valid json: {e}"), + )?) + }, + GenesisBuildAction::Full(FullCmd { ref config_path }) => { + let config = fs::read(config_path.as_path()) + .map_err(|e| format!("config file {config_path:?} shall be readable: {e}"))?; + builder.with_genesis_config(serde_json::from_slice::(&config[..]).map_err( + |e| format!("config file {config_path:?} shall contain a valid json: {e}"), + )?) + }, + GenesisBuildAction::Default(DefaultCmd { ref default_config_path }) => { + let caller = GenesisConfigBuilderRuntimeCaller::new(&code[..]); + let default_config = caller + .get_default_config() + .expect("getting default config from runtime should work"); + default_config_path.clone().map(|path| { + fs::write(path.as_path(), serde_json::to_string_pretty(&default_config).unwrap()) + .map_err(|err| err.to_string()) + }); + builder.with_genesis_config(default_config) + }, + }; + + let chain_spec = builder.build(); + + match (cmd.verify, cmd.raw_storage) { + (_, true) => chain_spec.as_json(true), + (true, false) => { + chain_spec.as_json(true)?; + println!("Genesis config verification: OK"); + chain_spec.as_json(false) + }, + (false, false) => chain_spec.as_json(false), + } +} diff --git a/bin/utils/chain-spec-builder/src/main.rs b/bin/utils/chain-spec-builder/src/main.rs deleted file mode 100644 index 88b18d1110b55..0000000000000 --- a/bin/utils/chain-spec-builder/src/main.rs +++ /dev/null @@ -1,298 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -//! Substrate's chain spec builder utility. -//! -//! A chain-spec is short for `chain-configuration`. See the [`sc-chain-spec`] for more information. -//! -//! See [`ChainSpecBuilder`] for a list of available commands. - -use std::{ - fs, - path::{Path, PathBuf}, -}; - -use ansi_term::Style; -use clap::Parser; -use rand::{distributions::Alphanumeric, rngs::OsRng, Rng}; - -use node_cli::chain_spec::{self, AccountId}; -use sc_keystore::LocalKeystore; -use sp_core::{ - crypto::{ByteArray, Ss58Codec}, - sr25519, -}; -use sp_keystore::KeystorePtr; - -/// A utility to easily create a testnet chain spec definition with a given set -/// of authorities and endowed accounts and/or generate random accounts. -#[derive(Parser)] -#[command(rename_all = "kebab-case")] -pub enum ChainSpecBuilder { - /// Create a new chain spec with the given authorities, endowed and sudo - /// accounts. - New { - /// Authority key seed. - #[arg(long, short, required = true)] - authority_seeds: Vec, - /// Active nominators (SS58 format), each backing a random subset of the aforementioned - /// authorities. - #[arg(long, short, default_value = "0")] - nominator_accounts: Vec, - /// Endowed account address (SS58 format). - #[arg(long, short)] - endowed_accounts: Vec, - /// Sudo account address (SS58 format). - #[arg(long, short)] - sudo_account: String, - /// The path where the chain spec should be saved. - #[arg(long, short, default_value = "./chain_spec.json")] - chain_spec_path: PathBuf, - }, - /// Create a new chain spec with the given number of authorities and endowed - /// accounts. Random keys will be generated as required. - Generate { - /// The number of authorities. - #[arg(long, short)] - authorities: usize, - /// The number of nominators backing the aforementioned authorities. - /// - /// Will nominate a random subset of `authorities`. - #[arg(long, short, default_value_t = 0)] - nominators: usize, - /// The number of endowed accounts. - #[arg(long, short, default_value_t = 0)] - endowed: usize, - /// The path where the chain spec should be saved. - #[arg(long, short, default_value = "./chain_spec.json")] - chain_spec_path: PathBuf, - /// Path to use when saving generated keystores for each authority. - /// - /// At this path, a new folder will be created for each authority's - /// keystore named `auth-$i` where `i` is the authority index, i.e. - /// `auth-0`, `auth-1`, etc. - #[arg(long, short)] - keystore_path: Option, - }, -} - -impl ChainSpecBuilder { - /// Returns the path where the chain spec should be saved. - fn chain_spec_path(&self) -> &Path { - match self { - ChainSpecBuilder::New { chain_spec_path, .. } => chain_spec_path.as_path(), - ChainSpecBuilder::Generate { chain_spec_path, .. } => chain_spec_path.as_path(), - } - } -} - -fn genesis_constructor( - authority_seeds: &[String], - nominator_accounts: &[AccountId], - endowed_accounts: &[AccountId], - sudo_account: &AccountId, -) -> chain_spec::RuntimeGenesisConfig { - let authorities = authority_seeds - .iter() - .map(AsRef::as_ref) - .map(chain_spec::authority_keys_from_seed) - .collect::>(); - - chain_spec::testnet_genesis( - authorities, - nominator_accounts.to_vec(), - sudo_account.clone(), - Some(endowed_accounts.to_vec()), - ) -} - -fn generate_chain_spec( - authority_seeds: Vec, - nominator_accounts: Vec, - endowed_accounts: Vec, - sudo_account: String, -) -> Result { - let parse_account = |address: String| { - AccountId::from_string(&address) - .map_err(|err| format!("Failed to parse account address: {:?}", err)) - }; - - let nominator_accounts = nominator_accounts - .into_iter() - .map(parse_account) - .collect::, String>>()?; - - let endowed_accounts = endowed_accounts - .into_iter() - .map(parse_account) - .collect::, String>>()?; - - let sudo_account = parse_account(sudo_account)?; - - let chain_spec = chain_spec::ChainSpec::from_genesis( - "Custom", - "custom", - sc_chain_spec::ChainType::Live, - move || { - genesis_constructor( - &authority_seeds, - &nominator_accounts, - &endowed_accounts, - &sudo_account, - ) - }, - vec![], - None, - None, - None, - None, - Default::default(), - ); - - chain_spec.as_json(false) -} - -fn generate_authority_keys_and_store(seeds: &[String], keystore_path: &Path) -> Result<(), String> { - for (n, seed) in seeds.iter().enumerate() { - let keystore: KeystorePtr = - LocalKeystore::open(keystore_path.join(format!("auth-{}", n)), None) - .map_err(|err| err.to_string())? - .into(); - - let (_, _, grandpa, babe, im_online, authority_discovery) = - chain_spec::authority_keys_from_seed(seed); - - let insert_key = |key_type, public| { - keystore - .insert(key_type, &format!("//{}", seed), public) - .map_err(|_| format!("Failed to insert key: {}", grandpa)) - }; - - insert_key(sp_core::crypto::key_types::BABE, babe.as_slice())?; - - insert_key(sp_core::crypto::key_types::GRANDPA, grandpa.as_slice())?; - - insert_key(sp_core::crypto::key_types::IM_ONLINE, im_online.as_slice())?; - - insert_key( - sp_core::crypto::key_types::AUTHORITY_DISCOVERY, - authority_discovery.as_slice(), - )?; - } - - Ok(()) -} - -fn print_seeds( - authority_seeds: &[String], - nominator_seeds: &[String], - endowed_seeds: &[String], - sudo_seed: &str, -) { - let header = Style::new().bold().underline(); - let entry = Style::new().bold(); - - println!("{}", header.paint("Authority seeds")); - - for (n, seed) in authority_seeds.iter().enumerate() { - println!("{} //{}", entry.paint(format!("auth-{}:", n)), seed); - } - - println!("{}", header.paint("Nominator seeds")); - - for (n, seed) in nominator_seeds.iter().enumerate() { - println!("{} //{}", entry.paint(format!("nom-{}:", n)), seed); - } - - println!(); - - if !endowed_seeds.is_empty() { - println!("{}", header.paint("Endowed seeds")); - for (n, seed) in endowed_seeds.iter().enumerate() { - println!("{} //{}", entry.paint(format!("endowed-{}:", n)), seed); - } - - println!(); - } - - println!("{}", header.paint("Sudo seed")); - println!("//{}", sudo_seed); -} - -fn main() -> Result<(), String> { - #[cfg(build_type = "debug")] - println!( - "The chain spec builder builds a chain specification that includes a Substrate runtime \ - compiled as WASM. To ensure proper functioning of the included runtime compile (or run) \ - the chain spec builder binary in `--release` mode.\n", - ); - - let builder = ChainSpecBuilder::parse(); - let chain_spec_path = builder.chain_spec_path().to_path_buf(); - - let (authority_seeds, nominator_accounts, endowed_accounts, sudo_account) = match builder { - ChainSpecBuilder::Generate { authorities, nominators, endowed, keystore_path, .. } => { - let authorities = authorities.max(1); - let rand_str = || -> String { - OsRng.sample_iter(&Alphanumeric).take(32).map(char::from).collect() - }; - - let authority_seeds = (0..authorities).map(|_| rand_str()).collect::>(); - let nominator_seeds = (0..nominators).map(|_| rand_str()).collect::>(); - let endowed_seeds = (0..endowed).map(|_| rand_str()).collect::>(); - let sudo_seed = rand_str(); - - print_seeds(&authority_seeds, &nominator_seeds, &endowed_seeds, &sudo_seed); - - if let Some(keystore_path) = keystore_path { - generate_authority_keys_and_store(&authority_seeds, &keystore_path)?; - } - - let nominator_accounts = nominator_seeds - .into_iter() - .map(|seed| { - chain_spec::get_account_id_from_seed::(&seed).to_ss58check() - }) - .collect(); - - let endowed_accounts = endowed_seeds - .into_iter() - .map(|seed| { - chain_spec::get_account_id_from_seed::(&seed).to_ss58check() - }) - .collect(); - - let sudo_account = - chain_spec::get_account_id_from_seed::(&sudo_seed).to_ss58check(); - - (authority_seeds, nominator_accounts, endowed_accounts, sudo_account) - }, - ChainSpecBuilder::New { - authority_seeds, - nominator_accounts, - endowed_accounts, - sudo_account, - .. - } => (authority_seeds, nominator_accounts, endowed_accounts, sudo_account), - }; - - let json = - generate_chain_spec(authority_seeds, nominator_accounts, endowed_accounts, sudo_account)?; - - fs::write(chain_spec_path, json).map_err(|err| err.to_string()) -} From bbfabb51db919b9d02b3f90c70d06a29e4768d53 Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Mon, 24 Jul 2023 18:25:59 +0200 Subject: [PATCH 28/47] Cargo.lock updated --- Cargo.lock | 128 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 75 insertions(+), 53 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ed6a4c356d10f..f7050759892a2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -685,7 +685,7 @@ checksum = "0e97ce7de6cf12de5d7226c73f5ba9811622f4db3a5b91b55c53e987e5f91cba" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -707,7 +707,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -718,7 +718,7 @@ checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -865,7 +865,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -1190,12 +1190,16 @@ version = "2.0.0" dependencies = [ "ansi_term", "clap 4.3.2", + "kitchensink-runtime", + "log", "node-cli", "rand 0.8.5", "sc-chain-spec", "sc-keystore", + "serde_json", "sp-core", "sp-keystore", + "sp-tracing", ] [[package]] @@ -1355,7 +1359,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -1845,7 +1849,7 @@ dependencies = [ "proc-macro2", "quote", "scratch", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -1862,7 +1866,7 @@ checksum = "4a076022ece33e7686fb76513518e219cca4fce5750a8ae6d1ce6c0f48fd1af9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -2130,7 +2134,7 @@ checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -2175,7 +2179,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.18", + "syn 2.0.27", "termcolor", "walkdir", ] @@ -2192,7 +2196,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.18", + "syn 2.0.27", "termcolor", "walkdir", ] @@ -2387,7 +2391,7 @@ checksum = "5e9a1f9f7d83e59740248a6e14ecf93929ade55027844dfcea78beafccc15745" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -2468,7 +2472,7 @@ dependencies = [ "fs-err", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -2742,7 +2746,7 @@ dependencies = [ "quote", "scale-info", "sp-arithmetic", - "syn 2.0.18", + "syn 2.0.27", "trybuild", ] @@ -2891,7 +2895,7 @@ dependencies = [ "proc-macro-warning", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -2902,7 +2906,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -2911,7 +2915,7 @@ version = "3.0.0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -3132,7 +3136,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -4194,12 +4198,14 @@ dependencies = [ "parity-scale-codec", "primitive-types", "scale-info", + "serde_json", "sp-api", "sp-authority-discovery", "sp-block-builder", "sp-consensus-babe", "sp-consensus-grandpa", "sp-core", + "sp-genesis-builder", "sp-inherents", "sp-io", "sp-offchain", @@ -4927,7 +4933,7 @@ dependencies = [ "macro_magic_core", "macro_magic_macros", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -4940,7 +4946,7 @@ dependencies = [ "macro_magic_core_macros", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -4951,7 +4957,7 @@ checksum = "93d7d9e6e234c040dafc745c7592738d56a03ad04b1fa04ab60821deb597466a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -4962,7 +4968,7 @@ checksum = "ffd19f13cfd2bfbd83692adfef8c244fe5109b3eb822a1fb4e0a6253b406cd81" dependencies = [ "macro_magic_core", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -5538,6 +5544,7 @@ dependencies = [ "frame-system", "futures", "kitchensink-runtime", + "log", "node-primitives", "node-testing", "pallet-balances", @@ -5656,6 +5663,7 @@ dependencies = [ "sc-telemetry", "sc-transaction-pool", "sc-transaction-pool-api", + "serde_json", "sp-api", "sp-block-builder", "sp-blockchain", @@ -5707,6 +5715,7 @@ dependencies = [ "pallet-transaction-payment-rpc-runtime-api", "parity-scale-codec", "scale-info", + "serde_json", "sp-api", "sp-block-builder", "sp-consensus-aura", @@ -6437,7 +6446,7 @@ version = "4.0.0-dev" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -7472,7 +7481,7 @@ dependencies = [ "proc-macro2", "quote", "sp-runtime", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -7958,7 +7967,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -7999,7 +8008,7 @@ checksum = "39407670928234ebc5e6e580247dd567ad73a3578460c5990f9503df207e8f07" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -8214,7 +8223,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b69d39aab54d069e7f2fe8cb970493e7834601ca2d8c65fd7bbd183578080d1" dependencies = [ "proc-macro2", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -8273,14 +8282,14 @@ checksum = "70550716265d1ec349c41f70dd4f964b4fd88394efe4405f0c1da679c4799a07" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] name = "proc-macro2" -version = "1.0.60" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dec2b086b7a862cf4de201096214fa870344cf922b2b30c167badb3af3195406" +checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" dependencies = [ "unicode-ident", ] @@ -8655,7 +8664,7 @@ checksum = "8d2275aab483050ab2a7364c1a46604865ee7d6906684e08db0f090acf74f9e7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -9106,18 +9115,31 @@ dependencies = [ name = "sc-chain-spec" version = "4.0.0-dev" dependencies = [ + "array-bytes", + "json-patch", + "kitchensink-runtime", + "log", "memmap2", + "parity-scale-codec", "sc-chain-spec-derive", "sc-client-api", "sc-executor", + "sc-executor-common", "sc-network", "sc-telemetry", "serde", "serde_json", + "sp-application-crypto", "sp-blockchain", + "sp-consensus-babe", "sp-core", + "sp-genesis-builder", + "sp-io", + "sp-keyring", "sp-runtime", "sp-state-machine", + "sp-tracing", + "substrate-test-runtime", ] [[package]] @@ -9127,7 +9149,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -10340,7 +10362,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -10628,29 +10650,29 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.164" +version = "1.0.175" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e8c8cf938e98f769bc164923b06dce91cea1751522f46f8466461af04c9027d" +checksum = "5d25439cd7397d044e2748a6fe2432b5e85db703d6d097bd014b3c0ad1ebff0b" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.164" +version = "1.0.175" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9735b638ccc51c28bf6914d90a2e9725b377144fc612c49a611fddd1b631d68" +checksum = "b23f7ade6f110613c0d63858ddb8b94c1041f550eab58a16b371bdf2c9c80ab4" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] name = "serde_json" -version = "1.0.96" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" +checksum = "d03b412469450d4404fe8499a268edd7f8b79fecb074b0d812ad64ca21f4031b" dependencies = [ "itoa", "ryu", @@ -10902,7 +10924,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -11287,7 +11309,7 @@ version = "9.0.0" dependencies = [ "quote", "sp-core-hashing", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -11331,7 +11353,7 @@ version = "8.0.0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -11563,7 +11585,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -11803,7 +11825,7 @@ dependencies = [ "proc-macro2", "quote", "sp-version", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -12255,7 +12277,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -12312,9 +12334,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.18" +version = "2.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" +checksum = "b60f673f44a8255b9c8c657daf66a596d435f2da81a555b06dc644d080ba45e0" dependencies = [ "proc-macro2", "quote", @@ -12429,7 +12451,7 @@ checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -12585,7 +12607,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -12765,7 +12787,7 @@ checksum = "0f57e3ca2a01450b1a921183a9c9cbfda207fd822cef4ccb00a65402cbba7a74" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -13296,7 +13318,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", "wasm-bindgen-shared", ] @@ -13330,7 +13352,7 @@ checksum = "e128beba882dd1eb6200e1dc92ae6c5dbaa4311aa7bb211ca035779e5efc39f8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -14321,7 +14343,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] From ce2b20a3bcc850312110e53ba321d9d797fdcd6e Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Thu, 6 Jul 2023 13:50:32 +0200 Subject: [PATCH 29/47] test-runtime: frame_system::code removed --- test-utils/runtime/Cargo.toml | 4 +- test-utils/runtime/src/genesismap.rs | 10 +- test-utils/runtime/src/lib.rs | 37 ++++-- .../src/test_json/default_genesis_config.json | 4 +- .../default_genesis_config_incomplete.json | 4 +- .../default_genesis_config_invalid.json | 4 +- .../default_genesis_config_invalid_2.json | 113 ++++++++++++++++++ 7 files changed, 150 insertions(+), 26 deletions(-) create mode 100644 test-utils/runtime/src/test_json/default_genesis_config_invalid_2.json diff --git a/test-utils/runtime/Cargo.toml b/test-utils/runtime/Cargo.toml index 2b80d1c64bc2d..4c50e4df764f2 100644 --- a/test-utils/runtime/Cargo.toml +++ b/test-utils/runtime/Cargo.toml @@ -48,8 +48,6 @@ sp-externalities = { version = "0.19.0", default-features = false, path = "../.. # 3rd party array-bytes = { version = "6.1", optional = true } log = { version = "0.4.17", default-features = false } -serde = { version = "1.0.163", features = ["alloc", "derive"], default-features = false } -serde_json = { version = "1.0.85", default-features = false, features = ["alloc"] } [dev-dependencies] futures = "0.3.21" @@ -60,6 +58,8 @@ sp-consensus = { version = "0.10.0-dev", path = "../../primitives/consensus/comm substrate-test-runtime-client = { version = "2.0.0", path = "./client" } sp-tracing = { version = "10.0.0", path = "../../primitives/tracing" } json-patch = { version = "1.0.0", default-features = false } +serde = { version = "1.0.163", features = ["alloc", "derive"], default-features = false } +serde_json = { version = "1.0.85", default-features = false, features = ["alloc"] } [build-dependencies] substrate-wasm-builder = { version = "5.0.0-dev", path = "../../utils/wasm-builder", optional = true } diff --git a/test-utils/runtime/src/genesismap.rs b/test-utils/runtime/src/genesismap.rs index 8a4d6dbe4a71a..5ed9c8a645886 100644 --- a/test-utils/runtime/src/genesismap.rs +++ b/test-utils/runtime/src/genesismap.rs @@ -117,10 +117,7 @@ impl GenesisStorageBuilder { .collect(); RuntimeGenesisConfig { - system: frame_system::GenesisConfig { - code: self.wasm_code.clone().unwrap_or(wasm_binary_unwrap().to_vec()), - ..Default::default() - }, + system: Default::default(), babe: pallet_babe::GenesisConfig { authorities: authorities_sr25519 .clone() @@ -149,6 +146,11 @@ impl GenesisStorageBuilder { storage.top.insert(well_known_keys::HEAP_PAGES.into(), heap_pages.encode()); } + storage.top.insert( + well_known_keys::CODE.into(), + self.wasm_code.clone().unwrap_or(wasm_binary_unwrap().to_vec()), + ); + storage.top.extend(self.extra_storage.top.clone()); storage.children_default.extend(self.extra_storage.children_default.clone()); diff --git a/test-utils/runtime/src/lib.rs b/test-utils/runtime/src/lib.rs index 0cc32e50956c8..64e51f0f1db83 100644 --- a/test-utils/runtime/src/lib.rs +++ b/test-utils/runtime/src/lib.rs @@ -1237,7 +1237,7 @@ mod tests { #[test] fn build_minimal_genesis_config_works() { sp_tracing::try_init_simple(); - let default_minimal_json = r#"{"system":{"code":"0x"},"babe":{"authorities":[],"epochConfig":{"c": [ 3, 10 ],"allowed_slots":"PrimaryAndSecondaryPlainSlots"}},"substrateTest":{"authorities":[]},"balances":{"balances":[]}}"#; + let default_minimal_json = r#"{"system":{},"babe":{"authorities":[],"epochConfig":{"c": [ 3, 10 ],"allowed_slots":"PrimaryAndSecondaryPlainSlots"}},"substrateTest":{"authorities":[]},"balances":{"balances":[]}}"#; let mut t = BasicExternalities::new_empty(); executor_call(&mut t, "GenesisBuilder_build_config", &default_minimal_json.encode()) @@ -1264,8 +1264,6 @@ mod tests { // System|LastRuntimeUpgrade "26aa394eea5630e07c48ae0c9558cef7f9cce9c888469bb1a0dceaa129672ef8", - // :code - "3a636f6465", // :extrinsic_index "3a65787472696e7369635f696e646578", // Balances|TotalIssuance @@ -1294,7 +1292,7 @@ mod tests { let r = Vec::::decode(&mut &r[..]).unwrap(); let json = String::from_utf8(r.into()).expect("returned value is json. qed."); - let expected = r#"{"system":{"code":"0x"},"babe":{"authorities":[],"epochConfig":null},"substrateTest":{"authorities":[]},"balances":{"balances":[]}}"#; + let expected = r#"{"system":{},"babe":{"authorities":[],"epochConfig":null},"substrateTest":{"authorities":[]},"balances":{"balances":[]}}"#; assert_eq!(expected.to_string(), json); } @@ -1308,7 +1306,13 @@ mod tests { let r = BuildResult::decode(&mut &r[..]); assert!(r.is_ok()); - let keys = t.into_storages().top.keys().cloned().map(hex).collect::>(); + let mut keys = t.into_storages().top.keys().cloned().map(hex).collect::>(); + + // following keys are not placed during `::build` + // process, add them `keys` to assert against known keys. + keys.push(hex(b":code")); + keys.sort(); + assert_eq!(keys, storage_key_generator::get_expected_storage_hashed_keys(false)); } @@ -1322,7 +1326,22 @@ mod tests { log::info!("result: {:#?}", r); assert_eq!(r, Err( sp_runtime::RuntimeString::Owned( - "Invalid JSON blob: unknown field `renamed_authorities`, expected `authorities` or `epochConfig` at line 6 column 25".to_string(), + "Invalid JSON blob: unknown field `renamed_authorities`, expected `authorities` or `epochConfig` at line 4 column 25".to_string(), + )) + ); + } + + #[test] + fn build_config_from_invalid_json_fails_2() { + sp_tracing::try_init_simple(); + let j = include_str!("test_json/default_genesis_config_invalid_2.json"); + let mut t = BasicExternalities::new_empty(); + let r = executor_call(&mut t, "GenesisBuilder_build_config", &j.encode()).unwrap(); + let r = BuildResult::decode(&mut &r[..]).unwrap(); + log::info!("result: {:#?}", r); + assert_eq!(r, Err( + sp_runtime::RuntimeString::Owned( + "Invalid JSON blob: unknown field `babex`, expected one of `system`, `babe`, `substrateTest`, `balances` at line 3 column 9".to_string(), )) ); } @@ -1339,7 +1358,7 @@ mod tests { assert_eq!( r, Err(sp_runtime::RuntimeString::Owned( - "Invalid JSON blob: missing field `authorities` at line 13 column 3" + "Invalid JSON blob: missing field `authorities` at line 11 column 3" .to_string() )) ); @@ -1438,10 +1457,6 @@ mod tests { ); assert_eq!(u64::decode(&mut &value[..]).unwrap(), 0); - // :code - let value: Vec = get_from_storage("3a636f6465"); - assert!(Vec::::decode(&mut &value[..]).is_err()); - //System|ParentHash let value: Vec = get_from_storage( "26aa394eea5630e07c48ae0c9558cef78a42f33323cb5ced3b44dd825fda9fcc", diff --git a/test-utils/runtime/src/test_json/default_genesis_config.json b/test-utils/runtime/src/test_json/default_genesis_config.json index b0218d417daa5..95c7799a033d8 100644 --- a/test-utils/runtime/src/test_json/default_genesis_config.json +++ b/test-utils/runtime/src/test_json/default_genesis_config.json @@ -1,7 +1,5 @@ { - "system": { - "code": "0x52" - }, + "system": {}, "babe": { "authorities": [ [ diff --git a/test-utils/runtime/src/test_json/default_genesis_config_incomplete.json b/test-utils/runtime/src/test_json/default_genesis_config_incomplete.json index e25730ee11cf0..510ed87c93c57 100644 --- a/test-utils/runtime/src/test_json/default_genesis_config_incomplete.json +++ b/test-utils/runtime/src/test_json/default_genesis_config_incomplete.json @@ -1,7 +1,5 @@ { - "system": { - "code": "0x52" - }, + "system": {}, "babe": { "epochConfig": { "c": [ diff --git a/test-utils/runtime/src/test_json/default_genesis_config_invalid.json b/test-utils/runtime/src/test_json/default_genesis_config_invalid.json index 00550efaeec9f..f8e06f91d6658 100644 --- a/test-utils/runtime/src/test_json/default_genesis_config_invalid.json +++ b/test-utils/runtime/src/test_json/default_genesis_config_invalid.json @@ -1,7 +1,5 @@ { - "system": { - "code": "0x52" - }, + "system": {}, "babe": { "renamed_authorities": [ [ diff --git a/test-utils/runtime/src/test_json/default_genesis_config_invalid_2.json b/test-utils/runtime/src/test_json/default_genesis_config_invalid_2.json new file mode 100644 index 0000000000000..a1345542bcda7 --- /dev/null +++ b/test-utils/runtime/src/test_json/default_genesis_config_invalid_2.json @@ -0,0 +1,113 @@ +{ + "system": {}, + "babex": { + "authorities": [ + [ + "5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY", + 1 + ], + [ + "5FHneW46xGXgs5mUiveU4sbTyGBzmstUspZC92UhjJM694ty", + 1 + ], + [ + "5FLSigC9HGRKVhB9FiEo4Y3koPsNmBmLJbpXg2mp1hXcS59Y", + 1 + ] + ], + "epochConfig": { + "c": [ + 3, + 10 + ], + "allowed_slots": "PrimaryAndSecondaryPlainSlots" + } + }, + "substrateTest": { + "authorities": [ + "5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY", + "5FHneW46xGXgs5mUiveU4sbTyGBzmstUspZC92UhjJM694ty", + "5FLSigC9HGRKVhB9FiEo4Y3koPsNmBmLJbpXg2mp1hXcS59Y" + ] + }, + "balances": { + "balances": [ + [ + "5D34dL5prEUaGNQtPPZ3yN5Y6BnkfXunKXXz6fo7ZJbLwRRH", + 100000000000000000 + ], + [ + "5GBNeWRhZc2jXu7D55rBimKYDk8PGk8itRYFTPfC8RJLKG5o", + 100000000000000000 + ], + [ + "5Dfis6XL8J2P6JHUnUtArnFWndn62SydeP8ee8sG2ky9nfm9", + 100000000000000000 + ], + [ + "5F4H97f7nQovyrbiq4ZetaaviNwThSVcFobcA5aGab6167dK", + 100000000000000000 + ], + [ + "5DiDShBWa1fQx6gLzpf3SFBhMinCoyvHM1BWjPNsmXS8hkrW", + 100000000000000000 + ], + [ + "5EFb84yH9tpcFuiKUcsmdoF7xeeY3ajG1ZLQimxQoFt9HMKR", + 100000000000000000 + ], + [ + "5DZLHESsfGrJ5YzT3HuRPXsSNb589xQ4Unubh1mYLodzKdVY", + 100000000000000000 + ], + [ + "5GHJzqvG6tXnngCpG7B12qjUvbo5e4e9z8Xjidk3CQZHxTPZ", + 100000000000000000 + ], + [ + "5CUnSsgAyLND3bxxnfNhgWXSe9Wn676JzLpGLgyJv858qhoX", + 100000000000000000 + ], + [ + "5CVKn7HAZW1Ky4r7Vkgsr7VEW88C2sHgUNDiwHY9Ct2hjU8q", + 100000000000000000 + ], + [ + "5H673aukQ4PeDe1U2nuv1bi32xDEziimh3PZz7hDdYUB7TNz", + 100000000000000000 + ], + [ + "5HTe9L15LJryjUAt1jZXZCBPnzbbGnpvFwbjE3NwCWaAqovf", + 100000000000000000 + ], + [ + "5D7LFzGpMwHPyDBavkRbWSKWTtJhCaPPZ379wWLT23bJwXJz", + 100000000000000000 + ], + [ + "5CLepMARnEgtVR1EkUuJVUvKh97gzergpSxUU3yKGx1v6EwC", + 100000000000000000 + ], + [ + "5Chb2UhfvZpmjjEziHbFbotM4quX32ZscRV6QJBt1rUKzz51", + 100000000000000000 + ], + [ + "5HmRp3i3ZZk7xsAvbi8hyXVP6whSMnBJGebVC4FsiZVhx52e", + 100000000000000000 + ], + [ + "5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY", + 100000000000000000 + ], + [ + "5FHneW46xGXgs5mUiveU4sbTyGBzmstUspZC92UhjJM694ty", + 100000000000000000 + ], + [ + "5FLSigC9HGRKVhB9FiEo4Y3koPsNmBmLJbpXg2mp1hXcS59Y", + 100000000000000000 + ] + ] + } +} From b214805b77893030f278fabd5c0c482e6e4337f9 Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Tue, 25 Jul 2023 10:59:49 +0200 Subject: [PATCH 30/47] node-bench: code addded to genesis storage --- bin/node/testing/src/bench.rs | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/bin/node/testing/src/bench.rs b/bin/node/testing/src/bench.rs index bbaf1877f6032..eef6abff61e94 100644 --- a/bin/node/testing/src/bench.rs +++ b/bin/node/testing/src/bench.rs @@ -398,7 +398,7 @@ impl BenchDb { let client_config = sc_service::ClientConfig::default(); let genesis_block_builder = sc_service::GenesisBlockBuilder::new( - &keyring.generate_genesis(), + keyring.as_storage_builder(), !client_config.no_genesis, backend.clone(), executor.clone(), @@ -585,9 +585,20 @@ impl BenchKeyring { } } - /// Generate genesis with accounts from this keyring endowed with some balance. - pub fn generate_genesis(&self) -> kitchensink_runtime::RuntimeGenesisConfig { - crate::genesis::config_endowed(self.collect_account_ids()) + /// Generate genesis with accounts from this keyring endowed with some balance and + /// kitchensink_runtime code blob. + pub fn as_storage_builder(&self) -> &dyn sp_runtime::BuildStorage { + self + } +} + +impl sp_runtime::BuildStorage for BenchKeyring { + fn assimilate_storage(&self, storage: &mut sp_core::storage::Storage) -> Result<(), String> { + storage.top.insert( + sp_core::storage::well_known_keys::CODE.to_vec(), + kitchensink_runtime::wasm_binary_unwrap().into(), + ); + crate::genesis::config_endowed(self.collect_account_ids()).assimilate_storage(storage) } } From 1fd42a0ac66c07d7b2b3317c73d7a2ec66c484b1 Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Tue, 25 Jul 2023 11:54:52 +0200 Subject: [PATCH 31/47] node-testing: code added to genesis storage --- bin/node/testing/src/client.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bin/node/testing/src/client.rs b/bin/node/testing/src/client.rs index 38ddb00dfe32f..a438d394dc220 100644 --- a/bin/node/testing/src/client.rs +++ b/bin/node/testing/src/client.rs @@ -45,7 +45,12 @@ pub struct GenesisParameters; impl substrate_test_client::GenesisInit for GenesisParameters { fn genesis_storage(&self) -> Storage { - crate::genesis::config().build_storage().unwrap() + let mut storage = crate::genesis::config().build_storage().unwrap(); + storage.top.insert( + sp_core::storage::well_known_keys::CODE.to_vec(), + kitchensink_runtime::wasm_binary_unwrap().into(), + ); + storage } } From b77b45ea14383e69801adc1ac8fee35514f364d7 Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Tue, 25 Jul 2023 12:11:51 +0200 Subject: [PATCH 32/47] bin/node: executor test fix --- bin/node/executor/tests/basic.rs | 10 ++++++---- .../executor/tests/res/default_genesis_config.json | 1 + 2 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 bin/node/executor/tests/res/default_genesis_config.json diff --git a/bin/node/executor/tests/basic.rs b/bin/node/executor/tests/basic.rs index 578cc285686ce..c925de2ea6d4c 100644 --- a/bin/node/executor/tests/basic.rs +++ b/bin/node/executor/tests/basic.rs @@ -863,13 +863,15 @@ fn default_config_as_json_works() { sp_tracing::try_init_simple(); // let mut t = BasicExternalities::new_empty(); let mut t = new_test_ext(compact_code_unwrap()); - let r = executor_call(&mut t, "GenesisBuilder_get_default_as_json", &vec![], false) + let r = executor_call(&mut t, "GenesisBuilder_create_default_config", &vec![], false) .0 .unwrap(); let r = Vec::::decode(&mut &r[..]).unwrap(); let json = String::from_utf8(r.into()).expect("returned value is json. qed."); + let mut expected = include_str!("res/default_genesis_config.json").to_string(); - let expected = r#"{"system":{"code":"0x"},"babe":{"authorities":[],"epochConfig":null},"indices":{"indices":[]},"balances":{"balances":[]},"transactionPayment":{"multiplier":"1000000000000000000"},"staking":{"validatorCount":0,"minimumValidatorCount":0,"invulnerables":[],"forceEra":"NotForcing","slashRewardFraction":0,"canceledPayout":0,"stakers":[],"minNominatorBond":0,"minValidatorBond":0,"maxValidatorCount":null,"maxNominatorCount":null},"session":{"keys":[]},"democracy":{},"council":{"members":[]},"technicalCommittee":{"members":[]},"elections":{"members":[]},"technicalMembership":{"members":[]},"grandpa":{"authorities":[]},"treasury":{},"sudo":{"key":null},"imOnline":{"keys":[]},"authorityDiscovery":{"keys":[]},"society":{"pot":0},"vesting":{"vesting":[]},"glutton":{"compute":"0","storage":"0","trashDataCount":0},"assets":{"assets":[],"metadata":[],"accounts":[]},"poolAssets":{"assets":[],"metadata":[],"accounts":[]},"transactionStorage":{"byteFee":10,"entryFee":1000,"storagePeriod":100800},"allianceMotion":{"members":[]},"alliance":{"fellows":[],"allies":[]},"nominationPools":{"minJoinBond":0,"minCreateBond":0,"maxPools":16,"maxMembersPerPool":32,"maxMembers":512,"globalMaxCommission":null}}"#; - log::info!("---> json: {:#?}", json); - assert_eq!(expected.to_string(), json); + //remove "\n": + expected.pop(); + + assert_eq!(expected, json); } diff --git a/bin/node/executor/tests/res/default_genesis_config.json b/bin/node/executor/tests/res/default_genesis_config.json new file mode 100644 index 0000000000000..2c7c7d5523a6a --- /dev/null +++ b/bin/node/executor/tests/res/default_genesis_config.json @@ -0,0 +1 @@ +{"system":{},"babe":{"authorities":[],"epochConfig":null},"indices":{"indices":[]},"balances":{"balances":[]},"transactionPayment":{"multiplier":"1000000000000000000"},"staking":{"validatorCount":0,"minimumValidatorCount":0,"invulnerables":[],"forceEra":"NotForcing","slashRewardFraction":0,"canceledPayout":0,"stakers":[],"minNominatorBond":0,"minValidatorBond":0,"maxValidatorCount":null,"maxNominatorCount":null},"session":{"keys":[]},"democracy":{},"council":{"members":[]},"technicalCommittee":{"members":[]},"elections":{"members":[]},"technicalMembership":{"members":[]},"grandpa":{"authorities":[]},"treasury":{},"sudo":{"key":null},"imOnline":{"keys":[]},"authorityDiscovery":{"keys":[]},"society":{"pot":0},"vesting":{"vesting":[]},"glutton":{"compute":"0","storage":"0","trashDataCount":0},"assets":{"assets":[],"metadata":[],"accounts":[]},"poolAssets":{"assets":[],"metadata":[],"accounts":[]},"transactionStorage":{"byteFee":10,"entryFee":1000,"storagePeriod":100800},"allianceMotion":{"members":[]},"alliance":{"fellows":[],"allies":[]},"nominationPools":{"minJoinBond":0,"minCreateBond":0,"maxPools":16,"maxMembersPerPool":32,"maxMembers":512,"globalMaxCommission":null}} From f869b0e2452533058d39dd7ee9bac14f6479682c Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Tue, 25 Jul 2023 15:33:50 +0200 Subject: [PATCH 33/47] Cargo.lock updated --- Cargo.lock | 128 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 75 insertions(+), 53 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 93a8e5e988305..1bb00c6f811d0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -571,7 +571,7 @@ checksum = "0e97ce7de6cf12de5d7226c73f5ba9811622f4db3a5b91b55c53e987e5f91cba" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -593,7 +593,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -604,7 +604,7 @@ checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -739,7 +739,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -1047,12 +1047,16 @@ version = "2.0.0" dependencies = [ "ansi_term", "clap 4.3.2", + "kitchensink-runtime", + "log", "node-cli", "rand 0.8.5", "sc-chain-spec", "sc-keystore", + "serde_json", "sp-core", "sp-keystore", + "sp-tracing", ] [[package]] @@ -1203,7 +1207,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -1688,7 +1692,7 @@ dependencies = [ "proc-macro2", "quote", "scratch", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -1705,7 +1709,7 @@ checksum = "4a076022ece33e7686fb76513518e219cca4fce5750a8ae6d1ce6c0f48fd1af9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -1902,7 +1906,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.18", + "syn 2.0.27", "termcolor", "walkdir", ] @@ -1919,7 +1923,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.18", + "syn 2.0.27", "termcolor", "walkdir", ] @@ -2080,7 +2084,7 @@ checksum = "5e9a1f9f7d83e59740248a6e14ecf93929ade55027844dfcea78beafccc15745" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -2161,7 +2165,7 @@ dependencies = [ "fs-err", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -2424,7 +2428,7 @@ dependencies = [ "quote", "scale-info", "sp-arithmetic", - "syn 2.0.18", + "syn 2.0.27", "trybuild", ] @@ -2573,7 +2577,7 @@ dependencies = [ "proc-macro-warning", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -2584,7 +2588,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -2593,7 +2597,7 @@ version = "3.0.0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -2814,7 +2818,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -3840,12 +3844,14 @@ dependencies = [ "parity-scale-codec", "primitive-types", "scale-info", + "serde_json", "sp-api", "sp-authority-discovery", "sp-block-builder", "sp-consensus-babe", "sp-consensus-grandpa", "sp-core", + "sp-genesis-builder", "sp-inherents", "sp-io", "sp-offchain", @@ -4237,7 +4243,7 @@ dependencies = [ "proc-macro-warning", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -4513,7 +4519,7 @@ dependencies = [ "macro_magic_core", "macro_magic_macros", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -4527,7 +4533,7 @@ dependencies = [ "macro_magic_core_macros", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -4538,7 +4544,7 @@ checksum = "c12469fc165526520dff2807c2975310ab47cf7190a45b99b49a7dc8befab17b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -4549,7 +4555,7 @@ checksum = "b8fb85ec1620619edf2984a7693497d4ec88a9665d8b87e942856884c92dbf2a" dependencies = [ "macro_magic_core", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -5160,6 +5166,7 @@ dependencies = [ "frame-system", "futures", "kitchensink-runtime", + "log", "node-primitives", "node-testing", "pallet-balances", @@ -5278,6 +5285,7 @@ dependencies = [ "sc-telemetry", "sc-transaction-pool", "sc-transaction-pool-api", + "serde_json", "sp-api", "sp-block-builder", "sp-blockchain", @@ -5329,6 +5337,7 @@ dependencies = [ "pallet-transaction-payment-rpc-runtime-api", "parity-scale-codec", "scale-info", + "serde_json", "sp-api", "sp-block-builder", "sp-consensus-aura", @@ -6019,7 +6028,7 @@ version = "4.0.0-dev" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -7054,7 +7063,7 @@ dependencies = [ "proc-macro2", "quote", "sp-runtime", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -7516,7 +7525,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -7557,7 +7566,7 @@ checksum = "39407670928234ebc5e6e580247dd567ad73a3578460c5990f9503df207e8f07" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -7762,7 +7771,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b69d39aab54d069e7f2fe8cb970493e7834601ca2d8c65fd7bbd183578080d1" dependencies = [ "proc-macro2", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -7827,14 +7836,14 @@ checksum = "70550716265d1ec349c41f70dd4f964b4fd88394efe4405f0c1da679c4799a07" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] name = "proc-macro2" -version = "1.0.60" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dec2b086b7a862cf4de201096214fa870344cf922b2b30c167badb3af3195406" +checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" dependencies = [ "unicode-ident", ] @@ -8166,7 +8175,7 @@ checksum = "8d2275aab483050ab2a7364c1a46604865ee7d6906684e08db0f090acf74f9e7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -8568,18 +8577,31 @@ dependencies = [ name = "sc-chain-spec" version = "4.0.0-dev" dependencies = [ + "array-bytes", + "json-patch", + "kitchensink-runtime", + "log", "memmap2", + "parity-scale-codec", "sc-chain-spec-derive", "sc-client-api", "sc-executor", + "sc-executor-common", "sc-network", "sc-telemetry", "serde", "serde_json", + "sp-application-crypto", "sp-blockchain", + "sp-consensus-babe", "sp-core", + "sp-genesis-builder", + "sp-io", + "sp-keyring", "sp-runtime", "sp-state-machine", + "sp-tracing", + "substrate-test-runtime", ] [[package]] @@ -8589,7 +8611,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -9805,7 +9827,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -10063,29 +10085,29 @@ checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" [[package]] name = "serde" -version = "1.0.164" +version = "1.0.175" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e8c8cf938e98f769bc164923b06dce91cea1751522f46f8466461af04c9027d" +checksum = "5d25439cd7397d044e2748a6fe2432b5e85db703d6d097bd014b3c0ad1ebff0b" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.164" +version = "1.0.175" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9735b638ccc51c28bf6914d90a2e9725b377144fc612c49a611fddd1b631d68" +checksum = "b23f7ade6f110613c0d63858ddb8b94c1041f550eab58a16b371bdf2c9c80ab4" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] name = "serde_json" -version = "1.0.96" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" +checksum = "d03b412469450d4404fe8499a268edd7f8b79fecb074b0d812ad64ca21f4031b" dependencies = [ "itoa", "ryu", @@ -10342,7 +10364,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -10727,7 +10749,7 @@ version = "9.0.0" dependencies = [ "quote", "sp-core-hashing", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -10771,7 +10793,7 @@ version = "8.0.0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -11003,7 +11025,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -11243,7 +11265,7 @@ dependencies = [ "proc-macro2", "quote", "sp-version", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -11679,7 +11701,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -11727,9 +11749,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.18" +version = "2.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" +checksum = "b60f673f44a8255b9c8c657daf66a596d435f2da81a555b06dc644d080ba45e0" dependencies = [ "proc-macro2", "quote", @@ -11844,7 +11866,7 @@ checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -11974,7 +11996,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -12154,7 +12176,7 @@ checksum = "0f57e3ca2a01450b1a921183a9c9cbfda207fd822cef4ccb00a65402cbba7a74" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] @@ -12648,7 +12670,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", "wasm-bindgen-shared", ] @@ -12682,7 +12704,7 @@ checksum = "e128beba882dd1eb6200e1dc92ae6c5dbaa4311aa7bb211ca035779e5efc39f8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -13417,7 +13439,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.27", ] [[package]] From e41a1612d319d7b491053200b0da04fa44b6714e Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Mon, 31 Jul 2023 15:08:02 +0200 Subject: [PATCH 34/47] chainspec::code field is better handled for raw specs --- client/chain-spec/res/raw_with_code.json | 19 ++ .../res/raw_with_code_no_encoded.json | 19 ++ .../substrate_test_runtime_from_config.json | 2 +- ...bstrate_test_runtime_from_config__raw.json | 4 +- .../substrate_test_runtime_from_patch.json | 2 +- ...ubstrate_test_runtime_from_patch__raw.json | 4 +- client/chain-spec/src/chain_spec.rs | 218 +++++++++++++----- 7 files changed, 205 insertions(+), 63 deletions(-) create mode 100644 client/chain-spec/res/raw_with_code.json create mode 100644 client/chain-spec/res/raw_with_code_no_encoded.json diff --git a/client/chain-spec/res/raw_with_code.json b/client/chain-spec/res/raw_with_code.json new file mode 100644 index 0000000000000..c12cec57ba7a2 --- /dev/null +++ b/client/chain-spec/res/raw_with_code.json @@ -0,0 +1,19 @@ +{ + "name": "TestName", + "id": "test_id", + "chainType": "Local", + "bootNodes": [], + "telemetryEndpoints": null, + "protocolId": null, + "properties": null, + "codeSubstitutes": {}, + "genesis": { + "raw": { + "top": { + "0x3a636f6465": "0x010101" + }, + "childrenDefault": {} + } + }, + "code": "0x060708" +} diff --git a/client/chain-spec/res/raw_with_code_no_encoded.json b/client/chain-spec/res/raw_with_code_no_encoded.json new file mode 100644 index 0000000000000..dc82ae6904b9c --- /dev/null +++ b/client/chain-spec/res/raw_with_code_no_encoded.json @@ -0,0 +1,19 @@ +{ + "name": "TestName", + "id": "test_id", + "chainType": "Local", + "bootNodes": [], + "telemetryEndpoints": null, + "protocolId": null, + "properties": null, + "codeSubstitutes": {}, + "genesis": { + "raw": { + "top": { + "0x3a65787472696e7369635f696e646578": "0x00000000" + }, + "childrenDefault": {} + } + }, + "code": "0x060708" +} diff --git a/client/chain-spec/res/substrate_test_runtime_from_config.json b/client/chain-spec/res/substrate_test_runtime_from_config.json index ae627cadf4326..7e4d491770196 100644 --- a/client/chain-spec/res/substrate_test_runtime_from_config.json +++ b/client/chain-spec/res/substrate_test_runtime_from_config.json @@ -122,5 +122,5 @@ "system": {} } }, - "code": "0x" + "code": "0x0" } diff --git a/client/chain-spec/res/substrate_test_runtime_from_config__raw.json b/client/chain-spec/res/substrate_test_runtime_from_config__raw.json index c667b22371d58..f033163ab19bc 100644 --- a/client/chain-spec/res/substrate_test_runtime_from_config__raw.json +++ b/client/chain-spec/res/substrate_test_runtime_from_config__raw.json @@ -42,12 +42,12 @@ "0x26aa394eea5630e07c48ae0c9558cef7b99d880ec681799c0cf30e8886371da9f5d6f1c082fe63eec7a71fcad00f4a892e3d43b7b0d04e776e69e7be35247cecdac65504c579195731eaf64b7940966e": "0x000000000000000000000000010000000000000000008a5d784563010000000000000000000000000000000000000000000000000000000000000080", "0x26aa394eea5630e07c48ae0c9558cef7b99d880ec681799c0cf30e8886371da9fbf0818841edf110e05228a6379763c4fc3c37459d9bdc61f58a5ebc01e9e2305a19d390c0543dc733861ec3cf1de01f": "0x000000000000000000000000010000000000000000008a5d784563010000000000000000000000000000000000000000000000000000000000000080", "0x26aa394eea5630e07c48ae0c9558cef7f9cce9c888469bb1a0dceaa129672ef8": "0x0000", + "0x3a636f6465": "0x0", "0x3a65787472696e7369635f696e646578": "0x00000000", "0xc2261276cc9d1f8598ea4b6a74b15c2f4e7b9012096b41c4eb3aaf947f6ea429": "0x0100", "0xc2261276cc9d1f8598ea4b6a74b15c2f57c875e4cff74148e4628f264b974c80": "0x00003ef1ee275e1a" }, "childrenDefault": {} } - }, - "code": "0x" + } } diff --git a/client/chain-spec/res/substrate_test_runtime_from_patch.json b/client/chain-spec/res/substrate_test_runtime_from_patch.json index c797fdda20799..2662020e6bad0 100644 --- a/client/chain-spec/res/substrate_test_runtime_from_patch.json +++ b/client/chain-spec/res/substrate_test_runtime_from_patch.json @@ -26,5 +26,5 @@ } } }, - "code": "0x" + "code": "0x0" } diff --git a/client/chain-spec/res/substrate_test_runtime_from_patch__raw.json b/client/chain-spec/res/substrate_test_runtime_from_patch__raw.json index 80668f20a36bd..9f4b2731649f3 100644 --- a/client/chain-spec/res/substrate_test_runtime_from_patch__raw.json +++ b/client/chain-spec/res/substrate_test_runtime_from_patch__raw.json @@ -21,12 +21,12 @@ "0x26aa394eea5630e07c48ae0c9558cef7a44704b568d21667356a5a050c118746bb1bdbcacd6ac9340000000000000000": "0x4545454545454545454545454545454545454545454545454545454545454545", "0x26aa394eea5630e07c48ae0c9558cef7a7fd6c28836b9a28522dc924110cf439": "0x01", "0x26aa394eea5630e07c48ae0c9558cef7f9cce9c888469bb1a0dceaa129672ef8": "0x0000", + "0x3a636f6465": "0x0", "0x3a65787472696e7369635f696e646578": "0x00000000", "0xc2261276cc9d1f8598ea4b6a74b15c2f4e7b9012096b41c4eb3aaf947f6ea429": "0x0100", "0xc2261276cc9d1f8598ea4b6a74b15c2f57c875e4cff74148e4628f264b974c80": "0x0000000000000000" }, "childrenDefault": {} } - }, - "code": "0x" + } } diff --git a/client/chain-spec/src/chain_spec.rs b/client/chain-spec/src/chain_spec.rs index 61d582228ac4d..23829d9bd69f1 100644 --- a/client/chain-spec/src/chain_spec.rs +++ b/client/chain-spec/src/chain_spec.rs @@ -40,9 +40,9 @@ use std::{ #[serde(rename_all = "camelCase")] enum GenesisBuildAction { #[serde(alias = "RuntimePatch")] - Patch(serde_json::Value), + Patch(json::Value), #[serde(alias = "Runtime")] - Full(serde_json::Value), + Full(json::Value), } #[allow(deprecated)] @@ -137,10 +137,6 @@ impl GenesisSource { impl BuildStorage for ChainSpec { fn assimilate_storage(&self, storage: &mut Storage) -> Result<(), String> { - storage - .top - .insert(sp_core::storage::well_known_keys::CODE.to_vec(), self.code.clone()); - match self.genesis.resolve()? { #[allow(deprecated)] Genesis::Runtime(gc) => gc.assimilate_storage(storage), @@ -169,6 +165,10 @@ impl BuildStorage for ChainSpec { .assimilate_storage(storage), }?; + storage + .top + .insert(sp_core::storage::well_known_keys::CODE.to_vec(), self.code.clone()); + Ok(()) } } @@ -216,9 +216,9 @@ enum Genesis { /// State root hash of the genesis storage. StateRootHash(StorageData), /// Full runtime genesis config. - RuntimeConfig(serde_json::Value), + RuntimeConfig(json::Value), /// Patch for default runtime genesis config. - RuntimeConfigPatch(serde_json::Value), + RuntimeConfigPatch(json::Value), } /// A configuration of a client. Does not include runtime storage initialization. @@ -348,12 +348,12 @@ impl ChainSpecBuilder { self } - pub fn with_genesis_config_patch(mut self, patch: serde_json::Value) -> Self { + pub fn with_genesis_config_patch(mut self, patch: json::Value) -> Self { self.genesis_build_action = Some(GenesisBuildAction::Patch(patch)); self } - pub fn with_genesis_config(mut self, config: serde_json::Value) -> Self { + pub fn with_genesis_config(mut self, config: json::Value) -> Self { self.genesis_build_action = Some(GenesisBuildAction::Full(config)); self } @@ -563,30 +563,45 @@ struct ChainSpecJsonContainer { #[serde(flatten)] client_spec: ClientSpec, genesis: Genesis, - #[serde(with = "sp_core::bytes")] + #[serde(with = "sp_core::bytes", skip_serializing_if = "Vec::is_empty")] code: Vec, } impl ChainSpec { fn json_container(&self, raw: bool) -> Result, String> { - let genesis = match (raw, self.genesis.resolve()?) { - (true, Genesis::RuntimeConfigPatch(patch)) => Genesis::Raw(RawGenesis::from( - RuntimeCaller::new(&self.code[..]).get_storage_for_patch(patch)?, - )), - (true, Genesis::RuntimeConfig(config)) => Genesis::Raw(RawGenesis::from( - RuntimeCaller::new(&self.code[..]).get_storage_for_config(config)?, - )), + let mut raw_genesis = match (raw, self.genesis.resolve()?) { + (true, Genesis::RuntimeConfigPatch(patch)) => { + let storage = RuntimeCaller::new(&self.code[..]).get_storage_for_patch(patch)?; + RawGenesis::from(storage) + }, + (true, Genesis::RuntimeConfig(config)) => { + let storage = RuntimeCaller::new(&self.code[..]).get_storage_for_config(config)?; + RawGenesis::from(storage) + }, #[allow(deprecated)] (true, Genesis::Runtime(g)) => { let storage = g.build_storage()?; - Genesis::Raw(RawGenesis::from(storage)) + RawGenesis::from(storage) }, - (_, genesis) => genesis, + (true, Genesis::Raw(raw)) => raw, + + (_, genesis) => + return Ok(ChainSpecJsonContainer { + client_spec: self.client_spec.clone(), + genesis, + code: self.code.clone(), + }), }; + + raw_genesis.top.insert( + StorageKey(sp_core::storage::well_known_keys::CODE.to_vec()), + StorageData(self.code.clone()), + ); + Ok(ChainSpecJsonContainer { client_spec: self.client_spec.clone(), - genesis, - code: self.code.clone(), + genesis: Genesis::Raw(raw_genesis), + code: vec![], }) } @@ -674,9 +689,11 @@ where #[cfg(test)] mod tests { use super::*; - use serde_json::json; + use serde_json::{from_str, json, Value}; use sp_application_crypto::Ss58Codec; + use sp_core::storage::well_known_keys; use sp_keyring::AccountKeyring; + use std::collections::VecDeque; #[derive(Debug, Serialize, Deserialize)] struct Genesis(BTreeMap); @@ -769,16 +786,38 @@ mod tests { } } - fn remove_code_key_from_json(json: &str) -> serde_json::Value { - serde_json::from_str::(json) - .map(|mut c| { - c.as_object_mut().map(|j| { - j.remove_entry("code").unwrap(); - // .expect("code field should be in blob returned by ChainSpec.as_json"); - }); - c + fn json_eval_value_at_key( + doc: &Value, + path: &mut VecDeque, + fun: &dyn Fn(&Value) -> bool, + ) -> bool { + if path.len() == 1 { + doc.as_object().map_or(false, |o| o.get(&path[0]).map_or(false, |v| fun(v))) + } else { + let key = path.pop_front().unwrap(); + doc.as_object().map_or(false, |o| { + o.get(&key).map_or(false, |v| json_eval_value_at_key(v, path, fun)) }) - .unwrap() + } + } + + fn json_contains_path(doc: &Value, path: &mut VecDeque) -> bool { + json_eval_value_at_key(doc, path, &|_| true) + } + + fn zeroize_code_key_in_json(encoded: bool, json: &str) -> Value { + let mut json = from_str::(json).unwrap(); + let (zeroing_patch, mut path) = if encoded { + ( + json!({"genesis":{"raw":{"top":{"0x3a636f6465":"0x0"}}}}), + ["genesis", "raw", "top", "0x3a636f6465"].map(String::from).into(), + ) + } else { + (json!({"code":"0x0"}), ["code"].map(String::from).into()) + }; + assert!(json_contains_path(&json, &mut path)); + json_patch::merge(&mut json, &zeroing_patch); + json } #[test] @@ -811,19 +850,20 @@ mod tests { // std::fs::write("/tmp/patch.json", output.as_json(false).unwrap()); // std::fs::write("/tmp/patch_raw.json", output.as_json(true).unwrap()); - let acutal = output.as_json(false).unwrap(); - let acutal_raw = output.as_json(true).unwrap(); - let expected = include_str!("../res/substrate_test_runtime_from_patch.json"); - let expected_raw = include_str!("../res/substrate_test_runtime_from_patch__raw.json"); + let actual = output.as_json(false).unwrap(); + let actual_raw = output.as_json(true).unwrap(); - //'code' may chang so let's remove it. Only ensure it is there: - let actual = remove_code_key_from_json(acutal.as_str()); - let actual_raw = remove_code_key_from_json(acutal_raw.as_str()); + let expected = + from_str::(include_str!("../res/substrate_test_runtime_from_patch.json")) + .unwrap(); + let expected_raw = + from_str::(include_str!("../res/substrate_test_runtime_from_patch__raw.json")) + .unwrap(); - // Removing key changes order of keys, so modify expected too: - let expected = remove_code_key_from_json(expected); - let expected_raw = remove_code_key_from_json(expected_raw); + //wasm blob may change overtime so let's zero it. Also ensure it is there: + let actual = zeroize_code_key_in_json(false, actual.as_str()); + let actual_raw = zeroize_code_key_in_json(true, actual_raw.as_str()); assert_eq!(actual, expected); assert_eq!(actual_raw, expected_raw); @@ -840,25 +880,25 @@ mod tests { .with_extensions(Default::default()) .with_chain_type(ChainType::Local) .with_code(substrate_test_runtime::wasm_binary_unwrap().into()) - .with_genesis_config(serde_json::from_str(j).unwrap()) + .with_genesis_config(from_str(j).unwrap()) .build(); // std::fs::write("/tmp/config.json", output.as_json(false).unwrap()); // std::fs::write("/tmp/config_raw.json", output.as_json(true).unwrap()); - let acutal = output.as_json(false).unwrap(); - let acutal_raw = output.as_json(true).unwrap(); + let actual = output.as_json(false).unwrap(); + let actual_raw = output.as_json(true).unwrap(); - let expected = include_str!("../res/substrate_test_runtime_from_config.json"); - let expected_raw = include_str!("../res/substrate_test_runtime_from_config__raw.json"); + let expected = + from_str::(include_str!("../res/substrate_test_runtime_from_config.json")) + .unwrap(); + let expected_raw = + from_str::(include_str!("../res/substrate_test_runtime_from_config__raw.json")) + .unwrap(); - //'code' may chang so let's remove it. Only ensure it is there: - let actual = remove_code_key_from_json(acutal.as_str()); - let actual_raw = remove_code_key_from_json(acutal_raw.as_str()); - - // Removing key changes order of keys, so modify expected too: - let expected = remove_code_key_from_json(expected); - let expected_raw = remove_code_key_from_json(expected_raw); + //wasm blob may change overtime so let's zero it. Also ensure it is there: + let actual = zeroize_code_key_in_json(false, actual.as_str()); + let actual_raw = zeroize_code_key_in_json(true, actual_raw.as_str()); assert_eq!(actual, expected); assert_eq!(actual_raw, expected_raw); @@ -868,7 +908,6 @@ mod tests { fn chain_spec_as_json_fails_with_invalid_config() { sp_tracing::try_init_simple(); let j = include_str!( - // "../../../test-utils/runtime/src/test_json/default_genesis_config_invalid_2.json" "../../../test-utils/runtime/src/test_json/default_genesis_config_invalid_2.json" ); let output: ChainSpec<()> = ChainSpecBuilder::new() @@ -877,10 +916,13 @@ mod tests { .with_extensions(Default::default()) .with_chain_type(ChainType::Local) .with_code(substrate_test_runtime::wasm_binary_unwrap().into()) - .with_genesis_config(serde_json::from_str(j).unwrap()) + .with_genesis_config(from_str(j).unwrap()) .build(); - assert_eq!( output.as_json(true), Err("Invalid JSON blob: unknown field `babex`, expected one of `system`, `babe`, `substrateTest`, `balances` at line 1 column 8".to_string()) ); + assert_eq!( + output.as_json(true), + Err("Invalid JSON blob: unknown field `babex`, expected one of `system`, `babe`, `substrateTest`, `balances` at line 1 column 8".to_string()) + ); } #[test] @@ -905,4 +947,66 @@ mod tests { assert!(output.as_json(true).unwrap_err().contains("Invalid JSON blob: unknown field `invalid_pallet`, expected one of `system`, `babe`, `substrateTest`, `balances`")); } + + #[test] + fn check_if_code_is_removed_from_raw_with_encoded() { + let spec = ChainSpec::<()>::from_json_bytes(Cow::Owned( + include_bytes!("../res/raw_with_code.json").to_vec(), + )) + .unwrap(); + + let j = from_str::(&spec.as_json(true).unwrap()).unwrap(); + + let mut path = VecDeque::from(["genesis", "raw", "top", "0x3a636f6465"].map(String::from)); + assert!(json_eval_value_at_key(&j, &mut path, &|v| { *v == "0x060708" })); + + let mut path = ["code"].map(String::from).into(); + assert!(!json_contains_path(&j, &mut path)); + } + + #[test] + fn check_if_code_is_removed_from_raw_without_encoded() { + let spec = ChainSpec::<()>::from_json_bytes(Cow::Owned( + include_bytes!("../res/raw_with_code_no_encoded.json").to_vec(), + )) + .unwrap(); + + let j = from_str::(&spec.as_json(true).unwrap()).unwrap(); + + let mut path = ["genesis", "raw", "top", "0x3a636f6465"].map(String::from).into(); + assert!(json_eval_value_at_key(&j, &mut path, &|v| { *v == "0x060708" })); + + let mut path = ["code"].map(String::from).into(); + assert!(!json_contains_path(&j, &mut path)); + } + + #[test] + fn check_code_in_assimilated_storage_for_raw_with_encoded() { + let spec = ChainSpec::<()>::from_json_bytes(Cow::Owned( + include_bytes!("../res/raw_with_code.json").to_vec(), + )) + .unwrap(); + + let storage = spec.build_storage().unwrap(); + assert!(storage + .top + .get(&well_known_keys::CODE.to_vec()) + .map(|v| *v == vec![6, 7, 8]) + .unwrap()) + } + + #[test] + fn check_code_in_assimilated_storage_for_raw_without_encoded() { + let spec = ChainSpec::<()>::from_json_bytes(Cow::Owned( + include_bytes!("../res/raw_with_code_no_encoded.json").to_vec(), + )) + .unwrap(); + + let storage = spec.build_storage().unwrap(); + assert!(storage + .top + .get(&well_known_keys::CODE.to_vec()) + .map(|v| *v == vec![6, 7, 8]) + .unwrap()) + } } From 991f9fa305e1aa2b2c80c682d166670ad4f66afe Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Mon, 31 Jul 2023 15:25:58 +0200 Subject: [PATCH 35/47] code field improvements --- client/chain-spec/res/raw_no_code.json | 18 ++++++++++ client/chain-spec/src/chain_spec.rs | 50 ++++++++++++++++++++++---- 2 files changed, 61 insertions(+), 7 deletions(-) create mode 100644 client/chain-spec/res/raw_no_code.json diff --git a/client/chain-spec/res/raw_no_code.json b/client/chain-spec/res/raw_no_code.json new file mode 100644 index 0000000000000..bac7c25582f96 --- /dev/null +++ b/client/chain-spec/res/raw_no_code.json @@ -0,0 +1,18 @@ +{ + "name": "TestName", + "id": "test_id", + "chainType": "Local", + "bootNodes": [], + "telemetryEndpoints": null, + "protocolId": null, + "properties": null, + "codeSubstitutes": {}, + "genesis": { + "raw": { + "top": { + "0x3a636f6465": "0x010101" + }, + "childrenDefault": {} + } + } +} diff --git a/client/chain-spec/src/chain_spec.rs b/client/chain-spec/src/chain_spec.rs index 23829d9bd69f1..caa3b01e789eb 100644 --- a/client/chain-spec/src/chain_spec.rs +++ b/client/chain-spec/src/chain_spec.rs @@ -165,9 +165,11 @@ impl BuildStorage for ChainSpec { .assimilate_storage(storage), }?; - storage - .top - .insert(sp_core::storage::well_known_keys::CODE.to_vec(), self.code.clone()); + if !self.code.is_empty() { + storage + .top + .insert(sp_core::storage::well_known_keys::CODE.to_vec(), self.code.clone()); + } Ok(()) } @@ -593,10 +595,12 @@ impl ChainSpec { }), }; - raw_genesis.top.insert( - StorageKey(sp_core::storage::well_known_keys::CODE.to_vec()), - StorageData(self.code.clone()), - ); + if !self.code.is_empty() { + raw_genesis.top.insert( + StorageKey(sp_core::storage::well_known_keys::CODE.to_vec()), + StorageData(self.code.clone()), + ); + } Ok(ChainSpecJsonContainer { client_spec: self.client_spec.clone(), @@ -948,6 +952,23 @@ mod tests { assert!(output.as_json(true).unwrap_err().contains("Invalid JSON blob: unknown field `invalid_pallet`, expected one of `system`, `babe`, `substrateTest`, `balances`")); } + #[test] + fn check_if_code_is_valid_for_raw_without_code() { + let spec = ChainSpec::<()>::from_json_bytes(Cow::Owned( + include_bytes!("../res/raw_no_code.json").to_vec(), + )) + .unwrap(); + + let j = from_str::(&spec.as_json(true).unwrap()).unwrap(); + std::fs::write("/tmp/11.json", serde_json::to_string_pretty(&j).unwrap()); + + let mut path = VecDeque::from(["genesis", "raw", "top", "0x3a636f6465"].map(String::from)); + assert!(json_eval_value_at_key(&j, &mut path, &|v| { *v == "0x010101" })); + + let mut path = ["code"].map(String::from).into(); + assert!(!json_contains_path(&j, &mut path)); + } + #[test] fn check_if_code_is_removed_from_raw_with_encoded() { let spec = ChainSpec::<()>::from_json_bytes(Cow::Owned( @@ -1009,4 +1030,19 @@ mod tests { .map(|v| *v == vec![6, 7, 8]) .unwrap()) } + + #[test] + fn check_code_in_assimilated_storage_for_raw_without_code() { + let spec = ChainSpec::<()>::from_json_bytes(Cow::Owned( + include_bytes!("../res/raw_no_code.json").to_vec(), + )) + .unwrap(); + + let storage = spec.build_storage().unwrap(); + assert!(storage + .top + .get(&well_known_keys::CODE.to_vec()) + .map(|v| *v == vec![1, 1, 1]) + .unwrap()) + } } From 0258f52f279e5a2f96802d89123aaf2d32491d3b Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Mon, 31 Jul 2023 22:24:31 +0200 Subject: [PATCH 36/47] doc + naming --- .../substrate_test_runtime_from_config.json | 2 +- .../substrate_test_runtime_from_patch.json | 2 +- client/chain-spec/src/chain_spec.rs | 118 +++++++++++++----- .../chain-spec/src/genesis_config_builder.rs | 24 ++-- 4 files changed, 93 insertions(+), 53 deletions(-) diff --git a/client/chain-spec/res/substrate_test_runtime_from_config.json b/client/chain-spec/res/substrate_test_runtime_from_config.json index 7e4d491770196..cb55d763512dc 100644 --- a/client/chain-spec/res/substrate_test_runtime_from_config.json +++ b/client/chain-spec/res/substrate_test_runtime_from_config.json @@ -8,7 +8,7 @@ "properties": null, "codeSubstitutes": {}, "genesis": { - "runtimeConfig": { + "runtimeGenesisConfig": { "babe": { "authorities": [ [ diff --git a/client/chain-spec/res/substrate_test_runtime_from_patch.json b/client/chain-spec/res/substrate_test_runtime_from_patch.json index 2662020e6bad0..4b90844d6f5ba 100644 --- a/client/chain-spec/res/substrate_test_runtime_from_patch.json +++ b/client/chain-spec/res/substrate_test_runtime_from_patch.json @@ -8,7 +8,7 @@ "properties": null, "codeSubstitutes": {}, "genesis": { - "runtimeConfigPatch": { + "runtimeGenesisConfigPatch": { "babe": { "epochConfig": { "allowed_slots": "PrimaryAndSecondaryPlainSlots", diff --git a/client/chain-spec/src/chain_spec.rs b/client/chain-spec/src/chain_spec.rs index caa3b01e789eb..3adfed90b3232 100644 --- a/client/chain-spec/src/chain_spec.rs +++ b/client/chain-spec/src/chain_spec.rs @@ -39,7 +39,6 @@ use std::{ #[derive(Serialize, Deserialize, Clone)] #[serde(rename_all = "camelCase")] enum GenesisBuildAction { - #[serde(alias = "RuntimePatch")] Patch(json::Value), #[serde(alias = "Runtime")] Full(json::Value), @@ -128,9 +127,9 @@ impl GenesisSource { }, // maybe a Factory for getting GenesisBuilderApi command? Self::GenesisBuilderApi(GenesisBuildAction::Full(config)) => - Ok(Genesis::RuntimeConfig(config.clone())), + Ok(Genesis::RuntimeGenesisConfig(config.clone())), Self::GenesisBuilderApi(GenesisBuildAction::Patch(patch)) => - Ok(Genesis::RuntimeConfigPatch(patch.clone())), + Ok(Genesis::RuntimeGenesisConfigPatch(patch.clone())), } } } @@ -157,10 +156,10 @@ impl BuildStorage for ChainSpec { // it, but Substrate itself isn't capable of loading chain specs with just a hash at the // moment. Genesis::StateRootHash(_) => Err("Genesis storage in hash format not supported".into()), - Genesis::RuntimeConfig(config) => RuntimeCaller::new(&self.code[..]) + Genesis::RuntimeGenesisConfig(config) => RuntimeCaller::new(&self.code[..]) .get_storage_for_config(config)? .assimilate_storage(storage), - Genesis::RuntimeConfigPatch(patch) => RuntimeCaller::new(&self.code[..]) + Genesis::RuntimeGenesisConfigPatch(patch) => RuntimeCaller::new(&self.code[..]) .get_storage_for_patch(patch)? .assimilate_storage(storage), }?; @@ -218,9 +217,9 @@ enum Genesis { /// State root hash of the genesis storage. StateRootHash(StorageData), /// Full runtime genesis config. - RuntimeConfig(json::Value), + RuntimeGenesisConfig(json::Value), /// Patch for default runtime genesis config. - RuntimeConfigPatch(json::Value), + RuntimeGenesisConfigPatch(json::Value), } /// A configuration of a client. Does not include runtime storage initialization. @@ -265,7 +264,7 @@ struct ClientSpec { /// We use `Option` here since `()` is not flattenable by serde. pub type NoExtension = Option<()>; -#[allow(missing_docs)] +/// Builder for creating [`ChainSpec`] instances. pub struct ChainSpecBuilder { name: Option, id: Option, @@ -281,8 +280,8 @@ pub struct ChainSpecBuilder { _genesis: PhantomData, } -#[allow(missing_docs)] //todo: add doc! impl ChainSpecBuilder { + /// Creates new builder instance with no defaults. pub fn new() -> Self { Self { name: None, @@ -300,66 +299,79 @@ impl ChainSpecBuilder { } } + /// Sets spec name. Must be called. pub fn with_name(mut self, name: &str) -> Self { self.name = Some(name.into()); self } + /// Sets spec id. Must be called. pub fn with_id(mut self, id: &str) -> Self { self.id = Some(id.into()); self } + /// Sets type of the chain. Must be called. pub fn with_chain_type(mut self, chain_type: ChainType) -> Self { self.chain_type = Some(chain_type); self } + /// Sets a list of bootnode addresses. pub fn with_boot_nodes(mut self, boot_nodes: Vec) -> Self { self.boot_nodes = Some(boot_nodes); self } + /// Sets telemetry endpoints. pub fn with_telemetry_endpoints(mut self, telemetry_endpoints: TelemetryEndpoints) -> Self { self.telemetry_endpoints = Some(telemetry_endpoints); self } + /// Sets network protocol id. pub fn with_protocol_id(mut self, protocol_id: &str) -> Self { self.protocol_id = Some(protocol_id.into()); self } + /// Sets optional network fork identifier. pub fn with_fork_id(mut self, fork_id: &str) -> Self { self.fork_id = Some(fork_id.into()); self } + /// Sets additional loosly-typed properties of the chain. pub fn with_properties(mut self, properties: Properties) -> Self { self.properties = Some(properties); self } + /// Sets chain spec extensions. Must be called. pub fn with_extensions(mut self, extensions: E) -> Self { self.extensions = Some(extensions); self } + /// Sets the code. Must be called. pub fn with_code(mut self, code: &[u8]) -> Self { self.code = Some(code.into()); self } + /// Sets the JSON patch for runtime's GenesisConfig. pub fn with_genesis_config_patch(mut self, patch: json::Value) -> Self { self.genesis_build_action = Some(GenesisBuildAction::Patch(patch)); self } + /// Sets the full runtime's GenesisConfig JSON. pub fn with_genesis_config(mut self, config: json::Value) -> Self { self.genesis_build_action = Some(GenesisBuildAction::Full(config)); self } + /// Builds the [`ChainSpec`] instance using provided settings pub fn build(self) -> ChainSpec { let client_spec = ClientSpec { name: self.name.expect("with_name must be called."), @@ -572,11 +584,11 @@ struct ChainSpecJsonContainer { impl ChainSpec { fn json_container(&self, raw: bool) -> Result, String> { let mut raw_genesis = match (raw, self.genesis.resolve()?) { - (true, Genesis::RuntimeConfigPatch(patch)) => { + (true, Genesis::RuntimeGenesisConfigPatch(patch)) => { let storage = RuntimeCaller::new(&self.code[..]).get_storage_for_patch(patch)?; RawGenesis::from(storage) }, - (true, Genesis::RuntimeConfig(config)) => { + (true, Genesis::RuntimeGenesisConfig(config)) => { let storage = RuntimeCaller::new(&self.code[..]).get_storage_for_config(config)?; RawGenesis::from(storage) }, @@ -790,7 +802,26 @@ mod tests { } } - fn json_eval_value_at_key( + macro_rules! json_path { + [ $($x:expr),+ ] => { + VecDeque::::from([$($x),+].map(String::from)) + }; + } + + /// The `fun` will be called with the value at `path`. + /// + /// If exists, the value at given `path` will be passed to the `fun` and the result of `fun` + /// call will be returned. Otherwise false is returned. + /// `path` will be modified. + /// + /// # Examples + /// ``` + /// use serde_json::{from_str, json, Value}; + /// let doc = json!({"a":{"b":{"c":"5"}}}); + /// let mut path = ["a", "b", "c"].map(String::from).into(); + /// assert!(json_eval_value_at_key(&doc, &mut path, &|v| { assert_eq!(v,"5"); true })); + /// ``` + pub fn json_eval_value_at_key( doc: &Value, path: &mut VecDeque, fun: &dyn Fn(&Value) -> bool, @@ -809,15 +840,36 @@ mod tests { json_eval_value_at_key(doc, path, &|_| true) } + #[test] + // some tests for json path utils + fn test_json_eval_value_at_key() { + let doc = json!({"a":{"b1":"20","b":{"c":{"d":"10"}}}}); + + assert!(json_eval_value_at_key(&doc, &mut json_path!["a", "b1"], &|v| { *v == "20" })); + assert!(json_eval_value_at_key(&doc, &mut json_path!["a", "b", "c", "d"], &|v| { + *v == "10" + })); + assert!(!json_eval_value_at_key(&doc, &mut json_path!["a", "c", "d"], &|_| { true })); + assert!(!json_eval_value_at_key(&doc, &mut json_path!["d"], &|_| { true })); + + assert!(json_contains_path(&doc, &mut json_path!["a", "b1"])); + assert!(json_contains_path(&doc, &mut json_path!["a", "b"])); + assert!(json_contains_path(&doc, &mut json_path!["a", "b", "c"])); + assert!(json_contains_path(&doc, &mut json_path!["a", "b", "c", "d"])); + assert!(!json_contains_path(&doc, &mut json_path!["a", "b", "c", "d", "e"])); + assert!(!json_contains_path(&doc, &mut json_path!["a", "b", "b1"])); + assert!(!json_contains_path(&doc, &mut json_path!["d"])); + } + fn zeroize_code_key_in_json(encoded: bool, json: &str) -> Value { let mut json = from_str::(json).unwrap(); let (zeroing_patch, mut path) = if encoded { ( json!({"genesis":{"raw":{"top":{"0x3a636f6465":"0x0"}}}}), - ["genesis", "raw", "top", "0x3a636f6465"].map(String::from).into(), + json_path!["genesis", "raw", "top", "0x3a636f6465"], ) } else { - (json!({"code":"0x0"}), ["code"].map(String::from).into()) + (json!({"code":"0x0"}), json_path!["code"]) }; assert!(json_contains_path(&json, &mut path)); json_patch::merge(&mut json, &zeroing_patch); @@ -852,9 +904,6 @@ mod tests { })) .build(); - // std::fs::write("/tmp/patch.json", output.as_json(false).unwrap()); - // std::fs::write("/tmp/patch_raw.json", output.as_json(true).unwrap()); - let actual = output.as_json(false).unwrap(); let actual_raw = output.as_json(true).unwrap(); @@ -887,9 +936,6 @@ mod tests { .with_genesis_config(from_str(j).unwrap()) .build(); - // std::fs::write("/tmp/config.json", output.as_json(false).unwrap()); - // std::fs::write("/tmp/config_raw.json", output.as_json(true).unwrap()); - let actual = output.as_json(false).unwrap(); let actual_raw = output.as_json(true).unwrap(); @@ -960,13 +1006,13 @@ mod tests { .unwrap(); let j = from_str::(&spec.as_json(true).unwrap()).unwrap(); - std::fs::write("/tmp/11.json", serde_json::to_string_pretty(&j).unwrap()); - - let mut path = VecDeque::from(["genesis", "raw", "top", "0x3a636f6465"].map(String::from)); - assert!(json_eval_value_at_key(&j, &mut path, &|v| { *v == "0x010101" })); - let mut path = ["code"].map(String::from).into(); - assert!(!json_contains_path(&j, &mut path)); + assert!(json_eval_value_at_key( + &j, + &mut json_path!["genesis", "raw", "top", "0x3a636f6465"], + &|v| { *v == "0x010101" } + )); + assert!(!json_contains_path(&j, &mut json_path!["code"])); } #[test] @@ -978,11 +1024,13 @@ mod tests { let j = from_str::(&spec.as_json(true).unwrap()).unwrap(); - let mut path = VecDeque::from(["genesis", "raw", "top", "0x3a636f6465"].map(String::from)); - assert!(json_eval_value_at_key(&j, &mut path, &|v| { *v == "0x060708" })); + assert!(json_eval_value_at_key( + &j, + &mut json_path!["genesis", "raw", "top", "0x3a636f6465"], + &|v| { *v == "0x060708" } + )); - let mut path = ["code"].map(String::from).into(); - assert!(!json_contains_path(&j, &mut path)); + assert!(!json_contains_path(&j, &mut json_path!["code"])); } #[test] @@ -994,11 +1042,13 @@ mod tests { let j = from_str::(&spec.as_json(true).unwrap()).unwrap(); - let mut path = ["genesis", "raw", "top", "0x3a636f6465"].map(String::from).into(); - assert!(json_eval_value_at_key(&j, &mut path, &|v| { *v == "0x060708" })); + assert!(json_eval_value_at_key( + &j, + &mut json_path!["genesis", "raw", "top", "0x3a636f6465"], + &|v| { *v == "0x060708" } + )); - let mut path = ["code"].map(String::from).into(); - assert!(!json_contains_path(&j, &mut path)); + assert!(!json_contains_path(&j, &mut json_path!["code"])); } #[test] diff --git a/client/chain-spec/src/genesis_config_builder.rs b/client/chain-spec/src/genesis_config_builder.rs index d132cfb74654e..92d32a0c6f5f8 100644 --- a/client/chain-spec/src/genesis_config_builder.rs +++ b/client/chain-spec/src/genesis_config_builder.rs @@ -70,11 +70,7 @@ impl<'a> GenesisConfigBuilderRuntimeCaller<'a> { } impl<'a> GenesisConfigBuilderRuntimeCaller<'a> { - /// Get the default `GenesisConfig` as a JSON blob. - /// - /// This function instantiates the default `GenesisConfig` struct for the runtime and serializes - /// it into a JSON blob. It returns a `Vec` containing the JSON representation of the - /// default `GenesisConfig`. + /// Calls [`sp_genesis_builder::GenesisBuilder::create_default_config`] provided by runtime. pub fn get_default_config(&self) -> core::result::Result { let mut t = BasicExternalities::new_empty(); let call_result = self @@ -85,14 +81,7 @@ impl<'a> GenesisConfigBuilderRuntimeCaller<'a> { Ok(from_slice(&default_config[..]).expect("returned value is json. qed.")) } - /// Build `GenesisConfig` from a JSON blob not using any defaults and store it in the storage. - /// - /// This function deserializes the full `GenesisConfig` from the given JSON blob and puts it - /// into the storage. If the provided JSON blob is incorrect or incomplete or the - /// deserialization fails, an error is returned. - /// - /// Please note that provided json blob must contain all `GenesisConfig` fields, no defaults - /// will be used. + /// Calls [`sp_genesis_builder::GenesisBuilder::build_config`] provided by runtime. pub fn get_storage_for_config(&self, config: Value) -> core::result::Result { let mut ext = BasicExternalities::new_empty(); @@ -118,10 +107,11 @@ impl<'a> GenesisConfigBuilderRuntimeCaller<'a> { /// /// The patching process modifies the default `GenesisConfig` according to the following rules: /// 1. Existing keys in the default configuration will be overridden by the corresponding values - /// in the patch. 2. If a key exists in the patch but not in the default configuration, it will - /// be added to the resulting `GenesisConfig`. 3. Keys in the default configuration that have - /// null values in the patch will be removed from the resulting `GenesisConfig`. This is - /// helpful for changing enum variant value. + /// in the patch. + /// 2. If a key exists in the patch but not in the default configuration, it will be added to + /// the resulting `GenesisConfig`. + /// 3. Keys in the default configuration that have null values in the patch will be removed from + /// the resulting `GenesisConfig`. This is helpful for changing enum variant value. /// /// Please note that the patch may contain full `GenesisConfig`. pub fn get_storage_for_patch(&self, patch: Value) -> core::result::Result { From 20bee680ed098be7239cf7a6b804cd4de267983e Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Tue, 1 Aug 2023 12:28:12 +0200 Subject: [PATCH 37/47] doc improvements --- client/chain-spec/src/chain_spec.rs | 73 ++++++++----------- .../chain-spec/src/genesis_config_builder.rs | 14 ++-- client/chain-spec/src/lib.rs | 9 ++- 3 files changed, 46 insertions(+), 50 deletions(-) diff --git a/client/chain-spec/src/chain_spec.rs b/client/chain-spec/src/chain_spec.rs index 3adfed90b3232..5beec136f1803 100644 --- a/client/chain-spec/src/chain_spec.rs +++ b/client/chain-spec/src/chain_spec.rs @@ -40,7 +40,6 @@ use std::{ #[serde(rename_all = "camelCase")] enum GenesisBuildAction { Patch(json::Value), - #[serde(alias = "Runtime")] Full(json::Value), } @@ -71,6 +70,7 @@ impl Clone for GenesisSource { impl GenesisSource { fn resolve(&self) -> Result, String> { + /// helper container for deserializeing genesis from the JSON file (ChainSpec JSON file is also supported here) #[derive(Serialize, Deserialize)] struct GenesisContainer { genesis: Genesis, @@ -92,40 +92,16 @@ impl GenesisSource { let genesis: GenesisContainer = json::from_slice(&bytes) .map_err(|e| format!("Error parsing spec file: {}", e))?; - Ok(genesis.genesis) //GenesisRuntime (typically) + Ok(genesis.genesis) }, Self::Binary(buf) => { let genesis: GenesisContainer = json::from_reader(buf.as_ref()) .map_err(|e| format!("Error parsing embedded file: {}", e))?; - Ok(genesis.genesis) //GenesisRuntime (typically) + Ok(genesis.genesis) }, #[allow(deprecated)] Self::Factory(f) => Ok(Genesis::Runtime(f())), - Self::Storage(storage) => { - let top = storage - .top - .iter() - .map(|(k, v)| (StorageKey(k.clone()), StorageData(v.clone()))) - .collect(); - - let children_default = storage - .children_default - .iter() - .map(|(k, child)| { - ( - StorageKey(k.clone()), - child - .data - .iter() - .map(|(k, v)| (StorageKey(k.clone()), StorageData(v.clone()))) - .collect(), - ) - }) - .collect(); - - Ok(Genesis::Raw(RawGenesis { top, children_default })) - }, - // maybe a Factory for getting GenesisBuilderApi command? + Self::Storage(storage) => Ok(Genesis::Raw(RawGenesis::from(storage.clone()))), Self::GenesisBuilderApi(GenesisBuildAction::Full(config)) => Ok(Genesis::RuntimeGenesisConfig(config.clone())), Self::GenesisBuilderApi(GenesisBuildAction::Patch(patch)) => @@ -207,18 +183,23 @@ impl From for RawGenesis { } } +/// Represents different options for the GenesisConfig configuration. #[derive(Serialize, Deserialize)] #[serde(rename_all = "camelCase")] #[serde(deny_unknown_fields)] enum Genesis { - /// note: this will be removed together with [`ChainSpec::from_genesis`] + /// [Deprecated] Contains the JSON representation of G (the natuve type represting the runtime GenesisConfig struct) + /// (will be removed with `ChainSpec::from_genesis`). Runtime(G), + /// The genesis storage as raw data. Raw(RawGenesis), /// State root hash of the genesis storage. StateRootHash(StorageData), - /// Full runtime genesis config. + /// Represents the full runtime genesis config in JSON format. + /// The contained object is a JSON blob that can be parsed by a compatible runtime. RuntimeGenesisConfig(json::Value), - /// Patch for default runtime genesis config. + /// Represents a patch for the default runtime genesis config in JSON format. + /// The contained value is a JSON object that can be parsed by a compatible runtime. RuntimeGenesisConfigPatch(json::Value), } @@ -281,7 +262,7 @@ pub struct ChainSpecBuilder { } impl ChainSpecBuilder { - /// Creates new builder instance with no defaults. + /// Creates a new builder instance with no defaults. pub fn new() -> Self { Self { name: None, @@ -299,19 +280,19 @@ impl ChainSpecBuilder { } } - /// Sets spec name. Must be called. + /// Sets the spec name. This method must be called. pub fn with_name(mut self, name: &str) -> Self { self.name = Some(name.into()); self } - /// Sets spec id. Must be called. + /// Sets the spec ID. This method must be called. pub fn with_id(mut self, id: &str) -> Self { self.id = Some(id.into()); self } - /// Sets type of the chain. Must be called. + /// Sets the type of the chain. This method must be called. pub fn with_chain_type(mut self, chain_type: ChainType) -> Self { self.chain_type = Some(chain_type); self @@ -329,31 +310,31 @@ impl ChainSpecBuilder { self } - /// Sets network protocol id. + /// Sets the network protocol ID. pub fn with_protocol_id(mut self, protocol_id: &str) -> Self { self.protocol_id = Some(protocol_id.into()); self } - /// Sets optional network fork identifier. + /// Sets an optional network fork identifier. pub fn with_fork_id(mut self, fork_id: &str) -> Self { self.fork_id = Some(fork_id.into()); self } - /// Sets additional loosly-typed properties of the chain. + /// Sets additional loosely-typed properties of the chain. pub fn with_properties(mut self, properties: Properties) -> Self { self.properties = Some(properties); self } - /// Sets chain spec extensions. Must be called. + /// Sets chain spec extensions. This method must be called. pub fn with_extensions(mut self, extensions: E) -> Self { self.extensions = Some(extensions); self } - /// Sets the code. Must be called. + /// Sets the code. This method must be called. pub fn with_code(mut self, code: &[u8]) -> Self { self.code = Some(code.into()); self @@ -371,7 +352,7 @@ impl ChainSpecBuilder { self } - /// Builds the [`ChainSpec`] instance using provided settings + /// Builds a [`ChainSpec`] instance using the provided settings. pub fn build(self) -> ChainSpec { let client_spec = ClientSpec { name: self.name.expect("with_name must be called."), @@ -528,6 +509,7 @@ impl ChainSpec { } } +/// Helper structure for deserializing optional `code` attribute from JSON file. #[derive(Serialize, Deserialize)] struct CodeContainer { #[serde(default, with = "sp_core::bytes")] @@ -571,6 +553,8 @@ impl ChainSpec { @@ -621,7 +605,12 @@ impl ChainSpec { }) } - /// Dump to json string. + /// Dump the configuration to JSON string. + /// + /// During conversion to `raw` format, the `ChainSpec::code` field will be removed and placed into `RawGenesis` as + /// `genesis::top::raw::0x3a636f6465` (which is [`sp_core::storage::well_known_keys::CODE`]). If the spec is already + /// in `raw` format, and contains `genesis::top::raw::0x3a636f6465` field it will be updated with content of `code` + /// field. pub fn as_json(&self, raw: bool) -> Result { let container = self.json_container(raw)?; json::to_string_pretty(&container).map_err(|e| format!("Error generating spec json: {}", e)) diff --git a/client/chain-spec/src/genesis_config_builder.rs b/client/chain-spec/src/genesis_config_builder.rs index 92d32a0c6f5f8..df6e50d65fa81 100644 --- a/client/chain-spec/src/genesis_config_builder.rs +++ b/client/chain-spec/src/genesis_config_builder.rs @@ -16,12 +16,10 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -///! Helper for calling GenesisBuilder API from arbitrary wasm blob. -use std::borrow::Cow; +//! A helper module for calling the GenesisBuilder API from arbitrary runtime wasm blobs. use codec::{Decode, Encode}; use sc_executor::{error::Result, WasmExecutor}; -// use sc_executor_common::runtime_blob::RuntimeBlob; use serde_json::{from_slice, Value}; use sp_core::{ storage::Storage, @@ -29,8 +27,9 @@ use sp_core::{ }; use sp_genesis_builder::Result as BuildResult; use sp_state_machine::BasicExternalities; +use std::borrow::Cow; -/// Util that allows to call GenesisBuilder API from the runtime wasm code blob. +/// A utility that facilitates calling the GenesisBuilder API from the runtime wasm code blob. pub struct GenesisConfigBuilderRuntimeCaller<'a> { code: Cow<'a, [u8]>, code_hash: Vec, @@ -44,7 +43,7 @@ impl<'a> FetchRuntimeCode for GenesisConfigBuilderRuntimeCaller<'a> { } impl<'a> GenesisConfigBuilderRuntimeCaller<'a> { - /// Creates new instance using provided code blob. + /// Creates new instance using the provided code blob. pub fn new(code: &'a [u8]) -> Self { GenesisConfigBuilderRuntimeCaller { code: code.into(), @@ -124,6 +123,7 @@ impl<'a> GenesisConfigBuilderRuntimeCaller<'a> { #[cfg(test)] mod tests { use super::*; + use serde_json::{from_str, json}; pub use sp_consensus_babe::{AllowedSlots, BabeEpochConfiguration, Slot}; #[test] @@ -134,14 +134,14 @@ mod tests { .get_default_config() .unwrap(); let expected = r#"{"system":{},"babe":{"authorities":[],"epochConfig":null},"substrateTest":{"authorities":[]},"balances":{"balances":[]}}"#; - assert_eq!(serde_json::from_str::(expected).unwrap(), config); + assert_eq!(from_str::(expected).unwrap(), config); } #[test] fn get_storage_for_patch_works() { sp_tracing::try_init_simple(); - let patch = serde_json::json!({ + let patch = json!({ "babe": { "epochConfig": { "c": [ diff --git a/client/chain-spec/src/lib.rs b/client/chain-spec/src/lib.rs index 93414c2e5ace8..5c78f98c2af7d 100644 --- a/client/chain-spec/src/lib.rs +++ b/client/chain-spec/src/lib.rs @@ -157,16 +157,23 @@ //! //! // The genesis declaration of the chain. //! // -//! // `runtime`, `raw`, `stateRootHash` denote the type of the genesis declaration. +//! // `runtime`, `runtimeGenesisConfig`, `runtimeGenesisConfigPatch`, `raw`, `stateRootHash` denote +//! // the type of the genesis declaration. //! // //! // These declarations are in the following formats: //! // - `runtime` is a `json` object that can be parsed by a compatible `GenesisConfig`. This //! // `GenesisConfig` is declared by a runtime and opaque to the node. +//! // - `runtimeGenesisConfig` is a json object that can be parsed by compatible runtime. It is a +//! // json object the represents full `GenesisConfig` of the runtime. Similar to `runtime`. +//! // - `runtimeGenesisConfigPatch` is a json object that can be parsed by compatible runtime. It +//! // contains a patch to default runtime's `GenesisConfig`. It is opaque to the node. //! // - `raw` is a `json` object with two fields `top` and `children_default`. Each of these //! // fields is a map of `key => value`. These key/value pairs represent the genesis storage. //! // - `stateRootHash` is a single hex encoded hash that represents the genesis hash. The hash //! // type depends on the hash used by the chain. //! // +//! // Runtime must support `GenesisBuilder` API in order to use `runtimeGenesisConfigPatch` or +//! // `runtimeGenesisConfig`. //! "genesis": { "runtime": {} }, //! //! /// Optional map of `block_number` to `wasm_code`. From 0bf081ecc69888bba6fa458063aaf0d236ae9639 Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Tue, 1 Aug 2023 13:45:43 +0200 Subject: [PATCH 38/47] improvements --- bin/node-template/node/src/chain_spec.rs | 22 ++++++++----------- bin/node-template/runtime/Cargo.toml | 6 +++++ bin/node-template/runtime/src/lib.rs | 13 +++++++++++ bin/node/executor/tests/basic.rs | 1 - client/chain-spec/Cargo.toml | 2 -- client/chain-spec/src/chain_spec.rs | 6 ++--- .../chain-spec/src/genesis_config_builder.rs | 3 --- 7 files changed, 30 insertions(+), 23 deletions(-) diff --git a/bin/node-template/node/src/chain_spec.rs b/bin/node-template/node/src/chain_spec.rs index c40ae9b197c1a..ee5a206e3b956 100644 --- a/bin/node-template/node/src/chain_spec.rs +++ b/bin/node-template/node/src/chain_spec.rs @@ -1,7 +1,4 @@ -use node_template_runtime::{ - AccountId, AuraConfig, BalancesConfig, GrandpaConfig, RuntimeGenesisConfig, Signature, - SudoConfig, WASM_BINARY, -}; +use node_template_runtime::{AccountId, RuntimeGenesisConfig, Signature, WASM_BINARY}; use sc_service::ChainType; use sp_consensus_aura::sr25519::AuthorityId as AuraId; use sp_consensus_grandpa::AuthorityId as GrandpaId; @@ -100,20 +97,19 @@ fn testnet_genesis( _enable_println: bool, ) -> serde_json::Value { serde_json::json!({ - "balances": BalancesConfig { + "balances": { // Configure endowed accounts with initial balance of 1 << 60. - balances: endowed_accounts.iter().cloned().map(|k| (k, 1 << 60)).collect(), + "balances": endowed_accounts.iter().cloned().map(|k| (k, 1u64 << 60)).collect::>(), }, - "aura": AuraConfig { - authorities: initial_authorities.iter().map(|x| (x.0.clone())).collect(), + "aura": { + "authorities": initial_authorities.iter().map(|x| (x.0.clone())).collect::>(), }, - "grandpa": GrandpaConfig { - authorities: initial_authorities.iter().map(|x| (x.1.clone(), 1)).collect(), - ..Default::default() + "grandpa": { + "authorities": initial_authorities.iter().map(|x| (x.1.clone(), 1)).collect::>(), }, - "sudo": SudoConfig { + "sudo": { // Assign network admin rights. - key: Some(root_key), + "key": Some(root_key), }, }) } diff --git a/bin/node-template/runtime/Cargo.toml b/bin/node-template/runtime/Cargo.toml index 84f2966930528..abe905a471f77 100644 --- a/bin/node-template/runtime/Cargo.toml +++ b/bin/node-template/runtime/Cargo.toml @@ -38,6 +38,7 @@ sp-std = { version = "8.0.0", default-features = false, path = "../../../primiti sp-transaction-pool = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/transaction-pool" } sp-version = { version = "22.0.0", default-features = false, path = "../../../primitives/version", features=["serde"] } serde_json = { version = "1.0.85", default-features = false, features = ["alloc"] } +sp-genesis-builder = { version = "0.1.0-dev", default-features = false, path = "../../../primitives/genesis-builder" } # Used for the node template's RPCs frame-system-rpc-runtime-api = { version = "4.0.0-dev", default-features = false, path = "../../../frame/system/rpc/runtime-api/" } @@ -87,6 +88,8 @@ std = [ "sp-transaction-pool/std", "sp-version/std", "substrate-wasm-builder", + "serde_json/std", + "sp-genesis-builder/std", ] runtime-benchmarks = [ "frame-benchmarking/runtime-benchmarks", @@ -113,3 +116,6 @@ try-runtime = [ "pallet-timestamp/try-runtime", "pallet-transaction-payment/try-runtime", ] + +#Enabling this flag will disable GenesisBuilder API implementation in runtime. +disable-genesis-builder = [] diff --git a/bin/node-template/runtime/src/lib.rs b/bin/node-template/runtime/src/lib.rs index 8fae73ef08571..c4df3fb8ed857 100644 --- a/bin/node-template/runtime/src/lib.rs +++ b/bin/node-template/runtime/src/lib.rs @@ -23,6 +23,8 @@ use sp_std::prelude::*; use sp_version::NativeVersion; use sp_version::RuntimeVersion; +#[cfg(not(feature = "disable-genesis-builder"))] +use frame_support::genesis_builder_helper::{build_config, create_default_config}; // A few exports that help ease life for downstream crates. pub use frame_support::{ construct_runtime, parameter_types, @@ -568,4 +570,15 @@ impl_runtime_apis! { Executive::try_execute_block(block, state_root_check, signature_check, select).expect("execute-block failed") } } + + #[cfg(not(feature = "disable-genesis-builder"))] + impl sp_genesis_builder::GenesisBuilder for Runtime { + fn create_default_config() -> Vec { + create_default_config::() + } + + fn build_config(config: Vec) -> sp_genesis_builder::Result { + build_config::(config) + } + } } diff --git a/bin/node/executor/tests/basic.rs b/bin/node/executor/tests/basic.rs index c925de2ea6d4c..c71095055e849 100644 --- a/bin/node/executor/tests/basic.rs +++ b/bin/node/executor/tests/basic.rs @@ -861,7 +861,6 @@ fn should_import_block_with_test_client() { #[test] fn default_config_as_json_works() { sp_tracing::try_init_simple(); - // let mut t = BasicExternalities::new_empty(); let mut t = new_test_ext(compact_code_unwrap()); let r = executor_call(&mut t, "GenesisBuilder_create_default_config", &vec![], false) .0 diff --git a/client/chain-spec/Cargo.toml b/client/chain-spec/Cargo.toml index 73c531aa82955..bf3886f50c27f 100644 --- a/client/chain-spec/Cargo.toml +++ b/client/chain-spec/Cargo.toml @@ -34,8 +34,6 @@ log = { version = "0.4.17", default-features = false } array-bytes = { version = "6.1" } #debug [dev-dependencies] -sp-tracing = { version = "10.0.0", path = "../../primitives/tracing" } -kitchensink-runtime = { path = "../../bin/node/runtime" } substrate-test-runtime = { path = "../../test-utils/runtime" } sp-keyring = { version = "24.0.0", path = "../../primitives/keyring" } sp-application-crypto = { version = "23.0.0", default-features = false, path = "../../primitives/application-crypto", features = ["serde"] } diff --git a/client/chain-spec/src/chain_spec.rs b/client/chain-spec/src/chain_spec.rs index 5beec136f1803..5eb960dfdc97a 100644 --- a/client/chain-spec/src/chain_spec.rs +++ b/client/chain-spec/src/chain_spec.rs @@ -204,6 +204,8 @@ enum Genesis { } /// A configuration of a client. Does not include runtime storage initialization. +/// Note: `genesis` and `code` are ignored due to way how the chain specification is serialized into JSON file. Refer to +/// [`ChainSpecJsonContainer`], which flattens [`ClientSpec`] and denies uknown fields. #[derive(Serialize, Deserialize, Clone, Debug)] #[serde(rename_all = "camelCase")] struct ClientSpec { @@ -867,7 +869,6 @@ mod tests { #[test] fn generate_chain_spec_with_patch_works() { - sp_tracing::try_init_simple(); let output: ChainSpec<()> = ChainSpecBuilder::new() .with_name("TestName") .with_id("test_id") @@ -913,7 +914,6 @@ mod tests { #[test] fn generate_chain_spec_with_full_config_works() { - sp_tracing::try_init_simple(); let j = include_str!("../../../test-utils/runtime/src/test_json/default_genesis_config.json"); let output: ChainSpec<()> = ChainSpecBuilder::new() @@ -945,7 +945,6 @@ mod tests { #[test] fn chain_spec_as_json_fails_with_invalid_config() { - sp_tracing::try_init_simple(); let j = include_str!( "../../../test-utils/runtime/src/test_json/default_genesis_config_invalid_2.json" ); @@ -966,7 +965,6 @@ mod tests { #[test] fn chain_spec_as_json_fails_with_invalid_patch() { - sp_tracing::try_init_simple(); let output: ChainSpec<()> = ChainSpecBuilder::new() .with_name("TestName") .with_id("test_id") diff --git a/client/chain-spec/src/genesis_config_builder.rs b/client/chain-spec/src/genesis_config_builder.rs index df6e50d65fa81..0507d3777796f 100644 --- a/client/chain-spec/src/genesis_config_builder.rs +++ b/client/chain-spec/src/genesis_config_builder.rs @@ -128,7 +128,6 @@ mod tests { #[test] fn get_default_config_works() { - sp_tracing::try_init_simple(); let config = GenesisConfigBuilderRuntimeCaller::new(substrate_test_runtime::wasm_binary_unwrap()) .get_default_config() @@ -139,8 +138,6 @@ mod tests { #[test] fn get_storage_for_patch_works() { - sp_tracing::try_init_simple(); - let patch = json!({ "babe": { "epochConfig": { From 67a77ad525ff3b37332d3fd30cd56d6d1a6e49a0 Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Tue, 1 Aug 2023 13:57:45 +0200 Subject: [PATCH 39/47] test-runtime: json files moved to res dir --- client/chain-spec/src/chain_spec.rs | 4 ++-- test-utils/runtime/{src/test_json => res}/README.md | 0 .../{src/test_json => res}/default_genesis_config.json | 0 .../default_genesis_config_incomplete.json | 0 .../test_json => res}/default_genesis_config_invalid.json | 0 .../default_genesis_config_invalid_2.json | 0 test-utils/runtime/src/lib.rs | 8 ++++---- 7 files changed, 6 insertions(+), 6 deletions(-) rename test-utils/runtime/{src/test_json => res}/README.md (100%) rename test-utils/runtime/{src/test_json => res}/default_genesis_config.json (100%) rename test-utils/runtime/{src/test_json => res}/default_genesis_config_incomplete.json (100%) rename test-utils/runtime/{src/test_json => res}/default_genesis_config_invalid.json (100%) rename test-utils/runtime/{src/test_json => res}/default_genesis_config_invalid_2.json (100%) diff --git a/client/chain-spec/src/chain_spec.rs b/client/chain-spec/src/chain_spec.rs index 5eb960dfdc97a..6c1019fe5c6c7 100644 --- a/client/chain-spec/src/chain_spec.rs +++ b/client/chain-spec/src/chain_spec.rs @@ -915,7 +915,7 @@ mod tests { #[test] fn generate_chain_spec_with_full_config_works() { let j = - include_str!("../../../test-utils/runtime/src/test_json/default_genesis_config.json"); + include_str!("../../../test-utils/runtime/res/default_genesis_config.json"); let output: ChainSpec<()> = ChainSpecBuilder::new() .with_name("TestName") .with_id("test_id") @@ -946,7 +946,7 @@ mod tests { #[test] fn chain_spec_as_json_fails_with_invalid_config() { let j = include_str!( - "../../../test-utils/runtime/src/test_json/default_genesis_config_invalid_2.json" + "../../../test-utils/runtime/res/default_genesis_config_invalid_2.json" ); let output: ChainSpec<()> = ChainSpecBuilder::new() .with_name("TestName") diff --git a/test-utils/runtime/src/test_json/README.md b/test-utils/runtime/res/README.md similarity index 100% rename from test-utils/runtime/src/test_json/README.md rename to test-utils/runtime/res/README.md diff --git a/test-utils/runtime/src/test_json/default_genesis_config.json b/test-utils/runtime/res/default_genesis_config.json similarity index 100% rename from test-utils/runtime/src/test_json/default_genesis_config.json rename to test-utils/runtime/res/default_genesis_config.json diff --git a/test-utils/runtime/src/test_json/default_genesis_config_incomplete.json b/test-utils/runtime/res/default_genesis_config_incomplete.json similarity index 100% rename from test-utils/runtime/src/test_json/default_genesis_config_incomplete.json rename to test-utils/runtime/res/default_genesis_config_incomplete.json diff --git a/test-utils/runtime/src/test_json/default_genesis_config_invalid.json b/test-utils/runtime/res/default_genesis_config_invalid.json similarity index 100% rename from test-utils/runtime/src/test_json/default_genesis_config_invalid.json rename to test-utils/runtime/res/default_genesis_config_invalid.json diff --git a/test-utils/runtime/src/test_json/default_genesis_config_invalid_2.json b/test-utils/runtime/res/default_genesis_config_invalid_2.json similarity index 100% rename from test-utils/runtime/src/test_json/default_genesis_config_invalid_2.json rename to test-utils/runtime/res/default_genesis_config_invalid_2.json diff --git a/test-utils/runtime/src/lib.rs b/test-utils/runtime/src/lib.rs index 64e51f0f1db83..8f6b9b18e902a 100644 --- a/test-utils/runtime/src/lib.rs +++ b/test-utils/runtime/src/lib.rs @@ -1299,7 +1299,7 @@ mod tests { #[test] fn build_config_from_json_works() { sp_tracing::try_init_simple(); - let j = include_str!("test_json/default_genesis_config.json"); + let j = include_str!("../res/default_genesis_config.json"); let mut t = BasicExternalities::new_empty(); let r = executor_call(&mut t, "GenesisBuilder_build_config", &j.encode()).unwrap(); @@ -1319,7 +1319,7 @@ mod tests { #[test] fn build_config_from_invalid_json_fails() { sp_tracing::try_init_simple(); - let j = include_str!("test_json/default_genesis_config_invalid.json"); + let j = include_str!("../res/default_genesis_config_invalid.json"); let mut t = BasicExternalities::new_empty(); let r = executor_call(&mut t, "GenesisBuilder_build_config", &j.encode()).unwrap(); let r = BuildResult::decode(&mut &r[..]).unwrap(); @@ -1334,7 +1334,7 @@ mod tests { #[test] fn build_config_from_invalid_json_fails_2() { sp_tracing::try_init_simple(); - let j = include_str!("test_json/default_genesis_config_invalid_2.json"); + let j = include_str!("../res/default_genesis_config_invalid_2.json"); let mut t = BasicExternalities::new_empty(); let r = executor_call(&mut t, "GenesisBuilder_build_config", &j.encode()).unwrap(); let r = BuildResult::decode(&mut &r[..]).unwrap(); @@ -1349,7 +1349,7 @@ mod tests { #[test] fn build_config_from_incomplete_json_fails() { sp_tracing::try_init_simple(); - let j = include_str!("test_json/default_genesis_config_incomplete.json"); + let j = include_str!("../res/default_genesis_config_incomplete.json"); let mut t = BasicExternalities::new_empty(); let r = executor_call(&mut t, "GenesisBuilder_build_config", &j.encode()).unwrap(); From da79f9642fec8a2d02c1cc456f352d87e183a2ac Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Tue, 1 Aug 2023 14:14:25 +0200 Subject: [PATCH 40/47] Cargo.toml fixes --- bin/node/cli/Cargo.toml | 4 ++-- bin/node/runtime/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/node/cli/Cargo.toml b/bin/node/cli/Cargo.toml index 381c96fce539f..5a78a9e675254 100644 --- a/bin/node/cli/Cargo.toml +++ b/bin/node/cli/Cargo.toml @@ -37,11 +37,10 @@ crate-type = ["cdylib", "rlib"] [dependencies] # third-party dependencies -array-bytes = { version = "6.1" } +array-bytes = "6.1" clap = { version = "4.2.5", features = ["derive"], optional = true } codec = { package = "parity-scale-codec", version = "3.6.1" } serde = { version = "1.0.163", features = ["derive"] } -serde_json = { version = "1.0.85", features = ["arbitrary_precision"] } jsonrpsee = { version = "0.16.2", features = ["server"] } futures = "0.3.21" log = "0.4.17" @@ -107,6 +106,7 @@ sc-cli = { version = "0.10.0-dev", optional = true, path = "../../../client/cli" frame-benchmarking-cli = { version = "4.0.0-dev", optional = true, path = "../../../utils/frame/benchmarking-cli" } node-inspect = { version = "0.9.0-dev", optional = true, path = "../inspect" } try-runtime-cli = { version = "0.10.0-dev", optional = true, path = "../../../utils/frame/try-runtime/cli" } +serde_json = "1.0.85" [dev-dependencies] sc-keystore = { version = "4.0.0-dev", path = "../../../client/keystore" } diff --git a/bin/node/runtime/Cargo.toml b/bin/node/runtime/Cargo.toml index 066bb595ce7f3..4372bd25475df 100644 --- a/bin/node/runtime/Cargo.toml +++ b/bin/node/runtime/Cargo.toml @@ -128,7 +128,7 @@ pallet-transaction-storage = { version = "4.0.0-dev", default-features = false, pallet-uniques = { version = "4.0.0-dev", default-features = false, path = "../../../frame/uniques" } pallet-vesting = { version = "4.0.0-dev", default-features = false, path = "../../../frame/vesting" } pallet-whitelist = { version = "4.0.0-dev", default-features = false, path = "../../../frame/whitelist" } -serde_json = { version = "1.0.85", default-features = false, features = ["alloc","arbitrary_precision"] } +serde_json = { version = "1.0.85", default-features = false, features = ["alloc", "arbitrary_precision"] } [build-dependencies] substrate-wasm-builder = { version = "5.0.0-dev", path = "../../../utils/wasm-builder", optional = true } From 2bd3fab5731764434d17f2959dc22e4a8477a144 Mon Sep 17 00:00:00 2001 From: command-bot <> Date: Tue, 1 Aug 2023 12:17:32 +0000 Subject: [PATCH 41/47] ".git/.scripts/commands/fmt/fmt.sh" --- client/chain-spec/src/chain_spec.rs | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/client/chain-spec/src/chain_spec.rs b/client/chain-spec/src/chain_spec.rs index 6c1019fe5c6c7..e8b60fa3ee39a 100644 --- a/client/chain-spec/src/chain_spec.rs +++ b/client/chain-spec/src/chain_spec.rs @@ -70,7 +70,8 @@ impl Clone for GenesisSource { impl GenesisSource { fn resolve(&self) -> Result, String> { - /// helper container for deserializeing genesis from the JSON file (ChainSpec JSON file is also supported here) + /// helper container for deserializeing genesis from the JSON file (ChainSpec JSON file is + /// also supported here) #[derive(Serialize, Deserialize)] struct GenesisContainer { genesis: Genesis, @@ -188,15 +189,15 @@ impl From for RawGenesis { #[serde(rename_all = "camelCase")] #[serde(deny_unknown_fields)] enum Genesis { - /// [Deprecated] Contains the JSON representation of G (the natuve type represting the runtime GenesisConfig struct) - /// (will be removed with `ChainSpec::from_genesis`). + /// [Deprecated] Contains the JSON representation of G (the natuve type represting the runtime + /// GenesisConfig struct) (will be removed with `ChainSpec::from_genesis`). Runtime(G), /// The genesis storage as raw data. Raw(RawGenesis), /// State root hash of the genesis storage. StateRootHash(StorageData), /// Represents the full runtime genesis config in JSON format. - /// The contained object is a JSON blob that can be parsed by a compatible runtime. + /// The contained object is a JSON blob that can be parsed by a compatible runtime. RuntimeGenesisConfig(json::Value), /// Represents a patch for the default runtime genesis config in JSON format. /// The contained value is a JSON object that can be parsed by a compatible runtime. @@ -204,8 +205,9 @@ enum Genesis { } /// A configuration of a client. Does not include runtime storage initialization. -/// Note: `genesis` and `code` are ignored due to way how the chain specification is serialized into JSON file. Refer to -/// [`ChainSpecJsonContainer`], which flattens [`ClientSpec`] and denies uknown fields. +/// Note: `genesis` and `code` are ignored due to way how the chain specification is serialized into +/// JSON file. Refer to [`ChainSpecJsonContainer`], which flattens [`ClientSpec`] and denies uknown +/// fields. #[derive(Serialize, Deserialize, Clone, Debug)] #[serde(rename_all = "camelCase")] struct ClientSpec { @@ -609,9 +611,10 @@ impl ChainSpec { /// Dump the configuration to JSON string. /// - /// During conversion to `raw` format, the `ChainSpec::code` field will be removed and placed into `RawGenesis` as - /// `genesis::top::raw::0x3a636f6465` (which is [`sp_core::storage::well_known_keys::CODE`]). If the spec is already - /// in `raw` format, and contains `genesis::top::raw::0x3a636f6465` field it will be updated with content of `code` + /// During conversion to `raw` format, the `ChainSpec::code` field will be removed and placed + /// into `RawGenesis` as `genesis::top::raw::0x3a636f6465` (which is + /// [`sp_core::storage::well_known_keys::CODE`]). If the spec is already in `raw` format, and + /// contains `genesis::top::raw::0x3a636f6465` field it will be updated with content of `code` /// field. pub fn as_json(&self, raw: bool) -> Result { let container = self.json_container(raw)?; @@ -914,8 +917,7 @@ mod tests { #[test] fn generate_chain_spec_with_full_config_works() { - let j = - include_str!("../../../test-utils/runtime/res/default_genesis_config.json"); + let j = include_str!("../../../test-utils/runtime/res/default_genesis_config.json"); let output: ChainSpec<()> = ChainSpecBuilder::new() .with_name("TestName") .with_id("test_id") @@ -945,9 +947,8 @@ mod tests { #[test] fn chain_spec_as_json_fails_with_invalid_config() { - let j = include_str!( - "../../../test-utils/runtime/res/default_genesis_config_invalid_2.json" - ); + let j = + include_str!("../../../test-utils/runtime/res/default_genesis_config_invalid_2.json"); let output: ChainSpec<()> = ChainSpecBuilder::new() .with_name("TestName") .with_id("test_id") From d788003b6bd185c47592b31462ca3275eac4eceb Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Tue, 1 Aug 2023 14:24:46 +0200 Subject: [PATCH 42/47] Cargo.lock updated --- Cargo.lock | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1bb00c6f811d0..08121db744991 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5343,6 +5343,7 @@ dependencies = [ "sp-consensus-aura", "sp-consensus-grandpa", "sp-core", + "sp-genesis-builder", "sp-inherents", "sp-offchain", "sp-runtime", @@ -8579,7 +8580,6 @@ version = "4.0.0-dev" dependencies = [ "array-bytes", "json-patch", - "kitchensink-runtime", "log", "memmap2", "parity-scale-codec", @@ -8600,7 +8600,6 @@ dependencies = [ "sp-keyring", "sp-runtime", "sp-state-machine", - "sp-tracing", "substrate-test-runtime", ] From 298305b2111bbff191d38a924be116c493f147bf Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Tue, 1 Aug 2023 16:17:47 +0200 Subject: [PATCH 43/47] minor fixes --- client/chain-spec/src/chain_spec.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/chain-spec/src/chain_spec.rs b/client/chain-spec/src/chain_spec.rs index e8b60fa3ee39a..b03e8db659ca2 100644 --- a/client/chain-spec/src/chain_spec.rs +++ b/client/chain-spec/src/chain_spec.rs @@ -457,7 +457,7 @@ impl ChainSpec { /// Create hardcoded spec. #[deprecated( - note = "`from_genesis` is planned to be removed in December 2023. Use [`builder()`] instead." + note = "`from_genesis` is planned to be removed in December 2023. Use `builder()` instead." )] // deprecated note: Genesis::Runtime + GenesisSource::Factory shall also be removed pub fn from_genesis G + 'static + Send + Sync>( @@ -609,13 +609,13 @@ impl ChainSpec { }) } - /// Dump the configuration to JSON string. + /// Dump the chain specification to JSON string. /// /// During conversion to `raw` format, the `ChainSpec::code` field will be removed and placed /// into `RawGenesis` as `genesis::top::raw::0x3a636f6465` (which is /// [`sp_core::storage::well_known_keys::CODE`]). If the spec is already in `raw` format, and /// contains `genesis::top::raw::0x3a636f6465` field it will be updated with content of `code` - /// field. + /// field (if present). pub fn as_json(&self, raw: bool) -> Result { let container = self.json_container(raw)?; json::to_string_pretty(&container).map_err(|e| format!("Error generating spec json: {}", e)) From 97e5b1b3f53830fe931f13b7857b008b9c874f4d Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Wed, 2 Aug 2023 18:07:07 +0200 Subject: [PATCH 44/47] Apply suggestions from code review Co-authored-by: Sebastian Kunert --- bin/node/runtime/src/lib.rs | 1 - client/chain-spec/src/chain_spec.rs | 6 +++--- client/chain-spec/src/genesis_config_builder.rs | 6 +++--- test-utils/runtime/src/lib.rs | 1 - 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index ed23d5bbc01b3..c3289dd16d271 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -66,7 +66,6 @@ use sp_api::impl_runtime_apis; use sp_authority_discovery::AuthorityId as AuthorityDiscoveryId; use sp_consensus_grandpa::AuthorityId as GrandpaId; use sp_core::{crypto::KeyTypeId, OpaqueMetadata}; -// #[cfg(feature = "genesis-builder")] use frame_support::genesis_builder_helper::{build_config, create_default_config}; use sp_inherents::{CheckInherentsResult, InherentData}; use sp_runtime::{ diff --git a/client/chain-spec/src/chain_spec.rs b/client/chain-spec/src/chain_spec.rs index b03e8db659ca2..b1dfe804fd526 100644 --- a/client/chain-spec/src/chain_spec.rs +++ b/client/chain-spec/src/chain_spec.rs @@ -70,7 +70,7 @@ impl Clone for GenesisSource { impl GenesisSource { fn resolve(&self) -> Result, String> { - /// helper container for deserializeing genesis from the JSON file (ChainSpec JSON file is + /// helper container for deserializing genesis from the JSON file (ChainSpec JSON file is /// also supported here) #[derive(Serialize, Deserialize)] struct GenesisContainer { @@ -189,7 +189,7 @@ impl From for RawGenesis { #[serde(rename_all = "camelCase")] #[serde(deny_unknown_fields)] enum Genesis { - /// [Deprecated] Contains the JSON representation of G (the natuve type represting the runtime + /// [Deprecated] Contains the JSON representation of G (the native type representing the runtime /// GenesisConfig struct) (will be removed with `ChainSpec::from_genesis`). Runtime(G), /// The genesis storage as raw data. @@ -378,7 +378,7 @@ impl ChainSpecBuilder { client_spec, genesis: GenesisSource::GenesisBuilderApi( self.genesis_build_action - .expect("with_genesis_patch or with_no_genesis_defaults must be called."), + .expect("with_genesis_config_patch or with_genesis_config must be called."), ), code: self.code.expect("with code must be called.").into(), } diff --git a/client/chain-spec/src/genesis_config_builder.rs b/client/chain-spec/src/genesis_config_builder.rs index 0507d3777796f..ddd9293d40022 100644 --- a/client/chain-spec/src/genesis_config_builder.rs +++ b/client/chain-spec/src/genesis_config_builder.rs @@ -88,10 +88,10 @@ impl<'a> GenesisConfigBuilderRuntimeCaller<'a> { .call(&mut ext, "GenesisBuilder_build_config", &config.to_string().encode()) .map_err(|e| format!("wasm call error {e}"))?; - let build_result = BuildResult::decode(&mut &call_result[..]) - .map_err(|e| format!("scale codec error: {e}"))?; + BuildResult::decode(&mut &call_result[..]) + .map_err(|e| format!("scale codec error: {e}"))??; - Ok(build_result.map(|_| ext.into_storages())?) + Ok(ext.into_storages()) } /// Patch default `GenesisConfig` using given JSON patch and store it in the storage. diff --git a/test-utils/runtime/src/lib.rs b/test-utils/runtime/src/lib.rs index 8f6b9b18e902a..e5f8d3ea4f281 100644 --- a/test-utils/runtime/src/lib.rs +++ b/test-utils/runtime/src/lib.rs @@ -1338,7 +1338,6 @@ mod tests { let mut t = BasicExternalities::new_empty(); let r = executor_call(&mut t, "GenesisBuilder_build_config", &j.encode()).unwrap(); let r = BuildResult::decode(&mut &r[..]).unwrap(); - log::info!("result: {:#?}", r); assert_eq!(r, Err( sp_runtime::RuntimeString::Owned( "Invalid JSON blob: unknown field `babex`, expected one of `system`, `babe`, `substrateTest`, `balances` at line 3 column 9".to_string(), From 5f60a4bc8e35f8614fc1cb9c6c22f6c3b4bd263a Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Wed, 2 Aug 2023 18:50:41 +0200 Subject: [PATCH 45/47] cleanup --- bin/node/cli/src/chain_spec.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/bin/node/cli/src/chain_spec.rs b/bin/node/cli/src/chain_spec.rs index 9b01b2c8111e9..dadfddb28ad08 100644 --- a/bin/node/cli/src/chain_spec.rs +++ b/bin/node/cli/src/chain_spec.rs @@ -634,8 +634,6 @@ pub(crate) mod tests { sp_tracing::try_init_simple(); let j1 = development_config().as_json(true).unwrap(); let j2 = development_config_legacy().as_json(true).unwrap(); - std::fs::write("/tmp/j1.json", j1.clone()).unwrap(); - std::fs::write("/tmp/j2.json", j2.clone()).unwrap(); assert_eq!(j1, j2); } @@ -670,8 +668,6 @@ pub(crate) mod tests { sp_tracing::try_init_simple(); let j1 = local_testnet_config().as_json(true).unwrap(); let j2 = local_testnet_config_legacy().as_json(true).unwrap(); - std::fs::write("/tmp/i1.json", j1.clone()).unwrap(); - std::fs::write("/tmp/i2.json", j2.clone()).unwrap(); assert_eq!(j1, j2); } @@ -715,13 +711,11 @@ pub(crate) mod tests { sp_tracing::try_init_simple(); let chain_spec = staging_testnet_config(); - std::fs::write("/tmp/k1.json", chain_spec.as_json(false).unwrap()).unwrap(); let storage = chain_spec.build_storage().unwrap(); let mut keys = storage.top.keys().cloned().map(hex).collect::>(); keys.sort(); let chain_spec = staging_testnet_config_legacy(); - std::fs::write("/tmp/k2.json", chain_spec.as_json(false).unwrap()).unwrap(); let storage = chain_spec.build_storage().unwrap(); let mut keys_expected = storage.top.keys().cloned().map(hex).collect::>(); keys_expected.sort(); From 0f58273e278429868f933cb965129eae6c6602b3 Mon Sep 17 00:00:00 2001 From: command-bot <> Date: Wed, 2 Aug 2023 16:55:21 +0000 Subject: [PATCH 46/47] ".git/.scripts/commands/fmt/fmt.sh" --- bin/node/runtime/src/lib.rs | 2 +- client/chain-spec/src/chain_spec.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index c3289dd16d271..a36416b738e41 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -29,6 +29,7 @@ use frame_election_provider_support::{ use frame_support::{ construct_runtime, dispatch::DispatchClass, + genesis_builder_helper::{build_config, create_default_config}, instances::{Instance1, Instance2}, ord_parameter_types, pallet_prelude::Get, @@ -66,7 +67,6 @@ use sp_api::impl_runtime_apis; use sp_authority_discovery::AuthorityId as AuthorityDiscoveryId; use sp_consensus_grandpa::AuthorityId as GrandpaId; use sp_core::{crypto::KeyTypeId, OpaqueMetadata}; -use frame_support::genesis_builder_helper::{build_config, create_default_config}; use sp_inherents::{CheckInherentsResult, InherentData}; use sp_runtime::{ create_runtime_str, diff --git a/client/chain-spec/src/chain_spec.rs b/client/chain-spec/src/chain_spec.rs index b1dfe804fd526..c4287e53b370f 100644 --- a/client/chain-spec/src/chain_spec.rs +++ b/client/chain-spec/src/chain_spec.rs @@ -189,8 +189,8 @@ impl From for RawGenesis { #[serde(rename_all = "camelCase")] #[serde(deny_unknown_fields)] enum Genesis { - /// [Deprecated] Contains the JSON representation of G (the native type representing the runtime - /// GenesisConfig struct) (will be removed with `ChainSpec::from_genesis`). + /// [Deprecated] Contains the JSON representation of G (the native type representing the + /// runtime GenesisConfig struct) (will be removed with `ChainSpec::from_genesis`). Runtime(G), /// The genesis storage as raw data. Raw(RawGenesis), From e1c7e11709fe2d89ae62b5090b37fb4491f130e9 Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Thu, 3 Aug 2023 08:25:54 +0200 Subject: [PATCH 47/47] removing unneeded refs to runtime PalletGenesisConfig types --- bin/node/cli/src/chain_spec.rs | 69 ++++++++++++++++------------------ 1 file changed, 32 insertions(+), 37 deletions(-) diff --git a/bin/node/cli/src/chain_spec.rs b/bin/node/cli/src/chain_spec.rs index dadfddb28ad08..05007ebcda2e9 100644 --- a/bin/node/cli/src/chain_spec.rs +++ b/bin/node/cli/src/chain_spec.rs @@ -20,9 +20,7 @@ use grandpa_primitives::AuthorityId as GrandpaId; use kitchensink_runtime::{ - constants::currency::*, wasm_binary_unwrap, AssetsConfig, BabeConfig, BalancesConfig, Block, - ElectionsConfig, ImOnlineConfig, MaxNominations, NominationPoolsConfig, SessionConfig, - SessionKeys, SocietyConfig, StakerStatus, StakingConfig, SudoConfig, TechnicalCommitteeConfig, + constants::currency::*, wasm_binary_unwrap, Block, MaxNominations, SessionKeys, StakerStatus, }; use pallet_im_online::sr25519::AuthorityId as ImOnlineId; use sc_chain_spec::ChainSpecExtension; @@ -321,11 +319,11 @@ pub fn testnet_genesis( configure_accounts(initial_authorities, initial_nominators, endowed_accounts, STASH); let json = serde_json::json!({ - "balances": BalancesConfig { - balances: endowed_accounts.iter().cloned().map(|x| (x, ENDOWMENT)).collect(), + "balances": { + "balances": endowed_accounts.iter().cloned().map(|x| (x, ENDOWMENT)).collect::>(), }, - "session": SessionConfig { - keys: initial_authorities + "session": { + "keys": initial_authorities .iter() .map(|x| { ( @@ -336,49 +334,42 @@ pub fn testnet_genesis( }) .collect::>(), }, - "staking": StakingConfig { - validator_count: initial_authorities.len() as u32, - minimum_validator_count: initial_authorities.len() as u32, - invulnerables: initial_authorities.iter().map(|x| x.0.clone()).collect(), - slash_reward_fraction: Perbill::from_percent(10), - stakers: stakers.clone(), - ..Default::default() + "staking": { + "validatorCount": initial_authorities.len() as u32, + "minimumValidatorCount": initial_authorities.len() as u32, + "invulnerables": initial_authorities.iter().map(|x| x.0.clone()).collect::>(), + "slashRewardFraction": Perbill::from_percent(10), + "stakers": stakers.clone(), }, - "elections": ElectionsConfig { - members: endowed_accounts + "elections": { + "members": endowed_accounts .iter() .take((num_endowed_accounts + 1) / 2) .cloned() .map(|member| (member, STASH)) - .collect(), + .collect::>(), }, - "technicalCommittee": TechnicalCommitteeConfig { - members: endowed_accounts + "technicalCommittee": { + "members": endowed_accounts .iter() .take((num_endowed_accounts + 1) / 2) .cloned() - .collect(), - ..Default::default() + .collect::>(), }, - "sudo": SudoConfig { key: Some(root_key.clone()) }, - "babe": BabeConfig { - epoch_config: Some(kitchensink_runtime::BABE_GENESIS_EPOCH_CONFIG), - ..Default::default() + "sudo": { "key": Some(root_key.clone()) }, + "babe": { + "epochConfig": Some(kitchensink_runtime::BABE_GENESIS_EPOCH_CONFIG), }, - "imOnline": ImOnlineConfig { keys: vec![] }, - "society": SocietyConfig { pot: 0 }, - "assets": AssetsConfig { + "society": { "pot": 0 }, + "assets": { // This asset is used by the NIS pallet as counterpart currency. - assets: vec![(9, get_account_id_from_seed::("Alice"), true, 1)], - ..Default::default() + "assets": vec![(9, get_account_id_from_seed::("Alice"), true, 1)], }, - "nominationPools": NominationPoolsConfig { - min_create_bond: 10 * DOLLARS, - min_join_bond: 1 * DOLLARS, - ..Default::default() + "nominationPools": { + "minCreateBond": 10 * DOLLARS, + "minJoinBond": 1 * DOLLARS, }, }); - log::info!("json -> {}", json); json } @@ -494,11 +485,15 @@ pub(crate) mod tests { } // This compares the output of RuntimeGenesisConfig-based ChainSpec building against JSON-based - // ChainSpec building. Once RuntimeGenesisConfig is removed from client-sode, entire mod can be + // ChainSpec building. Once RuntimeGenesisConfig is removed from client-side, entire mod can be // removed. mod json_vs_legacy { use crate::chain_spec::*; - use kitchensink_runtime::{CouncilConfig, DemocracyConfig, IndicesConfig}; + use kitchensink_runtime::{ + wasm_binary_unwrap, BabeConfig, BalancesConfig, CouncilConfig, DemocracyConfig, + ElectionsConfig, ImOnlineConfig, IndicesConfig, NominationPoolsConfig, SessionConfig, + SocietyConfig, StakingConfig, SudoConfig, TechnicalCommitteeConfig, + }; use sp_runtime::BuildStorage; /// Helper function to create RuntimeGenesisConfig for testing pub(super) fn testnet_genesis_legacy(