diff --git a/Cargo.lock b/Cargo.lock index 39f0b306761..32ca6b6a545 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5846,10 +5846,16 @@ dependencies = [ "base64 0.22.1", "chrono", "clap", + "dapi-grpc", "data-contracts", "dpp", "hex", "platform-version", + "serde", + "serde_json", + "tokio", + "tonic 0.14.5", + "ureq", ] [[package]] diff --git a/packages/rs-dpp/schema/meta_schemas/document/v1/document-meta.json b/packages/rs-dpp/schema/meta_schemas/document/v1/document-meta.json new file mode 100644 index 00000000000..9dfb6ad7499 --- /dev/null +++ b/packages/rs-dpp/schema/meta_schemas/document/v1/document-meta.json @@ -0,0 +1,645 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/dashpay/platform/blob/master/packages/rs-dpp/schema/meta_schemas/document/v1/document-meta.json", + "type": "object", + "$defs": { + "documentProperties": { + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9-_]{1,64}$": { + "type": "object", + "allOf": [ + { + "$ref": "#/$defs/documentSchema" + } + ], + "unevaluatedProperties": false + } + }, + "propertyNames": { + "pattern": "^[a-zA-Z0-9-_]{1,64}$" + }, + "minProperties": 1, + "maxProperties": 100 + }, + "documentSchemaArray": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "allOf": [ + { + "$ref": "#/$defs/documentSchema" + } + ], + "unevaluatedProperties": false + } + }, + "documentSchema": { + "type": "object", + "properties": { + "$id": { + "type": "string", + "pattern": "^#", + "minLength": 1 + }, + "$ref": { + "type": "string", + "pattern": "^#", + "minLength": 1 + }, + "$comment": { + "$ref": "https://json-schema.org/draft/2020-12/meta/core#/properties/$comment" + }, + "description": { + "$ref": "https://json-schema.org/draft/2020-12/meta/meta-data#/properties/description" + }, + "examples": { + "$ref": "https://json-schema.org/draft/2020-12/meta/meta-data#/properties/examples" + }, + "multipleOf": { + "$ref": "https://json-schema.org/draft/2020-12/meta/validation#/properties/multipleOf" + }, + "maximum": { + "$ref": "https://json-schema.org/draft/2020-12/meta/validation#/properties/maximum" + }, + "exclusiveMaximum": { + "$ref": "https://json-schema.org/draft/2020-12/meta/validation#/properties/exclusiveMaximum" + }, + "minimum": { + "$ref": "https://json-schema.org/draft/2020-12/meta/validation#/properties/minimum" + }, + "exclusiveMinimum": { + "$ref": "https://json-schema.org/draft/2020-12/meta/validation#/properties/exclusiveMinimum" + }, + "maxLength": { + "$ref": "https://json-schema.org/draft/2020-12/meta/validation#/properties/maxLength" + }, + "minLength": { + "$ref": "https://json-schema.org/draft/2020-12/meta/validation#/properties/minLength" + }, + "pattern": { + "$ref": "https://json-schema.org/draft/2020-12/meta/validation#/properties/pattern" + }, + "maxItems": { + "$ref": "https://json-schema.org/draft/2020-12/meta/validation#/properties/maxItems" + }, + "minItems": { + "$ref": "https://json-schema.org/draft/2020-12/meta/validation#/properties/minItems" + }, + "uniqueItems": { + "$ref": "https://json-schema.org/draft/2020-12/meta/validation#/properties/uniqueItems" + }, + "contains": { + "$ref": "https://json-schema.org/draft/2020-12/meta/applicator#/properties/contains" + }, + "maxProperties": { + "$ref": "https://json-schema.org/draft/2020-12/meta/validation#/properties/maxProperties" + }, + "minProperties": { + "$ref": "https://json-schema.org/draft/2020-12/meta/validation#/properties/minProperties" + }, + "required": { + "$ref": "https://json-schema.org/draft/2020-12/meta/validation#/properties/required" + }, + "additionalProperties": { + "type": "boolean", + "const": false + }, + "properties": { + "$ref": "#/$defs/documentProperties" + }, + "dependentRequired": { + "type": "object", + "minProperties": 1, + "additionalProperties": { + "$ref": "https://json-schema.org/draft/2020-12/meta/validation#/$defs/stringArray" + } + }, + "const": true, + "enum": { + "type": "array", + "items": true, + "minItems": 1, + "uniqueItems": true + }, + "type": { + "$ref": "https://json-schema.org/draft/2020-12/meta/validation#/properties/type" + }, + "format": { + "$ref": "https://json-schema.org/draft/2020-12/meta/format-annotation#/properties/format" + }, + "contentMediaType": { + "$ref": "https://json-schema.org/draft/2020-12/meta/content#/properties/contentMediaType" + }, + "byteArray": { + "type": "boolean", + "const": true + }, + "position": { + "type": "integer", + "minimum": 0 + } + }, + "dependentSchemas": { + "byteArray": { + "description": "should be used only with array type", + "properties": { + "type": { + "type": "string", + "const": "array" + } + } + }, + "contentMediaType": { + "if": { + "properties": { + "contentMediaType": { + "const": "application/x.dash.dpp.identifier" + } + } + }, + "then": { + "properties": { + "byteArray": { + "const": true + }, + "minItems": { + "const": 32 + }, + "maxItems": { + "const": 32 + } + }, + "required": [ + "byteArray", + "minItems", + "maxItems" + ] + } + }, + "pattern": { + "description": "prevent slow pattern matching of large strings", + "properties": { + "maxLength": { + "type": "integer", + "minimum": 0, + "maximum": 50000 + } + }, + "required": [ + "maxLength" + ] + }, + "format": { + "description": "prevent slow format validation of large strings", + "properties": { + "maxLength": { + "type": "integer", + "minimum": 0, + "maximum": 50000 + } + }, + "required": [ + "maxLength" + ] + } + }, + "allOf": [ + { + "$comment": "require index for object properties", + "if": { + "properties": { + "type": { + "const": "object" + } + }, + "required": [ + "type" + ] + }, + "then": { + "properties": { + "properties": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "position": true + }, + "required": [ + "position" + ] + } + } + } + } + }, + { + "$comment": "allow only byte arrays", + "if": { + "properties": { + "type": { + "const": "array" + } + }, + "required": [ + "type" + ] + }, + "then": { + "properties": { + "byteArray": true + }, + "required": [ + "byteArray" + ] + } + }, + { + "$comment": "all object properties must be defined", + "if": { + "properties": { + "type": { + "const": "object" + } + }, + "not": { + "properties": { + "$ref": true + }, + "required": [ + "$ref" + ] + } + }, + "then": { + "properties": { + "properties": { + "$ref": "#/$defs/documentProperties" + }, + "additionalProperties": { + "$ref": "#/$defs/documentSchema/properties/additionalProperties" + } + }, + "required": [ + "properties", + "additionalProperties" + ] + } + } + ] + }, + "documentActionTokenCost": { + "type": "object", + "properties": { + "contractId": { + "type": "array", + "contentMediaType": "application/x.dash.dpp.identifier", + "byteArray": true, + "minItems": 32, + "maxItems": 32 + }, + "tokenPosition": { + "type": "integer", + "minimum": 0, + "maximum": 65535 + }, + "amount": { + "type": "integer", + "minimum": 1, + "maximum": 281474976710655 + }, + "effect": { + "type": "integer", + "enum": [ + 0, + 1 + ], + "description": "0 - TransferTokenToContractOwner (default), 1 - Burn" + }, + "gasFeesPaidBy": { + "type": "integer", + "enum": [ + 0, + 1, + 2 + ], + "description": "0 - DocumentOwner (default), 1 - ContractOwner, 2 - PreferContractOwner" + } + }, + "required": [ + "tokenPosition", + "amount" + ], + "additionalProperties": false + } + }, + "properties": { + "type": { + "type": "string", + "const": "object" + }, + "$schema": { + "type": "string", + "const": "https://github.com/dashpay/platform/blob/master/packages/rs-dpp/schema/meta_schemas/document/v1/document-meta.json" + }, + "$defs": { + "$ref": "#/$defs/documentProperties" + }, + "indices": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 32 + }, + "properties": { + "type": "array", + "items": { + "type": "object", + "propertyNames": { + "maxLength": 256 + }, + "additionalProperties": { + "type": "string", + "enum": [ + "asc" + ] + }, + "minProperties": 1, + "maxProperties": 1 + }, + "minItems": 1, + "maxItems": 10 + }, + "unique": { + "type": "boolean" + }, + "nullSearchable": { + "type": "boolean" + }, + "contested": { + "type": "object", + "properties": { + "fieldMatches": { + "type": "array", + "items": { + "type": "object", + "properties": { + "field": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "regexPattern": { + "type": "string", + "minLength": 1, + "maxLength": 256 + } + }, + "additionalProperties": false, + "required": [ + "field", + "regexPattern" + ] + }, + "minItems": 1 + }, + "resolution": { + "type": "integer", + "enum": [ + 0 + ], + "description": "Resolution. 0 - Masternode Vote" + }, + "description": { + "type": "string", + "minLength": 1, + "maxLength": 256 + } + }, + "required": [ + "resolution" + ], + "additionalProperties": false + }, + "countable": { + "type": "boolean", + "description": "Enables countable operations on the index. Adds extra costs for documents storage" + } + }, + "required": [ + "properties", + "name" + ], + "additionalProperties": false + }, + "minItems": 1, + "maxItems": 10 + }, + "signatureSecurityLevelRequirement": { + "type": "integer", + "enum": [ + 1, + 2, + 3 + ], + "description": "Public key security level. 1 - Critical, 2 - High, 3 - Medium. If none specified, High level is used" + }, + "documentsKeepHistory": { + "type": "boolean", + "description": "True if the documents keep all their history, default is false" + }, + "documentsMutable": { + "type": "boolean", + "description": "True if the documents are mutable, default is true" + }, + "canBeDeleted": { + "type": "boolean", + "description": "True if the documents can be deleted, default is true" + }, + "transferable": { + "type": "integer", + "enum": [ + 0, + 1 + ], + "description": "Transferable without a marketplace sell. 0 - Never, 1 - Always" + }, + "tradeMode": { + "type": "integer", + "enum": [ + 0, + 1 + ], + "description": "Built in marketplace system. 0 - None, 1 - Direct purchase (The user can buy the item without the need for an approval)" + }, + "creationRestrictionMode": { + "type": "integer", + "enum": [ + 0, + 1, + 2 + ], + "description": "Restrictions of document creation. 0 - No restrictions, 1 - Owner only, 2 - No creation (System Only)" + }, + "requiresIdentityEncryptionBoundedKey": { + "type": "integer", + "enum": [ + 0, + 1, + 2 + ], + "description": "Key requirements. 0 - Unique Non Replaceable, 1 - Multiple, 2 - Multiple with reference to latest key." + }, + "requiresIdentityDecryptionBoundedKey": { + "type": "integer", + "enum": [ + 0, + 1, + 2 + ], + "description": "Key requirements. 0 - Unique Non Replaceable, 1 - Multiple, 2 - Multiple with reference to latest key." + }, + "tokenCost": { + "type": "object", + "properties": { + "create": { + "$ref": "#/$defs/documentActionTokenCost" + }, + "replace": { + "$ref": "#/$defs/documentActionTokenCost" + }, + "delete": { + "$ref": "#/$defs/documentActionTokenCost" + }, + "transfer": { + "$ref": "#/$defs/documentActionTokenCost" + }, + "update_price": { + "$ref": "#/$defs/documentActionTokenCost" + }, + "purchase": { + "$ref": "#/$defs/documentActionTokenCost" + } + }, + "additionalProperties": false + }, + "properties": { + "type": "object", + "additionalProperties": { + "type": "object", + "allOf": [ + { + "$ref": "#/$defs/documentSchema" + } + ], + "unevaluatedProperties": false + }, + "properties": { + "$id": true, + "$ownerId": true, + "$revision": true, + "$createdAt": true, + "$updatedAt": true, + "$transferredAt": true, + "$createdAtBlockHeight": true, + "$updatedAtBlockHeight": true, + "$transferredAtBlockHeight": true, + "$createdAtCoreBlockHeight": true, + "$updatedAtCoreBlockHeight": true, + "$transferredAtCoreBlockHeight": true + }, + "propertyNames": { + "oneOf": [ + { + "type": "string", + "pattern": "^[a-zA-Z0-9-_]{1,64}$" + }, + { + "type": "string", + "enum": [ + "$id", + "$ownerId", + "$revision", + "$createdAt", + "$updatedAt", + "$transferredAt", + "$createdAtBlockHeight", + "$updatedAtBlockHeight", + "$transferredAtBlockHeight", + "$createdAtCoreBlockHeight", + "$updatedAtCoreBlockHeight", + "$transferredAtCoreBlockHeight" + ] + } + ] + }, + "minProperties": 1, + "maxProperties": 100 + }, + "transient": { + "type": "array", + "items": { + "type": "string" + } + }, + "keywords": { + "type": "array", + "description": "List of up to 20 descriptive keywords for the contract, used in the Keyword Search contract", + "items": { + "type": "string", + "minLength": 3, + "maxLength": 50 + }, + "maxItems": 20, + "uniqueItems": true + }, + "additionalProperties": { + "type": "boolean", + "const": false + }, + "required": { + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "$comment": { + "type": "string" + }, + "description": { + "type": "string" + }, + "minProperties": { + "type": "integer", + "minimum": 0 + }, + "maxProperties": { + "type": "integer", + "minimum": 0 + }, + "dependentRequired": { + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + } + } + }, + "required": [ + "$schema", + "type", + "properties", + "additionalProperties" + ], + "additionalProperties": false +} diff --git a/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/v0/mod.rs b/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/v0/mod.rs index ac5c6258cc1..aa1ae0fe9b4 100644 --- a/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/v0/mod.rs +++ b/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/v0/mod.rs @@ -51,7 +51,7 @@ use crate::data_contract::errors::DataContractError; use crate::data_contract::storage_requirements::keys_for_document_type::StorageKeyRequirements; use crate::identity::SecurityLevel; #[cfg(feature = "validation")] -use crate::validation::meta_validators::DOCUMENT_META_SCHEMA_V0; +use crate::validation::meta_validators::{DOCUMENT_META_SCHEMA_V0, DOCUMENT_META_SCHEMA_V1}; use crate::validation::operations::ProtocolValidationOperation; use crate::version::PlatformVersion; use crate::ProtocolError; @@ -132,8 +132,28 @@ impl DocumentTypeV0 { ) })?; + // Select the appropriate document meta-schema based on platform version + let meta_schema = match platform_version + .dpp + .contract_versions + .document_type_versions + .schema + .document_type_schema + { + 0 => &*DOCUMENT_META_SCHEMA_V0, + 1 => &*DOCUMENT_META_SCHEMA_V1, + version => { + return Err(ProtocolError::UnknownVersionMismatch { + method: "DocumentTypeV0::try_from_schema (document_type_schema)" + .to_string(), + known_versions: vec![0, 1], + received: version, + }) + } + }; + // Validate against JSON Schema - DOCUMENT_META_SCHEMA_V0 + meta_schema .validate(&root_json_schema) .map_err(|mut errs| ConsensusError::from(errs.next().unwrap()))?; diff --git a/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/v1/mod.rs b/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/v1/mod.rs index 39e29550162..0b956f8c917 100644 --- a/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/v1/mod.rs +++ b/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/v1/mod.rs @@ -69,7 +69,7 @@ use crate::tokens::token_amount_on_contract_token::{ DocumentActionTokenCost, DocumentActionTokenEffect, }; #[cfg(feature = "validation")] -use crate::validation::meta_validators::DOCUMENT_META_SCHEMA_V0; +use crate::validation::meta_validators::{DOCUMENT_META_SCHEMA_V0, DOCUMENT_META_SCHEMA_V1}; use crate::validation::operations::ProtocolValidationOperation; use crate::version::PlatformVersion; use crate::ProtocolError; @@ -151,8 +151,28 @@ impl DocumentTypeV1 { ) })?; + // Select the appropriate document meta-schema based on platform version + let meta_schema = match platform_version + .dpp + .contract_versions + .document_type_versions + .schema + .document_type_schema + { + 0 => &*DOCUMENT_META_SCHEMA_V0, + 1 => &*DOCUMENT_META_SCHEMA_V1, + version => { + return Err(ProtocolError::UnknownVersionMismatch { + method: "DocumentTypeV1::try_from_schema (document_type_schema)" + .to_string(), + known_versions: vec![0, 1], + received: version, + }) + } + }; + // Validate against JSON Schema - DOCUMENT_META_SCHEMA_V0 + meta_schema .validate(&root_json_schema) .map_err(|mut errs| ConsensusError::from(errs.next().unwrap()))?; @@ -708,6 +728,139 @@ mod tests { use assert_matches::assert_matches; use platform_value::platform_value; + mod document_meta_schema_version { + use super::*; + + #[test] + fn v0_schema_allows_unknown_properties() { + let platform_version = PlatformVersion::first(); + + let schema = platform_value!({ + "type": "object", + "properties": { + "test_field": { + "type": "string", + "position": 0 + } + }, + "additionalProperties": false, + "unknownProp": true + }); + + let config = DataContractConfig::default_for_version(platform_version) + .expect("should create a default config"); + + let result = DocumentTypeV1::try_from_schema( + Identifier::new([1; 32]), + 1, + config.version(), + "test_doc", + schema, + None, + &BTreeMap::new(), + &config, + true, + &mut vec![], + platform_version, + ); + + assert!( + result.is_ok(), + "v0 schema should allow unknown top-level properties, got error: {:?}", + result.err() + ); + } + + #[test] + fn v1_schema_rejects_unknown_properties() { + let platform_version = PlatformVersion::latest(); + + let schema = platform_value!({ + "type": "object", + "properties": { + "test_field": { + "type": "string", + "position": 0 + } + }, + "additionalProperties": false, + "unknownProp": true + }); + + let config = DataContractConfig::default_for_version(platform_version) + .expect("should create a default config"); + + let result = DocumentTypeV1::try_from_schema( + Identifier::new([1; 32]), + 1, + config.version(), + "test_doc", + schema, + None, + &BTreeMap::new(), + &config, + true, + &mut vec![], + platform_version, + ); + + assert!( + result.is_err(), + "v1 schema should reject unknown top-level properties" + ); + + let err = result.unwrap_err(); + let err_str = format!("{:?}", err); + let err_str_lower = err_str.to_lowercase(); + assert!( + err_str_lower.contains("additional properties"), + "Error should mention additional properties, got: {}", + err_str + ); + } + + #[test] + fn v1_schema_accepts_known_properties() { + let platform_version = PlatformVersion::latest(); + + let schema = platform_value!({ + "type": "object", + "properties": { + "test_field": { + "type": "string", + "position": 0 + } + }, + "additionalProperties": false, + "required": ["test_field"], + "$comment": "hello" + }); + + let config = DataContractConfig::default_for_version(platform_version) + .expect("should create a default config"); + + let result = DocumentTypeV1::try_from_schema( + Identifier::new([1; 32]), + 1, + config.version(), + "test_doc", + schema, + None, + &BTreeMap::new(), + &config, + true, + &mut vec![], + platform_version, + ); + + assert!( + result.is_ok(), + "v1 schema should accept known properties like required and $comment, got error: {:?}", + result.err() + ); + } + } + mod document_type_name { use super::*; diff --git a/packages/rs-dpp/src/data_contract/document_type/schema/allowed_top_level_properties.rs b/packages/rs-dpp/src/data_contract/document_type/schema/allowed_top_level_properties.rs new file mode 100644 index 00000000000..2d3f6ced6df --- /dev/null +++ b/packages/rs-dpp/src/data_contract/document_type/schema/allowed_top_level_properties.rs @@ -0,0 +1,139 @@ +use platform_value::Value; + +/// The set of top-level property names allowed on a document type schema object +/// as defined by the v1 document meta-schema. +/// +/// Any key not in this list should be stripped from document type schemas +/// during the v12 protocol upgrade migration to prevent unknown properties +/// from changing storage semantics. +pub const ALLOWED_DOCUMENT_SCHEMA_V1_PROPERTIES: &[&str] = &[ + "type", + "$schema", + "$defs", + "indices", + "signatureSecurityLevelRequirement", + "documentsKeepHistory", + "documentsMutable", + "canBeDeleted", + "transferable", + "tradeMode", + "creationRestrictionMode", + "requiresIdentityEncryptionBoundedKey", + "requiresIdentityDecryptionBoundedKey", + "tokenCost", + "properties", + "transient", + "keywords", + "additionalProperties", + "required", + "$comment", + "description", + "minProperties", + "maxProperties", + "dependentRequired", +]; + +/// Strips any top-level key from the document type schema `Value::Map` that +/// is not in the allowed set. Returns `true` if any keys were removed. +pub fn strip_unknown_properties_from_document_schema(schema: &mut Value) -> bool { + let map = match schema { + Value::Map(map) => map, + _ => return false, + }; + + let before = map.len(); + map.retain(|(key, _)| { + let key_str = match key { + Value::Text(s) => s.as_str(), + _ => return true, // keep non-string keys (shouldn't happen but safe) + }; + ALLOWED_DOCUMENT_SCHEMA_V1_PROPERTIES.contains(&key_str) + }); + map.len() != before +} + +#[cfg(test)] +mod tests { + use super::*; + use platform_value::platform_value; + + #[test] + fn strips_unknown_properties() { + let mut schema = platform_value!({ + "type": "object", + "properties": {}, + "additionalProperties": false, + "unknownProp": true, + "anotherUnknown": 42 + }); + + let changed = strip_unknown_properties_from_document_schema(&mut schema); + assert!(changed); + + let map = schema.as_map().unwrap(); + let keys: Vec<&str> = map.iter().filter_map(|(k, _)| k.as_text()).collect(); + assert!(!keys.contains(&"unknownProp")); + assert!(!keys.contains(&"anotherUnknown")); + assert!(keys.contains(&"type")); + assert!(keys.contains(&"properties")); + assert!(keys.contains(&"additionalProperties")); + } + + #[test] + fn no_change_when_all_properties_are_known() { + let mut schema = platform_value!({ + "type": "object", + "properties": {}, + "additionalProperties": false, + "required": ["foo"], + "$comment": "test" + }); + + let changed = strip_unknown_properties_from_document_schema(&mut schema); + assert!(!changed); + } + + #[test] + fn handles_non_map_value() { + let mut schema = Value::Text("not a map".to_string()); + let changed = strip_unknown_properties_from_document_schema(&mut schema); + assert!(!changed); + } + + #[test] + fn allowlist_matches_v1_meta_schema_properties() { + let v1_schema: serde_json::Value = serde_json::from_str(include_str!( + "../../../../schema/meta_schemas/document/v1/document-meta.json" + )) + .expect("v1 document meta-schema JSON must be valid"); + + let schema_properties: std::collections::BTreeSet<&str> = v1_schema + .get("properties") + .and_then(|p| p.as_object()) + .expect("v1 meta-schema must have a 'properties' object") + .keys() + .map(|k| k.as_str()) + .collect(); + + let allowlist: std::collections::BTreeSet<&str> = ALLOWED_DOCUMENT_SCHEMA_V1_PROPERTIES + .iter() + .copied() + .collect(); + + let in_allowlist_not_schema: Vec<&&str> = + allowlist.difference(&schema_properties).collect(); + let in_schema_not_allowlist: Vec<&&str> = + schema_properties.difference(&allowlist).collect(); + + assert!( + in_allowlist_not_schema.is_empty(), + "Properties in ALLOWED_DOCUMENT_SCHEMA_V1_PROPERTIES but not in v1 meta-schema: {:?}", + in_allowlist_not_schema + ); + assert!( + in_schema_not_allowlist.is_empty(), + "Properties in v1 meta-schema but not in ALLOWED_DOCUMENT_SCHEMA_V1_PROPERTIES: {:?}", + in_schema_not_allowlist + ); + } +} diff --git a/packages/rs-dpp/src/data_contract/document_type/schema/enrich_with_base_schema/mod.rs b/packages/rs-dpp/src/data_contract/document_type/schema/enrich_with_base_schema/mod.rs index dc7019cf7a2..468c9594fb6 100644 --- a/packages/rs-dpp/src/data_contract/document_type/schema/enrich_with_base_schema/mod.rs +++ b/packages/rs-dpp/src/data_contract/document_type/schema/enrich_with_base_schema/mod.rs @@ -1,4 +1,5 @@ mod v0; +mod v1; use crate::consensus::basic::BasicError; use crate::consensus::ConsensusError; @@ -27,11 +28,72 @@ impl DocumentType { ) })?, ), + 1 => Ok( + v1::enrich_with_base_schema_v1(schema, schema_defs).map_err(|e| { + ProtocolError::ConsensusError( + ConsensusError::BasicError(BasicError::ContractError(e)).into(), + ) + })?, + ), version => Err(ProtocolError::UnknownVersionMismatch { method: "enrich_with_base_schema".to_string(), - known_versions: vec![0], + known_versions: vec![0, 1], received: version, }), } } } + +#[cfg(test)] +mod tests { + use super::*; + use platform_value::{platform_value, ValueMapHelper}; + + fn minimal_schema() -> Value { + platform_value!({ + "type": "object", + "properties": { + "name": { "type": "string" } + }, + "additionalProperties": false + }) + } + + #[test] + fn v0_enrichment_injects_v0_schema_uri() { + let platform_version = PlatformVersion::get(11).expect("expected v11"); + let enriched = + DocumentType::enrich_with_base_schema(minimal_schema(), None, platform_version) + .expect("enrichment should succeed"); + + let map = enriched.to_map_ref().expect("should be map"); + let schema_value = map + .get_optional_key("$schema") + .expect("should have $schema"); + let schema_uri = schema_value.as_text().expect("should be text"); + + assert!( + schema_uri.contains("/v0/document-meta.json"), + "pre-v12 should use v0 URI, got: {schema_uri}" + ); + } + + #[test] + fn v1_enrichment_injects_v1_schema_uri() { + let platform_version = PlatformVersion::latest(); + let enriched = + DocumentType::enrich_with_base_schema(minimal_schema(), None, platform_version) + .expect("enrichment should succeed"); + + let map = enriched.to_map_ref().expect("should be map"); + let schema_value = map + .get_optional_key("$schema") + .expect("should have $schema"); + let schema_uri = schema_value.as_text().expect("should be text"); + + assert!( + schema_uri.contains("/v1/document-meta.json"), + "v12+ should use v1 URI, got: {schema_uri}" + ); + } +} diff --git a/packages/rs-dpp/src/data_contract/document_type/schema/enrich_with_base_schema/v1/mod.rs b/packages/rs-dpp/src/data_contract/document_type/schema/enrich_with_base_schema/v1/mod.rs new file mode 100644 index 00000000000..a4c37833b5d --- /dev/null +++ b/packages/rs-dpp/src/data_contract/document_type/schema/enrich_with_base_schema/v1/mod.rs @@ -0,0 +1,78 @@ +use crate::data_contract::document_type::property_names; +use crate::data_contract::errors::DataContractError; +use crate::data_contract::serialized_version::property_names as contract_property_names; +use platform_value::{Value, ValueMapHelper}; + +pub const DATA_CONTRACT_SCHEMA_URI_V1: &str = + "https://github.com/dashpay/platform/blob/master/packages/rs-dpp/schema/meta_schemas/document/v1/document-meta.json"; + +pub const PROPERTY_SCHEMA: &str = "$schema"; + +const SYSTEM_GENERATED_FIELDS: [&str; 9] = [ + "$createdAt", + "$updatedAt", + "$transferredAt", + "$createdAtBlockHeight", + "$updatedAtBlockHeight", + "$transferredAtBlockHeight", + "$createdAtCoreBlockHeight", + "$updatedAtCoreBlockHeight", + "$transferredAtCoreBlockHeight", +]; + +#[inline(always)] +pub(super) fn enrich_with_base_schema_v1( + mut schema: Value, + schema_defs: Option, +) -> Result { + let schema_map = schema.to_map_mut().map_err(|err| { + DataContractError::InvalidContractStructure(format!( + "document schema must be an object: {err}" + )) + })?; + + // Add $schema + if schema_map.get_optional_key(PROPERTY_SCHEMA).is_some() { + return Err(DataContractError::InvalidContractStructure( + "document schema shouldn't contain '$schema' property".to_string(), + )); + } + + schema_map.insert_string_key_value( + PROPERTY_SCHEMA.to_string(), + DATA_CONTRACT_SCHEMA_URI_V1.into(), + ); + + // Add $defs + if schema_map + .get_optional_key(contract_property_names::DEFINITIONS) + .is_some() + { + return Err(DataContractError::InvalidContractStructure( + "document schema shouldn't contain '$defs' property".to_string(), + )); + } + + if let Some(schema_defs) = schema_defs { + schema_map.insert_string_key_value( + contract_property_names::DEFINITIONS.to_string(), + schema_defs, + ) + } + + // Remove system-generated fields from required since they aren't part of + // dynamic (user defined) document data which is validating against the schema + if let Some(required) = schema_map.get_optional_key_mut(property_names::REQUIRED) { + if let Some(required_array) = required.as_array_mut() { + required_array.retain(|field_value| { + if let Some(field) = field_value.as_text() { + !SYSTEM_GENERATED_FIELDS.contains(&field) + } else { + true + } + }); + } + } + + Ok(schema) +} diff --git a/packages/rs-dpp/src/data_contract/document_type/schema/mod.rs b/packages/rs-dpp/src/data_contract/document_type/schema/mod.rs index 169953690a5..7753caf844d 100644 --- a/packages/rs-dpp/src/data_contract/document_type/schema/mod.rs +++ b/packages/rs-dpp/src/data_contract/document_type/schema/mod.rs @@ -1,3 +1,5 @@ +pub mod allowed_top_level_properties; + mod enrich_with_base_schema; mod find_identifier_and_binary_paths; diff --git a/packages/rs-dpp/src/data_contract/serialized_version/mod.rs b/packages/rs-dpp/src/data_contract/serialized_version/mod.rs index d1188a2899a..188c8a7a841 100644 --- a/packages/rs-dpp/src/data_contract/serialized_version/mod.rs +++ b/packages/rs-dpp/src/data_contract/serialized_version/mod.rs @@ -134,6 +134,13 @@ impl DataContractInSerializationFormat { } } + pub fn document_schemas_mut(&mut self) -> &mut BTreeMap { + match self { + DataContractInSerializationFormat::V0(v0) => &mut v0.document_schemas, + DataContractInSerializationFormat::V1(v1) => &mut v1.document_schemas, + } + } + pub fn schema_defs(&self) -> Option<&BTreeMap> { match self { DataContractInSerializationFormat::V0(v0) => v0.schema_defs.as_ref(), diff --git a/packages/rs-dpp/src/validation/meta_validators/mod.rs b/packages/rs-dpp/src/validation/meta_validators/mod.rs index fd161724f20..9b31de0f020 100644 --- a/packages/rs-dpp/src/validation/meta_validators/mod.rs +++ b/packages/rs-dpp/src/validation/meta_validators/mod.rs @@ -39,7 +39,11 @@ lazy_static! { static ref DOCUMENT_META_JSON_V0: Value = serde_json::from_str::(include_str!( "../../../schema/meta_schemas/document/v0/document-meta.json" )) - .unwrap(); + .expect("v0 document meta-schema JSON must be valid"); + static ref DOCUMENT_META_JSON_V1: Value = serde_json::from_str::(include_str!( + "../../../schema/meta_schemas/document/v1/document-meta.json" + )) + .expect("v1 document meta-schema JSON must be valid"); pub static ref DRAFT_202012_META_SCHEMA: JSONSchema = JSONSchema::options() .with_draft(Draft::Draft202012) @@ -57,10 +61,6 @@ lazy_static! { "https://json-schema.org/draft/2020-12/meta/core".to_string(), DRAFT202012_CORE.clone(), ) - .with_document( - "https://json-schema.org/draft/2020-12/meta/applicator".to_string(), - DRAFT202012_APPLICATOR.clone(), - ) .with_document( "https://json-schema.org/draft/2020-12/meta/unevaluated".to_string(), DRAFT202012_UNEVALUATED.clone(), @@ -107,10 +107,56 @@ lazy_static! { "https://json-schema.org/draft/2020-12/meta/core".to_string(), DRAFT202012_CORE.clone(), ) + .with_document( + "https://json-schema.org/draft/2020-12/meta/unevaluated".to_string(), + DRAFT202012_UNEVALUATED.clone(), + ) + .with_document( + "https://json-schema.org/draft/2020-12/meta/validation".to_string(), + DRAFT202012_VALIDATION.clone(), + ) + .with_document( + "https://json-schema.org/draft/2020-12/meta/meta-data".to_string(), + DRAFT202012_META_DATA.clone(), + ) + .with_document( + "https://json-schema.org/draft/2020-12/meta/format-annotation".to_string(), + DRAFT202012_FORMAT_ANNOTATION.clone(), + ) + .with_document( + "https://json-schema.org/draft/2020-12/meta/content".to_string(), + DRAFT202012_CONTENT.clone(), + ) + .with_document( + "https://json-schema.org/draft/2020-12/schema".to_string(), + DRAFT202012.clone(), + ) + .to_owned() + .compile(&DOCUMENT_META_JSON_V0) + .expect("Invalid data contract schema"); + + // Compiled version of document meta schema v1 + // This version adds additionalProperties: false at the top level + pub static ref DOCUMENT_META_SCHEMA_V1: JSONSchema = JSONSchema::options() + .with_keyword( + "byteArray", + |_, _, _| Ok(Box::new(ByteArrayKeyword)), + ) + .with_patterns_regex_engine(RegexEngine::Regex(RegexOptions { + size_limit: Some(5 * (1 << 20)), + ..Default::default() + })) + .should_ignore_unknown_formats(false) + .should_validate_formats(true) + .with_draft(Draft::Draft202012) .with_document( "https://json-schema.org/draft/2020-12/meta/applicator".to_string(), DRAFT202012_APPLICATOR.clone(), ) + .with_document( + "https://json-schema.org/draft/2020-12/meta/core".to_string(), + DRAFT202012_CORE.clone(), + ) .with_document( "https://json-schema.org/draft/2020-12/meta/unevaluated".to_string(), DRAFT202012_UNEVALUATED.clone(), @@ -136,7 +182,7 @@ lazy_static! { DRAFT202012.clone(), ) .to_owned() - .compile(&DOCUMENT_META_JSON_V0) + .compile(&DOCUMENT_META_JSON_V1) .expect("Invalid data contract schema"); } diff --git a/packages/rs-drive-abci/src/execution/platform_events/protocol_upgrade/perform_events_on_first_block_of_protocol_change/v0/mod.rs b/packages/rs-drive-abci/src/execution/platform_events/protocol_upgrade/perform_events_on_first_block_of_protocol_change/v0/mod.rs index dff4cc3eb28..d4a45a70472 100644 --- a/packages/rs-drive-abci/src/execution/platform_events/protocol_upgrade/perform_events_on_first_block_of_protocol_change/v0/mod.rs +++ b/packages/rs-drive-abci/src/execution/platform_events/protocol_upgrade/perform_events_on_first_block_of_protocol_change/v0/mod.rs @@ -668,6 +668,15 @@ impl Platform { &platform_version.drive, )?; + // Strip unknown top-level properties from all contract document type schemas. + // The v1 document meta-schema enforces additionalProperties: false at the + // document-type level. Contracts created under the v0 meta-schema (which did + // NOT forbid unknown keys) may carry stale properties that would fail + // validation under v1. We clean them up here so every stored contract + // conforms to the v1 meta-schema going forward. + self.drive + .strip_unknown_document_schema_properties(transaction, &platform_version.drive)?; + Ok(()) } } @@ -938,4 +947,759 @@ mod tests { let result = platform.transition_to_version_6(&block_info, &transaction, platform_version); assert!(result.is_ok()); } + + #[test] + fn test_v12_migration_strips_unknown_document_schema_properties() { + use dpp::data_contract::accessors::v0::{DataContractV0Getters, DataContractV0Setters}; + use dpp::data_contract::document_type::accessors::DocumentTypeV0Getters; + use dpp::data_contract::serialized_version::DataContractInSerializationFormat; + use dpp::identity::accessors::IdentityGettersV0; + use dpp::identity::Identity; + use dpp::platform_value::Value as PlatformValue; + use dpp::serialization::PlatformSerializableWithPlatformVersion; + use dpp::tests::json_document::json_document_to_contract_with_ids; + use platform_version::TryFromPlatformVersioned; + + let platform_version_11 = PlatformVersion::get(11).expect("expected v11"); + let platform_version_12 = PlatformVersion::latest(); + + // 1. Set up platform at v11 + let mut platform = TestPlatformBuilder::new() + .build_with_mock_rpc() + .set_genesis_state(); + + let transaction = platform.drive.grove.start_transaction(); + + // 2. Create an identity to own the contract + let identity = Identity::random_identity(3, Some(42), platform_version_11) + .expect("expected a random identity"); + + platform + .drive + .add_new_identity( + identity.clone(), + false, + &BlockInfo::default(), + true, + Some(&transaction), + platform_version_11, + ) + .expect("expected to add identity"); + + // 3. Create a contract and manually inject unknown properties into the schema + let mut data_contract = json_document_to_contract_with_ids( + "../rs-drive/tests/supporting_files/contract/family/family-contract.json", + None, + None, + false, // no validation — we want to inject unknown props + platform_version_11, + ) + .expect("expected to get contract"); + + data_contract.set_owner_id(identity.id()); + + // Get the raw serialized format and inject unknown properties + let mut serialization_format = + DataContractInSerializationFormat::try_from_platform_versioned( + data_contract.clone(), + platform_version_11, + ) + .expect("expected to convert to serialization format"); + + // Inject unknown property into the "person" document schema + for (_doc_type_name, schema_value) in serialization_format.document_schemas_mut().iter_mut() + { + if let Some(map) = schema_value.as_map_mut() { + map.push(( + PlatformValue::Text("unknownSmuggled".to_string()), + PlatformValue::Bool(true), + )); + } + } + + // Serialize the modified contract and insert directly into Drive + let bincode_config = bincode::config::standard() + .with_big_endian() + .with_no_limit(); + let contract_bytes = bincode::encode_to_vec(&serialization_format, bincode_config) + .expect("expected to serialize"); + + let contract_id = data_contract.id(); + let contract_path = + drive::drive::contract::paths::contract_root_path(contract_id.as_bytes()); + + // Insert the contract storage tree + platform + .drive + .grove_insert_if_not_exists( + SubtreePath::from(&[&[RootTree::DataContractDocuments as u8] as &[u8]][..]), + contract_id.as_bytes(), + Element::empty_tree(), + Some(&transaction), + None, + &platform_version_11.drive, + ) + .expect("insert contract tree"); + + // Insert the contract data at key [0] + platform + .drive + .grove_insert( + (&contract_path).into(), + &[0], + Element::Item(contract_bytes, None), + Some(&transaction), + None, + &mut vec![], + &platform_version_11.drive, + ) + .expect("insert contract data"); + + // 4. Verify the unknown property is there before migration + let raw_before = platform + .drive + .grove_get_raw( + (&contract_path).into(), + &[0], + drive::util::grove_operations::DirectQueryType::StatefulDirectQuery, + Some(&transaction), + &mut vec![], + &platform_version_11.drive, + ) + .expect("get raw") + .expect("element should exist"); + + let bytes_before = match &raw_before { + Element::Item(bytes, _) => bytes.clone(), + _ => panic!("expected Item"), + }; + + let format_before: DataContractInSerializationFormat = bincode::borrow_decode_from_slice( + &bytes_before, + bincode::config::standard() + .with_big_endian() + .with_no_limit(), + ) + .expect("deserialize") + .0; + + let has_unknown_before = format_before.document_schemas().values().any(|schema| { + schema + .as_map() + .map(|map| { + map.iter() + .any(|(k, _)| k.as_text() == Some("unknownSmuggled")) + }) + .unwrap_or(false) + }); + assert!( + has_unknown_before, + "Contract should have unknownSmuggled property before migration" + ); + + // 5. Run the v12 migration + // First need v11 trees + platform + .transition_to_version_11(&transaction, platform_version_12) + .expect("v11 transition"); + + platform + .transition_to_version_12(&transaction, platform_version_12) + .expect("v12 transition should succeed and strip unknown properties"); + + // 6. Verify the unknown property is gone from disk + let raw_after = platform + .drive + .grove_get_raw( + (&contract_path).into(), + &[0], + drive::util::grove_operations::DirectQueryType::StatefulDirectQuery, + Some(&transaction), + &mut vec![], + &platform_version_12.drive, + ) + .expect("get raw") + .expect("element should exist"); + + let bytes_after = match &raw_after { + Element::Item(bytes, _) => bytes.clone(), + _ => panic!("expected Item"), + }; + + let format_after: DataContractInSerializationFormat = bincode::borrow_decode_from_slice( + &bytes_after, + bincode::config::standard() + .with_big_endian() + .with_no_limit(), + ) + .expect("deserialize") + .0; + + let has_unknown_after = format_after.document_schemas().values().any(|schema| { + schema + .as_map() + .map(|map| { + map.iter() + .any(|(k, _)| k.as_text() == Some("unknownSmuggled")) + }) + .unwrap_or(false) + }); + assert!( + !has_unknown_after, + "Contract should NOT have unknownSmuggled property after v12 migration" + ); + + // 7. Verify known properties are still present + let has_type = format_after.document_schemas().values().any(|schema| { + schema + .as_map() + .map(|map| map.iter().any(|(k, _)| k.as_text() == Some("type"))) + .unwrap_or(false) + }); + assert!( + has_type, + "Contract should still have 'type' property after migration" + ); + + // 8. Verify the cache was cleared by fetching the contract through the + // Drive API. The cache was populated implicitly during migration (or + // could have been from prior block processing). After the migration + // clears the global cache, a fresh fetch should reload from disk and + // return the cleaned contract. + platform + .drive + .grove + .commit_transaction(transaction) + .unwrap() + .expect("commit"); + + let (_fee, fetched): ( + _, + Option>, + ) = platform + .drive + .get_contract_with_fetch_info_and_fee( + *contract_id.as_bytes(), + None, + false, + None, // no transaction — reads committed state + platform_version_12, + ) + .expect("fetch from cache/disk"); + + let fetch_info = fetched.expect("contract should exist"); + let fetched_contract = &fetch_info.contract; + + // Check the fetched contract's raw serialization doesn't have the unknown property + // (This verifies both cache invalidation and disk update) + let refetched_format = DataContractInSerializationFormat::try_from_platform_versioned( + fetched_contract.clone(), + platform_version_12, + ) + .expect("convert to serialization format"); + + let has_unknown_refetched = refetched_format.document_schemas().values().any(|schema| { + schema + .as_map() + .map(|map| { + map.iter() + .any(|(k, _)| k.as_text() == Some("unknownSmuggled")) + }) + .unwrap_or(false) + }); + assert!( + !has_unknown_refetched, + "Contract fetched through Drive API after migration should not have unknownSmuggled" + ); + } + + #[test] + fn test_v12_migration_strips_unknown_properties_from_historical_contract() { + use dpp::data_contract::accessors::v0::{DataContractV0Getters, DataContractV0Setters}; + use dpp::data_contract::serialized_version::DataContractInSerializationFormat; + use dpp::identity::accessors::IdentityGettersV0; + use dpp::identity::Identity; + use dpp::platform_value::Value as PlatformValue; + use dpp::tests::json_document::json_document_to_contract_with_ids; + use drive::grovedb::reference_path::ReferencePathType; + use platform_version::TryFromPlatformVersioned; + + let platform_version_11 = PlatformVersion::get(11).expect("expected v11"); + let platform_version_12 = PlatformVersion::latest(); + + let platform = TestPlatformBuilder::new() + .build_with_mock_rpc() + .set_genesis_state(); + + let transaction = platform.drive.grove.start_transaction(); + + // Create identity + let identity = Identity::random_identity(3, Some(99), platform_version_11) + .expect("expected a random identity"); + platform + .drive + .add_new_identity( + identity.clone(), + false, + &BlockInfo::default(), + true, + Some(&transaction), + platform_version_11, + ) + .expect("expected to add identity"); + + // Create contract + let mut data_contract = json_document_to_contract_with_ids( + "../rs-drive/tests/supporting_files/contract/family/family-contract.json", + None, + None, + false, + platform_version_11, + ) + .expect("expected to get contract"); + data_contract.set_owner_id(identity.id()); + + // Serialize and inject unknown property + let mut serialization_format = + DataContractInSerializationFormat::try_from_platform_versioned( + data_contract.clone(), + platform_version_11, + ) + .expect("convert to serialization format"); + + for (_name, schema_value) in serialization_format.document_schemas_mut().iter_mut() { + if let Some(map) = schema_value.as_map_mut() { + map.push(( + PlatformValue::Text("historicalSmuggled".to_string()), + PlatformValue::Bool(true), + )); + } + } + + let bincode_config = bincode::config::standard() + .with_big_endian() + .with_no_limit(); + let contract_bytes = + bincode::encode_to_vec(&serialization_format, bincode_config).expect("serialize"); + + let contract_id = data_contract.id(); + + // Build historical contract storage layout: + // [DataContractDocuments, contract_id] = tree (contract root) + // [DataContractDocuments, contract_id, 0] = tree (history root) + // [DataContractDocuments, contract_id, 0, encoded_time] = Item(contract_bytes) + // [DataContractDocuments, contract_id, 0, 0] = Reference(SiblingReference(encoded_time)) + + let root_tree_key = &[RootTree::DataContractDocuments as u8]; + let encoded_time = drive::util::common::encode::encode_u64(1000000); + + // Insert contract root tree + platform + .drive + .grove_insert_if_not_exists( + SubtreePath::from(&[root_tree_key as &[u8]][..]), + contract_id.as_bytes(), + Element::empty_tree(), + Some(&transaction), + None, + &platform_version_11.drive, + ) + .expect("insert contract root"); + + // Insert history root tree at key [0] + let contract_root_path = + drive::drive::contract::paths::contract_root_path(contract_id.as_bytes()); + platform + .drive + .grove_insert( + (&contract_root_path).into(), + &[0], + Element::empty_tree(), + Some(&transaction), + None, + &mut vec![], + &platform_version_11.drive, + ) + .expect("insert history tree"); + + // Insert contract data at the timestamp key + let history_path = drive::drive::contract::paths::contract_keeping_history_root_path( + contract_id.as_bytes(), + ); + platform + .drive + .grove_insert( + (&history_path).into(), + encoded_time.as_slice(), + Element::Item(contract_bytes, None), + Some(&transaction), + None, + &mut vec![], + &platform_version_11.drive, + ) + .expect("insert contract at timestamp"); + + // Insert reference at key [0] pointing to the timestamp key + platform + .drive + .grove_insert( + (&history_path).into(), + &[0], + Element::Reference( + ReferencePathType::SiblingReference(encoded_time.clone()), + Some(1), + None, + ), + Some(&transaction), + None, + &mut vec![], + &platform_version_11.drive, + ) + .expect("insert reference"); + + // Verify unknown property exists before migration + let raw_element = platform + .drive + .grove_get_raw( + (&history_path).into(), + encoded_time.as_slice(), + drive::util::grove_operations::DirectQueryType::StatefulDirectQuery, + Some(&transaction), + &mut vec![], + &platform_version_11.drive, + ) + .expect("get raw") + .expect("element"); + + let bytes_before = match &raw_element { + Element::Item(bytes, _) => bytes.clone(), + _ => panic!("expected Item"), + }; + let format_before: DataContractInSerializationFormat = + bincode::borrow_decode_from_slice(&bytes_before, bincode_config) + .expect("deserialize") + .0; + + let has_smuggled = format_before.document_schemas().values().any(|s| { + s.as_map() + .map(|m| { + m.iter() + .any(|(k, _)| k.as_text() == Some("historicalSmuggled")) + }) + .unwrap_or(false) + }); + assert!( + has_smuggled, + "historical contract should have smuggled property before migration" + ); + + // Run v12 migration + platform + .transition_to_version_11(&transaction, platform_version_12) + .expect("v11 transition"); + platform + .transition_to_version_12(&transaction, platform_version_12) + .expect("v12 transition"); + + // Verify property is stripped from disk (at the timestamp key) + let raw_after = platform + .drive + .grove_get_raw( + (&history_path).into(), + encoded_time.as_slice(), + drive::util::grove_operations::DirectQueryType::StatefulDirectQuery, + Some(&transaction), + &mut vec![], + &platform_version_12.drive, + ) + .expect("get raw") + .expect("element"); + + let bytes_after = match &raw_after { + Element::Item(bytes, _) => bytes.clone(), + _ => panic!("expected Item"), + }; + let format_after: DataContractInSerializationFormat = + bincode::borrow_decode_from_slice(&bytes_after, bincode_config) + .expect("deserialize") + .0; + + let has_smuggled_after = format_after.document_schemas().values().any(|s| { + s.as_map() + .map(|m| { + m.iter() + .any(|(k, _)| k.as_text() == Some("historicalSmuggled")) + }) + .unwrap_or(false) + }); + assert!( + !has_smuggled_after, + "historical contract should NOT have smuggled property after v12 migration" + ); + } + + #[test] + fn test_v12_migration_strips_unknown_properties_from_all_historical_revisions() { + use dpp::data_contract::accessors::v0::{DataContractV0Getters, DataContractV0Setters}; + use dpp::data_contract::serialized_version::DataContractInSerializationFormat; + use dpp::identity::accessors::IdentityGettersV0; + use dpp::identity::Identity; + use dpp::platform_value::Value as PlatformValue; + use drive::grovedb::reference_path::ReferencePathType; + use platform_version::TryFromPlatformVersioned; + + let platform_version_11 = PlatformVersion::get(11).expect("expected v11"); + let platform_version_12 = PlatformVersion::latest(); + + let platform = TestPlatformBuilder::new() + .build_with_mock_rpc() + .set_genesis_state(); + + let transaction = platform.drive.grove.start_transaction(); + + // Create identity + let identity = Identity::random_identity(3, Some(101), platform_version_11) + .expect("expected a random identity"); + platform + .drive + .add_new_identity( + identity.clone(), + false, + &BlockInfo::default(), + true, + Some(&transaction), + platform_version_11, + ) + .expect("expected to add identity"); + + // Create base contract + let mut data_contract = dpp::tests::json_document::json_document_to_contract_with_ids( + "../rs-drive/tests/supporting_files/contract/family/family-contract.json", + None, + None, + false, + platform_version_11, + ) + .expect("expected to get contract"); + data_contract.set_owner_id(identity.id()); + + let contract_id = data_contract.id(); + let bincode_config = bincode::config::standard() + .with_big_endian() + .with_no_limit(); + + // Build 3 revisions, each with a different unknown property + let timestamps: Vec = vec![1_000_000, 2_000_000, 3_000_000]; + let smuggled_keys: Vec<&str> = vec!["smuggled_v1", "smuggled_v2", "smuggled_v3"]; + + let mut revision_bytes = Vec::new(); + for (i, smuggled_key) in smuggled_keys.iter().enumerate() { + let mut serialization_format = + DataContractInSerializationFormat::try_from_platform_versioned( + data_contract.clone(), + platform_version_11, + ) + .expect("convert to serialization format"); + + for (_name, schema_value) in serialization_format.document_schemas_mut().iter_mut() { + if let Some(map) = schema_value.as_map_mut() { + map.push(( + PlatformValue::Text(smuggled_key.to_string()), + PlatformValue::U32(i as u32), + )); + } + } + + let bytes = + bincode::encode_to_vec(&serialization_format, bincode_config).expect("serialize"); + revision_bytes.push(bytes); + } + + // Set up the historical contract storage layout + let root_tree_key = &[RootTree::DataContractDocuments as u8]; + + // Contract root tree + platform + .drive + .grove_insert_if_not_exists( + SubtreePath::from(&[root_tree_key as &[u8]][..]), + contract_id.as_bytes(), + Element::empty_tree(), + Some(&transaction), + None, + &platform_version_11.drive, + ) + .expect("insert contract root"); + + // History root tree at key [0] + let contract_root_path = + drive::drive::contract::paths::contract_root_path(contract_id.as_bytes()); + platform + .drive + .grove_insert( + (&contract_root_path).into(), + &[0], + Element::empty_tree(), + Some(&transaction), + None, + &mut vec![], + &platform_version_11.drive, + ) + .expect("insert history tree"); + + let history_path = drive::drive::contract::paths::contract_keeping_history_root_path( + contract_id.as_bytes(), + ); + + // Insert all 3 revisions at their timestamp keys, each with distinct flags + let revision_flags: Vec>> = + vec![Some(vec![1, 10]), Some(vec![2, 20]), Some(vec![3, 30])]; + let mut encoded_times = Vec::new(); + for (i, ts) in timestamps.iter().enumerate() { + let encoded_time = drive::util::common::encode::encode_u64(*ts); + platform + .drive + .grove_insert( + (&history_path).into(), + encoded_time.as_slice(), + Element::Item(revision_bytes[i].clone(), revision_flags[i].clone()), + Some(&transaction), + None, + &mut vec![], + &platform_version_11.drive, + ) + .expect("insert revision"); + encoded_times.push(encoded_time); + } + + // Reference at [0] pointing to the latest (3rd) revision + platform + .drive + .grove_insert( + (&history_path).into(), + &[0], + Element::Reference( + ReferencePathType::SiblingReference(encoded_times[2].clone()), + Some(1), + None, + ), + Some(&transaction), + None, + &mut vec![], + &platform_version_11.drive, + ) + .expect("insert reference"); + + // Verify all 3 revisions have their smuggled property before migration + for (i, encoded_time) in encoded_times.iter().enumerate() { + let raw = platform + .drive + .grove_get_raw( + (&history_path).into(), + encoded_time.as_slice(), + drive::util::grove_operations::DirectQueryType::StatefulDirectQuery, + Some(&transaction), + &mut vec![], + &platform_version_11.drive, + ) + .expect("get raw") + .expect("element"); + + let bytes = match &raw { + Element::Item(bytes, _) => bytes.clone(), + _ => panic!("expected Item"), + }; + let format: DataContractInSerializationFormat = + bincode::borrow_decode_from_slice(&bytes, bincode_config) + .expect("deserialize") + .0; + + let has_smuggled = format.document_schemas().values().any(|s| { + s.as_map() + .map(|m| m.iter().any(|(k, _)| k.as_text() == Some(smuggled_keys[i]))) + .unwrap_or(false) + }); + assert!( + has_smuggled, + "revision {} should have '{}' before migration", + i, smuggled_keys[i] + ); + } + + // Run v12 migration + platform + .transition_to_version_11(&transaction, platform_version_12) + .expect("v11 transition"); + platform + .transition_to_version_12(&transaction, platform_version_12) + .expect("v12 transition"); + + // Verify ALL 3 revisions are cleaned + for (i, encoded_time) in encoded_times.iter().enumerate() { + let raw = platform + .drive + .grove_get_raw( + (&history_path).into(), + encoded_time.as_slice(), + drive::util::grove_operations::DirectQueryType::StatefulDirectQuery, + Some(&transaction), + &mut vec![], + &platform_version_12.drive, + ) + .expect("get raw") + .expect("element"); + + let (bytes, flags) = match &raw { + Element::Item(bytes, flags) => (bytes.clone(), flags.clone()), + _ => panic!("expected Item"), + }; + let format: DataContractInSerializationFormat = + bincode::borrow_decode_from_slice(&bytes, bincode_config) + .expect("deserialize") + .0; + + let has_any_smuggled = format.document_schemas().values().any(|s| { + s.as_map() + .map(|m| { + m.iter().any(|(k, _)| { + k.as_text() + .map(|t| t.starts_with("smuggled_")) + .unwrap_or(false) + }) + }) + .unwrap_or(false) + }); + assert!( + !has_any_smuggled, + "revision {} should NOT have any smuggled properties after migration", + i + ); + + // Verify element flags are preserved + assert_eq!( + flags, revision_flags[i], + "revision {} should preserve its original element flags after migration", + i + ); + } + + // Verify the reference at [0] still exists and is intact + let ref_element = platform + .drive + .grove_get_raw( + (&history_path).into(), + &[0], + drive::util::grove_operations::DirectQueryType::StatefulDirectQuery, + Some(&transaction), + &mut vec![], + &platform_version_12.drive, + ) + .expect("get raw") + .expect("reference should still exist"); + + assert!( + matches!(ref_element, Element::Reference(..)), + "reference element should still be a Reference after migration" + ); + } } diff --git a/packages/rs-drive/src/drive/contract/migration/mod.rs b/packages/rs-drive/src/drive/contract/migration/mod.rs new file mode 100644 index 00000000000..df8449cd693 --- /dev/null +++ b/packages/rs-drive/src/drive/contract/migration/mod.rs @@ -0,0 +1 @@ +mod strip_unknown_document_schema_properties; diff --git a/packages/rs-drive/src/drive/contract/migration/strip_unknown_document_schema_properties.rs b/packages/rs-drive/src/drive/contract/migration/strip_unknown_document_schema_properties.rs new file mode 100644 index 00000000000..bb50e2cd38e --- /dev/null +++ b/packages/rs-drive/src/drive/contract/migration/strip_unknown_document_schema_properties.rs @@ -0,0 +1,212 @@ +use crate::drive::contract::paths::{contract_keeping_history_root_path, contract_root_path}; +use crate::drive::Drive; +use crate::error::drive::DriveError; +use crate::error::Error; +use crate::query::QueryResultType; +use crate::util::grove_operations::DirectQueryType; +use dpp::data_contract::document_type::schema::allowed_top_level_properties::strip_unknown_properties_from_document_schema; +use dpp::data_contract::serialized_version::DataContractInSerializationFormat; +use dpp::version::drive_versions::DriveVersion; +use grovedb::{Element, PathQuery, Query, SizedQuery, Transaction}; +use grovedb_path::SubtreePath; + +impl Drive { + /// Iterates every data contract in state, checks each document type schema for + /// top-level properties not listed in the v1 document meta-schema, removes them, + /// and re-serializes the contract if anything changed. + /// + /// For historical contracts, all stored revisions are cleaned (not just the latest). + /// + /// Also clears the data contract cache so that subsequent fetches reload + /// the cleaned contracts from disk. + pub fn strip_unknown_document_schema_properties( + &self, + transaction: &Transaction, + drive_version: &DriveVersion, + ) -> Result<(), Error> { + // 1. Fetch all contract IDs. + let contract_ids = + self.fetch_contract_ids_v0(None, u16::MAX, Some(transaction), drive_version)?; + + tracing::debug!( + contract_count = contract_ids.len(), + "Checking contracts for unknown document schema properties" + ); + + let bincode_config = bincode::config::standard() + .with_big_endian() + .with_no_limit(); + + // 2. For each contract, read the raw element, check, and possibly update. + for contract_id_bytes in &contract_ids { + let contract_path = contract_root_path(contract_id_bytes.as_slice()); + + // Try reading the element at key [0] under the contract root. + let maybe_element = self.grove_get_raw( + (&contract_path).into(), + &[0], + DirectQueryType::StatefulDirectQuery, + Some(transaction), + &mut vec![], + drive_version, + )?; + + match maybe_element { + Some(Element::Item(bytes, flags)) => { + // Non-historical contract: stored directly at [root, id, [0]] + let path_vec: Vec> = contract_path.iter().map(|s| s.to_vec()).collect(); + self.strip_and_rewrite_contract_element( + &bytes, + flags, + &path_vec, + &[0], + contract_id_bytes, + bincode_config, + transaction, + drive_version, + )?; + } + Some(Element::Tree(..)) => { + // Historical contract: iterate ALL revisions in the history + // subtree at [root, id, 0]. Each revision is an Item keyed + // by an encoded timestamp. Key [0] is a Reference to the + // latest — we skip references and only process Items. + let history_path = + contract_keeping_history_root_path(contract_id_bytes.as_slice()); + let history_path_vec: Vec> = + history_path.iter().map(|s| s.to_vec()).collect(); + + let mut query = Query::new(); + query.insert_all(); + let path_query = PathQuery::new( + history_path_vec.clone(), + SizedQuery::new(query, None, None), + ); + + let (result_items, _) = self.grove_get_raw_path_query( + &path_query, + Some(transaction), + QueryResultType::QueryKeyElementPairResultType, + &mut vec![], + drive_version, + )?; + + for (key, element) in result_items.to_key_elements() { + match element { + Element::Item(bytes, flags) => { + self.strip_and_rewrite_contract_element( + &bytes, + flags, + &history_path_vec, + &key, + contract_id_bytes, + bincode_config, + transaction, + drive_version, + )?; + } + Element::Reference(..) => { + // The [0] reference to the latest revision — skip it. + } + _ => { + return Err(Error::Drive(DriveError::CorruptedDriveState( + format!( + "Unexpected element type in historical contract {} at key {}", + hex::encode(contract_id_bytes), + hex::encode(&key) + ), + ))); + } + } + } + } + _ => { + return Err(Error::Drive(DriveError::CorruptedDriveState(format!( + "No element or unexpected type at contract root for {}", + hex::encode(contract_id_bytes) + )))); + } + } + } + + // Clear the global data contract cache so that subsequent fetches + // reload the cleaned contracts from disk rather than serving stale + // cached versions with the unknown properties still present. + self.cache.data_contracts.clear(); + + Ok(()) + } + + /// Deserializes a contract element, strips unknown top-level properties + /// from its document schemas, and writes back if anything changed. + #[allow(clippy::too_many_arguments)] + fn strip_and_rewrite_contract_element( + &self, + stored_bytes: &[u8], + element_flags: Option, + storage_path_vec: &[Vec], + storage_key: &[u8], + contract_id_bytes: &[u8; 32], + bincode_config: impl bincode::config::Config, + transaction: &Transaction, + drive_version: &DriveVersion, + ) -> Result<(), Error> { + let mut serialization_format: DataContractInSerializationFormat = + match bincode::borrow_decode_from_slice(stored_bytes, bincode_config) { + Ok((format, _len)) => format, + Err(e) => { + return Err(Error::Drive(DriveError::CorruptedSerialization(format!( + "Failed to deserialize contract {} during migration: {}", + hex::encode(contract_id_bytes), + e + )))); + } + }; + + let mut contract_modified = false; + for (doc_type_name, schema_value) in serialization_format.document_schemas_mut().iter_mut() + { + if strip_unknown_properties_from_document_schema(schema_value) { + tracing::info!( + contract_id = hex::encode(contract_id_bytes), + document_type = %doc_type_name, + "Stripped unknown top-level properties from document schema" + ); + contract_modified = true; + } + } + + if !contract_modified { + return Ok(()); + } + + let new_bytes = + bincode::encode_to_vec(&serialization_format, bincode_config).map_err(|e| { + Error::Drive(DriveError::CorruptedSerialization(format!( + "Failed to re-serialize contract {}: {}", + hex::encode(contract_id_bytes), + e + ))) + })?; + + let new_element = Element::Item(new_bytes, element_flags); + + let path_slices: Vec<&[u8]> = storage_path_vec.iter().map(|v| v.as_slice()).collect(); + self.grove_insert( + SubtreePath::from(path_slices.as_slice()), + storage_key, + new_element, + Some(transaction), + None, + &mut vec![], + drive_version, + )?; + + tracing::info!( + contract_id = hex::encode(contract_id_bytes), + "Updated contract after stripping unknown document schema properties" + ); + + Ok(()) + } +} diff --git a/packages/rs-drive/src/drive/contract/mod.rs b/packages/rs-drive/src/drive/contract/mod.rs index 45c9da4c081..d84f33de0a5 100644 --- a/packages/rs-drive/src/drive/contract/mod.rs +++ b/packages/rs-drive/src/drive/contract/mod.rs @@ -13,6 +13,8 @@ mod estimation_costs; mod get_fetch; #[cfg(feature = "server")] mod insert; +#[cfg(feature = "server")] +mod migration; /// Various paths for contract operations #[cfg(any(feature = "server", feature = "verify"))] pub mod paths; diff --git a/packages/rs-platform-version/src/version/dpp_versions/dpp_contract_versions/mod.rs b/packages/rs-platform-version/src/version/dpp_versions/dpp_contract_versions/mod.rs index ea970e7d982..64a33b735c5 100644 --- a/packages/rs-platform-version/src/version/dpp_versions/dpp_contract_versions/mod.rs +++ b/packages/rs-platform-version/src/version/dpp_versions/dpp_contract_versions/mod.rs @@ -2,6 +2,7 @@ use versioned_feature_core::{FeatureVersion, FeatureVersionBounds}; pub mod v1; pub mod v2; pub mod v3; +pub mod v4; #[derive(Clone, Debug, Default)] pub struct DPPContractVersions { @@ -71,6 +72,7 @@ pub struct DocumentTypeMethodVersions { #[derive(Clone, Debug, Default)] pub struct DocumentTypeSchemaVersions { + pub document_type_schema: FeatureVersion, pub should_add_creator_id: FeatureVersion, pub enrich_with_base_schema: FeatureVersion, pub find_identifier_and_binary_paths: FeatureVersion, diff --git a/packages/rs-platform-version/src/version/dpp_versions/dpp_contract_versions/v1.rs b/packages/rs-platform-version/src/version/dpp_versions/dpp_contract_versions/v1.rs index 77619d972df..e4761c569c9 100644 --- a/packages/rs-platform-version/src/version/dpp_versions/dpp_contract_versions/v1.rs +++ b/packages/rs-platform-version/src/version/dpp_versions/dpp_contract_versions/v1.rs @@ -37,6 +37,7 @@ pub const CONTRACT_VERSIONS_V1: DPPContractVersions = DPPContractVersions { }, structure_version: 0, schema: DocumentTypeSchemaVersions { + document_type_schema: 0, should_add_creator_id: 0, enrich_with_base_schema: 0, find_identifier_and_binary_paths: 0, diff --git a/packages/rs-platform-version/src/version/dpp_versions/dpp_contract_versions/v2.rs b/packages/rs-platform-version/src/version/dpp_versions/dpp_contract_versions/v2.rs index 891323d771b..1928ac74b0c 100644 --- a/packages/rs-platform-version/src/version/dpp_versions/dpp_contract_versions/v2.rs +++ b/packages/rs-platform-version/src/version/dpp_versions/dpp_contract_versions/v2.rs @@ -37,6 +37,7 @@ pub const CONTRACT_VERSIONS_V2: DPPContractVersions = DPPContractVersions { }, structure_version: 0, schema: DocumentTypeSchemaVersions { + document_type_schema: 0, should_add_creator_id: 0, enrich_with_base_schema: 0, find_identifier_and_binary_paths: 0, diff --git a/packages/rs-platform-version/src/version/dpp_versions/dpp_contract_versions/v3.rs b/packages/rs-platform-version/src/version/dpp_versions/dpp_contract_versions/v3.rs index af22004dc52..9e50775c78a 100644 --- a/packages/rs-platform-version/src/version/dpp_versions/dpp_contract_versions/v3.rs +++ b/packages/rs-platform-version/src/version/dpp_versions/dpp_contract_versions/v3.rs @@ -39,6 +39,7 @@ pub const CONTRACT_VERSIONS_V3: DPPContractVersions = DPPContractVersions { }, structure_version: 0, schema: DocumentTypeSchemaVersions { + document_type_schema: 0, should_add_creator_id: 1, //changed enrich_with_base_schema: 0, find_identifier_and_binary_paths: 0, diff --git a/packages/rs-platform-version/src/version/dpp_versions/dpp_contract_versions/v4.rs b/packages/rs-platform-version/src/version/dpp_versions/dpp_contract_versions/v4.rs new file mode 100644 index 00000000000..a36bbeccc26 --- /dev/null +++ b/packages/rs-platform-version/src/version/dpp_versions/dpp_contract_versions/v4.rs @@ -0,0 +1,68 @@ +use crate::version::dpp_versions::dpp_contract_versions::{ + DPPContractVersions, DataContractMethodVersions, DocumentTypeClassMethodVersions, + DocumentTypeIndexVersions, DocumentTypeMethodVersions, DocumentTypeSchemaVersions, + DocumentTypeVersions, RecursiveSchemaValidatorVersions, TokenVersions, +}; +use versioned_feature_core::FeatureVersionBounds; + +// Introduced in protocol version 12, document_type_schema is changed to v1 +// which adds additionalProperties: false at the top level of document meta-schema +pub const CONTRACT_VERSIONS_V4: DPPContractVersions = DPPContractVersions { + max_serialized_size: 65000, + contract_serialization_version: FeatureVersionBounds { + min_version: 0, + max_version: 1, + default_current_version: 1, + }, + contract_structure_version: 1, + created_data_contract_structure: 0, + config: FeatureVersionBounds { + min_version: 1, + max_version: 1, + default_current_version: 1, + }, + methods: DataContractMethodVersions { + validate_document: 0, + validate_update: 0, + schema: 0, + validate_groups: 0, + equal_ignoring_time_fields: 0, + registration_cost: 1, + }, + document_type_versions: DocumentTypeVersions { + index_versions: DocumentTypeIndexVersions { + index_levels_from_indices: 0, + }, + class_method_versions: DocumentTypeClassMethodVersions { + try_from_schema: 1, + create_document_types_from_document_schemas: 1, + }, + structure_version: 0, + schema: DocumentTypeSchemaVersions { + document_type_schema: 1, // changed: use v1 document meta-schema with additionalProperties: false + should_add_creator_id: 1, + enrich_with_base_schema: 1, // changed: inject v1 schema URI + find_identifier_and_binary_paths: 0, + validate_max_depth: 0, + max_depth: 256, + recursive_schema_validator_versions: RecursiveSchemaValidatorVersions { + traversal_validator: 0, + }, + validate_schema_compatibility: 0, + }, + methods: DocumentTypeMethodVersions { + create_document_from_data: 0, + create_document_with_prevalidated_properties: 0, + prefunded_voting_balance_for_document: 0, + contested_vote_poll_for_document: 0, + estimated_size: 0, + index_for_types: 0, + max_size: 0, + serialize_value_for_key: 0, + deserialize_value_for_key: 0, + }, + }, + token_versions: TokenVersions { + validate_structure_interval: 0, + }, +}; diff --git a/packages/rs-platform-version/src/version/v12.rs b/packages/rs-platform-version/src/version/v12.rs index fbe019d3afd..901dae3905c 100644 --- a/packages/rs-platform-version/src/version/v12.rs +++ b/packages/rs-platform-version/src/version/v12.rs @@ -1,6 +1,6 @@ use crate::version::consensus_versions::ConsensusVersions; use crate::version::dpp_versions::dpp_asset_lock_versions::v1::DPP_ASSET_LOCK_VERSIONS_V1; -use crate::version::dpp_versions::dpp_contract_versions::v3::CONTRACT_VERSIONS_V3; +use crate::version::dpp_versions::dpp_contract_versions::v4::CONTRACT_VERSIONS_V4; use crate::version::dpp_versions::dpp_costs_versions::v1::DPP_COSTS_VERSIONS_V1; use crate::version::dpp_versions::dpp_document_versions::v3::DOCUMENT_VERSIONS_V3; use crate::version::dpp_versions::dpp_factory_versions::v1::DPP_FACTORY_VERSIONS_V1; @@ -51,7 +51,7 @@ pub const PLATFORM_V12: PlatformVersion = PlatformVersion { state_transition_conversion_versions: STATE_TRANSITION_CONVERSION_VERSIONS_V2, state_transition_method_versions: STATE_TRANSITION_METHOD_VERSIONS_V1, state_transitions: STATE_TRANSITION_VERSIONS_V3, - contract_versions: CONTRACT_VERSIONS_V3, + contract_versions: CONTRACT_VERSIONS_V4, // changed: use v1 document meta-schema with additionalProperties: false document_versions: DOCUMENT_VERSIONS_V3, identity_versions: IDENTITY_VERSIONS_V1, voting_versions: VOTING_VERSION_V2, diff --git a/packages/rs-scripts/Cargo.toml b/packages/rs-scripts/Cargo.toml index 18bb06e33b9..a6a3abcc4ba 100644 --- a/packages/rs-scripts/Cargo.toml +++ b/packages/rs-scripts/Cargo.toml @@ -7,11 +7,21 @@ edition = "2021" name = "decode-document" path = "src/bin/decode_document.rs" +[[bin]] +name = "check-contract-properties" +path = "src/bin/check_contract_properties.rs" + [dependencies] dpp = { path = "../rs-dpp", features = ["system_contracts"] } +dapi-grpc = { path = "../dapi-grpc", features = ["client"] } data-contracts = { path = "../data-contracts" } platform-version = { path = "../rs-platform-version" } base64 = "0.22" chrono = "0.4" hex = "0.4" clap = { version = "4", features = ["derive"] } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +tokio = { version = "1.40", features = ["macros", "rt-multi-thread"] } +tonic = { version = "0.14.2", features = ["channel", "transport", "tls-native-roots", "tls-webpki-roots", "tls-ring"], default-features = false } +ureq = "3.3" diff --git a/packages/rs-scripts/src/bin/check_contract_properties.rs b/packages/rs-scripts/src/bin/check_contract_properties.rs new file mode 100644 index 00000000000..cb588f57e35 --- /dev/null +++ b/packages/rs-scripts/src/bin/check_contract_properties.rs @@ -0,0 +1,375 @@ +use clap::Parser; +use dapi_grpc::platform::v0 as platform_proto; +use dapi_grpc::platform::v0::platform_client::PlatformClient; +use dpp::data_contract::document_type::schema::allowed_top_level_properties::ALLOWED_DOCUMENT_SCHEMA_V1_PROPERTIES; +use dpp::data_contract::serialized_version::DataContractInSerializationFormat; +use dpp::platform_value::Identifier; +use serde::Deserialize; +use std::time::Duration; +use tonic::transport::Channel; + +const EXPLORER_MAINNET: &str = "https://platform-explorer.pshenmic.dev"; +const EXPLORER_TESTNET: &str = "https://testnet.platform-explorer.pshenmic.dev"; + +const MAX_VALIDATORS_TO_TRY: usize = 5; + +#[derive(Parser)] +#[command( + name = "check-contract-properties", + about = "Fetch all contracts from mainnet/testnet and check for unknown top-level document schema properties" +)] +struct Args { + /// Network: "mainnet", "testnet" + #[arg(short, long, default_value = "mainnet")] + network: String, + + /// Override the DAPI gRPC URI (e.g. "https://1.2.3.4:1443") + #[arg(long)] + dapi_uri: Option, + + /// Override the Platform Explorer API URI (e.g. "https://platform-explorer.pshenmic.dev") + #[arg(long)] + explorer_uri: Option, + + /// Contract IDs to check (base58 or hex). If none given, fetches all from the explorer. + #[arg(trailing_var_arg = true)] + contract_ids: Vec, +} + +#[derive(Deserialize)] +struct ExplorerResponse { + #[serde(rename = "resultSet")] + result_set: Vec, + pagination: ExplorerPagination, +} + +#[derive(Deserialize)] +struct ExplorerContract { + identifier: String, + name: Option, +} + +#[derive(Deserialize)] +struct ExplorerPagination { + #[allow(dead_code)] + page: u32, + total: u32, +} + +/// Fetches DAPI gRPC URIs by querying active validators from the explorer API. +fn fetch_dapi_uris_from_explorer(explorer_base: &str) -> Result, String> { + let url = format!( + "{}/validators?page=1&limit={}&isActive=true", + explorer_base, MAX_VALIDATORS_TO_TRY + ); + + let body: String = ureq::get(&url) + .call() + .map_err(|e| format!("Explorer API request failed: {e}"))? + .body_mut() + .read_to_string() + .map_err(|e| format!("Failed to read explorer response: {e}"))?; + + let resp: serde_json::Value = + serde_json::from_str(&body).map_err(|e| format!("Failed to parse JSON: {e}"))?; + + let mut uris = Vec::new(); + if let Some(result_set) = resp.get("resultSet").and_then(|v| v.as_array()) { + for validator in result_set { + if let Some(state) = validator.get("proTxInfo").and_then(|p| p.get("state")) { + let service = state.get("service").and_then(|v| v.as_str()); + let http_port = state.get("platformHTTPPort").and_then(|v| v.as_u64()); + + if let (Some(service), Some(port)) = (service, http_port) { + // service is "ip:core_port", we only need the IP + let ip = service + .rsplit_once(':') + .map(|(ip, _)| ip) + .unwrap_or(service); + uris.push(format!("https://{ip}:{port}")); + } + } + } + } + + if uris.is_empty() { + return Err("No active validators found with platform HTTP ports".to_string()); + } + Ok(uris) +} + +/// Fetches all contract identifiers from the Platform Explorer API. +fn fetch_all_contract_ids_from_explorer( + explorer_base: &str, +) -> Result, String> { + let mut all = Vec::new(); + let mut page = 1u32; + let limit = 100u32; + + loop { + let url = format!( + "{}/dataContracts?page={}&limit={}&order=asc&order_by=block_height", + explorer_base, page, limit + ); + + let body: String = ureq::get(&url) + .call() + .map_err(|e| format!("Explorer API request failed: {e}"))? + .body_mut() + .read_to_string() + .map_err(|e| format!("Failed to read explorer response: {e}"))?; + + let resp: ExplorerResponse = serde_json::from_str(&body) + .map_err(|e| format!("Failed to parse explorer JSON: {e}"))?; + + for contract in &resp.result_set { + let label = contract + .name + .clone() + .unwrap_or_else(|| contract.identifier.clone()); + let id = Identifier::from_string( + &contract.identifier, + dpp::platform_value::string_encoding::Encoding::Base58, + ) + .map_err(|e| format!("Failed to parse contract ID '{}': {e}", contract.identifier))?; + all.push((label, id)); + } + + let fetched_so_far = page * limit; + if fetched_so_far >= resp.pagination.total { + break; + } + page += 1; + } + + Ok(all) +} + +fn parse_contract_id(raw: &str) -> Identifier { + Identifier::from_string(raw, dpp::platform_value::string_encoding::Encoding::Base58) + .or_else(|_| { + let bytes = hex::decode(raw).unwrap_or_else(|e| { + eprintln!("Cannot parse '{raw}' as base58 or hex: {e}"); + std::process::exit(1); + }); + if bytes.len() != 32 { + eprintln!("ID '{raw}' decoded to {} bytes, expected 32", bytes.len()); + std::process::exit(1); + } + let arr: [u8; 32] = bytes.try_into().unwrap(); + Ok::<_, dpp::ProtocolError>(Identifier::new(arr)) + }) + .unwrap() +} + +#[tokio::main] +async fn main() { + let args = Args::parse(); + + let explorer_base = args.explorer_uri.clone().unwrap_or_else(|| { + match args.network.as_str() { + "testnet" => EXPLORER_TESTNET, + _ => EXPLORER_MAINNET, + } + .to_string() + }); + + let dapi_seeds: Vec = if let Some(uri) = args.dapi_uri.clone() { + vec![uri] + } else { + println!("Discovering DAPI nodes from explorer ({explorer_base}) ..."); + match fetch_dapi_uris_from_explorer(&explorer_base) { + Ok(uris) => { + println!("Found {} DAPI node(s).", uris.len()); + uris + } + Err(e) => { + eprintln!("Error discovering DAPI nodes: {e}"); + std::process::exit(1); + } + } + }; + + // Build list of (label, id) pairs to check + let contracts_to_check: Vec<(String, Identifier)> = if args.contract_ids.is_empty() { + println!("Fetching all contract IDs from explorer ({explorer_base}) ..."); + match fetch_all_contract_ids_from_explorer(&explorer_base) { + Ok(ids) => { + println!("Found {} contracts.\n", ids.len()); + ids + } + Err(e) => { + eprintln!("Error: {e}"); + std::process::exit(1); + } + } + } else { + args.contract_ids + .iter() + .map(|raw| (raw.clone(), parse_contract_id(raw))) + .collect() + }; + + // Try each seed until one connects + let mut client = None; + for seed_uri in &dapi_seeds { + println!("Trying DAPI seed {seed_uri} ..."); + + let mut endpoint = Channel::from_shared(seed_uri.clone()) + .expect("invalid URI") + .connect_timeout(Duration::from_secs(5)) + .timeout(Duration::from_secs(30)); + + if seed_uri.starts_with("https://") { + endpoint = endpoint + .tls_config(tonic::transport::ClientTlsConfig::new().with_enabled_roots()) + .expect("TLS config failed"); + } + + match endpoint.connect().await { + Ok(channel) => { + println!("Connected to {seed_uri}"); + client = Some(PlatformClient::new(channel)); + break; + } + Err(e) => { + eprintln!(" Failed: {e}"); + } + } + } + + let mut client = client.unwrap_or_else(|| { + eprintln!("Could not connect to any DAPI seed node"); + std::process::exit(1); + }); + + println!( + "Checking {} contract(s) for unknown document schema properties...\n", + contracts_to_check.len() + ); + + let bincode_config = dpp::bincode::config::standard() + .with_big_endian() + .with_no_limit(); + + let mut total_issues = 0usize; + let mut checked = 0usize; + let mut errors = 0usize; + + for (label, id) in &contracts_to_check { + let idx = checked + errors + 1; + let request = platform_proto::GetDataContractRequest { + version: Some(platform_proto::get_data_contract_request::Version::V0( + platform_proto::get_data_contract_request::GetDataContractRequestV0 { + id: id.to_vec(), + prove: false, + }, + )), + }; + + let response = match client.get_data_contract(request).await { + Ok(resp) => resp.into_inner(), + Err(e) => { + eprintln!( + " [{idx}/{total}] [{label}] ({id}) - ERROR fetching: {e}", + total = contracts_to_check.len() + ); + errors += 1; + continue; + } + }; + + let contract_bytes = match response.version { + Some(platform_proto::get_data_contract_response::Version::V0(v0)) => match v0.result { + Some( + platform_proto::get_data_contract_response::get_data_contract_response_v0::Result::DataContract( + bytes, + ), + ) => bytes, + Some( + platform_proto::get_data_contract_response::get_data_contract_response_v0::Result::Proof( + _, + ), + ) => { + eprintln!(" [{label}] ({id}) - got proof instead of data"); + errors += 1; + continue; + } + None => { + eprintln!(" [{label}] ({id}) - empty response"); + errors += 1; + continue; + } + }, + None => { + eprintln!(" [{label}] ({id}) - no version in response"); + errors += 1; + continue; + } + }; + + let serialization_format: DataContractInSerializationFormat = + match dpp::bincode::borrow_decode_from_slice(contract_bytes.as_slice(), bincode_config) + { + Ok((format, _)) => format, + Err(e) => { + eprintln!(" [{label}] ({id}) - deserialization error: {e}"); + errors += 1; + continue; + } + }; + + checked += 1; + let contract_id = serialization_format.id(); + let mut contract_has_issues = false; + + for (doc_type_name, schema_value) in serialization_format.document_schemas() { + let map = match schema_value { + dpp::platform_value::Value::Map(map) => map, + _ => continue, + }; + + let unknown_keys: Vec<&str> = map + .iter() + .filter_map(|(key, _)| { + let key_str = match key { + dpp::platform_value::Value::Text(s) => s.as_str(), + _ => return None, + }; + if ALLOWED_DOCUMENT_SCHEMA_V1_PROPERTIES.contains(&key_str) { + None + } else { + Some(key_str) + } + }) + .collect(); + + if !unknown_keys.is_empty() { + if !contract_has_issues { + println!(" [{label}] ({contract_id}) - UNKNOWN PROPERTIES FOUND:"); + contract_has_issues = true; + } + println!(" document type \"{doc_type_name}\": {:?}", unknown_keys); + total_issues += unknown_keys.len(); + } + } + + if !contract_has_issues { + println!( + " [{idx}/{total}] [{label}] ({contract_id}) - OK", + total = contracts_to_check.len() + ); + } + } + + println!(); + if errors > 0 { + println!("{errors} contract(s) could not be fetched."); + } + if total_issues > 0 { + println!("Found {total_issues} unknown property occurrence(s) across {checked} checked contracts."); + std::process::exit(1); + } else { + println!("All {checked} checked contracts are clean."); + } +}