Merge pull request #1848 from radixdlt/feature/schema-comparisons-par…

…t-2a

feature: Add `BasicSborAssertion` and `ScryptoSborAssertion` macros

radix-common/src/data/manifest/definitions.rs

@@ -1,3 +1,5 @@
+use vec_traits::vec_decode_with_nice_error;
+
 use crate::internal_prelude::*;
 
 pub use crate::constants::MANIFEST_SBOR_V1_MAX_DEPTH;
@@ -103,44 +105,40 @@ pub fn manifest_decode<T: ManifestDecode>(buf: &[u8]) -> Result<T, DecodeError>
     manifest_decode_with_depth_limit(buf, MANIFEST_SBOR_V1_MAX_DEPTH)
 }
 
+pub fn manifest_decode_with_depth_limit<T: ManifestDecode>(
+    buf: &[u8],
+    depth_limit: usize,
+) -> Result<T, DecodeError> {
+    ManifestDecoder::new(buf, depth_limit).decode_payload(MANIFEST_SBOR_V1_PAYLOAD_PREFIX)
+}
+
 /// Decodes a data structure from a byte array.
 ///
 /// If an error occurs, the type's schema is exported and used to give a better error message.
 ///
 /// NOTE:
 /// * The error path runs very slowly. This should only be used where errors are NOT expected.
+/// * This should not be used where the size of compiled code is an issue, as it will pull
+///   in the schema aggregation code which is large.
 pub fn manifest_decode_with_nice_error<T: ManifestDecode + ScryptoDescribe>(
     buf: &[u8],
 ) -> Result<T, String> {
-    match manifest_decode(buf) {
-        Ok(value) => Ok(value),
-        Err(err) => {
-            let (local_type_id, schema) =
-                generate_full_schema_from_single_type::<T, ScryptoCustomSchema>();
-            let schema = schema.as_unique_version();
-            match validate_payload_against_schema::<ManifestCustomExtension, _>(
-                buf,
-                schema,
-                local_type_id,
-                &(),
-                SCRYPTO_SBOR_V1_MAX_DEPTH,
-            ) {
-                Ok(()) => {
-                    // This case is unexpected. We got a decode error, but it's valid against the schema.
-                    // In this case, let's just debug-print the DecodeError.
-                    Err(format!("{err:?}"))
-                }
-                Err(err) => Err(err.error_message(schema)),
-            }
-        }
-    }
+    vec_decode_with_nice_error::<ManifestCustomExtension, T>(buf, MANIFEST_SBOR_V1_MAX_DEPTH)
 }
 
-pub fn manifest_decode_with_depth_limit<T: ManifestDecode>(
+/// Decodes a data structure from a byte array.
+///
+/// If an error occurs, the type's schema is exported and used to give a better error message.
+///
+/// NOTE:
+/// * The error path runs very slowly. This should only be used where errors are NOT expected.
+/// * This should not be used where the size of compiled code is an issue, as it will pull
+///   in the schema aggregation code which is large.
+pub fn manifest_decode_with_depth_limit_and_nice_error<T: ManifestDecode + ScryptoDescribe>(
     buf: &[u8],
     depth_limit: usize,
-) -> Result<T, DecodeError> {
-    ManifestDecoder::new(buf, depth_limit).decode_payload(MANIFEST_SBOR_V1_PAYLOAD_PREFIX)
+) -> Result<T, String> {
+    vec_decode_with_nice_error::<ManifestCustomExtension, T>(buf, depth_limit)
 }
 
 pub fn to_manifest_value<T: ManifestEncode + ?Sized>(

radix-common/src/data/scrypto/definitions.rs

@@ -1,3 +1,5 @@
+use vec_traits::vec_decode_with_nice_error;
+
 use crate::internal_prelude::*;
 
 pub use crate::constants::SCRYPTO_SBOR_V1_MAX_DEPTH;
@@ -63,6 +65,13 @@ pub fn scrypto_decode<T: ScryptoDecode>(buf: &[u8]) -> Result<T, DecodeError> {
     scrypto_decode_with_depth_limit(buf, SCRYPTO_SBOR_V1_MAX_DEPTH)
 }
 
+pub fn scrypto_decode_with_depth_limit<T: ScryptoDecode>(
+    buf: &[u8],
+    depth_limit: usize,
+) -> Result<T, DecodeError> {
+    ScryptoDecoder::new(buf, depth_limit).decode_payload(SCRYPTO_SBOR_V1_PAYLOAD_PREFIX)
+}
+
 /// Decodes a data structure from a byte array.
 ///
 /// If an error occurs, the type's schema is exported and used to give a better error message.
@@ -73,33 +82,19 @@ pub fn scrypto_decode<T: ScryptoDecode>(buf: &[u8]) -> Result<T, DecodeError> {
 pub fn scrypto_decode_with_nice_error<T: ScryptoDecode + ScryptoDescribe>(
     buf: &[u8],
 ) -> Result<T, String> {
-    match scrypto_decode(buf) {
-        Ok(value) => Ok(value),
-        Err(err) => {
-            let (local_type_id, schema) =
-                generate_full_schema_from_single_type::<T, ScryptoCustomSchema>();
-            let schema = schema.as_unique_version();
-            match validate_payload_against_schema::<ScryptoCustomExtension, _>(
-                buf,
-                schema,
-                local_type_id,
-                &(),
-                SCRYPTO_SBOR_V1_MAX_DEPTH,
-            ) {
-                Ok(()) => {
-                    // This case is unexpected. We got a decode error, but it's valid against the schema.
-                    // In this case, let's just debug-print the DecodeError.
-                    Err(format!("{err:?}"))
-                }
-                Err(err) => Err(err.error_message(schema)),
-            }
-        }
-    }
+    vec_decode_with_nice_error::<ScryptoCustomExtension, T>(buf, SCRYPTO_SBOR_V1_MAX_DEPTH)
 }
 
-pub fn scrypto_decode_with_depth_limit<T: ScryptoDecode>(
+/// Decodes a data structure from a byte array.
+///
+/// If an error occurs, the type's schema is exported and used to give a better error message.
+///
+/// NOTE:
+/// * The error path runs very slowly. This should only be used where errors are NOT expected.
+/// * This should not be used in Scrypto, as it will pull in the schema aggregation code which is large.
+pub fn scrypto_decode_with_depth_limit_and_nice_error<T: ScryptoDecode + ScryptoDescribe>(
     buf: &[u8],
     depth_limit: usize,
-) -> Result<T, DecodeError> {
-    ScryptoDecoder::new(buf, depth_limit).decode_payload(SCRYPTO_SBOR_V1_PAYLOAD_PREFIX)
+) -> Result<T, String> {
+    vec_decode_with_nice_error::<ScryptoCustomExtension, T>(buf, depth_limit)
 }

radix-common/src/lib.rs

@@ -35,7 +35,7 @@ pub use sbor::{Categorize, Decode, Encode, Sbor};
 extern crate radix_sbor_derive;
 pub use radix_sbor_derive::{
     ManifestCategorize, ManifestDecode, ManifestEncode, ManifestSbor, ScryptoCategorize,
-    ScryptoDecode, ScryptoEncode, ScryptoEvent, ScryptoSbor,
+    ScryptoDecode, ScryptoEncode, ScryptoEvent, ScryptoSbor, ScryptoSborAssertion,
 };
 
 // extern crate self as X; in lib.rs allows ::X and X to resolve to this crate inside this crate.
@@ -60,7 +60,7 @@ pub mod prelude {
     // Exports from upstream libraries
     pub use radix_sbor_derive::{
         ManifestCategorize, ManifestDecode, ManifestEncode, ManifestSbor, ScryptoCategorize,
-        ScryptoDecode, ScryptoEncode, ScryptoEvent, ScryptoSbor,
+        ScryptoDecode, ScryptoEncode, ScryptoEvent, ScryptoSbor, ScryptoSborAssertion,
     };
     pub use sbor::prelude::*;
     pub use sbor::*;

radix-engine/src/errors.rs

@@ -279,7 +279,8 @@ pub enum SystemError {
 
     /// A panic that's occurred in the system-layer or below. We're calling it system panic since
     /// we're treating the system as a black-box here.
-    #[cfg(feature = "std")]
+    ///
+    /// Note that this is only used when feature std is used.
     SystemPanic(String),
 }
 

radix-engine/src/transaction/system_structure.rs

@@ -29,13 +29,6 @@ pub struct SystemFieldStructure {
     pub field_kind: SystemFieldKind,
 }
 
-// NOTE: Adding BootLoader and BootLoader(BootLoaderFieldKind) are both changes to the
-//       TransactionReceiptV1 encoding, BUT we only require the encoding to be consistent
-//       for the toolkit which parses it opaquely at the moment when doing transaction preview
-//       in the wallet. This change doesn't apply because BootLoader substates are only edited
-//       during transaction preview.
-//
-// Node depends on this structure, so append only.
 #[derive(Debug, Clone, ScryptoSbor, PartialEq, Eq)]
 pub enum SystemFieldKind {
     TypeInfo,
@@ -98,7 +91,12 @@ pub struct EventSystemStructure {
 pub type SubstateSystemStructures =
     IndexMap<NodeId, IndexMap<PartitionNumber, IndexMap<SubstateKey, SubstateSystemStructure>>>;
 
-#[derive(Default, Debug, Clone, ScryptoSbor, PartialEq, Eq)]
+#[derive(Default, Debug, Clone, ScryptoSbor, PartialEq, Eq, ScryptoSborAssertion)]
+// This is currently persisted in the node's `LocalTransactionExecution` store,
+// so needs to be backwards compatible to avoid breaking the Core API transaction stream.
+// This assertion can be removed when `radixdlt-scrypto` is merged into the node,
+// because there will be an equivalent assertion against the `LocalTransactionExecution` itself.
+#[sbor_assert(backwards_compatible(bottlenose = "FILE:system_structure_v1.txt"))]
 pub struct SystemStructure {
     pub substate_system_structures: SubstateSystemStructures,
     pub event_system_structures: IndexMap<EventTypeIdentifier, EventSystemStructure>,

radix-engine/src/transaction/system_structure_v1.txt

@@ -0,0 +1 @@
+5c2102220001210320221e0e0120220201010a010000000000000001010a190000000000000010022201010a02000000000000002201010a03000000000000000d01220001070710022201010a04000000000000002201010a0500000000000000070010022201010a06000000000000002201010a09000000000000000f0123072003002201000107070122010001074102220101010a07000000000000000e0120220201010a0800000000000000000107410d0122000107070f012307200700220101010a0a0000000000000001220002220101010a0c0000000000000003220101010a110000000000000004220101010a160000000000000005220101010a170000000000000006220101010a18000000000000000e0120220101010a0b000000000000000f01230720040022000122000222000322000e0120220201010a0d0000000000000001010a0d000000000000000e0120220301010a020000000000000001010a0e0000000000000001010a0f000000000000000d0122000107070f012307200200220101010a10000000000000000122010001070a07000e0120220101010a12000000000000000f012307200200220101010a130000000000000001220101010a15000000000000000e0120220101010a14000000000000000e012022030001078301010a0e0000000000000001010a0f000000000000000e012022020001070701010a0d000000000000000e0120220201010a120000000000000001010a12000000000000000e0120220201010a120000000000000001010a12000000000000000e0120220201010a120000000000000001010a120000000000000010022201010a1a000000000000002201010a1d000000000000000e0120220201010a1b000000000000000001070c0f012307200200220101010a1c0000000000000001220201010a0200000000000000000107f00e01202202000107830001070c0e0120220101010a130000000000000020211e022201010c0f53797374656d537472756374757265220101220001200c021a73756273746174655f73797374656d5f73747275637475726573176576656e745f73797374656d5f7374727563747572657302220000220000022201010c064e6f6465496422000002220000220000022201010c0f506172746974696f6e4e756d62657222000002220000220000022201010c0b53756273746174654b65792201012201012307210300022201010c054669656c6422000001022201010c034d617022000002022201010c06536f727465642200000222000022000002220000220000022201010c17537562737461746553797374656d5374727563747572652201012201012307210700022201010c0b53797374656d4669656c6422000001022201010c0c53797374656d536368656d6122000002022201010c124b657956616c756553746f7265456e74727922000003022201010c0b4f626a6563744669656c6422000004022201010c1c4f626a6563744b657956616c7565506172746974696f6e456e74727922000005022201010c194f626a656374496e646578506172746974696f6e456e74727922000006022201010c1f4f626a656374536f72746564496e646578506172746974696f6e456e747279220000022201010c1453797374656d4669656c64537472756374757265220101220001200c010a6669656c645f6b696e64022201010c0f53797374656d4669656c644b696e642201012201012307210400022201010c0854797065496e666f22000001022201010c06566d426f6f7422000002022201010c0a53797374656d426f6f7422000003022201010c0a4b65726e656c426f6f74220000022201010c1b4b657956616c756553746f7265456e747279537472756374757265220101220001200c02106b65795f66756c6c5f747970655f69641276616c75655f66756c6c5f747970655f6964022201010c1146756c6c7953636f706564547970654964220000022201010c0a536368656d6148617368220000022201010c0b4c6f63616c5479706549642201012201012307210200022201010c0957656c6c4b6e6f776e22000001022201010c10536368656d614c6f63616c496e646578220000022201010c0f57656c6c4b6e6f776e547970654964220000022201010c0e4669656c64537472756374757265220101220001200c010c76616c75655f736368656d61022201010c1b4f626a6563745375627374617465547970655265666572656e63652201012201012307210200022201010c075061636b61676522000001022201010c0e4f626a656374496e7374616e6365220000022201010c145061636b616765547970655265666572656e6365220101220001200c010c66756c6c5f747970655f6964022201010c1146756c6c7953636f706564547970654964220000022201010c1b4f626a656374496e7374616e6365547970655265666572656e6365220101220001200c0210696e7374616e63655f747970655f6964157265736f6c7665645f66756c6c5f747970655f6964022201010c1f4b657956616c7565506172746974696f6e456e747279537472756374757265220101220001200c020a6b65795f736368656d610c76616c75655f736368656d61022201010c1c496e646578506172746974696f6e456e747279537472756374757265220101220001200c020a6b65795f736368656d610c76616c75655f736368656d61022201010c22536f72746564496e646578506172746974696f6e456e747279537472756374757265220101220001200c020a6b65795f736368656d610c76616c75655f736368656d6102220000220000022201010c134576656e74547970654964656e746966696572220000022201010c07456d69747465722201012201012307210200022201010c0846756e6374696f6e22000001022201010c064d6574686f64220000022201010c0b426c75657072696e744964220101220001200c020f7061636b6167655f616464726573730e626c75657072696e745f6e616d65022201010c144576656e7453797374656d537472756374757265220101220001200c01167061636b6167655f747970655f7265666572656e636520221e000000000c012102220101091e000000220101091e000000000000000000000000000c01210222010109020000002201010902000000000000000000000000000c012102220101092000000022010109200000000000000000000000000000000000000000000000000000000000000000002201010a0000000000000000

radix-engine/src/transaction/transaction_receipt.rs

@@ -26,7 +26,20 @@ define_single_versioned! {
     /// receipt versions, allowing us to release a wallet ahead-of-time which is forward
     /// compatible with a new version of the engine (and so a new transaction receipt).
     #[derive(Clone, ScryptoSbor)]
-    pub VersionedTransactionReceipt(TransactionReceiptVersions) => TransactionReceipt = TransactionReceiptV1
+    pub VersionedTransactionReceipt(TransactionReceiptVersions) => TransactionReceipt = TransactionReceiptV1,
+    outer_attributes: [
+        // VersionedTransactionReceipt is currently encoded in the node's preview API.
+        // It is then decoded in lots of different mobile wallets as part of Transaction Review.
+        // We therefore can't make any changes/additions, without breaking this.
+        //
+        // In the interim, we are planning to:
+        // * Temporarily, serialize a PreviewTransactionReceipt which is fixed as just a single v1 versioned
+        //   VersionedTransactionReceipt, and have `impl From<VersionedTransactionReceipt> for PreviewTransactionReceipt`.
+        //   This will allow us to add new receipt versions, but ensuring they can still map to the preview model.
+        // * Change the API to return some kind of explicit extensible preview DTO.
+        #[derive(ScryptoSborAssertion)]
+        #[sbor_assert(fixed("FILE:receipt_schema_bottlenose.txt"))]
+    ],
 }
 
 #[derive(Clone, ScryptoSbor, PartialEq, Eq)]

radix-sbor-derive/src/lib.rs

@@ -169,6 +169,109 @@ pub fn scrypto_sbor(input: TokenStream) -> TokenStream {
         .into()
 }
 
+/// A macro for outputting tests and marker traits to assert that a type has maintained its shape over time.
+///
+/// There are two types of assertion modes:
+/// * "fixed" mode is used to ensure that a type is unchanged.
+/// * "backwards_compatible" mode is used when multiple versions of the type are permissible, but
+///   newer versions of the type must be compatible with the older version where they align.
+///   This mode (A) ensures that the type's current schema is equivalent to the latest version, and
+///   (B) ensures that each of the schemas is a strict extension of the previous mode.
+///
+/// There is also a "generate" mode which can be used to export the current schema. Upon running the generated test,
+/// the schema is either written to a file, or output in a panic message.
+///
+/// ## Initial schema generation
+///
+/// To output the generated schema to a file path relative to the source file, add an attribute like this -
+/// and then run the test which gets generated. If using Rust Analyzer this can be run from the IDE,
+/// or it can be run via `cargo test`.
+///
+/// ```no_run
+/// #[derive(ScryptoSbor, ScryptoSborAssertion)]
+/// #[sbor_assert(generate("FILE:MyType-schema-v1.txt"))]
+/// struct MyType {
+///     // ...
+/// }
+/// ```
+///
+/// The test should generate the given file and then panic. If you wish to only generate the schema
+/// in the panic message, you can with `#[sbor_assert(generate("INLINE"))]`.
+///
+/// ## Fixed schema verification
+///
+/// To verify the type's schema is unchanged, do:
+/// ```no_run
+/// #[derive(ScryptoSbor, ScryptoSborAssertion)]
+/// #[sbor_assert(fixed("FILE:MyType-schema-v1.txt"))]
+/// struct MyType {
+///     // ...
+/// }
+/// ```
+///
+/// Other supported options are `fixed("INLINE:<hex>")` and `fixed("CONST:<Constant>")`.
+///
+/// ## Backwards compatibility verification
+///
+/// To allow multiple backwards-compatible versions, you can do this:
+/// ```no_run
+/// #[derive(ScryptoSbor, ScryptoSborAssertion)]
+/// #[sbor_assert(backwards_compatible(
+///     version1 = "FILE:MyType-schema-v1.txt",
+///     version2 = "FILE:MyType-schema-v2.txt",
+/// ))]
+/// struct MyType {
+///     // ...
+/// }
+/// ```
+///
+/// Instead of `"FILE:X"`, you can also use `"INLINE:<hex>"` and `"CONST:<Constant>"`.
+///
+/// ## Custom settings
+/// By default, the `fixed` mode will use `SchemaComparisonSettings::require_equality()` and
+/// the `backwards_compatible` mode will use `SchemaComparisonSettings::allow_extension()`.
+///
+/// You may wish to change these:
+/// * If you just wish to ignore the equality of metadata such as names, you can use the
+///   `allow_name_changes` flag.
+/// * If you wish to override any settings, you can provide a constant containing your
+///   own SchemaComparisonSettings.
+///
+/// For example:
+/// ```no_run
+/// #[derive(ScryptoSbor, ScryptoSborAssertion)]
+/// #[sbor_assert(
+///     fixed("FILE:MyType-schema-v1.txt"),
+///     settings(allow_name_changes),
+/// )]
+/// struct MyType {
+///     // ...
+/// }
+///
+/// const CUSTOM_COMPARISON_SETTINGS: sbor::schema::SchemaComparisonSettings = sbor::schema::SchemaComparisonSettings::require_equality();
+///
+/// #[derive(ScryptoSbor, ScryptoSborAssertion)]
+/// #[sbor_assert(
+///     backwards_compatible(
+///         version1 = "FILE:MyType-schema-v1.txt",
+///         version2 = "FILE:MyType-schema-v2.txt",
+///     ),
+///     settings(CUSTOM_COMPARISON_SETTINGS),
+/// )]
+/// struct MyOtherType {
+///     // ...
+/// }
+/// ```
+#[proc_macro_derive(ScryptoSborAssertion, attributes(sbor_assert))]
+pub fn scrypto_sbor_assertion(input: TokenStream) -> TokenStream {
+    sbor_derive_common::sbor_assert::handle_sbor_assert_derive(
+        proc_macro2::TokenStream::from(input),
+        "radix_common::data::scrypto::ScryptoCustomSchema",
+    )
+    .unwrap_or_else(|err| err.to_compile_error())
+    .into()
+}
+
 /// Derive code for implementing the required logic to mark a type as being an event.
 ///
 /// # Example

sbor-derive-common/Cargo.toml

@@ -13,6 +13,7 @@ syn = { workspace = true, features = ["full", "extra-traits"] }
 quote = { workspace = true }
 const-sha1 = { workspace = true } # Chosen because of its small size and 0 transitive dependencies
 itertools = { workspace = true }
+indexmap = { workspace = true }
 
 [features]
 trace = []

sbor-derive-common/src/lib.rs

@@ -3,4 +3,5 @@ pub mod decode;
 pub mod describe;
 pub mod encode;
 pub mod sbor;
+pub mod sbor_assert;
 pub mod utils;