Introduce Blake3

frame/support/Cargo.toml

@@ -16,7 +16,7 @@ targets = ["x86_64-unknown-linux-gnu"]
 serde = { version = "1.0.136", optional = true, features = ["derive"] }
 codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive", "max-encoded-len"] }
 scale-info = { version = "2.1.1", default-features = false, features = ["derive"] }
-frame-metadata = { version = "15.0.0", default-features = false, features = ["v14"] }
+frame-metadata = { version = "15.1.0", default-features = false, features = ["v14"], git = "https://github.com/chiefbiiko/frame-metadata", branch = "blake3" }
 sp-api = { version = "4.0.0-dev", default-features = false, path = "../../primitives/api" }
 sp-std = { version = "5.0.0", default-features = false, path = "../../primitives/std" }
 sp-io = { version = "7.0.0", default-features = false, path = "../../primitives/io" }

frame/support/procedural/src/lib.rs

@@ -114,11 +114,15 @@ fn get_cargo_env_var<T: FromStr>(version_env: &str) -> std::result::Result<T, ()
 ///
 ///   `$hash` representing a choice of hashing algorithms available in the
 ///   [`Hashable`](../frame_support/trait.Hashable.html) trait. You will generally want to use one
-///   of three hashers:
+///   of four hashers:
 ///   * `blake2_128_concat`: The default, safe choice. Use if you are unsure or don't care. It is
 ///     secure against user-tainted keys, fairly fast and memory-efficient and supports iteration
-///     over its keys and values. This must be used if the keys of your map can be selected *en
-///     masse* by untrusted users.
+///     over its keys and values. A Blake* hash must be used if the keys of your map can be selected
+///     *en masse* by untrusted users.
+///   * `blake3_128_concat`: The future default, safe choice. Use if you care for performance. It is
+///     secure against user-tainted keys, super fast and memory-efficient and supports iteration
+///     over its keys and values. A Blake* hash must be used if the keys of your map can be selected
+///     *en masse* by untrusted users.
 ///   * `twox_64_concat`: This is an insecure hasher and can only be used safely if you know that
 ///     the preimages cannot be chosen at will by untrusted users. It is memory-efficient, extremely
 ///     performant and supports iteration over its keys and values. You can safely use this is the
@@ -174,6 +178,8 @@ fn get_cargo_env_var<T: FromStr>(version_env: &str) -> std::result::Result<T, ()
 ///   already. The most efficient and iterable over keys.
 /// * `twox_64_concat` - TwoX with 64bit + key concatenated. Use only when an untrusted source
 ///   cannot select and insert key values. Very efficient and iterable over keys.
+/// * `blake3_128_concat` - Blake3 with 128bit + key concatenated. Fast and safe to use in all
+///   circumstances. Iterable over keys.
 /// * `blake2_128_concat` - Blake2 with 128bit + key concatenated. Slower but safe to use in all
 ///   circumstances. Iterable over keys.
 ///

frame/support/procedural/src/storage/mod.rs

@@ -437,6 +437,9 @@ pub enum HasherKind {
 	Twox128,
 	Twox64Concat,
 	Identity,
+	Blake3_256,
+	Blake3_128,
+	Blake3_128Concat,
 }
 
 impl HasherKind {
@@ -445,6 +448,9 @@ impl HasherKind {
 			HasherKind::Blake2_256 => quote!(Blake2_256),
 			HasherKind::Blake2_128 => quote!(Blake2_128),
 			HasherKind::Blake2_128Concat => quote!(Blake2_128Concat),
+			HasherKind::Blake3_256 => quote!(Blake3_256),
+			HasherKind::Blake3_128 => quote!(Blake3_128),
+			HasherKind::Blake3_128Concat => quote!(Blake3_128Concat),
 			HasherKind::Twox256 => quote!(Twox256),
 			HasherKind::Twox128 => quote!(Twox128),
 			HasherKind::Twox64Concat => quote!(Twox64Concat),
@@ -457,6 +463,9 @@ impl HasherKind {
 			HasherKind::Blake2_256 => quote!(StorageHasher::Blake2_256),
 			HasherKind::Blake2_128 => quote!(StorageHasher::Blake2_128),
 			HasherKind::Blake2_128Concat => quote!(StorageHasher::Blake2_128Concat),
+			HasherKind::Blake3_256 => quote!(StorageHasher::Blake3_256),
+			HasherKind::Blake3_128 => quote!(StorageHasher::Blake3_128),
+			HasherKind::Blake3_128Concat => quote!(StorageHasher::Blake3_128Concat),
 			HasherKind::Twox256 => quote!(StorageHasher::Twox256),
 			HasherKind::Twox128 => quote!(StorageHasher::Twox128),
 			HasherKind::Twox64Concat => quote!(StorageHasher::Twox64Concat),

frame/support/procedural/src/storage/parse.rs

@@ -35,6 +35,9 @@ mod keyword {
 	syn::custom_keyword!(opaque_blake2_256);
 	syn::custom_keyword!(opaque_blake2_128);
 	syn::custom_keyword!(blake2_128_concat);
+	syn::custom_keyword!(opaque_blake3_256);
+	syn::custom_keyword!(opaque_blake3_128);
+	syn::custom_keyword!(blake3_128_concat);
 	syn::custom_keyword!(opaque_twox_256);
 	syn::custom_keyword!(opaque_twox_128);
 	syn::custom_keyword!(twox_64_concat);
@@ -275,6 +278,9 @@ enum Hasher {
 	Blake2_256(keyword::opaque_blake2_256),
 	Blake2_128(keyword::opaque_blake2_128),
 	Blake2_128Concat(keyword::blake2_128_concat),
+	Blake3_256(keyword::opaque_blake3_256),
+	Blake3_128(keyword::opaque_blake3_128),
+	Blake3_128Concat(keyword::blake3_128_concat),
 	Twox256(keyword::opaque_twox_256),
 	Twox128(keyword::opaque_twox_128),
 	Twox64Concat(keyword::twox_64_concat),
@@ -290,6 +296,12 @@ impl syn::parse::Parse for Hasher {
 			Ok(Self::Blake2_128(input.parse()?))
 		} else if lookahead.peek(keyword::blake2_128_concat) {
 			Ok(Self::Blake2_128Concat(input.parse()?))
+		} else if lookahead.peek(keyword::opaque_blake3_256) {
+			Ok(Self::Blake3_256(input.parse()?))
+		} else if lookahead.peek(keyword::opaque_blake3_128) {
+			Ok(Self::Blake3_128(input.parse()?))
+		} else if lookahead.peek(keyword::blake3_128_concat) {
+			Ok(Self::Blake3_128Concat(input.parse()?))
 		} else if lookahead.peek(keyword::opaque_twox_256) {
 			Ok(Self::Twox256(input.parse()?))
 		} else if lookahead.peek(keyword::opaque_twox_128) {
@@ -346,6 +358,9 @@ impl From<Hasher> for super::HasherKind {
 			Hasher::Blake2_256(_) => super::HasherKind::Blake2_256,
 			Hasher::Blake2_128(_) => super::HasherKind::Blake2_128,
 			Hasher::Blake2_128Concat(_) => super::HasherKind::Blake2_128Concat,
+			Hasher::Blake3_256(_) => super::HasherKind::Blake3_256,
+			Hasher::Blake3_128(_) => super::HasherKind::Blake3_128,
+			Hasher::Blake3_128Concat(_) => super::HasherKind::Blake3_128Concat,
 			Hasher::Twox256(_) => super::HasherKind::Twox256,
 			Hasher::Twox128(_) => super::HasherKind::Twox128,
 			Hasher::Twox64Concat(_) => super::HasherKind::Twox64Concat,

frame/support/src/hash.rs

@@ -19,14 +19,17 @@
 
 use crate::metadata;
 use codec::{Codec, MaxEncodedLen};
-use sp_io::hashing::{blake2_128, blake2_256, twox_128, twox_256, twox_64};
+use sp_io::hashing::{blake2_128, blake2_256, blake3_128, blake3_256, twox_128, twox_256, twox_64};
 use sp_std::prelude::Vec;
 
 // This trait must be kept coherent with frame-support-procedural HasherKind usage
 pub trait Hashable: Sized {
 	fn blake2_128(&self) -> [u8; 16];
 	fn blake2_256(&self) -> [u8; 32];
 	fn blake2_128_concat(&self) -> Vec<u8>;
+	fn blake3_128(&self) -> [u8; 16];
+	fn blake3_256(&self) -> [u8; 32];
+	fn blake3_128_concat(&self) -> Vec<u8>;
 	fn twox_128(&self) -> [u8; 16];
 	fn twox_256(&self) -> [u8; 32];
 	fn twox_64_concat(&self) -> Vec<u8>;
@@ -43,6 +46,15 @@ impl<T: Codec> Hashable for T {
 	fn blake2_128_concat(&self) -> Vec<u8> {
 		self.using_encoded(Blake2_128Concat::hash)
 	}
+	fn blake3_128(&self) -> [u8; 16] {
+		self.using_encoded(blake3_128)
+	}
+	fn blake3_256(&self) -> [u8; 32] {
+		self.using_encoded(blake3_256)
+	}
+	fn blake3_128_concat(&self) -> Vec<u8> {
+		self.using_encoded(Blake3_128Concat::hash)
+	}
 	fn twox_128(&self) -> [u8; 16] {
 		self.using_encoded(twox_128)
 	}
@@ -165,6 +177,54 @@ impl StorageHasher for Blake2_256 {
 	}
 }
 
+/// Hash storage keys with `concat(blake3_128(key), key)`
+pub struct Blake3_128Concat;
+impl StorageHasher for Blake3_128Concat {
+	const METADATA: metadata::StorageHasher = metadata::StorageHasher::Blake3_128Concat;
+	type Output = Vec<u8>;
+	fn hash(x: &[u8]) -> Vec<u8> {
+		blake3_128(x).iter().chain(x.iter()).cloned().collect::<Vec<_>>()
+	}
+	fn max_len<K: MaxEncodedLen>() -> usize {
+		K::max_encoded_len().saturating_add(16)
+	}
+}
+impl ReversibleStorageHasher for Blake3_128Concat {
+	fn reverse(x: &[u8]) -> &[u8] {
+		if x.len() < 16 {
+			log::error!("Invalid reverse: hash length too short");
+			return &[]
+		}
+		&x[16..]
+	}
+}
+
+/// Hash storage keys with blake3 128
+pub struct Blake3_128;
+impl StorageHasher for Blake3_128 {
+	const METADATA: metadata::StorageHasher = metadata::StorageHasher::Blake3_128;
+	type Output = [u8; 16];
+	fn hash(x: &[u8]) -> [u8; 16] {
+		blake3_128(x)
+	}
+	fn max_len<K: MaxEncodedLen>() -> usize {
+		16
+	}
+}
+
+/// Hash storage keys with blake3 256
+pub struct Blake3_256;
+impl StorageHasher for Blake3_256 {
+	const METADATA: metadata::StorageHasher = metadata::StorageHasher::Blake3_256;
+	type Output = [u8; 32];
+	fn hash(x: &[u8]) -> [u8; 32] {
+		blake3_256(x)
+	}
+	fn max_len<K: MaxEncodedLen>() -> usize {
+		32
+	}
+}
+
 /// Hash storage keys with twox 128
 pub struct Twox128;
 impl StorageHasher for Twox128 {
@@ -207,6 +267,12 @@ mod tests {
 		assert_eq!(r.split_at(16), (&blake2_128(b"foo")[..], &b"foo"[..]))
 	}
 
+	#[test]
+	fn test_blake3_128_concat() {
+		let r = Blake3_128Concat::hash(b"foo");
+		assert_eq!(r.split_at(16), (&blake3_128(b"foo")[..], &b"foo"[..]))
+	}
+
 	#[test]
 	fn max_lengths() {
 		use codec::Encode;
@@ -217,6 +283,9 @@ mod tests {
 		assert_eq!(Blake2_128::hash(encoded_0u32).len(), Blake2_128::max_len::<u32>());
 		assert_eq!(Blake2_128Concat::hash(encoded_0u32).len(), Blake2_128Concat::max_len::<u32>());
 		assert_eq!(Blake2_256::hash(encoded_0u32).len(), Blake2_256::max_len::<u32>());
+		assert_eq!(Blake3_128::hash(encoded_0u32).len(), Blake3_128::max_len::<u32>());
+		assert_eq!(Blake3_128Concat::hash(encoded_0u32).len(), Blake3_128Concat::max_len::<u32>());
+		assert_eq!(Blake3_256::hash(encoded_0u32).len(), Blake3_256::max_len::<u32>());
 		assert_eq!(Identity::hash(encoded_0u32).len(), Identity::max_len::<u32>());
 	}
 }

frame/support/src/lib.rs

@@ -98,8 +98,8 @@ pub use self::storage::storage_noop_guard::StorageNoopGuard;
 pub use self::{
 	dispatch::{Callable, Parameter},
 	hash::{
-		Blake2_128, Blake2_128Concat, Blake2_256, Hashable, Identity, ReversibleStorageHasher,
-		StorageHasher, Twox128, Twox256, Twox64Concat,
+		Blake2_128, Blake2_128Concat, Blake2_256, Blake3_128, Blake3_128Concat, Blake3_256,
+		Hashable, Identity, ReversibleStorageHasher, StorageHasher, Twox128, Twox256, Twox64Concat,
 	},
 	storage::{
 		bounded_btree_map::BoundedBTreeMap,
@@ -1404,8 +1404,9 @@ pub mod pallet_prelude {
 			ConstU32, EnsureOrigin, Get, GetDefault, GetStorageVersion, Hooks, IsType,
 			PalletInfoAccess, StorageInfoTrait, StorageVersion, TypedGet,
 		},
-		Blake2_128, Blake2_128Concat, Blake2_256, CloneNoBound, DebugNoBound, EqNoBound, Identity,
-		PartialEqNoBound, RuntimeDebug, RuntimeDebugNoBound, Twox128, Twox256, Twox64Concat,
+		Blake2_128, Blake2_128Concat, Blake2_256, Blake3_128, Blake3_128Concat, Blake3_256,
+		CloneNoBound, DebugNoBound, EqNoBound, Identity, PartialEqNoBound, RuntimeDebug,
+		RuntimeDebugNoBound, Twox128, Twox256, Twox64Concat,
 	};
 	pub use codec::{Decode, Encode, MaxEncodedLen};
 	pub use frame_support::pallet_macros::*;

frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen.stderr

@@ -69,7 +69,7 @@ error[E0277]: the trait bound `Bar: TypeInfo` is not satisfied
              (A, B, C, D)
              (A, B, C, D, E)
              (A, B, C, D, E, F)
-           and 161 others
+           and 162 others
    = note: required for `Bar` to implement `StaticTypeInfo`
    = note: required for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo<T>, Bar>` to implement `StorageEntryMetadataBuilder`
 

frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.stderr

@@ -69,7 +69,7 @@ error[E0277]: the trait bound `Bar: TypeInfo` is not satisfied
              (A, B, C, D)
              (A, B, C, D, E)
              (A, B, C, D, E, F)
-           and 161 others
+           and 162 others
    = note: required for `Bar` to implement `StaticTypeInfo`
    = note: required for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo<T>, Bar>` to implement `StorageEntryMetadataBuilder`
 

primitives/core/Cargo.toml

@@ -49,6 +49,7 @@ bitflags = "1.3"
 array-bytes = { version = "4.1", optional = true }
 ed25519-zebra = { version = "3.1.0", default-features = false, optional = true }
 blake2 = { version = "0.10.4", default-features = false, optional = true }
+blake3 = { version = "^1.3", default-features = false, optional = true }
 schnorrkel = { version = "0.9.1", features = [
 	"preaudit_deprecated",
 	"u64_backend",
@@ -96,6 +97,7 @@ std = [
 	"sp-std/std",
 	"serde",
 	"blake2/std",
+	"blake3/std",
 	"array-bytes",
 	"ed25519-zebra/std",
 	"base58",
@@ -129,6 +131,7 @@ full_crypto = [
 	"array-bytes",
 	"ed25519-zebra",
 	"blake2",
+	"blake3",
 	"schnorrkel",
 	"libsecp256k1",
 	"secp256k1",

primitives/core/hashing/Cargo.toml

@@ -14,6 +14,7 @@ targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
 blake2 = { version = "0.10.4", default-features = false }
+blake3 = { version = "^1.3", default-features = false }
 byteorder = { version = "1.3.2", default-features = false }
 digest = { version = "0.10.3", default-features = false }
 sha2 = { version = "0.10.2", default-features = false }
@@ -26,6 +27,7 @@ default = ["std"]
 std = [
 	"digest/std",
 	"blake2/std",
+	"blake3/std",
 	"byteorder/std",
 	"sha2/std",
 	"sha3/std",

primitives/core/hashing/proc-macro/src/impls.rs

@@ -102,6 +102,18 @@ pub(super) fn blake2b_64(bytes: Vec<u8>) -> TokenStream {
 	bytes_to_array(sp_core_hashing::blake2_64(bytes.as_slice()))
 }
 
+pub(super) fn blake3_512(bytes: Vec<u8>) -> TokenStream {
+	bytes_to_array(sp_core_hashing::blake3_512(bytes.as_slice()))
+}
+
+pub(super) fn blake3_256(bytes: Vec<u8>) -> TokenStream {
+	bytes_to_array(sp_core_hashing::blake3_256(bytes.as_slice()))
+}
+
+pub(super) fn blake3_64(bytes: Vec<u8>) -> TokenStream {
+	bytes_to_array(sp_core_hashing::blake3_64(bytes.as_slice()))
+}
+
 pub(super) fn keccak_256(bytes: Vec<u8>) -> TokenStream {
 	bytes_to_array(sp_core_hashing::keccak_256(bytes.as_slice()))
 }