From cc74d80a5c559efc56a69d202e0022c6836e8f47 Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Mon, 8 Jul 2019 18:48:56 +0200 Subject: [PATCH 1/6] Move Input and Output traits to a separate crate codec-io We will eventually face the issue of dealing with multiple binary encoding formats (such as ssz). In those situations, it is apparent that serde's model (using one `Serialize` and `Deserialize` trait for everything) is not sufficient. For example, U256 needs to be serialized as string in json, while as bytearray in SCALE and ssz. This is not possible to be represented by basic Rust structures. Instead, we need to define Encode/Decode traits for each format we want to encode (have `scale::Encode`, `ssz::Encode`, etc). Those definitions can share common traits `Input` and `Output`. In this PR, I moved those two traits into a separate crate `codec-io` to make it reusable. Note that I also generalized `Error` -- in some other encoding libraries we may not want to represent Error as solely `&str` or unit type and want to actually derive the actual error and pass it to library user. This requires changing `Decode` trait in parity-codec to have an extra bound `From`, which is a breaking change. --- Cargo.lock | 11 +++- Cargo.toml | 10 ++-- derive/Cargo.toml | 2 +- derive/src/encode.rs | 14 ++--- derive/src/lib.rs | 4 +- io/Cargo.toml | 13 ++++ io/src/lib.rs | 101 +++++++++++++++++++++++++++++++ src/codec.rs | 139 ++++++++++++++++++------------------------- src/compact.rs | 54 +++++++++++------ src/lib.rs | 131 ++++++++++++++++++++-------------------- 10 files changed, 298 insertions(+), 181 deletions(-) create mode 100644 io/Cargo.toml create mode 100644 io/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 1144f2df..6e02af9c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -71,6 +71,10 @@ dependencies = [ "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "codec-io" +version = "0.1.0" + [[package]] name = "criterion" version = "0.2.11" @@ -231,20 +235,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "parity-scale-codec" -version = "1.0.0" +version = "2.0.0" dependencies = [ "arrayvec 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", "bitvec 0.11.1 (registry+https://github.com/rust-lang/crates.io-index)", "byte-slice-cast 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "codec-io 0.1.0", "criterion 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", - "parity-scale-codec-derive 1.0.0", + "parity-scale-codec-derive 2.0.0", "serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "parity-scale-codec-derive" -version = "1.0.0" +version = "2.0.0" dependencies = [ "proc-macro-crate 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/Cargo.toml b/Cargo.toml index 1a0f46c1..879df8a9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "parity-scale-codec" description = "SCALE - Simple Concatenating Aggregated Little Endians" -version = "1.0.0" +version = "2.0.0" authors = ["Parity Technologies "] license = "Apache-2.0" repository = "https://github.com/paritytech/parity-scale-codec" @@ -11,13 +11,14 @@ edition = "2018" [dependencies] arrayvec = { version = "0.4", default-features = false, features = ["array-sizes-33-128", "array-sizes-129-255"] } serde = { version = "1.0", optional = true } -parity-scale-codec-derive = { path = "derive", version = "1.0", default-features = false, optional = true } +parity-scale-codec-derive = { path = "derive", version = "2.0", default-features = false, optional = true } +codec-io = { version = "0.1", path = "io", default-features = false } bitvec = { version = "0.11", default-features = false, features = ["alloc"], optional = true } byte-slice-cast = { version = "0.3.1", optional = true } [dev-dependencies] serde_derive = { version = "1.0" } -parity-scale-codec-derive = { path = "derive", version = "1.0", default-features = false } +parity-scale-codec-derive = { path = "derive", version = "2.0", default-features = false } criterion = "0.2" bitvec = "0.11" @@ -31,7 +32,7 @@ bench = false [features] default = ["std"] derive = ["parity-scale-codec-derive"] -std = ["serde", "bitvec/std"] +std = ["serde", "bitvec/std", "codec-io/std"] bit-vec = ["bitvec", "byte-slice-cast"] # WARNING: DO _NOT_ USE THIS FEATURE IF YOU ARE WORKING ON CONSENSUS CODE!* @@ -44,4 +45,5 @@ full = [] [workspace] members = [ "derive", + "io", ] diff --git a/derive/Cargo.toml b/derive/Cargo.toml index 95aca39f..48ac0723 100644 --- a/derive/Cargo.toml +++ b/derive/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "parity-scale-codec-derive" description = "Serialization and deserialization derive macro for Parity SCALE Codec" -version = "1.0.0" +version = "2.0.0" authors = ["Parity Technologies "] license = "Apache-2.0" edition = "2018" diff --git a/derive/src/encode.rs b/derive/src/encode.rs index e9134199..0d86888a 100644 --- a/derive/src/encode.rs +++ b/derive/src/encode.rs @@ -100,9 +100,10 @@ fn encode_fields( let field_type = &f.ty; quote_spanned! { f.span() => { - #dest.push( + _parity_scale_codec::Encode::encode_to( &<<#field_type as _parity_scale_codec::HasCompact>::Type as - _parity_scale_codec::EncodeAsRef<'_, #field_type>>::RefType::from(#field) + _parity_scale_codec::EncodeAsRef<'_, #field_type>>::RefType::from(#field), + #dest ); } } @@ -110,18 +111,17 @@ fn encode_fields( let field_type = &f.ty; quote_spanned! { f.span() => { - #dest.push( + _parity_scale_codec::Encode::encode_to( &<#encoded_as as - _parity_scale_codec::EncodeAsRef<'_, #field_type>>::RefType::from(#field) + _parity_scale_codec::EncodeAsRef<'_, #field_type>>::RefType::from(#field), + #dest ); } } } else if skip { quote! {} } else { - quote_spanned! { f.span() => - #dest.push(#field); - } + quote_spanned! { f.span() => _parity_scale_codec::Encode::encode_to(#field, #dest); } } }); diff --git a/derive/src/lib.rs b/derive/src/lib.rs index f2994eea..cbb3570d 100644 --- a/derive/src/lib.rs +++ b/derive/src/lib.rs @@ -138,7 +138,9 @@ pub fn decode_derive(input: TokenStream) -> TokenStream { impl #impl_generics _parity_scale_codec::Decode for #name #ty_generics #where_clause { fn decode( #input_: &mut DecIn - ) -> Result { + ) -> Result where + _parity_scale_codec::Error: From + { #decoding } } diff --git a/io/Cargo.toml b/io/Cargo.toml new file mode 100644 index 00000000..e4ac6993 --- /dev/null +++ b/io/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "codec-io" +description = "Common Input/Output trait definition shared by encoding libraries." +version = "0.1.0" +authors = ["Parity Technologies "] +license = "Apache-2.0" +repository = "https://github.com/paritytech/parity-scale-codec" +categories = ["encoding"] +edition = "2018" + +[features] +default = ["std"] +std = [] diff --git a/io/src/lib.rs b/io/src/lib.rs new file mode 100644 index 00000000..e9c130d6 --- /dev/null +++ b/io/src/lib.rs @@ -0,0 +1,101 @@ +// Copyright 2017, 2018 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! # Input/Output Trait Definition for Encoding Libraries +//! +//! This library defines `Input` and `Output` traits that can be used for +//! encoding libraries to define their own `Encode` and `Decode` traits. + +#![warn(missing_docs)] + +#![cfg_attr(not(feature = "std"), no_std)] + +extern crate alloc; + +/// Trait that allows reading of data into a slice. +pub trait Input { + /// Error type of this input. + type Error; + + /// Read the exact number of bytes required to fill the given buffer. + /// + /// Note that this function is similar to `std::io::Read::read_exact` and not + /// `std::io::Read::read`. + fn read(&mut self, into: &mut [u8]) -> Result<(), Self::Error>; + + /// Read a single byte from the input. + fn read_byte(&mut self) -> Result { + let mut buf = [0u8]; + self.read(&mut buf[..])?; + Ok(buf[0]) + } +} + +/// Error for slice-based input. Only used in `no_std` environments. +#[cfg(not(feature = "std"))] +#[derive(PartialEq, Eq, Clone)] +pub enum SliceInputError { + /// Not enough data to fill the buffer. + NotEnoughData, +} + +#[cfg(not(feature = "std"))] +impl<'a> Input for &'a [u8] { + type Error = SliceInputError; + + fn read(&mut self, into: &mut [u8]) -> Result<(), SliceInputError> { + if into.len() > self.len() { + return Err(SliceInputError::NotEnoughData); + } + let len = into.len(); + into.copy_from_slice(&self[..len]); + *self = &self[len..]; + Ok(()) + } +} + +#[cfg(feature = "std")] +impl Input for R { + type Error = std::io::Error; + + fn read(&mut self, into: &mut [u8]) -> Result<(), std::io::Error> { + (self as &mut dyn std::io::Read).read_exact(into)?; + Ok(()) + } +} + +/// Trait that allows writing of data. +pub trait Output: Sized { + /// Write to the output. + fn write(&mut self, bytes: &[u8]); + + /// Write a single byte to the output. + fn push_byte(&mut self, byte: u8) { + self.write(&[byte]); + } +} + +#[cfg(not(feature = "std"))] +impl Output for alloc::vec::Vec { + fn write(&mut self, bytes: &[u8]) { + self.extend_from_slice(bytes) + } +} + +#[cfg(feature = "std")] +impl Output for W { + fn write(&mut self, bytes: &[u8]) { + (self as &mut dyn std::io::Write).write_all(bytes).expect("Codec outputs are infallible"); + } +} diff --git a/src/codec.rs b/src/codec.rs index 712eafa0..0f48bafe 100644 --- a/src/codec.rs +++ b/src/codec.rs @@ -29,6 +29,7 @@ use crate::alloc::{ use core::{mem, slice, ops::Deref}; use core::marker::PhantomData; use core::iter::FromIterator; +use codec_io::{Input, Output}; use arrayvec::ArrayVec; #[cfg(feature = "std")] @@ -44,6 +45,7 @@ pub struct Error(&'static str); #[cfg(not(feature = "std"))] #[derive(PartialEq)] +/// Error type for `no_std` environment pub struct Error; impl Error { @@ -92,32 +94,12 @@ impl From<&'static str> for Error { } } -/// Trait that allows reading of data into a slice. -pub trait Input { - /// Read the exact number of bytes required to fill the given buffer. - /// - /// Note that this function is similar to `std::io::Read::read_exact` and not - /// `std::io::Read::read`. - fn read(&mut self, into: &mut [u8]) -> Result<(), Error>; - - /// Read a single byte from the input. - fn read_byte(&mut self) -> Result { - let mut buf = [0u8]; - self.read(&mut buf[..])?; - Ok(buf[0]) - } -} - #[cfg(not(feature = "std"))] -impl<'a> Input for &'a [u8] { - fn read(&mut self, into: &mut [u8]) -> Result<(), Error> { - if into.len() > self.len() { - return Err("Not enough data to fill buffer".into()); +impl From for Error { + fn from(err: codec_io::SliceInputError) -> Self { + match err { + codec_io::SliceInputError::NotEnoughData => "not enough data to fill the buffer".into(), } - let len = into.len(); - into.copy_from_slice(&self[..len]); - *self = &self[len..]; - Ok(()) } } @@ -128,44 +110,6 @@ impl From for Error { } } -#[cfg(feature = "std")] -impl Input for R { - fn read(&mut self, into: &mut [u8]) -> Result<(), Error> { - (self as &mut dyn std::io::Read).read_exact(into)?; - Ok(()) - } -} - -/// Trait that allows writing of data. -pub trait Output: Sized { - /// Write to the output. - fn write(&mut self, bytes: &[u8]); - - /// Write a single byte to the output. - fn push_byte(&mut self, byte: u8) { - self.write(&[byte]); - } - - /// Write encoding of given value to the output. - fn push(&mut self, value: &V) { - value.encode_to(self); - } -} - -#[cfg(not(feature = "std"))] -impl Output for Vec { - fn write(&mut self, bytes: &[u8]) { - self.extend_from_slice(bytes) - } -} - -#[cfg(feature = "std")] -impl Output for W { - fn write(&mut self, bytes: &[u8]) { - (self as &mut dyn std::io::Write).write_all(bytes).expect("Codec outputs are infallible"); - } -} - /// This enum must not be exported and must only be instantiable by parity-scale-codec. /// Because implementation of Encode and Decode for u8 is done in this crate /// and there is not other usage. @@ -232,7 +176,8 @@ pub trait Decode: Sized { const IS_U8: IsU8 = IsU8::No; /// Attempt to deserialise the value from input. - fn decode(value: &mut I) -> Result; + fn decode(value: &mut I) -> Result where + Error: From; } /// Trait that allows zero-copy read/write of value-references to/from slices in LE format. @@ -304,7 +249,9 @@ impl Decode for X where T: Decode + Into, X: WrapperTypeDecode, { - fn decode(input: &mut I) -> Result { + fn decode(input: &mut I) -> Result where + Error: From + { Ok(T::decode(input)?.into()) } } @@ -338,7 +285,9 @@ impl Encode for Result { } impl Decode for Result { - fn decode(input: &mut I) -> Result { + fn decode(input: &mut I) -> Result where + Error: From + { match input.read_byte()? { 0 => Ok(Ok(T::decode(input)?)), 1 => Ok(Err(E::decode(input)?)), @@ -372,7 +321,9 @@ impl Encode for OptionBool { } impl Decode for OptionBool { - fn decode(input: &mut I) -> Result { + fn decode(input: &mut I) -> Result where + Error: From + { match input.read_byte()? { 0 => Ok(OptionBool(None)), 1 => Ok(OptionBool(Some(true))), @@ -402,7 +353,9 @@ impl Encode for Option { } impl Decode for Option { - fn decode(input: &mut I) -> Result { + fn decode(input: &mut I) -> Result where + Error: From + { match input.read_byte()? { 0 => Ok(None), 1 => Ok(Some(T::decode(input)?)), @@ -422,7 +375,9 @@ macro_rules! impl_array { } impl Decode for [T; $n] { - fn decode(input: &mut I) -> Result { + fn decode(input: &mut I) -> Result where + Error: From + { let mut r = ArrayVec::new(); for _ in 0..$n { r.push(T::decode(input)?); @@ -479,7 +434,9 @@ impl Encode for str { impl<'a, T: ToOwned + ?Sized> Decode for Cow<'a, T> where ::Owned: Decode, { - fn decode(input: &mut I) -> Result { + fn decode(input: &mut I) -> Result where + Error: From + { Ok(Cow::Owned(Decode::decode(input)?)) } } @@ -496,7 +453,9 @@ impl Decode for PhantomData { #[cfg(any(feature = "std", feature = "full"))] impl Decode for String { - fn decode(input: &mut I) -> Result { + fn decode(input: &mut I) -> Result where + Error: From + { Ok(Self::from_utf8_lossy(&Vec::decode(input)?).into()) } } @@ -516,7 +475,7 @@ impl Encode for [T] { Compact(len as u32).encode_to(dest); if let IsU8::Yes= ::IS_U8 { let self_transmute = unsafe { - std::mem::transmute::<&[T], &[u8]>(self) + core::mem::transmute::<&[T], &[u8]>(self) }; dest.write(self_transmute) } else { @@ -528,14 +487,16 @@ impl Encode for [T] { } impl Decode for Vec { - fn decode(input: &mut I) -> Result { + fn decode(input: &mut I) -> Result where + Error: From + { >::decode(input).and_then(move |Compact(len)| { let len = len as usize; if let IsU8::Yes = ::IS_U8 { let mut r = vec![0; len]; input.read(&mut r[..len])?; - let r = unsafe { std::mem::transmute::, Vec>(r) }; + let r = unsafe { core::mem::transmute::, Vec>(r) }; Ok(r) } else { let mut r = Vec::with_capacity(len); @@ -615,7 +576,9 @@ macro_rules! impl_codec_through_iterator { } impl<$($decode_generics)*> Decode for $type { - fn decode(input: &mut I) -> Result { + fn decode(input: &mut I) -> Result where + Error: From + { >::decode(input).and_then(move |Compact(len)| { Result::from_iter((0..len).map(|_| Decode::decode(input))) }) @@ -640,7 +603,7 @@ impl Encode for VecDeque { if let IsU8::Yes = ::IS_U8 { let slices = self.as_slices(); let slices_transmute = unsafe { - std::mem::transmute::<(&[T], &[T]), (&[u8], &[u8])>(slices) + core::mem::transmute::<(&[T], &[T]), (&[u8], &[u8])>(slices) }; dest.write(slices_transmute.0); dest.write(slices_transmute.1); @@ -653,7 +616,9 @@ impl Encode for VecDeque { } impl Decode for VecDeque { - fn decode(input: &mut I) -> Result { + fn decode(input: &mut I) -> Result where + Error: From + { Ok(>::decode(input)?.into()) } } @@ -672,7 +637,9 @@ impl Encode for () { } impl Decode for () { - fn decode(_: &mut I) -> Result<(), Error> { + fn decode(_: &mut I) -> Result<(), Error> where + Error: From + { Ok(()) } } @@ -712,7 +679,9 @@ macro_rules! tuple_impl { } impl<$one: Decode> Decode for ($one,) { - fn decode(input: &mut I) -> Result { + fn decode(input: &mut I) -> Result where + Error: From + { match $one::decode(input) { Err(e) => Err(e), Ok($one) => Ok(($one,)), @@ -747,7 +716,9 @@ macro_rules! tuple_impl { impl<$first: Decode, $($rest: Decode),+> Decode for ($first, $($rest),+) { - fn decode(input: &mut INPUT) -> Result { + fn decode(input: &mut INPUT) -> Result where + Error: From + { Ok(( match $first::decode(input) { Ok(x) => x, @@ -767,7 +738,7 @@ macro_rules! tuple_impl { #[allow(non_snake_case)] mod inner_tuple_impl { - use super::{Error, Input, Output, Decode, Encode}; + use super::{Error, Input, Output, Decode, Encode, Vec}; tuple_impl!(A, B, C, D, E, F, G, H, I, J, K,); } @@ -821,7 +792,9 @@ macro_rules! impl_endians { } impl Decode for $t { - fn decode(input: &mut I) -> Result { + fn decode(input: &mut I) -> Result where + Error: From + { let size = mem::size_of::<$t>(); assert!(size > 0, "EndianSensitive can never be implemented for a zero-sized type."); let mut val: $t = unsafe { mem::zeroed() }; @@ -869,7 +842,9 @@ macro_rules! impl_non_endians { impl Decode for $t { $( const $is_u8: IsU8 = IsU8::Yes; )? - fn decode(input: &mut I) -> Result { + fn decode(input: &mut I) -> Result where + Error: From + { let size = mem::size_of::<$t>(); assert!(size > 0, "EndianSensitive can never be implemented for a zero-sized type."); let mut val: $t = unsafe { mem::zeroed() }; diff --git a/src/compact.rs b/src/compact.rs index 9be7bf41..e62464cc 100644 --- a/src/compact.rs +++ b/src/compact.rs @@ -14,7 +14,9 @@ //! [Compact encoding](https://substrate.dev/docs/en/overview/low-level-data-format#compact-general-integers) -use crate::codec::{Input, Output, Error, Encode, Decode, EncodeAsRef}; +use crate::codec::{Error, Encode, Decode, EncodeAsRef}; +use crate::alloc::vec::Vec; +use codec_io::{Input, Output}; use arrayvec::ArrayVec; @@ -44,7 +46,11 @@ struct PrefixInput<'a, T> { input: &'a mut T, } -impl<'a, T: 'a + Input> Input for PrefixInput<'a, T> { +impl<'a, T: 'a + Input> Input for PrefixInput<'a, T> where + Error: From +{ + type Error = Error; + fn read(&mut self, buffer: &mut [u8]) -> Result<(), Error> { match self.prefix.take() { Some(v) if !buffer.is_empty() => { @@ -52,7 +58,7 @@ impl<'a, T: 'a + Input> Input for PrefixInput<'a, T> { self.input.read(&mut buffer[1..])?; Ok(()) } - _ => self.input.read(buffer) + _ => Ok(self.input.read(buffer)?) } } } @@ -135,7 +141,9 @@ where T: CompactAs, Compact: Decode, { - fn decode(input: &mut I) -> Result { + fn decode(input: &mut I) -> Result where + Error: From + { Compact::::decode(input) .map(|x| Compact(::decode_from(x.0))) } @@ -416,12 +424,14 @@ const U64_OUT_OF_RANGE: &'static str = "out of range decoding Compact"; const U128_OUT_OF_RANGE: &'static str = "out of range decoding Compact"; impl Decode for Compact { - fn decode(input: &mut I) -> Result { + fn decode(input: &mut I) -> Result where + Error: From + { let prefix = input.read_byte()?; Ok(Compact(match prefix % 4 { 0 => prefix >> 2, 1 => { - let x = u16::decode(&mut PrefixInput{prefix: Some(prefix), input})? >> 2; + let x = u16::decode::>(&mut PrefixInput{prefix: Some(prefix), input})? >> 2; if x > 0b00111111 && x <= 255 { x as u8 } else { @@ -434,12 +444,14 @@ impl Decode for Compact { } impl Decode for Compact { - fn decode(input: &mut I) -> Result { + fn decode(input: &mut I) -> Result where + Error: From + { let prefix = input.read_byte()?; Ok(Compact(match prefix % 4 { 0 => u16::from(prefix) >> 2, 1 => { - let x = u16::decode(&mut PrefixInput{prefix: Some(prefix), input})? >> 2; + let x = u16::decode::>(&mut PrefixInput{prefix: Some(prefix), input})? >> 2; if x > 0b00111111 && x <= 0b00111111_11111111 { u16::from(x) } else { @@ -447,7 +459,7 @@ impl Decode for Compact { } }, 2 => { - let x = u32::decode(&mut PrefixInput{prefix: Some(prefix), input})? >> 2; + let x = u32::decode::>(&mut PrefixInput{prefix: Some(prefix), input})? >> 2; if x > 0b00111111_11111111 && x < 65536 { x as u16 } else { @@ -460,12 +472,14 @@ impl Decode for Compact { } impl Decode for Compact { - fn decode(input: &mut I) -> Result { + fn decode(input: &mut I) -> Result where + Error: From + { let prefix = input.read_byte()?; Ok(Compact(match prefix % 4 { 0 => u32::from(prefix) >> 2, 1 => { - let x = u16::decode(&mut PrefixInput{prefix: Some(prefix), input})? >> 2; + let x = u16::decode::>(&mut PrefixInput{prefix: Some(prefix), input})? >> 2; if x > 0b00111111 && x <= 0b00111111_11111111 { u32::from(x) } else { @@ -473,7 +487,7 @@ impl Decode for Compact { } }, 2 => { - let x = u32::decode(&mut PrefixInput{prefix: Some(prefix), input})? >> 2; + let x = u32::decode::>(&mut PrefixInput{prefix: Some(prefix), input})? >> 2; if x > 0b00111111_11111111 && x <= u32::max_value() >> 2 { u32::from(x) } else { @@ -499,12 +513,14 @@ impl Decode for Compact { } impl Decode for Compact { - fn decode(input: &mut I) -> Result { + fn decode(input: &mut I) -> Result where + Error: From + { let prefix = input.read_byte()?; Ok(Compact(match prefix % 4 { 0 => u64::from(prefix) >> 2, 1 => { - let x = u16::decode(&mut PrefixInput{prefix: Some(prefix), input})? >> 2; + let x = u16::decode::>(&mut PrefixInput{prefix: Some(prefix), input})? >> 2; if x > 0b00111111 && x <= 0b00111111_11111111 { u64::from(x) } else { @@ -512,7 +528,7 @@ impl Decode for Compact { } }, 2 => { - let x = u32::decode(&mut PrefixInput{prefix: Some(prefix), input})? >> 2; + let x = u32::decode::>(&mut PrefixInput{prefix: Some(prefix), input})? >> 2; if x > 0b00111111_11111111 && x <= u32::max_value() >> 2 { u64::from(x) } else { @@ -554,12 +570,14 @@ impl Decode for Compact { } impl Decode for Compact { - fn decode(input: &mut I) -> Result { + fn decode(input: &mut I) -> Result where + Error: From + { let prefix = input.read_byte()?; Ok(Compact(match prefix % 4 { 0 => u128::from(prefix) >> 2, 1 => { - let x = u16::decode(&mut PrefixInput{prefix: Some(prefix), input})? >> 2; + let x = u16::decode::>(&mut PrefixInput{prefix: Some(prefix), input})? >> 2; if x > 0b00111111 && x <= 0b00111111_11111111 { u128::from(x) } else { @@ -567,7 +585,7 @@ impl Decode for Compact { } }, 2 => { - let x = u32::decode(&mut PrefixInput{prefix: Some(prefix), input})? >> 2; + let x = u32::decode::>(&mut PrefixInput{prefix: Some(prefix), input})? >> 2; if x > 0b00111111_11111111 && x <= u32::max_value() >> 2 { u128::from(x) } else { diff --git a/src/lib.rs b/src/lib.rs index 27db9a91..a0de6753 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,73 +13,73 @@ // limitations under the License. //! # Parity SCALE Codec -//! -//! Rust implementation of the SCALE (Simple Concatenated Aggregate Little-Endian) data format +//! +//! Rust implementation of the SCALE (Simple Concatenated Aggregate Little-Endian) data format //! for types used in the Parity Substrate framework. -//! -//! SCALE is a light-weight format which allows encoding (and decoding) which makes it highly -//! suitable for resource-constrained execution environments like blockchain runtimes and low-power, +//! +//! SCALE is a light-weight format which allows encoding (and decoding) which makes it highly +//! suitable for resource-constrained execution environments like blockchain runtimes and low-power, //! low-memory devices. -//! -//! It is important to note that the encoding context (knowledge of how the types and data structures look) -//! needs to be known separately at both encoding and decoding ends. +//! +//! It is important to note that the encoding context (knowledge of how the types and data structures look) +//! needs to be known separately at both encoding and decoding ends. //! The encoded data does not include this contextual information. -//! -//! To get a better understanding of how the encoding is done for different types, -//! take a look at the +//! +//! To get a better understanding of how the encoding is done for different types, +//! take a look at the //! [low-level data formats overview page at the Substrate docs site](https://substrate.dev/docs/en/overview/low-level-data-format). -//! +//! //! ## Implementation -//! +//! //! The codec is implemented using the following traits: -//! +//! //! ### Encode -//! +//! //! The `Encode` trait is used for encoding of data into the SCALE format. The `Encode` trait contains the following functions: -//! * `size_hint(&self) -> usize`: Gets the capacity (in bytes) required for the encoded data. -//! This is to avoid double-allocation of memory needed for the encoding. -//! It can be an estimate and does not need to be an exact number. +//! * `size_hint(&self) -> usize`: Gets the capacity (in bytes) required for the encoded data. +//! This is to avoid double-allocation of memory needed for the encoding. +//! It can be an estimate and does not need to be an exact number. //! If the size is not known, even no good maximum, then we can skip this function from the trait implementation. //! This is required to be a cheap operation, so should not involve iterations etc. //! * `encode_to(&self, dest: &mut T)`: Encodes the value and appends it to a destination buffer. //! * `encode(&self) -> Vec`: Encodes the type data and returns a slice. -//! * `using_encoded R>(&self, f: F) -> R`: Encodes the type data and executes a closure on the encoded value. +//! * `using_encoded R>(&self, f: F) -> R`: Encodes the type data and executes a closure on the encoded value. //! Returns the result from the executed closure. -//! -//! **Note:** Implementations should override `using_encoded` for value types and `encode_to` for allocating types. +//! +//! **Note:** Implementations should override `using_encoded` for value types and `encode_to` for allocating types. //! `size_hint` should be implemented for all types, wherever possible. Wrapper types should override all methods. -//! +//! //! ### Decode -//! +//! //! The `Decode` trait is used for deserialization/decoding of encoded data into the respective types. -//! -//! * `fn decode(value: &mut I) -> Result`: Tries to decode the value from SCALE format to the type it is called on. +//! +//! * `fn decode(value: &mut I) -> Result`: Tries to decode the value from SCALE format to the type it is called on. //! Returns an `Err` if the decoding fails. -//! +//! //! ### CompactAs -//! -//! The `CompactAs` trait is used for wrapping custom types/structs as compact types, which makes them even more space/memory efficient. +//! +//! The `CompactAs` trait is used for wrapping custom types/structs as compact types, which makes them even more space/memory efficient. //! The compact encoding is described [here](https://substrate.dev/docs/en/overview/low-level-data-format#compact-general-integers). -//! -//! * `encode_as(&self) -> &Self::As`: Encodes the type (self) as a compact type. +//! +//! * `encode_as(&self) -> &Self::As`: Encodes the type (self) as a compact type. //! The type `As` is defined in the same trait and its implementation should be compact encode-able. //! * `decode_from(_: Self::As) -> Self`: Decodes the type (self) from a compact encode-able type. -//! +//! //! ### HasCompact -//! +//! //! The `HasCompact` trait, if implemented, tells that the corresponding type is a compact encode-able type. -//! +//! //! ## Usage Examples -//! +//! //! Following are some examples to demonstrate usage of the codec. -//! +//! //! ### Simple types -//! +//! //! ``` //! use parity_scale_codec_derive::{Encode, Decode}; //! use parity_scale_codec::{Encode, Decode}; -//! +//! //! #[derive(Debug, PartialEq, Encode, Decode)] //! enum EnumType { //! #[codec(index = "15")] @@ -90,108 +90,108 @@ //! b: u64, //! }, //! } -//! +//! //! let a = EnumType::A; //! let b = EnumType::B(1, 2); //! let c = EnumType::C { a: 1, b: 2 }; -//! +//! //! a.using_encoded(|ref slice| { //! assert_eq!(slice, &b"\x0f"); //! }); -//! +//! //! b.using_encoded(|ref slice| { //! assert_eq!(slice, &b"\x01\x01\0\0\0\x02\0\0\0\0\0\0\0"); //! }); -//! +//! //! c.using_encoded(|ref slice| { //! assert_eq!(slice, &b"\x02\x01\0\0\0\x02\0\0\0\0\0\0\0"); //! }); -//! +//! //! let mut da: &[u8] = b"\x0f"; //! assert_eq!(EnumType::decode(&mut da).ok(), Some(a)); -//! +//! //! let mut db: &[u8] = b"\x01\x01\0\0\0\x02\0\0\0\0\0\0\0"; //! assert_eq!(EnumType::decode(&mut db).ok(), Some(b)); -//! +//! //! let mut dc: &[u8] = b"\x02\x01\0\0\0\x02\0\0\0\0\0\0\0"; //! assert_eq!(EnumType::decode(&mut dc).ok(), Some(c)); -//! +//! //! let mut dz: &[u8] = &[0]; //! assert_eq!(EnumType::decode(&mut dz).ok(), None); -//! +//! //! # fn main() { } //! ``` -//! +//! //! ### Compact type with HasCompact -//! +//! //! ``` //! use parity_scale_codec_derive::{Encode, Decode};; //! use parity_scale_codec::{Encode, Decode, Compact, HasCompact}; -//! +//! //! #[derive(Debug, PartialEq, Encode, Decode)] //! struct Test1CompactHasCompact { //! #[codec(compact)] //! bar: T, //! } -//! +//! //! #[derive(Debug, PartialEq, Encode, Decode)] //! struct Test1HasCompact { //! #[codec(encoded_as = "::Type")] //! bar: T, //! } -//! +//! //! let test_val: (u64, usize) = (0u64, 1usize); -//! +//! //! let encoded = Test1HasCompact { bar: test_val.0 }.encode(); //! assert_eq!(encoded.len(), test_val.1); //! assert_eq!(>::decode(&mut &encoded[..]).unwrap().bar, test_val.0); -//! +//! //! # fn main() { } //! ``` //! ### Type with CompactAs -//! +//! //! ```rust -//! +//! //! use serde_derive::{Serialize, Deserialize}; //! use parity_scale_codec_derive::{Encode, Decode};; //! use parity_scale_codec::{Encode, Decode, Compact, HasCompact, CompactAs}; -//! +//! //! #[cfg_attr(feature = "std", derive(Serialize, Deserialize, Debug))] //! #[derive(PartialEq, Eq, Clone)] //! struct StructHasCompact(u32); -//! +//! //! impl CompactAs for StructHasCompact { //! type As = u32; -//! +//! //! fn encode_as(&self) -> &Self::As { //! &12 //! } -//! +//! //! fn decode_from(_: Self::As) -> Self { //! StructHasCompact(12) //! } //! } -//! +//! //! impl From> for StructHasCompact { //! fn from(_: Compact) -> Self { //! StructHasCompact(12) //! } //! } -//! +//! //! #[derive(Debug, PartialEq, Encode, Decode)] //! enum TestGenericHasCompact { //! A { //! #[codec(compact)] a: T //! }, //! } -//! +//! //! let a = TestGenericHasCompact::A:: { //! a: StructHasCompact(12325678), //! }; -//! +//! //! let encoded = a.encode(); //! assert_eq!(encoded.len(), 2); -//! +//! //! # fn main() { } //! ``` @@ -241,8 +241,9 @@ mod keyedvec; #[cfg(feature = "bit-vec")] mod bit_vec; +pub use codec_io::{Input, Output}; pub use self::codec::{ - Input, Output, Error, Encode, Decode, Codec, EncodeAsRef, EncodeAppend, WrapperTypeEncode, + Error, Encode, Decode, Codec, EncodeAsRef, EncodeAppend, WrapperTypeEncode, WrapperTypeDecode, OptionBool, }; pub use self::compact::{Compact, HasCompact, CompactAs}; From 2f3a8ec0c6455f59674b6bce8d623433f3b72925 Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Mon, 8 Jul 2019 20:30:10 +0200 Subject: [PATCH 2/6] Update io/src/lib.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Bastian Köcher --- io/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/io/src/lib.rs b/io/src/lib.rs index e9c130d6..ce548f0c 100644 --- a/io/src/lib.rs +++ b/io/src/lib.rs @@ -1,4 +1,4 @@ -// Copyright 2017, 2018 Parity Technologies +// Copyright 2019 Parity Technologies // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. From ca4de9bcda11cfb8571ef3e3e7c000efcf2228a6 Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Mon, 8 Jul 2019 20:30:17 +0200 Subject: [PATCH 3/6] Update derive/src/encode.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Bastian Köcher --- derive/src/encode.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/derive/src/encode.rs b/derive/src/encode.rs index 0d86888a..495b8188 100644 --- a/derive/src/encode.rs +++ b/derive/src/encode.rs @@ -103,7 +103,7 @@ fn encode_fields( _parity_scale_codec::Encode::encode_to( &<<#field_type as _parity_scale_codec::HasCompact>::Type as _parity_scale_codec::EncodeAsRef<'_, #field_type>>::RefType::from(#field), - #dest + #dest, ); } } From b11181be8e15d9d64bf61e384af9f52f64fabbf0 Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Mon, 8 Jul 2019 20:30:28 +0200 Subject: [PATCH 4/6] Update derive/src/encode.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Bastian Köcher --- derive/src/encode.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/derive/src/encode.rs b/derive/src/encode.rs index 495b8188..8f021800 100644 --- a/derive/src/encode.rs +++ b/derive/src/encode.rs @@ -114,7 +114,7 @@ fn encode_fields( _parity_scale_codec::Encode::encode_to( &<#encoded_as as _parity_scale_codec::EncodeAsRef<'_, #field_type>>::RefType::from(#field), - #dest + #dest, ); } } From 1aabfbce679b1f4216eafd00dbfcec89659f5a46 Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Mon, 8 Jul 2019 20:32:08 +0200 Subject: [PATCH 5/6] Revert version bump --- Cargo.lock | 6 +++--- Cargo.toml | 6 +++--- derive/Cargo.toml | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6e02af9c..997b97df 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -235,21 +235,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "parity-scale-codec" -version = "2.0.0" +version = "1.0.0" dependencies = [ "arrayvec 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", "bitvec 0.11.1 (registry+https://github.com/rust-lang/crates.io-index)", "byte-slice-cast 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "codec-io 0.1.0", "criterion 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", - "parity-scale-codec-derive 2.0.0", + "parity-scale-codec-derive 1.0.0", "serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "parity-scale-codec-derive" -version = "2.0.0" +version = "1.0.0" dependencies = [ "proc-macro-crate 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/Cargo.toml b/Cargo.toml index 879df8a9..19dffc68 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "parity-scale-codec" description = "SCALE - Simple Concatenating Aggregated Little Endians" -version = "2.0.0" +version = "1.0.0" authors = ["Parity Technologies "] license = "Apache-2.0" repository = "https://github.com/paritytech/parity-scale-codec" @@ -11,14 +11,14 @@ edition = "2018" [dependencies] arrayvec = { version = "0.4", default-features = false, features = ["array-sizes-33-128", "array-sizes-129-255"] } serde = { version = "1.0", optional = true } -parity-scale-codec-derive = { path = "derive", version = "2.0", default-features = false, optional = true } +parity-scale-codec-derive = { path = "derive", version = "1.0", default-features = false, optional = true } codec-io = { version = "0.1", path = "io", default-features = false } bitvec = { version = "0.11", default-features = false, features = ["alloc"], optional = true } byte-slice-cast = { version = "0.3.1", optional = true } [dev-dependencies] serde_derive = { version = "1.0" } -parity-scale-codec-derive = { path = "derive", version = "2.0", default-features = false } +parity-scale-codec-derive = { path = "derive", version = "1.0", default-features = false } criterion = "0.2" bitvec = "0.11" diff --git a/derive/Cargo.toml b/derive/Cargo.toml index 48ac0723..95aca39f 100644 --- a/derive/Cargo.toml +++ b/derive/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "parity-scale-codec-derive" description = "Serialization and deserialization derive macro for Parity SCALE Codec" -version = "2.0.0" +version = "1.0.0" authors = ["Parity Technologies "] license = "Apache-2.0" edition = "2018" From 760695e7415e32c61ccfd85d34d730abcf37d27a Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Mon, 8 Jul 2019 23:30:10 +0200 Subject: [PATCH 6/6] Use unified Error type in codec_io, with `std` alias to `std::io::Error`. --- io/src/lib.rs | 34 +++++++++++++++------------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/io/src/lib.rs b/io/src/lib.rs index ce548f0c..5e70e894 100644 --- a/io/src/lib.rs +++ b/io/src/lib.rs @@ -23,40 +23,38 @@ extern crate alloc; +/// I/O error type for basic `no_std` environment. +#[cfg(not(feature = "std"))] +pub enum Error { + /// Not enough data to fill the buffer. + NotEnoughData, +} + +/// I/O error type for `std`. Alias of std::io::Error. +#[cfg(feature = "std")] +pub type Error = std::io::Error; + /// Trait that allows reading of data into a slice. pub trait Input { - /// Error type of this input. - type Error; - /// Read the exact number of bytes required to fill the given buffer. /// /// Note that this function is similar to `std::io::Read::read_exact` and not /// `std::io::Read::read`. - fn read(&mut self, into: &mut [u8]) -> Result<(), Self::Error>; + fn read(&mut self, into: &mut [u8]) -> Result<(), Error>; /// Read a single byte from the input. - fn read_byte(&mut self) -> Result { + fn read_byte(&mut self) -> Result { let mut buf = [0u8]; self.read(&mut buf[..])?; Ok(buf[0]) } } -/// Error for slice-based input. Only used in `no_std` environments. -#[cfg(not(feature = "std"))] -#[derive(PartialEq, Eq, Clone)] -pub enum SliceInputError { - /// Not enough data to fill the buffer. - NotEnoughData, -} - #[cfg(not(feature = "std"))] impl<'a> Input for &'a [u8] { - type Error = SliceInputError; - - fn read(&mut self, into: &mut [u8]) -> Result<(), SliceInputError> { + fn read(&mut self, into: &mut [u8]) -> Result<(), Error> { if into.len() > self.len() { - return Err(SliceInputError::NotEnoughData); + return Err(Error::NotEnoughData); } let len = into.len(); into.copy_from_slice(&self[..len]); @@ -67,8 +65,6 @@ impl<'a> Input for &'a [u8] { #[cfg(feature = "std")] impl Input for R { - type Error = std::io::Error; - fn read(&mut self, into: &mut [u8]) -> Result<(), std::io::Error> { (self as &mut dyn std::io::Read).read_exact(into)?; Ok(())