From a0ef8a2f90f53f1c30a554624c187a9672d098c9 Mon Sep 17 00:00:00 2001 From: Christopher Hunt Date: Sat, 22 Apr 2023 10:25:51 +1000 Subject: [PATCH] Introduces CRCs (#98) * Introduces CRCs A `crc` feature has been added that brings in the `crc` crate and a new `CrcModifier` flavour. This modifier flavour is similar to the COBS implementation in how it wraps another flavour. * Convenience functions Convenience functions for serde usage * Optionally include paste The paste crate is included now only if `use-crc` feature is enabled. This may help improve build performance where CRCs are not required. * Fix use of paste macro This uses the correct digest sizes, and adds a test for a random CRC8 variant. --------- Co-authored-by: James Munns --- .gitignore | 3 +- Cargo.toml | 9 ++ src/de/flavors.rs | 215 ++++++++++++++++++++++++--------------------- src/de/mod.rs | 20 ++++- src/lib.rs | 15 ++++ src/ser/flavors.rs | 131 ++++++++++++++++++++++++++- src/ser/mod.rs | 101 +++++++++++++++++++++ tests/crc.rs | 39 ++++++++ 8 files changed, 430 insertions(+), 103 deletions(-) create mode 100644 tests/crc.rs diff --git a/.gitignore b/.gitignore index 2f88dba..5cb9f6a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ /target **/*.rs.bk -Cargo.lock \ No newline at end of file +Cargo.lock +.vscode \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 6498e55..4fd0c4a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,6 +52,14 @@ path = "./postcard-derive" version = "0.1.1" optional = true +[dependencies.crc] +version = "3.0.1" +optional = true + +[dependencies.paste] +version = "1.0.12" +optional = true + [features] default = ["heapless-cas"] @@ -59,6 +67,7 @@ use-std = ["serde/std", "alloc"] heapless-cas = ["heapless", "heapless/cas"] alloc = ["serde/alloc"] use-defmt = ["defmt"] +use-crc = ["crc", "paste"] # Experimental features! # diff --git a/src/de/flavors.rs b/src/de/flavors.rs index a81fdb2..ff7efca 100644 --- a/src/de/flavors.rs +++ b/src/de/flavors.rs @@ -163,120 +163,139 @@ impl<'de> Flavor<'de> for Slice<'de> { } } -// This is a terrible checksum implementation to make sure that we can effectively -// use the deserialization flavor. This is kept as a test (and not published) -// because an 8-bit checksum is not ACTUALLY useful for almost anything. -// -// You could certainly do something similar with a CRC32, cryptographic sig, -// or something else -#[cfg(test)] -mod test { - use super::*; - use serde::{Deserialize, Serialize}; +//////////////////////////////////////// +// CRC +//////////////////////////////////////// - struct Checksum<'de, F> - where - F: Flavor<'de> + 'de, - { - flav: F, - checksum: u8, - _plt: PhantomData<&'de ()>, - } +/// This Cyclic Redundancy Check flavor applies [the CRC crate's `Algorithm`](https://docs.rs/crc/latest/crc/struct.Algorithm.html) struct on +/// the serialized data. The flavor will check the CRC assuming that it has been appended to the bytes. +/// +/// CRCs are used for error detection when reading data back. +/// +/// The `crc` feature requires enabling to use this module. +/// +/// More on CRCs: . +#[cfg(feature = "use-crc")] +pub mod crc { + use core::convert::TryInto; + + use crc::Digest; + use crc::Width; + use serde::Deserialize; - impl<'de, F> Checksum<'de, F> + use super::Flavor; + use super::Slice; + + use crate::Deserializer; + use crate::Error; + use crate::Result; + use paste::paste; + + /// Manages CRC modifications as a flavor. + pub struct CrcModifier<'de, B, W> where - F: Flavor<'de> + 'de, + B: Flavor<'de>, + W: Width, { - pub fn from_flav(flav: F) -> Self { - Self { - flav, - checksum: 0, - _plt: PhantomData, - } - } + flav: B, + digest: Digest<'de, W>, } - impl<'de, F> Flavor<'de> for Checksum<'de, F> + impl<'de, B, W> CrcModifier<'de, B, W> where - F: Flavor<'de> + 'de, + B: Flavor<'de>, + W: Width, { - type Remainder = (>::Remainder, u8); - type Source = F; - - fn pop(&mut self) -> Result { - match self.flav.pop() { - Ok(u) => { - self.checksum = self.checksum.wrapping_add(u); - Ok(u) - } - Err(e) => Err(e), - } - } - fn try_take_n(&mut self, ct: usize) -> Result<&'de [u8]> { - match self.flav.try_take_n(ct) { - Ok(u) => { - u.iter().for_each(|u| { - self.checksum = self.checksum.wrapping_add(*u); - }); - Ok(u) - } - Err(e) => Err(e), - } - } - fn finalize(self) -> Result { - Ok((self.flav.finalize()?, self.checksum)) + /// Create a new Crc modifier Flavor. + pub fn new(bee: B, digest: Digest<'de, W>) -> Self { + Self { flav: bee, digest } } } - #[derive(Serialize, Deserialize, PartialEq, Eq, Debug)] - pub struct SomeData<'a> { - #[serde(borrow)] - sli: &'a [u8], - sts: &'a str, - foo: u64, - bar: u128, - } + macro_rules! impl_flavor { + ($( $int:ty ),*) => { + $( + paste! { + impl<'de, B> Flavor<'de> for CrcModifier<'de, B, $int> + where + B: Flavor<'de>, + { + type Remainder = B::Remainder; - #[test] - fn smoke() { - const EXPECTED: &[u8] = &[ - 4, 255, 1, 34, 51, 19, 116, 104, 105, 115, 32, 105, 115, 32, 97, 32, 103, 111, 111, - 100, 32, 116, 101, 115, 116, 170, 213, 170, 213, 170, 213, 170, 213, 170, 1, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 127, - ]; + type Source = B::Source; - // Calculate simple 8-bit checksum - let mut check: u8 = 0; - EXPECTED.iter().for_each(|u| check = check.wrapping_add(*u)); + #[inline] + fn pop(&mut self) -> Result { + match self.flav.pop() { + Ok(byte) => { + self.digest.update(&[byte]); + Ok(byte) + } + e @ Err(_) => e, + } + } - let mut buf = [0u8; 256]; - let data = SomeData { - sli: &[0xFF, 0x01, 0x22, 0x33], - sts: "this is a good test", - foo: (u64::MAX / 3) * 2, - bar: u128::MAX / 4, - }; - let used = crate::to_slice(&data, &mut buf).unwrap(); - assert_eq!(used, EXPECTED); - let used = used.len(); + #[inline] + fn try_take_n(&mut self, ct: usize) -> Result<&'de [u8]> { + match self.flav.try_take_n(ct) { + Ok(bytes) => { + self.digest.update(bytes); + Ok(bytes) + } + e @ Err(_) => e, + } + } - // Put the checksum at the end - buf[used] = check; + fn finalize(mut self) -> Result { + match self.flav.try_take_n(core::mem::size_of::<$int>()) { + Ok(prev_crc_bytes) => match self.flav.finalize() { + Ok(remainder) => { + let crc = self.digest.finalize(); + let le_bytes = prev_crc_bytes + .try_into() + .map_err(|_| Error::DeserializeBadEncoding)?; + let prev_crc = <$int>::from_le_bytes(le_bytes); + if crc == prev_crc { + Ok(remainder) + } else { + Err(Error::DeserializeBadEncoding) + } + } + e @ Err(_) => e, + }, + Err(e) => Err(e), + } + } + } - let mut deser = crate::de::Deserializer::from_flavor(Checksum::from_flav(Slice::new(&buf))); + /// Deserialize a message of type `T` from a byte slice with a Crc. The unused portion (if any) + /// of the byte slice is not returned. + pub fn []<'a, T>(s: &'a [u8], digest: Digest<'a, $int>) -> Result + where + T: Deserialize<'a>, + { + let flav = CrcModifier::new(Slice::new(s), digest); + let mut deserializer = Deserializer::from_flavor(flav); + let r = T::deserialize(&mut deserializer)?; + let _ = deserializer.finalize()?; + Ok(r) + } - let t = SomeData::<'_>::deserialize(&mut deser).unwrap(); - assert_eq!(t, data); - - // Normally, you'd probably expect the check - let (rem, cksm) = deser.finalize().unwrap(); - - // The pre-calculated checksum we stuffed at the end is the - // first "unused" byte - assert_eq!(rem[0], check); - - // the one we calculated during serialization matches the - // pre-calculated one - assert_eq!(cksm, check); + /// Deserialize a message of type `T` from a byte slice with a Crc. The unused portion (if any) + /// of the byte slice is returned for further usage + pub fn []<'a, T>(s: &'a [u8], digest: Digest<'a, $int>) -> Result<(T, &'a [u8])> + where + T: Deserialize<'a>, + { + let flav = CrcModifier::new(Slice::new(s), digest); + let mut deserializer = Deserializer::from_flavor(flav); + let t = T::deserialize(&mut deserializer)?; + Ok((t, deserializer.finalize()?)) + } + } + )* + }; } + + impl_flavor![u8, u16, u32, u64, u128]; } diff --git a/src/de/mod.rs b/src/de/mod.rs index c17751b..a30c9ce 100644 --- a/src/de/mod.rs +++ b/src/de/mod.rs @@ -20,7 +20,7 @@ where /// Deserialize a message of type `T` from a cobs-encoded byte slice. The /// unused portion (if any) of the byte slice is not returned. -/// The used portion of the input slice is modified during deserialization (even if an error is returned). +/// The used portion of the input slice is modified during deserialization (even if an error is returned). /// Therefore, if this is not desired, pass a clone of the original slice. pub fn from_bytes_cobs<'a, T>(s: &'a mut [u8]) -> Result where @@ -32,7 +32,7 @@ where /// Deserialize a message of type `T` from a cobs-encoded byte slice. The /// unused portion (if any) of the byte slice is returned for further usage. -/// The used portion of the input slice is modified during deserialization (even if an error is returned). +/// The used portion of the input slice is modified during deserialization (even if an error is returned). /// Therefore, if this is not desired, pass a clone of the original slice. pub fn take_from_bytes_cobs<'a, T>(s: &'a mut [u8]) -> Result<(T, &'a mut [u8])> where @@ -67,6 +67,22 @@ where Ok((t, deserializer.finalize()?)) } +/// Conveniently deserialize a message of type `T` from a byte slice with a Crc. The unused portion (if any) +/// of the byte slice is not returned. +/// +/// See the `de_flavors::crc` module for the complete set of functions. +/// +#[cfg(feature = "use-crc")] +pub use flavors::crc::from_bytes_u32 as from_bytes_crc32; + +/// Conveniently deserialize a message of type `T` from a byte slice with a Crc. The unused portion (if any) +/// of the byte slice is returned for further usage +/// +/// See the `de_flavors::crc` module for the complete set of functions. +/// +#[cfg(feature = "use-crc")] +pub use flavors::crc::take_from_bytes_u32 as take_from_bytes_crc32; + //////////////////////////////////////////////////////////////////////////////// #[cfg(feature = "heapless")] diff --git a/src/lib.rs b/src/lib.rs index 96c6422..3688e37 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -91,6 +91,21 @@ pub use ser::{to_stdvec, to_stdvec_cobs}; #[cfg(feature = "alloc")] pub use ser::{to_allocvec, to_allocvec_cobs}; +#[cfg(feature = "use-crc")] +pub use { + de::{from_bytes_crc32, take_from_bytes_crc32}, + ser::to_slice_crc32, +}; + +#[cfg(all(feature = "use-crc", feature = "heapless"))] +pub use ser::to_vec_crc32; + +#[cfg(all(feature = "use-crc", feature = "use-std"))] +pub use ser::to_stdvec_crc32; + +#[cfg(all(feature = "use-crc", feature = "alloc"))] +pub use ser::to_allocvec_crc32; + #[cfg(test)] mod test { #[test] diff --git a/src/ser/flavors.rs b/src/ser/flavors.rs index bcd8dbc..1d77e79 100644 --- a/src/ser/flavors.rs +++ b/src/ser/flavors.rs @@ -101,6 +101,9 @@ pub use std_vec::*; #[cfg(feature = "alloc")] pub use alloc_vec::*; +#[cfg(feature = "alloc")] +extern crate alloc; + /// The serialization Flavor trait /// /// This is used as the primary way to encode serialized data into some kind of buffer, @@ -237,7 +240,7 @@ mod heapless_vec { } } - impl<'a, const B: usize> Flavor for HVec { + impl Flavor for HVec { type Output = Vec; #[inline(always)] @@ -381,7 +384,7 @@ where } } -impl<'a, B> Flavor for Cobs +impl Flavor for Cobs where B: Flavor + IndexMut, { @@ -412,6 +415,130 @@ where } } +//////////////////////////////////////// +// CRC +//////////////////////////////////////// + +/// This Cyclic Redundancy Check flavor applies [the CRC crate's `Algorithm`](https://docs.rs/crc/latest/crc/struct.Algorithm.html) struct on +/// the serialized data. The output of this flavor receives the CRC appended to the bytes. +/// +/// CRCs are used for error detection when reading data back. +/// +/// The `crc` feature requires enabling to use this module. +/// +/// More on CRCs: . +#[cfg(feature = "use-crc")] +pub mod crc { + use crc::Digest; + use crc::Width; + use serde::Serialize; + + #[cfg(feature = "alloc")] + use super::alloc; + use super::Flavor; + use super::Slice; + + use crate::serialize_with_flavor; + use crate::Result; + use paste::paste; + + /// Manages CRC modifications as a flavor. + pub struct CrcModifier<'a, B, W> + where + B: Flavor, + W: Width, + { + flav: B, + digest: Digest<'a, W>, + } + + impl<'a, B, W> CrcModifier<'a, B, W> + where + B: Flavor, + W: Width, + { + /// Create a new CRC modifier Flavor. + pub fn new(bee: B, digest: Digest<'a, W>) -> Self { + Self { flav: bee, digest } + } + } + + macro_rules! impl_flavor { + ($( $int:ty ),*) => { + $( + paste! { + impl<'a, B> Flavor for CrcModifier<'a, B, $int> + where + B: Flavor, + { + type Output = ::Output; + + #[inline(always)] + fn try_push(&mut self, data: u8) -> Result<()> { + self.digest.update(&[data]); + self.flav.try_push(data) + } + + fn finalize(mut self) -> Result { + let crc = self.digest.finalize(); + for byte in crc.to_le_bytes() { + self.flav.try_push(byte)?; + } + self.flav.finalize() + } + } + + /// Serialize a `T` to the given slice, with the resulting slice containing + /// data followed by a CRC. The CRC bytes are included in the output buffer. + /// + /// When successful, this function returns the slice containing the + /// serialized and encoded message. + pub fn []<'a, 'b, T>( + value: &'b T, + buf: &'a mut [u8], + digest: Digest<'a, $int>, + ) -> Result<&'a mut [u8]> + where + T: Serialize + ?Sized, + { + serialize_with_flavor(value, CrcModifier::new(Slice::new(buf), digest)) + } + + /// Serialize a `T` to a `heapless::Vec`, with the `Vec` containing + /// data followed by a CRC. The CRC bytes are included in the output `Vec`. + /// Requires the (default) `heapless` feature. + #[cfg(feature = "heapless")] + pub fn []<'a, T, const B: usize>( + value: &T, + digest: Digest<'a, $int>, + ) -> Result> + where + T: Serialize + ?Sized, + { + use super::HVec; + + serialize_with_flavor(value, CrcModifier::new(HVec::default(), digest)) + } + + /// Serialize a `T` to a `heapless::Vec`, with the `Vec` containing + /// data followed by a CRC. The CRC bytes are included in the output `Vec`. + #[cfg(feature = "alloc")] + pub fn []<'a, T>(value: &T, digest: Digest<'a, $int>) -> Result> + where + T: Serialize + ?Sized, + { + use super::AllocVec; + + serialize_with_flavor(value, CrcModifier::new(AllocVec::new(), digest)) + } + } + )* + }; + } + + impl_flavor![u8, u16, u32, u64, u128]; +} + /// The `Size` flavor is a measurement flavor, which accumulates the number of bytes needed to /// serialize the data. /// diff --git a/src/ser/mod.rs b/src/ser/mod.rs index 95d330e..7d37a83 100644 --- a/src/ser/mod.rs +++ b/src/ser/mod.rs @@ -237,6 +237,107 @@ where ) } +/// Conveniently serialize a `T` to the given slice, with the resulting slice containing +/// data followed by a 32-bit CRC. The CRC bytes are included in the output buffer. +/// +/// When successful, this function returns the slice containing the +/// serialized and encoded message. +/// +/// ## Example +/// +/// ```rust +/// use crc::{Crc, CRC_32_ISCSI}; +/// +/// let mut buf = [0; 9]; +/// +/// let data: &[u8] = &[0x01, 0x00, 0x20, 0x30]; +/// let crc = Crc::::new(&CRC_32_ISCSI); +/// let used = postcard::to_slice_crc32(data, &mut buf, crc.digest()).unwrap(); +/// assert_eq!(used, &[0x04, 0x01, 0x00, 0x20, 0x30, 0x8E, 0xC8, 0x1A, 0x37]); +/// ``` +/// +/// See the `ser_flavors::crc` module for the complete set of functions. +/// +#[cfg(feature = "use-crc")] +pub use flavors::crc::to_slice_u32 as to_slice_crc32; + +/// Conveniently serialize a `T` to a `heapless::Vec`, with the `Vec` containing +/// data followed by a 32-bit CRC. The CRC bytes are included in the output `Vec`. +/// Requires the (default) `heapless` feature. +/// +/// ## Example +/// +/// ```rust +/// use crc::{Crc, CRC_32_ISCSI}; +/// use heapless::Vec; +/// use core::ops::Deref; +/// +/// // NOTE: postcard handles `&[u8]` and `&[u8; N]` differently. +/// let data: &[u8] = &[0x01u8, 0x00, 0x20, 0x30]; +/// let crc = Crc::::new(&CRC_32_ISCSI); +/// let ser: Vec = postcard::to_vec_crc32(data, crc.digest()).unwrap(); +/// assert_eq!(ser.deref(), &[0x04, 0x01, 0x00, 0x20, 0x30, 0x8E, 0xC8, 0x1A, 0x37]); +/// +/// let data: &[u8; 4] = &[0x01u8, 0x00, 0x20, 0x30]; +/// let ser: Vec = postcard::to_vec_crc32(data, crc.digest()).unwrap(); +/// assert_eq!(ser.deref(), &[0x01, 0x00, 0x20, 0x30, 0xCC, 0x4B, 0x4A, 0xDA]); +/// ``` +/// +/// See the `ser_flavors::crc` module for the complete set of functions. +/// +#[cfg(all(feature = "use-crc", feature = "heapless"))] +pub use flavors::crc::to_vec_u32 as to_vec_crc32; + +/// Conveniently serialize a `T` to a `heapless::Vec`, with the `Vec` containing +/// data followed by a 32-bit CRC. The CRC bytes are included in the output `Vec`. +/// +/// ## Example +/// +/// ```rust +/// use crc::{Crc, CRC_32_ISCSI}; +/// use core::ops::Deref; +/// +/// // NOTE: postcard handles `&[u8]` and `&[u8; N]` differently. +/// let data: &[u8] = &[0x01u8, 0x00, 0x20, 0x30]; +/// let crc = Crc::::new(&CRC_32_ISCSI); +/// let ser: Vec = postcard::to_stdvec_crc32(data, crc.digest()).unwrap(); +/// assert_eq!(ser.deref(), &[0x04, 0x01, 0x00, 0x20, 0x30, 0x8E, 0xC8, 0x1A, 0x37]); +/// +/// let data: &[u8; 4] = &[0x01u8, 0x00, 0x20, 0x30]; +/// let ser: Vec = postcard::to_stdvec_crc32(data, crc.digest()).unwrap(); +/// assert_eq!(ser.deref(), &[0x01, 0x00, 0x20, 0x30, 0xCC, 0x4B, 0x4A, 0xDA]); +/// ``` +/// +/// See the `ser_flavors::crc` module for the complete set of functions. +/// +#[cfg(all(feature = "use-crc", feature = "use-std"))] +pub use flavors::crc::to_allocvec_u32 as to_stdvec_crc32; + +/// Conveniently serialize a `T` to a `heapless::Vec`, with the `Vec` containing +/// data followed by a 32-bit CRC. The CRC bytes are included in the output `Vec`. +/// +/// ## Example +/// +/// ```rust +/// use crc::{Crc, CRC_32_ISCSI}; +/// use core::ops::Deref; +/// +/// // NOTE: postcard handles `&[u8]` and `&[u8; N]` differently. +/// let data: &[u8] = &[0x01u8, 0x00, 0x20, 0x30]; +/// let crc = Crc::::new(&CRC_32_ISCSI); +/// let ser: Vec = postcard::to_allocvec_crc32(data, crc.digest()).unwrap(); +/// assert_eq!(ser.deref(), &[0x04, 0x01, 0x00, 0x20, 0x30, 0x8E, 0xC8, 0x1A, 0x37]); +/// +/// let data: &[u8; 4] = &[0x01u8, 0x00, 0x20, 0x30]; +/// let ser: Vec = postcard::to_allocvec_crc32(data, crc.digest()).unwrap(); +/// assert_eq!(ser.deref(), &[0x01, 0x00, 0x20, 0x30, 0xCC, 0x4B, 0x4A, 0xDA]); +/// ``` +/// +/// See the `ser_flavors::crc` module for the complete set of functions. +/// +#[cfg(all(feature = "use-crc", feature = "alloc"))] +pub use flavors::crc::to_allocvec_u32 as to_allocvec_crc32; + /// `serialize_with_flavor()` has three generic parameters, `T, F, O`. /// /// * `T`: This is the type that is being serialized diff --git a/tests/crc.rs b/tests/crc.rs new file mode 100644 index 0000000..e0f274d --- /dev/null +++ b/tests/crc.rs @@ -0,0 +1,39 @@ +#[test] +#[cfg(feature = "use-crc")] +fn test_crc() { + use crc::{Crc, CRC_32_ISCSI}; + + let data: &[u8] = &[0x01, 0x00, 0x20, 0x30]; + let buffer = &mut [0u8; 32]; + let crc = Crc::::new(&CRC_32_ISCSI); + let digest = crc.digest(); + let res = postcard::to_slice_crc32(data, buffer, digest).unwrap(); + assert_eq!(res, &[0x04, 0x01, 0x00, 0x20, 0x30, 0x8E, 0xC8, 0x1A, 0x37]); + + let digest = crc.digest(); + let res = postcard::take_from_bytes_crc32::<[u8; 5]>(&res, digest).unwrap(); + + let expected_bytes = [0x04, 0x01, 0x00, 0x20, 0x30]; + let remaining_bytes = []; + assert_eq!(res, (expected_bytes, remaining_bytes.as_slice())); +} + +#[test] +#[cfg(feature = "use-crc")] +fn test_crc_8() { + use crc::{Crc, CRC_8_SMBUS}; + + let data: &[u8] = &[0x01, 0x00, 0x20, 0x30]; + let buffer = &mut [0u8; 32]; + let crc = Crc::::new(&CRC_8_SMBUS); + let digest = crc.digest(); + let res = postcard::ser_flavors::crc::to_slice_u8(data, buffer, digest).unwrap(); + assert_eq!(res, &[0x04, 0x01, 0x00, 0x20, 0x30, 167]); + + let digest = crc.digest(); + let res = postcard::de_flavors::crc::take_from_bytes_u8::<[u8; 5]>(&res, digest).unwrap(); + + let expected_bytes = [0x04, 0x01, 0x00, 0x20, 0x30]; + let remaining_bytes = []; + assert_eq!(res, (expected_bytes, remaining_bytes.as_slice())); +}