Skip to content

Commit

Permalink
Adding support for crc8, crc16 and crc32 bit-reflected simd, x86 targets
Browse files Browse the repository at this point in the history
  • Loading branch information
valaphee committed Mar 22, 2024
1 parent 5e38fdd commit 4b9e109
Show file tree
Hide file tree
Showing 12 changed files with 456 additions and 14 deletions.
12 changes: 11 additions & 1 deletion benches/bench.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,17 @@ pub const BLUETOOTH: Crc<u8> = Crc::<u8>::new(&CRC_8_BLUETOOTH);
pub const BLUETOOTH_SLICE16: Crc<Slice16<u8>> = Crc::<Slice16<u8>>::new(&CRC_8_BLUETOOTH);
pub const BLUETOOTH_BYTEWISE: Crc<Bytewise<u8>> = Crc::<Bytewise<u8>>::new(&CRC_8_BLUETOOTH);
pub const BLUETOOTH_NOLOOKUP: Crc<NoTable<u8>> = Crc::<NoTable<u8>>::new(&CRC_8_BLUETOOTH);
pub const BLUETOOTH_SIMD: Crc<Simd<u8>> = Crc::<Simd<u8>>::new(&CRC_8_BLUETOOTH);
pub const X25: Crc<u16> = Crc::<u16>::new(&CRC_16_IBM_SDLC);
pub const X25_SLICE16: Crc<Slice16<u16>> = Crc::<Slice16<u16>>::new(&CRC_16_IBM_SDLC);
pub const X25_BYTEWISE: Crc<Bytewise<u16>> = Crc::<Bytewise<u16>>::new(&CRC_16_IBM_SDLC);
pub const X25_NOLOOKUP: Crc<NoTable<u16>> = Crc::<NoTable<u16>>::new(&CRC_16_IBM_SDLC);
pub const X25_SIMD: Crc<Simd<u16>> = Crc::<Simd<u16>>::new(&CRC_16_IBM_SDLC);
pub const ISCSI: Crc<u32> = Crc::<u32>::new(&CRC_32_ISCSI);
pub const ISCSI_SLICE16: Crc<Slice16<u32>> = Crc::<Slice16<u32>>::new(&CRC_32_ISCSI);
pub const ISCSI_BYTEWISE: Crc<Bytewise<u32>> = Crc::<Bytewise<u32>>::new(&CRC_32_ISCSI);
pub const ISCSI_NOLOOKUP: Crc<NoTable<u32>> = Crc::<NoTable<u32>>::new(&CRC_32_ISCSI);
pub const ISCSI_SIMD: Crc<Simd<u32>> = Crc::<Simd<u32>>::new(&CRC_32_ISCSI);
pub const GSM_40: Crc<u64> = Crc::<u64>::new(&CRC_40_GSM);
pub const ECMA: Crc<u64> = Crc::<u64>::new(&CRC_64_ECMA_182);
pub const ECMA_SLICE16: Crc<Slice16<u64>> = Crc::<Slice16<u64>>::new(&CRC_64_ECMA_182);
Expand Down Expand Up @@ -51,6 +54,9 @@ fn checksum(c: &mut Criterion) {
})
.bench_function("slice16", |b| {
b.iter(|| BLUETOOTH_SLICE16.checksum(black_box(&bytes)))
})
.bench_function("simd", |b| {
b.iter(|| BLUETOOTH_SIMD.checksum(black_box(&bytes)))
});

c.benchmark_group("crc16")
Expand All @@ -64,7 +70,8 @@ fn checksum(c: &mut Criterion) {
})
.bench_function("slice16", |b| {
b.iter(|| X25_SLICE16.checksum(black_box(&bytes)))
});
})
.bench_function("simd", |b| b.iter(|| X25_SIMD.checksum(black_box(&bytes))));

c.benchmark_group("crc32")
.throughput(Throughput::Bytes(size as u64))
Expand All @@ -77,6 +84,9 @@ fn checksum(c: &mut Criterion) {
})
.bench_function("slice16", |b| {
b.iter(|| ISCSI_SLICE16.checksum(black_box(&bytes)))
})
.bench_function("simd", |b| {
b.iter(|| ISCSI_SIMD.checksum(black_box(&bytes)))
});

c.benchmark_group("crc64")
Expand Down
13 changes: 12 additions & 1 deletion src/crc16.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ use crc_catalog::Algorithm;
mod bytewise;
mod default;
mod nolookup;
mod simd;
mod slice16;

const fn init(algorithm: &Algorithm<u16>, initial: u16) -> u16 {
Expand Down Expand Up @@ -141,7 +142,7 @@ const fn update_slice16(

#[cfg(test)]
mod test {
use crate::{Bytewise, Crc, Implementation, NoTable, Slice16};
use crate::{Bytewise, Crc, Implementation, NoTable, Simd, Slice16};
use crc_catalog::{Algorithm, CRC_16_IBM_SDLC};

#[test]
Expand Down Expand Up @@ -265,11 +266,13 @@ mod test {
for data in data {
let crc_slice16 = Crc::<Slice16<u16>>::new(alg);
let crc_nolookup = Crc::<NoTable<u16>>::new(alg);
let crc_simd = Crc::<Simd<u16>>::new(alg);
let expected = Crc::<Bytewise<u16>>::new(alg).checksum(data.as_bytes());

// Check that doing all at once works as expected
assert_eq!(crc_slice16.checksum(data.as_bytes()), expected);
assert_eq!(crc_nolookup.checksum(data.as_bytes()), expected);
assert_eq!(crc_simd.checksum(data.as_bytes()), expected);

let mut digest = crc_slice16.digest();
digest.update(data.as_bytes());
Expand All @@ -279,6 +282,10 @@ mod test {
digest.update(data.as_bytes());
assert_eq!(digest.finalize(), expected);

let mut digest = crc_simd.digest();
digest.update(data.as_bytes());
assert_eq!(digest.finalize(), expected);

// Check that we didn't break updating from multiple sources
if data.len() > 2 {
let data = data.as_bytes();
Expand All @@ -292,6 +299,10 @@ mod test {
digest.update(data1);
digest.update(data2);
assert_eq!(digest.finalize(), expected);
let mut digest = crc_simd.digest();
digest.update(data1);
digest.update(data2);
assert_eq!(digest.finalize(), expected);
}
}
}
Expand Down
72 changes: 72 additions & 0 deletions src/crc16/simd.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
use crate::crc32::update_simd;
use crate::simd::{SimdValue, SimdValueOps};
use crate::table::{crc16_table_slice_16, crc32_simd_coefficients};
use crate::{Algorithm, Crc, Digest, Simd};

use super::{finalize, init, update_slice16};

impl Crc<Simd<u16>> {
pub const fn new(algorithm: &'static Algorithm<u16>) -> Self {
Self {
algorithm,
table: (
crc16_table_slice_16(algorithm.width, algorithm.poly, algorithm.refin),
unsafe {
// SAFETY: SimdValue is the same as u64x2 and this only changes the representation of 8*u64 to 4*u64x2.
core::mem::transmute(crc32_simd_coefficients(
algorithm.width,
algorithm.poly as u32,
))
},
),
}
}

pub fn checksum(&self, bytes: &[u8]) -> u16 {
let mut crc = init(self.algorithm, self.algorithm.init);
crc = self.update(crc, bytes);
finalize(self.algorithm, crc)
}

fn update(&self, mut crc: u16, bytes: &[u8]) -> u16 {
if !SimdValue::is_supported() || !self.algorithm.refin {
return update_slice16(crc, self.algorithm.refin, &self.table.0, bytes);
}

let (bytes_before, chunks, bytes_after) = unsafe { bytes.align_to::<[SimdValue; 4]>() };
crc = update_slice16(crc, self.algorithm.refin, &self.table.0, bytes_before);
if let Some(first_chunk) = chunks.first() {
// SAFETY: All required features are supported, by checking SimdValue::is_supported.
crc = unsafe { update_simd(crc as u32, &self.table.1, first_chunk, chunks) } as u16;
}
update_slice16(crc, self.algorithm.refin, &self.table.0, bytes_after)
}

pub fn digest(&self) -> Digest<Simd<u16>> {
self.digest_with_initial(self.algorithm.init)
}

/// Construct a `Digest` with a given initial value.
///
/// This overrides the initial value specified by the algorithm.
/// The effects of the algorithm's properties `refin` and `width`
/// are applied to the custom initial value.
pub fn digest_with_initial(&self, initial: u16) -> Digest<Simd<u16>> {
let value = init(self.algorithm, initial);
Digest::new(self, value)
}
}

impl<'a> Digest<'a, Simd<u16>> {
const fn new(crc: &'a Crc<Simd<u16>>, value: u16) -> Self {
Digest { crc, value }
}

pub fn update(&mut self, bytes: &[u8]) {
self.value = self.crc.update(self.value, bytes);
}

pub const fn finalize(self) -> u16 {
finalize(self.crc.algorithm, self.value)
}
}
66 changes: 57 additions & 9 deletions src/crc32.rs
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
use crate::simd::{SimdValue, SimdValueOps};
use crate::util::crc32;
use crc_catalog::Algorithm;

mod bytewise;
mod default;
mod nolookup;
mod simd;
mod slice16;

// init is shared between all impls
Expand Down Expand Up @@ -150,9 +152,45 @@ const fn update_slice16(
crc
}

#[target_feature(enable = "sse2", enable = "sse4.1", enable = "pclmulqdq")]
pub(crate) unsafe fn update_simd(
crc: u32,
coefficients: &[SimdValue; 4],
first_chunk: &[SimdValue; 4],
chunks: &[[SimdValue; 4]],
) -> u32 {
let mut x4 = *first_chunk;

// Apply initial crc value
x4[0] = x4[0].xor(crc as u64);

// Iteratively Fold by 4:
let k1_k2 = coefficients[0];
for chunk in chunks {
for (x, value) in x4.iter_mut().zip(chunk.iter()) {
*x = x.fold_16(k1_k2, *value)
}
}

// Iteratively Fold by 1:
let k3_k4 = coefficients[1];
let mut x = x4[0].fold_16(k3_k4, x4[1]);
x = x.fold_16(k3_k4, x4[2]);
x = x.fold_16(k3_k4, x4[3]);

// Final Reduction of 128-bits
let k5_k6 = coefficients[2];
x = x.fold_8(k3_k4);
x = x.fold_4(k5_k6);

// Barret Reduction
let px_u = coefficients[3];
x.barret_reduction_32(px_u)
}

#[cfg(test)]
mod test {
use crate::{Bytewise, Crc, Implementation, NoTable, Slice16};
use crate::{Bytewise, Crc, Implementation, NoTable, Simd, Slice16};
use crc_catalog::{Algorithm, CRC_32_ISCSI};

#[test]
Expand Down Expand Up @@ -250,14 +288,14 @@ mod test {
#[test]
fn correctness() {
let data: &[&str] = &[
"",
"1",
"1234",
"123456789",
"0123456789ABCDE",
"01234567890ABCDEFGHIJK",
"01234567890ABCDEFGHIJK01234567890ABCDEFGHIJK01234567890ABCDEFGHIJK01234567890ABCDEFGHIJK01234567890ABCDEFGHIJK01234567890ABCDEFGHIJK01234567890ABCDEFGHIJK01234567890ABCDEFGHIJK01234567890ABCDEFGHIJK01234567890ABCDEFGHIJK01234567890ABCDEFGHIJK01234567890ABCDEFGHIJK",
];
"",
"1",
"1234",
"123456789",
"0123456789A",
"01234567890ABCDEFGHIJK",
"01234567890ABCDEFGHIJK01234567890ABCDEFGHIJK01234567890ABCDEFGHIJK01234567890ABCDEFGHIJK01234567890ABCDEFGHIJK01234567890ABCDEFGHIJK01234567890ABCDEFGHIJK01234567890ABCDEFGHIJK01234567890ABCDEFGHIJK01234567890ABCDEFGHIJK01234567890ABCDEFGHIJK01234567890ABCDEFGHIJK",
];

pub const CRC_32_ISCSI_NONREFLEX: Algorithm<u32> = Algorithm {
width: 32,
Expand All @@ -277,11 +315,13 @@ mod test {
for data in data {
let crc_slice16 = Crc::<Slice16<u32>>::new(alg);
let crc_nolookup = Crc::<NoTable<u32>>::new(alg);
let crc_simd = Crc::<Simd<u32>>::new(alg);
let expected = Crc::<Bytewise<u32>>::new(alg).checksum(data.as_bytes());

// Check that doing all at once works as expected
assert_eq!(crc_slice16.checksum(data.as_bytes()), expected);
assert_eq!(crc_nolookup.checksum(data.as_bytes()), expected);
assert_eq!(crc_simd.checksum(data.as_bytes()), expected);

let mut digest = crc_slice16.digest();
digest.update(data.as_bytes());
Expand All @@ -291,6 +331,10 @@ mod test {
digest.update(data.as_bytes());
assert_eq!(digest.finalize(), expected);

let mut digest = crc_simd.digest();
digest.update(data.as_bytes());
assert_eq!(digest.finalize(), expected);

// Check that we didn't break updating from multiple sources
if data.len() > 2 {
let data = data.as_bytes();
Expand All @@ -304,6 +348,10 @@ mod test {
digest.update(data1);
digest.update(data2);
assert_eq!(digest.finalize(), expected);
let mut digest = crc_simd.digest();
digest.update(data1);
digest.update(data2);
assert_eq!(digest.finalize(), expected);
}
}
}
Expand Down
69 changes: 69 additions & 0 deletions src/crc32/simd.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
use crate::simd::{SimdValue, SimdValueOps};
use crate::table::{crc32_simd_coefficients, crc32_table_slice_16};
use crate::{Algorithm, Crc, Digest, Simd};

use super::{finalize, init, update_simd, update_slice16};

impl Crc<Simd<u32>> {
pub const fn new(algorithm: &'static Algorithm<u32>) -> Self {
Self {
algorithm,
table: (
crc32_table_slice_16(algorithm.width, algorithm.poly, algorithm.refin),
unsafe {
// SAFETY: Both represent numbers
core::mem::transmute(crc32_simd_coefficients(algorithm.width, algorithm.poly))
},
),
}
}

pub fn checksum(&self, bytes: &[u8]) -> u32 {
let mut crc = init(self.algorithm, self.algorithm.init);
crc = self.update(crc, bytes);
finalize(self.algorithm, crc)
}

fn update(&self, mut crc: u32, bytes: &[u8]) -> u32 {
if !SimdValue::is_supported() || !self.algorithm.refin {
return update_slice16(crc, self.algorithm.refin, &self.table.0, bytes);
}

// SAFETY: Both represent numbers
let (bytes_before, chunks, bytes_after) = unsafe { bytes.align_to::<[SimdValue; 4]>() };
crc = update_slice16(crc, self.algorithm.refin, &self.table.0, bytes_before);
if let Some(first_chunk) = chunks.first() {
// SAFETY: All required features are supported, by checking SimdValue::is_supported.
crc = unsafe { update_simd(crc, &self.table.1, first_chunk, &chunks[1..]) };
}
update_slice16(crc, self.algorithm.refin, &self.table.0, bytes_after)
}

pub fn digest(&self) -> Digest<Simd<u32>> {
self.digest_with_initial(self.algorithm.init)
}

/// Construct a `Digest` with a given initial value.
///
/// This overrides the initial value specified by the algorithm.
/// The effects of the algorithm's properties `refin` and `width`
/// are applied to the custom initial value.
pub fn digest_with_initial(&self, initial: u32) -> Digest<Simd<u32>> {
let value = init(self.algorithm, initial);
Digest::new(self, value)
}
}

impl<'a> Digest<'a, Simd<u32>> {
const fn new(crc: &'a Crc<Simd<u32>>, value: u32) -> Self {
Digest { crc, value }
}

pub fn update(&mut self, bytes: &[u8]) {
self.value = self.crc.update(self.value, bytes);
}

pub const fn finalize(self) -> u32 {
finalize(self.crc.algorithm, self.value)
}
}
2 changes: 1 addition & 1 deletion src/crc64.rs
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ const fn update_slice16(

#[cfg(test)]
mod test {
use crate::{Bytewise, Crc, Implementation, NoTable, Slice16};
use crate::{Bytewise, Crc, Implementation, NoTable, Simd, Slice16};
use crc_catalog::{Algorithm, CRC_64_ECMA_182};

#[test]
Expand Down
Loading

0 comments on commit 4b9e109

Please sign in to comment.