diff --git a/libcrux-ml-dsa/src/hash_functions.rs b/libcrux-ml-dsa/src/hash_functions.rs index fe32126b..9b0fe321 100644 --- a/libcrux-ml-dsa/src/hash_functions.rs +++ b/libcrux-ml-dsa/src/hash_functions.rs @@ -299,12 +299,14 @@ pub(crate) mod simd256 { impl shake128::XofX4 for Shake128x4 { /// Init the state and absorb 4 blocks in parallel. + #[inline(always)] fn init_absorb(input0: &[u8], input1: &[u8], input2: &[u8], input3: &[u8]) -> Self { let mut state = x4::incremental::init(); x4::incremental::shake128_absorb_final(&mut state, &input0, &input1, &input2, &input3); Self { state } } + #[inline(always)] fn squeeze_first_five_blocks( &mut self, out0: &mut [u8; shake128::FIVE_BLOCKS_SIZE], @@ -321,6 +323,7 @@ pub(crate) mod simd256 { ); } + #[inline(always)] fn squeeze_next_block( &mut self, ) -> ( @@ -353,10 +356,12 @@ pub(crate) mod simd256 { state: portable::KeccakState, } impl shake256::Xof for Shake256 { + #[inline(always)] fn shake256(input: &[u8], out: &mut [u8; OUTPUT_LENGTH]) { portable::shake256(out, input); } + #[inline(always)] fn init_absorb(input: &[u8]) -> Self { let mut state = portable::incremental::shake256_init(); portable::incremental::shake256_absorb_final(&mut state, input); @@ -364,12 +369,14 @@ pub(crate) mod simd256 { Self { state } } + #[inline(always)] fn squeeze_first_block(&mut self) -> [u8; shake256::BLOCK_SIZE] { let mut out = [0u8; shake256::BLOCK_SIZE]; portable::incremental::shake256_squeeze_first_block(&mut self.state, &mut out); out } + #[inline(always)] fn squeeze_next_block(&mut self) -> [u8; shake256::BLOCK_SIZE] { let mut out = [0u8; shake256::BLOCK_SIZE]; portable::incremental::shake256_squeeze_next_block(&mut self.state, &mut out); @@ -383,12 +390,14 @@ pub(crate) mod simd256 { } impl shake256::XofX4 for Shake256x4 { + #[inline(always)] fn init_absorb(input0: &[u8], input1: &[u8], input2: &[u8], input3: &[u8]) -> Self { let mut state = x4::incremental::init(); x4::incremental::shake256_absorb_final(&mut state, &input0, &input1, &input2, &input3); Self { state } } + #[inline(always)] fn squeeze_first_block( &mut self, ) -> ( @@ -412,6 +421,7 @@ pub(crate) mod simd256 { (out0, out1, out2, out3) } + #[inline(always)] fn squeeze_next_block( &mut self, ) -> ( @@ -435,6 +445,7 @@ pub(crate) mod simd256 { (out0, out1, out2, out3) } + #[inline(always)] fn shake256( input0: &[u8], input1: &[u8], diff --git a/libcrux-ml-dsa/src/lib.rs b/libcrux-ml-dsa/src/lib.rs index c83f0ce2..0923457f 100644 --- a/libcrux-ml-dsa/src/lib.rs +++ b/libcrux-ml-dsa/src/lib.rs @@ -1,4 +1,5 @@ #![no_std] +#![deny(unsafe_code)] mod arithmetic; mod constants; diff --git a/libcrux-ml-dsa/src/ml_dsa_generic.rs b/libcrux-ml-dsa/src/ml_dsa_generic.rs index d13930b0..0aea9706 100644 --- a/libcrux-ml-dsa/src/ml_dsa_generic.rs +++ b/libcrux-ml-dsa/src/ml_dsa_generic.rs @@ -36,6 +36,7 @@ pub(crate) struct Signature< } /// Generate a key pair. +#[inline(always)] pub(crate) fn generate_key_pair< SIMDUnit: Operations, Shake128X4: shake128::XofX4, @@ -114,6 +115,7 @@ pub enum SigningError { } #[allow(non_snake_case)] +#[inline(always)] pub(crate) fn sign_pre_hashed< SIMDUnit: Operations, Shake128X4: shake128::XofX4, @@ -174,6 +176,7 @@ pub(crate) fn sign_pre_hashed< } #[allow(non_snake_case)] +#[inline(always)] pub(crate) fn sign< SIMDUnit: Operations, Shake128X4: shake128::XofX4, @@ -231,6 +234,7 @@ pub(crate) fn sign< /// If no `domain_separation_context` is supplied, it is assumed that /// `message` already contains the domain separation. #[allow(non_snake_case)] +#[inline(always)] pub(crate) fn sign_internal< SIMDUnit: Operations, Shake128X4: shake128::XofX4, @@ -437,6 +441,7 @@ pub(crate) fn sign_internal< /// for details on the domain separation for regular ML-DSA. Line /// 23 of Algorithm 4 (and line 18 of Algorithm 5,resp.) describe domain separation for the HashMl-DSA /// variant. +#[inline(always)] fn derive_message_representative( verification_key_hash: [u8; 64], domain_separation_context: Option, @@ -463,6 +468,7 @@ fn derive_message_representative( /// If no `domain_separation_context` is supplied, it is assumed that /// `message` already contains the domain separation. #[allow(non_snake_case)] +#[inline(always)] pub(crate) fn verify_internal< SIMDUnit: Operations, Shake128X4: shake128::XofX4, @@ -563,6 +569,7 @@ pub(crate) fn verify_internal< } #[allow(non_snake_case)] +#[inline(always)] pub(crate) fn verify< SIMDUnit: Operations, Shake128X4: shake128::XofX4, @@ -612,6 +619,7 @@ pub(crate) fn verify< } #[allow(non_snake_case)] +#[inline(always)] pub(crate) fn verify_pre_hashed< SIMDUnit: Operations, Shake128X4: shake128::XofX4, diff --git a/libcrux-ml-dsa/src/ml_dsa_generic/instantiations.rs b/libcrux-ml-dsa/src/ml_dsa_generic/instantiations.rs index 1718f6c0..60507636 100644 --- a/libcrux-ml-dsa/src/ml_dsa_generic/instantiations.rs +++ b/libcrux-ml-dsa/src/ml_dsa_generic/instantiations.rs @@ -305,12 +305,7 @@ instantiate! {portable, // AVX2 generic implementation. #[cfg(feature = "simd256")] -instantiate! {avx2, - crate::simd::avx2::AVX2SIMDUnit, - crate::hash_functions::simd256::Shake128x4, - crate::hash_functions::simd256::Shake256, - crate::hash_functions::simd256::Shake256x4 -} +pub mod avx2; // NEON generic implementation. #[cfg(feature = "simd128")] diff --git a/libcrux-ml-dsa/src/ml_dsa_generic/instantiations/avx2.rs b/libcrux-ml-dsa/src/ml_dsa_generic/instantiations/avx2.rs new file mode 100644 index 00000000..716fd2f5 --- /dev/null +++ b/libcrux-ml-dsa/src/ml_dsa_generic/instantiations/avx2.rs @@ -0,0 +1,584 @@ +use crate::{ + constants::*, + ml_dsa_generic::{SigningError, VerificationError}, + pre_hash::SHAKE128_PH, + types::*, +}; + +mod avx2_feature { + use super::*; + + /// Generate key pair. + #[target_feature(enable = "avx2")] + #[allow(unsafe_code)] + pub(super) unsafe fn generate_key_pair< + const ROWS_IN_A: usize, + const COLUMNS_IN_A: usize, + const ETA: usize, + const ERROR_RING_ELEMENT_SIZE: usize, + const SIGNING_KEY_SIZE: usize, + const VERIFICATION_KEY_SIZE: usize, + >( + randomness: [u8; KEY_GENERATION_RANDOMNESS_SIZE], + ) -> ([u8; SIGNING_KEY_SIZE], [u8; VERIFICATION_KEY_SIZE]) { + crate::ml_dsa_generic::generate_key_pair::< + crate::simd::avx2::AVX2SIMDUnit, + crate::hash_functions::simd256::Shake128x4, + crate::hash_functions::simd256::Shake256, + crate::hash_functions::simd256::Shake256x4, + ROWS_IN_A, + COLUMNS_IN_A, + ETA, + ERROR_RING_ELEMENT_SIZE, + SIGNING_KEY_SIZE, + VERIFICATION_KEY_SIZE, + >(randomness) + } + + /// Sign. + #[target_feature(enable = "avx2")] + #[allow(unsafe_code)] + pub(super) unsafe fn sign< + const ROWS_IN_A: usize, + const COLUMNS_IN_A: usize, + const ETA: usize, + const ERROR_RING_ELEMENT_SIZE: usize, + const GAMMA1_EXPONENT: usize, + const GAMMA2: i32, + const COMMITMENT_RING_ELEMENT_SIZE: usize, + const COMMITMENT_VECTOR_SIZE: usize, + const COMMITMENT_HASH_SIZE: usize, + const ONES_IN_VERIFIER_CHALLENGE: usize, + const MAX_ONES_IN_HINT: usize, + const GAMMA1_RING_ELEMENT_SIZE: usize, + const SIGNING_KEY_SIZE: usize, + const SIGNATURE_SIZE: usize, + >( + signing_key: &[u8; SIGNING_KEY_SIZE], + message: &[u8], + context: &[u8], + randomness: [u8; SIGNING_RANDOMNESS_SIZE], + ) -> Result, SigningError> { + crate::ml_dsa_generic::sign::< + crate::simd::avx2::AVX2SIMDUnit, + crate::hash_functions::simd256::Shake128x4, + crate::hash_functions::simd256::Shake256, + crate::hash_functions::simd256::Shake256x4, + ROWS_IN_A, + COLUMNS_IN_A, + ETA, + ERROR_RING_ELEMENT_SIZE, + GAMMA1_EXPONENT, + GAMMA2, + COMMITMENT_RING_ELEMENT_SIZE, + COMMITMENT_VECTOR_SIZE, + COMMITMENT_HASH_SIZE, + ONES_IN_VERIFIER_CHALLENGE, + MAX_ONES_IN_HINT, + GAMMA1_RING_ELEMENT_SIZE, + SIGNING_KEY_SIZE, + SIGNATURE_SIZE, + >(&signing_key, message, context, randomness) + } + + /// Sign (internal API) + #[cfg(feature = "acvp")] + #[target_feature(enable = "avx2")] + #[allow(unsafe_code)] + pub(super) unsafe fn sign_internal< + const ROWS_IN_A: usize, + const COLUMNS_IN_A: usize, + const ETA: usize, + const ERROR_RING_ELEMENT_SIZE: usize, + const GAMMA1_EXPONENT: usize, + const GAMMA2: i32, + const COMMITMENT_RING_ELEMENT_SIZE: usize, + const COMMITMENT_VECTOR_SIZE: usize, + const COMMITMENT_HASH_SIZE: usize, + const ONES_IN_VERIFIER_CHALLENGE: usize, + const MAX_ONES_IN_HINT: usize, + const GAMMA1_RING_ELEMENT_SIZE: usize, + const SIGNING_KEY_SIZE: usize, + const SIGNATURE_SIZE: usize, + >( + signing_key: &[u8; SIGNING_KEY_SIZE], + message: &[u8], + randomness: [u8; SIGNING_RANDOMNESS_SIZE], + ) -> Result, SigningError> { + crate::ml_dsa_generic::sign_internal::< + crate::simd::avx2::AVX2SIMDUnit, + crate::hash_functions::simd256::Shake128x4, + crate::hash_functions::simd256::Shake256, + crate::hash_functions::simd256::Shake256x4, + ROWS_IN_A, + COLUMNS_IN_A, + ETA, + ERROR_RING_ELEMENT_SIZE, + GAMMA1_EXPONENT, + GAMMA2, + COMMITMENT_RING_ELEMENT_SIZE, + COMMITMENT_VECTOR_SIZE, + COMMITMENT_HASH_SIZE, + ONES_IN_VERIFIER_CHALLENGE, + MAX_ONES_IN_HINT, + GAMMA1_RING_ELEMENT_SIZE, + SIGNING_KEY_SIZE, + SIGNATURE_SIZE, + >(&signing_key, message, None, randomness) + } + + /// Sign (pre-hashed). + #[target_feature(enable = "avx2")] + #[allow(unsafe_code)] + pub(super) unsafe fn sign_pre_hashed_shake128< + const ROWS_IN_A: usize, + const COLUMNS_IN_A: usize, + const ETA: usize, + const ERROR_RING_ELEMENT_SIZE: usize, + const GAMMA1_EXPONENT: usize, + const GAMMA2: i32, + const COMMITMENT_RING_ELEMENT_SIZE: usize, + const COMMITMENT_VECTOR_SIZE: usize, + const COMMITMENT_HASH_SIZE: usize, + const ONES_IN_VERIFIER_CHALLENGE: usize, + const MAX_ONES_IN_HINT: usize, + const GAMMA1_RING_ELEMENT_SIZE: usize, + const SIGNING_KEY_SIZE: usize, + const SIGNATURE_SIZE: usize, + >( + signing_key: &[u8; SIGNING_KEY_SIZE], + message: &[u8], + context: &[u8], + randomness: [u8; SIGNING_RANDOMNESS_SIZE], + ) -> Result, SigningError> { + crate::ml_dsa_generic::sign_pre_hashed::< + crate::simd::avx2::AVX2SIMDUnit, + crate::hash_functions::simd256::Shake128x4, + crate::hash_functions::simd256::Shake256, + crate::hash_functions::simd256::Shake256x4, + SHAKE128_PH, + 256, + ROWS_IN_A, + COLUMNS_IN_A, + ETA, + ERROR_RING_ELEMENT_SIZE, + GAMMA1_EXPONENT, + GAMMA2, + COMMITMENT_RING_ELEMENT_SIZE, + COMMITMENT_VECTOR_SIZE, + COMMITMENT_HASH_SIZE, + ONES_IN_VERIFIER_CHALLENGE, + MAX_ONES_IN_HINT, + GAMMA1_RING_ELEMENT_SIZE, + SIGNING_KEY_SIZE, + SIGNATURE_SIZE, + >(&signing_key, message, context, randomness) + } + + /// Verify. + #[target_feature(enable = "avx2")] + #[allow(unsafe_code)] + pub(super) unsafe fn verify< + const ROWS_IN_A: usize, + const COLUMNS_IN_A: usize, + const SIGNATURE_SIZE: usize, + const VERIFICATION_KEY_SIZE: usize, + const GAMMA1_EXPONENT: usize, + const GAMMA1_RING_ELEMENT_SIZE: usize, + const GAMMA2: i32, + const BETA: i32, + const COMMITMENT_RING_ELEMENT_SIZE: usize, + const COMMITMENT_VECTOR_SIZE: usize, + const COMMITMENT_HASH_SIZE: usize, + const ONES_IN_VERIFIER_CHALLENGE: usize, + const MAX_ONES_IN_HINT: usize, + >( + verification_key: &[u8; VERIFICATION_KEY_SIZE], + message: &[u8], + context: &[u8], + signature: &[u8; SIGNATURE_SIZE], + ) -> Result<(), VerificationError> { + crate::ml_dsa_generic::verify::< + crate::simd::avx2::AVX2SIMDUnit, + crate::hash_functions::simd256::Shake128x4, + crate::hash_functions::simd256::Shake256, + ROWS_IN_A, + COLUMNS_IN_A, + SIGNATURE_SIZE, + VERIFICATION_KEY_SIZE, + GAMMA1_EXPONENT, + GAMMA1_RING_ELEMENT_SIZE, + GAMMA2, + BETA, + COMMITMENT_RING_ELEMENT_SIZE, + COMMITMENT_VECTOR_SIZE, + COMMITMENT_HASH_SIZE, + ONES_IN_VERIFIER_CHALLENGE, + MAX_ONES_IN_HINT, + >(verification_key, message, context, signature) + } + + /// Verify (internal API). + #[cfg(feature = "acvp")] + #[target_feature(enable = "avx2")] + #[allow(unsafe_code)] + pub(super) unsafe fn verify_internal< + const ROWS_IN_A: usize, + const COLUMNS_IN_A: usize, + const SIGNATURE_SIZE: usize, + const VERIFICATION_KEY_SIZE: usize, + const GAMMA1_EXPONENT: usize, + const GAMMA1_RING_ELEMENT_SIZE: usize, + const GAMMA2: i32, + const BETA: i32, + const COMMITMENT_RING_ELEMENT_SIZE: usize, + const COMMITMENT_VECTOR_SIZE: usize, + const COMMITMENT_HASH_SIZE: usize, + const ONES_IN_VERIFIER_CHALLENGE: usize, + const MAX_ONES_IN_HINT: usize, + >( + verification_key: &[u8; VERIFICATION_KEY_SIZE], + message: &[u8], + signature: &[u8; SIGNATURE_SIZE], + ) -> Result<(), VerificationError> { + crate::ml_dsa_generic::verify_internal::< + crate::simd::avx2::AVX2SIMDUnit, + crate::hash_functions::simd256::Shake128x4, + crate::hash_functions::simd256::Shake256, + ROWS_IN_A, + COLUMNS_IN_A, + SIGNATURE_SIZE, + VERIFICATION_KEY_SIZE, + GAMMA1_EXPONENT, + GAMMA1_RING_ELEMENT_SIZE, + GAMMA2, + BETA, + COMMITMENT_RING_ELEMENT_SIZE, + COMMITMENT_VECTOR_SIZE, + COMMITMENT_HASH_SIZE, + ONES_IN_VERIFIER_CHALLENGE, + MAX_ONES_IN_HINT, + >(verification_key, message, None, signature) + } + + /// Verify (pre-hashed with SHAKE-128). + #[target_feature(enable = "avx2")] + #[allow(unsafe_code)] + pub(super) unsafe fn verify_pre_hashed_shake128< + const ROWS_IN_A: usize, + const COLUMNS_IN_A: usize, + const SIGNATURE_SIZE: usize, + const VERIFICATION_KEY_SIZE: usize, + const GAMMA1_EXPONENT: usize, + const GAMMA1_RING_ELEMENT_SIZE: usize, + const GAMMA2: i32, + const BETA: i32, + const COMMITMENT_RING_ELEMENT_SIZE: usize, + const COMMITMENT_VECTOR_SIZE: usize, + const COMMITMENT_HASH_SIZE: usize, + const ONES_IN_VERIFIER_CHALLENGE: usize, + const MAX_ONES_IN_HINT: usize, + >( + verification_key: &[u8; VERIFICATION_KEY_SIZE], + message: &[u8], + context: &[u8], + signature: &[u8; SIGNATURE_SIZE], + ) -> Result<(), VerificationError> { + crate::ml_dsa_generic::verify_pre_hashed::< + crate::simd::avx2::AVX2SIMDUnit, + crate::hash_functions::simd256::Shake128x4, + crate::hash_functions::simd256::Shake256, + SHAKE128_PH, + 256, + ROWS_IN_A, + COLUMNS_IN_A, + SIGNATURE_SIZE, + VERIFICATION_KEY_SIZE, + GAMMA1_EXPONENT, + GAMMA1_RING_ELEMENT_SIZE, + GAMMA2, + BETA, + COMMITMENT_RING_ELEMENT_SIZE, + COMMITMENT_VECTOR_SIZE, + COMMITMENT_HASH_SIZE, + ONES_IN_VERIFIER_CHALLENGE, + MAX_ONES_IN_HINT, + >(verification_key, message, context, signature) + } +} + +/// Generate key pair. +#[allow(unsafe_code)] +pub(crate) fn generate_key_pair< + const ROWS_IN_A: usize, + const COLUMNS_IN_A: usize, + const ETA: usize, + const ERROR_RING_ELEMENT_SIZE: usize, + const SIGNING_KEY_SIZE: usize, + const VERIFICATION_KEY_SIZE: usize, +>( + randomness: [u8; KEY_GENERATION_RANDOMNESS_SIZE], +) -> ([u8; SIGNING_KEY_SIZE], [u8; VERIFICATION_KEY_SIZE]) { + unsafe { + avx2_feature::generate_key_pair::< + ROWS_IN_A, + COLUMNS_IN_A, + ETA, + ERROR_RING_ELEMENT_SIZE, + SIGNING_KEY_SIZE, + VERIFICATION_KEY_SIZE, + >(randomness) + } +} + +/// Sign. +#[allow(unsafe_code)] +pub(crate) fn sign< + const ROWS_IN_A: usize, + const COLUMNS_IN_A: usize, + const ETA: usize, + const ERROR_RING_ELEMENT_SIZE: usize, + const GAMMA1_EXPONENT: usize, + const GAMMA2: i32, + const COMMITMENT_RING_ELEMENT_SIZE: usize, + const COMMITMENT_VECTOR_SIZE: usize, + const COMMITMENT_HASH_SIZE: usize, + const ONES_IN_VERIFIER_CHALLENGE: usize, + const MAX_ONES_IN_HINT: usize, + const GAMMA1_RING_ELEMENT_SIZE: usize, + const SIGNING_KEY_SIZE: usize, + const SIGNATURE_SIZE: usize, +>( + signing_key: &[u8; SIGNING_KEY_SIZE], + message: &[u8], + context: &[u8], + randomness: [u8; SIGNING_RANDOMNESS_SIZE], +) -> Result, SigningError> { + unsafe { + avx2_feature::sign::< + ROWS_IN_A, + COLUMNS_IN_A, + ETA, + ERROR_RING_ELEMENT_SIZE, + GAMMA1_EXPONENT, + GAMMA2, + COMMITMENT_RING_ELEMENT_SIZE, + COMMITMENT_VECTOR_SIZE, + COMMITMENT_HASH_SIZE, + ONES_IN_VERIFIER_CHALLENGE, + MAX_ONES_IN_HINT, + GAMMA1_RING_ELEMENT_SIZE, + SIGNING_KEY_SIZE, + SIGNATURE_SIZE, + >(signing_key, message, context, randomness) + } +} + +/// Sign (internal API) +#[cfg(feature = "acvp")] +#[allow(unsafe_code)] +pub(crate) fn sign_internal< + const ROWS_IN_A: usize, + const COLUMNS_IN_A: usize, + const ETA: usize, + const ERROR_RING_ELEMENT_SIZE: usize, + const GAMMA1_EXPONENT: usize, + const GAMMA2: i32, + const COMMITMENT_RING_ELEMENT_SIZE: usize, + const COMMITMENT_VECTOR_SIZE: usize, + const COMMITMENT_HASH_SIZE: usize, + const ONES_IN_VERIFIER_CHALLENGE: usize, + const MAX_ONES_IN_HINT: usize, + const GAMMA1_RING_ELEMENT_SIZE: usize, + const SIGNING_KEY_SIZE: usize, + const SIGNATURE_SIZE: usize, +>( + signing_key: &[u8; SIGNING_KEY_SIZE], + message: &[u8], + randomness: [u8; SIGNING_RANDOMNESS_SIZE], +) -> Result, SigningError> { + unsafe { + avx2_feature::sign_internal::< + ROWS_IN_A, + COLUMNS_IN_A, + ETA, + ERROR_RING_ELEMENT_SIZE, + GAMMA1_EXPONENT, + GAMMA2, + COMMITMENT_RING_ELEMENT_SIZE, + COMMITMENT_VECTOR_SIZE, + COMMITMENT_HASH_SIZE, + ONES_IN_VERIFIER_CHALLENGE, + MAX_ONES_IN_HINT, + GAMMA1_RING_ELEMENT_SIZE, + SIGNING_KEY_SIZE, + SIGNATURE_SIZE, + >(signing_key, message, randomness) + } +} + +/// Sign (pre-hashed). +#[allow(unsafe_code)] +pub(crate) fn sign_pre_hashed_shake128< + const ROWS_IN_A: usize, + const COLUMNS_IN_A: usize, + const ETA: usize, + const ERROR_RING_ELEMENT_SIZE: usize, + const GAMMA1_EXPONENT: usize, + const GAMMA2: i32, + const COMMITMENT_RING_ELEMENT_SIZE: usize, + const COMMITMENT_VECTOR_SIZE: usize, + const COMMITMENT_HASH_SIZE: usize, + const ONES_IN_VERIFIER_CHALLENGE: usize, + const MAX_ONES_IN_HINT: usize, + const GAMMA1_RING_ELEMENT_SIZE: usize, + const SIGNING_KEY_SIZE: usize, + const SIGNATURE_SIZE: usize, +>( + signing_key: &[u8; SIGNING_KEY_SIZE], + message: &[u8], + context: &[u8], + randomness: [u8; SIGNING_RANDOMNESS_SIZE], +) -> Result, SigningError> { + unsafe { + avx2_feature::sign_pre_hashed_shake128::< + ROWS_IN_A, + COLUMNS_IN_A, + ETA, + ERROR_RING_ELEMENT_SIZE, + GAMMA1_EXPONENT, + GAMMA2, + COMMITMENT_RING_ELEMENT_SIZE, + COMMITMENT_VECTOR_SIZE, + COMMITMENT_HASH_SIZE, + ONES_IN_VERIFIER_CHALLENGE, + MAX_ONES_IN_HINT, + GAMMA1_RING_ELEMENT_SIZE, + SIGNING_KEY_SIZE, + SIGNATURE_SIZE, + >(signing_key, message, context, randomness) + } +} + +/// Verify. +#[allow(unsafe_code)] +pub(crate) fn verify< + const ROWS_IN_A: usize, + const COLUMNS_IN_A: usize, + const SIGNATURE_SIZE: usize, + const VERIFICATION_KEY_SIZE: usize, + const GAMMA1_EXPONENT: usize, + const GAMMA1_RING_ELEMENT_SIZE: usize, + const GAMMA2: i32, + const BETA: i32, + const COMMITMENT_RING_ELEMENT_SIZE: usize, + const COMMITMENT_VECTOR_SIZE: usize, + const COMMITMENT_HASH_SIZE: usize, + const ONES_IN_VERIFIER_CHALLENGE: usize, + const MAX_ONES_IN_HINT: usize, +>( + verification_key: &[u8; VERIFICATION_KEY_SIZE], + message: &[u8], + context: &[u8], + signature: &[u8; SIGNATURE_SIZE], +) -> Result<(), VerificationError> { + unsafe { + avx2_feature::verify::< + ROWS_IN_A, + COLUMNS_IN_A, + SIGNATURE_SIZE, + VERIFICATION_KEY_SIZE, + GAMMA1_EXPONENT, + GAMMA1_RING_ELEMENT_SIZE, + GAMMA2, + BETA, + COMMITMENT_RING_ELEMENT_SIZE, + COMMITMENT_VECTOR_SIZE, + COMMITMENT_HASH_SIZE, + ONES_IN_VERIFIER_CHALLENGE, + MAX_ONES_IN_HINT, + >(verification_key, message, context, signature) + } +} + +/// Verify (internal API). +#[cfg(feature = "acvp")] +#[allow(unsafe_code)] +pub(crate) fn verify_internal< + const ROWS_IN_A: usize, + const COLUMNS_IN_A: usize, + const SIGNATURE_SIZE: usize, + const VERIFICATION_KEY_SIZE: usize, + const GAMMA1_EXPONENT: usize, + const GAMMA1_RING_ELEMENT_SIZE: usize, + const GAMMA2: i32, + const BETA: i32, + const COMMITMENT_RING_ELEMENT_SIZE: usize, + const COMMITMENT_VECTOR_SIZE: usize, + const COMMITMENT_HASH_SIZE: usize, + const ONES_IN_VERIFIER_CHALLENGE: usize, + const MAX_ONES_IN_HINT: usize, +>( + verification_key: &[u8; VERIFICATION_KEY_SIZE], + message: &[u8], + signature: &[u8; SIGNATURE_SIZE], +) -> Result<(), VerificationError> { + unsafe { + avx2_feature::verify_internal::< + ROWS_IN_A, + COLUMNS_IN_A, + SIGNATURE_SIZE, + VERIFICATION_KEY_SIZE, + GAMMA1_EXPONENT, + GAMMA1_RING_ELEMENT_SIZE, + GAMMA2, + BETA, + COMMITMENT_RING_ELEMENT_SIZE, + COMMITMENT_VECTOR_SIZE, + COMMITMENT_HASH_SIZE, + ONES_IN_VERIFIER_CHALLENGE, + MAX_ONES_IN_HINT, + >(verification_key, message, signature) + } +} + +/// Verify (pre-hashed with SHAKE-128). +#[allow(unsafe_code)] +pub(crate) fn verify_pre_hashed_shake128< + const ROWS_IN_A: usize, + const COLUMNS_IN_A: usize, + const SIGNATURE_SIZE: usize, + const VERIFICATION_KEY_SIZE: usize, + const GAMMA1_EXPONENT: usize, + const GAMMA1_RING_ELEMENT_SIZE: usize, + const GAMMA2: i32, + const BETA: i32, + const COMMITMENT_RING_ELEMENT_SIZE: usize, + const COMMITMENT_VECTOR_SIZE: usize, + const COMMITMENT_HASH_SIZE: usize, + const ONES_IN_VERIFIER_CHALLENGE: usize, + const MAX_ONES_IN_HINT: usize, +>( + verification_key: &[u8; VERIFICATION_KEY_SIZE], + message: &[u8], + context: &[u8], + signature: &[u8; SIGNATURE_SIZE], +) -> Result<(), VerificationError> { + unsafe { + avx2_feature::verify_pre_hashed_shake128::< + ROWS_IN_A, + COLUMNS_IN_A, + SIGNATURE_SIZE, + VERIFICATION_KEY_SIZE, + GAMMA1_EXPONENT, + GAMMA1_RING_ELEMENT_SIZE, + GAMMA2, + BETA, + COMMITMENT_RING_ELEMENT_SIZE, + COMMITMENT_VECTOR_SIZE, + COMMITMENT_HASH_SIZE, + ONES_IN_VERIFIER_CHALLENGE, + MAX_ONES_IN_HINT, + >(verification_key, message, context, signature) + } +} diff --git a/libcrux-ml-dsa/src/sample.rs b/libcrux-ml-dsa/src/sample.rs index dfbb5b55..b9736158 100644 --- a/libcrux-ml-dsa/src/sample.rs +++ b/libcrux-ml-dsa/src/sample.rs @@ -322,6 +322,7 @@ pub(crate) fn sample_four_error_ring_elements< ) } +#[inline(always)] fn update_seed(mut seed: [u8; 66], domain_separator: &mut u16) -> [u8; 66] { seed[64] = *domain_separator as u8; seed[65] = (*domain_separator >> 8) as u8; diff --git a/libcrux-ml-dsa/src/simd/avx2.rs b/libcrux-ml-dsa/src/simd/avx2.rs index 82192638..73a55a0a 100644 --- a/libcrux-ml-dsa/src/simd/avx2.rs +++ b/libcrux-ml-dsa/src/simd/avx2.rs @@ -18,112 +18,136 @@ impl From for AVX2SIMDUnit { } impl Operations for AVX2SIMDUnit { + #[inline(always)] fn ZERO() -> Self { libcrux_intrinsics::avx2::mm256_setzero_si256().into() } + #[inline(always)] fn from_coefficient_array(coefficient_array: &[i32]) -> Self { libcrux_intrinsics::avx2::mm256_loadu_si256_i32(coefficient_array).into() } + #[inline(always)] fn to_coefficient_array(&self) -> [i32; 8] { let mut coefficient_array = [0i32; 8]; libcrux_intrinsics::avx2::mm256_storeu_si256_i32(&mut coefficient_array, self.coefficients); coefficient_array } - + #[inline(always)] fn add(lhs: &Self, rhs: &Self) -> Self { arithmetic::add(lhs.coefficients, rhs.coefficients).into() } - + #[inline(always)] fn subtract(lhs: &Self, rhs: &Self) -> Self { arithmetic::subtract(lhs.coefficients, rhs.coefficients).into() } - + #[inline(always)] fn montgomery_multiply_by_constant(simd_unit: Self, constant: i32) -> Self { arithmetic::montgomery_multiply_by_constant(simd_unit.coefficients, constant).into() } + #[inline(always)] fn montgomery_multiply(lhs: Self, rhs: Self) -> Self { arithmetic::montgomery_multiply(lhs.coefficients, rhs.coefficients).into() } + #[inline(always)] fn shift_left_then_reduce(simd_unit: Self) -> Self { arithmetic::shift_left_then_reduce::(simd_unit.coefficients).into() } + #[inline(always)] fn power2round(simd_unit: Self) -> (Self, Self) { let (lower, upper) = arithmetic::power2round(simd_unit.coefficients); (lower.into(), upper.into()) } + #[inline(always)] fn infinity_norm_exceeds(simd_unit: Self, bound: i32) -> bool { arithmetic::infinity_norm_exceeds(simd_unit.coefficients, bound) } + #[inline(always)] fn decompose(simd_unit: Self) -> (Self, Self) { let (lower, upper) = arithmetic::decompose::(simd_unit.coefficients); (lower.into(), upper.into()) } + #[inline(always)] fn compute_hint(low: Self, high: Self) -> (usize, Self) { let (count, hint) = arithmetic::compute_hint::(low.coefficients, high.coefficients); (count, hint.into()) } + #[inline(always)] fn use_hint(simd_unit: Self, hint: Self) -> Self { arithmetic::use_hint::(simd_unit.coefficients, hint.coefficients).into() } + #[inline(always)] fn rejection_sample_less_than_field_modulus(randomness: &[u8], out: &mut [i32]) -> usize { rejection_sample::less_than_field_modulus::sample(randomness, out) } + #[inline(always)] fn rejection_sample_less_than_eta_equals_2(randomness: &[u8], out: &mut [i32]) -> usize { rejection_sample::less_than_eta::sample::<2>(randomness, out) } + #[inline(always)] fn rejection_sample_less_than_eta_equals_4(randomness: &[u8], out: &mut [i32]) -> usize { rejection_sample::less_than_eta::sample::<4>(randomness, out) } + #[inline(always)] fn gamma1_serialize(simd_unit: Self) -> [u8; OUTPUT_SIZE] { encoding::gamma1::serialize::(simd_unit.coefficients) } + #[inline(always)] fn gamma1_deserialize(serialized: &[u8]) -> Self { encoding::gamma1::deserialize::(serialized).into() } + #[inline(always)] fn commitment_serialize(simd_unit: Self) -> [u8; OUTPUT_SIZE] { encoding::commitment::serialize::(simd_unit.coefficients) } + #[inline(always)] fn error_serialize(simd_unit: Self) -> [u8; OUTPUT_SIZE] { encoding::error::serialize::(simd_unit.coefficients) } + #[inline(always)] fn error_deserialize(serialized: &[u8]) -> Self { encoding::error::deserialize::(serialized).into() } + #[inline(always)] fn t0_serialize(simd_unit: Self) -> [u8; 13] { encoding::t0::serialize(simd_unit.coefficients) } + #[inline(always)] fn t0_deserialize(serialized: &[u8]) -> Self { encoding::t0::deserialize(serialized).into() } + #[inline(always)] fn t1_serialize(simd_unit: Self) -> [u8; 10] { encoding::t1::serialize(simd_unit.coefficients) } + #[inline(always)] fn t1_deserialize(serialized: &[u8]) -> Self { encoding::t1::deserialize(serialized).into() } + #[inline(always)] fn ntt(simd_units: [Self; SIMD_UNITS_IN_RING_ELEMENT]) -> [Self; SIMD_UNITS_IN_RING_ELEMENT] { let result = ntt::ntt(simd_units.map(|x| x.coefficients)); result.map(|x| x.into()) } + #[inline(always)] fn invert_ntt_at_layer_0( simd_unit: Self, zeta0: i32, @@ -133,9 +157,11 @@ impl Operations for AVX2SIMDUnit { ) -> Self { ntt::invert_ntt_at_layer_0(simd_unit.coefficients, zeta0, zeta1, zeta2, zeta3).into() } + #[inline(always)] fn invert_ntt_at_layer_1(simd_unit: Self, zeta0: i32, zeta1: i32) -> Self { ntt::invert_ntt_at_layer_1(simd_unit.coefficients, zeta0, zeta1).into() } + #[inline(always)] fn invert_ntt_at_layer_2(simd_unit: Self, zeta: i32) -> Self { ntt::invert_ntt_at_layer_2(simd_unit.coefficients, zeta).into() }