Update Dilithium from crystals upstream (#1894)

### Issues:
Resolves #CryptoAlg-2722

### Description of changes: 
This PR updates the implementation of Dilithium (v3.1) with subsequent
changes made in the official upstream repository
(https://github.com/pq-crystals/dilithium).

As there are significant algorithmic changes between pre-FIPS 204
Dilithium (v3.1) and FIPS 204 IPD/Final we note that the integration of
this code will cause Dilithium v3.1 signatures to fail. Consumers of
AWS-LC were made aware of the unstable nature of the Dilithium
implementation and API during the integration into the library (see
https://github.com/aws/aws-lc/blob/8a1ee93969d8df64c4c51b2d6ddffb26a54adea9/crypto/dilithium/README.md).

As part of our due diligence we have verified that there are no existing
deployments contingent on the stability of Dilithium. As such, we will
continue to support the most up to date version of the algorithm from
the authors upstream.

### Call-outs:
Among the changes are modifications made per the FIPS 204 ML-DSA
standard:
- changes to signature and key sizes
- addition of signing context "ctx" added to signature
- removal of AES-based modes
- new KATs
- we maintain the `fqmul` refactor added in
#1748
- small changes to documentation (e.g. reduce.c#L27, poly.c#L24,
polyvec.c#L164)

### Testing:
As the signature size and private key size of ML-DSA has changed since
v3.1, so too must the tests `EVPExtraTest.d2i_PrivateKey` for testing
the parsing of ML-DSA private keys, as well as the certificates used for
`X509Test.TestDilithium3`.

I have updated the private key `kExampleDilithium3KeyDER` and test
certificate `kDilithium3Cert` to reflect changes to the signature/key
lengths, but will need to manually reproduce test-case certificates
`kDilithium3CertNull`, `kDilithium3CertParam`.

By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache 2.0 license and the ISC license.

crypto/dilithium/p_dilithium3.c

@@ -74,7 +74,7 @@ static int pkey_dilithium3_sign_message(EVP_PKEY_CTX *ctx, uint8_t *sig,
     return 0;
   }
 
-  if (DILITHIUM3_sign(sig, siglen, tbs, tbslen, key->priv) != 0) {
+  if (DILITHIUM3_sign(sig, siglen, tbs, tbslen, NULL, 0, key->priv) != 0) {
     OPENSSL_PUT_ERROR(EVP, ERR_R_INTERNAL_ERROR);
     return 0;
   }
@@ -92,7 +92,7 @@ static int pkey_dilithium3_verify_message(EVP_PKEY_CTX *ctx, const uint8_t *sig,
   DILITHIUM3_KEY *key = ctx->pkey->pkey.ptr;
 
   if (siglen != DILITHIUM3_SIGNATURE_BYTES ||
-      DILITHIUM3_verify(tbs, tbslen, sig, siglen, key->pub) != 0) {
+      DILITHIUM3_verify(tbs, tbslen, sig, siglen, NULL, 0, key->pub) != 0) {
     OPENSSL_PUT_ERROR(EVP, EVP_R_INVALID_SIGNATURE);
     return 0;
   }

crypto/dilithium/p_dilithium_test.cc

@@ -560,10 +560,10 @@ TEST(Dilithium3Test, Encoding) {
   EXPECT_EQ(EVP_PKEY_DILITHIUM3, EVP_PKEY_id(privkey.get()));
 
   // The private key must be extractable.
-  uint8_t priv_buf[4000];
+  uint8_t priv_buf[4032];
   size_t priv_len;
   ASSERT_TRUE(EVP_PKEY_get_raw_private_key(privkey.get(), nullptr, &priv_len));
-  EXPECT_EQ(priv_len, 4000u);
+  EXPECT_EQ(priv_len, 4032u);
   ASSERT_TRUE(EVP_PKEY_get_raw_private_key(privkey.get(), priv_buf, &priv_len));
 
   // The private key must encode properly.

crypto/dilithium/pqcrystals_dilithium_ref_common/README.md

@@ -0,0 +1,21 @@
+# AWS-LC ML-DSA readme file
+
+The source code in this folder implements ML-DSA as defined in FIPS 204 Module-Lattice-Based Digital Signature Standard [link](https://csrc.nist.gov/pubs/fips/204/final).
+
+**Source code origin and modifications** 
+
+The source code was imported from a branch of the official repository of the Crystals-Dilithium team: https://github.com/pq-crystals/dilithium. The code was taken at [commit](https://github.com/pq-crystals/dilithium/commit/cbcd8753a43402885c90343cd6335fb54712cda1) as of 10/01/2024. At the moment, only the reference C implementation is imported.
+
+The `api.h`, `fips202.h` and `params.h` header files were modified to support our [prefixed symbols build](https://github.com/awslabs/aws-lc/blob/main/BUILDING.md#building-with-prefixed-symbols).
+
+- `randombytes.{h|c}` are deleted because we are using the randomness generation functions provided by AWS-LC.
+- `sign.c`: calls to `randombytes` function is replaced with calls to `pq_custom_randombytes` and the appropriate header file is included (`crypto/rand_extra/pq_custom_randombytes.h`).
+- `ntt.c`, `poly.c`, `reduce.c`, `reduce.h`: have been modified with a code refactor. The function `fqmul` has been added to bring mode code consistency with Kyber/ML-KEM. See https://github.com/aws/aws-lc/pull/1748 for more details on this change.
+- `reduce.c`: a small fix to documentation has been made on the bounds of `reduce32`.
+- `poly.c`: a small fix to documentation has been made on the bounds of `poly_reduce`.
+- `polyvec.c`: a small fix to documentation has been made on the bounds of `polyveck_reduce`.
+
+**Testing** 
+
+The KATs were obtained from https://github.com/pq-crystals/dilithium/tree/master/ref/nistkat.
+To compile the KAT programs on Linux or macOS, go to the `ref/` directory and run `make nistkat`. This will produce executables within `nistkat` which once executed will produce the KATs: `PQCsignKAT_Dilithium2.rsp`, `PQCsignKAT_Dilithium3.rsp`,`PQCsignKAT_Dilithium5.rsp`.

crypto/dilithium/pqcrystals_dilithium_ref_common/api.h

@@ -6,7 +6,7 @@
 #include "openssl/base.h"
 
 #define pqcrystals_dilithium2_PUBLICKEYBYTES 1312
-#define pqcrystals_dilithium2_SECRETKEYBYTES 2528
+#define pqcrystals_dilithium2_SECRETKEYBYTES 2560
 #define pqcrystals_dilithium2_BYTES 2420
 
 #define pqcrystals_dilithium2_ref_PUBLICKEYBYTES pqcrystals_dilithium2_PUBLICKEYBYTES
@@ -25,53 +25,27 @@ int pqcrystals_dilithium2_ref_keypair(uint8_t *pk, uint8_t *sk);
 
 int pqcrystals_dilithium2_ref_signature(uint8_t *sig, size_t *siglen,
                                         const uint8_t *m, size_t mlen,
+                                        const uint8_t *ctx, size_t ctxlen,
                                         const uint8_t *sk);
 
 int pqcrystals_dilithium2_ref(uint8_t *sm, size_t *smlen,
                               const uint8_t *m, size_t mlen,
+                              const uint8_t *ctx, size_t ctxlen,
                               const uint8_t *sk);
 
 int pqcrystals_dilithium2_ref_verify(const uint8_t *sig, size_t siglen,
                                      const uint8_t *m, size_t mlen,
+                                     const uint8_t *ctx, size_t ctxlen,
                                      const uint8_t *pk);
 
 int pqcrystals_dilithium2_ref_open(uint8_t *m, size_t *mlen,
                                    const uint8_t *sm, size_t smlen,
+                                   const uint8_t *ctx, size_t ctxlen,
                                    const uint8_t *pk);
 
-#define pqcrystals_dilithium2aes_ref_PUBLICKEYBYTES pqcrystals_dilithium2_ref_PUBLICKEYBYTES
-#define pqcrystals_dilithium2aes_ref_SECRETKEYBYTES pqcrystals_dilithium2_ref_SECRETKEYBYTES
-#define pqcrystals_dilithium2aes_ref_BYTES pqcrystals_dilithium2_ref_BYTES
-
-#ifdef BORINGSSL_PREFIX
-#define pqcrystals_dilithium2aes_ref_keypair BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, pqcrystals_dilithium2aes_ref_keypair)
-#define pqcrystals_dilithium2aes_ref_signature BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, pqcrystals_dilithium2aes_ref_signature)
-#define pqcrystals_dilithium2aes_ref BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, pqcrystals_dilithium2aes_ref)
-#define pqcrystals_dilithium2aes_ref_verify BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, pqcrystals_dilithium2aes_ref_verify)
-#define pqcrystals_dilithium2aes_ref_open BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, pqcrystals_dilithium2aes_ref_open)
-#endif
-
-int pqcrystals_dilithium2aes_ref_keypair(uint8_t *pk, uint8_t *sk);
-
-int pqcrystals_dilithium2aes_ref_signature(uint8_t *sig, size_t *siglen,
-                                           const uint8_t *m, size_t mlen,
-                                           const uint8_t *sk);
-
-int pqcrystals_dilithium2aes_ref(uint8_t *sm, size_t *smlen,
-                                 const uint8_t *m, size_t mlen,
-                                 const uint8_t *sk);
-
-int pqcrystals_dilithium2aes_ref_verify(const uint8_t *sig, size_t siglen,
-                                        const uint8_t *m, size_t mlen,
-                                        const uint8_t *pk);
-
-int pqcrystals_dilithium2aes_ref_open(uint8_t *m, size_t *mlen,
-                                      const uint8_t *sm, size_t smlen,
-                                      const uint8_t *pk);
-
 #define pqcrystals_dilithium3_PUBLICKEYBYTES 1952
-#define pqcrystals_dilithium3_SECRETKEYBYTES 4000
-#define pqcrystals_dilithium3_BYTES 3293
+#define pqcrystals_dilithium3_SECRETKEYBYTES 4032
+#define pqcrystals_dilithium3_BYTES 3309
 
 #define pqcrystals_dilithium3_ref_PUBLICKEYBYTES pqcrystals_dilithium3_PUBLICKEYBYTES
 #define pqcrystals_dilithium3_ref_SECRETKEYBYTES pqcrystals_dilithium3_SECRETKEYBYTES
@@ -89,54 +63,27 @@ int pqcrystals_dilithium3_ref_keypair(uint8_t *pk, uint8_t *sk);
 
 int pqcrystals_dilithium3_ref_signature(uint8_t *sig, size_t *siglen,
                                         const uint8_t *m, size_t mlen,
+                                        const uint8_t *ctx, size_t ctxlen,
                                         const uint8_t *sk);
 
 int pqcrystals_dilithium3_ref(uint8_t *sm, size_t *smlen,
                               const uint8_t *m, size_t mlen,
+                              const uint8_t *ctx, size_t ctxlen,
                               const uint8_t *sk);
 
 int pqcrystals_dilithium3_ref_verify(const uint8_t *sig, size_t siglen,
                                      const uint8_t *m, size_t mlen,
+                                     const uint8_t *ctx, size_t ctxlen,
                                      const uint8_t *pk);
 
 int pqcrystals_dilithium3_ref_open(uint8_t *m, size_t *mlen,
                                    const uint8_t *sm, size_t smlen,
+                                   const uint8_t *ctx, size_t ctxlen,
                                    const uint8_t *pk);
 
-#define pqcrystals_dilithium3aes_ref_PUBLICKEYBYTES pqcrystals_dilithium3_ref_PUBLICKEYBYTES
-#define pqcrystals_dilithium3aes_ref_SECRETKEYBYTES pqcrystals_dilithium3_ref_SECRETKEYBYTES
-#define pqcrystals_dilithium3aes_ref_BYTES pqcrystals_dilithium3_ref_BYTES
-
-#ifdef BORINGSSL_PREFIX
-#define pqcrystals_dilithium3aes_ref_keypair BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, pqcrystals_dilithium3aes_ref_keypair)
-#define pqcrystals_dilithium3aes_ref_signature BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, pqcrystals_dilithium3aes_ref_signature)
-#define pqcrystals_dilithium3aes_ref BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, pqcrystals_dilithium3aes_ref)
-#define pqcrystals_dilithium3aes_ref_verify BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, pqcrystals_dilithium3aes_ref_verify)
-#define pqcrystals_dilithium3aes_ref_open BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, pqcrystals_dilithium3aes_ref_open)
-#endif
-
-
-int pqcrystals_dilithium3aes_ref_keypair(uint8_t *pk, uint8_t *sk);
-
-int pqcrystals_dilithium3aes_ref_signature(uint8_t *sig, size_t *siglen,
-                                           const uint8_t *m, size_t mlen,
-                                           const uint8_t *sk);
-
-int pqcrystals_dilithium3aes_ref(uint8_t *sm, size_t *smlen,
-                                 const uint8_t *m, size_t mlen,
-                                 const uint8_t *sk);
-
-int pqcrystals_dilithium3aes_ref_verify(const uint8_t *sig, size_t siglen,
-                                        const uint8_t *m, size_t mlen,
-                                        const uint8_t *pk);
-
-int pqcrystals_dilithium3aes_ref_open(uint8_t *m, size_t *mlen,
-                                      const uint8_t *sm, size_t smlen,
-                                      const uint8_t *pk);
-
 #define pqcrystals_dilithium5_PUBLICKEYBYTES 2592
-#define pqcrystals_dilithium5_SECRETKEYBYTES 4864
-#define pqcrystals_dilithium5_BYTES 4595
+#define pqcrystals_dilithium5_SECRETKEYBYTES 4896
+#define pqcrystals_dilithium5_BYTES 4627
 
 #define pqcrystals_dilithium5_ref_PUBLICKEYBYTES pqcrystals_dilithium5_PUBLICKEYBYTES
 #define pqcrystals_dilithium5_ref_SECRETKEYBYTES pqcrystals_dilithium5_SECRETKEYBYTES
@@ -154,50 +101,22 @@ int pqcrystals_dilithium5_ref_keypair(uint8_t *pk, uint8_t *sk);
 
 int pqcrystals_dilithium5_ref_signature(uint8_t *sig, size_t *siglen,
                                         const uint8_t *m, size_t mlen,
+                                        const uint8_t *ctx, size_t ctxlen,
                                         const uint8_t *sk);
 
 int pqcrystals_dilithium5_ref(uint8_t *sm, size_t *smlen,
                               const uint8_t *m, size_t mlen,
+                              const uint8_t *ctx, size_t ctxlen,
                               const uint8_t *sk);
 
 int pqcrystals_dilithium5_ref_verify(const uint8_t *sig, size_t siglen,
                                      const uint8_t *m, size_t mlen,
+                                     const uint8_t *ctx, size_t ctxlen,
                                      const uint8_t *pk);
 
 int pqcrystals_dilithium5_ref_open(uint8_t *m, size_t *mlen,
                                    const uint8_t *sm, size_t smlen,
+                                   const uint8_t *ctx, size_t ctxlen,
                                    const uint8_t *pk);
 
-#define pqcrystals_dilithium5aes_ref_PUBLICKEYBYTES pqcrystals_dilithium5_ref_PUBLICKEYBYTES
-#define pqcrystals_dilithium5aes_ref_SECRETKEYBYTES pqcrystals_dilithium5_ref_SECRETKEYBYTES
-#define pqcrystals_dilithium5aes_ref_BYTES pqcrystals_dilithium5_ref_BYTES
-
-#ifdef BORINGSSL_PREFIX
-#define pqcrystals_dilithium5aes_ref_keypair BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, pqcrystals_dilithium5aes_ref_keypair)
-#define pqcrystals_dilithium5aes_ref_signature BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, pqcrystals_dilithium5aes_ref_signature)
-#define pqcrystals_dilithium5aes_ref BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, pqcrystals_dilithium5aes_ref)
-#define pqcrystals_dilithium5aes_ref_verify BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, pqcrystals_dilithium5aes_ref_verify)
-#define pqcrystals_dilithium5aes_ref_open BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, pqcrystals_dilithium5aes_ref_open)
-#endif
-
-
-int pqcrystals_dilithium5aes_ref_keypair(uint8_t *pk, uint8_t *sk);
-
-int pqcrystals_dilithium5aes_ref_signature(uint8_t *sig, size_t *siglen,
-                                           const uint8_t *m, size_t mlen,
-                                           const uint8_t *sk);
-
-int pqcrystals_dilithium5aes_ref(uint8_t *sm, size_t *smlen,
-                                 const uint8_t *m, size_t mlen,
-                                 const uint8_t *sk);
-
-int pqcrystals_dilithium5aes_ref_verify(const uint8_t *sig, size_t siglen,
-                                        const uint8_t *m, size_t mlen,
-                                        const uint8_t *pk);
-
-int pqcrystals_dilithium5aes_ref_open(uint8_t *m, size_t *mlen,
-                                      const uint8_t *sm, size_t smlen,
-                                      const uint8_t *pk);
-
-
 #endif

crypto/dilithium/pqcrystals_dilithium_ref_common/config.h

@@ -3,6 +3,8 @@
 
 #include <openssl/base.h>
 
+#define DILITHIUM_RANDOMIZED_SIGNING
+
 #ifndef DILITHIUM_MODE
 #define DILITHIUM_MODE 3
 #endif

crypto/dilithium/pqcrystals_dilithium_ref_common/packing.c

@@ -64,7 +64,7 @@ void unpack_pk(uint8_t rho[SEEDBYTES],
 **************************************************/
 void pack_sk(uint8_t sk[CRYPTO_SECRETKEYBYTES],
              const uint8_t rho[SEEDBYTES],
-             const uint8_t tr[SEEDBYTES],
+             const uint8_t tr[TRBYTES],
              const uint8_t key[SEEDBYTES],
              const polyveck *t0,
              const polyvecl *s1,
@@ -80,9 +80,9 @@ void pack_sk(uint8_t sk[CRYPTO_SECRETKEYBYTES],
     sk[i] = key[i];
   sk += SEEDBYTES;
 
-  for(i = 0; i < SEEDBYTES; ++i)
+  for(i = 0; i < TRBYTES; ++i)
     sk[i] = tr[i];
-  sk += SEEDBYTES;
+  sk += TRBYTES;
 
   for(i = 0; i < L; ++i)
     polyeta_pack(sk + i*POLYETA_PACKEDBYTES, &s1->vec[i]);
@@ -110,7 +110,7 @@ void pack_sk(uint8_t sk[CRYPTO_SECRETKEYBYTES],
 *              - uint8_t sk[]: byte array containing bit-packed sk
 **************************************************/
 void unpack_sk(uint8_t rho[SEEDBYTES],
-               uint8_t tr[SEEDBYTES],
+               uint8_t tr[TRBYTES],
                uint8_t key[SEEDBYTES],
                polyveck *t0,
                polyvecl *s1,
@@ -127,9 +127,9 @@ void unpack_sk(uint8_t rho[SEEDBYTES],
     key[i] = sk[i];
   sk += SEEDBYTES;
 
-  for(i = 0; i < SEEDBYTES; ++i)
+  for(i = 0; i < TRBYTES; ++i)
     tr[i] = sk[i];
-  sk += SEEDBYTES;
+  sk += TRBYTES;
 
   for(i=0; i < L; ++i)
     polyeta_unpack(&s1->vec[i], sk + i*POLYETA_PACKEDBYTES);
@@ -154,15 +154,15 @@ void unpack_sk(uint8_t rho[SEEDBYTES],
 *              - const polyveck *h: pointer to hint vector h
 **************************************************/
 void pack_sig(uint8_t sig[CRYPTO_BYTES],
-              const uint8_t c[SEEDBYTES],
+              const uint8_t c[CTILDEBYTES],
               const polyvecl *z,
               const polyveck *h)
 {
   unsigned int i, j, k;
 
-  for(i=0; i < SEEDBYTES; ++i)
+  for(i=0; i < CTILDEBYTES; ++i)
     sig[i] = c[i];
-  sig += SEEDBYTES;
+  sig += CTILDEBYTES;
 
   for(i = 0; i < L; ++i)
     polyz_pack(sig + i*POLYZ_PACKEDBYTES, &z->vec[i]);
@@ -195,16 +195,16 @@ void pack_sig(uint8_t sig[CRYPTO_BYTES],
 *
 * Returns 1 in case of malformed signature; otherwise 0.
 **************************************************/
-int unpack_sig(uint8_t c[SEEDBYTES],
+int unpack_sig(uint8_t c[CTILDEBYTES],
                polyvecl *z,
                polyveck *h,
                const uint8_t sig[CRYPTO_BYTES])
 {
   unsigned int i, j, k;
 
-  for(i = 0; i < SEEDBYTES; ++i)
+  for(i = 0; i < CTILDEBYTES; ++i)
     c[i] = sig[i];
-  sig += SEEDBYTES;
+  sig += CTILDEBYTES;
 
   for(i = 0; i < L; ++i)
     polyz_unpack(&z->vec[i], sig + i*POLYZ_PACKEDBYTES);

crypto/dilithium/pqcrystals_dilithium_ref_common/packing.h

@@ -11,28 +11,28 @@ void pack_pk(uint8_t pk[CRYPTO_PUBLICKEYBYTES], const uint8_t rho[SEEDBYTES], co
 #define pack_sk DILITHIUM_NAMESPACE(pack_sk)
 void pack_sk(uint8_t sk[CRYPTO_SECRETKEYBYTES],
              const uint8_t rho[SEEDBYTES],
-             const uint8_t tr[SEEDBYTES],
+             const uint8_t tr[TRBYTES],
              const uint8_t key[SEEDBYTES],
              const polyveck *t0,
              const polyvecl *s1,
              const polyveck *s2);
 
 #define pack_sig DILITHIUM_NAMESPACE(pack_sig)
-void pack_sig(uint8_t sig[CRYPTO_BYTES], const uint8_t c[SEEDBYTES], const polyvecl *z, const polyveck *h);
+void pack_sig(uint8_t sig[CRYPTO_BYTES], const uint8_t c[CTILDEBYTES], const polyvecl *z, const polyveck *h);
 
 #define unpack_pk DILITHIUM_NAMESPACE(unpack_pk)
 void unpack_pk(uint8_t rho[SEEDBYTES], polyveck *t1, const uint8_t pk[CRYPTO_PUBLICKEYBYTES]);
 
 #define unpack_sk DILITHIUM_NAMESPACE(unpack_sk)
 void unpack_sk(uint8_t rho[SEEDBYTES],
-               uint8_t tr[SEEDBYTES],
+               uint8_t tr[TRBYTES],
                uint8_t key[SEEDBYTES],
                polyveck *t0,
                polyvecl *s1,
                polyveck *s2,
                const uint8_t sk[CRYPTO_SECRETKEYBYTES]);
 
 #define unpack_sig DILITHIUM_NAMESPACE(unpack_sig)
-int unpack_sig(uint8_t c[SEEDBYTES], polyvecl *z, polyveck *h, const uint8_t sig[CRYPTO_BYTES]);
+int unpack_sig(uint8_t c[CTILDEBYTES], polyvecl *z, polyveck *h, const uint8_t sig[CRYPTO_BYTES]);
 
 #endif