Replace SHAKE128 with TurboSHAKE128

The reference code uses the reference implementation of TurboSHAKE128.
This code is unoptimized, so care is needed to ensure our tests run in a
reasonable amount of time.

Each time `XofTurboShake128` is constructed we call `TurboSHAKE128()`
once and fill a buffer with the output stream. The size of the buffer is
a constant, `MAX_XOF_OUT_STREAM_BYTES`, chosen to be sufficiently long
for every test that we have. So that we don't have to make this value
too large, some of tests in `vdaf_poplar1.py` have been modified.

.github/workflows/test.yml

@@ -25,6 +25,9 @@ jobs:
         sudo apt-get update
         sudo apt-get install -y sagemath python3-pycryptodome python3-cffi
 
+    - with:
+      submodules: 'true'
+
     - name: Run tests
       working-directory: poc
       run: make test

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "poc/draft-irtf-cfrg-kangarootwelve"]
+	path = poc/draft-irtf-cfrg-kangarootwelve
+	url = https://github.com/cfrg/draft-irtf-cfrg-kangarootwelve

draft-irtf-cfrg-vdaf.md

@@ -31,13 +31,6 @@ author:
     organization: Google
     email: [email protected]
 
-normative:
-
-  FIPS202:
-    title: "SHA-3 Standard: Permutation-Based Hash and Extendable-Output Functions"
-    date: August 2015
-    seriesinfo: NIST FIPS PUB 202
-
 informative:
 
   AGJOP21:
@@ -1831,51 +1824,50 @@ def expand_into_vec(Xof,
 ~~~
 {: #xof-derived-methods title="Derived methods for XOFs."}
 
-### XofShake128 {#xof-shake128}
+### XofTurboShake128 {#xof-turboshake128}
 
-This section describes XofShake128, a XOF based on the SHAKE128 mode of
-operation for the Keccak permutation {{FIPS202}}. This XOF is RECOMMENDED for
-all use cases within VDAFs. The length of the domain separation string `dst`
-passed to XofShake128 MUST NOT exceed 255 bytes.
+This section describes XofTurboShake128, an XOF based on the
+TurboSHAKE128 {{!TurboSHAKE=I-D.draft-irtf-cfrg-kangarootwelve}}. This
+XOF is RECOMMENDED for all use cases within VDAFs. The length of the
+domain separation string `dst` passed to XofTurboShake128 MUST NOT
+exceed 255 bytes.
 
 ~~~
-class XofShake128(Xof):
-    """XOF based on SHA-3 (SHAKE128)."""
+class XofTurboShake128(Xof):
+    """XOF wrapper for TurboSHAKE128."""
 
     # Associated parameters
     SEED_SIZE = 16
 
     def __init__(self, seed, dst, binder):
         self.l = 0
-        self.x = seed + binder
-        self.s = dst
+        self.m = to_le_bytes(len(dst), 1) + dst + seed + binder
 
     def next(self, length: Unsigned) -> Bytes:
         self.l += length
 
-        # Function `SHAKE128(x, l)` is as defined in
-        # [FIPS 202, Section 6.2].
+        # Function `TurboSHAKE128(M, D, L)` is as defined in
+        # Section 2.2 of [TurboSHAKE].
         #
         # Implementation note: Rather than re-generate the output
         # stream each time `next()` is invoked, most implementations
-        # of SHA-3 will expose an "absorb-then-squeeze" API that
+        # of TurboSHAKE128 will expose an "absorb-then-squeeze" API that
         # allows stateful handling of the stream.
-        dst_length = to_le_bytes(len(self.s), 1)
-        stream = SHAKE128(dst_length + self.s + self.x, self.l)
+        stream = TurboSHAKE128(self.m, 1, self.l)
         return stream[-length:]
 ~~~
-{: title="Definition of XOF XofShake128."}
+{: title="Definition of XOF XofTurboShake128."}
 
 ### XofFixedKeyAes128 {#xof-fixed-key-aes128}
 
-While XofShake128 as described above can be securely used in all cases where a XOF
-is needed in the VDAFs described in this document, there are some cases where
-a more efficient instantiation based on fixed-key AES is possible. For now, this
-is limited to the XOF used inside the Idpf {{idpf}} implementation in Poplar1
-{{idpf-poplar}}. It is NOT RECOMMENDED to use this XOF anywhere else.
-The length of the domain separation string `dst` passed to XofFixedKeyAes128
-MUST NOT exceed 255 bytes. See Security Considerations {{security}} for a more
-detailed discussion.
+While XofTurboShake128 as described above can be securely used in all cases
+where a XOF is needed in the VDAFs described in this document, there are some
+cases where a more efficient instantiation based on fixed-key AES is possible.
+For now, this is limited to the XOF used inside the Idpf {{idpf}}
+implementation in Poplar1 {{idpf-poplar}}. It is NOT RECOMMENDED to use this
+XOF anywhere else. The length of the domain separation string `dst` passed to
+XofFixedKeyAes128 MUST NOT exceed 255 bytes. See Security Considerations
+{{security}} for a more detailed discussion.
 
 ~~~
 class XofFixedKeyAes128(Xof):
@@ -1890,15 +1882,15 @@ class XofFixedKeyAes128(Xof):
     def __init__(self, seed, dst, binder):
         self.length_consumed = 0
 
-        # Use SHA-3 to derive a key from the binder string and domain
-        # separation tag. Note that the AES key does not need to be
-        # kept secret from any party. However, when used with
+        # Use TurboSHAKE128 to derive a key from the binder string and
+        # domain separation tag. Note that the AES key does not need
+        # to be kept secret from any party. However, when used with
         # IdpfPoplar, we require the binder to be a random nonce.
         #
         # Implementation note: This step can be cached across XOF
         # evaluations with many different seeds.
         dst_length = to_le_bytes(len(dst), 1)
-        self.fixed_key = SHAKE128(dst_length + dst + binder, 16)
+        self.fixed_key = TurboSHAKE128(dst_length + dst + binder, 2, 16)
         self.seed = seed
 
     def next(self, length: Unsigned) -> Bytes:
@@ -3178,9 +3170,9 @@ each can be found in {{test-vectors}}.
 Our first instance of Prio3 is for a simple counter: Each measurement is either
 one or zero and the aggregate result is the sum of the measurements.
 
-This instance uses XofShake128 ({{xof-shake128}}) as its XOF. Its validity
-circuit, denoted `Count`, uses `Field64` ({{fields}}) as its finite field. Its
-gadget, denoted `Mul`, is the degree-2, arity-2 gadget defined as
+This instance uses XofTurboShake128 ({{xof-turboshake128}}) as its XOF. Its
+validity circuit, denoted `Count`, uses `Field64` ({{fields}}) as its finite
+field. Its gadget, denoted `Mul`, is the degree-2, arity-2 gadget defined as
 
 ~~~
 def eval(self, Field, inp):
@@ -3220,10 +3212,11 @@ The next instance of Prio3 supports summing of integers in a pre-determined
 range. Each measurement is an integer in range `[0, 2^bits)`, where `bits` is an
 associated parameter.
 
-This instance of Prio3 uses XofShake128 ({{xof-shake128}}) as its XOF. Its validity
-circuit, denoted `Sum`, uses `Field128` ({{fields}}) as its finite field. The
-measurement is encoded as a length-`bits` vector of field elements, where the
-`l`th element of the vector represents the `l`th bit of the summand:
+This instance of Prio3 uses XofTurboShake128 ({{xof-turboshake128}}) as its
+XOF. Its validity circuit, denoted `Sum`, uses `Field128` ({{fields}}) as its
+finite field. The measurement is encoded as a length-`bits` vector of field
+elements, where the `l`th element of the vector represents the `l`th bit of the
+summand:
 
 ~~~
 def encode(self, measurement):
@@ -3283,8 +3276,9 @@ of the measurement is an integer in the range `[0, 2^bits)`. It is RECOMMENDED
 to set `chunk_length` to an integer near the square root of `length * bits`
 (see {{parallel-sum-chunk-length}}).
 
-This instance uses XofShake128 ({{xof-shake128}}) as its XOF. Its validity circuit,
-denoted `SumVec`, uses `Field128` ({{fields}}) as its finite field.
+This instance uses XofTurboShake128 ({{xof-turboshake128}}) as its XOF. Its
+validity circuit, denoted `SumVec`, uses `Field128` ({{fields}}) as its finite
+field.
 
 Measurements are encoded as a vector of field elements with length `length *
 bits`. The field elements in the encoded vector represent all the bits of the
@@ -3417,12 +3411,12 @@ example, the buckets might quantize the real numbers, and each measurement would
 report the bucket that the corresponding client's real-numbered value falls
 into. The aggregate result counts the number of measurements in each bucket.
 
-This instance of Prio3 uses XofShake128 ({{xof-shake128}}) as its XOF. Its validity
-circuit, denoted `Histogram`, uses `Field128` ({{fields}}) as its finite field.
-It has two parameters, `length`, the number of histogram buckets, and
-`chunk_length`, which is used by by a circuit optimization described below. It
-is RECOMMENDED to set `chunk_length` to an integer near the square root of
-`length` (see {{parallel-sum-chunk-length}}).
+This instance of Prio3 uses XofTurboShake128 ({{xof-turboshake128}}) as its
+XOF. Its validity circuit, denoted `Histogram`, uses `Field128` ({{fields}}) as
+its finite field. It has two parameters, `length`, the number of histogram
+buckets, and `chunk_length`, which is used by by a circuit optimization
+described below. It is RECOMMENDED to set `chunk_length` to an integer near the
+square root of `length` (see {{parallel-sum-chunk-length}}).
 
 The measurement is encoded as a one-hot vector representing the bucket into
 which the measurement falls:
@@ -4438,8 +4432,8 @@ throws an error.
 ## Instantiation {#poplar1-inst}
 
 By default, Poplar1 is instantiated with IdpfPoplar (`VALUE_LEN == 2`) and
-XofShake128 ({{xof-shake128}}). This VDAF is suitable for any positive value of
-`BITS`. Test vectors can be found in {{test-vectors}}.
+XofTurboShake128 ({{xof-turboshake128}}). This VDAF is suitable for any
+positive value of `BITS`. Test vectors can be found in {{test-vectors}}.
 
 # Security Considerations {#security}
 
@@ -4591,7 +4585,7 @@ differential privacy.
 
 As described in {{xof}}, our constructions rely on eXtendable
 Output Functions (XOFs). In the security analyses of our protocols, these are
-usually modeled as random oracles. XofShake128 is designed to be
+usually modeled as random oracles. XofTurboShake128 is designed to be
 indifferentiable from a random oracle {{MRH04}}, making it a suitable choice
 for most situations.
 

poc/daf.py

@@ -6,7 +6,7 @@
 
 import field
 from common import Bool, Unsigned, gen_rand
-from xof import XofShake128
+from xof import XofTurboShake128
 
 
 class Daf:
@@ -166,11 +166,11 @@ class TestDaf(Daf):
 
     @classmethod
     def shard(cls, measurement, _nonce, rand):
-        helper_shares = XofShake128.expand_into_vec(cls.Field,
-                                                    rand,
-                                                    b'',
-                                                    b'',
-                                                    cls.SHARES-1)
+        helper_shares = XofTurboShake128.expand_into_vec(cls.Field,
+                                                         rand,
+                                                         b'',
+                                                         b'',
+                                                         cls.SHARES-1)
         leader_share = cls.Field(measurement)
         for helper_share in helper_shares:
             leader_share -= helper_share

poc/vdaf_poplar1.py

@@ -328,7 +328,7 @@ def with_bits(Poplar1, bits: Unsigned):
         TheIdpf = idpf_poplar.IdpfPoplar \
             .with_value_len(2) \
             .with_bits(bits)
-        TheXof = xof.XofShake128
+        TheXof = xof.XofTurboShake128
 
         class Poplar1WithBits(Poplar1):
             Idpf = TheIdpf
@@ -414,28 +414,28 @@ def encode_idpf_field_vec(vec):
         [2],
     )
     test_vdaf(
-        Poplar1.with_bits(128),
+        Poplar1.with_bits(64),
         (
-            127,
-            (from_be_bytes(b'0123456789abcdef'),),
+            63,
+            (from_be_bytes(b'01234567'),),
         ),
         [
-            from_be_bytes(b'0123456789abcdef'),
+            from_be_bytes(b'01234567'),
         ],
         [1],
     )
     test_vdaf(
-        Poplar1.with_bits(256),
+        Poplar1.with_bits(64),
         (
-            63,
+            31,
             (
-                from_be_bytes(b'00000000'),
-                from_be_bytes(b'01234567'),
+                from_be_bytes(b'0000'),
+                from_be_bytes(b'0123'),
             ),
         ),
         [
-            from_be_bytes(b'0123456789abcdef0123456789abcdef'),
-            from_be_bytes(b'01234567890000000000000000000000'),
+            from_be_bytes(b'01234567'),
+            from_be_bytes(b'01234000'),
         ],
         [0, 2],
     )

poc/vdaf_prio3.py

@@ -443,23 +443,23 @@ def test_vec_encode_prep_msg(Prio3, k_joint_rand):
 
 class Prio3Count(Prio3):
     # Generic types required by `Prio3`
-    Xof = xof.XofShake128
+    Xof = xof.XofTurboShake128
     Flp = flp_generic.FlpGeneric(flp_generic.Count())
 
     # Associated parameters.
     ID = 0x00000000
-    VERIFY_KEY_SIZE = xof.XofShake128.SEED_SIZE
+    VERIFY_KEY_SIZE = xof.XofTurboShake128.SEED_SIZE
 
     # Operational parameters.
     test_vec_name = 'Prio3Count'
 
 
 class Prio3Sum(Prio3):
     # Generic types required by `Prio3`
-    Xof = xof.XofShake128
+    Xof = xof.XofTurboShake128
 
     # Associated parameters.
-    VERIFY_KEY_SIZE = xof.XofShake128.SEED_SIZE
+    VERIFY_KEY_SIZE = xof.XofTurboShake128.SEED_SIZE
     ID = 0x00000001
 
     # Operational parameters.
@@ -474,10 +474,10 @@ class Prio3SumWithBits(Prio3Sum):
 
 class Prio3SumVec(Prio3):
     # Generic types required by `Prio3`
-    Xof = xof.XofShake128
+    Xof = xof.XofTurboShake128
 
     # Associated parameters.
-    VERIFY_KEY_SIZE = xof.XofShake128.SEED_SIZE
+    VERIFY_KEY_SIZE = xof.XofTurboShake128.SEED_SIZE
     ID = 0x00000002
 
     # Operational parameters.
@@ -495,10 +495,10 @@ class Prio3SumVecWithParams(Prio3SumVec):
 
 class Prio3Histogram(Prio3):
     # Generic types required by `Prio3`
-    Xof = xof.XofShake128
+    Xof = xof.XofTurboShake128
 
     # Associated parameters.
-    VERIFY_KEY_SIZE = xof.XofShake128.SEED_SIZE
+    VERIFY_KEY_SIZE = xof.XofTurboShake128.SEED_SIZE
     ID = 0x00000003
 
     # Operational parameters.
@@ -552,10 +552,10 @@ class Prio3SumVecWithMultiproofAndParams(cls):
 
 class Prio3MultiHotHistogram(Prio3):
     # Generic types required by `Prio3`
-    Xof = xof.XofShake128
+    Xof = xof.XofTurboShake128
 
     # Associated parameters.
-    VERIFY_KEY_SIZE = xof.XofShake128.SEED_SIZE
+    VERIFY_KEY_SIZE = xof.XofTurboShake128.SEED_SIZE
     # Private codepoint just for testing.
     ID = 0xFFFFFFFF
 
@@ -584,11 +584,11 @@ class TestPrio3Average(Prio3):
     class's decode() method.
     """
 
-    Xof = xof.XofShake128
+    Xof = xof.XofTurboShake128
     # NOTE 0xFFFFFFFF is reserved for testing. If we decide to standardize this
     # Prio3 variant, then we'll need to pick a real codepoint for it.
     ID = 0xFFFFFFFF
-    VERIFY_KEY_SIZE = xof.XofShake128.SEED_SIZE
+    VERIFY_KEY_SIZE = xof.XofTurboShake128.SEED_SIZE
 
     @classmethod
     def with_bits(cls, bits: Unsigned):
@@ -650,7 +650,7 @@ def test_prio3sumvec_with_multiproof():
     num_shares = 2  # Must be in range `[2, 256)`
 
     cls = Prio3 \
-        .with_xof(xof.XofShake128) \
+        .with_xof(xof.XofTurboShake128) \
         .with_flp(flp.FlpTestField128()) \
         .with_shares(num_shares)
     cls.ID = 0xFFFFFFFF
@@ -659,7 +659,7 @@ def test_prio3sumvec_with_multiproof():
     # If JOINT_RAND_LEN == 0, then Fiat-Shamir isn't needed and we can skip
     # generating the joint randomness.
     cls = Prio3 \
-        .with_xof(xof.XofShake128) \
+        .with_xof(xof.XofTurboShake128) \
         .with_flp(flp.FlpTestField128.with_joint_rand_len(0)) \
         .with_shares(num_shares)
     cls.ID = 0xFFFFFFFF