v20-ledger: new test and shred fixes associated with it

firedancer-io · Apr 30, 2024 · d0bb44d · d0bb44d
1 parent 75b9b6f
commit d0bb44d
Show file tree

Hide file tree

Showing 6 changed files with 74 additions and 28 deletions.
diff --git a/src/ballet/shred/fd_deshredder.c b/src/ballet/shred/fd_deshredder.c
@@ -32,7 +32,8 @@ fd_deshredder_next( fd_deshredder_t * const shredder ) {
     uchar shred_type = fd_shred_type( shred->variant );
     if( FD_UNLIKELY( shred_type!=FD_SHRED_TYPE_LEGACY_DATA
                   && shred_type!=FD_SHRED_TYPE_MERKLE_DATA
-                  && shred_type!=FD_SHRED_TYPE_MERKLE_DATA_CHAINED ) )
+                  && shred_type!=FD_SHRED_TYPE_MERKLE_DATA_CHAINED
+                  && shred_type!=FD_SHRED_TYPE_MERKLE_DATA_CHAINED_RESIGNED ) )
       return -FD_SHRED_EINVAL;
 
     /* Ensure entry fits next shred */

diff --git a/src/ballet/shred/fd_shred.c b/src/ballet/shred/fd_shred.c
@@ -17,6 +17,8 @@ fd_shred_parse( uchar const * const buf,
                    (type!=FD_SHRED_TYPE_MERKLE_CODE) &
                    (type!=FD_SHRED_TYPE_MERKLE_DATA_CHAINED) &
                    (type!=FD_SHRED_TYPE_MERKLE_CODE_CHAINED) &
+                   (type!=FD_SHRED_TYPE_MERKLE_DATA_CHAINED_RESIGNED) &
+                   (type!=FD_SHRED_TYPE_MERKLE_CODE_CHAINED_RESIGNED) &
                    (variant!=0xa5 /*FD_SHRED_TYPE_LEGACY_DATA*/ ) &
                    (variant!=0x5a /*FD_SHRED_TYPE_LEGACY_CODE*/ ) ) )
     return NULL;
@@ -31,9 +33,11 @@ fd_shred_parse( uchar const * const buf,
   ulong payload_sz;
 
   /* only present for chained merkle shreds */
-  ulong previous_merkle_root_sz = fd_ulong_if( 
-   (type == FD_SHRED_TYPE_MERKLE_DATA_CHAINED) | (type == FD_SHRED_TYPE_MERKLE_CODE_CHAINED), FD_SHRED_MERKLE_ROOT_SZ, 0UL );
-
+  ulong previous_merkle_root_sz = fd_ulong_if(
+   (type == FD_SHRED_TYPE_MERKLE_DATA_CHAINED) | (type == FD_SHRED_TYPE_MERKLE_CODE_CHAINED) |
+     (type == FD_SHRED_TYPE_MERKLE_DATA_CHAINED_RESIGNED) | (type == FD_SHRED_TYPE_MERKLE_CODE_CHAINED_RESIGNED)
+     , FD_SHRED_MERKLE_ROOT_SZ, 0UL );
+
   if( FD_LIKELY( type & FD_SHRED_TYPEMASK_DATA ) ) {
     if( FD_UNLIKELY( shred->data.size<header_sz ) ) return NULL;
     payload_sz = (ulong)shred->data.size - header_sz; /* between 0 and USHORT_MAX */

diff --git a/src/ballet/shred/fd_shred.h b/src/ballet/shred/fd_shred.h
@@ -31,6 +31,12 @@
       | Merkle node #1         | 20 bytes
       ..........................
 
+       for resigned shreds shreds, followed by:
+
+      +------------------------+
+      | signature              | 64 bytes
+      ..........................
+
    ### Shredding
 
    For a given input data blob (usually an entry batch),
@@ -58,6 +64,11 @@
 
    The length of the inclusion proof is indicated by the variant field.
 
+   ### resigned shreds
+
+   Resigned shreds allow for an additional signature to be added on to lock down
+   the retransmitter for turbine propagation
+
    ### Authentication
 
    Shreds are signed by the block producer.
@@ -95,6 +106,11 @@
 /* FD_SHRED_TYPE_MERKLE_CODE_CHAINED: A shred carrying Reed-Solomon ECC and a chained merkle inclusion proof. */
 #define FD_SHRED_TYPE_MERKLE_CODE_CHAINED ((uchar)0x60)
 
+/* FD_SHRED_TYPE_MERKLE_DATA_CHAINED_RESIGNED: A shred carrying raw binary data and a chained merkle inclusion proof and resigned. */
+#define FD_SHRED_TYPE_MERKLE_DATA_CHAINED_RESIGNED ((uchar)0xB0)
+/* FD_SHRED_TYPE_MERKLE_CODE_CHAINED_RESIGNED: A shred carrying Reed-Solomon ECC and a chained merkle inclusion proof and resigned. */
+#define FD_SHRED_TYPE_MERKLE_CODE_CHAINED_RESIGNED ((uchar)0x70)
+
 /* FD_SHRED_TYPEMASK_DATA: bitwise AND with type matches data shred */
 #define FD_SHRED_TYPEMASK_DATA FD_SHRED_TYPE_MERKLE_DATA
 /* FD_SHRED_TYPEMASK_CODE: bitwise AND with type matches code shred */
@@ -104,6 +120,8 @@
 #define FD_SHRED_MERKLE_ROOT_SZ (32UL)
 /* FD_SHRED_MERKLE_NODE_SZ: the size of a merkle inclusion proof node in bytes. */
 #define FD_SHRED_MERKLE_NODE_SZ (20UL)
+/* FD_SHRED_SIGNATURE_SZ: the size of a signature in a shred. */
+#define FD_SHRED_SIGNATURE_SZ (64UL)
 /* A merkle inclusion proof node. */
 typedef uchar fd_shred_merkle_t[FD_SHRED_MERKLE_NODE_SZ];
 
@@ -149,14 +167,18 @@ struct __attribute__((packed)) fd_shred {
      - 1010: legacy data
      - 0100: merkle code
      - 0110: merkle code (chained)
+     - 0111: merkle code (chained resigned)
      - 1000: merkle data
      - 1001: merkle data (chained)
+     - 1011: merkle data (chained resigned)
 
      For legacy type shreds, the low four bits are set to static patterns.
      For merkle type shreds, the low four bits are set to the number of non-root nodes in the inclusion proof.
      For chained merkle code type shreds, the 3rd highest bit represents if the merkle tree is chained.
-     For chained merkle data type shreds, the 4th highest bit represents if the merkle tree is 
-     chained. */
+     For chained merkle data type shreds, the 4th highest bit represents if the merkle tree is chained.
+     For resigned chained merkle code type shreds, the 4th highest bit represents if the shred is signed
+     For resigned chained merkle data type shreds, the 3th highest bit represents if the shred is signed
+*/
   /* 0x40 */ uchar  variant;
 
   /* Slot number that this shred is part of */
@@ -237,12 +259,11 @@ fd_shred_variant( uchar type,
 FD_FN_PURE static inline ulong
 fd_shred_sz( fd_shred_t const * shred ) {
   uchar type = fd_shred_type( shred->variant );
-  return fd_ulong_if( 
+  return fd_ulong_if(
     type & FD_SHRED_TYPEMASK_CODE,
     FD_SHRED_MAX_SZ,
-    fd_ulong_if( ( type==FD_SHRED_TYPE_MERKLE_DATA ) | ( type==FD_SHRED_TYPE_MERKLE_DATA_CHAINED ),
-      FD_SHRED_MIN_SZ,
-      shred->data.size ) ); /* Legacy data */
+    fd_ulong_if( type==FD_SHRED_TYPE_LEGACY_DATA, shred->data.size, FD_SHRED_MIN_SZ)
+  ); /* Legacy data */
 }
 
 /* fd_shred_header_sz: Returns the header size of a shred.
@@ -281,8 +302,17 @@ fd_shred_merkle_sz( uchar variant ) {
 /* fd_is_chained_shred: Returns true if the shred is a chained merkle data or code shred. */
 FD_FN_CONST static inline uchar
 fd_is_chained_shred( ulong type ) {
-  return ( type == FD_SHRED_TYPE_MERKLE_DATA_CHAINED ) 
-       | ( type == FD_SHRED_TYPE_MERKLE_CODE_CHAINED );
+  return ( type == FD_SHRED_TYPE_MERKLE_DATA_CHAINED )
+       | ( type == FD_SHRED_TYPE_MERKLE_CODE_CHAINED )
+       | ( type == FD_SHRED_TYPE_MERKLE_DATA_CHAINED_RESIGNED )
+       | ( type == FD_SHRED_TYPE_MERKLE_CODE_CHAINED_RESIGNED );
+}
+
+/* fd_is_resigned_shred: Returns true if the shred is resigned by the retransmitter */
+FD_FN_CONST static inline uchar
+fd_is_resigned_shred( ulong type ) {
+  return ( type == FD_SHRED_TYPE_MERKLE_DATA_CHAINED_RESIGNED )
+       | ( type == FD_SHRED_TYPE_MERKLE_CODE_CHAINED_RESIGNED );
 }
 
 /* fd_shred_payload_sz: Returns the payload size of a shred.
@@ -293,19 +323,22 @@ fd_shred_payload_sz( fd_shred_t const * shred ) {
   if( FD_LIKELY( type & FD_SHRED_TYPEMASK_DATA ) ) {
     return shred->data.size - FD_SHRED_DATA_HEADER_SZ;
   } else {
-    return fd_shred_sz( shred ) - FD_SHRED_CODE_HEADER_SZ 
-      - fd_shred_merkle_sz( shred->variant ) 
-      - fd_ulong_if( fd_is_chained_shred( type ), FD_SHRED_MERKLE_ROOT_SZ, 0 );
+    return fd_shred_sz( shred ) - FD_SHRED_CODE_HEADER_SZ
+      - fd_shred_merkle_sz( shred->variant )
+      - fd_ulong_if( fd_is_chained_shred( type ), FD_SHRED_MERKLE_ROOT_SZ, 0 )
+      - fd_ulong_if( fd_is_resigned_shred( type ), FD_SHRED_SIGNATURE_SZ, 0 );
   }
-
 }
 
 /* fd_shred_merkle_off: Returns the byte offset of the merkle inclusion proof of a shred.
 
    The provided shred must have passed validation in fd_shred_parse(). */
 FD_FN_PURE static inline ulong
 fd_shred_merkle_off( fd_shred_t const * shred ) {
-  return fd_shred_sz( shred ) - fd_shred_merkle_sz( shred->variant );
+  ulong type = fd_shred_type( shred->variant );
+  return fd_shred_sz( shred )
+    - fd_shred_merkle_sz( shred->variant )
+    - fd_ulong_if( fd_is_resigned_shred( type ), FD_SHRED_SIGNATURE_SZ, 0 );
 }
 
 /* fd_shred_merkle_nodes: Returns a pointer to the shred's merkle proof data.

diff --git a/src/ballet/shred/test_shred.c b/src/ballet/shred/test_shred.c
@@ -73,21 +73,26 @@ main( int     argc,
 
     /* Test type detection */
     int is_legacy  =  i      ==0x5a ||  i      ==0xa5;
-    int is_merkle  = (i&0xf0)==0x40 || (i&0xf0)==0x80 || (i&0xf0)==0x90 || (i&0xf0)==0x60;
-    int is_data    =  i      ==0xa5 || (i&0xf0)==0x80 || (i&0xf0)==0x90;
-    int is_code    = (i      ==0x5a || (i&0xf0)==0x40 || (i&0xf0)==0x60);
-    int is_chained = ((i&0xf0)==0x90 || (i&0xf0)==0x60 );
-    int is_valid   = (is_legacy^is_merkle) && (is_data^is_code) && (!is_chained || is_merkle);
+    int is_merkle  = (i&0xf0)==FD_SHRED_TYPE_MERKLE_CODE || (i&0xf0)==FD_SHRED_TYPE_MERKLE_DATA
+      || (i&0xf0)==FD_SHRED_TYPE_MERKLE_DATA_CHAINED || (i&0xf0)==FD_SHRED_TYPE_MERKLE_CODE_CHAINED
+      || (i&0xf0)==FD_SHRED_TYPE_MERKLE_DATA_CHAINED_RESIGNED || (i&0xf0)==FD_SHRED_TYPE_MERKLE_CODE_CHAINED_RESIGNED;
+    int is_data    =  i      ==0xa5 || (i&0xf0)==FD_SHRED_TYPE_MERKLE_DATA  || (i&0xf0)==FD_SHRED_TYPE_MERKLE_DATA_CHAINED || (i&0xf0)==FD_SHRED_TYPE_MERKLE_DATA_CHAINED_RESIGNED;
+    int is_code    =  i      ==0x5a || (i&0xf0)==FD_SHRED_TYPE_MERKLE_CODE  || (i&0xf0)==FD_SHRED_TYPE_MERKLE_CODE_CHAINED || (i&0xf0)==FD_SHRED_TYPE_MERKLE_CODE_CHAINED_RESIGNED;
+    int is_resigned = ((i&0xf0)==FD_SHRED_TYPE_MERKLE_DATA_CHAINED_RESIGNED || (i&0xf0)==FD_SHRED_TYPE_MERKLE_CODE_CHAINED_RESIGNED );
+    int is_chained = ((i&0xf0)==FD_SHRED_TYPE_MERKLE_DATA_CHAINED || (i&0xf0)==FD_SHRED_TYPE_MERKLE_CODE_CHAINED ) || is_resigned;
+    int is_valid   = ((is_legacy^is_merkle) && (is_data^is_code) && (!is_chained || is_merkle)) || (is_resigned && is_chained && !is_legacy);
 
     /* Find sizes for shred type */
     ulong header_sz = 0;
-    if( FD_LIKELY( is_data   ) ) header_sz = 0x58;
-    if( FD_LIKELY( is_code   ) ) header_sz = 0x59;
+    if( FD_LIKELY( is_data   ) ) header_sz = FD_SHRED_DATA_HEADER_SZ;
+    if( FD_LIKELY( is_code   ) ) header_sz = FD_SHRED_CODE_HEADER_SZ;
     ulong merkle_sz = 0;
     if( FD_LIKELY( is_merkle ) ) merkle_sz = (i&0x0f)*FD_SHRED_MERKLE_NODE_SZ;
     ulong chained_root_sz = 0;
     if( FD_LIKELY( is_chained ) ) chained_root_sz = FD_SHRED_MERKLE_ROOT_SZ;
-    ulong payload_sz = ((is_merkle&is_data) ? FD_SHRED_MIN_SZ : FD_SHRED_MAX_SZ) - header_sz - merkle_sz - chained_root_sz;
+    ulong resigned_sz = 0;
+    if( FD_LIKELY( is_resigned ) ) resigned_sz = FD_SHRED_SIGNATURE_SZ;
+    ulong payload_sz = ((is_merkle&is_data) ? FD_SHRED_MIN_SZ : FD_SHRED_MAX_SZ) - header_sz - merkle_sz - chained_root_sz - resigned_sz;
 
     if( is_data )
       *(ushort*)(buf+0x56) = (ushort)(payload_sz + header_sz); /* write data.size */

diff --git a/src/flamenco/runtime/fd_blockstore.c b/src/flamenco/runtime/fd_blockstore.c
@@ -597,6 +597,7 @@ fd_blockstore_deshred( fd_blockstore_t * blockstore, ulong slot ) {
     deshredder.shreds        = &shred;
     deshredder.shred_cnt     = 1;
     rc                       = fd_deshredder_next( &deshredder );
+    FD_TEST( rc >= 0 );
 
     shreds_laddr[i].hdr = *shred;
     shreds_laddr[i].off = off;

diff --git a/src/flamenco/runtime/tests/Local.mk b/src/flamenco/runtime/tests/Local.mk
@@ -9,11 +9,12 @@ $(call make-unit-test,test_exec_instr,test_exec_instr,fd_flamenco fd_funk fd_bal
 $(call make-shared,libfd_exec_sol_compat.so,fd_exec_sol_compat,fd_flamenco fd_funk fd_ballet fd_util,$(SECP256K1_LIBS))
 endif
 
-# TODO: add run-runtime-test-3 to the list of run-runtime-test after big merge is done
 run-runtime-test: run-runtime-test-1 run-runtime-test-2 run-runtime-test-3
 
-run-runtime-test-big: $(OBJDIR)/unit-test/test_native_programs $(OBJDIR)/unit-test/test_runtime $(OBJDIR)/bin/fd_frank_ledger
-	OBJDIR=$(OBJDIR) src/flamenco/runtime/tests/run_ledger_tests.sh -l bad-incremental2 -s snapshot-262497545-3sFmKsyF32p4V2HMKaM6s2ymCG64NVcjuxYmen1aKky2.tar.zst  -i incremental-snapshot-262497545-262507921-Asuwpa3yuxsBZuVwsad41S3QHYejcdTdeNcqSHKbxvG1.tar.zst -p 250 -m 80000000 -e 255312010
+run-runtime-test-small: $(OBJDIR)/unit-test/test_runtime $(OBJDIR)/bin/fd_frank_ledger
+#	OBJDIR=$(OBJDIR) src/flamenco/runtime/tests/run_ledger_tests.sh -l v20-ledger
+#	OBJDIR=$(OBJDIR) src/flamenco/runtime/tests/run_ledger_tests.sh -l test-ledger-alt-bn128-1.18.2
+#	OBJDIR=$(OBJDIR) src/flamenco/runtime/tests/run_ledger_tests.sh -l test-ledger-alt-bn128-1.18.7
 
 run-runtime-native: $(OBJDIR)/unit-test/test_native_programs
 	OBJDIR=$(OBJDIR) src/flamenco/runtime/tests/run_native_tests.sh
@@ -45,3 +46,4 @@ run-runtime-test-3: $(OBJDIR)/unit-test/test_runtime $(OBJDIR)/bin/fd_frank_ledg
 	OBJDIR=$(OBJDIR) src/flamenco/runtime/tests/run_ledger_tests.sh -l mainnet-257058865 -s snapshot-257058865-6SFEm7u5pLAhkm4vfiHiN3vMNkmZuyL2ACuaHznU52fi.tar.zst -p 16 -m 5000000 -e 257058870 --zst
 	OBJDIR=$(OBJDIR) src/flamenco/runtime/tests/run_ledger_tests.sh -l mainnet-257059815 -s snapshot-257059815-AmWkVebTmg6ih2VTEjMmU9WtXhT3RygEoSJBHfDpyAG3.tar.zst -p 16 -m 5000000 -e 257059818 --zst
 	OBJDIR=$(OBJDIR) src/flamenco/runtime/tests/run_ledger_tests.sh -l mainnet-257061172 -s snapshot-257061172-8e6cUSMUx2VZZBDzwXjEY6bGkzPgnUmqrDyr4uErG8BF.tar.zst -p 16 -m 5000000 -e 257061175 --zst
+#	OBJDIR=$(OBJDIR) src/flamenco/runtime/tests/run_ledger_tests.sh -l v20-ledger