From d0bb44d9a31fb2dfc8f2d724345b823b7bef467a Mon Sep 17 00:00:00 2001 From: Josh Siegel Date: Tue, 30 Apr 2024 15:49:02 +0000 Subject: [PATCH] v20-ledger: new test and shred fixes associated with it --- src/ballet/shred/fd_deshredder.c | 3 +- src/ballet/shred/fd_shred.c | 10 +++-- src/ballet/shred/fd_shred.h | 59 ++++++++++++++++++++++------ src/ballet/shred/test_shred.c | 21 ++++++---- src/flamenco/runtime/fd_blockstore.c | 1 + src/flamenco/runtime/tests/Local.mk | 8 ++-- 6 files changed, 74 insertions(+), 28 deletions(-) diff --git a/src/ballet/shred/fd_deshredder.c b/src/ballet/shred/fd_deshredder.c index 2c5254d900..e5ce988c7f 100644 --- a/src/ballet/shred/fd_deshredder.c +++ b/src/ballet/shred/fd_deshredder.c @@ -32,7 +32,8 @@ fd_deshredder_next( fd_deshredder_t * const shredder ) { uchar shred_type = fd_shred_type( shred->variant ); if( FD_UNLIKELY( shred_type!=FD_SHRED_TYPE_LEGACY_DATA && shred_type!=FD_SHRED_TYPE_MERKLE_DATA - && shred_type!=FD_SHRED_TYPE_MERKLE_DATA_CHAINED ) ) + && shred_type!=FD_SHRED_TYPE_MERKLE_DATA_CHAINED + && shred_type!=FD_SHRED_TYPE_MERKLE_DATA_CHAINED_RESIGNED ) ) return -FD_SHRED_EINVAL; /* Ensure entry fits next shred */ diff --git a/src/ballet/shred/fd_shred.c b/src/ballet/shred/fd_shred.c index ece7ae21d4..4ed4f4ad01 100644 --- a/src/ballet/shred/fd_shred.c +++ b/src/ballet/shred/fd_shred.c @@ -17,6 +17,8 @@ fd_shred_parse( uchar const * const buf, (type!=FD_SHRED_TYPE_MERKLE_CODE) & (type!=FD_SHRED_TYPE_MERKLE_DATA_CHAINED) & (type!=FD_SHRED_TYPE_MERKLE_CODE_CHAINED) & + (type!=FD_SHRED_TYPE_MERKLE_DATA_CHAINED_RESIGNED) & + (type!=FD_SHRED_TYPE_MERKLE_CODE_CHAINED_RESIGNED) & (variant!=0xa5 /*FD_SHRED_TYPE_LEGACY_DATA*/ ) & (variant!=0x5a /*FD_SHRED_TYPE_LEGACY_CODE*/ ) ) ) return NULL; @@ -31,9 +33,11 @@ fd_shred_parse( uchar const * const buf, ulong payload_sz; /* only present for chained merkle shreds */ - ulong previous_merkle_root_sz = fd_ulong_if( - (type == FD_SHRED_TYPE_MERKLE_DATA_CHAINED) | (type == FD_SHRED_TYPE_MERKLE_CODE_CHAINED), FD_SHRED_MERKLE_ROOT_SZ, 0UL ); - + ulong previous_merkle_root_sz = fd_ulong_if( + (type == FD_SHRED_TYPE_MERKLE_DATA_CHAINED) | (type == FD_SHRED_TYPE_MERKLE_CODE_CHAINED) | + (type == FD_SHRED_TYPE_MERKLE_DATA_CHAINED_RESIGNED) | (type == FD_SHRED_TYPE_MERKLE_CODE_CHAINED_RESIGNED) + , FD_SHRED_MERKLE_ROOT_SZ, 0UL ); + if( FD_LIKELY( type & FD_SHRED_TYPEMASK_DATA ) ) { if( FD_UNLIKELY( shred->data.sizedata.size - header_sz; /* between 0 and USHORT_MAX */ diff --git a/src/ballet/shred/fd_shred.h b/src/ballet/shred/fd_shred.h index 2ff224bab6..a56dac48f1 100644 --- a/src/ballet/shred/fd_shred.h +++ b/src/ballet/shred/fd_shred.h @@ -31,6 +31,12 @@ | Merkle node #1 | 20 bytes .......................... + for resigned shreds shreds, followed by: + + +------------------------+ + | signature | 64 bytes + .......................... + ### Shredding For a given input data blob (usually an entry batch), @@ -58,6 +64,11 @@ The length of the inclusion proof is indicated by the variant field. + ### resigned shreds + + Resigned shreds allow for an additional signature to be added on to lock down + the retransmitter for turbine propagation + ### Authentication Shreds are signed by the block producer. @@ -95,6 +106,11 @@ /* FD_SHRED_TYPE_MERKLE_CODE_CHAINED: A shred carrying Reed-Solomon ECC and a chained merkle inclusion proof. */ #define FD_SHRED_TYPE_MERKLE_CODE_CHAINED ((uchar)0x60) +/* FD_SHRED_TYPE_MERKLE_DATA_CHAINED_RESIGNED: A shred carrying raw binary data and a chained merkle inclusion proof and resigned. */ +#define FD_SHRED_TYPE_MERKLE_DATA_CHAINED_RESIGNED ((uchar)0xB0) +/* FD_SHRED_TYPE_MERKLE_CODE_CHAINED_RESIGNED: A shred carrying Reed-Solomon ECC and a chained merkle inclusion proof and resigned. */ +#define FD_SHRED_TYPE_MERKLE_CODE_CHAINED_RESIGNED ((uchar)0x70) + /* FD_SHRED_TYPEMASK_DATA: bitwise AND with type matches data shred */ #define FD_SHRED_TYPEMASK_DATA FD_SHRED_TYPE_MERKLE_DATA /* FD_SHRED_TYPEMASK_CODE: bitwise AND with type matches code shred */ @@ -104,6 +120,8 @@ #define FD_SHRED_MERKLE_ROOT_SZ (32UL) /* FD_SHRED_MERKLE_NODE_SZ: the size of a merkle inclusion proof node in bytes. */ #define FD_SHRED_MERKLE_NODE_SZ (20UL) +/* FD_SHRED_SIGNATURE_SZ: the size of a signature in a shred. */ +#define FD_SHRED_SIGNATURE_SZ (64UL) /* A merkle inclusion proof node. */ typedef uchar fd_shred_merkle_t[FD_SHRED_MERKLE_NODE_SZ]; @@ -149,14 +167,18 @@ struct __attribute__((packed)) fd_shred { - 1010: legacy data - 0100: merkle code - 0110: merkle code (chained) + - 0111: merkle code (chained resigned) - 1000: merkle data - 1001: merkle data (chained) + - 1011: merkle data (chained resigned) For legacy type shreds, the low four bits are set to static patterns. For merkle type shreds, the low four bits are set to the number of non-root nodes in the inclusion proof. For chained merkle code type shreds, the 3rd highest bit represents if the merkle tree is chained. - For chained merkle data type shreds, the 4th highest bit represents if the merkle tree is - chained. */ + For chained merkle data type shreds, the 4th highest bit represents if the merkle tree is chained. + For resigned chained merkle code type shreds, the 4th highest bit represents if the shred is signed + For resigned chained merkle data type shreds, the 3th highest bit represents if the shred is signed +*/ /* 0x40 */ uchar variant; /* Slot number that this shred is part of */ @@ -237,12 +259,11 @@ fd_shred_variant( uchar type, FD_FN_PURE static inline ulong fd_shred_sz( fd_shred_t const * shred ) { uchar type = fd_shred_type( shred->variant ); - return fd_ulong_if( + return fd_ulong_if( type & FD_SHRED_TYPEMASK_CODE, FD_SHRED_MAX_SZ, - fd_ulong_if( ( type==FD_SHRED_TYPE_MERKLE_DATA ) | ( type==FD_SHRED_TYPE_MERKLE_DATA_CHAINED ), - FD_SHRED_MIN_SZ, - shred->data.size ) ); /* Legacy data */ + fd_ulong_if( type==FD_SHRED_TYPE_LEGACY_DATA, shred->data.size, FD_SHRED_MIN_SZ) + ); /* Legacy data */ } /* fd_shred_header_sz: Returns the header size of a shred. @@ -281,8 +302,17 @@ fd_shred_merkle_sz( uchar variant ) { /* fd_is_chained_shred: Returns true if the shred is a chained merkle data or code shred. */ FD_FN_CONST static inline uchar fd_is_chained_shred( ulong type ) { - return ( type == FD_SHRED_TYPE_MERKLE_DATA_CHAINED ) - | ( type == FD_SHRED_TYPE_MERKLE_CODE_CHAINED ); + return ( type == FD_SHRED_TYPE_MERKLE_DATA_CHAINED ) + | ( type == FD_SHRED_TYPE_MERKLE_CODE_CHAINED ) + | ( type == FD_SHRED_TYPE_MERKLE_DATA_CHAINED_RESIGNED ) + | ( type == FD_SHRED_TYPE_MERKLE_CODE_CHAINED_RESIGNED ); +} + +/* fd_is_resigned_shred: Returns true if the shred is resigned by the retransmitter */ +FD_FN_CONST static inline uchar +fd_is_resigned_shred( ulong type ) { + return ( type == FD_SHRED_TYPE_MERKLE_DATA_CHAINED_RESIGNED ) + | ( type == FD_SHRED_TYPE_MERKLE_CODE_CHAINED_RESIGNED ); } /* fd_shred_payload_sz: Returns the payload size of a shred. @@ -293,11 +323,11 @@ fd_shred_payload_sz( fd_shred_t const * shred ) { if( FD_LIKELY( type & FD_SHRED_TYPEMASK_DATA ) ) { return shred->data.size - FD_SHRED_DATA_HEADER_SZ; } else { - return fd_shred_sz( shred ) - FD_SHRED_CODE_HEADER_SZ - - fd_shred_merkle_sz( shred->variant ) - - fd_ulong_if( fd_is_chained_shred( type ), FD_SHRED_MERKLE_ROOT_SZ, 0 ); + return fd_shred_sz( shred ) - FD_SHRED_CODE_HEADER_SZ + - fd_shred_merkle_sz( shred->variant ) + - fd_ulong_if( fd_is_chained_shred( type ), FD_SHRED_MERKLE_ROOT_SZ, 0 ) + - fd_ulong_if( fd_is_resigned_shred( type ), FD_SHRED_SIGNATURE_SZ, 0 ); } - } /* fd_shred_merkle_off: Returns the byte offset of the merkle inclusion proof of a shred. @@ -305,7 +335,10 @@ fd_shred_payload_sz( fd_shred_t const * shred ) { The provided shred must have passed validation in fd_shred_parse(). */ FD_FN_PURE static inline ulong fd_shred_merkle_off( fd_shred_t const * shred ) { - return fd_shred_sz( shred ) - fd_shred_merkle_sz( shred->variant ); + ulong type = fd_shred_type( shred->variant ); + return fd_shred_sz( shred ) + - fd_shred_merkle_sz( shred->variant ) + - fd_ulong_if( fd_is_resigned_shred( type ), FD_SHRED_SIGNATURE_SZ, 0 ); } /* fd_shred_merkle_nodes: Returns a pointer to the shred's merkle proof data. diff --git a/src/ballet/shred/test_shred.c b/src/ballet/shred/test_shred.c index 108842b756..abd161e2d8 100644 --- a/src/ballet/shred/test_shred.c +++ b/src/ballet/shred/test_shred.c @@ -73,21 +73,26 @@ main( int argc, /* Test type detection */ int is_legacy = i ==0x5a || i ==0xa5; - int is_merkle = (i&0xf0)==0x40 || (i&0xf0)==0x80 || (i&0xf0)==0x90 || (i&0xf0)==0x60; - int is_data = i ==0xa5 || (i&0xf0)==0x80 || (i&0xf0)==0x90; - int is_code = (i ==0x5a || (i&0xf0)==0x40 || (i&0xf0)==0x60); - int is_chained = ((i&0xf0)==0x90 || (i&0xf0)==0x60 ); - int is_valid = (is_legacy^is_merkle) && (is_data^is_code) && (!is_chained || is_merkle); + int is_merkle = (i&0xf0)==FD_SHRED_TYPE_MERKLE_CODE || (i&0xf0)==FD_SHRED_TYPE_MERKLE_DATA + || (i&0xf0)==FD_SHRED_TYPE_MERKLE_DATA_CHAINED || (i&0xf0)==FD_SHRED_TYPE_MERKLE_CODE_CHAINED + || (i&0xf0)==FD_SHRED_TYPE_MERKLE_DATA_CHAINED_RESIGNED || (i&0xf0)==FD_SHRED_TYPE_MERKLE_CODE_CHAINED_RESIGNED; + int is_data = i ==0xa5 || (i&0xf0)==FD_SHRED_TYPE_MERKLE_DATA || (i&0xf0)==FD_SHRED_TYPE_MERKLE_DATA_CHAINED || (i&0xf0)==FD_SHRED_TYPE_MERKLE_DATA_CHAINED_RESIGNED; + int is_code = i ==0x5a || (i&0xf0)==FD_SHRED_TYPE_MERKLE_CODE || (i&0xf0)==FD_SHRED_TYPE_MERKLE_CODE_CHAINED || (i&0xf0)==FD_SHRED_TYPE_MERKLE_CODE_CHAINED_RESIGNED; + int is_resigned = ((i&0xf0)==FD_SHRED_TYPE_MERKLE_DATA_CHAINED_RESIGNED || (i&0xf0)==FD_SHRED_TYPE_MERKLE_CODE_CHAINED_RESIGNED ); + int is_chained = ((i&0xf0)==FD_SHRED_TYPE_MERKLE_DATA_CHAINED || (i&0xf0)==FD_SHRED_TYPE_MERKLE_CODE_CHAINED ) || is_resigned; + int is_valid = ((is_legacy^is_merkle) && (is_data^is_code) && (!is_chained || is_merkle)) || (is_resigned && is_chained && !is_legacy); /* Find sizes for shred type */ ulong header_sz = 0; - if( FD_LIKELY( is_data ) ) header_sz = 0x58; - if( FD_LIKELY( is_code ) ) header_sz = 0x59; + if( FD_LIKELY( is_data ) ) header_sz = FD_SHRED_DATA_HEADER_SZ; + if( FD_LIKELY( is_code ) ) header_sz = FD_SHRED_CODE_HEADER_SZ; ulong merkle_sz = 0; if( FD_LIKELY( is_merkle ) ) merkle_sz = (i&0x0f)*FD_SHRED_MERKLE_NODE_SZ; ulong chained_root_sz = 0; if( FD_LIKELY( is_chained ) ) chained_root_sz = FD_SHRED_MERKLE_ROOT_SZ; - ulong payload_sz = ((is_merkle&is_data) ? FD_SHRED_MIN_SZ : FD_SHRED_MAX_SZ) - header_sz - merkle_sz - chained_root_sz; + ulong resigned_sz = 0; + if( FD_LIKELY( is_resigned ) ) resigned_sz = FD_SHRED_SIGNATURE_SZ; + ulong payload_sz = ((is_merkle&is_data) ? FD_SHRED_MIN_SZ : FD_SHRED_MAX_SZ) - header_sz - merkle_sz - chained_root_sz - resigned_sz; if( is_data ) *(ushort*)(buf+0x56) = (ushort)(payload_sz + header_sz); /* write data.size */ diff --git a/src/flamenco/runtime/fd_blockstore.c b/src/flamenco/runtime/fd_blockstore.c index b6e0925a99..bd91ce1829 100644 --- a/src/flamenco/runtime/fd_blockstore.c +++ b/src/flamenco/runtime/fd_blockstore.c @@ -597,6 +597,7 @@ fd_blockstore_deshred( fd_blockstore_t * blockstore, ulong slot ) { deshredder.shreds = &shred; deshredder.shred_cnt = 1; rc = fd_deshredder_next( &deshredder ); + FD_TEST( rc >= 0 ); shreds_laddr[i].hdr = *shred; shreds_laddr[i].off = off; diff --git a/src/flamenco/runtime/tests/Local.mk b/src/flamenco/runtime/tests/Local.mk index b333d62832..fcf2ec28ce 100644 --- a/src/flamenco/runtime/tests/Local.mk +++ b/src/flamenco/runtime/tests/Local.mk @@ -9,11 +9,12 @@ $(call make-unit-test,test_exec_instr,test_exec_instr,fd_flamenco fd_funk fd_bal $(call make-shared,libfd_exec_sol_compat.so,fd_exec_sol_compat,fd_flamenco fd_funk fd_ballet fd_util,$(SECP256K1_LIBS)) endif -# TODO: add run-runtime-test-3 to the list of run-runtime-test after big merge is done run-runtime-test: run-runtime-test-1 run-runtime-test-2 run-runtime-test-3 -run-runtime-test-big: $(OBJDIR)/unit-test/test_native_programs $(OBJDIR)/unit-test/test_runtime $(OBJDIR)/bin/fd_frank_ledger - OBJDIR=$(OBJDIR) src/flamenco/runtime/tests/run_ledger_tests.sh -l bad-incremental2 -s snapshot-262497545-3sFmKsyF32p4V2HMKaM6s2ymCG64NVcjuxYmen1aKky2.tar.zst -i incremental-snapshot-262497545-262507921-Asuwpa3yuxsBZuVwsad41S3QHYejcdTdeNcqSHKbxvG1.tar.zst -p 250 -m 80000000 -e 255312010 +run-runtime-test-small: $(OBJDIR)/unit-test/test_runtime $(OBJDIR)/bin/fd_frank_ledger +# OBJDIR=$(OBJDIR) src/flamenco/runtime/tests/run_ledger_tests.sh -l v20-ledger +# OBJDIR=$(OBJDIR) src/flamenco/runtime/tests/run_ledger_tests.sh -l test-ledger-alt-bn128-1.18.2 +# OBJDIR=$(OBJDIR) src/flamenco/runtime/tests/run_ledger_tests.sh -l test-ledger-alt-bn128-1.18.7 run-runtime-native: $(OBJDIR)/unit-test/test_native_programs OBJDIR=$(OBJDIR) src/flamenco/runtime/tests/run_native_tests.sh @@ -45,3 +46,4 @@ run-runtime-test-3: $(OBJDIR)/unit-test/test_runtime $(OBJDIR)/bin/fd_frank_ledg OBJDIR=$(OBJDIR) src/flamenco/runtime/tests/run_ledger_tests.sh -l mainnet-257058865 -s snapshot-257058865-6SFEm7u5pLAhkm4vfiHiN3vMNkmZuyL2ACuaHznU52fi.tar.zst -p 16 -m 5000000 -e 257058870 --zst OBJDIR=$(OBJDIR) src/flamenco/runtime/tests/run_ledger_tests.sh -l mainnet-257059815 -s snapshot-257059815-AmWkVebTmg6ih2VTEjMmU9WtXhT3RygEoSJBHfDpyAG3.tar.zst -p 16 -m 5000000 -e 257059818 --zst OBJDIR=$(OBJDIR) src/flamenco/runtime/tests/run_ledger_tests.sh -l mainnet-257061172 -s snapshot-257061172-8e6cUSMUx2VZZBDzwXjEY6bGkzPgnUmqrDyr4uErG8BF.tar.zst -p 16 -m 5000000 -e 257061175 --zst +# OBJDIR=$(OBJDIR) src/flamenco/runtime/tests/run_ledger_tests.sh -l v20-ledger