diff --git a/contrib/checkptify/autodownload.py b/contrib/checkptify/autodownload.py index bf8a168e23..9ab02e079f 100644 --- a/contrib/checkptify/autodownload.py +++ b/contrib/checkptify/autodownload.py @@ -21,31 +21,8 @@ print( f"solana endpoint: {solana_url}" ) def download(url): - while True: - print( f'trying {url}' ) - cmd = f'curl --max-redirs 0 --silent {url}' - proc = subprocess.Popen( cmd, shell=True, stdout=subprocess.PIPE ) - newname = proc.stdout.read().decode("utf-8").split('/')[-1] - if os.path.exists(newname) and os.stat(newname).st_size > 0: - return (newname,False) - if len(newname) == 0: - # We are temporarily banned - print( f'"{cmd}" failed' ) - time.sleep( 10 ) - continue - - print( f'downloading {newname} ...' ) - subprocess.run( 'rm -f tmp', shell=True ) - cmd = f'wget --output-document=tmp --quiet {url}' - subprocess.run( cmd, shell=True ) - if not (os.path.exists('tmp') and os.stat('tmp').st_size > 0): - print( f'"{cmd}" failed' ) - time.sleep( 10 ) - continue - - subprocess.run( f'mv -f tmp {newname}', shell=True ) - print( f'downloaded {newname}' ) - return (newname,True) + cmd = f'wget --no-clobber --trust-server-names {url}' + subprocess.run( cmd, shell=True ) def relink(snap, link): subprocess.run( f'rm -f tmp-link', shell=True ) @@ -54,18 +31,13 @@ def relink(snap, link): print( f'linked {link} to {snap}' ) def rmold(files, keep): - files.sort( key=os.path.getmtime, reverse=True) for i in range(keep, len(files)): os.remove( files[i] ) print( f'removed {files[i]}' ) while True: - (fullsnap,fullsnapnew) = download( f'{solana_url}/snapshot.tar.bz2' ) - (incsnap,incsnapnew) = download( f'{solana_url}/incremental-snapshot.tar.bz2' ) - - if (fullsnapnew or incsnapnew) and (fullsnap.split('-')[1] == incsnap.split('-')[2]): - relink( fullsnap, 'snapshot.tar.bz2' ) - relink( incsnap, 'incremental-snapshot.tar.bz2' ) + download( f'{solana_url}/snapshot.tar.bz2' ) + download( f'{solana_url}/incremental-snapshot.tar.bz2' ) fullfiles = [] incfiles = [] @@ -76,7 +48,20 @@ def rmold(files, keep): elif "snapshot" in file: if file != 'snapshot.tar.bz2': fullfiles.append(file) + + fullfiles.sort( key=(lambda n: int(n.split('-')[1])), reverse=True ); + incfiles.sort( key=(lambda n: int(n.split('-')[3])), reverse=True ); + rmold(fullfiles, 2) rmold(incfiles, 3) + if fullfiles[0].split('-')[1] == incfiles[0].split('-')[2]: + fullname = os.path.realpath(fullfiles[0]) + incname = os.path.realpath(incfiles[0]) + print(f'FULLSNAP={fullname}') + print(f'INCSNAP={incname}') + with open('latest', 'w') as fd: + fd.write(f'FULLSNAP={fullname}\n') + fd.write(f'INCSNAP={incname}\n') + time.sleep(30) diff --git a/src/flamenco/runtime/program/fd_bpf_program_util.c b/src/flamenco/runtime/program/fd_bpf_program_util.c index 5e0fd02626..04e52f965c 100644 --- a/src/flamenco/runtime/program/fd_bpf_program_util.c +++ b/src/flamenco/runtime/program/fd_bpf_program_util.c @@ -228,9 +228,7 @@ fd_bpf_scan_and_create_bpf_program_cache_entry_tpool( fd_exec_slot_ctx_t * slot_ ulong cached_cnt = 0; /* Use random-ish xid to avoid concurrency issues */ - fd_funk_txn_xid_t cache_xid; - cache_xid.ul[0] = fd_log_cpu_id() + 1; - cache_xid.ul[1] = fd_log_thread_id() + 1; + fd_funk_txn_xid_t cache_xid = fd_funk_generate_xid(); fd_funk_txn_t * cache_txn = fd_funk_txn_prepare( funk, slot_ctx->funk_txn, &cache_xid, 1 ); if( !cache_txn ) { @@ -297,9 +295,7 @@ fd_bpf_scan_and_create_bpf_program_cache_entry( fd_exec_slot_ctx_t * slot_ctx, ulong cnt = 0; /* Use random-ish xid to avoid concurrency issues */ - fd_funk_txn_xid_t cache_xid; - cache_xid.ul[0] = fd_log_cpu_id() + 1; - cache_xid.ul[1] = fd_log_thread_id() + 1; + fd_funk_txn_xid_t cache_xid = fd_funk_generate_xid(); fd_funk_txn_t * cache_txn = fd_funk_txn_prepare( funk, slot_ctx->funk_txn, &cache_xid, 1 ); if( !cache_txn ) { diff --git a/src/flamenco/runtime/tests/fd_exec_instr_test.c b/src/flamenco/runtime/tests/fd_exec_instr_test.c index fa282f904a..1475be9474 100644 --- a/src/flamenco/runtime/tests/fd_exec_instr_test.c +++ b/src/flamenco/runtime/tests/fd_exec_instr_test.c @@ -209,11 +209,8 @@ _instr_context_create( fd_exec_instr_test_runner_t * runner, /* Generate unique ID for funk txn */ - static FD_TL ulong xid_seq = 0UL; - fd_funk_txn_xid_t xid[1] = {0}; - xid->ul[0] = xid_seq++; - xid->ul[1] = (ulong)fd_tickcount(); + xid[0] = fd_funk_generate_xid(); /* Create temporary funk transaction and scratch contexts */ @@ -575,11 +572,9 @@ _txn_context_create( fd_exec_instr_test_runner_t * runner, fd_funk_t * funk = runner->funk; /* Generate unique ID for funk txn */ - static FD_TL ulong xid_seq = 0UL; fd_funk_txn_xid_t xid[1] = {0}; - xid->ul[0] = xid_seq++; - xid->ul[1] = (ulong)fd_tickcount(); + xid[0] = fd_funk_generate_xid(); /* Create temporary funk transaction and scratch contexts */ diff --git a/src/funk/fd_funk_txn.c b/src/funk/fd_funk_txn.c index ce2e54e26b..f69527d6c8 100644 --- a/src/funk/fd_funk_txn.c +++ b/src/funk/fd_funk_txn.c @@ -939,6 +939,18 @@ fd_funk_txn_next_rec( fd_funk_t * funk, return rec_map + rec_idx; } +fd_funk_txn_xid_t +fd_funk_generate_xid(void) { + fd_funk_txn_xid_t xid; + static FD_TL ulong seq = 0; + xid.ul[0] = + (fd_log_cpu_id() + 1U)*3138831853UL + + (fd_log_thread_id() + 1U)*9180195821UL + + (++seq)*6208101967UL; + xid.ul[1] = ((ulong)fd_tickcount())*2810745731UL; + return xid; +} + int fd_funk_txn_verify( fd_funk_t * funk ) { fd_wksp_t * wksp = fd_funk_wksp( funk ); /* Previously verified */ diff --git a/src/funk/fd_funk_txn.h b/src/funk/fd_funk_txn.h index 4fdf127b25..5e3a6044c8 100644 --- a/src/funk/fd_funk_txn.h +++ b/src/funk/fd_funk_txn.h @@ -80,6 +80,9 @@ static inline ulong fd_funk_txn_idx ( uint idx ) { return (ulong)idx; } static inline int fd_funk_txn_idx_is_null( ulong idx ) { return idx==FD_FUNK_TXN_IDX_NULL; } +/* Generate a globally unique psuedo-random xid */ +fd_funk_txn_xid_t fd_funk_generate_xid(void); + /* Accessors */ /* fd_funk_txn_cnt returns the number of transactions currently in diff --git a/src/funk/test_funk_base.c b/src/funk/test_funk_base.c index f3dac4356d..8796c8d353 100644 --- a/src/funk/test_funk_base.c +++ b/src/funk/test_funk_base.c @@ -43,20 +43,9 @@ fd_funk_rec_key_set_unique( fd_funk_rec_key_t * key ) { return key; } -static fd_funk_txn_xid_t * -fd_funk_txn_xid_set_unique( fd_funk_txn_xid_t * xid ) { - xid->ul[0] = ++unique_tag; -# if FD_HAS_X86 - xid->ul[1] = (ulong)fd_tickcount(); -# else - xid->ul[1] = 0UL; -# endif - return xid; -} - static fd_funk_xid_key_pair_t * fd_funk_xid_key_pair_set_unique( fd_funk_xid_key_pair_t * pair ) { - fd_funk_txn_xid_set_unique( pair->xid ); + pair->xid[0] = fd_funk_generate_xid(); fd_funk_rec_key_set_unique( pair->key ); return pair; } @@ -98,8 +87,8 @@ main( int argc, FD_TEST( !(z->ul[0] | z->ul[1]) ); for( ulong rem=1000000UL; rem; rem-- ) { - fd_funk_txn_xid_t a[1]; fd_funk_txn_xid_set_unique( a ); - fd_funk_txn_xid_t b[1]; fd_funk_txn_xid_set_unique( b ); + fd_funk_txn_xid_t a[1]; a[0] = fd_funk_generate_xid(); + fd_funk_txn_xid_t b[1]; b[0] = fd_funk_generate_xid(); ulong hash = fd_funk_txn_xid_hash( a, 1234UL ); FD_COMPILER_FORGET( hash ); /**/ hash = fd_funk_txn_xid_hash( b, 1234UL ); FD_COMPILER_FORGET( hash ); diff --git a/src/funk/test_funk_part.c b/src/funk/test_funk_part.c index bef7241430..8e18d6fccf 100644 --- a/src/funk/test_funk_part.c +++ b/src/funk/test_funk_part.c @@ -3,14 +3,6 @@ #if FD_HAS_HOSTED -static fd_funk_txn_xid_t * -fd_funk_txn_xid_set_unique( fd_funk_txn_xid_t * xid ) { - static FD_TL ulong tag = 0UL; - xid->ul[0] = fd_log_thread_id(); - xid->ul[1] = ++tag; - return xid; -} - static fd_funk_rec_key_t * fd_funk_rec_key_set_unique( fd_funk_rec_key_t * key ) { static FD_TL ulong tag = 0UL; @@ -89,7 +81,8 @@ main( int argc, case 0: { /* commit/create a transaction */ if ( txn == NULL || (r&3) ) { fd_funk_txn_xid_t xid[1]; - txn = fd_funk_txn_prepare(funk, txn, fd_funk_txn_xid_set_unique(xid), 0); + xid[0] = fd_funk_generate_xid(); + txn = fd_funk_txn_prepare(funk, txn, xid, 0); FD_TEST(txn); } else { fd_funk_txn_publish(funk, txn, 0); diff --git a/src/funk/test_funk_txn.c b/src/funk/test_funk_txn.c index 471cfc821e..0da4103fdf 100644 --- a/src/funk/test_funk_txn.c +++ b/src/funk/test_funk_txn.c @@ -14,14 +14,6 @@ FD_STATIC_ASSERT( FD_FUNK_TXN_FOOTPRINT==sizeof (fd_funk_txn_t), unit_test ); FD_STATIC_ASSERT( FD_FUNK_TXN_IDX_NULL==(ulong)UINT_MAX, unit_test ); -static fd_funk_txn_xid_t * -fd_funk_txn_xid_set_unique( fd_funk_txn_xid_t * xid ) { - static FD_TL ulong tag = 0UL; - xid->ul[0] = tag++; - xid->ul[1] = fd_log_thread_id(); - return xid; -} - int main( int argc, char ** argv ) { @@ -76,7 +68,7 @@ main( int argc, FD_TEST( !fd_funk_txn_cnt ( map ) ); FD_TEST( !fd_funk_txn_is_full( map ) ); - fd_funk_txn_xid_t recent_xid[ 64 ]; for( ulong idx=0UL; idx<64UL; idx++ ) fd_funk_txn_xid_set_unique( &recent_xid[ idx ] ); + fd_funk_txn_xid_t recent_xid[ 64 ]; for( ulong idx=0UL; idx<64UL; idx++ ) recent_xid[ idx ] = fd_funk_generate_xid(); ulong recent_cursor = 0UL; for( ulong iter=0UL; iter>= 6; fd_funk_txn_t * txn = fd_funk_txn_query( &recent_xid[idx], map ); - fd_funk_txn_xid_t xid[1]; fd_funk_txn_xid_set_unique( xid ); + fd_funk_txn_xid_t xid[1]; + xid[0] = fd_funk_generate_xid(); fd_funk_txn_t * dead = NULL; if( txn_max && !fd_funk_txn_query( fd_funk_txn_xid( &map[0] ), map ) ) dead = &map[0];