added funk function to generate unique xid

contrib/checkptify/autodownload.py

@@ -21,31 +21,8 @@
 print( f"solana endpoint: {solana_url}" )
 
 def download(url):
-    while True:
-        print( f'trying {url}' )
-        cmd = f'curl --max-redirs 0 --silent {url}'
-        proc = subprocess.Popen( cmd, shell=True, stdout=subprocess.PIPE )
-        newname = proc.stdout.read().decode("utf-8").split('/')[-1]
-        if os.path.exists(newname) and os.stat(newname).st_size > 0:
-            return (newname,False)
-        if len(newname) == 0:
-            # We are temporarily banned
-            print( f'"{cmd}" failed' )
-            time.sleep( 10 )
-            continue
-
-        print( f'downloading {newname} ...' )
-        subprocess.run( 'rm -f tmp', shell=True )
-        cmd = f'wget --output-document=tmp --quiet {url}'
-        subprocess.run( cmd, shell=True )
-        if not (os.path.exists('tmp') and os.stat('tmp').st_size > 0):
-            print( f'"{cmd}" failed' )
-            time.sleep( 10 )
-            continue
-
-        subprocess.run( f'mv -f tmp {newname}', shell=True )
-        print( f'downloaded {newname}' )
-        return (newname,True)
+    cmd = f'wget --no-clobber --trust-server-names {url}'
+    subprocess.run( cmd, shell=True )
 
 def relink(snap, link):
     subprocess.run( f'rm -f tmp-link', shell=True )
@@ -54,18 +31,13 @@ def relink(snap, link):
     print( f'linked {link} to {snap}' )
 
 def rmold(files, keep):
-    files.sort( key=os.path.getmtime, reverse=True)
     for i in range(keep, len(files)):
         os.remove( files[i] )
         print( f'removed {files[i]}' )
 
 while True:
-    (fullsnap,fullsnapnew) = download( f'{solana_url}/snapshot.tar.bz2' )
-    (incsnap,incsnapnew) = download( f'{solana_url}/incremental-snapshot.tar.bz2' )
-
-    if (fullsnapnew or incsnapnew) and (fullsnap.split('-')[1] == incsnap.split('-')[2]):
-        relink( fullsnap, 'snapshot.tar.bz2' )
-        relink( incsnap, 'incremental-snapshot.tar.bz2' )
+    download( f'{solana_url}/snapshot.tar.bz2' )
+    download( f'{solana_url}/incremental-snapshot.tar.bz2' )
 
     fullfiles = []
     incfiles = []
@@ -76,7 +48,20 @@ def rmold(files, keep):
         elif "snapshot" in file:
             if file != 'snapshot.tar.bz2':
                 fullfiles.append(file)
+
+    fullfiles.sort( key=(lambda n: int(n.split('-')[1])), reverse=True );
+    incfiles.sort( key=(lambda n: int(n.split('-')[3])), reverse=True );
+
     rmold(fullfiles, 2)
     rmold(incfiles, 3)
 
+    if fullfiles[0].split('-')[1] == incfiles[0].split('-')[2]:
+        fullname = os.path.realpath(fullfiles[0])
+        incname = os.path.realpath(incfiles[0])
+        print(f'FULLSNAP={fullname}')
+        print(f'INCSNAP={incname}')
+        with open('latest', 'w') as fd:
+            fd.write(f'FULLSNAP={fullname}\n')
+            fd.write(f'INCSNAP={incname}\n')
+
     time.sleep(30)

src/flamenco/runtime/program/fd_bpf_program_util.c

@@ -228,9 +228,7 @@ fd_bpf_scan_and_create_bpf_program_cache_entry_tpool( fd_exec_slot_ctx_t * slot_
   ulong cached_cnt = 0;
 
   /* Use random-ish xid to avoid concurrency issues */
-  fd_funk_txn_xid_t cache_xid;
-  cache_xid.ul[0] = fd_log_cpu_id() + 1;
-  cache_xid.ul[1] = fd_log_thread_id() + 1;
+  fd_funk_txn_xid_t cache_xid = fd_funk_generate_xid();
 
   fd_funk_txn_t * cache_txn = fd_funk_txn_prepare( funk, slot_ctx->funk_txn, &cache_xid, 1 );
   if( !cache_txn ) {
@@ -297,9 +295,7 @@ fd_bpf_scan_and_create_bpf_program_cache_entry( fd_exec_slot_ctx_t * slot_ctx,
   ulong cnt = 0;
 
   /* Use random-ish xid to avoid concurrency issues */
-  fd_funk_txn_xid_t cache_xid;
-  cache_xid.ul[0] = fd_log_cpu_id() + 1;
-  cache_xid.ul[1] = fd_log_thread_id() + 1;
+  fd_funk_txn_xid_t cache_xid = fd_funk_generate_xid();
 
   fd_funk_txn_t * cache_txn = fd_funk_txn_prepare( funk, slot_ctx->funk_txn, &cache_xid, 1 );
   if( !cache_txn ) {

src/flamenco/runtime/tests/fd_exec_instr_test.c

@@ -209,11 +209,8 @@ _instr_context_create( fd_exec_instr_test_runner_t *        runner,
 
   /* Generate unique ID for funk txn */
 
-  static FD_TL ulong xid_seq = 0UL;
-
   fd_funk_txn_xid_t xid[1] = {0};
-  xid->ul[0] = xid_seq++;
-  xid->ul[1] = (ulong)fd_tickcount();
+  xid[0] = fd_funk_generate_xid();
 
   /* Create temporary funk transaction and scratch contexts */
 
@@ -575,11 +572,9 @@ _txn_context_create( fd_exec_instr_test_runner_t *      runner,
   fd_funk_t * funk = runner->funk;
 
   /* Generate unique ID for funk txn */
-  static FD_TL ulong xid_seq = 0UL;
 
   fd_funk_txn_xid_t xid[1] = {0};
-  xid->ul[0] = xid_seq++;
-  xid->ul[1] = (ulong)fd_tickcount();
+  xid[0] = fd_funk_generate_xid();
 
   /* Create temporary funk transaction and scratch contexts */
 

src/funk/fd_funk_txn.c

@@ -939,6 +939,18 @@ fd_funk_txn_next_rec( fd_funk_t *           funk,
   return rec_map + rec_idx;
 }
 
+fd_funk_txn_xid_t
+fd_funk_generate_xid(void) {
+  fd_funk_txn_xid_t xid;
+  static FD_TL ulong seq = 0;
+  xid.ul[0] =
+    (fd_log_cpu_id() + 1U)*3138831853UL +
+    (fd_log_thread_id() + 1U)*9180195821UL +
+    (++seq)*6208101967UL;
+  xid.ul[1] = ((ulong)fd_tickcount())*2810745731UL;
+  return xid;
+}
+
 int
 fd_funk_txn_verify( fd_funk_t * funk ) {
   fd_wksp_t *     wksp    = fd_funk_wksp( funk );          /* Previously verified */

src/funk/fd_funk_txn.h

@@ -80,6 +80,9 @@ static inline ulong fd_funk_txn_idx ( uint  idx ) { return (ulong)idx; }
 
 static inline int fd_funk_txn_idx_is_null( ulong idx ) { return idx==FD_FUNK_TXN_IDX_NULL; }
 
+/* Generate a globally unique psuedo-random xid */
+fd_funk_txn_xid_t fd_funk_generate_xid(void);
+
 /* Accessors */
 
 /* fd_funk_txn_cnt returns the number of transactions currently in

src/funk/test_funk_base.c

@@ -43,20 +43,9 @@ fd_funk_rec_key_set_unique( fd_funk_rec_key_t * key ) {
   return key;
 }
 
-static fd_funk_txn_xid_t *
-fd_funk_txn_xid_set_unique( fd_funk_txn_xid_t * xid ) {
-  xid->ul[0] = ++unique_tag;
-# if FD_HAS_X86
-  xid->ul[1] = (ulong)fd_tickcount();
-# else
-  xid->ul[1] = 0UL;
-# endif
-  return xid;
-}
-
 static fd_funk_xid_key_pair_t *
 fd_funk_xid_key_pair_set_unique( fd_funk_xid_key_pair_t * pair ) {
-  fd_funk_txn_xid_set_unique( pair->xid );
+  pair->xid[0] = fd_funk_generate_xid();
   fd_funk_rec_key_set_unique( pair->key );
   return pair;
 }
@@ -98,8 +87,8 @@ main( int     argc,
   FD_TEST( !(z->ul[0] | z->ul[1]) );
 
   for( ulong rem=1000000UL; rem; rem-- ) {
-    fd_funk_txn_xid_t a[1]; fd_funk_txn_xid_set_unique( a );
-    fd_funk_txn_xid_t b[1]; fd_funk_txn_xid_set_unique( b );
+    fd_funk_txn_xid_t a[1]; a[0] = fd_funk_generate_xid();
+    fd_funk_txn_xid_t b[1]; b[0] = fd_funk_generate_xid();
 
     ulong hash = fd_funk_txn_xid_hash( a, 1234UL ); FD_COMPILER_FORGET( hash );
     /**/  hash = fd_funk_txn_xid_hash( b, 1234UL ); FD_COMPILER_FORGET( hash );

src/funk/test_funk_part.c

@@ -3,14 +3,6 @@
 
 #if FD_HAS_HOSTED
 
-static fd_funk_txn_xid_t *
-fd_funk_txn_xid_set_unique( fd_funk_txn_xid_t * xid ) {
-  static FD_TL ulong tag = 0UL;
-  xid->ul[0] = fd_log_thread_id();
-  xid->ul[1] = ++tag;
-  return xid;
-}
-
 static fd_funk_rec_key_t *
 fd_funk_rec_key_set_unique( fd_funk_rec_key_t * key ) {
   static FD_TL ulong tag = 0UL;
@@ -89,7 +81,8 @@ main( int     argc,
     case 0: { /* commit/create a transaction */
       if ( txn == NULL || (r&3) ) {
         fd_funk_txn_xid_t xid[1];
-        txn = fd_funk_txn_prepare(funk, txn, fd_funk_txn_xid_set_unique(xid), 0);
+        xid[0] = fd_funk_generate_xid();
+        txn = fd_funk_txn_prepare(funk, txn, xid, 0);
         FD_TEST(txn);
       } else {
         fd_funk_txn_publish(funk, txn, 0);

src/funk/test_funk_txn.c

@@ -14,14 +14,6 @@ FD_STATIC_ASSERT( FD_FUNK_TXN_FOOTPRINT==sizeof (fd_funk_txn_t), unit_test );
 
 FD_STATIC_ASSERT( FD_FUNK_TXN_IDX_NULL==(ulong)UINT_MAX, unit_test );
 
-static fd_funk_txn_xid_t *
-fd_funk_txn_xid_set_unique( fd_funk_txn_xid_t * xid ) {
-  static FD_TL ulong tag = 0UL;
-  xid->ul[0] = tag++;
-  xid->ul[1] = fd_log_thread_id();
-  return xid;
-}
-
 int
 main( int     argc,
       char ** argv ) {
@@ -76,7 +68,7 @@ main( int     argc,
   FD_TEST( !fd_funk_txn_cnt    ( map ) );
   FD_TEST( !fd_funk_txn_is_full( map ) );
 
-  fd_funk_txn_xid_t recent_xid[ 64 ]; for( ulong idx=0UL; idx<64UL; idx++ ) fd_funk_txn_xid_set_unique( &recent_xid[ idx ] );
+  fd_funk_txn_xid_t recent_xid[ 64 ]; for( ulong idx=0UL; idx<64UL; idx++ ) recent_xid[ idx ] = fd_funk_generate_xid();
   ulong recent_cursor = 0UL;
 
   for( ulong iter=0UL; iter<iter_max; iter++ ) {
@@ -111,7 +103,8 @@ main( int     argc,
     }
 
     case 2: { /* look up never seen xid (always fail) */
-      fd_funk_txn_xid_t xid[1]; fd_funk_txn_xid_set_unique( xid );
+      fd_funk_txn_xid_t xid[1];
+      xid[0] = fd_funk_generate_xid();
       fd_funk_txn_t * txn = fd_funk_txn_query( xid, map );
       FD_TEST( !txn );
       break;
@@ -136,7 +129,8 @@ main( int     argc,
     }
 
     case 5: { /* prepare from most recent published never seen xid (succeed if not full) */
-      fd_funk_txn_xid_t const * xid = fd_funk_txn_xid_set_unique( &recent_xid[ recent_cursor ] );
+      fd_funk_txn_xid_t * xid = &recent_xid[ recent_cursor ];
+      *xid = fd_funk_generate_xid();
       recent_cursor = (recent_cursor+1UL) & 63UL;
       int is_full = fd_funk_txn_is_full( map );
       fd_funk_txn_t * txn = fd_funk_txn_prepare( funk, NULL, xid, verbose );
@@ -172,7 +166,8 @@ main( int     argc,
       uint idx; RANDOM_SET_BIT_IDX( live_pmap );
       fd_funk_txn_t * parent = fd_funk_txn_query( &recent_xid[idx], map );
       FD_TEST( parent && fd_funk_txn_xid_eq( fd_funk_txn_xid( parent ), &recent_xid[idx] ) );
-      fd_funk_txn_xid_t const * xid = fd_funk_txn_xid_set_unique( &recent_xid[ recent_cursor ] );
+      fd_funk_txn_xid_t * xid = &recent_xid[ recent_cursor ];
+      *xid = fd_funk_generate_xid();
       recent_cursor = (recent_cursor+1UL) & 63UL;
       int is_full = fd_funk_txn_is_full( map );
       fd_funk_txn_t * txn = fd_funk_txn_prepare( funk, parent, xid, verbose );
@@ -200,7 +195,8 @@ main( int     argc,
     }
 
     case 11: { /* cancel a never seen xid (should always be 0) */
-      fd_funk_txn_xid_t xid[1]; fd_funk_txn_xid_set_unique( xid );
+      fd_funk_txn_xid_t xid[1];
+      xid[0] = fd_funk_generate_xid();
       fd_funk_txn_t * txn = fd_funk_txn_query( xid, map );
       FD_TEST( !txn );
       FD_TEST( fd_funk_txn_cancel( funk, txn, verbose )==0UL );
@@ -227,7 +223,8 @@ main( int     argc,
     }
 
     case 14: { /* publish a never seen xid (should always be 0) */
-      fd_funk_txn_xid_t xid[1]; fd_funk_txn_xid_set_unique( xid );
+      fd_funk_txn_xid_t xid[1];
+      xid[0] = fd_funk_generate_xid();
       fd_funk_txn_t * txn = fd_funk_txn_query( xid, map );
       FD_TEST( !txn );
       FD_TEST( fd_funk_txn_publish( funk, txn, verbose )==0UL );
@@ -237,7 +234,8 @@ main( int     argc,
     default: { /* various sanity checks */
       uint idx = r & 63U; r >>= 6;
       fd_funk_txn_t * txn = fd_funk_txn_query( &recent_xid[idx], map );
-      fd_funk_txn_xid_t xid[1]; fd_funk_txn_xid_set_unique( xid );
+      fd_funk_txn_xid_t xid[1];
+      xid[0] = fd_funk_generate_xid();
 
       fd_funk_txn_t * dead = NULL;
       if( txn_max && !fd_funk_txn_query( fd_funk_txn_xid( &map[0] ), map ) ) dead = &map[0];