Merge branch 'main' into ripatel/openssl-3.1.2

.github/workflows-disabled/fuzz_artifacts.yml → .github/workflows/fuzz_artifacts.yml

@@ -2,8 +2,6 @@ name: Make Fuzz Artifacts
 on:
   pull_request:
   push:
-    branches:
-      - main
   workflow_dispatch:
   merge_group:
 jobs:
@@ -21,16 +19,16 @@ jobs:
 
       - uses: dtolnay/[email protected]
 
-      - name: Set extras
-        run: |
-          echo "EXTRAS=static" >> "$GITHUB_ENV"
+      - uses: firedancer-io/fuzzbot-builder@main
+        name: Build with ASAN on Ubuntu 20.04
+        env:
+          MACHINE: linux_clang_x86_64_fuzz_asan
+        with:
+          command: ./deps.sh install && make --output-sync=target -j fuzz-test
 
-      - name: Prepare asan
+      - name: List Artifacts
         run: |
-          echo "MACHINE=linux_clang_x86_64_fuzz_asan" >> "$GITHUB_ENV"
-
-      - uses: firedancer-io/alpine-builder@main
-        name: Build with asan on Alpine
+          ls build/linux/clang/x86_64_fuzz_asan/fuzz-test
 
       - uses: firedancer-io/clusterfuzz-action@main
         if: ${{ github.ref == 'refs/heads/main' }}

.github/workflows/make_test.yml

@@ -69,3 +69,21 @@ jobs:
 
       - name: Run unit tests
         run: make -k -j2 --output-sync=target run-unit-test
+
+  frank-single-transaction:
+    runs-on: [self-hosted, Linux, X64]
+    strategy:
+      matrix:
+        compiler: [gcc]
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: recursive
+
+      - uses: dtolnay/[email protected]
+
+      - name: Run
+        run: ./src/test/frank-single-transaction.sh

activate

@@ -6,7 +6,7 @@ if ! command -v module >/dev/null 2>&1; then
 fi
 
 module purge
-module load gcc-9.3.0
+module load gcc-12.2.0
 module load Python-3.9
 module load openssl-1.1.1s+quic1
 module list

activate-gcc

@@ -5,9 +5,9 @@
 if [ $# -gt 0 ] && [ "$1" = "install" ]; then
   # Install Packages needed for dev hosts
   sudo yum install                        \
-    jump_module_gcc-9.3.0
+    jump_module_gcc-12.2.0
 fi
 
 module purge
-module load gcc-9.3.0
+module load gcc-12.2.0
 module list

src/app/fdctl/Local.mk

@@ -5,7 +5,8 @@ ifdef FD_HAS_DOUBLE
 $(call make-lib,fd_fdctl)
 $(call add-objs,main config security utility run keygen monitor/monitor monitor/helper configure/configure configure/large_pages configure/sysctl configure/shmem configure/xdp configure/xdp_leftover configure/ethtool configure/workspace_leftover configure/workspace,fd_fdctl)
 $(call make-bin,fdctl,main1,fd_fdctl fd_frank fd_disco fd_ballet fd_tango fd_util fd_quic solana_validator_fd)
-$(OBJDIR)/bin/fdctl: src/app/fdctl/config/default.toml
+$(OBJDIR)/obj/app/fdctl/configure/xdp.o: src/tango/xdp/fd_xdp_redirect_prog.o
+$(OBJDIR)/obj/app/fdctl/config.o: src/app/fdctl/config/default.toml
 endif
 endif
 endif

src/app/fdctl/monitor/helper.c

@@ -112,9 +112,9 @@ printf_seq( char ** buf,
                      : (delta>0L) ? TEXT_GREEN  /* new sequence numbers published */
                      : (seq_now)  ? TEXT_RED    /* sequence number went backward */
                      :              TEXT_BLUE;  /* sequence number reset */
-  if(      delta> 99999L ) PRINT( "%16lx(%s>+99999" TEXT_NORMAL ")", seq_now, color        );
-  else if( delta<-99999L ) PRINT( "%16lx(%s<-99999" TEXT_NORMAL ")", seq_now, color        );
-  else                     PRINT( "%16lx(%s %+6li"  TEXT_NORMAL ")", seq_now, color, delta );
+  if(      delta> 99999L ) PRINT( "%10lu(%s>+99999" TEXT_NORMAL ")", seq_now, color        );
+  else if( delta<-99999L ) PRINT( "%10lu(%s<-99999" TEXT_NORMAL ")", seq_now, color        );
+  else                     PRINT( "%10lu(%s %+6li"  TEXT_NORMAL ")", seq_now, color, delta );
 }
 
 void

src/app/fdctl/monitor/monitor.c

@@ -334,8 +334,8 @@ run_monitor( config_t * const config,
       PRINT( TEXT_NEWLINE );
     }
     PRINT( TEXT_NEWLINE );
-    PRINT( "           link |  tot TPS |  tot bps | uniq TPS | uniq bps |   ha tr%% | uniq bw%% | filt tr%% | filt bw%% |           ovrnp cnt |           ovrnr cnt |            slow cnt" TEXT_NEWLINE );
-    PRINT( "----------------+----------+----------+----------+----------+----------+----------+----------+----------+---------------------+---------------------+---------------------"    TEXT_NEWLINE );
+    PRINT( "           link |  tot TPS |  tot bps | uniq TPS | uniq bps |   ha tr%% | uniq bw%% | filt tr%% | filt bw%% |           ovrnp cnt |           ovrnr cnt |            slow cnt |             tx seq" TEXT_NEWLINE );
+    PRINT( "----------------+----------+----------+----------+----------+----------+----------+----------+----------+---------------------+---------------------+---------------------+-------------------"    TEXT_NEWLINE );
     long dt = now-then;
     for( ulong link_idx=0UL; link_idx<link_cnt; link_idx++ ) {
       link_snap_t * prv = &link_snap_prv[ link_idx ];
@@ -363,6 +363,7 @@ run_monitor( config_t * const config,
       PRINT( " | " ); printf_err_cnt( &buf, &buf_sz, cur->fseq_diag_ovrnp_cnt, prv->fseq_diag_ovrnp_cnt );
       PRINT( " | " ); printf_err_cnt( &buf, &buf_sz, cur->fseq_diag_ovrnr_cnt, prv->fseq_diag_ovrnr_cnt );
       PRINT( " | " ); printf_err_cnt( &buf, &buf_sz, cur->fseq_diag_slow_cnt,  prv->fseq_diag_slow_cnt  );
+      PRINT( " | " ); printf_seq(     &buf, &buf_sz, cur->mcache_seq,          prv->mcache_seq  );
       PRINT( TEXT_NEWLINE );
     }
 

src/ballet/base64/Local.mk

@@ -0,0 +1,3 @@
+$(call add-hdrs,fd_base64.h)
+$(call add-objs,fd_base64,fd_ballet)
+$(call make-unit-test,test_base64,test_base64,fd_ballet fd_util)

src/ballet/base64/fd_base64.c

@@ -0,0 +1,84 @@
+#include "fd_base64.h"
+
+/* Function to get the index of a character in the Base64 alphabet */
+static inline int
+base64_decode_char( char c ) {
+  if( c >= 'A' && c <= 'Z' ) return c - 'A';
+  if( c >= 'a' && c <= 'z' ) return c - 'a' + 26;
+  if( c >= '0' && c <= '9' ) return c - '0' + 52;
+  if( c == '+' ) return 62;
+  if( c == '/' ) return 63;
+  return -1; // Invalid character
+}
+
+/* Function to decode a base64 encoded string into an unsigned char array
+   The function returns the length of the decoded array */
+int
+fd_base64_decode( const char *  encoded,
+                  uchar *       decoded ) {
+  int    len = 0;
+  int    bits_collected = 0;
+  uint   accumulator = 0;
+
+  while ( *encoded ) {
+    char c = *encoded++;
+    int value = base64_decode_char(c);
+
+    if( value >= 0 ) {
+      accumulator = ( accumulator << 6 ) | ( uint ) value;
+      bits_collected += 6;
+
+      if( bits_collected >= 8 ) {
+        bits_collected -= 8;
+        decoded[ len++ ] = ( uchar )( accumulator >> bits_collected );
+        accumulator &= ( 1U << bits_collected ) - 1;
+      }
+    } else if( c == '=' ) {
+      /* Padding character, ignore and break the loop */
+      break;
+    } else {
+      /* Fail with invalid characters (e.g., whitespace, padding) */
+      return -1;
+    }
+  }
+
+  return len;
+}
+
+ulong
+fd_base64_encode( const uchar * data,
+                  int           data_len,
+                  char *        encoded ) {
+  static const char base64_alphabet[] =
+      "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+
+  uint encoded_len = 0;
+  uint accumulator = 0;
+  int bits_collected = 0;
+
+  while( data_len-- ) {
+    accumulator = ( accumulator << 8 ) | *data++;
+    bits_collected += 8;
+
+    while( bits_collected >= 6 ) {
+      encoded[ encoded_len++ ] = base64_alphabet[ ( accumulator >> ( bits_collected - 6) ) & 0x3F ];
+      bits_collected -= 6;
+    }
+  }
+
+  if( bits_collected > 0 ) {
+    // If there are remaining bits, pad the last Base64 character with zeroes
+    accumulator <<= 6 - bits_collected;
+    encoded[ encoded_len++ ] = base64_alphabet[accumulator & 0x3F ];
+  }
+
+  // Add padding characters if necessary
+  while( encoded_len % 4 != 0 ) {
+    encoded[ encoded_len++ ] = '=';
+  }
+
+  // Null-terminate the encoded string
+  encoded[ encoded_len ] = '\0';
+
+  return encoded_len;
+}

src/ballet/base64/fd_base64.h

@@ -0,0 +1,15 @@
+#ifndef HEADER_fd_src_ballet_base64_fd_base64_h
+#define HEADER_fd_src_ballet_base64_fd_base64_h
+
+/* fd_base64.h provides methods for converting between binary and base64. */
+#include "../fd_ballet_base.h"
+int
+fd_base64_decode( const char *  encoded,
+                  uchar *       decoded );
+
+ulong
+fd_base64_encode( const uchar * data,
+                  int           data_len,
+                  char *        encoded );
+
+#endif /* HEADER_fd_src_ballet_base64_fd_base64_h */

src/ballet/base64/test_base64.c

@@ -0,0 +1,49 @@
+#include "fd_base64.h"
+#include <string.h>
+#include <stdio.h>
+
+void
+decode_test( void ) {
+  const char * expected_string = "Hello World!";
+  const char * encoded_string  = "SGVsbG8gV29ybGQh";
+  uchar decoded[ 100 ]; /* Assuming the decoded data won't exceed 100 bytes */
+
+  int decoded_length = fd_base64_decode( encoded_string, decoded );
+
+  FD_TEST( (uint)decoded_length == strlen( expected_string ) );
+  FD_TEST( memcmp( decoded, expected_string, strlen(expected_string) ) == 0 );
+}
+
+void
+decode_test_equals( void ) {
+  const char * encoded_string  = "AZCML352XGjOwgIwMGsRf8oa2IoWzSvgWlJwcAEtLtwk3/h2VIe7n+YbPrAwpbIiK3KOM/G4XiNAKyhHbn2VBQ0BAAEGUn3G2+sjJ+xarkiI77ZYW6CEGHzEjzovKWoUG3/TSKAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACdPJdmqIA5PfVdI4dCMAMKH7z7U0fpkodPhLfE54yrfhMa9sJylZdraDb38lv6aISwi7GkOXsRZ8PQnKkkbFarB08LBYexRVX/v3EVfNeQk9z+WMTKqR0tc/WtXBjQP+v5pDHP1tYUMPTA8WZARvn8XTCjSs+9iPlPPYBWQrEspMZwcluFwA/afVpAczCo7+IJMw5/a0W/kR2EsJRNuF3IBBQUCAwAEAYgEJgW58JTT/VUQEAERERERARABAREBEBERERERABEBEQAQABARAAAREQEREBAQEBERERABERABEBAREBEQEQARAAERABARAQEQABABABAQAQABABEBAREQAAARAAAREREREAERARAAAREQEBAAEAAAEAAQEBEAEREAAAEQEBERAQAAAQEAABABABAQAAEBAAAQEBEAERAQEAAQAREBEQAREREAARAAAREAEAAAAREBEQEQEAAAEQEBEREREBABAQEQAQEQAAEQAREAERAAEAEQARABEBAAAQAQEAABEQEBEAAAEAABEBAAEAEAAQAQEBEQAQABABAREREBAQEAABEBARAAEAABEBAQEREAEBAREQEAAAEBEAEREREQARERAQEQAAEBEQEQARABEBAQAAAAEBEBAQEREQEQEQEAEQAQEBERARAQAQEBABAAAAEAERABAAAAAAEBEAEAAREREBEQEQARAREBEBEREAABABEAEBEBERAQABAAEQEAEAAQABEBEAAAABABAAABEAAAEQAQAQEBAQARAAEREAAAAQAQAREAAQAQAAEQEAEQEAAAARABEAEREBEBAQAQAQEREQEAAAEBAAAQEQABEAABEAEBEBAQABEREAAAABEBAAAQEQEQAAARERAQABERERABEAARABABEBAQAQEAEAARARERABABAREQEBAAAA==";
+  uchar decoded[ 1500 ]; /* Assuming the decoded data won't exceed 1500 bytes */
+
+  int decoded_length = fd_base64_decode( encoded_string, decoded );
+
+  FD_TEST( decoded_length != -1 );
+}
+
+void encode_test( void ) {
+  uchar binary_data[] = {72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33}; /* "Hello World!" */
+  const char * expected_string  = "SGVsbG8gV29ybGQh";
+
+  char encoded[100]; /* Assuming the encoded data won't exceed 100 characters */
+
+  ulong encoded_length = fd_base64_encode(binary_data, sizeof(binary_data), encoded);
+  printf("%lu %ld\n", sizeof(binary_data), strlen((char*)binary_data));
+
+  FD_TEST( (uint)encoded_length == strlen( expected_string ) );
+  FD_TEST( memcmp( encoded, expected_string, strlen(encoded) ) == 0 );
+}
+
+int
+main( int     argc,
+      char ** argv ) {
+  fd_boot( &argc, &argv );
+  decode_test();
+  decode_test_equals();
+  encode_test();
+  fd_halt();
+  return 0;
+}

src/tango/aio/fd_aio_pcapng.c

@@ -23,7 +23,12 @@ fd_aio_pcapng_send( void *                    ctx,
     }
   }
 
-  return fd_aio_send( mitm->dst, batch, batch_cnt, opt_batch_idx, flush );
+  /* pcaping doesn't require any additional destination */
+  if( mitm->dst ) {
+    return fd_aio_send( mitm->dst, batch, batch_cnt, opt_batch_idx, flush );
+  }
+
+  return FD_AIO_SUCCESS;
 }
 
 FD_FN_CONST fd_aio_t const *