fddev: add debugging wrapper

.vscode/launch.json

@@ -0,0 +1,37 @@
+{
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "fddev",
+      "type": "cppdbg",
+      "request": "launch",
+      "cwd": "${workspaceFolder}/build/native/gcc/bin",
+      "program": "${workspaceFolder}/build/native/gcc/bin/fddev",
+      "args": ["--no-sandbox"],
+      "miDebuggerPath": "${workspaceFolder}/build/native/gcc/bin/fddbg",
+      "MIMode": "gdb",
+      "setupCommands": [
+          {
+              "description": "Enable pretty-printing for gdb",
+              "text": "-enable-pretty-printing",
+              "ignoreFailures": true
+          },
+          {
+              "description": "Set Disassembly Flavor to Intel",
+              "text": "-gdb-set disassembly-flavor intel",
+              "ignoreFailures": true
+          },
+          {
+              "description": "Don't detach on fork",
+              "text": "-gdb-set detach-on-fork off",
+              "ignoreFailures": false
+          },
+          {
+            "description": "Stay parent after fork",
+            "text": "-gdb-set follow-fork-mode parent",
+            "ignoreFailures": false
+        }
+      ]
+    }
+  ]
+}

config/with-optimization.mk

@@ -1,3 +1,3 @@
-CPPFLAGS+=-O3 -ffast-math -fno-associative-math -fno-reciprocal-math
-CPPFLAGS+=-DFD_HAS_OPTIMIZATION=1
-FD_HAS_OPTIMIZATION:=1
+# CPPFLAGS+=-O3 -ffast-math -fno-associative-math -fno-reciprocal-math
+# CPPFLAGS+=-DFD_HAS_OPTIMIZATION=1
+# FD_HAS_OPTIMIZATION:=1

ffi/rust/firedancer-sys/build.rs

@@ -47,7 +47,10 @@ fn main() {
         //
         // We would need to `cargo clean` if any Makefile changes, which
         // isn't possible now.
-        println!("cargo:rerun-if-changed=../../../src");
+        println!("cargo:rerun-if-changed=../../../src/util");
+        println!("cargo:rerun-if-changed=../../../src/ballet");
+        println!("cargo:rerun-if-changed=../../../src/tango");
+        println!("cargo:rerun-if-changed=../../../src/disco");
 
         "../../../"
     };

ffi/rust/firedancer-sys/src/tango/cnc.rs

@@ -13,7 +13,9 @@ pub use crate::generated::{
     fd_cnc_leave,
     fd_cnc_new,
     fd_cnc_open,
+    fd_cnc_signal,
     fd_cnc_signal_cstr,
+    fd_cnc_signal_query,
     fd_cnc_strerror,
     fd_cnc_t,
     fd_cnc_type,

ffi/rust/firedancer-sys/src/tango/fctl.rs

@@ -1,8 +1,14 @@
 pub use crate::generated::{
     fd_fctl_cfg_done,
     fd_fctl_cfg_rx_add,
+    fd_fctl_delete,
+    fd_fctl_join,
+    fd_fctl_leave,
     fd_fctl_new,
+    fd_fctl_private_rx_t,
+    fd_fctl_rx_cr_return,
     fd_fctl_t,
+    fd_fctl_tx_cr_update,
     FD_FCTL_ALIGN,
     FD_FCTL_RX_MAX_MAX,
 };

ffi/rust/firedancer-sys/src/tango/fseq.rs

@@ -1,5 +1,6 @@
 pub use crate::generated::{
     fd_fseq_align,
+    fd_fseq_app_laddr,
     fd_fseq_app_laddr_const,
     fd_fseq_delete,
     fd_fseq_footprint,

ffi/rust/firedancer-sys/src/tango/mod.rs

@@ -13,3 +13,6 @@ pub use fseq::*;
 pub use mcache::*;
 pub use tcache::*;
 pub use xdp::*;
+
+pub use crate::generated::fd_chunk_to_laddr;
+pub use crate::generated::fd_chunk_to_laddr_const;

ffi/rust/firedancer-sys/src/util/mod.rs

@@ -1,14 +1,18 @@
 mod bits;
 mod log;
 mod pod;
+mod rng;
 mod shmem;
+mod tempo;
 mod tile;
 mod wksp;
 
 pub use bits::*;
 pub use log::*;
 pub use pod::*;
+pub use rng::*;
 pub use shmem::*;
+pub use tempo::*;
 pub use tile::*;
 pub use wksp::*;
 

ffi/rust/firedancer-sys/src/util/pod.rs

@@ -1,6 +1,8 @@
 pub use crate::generated::{
     fd_pod_cnt_subpod,
     fd_pod_info_t,
+    fd_pod_join,
+    fd_pod_leave,
     fd_pod_query,
     fd_pod_query_buf,
     fd_pod_query_char,

ffi/rust/firedancer-sys/src/util/wksp.rs

@@ -27,5 +27,6 @@ pub use crate::generated::{
     fd_wksp_pod_map,
     fd_wksp_pod_unmap,
     fd_wksp_t,
+    fd_wksp_private,
     fd_wksp_unmap,
 };

ffi/rust/firedancer-sys/wrapper_util.h

@@ -1,4 +1,5 @@
 #include "src/util/fd_util.h"
+#include "src/util/wksp/fd_wksp_private.h"
 
 #if FD_MCACHE_LG_INTERLEAVE
 #error "FD_MCACHE_LG_INTERLEAVE unsupported"

src/app/fdctl/config.c

@@ -26,8 +26,9 @@ find_wksp( config_t * const config,
 
 /* partial frank_bank definition since the tile doesn't really exist */
 static fd_frank_task_t frank_bank = {
-   .in_wksp = "pack_bank",
-   .out_wksp = "bank_shred",
+   .in_wksp    = "pack_bank",
+   .out_wksp   = "bank_shred",
+   .extra_wksp = NULL,
 };
 
 ulong
@@ -36,21 +37,29 @@ memlock_max_bytes( config_t * const config ) {
   for( ulong j=0; j<config->shmem.workspaces_cnt; j++ ) {
     workspace_config_t * wksp = &config->shmem.workspaces[ j ];
 
-#define TILE_MAX( tile ) do {                                                                     \
-    ulong in_bytes = 0, out_bytes = 0;                                                            \
-    if( FD_LIKELY( tile.in_wksp ) ) {                                                             \
-      workspace_config_t * in_wksp = find_wksp( config, tile.in_wksp );                           \
-      in_bytes = in_wksp->num_pages * in_wksp->page_size;                                         \
-    }                                                                                             \
-    if( FD_LIKELY( tile.out_wksp ) ) {                                                            \
-      workspace_config_t * out_wksp = find_wksp( config, tile.out_wksp );                         \
-      out_bytes = out_wksp->num_pages * out_wksp->page_size;                                      \
-    }                                                                                             \
-    memlock_max_bytes = fd_ulong_max( memlock_max_bytes,                                          \
-                                      wksp->page_size * wksp->num_pages + in_bytes + out_bytes ); \
+#define TILE_MAX( tile ) do {                                                 \
+    ulong in_bytes = 0, out_bytes = 0, extra_bytes = 0;                       \
+    if( FD_LIKELY( tile.in_wksp ) ) {                                         \
+      workspace_config_t * in_wksp = find_wksp( config, tile.in_wksp );       \
+      in_bytes = in_wksp->num_pages * in_wksp->page_size;                     \
+    }                                                                         \
+    if( FD_LIKELY( tile.out_wksp ) ) {                                        \
+      workspace_config_t * out_wksp = find_wksp( config, tile.out_wksp );     \
+      out_bytes = out_wksp->num_pages * out_wksp->page_size;                  \
+    }                                                                         \
+    if( FD_LIKELY( tile.extra_wksp ) ) {                                      \
+      workspace_config_t * extra_wksp = find_wksp( config, tile.extra_wksp ); \
+      extra_bytes = extra_wksp->num_pages * extra_wksp->page_size;            \
+    }                                                                         \
+    memlock_max_bytes = fd_ulong_max( memlock_max_bytes,                      \
+                                      wksp->page_size * wksp->num_pages +     \
+                                      in_bytes +                              \
+                                      out_bytes +                             \
+                                      extra_bytes );                          \
   } while(0)
 
     switch ( wksp->kind ) {
+      case wksp_tpu_txn_data:
       case wksp_quic_verify:
       case wksp_verify_dedup:
       case wksp_dedup_pack:
@@ -461,21 +470,27 @@ static void
 init_workspaces( config_t * config ) {
   ulong idx = 0;
 
-  config->shmem.workspaces[ idx ].kind      = wksp_quic_verify;
-  config->shmem.workspaces[ idx ].name      = "quic_verify";
+  config->shmem.workspaces[ idx ].kind      = wksp_tpu_txn_data;
+  config->shmem.workspaces[ idx ].name      = "tpu_txn_data";
   config->shmem.workspaces[ idx ].page_size = FD_SHMEM_GIGANTIC_PAGE_SZ;
   config->shmem.workspaces[ idx ].num_pages = 1;
   idx++;
 
+  config->shmem.workspaces[ idx ].kind      = wksp_quic_verify;
+  config->shmem.workspaces[ idx ].name      = "quic_verify";
+  config->shmem.workspaces[ idx ].page_size = FD_SHMEM_HUGE_PAGE_SZ;
+  config->shmem.workspaces[ idx ].num_pages = 2;
+  idx++;
+
   config->shmem.workspaces[ idx ].kind      = wksp_verify_dedup;
   config->shmem.workspaces[ idx ].name      = "verify_dedup";
-  config->shmem.workspaces[ idx ].page_size = FD_SHMEM_GIGANTIC_PAGE_SZ;
-  config->shmem.workspaces[ idx ].num_pages = 1;
+  config->shmem.workspaces[ idx ].page_size = FD_SHMEM_HUGE_PAGE_SZ;
+  config->shmem.workspaces[ idx ].num_pages = 2;
   idx++;
 
   config->shmem.workspaces[ idx ].kind      = wksp_dedup_pack;
   config->shmem.workspaces[ idx ].name      = "dedup_pack";
-  config->shmem.workspaces[ idx ].page_size = FD_SHMEM_GIGANTIC_PAGE_SZ;
+  config->shmem.workspaces[ idx ].page_size = FD_SHMEM_HUGE_PAGE_SZ;
   config->shmem.workspaces[ idx ].num_pages = 1;
   idx++;
 
@@ -517,7 +532,7 @@ init_workspaces( config_t * config ) {
 
   config->shmem.workspaces[ idx ].kind      = wksp_pack;
   config->shmem.workspaces[ idx ].name      = "pack";
-  config->shmem.workspaces[ idx ].page_size = FD_SHMEM_HUGE_PAGE_SZ;
+  config->shmem.workspaces[ idx ].page_size = FD_SHMEM_GIGANTIC_PAGE_SZ;
   config->shmem.workspaces[ idx ].num_pages = 1;
   idx++;
 

src/app/fdctl/config.h

@@ -14,6 +14,7 @@
 
 typedef struct {
   enum {
+    wksp_tpu_txn_data,
     wksp_quic_verify,
     wksp_verify_dedup,
     wksp_dedup_pack,

src/app/fdctl/config/default.toml

@@ -112,13 +112,13 @@ dynamic_port_range = "8000-10000"
     # If nonzero, enable JSON RPC on this port, and use the next port for the
     # RPC websocket. If zero, disable JSON RPC. This option is passed to the
     # Solana Labs client with the `--rpc-port` argument.
-    port = 0
+    port = 8899
 
     # If true, all RPC operations are enabled on this validator, including
     # non-default RPC methods for querying chain state and transaction history.
     # This option is passed to the Solana Labs client with the `--full-rpc-api`
     # argument.
-    full_api = false
+    full_api = true
 
     # If the RPC is private, the valdiator's open RPC port is not published in
     # the `solana gossip` command for use by others. This option is passed to
@@ -129,7 +129,7 @@ dynamic_port_range = "8000-10000"
     # `getConfirmedBlock` API. This will cause an increase in disk usage and
     # IOPS. This option is passed to the Solana Labs client with the
     # `--enable-rpc-transaction-history` argument.
-    transaction_history = false
+    transaction_history = true
 
     # If enabled, include CPI inner instructions, logs, and return data in the
     # historical transaction info stored. This option is passed to the Solana

src/app/fdctl/configure/workspace.c

@@ -216,18 +216,21 @@ init( config_t * const config ) {
     WKSP_BEGIN( config, wksp1, 0 );
 
     switch( wksp1->kind ) {
+      case wksp_tpu_txn_data:
+        for( ulong i=0; i<config->layout.verify_tile_count; i++ ) {
+          dcache( pod, "dcache%lu", config->tiles.verify.mtu, config->tiles.verify.receive_buffer_size, config->tiles.verify.receive_buffer_size * 32, i );
+        }
+        break;
       case wksp_quic_verify:
         for( ulong i=0; i<config->layout.verify_tile_count; i++ ) {
           mcache( pod, "mcache%lu", config->tiles.verify.receive_buffer_size, i );
-          dcache( pod, "dcache%lu", config->tiles.verify.mtu, config->tiles.verify.receive_buffer_size, config->tiles.verify.receive_buffer_size * 32, i );
           fseq  ( pod, "fseq%lu", i );
         }
         break;
       case wksp_verify_dedup:
         ulong1( pod, "cnt", config->layout.verify_tile_count );
         for( ulong i=0; i<config->layout.verify_tile_count; i++ ) {
           mcache( pod, "mcache%lu", config->tiles.verify.receive_buffer_size, i );
-          dcache( pod, "dcache%lu", config->tiles.verify.mtu, config->tiles.verify.receive_buffer_size, 0, i );
           fseq  ( pod, "fseq%lu",   i );
         }
         break;
@@ -241,6 +244,8 @@ init( config_t * const config ) {
           mcache( pod, "mcache%lu", config->tiles.bank.receive_buffer_size, i );
           dcache( pod, "dcache%lu", USHORT_MAX, config->layout.bank_tile_count * (ulong)config->tiles.bank.receive_buffer_size, 0, i );
           fseq  ( pod, "fseq%lu", i );
+          mcache( pod, "mcache-back%lu", config->tiles.bank.receive_buffer_size, i );
+          fseq  ( pod, "fseq-back%lu", i );
         }
         break;
       case wksp_bank_shred:

src/app/fdctl/fdctl.h

@@ -33,6 +33,16 @@ typedef union {
   struct {
     int monitor;
   } dev;
+  struct {
+    const char * payload_base64;
+    ulong  count;
+    const char * dst_ip;
+    ushort dst_port;
+  } txn;
+  struct {
+    int setcap;
+    int withcap;
+  } gdb;
 } args_t;
 
 typedef struct security security_t;

src/app/fdctl/monitor/monitor.c

@@ -206,7 +206,8 @@ run_monitor( config_t * const config,
     config->layout.verify_tile_count + // quic <-> verify
     config->layout.verify_tile_count + // verify <-> dedup
     1 +                                // dedup <-> pack
-    config->layout.bank_tile_count;    // pack <-> bank
+    config->layout.bank_tile_count +   // pack <-> bank
+    config->layout.bank_tile_count;    // bank <-> pack
 
   tile_t * tiles = fd_alloca( alignof(tile_t *), sizeof(tile_t)*tile_cnt );
   link_t * links = fd_alloca( alignof(link_t *), sizeof(link_t)*link_cnt );
@@ -220,6 +221,8 @@ run_monitor( config_t * const config,
 
     char buf[ 64 ];
     switch( wksp->kind ) {
+      case wksp_tpu_txn_data:
+        break;
       case wksp_quic_verify:
         for( ulong i=0; i<config->layout.verify_tile_count; i++ ) {
           links[ link_idx ].src_name = "quic";
@@ -260,6 +263,14 @@ run_monitor( config_t * const config,
           links[ link_idx ].fseq = fd_fseq_join( fd_wksp_pod_map( pods[ j ], snprintf1( buf, 64, "fseq%lu", i ) ) );
           if( FD_UNLIKELY( !links[ link_idx ].fseq ) ) FD_LOG_ERR(( "fd_fseq_join failed" ));
           link_idx++;
+
+          links[ link_idx ].src_name = "bank";
+          links[ link_idx ].dst_name = "pack";
+          links[ link_idx ].mcache = fd_mcache_join( fd_wksp_pod_map( pods[ j ], snprintf1( buf, 64, "mcache-back%lu", i ) ) );
+          if( FD_UNLIKELY( !links[ link_idx ].mcache ) ) FD_LOG_ERR(( "fd_mcache_join failed" ));
+          links[ link_idx ].fseq = fd_fseq_join( fd_wksp_pod_map( pods[ j ], snprintf1( buf, 64, "fseq-back%lu", i ) ) );
+          if( FD_UNLIKELY( !links[ link_idx ].fseq ) ) FD_LOG_ERR(( "fd_fseq_join failed" ));
+          link_idx++;
         }
         break;
       case wksp_bank_shred: