From 1e2d60e5b4c7f97d56c69c4235875d3b54e11bc4 Mon Sep 17 00:00:00 2001 From: Michael McGee Date: Tue, 22 Aug 2023 20:46:33 +0000 Subject: [PATCH] solana: bank --- .vscode/launch.json | 10 +- config/with-optimization.mk | 8 +- ffi/rust/Cargo.toml | 2 +- ffi/rust/firedancer-sys/src/tango/cnc.rs | 2 + ffi/rust/firedancer-sys/src/tango/fctl.rs | 6 + ffi/rust/firedancer-sys/src/tango/fseq.rs | 1 + ffi/rust/firedancer-sys/src/tango/mod.rs | 3 + ffi/rust/firedancer-sys/src/tango/xdp.rs | 2 +- ffi/rust/firedancer-sys/src/util/mod.rs | 4 + ffi/rust/firedancer-sys/src/util/pod.rs | 2 + ffi/rust/firedancer-sys/src/util/rng.rs | 19 ++ ffi/rust/firedancer-sys/src/util/tempo.rs | 6 + ffi/rust/firedancer-sys/src/util/wksp.rs | 1 + ffi/rust/firedancer-sys/wrapper_util.h | 1 + ffi/rust/firedancer/Cargo.toml | 11 ++ ffi/rust/firedancer/src/bits.rs | 42 +++++ ffi/rust/firedancer/src/cnc.rs | 78 ++++++++ ffi/rust/firedancer/src/dcache.rs | 39 ++++ ffi/rust/firedancer/src/fctl.rs | 99 ++++++++++ ffi/rust/firedancer/src/fseq.rs | 67 +++++++ ffi/rust/firedancer/src/gaddr.rs | 41 +++++ ffi/rust/firedancer/src/lib.rs | 21 +++ ffi/rust/firedancer/src/mcache.rs | 122 +++++++++++++ ffi/rust/firedancer/src/pod.rs | 156 ++++++++++++++++ ffi/rust/firedancer/src/rng.rs | 44 +++++ ffi/rust/firedancer/src/workspace.rs | 37 ++++ ffi/rust/firedancer/tests/linker.rs | 7 + solana | 2 +- src/app/fdctl/config.c | 12 +- src/app/fdctl/config.h | 6 +- src/app/fdctl/config/default.toml | 31 +++- src/app/fdctl/configure/configure.c | 15 +- src/app/fdctl/configure/workspace.c | 23 ++- src/app/fdctl/configure/xdp.c | 64 +++++-- src/app/fdctl/fdctl.h | 10 + src/app/fdctl/main1.c | 8 + src/app/fdctl/run.c | 27 ++- src/app/fddev/Local.mk | 2 +- src/app/fddev/configure/cluster.c | 65 +++++-- src/app/fddev/dev.c | 3 - src/app/fddev/dev1.c | 6 + src/app/fddev/fddev.h | 14 ++ src/app/fddev/main.c | 7 + src/app/fddev/txn.c | 193 ++++++++++++++++++++ src/app/frank/fd_frank.h | 1 + src/app/frank/fd_frank_quic.c | 126 ++++++++++++- src/disco/quic/fd_quic.h | 1 + src/disco/quic/fd_quic_tile.c | 82 +++++++-- src/disco/quic/test_quic_tile.c | 7 +- src/tango/quic/tests/fd_quic_test_helpers.c | 2 +- src/tango/xdp/fd_xdp_ctl.c | 5 +- src/tango/xdp/fd_xdp_license.h | 2 +- src/tango/xdp/fd_xdp_redirect_prog.c | 11 +- src/tango/xdp/fd_xdp_redirect_prog.o | Bin 1272 -> 1704 bytes src/tango/xdp/fd_xdp_redirect_user.c | 40 ++-- src/tango/xdp/fd_xdp_redirect_user.h | 19 +- src/util/log/fd_log.c | 20 +- src/util/net/fd_ip4.c | 11 ++ src/util/net/fd_ip4.h | 8 + 59 files changed, 1514 insertions(+), 140 deletions(-) create mode 100644 ffi/rust/firedancer-sys/src/util/rng.rs create mode 100644 ffi/rust/firedancer-sys/src/util/tempo.rs create mode 100644 ffi/rust/firedancer/Cargo.toml create mode 100644 ffi/rust/firedancer/src/bits.rs create mode 100644 ffi/rust/firedancer/src/cnc.rs create mode 100644 ffi/rust/firedancer/src/dcache.rs create mode 100644 ffi/rust/firedancer/src/fctl.rs create mode 100644 ffi/rust/firedancer/src/fseq.rs create mode 100644 ffi/rust/firedancer/src/gaddr.rs create mode 100644 ffi/rust/firedancer/src/lib.rs create mode 100644 ffi/rust/firedancer/src/mcache.rs create mode 100644 ffi/rust/firedancer/src/pod.rs create mode 100644 ffi/rust/firedancer/src/rng.rs create mode 100644 ffi/rust/firedancer/src/workspace.rs create mode 100644 ffi/rust/firedancer/tests/linker.rs create mode 100644 src/app/fddev/txn.c diff --git a/.vscode/launch.json b/.vscode/launch.json index 5a3262759d..f64766603d 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -4,10 +4,10 @@ { "name": "fddev", "type": "cppdbg", - "request": "launch", + "request": "attach", "cwd": "${workspaceFolder}/build/native/gcc/bin", "program": "${workspaceFolder}/build/native/gcc/bin/fddev", - "args": ["--no-sandbox"], + "args": ["dev1", "solana", "--no-sandbox"], "miDebuggerPath": "${workspaceFolder}/build/native/gcc/bin/fddbg", "MIMode": "gdb", "setupCommands": [ @@ -27,9 +27,9 @@ "ignoreFailures": false }, { - "description": "Stay parent after fork", - "text": "-gdb-set follow-fork-mode parent", - "ignoreFailures": false + "description": "Stay parent after fork", + "text": "-gdb-set follow-fork-mode parent", + "ignoreFailures": false } ] } diff --git a/config/with-optimization.mk b/config/with-optimization.mk index 70444fa0f1..5ca04eab71 100644 --- a/config/with-optimization.mk +++ b/config/with-optimization.mk @@ -1,4 +1,4 @@ -CPPFLAGS+=-O3 -ffast-math -fno-associative-math -fno-reciprocal-math -CPPFLAGS+=-DFD_HAS_OPTIMIZATION=1 -FD_HAS_OPTIMIZATION:=1 -RUST_PROFILE:=release +# CPPFLAGS+=-O3 -ffast-math -fno-associative-math -fno-reciprocal-math +# CPPFLAGS+=-DFD_HAS_OPTIMIZATION=1 +# FD_HAS_OPTIMIZATION:=1 +# RUST_PROFILE:=release diff --git a/ffi/rust/Cargo.toml b/ffi/rust/Cargo.toml index 6fe709b2a3..9cf1e2e908 100644 --- a/ffi/rust/Cargo.toml +++ b/ffi/rust/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["ffi/rust/firedancer-sys", "ffi/rust/firedancer-diff"] +members = ["ffi/rust/firedancer", "ffi/rust/firedancer-sys", "ffi/rust/firedancer-diff"] resolver = "2" [profile.dev] diff --git a/ffi/rust/firedancer-sys/src/tango/cnc.rs b/ffi/rust/firedancer-sys/src/tango/cnc.rs index fe4bdfa29a..17d19c1033 100644 --- a/ffi/rust/firedancer-sys/src/tango/cnc.rs +++ b/ffi/rust/firedancer-sys/src/tango/cnc.rs @@ -13,7 +13,9 @@ pub use crate::generated::{ fd_cnc_leave, fd_cnc_new, fd_cnc_open, + fd_cnc_signal, fd_cnc_signal_cstr, + fd_cnc_signal_query, fd_cnc_strerror, fd_cnc_t, fd_cnc_type, diff --git a/ffi/rust/firedancer-sys/src/tango/fctl.rs b/ffi/rust/firedancer-sys/src/tango/fctl.rs index eb32ef5094..654a6b4719 100644 --- a/ffi/rust/firedancer-sys/src/tango/fctl.rs +++ b/ffi/rust/firedancer-sys/src/tango/fctl.rs @@ -1,8 +1,14 @@ pub use crate::generated::{ fd_fctl_cfg_done, fd_fctl_cfg_rx_add, + fd_fctl_delete, + fd_fctl_join, + fd_fctl_leave, fd_fctl_new, + fd_fctl_private_rx_t, + fd_fctl_rx_cr_return, fd_fctl_t, + fd_fctl_tx_cr_update, FD_FCTL_ALIGN, FD_FCTL_RX_MAX_MAX, }; diff --git a/ffi/rust/firedancer-sys/src/tango/fseq.rs b/ffi/rust/firedancer-sys/src/tango/fseq.rs index fbdbc34290..605d620b94 100644 --- a/ffi/rust/firedancer-sys/src/tango/fseq.rs +++ b/ffi/rust/firedancer-sys/src/tango/fseq.rs @@ -1,5 +1,6 @@ pub use crate::generated::{ fd_fseq_align, + fd_fseq_app_laddr, fd_fseq_app_laddr_const, fd_fseq_delete, fd_fseq_footprint, diff --git a/ffi/rust/firedancer-sys/src/tango/mod.rs b/ffi/rust/firedancer-sys/src/tango/mod.rs index 77d451c0a7..5a9762ef9f 100644 --- a/ffi/rust/firedancer-sys/src/tango/mod.rs +++ b/ffi/rust/firedancer-sys/src/tango/mod.rs @@ -13,3 +13,6 @@ pub use fseq::*; pub use mcache::*; pub use tcache::*; pub use xdp::*; + +pub use crate::generated::fd_chunk_to_laddr; +pub use crate::generated::fd_chunk_to_laddr_const; diff --git a/ffi/rust/firedancer-sys/src/tango/xdp.rs b/ffi/rust/firedancer-sys/src/tango/xdp.rs index b06d21e9e3..ee9a2283dc 100644 --- a/ffi/rust/firedancer-sys/src/tango/xdp.rs +++ b/ffi/rust/firedancer-sys/src/tango/xdp.rs @@ -1,5 +1,5 @@ pub use crate::generated::{ fd_xdp_clear_listeners, - fd_xdp_listen_udp_port, + fd_xdp_listen_udp_ports, fd_xdp_release_udp_port, }; diff --git a/ffi/rust/firedancer-sys/src/util/mod.rs b/ffi/rust/firedancer-sys/src/util/mod.rs index 3d99fc0fbb..dbfd9e5ee7 100644 --- a/ffi/rust/firedancer-sys/src/util/mod.rs +++ b/ffi/rust/firedancer-sys/src/util/mod.rs @@ -1,14 +1,18 @@ mod bits; mod log; mod pod; +mod rng; mod shmem; +mod tempo; mod tile; mod wksp; pub use bits::*; pub use log::*; pub use pod::*; +pub use rng::*; pub use shmem::*; +pub use tempo::*; pub use tile::*; pub use wksp::*; diff --git a/ffi/rust/firedancer-sys/src/util/pod.rs b/ffi/rust/firedancer-sys/src/util/pod.rs index 16687f7bf8..44a9150bf2 100644 --- a/ffi/rust/firedancer-sys/src/util/pod.rs +++ b/ffi/rust/firedancer-sys/src/util/pod.rs @@ -1,6 +1,8 @@ pub use crate::generated::{ fd_pod_cnt_subpod, fd_pod_info_t, + fd_pod_join, + fd_pod_leave, fd_pod_query, fd_pod_query_buf, fd_pod_query_char, diff --git a/ffi/rust/firedancer-sys/src/util/rng.rs b/ffi/rust/firedancer-sys/src/util/rng.rs new file mode 100644 index 0000000000..3648ddb24a --- /dev/null +++ b/ffi/rust/firedancer-sys/src/util/rng.rs @@ -0,0 +1,19 @@ +pub use crate::generated::{ + fd_rng_delete, + fd_rng_idx, + fd_rng_idx_set, + fd_rng_int, + fd_rng_join, + fd_rng_leave, + fd_rng_long, + fd_rng_new, + fd_rng_schar, + fd_rng_seq, + fd_rng_seq_set, + fd_rng_short, + fd_rng_t, + fd_rng_uchar, + fd_rng_uint, + fd_rng_ulong, + fd_rng_ushort, +}; diff --git a/ffi/rust/firedancer-sys/src/util/tempo.rs b/ffi/rust/firedancer-sys/src/util/tempo.rs new file mode 100644 index 0000000000..1bc091dd7e --- /dev/null +++ b/ffi/rust/firedancer-sys/src/util/tempo.rs @@ -0,0 +1,6 @@ +pub use crate::generated::{ + fd_tempo_async_min, + fd_tempo_async_reload, + fd_tempo_lazy_default, + fd_tempo_tick_per_ns, +}; diff --git a/ffi/rust/firedancer-sys/src/util/wksp.rs b/ffi/rust/firedancer-sys/src/util/wksp.rs index f43cad4483..83da7cc153 100644 --- a/ffi/rust/firedancer-sys/src/util/wksp.rs +++ b/ffi/rust/firedancer-sys/src/util/wksp.rs @@ -27,5 +27,6 @@ pub use crate::generated::{ fd_wksp_pod_map, fd_wksp_pod_unmap, fd_wksp_t, + fd_wksp_private, fd_wksp_unmap, }; diff --git a/ffi/rust/firedancer-sys/wrapper_util.h b/ffi/rust/firedancer-sys/wrapper_util.h index 01c6b72bc2..948fcfea45 100644 --- a/ffi/rust/firedancer-sys/wrapper_util.h +++ b/ffi/rust/firedancer-sys/wrapper_util.h @@ -1,4 +1,5 @@ #include "src/util/fd_util.h" +#include "src/util/wksp/fd_wksp_private.h" #if FD_MCACHE_LG_INTERLEAVE #error "FD_MCACHE_LG_INTERLEAVE unsupported" diff --git a/ffi/rust/firedancer/Cargo.toml b/ffi/rust/firedancer/Cargo.toml new file mode 100644 index 0000000000..2656000b33 --- /dev/null +++ b/ffi/rust/firedancer/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "firedancer" +version = "0.1.0" +edition = "2021" +authors = ["Firedancer Contributors "] +description = "Safe high level wrappers for Firedancer" +license = "Apache-2.0" + +[dependencies] +paste = "1.0.12" +firedancer-sys = { path = "../firedancer-sys" } diff --git a/ffi/rust/firedancer/src/bits.rs b/ffi/rust/firedancer/src/bits.rs new file mode 100644 index 0000000000..587dae92ec --- /dev/null +++ b/ffi/rust/firedancer/src/bits.rs @@ -0,0 +1,42 @@ +macro_rules! align_up { + ( $x:expr, $a:expr ) => { + ($x + ($a - 1)) & !($a - 1) + }; +} + +macro_rules! layout { + ( value=$value:expr, ) => { $value }; + ( value=$value:expr, ($align:expr, $size:expr), $($tail:tt)*) => { + layout!( value=align_up!($value, $align) + $size, $($tail)*) + }; + ( align=$align:expr, [ $($tail:tt)* ]) => { + align_up!(layout!(value = 0, $($tail)*), $align) + }; +} + +pub(crate) use { + align_up, + layout, +}; + +#[cfg(test)] +mod tests { + #[test] + fn test_align_up() { + let zeros = 0u64; + let ones = u64::MAX; + + for i in 0..64 { + let align = 1u64 << i; + let lo = (1u64 << i) - 1; + let hi = !lo; + + assert_eq!(align_up!(zeros, align), zeros); + assert_eq!(align_up!(ones, align), if i == 0 { ones } else { zeros }); + for j in 0..64 { + let x = 1u64 << j; + assert_eq!(align_up!(x, align), (x + lo) & hi); + } + } + } +} diff --git a/ffi/rust/firedancer/src/cnc.rs b/ffi/rust/firedancer/src/cnc.rs new file mode 100644 index 0000000000..e7586b0fbc --- /dev/null +++ b/ffi/rust/firedancer/src/cnc.rs @@ -0,0 +1,78 @@ +use std::ffi::c_ulong; +use std::ptr::{ + read_volatile, + write_volatile, +}; +use firedancer_sys::*; + +use crate::*; + +pub struct Cnc { + laddr: *mut tango::fd_cnc_t, + diagnostic: *mut c_ulong, + _workspace: Workspace, +} + +impl Drop for Cnc { + fn drop(&mut self) { + unsafe { tango::fd_cnc_leave(self.laddr) }; + } +} + +#[repr(u32)] +#[derive(Copy, Clone, Debug)] +pub enum CncSignal { + Run = tango::FD_CNC_SIGNAL_RUN, + Boot = tango::FD_CNC_SIGNAL_BOOT, + Fail = tango::FD_CNC_SIGNAL_FAIL, + Halt = tango::FD_CNC_SIGNAL_HALT, +} + +#[repr(u32)] +#[derive(Copy, Clone, Debug)] +pub enum CncDiag { + InBackpressure = tango::FD_CNC_DIAG_IN_BACKP, + BackpressureCount = tango::FD_CNC_DIAG_BACKP_CNT, +} + +impl Cnc { + pub unsafe fn join>(gaddr: T) -> Result { + let workspace = Workspace::map(gaddr)?; + let laddr = tango::fd_cnc_join(workspace.laddr.as_ptr()); + if laddr.is_null() { + Err(()) + } else { + let diagnostic = tango::fd_cnc_app_laddr(laddr) as *mut c_ulong; + if diagnostic.is_null() { + Err(()) + } else { + Ok(Self { + laddr, + diagnostic, + _workspace: workspace, + }) + } + } + } + + pub fn query(&self) -> u64 { + unsafe { tango::fd_cnc_signal_query(self.laddr) } + } + + pub fn signal(&self, signal: u64) { + unsafe { tango::fd_cnc_signal(self.laddr, signal) } + } + + pub unsafe fn set(&self, diag: u64, value: u64) { + write_volatile(self.diagnostic.offset(diag as isize), value) + } + + pub unsafe fn increment(&self, diag: u64, value: u64) { + let offset = self.diagnostic.offset(diag as isize); + write_volatile(offset, read_volatile(offset) + value) + } + + pub fn heartbeat(&self, now: i64) { + unsafe { tango::fd_cnc_heartbeat(self.laddr, now) } + } +} diff --git a/ffi/rust/firedancer/src/dcache.rs b/ffi/rust/firedancer/src/dcache.rs new file mode 100644 index 0000000000..8510e34f08 --- /dev/null +++ b/ffi/rust/firedancer/src/dcache.rs @@ -0,0 +1,39 @@ +use firedancer_sys::*; + +use std::ffi::c_void; + +use crate::*; + +pub struct DCache { + laddr: *mut u8, + wksp: *mut util::fd_wksp_t, + _workspace: Workspace, +} + +impl Drop for DCache { + fn drop(&mut self) { + unsafe { tango::fd_dcache_leave(self.laddr) }; + } +} + +impl DCache { + pub unsafe fn join>(gaddr: T) -> Result { + let workspace = Workspace::map(gaddr)?; + let laddr = tango::fd_dcache_join(workspace.laddr.as_ptr()); + if laddr.is_null() { + return Err(()); + } + + let wksp = util::fd_wksp_containing(laddr as *const c_void); + Ok(Self { + laddr, + wksp, + _workspace: workspace, + }) + } + + pub unsafe fn slice<'a>(&self, chunk: u64, offset: u64, len: u64) -> &'a[u8] { + let laddr = tango::fd_chunk_to_laddr_const(self.wksp as *const c_void, chunk); + std::slice::from_raw_parts(laddr.offset(offset as isize) as *const u8, len as usize) + } +} diff --git a/ffi/rust/firedancer/src/fctl.rs b/ffi/rust/firedancer/src/fctl.rs new file mode 100644 index 0000000000..a308206041 --- /dev/null +++ b/ffi/rust/firedancer/src/fctl.rs @@ -0,0 +1,99 @@ +use std::ffi::c_void; +use std::marker::PhantomData; +use std::mem::{ + align_of, + size_of, +}; +use std::ptr::null_mut; + +use firedancer_sys::*; + +use crate::*; + +macro_rules! footprint { + ( $rx_max:expr ) => { + layout!( + align = tango::FD_FCTL_ALIGN as usize, + [ + ( + align_of::(), + size_of::() + ), + ( + align_of::(), + $rx_max * size_of::() + ), + ] + ) + }; +} + +pub struct FCtl<'a, 'b> { + _stack: Vec, + shmem: *mut c_void, + fctl: *mut tango::fd_fctl_t, + _seq: PhantomData<&'a u64>, + _slow: PhantomData<&'b mut u64>, +} + +impl<'a, 'b> Drop for FCtl<'a, 'b> { + fn drop(&mut self) { + unsafe { tango::fd_fctl_leave(self.fctl) }; + unsafe { tango::fd_fctl_delete(self.shmem) }; + } +} + +impl<'a, 'b> FCtl<'a, 'b> { + pub fn new( + cr_burst: u64, + cr_max: u64, + cr_resume: u64, + cr_refill: u64, + fseq: &FSeq, + ) -> Result { + let mut stack = vec![0; footprint!(1usize)]; + let shmem = unsafe { tango::fd_fctl_new(stack.as_mut_ptr() as *mut _, 1) }; + if shmem.is_null() { + return Err(()); + } + + let fctl = unsafe { tango::fd_fctl_join(shmem) }; + if fctl.is_null() { + return Err(()); + } + + let fctl = unsafe { + tango::fd_fctl_cfg_rx_add( + fctl, + cr_max, + fseq.laddr, + fseq.diagnostic.offset(FSeqDiag::SlowCount as isize), + ) + }; + if fctl.is_null() { + return Err(()); + } + + let fctl = unsafe { tango::fd_fctl_cfg_done(fctl, cr_burst, cr_max, cr_resume, cr_refill) }; + + Ok(FCtl { + _stack: stack, + shmem, + fctl, + _seq: PhantomData, + _slow: PhantomData, + }) + } + + pub fn tx_cr_update(&self, cr_avail: u64, mcache: &MCache) -> u64 { + unsafe { tango::fd_fctl_tx_cr_update(self.fctl, cr_avail, mcache.sequence_number) } + } +} + +pub fn housekeeping_default_interval_nanos(cr_max: u64) -> i64 { + unsafe { util::fd_tempo_lazy_default(cr_max) } +} + +pub fn minimum_housekeeping_tick_interval(lazy: i64) -> u64 { + unsafe { util::fd_tempo_async_min(lazy, 1, util::fd_tempo_tick_per_ns(null_mut()) as f32) } +} diff --git a/ffi/rust/firedancer/src/fseq.rs b/ffi/rust/firedancer/src/fseq.rs new file mode 100644 index 0000000000..8792a8b57d --- /dev/null +++ b/ffi/rust/firedancer/src/fseq.rs @@ -0,0 +1,67 @@ +use std::ffi::c_ulong; +use std::ptr::{ + read_volatile, + write_volatile, +}; + +use firedancer_sys::*; + +use crate::*; + +pub struct FSeq { + pub(crate) laddr: *mut c_ulong, + pub(crate) diagnostic: *mut c_ulong, + _workspace: Workspace, +} + +impl Drop for FSeq { + fn drop(&mut self) { + unsafe { tango::fd_fseq_leave(self.laddr) }; + } +} + +#[repr(u32)] +#[derive(Copy, Clone, Debug)] +pub enum FSeqDiag { + PublishedCount = tango::FD_FSEQ_DIAG_PUB_CNT, + PublishedSize = tango::FD_FSEQ_DIAG_PUB_SZ, + FilteredCount = tango::FD_FSEQ_DIAG_FILT_CNT, + FilteredSize = tango::FD_FSEQ_DIAG_FILT_SZ, + OverrunPollingCount = tango::FD_FSEQ_DIAG_OVRNP_CNT, + OverrunReadingCount = tango::FD_FSEQ_DIAG_OVRNR_CNT, + SlowCount = tango::FD_FSEQ_DIAG_SLOW_CNT, +} + +impl FSeq { + pub unsafe fn join>(gaddr: T) -> Result { + let workspace = Workspace::map(gaddr)?; + let laddr = tango::fd_fseq_join(workspace.laddr.as_ptr()); + if laddr.is_null() { + Err(()) + } else { + let diagnostic = tango::fd_fseq_app_laddr(laddr) as *mut c_ulong; + if diagnostic.is_null() { + Err(()) + } else { + Ok(Self { + laddr, + diagnostic, + _workspace: workspace, + }) + } + } + } + + pub unsafe fn set(&self, diag: u64, value: u64) { + write_volatile(self.diagnostic.offset(diag as isize), value) + } + + pub unsafe fn increment(&self, diag: u64, value: u64) { + let offset = self.diagnostic.offset(diag as isize); + write_volatile(offset, read_volatile(offset) + value) + } + + pub fn rx_cr_return(&self, mcache: &MCache) { + unsafe { tango::fd_fctl_rx_cr_return(self.laddr, mcache.sequence_number) } + } +} \ No newline at end of file diff --git a/ffi/rust/firedancer/src/gaddr.rs b/ffi/rust/firedancer/src/gaddr.rs new file mode 100644 index 0000000000..427d90ea35 --- /dev/null +++ b/ffi/rust/firedancer/src/gaddr.rs @@ -0,0 +1,41 @@ +use std::ffi::{ + c_char, + CString, +}; + +pub struct GlobalAddress { + gaddr: CString, + _offset: u64, +} + +impl GlobalAddress { + pub(crate) fn as_ptr(&self) -> *const c_char { + self.gaddr.as_ptr() + } +} + +impl TryFrom for GlobalAddress { + type Error = (); + + fn try_from(value: String) -> Result { + let (_workspace, offset) = match value.split_once(':') { + None => return Err(()), + Some(parts) => parts, + }; + + let offset = match offset.parse() { + Err(_) => return Err(()), + Ok(offset) => offset, + }; + + let gaddr = match CString::new(value) { + Err(_) => return Err(()), + Ok(path) => path, + }; + + Ok(GlobalAddress { + gaddr, + _offset: offset, + }) + } +} diff --git a/ffi/rust/firedancer/src/lib.rs b/ffi/rust/firedancer/src/lib.rs new file mode 100644 index 0000000000..7adcde9139 --- /dev/null +++ b/ffi/rust/firedancer/src/lib.rs @@ -0,0 +1,21 @@ +mod bits; +mod cnc; +mod dcache; +mod fctl; +mod fseq; +mod gaddr; +mod mcache; +mod pod; +mod rng; +mod workspace; + +use bits::*; +pub use cnc::*; +pub use dcache::*; +pub use fctl::*; +pub use fseq::*; +pub use gaddr::*; +pub use mcache::*; +pub use pod::*; +pub use rng::*; +pub use workspace::*; diff --git a/ffi/rust/firedancer/src/mcache.rs b/ffi/rust/firedancer/src/mcache.rs new file mode 100644 index 0000000000..959ac26c66 --- /dev/null +++ b/ffi/rust/firedancer/src/mcache.rs @@ -0,0 +1,122 @@ +use std::sync::atomic::{ + compiler_fence, + Ordering, +}; + +use firedancer_sys::*; + +use crate::*; + +pub struct MCache { + laddr: *mut tango::fd_frag_meta_t, + mline: *mut tango::fd_frag_meta_t, + pub(crate) sequence_number: u64, + depth: u64, + + sync: *mut u64, + _workspace: Workspace, +} + +#[derive(Copy, Clone)] +pub enum MCacheCtl { + None +} + +impl MCacheCtl { + fn ctl(&self) -> u64 { + match self { + MCacheCtl::None => 0, + } + } +} + +#[derive(Copy, Clone, Debug)] +pub enum Poll { + CaughtUp, + Overrun, + Ready, +} + +#[derive(Copy, Clone)] + pub enum Advance { + Overrun, + Normal, +} + +impl MCache { + pub unsafe fn join>(gaddr: T) -> Result { + let workspace = Workspace::map(gaddr)?; + let laddr = tango::fd_mcache_join(workspace.laddr.as_ptr()); + if laddr.is_null() { + return Err(()); + } + + let depth = tango::fd_mcache_depth(laddr); + let sync = tango::fd_mcache_seq_laddr(laddr); + let sequence_number = tango::fd_mcache_seq_query(sync); + let mline = laddr.offset(tango::fd_mcache_line_idx(sequence_number, depth) as isize); + + Ok(Self { + laddr, + mline, + sequence_number, + depth, + sync, + _workspace: workspace, + }) + } + + pub fn depth(&self) -> u64 { + self.depth + } + + pub fn chunk(&self) -> u32 { + unsafe { (*self.mline).__bindgen_anon_1.chunk } + } + + pub fn housekeep(&self) { + unsafe { tango::fd_mcache_seq_update(self.sync, self.sequence_number) } + } + + pub fn poll(&mut self) -> Poll { + compiler_fence(Ordering::AcqRel); + let sequence_number_found = unsafe { (*self.mline).__bindgen_anon_1.seq }; + compiler_fence(Ordering::AcqRel); + + let result: i64 = sequence_number_found.wrapping_sub(self.sequence_number) as i64; + if result < 0 { + Poll::CaughtUp + } else if result == 0 { + Poll::Ready + } else { + self.sequence_number = sequence_number_found; + Poll::Overrun + } + } + + pub fn size(&self) -> u16 { + unsafe { (*self.mline).__bindgen_anon_1.sz } + } + + pub fn advance(&mut self) -> Advance { + compiler_fence(Ordering::AcqRel); + let sequence_number_found = unsafe { (*self.mline).__bindgen_anon_1.seq }; + compiler_fence(Ordering::AcqRel); + + if sequence_number_found != self.sequence_number { + self.sequence_number = sequence_number_found; + Advance::Overrun + } else { + self.sequence_number += 1; + self.mline = unsafe { + self.laddr + .offset(tango::fd_mcache_line_idx(self.sequence_number, self.depth) as isize) + }; + Advance::Normal + } + } + + pub fn publish(&mut self, sig: u64, chunk: u64, sz: u64, ctl: MCacheCtl, tsorig: u64, tspub: u64) { + unsafe { tango::fd_mcache_publish(self.mline, self.depth, self.sequence_number, sig, chunk, sz, ctl.ctl(), tsorig, tspub) } + } +} diff --git a/ffi/rust/firedancer/src/pod.rs b/ffi/rust/firedancer/src/pod.rs new file mode 100644 index 0000000000..2fb1cd9155 --- /dev/null +++ b/ffi/rust/firedancer/src/pod.rs @@ -0,0 +1,156 @@ +use std::ffi::{ + c_char, + CStr, + CString, c_void, +}; +use std::marker::PhantomData; +use std::ptr::{null, NonNull}; +use std::sync::Arc; + +use firedancer_sys::*; +use paste::paste; + +use crate::*; + +#[derive(Clone)] +pub struct Pod { + laddr: *const u8, + workspace: Arc, +} + +unsafe impl Send for Pod {} +unsafe impl Sync for Pod {} + +impl Drop for Pod { + fn drop(&mut self) { + unsafe { util::fd_pod_leave(self.laddr) }; + } +} + +impl Pod { + pub unsafe fn join>(gaddr: T) -> Result { + let workspace = Workspace::map(gaddr)?; + let laddr = util::fd_pod_join(workspace.laddr.as_ptr()); + if laddr.is_null() { + Err(()) + } else { + Ok(Self { + laddr, + workspace: Arc::new(workspace), + }) + } + } + + pub unsafe fn join_default>(wksp: T) -> Result { + let wksp_str = CString::new(wksp.as_ref()).unwrap(); + let wksp = util::fd_wksp_attach(wksp_str.as_ptr()); + if wksp.is_null() { + return Err(()); + } + + let pod = util::fd_wksp_laddr( wksp, (*wksp).gaddr_lo ); + if pod.is_null() { + return Err(()); + } + + let laddr = util::fd_pod_join(pod); + + if laddr.is_null() { + Err(()) + } else { + Ok(Self { + laddr, + workspace: Arc::new(Workspace { + laddr: NonNull::new(laddr as *mut c_void).unwrap(), + _marker: PhantomData, + }), + }) + } + } + + pub fn try_query>(&self, key: S) -> Option { + let key = match CString::new(key.as_ref()) { + Ok(key) => key, + _ => return None, + }; + + T::try_query(self, key.as_ptr()) + } + + pub fn query>(&self, key: S) -> T { + let key = match CString::new(key.as_ref()) { + Ok(key) => key, + _ => return T::default(), + }; + + T::query(self, key.as_ptr()) + } +} + +pub trait FromPod: Sized { + fn try_query(pod: &Pod, key: *const c_char) -> Option; + + fn query(pod: &Pod, key: *const c_char) -> Self { + FromPod::try_query(pod, key).unwrap() + } +} + +impl FromPod for String { + fn try_query(pod: &Pod, key: *const c_char) -> Option { + let value = unsafe { util::fd_pod_query_cstr(pod.laddr, key, null()) } as *const i8; + + if value.is_null() { + return None; + } + + match unsafe { CStr::from_ptr(value).to_str() } { + Ok(str) => Some(str.to_owned()), + _ => None, + } + } +} + +impl FromPod for Pod { + fn try_query(pod: &Pod, key: *const c_char) -> Option { + let laddr = unsafe { util::fd_pod_query_subpod(pod.laddr, key) }; + + if laddr.is_null() { + return None; + } + + Some(Pod { + laddr, + workspace: pod.workspace.clone(), + }) + } +} + +impl FromPod for GlobalAddress { + fn try_query(pod: &Pod, key: *const c_char) -> Option { + let string: String = FromPod::try_query(pod, key)?; + string.try_into().ok() + } +} + +macro_rules! impl_from_pod { + ($ty:ty, $id:ident) => { + impl FromPod for $ty { + fn try_query(pod: &Pod, key: *const c_char) -> Option { + paste! { + unsafe { + Some(util::[](pod.laddr, key, $ty::default())) + } + } + } + } + }; +} + +impl_from_pod!(i8, char); +impl_from_pod!(i16, short); +impl_from_pod!(i32, int); +impl_from_pod!(i64, long); +impl_from_pod!(u8, uchar); +impl_from_pod!(u16, ushort); +impl_from_pod!(u32, uint); +impl_from_pod!(u64, ulong); diff --git a/ffi/rust/firedancer/src/rng.rs b/ffi/rust/firedancer/src/rng.rs new file mode 100644 index 0000000000..09a0f3a9e8 --- /dev/null +++ b/ffi/rust/firedancer/src/rng.rs @@ -0,0 +1,44 @@ +use std::ffi::c_void; + +use firedancer_sys::*; + +pub struct Rng { + inner: util::fd_rng_t, + shmem: *mut c_void, +} + +impl Drop for Rng { + fn drop(&mut self) { + unsafe { util::fd_rng_leave(&mut self.inner) }; + unsafe { util::fd_rng_delete(self.shmem) }; + } +} + +impl Rng { + pub fn new(seed: u32, id: u64) -> Result { + let mut inner = util::fd_rng_t { seq: 0, idx: 0 }; + + let shmem = unsafe { util::fd_rng_new(&mut inner as *mut _ as *mut c_void, seed, id) }; + if shmem.is_null() { + return Err(()); + } + + let rng = unsafe { util::fd_rng_join(shmem) }; + if rng.is_null() { + return Err(()); + } + + Ok(Rng { inner, shmem }) + } + + pub fn async_reload(&mut self, min: u64) -> u64 { + unsafe { util::fd_tempo_async_reload(&mut self.inner, min) } + } +} + +pub fn boot() { + let mut argc: i32 = 0; + unsafe { + util::fd_boot(&mut argc as *mut _, std::ptr::null_mut()); + } +} diff --git a/ffi/rust/firedancer/src/workspace.rs b/ffi/rust/firedancer/src/workspace.rs new file mode 100644 index 0000000000..f1cefd18e3 --- /dev/null +++ b/ffi/rust/firedancer/src/workspace.rs @@ -0,0 +1,37 @@ +use std::cell::Cell; +use std::ffi::c_void; +use std::marker::PhantomData; +use std::ptr::NonNull; + +use firedancer_sys::*; + +use crate::*; + +pub struct Workspace { + pub(crate) laddr: NonNull, + pub(crate) _marker: PhantomData>, // Not covariant +} + +impl Drop for Workspace { + fn drop(&mut self) { + unsafe { firedancer_sys::util::fd_wksp_unmap(self.laddr.as_ptr()) } + } +} + +impl Workspace { + pub(crate) unsafe fn map>(gaddr: G) -> Result { + let addr: GlobalAddress = match gaddr.try_into() { + Ok(addr) => addr, + _ => return Err(()), + }; + let laddr = unsafe { util::fd_wksp_map(addr.as_ptr()) }; + if laddr.is_null() { + Err(()) + } else { + Ok(Self { + laddr: NonNull::new(laddr).unwrap(), + _marker: PhantomData, + }) + } + } +} diff --git a/ffi/rust/firedancer/tests/linker.rs b/ffi/rust/firedancer/tests/linker.rs new file mode 100644 index 0000000000..c7a80d077d --- /dev/null +++ b/ffi/rust/firedancer/tests/linker.rs @@ -0,0 +1,7 @@ +use firedancer::*; + +#[test] +fn links_properly() { + let gaddr = GlobalAddress::try_from("".to_string()).unwrap(); + assert!(unsafe { MCache::join(gaddr) }.is_err()); +} diff --git a/solana b/solana index 354d6262d6..3c4854244f 160000 --- a/solana +++ b/solana @@ -1 +1 @@ -Subproject commit 354d6262d62e95df27aefa1b8e5d24f1e5f415e2 +Subproject commit 3c4854244f38c925fe7ab4c641f9d64f4bd2f0a0 diff --git a/src/app/fdctl/config.c b/src/app/fdctl/config.c index 65dd7cba71..54fee7b062 100644 --- a/src/app/fdctl/config.c +++ b/src/app/fdctl/config.c @@ -227,6 +227,8 @@ static void parse_key_value( config_t * config, ENTRY_BOOL ( ., ledger, bigtable_storage ); ENTRY_VSTR ( ., ledger, account_indexes ); ENTRY_VSTR ( ., ledger, account_index_exclude_keys ); + ENTRY_STR ( ., ledger, snapshot_compression ); + ENTRY_BOOL ( ., ledger, require_tower ); ENTRY_VSTR ( ., gossip, entrypoints ); ENTRY_BOOL ( ., gossip, port_check ); @@ -245,6 +247,7 @@ static void parse_key_value( config_t * config, ENTRY_BOOL ( ., consensus, wait_for_vote_to_start_leader ); ENTRY_VUINT ( ., consensus, hard_fork_at_slots ); ENTRY_VSTR ( ., consensus, known_validators ); + ENTRY_BOOL ( ., consensus, os_network_limits_test ); ENTRY_USHORT( ., rpc, port ); ENTRY_BOOL ( ., rpc, full_api ); @@ -274,7 +277,8 @@ static void parse_key_value( config_t * config, ENTRY_STR ( ., development.netns, interface1_addr ); ENTRY_STR ( ., tiles.quic, interface ); - ENTRY_USHORT( ., tiles.quic, listen_port ); + ENTRY_USHORT( ., tiles.quic, transaction_listen_port ); + ENTRY_USHORT( ., tiles.quic, quic_transaction_listen_port ); ENTRY_UINT ( ., tiles.quic, max_concurrent_connections ); ENTRY_UINT ( ., tiles.quic, max_concurrent_connection_ids_per_connection ); ENTRY_UINT ( ., tiles.quic, max_concurrent_streams_per_connection ); @@ -712,6 +716,12 @@ config_parse( int * pargc, FD_LOG_ERR(( "trying to join a live cluster, but configuration enables [development.netns] which is a development only feature" )); } + if( FD_UNLIKELY( result.tiles.quic.quic_transaction_listen_port != result.tiles.quic.transaction_listen_port + 6 ) ) + FD_LOG_ERR(( "configuration specifies invalid [tiles.quic.quic_transaction_listen_port] `%hu`. " + "This must be 6 more than [tiles.quic.transaction_listen_port] `%hu`", + result.tiles.quic.quic_transaction_listen_port, + result.tiles.quic.transaction_listen_port )); + init_workspaces( &result ); return result; diff --git a/src/app/fdctl/config.h b/src/app/fdctl/config.h index 4ba40671b2..895d3b2061 100644 --- a/src/app/fdctl/config.h +++ b/src/app/fdctl/config.h @@ -57,6 +57,8 @@ typedef struct { char account_indexes[ 4 ][ 32 ]; ulong account_index_exclude_keys_cnt; char account_index_exclude_keys[ 32 ][ 32 ]; + int require_tower; + char snapshot_compression[ 10 ]; } ledger; struct { @@ -82,6 +84,7 @@ typedef struct { uint hard_fork_at_slots[ 32 ]; ulong known_validators_cnt; char known_validators[ 16 ][ 256 ]; + int os_network_limits_test; } consensus; struct { @@ -129,7 +132,8 @@ typedef struct { char interface[ IF_NAMESIZE ]; uint ip_addr; uchar mac_addr[6]; - ushort listen_port; + ushort transaction_listen_port; + ushort quic_transaction_listen_port; char xdp_mode[ 8 ]; uint max_concurrent_connections; diff --git a/src/app/fdctl/config/default.toml b/src/app/fdctl/config/default.toml index 83cc82da4b..c40aaf456c 100644 --- a/src/app/fdctl/config/default.toml +++ b/src/app/fdctl/config/default.toml @@ -29,7 +29,7 @@ scratch_directory = "/home/{user}/.firedancer/{name}" # [min, max). Ports are used for receiving transactions and votes from clients # and other validators. This option is passed to the Solana Labs client with the # `--dynamic-port-range` argument. -dynamic_port_range = "8000-10000" +dynamic_port_range = "" # TODO: Describe ledger. [ledger] @@ -63,7 +63,7 @@ dynamic_port_range = "8000-10000" # # This option is passed to the Solana Labs client with the # `--limit-ledger-size` argument. - limit_size = 200_000_000 + limit_size = 0 # If enabled, fetch historical transaction info from a BigTable instance as # a fallback to local ledger data. The `GOOGLE_APPLICATION_CREDENTIALS` @@ -83,6 +83,14 @@ dynamic_port_range = "8000-10000" # `--account-index-exclude-key` argument. account_index_exclude_keys = [] + # Whether to use compression when storing snapshots. This option is passed + # to the Solana Labs client with the `--snapshot-compression` argument. + snapshot_compression = "none" + + # Refuse to start if saved tower state is not found. This option is passed + # to the Solana Labs client with the `--require-tower` argument. + require_tower = true + [gossip] # Routable DNS name or IP address and port number to use to rendezvous with # the gossip cluster. These entrypoints are passed to the Solana Labs client @@ -123,7 +131,7 @@ dynamic_port_range = "8000-10000" # If the RPC is private, the valdiator's open RPC port is not published in # the `solana gossip` command for use by others. This option is passed to # the Solana Labs client with the `--private-rpc` argument. - private = true + private = false # Enable historical transaction info over JSON RPC, including the # `getConfirmedBlock` API. This will cause an increase in disk usage and @@ -134,7 +142,7 @@ dynamic_port_range = "8000-10000" # If enabled, include CPI inner instructions, logs, and return data in the # historical transaction info stored. This option is passed to the Solana # Labs client with the `--enable-extended-tx-metadata-storage` argument. - extended_tx_metadata_storage = false + extended_tx_metadata_storage = true # If true, use the RPC service of known validators only. This option is # passed to the Solana Labs client with the `--only-known-rpc` argument. @@ -225,6 +233,12 @@ dynamic_port_range = "8000-10000" # `--no-wait-for-vote-to-start-leader` argument. wait_for_vote_to_start_leader = true + # Don't perform a network speed test on starting up the validator. If this + # is not disabled, and the speed test fails, the validator will refuse to + # start. This option is passed to the Solana Labs client (inverted) with the + # `--no-os-network-limits-test` argument. + os_network_limits_test = false + # If nonempty, add a hard fork at each of the provided slots. These options # are passed to the Solana Labs client with the `--hard-fork` argument. hard_fork_at_slots = [] @@ -370,8 +384,13 @@ dynamic_port_range = "8000-10000" # you can check what this is with `ip route get 8.8.8.8` interface = "" - # Which port to listen on. - listen_port = 9001 + # Which port to listen on for incoming transactions. This could be votes, + # user transactions, or transactions forwarded from another validator. + transaction_listen_port = 9001 + + # Which port to listen on for incoming quic transactions. Currently + # this must be exactly 6 more than the transaction_listen_port. + quic_transaction_listen_port = 9007 # Maximum number of simultaneous QUIC connections which can be open. New # connections which would exceed this limit will not be accepted. diff --git a/src/app/fdctl/configure/configure.c b/src/app/fdctl/configure/configure.c index e53c49898b..60a19c4341 100644 --- a/src/app/fdctl/configure/configure.c +++ b/src/app/fdctl/configure/configure.c @@ -161,10 +161,21 @@ configure_cmd_fn( args_t * args, config_t * const config ) { int error = 0; - for( configure_stage_t ** stage = args->configure.stages; *stage; stage++ ) { - if( FD_UNLIKELY( configure_stage( *stage, (configure_cmd_t)args->configure.command, config ) ) ) error = 1; + if( FD_LIKELY( (configure_cmd_t)args->configure.command != CONFIGURE_CMD_FINI ) ) { + for( configure_stage_t ** stage = args->configure.stages; *stage; stage++ ) { + if( FD_UNLIKELY( configure_stage( *stage, (configure_cmd_t)args->configure.command, config ) ) ) error = 1; + } + } else { + ulong i; + for( i=0; args->configure.stages[ i ]; i++ ) ; + if( FD_LIKELY( i > 0 ) ) { + for( ulong j=0; jconfigure.stages[ i-1-j ], (configure_cmd_t)args->configure.command, config ) ) ) error = 1; + } + } } + if( FD_UNLIKELY( error ) ) FD_LOG_ERR(( "failed to configure some stages" )); } diff --git a/src/app/fdctl/configure/workspace.c b/src/app/fdctl/configure/workspace.c index e8c5638d50..700d4ccb3b 100644 --- a/src/app/fdctl/configure/workspace.c +++ b/src/app/fdctl/configure/workspace.c @@ -267,11 +267,24 @@ init( config_t * const config ) { FD_LOG_ERR(( "failed to bind xsk for quic tile %lu", wksp1->kind_idx )); fd_wksp_unmap( shmem ); - uint1 ( pod, "ip_addr", config->tiles.quic.ip_addr ); - ushort1( pod, "listen_port", config->tiles.quic.listen_port, 0 ); - buf ( pod, "src_mac_addr", config->tiles.quic.mac_addr, 6 ); - ulong1 ( pod, "idle_timeout_ms", 1000 ); - ulong1 ( pod, "initial_rx_max_stream_data", 1<<15 ); + if( FD_UNLIKELY( strcmp( config->tiles.quic.interface, "lo") && !wksp1->kind_idx ) ) { + // First QUIC tile (0) can also listen to loopback XSK. + xsk ( pod, "lo_xsk", 2048, config->tiles.quic.xdp_rx_queue_size, config->tiles.quic.xdp_tx_queue_size ); + xsk_aio( pod, "lo_xsk_aio", config->tiles.quic.xdp_tx_queue_size, config->tiles.quic.xdp_aio_depth ); + + char const * lo_xsk_gaddr = fd_pod_query_cstr( pod, "lo_xsk", NULL ); + void * lo_shmem = fd_wksp_map ( lo_xsk_gaddr ); + if( FD_UNLIKELY( !fd_xsk_bind( lo_shmem, config->name, "lo", (uint)wksp1->kind_idx ) ) ) + FD_LOG_ERR(( "failed to bind lo_xsk for quic tile %lu", wksp1->kind_idx )); + fd_wksp_unmap( lo_shmem ); + } + + uint1 ( pod, "ip_addr", config->tiles.quic.ip_addr ); + ushort1( pod, "transaction_listen_port", config->tiles.quic.transaction_listen_port, 0 ); + ushort1( pod, "quic_transaction_listen_port", config->tiles.quic.quic_transaction_listen_port, 0 ); + buf ( pod, "src_mac_addr", config->tiles.quic.mac_addr, 6 ); + ulong1 ( pod, "idle_timeout_ms", 1000 ); + ulong1 ( pod, "initial_rx_max_stream_data", 1<<15 ); break; case wksp_verify: cnc( pod, "cnc" ); diff --git a/src/app/fdctl/configure/xdp.c b/src/app/fdctl/configure/xdp.c index b5d18b3c37..ade593cc93 100644 --- a/src/app/fdctl/configure/xdp.c +++ b/src/app/fdctl/configure/xdp.c @@ -41,16 +41,52 @@ init( config_t * const config ) { (int)config->uid, (int)config->uid ) ) ) FD_LOG_ERR(( "fd_xdp_init failed" )); + if( FD_UNLIKELY( fd_xdp_hook_iface( config->name, config->tiles.quic.interface, mode, fd_xdp_redirect_prog, fd_xdp_redirect_prog_sz ) ) ) FD_LOG_ERR(( "fd_xdp_hook_iface failed" )); - if( FD_UNLIKELY( fd_xdp_listen_udp_port( config->name, - config->tiles.quic.ip_addr, - config->tiles.quic.listen_port, 1 ) ) ) - FD_LOG_ERR(( "fd_xdp_listen_udp_port failed" )); + + /* The Linux kernel does some short circuiting optimizations + when sending packets to an IP address that's owned by the + same host. The optimization is basically to route them over + to the loopback interface directly, bypassing the network + hardware. + + This redirection to the loopback interface happens before + XDP programs are executed, so local traffic destined for + our listen addresses will not get ingested correctly. + + There are two reasons we send traffic locally, + + * For testing and development. + * The Solana Labs code sends local traffic to itself to + as part of routine operation (eg, when it's the leader + it sends votes to its own TPU socket). + + So for now we need to also bind to loopback. This is a + small performance hit for other traffic, but we only + redirect packets destined for our target IP and port so + it will not otherwise interfere. */ + if( FD_LIKELY( strcmp( config->tiles.quic.interface, "lo" ) ) ) { + if( FD_UNLIKELY( fd_xdp_hook_iface( config->name, + "lo", + mode, + fd_xdp_redirect_prog, + fd_xdp_redirect_prog_sz ) ) ) + FD_LOG_ERR(( "fd_xdp_hook_iface failed" )); + } + + + ushort udp_ports[] = { config->tiles.quic.transaction_listen_port, config->tiles.quic.quic_transaction_listen_port }; + if( FD_UNLIKELY( fd_xdp_listen_udp_ports( config->name, + config->tiles.quic.ip_addr, + 2, + udp_ports, + 1 ) ) ) + FD_LOG_ERR(( "fd_xdp_listen_udp_ports failed" )); } static void @@ -75,6 +111,8 @@ fini( config_t * const config ) { char path[ PATH_MAX ]; snprintf1( path, PATH_MAX, "/sys/fs/bpf/%s/%s", config->name, config->tiles.quic.interface ); if( FD_UNLIKELY( rmdir( path ) && errno != ENOENT ) ) FD_LOG_ERR(( "rmdir failed (%i-%s)", errno, fd_io_strerror( errno ) )); + snprintf1( path, PATH_MAX, "/sys/fs/bpf/%s/lo", config->name ); + if( FD_UNLIKELY( rmdir( path ) && errno != ENOENT ) ) FD_LOG_ERR(( "rmdir failed (%i-%s)", errno, fd_io_strerror( errno ) )); snprintf1( path, PATH_MAX, "/sys/fs/bpf/%s", config->name ); if( FD_UNLIKELY( rmdir( path ) && errno != ENOENT ) ) FD_LOG_ERR(( "rmdir failed (%i-%s)", errno, fd_io_strerror( errno ) )); } @@ -95,17 +133,21 @@ check( config_t * const config ) { snprintf1( xdp_path, PATH_MAX, "/sys/fs/bpf/%s/udp_dsts", config->name ); CHECK( check_file( xdp_path, config->uid, config->uid, S_IFREG | S_IRUSR | S_IWUSR | S_IRGRP ) ); - snprintf1( xdp_path, PATH_MAX, "/sys/fs/bpf/%s/%s/xdp_link", config->name, config->tiles.quic.interface ); - CHECK( check_file( xdp_path, config->uid, config->uid, S_IFREG | S_IRUSR | S_IWUSR | S_IRGRP ) ); + char * interfaces[] = { config->tiles.quic.interface, "lo" }; + ulong interfaces_sz = !strcmp( config->tiles.quic.interface, "lo" ) ? 1 : 2; + for( ulong i=0; iname, interfaces[i] ); + CHECK( check_file( xdp_path, config->uid, config->uid, S_IFREG | S_IRUSR | S_IWUSR | S_IRGRP ) ); - snprintf1( xdp_path, PATH_MAX, "/sys/fs/bpf/%s/%s/xdp_prog", config->name, config->tiles.quic.interface ); - CHECK( check_file( xdp_path, config->uid, config->uid, S_IFREG | S_IRUSR | S_IWUSR | S_IRGRP ) ); + snprintf1( xdp_path, PATH_MAX, "/sys/fs/bpf/%s/%s/xdp_prog", config->name, interfaces[i] ); + CHECK( check_file( xdp_path, config->uid, config->uid, S_IFREG | S_IRUSR | S_IWUSR | S_IRGRP ) ); - snprintf1( xdp_path, PATH_MAX, "/sys/fs/bpf/%s/%s/xsks", config->name, config->tiles.quic.interface ); - CHECK( check_file( xdp_path, config->uid, config->uid, S_IFREG | S_IRUSR | S_IWUSR | S_IRGRP ) ); + snprintf1( xdp_path, PATH_MAX, "/sys/fs/bpf/%s/%s/xsks", config->name, interfaces[i] ); + CHECK( check_file( xdp_path, config->uid, config->uid, S_IFREG | S_IRUSR | S_IWUSR | S_IRGRP ) ); + } /* todo: step into these links and make sure the interior data is - correct */ + correct, eg, port numbers still match */ CONFIGURE_OK(); } diff --git a/src/app/fdctl/fdctl.h b/src/app/fdctl/fdctl.h index cc91437ea2..589e8a25c3 100644 --- a/src/app/fdctl/fdctl.h +++ b/src/app/fdctl/fdctl.h @@ -33,6 +33,16 @@ typedef union { struct { int monitor; } dev; + struct { + const char * payload_base64; + ulong count; + const char * dst_ip; + ushort dst_port; + } txn; + struct { + int setcap; + int withcap; + } gdb; } args_t; typedef struct security security_t; diff --git a/src/app/fdctl/main1.c b/src/app/fdctl/main1.c index 16aa41059a..31d61713d6 100644 --- a/src/app/fdctl/main1.c +++ b/src/app/fdctl/main1.c @@ -1,5 +1,8 @@ +#define _GNU_SOURCE #include "fdctl.h" +#include + action_t ACTIONS[ 4 ] = { { .name = "run", .args = NULL, .fn = run_cmd_fn, .perm = run_cmd_perm }, { .name = "configure", .args = configure_cmd_args, .fn = configure_cmd_fn, .perm = configure_cmd_perm }, @@ -7,9 +10,14 @@ action_t ACTIONS[ 4 ] = { { .name = "keygen", .args = NULL, .fn = keygen_cmd_fn, .perm = NULL }, }; +extern int * fd_log_private_shared_lock; + int main1( int argc, char ** _argv ) { + fd_log_private_shared_lock = (int*)mmap( NULL, sizeof(int), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, (off_t)0 ); + if( FD_UNLIKELY( !fd_log_private_shared_lock ) ) exit(1); + fd_boot( &argc, &_argv ); fd_log_thread_set( "main" ); diff --git a/src/app/fdctl/run.c b/src/app/fdctl/run.c index 3bc39bc8e6..982205968d 100644 --- a/src/app/fdctl/run.c +++ b/src/app/fdctl/run.c @@ -198,6 +198,9 @@ int solana_labs_main( void * args ) { config_t * const config = args; + FD_LOG_NOTICE(( "solana pid %d", getpid() )); + // sleep( 60 ); + gid_t gid, egid, sgid; if( FD_UNLIKELY( getresgid( &gid, &egid, &sgid ) ) ) FD_LOG_ERR(( "getresgid() failed (%i-%s)", errno, fd_io_strerror( errno ) )); @@ -241,13 +244,24 @@ solana_labs_main( void * args ) { ADD1( "fdctl" ); ADD( "--log", "-" ); + ADD( "--firedancer-app-name", config->name ); + + ADD1("--no-poh-speed-test" ); // TODO: Only if we built debug? + if( FD_UNLIKELY( strcmp( config->dynamic_port_range, "" ) ) ) + ADD( "--dynamic-port-range", config->dynamic_port_range ); + + ADDU( "--tpu-port", config->tiles.quic.transaction_listen_port ); - ADD( "--dynamic-port-range", config->dynamic_port_range ); + char ip_addr[16]; + fd_ip4_addr_to_cstr( config->tiles.quic.ip_addr, ip_addr ); + ADD( "--gossip-host", ip_addr ); /* consensus */ + (void)identity_path; ADD( "--identity", identity_path ); if( strcmp( config->consensus.vote_account_path, "" ) ) ADD( "--vote-account", config->consensus.vote_account_path ); + if( !config->consensus.snapshot_fetch ) ADD1( "--no-snapshot-fetch" ); if( !config->consensus.genesis_fetch ) ADD1( "--no-genesis-fetch" ); if( !config->consensus.poh_speed_test ) ADD1( "--no-poh-speed-test" ); @@ -266,9 +280,14 @@ solana_labs_main( void * args ) { for( ulong i=0; iconsensus.known_validators_cnt; i++ ) ADD( "--known_validator", config->consensus.known_validators[ i ] ); + ADD( "--snapshot-compression", config->ledger.snapshot_compression ); + if( FD_UNLIKELY( config->ledger.require_tower ) ) ADD1( "--require-tower" ); + + if( FD_UNLIKELY( !config->consensus.os_network_limits_test ) ) ADD1( "--no-os-network-limits-test" ); + /* ledger */ ADD( "--ledger", config->ledger.path ); - ADDU( "--limit-ledger-size", config->ledger.limit_size ); + if( FD_LIKELY( config->ledger.limit_size ) ) ADDU( "--limit-ledger-size", config->ledger.limit_size ); if( config->ledger.bigtable_storage ) ADD1( "--enable-rpc-bigtable-ledger-storage" ); for( ulong i=0; iledger.account_indexes_cnt; i++ ) ADD( "--account-index", config->ledger.account_indexes[ i ] ); @@ -282,7 +301,7 @@ solana_labs_main( void * args ) { if( strcmp( config->gossip.host, "" ) ) ADD( "--gossip-host", config->gossip.host ); - /* rpc */ + // /* rpc */ if( config->rpc.port ) ADDH( "--rpc-port", config->rpc.port ); if( config->rpc.full_api ) ADD1( "--full-rpc-api" ); if( config->rpc.private ) ADD1( "--private-rpc" ); @@ -296,7 +315,7 @@ solana_labs_main( void * args ) { argv[ idx ] = NULL; /* silence a bunch of solana_metrics INFO spam */ - if( FD_UNLIKELY( setenv( "RUST_LOG", "solana=info,solana_metrics::metrics=warn", 1 ) ) ) + if( FD_UNLIKELY( setenv( "RUST_LOG", "solana=info,solana_metrics::metrics=warn,solana_gossip::cluster_info=trace", 1 ) ) ) FD_LOG_ERR(( "setenv() failed (%i-%s)", errno, fd_io_strerror( errno ) )); FD_LOG_INFO(( "Running Solana Labs validator with the following arguments:" )); diff --git a/src/app/fddev/Local.mk b/src/app/fddev/Local.mk index 5a1ea0ded0..71a211e427 100644 --- a/src/app/fddev/Local.mk +++ b/src/app/fddev/Local.mk @@ -4,7 +4,7 @@ ifdef FD_HAS_X86 ifdef FD_HAS_DOUBLE .PHONY: fddev -$(call make-bin-rust,fddev,main dev dev1 configure/netns configure/cluster,fd_fdctl fd_frank fd_disco fd_ballet fd_tango fd_util fd_quic solana_validator_fd) +$(call make-bin-rust,fddev,main dev dev1 txn configure/netns configure/cluster,fd_fdctl fd_frank fd_disco fd_ballet fd_tango fd_util fd_quic solana_validator_fd) ifeq (run,$(firstword $(MAKECMDGOALS))) RUN_ARGS := $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS)) diff --git a/src/app/fddev/configure/cluster.c b/src/app/fddev/configure/cluster.c index 1d75e90231..4322ddb0ce 100644 --- a/src/app/fddev/configure/cluster.c +++ b/src/app/fddev/configure/cluster.c @@ -2,6 +2,7 @@ #include "../../fdctl/configure/configure.h" #include +#include #include #define NAME "cluster" @@ -48,17 +49,11 @@ init( config_t * const config ) { ADD1( "fddev" ); - ADDU( "--max-genesis-archive-unpacked-size", 1073741824 ); - ADD1( "--enable-warmup-epochs" ); - ADD( "--bootstrap-validator", config->consensus.identity_path ); - ADD1( vote ); - ADD1( stake ); - ADD( "--bootstrap-stake-authorized-pubkey", config->consensus.identity_path ); - - ADD( "--ledger", config->ledger.path ); ADD( "--faucet-pubkey", faucet ); - ADDU( "--faucet-lamports", 500000000000000000 ); - ADD( "--hashes-per-tick", "auto" ); + ADD( "--hashes-per-tick", "sleep" ); + ADDU( "--faucet-lamports", 500000000000000000UL ); + ADD( "--bootstrap-validator", config->consensus.identity_path ); ADD1( vote ); ADD1( stake ); + ADD( "--ledger", config->ledger.path ); ADD( "--cluster-type", "development" ); /* these are copied out of the output of `solana/fetch-spl.sh` ... need to @@ -108,6 +103,42 @@ init( config_t * const config ) { if( FD_UNLIKELY( setegid( gid ) ) ) FD_LOG_ERR(( "setegid() failed (%i-%s)", errno, fd_io_strerror( errno ) )); } +static void +rmtree( char * path ) { + DIR * dir = opendir( path ); + if( FD_UNLIKELY( !dir ) ) { + if( errno == ENOENT ) return; + FD_LOG_ERR(( "opendir `%s` failed (%i-%s)", path, errno, fd_io_strerror( errno ) )); + } + + struct dirent * entry; + errno = 0; + while(( entry = readdir( dir ) )) { + if( FD_LIKELY( !strcmp( entry->d_name, "." ) || !strcmp( entry->d_name, ".." ) ) ) continue; + + char path1[ PATH_MAX ]; + snprintf1( path1, PATH_MAX, "%s/%s", path, entry->d_name ); + + struct stat st; + if( FD_UNLIKELY( lstat( path1, &st ) ) ) { + if( FD_LIKELY( errno == ENOENT ) ) continue; + FD_LOG_ERR(( "stat `%s` failed (%i-%s)", path1, errno, fd_io_strerror( errno ) )); + } + + if( FD_UNLIKELY( S_ISDIR( st.st_mode ) ) ) { + rmtree( path1 ); + } else { + if( FD_UNLIKELY( unlink( path1 ) && errno != ENOENT ) ) + FD_LOG_ERR(( "unlink `%s` failed (%i-%s)", path1, errno, fd_io_strerror( errno ) )); + } + } + + if( FD_UNLIKELY( errno && errno != ENOENT ) ) FD_LOG_ERR(( "readdir `%s` failed (%i-%s)", path, errno, fd_io_strerror( errno ) )); + + if( FD_UNLIKELY( rmdir( path ) ) ) FD_LOG_ERR(( "rmdir `%s` failed (%i-%s)", path, errno, fd_io_strerror( errno ) )); + if( FD_UNLIKELY( closedir( dir ) ) ) FD_LOG_ERR(( "closedir `%s` failed (%i-%s)", path, errno, fd_io_strerror( errno ) )); +} + static void fini( config_t * const config ) { char path[ PATH_MAX ]; @@ -120,26 +151,23 @@ fini( config_t * const config ) { snprintf1( path, PATH_MAX, "%s/vote-account.json", config->scratch_directory ); if( FD_UNLIKELY( unlink( path ) && errno != ENOENT ) ) FD_LOG_ERR(( "could not remove cluster file `%s` (%i-%s)", path, errno, fd_io_strerror( errno ) )); - snprintf1( path, PATH_MAX, "%s/genesis.bin", config->ledger.path ); - if( FD_UNLIKELY( unlink( path ) && errno != ENOENT ) ) - FD_LOG_ERR(( "could not remove cluster file `%s` (%i-%s)", path, errno, fd_io_strerror( errno ) )); + rmtree( config->ledger.path ); } static configure_result_t check( config_t * const config ) { - char faucet[ PATH_MAX ], stake[ PATH_MAX ], vote[ PATH_MAX ], genesis[ PATH_MAX ]; + char faucet[ PATH_MAX ], stake[ PATH_MAX ], vote[ PATH_MAX ]; snprintf1( faucet, PATH_MAX, "%s/faucet.json", config->scratch_directory ); snprintf1( stake, PATH_MAX, "%s/stake-account.json", config->scratch_directory ); snprintf1( vote, PATH_MAX, "%s/vote-account.json", config->scratch_directory ); - snprintf1( genesis, PATH_MAX, "%s/genesis.bin", config->ledger.path ); struct stat st; if( FD_UNLIKELY( stat( faucet, &st ) && errno == ENOENT && stat( stake, &st ) && errno == ENOENT && stat( vote, &st ) && errno == ENOENT && - stat( genesis, &st ) && errno == ENOENT ) ) - NOT_CONFIGURED( "faucet.json, stake-account.json, vote-account.json, and genesis.bin do not exist" ); + stat( config->ledger.path, &st ) && errno == ENOENT ) ) + NOT_CONFIGURED( "faucet.json, stake-account.json, vote-account.json, and `%s` do not exist", config->ledger.path ); CHECK( check_dir( config->ledger.path, config->uid, config->gid, S_IFDIR | S_IRUSR | S_IWUSR | S_IXUSR ) ); CHECK( check_dir( config->scratch_directory, config->uid, config->gid, S_IFDIR | S_IRUSR | S_IWUSR | S_IXUSR ) ); @@ -147,9 +175,8 @@ check( config_t * const config ) { CHECK( check_file( faucet, config->uid, config->gid, S_IFREG | S_IRUSR | S_IWUSR ) ); CHECK( check_file( stake, config->uid, config->gid, S_IFREG | S_IRUSR | S_IWUSR ) ); CHECK( check_file( vote, config->uid, config->gid, S_IFREG | S_IRUSR | S_IWUSR ) ); - CHECK( check_file( genesis, config->uid, config->gid, S_IFREG | S_IRUSR | S_IWUSR ) ); - CONFIGURE_OK(); + PARTIALLY_CONFIGURED( "genesis directory exists at `%s`", config->ledger.path ); } configure_stage_t cluster = { diff --git a/src/app/fddev/dev.c b/src/app/fddev/dev.c index e290d81770..905c981641 100644 --- a/src/app/fddev/dev.c +++ b/src/app/fddev/dev.c @@ -74,9 +74,6 @@ dev_cmd_fn( args_t * args, validator will get stuck forever. */ config->consensus.wait_for_vote_to_start_leader = 0; - config->consensus.genesis_fetch = 0; - config->consensus.snapshot_fetch = 0; - if( FD_LIKELY( !strcmp( config->consensus.vote_account_path, "" ) ) ) snprintf1( config->consensus.vote_account_path, sizeof( config->consensus.vote_account_path ), diff --git a/src/app/fddev/dev1.c b/src/app/fddev/dev1.c index f3b91d7e11..97df528069 100644 --- a/src/app/fddev/dev1.c +++ b/src/app/fddev/dev1.c @@ -53,6 +53,12 @@ dev1_cmd_fn( args_t * args, validator will get stuck forever. */ config->consensus.wait_for_vote_to_start_leader = 0; + if( FD_LIKELY( !strcmp( config->consensus.vote_account_path, "" ) ) ) + snprintf1( config->consensus.vote_account_path, + sizeof( config->consensus.vote_account_path ), + "%s/vote-account.json", + config->scratch_directory ); + tile_main_args_t tile_args = { .app_name = config->name, .uid = config->uid, diff --git a/src/app/fddev/fddev.h b/src/app/fddev/fddev.h index a8858ee0a7..a8a2ccb840 100644 --- a/src/app/fddev/fddev.h +++ b/src/app/fddev/fddev.h @@ -26,4 +26,18 @@ void dev1_cmd_fn( args_t * args, config_t * const config ); +void +txn_cmd_perm( args_t * args, + security_t * security, + config_t * const config ); + +void +txn_cmd_args( int * pargc, + char *** pargv, + args_t * args); + +void +txn_cmd_fn( args_t * args, + config_t * const config ); + #endif /* HEADER_fd_src_app_fddev_fddev_h */ diff --git a/src/app/fddev/main.c b/src/app/fddev/main.c index e485c34360..d5d855c9ff 100644 --- a/src/app/fddev/main.c +++ b/src/app/fddev/main.c @@ -8,6 +8,7 @@ #include #include #include +#include configure_stage_t * STAGES[ CONFIGURE_STAGE_COUNT ] = { &netns, @@ -26,6 +27,7 @@ configure_stage_t * STAGES[ CONFIGURE_STAGE_COUNT ] = { static action_t DEV_ACTIONS[] = { { .name = "dev", .args = dev_cmd_args, .fn = dev_cmd_fn, .perm = dev_cmd_perm }, { .name = "dev1", .args = dev1_cmd_args, .fn = dev1_cmd_fn, .perm = dev_cmd_perm }, + { .name = "txn", .args = txn_cmd_args, .fn = txn_cmd_fn, .perm = txn_cmd_perm }, }; #define MAX_ARGC 32 @@ -67,9 +69,14 @@ execve_as_root( int argc, FD_LOG_ERR(( "execve(sudo) failed (%i-%s)", errno, fd_io_strerror( errno ) )); } +extern int * fd_log_private_shared_lock; + int main( int argc, char ** _argv ) { + fd_log_private_shared_lock = (int*)mmap( NULL, sizeof(int), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, (off_t)0 ); + if( FD_UNLIKELY( !fd_log_private_shared_lock ) ) exit(1); + /* save original arguments list in case we need to respawn the process as privileged */ int orig_argc = argc; diff --git a/src/app/fddev/txn.c b/src/app/fddev/txn.c new file mode 100644 index 0000000000..1fa67f1b5c --- /dev/null +++ b/src/app/fddev/txn.c @@ -0,0 +1,193 @@ +#include "fddev.h" + +#include "../fdctl/configure/configure.h" +#include "../../ballet/base64/fd_base64.h" +#include "../../tango/quic/fd_quic.h" +#include "../../tango/quic/tests/fd_quic_test_helpers.h" + +#include + +FD_IMPORT_BINARY(sample_transaction, "src/tango/quic/tests/quic_txn.bin"); + +static int g_conn_hs_complete = 0; +static int g_conn_final = 0; +static int g_stream_notify = 0; + +#define MAX_TXN_COUNT 128 + +void +txn_cmd_perm( args_t * args, + security_t * security, + config_t * const config ) { + (void)args; + + if( FD_UNLIKELY( config->development.netns.enabled ) ) + check_cap( security, "txn", CAP_SYS_ADMIN, "enter a network namespace by calling `setns(2)`" ); +} + +void +txn_cmd_args( int * pargc, + char *** pargv, + args_t * args ) { + args->txn.payload_base64 = fd_env_strip_cmdline_cstr( pargc, pargv, "--payload-base64-encoded", NULL, NULL ); + args->txn.count = fd_env_strip_cmdline_ulong( pargc, pargv, "--count", NULL, 1 ); + if( FD_UNLIKELY( !args->txn.count || args->txn.count > MAX_TXN_COUNT ) ) + FD_LOG_ERR(( "count must be between 1 and %d", MAX_TXN_COUNT )); + + args->txn.dst_ip = fd_env_strip_cmdline_cstr( pargc, pargv, "--dst-ip", NULL, 0 ); + args->txn.dst_port = fd_env_strip_cmdline_ushort( pargc, pargv, "--dst-port", NULL, 0 ); +} + +static ulong +cb_now( void * context ) { + (void)context; + return (ulong)fd_log_wallclock(); +} + +static void +cb_conn_hs_complete( fd_quic_conn_t * conn, + void * quic_ctx ) { + (void)conn; + (void)quic_ctx; + g_conn_hs_complete = 1; +} + +static void +cb_conn_final( fd_quic_conn_t * conn, + void * quic_ctx ) { + (void)conn; + (void)quic_ctx; + g_conn_final = 1; +} + +static void +cb_stream_notify( fd_quic_stream_t * stream, + void * stream_ctx, + int notify_type ) { + (void)stream; + (void)stream_ctx; + (void)notify_type; + g_stream_notify = 1; +} + +static void +send_quic_transactions( fd_quic_t * quic, + fd_quic_udpsock_t * udpsock, + ulong count, + uint dst_ip, + ushort dst_port, + fd_aio_pkt_info_t * pkt ) { + fd_quic_set_aio_net_tx( quic, udpsock->aio ); + FD_TEST( fd_quic_init( quic ) ); + + quic->cb.now = cb_now; + quic->cb.conn_final = cb_conn_final; + quic->cb.conn_hs_complete = cb_conn_hs_complete; + quic->cb.stream_notify = cb_stream_notify; + + fd_quic_conn_t * conn = fd_quic_connect( quic, dst_ip, dst_port, NULL ); + while ( FD_LIKELY( !( g_conn_hs_complete || g_conn_final ) ) ) { + fd_quic_service( quic ); + fd_quic_udpsock_service( udpsock ); + } + FD_TEST( conn ); + if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_ACTIVE ) ) + FD_LOG_ERR(( "unable to connect to QUIC endpoint on port %hu, is it running? state is %d", dst_port, conn->state )); + + fd_quic_stream_t * stream = fd_quic_conn_new_stream( conn, FD_QUIC_TYPE_UNIDIR ); + FD_TEST( stream ); + + ulong sent = 0; + while( sent < count ) { + int res = fd_quic_stream_send( stream, pkt + sent, count - sent, 1 ); + if( FD_UNLIKELY( res < 0 ) ) FD_LOG_ERR(( "fd_quic_stream_send failed (%d)", res )); + sent += (ulong)res; + + fd_quic_service( quic ); + fd_quic_udpsock_service( udpsock ); + } + + while ( FD_UNLIKELY( !( g_stream_notify || g_conn_final ) ) ) { + fd_quic_service( quic ); + fd_quic_udpsock_service( udpsock ); + } + + fd_quic_conn_close( conn, 0 ); + fd_quic_fini( quic ); +} + +void +txn_cmd_fn( args_t * args, + config_t * const config ) { + if( FD_UNLIKELY( config->development.netns.enabled ) ) + enter_network_namespace( config->development.netns.interface1 ); + + fd_quic_limits_t quic_limits = { + .conn_cnt = 1UL, + .handshake_cnt = 1UL, + .conn_id_cnt = 4UL, + .conn_id_sparsity = 4.0, + .stream_cnt = { 0UL, // FD_QUIC_STREAM_TYPE_BIDI_CLIENT + 0UL, // FD_QUIC_STREAM_TYPE_BIDI_SERVER + 1UL, // FD_QUIC_STREAM_TYPE_UNI_CLIENT + 0UL }, // FD_QUIC_STREAM_TYPE_UNI_SERVER + .stream_sparsity = 4.0, + .inflight_pkt_cnt = 64UL, + .tx_buf_sz = 1UL<<15UL + }; + ulong quic_footprint = fd_quic_footprint( &quic_limits ); + FD_TEST( quic_footprint ); + + fd_wksp_t * wksp = fd_wksp_new_anonymous( fd_cstr_to_shmem_page_sz("normal"), + 1UL << 10, + fd_shmem_cpu_idx( 0 ), + "wksp", + 0UL ); + FD_TEST( wksp ); + void * mem = fd_wksp_alloc_laddr( wksp, fd_quic_align(), quic_footprint, 1UL ); + fd_quic_t * quic = fd_quic_new( mem, &quic_limits ); + FD_TEST( quic ); + + fd_quic_udpsock_t _udpsock; + fd_quic_udpsock_t * udpsock = fd_quic_client_create_udpsock( &_udpsock, wksp, fd_quic_get_aio_net_rx( quic ), 0 ); + FD_TEST( udpsock == &_udpsock ); + + fd_quic_config_t * client_cfg = &quic->config; + client_cfg->role = FD_QUIC_ROLE_CLIENT; + memcpy( client_cfg->alpns, "\xasolana-tpu", 11UL ); + client_cfg->alpns_sz = 11U; + memcpy( client_cfg->link.dst_mac_addr, config->tiles.quic.mac_addr, 6UL ); + client_cfg->net.ip_addr = udpsock->listen_ip; + client_cfg->net.ephem_udp_port.lo = (ushort)udpsock->listen_port; + client_cfg->net.ephem_udp_port.hi = (ushort)(udpsock->listen_port + 1); + client_cfg->initial_rx_max_stream_data = 1<<15; + client_cfg->idle_timeout = 100; + client_cfg->initial_rx_max_stream_data = FD_QUIC_DEFAULT_INITIAL_RX_MAX_STREAM_DATA; + + fd_aio_pkt_info_t pkt[ MAX_TXN_COUNT ]; + + if( FD_LIKELY( !args->txn.payload_base64 ) ) { + for( ulong i=0; itxn.count; i++ ) { + pkt[ i ].buf = (void * )sample_transaction; + pkt[ i ].buf_sz = (ushort )sample_transaction_sz; + } + } else { + uchar buf[1300]; + int buf_sz = fd_base64_decode( args->txn.payload_base64, buf ); + if( FD_UNLIKELY( buf_sz == -1 ) ) FD_LOG_ERR(( "bad payload input `%s`", args->txn.payload_base64 )); + for( ulong i=0; itxn.count; i++ ) { + pkt[ i ].buf = (void * )buf; + pkt[ i ].buf_sz = (ushort )buf_sz; + } + } + + uint dst_ip = config->tiles.quic.ip_addr; + if( FD_UNLIKELY( args->txn.dst_ip ) ) + if( FD_UNLIKELY( !fd_cstr_to_ip4_addr( args->txn.dst_ip, &dst_ip ) ) ) FD_LOG_ERR(( "invalid --dst-ip" )); + + ushort dst_port = config->tiles.quic.transaction_listen_port; + if( FD_UNLIKELY( args->txn.dst_port ) ) dst_port = args->txn.dst_port; + + send_quic_transactions( quic, udpsock, args->txn.count, dst_ip, dst_port, pkt ); + exit_group( 0 ); +} diff --git a/src/app/frank/fd_frank.h b/src/app/frank/fd_frank.h index e0488848e6..b774487bc0 100644 --- a/src/app/frank/fd_frank.h +++ b/src/app/frank/fd_frank.h @@ -37,6 +37,7 @@ typedef struct { uchar const * out_pod; uchar const * extra_pod; fd_xsk_t * xsk; + fd_xsk_t * lo_xsk; double tick_per_ns; } fd_frank_args_t; diff --git a/src/app/frank/fd_frank_quic.c b/src/app/frank/fd_frank_quic.c index 1229c532b6..e116edbf19 100644 --- a/src/app/frank/fd_frank_quic.c +++ b/src/app/frank/fd_frank_quic.c @@ -20,6 +20,13 @@ init( fd_frank_args_t * args ) { args->xsk = fd_xsk_join( fd_wksp_pod_map( args->tile_pod, "xsk" ) ); if( FD_UNLIKELY( !args->xsk ) ) FD_LOG_ERR(( "fd_xsk_join failed" )); + args->lo_xsk = NULL; + if( FD_UNLIKELY( fd_pod_query_cstr( args->tile_pod, "lo_xsk", NULL ) ) ) { + FD_LOG_INFO(( "loading %s", "lo_xsk" )); + args->lo_xsk = fd_xsk_join( fd_wksp_pod_map( args->tile_pod, "lo_xsk" ) ); + if( FD_UNLIKELY( !args->lo_xsk ) ) FD_LOG_ERR(( "fd_xsk_join (lo) failed" )); + } + /* OpenSSL goes and tries to read files and allocate memory and other dumb things on a thread local basis, so we need a special initializer to do it before seccomp happens in the process. */ @@ -31,6 +38,79 @@ init( fd_frank_args_t * args ) { FD_LOG_ERR(( "OPENSSL_init_crypto failed" )); } +struct fd_quic_tpu_ctx; + +struct root_aio_ctx { + ushort transaction_listen_port; + ushort quic_transaction_listen_port; + + const fd_aio_t * quic_aio; + void (*transaction_callback)( struct fd_quic_tpu_ctx * ctx, uchar const * packet, uint packet_sz ); +}; + +static int +root_aio_net_rx( void * ctx, + fd_aio_pkt_info_t const * batch, + ulong batch_cnt, + ulong * opt_batch_idx, + int flush ) { + struct root_aio_ctx * root_ctx = ctx; + + for( ulong i=0; i packet_end ) ) continue; + + /* Extract IP dest addr and UDP dest port */ + ulong ip_dstaddr = *(uint *)( iphdr+16UL ); + (void)ip_dstaddr; + ushort udp_dstport = *(ushort *)( udp+2UL ); + + uchar const * data = udp + 8U; + uint data_sz = (uint)(packet_end - data); + + ulong ignored; + if( FD_LIKELY( fd_ushort_bswap( udp_dstport ) == root_ctx->quic_transaction_listen_port ) ) + root_ctx->quic_aio->send_func( root_ctx->quic_aio->ctx, batch + i, 1, &ignored, flush ); + else if( FD_LIKELY( fd_ushort_bswap( udp_dstport ) == root_ctx->transaction_listen_port ) ) + root_ctx->transaction_callback( root_ctx->quic_aio->ctx, data, data_sz ); + else + FD_LOG_ERR(( "Firedancer received a UDP packet on port %hu which was not expected. " + "Only ports %hu and %hu should be configured to forward packets. Do " + "you need to reload the XDP program?", + fd_ushort_bswap( udp_dstport ), root_ctx->transaction_listen_port, root_ctx->quic_transaction_listen_port )); + } + + /* the assumption here at present is that any packet that could not be processed + is simply dropped hence, all packets were consumed */ + if( FD_LIKELY( opt_batch_idx ) ) { + *opt_batch_idx = batch_cnt; + } + + return FD_AIO_SUCCESS; +} + +extern void +fd_quic_transaction_receive( struct fd_quic_tpu_ctx * ctx, + uchar const * packet, + uint packet_sz ); + static void run( fd_frank_args_t * args ) { FD_LOG_INFO(( "quic.%lu init", args->idx )); @@ -64,6 +144,13 @@ run( fd_frank_args_t * args ) { fd_xsk_aio_t * xsk_aio = fd_xsk_aio_join( fd_wksp_pod_map( args->tile_pod, "xsk_aio" ), args->xsk ); if( FD_UNLIKELY( !xsk_aio ) ) FD_LOG_ERR(( "fd_xsk_aio_join failed" )); + fd_xsk_aio_t * lo_xsk_aio = NULL; + if( FD_UNLIKELY( args->lo_xsk ) ) { + FD_LOG_INFO(( "loading lo xsk_aio" )); + lo_xsk_aio = fd_xsk_aio_join( fd_wksp_pod_map( args->tile_pod, "lo_xsk_aio" ), args->lo_xsk ); + if( FD_UNLIKELY( !lo_xsk_aio ) ) FD_LOG_ERR(( "fd_xsk_aio_join failed" )); + } + /* Setup local objects used by this tile */ FD_LOG_INFO(( "configuring flow control" )); @@ -106,9 +193,12 @@ run( fd_frank_args_t * args ) { if( FD_UNLIKELY( !src_mac ) ) FD_LOG_ERR(( "src_mac_addr not set" )); fd_memcpy( quic_cfg->link.src_mac_addr, src_mac, 6 ); - ushort listen_port = fd_pod_query_ushort( args->tile_pod, "listen_port", 0 ); - if( FD_UNLIKELY( !listen_port ) ) FD_LOG_ERR(( "listen_port not set" )); - quic_cfg->net.listen_udp_port = listen_port; + ushort transaction_listen_port = fd_pod_query_ushort( args->tile_pod, "transaction_listen_port", 0 ); + if( FD_UNLIKELY( !transaction_listen_port ) ) FD_LOG_ERR(( "transaction_listen_port not set" )); + + ushort quic_transaction_listen_port = fd_pod_query_ushort( args->tile_pod, "quic_transaction_listen_port", 0 ); + if( FD_UNLIKELY( !quic_transaction_listen_port ) ) FD_LOG_ERR(( "quic_transaction_listen_port not set" )); + quic_cfg->net.listen_udp_port = quic_transaction_listen_port; ulong idle_timeout_ms = fd_pod_query_ulong( args->tile_pod, "idle_timeout_ms", 0 ); if( FD_UNLIKELY( !idle_timeout_ms ) ) FD_LOG_ERR(( "idle_timeout_ms not set" )); @@ -120,13 +210,28 @@ run( fd_frank_args_t * args ) { /* Attach to XSK */ - fd_xsk_aio_set_rx ( xsk_aio, fd_quic_get_aio_net_rx( quic ) ); - fd_quic_set_aio_net_tx( quic, fd_xsk_aio_get_tx ( xsk_aio ) ); + const fd_aio_t * quic_aio = fd_quic_get_aio_net_rx( quic ); + + struct root_aio_ctx root_ctx = { + .quic_aio = quic_aio, + .transaction_callback = fd_quic_transaction_receive, + .transaction_listen_port = transaction_listen_port, + .quic_transaction_listen_port = quic_transaction_listen_port, + }; + + fd_aio_t root_aio = { + .ctx = &root_ctx, + .send_func = root_aio_net_rx, + }; + + if( FD_UNLIKELY( lo_xsk_aio) ) fd_xsk_aio_set_rx( lo_xsk_aio, &root_aio ); + fd_xsk_aio_set_rx ( xsk_aio, &root_aio ); + fd_quic_set_aio_net_tx( quic, fd_xsk_aio_get_tx( xsk_aio ) ); /* Start serving */ FD_LOG_INFO(( "%s(%lu) run", args->tile_name, args->tile_idx )); - int err = fd_quic_tile( cnc, quic, xsk_aio, mcache, dcache, lazy, rng, scratch, args->tick_per_ns ); + int err = fd_quic_tile( cnc, quic, xsk_aio, lo_xsk_aio, mcache, dcache, lazy, rng, scratch, args->tick_per_ns ); if( FD_UNLIKELY( err ) ) FD_LOG_ERR(( "fd_quic_tile failed (%i)", err )); } @@ -141,13 +246,14 @@ static long allow_syscalls[] = { static ulong allow_fds( fd_frank_args_t * args, - ulong out_fds_sz, - int * out_fds ) { - if( FD_UNLIKELY( out_fds_sz < 3 ) ) FD_LOG_ERR(( "out_fds_sz %lu", out_fds_sz )); + ulong out_fds_sz, + int * out_fds ) { + if( FD_UNLIKELY( out_fds_sz < 4 ) ) FD_LOG_ERR(( "out_fds_sz %lu", out_fds_sz )); out_fds[ 0 ] = 2; /* stderr */ out_fds[ 1 ] = 3; /* logfile */ out_fds[ 2 ] = args->xsk->xsk_fd; - return 3; + out_fds[ 3 ] = args->lo_xsk ? args->lo_xsk->xsk_fd : -1; + return args->lo_xsk ? 4 : 3; } fd_frank_task_t frank_quic = { diff --git a/src/disco/quic/fd_quic.h b/src/disco/quic/fd_quic.h index 9daebbba94..84d5f15629 100644 --- a/src/disco/quic/fd_quic.h +++ b/src/disco/quic/fd_quic.h @@ -121,6 +121,7 @@ int fd_quic_tile( fd_cnc_t * cnc, /* Local join to the tile's command-and-control */ fd_quic_t * quic, /* QUIC without active join */ fd_xsk_aio_t * xsk_aio, /* Local join to QUIC XSK aio */ + fd_xsk_aio_t * lo_xsk_aio, /* Local join to QUIC XSK aio for loopback interface */ fd_frag_meta_t * mcache, /* Local join to the tile's txn output mcache */ uchar * dcache, /* Local join to the tile's txn output dcache */ long lazy, /* Laziness, <=0 means use a reasonable default */ diff --git a/src/disco/quic/fd_quic_tile.c b/src/disco/quic/fd_quic_tile.c index 0f3865bfe6..d06b53636e 100644 --- a/src/disco/quic/fd_quic_tile.c +++ b/src/disco/quic/fd_quic_tile.c @@ -104,6 +104,29 @@ fd_tpu_conn_destroy( fd_quic_conn_t * conn, ctx->cnc_diag_tpu_conn_live_cnt--; } +static void +fd_tpu_dummy_dcache( fd_quic_tpu_ctx_t * ctx ) { + /* By default the dcache only has headroom for one in-flight fragment, but + QUIC might have many. If we exceed the headroom, we publish a dummy + mcache entry to evict the reader from this fragment we want to use so we + can start using it. + + This is not ideal because if the reader is already done with the fragment + we are writing a useless mcache entry, so we try and do it only when + needed. + + The QUIC receive path might typically execute stream_create, + stream_receive, and stream_notice serially, so it is often the case that + even if we are handling multiple new connections in one receive batch, + the in-flight count remains zero or one. */ + if( ctx->inflight_streams > 0 ) { + ulong ctl = fd_frag_meta_ctl( 0, 1 /* som */, 1 /* eom */, 0 /* err */ ); + ulong tsnow = fd_frag_meta_ts_comp( fd_tickcount() ); + fd_mcache_publish( ctx->mcache, ctx->depth, *ctx->seq, 1, 0, 0, ctl, tsnow, tsnow ); + *ctx->seq = fd_seq_inc( *ctx->seq, 1UL ); + } +} + /* fd_tpu_stream_create implements fd_quic_cb_stream_new_t */ static void fd_tpu_stream_create( fd_quic_stream_t * stream, @@ -143,25 +166,7 @@ fd_tpu_stream_create( fd_quic_stream_t * stream, msg_ctx->sz = 0U; msg_ctx->tsorig = (uint)fd_frag_meta_ts_comp( fd_tickcount() ); - /* By default the dcache only has headroom for one in-flight fragment, but - QUIC might have many. If we exceed the headroom, we publish a dummy - mcache entry to evict the reader from this fragment we want to use so we - can start using it. - - This is not ideal because if the reader is already done with the fragment - we are writing a useless mcache entry, so we try and do it only when - needed. - - The QUIC receive path might typically execute stream_create, - stream_receive, and stream_notice serially, so it is often the case that - even if we are handling multiple new connections in one receive batch, - the in-flight count remains zero or one. */ - if( ctx->inflight_streams > 0 ) { - ulong ctl = fd_frag_meta_ctl( 0, 1 /* som */, 1 /* eom */, 0 /* err */ ); - ulong tsnow = fd_frag_meta_ts_comp( fd_tickcount() ); - fd_mcache_publish( ctx->mcache, ctx->depth, *ctx->seq, 1, 0, 0, ctl, tsnow, tsnow ); - *ctx->seq = fd_seq_inc( *ctx->seq, 1UL ); - } + fd_tpu_dummy_dcache( ctx ); ctx->inflight_streams += 1; @@ -171,6 +176,43 @@ fd_tpu_stream_create( fd_quic_stream_t * stream, stream->context = msg_ctx; /* Update stream dcache entry */ } +void +fd_quic_transaction_receive( fd_quic_t * _ctx, + uchar const * packet, + uint packet_sz ) { + fd_quic_tpu_ctx_t * ctx = _ctx->cb.quic_ctx; + + /* Load dcache info */ + uchar * const base = ctx->base; + uchar * const dcache_app = ctx->dcache_app; + ulong const chunk0 = ctx->chunk0; + ulong const wmark = ctx->wmark; + ulong chunk = ctx->chunk; + + /* Allocate new dcache entry */ + chunk = fd_dcache_compact_next( chunk, FD_TPU_DCACHE_MTU, chunk0, wmark ); + + fd_quic_tpu_msg_ctx_t * msg_ctx = fd_quic_dcache_msg_ctx( dcache_app, chunk0, chunk ); + msg_ctx->conn_id = ULONG_MAX; + msg_ctx->stream_id = ULONG_MAX; + msg_ctx->data = fd_chunk_to_laddr( base, chunk ); + msg_ctx->sz = packet_sz; + msg_ctx->tsorig = (uint)fd_frag_meta_ts_comp( fd_tickcount() ); + + fd_tpu_dummy_dcache( ctx ); + + /* Add to local publish queue */ + if( FD_UNLIKELY( pubq_full( ctx->pubq ) ) ) { + FD_LOG_WARNING(( "pubq full, dropping" )); + return; + } + + fd_memcpy( msg_ctx->data, packet, packet_sz ); + pubq_push( ctx->pubq, msg_ctx ); + + ctx->chunk = chunk; /* Update dcache chunk index */ +} + /* fd_tpu_stream_receive implements fd_quic_cb_stream_receive_t */ static void fd_tpu_stream_receive( fd_quic_stream_t * stream, @@ -259,6 +301,7 @@ int fd_quic_tile( fd_cnc_t * cnc, fd_quic_t * quic, fd_xsk_aio_t * xsk_aio, + fd_xsk_aio_t * lo_xsk_aio, fd_frag_meta_t * mcache, uchar * dcache, long lazy, @@ -448,6 +491,7 @@ fd_quic_tile( fd_cnc_t * cnc, /* Poll network backend */ fd_xsk_aio_service( xsk_aio ); + if( FD_UNLIKELY( lo_xsk_aio ) ) fd_xsk_aio_service( lo_xsk_aio ); /* Service QUIC clients */ fd_quic_service( quic ); diff --git a/src/disco/quic/test_quic_tile.c b/src/disco/quic/test_quic_tile.c index 7e32b69691..e6a805ae89 100644 --- a/src/disco/quic/test_quic_tile.c +++ b/src/disco/quic/test_quic_tile.c @@ -188,6 +188,7 @@ tx_tile_main( int argc, cfg->tx_cnc, cfg->tx_quic, cfg->xsk_aio, + NULL, cfg->tx_mcache, cfg->tx_dcache, cfg->tx_lazy, @@ -225,7 +226,7 @@ int main( int argc, char const * iface = fd_env_strip_cmdline_cstr ( &argc, &argv, "--iface", NULL, NULL ); uint ifqueue = fd_env_strip_cmdline_uint ( &argc, &argv, "--ifqueue", NULL, 0U ); char const * _listen_addr = fd_env_strip_cmdline_cstr ( &argc, &argv, "--listen", NULL, NULL ); - uint udp_port = fd_env_strip_cmdline_uint ( &argc, &argv, "--port", NULL, 8080U ); + ushort udp_port = fd_env_strip_cmdline_ushort( &argc, &argv, "--port", NULL, 8080U ); char const * _hwaddr = fd_env_strip_cmdline_cstr ( &argc, &argv, "--hwaddr", NULL, NULL ); char const * bpf_dir = fd_env_strip_cmdline_cstr ( &argc, &argv, "--bpf-dir", NULL, "test_quic" ); @@ -245,7 +246,7 @@ int main( int argc, if( FD_UNLIKELY( !fd_cstr_to_ip4_addr( _listen_addr, &listen_addr ) ) ) FD_LOG_ERR(( "invalid IPv4 address \"%s\"", _listen_addr )); - if( FD_UNLIKELY( udp_port<=0 || udp_port>USHORT_MAX ) ) + if( FD_UNLIKELY( udp_port<=0 ) ) FD_LOG_ERR(( "invalid UDP port %d", udp_port )); if( FD_UNLIKELY( !_hwaddr ) ) FD_LOG_ERR(( "missing --hwaddr" )); @@ -314,7 +315,7 @@ int main( int argc, FD_LOG_NOTICE(( "Listening on " FD_IP4_ADDR_FMT ":%u", FD_IP4_ADDR_FMT_ARGS( listen_addr ), udp_port )); - FD_TEST( 0==fd_xdp_listen_udp_port( bpf_dir, listen_addr, udp_port, 0U ) ); + FD_TEST( 0==fd_xdp_listen_udp_ports( bpf_dir, listen_addr, 1, &udp_port, 0U ) ); FD_LOG_NOTICE(( "Joining xsk" )); cfg->xsk = fd_xsk_join( shxsk ); diff --git a/src/tango/quic/tests/fd_quic_test_helpers.c b/src/tango/quic/tests/fd_quic_test_helpers.c index e356dafc0a..7a0a68e1aa 100644 --- a/src/tango/quic/tests/fd_quic_test_helpers.c +++ b/src/tango/quic/tests/fd_quic_test_helpers.c @@ -380,7 +380,7 @@ fd_quic_udpsock_create( void * _sock, FD_LOG_NOTICE(( "Adding UDP listener (" FD_IP4_ADDR_FMT ":%u)", FD_IP4_ADDR_FMT_ARGS( quic_sock->listen_ip ), quic_sock->listen_port )); - if( FD_UNLIKELY( 0!=fd_xdp_listen_udp_port( xdp_app_name, quic_sock->listen_ip, quic_sock->listen_port, 0 ) ) ) { + if( FD_UNLIKELY( 0!=fd_xdp_listen_udp_ports( xdp_app_name, quic_sock->listen_ip, 1, &quic_sock->listen_port, 0 ) ) ) { FD_LOG_WARNING(( "failed to add UDP listener" )); fd_xsk_aio_leave( xsk_aio ); fd_xsk_leave( xsk ); diff --git a/src/tango/xdp/fd_xdp_ctl.c b/src/tango/xdp/fd_xdp_ctl.c index f0e215b6f8..851a8aa137 100644 --- a/src/tango/xdp/fd_xdp_ctl.c +++ b/src/tango/xdp/fd_xdp_ctl.c @@ -144,8 +144,9 @@ main( int argc, (void)_proto; uint proto = 1UL; - if( FD_UNLIKELY( 0!=fd_xdp_listen_udp_port( _wksp, ip_addr, (uint)udp_port, proto ) ) ) - FD_LOG_ERR(( "%i: %s: fd_xdp_listen_udp_port(%s,%s,%lu,%s) failed\n\tDo %s help for help", + ushort port = (ushort)udp_port; + if( FD_UNLIKELY( 0!=fd_xdp_listen_udp_ports( _wksp, ip_addr, 1, &port, proto ) ) ) + FD_LOG_ERR(( "%i: %s: fd_xdp_listen_udp_ports(%s,%s,%lu,%s) failed\n\tDo %s help for help", cnt, cmd, _wksp, _ip_addr, udp_port, _proto, bin )); FD_LOG_NOTICE(( "%i: %s %s %s %lu %s: success", cnt, cmd, _wksp, _ip_addr, udp_port, _proto )); diff --git a/src/tango/xdp/fd_xdp_license.h b/src/tango/xdp/fd_xdp_license.h index 62e51d36a3..2579b93f4b 100644 --- a/src/tango/xdp/fd_xdp_license.h +++ b/src/tango/xdp/fd_xdp_license.h @@ -1 +1 @@ -#define FD_LICENSE "Apache-2.0" +#define FD_LICENSE "GPL" diff --git a/src/tango/xdp/fd_xdp_redirect_prog.c b/src/tango/xdp/fd_xdp_redirect_prog.c index 28912e8606..c0376163d9 100644 --- a/src/tango/xdp/fd_xdp_redirect_prog.c +++ b/src/tango/xdp/fd_xdp_redirect_prog.c @@ -47,7 +47,7 @@ static long ulong flags ) = (void *)51U; -#ifdef FD_XDP_LOGGING +#ifndef FD_XDP_LOGGING /* To do logging, you should enable this import and then call it like @@ -112,23 +112,28 @@ int fd_xdp_redirect( struct xdp_md *ctx ) { uchar const * udp = iphdr + iplen; /* Ignore if UDP header is too short */ - if( udp+4U > data_end ) return XDP_PASS; + if( FD_UNLIKELY( udp+4U > data_end ) ) return XDP_PASS; /* Extract IP dest addr and UDP dest port */ ulong ip_dstaddr = *(uint *)( iphdr+16UL ); ulong udp_dstport = *(ushort *)( udp+2UL ); ulong flow_key = (ip_dstaddr<<16) | udp_dstport; + char fmt[] = "len is %d, port is %lu addr is %lu"; + bpf_trace_printk( fmt, sizeof(fmt), data_end - data, udp_dstport, ip_dstaddr ); + /* Filter for known UDP dest ports of interest */ /* FIXME: This generates invalid asm. The lddw instruction for loading the fd_xdp_udp_dsts has src_reg==0, but it should be src_reg==1 */ /* TODO: Consider using inline asm instead */ uint * udp_value = bpf_map_lookup_elem( &fd_xdp_udp_dsts, &flow_key ); - if( !udp_value ) return XDP_PASS; + if( FD_UNLIKELY( !udp_value ) ) return XDP_PASS; /* Look up the interface queue to find the socket to forward to */ uint socket_key = ctx->rx_queue_index; + char fmt3[] = "found udp valuekey is %u"; + bpf_trace_printk( fmt3, sizeof(fmt3), socket_key ); return bpf_redirect_map( &fd_xdp_xsks, socket_key, 0 ); } diff --git a/src/tango/xdp/fd_xdp_redirect_prog.o b/src/tango/xdp/fd_xdp_redirect_prog.o index 9371b53d35fae25e5babf6fb738dbdda53ab97b0..140c62ca7ad8752a2709095738186aa71136bd19 100644 GIT binary patch literal 1704 zcmb`Hzi-n}5XUb~N?M6u4HQBKutX73L?c>2Ai)qS0YXG70wLHeH+Dj(+a$JAwe>(5 zkQiC`17PadEo^4&z)*>Wk&zKt=oaq#emGC5VB#s(-F@!v`RjXjUf!wQ9m`}$uMGXB zVWcV1^W4y{3b#s=T$!Abyw7HgFUOIwQN3&ONh(v1! z)vSB^tgy)=VMyy*RQ+*ze}GIsFZ1E%FBx;Y-y>Qs{}6GJW7iEVRJB6uLR|iya>U?% z&vN4Ox0Jh=tPXfPh^)B$HRZmyYJtmr_5?q70)!}eFKwO=Yi}?Z^!dK$8NKG~5b_^X zp7}oKizA1MI-o>xeSXuKY0>C3a4JOT^ERv&e3svfOP{1Z<{fLz0rz{}a$NeD@=^TMn!Ps|cN zW{vZZK0z8}K{{mP(ePMmmxJapqa6Jd+Rf~y2(q-r^EDSw z=EA!=$ye2c?C&W3mg49G(*63KdB63(f* paxz;RwApN#b$d#X)}?jpK3yBIqi&^3xcd{{cQ!xeWjS delta 472 zcmZ3%`-5|WhUgC_1~_m4vm6*WCfdr>C$WHe`-LIW?92=dT0o|*vNBkpP?!fyrvt?W zfUH6hUI-tkjRVN?Hvw}Q3WdeN3>ZI@MVdhnr~=tsKA?(1sQPrEevp%V!+{JNAcNl- zte;^&Q~^6s9EccnRfND2nMM#FBnrxb`RR-h2l*>P^i@m!_|Gu;Dx<9gD_9^g3FKS` z1__`#2$;NzNsi00ATc>3Ro6(*U~(ag+++o21t*Z+2jTmzR4?CBshzq;odm;BuhP01mk2cRx`l> tsGtH=Kl|i~toocYp!RSBmC8@nV3QCmfU1MJg?(}$n?BPD#>q`=Y5=+PH3$Fz diff --git a/src/tango/xdp/fd_xdp_redirect_user.c b/src/tango/xdp/fd_xdp_redirect_user.c index 8959bfa4a3..fe50ad694f 100644 --- a/src/tango/xdp/fd_xdp_redirect_user.c +++ b/src/tango/xdp/fd_xdp_redirect_user.c @@ -417,7 +417,7 @@ fd_xdp_unhook_iface( char const * app_name, /* Remove pinned maps */ - if( FD_UNLIKELY( 0!=unlinkat( dir_fd, "xsks", 0 ) ) ) { + if( FD_UNLIKELY( 0!=unlinkat( dir_fd, "xsks", 0 ) && errno != ENOENT ) ) { FD_LOG_WARNING(( "unlinkat(\"%s\",\"xsks\",0) failed (%i-%s)", path, errno, fd_io_strerror( errno ) )); close( dir_fd ); return -1; @@ -425,7 +425,7 @@ fd_xdp_unhook_iface( char const * app_name, /* Remove pinned program */ - if( FD_UNLIKELY( 0!=unlinkat( dir_fd, "xdp_prog", 0 ) ) ) { + if( FD_UNLIKELY( 0!=unlinkat( dir_fd, "xdp_prog", 0 ) && errno != ENOENT ) ) { FD_LOG_WARNING(( "unlinkat(\"%s\",\"xdp_prog\",0) failed (%i-%s)", path, errno, fd_io_strerror( errno ) )); close( dir_fd ); return -1; @@ -433,16 +433,23 @@ fd_xdp_unhook_iface( char const * app_name, /* Remove pinned program link */ - if( FD_UNLIKELY( 0!=unlinkat( dir_fd, "xdp_link", 0 ) ) ) { + if( FD_UNLIKELY( 0!=unlinkat( dir_fd, "xdp_link", 0 ) && errno != ENOENT ) ) { FD_LOG_WARNING(( "unlinkat(\"%s\",\"xdp_link\",0) failed (%i-%s)", path, errno, fd_io_strerror( errno ) )); close( dir_fd ); return -1; } + if( FD_UNLIKELY( 0!=unlinkat( dir_fd, "xdp_link2", 0 ) && errno != ENOENT ) ) { + FD_LOG_WARNING(( "unlinkat(\"%s\",\"xdp_link2\",0) failed (%i-%s)", path, errno, fd_io_strerror( errno ) )); + close( dir_fd ); + return -1; + } + /* Clean up */ close( dir_fd ); - rmdir( path ); + if( FD_UNLIKELY( -1==rmdir( path ) ) ) + FD_LOG_WARNING(( "rmdir(%s) failed (%i-%s)", path, errno, fd_io_strerror( errno ) )); return 0; } @@ -461,10 +468,11 @@ fd_xdp_get_udp_dsts_map( char const * app_name ) { } int -fd_xdp_listen_udp_port( char const * app_name, - uint ip4_dst_addr, - uint udp_dst_port, - uint proto ) { +fd_xdp_listen_udp_ports( char const * app_name, + uint ip4_dst_addr, + ulong udp_dst_ports_sz, + ushort * udp_dst_ports, + uint proto ) { /* Validate arguments */ if( FD_UNLIKELY( 0!=fd_xdp_validate_name_cstr( app_name, NAME_MAX, "app_name" ) ) ) @@ -477,14 +485,16 @@ fd_xdp_listen_udp_port( char const * app_name, /* Insert element */ - ulong key = fd_xdp_udp_dst_key( ip4_dst_addr, udp_dst_port ); - uint value = proto; + uint value = proto; + for( ulong i=0; i +#include /* FIXME add bounds checks to fd_util version */ @@ -34,3 +35,13 @@ fd_cstr_to_ip4_addr( char const * s, *out = FD_IP4_ADDR( v0, v1, v2, v3 ); return 1; } + +void +fd_ip4_addr_to_cstr( uint addr, + char * buf ) { + sprintf( buf, "%u.%u.%u.%u", + (addr )&0xFF, + (addr>> 8)&0xFF, + (addr>>16)&0xFF, + (addr>>24)&0xFF ); +} diff --git a/src/util/net/fd_ip4.h b/src/util/net/fd_ip4.h index 792ae6236a..3f9af64486 100644 --- a/src/util/net/fd_ip4.h +++ b/src/util/net/fd_ip4.h @@ -124,6 +124,14 @@ int fd_cstr_to_ip4_addr( char const * s, uint * addr ); + +/* fd_ip4_addr_to_cstr converts an ip address back to a formatted + string like %u.%u.%u.%u The input buffer is assumed to be at + least 16 bytes, a trailing NUL byte is also written. */ +void +fd_ip4_addr_to_cstr( uint addr, + char * buf ); + /* fd_ip4_hdr_bswap reverses the endianness of all fields in the IPv4 header. */