Merge branch 'main' into chali/feat/quic-tpu-stake-weighted-flow-control

.github/workflows/codeql.yml

@@ -28,7 +28,7 @@ jobs:
         # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
     defaults:
       run:
-        shell: bash --noprofile --norc -eo pipefail -c "set -ex; export MACHINE=linux_clang_x86_64; source /home/runner/firedancer-opts/activate-opt && chmod +x {0} && {0}"
+        shell: bash --noprofile --norc -eo pipefail -c "set -ex; export MACHINE=linux_clang_x86_64; chmod +x {0} && {0}"
     steps:
     - name: Checkout repository
       uses: actions/checkout@v3

.github/workflows/fuzz_artifacts.yml

@@ -0,0 +1,39 @@
+name: Make Fuzz Artifacts
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+  merge_group:
+jobs:
+  make-with-sanitizer:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install zip
+        run: |
+          sudo apt update
+          sudo apt install -y zip
+
+      - uses: actions/checkout@v3
+
+      - name: Set extras
+        run: |
+          echo "EXTRAS=static" >> "$GITHUB_ENV"
+
+      - name: Prepare asan
+        run: |
+          echo "MACHINE=linux_clang_x86_64_fuzz_asan" >> "$GITHUB_ENV"
+
+      - uses: firedancer-io/alpine-builder@main
+        name: Build with asan on Alpine
+
+      - uses: firedancer-io/clusterfuzz-action@main
+        if: ${{ github.ref == 'refs/heads/main' }}
+        name: Publish asan artifacts to ClusterFuzz
+        with:
+          bucket-name: firedancer-builds.isol-clusterfuzz.appspot.com
+          artifact-dir: build/linux/clang/x86_64_fuzz_asan/fuzz-test
+          object-prefix: main/libfuzzer-asan/firedancer
+          project-id: isol-clusterfuzz
+          service-account-credentials: ${{ secrets.FUZZ_SERVICE_ACCT_JSON_BUNDLE }}

.github/workflows/make_build.yml

@@ -12,12 +12,8 @@ jobs:
       matrix:
         compiler: [gcc, clang]
     runs-on: [self-hosted, Linux, X64]
-    defaults:
-      run:
-        shell: bash --noprofile --norc -eo pipefail -c "set -ex; source /home/runner/firedancer-opts/activate-opt && echo `pwd` && ls -lah {0} && chmod +x {0} && {0}"
-
     env:
-      MACHINE: linux_${{ matrix.compiler }}_x86_64
+      CC: ${{ matrix.compiler }}
     steps:
       - uses: actions/checkout@v3
 

.github/workflows/make_coverage.yml

@@ -10,7 +10,7 @@ jobs:
     runs-on: [self-hosted, Linux, X64]
     defaults:
       run:
-        shell: sudo bash --noprofile --norc -eo pipefail -c "export MACHINE=linux_clang_x86_64; export EXTRAS=llvm-cov; set -ex; source /home/runner/firedancer-opts/activate-opt && chmod +x {0} && {0}"
+        shell: sudo bash --noprofile --norc -eo pipefail -c "export CC=clang; export EXTRAS=llvm-cov; set -ex; chmod +x {0} && {0}"
     steps:
       - uses: actions/checkout@v3
 
@@ -26,7 +26,7 @@ jobs:
       - name: Make coverage report
         run: |
           make cov-report
-          mv build/linux/clang/x86_64/cov/html coverage-report
+          mv build/native/clang/cov/html coverage-report
 
       - name: Upload coverage HTML report as GitHub artifact
         uses: actions/upload-artifact@v3
@@ -38,8 +38,8 @@ jobs:
         uses: codecov/codecov-action@v3
         timeout-minutes: 5
         with:
-          files: build/linux/clang/x86_64/cov/cov.lcov
-          name: codecov-make-linux_clang-14_x86_64
+          files: build/native/clang/cov/cov.lcov
+          name: codecov-make-linux_clang-14_native
           fail_ci_if_error: false
           functionalities: search
 

.github/workflows/make_test.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: [self-hosted, Linux, X64]
     defaults:
       run:
-        shell: sudo bash --noprofile --norc -eo pipefail -c "set -ex; source /home/runner/firedancer-opts/activate-opt && chmod +x {0} && {0}"
+        shell: sudo bash --noprofile --norc -eo pipefail -c "set -ex; export MACHINE=linux_clang_x86_64; chmod +x {0} && {0}"
     steps:
       - uses: actions/checkout@v3
 
@@ -22,13 +22,13 @@ jobs:
         run: make -j
 
       - name: Run unit tests
-        run: make -k -j4 --output-sync=target run-unit-test
+        run: make -k -j2 --output-sync=target run-unit-test
 
   make-test-asan:
     runs-on: [self-hosted, Linux, X64]
     defaults:
       run:
-        shell: sudo bash --noprofile --norc -eo pipefail -c "set -ex; export MACHINE=linux_clang_x86_64_asan; source /home/runner/firedancer-opts/activate-opt && chmod +x {0} && {0}"
+        shell: sudo bash --noprofile --norc -eo pipefail -c "set -ex; export MACHINE=linux_clang_x86_64_asan; chmod +x {0} && {0}"
     steps:
       - uses: actions/checkout@v3
 
@@ -39,16 +39,16 @@ jobs:
         run: make -j
 
       - name: Run unit tests
-        run: make -k -j4 --output-sync=target run-unit-test
-  
+        run: make -k -j2 --output-sync=target run-unit-test
+
   make-test-ubsan:
     runs-on: [self-hosted, Linux, X64]
     strategy:
       matrix:
         compiler: [gcc, clang]
     defaults:
       run:
-        shell: sudo bash --noprofile --norc -eo pipefail -c "set -ex; export MACHINE=linux_${{ matrix.compiler }}_x86_64_ubsan; source /home/runner/firedancer-opts/activate-opt && chmod +x {0} && {0}"
+        shell: sudo bash --noprofile --norc -eo pipefail -c "set -ex; export MACHINE=linux_${{ matrix.compiler }}_x86_64_ubsan; chmod +x {0} && {0}"
     steps:
       - uses: actions/checkout@v3
 
@@ -59,5 +59,5 @@ jobs:
         run: make -j
 
       - name: Run unit tests
-        run: make -k -j4 --output-sync=target run-unit-test
+        run: make -k -j2 --output-sync=target run-unit-test
 

Cargo.toml

@@ -1,6 +1,5 @@
 [workspace]
-members = ["src/app/fdctl", "ffi/rust/firedancer-sys", "ffi/rust/firedancer-diff"]
-default-members = ["src/app/fdctl"]
+members = ["ffi/rust/firedancer-sys", "ffi/rust/firedancer-diff"]
 resolver = "2"
 
 [profile.dev]

Makefile

@@ -2,7 +2,7 @@
 #
 #  make -j
 #
-# will do a parallel make all targets for the default machine.
+# will do a parallel make all targets for the native machine.
 #
 # The environment variable MACHINE allows building for different
 # machines.  As such, the above is equivalent to running:
@@ -45,8 +45,7 @@
 # help building efficiently on systems with cpu isolation enabled.
 
 ifndef MACHINE
-$(warning MACHINE not specified, using default, run make help for more info)
-MACHINE=linux_gcc_x86_64
+MACHINE=native
 endif
 
 $(info Using MACHINE=$(MACHINE))

README.md

@@ -26,33 +26,41 @@ setup instructions.
 $ git clone https://github.com/firedancer-io/firedancer.git
 $ cd firedancer
 $ ./deps.sh
-$ make -j
+$ make -j run
 ```
 
-## Running
-
 Firedancer uses several privileged operating system features to improve
 performance and security. These can be configured manually in your
-environment, but we also have a command line utility `fdctl` that lets a
-user set up and tear down the environment easily.
+environment, but we also provide a tool for automatic configuration.
 
-```bash
-$ cargo run run --configure # Run the Firedancer program, configuring required system knobs
-```
+The default `make run` target will ensure your system is configured
+correctly before running Firedancer, using `sudo` to make privileged
+changes where necessary.
+
+## Running
 
-When done, the environment can be returned to normal.
+In production, it is recommended to configure the system immediately at
+boot time rather than when running Firedancer. This ensures we can
+allocate the contiguous memory we need, as over time memory may become
+fragmented.
 
 ```bash
-$ cargo run configure all fini # Remove any special configuration we installed
+$ fdctl configure init all
 ```
 
-`fdctl` reads from a `FIREDANCER_CONFIG_TOML` environment variable to
-determine all options needed in configuring and running the program. A
-complete list of configuration options is provided in
+`fdctl` reads from an optional `FIREDANCER_CONFIG_TOML` environment
+variable to determine all configuration. A complete list of
+configuration options and their defaults are provided in
 [default.toml](src/app/fdctl/config/default.toml)
 
-Some of the privileged system configuration steps performed by `fdctl`
-are,
+Later, when you wish to start the validator, you can run
+
+```bash
+$ fdctl run
+```
+
+Some of the privileged system configuration steps performed by `fdctl
+configure` are,
 
 * **Huge pages** Memory needed for Firedancer must be pre-allocated
 before launching it. Firedancer uses `huge` and `gigantic` memory pages,
@@ -70,8 +78,9 @@ debugging when using XDP. Performing these initial restrictions can
 require additional capabilities.
 
 A good way to see what privileges are needed to configure the
-environment is to run `fdctl configure` as a non-privileged user, which
-will display information about the operations it wishes to perform.
+environment is to run `fdctl configure init all` as a non-privileged
+user, which will display information about the operations it wishes to
+perform.
 
 ## License
 Firedancer is available under the [Apache 2

config/base.mk

@@ -7,7 +7,7 @@ CFLAGS:=-std=c17
 CXX:=g++
 CXXFLAGS:=-std=c++17
 LD:=g++
-LDFLAGS:=-lm
+LDFLAGS:=-lm -L./opt/lib -L./opt/lib64
 AR:=ar
 ARFLAGS:=rv
 RANLIB:=ranlib

config/everything.mk

@@ -2,7 +2,7 @@ MAKEFLAGS += --no-builtin-rules
 MAKEFLAGS += --no-builtin-variables
 .SUFFIXES:
 .SUFFIXES: .h .hxx .c .cxx .o .a .d .S .i
-.PHONY: all bin include lib unit-test fuzz-test run-unit-test help clean distclean asm ppp show-deps lint check-lint
+.PHONY: all bin run monitor include lib unit-test fuzz-test run-unit-test help clean distclean asm ppp show-deps lint check-lint
 .SECONDARY:
 .SECONDEXPANSION:
 
@@ -82,6 +82,28 @@ check-lint:
 	#######################################################################
 	$(FIND) src/ -iname "*.c" -or -iname "*.h" | uncrustify -c lint.cfg -F - --check
 
+ifeq (run,$(firstword $(MAKECMDGOALS)))
+  RUN_ARGS := $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS))
+  ifeq ($(RUN_ARGS),)
+    RUN_ARGS := --configure --sudo
+  endif
+  $(eval $(RUN_ARGS):;@:)
+endif
+
+run: bin
+	$(OBJDIR)/bin/fdctl $(RUN_ARGS)
+
+ifeq (monitor,$(firstword $(MAKECMDGOALS)))
+  MONITOR_ARGS := $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS))
+  ifeq ($(MONITOR_ARGS),)
+    MONITOR_ARGS := --sudo
+  endif
+  $(eval $(MONITOR_ARGS):;@:)
+endif
+
+monitor: bin
+	$(OBJDIR)/bin/fdctl monitor $(MONITOR_ARGS)
+
 ##############################
 # Usage: $(call make-lib,name)
 

config/linux_clang_x86_64.mk

@@ -8,16 +8,18 @@ include config/with-optimization.mk
 include config/with-threads.mk
 include config/with-openssl.mk
 
+include config/x86-64-clang-flags.mk
+include config/x86-64-flags.mk
+
 # Clang sadly doesn't support important optimizations.  This practically
 # limits clang usage to code hygenine usage for the time being.  Here,
 # ideally would do:
 #
 # -falign-functions=32 -falign-jumps=32 -falign-labels=32 -falign-loops=32
 # -mbranch-cost=5
 
-CPPFLAGS+=-fomit-frame-pointer -march=haswell -mtune=skylake -mfpmath=sse \
-	  -DFD_HAS_INT128=1 -DFD_HAS_DOUBLE=1 -DFD_HAS_ALLOCA=1 -DFD_HAS_X86=1 -DFD_HAS_SSE=1 -DFD_HAS_AVX=1
-LDFLAGS+=-lrt
+CPPFLAGS+=-march=haswell -mtune=skylake
+CPPFLAGS+=-DFD_HAS_INT128=1 -DFD_HAS_DOUBLE=1 -DFD_HAS_ALLOCA=1 -DFD_HAS_X86=1 -DFD_HAS_SSE=1 -DFD_HAS_AVX=1
 
 FD_HAS_INT128:=1
 FD_HAS_DOUBLE:=1

config/linux_gcc_x86_64.mk

@@ -8,9 +8,11 @@ include config/with-optimization.mk
 include config/with-threads.mk
 include config/with-openssl.mk
 
-CPPFLAGS+=-fomit-frame-pointer -falign-functions=32 -falign-jumps=32 -falign-labels=32 -falign-loops=32 \
-          -march=haswell -mtune=skylake -mfpmath=sse -mbranch-cost=5 \
-	  -DFD_HAS_INT128=1 -DFD_HAS_DOUBLE=1 -DFD_HAS_ALLOCA=1 -DFD_HAS_X86=1 -DFD_HAS_SSE=1 -DFD_HAS_AVX=1
+include config/x86-64-flags.mk
+include config/x86-64-gcc-flags.mk
+
+CPPFLAGS+=-march=haswell -mtune=skylake
+CPPFLAGS+=-DFD_HAS_INT128=1 -DFD_HAS_DOUBLE=1 -DFD_HAS_ALLOCA=1 -DFD_HAS_X86=1 -DFD_HAS_SSE=1 -DFD_HAS_AVX=1
 
 FD_HAS_INT128:=1
 FD_HAS_DOUBLE:=1