Add first integration

.github/actions/download_playground/action.yaml

@@ -0,0 +1,30 @@
+name: "Download fake relay"
+description: "Download fake relay for integration tests"
+runs:
+  using: "composite"
+  steps:
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Set up Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.22
+
+    - name: Clone remote repository
+      shell: bash
+      run: |
+        git clone https://github.com/flashbots/suave-playground.git
+
+    - name: Build and run
+      shell: bash
+      run: |
+        cd suave-playground
+        go build -v -o suave-playground .
+        echo "$PWD" >> $GITHUB_PATH
+
+    - name: Install reth & lighthouse
+      shell: bash
+      run: |
+        ./suave-playground download-artifacts
+        echo "$PWD" >> $GITHUB_PATH/output

.github/workflows/integration.yaml

@@ -0,0 +1,61 @@
+name: Checks
+
+on:
+  workflow_dispatch:
+  pull_request:
+  merge_group:
+  push:
+    branches: [develop]
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  lint_and_test:
+    name: Lint and test
+    runs-on: warp-ubuntu-latest-x64-16x
+    strategy:
+      matrix:
+        toolchain:
+          - stable
+          #- beta
+          #- nightly
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v4
+
+      - name: Download suave playground
+        uses: ./.github/actions/download_playground
+
+      - name: Install Foundry toolchain
+        uses: foundry-rs/foundry-toolchain@v1
+        with:
+          version: nightly
+
+      - name: Install native dependencies
+        run: sudo apt-get install -y libsqlite3-dev
+
+      - name: Build the rbuilder
+        run: cargo build
+
+      - name: Check reth
+        run: |
+          which reth
+          reth --version
+
+      - name: Check lighthouse
+        run: |
+          which lighthouse
+          lighthouse --version
+
+      - name: Run the playground
+        run: suave-playground --output /tmp/playground &
+
+      - name: Run integration tests
+        run: cargo test --package rbuilder --lib -- integration::tests::test_simple_example --exact
+
+      - name: Upload playground artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: playground
+          path: /tmp/playground

.gitignore

@@ -5,6 +5,7 @@
 /target
 /scripts/benchmark-results.*
 /test/
+/integration_logs
 
 # editors
 .code

crates/rbuilder/src/integration/config.rs

@@ -0,0 +1,49 @@
+pub const CONFIG: &str = r#"
+log_json = false
+log_level = "info,rbuilder=debug"
+telemetry_port = 6060
+telemetry_ip = "0.0.0.0"
+
+relay_secret_key = "5eae315483f028b5cdd5d1090ff0c7618b18737ea9bf3c35047189db22835c48"
+coinbase_secret_key = "ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"
+
+cl_node_url = ["http://localhost:3500"]
+jsonrpc_server_port = 8645
+jsonrpc_server_ip = "0.0.0.0"
+el_node_ipc_path = "/tmp/reth.ipc"
+extra_data = "⚡🤖"
+
+dry_run = false
+dry_run_validation_url = "http://localhost:8545"
+
+blocks_processor_url = "http://block_processor.internal"
+ignore_cancellable_orders = true
+
+sbundle_mergeabe_signers = []
+# slot_delta_to_start_submits_ms is usually negative since we start bidding BEFORE the slot start
+#slot_delta_to_start_submits_ms = -5000
+live_builders = ["mp-ordering"]
+
+[[relays]]
+name = "custom"
+url = "http://0xac6e77dfe25ecd6110b8e780608cce0dab71fdd5ebea22a16c0205200f2f8e2e3ad3b71d3499c54ad14d6c21b41a37ae@localhost:5555"
+priority = 0
+use_ssz_for_submit = false
+use_gzip_for_submit = false
+
+[[builders]]
+name = "mgp-ordering"
+algo = "ordering-builder"
+discard_txs = true
+sorting = "mev-gas-price"
+failed_order_retries = 1
+drop_failed_orders = true
+
+[[builders]]
+name = "mp-ordering"
+algo = "ordering-builder"
+discard_txs = true
+sorting = "max-profit"
+failed_order_retries = 1
+drop_failed_orders = true
+"#;

crates/rbuilder/src/integration/mod.rs

@@ -0,0 +1,226 @@
+use crate::{
+    beacon_api_client::{Client, PayloadAttributesTopic},
+    mev_boost::{ProposerPayloadDelivered, RelayClient, RelayError},
+};
+use alloy_network::EthereumWallet;
+use alloy_primitives::{address, Address};
+use alloy_signer_local::PrivateKeySigner;
+use futures::StreamExt;
+use primitive_types::H384;
+use reth::rpc::types::beacon::events::PayloadAttributesEvent;
+use serde::{Deserialize, Serialize};
+use std::{
+    fs::{File, OpenOptions},
+    io,
+    io::prelude::*,
+    path::PathBuf,
+    process::{Child, Command},
+    str::FromStr,
+    time::SystemTime,
+};
+use time::{format_description::well_known, OffsetDateTime};
+use url::Url;
+
+mod config;
+use config::CONFIG;
+
+#[derive(Debug)]
+pub enum FakeMevBoostRelayError {
+    SpawnError,
+    BinaryNotFound,
+}
+
+pub struct FakeMevBoostRelay {}
+
+impl Default for FakeMevBoostRelay {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+fn open_log_file(path: PathBuf) -> File {
+    let prefix = path.parent().unwrap();
+    std::fs::create_dir_all(prefix).unwrap();
+
+    OpenOptions::new()
+        .append(true)
+        .create(true)
+        .open(path)
+        .unwrap()
+}
+
+impl FakeMevBoostRelay {
+    pub fn new() -> Self {
+        Self {}
+    }
+
+    pub fn spawn(self) -> Option<FakeMevBoostRelayInstance> {
+        self.try_spawn().unwrap()
+    }
+
+    fn try_spawn(self) -> Result<Option<FakeMevBoostRelayInstance>, FakeMevBoostRelayError> {
+        let playground_dir = std::env::var("PLAYGROUND_DIR").unwrap();
+
+        // append to the config template the paths to the playground
+        let mut config = CONFIG.to_string();
+        config.insert_str(
+            0,
+            format!("chain = \"{}/genesis.json\"\n", playground_dir).as_str(),
+        );
+        config.insert_str(
+            0,
+            format!("reth_datadir = \"{}/data_reth\"\n", playground_dir).as_str(),
+        );
+
+        // write the config into /tmp/rbuilder.toml
+        let mut file = File::create("/tmp/rbuilder.toml").unwrap();
+        file.write_all(config.as_bytes()).unwrap();
+
+        // load the binary from the cargo_dir
+        let mut bin_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+        bin_path.push("../../target/debug/rbuilder");
+
+        let dt: OffsetDateTime = SystemTime::now().into();
+        let name = dt.format(&well_known::Rfc2822).unwrap();
+
+        let mut log_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+        log_path.push(format!("../../integration_logs/{:?}.log", name));
+
+        let log = open_log_file(log_path);
+
+        let mut cmd = Command::new(bin_path.clone());
+
+        cmd.arg("run").arg("/tmp/rbuilder.toml");
+        cmd.stdout(log.try_clone().unwrap())
+            .stderr(log.try_clone().unwrap());
+
+        match cmd.spawn() {
+            Ok(child) => Ok(Some(FakeMevBoostRelayInstance { child })),
+            Err(e) => match e.kind() {
+                io::ErrorKind::NotFound => Err(FakeMevBoostRelayError::BinaryNotFound),
+                _ => Err(FakeMevBoostRelayError::SpawnError),
+            },
+        }
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+struct PayloadDelivered {
+    slot: String,
+    block_number: String,
+    builder_pubkey: String,
+}
+
+#[derive(Debug)]
+enum PayloadDeliveredError {
+    Empty,
+    IncorrectBuilder(H384),
+    RelayError(RelayError),
+}
+
+#[derive(Debug)]
+pub struct FakeMevBoostRelayInstance {
+    child: Child,
+}
+
+impl Drop for FakeMevBoostRelayInstance {
+    fn drop(&mut self) {
+        self.child.kill().expect("could not kill mev-boost-server");
+    }
+}
+
+impl FakeMevBoostRelayInstance {
+    async fn wait_for_next_slot(
+        &self,
+    ) -> Result<PayloadAttributesEvent, Box<dyn std::error::Error>> {
+        let client = Client::new(Url::parse("http://localhost:3500")?);
+        let mut stream = client.get_events::<PayloadAttributesTopic>().await?;
+
+        // wait for the next slot to send it so that it has enough time to build it.
+        let event = stream.next().await.unwrap()?; // Fix unwrap
+        Ok(event)
+    }
+
+    fn el_url(&self) -> &str {
+        "http://localhost:8545"
+    }
+
+    fn prefunded_keys(&self, indx: usize) -> EthereumWallet {
+        let signer: PrivateKeySigner =
+            "59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d"
+                .parse()
+                .unwrap();
+        let wallet = EthereumWallet::from(signer);
+        vec![wallet][indx].clone()
+    }
+
+    fn builder_address(&self) -> Address {
+        address!("f39Fd6e51aad88F6F4ce6aB8827279cffFb92266")
+    }
+
+    async fn validate_block_built(
+        &self,
+        block_number: u64,
+    ) -> Result<ProposerPayloadDelivered, PayloadDeliveredError> {
+        tokio::time::sleep(std::time::Duration::from_secs(1)).await; // FIX,
+
+        let client = RelayClient::from_url(
+            Url::parse("http://localhost:5555").unwrap(),
+            None,
+            None,
+            None,
+        );
+
+        let payload = client
+            .proposer_payload_delivered_block_number(block_number)
+            .await
+            .map_err(|err| PayloadDeliveredError::RelayError(err))?
+            .ok_or(PayloadDeliveredError::Empty)?;
+
+        let builder_pubkey = H384::from_str("0xa1885d66bef164889a2e35845c3b626545d7b0e513efe335e97c3a45e534013fa3bc38c3b7e6143695aecc4872ac52c4").unwrap();
+        if payload.builder_pubkey == builder_pubkey {
+            Ok(payload)
+        } else {
+            Err(PayloadDeliveredError::IncorrectBuilder(
+                payload.builder_pubkey,
+            ))
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use alloy_network::TransactionBuilder;
+    use alloy_primitives::U256;
+    use alloy_provider::{Provider, ProviderBuilder};
+    use alloy_rpc_types::TransactionRequest;
+    use url::Url;
+
+    #[tokio::test]
+    async fn test_simple_example() {
+        let srv = FakeMevBoostRelay::new().spawn().unwrap();
+        srv.wait_for_next_slot().await.unwrap();
+
+        // send a transfer to the builder
+        let provider = ProviderBuilder::new()
+            .with_recommended_fillers()
+            .wallet(srv.prefunded_keys(0))
+            .on_http(Url::parse(srv.el_url()).unwrap());
+
+        let gas_price = provider.get_gas_price().await.unwrap();
+
+        let tx = TransactionRequest::default()
+            .with_to(srv.builder_address())
+            .with_value(U256::from_str("10000000000000000000").unwrap())
+            .with_gas_price(gas_price)
+            .with_gas_limit(21000);
+
+        let pending_tx = provider.send_transaction(tx).await.unwrap();
+        let receipt = pending_tx.get_receipt().await.unwrap();
+
+        srv.validate_block_built(receipt.block_number.unwrap())
+            .await
+            .unwrap();
+    }
+}

crates/rbuilder/src/lib.rs

@@ -2,6 +2,7 @@ pub mod backtest;
 pub mod beacon_api_client;
 pub mod building;
 pub mod flashbots;
+pub mod integration;
 pub mod live_builder;
 pub mod mev_boost;
 pub mod primitives;

crates/rbuilder/src/live_builder/order_input/mod.rs

@@ -194,8 +194,6 @@ pub async fn start_orderpool_jobs<DB: Database + Clone + 'static>(
     .await?;
 
     let handle = tokio::spawn(async move {
-        info!("OrderPoolJobs: started");
-
         // @Maybe we should add sleep here because each new order will trigger locking
         let mut new_commands = Vec::new();
         let mut order_receiver = order_receiver;