chore(host): Consolidate fetchers into HintHandlers

Cargo.toml

@@ -104,6 +104,7 @@ alloy-sol-types = { version = "0.8.19", default-features = false }
 # OP Alloy
 op-alloy-consensus = { version = "0.10.0", default-features = false }
 op-alloy-rpc-types-engine = { version = "0.10.0", default-features = false }
+op-alloy-network = { version = "0.10.0", default-features = false }
 
 # General
 lru = "0.13.0"

bin/host/Cargo.toml

@@ -44,6 +44,7 @@ alloy-rpc-types-beacon.workspace = true
 
 # Op Alloy
 op-alloy-rpc-types-engine = { workspace = true, features = ["serde"] }
+op-alloy-network.workspace = true
 
 # Revm
 revm = { workspace = true, features = ["std", "c-kzg", "secp256k1", "portable", "blst"] }

bin/host/src/backend/mod.rs

@@ -0,0 +1,9 @@
+//! Backend for the preimage server.
+
+mod offline;
+pub use offline::OfflineHostBackend;
+
+mod online;
+pub use online::{HintHandler, OnlineHostBackend, OnlineHostBackendCfg};
+
+pub(crate) mod util;

bin/host/src/backend/online.rs

@@ -0,0 +1,133 @@
+//! Contains the [OnlineHostBackend] definition.
+
+use crate::SharedKeyValueStore;
+use anyhow::Result;
+use async_trait::async_trait;
+use kona_preimage::{
+    errors::{PreimageOracleError, PreimageOracleResult},
+    HintRouter, PreimageFetcher, PreimageKey,
+};
+use std::{hash::Hash, str::FromStr, sync::Arc};
+use tokio::sync::RwLock;
+use tracing::{error, trace, warn};
+
+/// The [OnlineHostBackendCfg] trait is used to define the type configuration for the
+/// [OnlineHostBackend].
+pub trait OnlineHostBackendCfg {
+    /// The hint type describing the range of hints that can be received.
+    type Hint: FromStr + Hash + Eq + PartialEq + Send + Sync;
+
+    /// The providers that are used to fetch data in response to hints.
+    type Providers: Send + Sync;
+}
+
+/// A [HintHandler] is an interface for receiving hints, fetching remote data, and storing it in the
+/// key-value store.
+#[async_trait]
+pub trait HintHandler {
+    /// The type configuration for the [HintHandler].
+    type Cfg: OnlineHostBackendCfg;
+
+    /// Fetches data in response to a hint.
+    async fn fetch_hint(
+        hint: <Self::Cfg as OnlineHostBackendCfg>::Hint,
+        cfg: &Self::Cfg,
+        providers: &<Self::Cfg as OnlineHostBackendCfg>::Providers,
+        kv: SharedKeyValueStore,
+    ) -> Result<()>;
+}
+
+/// The [OnlineHostBackend] is a [HintRouter] and [PreimageFetcher] that is used to fetch data from
+/// remote sources in response to hints.
+///
+/// [PreimageKey]: kona_preimage::PreimageKey
+#[allow(missing_debug_implementations)]
+pub struct OnlineHostBackend<C, H>
+where
+    C: OnlineHostBackendCfg,
+    H: HintHandler,
+{
+    /// The configuration that is used to route hints.
+    cfg: C,
+    /// The key-value store that is used to store preimages.
+    kv: SharedKeyValueStore,
+    /// The providers that are used to fetch data in response to hints.
+    providers: C::Providers,
+    /// The last hint that was received.
+    last_hint: Arc<RwLock<Option<String>>>,
+    /// Phantom marker for the [HintHandler].
+    _hint_handler: std::marker::PhantomData<H>,
+}
+
+impl<C, H> OnlineHostBackend<C, H>
+where
+    C: OnlineHostBackendCfg,
+    H: HintHandler,
+{
+    /// Creates a new [HintHandler] with the given configuration, key-value store, providers, and
+    /// external configuration.
+    pub fn new(cfg: C, kv: SharedKeyValueStore, providers: C::Providers, _: H) -> Self {
+        Self {
+            cfg,
+            kv,
+            providers,
+            last_hint: Arc::new(RwLock::new(None)),
+            _hint_handler: std::marker::PhantomData,
+        }
+    }
+}
+
+#[async_trait]
+impl<C, H> HintRouter for OnlineHostBackend<C, H>
+where
+    C: OnlineHostBackendCfg + Send + Sync,
+    H: HintHandler<Cfg = C> + Send + Sync,
+{
+    /// Set the last hint to be received.
+    async fn route_hint(&self, hint: String) -> PreimageOracleResult<()> {
+        trace!(target: "host-backend", "Received hint: {hint}");
+        let mut hint_lock = self.last_hint.write().await;
+        hint_lock.replace(hint);
+        Ok(())
+    }
+}
+
+#[async_trait]
+impl<C, H> PreimageFetcher for OnlineHostBackend<C, H>
+where
+    C: OnlineHostBackendCfg + Send + Sync,
+    H: HintHandler<Cfg = C> + Send + Sync,
+{
+    /// Get the preimage for the given key.
+    async fn get_preimage(&self, key: PreimageKey) -> PreimageOracleResult<Vec<u8>> {
+        trace!(target: "host-backend", "Pre-image requested. Key: {key}");
+
+        // Acquire a read lock on the key-value store.
+        let kv_lock = self.kv.read().await;
+        let mut preimage = kv_lock.get(key.into());
+
+        // Drop the read lock before beginning the retry loop.
+        drop(kv_lock);
+
+        // Use a loop to keep retrying the prefetch as long as the key is not found
+        while preimage.is_none() {
+            if let Some(hint) = self.last_hint.read().await.as_ref() {
+                let parsed_hint =
+                    hint.parse::<C::Hint>().map_err(|_| PreimageOracleError::KeyNotFound)?;
+                let value =
+                    H::fetch_hint(parsed_hint, &self.cfg, &self.providers, self.kv.clone()).await;
+
+                if let Err(e) = value {
+                    error!(target: "host-backend", "Failed to prefetch hint: {e}");
+                    warn!(target: "host-backend", "Retrying hint fetch: {hint}");
+                    continue;
+                }
+
+                let kv_lock = self.kv.read().await;
+                preimage = kv_lock.get(key.into());
+            }
+        }
+
+        preimage.ok_or(PreimageOracleError::KeyNotFound)
+    }
+}

bin/host/src/backend/util.rs

@@ -0,0 +1,41 @@
+//! Utilities for the preimage server backend.
+
+use crate::KeyValueStore;
+use alloy_consensus::EMPTY_ROOT_HASH;
+use alloy_primitives::keccak256;
+use alloy_rlp::EMPTY_STRING_CODE;
+use anyhow::Result;
+use kona_preimage::{PreimageKey, PreimageKeyType};
+use tokio::sync::RwLock;
+
+/// Constructs a merkle patricia trie from the ordered list passed and stores all encoded
+/// intermediate nodes of the trie in the [KeyValueStore].
+pub(crate) async fn store_ordered_trie<KV: KeyValueStore + ?Sized, T: AsRef<[u8]>>(
+    kv: &RwLock<KV>,
+    values: &[T],
+) -> Result<()> {
+    let mut kv_write_lock = kv.write().await;
+
+    // If the list of nodes is empty, store the empty root hash and exit early.
+    // The `HashBuilder` will not push the preimage of the empty root hash to the
+    // `ProofRetainer` in the event that there are no leaves inserted.
+    if values.is_empty() {
+        let empty_key = PreimageKey::new(*EMPTY_ROOT_HASH, PreimageKeyType::Keccak256);
+        return kv_write_lock.set(empty_key.into(), [EMPTY_STRING_CODE].into());
+    }
+
+    let mut hb = kona_mpt::ordered_trie_with_encoder(values, |node, buf| {
+        buf.put_slice(node.as_ref());
+    });
+    hb.root();
+    let intermediates = hb.take_proof_nodes().into_inner();
+
+    for (_, value) in intermediates.into_iter() {
+        let value_hash = keccak256(value.as_ref());
+        let key = PreimageKey::new(*value_hash, PreimageKeyType::Keccak256);
+
+        kv_write_lock.set(key.into(), value.into())?;
+    }
+
+    Ok(())
+}

bin/host/src/eth/mod.rs

@@ -1,6 +1,6 @@
 //! Ethereum utilities for the host binary.
 
-use alloy_provider::RootProvider;
+use alloy_provider::{Network, RootProvider};
 use alloy_rpc_client::RpcClient;
 use alloy_transport_http::Http;
 use reqwest::Client;
@@ -9,7 +9,7 @@ mod precompiles;
 pub(crate) use precompiles::execute;
 
 /// Returns an HTTP provider for the given URL.
-pub fn http_provider(url: &str) -> RootProvider {
+pub fn http_provider<N: Network>(url: &str) -> RootProvider<N> {
     let url = url.parse().unwrap();
     let http = Http::<Client>::new(url);
     RootProvider::new(RpcClient::new(http, true))

bin/host/src/interop/cfg.rs

@@ -1,14 +1,14 @@
 //! This module contains all CLI-specific code for the interop entrypoint.
 
-use super::{InteropFetcher, InteropLocalInputs};
+use super::{InteropHintHandler, InteropLocalInputs};
 use crate::{
     cli::{
         cli_styles,
         parser::{parse_b256, parse_bytes},
     },
     eth::http_provider,
-    DiskKeyValueStore, MemoryKeyValueStore, OfflineHostBackend, PreimageServer,
-    SharedKeyValueStore, SplitKeyValueStore,
+    DiskKeyValueStore, MemoryKeyValueStore, OfflineHostBackend, OnlineHostBackend,
+    OnlineHostBackendCfg, PreimageServer, SharedKeyValueStore, SplitKeyValueStore,
 };
 use alloy_primitives::{Bytes, B256};
 use alloy_provider::{Provider, RootProvider};
@@ -18,10 +18,12 @@ use clap::Parser;
 use kona_preimage::{
     BidirectionalChannel, Channel, HintReader, HintWriter, OracleReader, OracleServer,
 };
-use kona_proof_interop::PreState;
+use kona_proof::Hint;
+use kona_proof_interop::{HintType, PreState};
 use kona_providers_alloy::{OnlineBeaconClient, OnlineBlobProvider};
 use kona_std_fpvm::{FileChannel, FileDescriptor};
 use maili_genesis::RollupConfig;
+use op_alloy_network::Optimism;
 use serde::Serialize;
 use std::{collections::HashMap, path::PathBuf, sync::Arc};
 use tokio::{
@@ -131,12 +133,11 @@ impl InteropHost {
             )
         } else {
             let providers = self.create_providers().await?;
-            let backend = InteropFetcher::new(
+            let backend = OnlineHostBackend::new(
                 self.clone(),
                 kv_store.clone(),
-                providers.l1_provider,
-                providers.blob_provider,
-                providers.l2_providers,
+                providers,
+                InteropHintHandler,
             );
 
             task::spawn(
@@ -260,21 +261,36 @@ impl InteropHost {
             self.l2_node_addresses.as_ref().ok_or(anyhow!("L2 node addresses must be set"))?;
         let mut l2_providers = HashMap::default();
         for l2_node_address in l2_node_addresses {
-            let l2_provider = http_provider(l2_node_address);
+            let l2_provider = http_provider::<Optimism>(l2_node_address);
             let chain_id = l2_provider.get_chain_id().await?;
             l2_providers.insert(chain_id, l2_provider);
         }
 
-        Ok(InteropProviders { l1_provider, blob_provider, l2_providers })
+        Ok(InteropProviders { l1: l1_provider, blobs: blob_provider, l2s: l2_providers })
     }
 }
+
+impl OnlineHostBackendCfg for InteropHost {
+    type Hint = Hint<HintType>;
+    type Providers = InteropProviders;
+}
+
 /// The providers required for the single chain host.
 #[derive(Debug)]
 pub struct InteropProviders {
     /// The L1 EL provider.
-    l1_provider: RootProvider,
+    pub l1: RootProvider,
     /// The L1 beacon node provider.
-    blob_provider: OnlineBlobProvider<OnlineBeaconClient>,
+    pub blobs: OnlineBlobProvider<OnlineBeaconClient>,
     /// The L2 EL providers, keyed by chain ID.
-    l2_providers: HashMap<u64, RootProvider>,
+    pub l2s: HashMap<u64, RootProvider<Optimism>>,
+}
+
+impl InteropProviders {
+    /// Returns the L2 [RootProvider] for the given chain ID.
+    pub fn l2(&self, chain_id: &u64) -> Result<&RootProvider<Optimism>> {
+        self.l2s
+            .get(chain_id)
+            .ok_or_else(|| anyhow!("No provider found for chain ID: {}", chain_id))
+    }
 }