import autonomys/subspace#1645 + Cuckoo filter addition

Cargo.toml

@@ -12,7 +12,7 @@ sdk-dsn = { path = "dsn" }
 sdk-substrate = { path = "substrate" }
 sdk-farmer = { path = "farmer" }
 
-subspace-proof-of-space = { git = "https://github.com/subspace/subspace", rev = "ba0ee8ba6f9f00be441c6ec07eb229b486d547b8" }
+subspace-proof-of-space = { git = "https://github.com/subspace/subspace", rev = "6d10df7fcab2c2279fc45bb607ae2a1dca8d2c74" }
 
 # The only triple tested and confirmed as working in `jemallocator` crate is `x86_64-unknown-linux-gnu`
 [target.'cfg(all(target_arch = "x86_64", target_vendor = "unknown", target_os = "linux", target_env = "gnu"))'.dev-dependencies]
@@ -35,7 +35,7 @@ tracing = "0.1"
 tracing-futures = "0.2"
 tracing-subscriber = "0.3"
 
-subspace-farmer-components = { git = "https://github.com/subspace/subspace", rev = "ba0ee8ba6f9f00be441c6ec07eb229b486d547b8" }
+subspace-farmer-components = { git = "https://github.com/subspace/subspace", rev = "6d10df7fcab2c2279fc45bb607ae2a1dca8d2c74" }
 
 # The list of dependencies below (which can be both direct and indirect dependencies) are crates
 # that are suspected to be CPU-intensive, and that are unlikely to require debugging (as some of

dsn/Cargo.toml

@@ -25,9 +25,9 @@ sc-client-api = { version = "4.0.0-dev", git = "https://github.com/subspace/subs
 sp-runtime = { version = "24.0.0", git = "https://github.com/subspace/substrate", rev = "55c157cff49b638a59d81a9f971f0f9a66829c71" }
 sp-blockchain = { version = "4.0.0-dev", git = "https://github.com/subspace/substrate", rev = "55c157cff49b638a59d81a9f971f0f9a66829c71" }
 
-subspace-networking = { git = "https://github.com/subspace/subspace", rev = "ba0ee8ba6f9f00be441c6ec07eb229b486d547b8" }
-subspace-core-primitives = { git = "https://github.com/subspace/subspace", rev = "ba0ee8ba6f9f00be441c6ec07eb229b486d547b8" }
-subspace-farmer = { git = "https://github.com/subspace/subspace", rev = "ba0ee8ba6f9f00be441c6ec07eb229b486d547b8" }
-subspace-farmer-components = { git = "https://github.com/subspace/subspace", rev = "ba0ee8ba6f9f00be441c6ec07eb229b486d547b8" }
-subspace-service = { git = "https://github.com/subspace/subspace", rev = "ba0ee8ba6f9f00be441c6ec07eb229b486d547b8" }
-sc-consensus-subspace = { git = "https://github.com/subspace/subspace", rev = "ba0ee8ba6f9f00be441c6ec07eb229b486d547b8" }
+subspace-networking = { git = "https://github.com/subspace/subspace", rev = "6d10df7fcab2c2279fc45bb607ae2a1dca8d2c74" }
+subspace-core-primitives = { git = "https://github.com/subspace/subspace", rev = "6d10df7fcab2c2279fc45bb607ae2a1dca8d2c74" }
+subspace-farmer = { git = "https://github.com/subspace/subspace", rev = "6d10df7fcab2c2279fc45bb607ae2a1dca8d2c74" }
+subspace-farmer-components = { git = "https://github.com/subspace/subspace", rev = "6d10df7fcab2c2279fc45bb607ae2a1dca8d2c74" }
+subspace-service = { git = "https://github.com/subspace/subspace", rev = "6d10df7fcab2c2279fc45bb607ae2a1dca8d2c74" }
+sc-consensus-subspace = { git = "https://github.com/subspace/subspace", rev = "6d10df7fcab2c2279fc45bb607ae2a1dca8d2c74" }

dsn/src/builder.rs

@@ -10,6 +10,8 @@ use either::*;
 use futures::prelude::*;
 use sdk_utils::{self, DropCollection, Multiaddr, MultiaddrWithPeerId};
 use serde::{Deserialize, Serialize};
+use subspace_core_primitives::Piece;
+use subspace_farmer::utils::archival_storage_pieces::ArchivalStoragePieces;
 use subspace_farmer::utils::readers_and_pieces::ReadersAndPieces;
 use subspace_networking::libp2p::kad::ProviderRecord;
 use subspace_networking::{
@@ -175,6 +177,8 @@ pub struct DsnOptions<C, ASNS, PieceByHash, SegmentHeaderByIndexes> {
     pub get_piece_by_hash: PieceByHash,
     /// Get segment header by segment indexes handler
     pub get_segment_header_by_segment_indexes: SegmentHeaderByIndexes,
+    /// Farmer total allocated space across all plots
+    pub farmer_total_space_pledged: usize,
 }
 
 /// Farmer piece store
@@ -199,6 +203,8 @@ pub struct DsnShared<C: sc_client_api::AuxStore + Send + Sync + 'static> {
         parking_lot::Mutex<Option<futures::channel::mpsc::Receiver<ProviderRecord>>>,
     /// Farmer provider storage
     pub farmer_provider_storage: MaybeProviderStorage<FarmerProviderStorage>,
+    /// Farmer archival storage pieces
+    pub farmer_archival_storage_pieces: ArchivalStoragePieces,
     /// Farmer piece cache
     #[derivative(Debug = "ignore")]
     pub piece_cache: NodePieceCache<C>,
@@ -242,12 +248,16 @@ impl Dsn {
             piece_cache_size,
             get_piece_by_hash,
             get_segment_header_by_segment_indexes,
+            farmer_total_space_pledged,
         } = options;
         let farmer_readers_and_pieces = Arc::new(parking_lot::Mutex::new(None));
         let farmer_piece_store = Arc::new(tokio::sync::Mutex::new(None));
         let farmer_provider_storage = MaybeProviderStorage::none();
         let protocol_version = hex::encode(client.info().genesis_hash);
 
+        let cuckoo_filter_size = farmer_total_space_pledged / Piece::SIZE + 1usize;
+        let farmer_archival_storage_pieces = ArchivalStoragePieces::new(cuckoo_filter_size);
+
         tracing::debug!(genesis_hash = protocol_version, "Setting DSN protocol version...");
 
         let piece_cache = NodePieceCache::new(
@@ -329,7 +339,7 @@ impl Dsn {
             protocol_version,
             keypair,
             provider_storage.clone(),
-            PeerInfoProvider::new_node(),
+            PeerInfoProvider::new_farmer(Box::new(farmer_archival_storage_pieces.clone())),
         );
         default_networking_config.kademlia.set_provider_record_ttl(KADEMLIA_PROVIDER_TTL_IN_SECS);
 
@@ -454,6 +464,7 @@ impl Dsn {
                 farmer_readers_and_pieces,
                 piece_cache,
                 _drop: drop_collection,
+                farmer_archival_storage_pieces,
             },
             runner,
         ))

examples/complete.rs

@@ -5,18 +5,20 @@ use subspace_sdk::{chain_spec, node, ByteSize, Farmer, Node, PlotDescription, Pu
 
 #[tokio::main]
 async fn main() {
+    let plots = [PlotDescription::new("plot", ByteSize::gb(10))];
+    let farmer_total_space_pledged =
+        plots.iter().map(|p| p.space_pledged.as_u64() as usize).sum::<usize>();
     let node: Node = Node::builder()
         .blocks_pruning(node::BlocksPruning::Some(1000))
         .state_pruning(node::PruningMode::ArchiveCanonical)
         .network(NetworkBuilder::new().name("i1i1"))
-        .build("node", chain_spec::dev_config())
+        .build("node", chain_spec::dev_config(), farmer_total_space_pledged)
         .await
         .expect("Failed to init a node");
 
     node.sync().await.unwrap();
 
     let reward_address = PublicKey::from([0; 32]);
-    let plots = [PlotDescription::new("plot", ByteSize::gb(10))];
     let farmer: Farmer = Farmer::builder()
         // .ws_rpc("127.0.0.1:9955".parse().unwrap())
         // .listen_on("/ip4/0.0.0.0/tcp/40333".parse().unwrap())
@@ -57,7 +59,7 @@ async fn main() {
     let node = Node::builder()
         .blocks_pruning(node::BlocksPruning::Some(1000))
         .state_pruning(node::PruningMode::ArchiveCanonical)
-        .build("node", chain_spec::dev_config())
+        .build("node", chain_spec::dev_config(), farmer_total_space_pledged)
         .await
         .expect("Failed to init a node");
     node.sync().await.unwrap();

examples/mini-farmer.rs

@@ -116,6 +116,7 @@ async fn main() -> anyhow::Result<()> {
                 Chain::Devnet => node::chain_spec::devnet_config(),
                 Chain::Dev => node::chain_spec::dev_config(),
             },
+            plot_size.as_u64() as usize,
         )
         .await?;
 

examples/simple.rs

@@ -3,16 +3,17 @@ use futures::prelude::*;
 #[tokio::main]
 async fn main() {
     tracing_subscriber::fmt().init();
-
+    let plots = [subspace_sdk::PlotDescription::new("plot", subspace_sdk::ByteSize::mb(100))];
+    let farmer_total_space_pledged =
+        plots.iter().map(|p| p.space_pledged.as_u64() as usize).sum::<usize>();
     let node = subspace_sdk::Node::builder()
         .force_authoring(true)
         .role(subspace_sdk::node::Role::Authority)
         // Starting a new chain
-        .build("node", subspace_sdk::chain_spec::dev_config())
+        .build("node", subspace_sdk::chain_spec::dev_config(), farmer_total_space_pledged)
         .await
         .unwrap();
 
-    let plots = [subspace_sdk::PlotDescription::new("plot", subspace_sdk::ByteSize::mb(100))];
     let cache =
         subspace_sdk::farmer::CacheDescription::new("cache", subspace_sdk::ByteSize::mb(10))
             .unwrap();

examples/sync.rs

@@ -36,6 +36,10 @@ enum Args {
         /// Path to the chain spec
         #[arg(short, long)]
         spec: PathBuf,
+
+        /// Total space pledged by farmer
+        #[arg(short, long)]
+        farmer_total_space_pledged: ByteSize,
     },
     GenerateSpec {
         path: PathBuf,
@@ -54,6 +58,9 @@ async fn main() -> anyhow::Result<()> {
             let chain_spec = serde_json::from_str(&tokio::fs::read_to_string(spec).await?)?;
             let (plot_size, cache_size) =
                 (ByteSize::b(plot_size.as_u64() * 9 / 10), ByteSize::b(plot_size.as_u64() / 10));
+            let plots = [PlotDescription::new(plot.join("plot"), plot_size)];
+            let farmer_total_space_pledged =
+                plots.iter().map(|p| p.space_pledged.as_u64() as usize).sum::<usize>();
             let node = Node::builder()
                 .network(
                     NetworkBuilder::new()
@@ -62,10 +69,9 @@ async fn main() -> anyhow::Result<()> {
                 )
                 .force_authoring(true)
                 .role(subspace_sdk::node::Role::Authority)
-                .build(node, chain_spec)
+                .build(node, chain_spec, farmer_total_space_pledged)
                 .await?;
 
-            let plots = [PlotDescription::new(plot.join("plot"), plot_size)];
             let _farmer: Farmer = Farmer::builder()
                 .build(
                     PublicKey::from([13; 32]),
@@ -83,14 +89,14 @@ async fn main() -> anyhow::Result<()> {
                 .for_each(|header| async move { tracing::info!(?header, "New block!") })
                 .await;
         }
-        Args::Sync { boot_nodes, spec } => {
+        Args::Sync { boot_nodes, spec, farmer_total_space_pledged } => {
             let node = TempDir::new()?;
             let chain_spec = serde_json::from_str(&tokio::fs::read_to_string(spec).await?)?;
             let node = Node::builder()
                 .force_authoring(true)
                 .role(subspace_sdk::node::Role::Authority)
                 .network(NetworkBuilder::new().boot_nodes(boot_nodes))
-                .build(node.as_ref(), chain_spec)
+                .build(node.as_ref(), chain_spec, farmer_total_space_pledged.as_u64() as usize)
                 .await?;
 
             node.sync().await.unwrap();

farmer/Cargo.toml

@@ -25,10 +25,10 @@ sdk-utils = { path = "../utils" }
 sdk-dsn = { path = "../dsn" }
 sdk-traits = { path = "../traits" }
 
-subspace-core-primitives = { git = "https://github.com/subspace/subspace", rev = "ba0ee8ba6f9f00be441c6ec07eb229b486d547b8" }
-subspace-erasure-coding = { git = "https://github.com/subspace/subspace", rev = "ba0ee8ba6f9f00be441c6ec07eb229b486d547b8" }
-subspace-farmer = { git = "https://github.com/subspace/subspace", rev = "ba0ee8ba6f9f00be441c6ec07eb229b486d547b8" }
-subspace-farmer-components = { git = "https://github.com/subspace/subspace", rev = "ba0ee8ba6f9f00be441c6ec07eb229b486d547b8" }
-subspace-networking = { git = "https://github.com/subspace/subspace", rev = "ba0ee8ba6f9f00be441c6ec07eb229b486d547b8" }
-subspace-proof-of-space = { git = "https://github.com/subspace/subspace", rev = "ba0ee8ba6f9f00be441c6ec07eb229b486d547b8", features = ["parallel", "chia"] }
-subspace-rpc-primitives = { git = "https://github.com/subspace/subspace", rev = "ba0ee8ba6f9f00be441c6ec07eb229b486d547b8" }
+subspace-core-primitives = { git = "https://github.com/subspace/subspace", rev = "6d10df7fcab2c2279fc45bb607ae2a1dca8d2c74" }
+subspace-erasure-coding = { git = "https://github.com/subspace/subspace", rev = "6d10df7fcab2c2279fc45bb607ae2a1dca8d2c74" }
+subspace-farmer = { git = "https://github.com/subspace/subspace", rev = "6d10df7fcab2c2279fc45bb607ae2a1dca8d2c74" }
+subspace-farmer-components = { git = "https://github.com/subspace/subspace", rev = "6d10df7fcab2c2279fc45bb607ae2a1dca8d2c74" }
+subspace-networking = { git = "https://github.com/subspace/subspace", rev = "6d10df7fcab2c2279fc45bb607ae2a1dca8d2c74" }
+subspace-proof-of-space = { git = "https://github.com/subspace/subspace", rev = "6d10df7fcab2c2279fc45bb607ae2a1dca8d2c74", features = ["parallel", "chia"] }
+subspace-rpc-primitives = { git = "https://github.com/subspace/subspace", rev = "6d10df7fcab2c2279fc45bb607ae2a1dca8d2c74" }