Skip to content
This repository has been archived by the owner on Feb 15, 2024. It is now read-only.

Commit

Permalink
Merge pull request #219 from subspace/sep-29
Browse files Browse the repository at this point in the history 
Update sdk to gemini-3f-2023-sep-29
  • Loading branch information
@ParthDesai
ParthDesai authored Sep 30, 2023
2 parents df71413 + e289af3 commit d1ecb61
Show file tree
Hide file tree
Showing 8 changed files with 166 additions and 99 deletions.
107 changes: 53 additions & 54 deletions Cargo.lock
View file

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ sdk-substrate = { path = "substrate" }
sdk-utils = { path = "utils" }
static_assertions = "1.1.0"

subspace-proof-of-space = { git = "https://github.com/subspace/subspace", rev = "2d04263a9d44e845b1c92e5ec92e86dfeed0f511" }
subspace-proof-of-space = { git = "https://github.com/subspace/subspace", rev = "d3914a13ea2d5a476b9b93749416ae960f2d105a" }

# The only triple tested and confirmed as working in `jemallocator` crate is `x86_64-unknown-linux-gnu`
[target.'cfg(all(target_arch = "x86_64", target_vendor = "unknown", target_os = "linux", target_env = "gnu"))'.dev-dependencies]
Expand All @@ -28,7 +28,7 @@ derive_more = "0.99"
fdlimit = "0.2"
futures = "0.3"
serde_json = "1"
subspace-farmer-components = { git = "https://github.com/subspace/subspace", rev = "2d04263a9d44e845b1c92e5ec92e86dfeed0f511" }
subspace-farmer-components = { git = "https://github.com/subspace/subspace", rev = "d3914a13ea2d5a476b9b93749416ae960f2d105a" }
tempfile = "3"
tokio = { version = "1.26", features = ["rt-multi-thread", "macros"] }
tracing = "0.1"
Expand Down
6 changes: 3 additions & 3 deletions dsn/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,13 @@ futures = "0.3"
hex = "0.4.3"
parking_lot = "0.12"
sc-client-api = { version = "4.0.0-dev", git = "https://github.com/subspace/substrate", rev = "55c157cff49b638a59d81a9f971f0f9a66829c71" }
sc-consensus-subspace = { git = "https://github.com/subspace/subspace", rev = "2d04263a9d44e845b1c92e5ec92e86dfeed0f511" }
sc-consensus-subspace = { git = "https://github.com/subspace/subspace", rev = "d3914a13ea2d5a476b9b93749416ae960f2d105a" }
sdk-utils = { path = "../utils" }
serde = { version = "1", features = ["derive"] }
sp-blockchain = { version = "4.0.0-dev", git = "https://github.com/subspace/substrate", rev = "55c157cff49b638a59d81a9f971f0f9a66829c71" }
sp-runtime = { version = "24.0.0", git = "https://github.com/subspace/substrate", rev = "55c157cff49b638a59d81a9f971f0f9a66829c71" }
subspace-farmer = { git = "https://github.com/subspace/subspace", rev = "2d04263a9d44e845b1c92e5ec92e86dfeed0f511" }
subspace-networking = { git = "https://github.com/subspace/subspace", rev = "2d04263a9d44e845b1c92e5ec92e86dfeed0f511" }
subspace-farmer = { git = "https://github.com/subspace/subspace", rev = "d3914a13ea2d5a476b9b93749416ae960f2d105a" }
subspace-networking = { git = "https://github.com/subspace/subspace", rev = "d3914a13ea2d5a476b9b93749416ae960f2d105a" }
tracing = "0.1"

[features]
Expand Down
15 changes: 8 additions & 7 deletions farmer/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,16 +15,17 @@ futures = "0.3"
lru = "0.10"
parking_lot = "0.12"
pin-project = "1"
rayon = "1.7.0"
sdk-traits = { path = "../traits" }
sdk-utils = { path = "../utils" }
serde = { version = "1", features = ["derive"] }
subspace-core-primitives = { git = "https://github.com/subspace/subspace", rev = "2d04263a9d44e845b1c92e5ec92e86dfeed0f511" }
subspace-erasure-coding = { git = "https://github.com/subspace/subspace", rev = "2d04263a9d44e845b1c92e5ec92e86dfeed0f511" }
subspace-farmer = { git = "https://github.com/subspace/subspace", rev = "2d04263a9d44e845b1c92e5ec92e86dfeed0f511" }
subspace-farmer-components = { git = "https://github.com/subspace/subspace", rev = "2d04263a9d44e845b1c92e5ec92e86dfeed0f511" }
subspace-networking = { git = "https://github.com/subspace/subspace", rev = "2d04263a9d44e845b1c92e5ec92e86dfeed0f511" }
subspace-proof-of-space = { git = "https://github.com/subspace/subspace", rev = "2d04263a9d44e845b1c92e5ec92e86dfeed0f511", features = ["parallel", "chia"] }
subspace-rpc-primitives = { git = "https://github.com/subspace/subspace", rev = "2d04263a9d44e845b1c92e5ec92e86dfeed0f511" }
subspace-core-primitives = { git = "https://github.com/subspace/subspace", rev = "d3914a13ea2d5a476b9b93749416ae960f2d105a" }
subspace-erasure-coding = { git = "https://github.com/subspace/subspace", rev = "d3914a13ea2d5a476b9b93749416ae960f2d105a" }
subspace-farmer = { git = "https://github.com/subspace/subspace", rev = "d3914a13ea2d5a476b9b93749416ae960f2d105a" }
subspace-farmer-components = { git = "https://github.com/subspace/subspace", rev = "d3914a13ea2d5a476b9b93749416ae960f2d105a" }
subspace-networking = { git = "https://github.com/subspace/subspace", rev = "d3914a13ea2d5a476b9b93749416ae960f2d105a" }
subspace-proof-of-space = { git = "https://github.com/subspace/subspace", rev = "d3914a13ea2d5a476b9b93749416ae960f2d105a", features = ["parallel", "chia"] }
subspace-rpc-primitives = { git = "https://github.com/subspace/subspace", rev = "d3914a13ea2d5a476b9b93749416ae960f2d105a" }
thiserror = "1"
tokio = { version = "1.28.2", features = ["fs", "rt", "tracing", "macros", "parking_lot", "rt-multi-thread", "signal"] }
tokio-stream = { version = "0.1", features = ["sync", "time"] }
Expand Down
71 changes: 69 additions & 2 deletions farmer/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ pub use builder::{Builder, Config};
use derivative::Derivative;
use futures::prelude::*;
use futures::stream::FuturesUnordered;
use rayon::{ThreadPool, ThreadPoolBuilder};
use sdk_traits::Node;
use sdk_utils::{ByteSize, DestructorSet, PublicKey, TaskOutput};
use serde::{Deserialize, Serialize};
Expand Down Expand Up @@ -77,10 +78,26 @@ mod builder {
use sdk_traits::Node;
use sdk_utils::{ByteSize, PublicKey};
use serde::{Deserialize, Serialize};
use tracing::warn;

use super::BuildError;
use crate::{FarmDescription, Farmer};

fn available_parallelism() -> usize {
match std::thread::available_parallelism() {
Ok(parallelism) => parallelism.get(),
Err(error) => {
warn!(
%error,
"Unable to identify available parallelism, you might want to configure thread pool sizes with CLI \
options manually"
);

0
}
}
}

#[derive(
Debug,
Clone,
Expand Down Expand Up @@ -157,6 +174,15 @@ mod builder {
/// Maximum number of pieces in single sector
#[builder(default)]
pub max_pieces_in_sector: Option<u16>,
/// Farming thread pool size
#[builder(default = "available_parallelism()")]
pub farming_thread_pool_size: usize,
/// Plotting thread pool size
#[builder(default = "available_parallelism()")]
pub plotting_thread_pool_size: usize,
/// Replotting thread pool size
#[builder(default = "available_parallelism()")]
pub replotting_thread_pool_size: usize,
}

impl Builder {
Expand Down Expand Up @@ -206,6 +232,9 @@ pub enum BuildError {
/// Failed to fetch data from the node
#[error("Failed to fetch data from node: {0}")]
RPCError(#[source] subspace_farmer::RpcClientError),
/// Failed to build thread pool
#[error("Failed to build thread pool: {0}")]
ThreadPoolError(#[from] rayon::ThreadPoolBuildError),
/// Other error
#[error("{0}")]
Other(#[from] anyhow::Error),
Expand Down Expand Up @@ -335,7 +364,14 @@ impl Config {

let mut destructors = DestructorSet::new("farmer-destructors");

let Self { max_concurrent_farms: _, provided_keys_limit: _, max_pieces_in_sector } = self;
let Self {
max_concurrent_farms: _,
provided_keys_limit: _,
max_pieces_in_sector,
farming_thread_pool_size,
plotting_thread_pool_size,
replotting_thread_pool_size,
} = self;

let mut single_disk_farms = Vec::with_capacity(farms.len());
let mut farm_info = HashMap::with_capacity(farms.len());
Expand Down Expand Up @@ -413,6 +449,25 @@ impl Config {
None => farmer_app_info.protocol_info.max_pieces_in_sector,
};

let farming_thread_pool = Arc::new(
ThreadPoolBuilder::new()
.thread_name(move |thread_index| format!("farming#{thread_index}"))
.num_threads(farming_thread_pool_size)
.build()?,
);
let plotting_thread_pool = Arc::new(
ThreadPoolBuilder::new()
.thread_name(move |thread_index| format!("plotting#{thread_index}"))
.num_threads(plotting_thread_pool_size)
.build()?,
);
let replotting_thread_pool = Arc::new(
ThreadPoolBuilder::new()
.thread_name(move |thread_index| format!("replotting#{thread_index}"))
.num_threads(replotting_thread_pool_size)
.build()?,
);

for (disk_farm_idx, description) in farms.iter().enumerate() {
let (farm, single_disk_farm) = Farm::new(FarmOptions {
disk_farm_idx,
Expand All @@ -424,6 +479,9 @@ impl Config {
description,
kzg: kzg.clone(),
erasure_coding: erasure_coding.clone(),
farming_thread_pool: farming_thread_pool.clone(),
plotting_thread_pool: plotting_thread_pool.clone(),
replotting_thread_pool: replotting_thread_pool.clone(),
})
.await?;
farm_info.insert(farm.directory.clone(), farm);
Expand Down Expand Up @@ -712,6 +770,9 @@ struct FarmOptions<'a, PG, N: sdk_traits::Node> {
pub kzg: kzg::Kzg,
pub erasure_coding: ErasureCoding,
pub max_pieces_in_sector: u16,
pub farming_thread_pool: Arc<ThreadPool>,
pub plotting_thread_pool: Arc<ThreadPool>,
pub replotting_thread_pool: Arc<ThreadPool>,
}

impl<T: subspace_proof_of_space::Table> Farm<T> {
Expand All @@ -726,9 +787,12 @@ impl<T: subspace_proof_of_space::Table> Farm<T> {
kzg,
erasure_coding,
max_pieces_in_sector,
farming_thread_pool,
plotting_thread_pool,
replotting_thread_pool,
}: FarmOptions<
'_,
impl subspace_farmer_components::plotting::PieceGetter + Send + 'static,
impl subspace_farmer_components::plotting::PieceGetter + Clone + Send + 'static,
impl sdk_traits::Node,
>,
) -> Result<(Self, SingleDiskFarm), BuildError> {
Expand All @@ -748,6 +812,9 @@ impl<T: subspace_proof_of_space::Table> Farm<T> {
erasure_coding,
piece_getter,
cache_percentage,
farming_thread_pool,
plotting_thread_pool,
replotting_thread_pool,
};
let single_disk_farm_fut = SingleDiskFarm::new::<_, _, T>(description, disk_farm_idx);
let single_disk_farm = match single_disk_farm_fut.await {
Expand Down
44 changes: 22 additions & 22 deletions node/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,36 +7,36 @@ edition = "2021"
[dependencies]
anyhow = "1"
backoff = "0.4"
cross-domain-message-gossip = { git = "https://github.com/subspace/subspace", rev = "2d04263a9d44e845b1c92e5ec92e86dfeed0f511" }
cross-domain-message-gossip = { git = "https://github.com/subspace/subspace", rev = "d3914a13ea2d5a476b9b93749416ae960f2d105a" }
derivative = "2.2.0"
derive_builder = "0.12"
derive_more = "0.99"
domain-client-operator = { git = "https://github.com/subspace/subspace", rev = "2d04263a9d44e845b1c92e5ec92e86dfeed0f511" }
domain-eth-service = { git = "https://github.com/subspace/subspace", rev = "2d04263a9d44e845b1c92e5ec92e86dfeed0f511" }
domain-runtime-primitives = { git = "https://github.com/subspace/subspace", rev = "2d04263a9d44e845b1c92e5ec92e86dfeed0f511" }
domain-service = { git = "https://github.com/subspace/subspace", rev = "2d04263a9d44e845b1c92e5ec92e86dfeed0f511" }
evm-domain-runtime = { git = "https://github.com/subspace/subspace", rev = "2d04263a9d44e845b1c92e5ec92e86dfeed0f511" }
domain-client-operator = { git = "https://github.com/subspace/subspace", rev = "d3914a13ea2d5a476b9b93749416ae960f2d105a" }
domain-eth-service = { git = "https://github.com/subspace/subspace", rev = "d3914a13ea2d5a476b9b93749416ae960f2d105a" }
domain-runtime-primitives = { git = "https://github.com/subspace/subspace", rev = "d3914a13ea2d5a476b9b93749416ae960f2d105a" }
domain-service = { git = "https://github.com/subspace/subspace", rev = "d3914a13ea2d5a476b9b93749416ae960f2d105a" }
evm-domain-runtime = { git = "https://github.com/subspace/subspace", rev = "d3914a13ea2d5a476b9b93749416ae960f2d105a" }
fp-evm = { version = "3.0.0-dev", git = "https://github.com/subspace/frontier", rev = "74483666645e121c0c5e6616f43fdfd8664ea0d3" }
frame-system = { version = "4.0.0-dev", git = "https://github.com/subspace/substrate", rev = "55c157cff49b638a59d81a9f971f0f9a66829c71" }
futures = "0.3"
hex-literal = "0.4"
once_cell = "1.18.0"
pallet-rewards = { git = "https://github.com/subspace/subspace", rev = "2d04263a9d44e845b1c92e5ec92e86dfeed0f511" }
pallet-subspace = { git = "https://github.com/subspace/subspace", rev = "2d04263a9d44e845b1c92e5ec92e86dfeed0f511" }
pallet-rewards = { git = "https://github.com/subspace/subspace", rev = "d3914a13ea2d5a476b9b93749416ae960f2d105a" }
pallet-subspace = { git = "https://github.com/subspace/subspace", rev = "d3914a13ea2d5a476b9b93749416ae960f2d105a" }
parity-scale-codec = "3.6.3"
parking_lot = "0.12"
pin-project = "1"
sc-client-api = { version = "4.0.0-dev", git = "https://github.com/subspace/substrate", rev = "55c157cff49b638a59d81a9f971f0f9a66829c71" }
sc-consensus-slots = { version = "0.10.0-dev", git = "https://github.com/subspace/substrate", rev = "55c157cff49b638a59d81a9f971f0f9a66829c71" }
sc-consensus-subspace = { git = "https://github.com/subspace/subspace", rev = "2d04263a9d44e845b1c92e5ec92e86dfeed0f511" }
sc-consensus-subspace = { git = "https://github.com/subspace/subspace", rev = "d3914a13ea2d5a476b9b93749416ae960f2d105a" }
sc-executor = { version = "0.10.0-dev", git = "https://github.com/subspace/substrate", rev = "55c157cff49b638a59d81a9f971f0f9a66829c71" }
sc-network = { version = "0.10.0-dev", git = "https://github.com/subspace/substrate", rev = "55c157cff49b638a59d81a9f971f0f9a66829c71" }
sc-network-sync = { version = "0.10.0-dev", git = "https://github.com/subspace/substrate", rev = "55c157cff49b638a59d81a9f971f0f9a66829c71" }
sc-proof-of-time = { git = "https://github.com/subspace/subspace", rev = "2d04263a9d44e845b1c92e5ec92e86dfeed0f511" }
sc-proof-of-time = { git = "https://github.com/subspace/subspace", rev = "d3914a13ea2d5a476b9b93749416ae960f2d105a" }
sc-rpc-api = { version = "0.10.0-dev", git = "https://github.com/subspace/substrate", rev = "55c157cff49b638a59d81a9f971f0f9a66829c71" }
sc-service = { version = "0.10.0-dev", git = "https://github.com/subspace/substrate", rev = "55c157cff49b638a59d81a9f971f0f9a66829c71", default-features = false }
sc-storage-monitor = { version = "0.1.0", git = "https://github.com/subspace/substrate", rev = "55c157cff49b638a59d81a9f971f0f9a66829c71", default-features = false }
sc-subspace-chain-specs = { git = "https://github.com/subspace/subspace", rev = "2d04263a9d44e845b1c92e5ec92e86dfeed0f511" }
sc-subspace-chain-specs = { git = "https://github.com/subspace/subspace", rev = "d3914a13ea2d5a476b9b93749416ae960f2d105a" }
sc-telemetry = { version = "4.0.0-dev", git = "https://github.com/subspace/substrate", rev = "55c157cff49b638a59d81a9f971f0f9a66829c71" }
sdk-dsn = { path = "../dsn" }
sdk-substrate = { path = "../substrate" }
Expand All @@ -46,20 +46,20 @@ serde = { version = "1", features = ["derive"] }
serde_json = "1"
sp-blockchain = { version = "4.0.0-dev", git = "https://github.com/subspace/substrate", rev = "55c157cff49b638a59d81a9f971f0f9a66829c71" }
sp-consensus = { version = "0.10.0-dev", git = "https://github.com/subspace/substrate", rev = "55c157cff49b638a59d81a9f971f0f9a66829c71" }
sp-consensus-subspace = { git = "https://github.com/subspace/subspace", rev = "2d04263a9d44e845b1c92e5ec92e86dfeed0f511" }
sp-consensus-subspace = { git = "https://github.com/subspace/subspace", rev = "d3914a13ea2d5a476b9b93749416ae960f2d105a" }
sp-core = { version = "21.0.0", git = "https://github.com/subspace/substrate", rev = "55c157cff49b638a59d81a9f971f0f9a66829c71" }
sp-domains = { git = "https://github.com/subspace/subspace", rev = "2d04263a9d44e845b1c92e5ec92e86dfeed0f511" }
sp-messenger = { git = "https://github.com/subspace/subspace", rev = "2d04263a9d44e845b1c92e5ec92e86dfeed0f511" }
sp-domains = { git = "https://github.com/subspace/subspace", rev = "d3914a13ea2d5a476b9b93749416ae960f2d105a" }
sp-messenger = { git = "https://github.com/subspace/subspace", rev = "d3914a13ea2d5a476b9b93749416ae960f2d105a" }
sp-runtime = { version = "24.0.0", git = "https://github.com/subspace/substrate", rev = "55c157cff49b638a59d81a9f971f0f9a66829c71" }
sp-version = { version = "22.0.0", git = "https://github.com/subspace/substrate", rev = "55c157cff49b638a59d81a9f971f0f9a66829c71" }
subspace-core-primitives = { git = "https://github.com/subspace/subspace", rev = "2d04263a9d44e845b1c92e5ec92e86dfeed0f511" }
subspace-farmer = { git = "https://github.com/subspace/subspace", rev = "2d04263a9d44e845b1c92e5ec92e86dfeed0f511" }
subspace-farmer-components = { git = "https://github.com/subspace/subspace", rev = "2d04263a9d44e845b1c92e5ec92e86dfeed0f511" }
subspace-networking = { git = "https://github.com/subspace/subspace", rev = "2d04263a9d44e845b1c92e5ec92e86dfeed0f511" }
subspace-rpc-primitives = { git = "https://github.com/subspace/subspace", rev = "2d04263a9d44e845b1c92e5ec92e86dfeed0f511" }
subspace-runtime = { git = "https://github.com/subspace/subspace", rev = "2d04263a9d44e845b1c92e5ec92e86dfeed0f511" }
subspace-runtime-primitives = { git = "https://github.com/subspace/subspace", rev = "2d04263a9d44e845b1c92e5ec92e86dfeed0f511" }
subspace-service = { git = "https://github.com/subspace/subspace", rev = "2d04263a9d44e845b1c92e5ec92e86dfeed0f511" }
subspace-core-primitives = { git = "https://github.com/subspace/subspace", rev = "d3914a13ea2d5a476b9b93749416ae960f2d105a" }
subspace-farmer = { git = "https://github.com/subspace/subspace", rev = "d3914a13ea2d5a476b9b93749416ae960f2d105a" }
subspace-farmer-components = { git = "https://github.com/subspace/subspace", rev = "d3914a13ea2d5a476b9b93749416ae960f2d105a" }
subspace-networking = { git = "https://github.com/subspace/subspace", rev = "d3914a13ea2d5a476b9b93749416ae960f2d105a" }
subspace-rpc-primitives = { git = "https://github.com/subspace/subspace", rev = "d3914a13ea2d5a476b9b93749416ae960f2d105a" }
subspace-runtime = { git = "https://github.com/subspace/subspace", rev = "d3914a13ea2d5a476b9b93749416ae960f2d105a" }
subspace-runtime-primitives = { git = "https://github.com/subspace/subspace", rev = "d3914a13ea2d5a476b9b93749416ae960f2d105a" }
subspace-service = { git = "https://github.com/subspace/subspace", rev = "d3914a13ea2d5a476b9b93749416ae960f2d105a" }
tokio = { version = "1.28.2", features = ["fs", "rt", "tracing", "macros", "parking_lot", "rt-multi-thread", "signal"] }
tokio-stream = { version = "0.1", features = ["sync", "time"] }
tracing = "0.1"
Expand Down
6 changes: 3 additions & 3 deletions traits/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,9 @@ async-trait = "0.1"
parking_lot = "0.12"
sc-client-api = { version = "4.0.0-dev", git = "https://github.com/subspace/substrate", rev = "55c157cff49b638a59d81a9f971f0f9a66829c71" }
sdk-dsn = { path = "../dsn" }
subspace-core-primitives = { git = "https://github.com/subspace/subspace", rev = "2d04263a9d44e845b1c92e5ec92e86dfeed0f511" }
subspace-farmer = { git = "https://github.com/subspace/subspace", rev = "2d04263a9d44e845b1c92e5ec92e86dfeed0f511" }
subspace-proof-of-space = { git = "https://github.com/subspace/subspace", rev = "2d04263a9d44e845b1c92e5ec92e86dfeed0f511" }
subspace-core-primitives = { git = "https://github.com/subspace/subspace", rev = "d3914a13ea2d5a476b9b93749416ae960f2d105a" }
subspace-farmer = { git = "https://github.com/subspace/subspace", rev = "d3914a13ea2d5a476b9b93749416ae960f2d105a" }
subspace-proof-of-space = { git = "https://github.com/subspace/subspace", rev = "d3914a13ea2d5a476b9b93749416ae960f2d105a" }

[features]
default = []
Expand Down
Loading

0 comments on commit d1ecb61

Please sign in to comment.