diff --git a/.gitignore b/.gitignore index 378be311..4caf66d4 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,6 @@ target/ # reproducible local environment .direnv + +# Visual Studio Code +.vscode/ \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index adef4928..dee871c7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1309,6 +1309,33 @@ dependencies = [ "windows-targets 0.52.5", ] +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "cid" version = "0.9.0" @@ -1319,7 +1346,7 @@ dependencies = [ "multibase", "multihash 0.17.0", "serde", - "unsigned-varint", + "unsigned-varint 0.7.2", ] [[package]] @@ -1332,7 +1359,21 @@ dependencies = [ "multibase", "multihash 0.18.1", "serde", - "unsigned-varint", + "unsigned-varint 0.7.2", +] + +[[package]] +name = "cid" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3147d8272e8fa0ccd29ce51194dd98f79ddfb8191ba9e3409884e751798acf3a" +dependencies = [ + "core2", + "multibase", + "multihash 0.19.1", + "serde", + "serde_bytes", + "unsigned-varint 0.8.0", ] [[package]] @@ -3943,6 +3984,16 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" +dependencies = [ + "cfg-if", + "crunchy", +] + [[package]] name = "handlebars" version = "5.1.2" @@ -4806,7 +4857,7 @@ dependencies = [ "rw-stream-sink", "smallvec", "thiserror", - "unsigned-varint", + "unsigned-varint 0.7.2", "void", ] @@ -4888,7 +4939,7 @@ dependencies = [ "smallvec", "thiserror", "uint", - "unsigned-varint", + "unsigned-varint 0.7.2", "void", ] @@ -5140,7 +5191,9 @@ dependencies = [ "glob", "libc", "libz-sys", + "lz4-sys", "tikv-jemalloc-sys", + "zstd-sys", ] [[package]] @@ -5311,7 +5364,7 @@ dependencies = [ "tracing", "trust-dns-resolver 0.23.2", "uint", - "unsigned-varint", + "unsigned-varint 0.7.2", "url", "webpki", "x25519-dalek 2.0.1", @@ -5669,7 +5722,7 @@ dependencies = [ "percent-encoding", "serde", "static_assertions", - "unsigned-varint", + "unsigned-varint 0.7.2", "url", ] @@ -5698,7 +5751,7 @@ dependencies = [ "multihash-derive 0.8.0", "sha2 0.10.8", "sha3", - "unsigned-varint", + "unsigned-varint 0.7.2", ] [[package]] @@ -5715,7 +5768,7 @@ dependencies = [ "multihash-derive 0.8.0", "sha2 0.10.8", "sha3", - "unsigned-varint", + "unsigned-varint 0.7.2", ] [[package]] @@ -5725,7 +5778,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "076d548d76a0e2a0d4ab471d0b1c36c577786dfc4471242035d97a12a735c492" dependencies = [ "core2", - "unsigned-varint", + "serde", + "unsigned-varint 0.7.2", ] [[package]] @@ -5804,7 +5858,7 @@ dependencies = [ "log", "pin-project", "smallvec", - "unsigned-varint", + "unsigned-varint 0.7.2", ] [[package]] @@ -7724,6 +7778,18 @@ version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db23d408679286588f4d4644f965003d056e3dd5abcaaa938116871d7ce2fee7" +[[package]] +name = "polka-index" +version = "0.1.0" +dependencies = [ + "ciborium", + "cid 0.11.1", + "rocksdb", + "serde", + "tempfile", + "thiserror", +] + [[package]] name = "polka-storage-node" version = "0.0.0" @@ -9381,7 +9447,7 @@ dependencies = [ "bytes", "quick-protobuf", "thiserror", - "unsigned-varint", + "unsigned-varint 0.7.2", ] [[package]] @@ -10865,7 +10931,7 @@ dependencies = [ "thiserror", "tokio", "tokio-stream", - "unsigned-varint", + "unsigned-varint 0.7.2", "void", "wasm-timer", "zeroize", @@ -13337,9 +13403,9 @@ checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" [[package]] name = "thiserror" -version = "1.0.60" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "579e9083ca58dd9dcf91a9923bb9054071b9ebbd800b342194c9feb0ee89fc18" +checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709" dependencies = [ "thiserror-impl", ] @@ -13366,9 +13432,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "1.0.60" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2470041c06ec3ac1ab38d0356a6119054dedaea53e12fbefc0de730a1c08524" +checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" dependencies = [ "proc-macro2", "quote", @@ -14035,6 +14101,12 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "unsigned-varint" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb066959b24b5196ae73cb057f45598450d2c5f71460e98c49b738086eff9c06" + [[package]] name = "untrusted" version = "0.7.1" diff --git a/Cargo.toml b/Cargo.toml index 21e5ab90..947942c1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,7 @@ license-file = "LICENSE" repository = "https://github.com/eigerco/polka-storage" [workspace] -members = ["node", "runtime"] +members = ["node", "runtime", "storage/polka-index"] resolver = "2" # FIXME(#@jmg-duarte,#7,14/5/24): remove the patch once something >1.11.0 is released @@ -28,6 +28,8 @@ panic = 'abort' # Use abort on panic to reduce binary size substrate-build-script-utils = { git = "https://github.com/paritytech/polkadot-sdk", tag = "polkadot-v1.11.0" } substrate-wasm-builder = { git = "https://github.com/paritytech/polkadot-sdk", tag = "polkadot-v1.11.0" } +ciborium = "0.2.2" +cid = { version = "0.11.1" } clap = { version = "4.5.3" } codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false } color-print = "0.3.4" @@ -39,6 +41,7 @@ polkavm = "0.9.3" polkavm-derive = "0.9.1" polkavm-linker = "0.9.2" quote = { version = "1.0.33" } +rocksdb = { version = "0.21" } scale-info = { version = "2.11.1", default-features = false } serde = { version = "1.0.197", default-features = false } serde-big-array = { version = "0.3.2" } @@ -47,6 +50,7 @@ serde_json = { version = "1.0.114", default-features = false } serde_yaml = { version = "0.9" } smallvec = "1.11.0" syn = { version = "2.0.53" } +tempfile = "3.10.1" thiserror = { version = "1.0.48" } tracing-subscriber = { version = "0.3.18" } diff --git a/storage/polka-index/.gitkeep b/storage/polka-index/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/storage/polka-index/Cargo.toml b/storage/polka-index/Cargo.toml new file mode 100644 index 00000000..335e9e4b --- /dev/null +++ b/storage/polka-index/Cargo.toml @@ -0,0 +1,21 @@ +[package] +authors.workspace = true +edition.workspace = true +homepage.workspace = true +license-file.workspace = true +name = "polka-index" +repository.workspace = true +version = "0.1.0" + +[dependencies] +ciborium = { workspace = true } +cid = { workspace = true, features = ["serde"] } +rocksdb = { workspace = true } +serde = { workspace = true } +thiserror = { workspace = true } + +[dev-dependencies] +tempfile = { workspace = true } + +[lints] +workspace = true diff --git a/storage/polka-index/src/lib.rs b/storage/polka-index/src/lib.rs new file mode 100644 index 00000000..26d67933 --- /dev/null +++ b/storage/polka-index/src/lib.rs @@ -0,0 +1 @@ +pub mod piecestore; diff --git a/storage/polka-index/src/main.rs b/storage/polka-index/src/main.rs new file mode 100644 index 00000000..e7a11a96 --- /dev/null +++ b/storage/polka-index/src/main.rs @@ -0,0 +1,3 @@ +fn main() { + println!("Hello, world!"); +} diff --git a/storage/polka-index/src/piecestore/README.md b/storage/polka-index/src/piecestore/README.md new file mode 100644 index 00000000..64a7cb55 --- /dev/null +++ b/storage/polka-index/src/piecestore/README.md @@ -0,0 +1,4 @@ +# piecestore + +The piecestore module is a simple encapsulation of two data stores, one for `PieceInfo` and +another for `CidInfo`. diff --git a/storage/polka-index/src/piecestore/mod.rs b/storage/polka-index/src/piecestore/mod.rs new file mode 100644 index 00000000..b62c6629 --- /dev/null +++ b/storage/polka-index/src/piecestore/mod.rs @@ -0,0 +1,69 @@ +use std::collections::HashMap; + +use cid::Cid; +use rocksdb::RocksDBError; +use thiserror::Error; + +use self::types::{BlockLocation, CidInfo, DealInfo, PieceInfo}; + +pub mod rocksdb; +pub mod types; + +pub trait PieceStore { + /// Implementation-specific configuration. + type Config; + + /// Initialize a new store. + fn new(config: Self::Config) -> Result + where + Self: Sized; + + /// Store [`DealInfo`] in the PieceStore with key piece [`Cid`]. + fn add_deal_for_piece( + &self, + piece_cid: &Cid, + deal_info: DealInfo, + ) -> Result<(), PieceStoreError>; + + /// Store the map of [`BlockLocation`] in the [`PieceStore`]'s [`CidInfo`] store, with + /// key piece [`Cid`]. + /// + /// Note: If a piece block location is already present in the [`CidInfo`], it + /// will be ignored. + fn add_piece_block_locations( + &self, + piece_cid: &Cid, + block_locations: &HashMap, + ) -> Result<(), PieceStoreError>; + + /// List all piece [`Cid`]s stored in the [`PieceStore`]. + fn list_piece_info_keys(&self) -> Result, PieceStoreError>; + + /// List all [`CidInfo`]s keys stored in the [`PieceStore`]. + fn list_cid_info_keys(&self) -> Result, PieceStoreError>; + + /// Retrieve the [`PieceInfo`] for a given piece [`Cid`]. + fn get_piece_info(&self, cid: &Cid) -> Result, PieceStoreError>; + + /// Retrieve the [`CidInfo`] associated with piece [`Cid`]. + fn get_cid_info(&self, cid: &Cid) -> Result, PieceStoreError>; +} + +/// Error that can occur when interacting with the [`PieceStore`]. +#[derive(Debug, Error)] +pub enum PieceStoreError { + #[error("Initialization error: {0}")] + Initialization(String), + + #[error("Deal already exists")] + DealExists, + + #[error("Serialization error: {0}")] + Serialization(String), + + #[error("Deserialization error: {0}")] + Deserialization(String), + + #[error(transparent)] + StoreError(#[from] RocksDBError), +} diff --git a/storage/polka-index/src/piecestore/rocksdb.rs b/storage/polka-index/src/piecestore/rocksdb.rs new file mode 100644 index 00000000..3371663e --- /dev/null +++ b/storage/polka-index/src/piecestore/rocksdb.rs @@ -0,0 +1,410 @@ +use std::{collections::HashMap, path::PathBuf}; + +use cid::Cid; +use rocksdb::{ColumnFamily, ColumnFamilyDescriptor, IteratorMode, Options, DB as RocksDB}; +use serde::{de::DeserializeOwned, Serialize}; + +use super::{ + types::{BlockLocation, CidInfo, DealInfo}, + PieceStore, PieceStoreError, +}; +use crate::piecestore::types::{PieceBlockLocation, PieceInfo}; + +/// Error type for RocksDB operations. +pub(crate) type RocksDBError = rocksdb::Error; + +/// Column family name used to store piece. +const PIECES_CF: &str = "pieces"; + +/// Column family name used to store CID infos. +const CID_INFOS_CF: &str = "cid_infos"; + +pub struct RocksDBStateStoreConfig { + pub path: PathBuf, +} + +/// A [`super::PieceStore`] implementation backed by RocksDB. +pub struct RocksDBPieceStore { + database: RocksDB, +} + +impl RocksDBPieceStore { + /// Get the column family handle for the given column family name. Panics if + /// the column family is not present. The column families needed and used + /// are created at initialization. They will always be present. + #[track_caller] + fn cf_handle(&self, cf_name: &str) -> &ColumnFamily { + self.database + .cf_handle(cf_name) + .expect("column family should be present") + } + + fn list_cids_in_cf(&self, cf_name: &str) -> Result, PieceStoreError> { + let iterator = self + .database + .iterator_cf(self.cf_handle(cf_name), IteratorMode::Start); + + iterator + .map(|line| { + let (key, _) = line?; + + let parsed_cid = Cid::try_from(key.as_ref()).map_err(|err| { + // We know that all stored CIDs are valid, so this + // should only happen if database is corrupted. + PieceStoreError::Deserialization(format!("invalid CID: {}", err)) + })?; + + Ok(parsed_cid) + }) + .collect() + } + + /// Get value at the specified key in the specified column family. + fn get_value_at_key( + &self, + key: Key, + cf_name: &str, + ) -> Result, PieceStoreError> + where + Key: AsRef<[u8]>, + Value: DeserializeOwned, + { + let Some(slice) = self.database.get_pinned_cf(self.cf_handle(cf_name), key)? else { + return Ok(None); + }; + + match ciborium::from_reader(slice.as_ref()) { + Ok(value) => Ok(Some(value)), + // ciborium error is bubbled up as a string because it is generic + // and we didn't want to add a generic type to the PieceStoreError + Err(err) => Err(PieceStoreError::Deserialization(err.to_string())), + } + } + + /// Put value at the specified key in the specified column family. + fn put_value_at_key( + &self, + key: Key, + value: &Value, + cf_name: &str, + ) -> Result<(), PieceStoreError> + where + Key: AsRef<[u8]>, + Value: Serialize, + { + let mut serialized = Vec::new(); + if let Err(err) = ciborium::into_writer(value, &mut serialized) { + // ciborium error is bubbled up as a string because it is generic + // and we didn't want to add a generic type to the PieceStoreError + return Err(PieceStoreError::Serialization(err.to_string())); + } + + self.database + .put_cf(self.cf_handle(cf_name), key, serialized)?; + + Ok(()) + } +} + +impl PieceStore for RocksDBPieceStore { + type Config = RocksDBStateStoreConfig; + + fn new(config: Self::Config) -> Result + where + Self: Sized, + { + let pieces_column = ColumnFamilyDescriptor::new(PIECES_CF, Options::default()); + let cid_infos_column = ColumnFamilyDescriptor::new(CID_INFOS_CF, Options::default()); + + let mut opts = Options::default(); + // Creates a new database if it doesn't exist + opts.create_if_missing(true); + // Create missing column families + opts.create_missing_column_families(true); + + let database = RocksDB::open_cf_descriptors( + &opts, + config.path, + vec![pieces_column, cid_infos_column], + )?; + + Ok(Self { database }) + } + + fn add_deal_for_piece( + &self, + piece_cid: &Cid, + deal_info: DealInfo, + ) -> Result<(), PieceStoreError> { + // Check if the piece exists + let mut piece_info = self + .get_value_at_key(piece_cid.to_bytes(), PIECES_CF)? + .unwrap_or_else(|| PieceInfo { + piece_cid: *piece_cid, + deals: Vec::new(), + }); + + // Check if deal already added for this piece + if piece_info.deals.iter().any(|d| *d == deal_info) { + return Err(PieceStoreError::DealExists); + } + + // Save the new deal + piece_info.deals.push(deal_info); + self.put_value_at_key(piece_cid.to_bytes(), &piece_info, PIECES_CF) + } + + fn add_piece_block_locations( + &self, + piece_cid: &Cid, + block_locations: &HashMap, + ) -> Result<(), PieceStoreError> { + for (cid, block_location) in block_locations { + let mut info = self + .get_value_at_key(cid.to_bytes(), CID_INFOS_CF)? + .unwrap_or_else(|| CidInfo { + cid: *cid, + piece_block_location: Vec::new(), + }); + + if info + .piece_block_location + .iter() + .any(|pbl| pbl.piece_cid == *piece_cid && pbl.location == *block_location) + { + continue; + } + + // Append the new block location + info.piece_block_location.push(PieceBlockLocation { + piece_cid: *piece_cid, + location: *block_location, + }); + + // Save the updated CidInfo + self.put_value_at_key(cid.to_bytes(), &info, CID_INFOS_CF)?; + } + + Ok(()) + } + + fn list_piece_info_keys(&self) -> Result, PieceStoreError> { + self.list_cids_in_cf(PIECES_CF) + } + + fn list_cid_info_keys(&self) -> Result, PieceStoreError> { + self.list_cids_in_cf(CID_INFOS_CF) + } + + fn get_piece_info(&self, cid: &Cid) -> Result, PieceStoreError> { + self.get_value_at_key(cid.to_bytes(), PIECES_CF) + } + + fn get_cid_info(&self, cid: &Cid) -> Result, PieceStoreError> { + self.get_value_at_key(cid.to_bytes(), CID_INFOS_CF) + } +} + +#[cfg(test)] +mod test { + use std::{collections::HashMap, str::FromStr}; + + use cid::Cid; + use tempfile::tempdir; + + use super::{RocksDBPieceStore, RocksDBStateStoreConfig}; + use crate::piecestore::{ + types::{BlockLocation, DealInfo, PieceBlockLocation}, + PieceStore, PieceStoreError, + }; + + fn init_database() -> RocksDBPieceStore { + let tmp_dir = tempdir().unwrap(); + let config = RocksDBStateStoreConfig { + path: tmp_dir.path().join("rocksdb"), + }; + + RocksDBPieceStore::new(config).unwrap() + } + + fn cids() -> (Cid, Cid, Cid) { + ( + Cid::from_str("QmawceGscqN4o8Y8Fv26UUmB454kn2bnkXV5tEQYc4jBd6").unwrap(), + Cid::from_str("QmbvrHYWXAU1BuxMPNRtfeF4DS2oPmo5hat7ocqAkNPr74").unwrap(), + Cid::from_str("QmfRL5b6fPZ851F6E2ZUWX1kC4opXzq9QDhamvU4tJGuyR").unwrap(), + ) + } + + fn rand_deal() -> DealInfo { + DealInfo { + deal_id: 1, + sector_id: 1, + offset: 0, + length: 100, + } + } + + fn block_location() -> BlockLocation { + BlockLocation { + rel_offset: 0, + block_size: 100, + } + } + + #[test] + fn test_piece_info_can_add_deals() { + let store = init_database(); + let (piece_cid, piece_cid2, _) = cids(); + let deal_info = rand_deal(); + + // add deal for piece + store.add_deal_for_piece(&piece_cid, deal_info).unwrap(); + + // get piece info + let info = store.get_piece_info(&piece_cid).unwrap().unwrap(); + assert_eq!(info.deals, vec![deal_info]); + + // verify that getting a piece with a non-existent CID return None + let info = store.get_piece_info(&piece_cid2).unwrap(); + assert!(info.is_none(), "expected None, got {:?}", info); + } + + #[test] + fn test_piece_adding_same_deal_twice_returns_error() { + let store = init_database(); + let (piece_cid, _, _) = cids(); + let deal_info = rand_deal(); + + // add deal for piece + store.add_deal_for_piece(&piece_cid, deal_info).unwrap(); + + // add deal for piece + let result = store.add_deal_for_piece(&piece_cid, deal_info); + assert!( + matches!(result, Err(PieceStoreError::DealExists)), + "expected error, got {:?}", + result + ); + } + + #[test] + fn test_cid_info_can_add_piece_block_locations() { + let store = init_database(); + let (piece_cid, _, _) = cids(); + let block_locations = [block_location(); 4]; + let test_cids = [ + Cid::from_str("QmW9pMY7fvbxVA2CaihgxJRzmSv15Re2TABte4HoZdfypo").unwrap(), + Cid::from_str("QmZbaU7GGuu9F7saPgVmPSK55of8QkzEwjPrj7xxWogxiY").unwrap(), + Cid::from_str("QmQSQYNn2K6xTDLhfNcoTjBExz5Q5gpHHBTqZZKdxsPRB9").unwrap(), + ]; + + let block_locations = test_cids + .iter() + .zip(block_locations.iter()) + .map(|(cid, block_location)| (*cid, *block_location)) + .collect::>(); + + // add piece block locations + store + .add_piece_block_locations(&piece_cid, &block_locations) + .unwrap(); + + // get cid info + let info = store.get_cid_info(&test_cids[0]).unwrap().unwrap(); + assert!( + info.piece_block_location.contains(&PieceBlockLocation { + piece_cid, + location: block_locations[&test_cids[0]] + }), + "block location not found in cid info" + ); + + let info = store.get_cid_info(&test_cids[1]).unwrap().unwrap(); + assert!( + info.piece_block_location.contains(&PieceBlockLocation { + piece_cid, + location: block_locations[&test_cids[1]] + }), + "block location not found in cid info" + ); + + let info = store.get_cid_info(&test_cids[2]).unwrap().unwrap(); + assert!( + info.piece_block_location.contains(&PieceBlockLocation { + piece_cid, + location: block_locations[&test_cids[2]] + }), + "block location not found in cid info" + ); + + // verify that getting a piece with a non-existent CID return None + let info = store + .get_cid_info(&Cid::from_str("QmW9pMY7fvbxVA2CaihgxJRzmSv15Re2TABte4HoZdfypa").unwrap()) + .unwrap(); + assert!(info.is_none(), "expected None, got {:?}", info); + } + + #[test] + fn test_cid_info_overlapping_adds() { + let store = init_database(); + let (piece_cid, _, _) = cids(); + let block_locations = [block_location(); 4]; + let test_cids = [ + Cid::from_str("QmW9pMY7fvbxVA2CaihgxJRzmSv15Re2TABte4HoZdfypo").unwrap(), + Cid::from_str("QmZbaU7GGuu9F7saPgVmPSK55of8QkzEwjPrj7xxWogxiY").unwrap(), + Cid::from_str("QmQSQYNn2K6xTDLhfNcoTjBExz5Q5gpHHBTqZZKdxsPRB9").unwrap(), + ]; + + // add piece block locations + let locations = [ + (test_cids[0], block_locations[0]), + (test_cids[1], block_locations[2]), + ] + .into_iter() + .collect::>(); + + store + .add_piece_block_locations(&piece_cid, &locations) + .unwrap(); + + // add piece block locations + let locations = [ + (test_cids[1], block_locations[1]), + (test_cids[2], block_locations[2]), + ] + .into_iter() + .collect::>(); + + store + .add_piece_block_locations(&piece_cid, &locations) + .unwrap(); + + // get cid info + let info = store.get_cid_info(&test_cids[0]).unwrap().unwrap(); + assert_eq!( + info.piece_block_location, + vec![PieceBlockLocation { + piece_cid, + location: block_locations[0] + }] + ); + + let info = store.get_cid_info(&test_cids[1]).unwrap().unwrap(); + assert_eq!( + info.piece_block_location, + vec![PieceBlockLocation { + piece_cid, + location: block_locations[1] + }] + ); + + let info = store.get_cid_info(&test_cids[2]).unwrap().unwrap(); + assert_eq!( + info.piece_block_location, + vec![PieceBlockLocation { + piece_cid, + location: block_locations[2] + }] + ); + } +} diff --git a/storage/polka-index/src/piecestore/types.rs b/storage/polka-index/src/piecestore/types.rs new file mode 100644 index 00000000..b89d7c30 --- /dev/null +++ b/storage/polka-index/src/piecestore/types.rs @@ -0,0 +1,48 @@ +use cid::Cid; +use serde::{Deserialize, Serialize}; + +/// Metadata about a piece that provider may be storing based on its [`Cid`]. So +/// that, given a [`Cid`] during retrieval, the miner can determine how to +/// unseal it if needed +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PieceInfo { + pub piece_cid: Cid, + pub deals: Vec, +} + +/// Identifier for a retrieval deal (unique to a client) +type DealId = u64; + +/// Numeric identifier for a sector. It is usually relative to a miner. +type SectorNumber = u64; + +/// Information about a single deal for a given piece +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct DealInfo { + pub deal_id: DealId, + pub sector_id: SectorNumber, + pub offset: u64, + pub length: u64, +} + +/// Information about where a given block is relative to the overall piece +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct BlockLocation { + pub rel_offset: u64, + pub block_size: u64, +} + +/// Contains block information along with the [`Cid`] of the piece the block is +/// inside of +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct PieceBlockLocation { + pub piece_cid: Cid, + pub location: BlockLocation, +} + +/// Information about where a given [`Cid`] will live inside a piece +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CidInfo { + pub cid: Cid, + pub piece_block_location: Vec, +}