From 3e150229007b010e73b1229c27ff921848b074e6 Mon Sep 17 00:00:00 2001 From: aidan Date: Thu, 23 May 2024 14:23:41 +0200 Subject: [PATCH 01/75] feat(storage-provider): Add pallet miner interface Set up scaffolding for the storage provider for miner indexing. This includes function definitions and events emitted. --- Cargo.lock | 11 ++ Cargo.toml | 3 +- Justfile | 7 +- pallets/storage-provider/.gitkeep | 0 pallets/storage-provider/Cargo.toml | 19 +++ pallets/storage-provider/src/lib.rs | 202 ++++++++++++++++++++++++++++ runtime/Cargo.toml | 3 + runtime/src/configs/mod.rs | 6 + runtime/src/lib.rs | 1 + 9 files changed, 245 insertions(+), 7 deletions(-) delete mode 100644 pallets/storage-provider/.gitkeep create mode 100644 pallets/storage-provider/Cargo.toml create mode 100644 pallets/storage-provider/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index adef49283..7cf7e0a6e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7836,6 +7836,7 @@ dependencies = [ "staging-xcm", "staging-xcm-builder", "staging-xcm-executor", + "storage-provider", "substrate-wasm-builder", ] @@ -13049,6 +13050,16 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "storage-provider" +version = "0.1.0" +dependencies = [ + "frame-support", + "frame-system", + "parity-scale-codec", + "scale-info", +] + [[package]] name = "str0m" version = "0.2.0" diff --git a/Cargo.toml b/Cargo.toml index 21e5ab904..c191043de 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,7 @@ license-file = "LICENSE" repository = "https://github.com/eigerco/polka-storage" [workspace] -members = ["node", "runtime"] +members = ["node", "pallets/storage-provider", "runtime"] resolver = "2" # FIXME(#@jmg-duarte,#7,14/5/24): remove the patch once something >1.11.0 is released @@ -52,6 +52,7 @@ tracing-subscriber = { version = "0.3.18" } # Local polka-storage-runtime = { path = "runtime" } +storage-provider = { path = "pallets/storage-provider" } # Substrate pallet-transaction-payment-rpc = { git = "https://github.com/paritytech/polkadot-sdk", tag = "polkadot-v1.11.0" } diff --git a/Justfile b/Justfile index 69bdeea3d..4c994d974 100644 --- a/Justfile +++ b/Justfile @@ -1,16 +1,11 @@ alias b := build -alias r := release alias t := testnet lint: - cargo clippy --locked --no-deps -- -D warnings taplo lint && taplo fmt --check build: lint - cargo build - -release: lint cargo build --release -testnet: release +testnet: build zombienet -p native spawn zombienet/local-testnet.toml diff --git a/pallets/storage-provider/.gitkeep b/pallets/storage-provider/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/pallets/storage-provider/Cargo.toml b/pallets/storage-provider/Cargo.toml new file mode 100644 index 000000000..6c40ea930 --- /dev/null +++ b/pallets/storage-provider/Cargo.toml @@ -0,0 +1,19 @@ +[package] +authors.workspace = true +edition.workspace = true +homepage.workspace = true +license-file.workspace = true +name = "storage-provider" +repository.workspace = true +version = "0.1.0" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +codec.workspace = true +frame-support = { workspace = true, default-features = false } +frame-system = { workspace = true, default-features = false } +scale-info.workspace = true + +[lints] +workspace = true diff --git a/pallets/storage-provider/src/lib.rs b/pallets/storage-provider/src/lib.rs new file mode 100644 index 000000000..9e6922d8a --- /dev/null +++ b/pallets/storage-provider/src/lib.rs @@ -0,0 +1,202 @@ +//! # Storage Provider Pallet +//! +//! This pallet holds information about storage providers and +//! provides an interface to modify information about miners. +//! +//! The Storage Provider Pallet is the source of truth for anything storage provider (the binary) related. +//! +//! At some point this pallet will have to verify proofs submitted by storage providers + +#![cfg_attr(not(feature = "std"), no_std)] + +pub use pallet::{Config, Pallet}; + +use codec::{Decode, Encode}; +use scale_info::TypeInfo; + +#[derive(Decode, Encode, TypeInfo)] +pub struct MinerInfo { + /// The owner of this miner. + owner: AccountId, + /// Worker of this miner + worker: WorkerAddress, + /// Miner's libp2p peer id in bytes. + peer_id: PeerId, +} + +#[frame_support::pallet(dev_mode)] +pub mod pallet { + use super::MinerInfo; + + use frame_support::dispatch::DispatchResultWithPostInfo; + use frame_support::pallet_prelude::{IsType, PhantomData, StorageMap}; + use frame_support::{Blake2_128Concat, Parameter}; + use frame_system::ensure_signed; + use frame_system::pallet_prelude::OriginFor; + use scale_info::prelude::vec::Vec; + + /// Peer ID is derived by hashing an encoded public key. + /// Usually represented in bytes. + /// https://github.com/libp2p/specs/blob/master/peer-ids/peer-ids.md#peer-ids + type PeerId = Vec; + + #[pallet::pallet] + #[pallet::without_storage_info] // Allows to define storage items without fixed size + pub struct Pallet(PhantomData); + + #[pallet::config] + pub trait Config: frame_system::Config { + /// Because this pallet emits events, it depends on the runtime's definition of an event. + type RuntimeEvent: From> + IsType<::RuntimeEvent>; + + /// This pallet handles registration of workers to an account. These workers have an address that is outside of substrate. + type WorkerAddress: Parameter; + + /// MinerAccountId identifies a miner + type MinerAccountId: Parameter; + } + + // Need some storage type that keeps track of sectors, deadlines and terminations. + // Could be added to this type maybe? + #[pallet::storage] + #[pallet::getter(fn miners)] + pub type Miners = StorageMap< + _, + Blake2_128Concat, + T::AccountId, + MinerInfo, + >; + + #[pallet::event] + #[pallet::generate_deposit(pub(super) fn deposit_event)] + pub enum Event { + MinerCreated { + owner: T::AccountId, + }, + ChangeWorkerAddressRequested { + miner: T::MinerAccountId, + new_worker: T::WorkerAddress, + }, + PeerIdChanged { + miner: T::MinerAccountId, + peer_id: PeerId, + }, + WorkerAddressChanged { + miner: T::MinerAccountId, + new_worker: T::WorkerAddress, + }, + OwnerAddressChanged { + miner: T::MinerAccountId, + new_owner: T::AccountId, + }, + } + + #[pallet::call] + impl Pallet { + /// Add a new miner information to `Miners` + #[pallet::call_index(0)] + pub fn create_miner( + origin: OriginFor, + owner: T::AccountId, + _worker: T::WorkerAddress, + _peer_id: PeerId, + ) -> DispatchResultWithPostInfo { + // Check that the extrinsic was signed and get the signer. + let _who = ensure_signed(origin)?; + + // Generate some miner id and insert into `Miners` storage + + // This probably inherits a `create_miner` function from a `Power` trait. + + Self::deposit_event(Event::MinerCreated { owner }); + todo!() + } + + /// Request to change a worker address for a given miner + #[pallet::call_index(1)] + pub fn change_worker_address( + origin: OriginFor, + miner: T::MinerAccountId, + new_worker: T::WorkerAddress, + ) -> DispatchResultWithPostInfo { + // Check that the extrinsic was signed and get the signer. + let _who = ensure_signed(origin)?; + + // Get miner info from `Miners` with `who` value + // let miner_info = Miners::::try_get(&miner); + + // Ensure who is the owner of the miner + // ensure!(who == miner_info.owner) + + // Flag miner as worker address change + + Self::deposit_event(Event::ChangeWorkerAddressRequested { miner, new_worker }); + todo!() + } + + /// Update PeerId associated with a given miner. + #[pallet::call_index(2)] + pub fn change_peer_id( + origin: OriginFor, + miner: T::MinerAccountId, + peer_id: PeerId, + ) -> DispatchResultWithPostInfo { + // Check that the extrinsic was signed and get the signer. + let _who = ensure_signed(origin)?; + + // Get miner info from `Miners` with `who` value + // let miner_info = Miners::::try_get(&miner); + + // Ensure who is the owner of the miner + // ensure!(who == miner_info.owner) + + // Update PeerId + + Self::deposit_event(Event::PeerIdChanged { miner, peer_id }); + todo!() + } + + /// Confirms the change of the worker address initiated by `change_worker_address` + #[pallet::call_index(3)] + pub fn confirm_change_worker_address( + origin: OriginFor, + _miner: T::MinerAccountId, + ) -> DispatchResultWithPostInfo { + // Check that the extrinsic was signed and get the signer. + let _who = ensure_signed(origin)?; + + // Get miner info from `Miners` with `who` value + // let miner_info = Miners::::try_get(&miner); + + // Ensure who is the owner of the miner + // ensure!(who == miner_info.owner) + + // Change worker address and extract `new_worker` for event emitted. + + // Self::deposit_event(Event::WorkerAddressChanged { miner, new_worker }); + todo!() + } + + // This function updates the owner address to the given `new_owner` for the given `miner` + #[pallet::call_index(4)] + pub fn change_owner_address( + origin: OriginFor, + miner: T::MinerAccountId, + new_owner: T::AccountId, + ) -> DispatchResultWithPostInfo { + // Check that the extrinsic was signed and get the signer. + let _who = ensure_signed(origin)?; + + // Get miner info from `Miners` with `who` value + // let miner_info = Miners::::try_get(&miner); + + // Ensure who is the owner of the miner + // ensure!(who == miner_info.owner) + + // Change owner address + + Self::deposit_event(Event::OwnerAddressChanged { miner, new_owner }); + todo!() + } + } +} diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index 6e22ebd23..8dfdec629 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -19,6 +19,9 @@ targets = ["x86_64-unknown-linux-gnu"] substrate-wasm-builder = { workspace = true, optional = true } [dependencies] +# Pallets +storage-provider.workspace = true + codec = { workspace = true, default-features = false, features = ["derive"] } hex-literal = { workspace = true, optional = true } log = { workspace = true } diff --git a/runtime/src/configs/mod.rs b/runtime/src/configs/mod.rs index 77ae6bf0a..dedbbbabf 100644 --- a/runtime/src/configs/mod.rs +++ b/runtime/src/configs/mod.rs @@ -303,3 +303,9 @@ impl pallet_collator_selection::Config for Runtime { type ValidatorRegistration = Session; type WeightInfo = (); } + +impl storage_provider::Config for Runtime { + type RuntimeEvent = RuntimeEvent; + type WorkerAddress = (); // Not sure what this should be yet + type MinerAccountId = AccountId; +} diff --git a/runtime/src/lib.rs b/runtime/src/lib.rs index 1fbe72296..6492bdfe9 100644 --- a/runtime/src/lib.rs +++ b/runtime/src/lib.rs @@ -245,6 +245,7 @@ construct_runtime!( PolkadotXcm: pallet_xcm = 31, CumulusXcm: cumulus_pallet_xcm = 32, MessageQueue: pallet_message_queue = 33, + StorageProvider: storage_provider::pallet = 34, } ); From 8166469fddf1fec36df3f17c6f25099a68565f41 Mon Sep 17 00:00:00 2001 From: aidan Date: Thu, 30 May 2024 12:46:50 +0200 Subject: [PATCH 02/75] style: Format rs files --- pallets/storage-provider/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pallets/storage-provider/src/lib.rs b/pallets/storage-provider/src/lib.rs index 9e6922d8a..065e943f4 100644 --- a/pallets/storage-provider/src/lib.rs +++ b/pallets/storage-provider/src/lib.rs @@ -2,9 +2,9 @@ //! //! This pallet holds information about storage providers and //! provides an interface to modify information about miners. -//! +//! //! The Storage Provider Pallet is the source of truth for anything storage provider (the binary) related. -//! +//! //! At some point this pallet will have to verify proofs submitted by storage providers #![cfg_attr(not(feature = "std"), no_std)] From b8f96d55e3456bb01e4328de01537cb4f0e89480 Mon Sep 17 00:00:00 2001 From: aidan Date: Thu, 30 May 2024 14:30:52 +0200 Subject: [PATCH 03/75] fix: Revert Justfile back to original --- Justfile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Justfile b/Justfile index 4c994d974..69bdeea3d 100644 --- a/Justfile +++ b/Justfile @@ -1,11 +1,16 @@ alias b := build +alias r := release alias t := testnet lint: + cargo clippy --locked --no-deps -- -D warnings taplo lint && taplo fmt --check build: lint + cargo build + +release: lint cargo build --release -testnet: build +testnet: release zombienet -p native spawn zombienet/local-testnet.toml From 6ca40cf1ed30ed246d8094221733b33f67140ca3 Mon Sep 17 00:00:00 2001 From: aidan Date: Thu, 30 May 2024 15:08:29 +0200 Subject: [PATCH 04/75] fix: Remove worker related functionality Since the storage providers will handle their own worker addresses it is not needed to include the worker addresses in the pallet implementation. --- pallets/storage-provider/src/lib.rs | 52 ----------------------------- 1 file changed, 52 deletions(-) diff --git a/pallets/storage-provider/src/lib.rs b/pallets/storage-provider/src/lib.rs index 065e943f4..a38b74064 100644 --- a/pallets/storage-provider/src/lib.rs +++ b/pallets/storage-provider/src/lib.rs @@ -73,18 +73,10 @@ pub mod pallet { MinerCreated { owner: T::AccountId, }, - ChangeWorkerAddressRequested { - miner: T::MinerAccountId, - new_worker: T::WorkerAddress, - }, PeerIdChanged { miner: T::MinerAccountId, peer_id: PeerId, }, - WorkerAddressChanged { - miner: T::MinerAccountId, - new_worker: T::WorkerAddress, - }, OwnerAddressChanged { miner: T::MinerAccountId, new_owner: T::AccountId, @@ -98,7 +90,6 @@ pub mod pallet { pub fn create_miner( origin: OriginFor, owner: T::AccountId, - _worker: T::WorkerAddress, _peer_id: PeerId, ) -> DispatchResultWithPostInfo { // Check that the extrinsic was signed and get the signer. @@ -112,28 +103,6 @@ pub mod pallet { todo!() } - /// Request to change a worker address for a given miner - #[pallet::call_index(1)] - pub fn change_worker_address( - origin: OriginFor, - miner: T::MinerAccountId, - new_worker: T::WorkerAddress, - ) -> DispatchResultWithPostInfo { - // Check that the extrinsic was signed and get the signer. - let _who = ensure_signed(origin)?; - - // Get miner info from `Miners` with `who` value - // let miner_info = Miners::::try_get(&miner); - - // Ensure who is the owner of the miner - // ensure!(who == miner_info.owner) - - // Flag miner as worker address change - - Self::deposit_event(Event::ChangeWorkerAddressRequested { miner, new_worker }); - todo!() - } - /// Update PeerId associated with a given miner. #[pallet::call_index(2)] pub fn change_peer_id( @@ -156,27 +125,6 @@ pub mod pallet { todo!() } - /// Confirms the change of the worker address initiated by `change_worker_address` - #[pallet::call_index(3)] - pub fn confirm_change_worker_address( - origin: OriginFor, - _miner: T::MinerAccountId, - ) -> DispatchResultWithPostInfo { - // Check that the extrinsic was signed and get the signer. - let _who = ensure_signed(origin)?; - - // Get miner info from `Miners` with `who` value - // let miner_info = Miners::::try_get(&miner); - - // Ensure who is the owner of the miner - // ensure!(who == miner_info.owner) - - // Change worker address and extract `new_worker` for event emitted. - - // Self::deposit_event(Event::WorkerAddressChanged { miner, new_worker }); - todo!() - } - // This function updates the owner address to the given `new_owner` for the given `miner` #[pallet::call_index(4)] pub fn change_owner_address( From 805f22b75db78271071d27cd96edb3d9073d1463 Mon Sep 17 00:00:00 2001 From: aidan Date: Thu, 30 May 2024 15:21:43 +0200 Subject: [PATCH 05/75] feat: Add `apply_reward` function --- pallets/storage-provider/src/lib.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/pallets/storage-provider/src/lib.rs b/pallets/storage-provider/src/lib.rs index a38b74064..27e666390 100644 --- a/pallets/storage-provider/src/lib.rs +++ b/pallets/storage-provider/src/lib.rs @@ -104,7 +104,7 @@ pub mod pallet { } /// Update PeerId associated with a given miner. - #[pallet::call_index(2)] + #[pallet::call_index(1)] pub fn change_peer_id( origin: OriginFor, miner: T::MinerAccountId, @@ -126,7 +126,7 @@ pub mod pallet { } // This function updates the owner address to the given `new_owner` for the given `miner` - #[pallet::call_index(4)] + #[pallet::call_index(2)] pub fn change_owner_address( origin: OriginFor, miner: T::MinerAccountId, @@ -146,5 +146,14 @@ pub mod pallet { Self::deposit_event(Event::OwnerAddressChanged { miner, new_owner }); todo!() } + + // Used by the reward pallet to award a block reward to a Miner. + // I am not sure if this should be implemented on this pallet. + // The reward pallet could be tightly coupled with the storage provider pallet + // so the reward pallet can take over this functionality. + #[pallet::call_index(3)] + pub fn apply_rewards(origin: OriginFor) -> DispatchResultWithPostInfo { + todo!() + } } } From 4f48cae25d32b0b69db449c34fa093f06f1326c1 Mon Sep 17 00:00:00 2001 From: aidan Date: Thu, 30 May 2024 15:32:49 +0200 Subject: [PATCH 06/75] feat: Add `report_consensus_fault` function --- pallets/storage-provider/src/lib.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pallets/storage-provider/src/lib.rs b/pallets/storage-provider/src/lib.rs index 27e666390..d5c03c66c 100644 --- a/pallets/storage-provider/src/lib.rs +++ b/pallets/storage-provider/src/lib.rs @@ -155,5 +155,10 @@ pub mod pallet { pub fn apply_rewards(origin: OriginFor) -> DispatchResultWithPostInfo { todo!() } - } + + // This method is used to report a consensus fault by a miner. + #[pallet::call_index(4)] + pub fn report_consensus_fault(origin: OriginFor) -> DispatchResultWithPostInfo { + todo!() + } } From 4c91a211618520b212be0d66780790d8dcae3880 Mon Sep 17 00:00:00 2001 From: aidan Date: Thu, 30 May 2024 16:05:25 +0200 Subject: [PATCH 07/75] feat: Add `withdraw_balance` function --- pallets/storage-provider/src/lib.rs | 39 +++++++++++++++++++++++------ runtime/src/configs/mod.rs | 2 +- 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/pallets/storage-provider/src/lib.rs b/pallets/storage-provider/src/lib.rs index d5c03c66c..f99ffbd5c 100644 --- a/pallets/storage-provider/src/lib.rs +++ b/pallets/storage-provider/src/lib.rs @@ -15,11 +15,9 @@ use codec::{Decode, Encode}; use scale_info::TypeInfo; #[derive(Decode, Encode, TypeInfo)] -pub struct MinerInfo { +pub struct MinerInfo { /// The owner of this miner. owner: AccountId, - /// Worker of this miner - worker: WorkerAddress, /// Miner's libp2p peer id in bytes. peer_id: PeerId, } @@ -30,11 +28,17 @@ pub mod pallet { use frame_support::dispatch::DispatchResultWithPostInfo; use frame_support::pallet_prelude::{IsType, PhantomData, StorageMap}; + use frame_support::traits::{Currency, ReservableCurrency}; use frame_support::{Blake2_128Concat, Parameter}; - use frame_system::ensure_signed; use frame_system::pallet_prelude::OriginFor; + use frame_system::{ensure_signed, Config as SystemConfig}; use scale_info::prelude::vec::Vec; + // Allows to extract Balance of an account via the Config::Currency associated type. + // BalanceOf is a sophisticated way of getting an u128. + type BalanceOf = + <::Currency as Currency<::AccountId>>::Balance; + /// Peer ID is derived by hashing an encoded public key. /// Usually represented in bytes. /// https://github.com/libp2p/specs/blob/master/peer-ids/peer-ids.md#peer-ids @@ -49,8 +53,10 @@ pub mod pallet { /// Because this pallet emits events, it depends on the runtime's definition of an event. type RuntimeEvent: From> + IsType<::RuntimeEvent>; - /// This pallet handles registration of workers to an account. These workers have an address that is outside of substrate. - type WorkerAddress: Parameter; + /// The currency mechanism. + /// Used for rewards, using `ReservableCurrency` over `Currency` because the rewards will be locked + /// in this pallet until the miner requests the funds through `withdraw_balance` + type Currency: ReservableCurrency; /// MinerAccountId identifies a miner type MinerAccountId: Parameter; @@ -64,7 +70,7 @@ pub mod pallet { _, Blake2_128Concat, T::AccountId, - MinerInfo, + MinerInfo, >; #[pallet::event] @@ -149,16 +155,33 @@ pub mod pallet { // Used by the reward pallet to award a block reward to a Miner. // I am not sure if this should be implemented on this pallet. - // The reward pallet could be tightly coupled with the storage provider pallet + // The reward pallet could be tightly coupled with the storage provider pallet // so the reward pallet can take over this functionality. #[pallet::call_index(3)] pub fn apply_rewards(origin: OriginFor) -> DispatchResultWithPostInfo { + // Check that the extrinsic was signed and get the signer. + let _who = ensure_signed(origin)?; todo!() } // This method is used to report a consensus fault by a miner. #[pallet::call_index(4)] pub fn report_consensus_fault(origin: OriginFor) -> DispatchResultWithPostInfo { + // Check that the extrinsic was signed and get the signer. + let _who = ensure_signed(origin)?; todo!() } + + // Used by the Miner's Owner to withdraw available funds earned from block rewards. + // If the amount to withdraw is larger than what is available the extrinsic will fail. + #[pallet::call_index(5)] + pub fn withdraw_balance( + origin: OriginFor, + _amount: BalanceOf, + ) -> DispatchResultWithPostInfo { + // Check that the extrinsic was signed and get the signer. + let _who = ensure_signed(origin)?; + todo!() + } + } } diff --git a/runtime/src/configs/mod.rs b/runtime/src/configs/mod.rs index dedbbbabf..ca6918866 100644 --- a/runtime/src/configs/mod.rs +++ b/runtime/src/configs/mod.rs @@ -306,6 +306,6 @@ impl pallet_collator_selection::Config for Runtime { impl storage_provider::Config for Runtime { type RuntimeEvent = RuntimeEvent; - type WorkerAddress = (); // Not sure what this should be yet + type Currency = Balances; type MinerAccountId = AccountId; } From 73d3f9894ea3cf79bb95a5bc9ac213bd1ee91699 Mon Sep 17 00:00:00 2001 From: aidan Date: Fri, 31 May 2024 15:31:07 +0200 Subject: [PATCH 08/75] feat: Add logic for change_peer_id In the Storage Provider pallet implement the logic needed to for the `change_peer_id` function. --- Cargo.lock | 1 + pallets/storage-provider/Cargo.toml | 1 + pallets/storage-provider/src/lib.rs | 69 ++++++++++++++++++----------- runtime/src/configs/mod.rs | 1 - 4 files changed, 44 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7cf7e0a6e..f53075843 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13056,6 +13056,7 @@ version = "0.1.0" dependencies = [ "frame-support", "frame-system", + "log", "parity-scale-codec", "scale-info", ] diff --git a/pallets/storage-provider/Cargo.toml b/pallets/storage-provider/Cargo.toml index 6c40ea930..ee9ee84a4 100644 --- a/pallets/storage-provider/Cargo.toml +++ b/pallets/storage-provider/Cargo.toml @@ -13,6 +13,7 @@ version = "0.1.0" codec.workspace = true frame-support = { workspace = true, default-features = false } frame-system = { workspace = true, default-features = false } +log.workspace = true scale-info.workspace = true [lints] diff --git a/pallets/storage-provider/src/lib.rs b/pallets/storage-provider/src/lib.rs index f99ffbd5c..83c6a102b 100644 --- a/pallets/storage-provider/src/lib.rs +++ b/pallets/storage-provider/src/lib.rs @@ -15,9 +15,14 @@ use codec::{Decode, Encode}; use scale_info::TypeInfo; #[derive(Decode, Encode, TypeInfo)] -pub struct MinerInfo { +pub struct MinerInfo< + AccountId: Encode + Decode + Eq + PartialEq, + PeerId: Encode + Decode + Eq + PartialEq, +> { /// The owner of this miner. owner: AccountId, + /// The miner address + miner: AccountId, /// Miner's libp2p peer id in bytes. peer_id: PeerId, } @@ -27,9 +32,9 @@ pub mod pallet { use super::MinerInfo; use frame_support::dispatch::DispatchResultWithPostInfo; + use frame_support::ensure; use frame_support::pallet_prelude::{IsType, PhantomData, StorageMap}; use frame_support::traits::{Currency, ReservableCurrency}; - use frame_support::{Blake2_128Concat, Parameter}; use frame_system::pallet_prelude::OriginFor; use frame_system::{ensure_signed, Config as SystemConfig}; use scale_info::prelude::vec::Vec; @@ -54,24 +59,16 @@ pub mod pallet { type RuntimeEvent: From> + IsType<::RuntimeEvent>; /// The currency mechanism. - /// Used for rewards, using `ReservableCurrency` over `Currency` because the rewards will be locked + /// Used for rewards, using `ReservableCurrency` over `Currency` because the rewards will be locked /// in this pallet until the miner requests the funds through `withdraw_balance` type Currency: ReservableCurrency; - - /// MinerAccountId identifies a miner - type MinerAccountId: Parameter; } // Need some storage type that keeps track of sectors, deadlines and terminations. // Could be added to this type maybe? #[pallet::storage] #[pallet::getter(fn miners)] - pub type Miners = StorageMap< - _, - Blake2_128Concat, - T::AccountId, - MinerInfo, - >; + pub type Miners = StorageMap<_, _, T::AccountId, MinerInfo>; #[pallet::event] #[pallet::generate_deposit(pub(super) fn deposit_event)] @@ -80,15 +77,21 @@ pub mod pallet { owner: T::AccountId, }, PeerIdChanged { - miner: T::MinerAccountId, - peer_id: PeerId, + miner: T::AccountId, + new_peer_id: PeerId, }, OwnerAddressChanged { - miner: T::MinerAccountId, + miner: T::AccountId, new_owner: T::AccountId, }, } + #[pallet::error] + pub enum Error { + MinerNotFound, + InvalidSigner, + } + #[pallet::call] impl Pallet { /// Add a new miner information to `Miners` @@ -113,29 +116,41 @@ pub mod pallet { #[pallet::call_index(1)] pub fn change_peer_id( origin: OriginFor, - miner: T::MinerAccountId, + miner: T::AccountId, peer_id: PeerId, ) -> DispatchResultWithPostInfo { // Check that the extrinsic was signed and get the signer. - let _who = ensure_signed(origin)?; + let who = ensure_signed(origin)?; // Get miner info from `Miners` with `who` value - // let miner_info = Miners::::try_get(&miner); - - // Ensure who is the owner of the miner - // ensure!(who == miner_info.owner) - - // Update PeerId - - Self::deposit_event(Event::PeerIdChanged { miner, peer_id }); - todo!() + Miners::::try_mutate(&miner, |maybe_miner| -> DispatchResultWithPostInfo { + let miner_info = match maybe_miner.as_mut().ok_or(Error::::MinerNotFound) { + Ok(info) => info, + Err(e) => { + log::warn!("Could not get info for miner: {miner:?}"); + return Err(e.into()); + } + }; + + // Ensure who is the owner of the miner + ensure!(who == miner_info.owner, Error::::InvalidSigner); + + log::debug!("Updating peer id for {miner:?}"); + // Update PeerId + miner_info.peer_id = peer_id.clone(); + Self::deposit_event(Event::PeerIdChanged { + miner: miner.clone(), + new_peer_id: peer_id, + }); + Ok(().into()) + }) } // This function updates the owner address to the given `new_owner` for the given `miner` #[pallet::call_index(2)] pub fn change_owner_address( origin: OriginFor, - miner: T::MinerAccountId, + miner: T::AccountId, new_owner: T::AccountId, ) -> DispatchResultWithPostInfo { // Check that the extrinsic was signed and get the signer. diff --git a/runtime/src/configs/mod.rs b/runtime/src/configs/mod.rs index ca6918866..6d63289a0 100644 --- a/runtime/src/configs/mod.rs +++ b/runtime/src/configs/mod.rs @@ -307,5 +307,4 @@ impl pallet_collator_selection::Config for Runtime { impl storage_provider::Config for Runtime { type RuntimeEvent = RuntimeEvent; type Currency = Balances; - type MinerAccountId = AccountId; } From be4d31c9b4297f094c3ddfaa6a95deae47a1fde9 Mon Sep 17 00:00:00 2001 From: aidan Date: Fri, 31 May 2024 16:04:24 +0200 Subject: [PATCH 09/75] feat: Add logic for change_owner_address In the Storage Provider pallet implement the logic needed to for the `change_owner_address` function. --- pallets/storage-provider/src/lib.rs | 36 ++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/pallets/storage-provider/src/lib.rs b/pallets/storage-provider/src/lib.rs index 83c6a102b..3dcd8c941 100644 --- a/pallets/storage-provider/src/lib.rs +++ b/pallets/storage-provider/src/lib.rs @@ -122,9 +122,8 @@ pub mod pallet { // Check that the extrinsic was signed and get the signer. let who = ensure_signed(origin)?; - // Get miner info from `Miners` with `who` value - Miners::::try_mutate(&miner, |maybe_miner| -> DispatchResultWithPostInfo { - let miner_info = match maybe_miner.as_mut().ok_or(Error::::MinerNotFound) { + Miners::::try_mutate(&miner, |info| -> DispatchResultWithPostInfo { + let miner_info = match info.as_mut().ok_or(Error::::MinerNotFound) { Ok(info) => info, Err(e) => { log::warn!("Could not get info for miner: {miner:?}"); @@ -136,8 +135,10 @@ pub mod pallet { ensure!(who == miner_info.owner, Error::::InvalidSigner); log::debug!("Updating peer id for {miner:?}"); + // Update PeerId miner_info.peer_id = peer_id.clone(); + Self::deposit_event(Event::PeerIdChanged { miner: miner.clone(), new_peer_id: peer_id, @@ -154,18 +155,31 @@ pub mod pallet { new_owner: T::AccountId, ) -> DispatchResultWithPostInfo { // Check that the extrinsic was signed and get the signer. - let _who = ensure_signed(origin)?; + let who = ensure_signed(origin)?; + + Miners::::try_mutate(&miner, |info| -> DispatchResultWithPostInfo { + let miner_info = match info.as_mut().ok_or(Error::::MinerNotFound) { + Ok(info) => info, + Err(e) => { + log::warn!("Could not get info for miner: {miner:?}"); + return Err(e.into()); + } + }; - // Get miner info from `Miners` with `who` value - // let miner_info = Miners::::try_get(&miner); + // Ensure who is the owner of the miner + ensure!(who == miner_info.owner, Error::::InvalidSigner); - // Ensure who is the owner of the miner - // ensure!(who == miner_info.owner) + log::debug!("Updating owner for {miner:?}"); - // Change owner address + // Update owner address + miner_info.owner = new_owner.clone(); - Self::deposit_event(Event::OwnerAddressChanged { miner, new_owner }); - todo!() + Self::deposit_event(Event::OwnerAddressChanged { + miner: miner.clone(), + new_owner, + }); + Ok(().into()) + }) } // Used by the reward pallet to award a block reward to a Miner. From 1a95bc7f7e430c52199b7d16285271e8ac5b6eb9 Mon Sep 17 00:00:00 2001 From: aidan Date: Mon, 3 Jun 2024 13:56:02 +0200 Subject: [PATCH 10/75] docs: Add README and DESIGN markdown --- pallets/storage-provider/DESIGN.md | 38 ++++++++++++++++++++++++++++++ pallets/storage-provider/README.md | 1 + 2 files changed, 39 insertions(+) create mode 100644 pallets/storage-provider/DESIGN.md create mode 100644 pallets/storage-provider/README.md diff --git a/pallets/storage-provider/DESIGN.md b/pallets/storage-provider/DESIGN.md new file mode 100644 index 000000000..02aa9eef8 --- /dev/null +++ b/pallets/storage-provider/DESIGN.md @@ -0,0 +1,38 @@ +# Storage Provider Pallet + +## Overview + +The `Storage Provider Pallet` handles the creation of storage providers and facilitates storage providers and client in creating storage deals. + +## Usage + +### Indexing storage providers + +A storage provider indexes in the storage provider pallet itself when it starts up by calling the `create_storage_provider` extrinsic with it's `PeerId` as an argument. The public key will be extracted from the origin and is used to modify on-chain information and receive rewards. The `PeerId` is given by the storage provider so clients can use that to connect to the storage provider. + +### Modifying storage provider information + +The `Storage Provider Pallet` allows storage providers to modify their information such as changing the peer id, through `change_peer_id` and changing owners, through `change_owner_address`. + +## Data structures + +```rust +pub struct StorageProviderInfo< + AccountId: Encode + Decode + Eq + PartialEq, + PeerId: Encode + Decode + Eq + PartialEq, +> { + /// The owner of this storage provider. + owner: AccountId, + /// Storage provider'ss libp2p peer id in bytes. + peer_id: PeerId, +} +``` + +The `StorageProviderInfo` structure holds information about a `StorageProvider`. + +```rust +pub type StorageProviders = + StorageMap<_, _, T::AccountId, StorageProviderInfo>; +``` + +The `StorageProviders` mapping `AccountId`'s to `PeerId`'s. diff --git a/pallets/storage-provider/README.md b/pallets/storage-provider/README.md new file mode 100644 index 000000000..f7be98816 --- /dev/null +++ b/pallets/storage-provider/README.md @@ -0,0 +1 @@ +# Storage Provider Pallet From f08c3914ec1b5fae3d1229173973f26ee3e029be Mon Sep 17 00:00:00 2001 From: aidan Date: Mon, 3 Jun 2024 13:57:11 +0200 Subject: [PATCH 11/75] Rename `miner` to `storage provider` --- pallets/storage-provider/src/lib.rs | 180 +++++++++++++++------------- runtime/src/configs/mod.rs | 2 + 2 files changed, 100 insertions(+), 82 deletions(-) diff --git a/pallets/storage-provider/src/lib.rs b/pallets/storage-provider/src/lib.rs index 3dcd8c941..bc3ad6cd0 100644 --- a/pallets/storage-provider/src/lib.rs +++ b/pallets/storage-provider/src/lib.rs @@ -1,7 +1,7 @@ //! # Storage Provider Pallet //! //! This pallet holds information about storage providers and -//! provides an interface to modify information about miners. +//! provides an interface to modify information about Storage Providers. //! //! The Storage Provider Pallet is the source of truth for anything storage provider (the binary) related. //! @@ -15,40 +15,35 @@ use codec::{Decode, Encode}; use scale_info::TypeInfo; #[derive(Decode, Encode, TypeInfo)] -pub struct MinerInfo< +pub struct StorageProviderInfo< AccountId: Encode + Decode + Eq + PartialEq, PeerId: Encode + Decode + Eq + PartialEq, > { - /// The owner of this miner. + /// The owner of this storage_provider. owner: AccountId, - /// The miner address - miner: AccountId, - /// Miner's libp2p peer id in bytes. + /// storage_provider's libp2p peer id in bytes. peer_id: PeerId, } #[frame_support::pallet(dev_mode)] pub mod pallet { - use super::MinerInfo; + use super::StorageProviderInfo; + use codec::{Decode, Encode}; + use core::fmt::Debug; use frame_support::dispatch::DispatchResultWithPostInfo; use frame_support::ensure; use frame_support::pallet_prelude::{IsType, PhantomData, StorageMap}; use frame_support::traits::{Currency, ReservableCurrency}; use frame_system::pallet_prelude::OriginFor; use frame_system::{ensure_signed, Config as SystemConfig}; - use scale_info::prelude::vec::Vec; + use scale_info::TypeInfo; // Allows to extract Balance of an account via the Config::Currency associated type. // BalanceOf is a sophisticated way of getting an u128. type BalanceOf = <::Currency as Currency<::AccountId>>::Balance; - /// Peer ID is derived by hashing an encoded public key. - /// Usually represented in bytes. - /// https://github.com/libp2p/specs/blob/master/peer-ids/peer-ids.md#peer-ids - type PeerId = Vec; - #[pallet::pallet] #[pallet::without_storage_info] // Allows to define storage items without fixed size pub struct Pallet(PhantomData); @@ -60,129 +55,150 @@ pub mod pallet { /// The currency mechanism. /// Used for rewards, using `ReservableCurrency` over `Currency` because the rewards will be locked - /// in this pallet until the miner requests the funds through `withdraw_balance` + /// in this pallet until the storage provider requests the funds through `withdraw_balance` type Currency: ReservableCurrency; + + /// Peer ID is derived by hashing an encoded public key. + /// Usually represented in bytes. + /// https://github.com/libp2p/specs/blob/master/peer-ids/peer-ids.md#peer-ids + type PeerId: Clone + Debug + Decode + Encode + Eq + TypeInfo; } // Need some storage type that keeps track of sectors, deadlines and terminations. // Could be added to this type maybe? #[pallet::storage] - #[pallet::getter(fn miners)] - pub type Miners = StorageMap<_, _, T::AccountId, MinerInfo>; + #[pallet::getter(fn storage_providers)] + pub type StorageProviders = + StorageMap<_, _, T::AccountId, StorageProviderInfo>; #[pallet::event] #[pallet::generate_deposit(pub(super) fn deposit_event)] pub enum Event { - MinerCreated { + StorageProviderCreated { owner: T::AccountId, }, PeerIdChanged { - miner: T::AccountId, - new_peer_id: PeerId, + storage_provider: T::AccountId, + new_peer_id: T::PeerId, }, OwnerAddressChanged { - miner: T::AccountId, + storage_provider: T::AccountId, new_owner: T::AccountId, }, } #[pallet::error] pub enum Error { - MinerNotFound, + StorageProviderNotFound, InvalidSigner, } #[pallet::call] impl Pallet { - /// Add a new miner information to `Miners` + /// Add a new storage_provider information to `StorageProviders` #[pallet::call_index(0)] - pub fn create_miner( + pub fn create_storage_provider( origin: OriginFor, - owner: T::AccountId, - _peer_id: PeerId, + _peer_id: T::PeerId, ) -> DispatchResultWithPostInfo { // Check that the extrinsic was signed and get the signer. let _who = ensure_signed(origin)?; - // Generate some miner id and insert into `Miners` storage + // Generate some storage_provider id and insert into `StorageProviders` storage - // This probably inherits a `create_miner` function from a `Power` trait. + // This probably inherits a `create_storage_provider` function from a `Power` trait. - Self::deposit_event(Event::MinerCreated { owner }); + Self::deposit_event(Event::StorageProviderCreated { owner }); todo!() } - /// Update PeerId associated with a given miner. + /// Update PeerId associated with a given storage_provider. #[pallet::call_index(1)] pub fn change_peer_id( origin: OriginFor, - miner: T::AccountId, - peer_id: PeerId, + peer_id: T::PeerId, ) -> DispatchResultWithPostInfo { // Check that the extrinsic was signed and get the signer. let who = ensure_signed(origin)?; - Miners::::try_mutate(&miner, |info| -> DispatchResultWithPostInfo { - let miner_info = match info.as_mut().ok_or(Error::::MinerNotFound) { - Ok(info) => info, - Err(e) => { - log::warn!("Could not get info for miner: {miner:?}"); - return Err(e.into()); - } - }; - - // Ensure who is the owner of the miner - ensure!(who == miner_info.owner, Error::::InvalidSigner); - - log::debug!("Updating peer id for {miner:?}"); - - // Update PeerId - miner_info.peer_id = peer_id.clone(); - - Self::deposit_event(Event::PeerIdChanged { - miner: miner.clone(), - new_peer_id: peer_id, - }); - Ok(().into()) - }) + StorageProviders::::try_mutate( + &storage_provider, + |info| -> DispatchResultWithPostInfo { + let storage_provider_info = + match info.as_mut().ok_or(Error::::StorageProviderNotFound) { + Ok(info) => info, + Err(e) => { + log::warn!( + "Could not get info for storage_provider: {storage_provider:?}" + ); + return Err(e.into()); + } + }; + + // Ensure who is the owner of the storage_provider + ensure!( + who == storage_provider_info.owner, + Error::::InvalidSigner + ); + + log::debug!("Updating peer id for {storage_provider:?}"); + + // Update PeerId + storage_provider_info.peer_id = peer_id.clone(); + + Self::deposit_event(Event::PeerIdChanged { + storage_provider: storage_provider.clone(), + new_peer_id: peer_id, + }); + Ok(().into()) + }, + ) } - // This function updates the owner address to the given `new_owner` for the given `miner` + // This function updates the owner address to the given `new_owner` for the given `storage_provider` #[pallet::call_index(2)] pub fn change_owner_address( origin: OriginFor, - miner: T::AccountId, new_owner: T::AccountId, ) -> DispatchResultWithPostInfo { // Check that the extrinsic was signed and get the signer. let who = ensure_signed(origin)?; - Miners::::try_mutate(&miner, |info| -> DispatchResultWithPostInfo { - let miner_info = match info.as_mut().ok_or(Error::::MinerNotFound) { - Ok(info) => info, - Err(e) => { - log::warn!("Could not get info for miner: {miner:?}"); - return Err(e.into()); - } - }; - - // Ensure who is the owner of the miner - ensure!(who == miner_info.owner, Error::::InvalidSigner); - - log::debug!("Updating owner for {miner:?}"); - - // Update owner address - miner_info.owner = new_owner.clone(); - - Self::deposit_event(Event::OwnerAddressChanged { - miner: miner.clone(), - new_owner, - }); - Ok(().into()) - }) + StorageProviders::::try_mutate( + &storage_provider, + |info| -> DispatchResultWithPostInfo { + let storage_provider_info = + match info.as_mut().ok_or(Error::::StorageProviderNotFound) { + Ok(info) => info, + Err(e) => { + log::warn!( + "Could not get info for storage_provider: {storage_provider:?}" + ); + return Err(e.into()); + } + }; + + // Ensure who is the owner of the storage_provider + ensure!( + who == storage_provider_info.owner, + Error::::InvalidSigner + ); + + log::debug!("Updating owner for {storage_provider:?}"); + + // Update owner address + storage_provider_info.owner = new_owner.clone(); + + Self::deposit_event(Event::OwnerAddressChanged { + storage_provider: storage_provider.clone(), + new_owner, + }); + Ok(().into()) + }, + ) } - // Used by the reward pallet to award a block reward to a Miner. + // Used by the reward pallet to award a block reward to a storage_provider. // I am not sure if this should be implemented on this pallet. // The reward pallet could be tightly coupled with the storage provider pallet // so the reward pallet can take over this functionality. @@ -193,7 +209,7 @@ pub mod pallet { todo!() } - // This method is used to report a consensus fault by a miner. + // This method is used to report a consensus fault by a storage_provider. #[pallet::call_index(4)] pub fn report_consensus_fault(origin: OriginFor) -> DispatchResultWithPostInfo { // Check that the extrinsic was signed and get the signer. @@ -201,7 +217,7 @@ pub mod pallet { todo!() } - // Used by the Miner's Owner to withdraw available funds earned from block rewards. + // Used by the storage_provider's Owner to withdraw available funds earned from block rewards. // If the amount to withdraw is larger than what is available the extrinsic will fail. #[pallet::call_index(5)] pub fn withdraw_balance( diff --git a/runtime/src/configs/mod.rs b/runtime/src/configs/mod.rs index 6d63289a0..d64eb8ffd 100644 --- a/runtime/src/configs/mod.rs +++ b/runtime/src/configs/mod.rs @@ -45,6 +45,7 @@ use parachains_common::message_queue::{NarrowOriginToSibling, ParaIdToSibling}; use polkadot_runtime_common::{ xcm_sender::NoPriceForMessageDelivery, BlockHashCount, SlowAdjustingFeeUpdate, }; +use scale_info::prelude::vec::Vec; use sp_consensus_aura::sr25519::AuthorityId as AuraId; use sp_runtime::Perbill; use sp_version::RuntimeVersion; @@ -307,4 +308,5 @@ impl pallet_collator_selection::Config for Runtime { impl storage_provider::Config for Runtime { type RuntimeEvent = RuntimeEvent; type Currency = Balances; + type PeerId = Vec; } From 86adf708c6f4ca08bdf2f8dd6370d738a4603a9a Mon Sep 17 00:00:00 2001 From: aidan Date: Mon, 3 Jun 2024 15:01:34 +0200 Subject: [PATCH 12/75] feat(storage-provider): Implement pallet functions Implement `create_storage_provider`, `change_peer_id` and `change_owner_address` functions in the Storage Provider pallet. --- pallets/storage-provider/src/lib.rs | 74 ++++++++++++++++------------- 1 file changed, 42 insertions(+), 32 deletions(-) diff --git a/pallets/storage-provider/src/lib.rs b/pallets/storage-provider/src/lib.rs index bc3ad6cd0..301bdf10f 100644 --- a/pallets/storage-provider/src/lib.rs +++ b/pallets/storage-provider/src/lib.rs @@ -91,6 +91,7 @@ pub mod pallet { pub enum Error { StorageProviderNotFound, InvalidSigner, + StorageProviderExists, } #[pallet::call] @@ -99,17 +100,25 @@ pub mod pallet { #[pallet::call_index(0)] pub fn create_storage_provider( origin: OriginFor, - _peer_id: T::PeerId, + peer_id: T::PeerId, ) -> DispatchResultWithPostInfo { // Check that the extrinsic was signed and get the signer. - let _who = ensure_signed(origin)?; + let owner = ensure_signed(origin)?; // Generate some storage_provider id and insert into `StorageProviders` storage - - // This probably inherits a `create_storage_provider` function from a `Power` trait. - + let storage_provider_info = StorageProviderInfo { + owner: owner.clone(), + peer_id: peer_id.clone(), + }; + // Probably need some check to make sure the storage provider is legit + // This means the storage provider exist + ensure!( + !StorageProviders::::contains_key(&owner), + Error::::StorageProviderExists + ); + StorageProviders::::insert(owner.clone(), storage_provider_info); Self::deposit_event(Event::StorageProviderCreated { owner }); - todo!() + Ok(().into()) } /// Update PeerId associated with a given storage_provider. @@ -119,7 +128,7 @@ pub mod pallet { peer_id: T::PeerId, ) -> DispatchResultWithPostInfo { // Check that the extrinsic was signed and get the signer. - let who = ensure_signed(origin)?; + let storage_provider = ensure_signed(origin)?; StorageProviders::::try_mutate( &storage_provider, @@ -128,16 +137,16 @@ pub mod pallet { match info.as_mut().ok_or(Error::::StorageProviderNotFound) { Ok(info) => info, Err(e) => { - log::warn!( + log::error!( "Could not get info for storage_provider: {storage_provider:?}" ); return Err(e.into()); } }; - // Ensure who is the owner of the storage_provider + // Ensure storage_provider is the owner of the storage_provider ensure!( - who == storage_provider_info.owner, + storage_provider == storage_provider_info.owner, Error::::InvalidSigner ); @@ -162,40 +171,41 @@ pub mod pallet { new_owner: T::AccountId, ) -> DispatchResultWithPostInfo { // Check that the extrinsic was signed and get the signer. - let who = ensure_signed(origin)?; + let storage_provider = ensure_signed(origin)?; - StorageProviders::::try_mutate( - &storage_provider, - |info| -> DispatchResultWithPostInfo { - let storage_provider_info = - match info.as_mut().ok_or(Error::::StorageProviderNotFound) { - Ok(info) => info, - Err(e) => { - log::warn!( - "Could not get info for storage_provider: {storage_provider:?}" - ); - return Err(e.into()); - } - }; + // Extract storage provider + match StorageProviders::::try_get(&storage_provider) { + Ok(info) => { + // Ensure storage_provider is the owner of the storage_provider + ensure!(storage_provider == info.owner, Error::::InvalidSigner); - // Ensure who is the owner of the storage_provider + let new_info = StorageProviderInfo { + owner: new_owner.clone(), + peer_id: info.peer_id, + }; + + // Ensure no storage provider is associated with the new owner ensure!( - who == storage_provider_info.owner, - Error::::InvalidSigner + !StorageProviders::::contains_key(&new_owner), + Error::::StorageProviderExists ); - log::debug!("Updating owner for {storage_provider:?}"); + // Insert new storage provider info + StorageProviders::::insert(new_owner.clone(), new_info); - // Update owner address - storage_provider_info.owner = new_owner.clone(); + // Remove old storage provider entry + StorageProviders::::remove(storage_provider.clone()); + // Emit event Self::deposit_event(Event::OwnerAddressChanged { storage_provider: storage_provider.clone(), new_owner, }); + Ok(().into()) - }, - ) + } + Err(..) => Err(Error::::StorageProviderNotFound.into()), + } } // Used by the reward pallet to award a block reward to a storage_provider. From 858a79d6f3d9e7d773eae8760e9ae3573337b3a2 Mon Sep 17 00:00:00 2001 From: aidan Date: Mon, 3 Jun 2024 15:04:32 +0200 Subject: [PATCH 13/75] docs: Update comments in storage provider pallet --- pallets/storage-provider/src/lib.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pallets/storage-provider/src/lib.rs b/pallets/storage-provider/src/lib.rs index 301bdf10f..fd192ca34 100644 --- a/pallets/storage-provider/src/lib.rs +++ b/pallets/storage-provider/src/lib.rs @@ -96,7 +96,7 @@ pub mod pallet { #[pallet::call] impl Pallet { - /// Add a new storage_provider information to `StorageProviders` + /// Index a new storage provider #[pallet::call_index(0)] pub fn create_storage_provider( origin: OriginFor, @@ -121,7 +121,7 @@ pub mod pallet { Ok(().into()) } - /// Update PeerId associated with a given storage_provider. + /// Update PeerId for a Storage Provider. #[pallet::call_index(1)] pub fn change_peer_id( origin: OriginFor, @@ -164,7 +164,7 @@ pub mod pallet { ) } - // This function updates the owner address to the given `new_owner` for the given `storage_provider` + /// Update the owner address for a Storage Provider. #[pallet::call_index(2)] pub fn change_owner_address( origin: OriginFor, From 1e5a4dda38d4a616bf3819a23104213b2696656e Mon Sep 17 00:00:00 2001 From: aidan Date: Tue, 4 Jun 2024 11:44:02 +0200 Subject: [PATCH 14/75] refactor: Rename storage provider pallet --- Cargo.lock | 24 ++++++++++++------------ Cargo.toml | 2 +- pallets/storage-provider/Cargo.toml | 2 +- runtime/Cargo.toml | 2 +- runtime/src/configs/mod.rs | 2 +- runtime/src/lib.rs | 2 +- 6 files changed, 17 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f4f127ab3..87c9a5701 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7322,6 +7322,17 @@ dependencies = [ "sp-std 14.0.0 (git+https://github.com/paritytech/polkadot-sdk?tag=polkadot-v1.11.0)", ] +[[package]] +name = "pallet-storage-provider" +version = "0.1.0" +dependencies = [ + "frame-support", + "frame-system", + "log", + "parity-scale-codec", + "scale-info", +] + [[package]] name = "pallet-sudo" version = "28.0.0" @@ -7964,6 +7975,7 @@ dependencies = [ "pallet-collator-selection", "pallet-message-queue", "pallet-session", + "pallet-storage-provider", "pallet-sudo", "pallet-timestamp", "pallet-transaction-payment", @@ -7991,7 +8003,6 @@ dependencies = [ "staging-xcm", "staging-xcm-builder", "staging-xcm-executor", - "storage-provider", "substrate-wasm-builder", ] @@ -13217,17 +13228,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "storage-provider" -version = "0.1.0" -dependencies = [ - "frame-support", - "frame-system", - "log", - "parity-scale-codec", - "scale-info", -] - [[package]] name = "str0m" version = "0.2.0" diff --git a/Cargo.toml b/Cargo.toml index 9a98bc7f6..2b88f2aaf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,8 +72,8 @@ tokio-util = "0.7.11" tracing-subscriber = { version = "0.3.18" } # Local +pallet-storage-provider = { path = "pallets/storage-provider" } polka-storage-runtime = { path = "runtime" } -storage-provider = { path = "pallets/storage-provider" } # Substrate pallet-transaction-payment-rpc = { git = "https://github.com/paritytech/polkadot-sdk", tag = "polkadot-v1.11.0" } diff --git a/pallets/storage-provider/Cargo.toml b/pallets/storage-provider/Cargo.toml index ee9ee84a4..6b1c820ab 100644 --- a/pallets/storage-provider/Cargo.toml +++ b/pallets/storage-provider/Cargo.toml @@ -3,7 +3,7 @@ authors.workspace = true edition.workspace = true homepage.workspace = true license-file.workspace = true -name = "storage-provider" +name = "pallet-storage-provider" repository.workspace = true version = "0.1.0" diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index 8dfdec629..2170b4d60 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -20,7 +20,7 @@ substrate-wasm-builder = { workspace = true, optional = true } [dependencies] # Pallets -storage-provider.workspace = true +pallet-storage-provider.workspace = true codec = { workspace = true, default-features = false, features = ["derive"] } hex-literal = { workspace = true, optional = true } diff --git a/runtime/src/configs/mod.rs b/runtime/src/configs/mod.rs index d64eb8ffd..616e27782 100644 --- a/runtime/src/configs/mod.rs +++ b/runtime/src/configs/mod.rs @@ -305,7 +305,7 @@ impl pallet_collator_selection::Config for Runtime { type WeightInfo = (); } -impl storage_provider::Config for Runtime { +impl pallet_storage_provider::Config for Runtime { type RuntimeEvent = RuntimeEvent; type Currency = Balances; type PeerId = Vec; diff --git a/runtime/src/lib.rs b/runtime/src/lib.rs index 6492bdfe9..7516207b5 100644 --- a/runtime/src/lib.rs +++ b/runtime/src/lib.rs @@ -245,7 +245,7 @@ construct_runtime!( PolkadotXcm: pallet_xcm = 31, CumulusXcm: cumulus_pallet_xcm = 32, MessageQueue: pallet_message_queue = 33, - StorageProvider: storage_provider::pallet = 34, + StorageProvider: pallet_storage_provider::pallet = 34, } ); From 3cca71d4ead486fc49ded772e58facd4bb8ddc30 Mon Sep 17 00:00:00 2001 From: aidan Date: Tue, 4 Jun 2024 12:39:16 +0200 Subject: [PATCH 15/75] feat(storage-provider): Add feature flags --- Cargo.lock | 6 +++- Cargo.toml | 2 +- Justfile | 2 +- pallets/storage-provider/Cargo.toml | 43 ++++++++++++++++++++++++----- runtime/Cargo.toml | 5 +++- 5 files changed, 47 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 87c9a5701..327748eab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7324,13 +7324,17 @@ dependencies = [ [[package]] name = "pallet-storage-provider" -version = "0.1.0" +version = "0.0.0" dependencies = [ + "frame-benchmarking", "frame-support", "frame-system", "log", "parity-scale-codec", "scale-info", + "sp-core", + "sp-io", + "sp-runtime", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 2b88f2aaf..a514b8bfc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,7 +72,7 @@ tokio-util = "0.7.11" tracing-subscriber = { version = "0.3.18" } # Local -pallet-storage-provider = { path = "pallets/storage-provider" } +pallet-storage-provider = { path = "pallets/storage-provider", default-features = false } polka-storage-runtime = { path = "runtime" } # Substrate diff --git a/Justfile b/Justfile index 69bdeea3d..57418e685 100644 --- a/Justfile +++ b/Justfile @@ -3,7 +3,7 @@ alias r := release alias t := testnet lint: - cargo clippy --locked --no-deps -- -D warnings + cargo clippy --no-deps -- -D warnings taplo lint && taplo fmt --check build: lint diff --git a/pallets/storage-provider/Cargo.toml b/pallets/storage-provider/Cargo.toml index 6b1c820ab..1aed701eb 100644 --- a/pallets/storage-provider/Cargo.toml +++ b/pallets/storage-provider/Cargo.toml @@ -5,16 +5,45 @@ homepage.workspace = true license-file.workspace = true name = "pallet-storage-provider" repository.workspace = true -version = "0.1.0" +version = "0.0.0" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[lints] +workspace = true + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec.workspace = true +codec = { workspace = true, default-features = false, features = ["derive"] } +log.workspace = true +scale-info = { workspace = true, default-features = false, features = ["derive"] } + +# Frame deps +frame-benchmarking = { workspace = true, default-features = false, optional = true } frame-support = { workspace = true, default-features = false } frame-system = { workspace = true, default-features = false } -log.workspace = true -scale-info.workspace = true -[lints] -workspace = true +[dev-dependencies] +sp-core = { workspace = true, default-features = false } +sp-io = { workspace = true } +sp-runtime = { workspace = true, default-features = false } + +[features] +default = ["std"] +runtime-benchmarks = [ + "frame-benchmarking/runtime-benchmarks", + "frame-support/runtime-benchmarks", + "frame-system/runtime-benchmarks", + "sp-runtime/runtime-benchmarks", +] +std = [ + "codec/std", + "frame-benchmarking?/std", + "frame-support/std", + "frame-system/std", + "scale-info/std", + "sp-core/std", + "sp-io/std", + "sp-runtime/std", +] +try-runtime = ["frame-support/try-runtime", "frame-system/try-runtime", "sp-runtime/try-runtime"] diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index 2170b4d60..53d4e5c83 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -20,7 +20,7 @@ substrate-wasm-builder = { workspace = true, optional = true } [dependencies] # Pallets -pallet-storage-provider.workspace = true +pallet-storage-provider = { workspace = true, default-features = false } codec = { workspace = true, default-features = false, features = ["derive"] } hex-literal = { workspace = true, optional = true } @@ -109,6 +109,7 @@ std = [ "pallet-collator-selection/std", "pallet-message-queue/std", "pallet-session/std", + "pallet-storage-provider/std", "pallet-sudo/std", "pallet-timestamp/std", "pallet-transaction-payment-rpc-runtime-api/std", @@ -151,6 +152,7 @@ runtime-benchmarks = [ "pallet-balances/runtime-benchmarks", "pallet-collator-selection/runtime-benchmarks", "pallet-message-queue/runtime-benchmarks", + "pallet-storage-provider/runtime-benchmarks", "pallet-sudo/runtime-benchmarks", "pallet-timestamp/runtime-benchmarks", "pallet-xcm/runtime-benchmarks", @@ -177,6 +179,7 @@ try-runtime = [ "pallet-collator-selection/try-runtime", "pallet-message-queue/try-runtime", "pallet-session/try-runtime", + "pallet-storage-provider/try-runtime", "pallet-sudo/try-runtime", "pallet-timestamp/try-runtime", "pallet-transaction-payment/try-runtime", From 84b47702f2ea0464fab538df47934fda7fae47cd Mon Sep 17 00:00:00 2001 From: aidan Date: Tue, 4 Jun 2024 12:54:17 +0200 Subject: [PATCH 16/75] refactor: Remove `PhantomData` --- pallets/storage-provider/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pallets/storage-provider/src/lib.rs b/pallets/storage-provider/src/lib.rs index fd192ca34..4c33e7fba 100644 --- a/pallets/storage-provider/src/lib.rs +++ b/pallets/storage-provider/src/lib.rs @@ -33,7 +33,7 @@ pub mod pallet { use core::fmt::Debug; use frame_support::dispatch::DispatchResultWithPostInfo; use frame_support::ensure; - use frame_support::pallet_prelude::{IsType, PhantomData, StorageMap}; + use frame_support::pallet_prelude::{IsType, StorageMap}; use frame_support::traits::{Currency, ReservableCurrency}; use frame_system::pallet_prelude::OriginFor; use frame_system::{ensure_signed, Config as SystemConfig}; @@ -46,7 +46,7 @@ pub mod pallet { #[pallet::pallet] #[pallet::without_storage_info] // Allows to define storage items without fixed size - pub struct Pallet(PhantomData); + pub struct Pallet(_); #[pallet::config] pub trait Config: frame_system::Config { From 60afc4c439dd79e6f20ec794843efaabbcfeac67 Mon Sep 17 00:00:00 2001 From: aidan Date: Tue, 4 Jun 2024 13:37:17 +0200 Subject: [PATCH 17/75] docs(storage-provider): Document error values --- pallets/storage-provider/src/lib.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pallets/storage-provider/src/lib.rs b/pallets/storage-provider/src/lib.rs index 4c33e7fba..5c0e2abb0 100644 --- a/pallets/storage-provider/src/lib.rs +++ b/pallets/storage-provider/src/lib.rs @@ -89,9 +89,12 @@ pub mod pallet { #[pallet::error] pub enum Error { + /// Error emitted when trying to get info on a storage provider that does not exist. StorageProviderNotFound, + /// Error emitted when doing a privileged call and the signer does not match. InvalidSigner, - StorageProviderExists, + /// Error emitted when trying to create a storage provider that is already indexed. + DuplicateStorageProvider, } #[pallet::call] @@ -114,7 +117,7 @@ pub mod pallet { // This means the storage provider exist ensure!( !StorageProviders::::contains_key(&owner), - Error::::StorageProviderExists + Error::::DuplicateStorageProvider ); StorageProviders::::insert(owner.clone(), storage_provider_info); Self::deposit_event(Event::StorageProviderCreated { owner }); @@ -187,7 +190,7 @@ pub mod pallet { // Ensure no storage provider is associated with the new owner ensure!( !StorageProviders::::contains_key(&new_owner), - Error::::StorageProviderExists + Error::::DuplicateStorageProvider ); // Insert new storage provider info From f1c89cd9e0cc3bceb485f898c6dc12d733bd278a Mon Sep 17 00:00:00 2001 From: aidan Date: Tue, 4 Jun 2024 13:41:06 +0200 Subject: [PATCH 18/75] style: Align todo's to agreed practice --- pallets/storage-provider/src/lib.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pallets/storage-provider/src/lib.rs b/pallets/storage-provider/src/lib.rs index 5c0e2abb0..b44a0a1b3 100644 --- a/pallets/storage-provider/src/lib.rs +++ b/pallets/storage-provider/src/lib.rs @@ -219,7 +219,8 @@ pub mod pallet { pub fn apply_rewards(origin: OriginFor) -> DispatchResultWithPostInfo { // Check that the extrinsic was signed and get the signer. let _who = ensure_signed(origin)?; - todo!() + // TODO(@aidan46, no-ref, 2024-06-04): Implement apply rewards functionality + unimplemented!("Apply rewards is not implemented yet") } // This method is used to report a consensus fault by a storage_provider. @@ -227,7 +228,8 @@ pub mod pallet { pub fn report_consensus_fault(origin: OriginFor) -> DispatchResultWithPostInfo { // Check that the extrinsic was signed and get the signer. let _who = ensure_signed(origin)?; - todo!() + // TODO(@aidan46, no-ref, 2024-06-04): Implement report consensus fault functionality + unimplemented!("Report consensus fault is not implemented yet") } // Used by the storage_provider's Owner to withdraw available funds earned from block rewards. @@ -239,7 +241,8 @@ pub mod pallet { ) -> DispatchResultWithPostInfo { // Check that the extrinsic was signed and get the signer. let _who = ensure_signed(origin)?; - todo!() + // TODO(@aidan46, no-ref, 2024-06-04): Implement withdraw balance functionality + unimplemented!("Withdraw balance is not implemented yet") } } } From 69c8cc229d6c1819d582d2740a075f26b0c197f6 Mon Sep 17 00:00:00 2001 From: aidan Date: Tue, 4 Jun 2024 13:43:12 +0200 Subject: [PATCH 19/75] fix(storage-provider): Add TODO to README --- pallets/storage-provider/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pallets/storage-provider/README.md b/pallets/storage-provider/README.md index f7be98816..687734315 100644 --- a/pallets/storage-provider/README.md +++ b/pallets/storage-provider/README.md @@ -1 +1,3 @@ # Storage Provider Pallet + +TODO(aidan46, no-ref, 2024-06-04): Add README docs for storage provider \ No newline at end of file From fdcdf58eee60032588e9d3139331e479b35de686 Mon Sep 17 00:00:00 2001 From: aidan Date: Tue, 4 Jun 2024 13:57:09 +0200 Subject: [PATCH 20/75] fix: Add locked flag to clippy in Justfile --- Justfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Justfile b/Justfile index 57418e685..69bdeea3d 100644 --- a/Justfile +++ b/Justfile @@ -3,7 +3,7 @@ alias r := release alias t := testnet lint: - cargo clippy --no-deps -- -D warnings + cargo clippy --locked --no-deps -- -D warnings taplo lint && taplo fmt --check build: lint From 05c2f7c2970d1f8ee4fb7d6bd51f2a3ddebb0b6c Mon Sep 17 00:00:00 2001 From: aidan Date: Tue, 4 Jun 2024 14:16:32 +0200 Subject: [PATCH 21/75] feat: Add scaffolding for storage provider benchmarks --- pallets/storage-provider/README.md | 2 +- pallets/storage-provider/src/benchmarks.rs | 7 +++++++ pallets/storage-provider/src/lib.rs | 3 +++ 3 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 pallets/storage-provider/src/benchmarks.rs diff --git a/pallets/storage-provider/README.md b/pallets/storage-provider/README.md index 687734315..124b30447 100644 --- a/pallets/storage-provider/README.md +++ b/pallets/storage-provider/README.md @@ -1,3 +1,3 @@ # Storage Provider Pallet -TODO(aidan46, no-ref, 2024-06-04): Add README docs for storage provider \ No newline at end of file +TODO(aidan46, no-ref, 2024-06-04): Add README docs for storage provider diff --git a/pallets/storage-provider/src/benchmarks.rs b/pallets/storage-provider/src/benchmarks.rs new file mode 100644 index 000000000..aa70b0c67 --- /dev/null +++ b/pallets/storage-provider/src/benchmarks.rs @@ -0,0 +1,7 @@ +//! Benchmarking setup for pallet-storage-provider +#![cfg(feature = "runtime-benchmarks")] + +#[benchmarks] +mod benchmarks { + // TODO(aidan46, no-ref, 2024-06-04): Add benchmarks for storage provider pallet. +} diff --git a/pallets/storage-provider/src/lib.rs b/pallets/storage-provider/src/lib.rs index b44a0a1b3..2ecf91b43 100644 --- a/pallets/storage-provider/src/lib.rs +++ b/pallets/storage-provider/src/lib.rs @@ -9,6 +9,9 @@ #![cfg_attr(not(feature = "std"), no_std)] +#[cfg(feature = "runtime-benchmarks")] +mod benchmarking; + pub use pallet::{Config, Pallet}; use codec::{Decode, Encode}; From 6b34d2f32b9d81eee224a8a1bc59a4cc29d2c422 Mon Sep 17 00:00:00 2001 From: aidan Date: Tue, 4 Jun 2024 14:29:43 +0200 Subject: [PATCH 22/75] feat: Add `StoragePower` to storage provider --- pallets/storage-provider/src/lib.rs | 18 ++++++++++++++++-- runtime/src/configs/mod.rs | 1 + 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/pallets/storage-provider/src/lib.rs b/pallets/storage-provider/src/lib.rs index 2ecf91b43..fe19900f9 100644 --- a/pallets/storage-provider/src/lib.rs +++ b/pallets/storage-provider/src/lib.rs @@ -21,11 +21,14 @@ use scale_info::TypeInfo; pub struct StorageProviderInfo< AccountId: Encode + Decode + Eq + PartialEq, PeerId: Encode + Decode + Eq + PartialEq, + StoragePower: Encode + Decode + Eq + PartialEq, > { /// The owner of this storage_provider. owner: AccountId, /// storage_provider's libp2p peer id in bytes. peer_id: PeerId, + /// The total power the storage provider has + total_raw_power: StoragePower, } #[frame_support::pallet(dev_mode)] @@ -65,14 +68,22 @@ pub mod pallet { /// Usually represented in bytes. /// https://github.com/libp2p/specs/blob/master/peer-ids/peer-ids.md#peer-ids type PeerId: Clone + Debug + Decode + Encode + Eq + TypeInfo; + + /// Unit of Storage Power of a Miner + /// E.g. `u128`, used as `number of bytes` for a given SP. + type StoragePower: Clone + Debug + Decode + Encode + Eq + TypeInfo; } // Need some storage type that keeps track of sectors, deadlines and terminations. // Could be added to this type maybe? #[pallet::storage] #[pallet::getter(fn storage_providers)] - pub type StorageProviders = - StorageMap<_, _, T::AccountId, StorageProviderInfo>; + pub type StorageProviders = StorageMap< + _, + _, + T::AccountId, + StorageProviderInfo, + >; #[pallet::event] #[pallet::generate_deposit(pub(super) fn deposit_event)] @@ -107,6 +118,7 @@ pub mod pallet { pub fn create_storage_provider( origin: OriginFor, peer_id: T::PeerId, + total_raw_power: T::StoragePower, ) -> DispatchResultWithPostInfo { // Check that the extrinsic was signed and get the signer. let owner = ensure_signed(origin)?; @@ -115,6 +127,7 @@ pub mod pallet { let storage_provider_info = StorageProviderInfo { owner: owner.clone(), peer_id: peer_id.clone(), + total_raw_power: total_raw_power, }; // Probably need some check to make sure the storage provider is legit // This means the storage provider exist @@ -188,6 +201,7 @@ pub mod pallet { let new_info = StorageProviderInfo { owner: new_owner.clone(), peer_id: info.peer_id, + total_raw_power: info.total_raw_power, }; // Ensure no storage provider is associated with the new owner diff --git a/runtime/src/configs/mod.rs b/runtime/src/configs/mod.rs index 616e27782..0c504bf21 100644 --- a/runtime/src/configs/mod.rs +++ b/runtime/src/configs/mod.rs @@ -309,4 +309,5 @@ impl pallet_storage_provider::Config for Runtime { type RuntimeEvent = RuntimeEvent; type Currency = Balances; type PeerId = Vec; + type StoragePower = u128; } From fa3869e21c8ae648037501974e4a43b61e44f729 Mon Sep 17 00:00:00 2001 From: aidan Date: Tue, 4 Jun 2024 14:33:35 +0200 Subject: [PATCH 23/75] docs(storage-provider): Add docs to `Event` --- pallets/storage-provider/src/lib.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pallets/storage-provider/src/lib.rs b/pallets/storage-provider/src/lib.rs index fe19900f9..d8675ed55 100644 --- a/pallets/storage-provider/src/lib.rs +++ b/pallets/storage-provider/src/lib.rs @@ -88,13 +88,14 @@ pub mod pallet { #[pallet::event] #[pallet::generate_deposit(pub(super) fn deposit_event)] pub enum Event { - StorageProviderCreated { - owner: T::AccountId, - }, + /// This event is emitted when a new storage provider is initialized. + StorageProviderCreated { owner: T::AccountId }, + /// This event is emitted when a storage provider changes its `PeerId`. PeerIdChanged { storage_provider: T::AccountId, new_peer_id: T::PeerId, }, + /// This event is emitted when a storage provider changes its owner. OwnerAddressChanged { storage_provider: T::AccountId, new_owner: T::AccountId, From 0246ab4c27717a4580d728b707481cc4695958a0 Mon Sep 17 00:00:00 2001 From: aidan Date: Tue, 4 Jun 2024 14:41:41 +0200 Subject: [PATCH 24/75] feat: Add TODO's for call weights --- pallets/storage-provider/src/lib.rs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/pallets/storage-provider/src/lib.rs b/pallets/storage-provider/src/lib.rs index d8675ed55..4ab51dc70 100644 --- a/pallets/storage-provider/src/lib.rs +++ b/pallets/storage-provider/src/lib.rs @@ -116,6 +116,8 @@ pub mod pallet { impl Pallet { /// Index a new storage provider #[pallet::call_index(0)] + // #[pallet::weight(Weight::from_parts(10_000, 0) + T::DbWeight::get().writes(1))] + // TODO(aidan46, no-ref, 2024-06-04): Determine applicable weights. pub fn create_storage_provider( origin: OriginFor, peer_id: T::PeerId, @@ -143,6 +145,8 @@ pub mod pallet { /// Update PeerId for a Storage Provider. #[pallet::call_index(1)] + // #[pallet::weight(Weight::from_parts(10_000, 0) + T::DbWeight::get().writes(1))] + // TODO(aidan46, no-ref, 2024-06-04): Determine applicable weights. pub fn change_peer_id( origin: OriginFor, peer_id: T::PeerId, @@ -186,6 +190,8 @@ pub mod pallet { /// Update the owner address for a Storage Provider. #[pallet::call_index(2)] + // #[pallet::weight(Weight::from_parts(10_000, 0) + T::DbWeight::get().writes(1))] + // TODO(aidan46, no-ref, 2024-06-04): Determine applicable weights. pub fn change_owner_address( origin: OriginFor, new_owner: T::AccountId, @@ -230,10 +236,9 @@ pub mod pallet { } // Used by the reward pallet to award a block reward to a storage_provider. - // I am not sure if this should be implemented on this pallet. - // The reward pallet could be tightly coupled with the storage provider pallet - // so the reward pallet can take over this functionality. #[pallet::call_index(3)] + // #[pallet::weight(Weight::from_parts(10_000, 0) + T::DbWeight::get().writes(1))] + // TODO(aidan46, no-ref, 2024-06-04): Determine applicable weights. pub fn apply_rewards(origin: OriginFor) -> DispatchResultWithPostInfo { // Check that the extrinsic was signed and get the signer. let _who = ensure_signed(origin)?; @@ -243,6 +248,8 @@ pub mod pallet { // This method is used to report a consensus fault by a storage_provider. #[pallet::call_index(4)] + // #[pallet::weight(Weight::from_parts(10_000, 0) + T::DbWeight::get().writes(1))] + // TODO(aidan46, no-ref, 2024-06-04): Determine applicable weights. pub fn report_consensus_fault(origin: OriginFor) -> DispatchResultWithPostInfo { // Check that the extrinsic was signed and get the signer. let _who = ensure_signed(origin)?; @@ -253,6 +260,8 @@ pub mod pallet { // Used by the storage_provider's Owner to withdraw available funds earned from block rewards. // If the amount to withdraw is larger than what is available the extrinsic will fail. #[pallet::call_index(5)] + // #[pallet::weight(Weight::from_parts(10_000, 0) + T::DbWeight::get().writes(1))] + // TODO(aidan46, no-ref, 2024-06-04): Determine applicable weights. pub fn withdraw_balance( origin: OriginFor, _amount: BalanceOf, From 7a10202081ef03b9b5f207c4e0b52e3462a588c2 Mon Sep 17 00:00:00 2001 From: aidan Date: Tue, 4 Jun 2024 16:08:22 +0200 Subject: [PATCH 25/75] feat(storage-provider): Add `price_per_block` to map --- pallets/storage-provider/src/lib.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pallets/storage-provider/src/lib.rs b/pallets/storage-provider/src/lib.rs index 4ab51dc70..98027625e 100644 --- a/pallets/storage-provider/src/lib.rs +++ b/pallets/storage-provider/src/lib.rs @@ -15,6 +15,7 @@ mod benchmarking; pub use pallet::{Config, Pallet}; use codec::{Decode, Encode}; +use scale_info::prelude::string::String; use scale_info::TypeInfo; #[derive(Decode, Encode, TypeInfo)] @@ -29,6 +30,9 @@ pub struct StorageProviderInfo< peer_id: PeerId, /// The total power the storage provider has total_raw_power: StoragePower, + /// The price of storage (in DOT) for each block the storage provider takes for storage. + // TODO(aidan46, no-ref, 2024-06-04): Use appropriate type + price_per_block: String, } #[frame_support::pallet(dev_mode)] @@ -43,6 +47,7 @@ pub mod pallet { use frame_support::traits::{Currency, ReservableCurrency}; use frame_system::pallet_prelude::OriginFor; use frame_system::{ensure_signed, Config as SystemConfig}; + use scale_info::prelude::string::String; use scale_info::TypeInfo; // Allows to extract Balance of an account via the Config::Currency associated type. @@ -122,6 +127,7 @@ pub mod pallet { origin: OriginFor, peer_id: T::PeerId, total_raw_power: T::StoragePower, + price_per_block: String, ) -> DispatchResultWithPostInfo { // Check that the extrinsic was signed and get the signer. let owner = ensure_signed(origin)?; @@ -130,7 +136,8 @@ pub mod pallet { let storage_provider_info = StorageProviderInfo { owner: owner.clone(), peer_id: peer_id.clone(), - total_raw_power: total_raw_power, + total_raw_power, + price_per_block, }; // Probably need some check to make sure the storage provider is legit // This means the storage provider exist @@ -209,6 +216,7 @@ pub mod pallet { owner: new_owner.clone(), peer_id: info.peer_id, total_raw_power: info.total_raw_power, + price_per_block: info.price_per_block, }; // Ensure no storage provider is associated with the new owner From f751e868329725bb17a3311b12b298ce523fc090 Mon Sep 17 00:00:00 2001 From: aidan Date: Tue, 4 Jun 2024 16:24:02 +0200 Subject: [PATCH 26/75] feat(storage-provider): Add `submit_windowed_post` --- pallets/storage-provider/src/lib.rs | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/pallets/storage-provider/src/lib.rs b/pallets/storage-provider/src/lib.rs index 98027625e..b20c6da75 100644 --- a/pallets/storage-provider/src/lib.rs +++ b/pallets/storage-provider/src/lib.rs @@ -254,7 +254,7 @@ pub mod pallet { unimplemented!("Apply rewards is not implemented yet") } - // This method is used to report a consensus fault by a storage_provider. + /// This method is used to report a consensus fault by a storage provider. #[pallet::call_index(4)] // #[pallet::weight(Weight::from_parts(10_000, 0) + T::DbWeight::get().writes(1))] // TODO(aidan46, no-ref, 2024-06-04): Determine applicable weights. @@ -265,8 +265,8 @@ pub mod pallet { unimplemented!("Report consensus fault is not implemented yet") } - // Used by the storage_provider's Owner to withdraw available funds earned from block rewards. - // If the amount to withdraw is larger than what is available the extrinsic will fail. + /// Used by the storage provider to withdraw available funds earned from block rewards. + /// If the amount to withdraw is larger than what is available the extrinsic will fail. #[pallet::call_index(5)] // #[pallet::weight(Weight::from_parts(10_000, 0) + T::DbWeight::get().writes(1))] // TODO(aidan46, no-ref, 2024-06-04): Determine applicable weights. @@ -279,5 +279,16 @@ pub mod pallet { // TODO(@aidan46, no-ref, 2024-06-04): Implement withdraw balance functionality unimplemented!("Withdraw balance is not implemented yet") } + + /// Used by the storage provider to submit their Proof-of-Spacetime + #[pallet::call_index(6)] + // #[pallet::weight(Weight::from_parts(10_000, 0) + T::DbWeight::get().writes(1))] + // TODO(aidan46, no-ref, 2024-06-04): Determine applicable weights. + pub fn submit_windowed_post(origin: OriginFor) -> DispatchResultWithPostInfo { + // Check that the extrinsic was signed and get the signer. + let _who = ensure_signed(origin)?; + // TODO(@aidan46, no-ref, 2024-06-04): Implement submit windowed PoSt functionality + unimplemented!("Submit windowed PoSt is not implemented yet") + } } } From 179108c5b1ebb5457092e7f9fdd9f5a993ebcd92 Mon Sep 17 00:00:00 2001 From: aidan Date: Wed, 5 Jun 2024 16:42:54 +0200 Subject: [PATCH 27/75] docs(storage-provider): Update pallet documentation --- pallets/storage-provider/src/lib.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pallets/storage-provider/src/lib.rs b/pallets/storage-provider/src/lib.rs index b20c6da75..8d4546ae4 100644 --- a/pallets/storage-provider/src/lib.rs +++ b/pallets/storage-provider/src/lib.rs @@ -1,11 +1,13 @@ //! # Storage Provider Pallet //! -//! This pallet holds information about storage providers and -//! provides an interface to modify information about Storage Providers. +//! This pallet is responsible for: +//! - Storage proving operations +//! - Used by the storage provider to generate and submit Proof-of-Replication (PoRep) and Proof-of-Spacetime (PoSt). +//! - Managing and handling collateral for storage deals, penalties, and rewards related to storage deal performance. //! -//! The Storage Provider Pallet is the source of truth for anything storage provider (the binary) related. +//! This pallet holds information about storage providers and provides an interface to modify that information. //! -//! At some point this pallet will have to verify proofs submitted by storage providers +//! The Storage Provider Pallet is the source of truth for anything storage provider related. #![cfg_attr(not(feature = "std"), no_std)] From 0a58259741bda86ada255a017c5f20fab79b178f Mon Sep 17 00:00:00 2001 From: aidan Date: Wed, 5 Jun 2024 17:01:16 +0200 Subject: [PATCH 28/75] feat(storage-provider): Add declare faults function --- pallets/storage-provider/src/lib.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pallets/storage-provider/src/lib.rs b/pallets/storage-provider/src/lib.rs index 8d4546ae4..36b3ca3ee 100644 --- a/pallets/storage-provider/src/lib.rs +++ b/pallets/storage-provider/src/lib.rs @@ -292,5 +292,17 @@ pub mod pallet { // TODO(@aidan46, no-ref, 2024-06-04): Implement submit windowed PoSt functionality unimplemented!("Submit windowed PoSt is not implemented yet") } + + /// Used to declare a set of sectors as "faulty," indicating that the next PoSt for those sectors' + /// deadline will not contain a proof for those sectors' existence. + #[pallet::call_index(7)] + // #[pallet::weight(Weight::from_parts(10_000, 0) + T::DbWeight::get().writes(1))] + // TODO(aidan46, no-ref, 2024-06-05): Determine applicable weights. + pub fn declare_faults(origin: OriginFor) -> DispatchResultWithPostInfo { + // Check that the extrinsic was signed and get the signer. + let _who = ensure_signed(origin)?; + // TODO(@aidan46, no-ref, 2024-06-05): Implement declare faults functionality + unimplemented!("Declare faults is not implemented yet") + } } } From 26165ca51d36b209d187941074235f86b50c7fbf Mon Sep 17 00:00:00 2001 From: aidan Date: Wed, 5 Jun 2024 17:01:44 +0200 Subject: [PATCH 29/75] feat: Add declare faults recovered function --- pallets/storage-provider/src/lib.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pallets/storage-provider/src/lib.rs b/pallets/storage-provider/src/lib.rs index 36b3ca3ee..33c04fa32 100644 --- a/pallets/storage-provider/src/lib.rs +++ b/pallets/storage-provider/src/lib.rs @@ -304,5 +304,17 @@ pub mod pallet { // TODO(@aidan46, no-ref, 2024-06-05): Implement declare faults functionality unimplemented!("Declare faults is not implemented yet") } + + /// Used by a Storage Provider to declare a set of faulty sectors as "recovering," indicating that the + /// next PoSt for those sectors' deadline will contain a proof for those sectors' existence. + #[pallet::call_index(8)] + // #[pallet::weight(Weight::from_parts(10_000, 0) + T::DbWeight::get().writes(1))] + // TODO(aidan46, no-ref, 2024-06-05): Determine applicable weights. + pub fn declare_faults_recovered(origin: OriginFor) -> DispatchResultWithPostInfo { + // Check that the extrinsic was signed and get the signer. + let _who = ensure_signed(origin)?; + // TODO(@aidan46, no-ref, 2024-06-05): Implement declare faults recovered functionality + unimplemented!("Declare faults recovered is not implemented yet") + } } } From d66aeb3c0536515b2340636fd3ce59724aec4c12 Mon Sep 17 00:00:00 2001 From: aidan Date: Wed, 5 Jun 2024 18:05:00 +0200 Subject: [PATCH 30/75] feat: Add pre commit sector function --- pallets/storage-provider/src/lib.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/pallets/storage-provider/src/lib.rs b/pallets/storage-provider/src/lib.rs index 33c04fa32..6e1893846 100644 --- a/pallets/storage-provider/src/lib.rs +++ b/pallets/storage-provider/src/lib.rs @@ -305,7 +305,7 @@ pub mod pallet { unimplemented!("Declare faults is not implemented yet") } - /// Used by a Storage Provider to declare a set of faulty sectors as "recovering," indicating that the + /// Used by a Storage Provider to declare a set of faulty sectors as "recovering," indicating that the /// next PoSt for those sectors' deadline will contain a proof for those sectors' existence. #[pallet::call_index(8)] // #[pallet::weight(Weight::from_parts(10_000, 0) + T::DbWeight::get().writes(1))] @@ -316,5 +316,16 @@ pub mod pallet { // TODO(@aidan46, no-ref, 2024-06-05): Implement declare faults recovered functionality unimplemented!("Declare faults recovered is not implemented yet") } + + /// Pledges the storage provider to seal and commit some new sectors. + #[pallet::call_index(9)] + // #[pallet::weight(Weight::from_parts(10_000, 0) + T::DbWeight::get().writes(1))] + // TODO(aidan46, no-ref, 2024-06-05): Determine applicable weights. + pub fn pre_commit_sector(origin: OriginFor) -> DispatchResultWithPostInfo { + // Check that the extrinsic was signed and get the signer. + let _who = ensure_signed(origin)?; + // TODO(@aidan46, no-ref, 2024-06-05): Implement pre commit sector functionality + unimplemented!("Pre commit sector is not implemented yet") + } } } From 07477cf6d98e0d1db88106c98b58892e413595dc Mon Sep 17 00:00:00 2001 From: aidan Date: Fri, 7 Jun 2024 17:11:49 +0200 Subject: [PATCH 31/75] docs: Add docs for proof of storage --- pallets/storage-provider/PROOF-OF-STORAGE.md | 86 ++++++++++++++++++++ pallets/storage-provider/src/lib.rs | 2 +- 2 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 pallets/storage-provider/PROOF-OF-STORAGE.md diff --git a/pallets/storage-provider/PROOF-OF-STORAGE.md b/pallets/storage-provider/PROOF-OF-STORAGE.md new file mode 100644 index 000000000..ff6205135 --- /dev/null +++ b/pallets/storage-provider/PROOF-OF-STORAGE.md @@ -0,0 +1,86 @@ +# Proof of Storage + +In our parachain within the Polkadot ecosystem, storage providers are required to prove that they hold a copy of the data they have committed to storing at any given point in time. This proof is achieved through a mechanism known as 'challenges.' The process involves the system posing specific questions to the storage providers, who must then provide correct answers to prove they are maintaining the data as promised. + +To ensure the integrity and reliability of these proofs, the challenges must: + +1. Target a Random Part of the Data: The challenge must be directed at a randomly selected portion of the stored data. +2. Be Timed Appropriately: Challenges must occur at intervals that make it infeasible, unprofitable, or irrational for the storage provider to discard the data and retrieve it only when challenged. + +General Proof-of-Storage (PoS) schemes are designed to allow users to verify that a storage provider is indeed storing the outsourced data at the time a challenge is issued. However, proving that data has been stored continuously over a period of time poses additional challenges. One method to address this is to require repeated challenges to the storage provider. However, this approach can lead to high communication complexity, which becomes a bottleneck, especially when storage providers must frequently submit proofs to the network. + +To overcome the limitations of continuous Proof-of-Storage, we introduce a novel proof mechanism inspired by Filecoin, called Proof-of-Spacetime (PoSt). PoSt allows a verifier to check whether a storage provider has consistently stored the committed data over Space (the storage capacity) and Time (the duration). This method provides a more efficient and reliable means of proving data storage over extended periods, reducing the need for constant interaction and lowering the overall communication overhead. + +By implementing PoSt, our parachain ensures that storage providers maintain the integrity of the data they store, providing a robust and scalable solution for decentralized storage within the Polkadot ecosystem. + +## Proof of Replication + +To register a storage sector with our parachain, the sector must undergo a sealing process. Sealing is a computationally intensive procedure that generates a unique proof called Proof-of-Replication (PoRep), which attests to the unique representation of the stored data. + +The PoRep proof links together: + +1. The data itself. +2. The storage provider who performs the sealing. +3. The time when the specific data was sealed by the specific storage provider. + +If the same storage provider attempts to seal the same data at a later time, a different PoRep proof will be produced. The time is recorded as the blockchain height at which sealing took place, with the corresponding chain reference termed SealRandomness. +Generating and Submitting PoRep Proofs + +Once the proof is generated, the storage provider compresses it using a SNARK (Succinct Non-interactive Argument of Knowledge) and submits the result to the blockchain. This submission certifies that the storage provider has indeed replicated a copy of the data they committed to store. +Phases of the PoRep Process + +The PoRep process is divided into two main phases: + +1. Sealing preCommit Phase 1: In this phase, the PoRep encoding and replication take place, ensuring that the data is uniquely tied to the storage provider and timestamp. +2. Sealing preCommit Phase 2: This phase involves the generation of Merkle proofs and trees using the Poseidon hashing algorithm, providing a secure and verifiable method of proof generation. + +By implementing PoRep within our parachain, we ensure that storage providers are accountable for the data they store, enhancing the integrity and reliability of our decentralized storage solution in the Polkadot ecosystem. + +## Proof of Spacetime + +From the point of committing to store data, storage providers must continuously prove that they maintain the data they pledged to store. Proof-of-Spacetime (PoSt) is a procedure during which storage providers are given cryptographic challenges that can only be correctly answered if they are actually storing a copy of the sealed data. + +There are two types of challenges (and their corresponding mechanisms) within the PoSt process: WinningPoSt and WindowPoSt, each serving a different purpose. + +- WinningPoSt: Proves that the storage provider has a replica of the data at the specific time they are challenged. A WinningPoSt challenge is issued to a storage provider only if they are selected through the Secret Leader Election algorithm to validate the next block. The answer to the WinningPoSt challenge must be submitted within a short deadline, making it impractical for the provider to reseal and find the answer on demand. This ensures that the provider maintains a copy of the data at the time of the challenge. +- WindowPoSt: Proves that a copy of the data has been continuously maintained over time. Providers must submit proofs regularly, making it irrational for them to reseal the data every time a WindowPoSt challenge is issued. + +### Constants & Terminology + +Before delving into the details of WinningPoSt and WindowPoSt algorithms, it is essential to clarify the following terms: + +- Partition: A group of 2349 sectors proven simultaneously. +- Proving Period: The average period for proving all sectors maintained by a provider (currently set to 24 hours). +- Deadline: One of the multiple points during a proving period when proofs for some partitions are due. +- Challenge Window: The period immediately before a deadline during which a challenge can be generated by the chain and the requisite proofs computed. +- Provider Size: The amount of proven storage maintained by a single storage provider. + +### WinningPoSt + +At the beginning of each epoch, a small number of storage providers are elected to validate new blocks through the Expected Consensus algorithm. Each elected provider must submit proof that they maintain a sealed copy of the data included in their proposed block before the end of the current epoch. This proof submission is known as WinningPoSt. Successfully submitting a WinningPoSt proof grants the provider a block reward and the opportunity to charge fees for including transactions in the block. Failing to meet the deadline results in the provider missing the opportunity to validate a block and earn rewards. + +### WindowPoSt + +WindowPoSt audits the commitments made by storage providers. Every 24-hour period, known as a proving period, is divided into 30-minute, non-overlapping deadlines, totalling 48 deadlines per period. Providers must demonstrate the availability of all claimed sectors within this time frame. Each proof is limited to 2349 sectors (a partition), with 10 challenges per partition. +Sectors are assigned to deadlines and grouped into partitions. At each deadline, providers must prove an entire partition rather than individual sectors. For each partition, the provider generates a SNARK-compressed proof and publishes it to the blockchain. This process ensures that each sector is audited at least once every 24 hours, creating a permanent, verifiable record of the provider's commitment. +The more sectors a provider has pledged to store, the more partitions they must prove per deadline. This setup necessitates ready access to sealed copies of each challenged sector, making it impractical for the provider to reseal data each time a WindowPoSt proof is required. + +### Design of Proof-of-Spacetime + +Each storage provider is allocated a 24-hour proving period upon creation, divided into 48 non-overlapping half-hour deadlines. Each sector is assigned to a specific deadline when proven to the chain and remains assigned to that deadline throughout its lifetime. Sectors are proven in partitions, and the set of sectors due at each deadline is recorded in a collection of 48 bitfields. + +- Open: Epoch from which a PoSt Proof for this deadline can be submitted. +- Close: Epoch after which a PoSt Proof for this deadline will be rejected. +- FaultCutoff: Epoch after which fault declarations for sectors in the upcoming deadline are rejected. +- Challenge: Epoch at which the randomness for the challenges is available. + +### PoSt Summary + +- Storage providers maintain their sectors by generating Proofs-of-Spacetime (PoSt) and submitting WindowPoSt proofs for their sectors on time. +- WindowPoSt ensures that sectors are persistently stored over time. +- Each provider proves all their sectors once per proving period, with each sector proven by a specific deadline. +- The proving period is a 24-hour cycle divided into deadlines, each assigned to specific sectors. +- To prove continuous storage of a sector, providers must submit a WindowPoSt for each deadline. +- Sectors are grouped into partitions, with each partition proven in a single SNARK proof. + +By implementing PoSt within our parachain, we ensure that storage providers are consistently accountable for the data they store, enhancing the integrity and reliability of our decentralized storage solution in the Polkadot ecosystem. diff --git a/pallets/storage-provider/src/lib.rs b/pallets/storage-provider/src/lib.rs index 6e1893846..599449ba6 100644 --- a/pallets/storage-provider/src/lib.rs +++ b/pallets/storage-provider/src/lib.rs @@ -76,7 +76,7 @@ pub mod pallet { /// https://github.com/libp2p/specs/blob/master/peer-ids/peer-ids.md#peer-ids type PeerId: Clone + Debug + Decode + Encode + Eq + TypeInfo; - /// Unit of Storage Power of a Miner + /// Unit of Storage Power of a Storage Provider /// E.g. `u128`, used as `number of bytes` for a given SP. type StoragePower: Clone + Debug + Decode + Encode + Eq + TypeInfo; } From 6e9b50ac2992cd478c7facbbdac19aec22d83104 Mon Sep 17 00:00:00 2001 From: aidan Date: Fri, 7 Jun 2024 17:15:57 +0200 Subject: [PATCH 32/75] feat: Add prove commit sector --- pallets/storage-provider/src/lib.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pallets/storage-provider/src/lib.rs b/pallets/storage-provider/src/lib.rs index 599449ba6..b1cfdbe0b 100644 --- a/pallets/storage-provider/src/lib.rs +++ b/pallets/storage-provider/src/lib.rs @@ -327,5 +327,17 @@ pub mod pallet { // TODO(@aidan46, no-ref, 2024-06-05): Implement pre commit sector functionality unimplemented!("Pre commit sector is not implemented yet") } + + /// Checks state of the corresponding sector pre-commitments and verifies aggregate proof of replication + /// of these sectors. If valid, the sectors' deals are activated. + #[pallet::call_index(10)] + // #[pallet::weight(Weight::from_parts(10_000, 0) + T::DbWeight::get().writes(1))] + // TODO(aidan46, no-ref, 2024-06-05): Determine applicable weights. + pub fn prove_commit_sector(origin: OriginFor) -> DispatchResultWithPostInfo { + // Check that the extrinsic was signed and get the signer. + let _who = ensure_signed(origin)?; + // TODO(@aidan46, no-ref, 2024-06-07): Implement prove commit sector functionality + unimplemented!("Prove commit sector is not implemented yet") + } } } From 1909e30f9f3cd17e90f53ce3ed2cee8108ea5774 Mon Sep 17 00:00:00 2001 From: aidan Date: Fri, 7 Jun 2024 17:42:10 +0200 Subject: [PATCH 33/75] docs: Update provider state management --- pallets/storage-provider/DESIGN.md | 82 +++++++++++++++++++++++++----- 1 file changed, 68 insertions(+), 14 deletions(-) diff --git a/pallets/storage-provider/DESIGN.md b/pallets/storage-provider/DESIGN.md index 02aa9eef8..9fb4836c6 100644 --- a/pallets/storage-provider/DESIGN.md +++ b/pallets/storage-provider/DESIGN.md @@ -14,25 +14,79 @@ A storage provider indexes in the storage provider pallet itself when it starts The `Storage Provider Pallet` allows storage providers to modify their information such as changing the peer id, through `change_peer_id` and changing owners, through `change_owner_address`. -## Data structures +## State management for Storage Providers + +In our parachain, the state management for all storage providers is handled collectively, unlike Filecoin, which manages the state for individual storage providers. + +### State Structure + +The State structure maintains all the necessary information about the storage providers. This structure includes details about funds, sectors, and deadlines. ```rust -pub struct StorageProviderInfo< - AccountId: Encode + Decode + Eq + PartialEq, - PeerId: Encode + Decode + Eq + PartialEq, -> { - /// The owner of this storage provider. - owner: AccountId, - /// Storage provider'ss libp2p peer id in bytes. - peer_id: PeerId, +#[derive(Encode, Decode, Default, Clone, PartialEq, Debug)] +pub struct ProviderInfo { + // Contains static information about the storage provider. + pub info: Cid, + /// Total funds locked as pre_commit_deposit + pub pre_commit_deposits: u128, + /// Total rewards and added funds locked in vesting table + pub locked_funds: u128, + /// Sum of initial pledge requirements of all active sectors + pub initial_pledge: u128, + /// Sectors that have been pre-committed but not yet proven + pub pre_committed_sectors: Cid, + /// Allocated sector IDs. + pub allocated_sectors: Cid, + /// Information for all proven and not-yet-garbage-collected sectors + pub sectors: Cid, + /// The first block number in this storage provider's current proving period + pub proving_period_start: u64, + /// Index of the deadline within the proving period beginning at ProvingPeriodStart that has not yet been finalized + pub current_deadline: u64, + /// The sector numbers due for PoSt at each deadline in the current proving period, frozen at period start + pub deadlines: Cid, } ``` -The `StorageProviderInfo` structure holds information about a `StorageProvider`. +### Static information about a Storage Provider + +The below struct and its fields ensure that all necessary static information about a Storage provider is encapsulated, allowing for efficient management and interaction within the parachain. ```rust -pub type StorageProviders = - StorageMap<_, _, T::AccountId, StorageProviderInfo>; -``` +#[derive(Debug, PartialEq, Eq, Serialize_tuple, Deserialize_tuple)] +pub struct StorageProviderInfo { + /// Account that owns this StorageProvider + /// - Income and returned collateral are paid to this address + /// + /// Rationale: The owner account is essential for economic transactions and permissions management. + /// By tying the income and collateral to this address, we ensure that the economic benefits and responsibilities + /// are correctly attributed. + pub owner: AccountId, -The `StorageProviders` mapping `AccountId`'s to `PeerId`'s. + /// Libp2p identity that should be used when connecting to this Storage Provider + pub peer_id: PeerId, + + /// The proof type used by this Storage provider for sealing sectors. + /// Rationale: Different StorageProviders may use different proof types for sealing sectors. By storing + /// the `window_post_proof_type`, we can ensure that the correct proof mechanisms are applied and verified + /// according to the provider's chosen method. This enhances compatibility and integrity in the proof-of-storage + /// processes. + pub window_post_proof_type: RegisteredPoStProof, + + /// Amount of space in each sector committed to the network by this Storage Provider + /// + /// Rationale: The `sector_size` indicates the amount of data each sector can hold. This information is crucial + /// for calculating storage capacity, economic incentives, and the validation process. It ensures that the storage + /// commitments made by the provider are transparent and verifiable. + pub sector_size: SectorSize, + + /// The number of sectors in each Window PoSt partition (proof). + /// This is computed from the proof type and represented here redundantly. + /// + /// Rationale: The `window_post_partition_sectors` field specifies the number of sectors included in each + /// Window PoSt proof partition. This redundancy ensures that partition calculations are consistent and + /// simplifies the process of generating and verifying proofs. By storing this value, we enhance the efficiency + /// of proof operations and reduce computational overhead during runtime. + pub window_post_partition_sectors: u64, +} +``` From 511e85f12c8260c59372d923e9b5552e7bca8fd1 Mon Sep 17 00:00:00 2001 From: aidan Date: Mon, 10 Jun 2024 14:40:00 +0200 Subject: [PATCH 34/75] docs: Add proof types --- pallets/storage-provider/DESIGN.md | 75 +++++++++++++- pallets/storage-provider/src/lib.rs | 25 +---- pallets/storage-provider/src/types.rs | 134 ++++++++++++++++++++++++++ 3 files changed, 210 insertions(+), 24 deletions(-) create mode 100644 pallets/storage-provider/src/types.rs diff --git a/pallets/storage-provider/DESIGN.md b/pallets/storage-provider/DESIGN.md index 9fb4836c6..b8da74273 100644 --- a/pallets/storage-provider/DESIGN.md +++ b/pallets/storage-provider/DESIGN.md @@ -23,7 +23,6 @@ In our parachain, the state management for all storage providers is handled coll The State structure maintains all the necessary information about the storage providers. This structure includes details about funds, sectors, and deadlines. ```rust -#[derive(Encode, Decode, Default, Clone, PartialEq, Debug)] pub struct ProviderInfo { // Contains static information about the storage provider. pub info: Cid, @@ -53,7 +52,6 @@ pub struct ProviderInfo { The below struct and its fields ensure that all necessary static information about a Storage provider is encapsulated, allowing for efficient management and interaction within the parachain. ```rust -#[derive(Debug, PartialEq, Eq, Serialize_tuple, Deserialize_tuple)] pub struct StorageProviderInfo { /// Account that owns this StorageProvider /// - Income and returned collateral are paid to this address @@ -90,3 +88,76 @@ pub struct StorageProviderInfo { pub window_post_partition_sectors: u64, } ``` + +## Data structures + +### Proof of spacetime + +Proof of spacetime indicates the version and the sector size of the proof. This type is used by the Storage Provider when initially starting up to indicate what PoSt version it will use to submit Window PoSt proof. + +```rust +pub enum RegisteredPoStProof { + StackedDRGWinning2KiBV1, + StackedDRGWinning8MiBV1, + StackedDRGWinning512MiBV1, + StackedDRGWinning32GiBV1, + StackedDRGWinning64GiBV1, + StackedDRGWindow2KiBV1P1, + StackedDRGWindow8MiBV1P1, + StackedDRGWindow512MiBV1P1, + StackedDRGWindow32GiBV1P1, + StackedDRGWindow64GiBV1P1, + Invalid(i64), +} +``` + +The `SectorSize` indicates one of a set of possible sizes in the network. + +```rust +#[repr(u64)] +pub enum SectorSize { + _2KiB = 2_048, + _8MiB = 8_388_608, + _512MiB = 536_870_912, + _32GiB = 34_359_738_368, + _64GiB = 68_719_476_736, +} +``` + +### Proof of replication + +Proof of replication is used when a Storage Provider wants to store data on behalf of a client and receives a piece of client data. The data will first be placed in a sector after which that sector is sealed by the storage provider. Then a unique encoding, which serves as proof that the Storage Provider has replicated a copy of the data they agreed to store, is generated. Finally, the proof is compressed and submitted to the network as certification of storage. + +```rust +/// This type indicates the seal proof type which defines the version and the sector size +pub enum RegisteredSealProof { + StackedDRG2KiBV1, + StackedDRG512MiBV1, + StackedDRG8MiBV1, + StackedDRG32GiBV1, + StackedDRG64GiBV1, + StackedDRG2KiBV1P1, + StackedDRG512MiBV1P1, + StackedDRG8MiBV1P1, + StackedDRG32GiBV1P1, + StackedDRG64GiBV1P1, + StackedDRG2KiBV1P1_Feat_SyntheticPoRep, + StackedDRG512MiBV1P1_Feat_SyntheticPoRep, + StackedDRG8MiBV1P1_Feat_SyntheticPoRep, + StackedDRG32GiBV1P1_Feat_SyntheticPoRep, + StackedDRG64GiBV1P1_Feat_SyntheticPoRep, + Invalid(i64), +} +``` + +The unique encoding created during the sealing process is generated using the sealed data, the storage provider who seals the data and the time at which the data was sealed. + +```rust +/// This type is passed into the pre commit function on the storage provider pallet +pub struct SectorPreCommitInfo { + pub seal_proof: RegisteredSealProof, + pub sector_number: SectorNumber, + pub sealed_cid: Cid, + pub expiration: u64, +} +``` diff --git a/pallets/storage-provider/src/lib.rs b/pallets/storage-provider/src/lib.rs index b1cfdbe0b..32dd13349 100644 --- a/pallets/storage-provider/src/lib.rs +++ b/pallets/storage-provider/src/lib.rs @@ -14,32 +14,13 @@ #[cfg(feature = "runtime-benchmarks")] mod benchmarking; -pub use pallet::{Config, Pallet}; +mod types; -use codec::{Decode, Encode}; -use scale_info::prelude::string::String; -use scale_info::TypeInfo; - -#[derive(Decode, Encode, TypeInfo)] -pub struct StorageProviderInfo< - AccountId: Encode + Decode + Eq + PartialEq, - PeerId: Encode + Decode + Eq + PartialEq, - StoragePower: Encode + Decode + Eq + PartialEq, -> { - /// The owner of this storage_provider. - owner: AccountId, - /// storage_provider's libp2p peer id in bytes. - peer_id: PeerId, - /// The total power the storage provider has - total_raw_power: StoragePower, - /// The price of storage (in DOT) for each block the storage provider takes for storage. - // TODO(aidan46, no-ref, 2024-06-04): Use appropriate type - price_per_block: String, -} +pub use pallet::{Config, Pallet}; #[frame_support::pallet(dev_mode)] pub mod pallet { - use super::StorageProviderInfo; + use crate::types::StorageProviderInfo; use codec::{Decode, Encode}; use core::fmt::Debug; diff --git a/pallets/storage-provider/src/types.rs b/pallets/storage-provider/src/types.rs new file mode 100644 index 000000000..bd4dbeb65 --- /dev/null +++ b/pallets/storage-provider/src/types.rs @@ -0,0 +1,134 @@ +use codec::{Decode, Encode}; +use scale_info::prelude::format; +use scale_info::prelude::string::String; +use scale_info::TypeInfo; + +/// SectorNumber is a numeric identifier for a sector. +pub type SectorNumber = u64; + +/// Content identifier +pub type Cid = String; + +#[derive(Decode, Encode, TypeInfo)] +pub struct StorageProviderInfo< + AccountId: Encode + Decode + Eq + PartialEq, + PeerId: Encode + Decode + Eq + PartialEq, + StoragePower: Encode + Decode + Eq + PartialEq, +> { + /// The owner of this storage_provider. + pub owner: AccountId, + /// storage_provider's libp2p peer id in bytes. + pub peer_id: PeerId, + /// The total power the storage provider has + pub total_raw_power: StoragePower, + /// The price of storage (in DOT) for each block the storage provider takes for storage. + // TODO(aidan46, no-ref, 2024-06-04): Use appropriate type + pub price_per_block: String, +} + +/// SectorSize indicates one of a set of possible sizes in the network. +#[repr(u64)] +pub enum SectorSize { + _2KiB = 2_048, + _8MiB = 8_388_608, + _512MiB = 536_870_912, + _32GiB = 34_359_738_368, + _64GiB = 68_719_476_736, +} + +/// Proof of spacetime type, indicating version and sector size of the proof. +#[derive(Decode, Encode, TypeInfo)] +pub enum RegisteredPoStProof { + StackedDRGWinning2KiBV1, + StackedDRGWinning8MiBV1, + StackedDRGWinning512MiBV1, + StackedDRGWinning32GiBV1, + StackedDRGWinning64GiBV1, + StackedDRGWindow2KiBV1P1, + StackedDRGWindow8MiBV1P1, + StackedDRGWindow512MiBV1P1, + StackedDRGWindow32GiBV1P1, + StackedDRGWindow64GiBV1P1, + Invalid(i64), +} + +impl RegisteredPoStProof { + /// Returns the sector size of the proof type, which is measured in bytes. + #[allow(unused)] + pub fn sector_size(self) -> Result { + use RegisteredPoStProof::*; + match self { + StackedDRGWindow2KiBV1P1 | StackedDRGWinning2KiBV1 => Ok(SectorSize::_2KiB), + StackedDRGWindow8MiBV1P1 | StackedDRGWinning8MiBV1 => Ok(SectorSize::_8MiB), + StackedDRGWindow512MiBV1P1 | StackedDRGWinning512MiBV1 => Ok(SectorSize::_512MiB), + StackedDRGWindow32GiBV1P1 | StackedDRGWinning32GiBV1 => Ok(SectorSize::_32GiB), + StackedDRGWindow64GiBV1P1 | StackedDRGWinning64GiBV1 => Ok(SectorSize::_64GiB), + Invalid(i) => Err(format!("unsupported proof type: {}", i)), + } + } + + /// Proof size for each PoStProof type + #[allow(unused)] + pub fn proof_size(self) -> Result { + use RegisteredPoStProof::*; + match self { + StackedDRGWinning2KiBV1 + | StackedDRGWinning8MiBV1 + | StackedDRGWinning512MiBV1 + | StackedDRGWinning32GiBV1 + | StackedDRGWinning64GiBV1 + | StackedDRGWindow2KiBV1P1 + | StackedDRGWindow8MiBV1P1 + | StackedDRGWindow512MiBV1P1 + | StackedDRGWindow32GiBV1P1 + | StackedDRGWindow64GiBV1P1 => Ok(192), + Invalid(i) => Err(format!("unsupported proof type: {}", i)), + } + } + /// Returns the partition size, in sectors, associated with a proof type. + /// The partition size is the number of sectors proven in a single PoSt proof. + #[allow(unused)] + pub fn window_post_partitions_sector(self) -> Result { + // Resolve to post proof and then compute size from that. + use RegisteredPoStProof::*; + match self { + StackedDRGWinning64GiBV1 | StackedDRGWindow64GiBV1P1 => Ok(2300), + StackedDRGWinning32GiBV1 | StackedDRGWindow32GiBV1P1 => Ok(2349), + StackedDRGWinning2KiBV1 | StackedDRGWindow2KiBV1P1 => Ok(2), + StackedDRGWinning8MiBV1 | StackedDRGWindow8MiBV1P1 => Ok(2), + StackedDRGWinning512MiBV1 | StackedDRGWindow512MiBV1P1 => Ok(2), + Invalid(i) => Err(format!("unsupported proof type: {}", i)), + } + } +} + +/// Seal proof type which defines the version and sector size. +#[allow(non_camel_case_types)] +#[derive(Decode, Encode, TypeInfo)] +pub enum RegisteredSealProof { + StackedDRG2KiBV1, + StackedDRG512MiBV1, + StackedDRG8MiBV1, + StackedDRG32GiBV1, + StackedDRG64GiBV1, + StackedDRG2KiBV1P1, + StackedDRG512MiBV1P1, + StackedDRG8MiBV1P1, + StackedDRG32GiBV1P1, + StackedDRG64GiBV1P1, + StackedDRG2KiBV1P1_Feat_SyntheticPoRep, + StackedDRG512MiBV1P1_Feat_SyntheticPoRep, + StackedDRG8MiBV1P1_Feat_SyntheticPoRep, + StackedDRG32GiBV1P1_Feat_SyntheticPoRep, + StackedDRG64GiBV1P1_Feat_SyntheticPoRep, + Invalid(i64), +} + +/// This type is passed into the pre commit function on the storage provider pallet +#[derive(Decode, Encode, TypeInfo)] +pub struct SectorPreCommitInfo { + pub seal_proof: RegisteredSealProof, + pub sector_number: SectorNumber, + pub sealed_cid: Cid, + pub expiration: u64, +} From 66c3e599d1a47d51e246c57036698dcae4f21773 Mon Sep 17 00:00:00 2001 From: aidan Date: Mon, 10 Jun 2024 16:21:40 +0200 Subject: [PATCH 35/75] docs: Add API docs Add API docs for the methods provider by the storage provider pallet. --- pallets/storage-provider/API.md | 74 +++++++++++++++ pallets/storage-provider/DESIGN.md | 19 ++-- pallets/storage-provider/src/lib.rs | 129 +++++++++----------------- pallets/storage-provider/src/types.rs | 102 ++++++++++++++++---- runtime/src/configs/mod.rs | 2 - 5 files changed, 214 insertions(+), 112 deletions(-) create mode 100644 pallets/storage-provider/API.md diff --git a/pallets/storage-provider/API.md b/pallets/storage-provider/API.md new file mode 100644 index 000000000..a575b9e74 --- /dev/null +++ b/pallets/storage-provider/API.md @@ -0,0 +1,74 @@ +# Storage provider API + +## Creating Storage Provider + +When a Storage provider first starts up it needs to index itself in the storage provider pallet. The `create_storage_provider` extrinsic is used for this. + +### Arguments + +`peer_id`: storage_provider's libp2p peer id in bytes. +`window_post_proof_type`: The total power the storage provider has + +## Change peer id + +The `change_peer_id` extrinsic is used by the Storage Provider to update its peer id. + +### Arguments + +`peer_id`: The new peer id for the Storage Provider. + +## Change owner address + +The `change_owner_address` extrinsic is used by the Storage Provider to update its owner. + +### Arguments + +`new_owner`: The address of the new owner. + +## Submitting Proof of Spacetime + +The `submit_windowed_post` is used by the storage provider to submit their Proof of Spacetime + +### Arguments + +`deadline`: The deadline index which the submission targets. +`partitions`: The partitions being proven. +`proofs`: An array of proofs, one per distinct registered proof type present in the sectors being proven. + +## Declaring faults + +Storage providers can use the `declare_faults` extrinsic to declare a set of sectors as 'faulty', indicating that the next PoSt for those sectors' deadline will not contain a proof for those sectors' existence. + +### Arguments + +`deadline`: The deadline to which the faulty sectors are assigned +`partition`: Partition index within the deadline containing the faulty sectors. +`sectors`: Sectors in the partition being declared faulty. + +## Declaring faults as recovered + +Storage providers can declare a set of faulty sectors as "recovering", indicating that the next PoSt for those sectors' deadline will contain a proof for those sectors' existence. + +### Arguments + +`deadline`: The deadline to which the recovered sectors are assigned +`partition`: Partition index within the deadline containing the recovered sectors. +`sectors`: Sectors in the partition being declared recovered. + +## Pre committing sectors + +The Storage Provider can use the `pre_commit_sector` extrinsic to pledge to seal and commit some new sectors. + +### Arguments + +`sectors`: Sectors to be committed. + + +## Prove commit sectors + +Storage providers can use the `prove_commit_sector` extrinsic to check the state of the corresponding sector pre-commitments and verifies aggregate proof of replication of these sectors. If valid, the sectors' deals are activated. + +### Arguments + +`sector_number`: The sector number to be proved. +`proof`: The proof, supplied by the storage provider. diff --git a/pallets/storage-provider/DESIGN.md b/pallets/storage-provider/DESIGN.md index b8da74273..5cb7f7966 100644 --- a/pallets/storage-provider/DESIGN.md +++ b/pallets/storage-provider/DESIGN.md @@ -53,14 +53,6 @@ The below struct and its fields ensure that all necessary static information abo ```rust pub struct StorageProviderInfo { - /// Account that owns this StorageProvider - /// - Income and returned collateral are paid to this address - /// - /// Rationale: The owner account is essential for economic transactions and permissions management. - /// By tying the income and collateral to this address, we ensure that the economic benefits and responsibilities - /// are correctly attributed. - pub owner: AccountId, - /// Libp2p identity that should be used when connecting to this Storage Provider pub peer_id: PeerId, @@ -124,6 +116,15 @@ pub enum SectorSize { } ``` +The `PoStProof` is the proof of spacetime data that is stored on chain + +```rust +pub struct PoStProof { + pub post_proof: RegisteredPoStProof, + pub proof_bytes: Vec, +} +``` + ### Proof of replication Proof of replication is used when a Storage Provider wants to store data on behalf of a client and receives a piece of client data. The data will first be placed in a sector after which that sector is sealed by the storage provider. Then a unique encoding, which serves as proof that the Storage Provider has replicated a copy of the data they agreed to store, is generated. Finally, the proof is compressed and submitted to the network as certification of storage. @@ -160,4 +161,4 @@ pub struct SectorPreCommitInfo { pub sealed_cid: Cid, pub expiration: u64, } -``` +``` \ No newline at end of file diff --git a/pallets/storage-provider/src/lib.rs b/pallets/storage-provider/src/lib.rs index 32dd13349..2ea695767 100644 --- a/pallets/storage-provider/src/lib.rs +++ b/pallets/storage-provider/src/lib.rs @@ -20,24 +20,20 @@ pub use pallet::{Config, Pallet}; #[frame_support::pallet(dev_mode)] pub mod pallet { - use crate::types::StorageProviderInfo; + use crate::types::{ + PoStProof, RegisteredPoStProof, SectorNumber, SectorPreCommitInfo, StorageProviderInfo, + }; use codec::{Decode, Encode}; use core::fmt::Debug; use frame_support::dispatch::DispatchResultWithPostInfo; use frame_support::ensure; use frame_support::pallet_prelude::{IsType, StorageMap}; - use frame_support::traits::{Currency, ReservableCurrency}; + use frame_system::ensure_signed; use frame_system::pallet_prelude::OriginFor; - use frame_system::{ensure_signed, Config as SystemConfig}; - use scale_info::prelude::string::String; + use scale_info::prelude::vec::Vec; use scale_info::TypeInfo; - // Allows to extract Balance of an account via the Config::Currency associated type. - // BalanceOf is a sophisticated way of getting an u128. - type BalanceOf = - <::Currency as Currency<::AccountId>>::Balance; - #[pallet::pallet] #[pallet::without_storage_info] // Allows to define storage items without fixed size pub struct Pallet(_); @@ -47,31 +43,18 @@ pub mod pallet { /// Because this pallet emits events, it depends on the runtime's definition of an event. type RuntimeEvent: From> + IsType<::RuntimeEvent>; - /// The currency mechanism. - /// Used for rewards, using `ReservableCurrency` over `Currency` because the rewards will be locked - /// in this pallet until the storage provider requests the funds through `withdraw_balance` - type Currency: ReservableCurrency; - /// Peer ID is derived by hashing an encoded public key. /// Usually represented in bytes. /// https://github.com/libp2p/specs/blob/master/peer-ids/peer-ids.md#peer-ids type PeerId: Clone + Debug + Decode + Encode + Eq + TypeInfo; - - /// Unit of Storage Power of a Storage Provider - /// E.g. `u128`, used as `number of bytes` for a given SP. - type StoragePower: Clone + Debug + Decode + Encode + Eq + TypeInfo; } // Need some storage type that keeps track of sectors, deadlines and terminations. // Could be added to this type maybe? #[pallet::storage] #[pallet::getter(fn storage_providers)] - pub type StorageProviders = StorageMap< - _, - _, - T::AccountId, - StorageProviderInfo, - >; + pub type StorageProviders = + StorageMap<_, _, T::AccountId, StorageProviderInfo>; #[pallet::event] #[pallet::generate_deposit(pub(super) fn deposit_event)] @@ -92,6 +75,8 @@ pub mod pallet { #[pallet::error] pub enum Error { + /// Error emitted when the provided information for the storage provider is invalid. + StorageProviderInfoError, /// Error emitted when trying to get info on a storage provider that does not exist. StorageProviderNotFound, /// Error emitted when doing a privileged call and the signer does not match. @@ -109,19 +94,15 @@ pub mod pallet { pub fn create_storage_provider( origin: OriginFor, peer_id: T::PeerId, - total_raw_power: T::StoragePower, - price_per_block: String, + window_post_proof_type: RegisteredPoStProof, ) -> DispatchResultWithPostInfo { // Check that the extrinsic was signed and get the signer. let owner = ensure_signed(origin)?; // Generate some storage_provider id and insert into `StorageProviders` storage - let storage_provider_info = StorageProviderInfo { - owner: owner.clone(), - peer_id: peer_id.clone(), - total_raw_power, - price_per_block, - }; + let storage_provider_info = + StorageProviderInfo::new(owner.clone(), peer_id.clone(), window_post_proof_type) + .map_err(|_| Error::::StorageProviderInfoError)?; // Probably need some check to make sure the storage provider is legit // This means the storage provider exist ensure!( @@ -195,19 +176,14 @@ pub mod pallet { // Ensure storage_provider is the owner of the storage_provider ensure!(storage_provider == info.owner, Error::::InvalidSigner); - let new_info = StorageProviderInfo { - owner: new_owner.clone(), - peer_id: info.peer_id, - total_raw_power: info.total_raw_power, - price_per_block: info.price_per_block, - }; - // Ensure no storage provider is associated with the new owner ensure!( !StorageProviders::::contains_key(&new_owner), Error::::DuplicateStorageProvider ); + let new_info = info.change_owner(new_owner.clone()); + // Insert new storage provider info StorageProviders::::insert(new_owner.clone(), new_info); @@ -226,48 +202,16 @@ pub mod pallet { } } - // Used by the reward pallet to award a block reward to a storage_provider. + /// Used by the storage provider to submit their Proof-of-Spacetime #[pallet::call_index(3)] // #[pallet::weight(Weight::from_parts(10_000, 0) + T::DbWeight::get().writes(1))] // TODO(aidan46, no-ref, 2024-06-04): Determine applicable weights. - pub fn apply_rewards(origin: OriginFor) -> DispatchResultWithPostInfo { - // Check that the extrinsic was signed and get the signer. - let _who = ensure_signed(origin)?; - // TODO(@aidan46, no-ref, 2024-06-04): Implement apply rewards functionality - unimplemented!("Apply rewards is not implemented yet") - } - - /// This method is used to report a consensus fault by a storage provider. - #[pallet::call_index(4)] - // #[pallet::weight(Weight::from_parts(10_000, 0) + T::DbWeight::get().writes(1))] - // TODO(aidan46, no-ref, 2024-06-04): Determine applicable weights. - pub fn report_consensus_fault(origin: OriginFor) -> DispatchResultWithPostInfo { - // Check that the extrinsic was signed and get the signer. - let _who = ensure_signed(origin)?; - // TODO(@aidan46, no-ref, 2024-06-04): Implement report consensus fault functionality - unimplemented!("Report consensus fault is not implemented yet") - } - - /// Used by the storage provider to withdraw available funds earned from block rewards. - /// If the amount to withdraw is larger than what is available the extrinsic will fail. - #[pallet::call_index(5)] - // #[pallet::weight(Weight::from_parts(10_000, 0) + T::DbWeight::get().writes(1))] - // TODO(aidan46, no-ref, 2024-06-04): Determine applicable weights. - pub fn withdraw_balance( + pub fn submit_windowed_post( origin: OriginFor, - _amount: BalanceOf, + _deadline: u64, + _partitions: Vec, + _proofs: Vec, ) -> DispatchResultWithPostInfo { - // Check that the extrinsic was signed and get the signer. - let _who = ensure_signed(origin)?; - // TODO(@aidan46, no-ref, 2024-06-04): Implement withdraw balance functionality - unimplemented!("Withdraw balance is not implemented yet") - } - - /// Used by the storage provider to submit their Proof-of-Spacetime - #[pallet::call_index(6)] - // #[pallet::weight(Weight::from_parts(10_000, 0) + T::DbWeight::get().writes(1))] - // TODO(aidan46, no-ref, 2024-06-04): Determine applicable weights. - pub fn submit_windowed_post(origin: OriginFor) -> DispatchResultWithPostInfo { // Check that the extrinsic was signed and get the signer. let _who = ensure_signed(origin)?; // TODO(@aidan46, no-ref, 2024-06-04): Implement submit windowed PoSt functionality @@ -276,10 +220,15 @@ pub mod pallet { /// Used to declare a set of sectors as "faulty," indicating that the next PoSt for those sectors' /// deadline will not contain a proof for those sectors' existence. - #[pallet::call_index(7)] + #[pallet::call_index(4)] // #[pallet::weight(Weight::from_parts(10_000, 0) + T::DbWeight::get().writes(1))] // TODO(aidan46, no-ref, 2024-06-05): Determine applicable weights. - pub fn declare_faults(origin: OriginFor) -> DispatchResultWithPostInfo { + pub fn declare_faults( + origin: OriginFor, + _deadline: u64, + _partition: u64, + _sectors: Vec, + ) -> DispatchResultWithPostInfo { // Check that the extrinsic was signed and get the signer. let _who = ensure_signed(origin)?; // TODO(@aidan46, no-ref, 2024-06-05): Implement declare faults functionality @@ -288,10 +237,15 @@ pub mod pallet { /// Used by a Storage Provider to declare a set of faulty sectors as "recovering," indicating that the /// next PoSt for those sectors' deadline will contain a proof for those sectors' existence. - #[pallet::call_index(8)] + #[pallet::call_index(5)] // #[pallet::weight(Weight::from_parts(10_000, 0) + T::DbWeight::get().writes(1))] // TODO(aidan46, no-ref, 2024-06-05): Determine applicable weights. - pub fn declare_faults_recovered(origin: OriginFor) -> DispatchResultWithPostInfo { + pub fn declare_faults_recovered( + origin: OriginFor, + _deadline: u64, + _partition: u64, + _sectors: Vec, + ) -> DispatchResultWithPostInfo { // Check that the extrinsic was signed and get the signer. let _who = ensure_signed(origin)?; // TODO(@aidan46, no-ref, 2024-06-05): Implement declare faults recovered functionality @@ -299,10 +253,13 @@ pub mod pallet { } /// Pledges the storage provider to seal and commit some new sectors. - #[pallet::call_index(9)] + #[pallet::call_index(6)] // #[pallet::weight(Weight::from_parts(10_000, 0) + T::DbWeight::get().writes(1))] // TODO(aidan46, no-ref, 2024-06-05): Determine applicable weights. - pub fn pre_commit_sector(origin: OriginFor) -> DispatchResultWithPostInfo { + pub fn pre_commit_sector( + origin: OriginFor, + _sectors: SectorPreCommitInfo, + ) -> DispatchResultWithPostInfo { // Check that the extrinsic was signed and get the signer. let _who = ensure_signed(origin)?; // TODO(@aidan46, no-ref, 2024-06-05): Implement pre commit sector functionality @@ -311,10 +268,14 @@ pub mod pallet { /// Checks state of the corresponding sector pre-commitments and verifies aggregate proof of replication /// of these sectors. If valid, the sectors' deals are activated. - #[pallet::call_index(10)] + #[pallet::call_index(8)] // #[pallet::weight(Weight::from_parts(10_000, 0) + T::DbWeight::get().writes(1))] // TODO(aidan46, no-ref, 2024-06-05): Determine applicable weights. - pub fn prove_commit_sector(origin: OriginFor) -> DispatchResultWithPostInfo { + pub fn prove_commit_sector( + origin: OriginFor, + _sector_number: SectorNumber, + _proof: Vec, + ) -> DispatchResultWithPostInfo { // Check that the extrinsic was signed and get the signer. let _who = ensure_signed(origin)?; // TODO(@aidan46, no-ref, 2024-06-07): Implement prove commit sector functionality diff --git a/pallets/storage-provider/src/types.rs b/pallets/storage-provider/src/types.rs index bd4dbeb65..5a2cdcfbd 100644 --- a/pallets/storage-provider/src/types.rs +++ b/pallets/storage-provider/src/types.rs @@ -1,6 +1,7 @@ use codec::{Decode, Encode}; use scale_info::prelude::format; use scale_info::prelude::string::String; +use scale_info::prelude::vec::Vec; use scale_info::TypeInfo; /// SectorNumber is a numeric identifier for a sector. @@ -13,31 +14,91 @@ pub type Cid = String; pub struct StorageProviderInfo< AccountId: Encode + Decode + Eq + PartialEq, PeerId: Encode + Decode + Eq + PartialEq, - StoragePower: Encode + Decode + Eq + PartialEq, > { - /// The owner of this storage_provider. + /// Account that owns this StorageProvider + /// - Income and returned collateral are paid to this address + /// + /// Rationale: The owner account is essential for economic transactions and permissions management. + /// By tying the income and collateral to this address, we ensure that the economic benefits and responsibilities + /// are correctly attributed. pub owner: AccountId, - /// storage_provider's libp2p peer id in bytes. + + /// Libp2p identity that should be used when connecting to this Storage Provider pub peer_id: PeerId, - /// The total power the storage provider has - pub total_raw_power: StoragePower, - /// The price of storage (in DOT) for each block the storage provider takes for storage. - // TODO(aidan46, no-ref, 2024-06-04): Use appropriate type - pub price_per_block: String, + + /// The proof type used by this Storage provider for sealing sectors. + /// Rationale: Different StorageProviders may use different proof types for sealing sectors. By storing + /// the `window_post_proof_type`, we can ensure that the correct proof mechanisms are applied and verified + /// according to the provider's chosen method. This enhances compatibility and integrity in the proof-of-storage + /// processes. + pub window_post_proof_type: RegisteredPoStProof, + + /// Amount of space in each sector committed to the network by this Storage Provider + /// + /// Rationale: The `sector_size` indicates the amount of data each sector can hold. This information is crucial + /// for calculating storage capacity, economic incentives, and the validation process. It ensures that the storage + /// commitments made by the provider are transparent and verifiable. + pub sector_size: SectorSize, + + /// The number of sectors in each Window PoSt partition (proof). + /// This is computed from the proof type and represented here redundantly. + /// + /// Rationale: The `window_post_partition_sectors` field specifies the number of sectors included in each + /// Window PoSt proof partition. This redundancy ensures that partition calculations are consistent and + /// simplifies the process of generating and verifying proofs. By storing this value, we enhance the efficiency + /// of proof operations and reduce computational overhead during runtime. + pub window_post_partition_sectors: u64, +} + +impl StorageProviderInfo +where + AccountId: Encode + Decode + Eq + PartialEq, + PeerId: Encode + Decode + Eq + PartialEq + Clone, +{ + /// Create a new instance of StorageProviderInfo + pub fn new( + owner: AccountId, + peer_id: PeerId, + window_post_proof_type: RegisteredPoStProof, + ) -> Result { + let sector_size = window_post_proof_type.sector_size()?; + + let window_post_partition_sectors = + window_post_proof_type.window_post_partitions_sector()?; + + Ok(Self { + owner, + peer_id, + window_post_proof_type, + sector_size, + window_post_partition_sectors, + }) + } + + /// Updates the owner address. + pub fn change_owner(&self, owner: AccountId) -> Self { + Self { + owner, + peer_id: self.peer_id.clone(), + window_post_proof_type: self.window_post_proof_type, + sector_size: self.sector_size, + window_post_partition_sectors: self.window_post_partition_sectors, + } + } } /// SectorSize indicates one of a set of possible sizes in the network. -#[repr(u64)] +#[derive(Encode, Decode, TypeInfo, Clone, Debug, PartialEq, Eq, Copy)] pub enum SectorSize { - _2KiB = 2_048, - _8MiB = 8_388_608, - _512MiB = 536_870_912, - _32GiB = 34_359_738_368, - _64GiB = 68_719_476_736, + _2KiB, + _8MiB, + _512MiB, + _32GiB, + _64GiB, } /// Proof of spacetime type, indicating version and sector size of the proof. -#[derive(Decode, Encode, TypeInfo)] +#[derive(Debug, Decode, Encode, TypeInfo, PartialEq, Eq, Clone, Copy)] pub enum RegisteredPoStProof { StackedDRGWinning2KiBV1, StackedDRGWinning8MiBV1, @@ -102,9 +163,16 @@ impl RegisteredPoStProof { } } +/// Proof of spacetime data stored on chain. +#[derive(Debug, Decode, Encode, TypeInfo, PartialEq, Eq, Clone)] +pub struct PoStProof { + pub post_proof: RegisteredPoStProof, + pub proof_bytes: Vec, +} + /// Seal proof type which defines the version and sector size. #[allow(non_camel_case_types)] -#[derive(Decode, Encode, TypeInfo)] +#[derive(Debug, Decode, Encode, TypeInfo, Eq, PartialEq, Clone)] pub enum RegisteredSealProof { StackedDRG2KiBV1, StackedDRG512MiBV1, @@ -125,7 +193,7 @@ pub enum RegisteredSealProof { } /// This type is passed into the pre commit function on the storage provider pallet -#[derive(Decode, Encode, TypeInfo)] +#[derive(Debug, Decode, Encode, TypeInfo, Eq, PartialEq, Clone)] pub struct SectorPreCommitInfo { pub seal_proof: RegisteredSealProof, pub sector_number: SectorNumber, diff --git a/runtime/src/configs/mod.rs b/runtime/src/configs/mod.rs index 0c504bf21..7e348cf46 100644 --- a/runtime/src/configs/mod.rs +++ b/runtime/src/configs/mod.rs @@ -307,7 +307,5 @@ impl pallet_collator_selection::Config for Runtime { impl pallet_storage_provider::Config for Runtime { type RuntimeEvent = RuntimeEvent; - type Currency = Balances; type PeerId = Vec; - type StoragePower = u128; } From 5598cef5f2c8bf7116a8a3cf267b343f993f78c1 Mon Sep 17 00:00:00 2001 From: aidan Date: Mon, 10 Jun 2024 16:36:49 +0200 Subject: [PATCH 36/75] docs: Add note to WinningPoSt WinningPost is not handled by the storage provider pallet as block rewards are earned by Collators --- pallets/storage-provider/PROOF-OF-STORAGE.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pallets/storage-provider/PROOF-OF-STORAGE.md b/pallets/storage-provider/PROOF-OF-STORAGE.md index ff6205135..99c2d577e 100644 --- a/pallets/storage-provider/PROOF-OF-STORAGE.md +++ b/pallets/storage-provider/PROOF-OF-STORAGE.md @@ -57,6 +57,9 @@ Before delving into the details of WinningPoSt and WindowPoSt algorithms, it is ### WinningPoSt +> [!NOTE] +> This is not relevant for our implementation as block rewards are earned by Collators. + At the beginning of each epoch, a small number of storage providers are elected to validate new blocks through the Expected Consensus algorithm. Each elected provider must submit proof that they maintain a sealed copy of the data included in their proposed block before the end of the current epoch. This proof submission is known as WinningPoSt. Successfully submitting a WinningPoSt proof grants the provider a block reward and the opportunity to charge fees for including transactions in the block. Failing to meet the deadline results in the provider missing the opportunity to validate a block and earn rewards. ### WindowPoSt From ab616dbedda0b95946f34f71a738af44d3028501 Mon Sep 17 00:00:00 2001 From: aidan Date: Mon, 10 Jun 2024 16:41:50 +0200 Subject: [PATCH 37/75] docs: Align design doc types Align the types mentioned in the design doc with the types in the pallet. --- pallets/storage-provider/DESIGN.md | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/pallets/storage-provider/DESIGN.md b/pallets/storage-provider/DESIGN.md index 5cb7f7966..957a53e58 100644 --- a/pallets/storage-provider/DESIGN.md +++ b/pallets/storage-provider/DESIGN.md @@ -18,35 +18,6 @@ The `Storage Provider Pallet` allows storage providers to modify their informati In our parachain, the state management for all storage providers is handled collectively, unlike Filecoin, which manages the state for individual storage providers. -### State Structure - -The State structure maintains all the necessary information about the storage providers. This structure includes details about funds, sectors, and deadlines. - -```rust -pub struct ProviderInfo { - // Contains static information about the storage provider. - pub info: Cid, - /// Total funds locked as pre_commit_deposit - pub pre_commit_deposits: u128, - /// Total rewards and added funds locked in vesting table - pub locked_funds: u128, - /// Sum of initial pledge requirements of all active sectors - pub initial_pledge: u128, - /// Sectors that have been pre-committed but not yet proven - pub pre_committed_sectors: Cid, - /// Allocated sector IDs. - pub allocated_sectors: Cid, - /// Information for all proven and not-yet-garbage-collected sectors - pub sectors: Cid, - /// The first block number in this storage provider's current proving period - pub proving_period_start: u64, - /// Index of the deadline within the proving period beginning at ProvingPeriodStart that has not yet been finalized - pub current_deadline: u64, - /// The sector numbers due for PoSt at each deadline in the current proving period, frozen at period start - pub deadlines: Cid, -} -``` - ### Static information about a Storage Provider The below struct and its fields ensure that all necessary static information about a Storage provider is encapsulated, allowing for efficient management and interaction within the parachain. From 31151798d311c1609bb8af52af48bc6d246fabc5 Mon Sep 17 00:00:00 2001 From: aidan Date: Mon, 10 Jun 2024 16:50:08 +0200 Subject: [PATCH 38/75] docs: Update docs --- pallets/storage-provider/DESIGN.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pallets/storage-provider/DESIGN.md b/pallets/storage-provider/DESIGN.md index 957a53e58..f2b52fd0e 100644 --- a/pallets/storage-provider/DESIGN.md +++ b/pallets/storage-provider/DESIGN.md @@ -79,11 +79,11 @@ The `SectorSize` indicates one of a set of possible sizes in the network. ```rust #[repr(u64)] pub enum SectorSize { - _2KiB = 2_048, - _8MiB = 8_388_608, - _512MiB = 536_870_912, - _32GiB = 34_359_738_368, - _64GiB = 68_719_476_736, + _2KiB + _8MiB, + _512MiB, + _32GiB, + _64GiB, } ``` From cc13ba8f4cd3e2e96d6876810eb522f73f44d913 Mon Sep 17 00:00:00 2001 From: aidan Date: Mon, 10 Jun 2024 18:02:37 +0200 Subject: [PATCH 39/75] fix: Remove unused attribute --- pallets/storage-provider/src/types.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/pallets/storage-provider/src/types.rs b/pallets/storage-provider/src/types.rs index 5a2cdcfbd..1fbd32105 100644 --- a/pallets/storage-provider/src/types.rs +++ b/pallets/storage-provider/src/types.rs @@ -115,7 +115,6 @@ pub enum RegisteredPoStProof { impl RegisteredPoStProof { /// Returns the sector size of the proof type, which is measured in bytes. - #[allow(unused)] pub fn sector_size(self) -> Result { use RegisteredPoStProof::*; match self { @@ -148,7 +147,6 @@ impl RegisteredPoStProof { } /// Returns the partition size, in sectors, associated with a proof type. /// The partition size is the number of sectors proven in a single PoSt proof. - #[allow(unused)] pub fn window_post_partitions_sector(self) -> Result { // Resolve to post proof and then compute size from that. use RegisteredPoStProof::*; From 03b96e6c4c09b91f0fab905ed204c9d7e7eae7df Mon Sep 17 00:00:00 2001 From: aidan Date: Tue, 11 Jun 2024 17:52:08 +0200 Subject: [PATCH 40/75] fix: Correct benchmark module name --- pallets/storage-provider/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pallets/storage-provider/src/lib.rs b/pallets/storage-provider/src/lib.rs index 2ea695767..2a489a64d 100644 --- a/pallets/storage-provider/src/lib.rs +++ b/pallets/storage-provider/src/lib.rs @@ -12,7 +12,7 @@ #![cfg_attr(not(feature = "std"), no_std)] #[cfg(feature = "runtime-benchmarks")] -mod benchmarking; +mod benchmarks; mod types; From 4fe085178947aa336a5f53266100ca26f9dd3fde Mon Sep 17 00:00:00 2001 From: aidan Date: Wed, 12 Jun 2024 14:29:46 +0200 Subject: [PATCH 41/75] docs: Change wording --- pallets/storage-provider/DESIGN.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pallets/storage-provider/DESIGN.md b/pallets/storage-provider/DESIGN.md index f2b52fd0e..64b3e7574 100644 --- a/pallets/storage-provider/DESIGN.md +++ b/pallets/storage-provider/DESIGN.md @@ -6,7 +6,7 @@ The `Storage Provider Pallet` handles the creation of storage providers and faci ## Usage -### Indexing storage providers +### Registering storage providers A storage provider indexes in the storage provider pallet itself when it starts up by calling the `create_storage_provider` extrinsic with it's `PeerId` as an argument. The public key will be extracted from the origin and is used to modify on-chain information and receive rewards. The `PeerId` is given by the storage provider so clients can use that to connect to the storage provider. From 812c7aba45ca97dbfe67cccda52c0768f55d75c6 Mon Sep 17 00:00:00 2001 From: aidan Date: Wed, 12 Jun 2024 14:30:37 +0200 Subject: [PATCH 42/75] docs: Clarify pallet API docs --- pallets/storage-provider/API.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pallets/storage-provider/API.md b/pallets/storage-provider/API.md index a575b9e74..fb3ab1eb6 100644 --- a/pallets/storage-provider/API.md +++ b/pallets/storage-provider/API.md @@ -1,4 +1,4 @@ -# Storage provider API +# Storage Provider pallet API ## Creating Storage Provider From e9aa1fa1038d343c6a38973311807d78fef4f898 Mon Sep 17 00:00:00 2001 From: aidan Date: Wed, 12 Jun 2024 14:32:02 +0200 Subject: [PATCH 43/75] docs: Update markdown headers --- pallets/storage-provider/DESIGN.md | 8 ++++---- pallets/storage-provider/src/types.rs | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pallets/storage-provider/DESIGN.md b/pallets/storage-provider/DESIGN.md index 64b3e7574..44888e8f2 100644 --- a/pallets/storage-provider/DESIGN.md +++ b/pallets/storage-provider/DESIGN.md @@ -54,9 +54,9 @@ pub struct StorageProviderInfo { ## Data structures -### Proof of spacetime +### Proof of Spacetime -Proof of spacetime indicates the version and the sector size of the proof. This type is used by the Storage Provider when initially starting up to indicate what PoSt version it will use to submit Window PoSt proof. +Proof of Spacetime indicates the version and the sector size of the proof. This type is used by the Storage Provider when initially starting up to indicate what PoSt version it will use to submit Window PoSt proof. ```rust pub enum RegisteredPoStProof { @@ -96,9 +96,9 @@ pub struct PoStProof { } ``` -### Proof of replication +### Proof of Replication -Proof of replication is used when a Storage Provider wants to store data on behalf of a client and receives a piece of client data. The data will first be placed in a sector after which that sector is sealed by the storage provider. Then a unique encoding, which serves as proof that the Storage Provider has replicated a copy of the data they agreed to store, is generated. Finally, the proof is compressed and submitted to the network as certification of storage. +Proof of Replication is used when a Storage Provider wants to store data on behalf of a client and receives a piece of client data. The data will first be placed in a sector after which that sector is sealed by the storage provider. Then a unique encoding, which serves as proof that the Storage Provider has replicated a copy of the data they agreed to store, is generated. Finally, the proof is compressed and submitted to the network as certification of storage. ```rust /// This type indicates the seal proof type which defines the version and the sector size diff --git a/pallets/storage-provider/src/types.rs b/pallets/storage-provider/src/types.rs index 1fbd32105..df091de85 100644 --- a/pallets/storage-provider/src/types.rs +++ b/pallets/storage-provider/src/types.rs @@ -97,7 +97,7 @@ pub enum SectorSize { _64GiB, } -/// Proof of spacetime type, indicating version and sector size of the proof. +/// Proof of Spacetime type, indicating version and sector size of the proof. #[derive(Debug, Decode, Encode, TypeInfo, PartialEq, Eq, Clone, Copy)] pub enum RegisteredPoStProof { StackedDRGWinning2KiBV1, @@ -161,7 +161,7 @@ impl RegisteredPoStProof { } } -/// Proof of spacetime data stored on chain. +/// Proof of Spacetime data stored on chain. #[derive(Debug, Decode, Encode, TypeInfo, PartialEq, Eq, Clone)] pub struct PoStProof { pub post_proof: RegisteredPoStProof, From 0dd19f9b1a3c0b991f58fd9baefa15798520e495 Mon Sep 17 00:00:00 2001 From: aidan Date: Wed, 12 Jun 2024 14:33:49 +0200 Subject: [PATCH 44/75] docs: Add reference to proof of storage docs --- pallets/storage-provider/DESIGN.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pallets/storage-provider/DESIGN.md b/pallets/storage-provider/DESIGN.md index 44888e8f2..544b6352c 100644 --- a/pallets/storage-provider/DESIGN.md +++ b/pallets/storage-provider/DESIGN.md @@ -56,6 +56,9 @@ pub struct StorageProviderInfo { ### Proof of Spacetime +> [!NOTE] +> For more information about proofs check out the [proof of storage docs](./PROOF-OF-STORAGE.md) + Proof of Spacetime indicates the version and the sector size of the proof. This type is used by the Storage Provider when initially starting up to indicate what PoSt version it will use to submit Window PoSt proof. ```rust @@ -98,6 +101,9 @@ pub struct PoStProof { ### Proof of Replication +> [!NOTE] +> For more information about proofs check out the [proof of storage docs](./PROOF-OF-STORAGE.md) + Proof of Replication is used when a Storage Provider wants to store data on behalf of a client and receives a piece of client data. The data will first be placed in a sector after which that sector is sealed by the storage provider. Then a unique encoding, which serves as proof that the Storage Provider has replicated a copy of the data they agreed to store, is generated. Finally, the proof is compressed and submitted to the network as certification of storage. ```rust From 011b068a20fe00e9a9ff9496c32f947bd209b41b Mon Sep 17 00:00:00 2001 From: aidan Date: Wed, 12 Jun 2024 14:37:10 +0200 Subject: [PATCH 45/75] fix: Remove 'winning' PoSt type --- pallets/storage-provider/DESIGN.md | 5 ----- pallets/storage-provider/src/types.rs | 32 +++++++++------------------ 2 files changed, 11 insertions(+), 26 deletions(-) diff --git a/pallets/storage-provider/DESIGN.md b/pallets/storage-provider/DESIGN.md index 544b6352c..41085dbe0 100644 --- a/pallets/storage-provider/DESIGN.md +++ b/pallets/storage-provider/DESIGN.md @@ -63,11 +63,6 @@ Proof of Spacetime indicates the version and the sector size of the proof. This ```rust pub enum RegisteredPoStProof { - StackedDRGWinning2KiBV1, - StackedDRGWinning8MiBV1, - StackedDRGWinning512MiBV1, - StackedDRGWinning32GiBV1, - StackedDRGWinning64GiBV1, StackedDRGWindow2KiBV1P1, StackedDRGWindow8MiBV1P1, StackedDRGWindow512MiBV1P1, diff --git a/pallets/storage-provider/src/types.rs b/pallets/storage-provider/src/types.rs index df091de85..f042923fb 100644 --- a/pallets/storage-provider/src/types.rs +++ b/pallets/storage-provider/src/types.rs @@ -100,11 +100,6 @@ pub enum SectorSize { /// Proof of Spacetime type, indicating version and sector size of the proof. #[derive(Debug, Decode, Encode, TypeInfo, PartialEq, Eq, Clone, Copy)] pub enum RegisteredPoStProof { - StackedDRGWinning2KiBV1, - StackedDRGWinning8MiBV1, - StackedDRGWinning512MiBV1, - StackedDRGWinning32GiBV1, - StackedDRGWinning64GiBV1, StackedDRGWindow2KiBV1P1, StackedDRGWindow8MiBV1P1, StackedDRGWindow512MiBV1P1, @@ -118,11 +113,11 @@ impl RegisteredPoStProof { pub fn sector_size(self) -> Result { use RegisteredPoStProof::*; match self { - StackedDRGWindow2KiBV1P1 | StackedDRGWinning2KiBV1 => Ok(SectorSize::_2KiB), - StackedDRGWindow8MiBV1P1 | StackedDRGWinning8MiBV1 => Ok(SectorSize::_8MiB), - StackedDRGWindow512MiBV1P1 | StackedDRGWinning512MiBV1 => Ok(SectorSize::_512MiB), - StackedDRGWindow32GiBV1P1 | StackedDRGWinning32GiBV1 => Ok(SectorSize::_32GiB), - StackedDRGWindow64GiBV1P1 | StackedDRGWinning64GiBV1 => Ok(SectorSize::_64GiB), + StackedDRGWindow2KiBV1P1 => Ok(SectorSize::_2KiB), + StackedDRGWindow8MiBV1P1 => Ok(SectorSize::_8MiB), + StackedDRGWindow512MiBV1P1 => Ok(SectorSize::_512MiB), + StackedDRGWindow32GiBV1P1 => Ok(SectorSize::_32GiB), + StackedDRGWindow64GiBV1P1 => Ok(SectorSize::_64GiB), Invalid(i) => Err(format!("unsupported proof type: {}", i)), } } @@ -132,12 +127,7 @@ impl RegisteredPoStProof { pub fn proof_size(self) -> Result { use RegisteredPoStProof::*; match self { - StackedDRGWinning2KiBV1 - | StackedDRGWinning8MiBV1 - | StackedDRGWinning512MiBV1 - | StackedDRGWinning32GiBV1 - | StackedDRGWinning64GiBV1 - | StackedDRGWindow2KiBV1P1 + StackedDRGWindow2KiBV1P1 | StackedDRGWindow8MiBV1P1 | StackedDRGWindow512MiBV1P1 | StackedDRGWindow32GiBV1P1 @@ -151,11 +141,11 @@ impl RegisteredPoStProof { // Resolve to post proof and then compute size from that. use RegisteredPoStProof::*; match self { - StackedDRGWinning64GiBV1 | StackedDRGWindow64GiBV1P1 => Ok(2300), - StackedDRGWinning32GiBV1 | StackedDRGWindow32GiBV1P1 => Ok(2349), - StackedDRGWinning2KiBV1 | StackedDRGWindow2KiBV1P1 => Ok(2), - StackedDRGWinning8MiBV1 | StackedDRGWindow8MiBV1P1 => Ok(2), - StackedDRGWinning512MiBV1 | StackedDRGWindow512MiBV1P1 => Ok(2), + StackedDRGWindow2KiBV1P1 => Ok(2), + StackedDRGWindow8MiBV1P1 => Ok(2), + StackedDRGWindow512MiBV1P1 => Ok(2), + StackedDRGWindow32GiBV1P1 => Ok(2349), + StackedDRGWindow64GiBV1P1 => Ok(2300), Invalid(i) => Err(format!("unsupported proof type: {}", i)), } } From 78cab9d5a50504b7d6babb7abd0b696eff200611 Mon Sep 17 00:00:00 2001 From: aidan Date: Wed, 12 Jun 2024 14:47:21 +0200 Subject: [PATCH 46/75] fix: Simplify PoSt proof for prototype --- pallets/storage-provider/src/types.rs | 38 ++++++--------------------- 1 file changed, 8 insertions(+), 30 deletions(-) diff --git a/pallets/storage-provider/src/types.rs b/pallets/storage-provider/src/types.rs index f042923fb..68e89a774 100644 --- a/pallets/storage-provider/src/types.rs +++ b/pallets/storage-provider/src/types.rs @@ -1,5 +1,4 @@ use codec::{Decode, Encode}; -use scale_info::prelude::format; use scale_info::prelude::string::String; use scale_info::prelude::vec::Vec; use scale_info::TypeInfo; @@ -61,10 +60,9 @@ where peer_id: PeerId, window_post_proof_type: RegisteredPoStProof, ) -> Result { - let sector_size = window_post_proof_type.sector_size()?; + let sector_size = window_post_proof_type.sector_size(); - let window_post_partition_sectors = - window_post_proof_type.window_post_partitions_sector()?; + let window_post_partition_sectors = window_post_proof_type.window_post_partitions_sector(); Ok(Self { owner, @@ -101,52 +99,32 @@ pub enum SectorSize { #[derive(Debug, Decode, Encode, TypeInfo, PartialEq, Eq, Clone, Copy)] pub enum RegisteredPoStProof { StackedDRGWindow2KiBV1P1, - StackedDRGWindow8MiBV1P1, - StackedDRGWindow512MiBV1P1, - StackedDRGWindow32GiBV1P1, - StackedDRGWindow64GiBV1P1, - Invalid(i64), } impl RegisteredPoStProof { /// Returns the sector size of the proof type, which is measured in bytes. - pub fn sector_size(self) -> Result { + pub fn sector_size(self) -> SectorSize { use RegisteredPoStProof::*; match self { - StackedDRGWindow2KiBV1P1 => Ok(SectorSize::_2KiB), - StackedDRGWindow8MiBV1P1 => Ok(SectorSize::_8MiB), - StackedDRGWindow512MiBV1P1 => Ok(SectorSize::_512MiB), - StackedDRGWindow32GiBV1P1 => Ok(SectorSize::_32GiB), - StackedDRGWindow64GiBV1P1 => Ok(SectorSize::_64GiB), - Invalid(i) => Err(format!("unsupported proof type: {}", i)), + StackedDRGWindow2KiBV1P1 => SectorSize::_2KiB, } } /// Proof size for each PoStProof type #[allow(unused)] - pub fn proof_size(self) -> Result { + pub fn proof_size(self) -> usize { use RegisteredPoStProof::*; match self { - StackedDRGWindow2KiBV1P1 - | StackedDRGWindow8MiBV1P1 - | StackedDRGWindow512MiBV1P1 - | StackedDRGWindow32GiBV1P1 - | StackedDRGWindow64GiBV1P1 => Ok(192), - Invalid(i) => Err(format!("unsupported proof type: {}", i)), + StackedDRGWindow2KiBV1P1 => 192, } } /// Returns the partition size, in sectors, associated with a proof type. /// The partition size is the number of sectors proven in a single PoSt proof. - pub fn window_post_partitions_sector(self) -> Result { + pub fn window_post_partitions_sector(self) -> u64 { // Resolve to post proof and then compute size from that. use RegisteredPoStProof::*; match self { - StackedDRGWindow2KiBV1P1 => Ok(2), - StackedDRGWindow8MiBV1P1 => Ok(2), - StackedDRGWindow512MiBV1P1 => Ok(2), - StackedDRGWindow32GiBV1P1 => Ok(2349), - StackedDRGWindow64GiBV1P1 => Ok(2300), - Invalid(i) => Err(format!("unsupported proof type: {}", i)), + StackedDRGWindow2KiBV1P1 => 2, } } } From b84bf82379b805c60f3a1c211f99bed4928a02fe Mon Sep 17 00:00:00 2001 From: aidan Date: Wed, 12 Jun 2024 14:50:53 +0200 Subject: [PATCH 47/75] docs: Fix mistake in create storage provider function --- pallets/storage-provider/API.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pallets/storage-provider/API.md b/pallets/storage-provider/API.md index fb3ab1eb6..390e1abca 100644 --- a/pallets/storage-provider/API.md +++ b/pallets/storage-provider/API.md @@ -7,7 +7,7 @@ When a Storage provider first starts up it needs to index itself in the storage ### Arguments `peer_id`: storage_provider's libp2p peer id in bytes. -`window_post_proof_type`: The total power the storage provider has +`window_post_proof_type`: The Proof of Spacetime type the storage provider will submit. ## Change peer id From 8c901533a7f036783fa4c562c4f5ca157f76229a Mon Sep 17 00:00:00 2001 From: aidan Date: Wed, 12 Jun 2024 15:03:16 +0200 Subject: [PATCH 48/75] docs: Add terminology section --- pallets/storage-provider/API.md | 3 +++ pallets/storage-provider/DESIGN.md | 9 +++++++++ pallets/storage-provider/PROOF-OF-STORAGE.md | 13 +++---------- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/pallets/storage-provider/API.md b/pallets/storage-provider/API.md index 390e1abca..ff89a2594 100644 --- a/pallets/storage-provider/API.md +++ b/pallets/storage-provider/API.md @@ -1,5 +1,8 @@ # Storage Provider pallet API +> [!NOTE] +> Some terms used in this document are described in the [design document](./DESIGN.md#constants--terminology) + ## Creating Storage Provider When a Storage provider first starts up it needs to index itself in the storage provider pallet. The `create_storage_provider` extrinsic is used for this. diff --git a/pallets/storage-provider/DESIGN.md b/pallets/storage-provider/DESIGN.md index 41085dbe0..810bbbc2b 100644 --- a/pallets/storage-provider/DESIGN.md +++ b/pallets/storage-provider/DESIGN.md @@ -4,6 +4,15 @@ The `Storage Provider Pallet` handles the creation of storage providers and facilitates storage providers and client in creating storage deals. +### Constants & Terminology + +- **Sector**: The sector is the default unit of storage that providers put in the network. A sector is a contiguous array of bytes that a storage provider puts together, seals, and performs Proofs of Spacetime on. Storage providers store data on the network in fixed-size sectors. +- **Partition**: A group of 2349 sectors proven simultaneously. +- **Proving Period**: The average period for proving all sectors maintained by a provider (currently set to 24 hours). +- **Deadline**: One of the multiple points during a proving period when proofs for some partitions are due. +- **Challenge Window**: The period immediately before a deadline during which a challenge can be generated by the chain and the requisite proofs computed. +- **Provider Size**: The amount of proven storage maintained by a single storage provider. + ## Usage ### Registering storage providers diff --git a/pallets/storage-provider/PROOF-OF-STORAGE.md b/pallets/storage-provider/PROOF-OF-STORAGE.md index 99c2d577e..7dfbce2f0 100644 --- a/pallets/storage-provider/PROOF-OF-STORAGE.md +++ b/pallets/storage-provider/PROOF-OF-STORAGE.md @@ -1,5 +1,8 @@ # Proof of Storage +> [!NOTE] +> Some terms used in this document are described in the [design document](./DESIGN.md#constants--terminology) + In our parachain within the Polkadot ecosystem, storage providers are required to prove that they hold a copy of the data they have committed to storing at any given point in time. This proof is achieved through a mechanism known as 'challenges.' The process involves the system posing specific questions to the storage providers, who must then provide correct answers to prove they are maintaining the data as promised. To ensure the integrity and reliability of these proofs, the challenges must: @@ -45,16 +48,6 @@ There are two types of challenges (and their corresponding mechanisms) within th - WinningPoSt: Proves that the storage provider has a replica of the data at the specific time they are challenged. A WinningPoSt challenge is issued to a storage provider only if they are selected through the Secret Leader Election algorithm to validate the next block. The answer to the WinningPoSt challenge must be submitted within a short deadline, making it impractical for the provider to reseal and find the answer on demand. This ensures that the provider maintains a copy of the data at the time of the challenge. - WindowPoSt: Proves that a copy of the data has been continuously maintained over time. Providers must submit proofs regularly, making it irrational for them to reseal the data every time a WindowPoSt challenge is issued. -### Constants & Terminology - -Before delving into the details of WinningPoSt and WindowPoSt algorithms, it is essential to clarify the following terms: - -- Partition: A group of 2349 sectors proven simultaneously. -- Proving Period: The average period for proving all sectors maintained by a provider (currently set to 24 hours). -- Deadline: One of the multiple points during a proving period when proofs for some partitions are due. -- Challenge Window: The period immediately before a deadline during which a challenge can be generated by the chain and the requisite proofs computed. -- Provider Size: The amount of proven storage maintained by a single storage provider. - ### WinningPoSt > [!NOTE] From d35d041e68e7944d930eb3a7e5b8cbb7f7ebef98 Mon Sep 17 00:00:00 2001 From: aidan Date: Wed, 12 Jun 2024 15:08:01 +0200 Subject: [PATCH 49/75] docs: Add references to terminology --- pallets/storage-provider/API.md | 33 ++++++++++---------- pallets/storage-provider/PROOF-OF-STORAGE.md | 30 +++++++++--------- 2 files changed, 31 insertions(+), 32 deletions(-) diff --git a/pallets/storage-provider/API.md b/pallets/storage-provider/API.md index ff89a2594..448f111e3 100644 --- a/pallets/storage-provider/API.md +++ b/pallets/storage-provider/API.md @@ -34,42 +34,41 @@ The `submit_windowed_post` is used by the storage provider to submit their Proof ### Arguments -`deadline`: The deadline index which the submission targets. -`partitions`: The partitions being proven. -`proofs`: An array of proofs, one per distinct registered proof type present in the sectors being proven. +`deadline`: The [deadline](./DESIGN.md#constants--terminology) at which the submission targets. +`partition`: The [partition](./DESIGN.md#constants--terminology)s being proven. +`proofs`: An array of proofs, one per distinct registered proof type present in the [sectors](./DESIGN.md#constants--terminology) being proven. ## Declaring faults -Storage providers can use the `declare_faults` extrinsic to declare a set of sectors as 'faulty', indicating that the next PoSt for those sectors' deadline will not contain a proof for those sectors' existence. +Storage providers can use the `declare_faults` extrinsic to declare a set of [sectors](./DESIGN.md#constants--terminology) as 'faulty', indicating that the next PoSt for those [sectors](./DESIGN.md#constants--terminology)' [deadline](./DESIGN.md#constants--terminology) will not contain a proof for those [sectors](./DESIGN.md#constants--terminology)' existence. ### Arguments -`deadline`: The deadline to which the faulty sectors are assigned -`partition`: Partition index within the deadline containing the faulty sectors. -`sectors`: Sectors in the partition being declared faulty. +`deadline`: The [deadline](./DESIGN.md#constants--terminology) to which the faulty [sectors](./DESIGN.md#constants--terminology) are assigned +`partition`: [Partition](./DESIGN.md#constants--terminology) index within the [deadline](./DESIGN.md#constants--terminology) containing the faulty [sectors](./DESIGN.md#constants--terminology). +`sectors`: [Sectors](./DESIGN.md#constants--terminology) in the [partition](./DESIGN.md#constants--terminology) being declared faulty. ## Declaring faults as recovered -Storage providers can declare a set of faulty sectors as "recovering", indicating that the next PoSt for those sectors' deadline will contain a proof for those sectors' existence. +Storage providers can declare a set of faulty [sectors](./DESIGN.md#constants--terminology) as "recovering", indicating that the next PoSt for those [sectors](./DESIGN.md#constants--terminology)' [deadline](./DESIGN.md#constants--terminology) will contain a proof for those [sectors](./DESIGN.md#constants--terminology)' existence. ### Arguments -`deadline`: The deadline to which the recovered sectors are assigned -`partition`: Partition index within the deadline containing the recovered sectors. -`sectors`: Sectors in the partition being declared recovered. +`deadline`: The [deadline](./DESIGN.md#constants--terminology) to which the recovered [sectors](./DESIGN.md#constants--terminology) are assigned +`partition`: [Partition](./DESIGN.md#constants--terminology) index within the [deadline](./DESIGN.md#constants--terminology) containing the recovered [sectors](./DESIGN.md#constants--terminology). +`sectors`: [Sectors](./DESIGN.md#constants--terminology) in the [partition](./DESIGN.md#constants--terminology) being declared recovered. -## Pre committing sectors +## Pre committing [sectors](./DESIGN.md#constants--terminology) -The Storage Provider can use the `pre_commit_sector` extrinsic to pledge to seal and commit some new sectors. +The Storage Provider can use the `pre_commit_sector` extrinsic to pledge to seal and commit some new [sectors](./DESIGN.md#constants--terminology). ### Arguments -`sectors`: Sectors to be committed. +`sectors`: [Sectors](./DESIGN.md#constants--terminology) to be committed. +## Prove commit [sectors](./DESIGN.md#constants--terminology) -## Prove commit sectors - -Storage providers can use the `prove_commit_sector` extrinsic to check the state of the corresponding sector pre-commitments and verifies aggregate proof of replication of these sectors. If valid, the sectors' deals are activated. +Storage providers can use the `prove_commit_sector` extrinsic to check the state of the corresponding sector pre-commitments and verifies aggregate proof of replication of these [sectors](./DESIGN.md#constants--terminology). If valid, the [sectors](./DESIGN.md#constants--terminology)' deals are activated. ### Arguments diff --git a/pallets/storage-provider/PROOF-OF-STORAGE.md b/pallets/storage-provider/PROOF-OF-STORAGE.md index 7dfbce2f0..57c12540f 100644 --- a/pallets/storage-provider/PROOF-OF-STORAGE.md +++ b/pallets/storage-provider/PROOF-OF-STORAGE.md @@ -45,7 +45,7 @@ From the point of committing to store data, storage providers must continuously There are two types of challenges (and their corresponding mechanisms) within the PoSt process: WinningPoSt and WindowPoSt, each serving a different purpose. -- WinningPoSt: Proves that the storage provider has a replica of the data at the specific time they are challenged. A WinningPoSt challenge is issued to a storage provider only if they are selected through the Secret Leader Election algorithm to validate the next block. The answer to the WinningPoSt challenge must be submitted within a short deadline, making it impractical for the provider to reseal and find the answer on demand. This ensures that the provider maintains a copy of the data at the time of the challenge. +- WinningPoSt: Proves that the storage provider has a replica of the data at the specific time they are challenged. A WinningPoSt challenge is issued to a storage provider only if they are selected through the Secret Leader Election algorithm to validate the next block. The answer to the WinningPoSt challenge must be submitted within a short [deadline](./DESIGN.md#constants--terminology), making it impractical for the provider to reseal and find the answer on demand. This ensures that the provider maintains a copy of the data at the time of the challenge. - WindowPoSt: Proves that a copy of the data has been continuously maintained over time. Providers must submit proofs regularly, making it irrational for them to reseal the data every time a WindowPoSt challenge is issued. ### WinningPoSt @@ -53,30 +53,30 @@ There are two types of challenges (and their corresponding mechanisms) within th > [!NOTE] > This is not relevant for our implementation as block rewards are earned by Collators. -At the beginning of each epoch, a small number of storage providers are elected to validate new blocks through the Expected Consensus algorithm. Each elected provider must submit proof that they maintain a sealed copy of the data included in their proposed block before the end of the current epoch. This proof submission is known as WinningPoSt. Successfully submitting a WinningPoSt proof grants the provider a block reward and the opportunity to charge fees for including transactions in the block. Failing to meet the deadline results in the provider missing the opportunity to validate a block and earn rewards. +At the beginning of each epoch, a small number of storage providers are elected to validate new blocks through the Expected Consensus algorithm. Each elected provider must submit proof that they maintain a sealed copy of the data included in their proposed block before the end of the current epoch. This proof submission is known as WinningPoSt. Successfully submitting a WinningPoSt proof grants the provider a block reward and the opportunity to charge fees for including transactions in the block. Failing to meet the [deadline](./DESIGN.md#constants--terminology) results in the provider missing the opportunity to validate a block and earn rewards. ### WindowPoSt -WindowPoSt audits the commitments made by storage providers. Every 24-hour period, known as a proving period, is divided into 30-minute, non-overlapping deadlines, totalling 48 deadlines per period. Providers must demonstrate the availability of all claimed sectors within this time frame. Each proof is limited to 2349 sectors (a partition), with 10 challenges per partition. -Sectors are assigned to deadlines and grouped into partitions. At each deadline, providers must prove an entire partition rather than individual sectors. For each partition, the provider generates a SNARK-compressed proof and publishes it to the blockchain. This process ensures that each sector is audited at least once every 24 hours, creating a permanent, verifiable record of the provider's commitment. -The more sectors a provider has pledged to store, the more partitions they must prove per deadline. This setup necessitates ready access to sealed copies of each challenged sector, making it impractical for the provider to reseal data each time a WindowPoSt proof is required. +WindowPoSt audits the commitments made by storage providers. Every 24-hour period, known as a [proving period](./DESIGN.md#constants--terminology), is divided into 30-minute, non-overlapping [deadline](./DESIGN.md#constants--terminology)s, totalling 48 [deadline](./DESIGN.md#constants--terminology)s per period. Providers must demonstrate the availability of all claimed [sectors](./DESIGN.md#constants--terminology) within this time frame. Each proof is limited to 2349 [sectors](./DESIGN.md#constants--terminology) (a partition), with 10 challenges per partition. +[Sectors](./DESIGN.md#constants--terminology) are assigned to [deadline](./DESIGN.md#constants--terminology)s and grouped into partitions. At each [deadline](./DESIGN.md#constants--terminology), providers must prove an entire partition rather than individual [sectors](./DESIGN.md#constants--terminology). For each partition, the provider generates a SNARK-compressed proof and publishes it to the blockchain. This process ensures that each sector is audited at least once every 24 hours, creating a permanent, verifiable record of the provider's commitment. +The more [sectors](./DESIGN.md#constants--terminology) a provider has pledged to store, the more partitions they must prove per [deadline](./DESIGN.md#constants--terminology). This setup necessitates ready access to sealed copies of each challenged sector, making it impractical for the provider to reseal data each time a WindowPoSt proof is required. ### Design of Proof-of-Spacetime -Each storage provider is allocated a 24-hour proving period upon creation, divided into 48 non-overlapping half-hour deadlines. Each sector is assigned to a specific deadline when proven to the chain and remains assigned to that deadline throughout its lifetime. Sectors are proven in partitions, and the set of sectors due at each deadline is recorded in a collection of 48 bitfields. +Each storage provider is allocated a 24-hour [proving period](./DESIGN.md#constants--terminology) upon creation, divided into 48 non-overlapping half-hour [deadline](./DESIGN.md#constants--terminology)s. Each sector is assigned to a specific [deadline](./DESIGN.md#constants--terminology) when proven to the chain and remains assigned to that [deadline](./DESIGN.md#constants--terminology) throughout its lifetime. [Sectors](./DESIGN.md#constants--terminology) are proven in partitions, and the set of [sectors](./DESIGN.md#constants--terminology) due at each [deadline](./DESIGN.md#constants--terminology) is recorded in a collection of 48 bitfields. -- Open: Epoch from which a PoSt Proof for this deadline can be submitted. -- Close: Epoch after which a PoSt Proof for this deadline will be rejected. -- FaultCutoff: Epoch after which fault declarations for sectors in the upcoming deadline are rejected. +- Open: Epoch from which a PoSt Proof for this [deadline](./DESIGN.md#constants--terminology) can be submitted. +- Close: Epoch after which a PoSt Proof for this [deadline](./DESIGN.md#constants--terminology) will be rejected. +- FaultCutoff: Epoch after which fault declarations for [sectors](./DESIGN.md#constants--terminology) in the upcoming [deadline](./DESIGN.md#constants--terminology) are rejected. - Challenge: Epoch at which the randomness for the challenges is available. ### PoSt Summary -- Storage providers maintain their sectors by generating Proofs-of-Spacetime (PoSt) and submitting WindowPoSt proofs for their sectors on time. -- WindowPoSt ensures that sectors are persistently stored over time. -- Each provider proves all their sectors once per proving period, with each sector proven by a specific deadline. -- The proving period is a 24-hour cycle divided into deadlines, each assigned to specific sectors. -- To prove continuous storage of a sector, providers must submit a WindowPoSt for each deadline. -- Sectors are grouped into partitions, with each partition proven in a single SNARK proof. +- Storage providers maintain their [sectors](./DESIGN.md#constants--terminology) by generating Proofs-of-Spacetime (PoSt) and submitting WindowPoSt proofs for their [sectors](./DESIGN.md#constants--terminology) on time. +- WindowPoSt ensures that [sectors](./DESIGN.md#constants--terminology) are persistently stored over time. +- Each provider proves all their [sectors](./DESIGN.md#constants--terminology) once per [proving period](./DESIGN.md#constants--terminology), with each sector proven by a specific [deadline](./DESIGN.md#constants--terminology). +- The [proving period](./DESIGN.md#constants--terminology) is a 24-hour cycle divided into [deadline](./DESIGN.md#constants--terminology)s, each assigned to specific [sectors](./DESIGN.md#constants--terminology). +- To prove continuous storage of a sector, providers must submit a WindowPoSt for each [deadline](./DESIGN.md#constants--terminology). +- [Sectors](./DESIGN.md#constants--terminology) are grouped into partitions, with each partition proven in a single SNARK proof. By implementing PoSt within our parachain, we ensure that storage providers are consistently accountable for the data they store, enhancing the integrity and reliability of our decentralized storage solution in the Polkadot ecosystem. From 9baefdbe262783abf6b61cc8f77e309ae5a65aaa Mon Sep 17 00:00:00 2001 From: aidan Date: Wed, 12 Jun 2024 16:37:36 +0200 Subject: [PATCH 50/75] docs: Add section about declaring faults --- pallets/storage-provider/DESIGN.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pallets/storage-provider/DESIGN.md b/pallets/storage-provider/DESIGN.md index 810bbbc2b..238309065 100644 --- a/pallets/storage-provider/DESIGN.md +++ b/pallets/storage-provider/DESIGN.md @@ -15,6 +15,9 @@ The `Storage Provider Pallet` handles the creation of storage providers and faci ## Usage +> [!NOTE] +> For more information about the storage provider pallet API check out [the designated docs](./API.md) + ### Registering storage providers A storage provider indexes in the storage provider pallet itself when it starts up by calling the `create_storage_provider` extrinsic with it's `PeerId` as an argument. The public key will be extracted from the origin and is used to modify on-chain information and receive rewards. The `PeerId` is given by the storage provider so clients can use that to connect to the storage provider. @@ -23,6 +26,14 @@ A storage provider indexes in the storage provider pallet itself when it starts The `Storage Provider Pallet` allows storage providers to modify their information such as changing the peer id, through `change_peer_id` and changing owners, through `change_owner_address`. +### Declaring storage faults + +A storage provider can declare sectors as faulty, through the `declare_faults`, for any sectors that it cannot generate `WindowPoSt` proofs. A storage provider has to declare the sector as faulty **before** the challenge window. Until the sectors are recovered they will be masked from proofs in subsequent proving periods. + +### Declaring storage faults recovered + +After a storage provider has declared some sectors as faulty, it can recover those sectors. The storage provider can use the `declare_faults_recovered` method to set the sectors it previously declared as faulty to recovering. + ## State management for Storage Providers In our parachain, the state management for all storage providers is handled collectively, unlike Filecoin, which manages the state for individual storage providers. From 5b5d171d2f7cca297352a9e99e538a4b4e468041 Mon Sep 17 00:00:00 2001 From: aidan Date: Wed, 12 Jun 2024 16:48:35 +0200 Subject: [PATCH 51/75] docs: Add section about slashing --- pallets/storage-provider/DESIGN.md | 43 ++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/pallets/storage-provider/DESIGN.md b/pallets/storage-provider/DESIGN.md index 238309065..ad8c6cb16 100644 --- a/pallets/storage-provider/DESIGN.md +++ b/pallets/storage-provider/DESIGN.md @@ -1,5 +1,22 @@ # Storage Provider Pallet +- [Overview](#overview) + - [Constants \& Terminology](#constants--terminology) +- [Usage](#usage) + - [Registering storage providers](#registering-storage-providers) + - [Modifying storage provider information](#modifying-storage-provider-information) + - [Declaring storage faults](#declaring-storage-faults) + - [Declaring storage faults recovered](#declaring-storage-faults-recovered) +- [Storage fault slashing](#storage-fault-slashing) + - [Fault Fee (FF)](#fault-fee-ff) + - [Sector Penalty (SP)](#sector-penalty-sp) + - [Termination Penalty (TP)](#termination-penalty-tp) + - [State management for Storage Providers](#state-management-for-storage-providers) + - [Static information about a Storage Provider](#static-information-about-a-storage-provider) +- [Data structures](#data-structures) + - [Proof of Spacetime](#proof-of-spacetime) + - [Proof of Replication](#proof-of-replication) + ## Overview The `Storage Provider Pallet` handles the creation of storage providers and facilitates storage providers and client in creating storage deals. @@ -34,7 +51,29 @@ A storage provider can declare sectors as faulty, through the `declare_faults`, After a storage provider has declared some sectors as faulty, it can recover those sectors. The storage provider can use the `declare_faults_recovered` method to set the sectors it previously declared as faulty to recovering. -## State management for Storage Providers +## Storage fault slashing + +Storage Fault Slashing refers to a set of penalties that storage providers may incur if they fail to maintain sector reliability or choose to voluntarily exit the network. These penalties include Fault Fees, Sector Penalties, and Termination Fees. Below is a detailed explanation of each type of penalty. + +### Fault Fee (FF) + +- **Description**: A penalty incurred by a storage provider for each day that a sector is offline. +- **Rationale**: Ensures that storage providers maintain high availability and reliability of their committed data. + +### Sector Penalty (SP) + +- **Description**: A penalty incurred by a storage provider for a sector that becomes faulted without being declared as such before a WindowPoSt (Proof-of-Spacetime) check. +- **Rationale**: Encourages storage providers to promptly declare any faults to avoid more severe penalties. +- **Details**: If a fault is detected during a WindowPoSt check, the sector will incur an SP and will continue to incur a FF until the fault is resolved. + +### Termination Penalty (TP) + +- **Description**: A penalty incurred when a sector is either voluntarily or involuntarily terminated and removed from the network. +- **Rationale**: Discourages storage providers from arbitrarily terminating sectors and ensures they fulfill their storage commitments. + +By implementing these penalties, storage providers are incentivised to maintain the reliability and availability of the data they store. This system of Storage Fault Slashing helps maintain the integrity and reliability of our decentralized storage network. + +### State management for Storage Providers In our parachain, the state management for all storage providers is handled collectively, unlike Filecoin, which manages the state for individual storage providers. @@ -153,4 +192,4 @@ pub struct SectorPreCommitInfo { pub sealed_cid: Cid, pub expiration: u64, } -``` \ No newline at end of file +``` From 994588864085bbd61a9f3c7a675dc1e7919d6aca Mon Sep 17 00:00:00 2001 From: aidan Date: Wed, 12 Jun 2024 16:52:34 +0200 Subject: [PATCH 52/75] fix: Move storage provider check --- pallets/storage-provider/src/lib.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pallets/storage-provider/src/lib.rs b/pallets/storage-provider/src/lib.rs index 2a489a64d..b0a38d327 100644 --- a/pallets/storage-provider/src/lib.rs +++ b/pallets/storage-provider/src/lib.rs @@ -57,7 +57,7 @@ pub mod pallet { StorageMap<_, _, T::AccountId, StorageProviderInfo>; #[pallet::event] - #[pallet::generate_deposit(pub(super) fn deposit_event)] + #[pallet::generate_deposit(fn deposit_event)] pub enum Event { /// This event is emitted when a new storage provider is initialized. StorageProviderCreated { owner: T::AccountId }, @@ -87,7 +87,7 @@ pub mod pallet { #[pallet::call] impl Pallet { - /// Index a new storage provider + /// Register a new storage provider #[pallet::call_index(0)] // #[pallet::weight(Weight::from_parts(10_000, 0) + T::DbWeight::get().writes(1))] // TODO(aidan46, no-ref, 2024-06-04): Determine applicable weights. @@ -99,16 +99,16 @@ pub mod pallet { // Check that the extrinsic was signed and get the signer. let owner = ensure_signed(origin)?; - // Generate some storage_provider id and insert into `StorageProviders` storage - let storage_provider_info = - StorageProviderInfo::new(owner.clone(), peer_id.clone(), window_post_proof_type) - .map_err(|_| Error::::StorageProviderInfoError)?; - // Probably need some check to make sure the storage provider is legit - // This means the storage provider exist + // This means the storage provider already exists ensure!( !StorageProviders::::contains_key(&owner), Error::::DuplicateStorageProvider ); + + // Generate some storage_provider id and insert into `StorageProviders` storage + let storage_provider_info = + StorageProviderInfo::new(owner.clone(), peer_id.clone(), window_post_proof_type) + .map_err(|_| Error::::StorageProviderInfoError)?; StorageProviders::::insert(owner.clone(), storage_provider_info); Self::deposit_event(Event::StorageProviderCreated { owner }); Ok(().into()) From b67214e0a7ddea95ccbdaf7c1f4ad673744949c0 Mon Sep 17 00:00:00 2001 From: aidan Date: Wed, 12 Jun 2024 17:07:50 +0200 Subject: [PATCH 53/75] refactor: Perform SP checks before mutating info --- pallets/storage-provider/src/lib.rs | 49 +++++++++++------------------ 1 file changed, 19 insertions(+), 30 deletions(-) diff --git a/pallets/storage-provider/src/lib.rs b/pallets/storage-provider/src/lib.rs index b0a38d327..e2b2e6c68 100644 --- a/pallets/storage-provider/src/lib.rs +++ b/pallets/storage-provider/src/lib.rs @@ -99,7 +99,7 @@ pub mod pallet { // Check that the extrinsic was signed and get the signer. let owner = ensure_signed(origin)?; - // This means the storage provider already exists + // This means the storage provider is already registered. ensure!( !StorageProviders::::contains_key(&owner), Error::::DuplicateStorageProvider @@ -123,40 +123,29 @@ pub mod pallet { peer_id: T::PeerId, ) -> DispatchResultWithPostInfo { // Check that the extrinsic was signed and get the signer. - let storage_provider = ensure_signed(origin)?; + let owner = ensure_signed(origin)?; - StorageProviders::::try_mutate( - &storage_provider, - |info| -> DispatchResultWithPostInfo { - let storage_provider_info = - match info.as_mut().ok_or(Error::::StorageProviderNotFound) { - Ok(info) => info, - Err(e) => { - log::error!( - "Could not get info for storage_provider: {storage_provider:?}" - ); - return Err(e.into()); - } - }; + // Fails if the SP has not been registered. + let sp = + StorageProviders::::get(&owner).ok_or(Error::::StorageProviderNotFound)?; - // Ensure storage_provider is the owner of the storage_provider - ensure!( - storage_provider == storage_provider_info.owner, - Error::::InvalidSigner - ); + // Ensure caller is the owner of SP + ensure!(owner == sp.owner, Error::::InvalidSigner); - log::debug!("Updating peer id for {storage_provider:?}"); + StorageProviders::::mutate(&owner, |info| { + // Can safely unwrap this because of previous `get` check + let sp_info = info.as_mut().unwrap(); - // Update PeerId - storage_provider_info.peer_id = peer_id.clone(); + log::debug!("Updating peer id for {owner:?}"); - Self::deposit_event(Event::PeerIdChanged { - storage_provider: storage_provider.clone(), - new_peer_id: peer_id, - }); - Ok(().into()) - }, - ) + sp_info.peer_id = peer_id.clone(); + + Self::deposit_event(Event::PeerIdChanged { + storage_provider: owner.clone(), + new_peer_id: peer_id, + }); + Ok(().into()) + }) } /// Update the owner address for a Storage Provider. From cf2bda176b832bf84b7dc04812076e00314038d9 Mon Sep 17 00:00:00 2001 From: aidan Date: Thu, 13 Jun 2024 13:09:54 +0200 Subject: [PATCH 54/75] docs: Correct API links --- pallets/storage-provider/DESIGN.md | 35 +++++++++++++++--------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/pallets/storage-provider/DESIGN.md b/pallets/storage-provider/DESIGN.md index ad8c6cb16..4a080ca2c 100644 --- a/pallets/storage-provider/DESIGN.md +++ b/pallets/storage-provider/DESIGN.md @@ -1,21 +1,22 @@ # Storage Provider Pallet -- [Overview](#overview) - - [Constants \& Terminology](#constants--terminology) -- [Usage](#usage) - - [Registering storage providers](#registering-storage-providers) - - [Modifying storage provider information](#modifying-storage-provider-information) - - [Declaring storage faults](#declaring-storage-faults) - - [Declaring storage faults recovered](#declaring-storage-faults-recovered) -- [Storage fault slashing](#storage-fault-slashing) - - [Fault Fee (FF)](#fault-fee-ff) - - [Sector Penalty (SP)](#sector-penalty-sp) - - [Termination Penalty (TP)](#termination-penalty-tp) - - [State management for Storage Providers](#state-management-for-storage-providers) - - [Static information about a Storage Provider](#static-information-about-a-storage-provider) -- [Data structures](#data-structures) - - [Proof of Spacetime](#proof-of-spacetime) - - [Proof of Replication](#proof-of-replication) +- [Storage Provider Pallet](#storage-provider-pallet) + - [Overview](#overview) + - [Constants \& Terminology](#constants--terminology) + - [Usage](#usage) + - [Registering storage providers](#registering-storage-providers) + - [Modifying storage provider information](#modifying-storage-provider-information) + - [Declaring storage faults](#declaring-storage-faults) + - [Declaring storage faults recovered](#declaring-storage-faults-recovered) + - [Storage fault slashing](#storage-fault-slashing) + - [Fault Fee (FF)](#fault-fee-ff) + - [Sector Penalty (SP)](#sector-penalty-sp) + - [Termination Penalty (TP)](#termination-penalty-tp) + - [State management for Storage Providers](#state-management-for-storage-providers) + - [Static information about a Storage Provider](#static-information-about-a-storage-provider) + - [Data structures](#data-structures) + - [Proof of Spacetime](#proof-of-spacetime) + - [Proof of Replication](#proof-of-replication) ## Overview @@ -33,7 +34,7 @@ The `Storage Provider Pallet` handles the creation of storage providers and faci ## Usage > [!NOTE] -> For more information about the storage provider pallet API check out [the designated docs](./API.md) +> For more information about the storage provider pallet API check out [the API docs](./API.md) ### Registering storage providers From 7e6583e48b2be84a073e7e32f4a563bb83a7a602 Mon Sep 17 00:00:00 2001 From: aidan Date: Thu, 13 Jun 2024 13:11:35 +0200 Subject: [PATCH 55/75] docs: Fix link pointing to master --- pallets/storage-provider/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pallets/storage-provider/src/lib.rs b/pallets/storage-provider/src/lib.rs index e2b2e6c68..d99b6da20 100644 --- a/pallets/storage-provider/src/lib.rs +++ b/pallets/storage-provider/src/lib.rs @@ -45,7 +45,7 @@ pub mod pallet { /// Peer ID is derived by hashing an encoded public key. /// Usually represented in bytes. - /// https://github.com/libp2p/specs/blob/master/peer-ids/peer-ids.md#peer-ids + /// https://github.com/libp2p/specs/blob/2ea41e8c769f1bead8e637a9d4ebf8c791976e8a/peer-ids/peer-ids.md#peer-ids type PeerId: Clone + Debug + Decode + Encode + Eq + TypeInfo; } From 342a5bfb4e504a5a7a96f8300bda8f2ce7ea666f Mon Sep 17 00:00:00 2001 From: aidan Date: Thu, 13 Jun 2024 13:12:20 +0200 Subject: [PATCH 56/75] fix: Rename error members --- pallets/storage-provider/src/lib.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pallets/storage-provider/src/lib.rs b/pallets/storage-provider/src/lib.rs index d99b6da20..a8a792926 100644 --- a/pallets/storage-provider/src/lib.rs +++ b/pallets/storage-provider/src/lib.rs @@ -75,13 +75,13 @@ pub mod pallet { #[pallet::error] pub enum Error { - /// Error emitted when the provided information for the storage provider is invalid. - StorageProviderInfoError, - /// Error emitted when trying to get info on a storage provider that does not exist. + /// Emitted when the provided information for the storage provider is invalid. + StorageProviderInfo, + /// Emitted when trying to get info on a storage provider that does not exist. StorageProviderNotFound, - /// Error emitted when doing a privileged call and the signer does not match. + /// Emitted when doing a privileged call and the signer does not match. InvalidSigner, - /// Error emitted when trying to create a storage provider that is already indexed. + /// Emitted when trying to create a storage provider that is already indexed. DuplicateStorageProvider, } From f26f2c7a7310b0693f391c3a11e777ea223f55c1 Mon Sep 17 00:00:00 2001 From: aidan Date: Thu, 13 Jun 2024 13:20:32 +0200 Subject: [PATCH 57/75] docs: Add docs about sealing --- pallets/storage-provider/DESIGN.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/pallets/storage-provider/DESIGN.md b/pallets/storage-provider/DESIGN.md index 4a080ca2c..42d94bbf7 100644 --- a/pallets/storage-provider/DESIGN.md +++ b/pallets/storage-provider/DESIGN.md @@ -14,6 +14,7 @@ - [Termination Penalty (TP)](#termination-penalty-tp) - [State management for Storage Providers](#state-management-for-storage-providers) - [Static information about a Storage Provider](#static-information-about-a-storage-provider) + - [Sector sealing](#sector-sealing) - [Data structures](#data-structures) - [Proof of Spacetime](#proof-of-spacetime) - [Proof of Replication](#proof-of-replication) @@ -112,6 +113,23 @@ pub struct StorageProviderInfo { } ``` +## Sector sealing + +Before a sector can be used, the storage provider must seal the sector, which involves encoding the data in the sector to prepare it for the proving process. + +- **Unsealed Sector**: An unsealed sector is a sector containing raw data that has not yet been sealed. +- **UnsealedCID (CommD)**: The root hash of the unsealed sector’s Merkle tree, also referred to as CommD or "data commitment." +- **Sealed Sector**: A sector that has been encoded and prepared for the proving process. +- **SealedCID (CommR)**: The root hash of the sealed sector’s Merkle tree, also referred to as CommR or "replica commitment." + +By sealing sectors, storage providers ensure that data is properly encoded and ready for the proof-of-storage process, maintaining the integrity and security of the stored data in the network. + +Sealing a sector using Proof-of-Replication (PoRep) is a computation-intensive process that results in a unique encoding of the sector. Once the data is sealed, storage providers follow these steps: + +- **Generate a Proof**: Create a proof that the data has been correctly sealed. +- **Run a SNARK on the Proof**: Compress the proof using a Succinct Non-interactive Argument of Knowledge (SNARK). +- **Submit the Compressed Proof:** Submit the result of the compression to the blockchain as certification of the storage commitment. + ## Data structures ### Proof of Spacetime From 5b009fb476b1e487df305ecf11b54ea358897bfc Mon Sep 17 00:00:00 2001 From: aidan Date: Thu, 13 Jun 2024 13:20:58 +0200 Subject: [PATCH 58/75] docs: Reword PoS docs --- pallets/storage-provider/PROOF-OF-STORAGE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pallets/storage-provider/PROOF-OF-STORAGE.md b/pallets/storage-provider/PROOF-OF-STORAGE.md index 57c12540f..a872fd671 100644 --- a/pallets/storage-provider/PROOF-OF-STORAGE.md +++ b/pallets/storage-provider/PROOF-OF-STORAGE.md @@ -12,7 +12,7 @@ To ensure the integrity and reliability of these proofs, the challenges must: General Proof-of-Storage (PoS) schemes are designed to allow users to verify that a storage provider is indeed storing the outsourced data at the time a challenge is issued. However, proving that data has been stored continuously over a period of time poses additional challenges. One method to address this is to require repeated challenges to the storage provider. However, this approach can lead to high communication complexity, which becomes a bottleneck, especially when storage providers must frequently submit proofs to the network. -To overcome the limitations of continuous Proof-of-Storage, we introduce a novel proof mechanism inspired by Filecoin, called Proof-of-Spacetime (PoSt). PoSt allows a verifier to check whether a storage provider has consistently stored the committed data over Space (the storage capacity) and Time (the duration). This method provides a more efficient and reliable means of proving data storage over extended periods, reducing the need for constant interaction and lowering the overall communication overhead. +To overcome the limitations of continuous Proof-of-Storage, there is proof called Proof-of-Spacetime (PoSt). PoSt allows a verifier to check whether a storage provider has consistently stored the committed data over Space (the storage capacity) and Time (the duration). This method provides a more efficient and reliable means of proving data storage over extended periods, reducing the need for constant interaction and lowering the overall communication overhead. By implementing PoSt, our parachain ensures that storage providers maintain the integrity of the data they store, providing a robust and scalable solution for decentralized storage within the Polkadot ecosystem. From 0e84fe41ea569b70915c742f90b5d0ce04d357ac Mon Sep 17 00:00:00 2001 From: aidan Date: Thu, 13 Jun 2024 13:22:50 +0200 Subject: [PATCH 59/75] docs: Add link to seal randomness --- pallets/storage-provider/PROOF-OF-STORAGE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pallets/storage-provider/PROOF-OF-STORAGE.md b/pallets/storage-provider/PROOF-OF-STORAGE.md index a872fd671..f62314597 100644 --- a/pallets/storage-provider/PROOF-OF-STORAGE.md +++ b/pallets/storage-provider/PROOF-OF-STORAGE.md @@ -26,7 +26,7 @@ The PoRep proof links together: 2. The storage provider who performs the sealing. 3. The time when the specific data was sealed by the specific storage provider. -If the same storage provider attempts to seal the same data at a later time, a different PoRep proof will be produced. The time is recorded as the blockchain height at which sealing took place, with the corresponding chain reference termed SealRandomness. +If the same storage provider attempts to seal the same data at a later time, a different PoRep proof will be produced. The time is recorded as the blockchain height at which sealing took place, with the corresponding chain reference termed [SealRandomness](https://spec.filecoin.io/systems/filecoin_mining/sector/sealing/#section-systems.filecoin_mining.sector.sealing.randomness). Generating and Submitting PoRep Proofs Once the proof is generated, the storage provider compresses it using a SNARK (Succinct Non-interactive Argument of Knowledge) and submits the result to the blockchain. This submission certifies that the storage provider has indeed replicated a copy of the data they committed to store. From 9181c815112275c7021e6e0cee872326f42e8b0b Mon Sep 17 00:00:00 2001 From: aidan Date: Thu, 13 Jun 2024 13:25:09 +0200 Subject: [PATCH 60/75] fix: Reduce seal proof types for prototype --- pallets/storage-provider/DESIGN.md | 26 +------------------------- pallets/storage-provider/src/types.rs | 23 ++--------------------- 2 files changed, 3 insertions(+), 46 deletions(-) diff --git a/pallets/storage-provider/DESIGN.md b/pallets/storage-provider/DESIGN.md index 42d94bbf7..bad8350cf 100644 --- a/pallets/storage-provider/DESIGN.md +++ b/pallets/storage-provider/DESIGN.md @@ -142,11 +142,6 @@ Proof of Spacetime indicates the version and the sector size of the proof. This ```rust pub enum RegisteredPoStProof { StackedDRGWindow2KiBV1P1, - StackedDRGWindow8MiBV1P1, - StackedDRGWindow512MiBV1P1, - StackedDRGWindow32GiBV1P1, - StackedDRGWindow64GiBV1P1, - Invalid(i64), } ``` @@ -155,11 +150,7 @@ The `SectorSize` indicates one of a set of possible sizes in the network. ```rust #[repr(u64)] pub enum SectorSize { - _2KiB - _8MiB, - _512MiB, - _32GiB, - _64GiB, + 2KiB } ``` @@ -182,22 +173,7 @@ Proof of Replication is used when a Storage Provider wants to store data on beha ```rust /// This type indicates the seal proof type which defines the version and the sector size pub enum RegisteredSealProof { - StackedDRG2KiBV1, - StackedDRG512MiBV1, - StackedDRG8MiBV1, - StackedDRG32GiBV1, - StackedDRG64GiBV1, StackedDRG2KiBV1P1, - StackedDRG512MiBV1P1, - StackedDRG8MiBV1P1, - StackedDRG32GiBV1P1, - StackedDRG64GiBV1P1, - StackedDRG2KiBV1P1_Feat_SyntheticPoRep, - StackedDRG512MiBV1P1_Feat_SyntheticPoRep, - StackedDRG8MiBV1P1_Feat_SyntheticPoRep, - StackedDRG32GiBV1P1_Feat_SyntheticPoRep, - StackedDRG64GiBV1P1_Feat_SyntheticPoRep, - Invalid(i64), } ``` diff --git a/pallets/storage-provider/src/types.rs b/pallets/storage-provider/src/types.rs index 68e89a774..d67ae7e7d 100644 --- a/pallets/storage-provider/src/types.rs +++ b/pallets/storage-provider/src/types.rs @@ -88,11 +88,7 @@ where /// SectorSize indicates one of a set of possible sizes in the network. #[derive(Encode, Decode, TypeInfo, Clone, Debug, PartialEq, Eq, Copy)] pub enum SectorSize { - _2KiB, - _8MiB, - _512MiB, - _32GiB, - _64GiB, + 2KiB, } /// Proof of Spacetime type, indicating version and sector size of the proof. @@ -106,7 +102,7 @@ impl RegisteredPoStProof { pub fn sector_size(self) -> SectorSize { use RegisteredPoStProof::*; match self { - StackedDRGWindow2KiBV1P1 => SectorSize::_2KiB, + StackedDRGWindow2KiBV1P1 => SectorSize::2KiB, } } @@ -140,22 +136,7 @@ pub struct PoStProof { #[allow(non_camel_case_types)] #[derive(Debug, Decode, Encode, TypeInfo, Eq, PartialEq, Clone)] pub enum RegisteredSealProof { - StackedDRG2KiBV1, - StackedDRG512MiBV1, - StackedDRG8MiBV1, - StackedDRG32GiBV1, - StackedDRG64GiBV1, StackedDRG2KiBV1P1, - StackedDRG512MiBV1P1, - StackedDRG8MiBV1P1, - StackedDRG32GiBV1P1, - StackedDRG64GiBV1P1, - StackedDRG2KiBV1P1_Feat_SyntheticPoRep, - StackedDRG512MiBV1P1_Feat_SyntheticPoRep, - StackedDRG8MiBV1P1_Feat_SyntheticPoRep, - StackedDRG32GiBV1P1_Feat_SyntheticPoRep, - StackedDRG64GiBV1P1_Feat_SyntheticPoRep, - Invalid(i64), } /// This type is passed into the pre commit function on the storage provider pallet From 688e2b711ef241ac4a151de13dd22bfb148ec3a6 Mon Sep 17 00:00:00 2001 From: aidan Date: Thu, 13 Jun 2024 13:32:52 +0200 Subject: [PATCH 61/75] fix: Error renaming --- pallets/storage-provider/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pallets/storage-provider/src/lib.rs b/pallets/storage-provider/src/lib.rs index a8a792926..aac8f6249 100644 --- a/pallets/storage-provider/src/lib.rs +++ b/pallets/storage-provider/src/lib.rs @@ -108,7 +108,7 @@ pub mod pallet { // Generate some storage_provider id and insert into `StorageProviders` storage let storage_provider_info = StorageProviderInfo::new(owner.clone(), peer_id.clone(), window_post_proof_type) - .map_err(|_| Error::::StorageProviderInfoError)?; + .map_err(|_| Error::::StorageProviderInfo)?; StorageProviders::::insert(owner.clone(), storage_provider_info); Self::deposit_event(Event::StorageProviderCreated { owner }); Ok(().into()) From 4300ab72298041896397368ba9a6a42bebf55b95 Mon Sep 17 00:00:00 2001 From: aidan Date: Thu, 13 Jun 2024 13:35:27 +0200 Subject: [PATCH 62/75] fix: Add _ prefix to SectorSize members --- pallets/storage-provider/DESIGN.md | 2 +- pallets/storage-provider/src/types.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pallets/storage-provider/DESIGN.md b/pallets/storage-provider/DESIGN.md index bad8350cf..2b764e5ca 100644 --- a/pallets/storage-provider/DESIGN.md +++ b/pallets/storage-provider/DESIGN.md @@ -150,7 +150,7 @@ The `SectorSize` indicates one of a set of possible sizes in the network. ```rust #[repr(u64)] pub enum SectorSize { - 2KiB + _2KiB, } ``` diff --git a/pallets/storage-provider/src/types.rs b/pallets/storage-provider/src/types.rs index d67ae7e7d..6b1f2b723 100644 --- a/pallets/storage-provider/src/types.rs +++ b/pallets/storage-provider/src/types.rs @@ -88,7 +88,7 @@ where /// SectorSize indicates one of a set of possible sizes in the network. #[derive(Encode, Decode, TypeInfo, Clone, Debug, PartialEq, Eq, Copy)] pub enum SectorSize { - 2KiB, + _2KiB, } /// Proof of Spacetime type, indicating version and sector size of the proof. @@ -102,7 +102,7 @@ impl RegisteredPoStProof { pub fn sector_size(self) -> SectorSize { use RegisteredPoStProof::*; match self { - StackedDRGWindow2KiBV1P1 => SectorSize::2KiB, + StackedDRGWindow2KiBV1P1 => SectorSize::_2KiB, } } From 5185e2247d63ed8fa0ccc5ec136d1cd172fb381b Mon Sep 17 00:00:00 2001 From: aidan Date: Fri, 14 Jun 2024 13:14:01 +0200 Subject: [PATCH 63/75] docs: Update Storage provider overview --- pallets/storage-provider/DESIGN.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pallets/storage-provider/DESIGN.md b/pallets/storage-provider/DESIGN.md index 2b764e5ca..0a962b980 100644 --- a/pallets/storage-provider/DESIGN.md +++ b/pallets/storage-provider/DESIGN.md @@ -21,7 +21,7 @@ ## Overview -The `Storage Provider Pallet` handles the creation of storage providers and facilitates storage providers and client in creating storage deals. +The `Storage Provider Pallet` handles the creation of storage providers and facilitates storage providers and client in creating storage deals. Storage providers must provide Proof of Spacetime and Proof of Replication to the `Storage Provider Pallet` in order to prevent the pallet impose penalties on the storage providers through [slashing](#storage-fault-slashing). ### Constants & Terminology From d0325b4d0dc445d7508328c593bfe64d609ad69c Mon Sep 17 00:00:00 2001 From: aidan Date: Fri, 14 Jun 2024 16:12:34 +0200 Subject: [PATCH 64/75] docs: Change wording for proving period --- pallets/storage-provider/DESIGN.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pallets/storage-provider/DESIGN.md b/pallets/storage-provider/DESIGN.md index 0a962b980..b6d9aae09 100644 --- a/pallets/storage-provider/DESIGN.md +++ b/pallets/storage-provider/DESIGN.md @@ -27,7 +27,7 @@ The `Storage Provider Pallet` handles the creation of storage providers and faci - **Sector**: The sector is the default unit of storage that providers put in the network. A sector is a contiguous array of bytes that a storage provider puts together, seals, and performs Proofs of Spacetime on. Storage providers store data on the network in fixed-size sectors. - **Partition**: A group of 2349 sectors proven simultaneously. -- **Proving Period**: The average period for proving all sectors maintained by a provider (currently set to 24 hours). +- **Proving Period**: The average period for proving all sectors maintained by a provider (default set to 24 hours). - **Deadline**: One of the multiple points during a proving period when proofs for some partitions are due. - **Challenge Window**: The period immediately before a deadline during which a challenge can be generated by the chain and the requisite proofs computed. - **Provider Size**: The amount of proven storage maintained by a single storage provider. From 031ded0f648fc0ef497e18a325b0bc8bf63f2c9d Mon Sep 17 00:00:00 2001 From: aidan Date: Fri, 14 Jun 2024 18:10:35 +0200 Subject: [PATCH 65/75] docs: Rename epoch to Blocknumber --- pallets/storage-provider/PROOF-OF-STORAGE.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pallets/storage-provider/PROOF-OF-STORAGE.md b/pallets/storage-provider/PROOF-OF-STORAGE.md index f62314597..d8eb1b4ff 100644 --- a/pallets/storage-provider/PROOF-OF-STORAGE.md +++ b/pallets/storage-provider/PROOF-OF-STORAGE.md @@ -53,7 +53,7 @@ There are two types of challenges (and their corresponding mechanisms) within th > [!NOTE] > This is not relevant for our implementation as block rewards are earned by Collators. -At the beginning of each epoch, a small number of storage providers are elected to validate new blocks through the Expected Consensus algorithm. Each elected provider must submit proof that they maintain a sealed copy of the data included in their proposed block before the end of the current epoch. This proof submission is known as WinningPoSt. Successfully submitting a WinningPoSt proof grants the provider a block reward and the opportunity to charge fees for including transactions in the block. Failing to meet the [deadline](./DESIGN.md#constants--terminology) results in the provider missing the opportunity to validate a block and earn rewards. +At the beginning of each block, a small number of storage providers are elected to validate new blocks through the Expected Consensus algorithm. Each elected provider must submit proof that they maintain a sealed copy of the data included in their proposed block before the end of the current block. This proof submission is known as WinningPoSt. Successfully submitting a WinningPoSt proof grants the provider a block reward and the opportunity to charge fees for including transactions in the block. Failing to meet the [deadline](./DESIGN.md#constants--terminology) results in the provider missing the opportunity to validate a block and earn rewards. ### WindowPoSt @@ -65,10 +65,10 @@ The more [sectors](./DESIGN.md#constants--terminology) a provider has pledged to Each storage provider is allocated a 24-hour [proving period](./DESIGN.md#constants--terminology) upon creation, divided into 48 non-overlapping half-hour [deadline](./DESIGN.md#constants--terminology)s. Each sector is assigned to a specific [deadline](./DESIGN.md#constants--terminology) when proven to the chain and remains assigned to that [deadline](./DESIGN.md#constants--terminology) throughout its lifetime. [Sectors](./DESIGN.md#constants--terminology) are proven in partitions, and the set of [sectors](./DESIGN.md#constants--terminology) due at each [deadline](./DESIGN.md#constants--terminology) is recorded in a collection of 48 bitfields. -- Open: Epoch from which a PoSt Proof for this [deadline](./DESIGN.md#constants--terminology) can be submitted. -- Close: Epoch after which a PoSt Proof for this [deadline](./DESIGN.md#constants--terminology) will be rejected. -- FaultCutoff: Epoch after which fault declarations for [sectors](./DESIGN.md#constants--terminology) in the upcoming [deadline](./DESIGN.md#constants--terminology) are rejected. -- Challenge: Epoch at which the randomness for the challenges is available. +- Open: BlockNumber from which a PoSt Proof for this [deadline](./DESIGN.md#constants--terminology) can be submitted. +- Close: BlockNumber after which a PoSt Proof for this [deadline](./DESIGN.md#constants--terminology) will be rejected. +- FaultCutoff: BlockNumber after which fault declarations for [sectors](./DESIGN.md#constants--terminology) in the upcoming [deadline](./DESIGN.md#constants--terminology) are rejected. +- Challenge: BlockNumber at which the randomness for the challenges is available. ### PoSt Summary From e94b7c3dda5ccfcaebbfe059b6d278a969d45bbf Mon Sep 17 00:00:00 2001 From: aidan Date: Sun, 16 Jun 2024 18:19:42 +0200 Subject: [PATCH 66/75] docs: Add storage provider flow --- pallets/storage-provider/DESIGN.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/pallets/storage-provider/DESIGN.md b/pallets/storage-provider/DESIGN.md index b6d9aae09..dd51c7e4f 100644 --- a/pallets/storage-provider/DESIGN.md +++ b/pallets/storage-provider/DESIGN.md @@ -18,6 +18,10 @@ - [Data structures](#data-structures) - [Proof of Spacetime](#proof-of-spacetime) - [Proof of Replication](#proof-of-replication) + - [Storage Provider Flow](#storage-provider-flow) + - [Registration](#registration) + - [Commit](#commit) + - [Proof of Spacetime submission](#proof-of-spacetime-submission) ## Overview @@ -188,3 +192,21 @@ pub struct SectorPreCommitInfo { pub expiration: u64, } ``` + +## Storage Provider Flow + +### Registration + +The first thing a storage provider must do is register itself by calling `storage_provider.create_storage_provider(peer_id: PeerId, window_post_proof_type: RegisteredPoStProof)`. At this point there are no funds locked in the storage provider pallet. The next step is to place storage market asks on the market, this is done through the market pallet. After that the storage provider needs to make deals with clients and begin filling up sectors with data. When they have a full sector they should seal the sector. + +### Commit + +When the storage provider has completed their first seal, they should post it to the storage provider pallet by calling `storage_provider.pre_commit_sector(sectors: SectorPreCommitInfo)`. If the storage provider had zero committed sectors before this call, this begins their proving period. The proving period is a fixed amount of time in which the storage provider must submit a Proof of Space Time to the network. +During this period, the storage provider may also commit to new sectors, but they will not be included in proofs of space time until the next proving period starts. + +### Proof of Spacetime submission + +When the storage provider has completed their PoSt, they must submit it to the network by calling `storage_provider.submit_windowed_post(deadline: u64, partitions: Vec, proofs: Vec)`. There are two different types of submissions: + +- **Standard Submission**: A standard submission is one that makes it on-chain before the end of the proving period. +- **Penalize Submission**:A penalized submission is one that makes it on-chain after the end of the proving period, but before the generation attack threshold. These submissions count as valid PoSt submissions, but the miner must pay a penalty for their late submission. See [storage fault slashing](#storage-fault-slashing). From 21caf6a971fe509b130f6a1bed61eac39456e0af Mon Sep 17 00:00:00 2001 From: aidan Date: Sun, 16 Jun 2024 18:20:13 +0200 Subject: [PATCH 67/75] docs: Remove API.md --- pallets/storage-provider/API.md | 76 --------------------------------- 1 file changed, 76 deletions(-) delete mode 100644 pallets/storage-provider/API.md diff --git a/pallets/storage-provider/API.md b/pallets/storage-provider/API.md deleted file mode 100644 index 448f111e3..000000000 --- a/pallets/storage-provider/API.md +++ /dev/null @@ -1,76 +0,0 @@ -# Storage Provider pallet API - -> [!NOTE] -> Some terms used in this document are described in the [design document](./DESIGN.md#constants--terminology) - -## Creating Storage Provider - -When a Storage provider first starts up it needs to index itself in the storage provider pallet. The `create_storage_provider` extrinsic is used for this. - -### Arguments - -`peer_id`: storage_provider's libp2p peer id in bytes. -`window_post_proof_type`: The Proof of Spacetime type the storage provider will submit. - -## Change peer id - -The `change_peer_id` extrinsic is used by the Storage Provider to update its peer id. - -### Arguments - -`peer_id`: The new peer id for the Storage Provider. - -## Change owner address - -The `change_owner_address` extrinsic is used by the Storage Provider to update its owner. - -### Arguments - -`new_owner`: The address of the new owner. - -## Submitting Proof of Spacetime - -The `submit_windowed_post` is used by the storage provider to submit their Proof of Spacetime - -### Arguments - -`deadline`: The [deadline](./DESIGN.md#constants--terminology) at which the submission targets. -`partition`: The [partition](./DESIGN.md#constants--terminology)s being proven. -`proofs`: An array of proofs, one per distinct registered proof type present in the [sectors](./DESIGN.md#constants--terminology) being proven. - -## Declaring faults - -Storage providers can use the `declare_faults` extrinsic to declare a set of [sectors](./DESIGN.md#constants--terminology) as 'faulty', indicating that the next PoSt for those [sectors](./DESIGN.md#constants--terminology)' [deadline](./DESIGN.md#constants--terminology) will not contain a proof for those [sectors](./DESIGN.md#constants--terminology)' existence. - -### Arguments - -`deadline`: The [deadline](./DESIGN.md#constants--terminology) to which the faulty [sectors](./DESIGN.md#constants--terminology) are assigned -`partition`: [Partition](./DESIGN.md#constants--terminology) index within the [deadline](./DESIGN.md#constants--terminology) containing the faulty [sectors](./DESIGN.md#constants--terminology). -`sectors`: [Sectors](./DESIGN.md#constants--terminology) in the [partition](./DESIGN.md#constants--terminology) being declared faulty. - -## Declaring faults as recovered - -Storage providers can declare a set of faulty [sectors](./DESIGN.md#constants--terminology) as "recovering", indicating that the next PoSt for those [sectors](./DESIGN.md#constants--terminology)' [deadline](./DESIGN.md#constants--terminology) will contain a proof for those [sectors](./DESIGN.md#constants--terminology)' existence. - -### Arguments - -`deadline`: The [deadline](./DESIGN.md#constants--terminology) to which the recovered [sectors](./DESIGN.md#constants--terminology) are assigned -`partition`: [Partition](./DESIGN.md#constants--terminology) index within the [deadline](./DESIGN.md#constants--terminology) containing the recovered [sectors](./DESIGN.md#constants--terminology). -`sectors`: [Sectors](./DESIGN.md#constants--terminology) in the [partition](./DESIGN.md#constants--terminology) being declared recovered. - -## Pre committing [sectors](./DESIGN.md#constants--terminology) - -The Storage Provider can use the `pre_commit_sector` extrinsic to pledge to seal and commit some new [sectors](./DESIGN.md#constants--terminology). - -### Arguments - -`sectors`: [Sectors](./DESIGN.md#constants--terminology) to be committed. - -## Prove commit [sectors](./DESIGN.md#constants--terminology) - -Storage providers can use the `prove_commit_sector` extrinsic to check the state of the corresponding sector pre-commitments and verifies aggregate proof of replication of these [sectors](./DESIGN.md#constants--terminology). If valid, the [sectors](./DESIGN.md#constants--terminology)' deals are activated. - -### Arguments - -`sector_number`: The sector number to be proved. -`proof`: The proof, supplied by the storage provider. From c9fa3ba315ee46b5b9223390a91cc471c1188599 Mon Sep 17 00:00:00 2001 From: aidan Date: Sun, 16 Jun 2024 19:56:14 +0200 Subject: [PATCH 68/75] fix: Remove code skeleton --- pallets/storage-provider/src/lib.rs | 222 +--------------------------- 1 file changed, 4 insertions(+), 218 deletions(-) diff --git a/pallets/storage-provider/src/lib.rs b/pallets/storage-provider/src/lib.rs index aac8f6249..54109acb7 100644 --- a/pallets/storage-provider/src/lib.rs +++ b/pallets/storage-provider/src/lib.rs @@ -20,18 +20,11 @@ pub use pallet::{Config, Pallet}; #[frame_support::pallet(dev_mode)] pub mod pallet { - use crate::types::{ - PoStProof, RegisteredPoStProof, SectorNumber, SectorPreCommitInfo, StorageProviderInfo, - }; + use crate::types::StorageProviderInfo; use codec::{Decode, Encode}; use core::fmt::Debug; - use frame_support::dispatch::DispatchResultWithPostInfo; - use frame_support::ensure; use frame_support::pallet_prelude::{IsType, StorageMap}; - use frame_system::ensure_signed; - use frame_system::pallet_prelude::OriginFor; - use scale_info::prelude::vec::Vec; use scale_info::TypeInfo; #[pallet::pallet] @@ -57,218 +50,11 @@ pub mod pallet { StorageMap<_, _, T::AccountId, StorageProviderInfo>; #[pallet::event] - #[pallet::generate_deposit(fn deposit_event)] - pub enum Event { - /// This event is emitted when a new storage provider is initialized. - StorageProviderCreated { owner: T::AccountId }, - /// This event is emitted when a storage provider changes its `PeerId`. - PeerIdChanged { - storage_provider: T::AccountId, - new_peer_id: T::PeerId, - }, - /// This event is emitted when a storage provider changes its owner. - OwnerAddressChanged { - storage_provider: T::AccountId, - new_owner: T::AccountId, - }, - } + pub enum Event {} #[pallet::error] - pub enum Error { - /// Emitted when the provided information for the storage provider is invalid. - StorageProviderInfo, - /// Emitted when trying to get info on a storage provider that does not exist. - StorageProviderNotFound, - /// Emitted when doing a privileged call and the signer does not match. - InvalidSigner, - /// Emitted when trying to create a storage provider that is already indexed. - DuplicateStorageProvider, - } + pub enum Error {} #[pallet::call] - impl Pallet { - /// Register a new storage provider - #[pallet::call_index(0)] - // #[pallet::weight(Weight::from_parts(10_000, 0) + T::DbWeight::get().writes(1))] - // TODO(aidan46, no-ref, 2024-06-04): Determine applicable weights. - pub fn create_storage_provider( - origin: OriginFor, - peer_id: T::PeerId, - window_post_proof_type: RegisteredPoStProof, - ) -> DispatchResultWithPostInfo { - // Check that the extrinsic was signed and get the signer. - let owner = ensure_signed(origin)?; - - // This means the storage provider is already registered. - ensure!( - !StorageProviders::::contains_key(&owner), - Error::::DuplicateStorageProvider - ); - - // Generate some storage_provider id and insert into `StorageProviders` storage - let storage_provider_info = - StorageProviderInfo::new(owner.clone(), peer_id.clone(), window_post_proof_type) - .map_err(|_| Error::::StorageProviderInfo)?; - StorageProviders::::insert(owner.clone(), storage_provider_info); - Self::deposit_event(Event::StorageProviderCreated { owner }); - Ok(().into()) - } - - /// Update PeerId for a Storage Provider. - #[pallet::call_index(1)] - // #[pallet::weight(Weight::from_parts(10_000, 0) + T::DbWeight::get().writes(1))] - // TODO(aidan46, no-ref, 2024-06-04): Determine applicable weights. - pub fn change_peer_id( - origin: OriginFor, - peer_id: T::PeerId, - ) -> DispatchResultWithPostInfo { - // Check that the extrinsic was signed and get the signer. - let owner = ensure_signed(origin)?; - - // Fails if the SP has not been registered. - let sp = - StorageProviders::::get(&owner).ok_or(Error::::StorageProviderNotFound)?; - - // Ensure caller is the owner of SP - ensure!(owner == sp.owner, Error::::InvalidSigner); - - StorageProviders::::mutate(&owner, |info| { - // Can safely unwrap this because of previous `get` check - let sp_info = info.as_mut().unwrap(); - - log::debug!("Updating peer id for {owner:?}"); - - sp_info.peer_id = peer_id.clone(); - - Self::deposit_event(Event::PeerIdChanged { - storage_provider: owner.clone(), - new_peer_id: peer_id, - }); - Ok(().into()) - }) - } - - /// Update the owner address for a Storage Provider. - #[pallet::call_index(2)] - // #[pallet::weight(Weight::from_parts(10_000, 0) + T::DbWeight::get().writes(1))] - // TODO(aidan46, no-ref, 2024-06-04): Determine applicable weights. - pub fn change_owner_address( - origin: OriginFor, - new_owner: T::AccountId, - ) -> DispatchResultWithPostInfo { - // Check that the extrinsic was signed and get the signer. - let storage_provider = ensure_signed(origin)?; - - // Extract storage provider - match StorageProviders::::try_get(&storage_provider) { - Ok(info) => { - // Ensure storage_provider is the owner of the storage_provider - ensure!(storage_provider == info.owner, Error::::InvalidSigner); - - // Ensure no storage provider is associated with the new owner - ensure!( - !StorageProviders::::contains_key(&new_owner), - Error::::DuplicateStorageProvider - ); - - let new_info = info.change_owner(new_owner.clone()); - - // Insert new storage provider info - StorageProviders::::insert(new_owner.clone(), new_info); - - // Remove old storage provider entry - StorageProviders::::remove(storage_provider.clone()); - - // Emit event - Self::deposit_event(Event::OwnerAddressChanged { - storage_provider: storage_provider.clone(), - new_owner, - }); - - Ok(().into()) - } - Err(..) => Err(Error::::StorageProviderNotFound.into()), - } - } - - /// Used by the storage provider to submit their Proof-of-Spacetime - #[pallet::call_index(3)] - // #[pallet::weight(Weight::from_parts(10_000, 0) + T::DbWeight::get().writes(1))] - // TODO(aidan46, no-ref, 2024-06-04): Determine applicable weights. - pub fn submit_windowed_post( - origin: OriginFor, - _deadline: u64, - _partitions: Vec, - _proofs: Vec, - ) -> DispatchResultWithPostInfo { - // Check that the extrinsic was signed and get the signer. - let _who = ensure_signed(origin)?; - // TODO(@aidan46, no-ref, 2024-06-04): Implement submit windowed PoSt functionality - unimplemented!("Submit windowed PoSt is not implemented yet") - } - - /// Used to declare a set of sectors as "faulty," indicating that the next PoSt for those sectors' - /// deadline will not contain a proof for those sectors' existence. - #[pallet::call_index(4)] - // #[pallet::weight(Weight::from_parts(10_000, 0) + T::DbWeight::get().writes(1))] - // TODO(aidan46, no-ref, 2024-06-05): Determine applicable weights. - pub fn declare_faults( - origin: OriginFor, - _deadline: u64, - _partition: u64, - _sectors: Vec, - ) -> DispatchResultWithPostInfo { - // Check that the extrinsic was signed and get the signer. - let _who = ensure_signed(origin)?; - // TODO(@aidan46, no-ref, 2024-06-05): Implement declare faults functionality - unimplemented!("Declare faults is not implemented yet") - } - - /// Used by a Storage Provider to declare a set of faulty sectors as "recovering," indicating that the - /// next PoSt for those sectors' deadline will contain a proof for those sectors' existence. - #[pallet::call_index(5)] - // #[pallet::weight(Weight::from_parts(10_000, 0) + T::DbWeight::get().writes(1))] - // TODO(aidan46, no-ref, 2024-06-05): Determine applicable weights. - pub fn declare_faults_recovered( - origin: OriginFor, - _deadline: u64, - _partition: u64, - _sectors: Vec, - ) -> DispatchResultWithPostInfo { - // Check that the extrinsic was signed and get the signer. - let _who = ensure_signed(origin)?; - // TODO(@aidan46, no-ref, 2024-06-05): Implement declare faults recovered functionality - unimplemented!("Declare faults recovered is not implemented yet") - } - - /// Pledges the storage provider to seal and commit some new sectors. - #[pallet::call_index(6)] - // #[pallet::weight(Weight::from_parts(10_000, 0) + T::DbWeight::get().writes(1))] - // TODO(aidan46, no-ref, 2024-06-05): Determine applicable weights. - pub fn pre_commit_sector( - origin: OriginFor, - _sectors: SectorPreCommitInfo, - ) -> DispatchResultWithPostInfo { - // Check that the extrinsic was signed and get the signer. - let _who = ensure_signed(origin)?; - // TODO(@aidan46, no-ref, 2024-06-05): Implement pre commit sector functionality - unimplemented!("Pre commit sector is not implemented yet") - } - - /// Checks state of the corresponding sector pre-commitments and verifies aggregate proof of replication - /// of these sectors. If valid, the sectors' deals are activated. - #[pallet::call_index(8)] - // #[pallet::weight(Weight::from_parts(10_000, 0) + T::DbWeight::get().writes(1))] - // TODO(aidan46, no-ref, 2024-06-05): Determine applicable weights. - pub fn prove_commit_sector( - origin: OriginFor, - _sector_number: SectorNumber, - _proof: Vec, - ) -> DispatchResultWithPostInfo { - // Check that the extrinsic was signed and get the signer. - let _who = ensure_signed(origin)?; - // TODO(@aidan46, no-ref, 2024-06-07): Implement prove commit sector functionality - unimplemented!("Prove commit sector is not implemented yet") - } - } + impl Pallet {} } From fa6c4f9731deb28c203f4ff5a8143f7b94ff4e06 Mon Sep 17 00:00:00 2001 From: aidan Date: Sun, 16 Jun 2024 20:04:20 +0200 Subject: [PATCH 69/75] docs: Add hooks docs The storage provider pallet will perform tasks, checking if storage providers are misbehaving. Add a section to the design on how to handle these periodic checks --- pallets/storage-provider/DESIGN.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pallets/storage-provider/DESIGN.md b/pallets/storage-provider/DESIGN.md index dd51c7e4f..a645554bf 100644 --- a/pallets/storage-provider/DESIGN.md +++ b/pallets/storage-provider/DESIGN.md @@ -22,6 +22,7 @@ - [Registration](#registration) - [Commit](#commit) - [Proof of Spacetime submission](#proof-of-spacetime-submission) + - [Storage provider pallet hooks](#storage-provider-pallet-hooks) ## Overview @@ -202,7 +203,7 @@ The first thing a storage provider must do is register itself by calling `storag ### Commit When the storage provider has completed their first seal, they should post it to the storage provider pallet by calling `storage_provider.pre_commit_sector(sectors: SectorPreCommitInfo)`. If the storage provider had zero committed sectors before this call, this begins their proving period. The proving period is a fixed amount of time in which the storage provider must submit a Proof of Space Time to the network. -During this period, the storage provider may also commit to new sectors, but they will not be included in proofs of space time until the next proving period starts. +During this period, the storage provider may also commit to new sectors, but they will not be included in proofs of space time until the next proving period starts. During the prove commit call, the storage provider pledges some collateral in case they fail to submit their PoSt on time. ### Proof of Spacetime submission @@ -210,3 +211,7 @@ When the storage provider has completed their PoSt, they must submit it to the n - **Standard Submission**: A standard submission is one that makes it on-chain before the end of the proving period. - **Penalize Submission**:A penalized submission is one that makes it on-chain after the end of the proving period, but before the generation attack threshold. These submissions count as valid PoSt submissions, but the miner must pay a penalty for their late submission. See [storage fault slashing](#storage-fault-slashing). + +## Storage provider pallet hooks + +Substrate pallet hooks execute some actions when certain conditions are met. We use these hooks, when a block finalizes, to check if storage providers are up to date with their proofs. If a proof needs to be submitted but isn't the storage provider pallet will penalize the storage provider accordingly [slash](#storage-fault-slashing) their collateral that the locked up during the [pre commit section](#commit). \ No newline at end of file From f4ec0fd666bf2d94f16a77715482e2accb667b5e Mon Sep 17 00:00:00 2001 From: aidan Date: Tue, 18 Jun 2024 18:25:27 +0200 Subject: [PATCH 70/75] docs: Add reference for WinningPoSt --- pallets/storage-provider/PROOF-OF-STORAGE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pallets/storage-provider/PROOF-OF-STORAGE.md b/pallets/storage-provider/PROOF-OF-STORAGE.md index d8eb1b4ff..7e99edf78 100644 --- a/pallets/storage-provider/PROOF-OF-STORAGE.md +++ b/pallets/storage-provider/PROOF-OF-STORAGE.md @@ -45,7 +45,7 @@ From the point of committing to store data, storage providers must continuously There are two types of challenges (and their corresponding mechanisms) within the PoSt process: WinningPoSt and WindowPoSt, each serving a different purpose. -- WinningPoSt: Proves that the storage provider has a replica of the data at the specific time they are challenged. A WinningPoSt challenge is issued to a storage provider only if they are selected through the Secret Leader Election algorithm to validate the next block. The answer to the WinningPoSt challenge must be submitted within a short [deadline](./DESIGN.md#constants--terminology), making it impractical for the provider to reseal and find the answer on demand. This ensures that the provider maintains a copy of the data at the time of the challenge. +- WinningPoSt: Proves that the storage provider has a replica of the data at the specific time they are challenged. A WinningPoSt challenge is issued to a storage provider only if they are selected through the [Secret Leader Election algorithm](https://eprint.iacr.org/2020/025.pdf) to validate the next block. The answer to the WinningPoSt challenge must be submitted within a short [deadline](./DESIGN.md#constants--terminology), making it impractical for the provider to reseal and find the answer on demand. This ensures that the provider maintains a copy of the data at the time of the challenge. - WindowPoSt: Proves that a copy of the data has been continuously maintained over time. Providers must submit proofs regularly, making it irrational for them to reseal the data every time a WindowPoSt challenge is issued. ### WinningPoSt From 670783a6ae436c7c6a4597d4e3925fe7f98a59b6 Mon Sep 17 00:00:00 2001 From: aidan Date: Tue, 18 Jun 2024 18:27:44 +0200 Subject: [PATCH 71/75] docs: Add missing header --- pallets/storage-provider/PROOF-OF-STORAGE.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pallets/storage-provider/PROOF-OF-STORAGE.md b/pallets/storage-provider/PROOF-OF-STORAGE.md index 7e99edf78..ab085e231 100644 --- a/pallets/storage-provider/PROOF-OF-STORAGE.md +++ b/pallets/storage-provider/PROOF-OF-STORAGE.md @@ -27,7 +27,8 @@ The PoRep proof links together: 3. The time when the specific data was sealed by the specific storage provider. If the same storage provider attempts to seal the same data at a later time, a different PoRep proof will be produced. The time is recorded as the blockchain height at which sealing took place, with the corresponding chain reference termed [SealRandomness](https://spec.filecoin.io/systems/filecoin_mining/sector/sealing/#section-systems.filecoin_mining.sector.sealing.randomness). -Generating and Submitting PoRep Proofs + +## Generating and Submitting PoRep Proofs Once the proof is generated, the storage provider compresses it using a SNARK (Succinct Non-interactive Argument of Knowledge) and submits the result to the blockchain. This submission certifies that the storage provider has indeed replicated a copy of the data they committed to store. Phases of the PoRep Process From 64d86c2bd97e9a63e45725c00f064c2def075599 Mon Sep 17 00:00:00 2001 From: aidan Date: Tue, 25 Jun 2024 12:02:54 +0200 Subject: [PATCH 72/75] format: Run cargo fmt --- pallets/storage-provider/src/lib.rs | 5 +++-- pallets/storage-provider/src/types.rs | 7 ++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/pallets/storage-provider/src/lib.rs b/pallets/storage-provider/src/lib.rs index 54109acb7..16834b8e3 100644 --- a/pallets/storage-provider/src/lib.rs +++ b/pallets/storage-provider/src/lib.rs @@ -20,13 +20,14 @@ pub use pallet::{Config, Pallet}; #[frame_support::pallet(dev_mode)] pub mod pallet { - use crate::types::StorageProviderInfo; + use core::fmt::Debug; use codec::{Decode, Encode}; - use core::fmt::Debug; use frame_support::pallet_prelude::{IsType, StorageMap}; use scale_info::TypeInfo; + use crate::types::StorageProviderInfo; + #[pallet::pallet] #[pallet::without_storage_info] // Allows to define storage items without fixed size pub struct Pallet(_); diff --git a/pallets/storage-provider/src/types.rs b/pallets/storage-provider/src/types.rs index 6b1f2b723..a9c966177 100644 --- a/pallets/storage-provider/src/types.rs +++ b/pallets/storage-provider/src/types.rs @@ -1,7 +1,8 @@ use codec::{Decode, Encode}; -use scale_info::prelude::string::String; -use scale_info::prelude::vec::Vec; -use scale_info::TypeInfo; +use scale_info::{ + prelude::{string::String, vec::Vec}, + TypeInfo, +}; /// SectorNumber is a numeric identifier for a sector. pub type SectorNumber = u64; From dea3b90903ca63e7b3a7599fc83bd9ad049bdee4 Mon Sep 17 00:00:00 2001 From: aidan46 <47111423+aidan46@users.noreply.github.com> Date: Thu, 27 Jun 2024 17:18:10 +0200 Subject: [PATCH 73/75] fix: Typo in proof of storage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: José Duarte --- pallets/storage-provider/PROOF-OF-STORAGE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pallets/storage-provider/PROOF-OF-STORAGE.md b/pallets/storage-provider/PROOF-OF-STORAGE.md index ab085e231..1e43ea2e2 100644 --- a/pallets/storage-provider/PROOF-OF-STORAGE.md +++ b/pallets/storage-provider/PROOF-OF-STORAGE.md @@ -3,7 +3,7 @@ > [!NOTE] > Some terms used in this document are described in the [design document](./DESIGN.md#constants--terminology) -In our parachain within the Polkadot ecosystem, storage providers are required to prove that they hold a copy of the data they have committed to storing at any given point in time. This proof is achieved through a mechanism known as 'challenges.' The process involves the system posing specific questions to the storage providers, who must then provide correct answers to prove they are maintaining the data as promised. +In our parachain within the Polkadot ecosystem, storage providers are required to prove that they hold a copy of the data they have committed to storing at any given point in time. This proof is achieved through a mechanism known as 'challenges'. The process involves the system posing specific questions to the storage providers, who must then provide correct answers to prove they are maintaining the data as promised. To ensure the integrity and reliability of these proofs, the challenges must: From 25e97a2a2bda95c78a243b53762f535a7947e621 Mon Sep 17 00:00:00 2001 From: aidan Date: Fri, 28 Jun 2024 13:25:55 +0200 Subject: [PATCH 74/75] docs: Add comment to peer id --- pallets/storage-provider/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/pallets/storage-provider/src/lib.rs b/pallets/storage-provider/src/lib.rs index 16834b8e3..ed4dd7465 100644 --- a/pallets/storage-provider/src/lib.rs +++ b/pallets/storage-provider/src/lib.rs @@ -40,6 +40,7 @@ pub mod pallet { /// Peer ID is derived by hashing an encoded public key. /// Usually represented in bytes. /// https://github.com/libp2p/specs/blob/2ea41e8c769f1bead8e637a9d4ebf8c791976e8a/peer-ids/peer-ids.md#peer-ids + /// More information about libp2p peer ids: https://docs.libp2p.io/concepts/fundamentals/peers/ type PeerId: Clone + Debug + Decode + Encode + Eq + TypeInfo; } From 9c19631ead0d362cd4d621e5dd7f6caa3bb1b141 Mon Sep 17 00:00:00 2001 From: aidan Date: Fri, 28 Jun 2024 13:27:08 +0200 Subject: [PATCH 75/75] docs: Remove broken link --- pallets/storage-provider/DESIGN.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/pallets/storage-provider/DESIGN.md b/pallets/storage-provider/DESIGN.md index a645554bf..227844e70 100644 --- a/pallets/storage-provider/DESIGN.md +++ b/pallets/storage-provider/DESIGN.md @@ -39,9 +39,6 @@ The `Storage Provider Pallet` handles the creation of storage providers and faci ## Usage -> [!NOTE] -> For more information about the storage provider pallet API check out [the API docs](./API.md) - ### Registering storage providers A storage provider indexes in the storage provider pallet itself when it starts up by calling the `create_storage_provider` extrinsic with it's `PeerId` as an argument. The public key will be extracted from the origin and is used to modify on-chain information and receive rewards. The `PeerId` is given by the storage provider so clients can use that to connect to the storage provider.