merge main into devnet-ready 7-12-2024

.github/workflows/e2e-bittensor-tests.yml

@@ -5,16 +5,10 @@ concurrency:
   cancel-in-progress: true
 
 on:
-  ## Run automatically for all PRs against main, regardless of what the changes are
-  ## to be safe and so we can more easily force re-run the CI when github is being
-  ## weird by using a blank commit
   push:
-    branches: [main, development, staging]
+    branches: [main, devnet-ready, devnet, testnet, finney]
 
-  ##
-  # Run automatically for PRs against default/main branch if Rust files change
   pull_request:
-    branches: [main, development, staging]
 
   ## Allow running workflow manually from the Actions tab
   workflow_dispatch:

.gitignore

@@ -40,4 +40,7 @@ specs/*.json
 .idea
 
 # Runtime upgrade snapshot
-bt.snap
+bt.snap
+
+# localnet spec
+scripts/specs/local.json

Dockerfile

@@ -38,6 +38,8 @@ COPY Cargo.lock Cargo.toml /subtensor/
 
 # Specs
 COPY ./snapshot.json /subtensor/snapshot.json
+COPY ./raw_spec_testfinney.json /subtensor/raw_spec_testfinney.json
+COPY ./raw_spec_finney.json /subtensor/raw_spec_finney.json
 
 # Copy our sources
 COPY ./node /subtensor/node
@@ -51,11 +53,13 @@ RUN /subtensor/scripts/init.sh
 
 # Cargo build
 WORKDIR /subtensor
-RUN cargo build --release --features runtime-benchmarks --locked
+RUN cargo build --profile production --features runtime-benchmarks --locked
 EXPOSE 30333 9933 9944
 
 
 FROM $BASE_IMAGE AS subtensor
 
 COPY --from=builder /subtensor/snapshot.json /
-COPY --from=builder /subtensor/target/release/node-subtensor /usr/local/bin
+COPY --from=builder /subtensor/raw_spec_testfinney.json /
+COPY --from=builder /subtensor/raw_spec_finney.json /
+COPY --from=builder /subtensor/target/production/node-subtensor /usr/local/bin

docs/running-subtensor-locally.md

@@ -162,7 +162,7 @@ rm -rf /tmp/blockchain
 Install subtensor by compiling with `cargo`:
 
 ```bash
-cargo build --release --features=runtime-benchmarks
+cargo build --profile production --features=runtime-benchmarks
 ```
 
 ## Run the subtensor node
@@ -174,31 +174,31 @@ You can now run the public subtensor node either as a lite node or as an archive
 To run a lite node connected to the mainchain, execute the below command (note the `--sync=warp` flag which runs the subtensor node in lite mode):
 
 ```bash title="With --sync=warp setting, for lite node"
-./target/release/node-subtensor --chain raw_spec_finney.json --base-path /tmp/blockchain --sync=warp --port 30333 --max-runtime-instances 32 --rpc-max-response-size 2048 --rpc-cors all --rpc-port 9944 --bootnodes /ip4/13.58.175.193/tcp/30333/p2p/12D3KooWDe7g2JbNETiKypcKT1KsCEZJbTzEHCn8hpd4PHZ6pdz5 --no-mdns --in-peers 8000 --out-peers 8000 --prometheus-external --rpc-external
+./target/production/node-subtensor --chain raw_spec.json --base-path /tmp/blockchain --sync=warp --port 30333 --max-runtime-instances 32 --rpc-max-response-size 2048 --rpc-cors all --rpc-port 9944 --bootnodes /ip4/13.58.175.193/tcp/30333/p2p/12D3KooWDe7g2JbNETiKypcKT1KsCEZJbTzEHCn8hpd4PHZ6pdz5 --no-mdns --in-peers 8000 --out-peers 8000 --prometheus-external --rpc-external
 ```
 
 ### Archive node on mainchain
 
 To run an archive node connected to the mainchain, execute the below command (note the `--sync=full` which syncs the node to the full chain and `--pruning archive` flags, which disables the node's automatic pruning of older historical data):
 
 ```bash title="With --sync=full and --pruning archive setting, for archive node"
-./target/release/node-subtensor --chain raw_spec_finney.json --base-path /tmp/blockchain --sync=full --pruning archive --port 30333 --max-runtime-instances 32 --rpc-max-response-size 2048 --rpc-cors all --rpc-port 9944 --bootnodes /ip4/13.58.175.193/tcp/30333/p2p/12D3KooWDe7g2JbNETiKypcKT1KsCEZJbTzEHCn8hpd4PHZ6pdz5 --no-mdns --in-peers 8000 --out-peers 8000 --prometheus-external --rpc-external
+./target/production/node-subtensor --chain raw_spec.json --base-path /tmp/blockchain --sync=full --pruning archive --port 30333 --max-runtime-instances 32 --rpc-max-response-size 2048 --rpc-cors all --rpc-port 9944 --bootnodes /ip4/13.58.175.193/tcp/30333/p2p/12D3KooWDe7g2JbNETiKypcKT1KsCEZJbTzEHCn8hpd4PHZ6pdz5 --no-mdns --in-peers 8000 --out-peers 8000 --prometheus-external --rpc-external
 ```
 
 ### Lite node on testchain
 
 To run a lite node connected to the testchain, execute the below command:
 
 ```bash title="With bootnodes set to testnet and --sync=warp setting, for lite node."
-./target/release/node-subtensor --chain raw_spec_testfinney.json --base-path /tmp/blockchain --sync=warp --port 30333 --max-runtime-instances 32 --rpc-max-response-size 2048 --rpc-cors all --rpc-port  9944 --bootnodes /dns/bootnode.test.finney.opentensor.ai/tcp/30333/p2p/12D3KooWPM4mLcKJGtyVtkggqdG84zWrd7Rij6PGQDoijh1X86Vr --no-mdns --in-peers 8000 --out-peers 8000 --prometheus-external --rpc-external
+./target/production/node-subtensor --chain raw_testspec.json --base-path /tmp/blockchain --sync=warp --port 30333 --max-runtime-instances 32 --rpc-max-response-size 2048 --rpc-cors all --rpc-port  9944 --bootnodes /dns/bootnode.test.finney.opentensor.ai/tcp/30333/p2p/12D3KooWPM4mLcKJGtyVtkggqdG84zWrd7Rij6PGQDoijh1X86Vr --no-mdns --in-peers 8000 --out-peers 8000 --prometheus-external --rpc-external
 ```
 
 ### Archive node on testchain
 
 To run an archive node connected to the testchain, execute the below command:
 
 ```bash title="With bootnodes set to testnet and --sync=full and --pruning archive setting, for archive node"
-./target/release/node-subtensor --chain raw_spec_testfinney.json --base-path /tmp/blockchain --sync=full --pruning archive --port 30333 --max-runtime-instances 32 --rpc-max-response-size 2048 --rpc-cors all --rpc-port 9944 --bootnodes /dns/bootnode.test.finney.opentensor.ai/tcp/30333/p2p/12D3KooWPM4mLcKJGtyVtkggqdG84zWrd7Rij6PGQDoijh1X86Vr --no-mdns --in-peers 8000 --out-peers 8000 --prometheus-external --rpc-external
+./target/production/node-subtensor --chain raw_testspec.json --base-path /tmp/blockchain --sync=full --pruning archive --port 30333 --max-runtime-instances 32 --rpc-max-response-size 2048 --rpc-cors all --rpc-port 9944 --bootnodes /dns/bootnode.test.finney.opentensor.ai/tcp/30333/p2p/12D3KooWPM4mLcKJGtyVtkggqdG84zWrd7Rij6PGQDoijh1X86Vr --no-mdns --in-peers 8000 --out-peers 8000 --prometheus-external --rpc-external
 ```
 
 ## Running on cloud

docs/rust-setup.md

@@ -1,8 +1,7 @@
 ---
 title: Installation
 ---
-
-This guide is for reference only, please check the latest information on getting starting with Substrate 
+This guide is for reference only, please check the latest information on getting starting with Substrate
 [here](https://docs.substrate.io/main-docs/install/).
 
 This page will guide you through the **2 steps** needed to prepare a computer for **Substrate** development.
@@ -207,7 +206,7 @@ Use the `WASM_BUILD_TOOLCHAIN` environment variable to specify the Rust nightly
 project should use for Wasm compilation:
 
 ```bash
-WASM_BUILD_TOOLCHAIN=nightly-<yyyy-MM-dd> cargo build --release
+WASM_BUILD_TOOLCHAIN=nightly-<yyyy-MM-dd> cargo build --profile production
 ```
 
 > Note that this only builds _the runtime_ with the specified nightly. The rest of project will be
@@ -223,3 +222,4 @@ rustup uninstall nightly
 rustup install nightly-<yyyy-MM-dd>
 rustup target add wasm32-unknown-unknown --toolchain nightly-<yyyy-MM-dd>
 ```
+

justfile

@@ -30,11 +30,12 @@ clippy:
                             -D clippy::unimplemented
 
 clippy-fix:
-  @echo "Running cargo clippy with automatic fixes on potentially dirty code..."
-  cargo +{{RUSTV}} clippy --workspace --all-targets --fix --allow-dirty --  \
-                                                      -A clippy::todo \
-                                                      -A clippy::unimplemented \
-                                                      -A clippy::indexing_slicing
+    @echo "Running cargo clippy with automatic fixes on potentially dirty code..."
+    cargo +{{RUSTV}} clippy --fix --allow-dirty --allow-staged --workspace --all-targets -- \
+        -A clippy::todo \
+        -A clippy::unimplemented \
+        -A clippy::indexing_slicing
+
 fix:
   @echo "Running cargo fix..."
   cargo +{{RUSTV}} fix --workspace

pallets/admin-utils/scripts/benchmark.sh

@@ -1,11 +1,9 @@
-cargo build --release --features runtime-benchmarks
-./target/release/node-subtensor benchmark pallet \
-    --chain=local \
-    --execution=wasm \
-    --wasm-execution=compiled \
-    --pallet=pallet_admin_utils \
-    --extrinsic="*" \
-    --steps 50 \
-    --repeat 20 \
-    --output=pallets/admin-utils/src/weights.rs \
-    --template=./.maintain/frame-weight-template.hbs
+cargo build --profile production --features runtime-benchmarks
+./target/production/node-subtensor benchmark pallet \
+  --chain=local \
+  --pallet=pallet_admin_utils \
+  --extrinsic="*" \
+  --steps 50 \
+  --repeat 20 \
+  --output=pallets/admin-utils/src/weights.rs \
+  --template=./.maintain/frame-weight-template.hbs

pallets/admin-utils/tests/mock.rs

@@ -114,6 +114,7 @@ parameter_types! {
     pub const InitialAlphaHigh: u16 = 58982; // Represents 0.9 as per the production default
     pub const InitialAlphaLow: u16 = 45875; // Represents 0.7 as per the production default
     pub const InitialLiquidAlphaOn: bool = false; // Default value for LiquidAlphaOn
+    pub const InitialBaseDifficulty: u64 = 10_000; // Base difficulty
 }
 
 impl pallet_subtensor::Config for Test {
@@ -169,6 +170,7 @@ impl pallet_subtensor::Config for Test {
     type AlphaHigh = InitialAlphaHigh;
     type AlphaLow = InitialAlphaLow;
     type LiquidAlphaOn = InitialLiquidAlphaOn;
+    type InitialBaseDifficulty = InitialBaseDifficulty;
 }
 
 #[derive_impl(frame_system::config_preludes::TestDefaultConfig)]

pallets/collective/src/lib.rs

@@ -151,7 +151,7 @@ impl<AccountId, I> GetBacking for RawOrigin<AccountId, I> {
 }
 
 /// Info for keeping track of a motion being voted on.
-#[freeze_struct("5959418cdb31993b")]
+#[freeze_struct("a8e7b0b34ad52b17")]
 #[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo)]
 pub struct Votes<AccountId, BlockNumber> {
     /// The proposal's unique index.

pallets/commitments/scripts/benchmark.sh

@@ -1,9 +1,7 @@
-cargo build --release --features runtime-benchmarks
-./target/release/node-subtensor benchmark pallet \
-    --chain=local \
-    --execution=wasm \
-    --wasm-execution=compiled \
-    --pallet=pallet_commitments \
-    --extrinsic="*" \
-    --output=pallets/commitments/src/weights.rs \
-    --template=./.maintain/frame-weight-template.hbs
+cargo build --profile production --features runtime-benchmarks
+./target/production/node-subtensor benchmark pallet \
+  --chain=local \
+  --pallet=pallet_commitments \
+  --extrinsic="*" \
+  --output=pallets/commitments/src/weights.rs \
+  --template=./.maintain/frame-weight-template.hbs

pallets/registry/scripts/benchmark.sh

@@ -1,9 +1,7 @@
 cargo build --release --features runtime-benchmarks
-./target/release/node-subtensor benchmark pallet \
-    --chain=local \
-    --execution=wasm \
-    --wasm-execution=compiled \
-    --pallet=pallet_registry \
-    --extrinsic="*" \
-    --output=pallets/registry/src/weights.rs \
-    --template=./.maintain/frame-weight-template.hbs
+./target/production/node-subtensor benchmark pallet \
+  --chain=local \
+  --pallet=pallet_registry \
+  --extrinsic="*" \
+  --output=pallets/registry/src/weights.rs \
+  --template=./.maintain/frame-weight-template.hbs

pallets/registry/src/types.rs

@@ -279,7 +279,7 @@ impl TypeInfo for IdentityFields {
 ///
 /// NOTE: This should be stored at the end of the storage item to facilitate the addition of extra
 /// fields in a backwards compatible way through a specialized `Decode` impl.
-#[freeze_struct("9d47309f8e379957")]
+#[freeze_struct("98e2d7fc7536226b")]
 #[derive(
     CloneNoBound, Encode, Decode, Eq, MaxEncodedLen, PartialEqNoBound, RuntimeDebugNoBound, TypeInfo,
 )]

pallets/subtensor/src/benchmarks.rs

@@ -429,28 +429,30 @@ reveal_weights {
 
   }: reveal_weights(RawOrigin::Signed(hotkey.clone()), netuid, uids, weight_values, salt, version_key)
 
+  schedule_coldkey_swap {
+    let seed: u32 = 1;
+    let old_coldkey: T::AccountId = account("OldColdkey", 0, seed);
+    let new_coldkey: T::AccountId = account("NewColdkey", 0, seed + 1);
+    let hotkey: T::AccountId = account("Hotkey", 0, seed);
 
-  adjust_senate {
-    migration::migrate_create_root_network::<T>();
-    let netuid: u16 = 1;
-    let tempo: u16 = 13;
-    let burn_cost = 1000;
-    let hotkey: T::AccountId = account("hot", 0, 1);
-    let coldkey: T::AccountId = account("cold", 0, 2);
-
-    Subtensor::<T>::init_new_network(netuid, 100);
-    Subtensor::<T>::set_burn(netuid, 1);
-    Subtensor::<T>::set_max_allowed_uids( netuid, 4096 );
-    assert_eq!(Subtensor::<T>::get_max_allowed_uids(netuid), 4096);
-    Subtensor::<T>::set_burn(netuid, burn_cost);
+    let netuid = 1u16;
+    let tempo = 1u16;
+    let block_number: u64 = Subtensor::<T>::get_current_block_as_u64();
+    let nonce = 0;
 
-    let amount_to_be_staked = 100_000_000_000;
-    Subtensor::<T>::add_balance_to_coldkey_account(&coldkey.clone(), amount_to_be_staked);
+    // Initialize the network
+    Subtensor::<T>::init_new_network(netuid, tempo);
+    Subtensor::<T>::set_network_registration_allowed(netuid, true);
 
-    assert_ok!(Subtensor::<T>::burned_register(RawOrigin::Signed(coldkey.clone()).into(), netuid, hotkey.clone()));
-    assert_ok!(Subtensor::<T>::become_delegate(RawOrigin::Signed(coldkey.clone()).into(), hotkey.clone()));
-    assert_ok!(Subtensor::<T>::root_register(RawOrigin::Signed(coldkey.clone()).into(), hotkey.clone()));
-    T::SenateMembers::remove_member(&hotkey).map_err(|_| "Failed to remove member")?;
+    // Add balance to the old coldkey account
+    let amount_to_be_staked: u64 = 1000000u32.into();
+    Subtensor::<T>::add_balance_to_coldkey_account(&old_coldkey.clone(), amount_to_be_staked+1000000000);
+       // Burned register the hotkey with the old coldkey
+       assert_ok!(Subtensor::<T>::burned_register(
+        RawOrigin::Signed(old_coldkey.clone()).into(),
+        netuid,
+        hotkey.clone()
+    ));
 
-  }: adjust_senate(RawOrigin::Signed(coldkey.clone()), hotkey.clone())
+  }: schedule_coldkey_swap(RawOrigin::Signed(old_coldkey.clone()), new_coldkey.clone(), vec![], block_number, nonce)
 }

pallets/subtensor/src/block_step.rs

@@ -118,13 +118,17 @@ impl<T: Config> Pallet<T> {
         // --- 1. Iterate across each network and add pending emission into stash.
         for (netuid, tempo) in <Tempo<T> as IterableStorageMap<u16, u16>>::iter() {
             // Skip the root network or subnets with registrations turned off
-            if netuid == Self::get_root_netuid() || !Self::is_registration_allowed(netuid) {
+            if netuid == Self::get_root_netuid() {
                 // Root emission or subnet emission is burned
                 continue;
             }
 
             // --- 2. Queue the emission due to this network.
-            let new_queued_emission: u64 = Self::get_subnet_emission_value(netuid);
+            let mut new_queued_emission: u64 = Self::get_subnet_emission_value(netuid);
+            if !Self::is_registration_allowed(netuid) {
+                new_queued_emission = 0; // No emission for this network if registration is off.
+            }
+
             log::debug!(
                 "generate_emission for netuid: {:?} with tempo: {:?} and emission: {:?}",
                 netuid,