diff --git a/.github/workflows/check-finney.yml b/.github/workflows/check-finney.yml index 4bb12caf2..3e9fb5994 100644 --- a/.github/workflows/check-finney.yml +++ b/.github/workflows/check-finney.yml @@ -11,6 +11,7 @@ jobs: check-spec-version: name: Check spec_version bump runs-on: SubtensorCI + if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-spec-version-bump') }} steps: - name: Dependencies run: | diff --git a/CODEOWNERS b/CODEOWNERS index 5fefbd608..ffc01511b 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1 +1 @@ -* @sacha-l @lisa-parity +* @unconst diff --git a/README.md b/README.md index 47639b640..30f36ffab 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ Requirements: ## For Subnet Development -If you are developing and testing subnet incentive mechanism, you will need to run a local subtensor node. Follow the detailed step-by-step instructions provided in the document [Running subtensor locally](./docs/running-subtensor-locally.md) to run either a lite node or an archive node. Also see the [**Subtensor Nodes** section in Bittensor Developer Documentation](https://docs.bittensor.com/subtensor-nodes). +If you are developing and testing subnet incentive mechanism, you will need to run a local subtensor node. Follow the detailed step-by-step instructions provided in the [**Subtensor Nodes** section in Bittensor Developer Documentation](https://docs.bittensor.com/subtensor-nodes). ### Lite node vs Archive node diff --git a/docs/running-subtensor-locally.md b/docs/running-subtensor-locally.md index 505fe2fb5..82bb87356 100644 --- a/docs/running-subtensor-locally.md +++ b/docs/running-subtensor-locally.md @@ -1,206 +1,3 @@ # Running subtensor node locally -- [Method 1: Using Docker](#method-1-using-docker) -- [Method 2: Using Source Code](#method-2-using-source-code) -- [Running on Cloud](#running-on-cloud) - -## Method 1: Using Docker - -To run a subtensor node with Docker, follow the below steps. - -If you are already running a subtensor node using Docker, then go directly to [Step 5 Prepare to Run](#step-5-prepare-to-run) to restart the Docker container. The below steps 1 through 4 are for first time users only. - -### Step 1: Install git - -Make sure you installed `git` on your machine. See [GitHub docs](https://docs.github.com/en/get-started). - -### Step 2: Install Docker - -Follow Docker's [official installation guides](https://docs.docker.com/engine/install/) and install Docker. - -**Run Docker first** -Before you proceed, make sure that Docker is running. - -### Step 3: Clone the subtensor repo - -Clone the subtensor repo: - -```bash -git clone https://github.com/opentensor/subtensor.git -``` - -### Step 4: Go into subtensor directory - -Then `cd` into the subtensor directory: - -```bash -cd subtensor -``` - -### Step 5: Prepare to run - -Execute the below three commands in this order: - -Make sure you are on the `main` branch. If not, switch to it: - -```bash -git checkout main -``` - -Pull the latest `main` branch contents: - -```bash -git pull -``` - -Stop the currently running Docker containers: - -```bash -docker compose down --volumes -``` - -### Run a lite node on mainchain - -To run a lite node connected to the Bittensor mainchain, run the below command. - -```bash -sudo ./scripts/run/subtensor.sh -e docker --network mainnet --node-type lite -``` - -### Run an archive node on mainchain - -To run an archive node connected to the Bittensor mainchain, run the below command. - -```bash -sudo ./scripts/run/subtensor.sh -e docker --network mainnet --node-type archive -``` - -### Run a lite node on testchain - -To run a lite node connected to the Bittensor testchain, run the below command. - -```bash -sudo ./scripts/run/subtensor.sh -e docker --network testnet --node-type lite -``` - -### Run an archive node on testchain - -To run an archive node connected to the Bittensor testchain, run the below command. - -```bash -sudo ./scripts/run/subtensor.sh -e docker --network testnet --node-type archive -``` - ---- - -## Method 2: Using Source Code - -To install and run a subtensor node by compiling the source code, follow the below steps. - -## Install basic packages - -Install the basic requirements by running the below commands on a Linux terminal. - -```bash title="On Linux" -sudo apt-get update -sudo apt install build-essential -sudo apt-get install clang -sudo apt-get install curl -sudo apt-get install git -sudo apt-get install make -sudo apt install --assume-yes git clang curl libssl-dev protobuf-compiler -sudo apt install --assume-yes git clang curl libssl-dev llvm libudev-dev make protobuf-compiler -``` - -## Install Rust - -Next, install Rust and update the environment by running the following commands: - -```bash -curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -source ~/.cargo/env -``` - -Next, install Rust toolchain: - -```bash -rustup default stable -rustup update -rustup target add wasm32-unknown-unknown -rustup toolchain install nightly -rustup target add --toolchain nightly wasm32-unknown-unknown -``` - -## Compile subtensor code - -Next, to compile the subtensor source code, follow the below steps: - -Clone the subtensor repo: - -```bash -git clone https://github.com/opentensor/subtensor.git -``` - -`cd` into the subtensor directory: - -```bash -cd subtensor -``` - -Make sure you are on the `main` branch. If not, switch to it: - -```bash -git checkout main -``` - -Remove previous chain state: - -```bash -rm -rf /tmp/blockchain -``` - -Install subtensor by compiling with `cargo`: - -```bash -cargo build --profile production --features=runtime-benchmarks -``` - -## Run the subtensor node - -You can now run the public subtensor node either as a lite node or as an archive node. See below: - -### Lite node on mainchain - -To run a lite node connected to the mainchain, execute the below command (note the `--sync=warp` flag which runs the subtensor node in lite mode): - -```bash title="With --sync=warp setting, for lite node" -./target/production/node-subtensor --chain raw_spec.json --base-path /tmp/blockchain --sync=warp --port 30333 --max-runtime-instances 32 --rpc-max-response-size 2048 --rpc-cors all --rpc-port 9944 --bootnodes /ip4/13.58.175.193/tcp/30333/p2p/12D3KooWDe7g2JbNETiKypcKT1KsCEZJbTzEHCn8hpd4PHZ6pdz5 --no-mdns --in-peers 8000 --out-peers 8000 --prometheus-external --rpc-external -``` - -### Archive node on mainchain - -To run an archive node connected to the mainchain, execute the below command (note the `--sync=full` which syncs the node to the full chain and `--pruning archive` flags, which disables the node's automatic pruning of older historical data): - -```bash title="With --sync=full and --pruning archive setting, for archive node" -./target/production/node-subtensor --chain raw_spec.json --base-path /tmp/blockchain --sync=full --pruning archive --port 30333 --max-runtime-instances 32 --rpc-max-response-size 2048 --rpc-cors all --rpc-port 9944 --bootnodes /ip4/13.58.175.193/tcp/30333/p2p/12D3KooWDe7g2JbNETiKypcKT1KsCEZJbTzEHCn8hpd4PHZ6pdz5 --no-mdns --in-peers 8000 --out-peers 8000 --prometheus-external --rpc-external -``` - -### Lite node on testchain - -To run a lite node connected to the testchain, execute the below command: - -```bash title="With bootnodes set to testnet and --sync=warp setting, for lite node." -./target/production/node-subtensor --chain raw_testspec.json --base-path /tmp/blockchain --sync=warp --port 30333 --max-runtime-instances 32 --rpc-max-response-size 2048 --rpc-cors all --rpc-port 9944 --bootnodes /dns/bootnode.test.finney.opentensor.ai/tcp/30333/p2p/12D3KooWPM4mLcKJGtyVtkggqdG84zWrd7Rij6PGQDoijh1X86Vr --no-mdns --in-peers 8000 --out-peers 8000 --prometheus-external --rpc-external -``` - -### Archive node on testchain - -To run an archive node connected to the testchain, execute the below command: - -```bash title="With bootnodes set to testnet and --sync=full and --pruning archive setting, for archive node" -./target/production/node-subtensor --chain raw_testspec.json --base-path /tmp/blockchain --sync=full --pruning archive --port 30333 --max-runtime-instances 32 --rpc-max-response-size 2048 --rpc-cors all --rpc-port 9944 --bootnodes /dns/bootnode.test.finney.opentensor.ai/tcp/30333/p2p/12D3KooWPM4mLcKJGtyVtkggqdG84zWrd7Rij6PGQDoijh1X86Vr --no-mdns --in-peers 8000 --out-peers 8000 --prometheus-external --rpc-external -``` - -## Running on cloud - -We have not tested these installation scripts on any cloud service. In addition, if you are using Runpod cloud service, then note that this service is already [containerized](https://docs.runpod.io/pods/overview). Hence, the only option available to you is to compile from the source, as described in the above [Method 2: Using Source Code](#method-2-using-source-code) section. Note that these scripts have not been tested on Runpod. +See the [**Subtensor Nodes** section in Bittensor Developer Documentation](https://docs.bittensor.com/subtensor-nodes). diff --git a/pallets/subtensor/src/subnets/registration.rs b/pallets/subtensor/src/subnets/registration.rs index 9319adcd1..750d93e04 100644 --- a/pallets/subtensor/src/subnets/registration.rs +++ b/pallets/subtensor/src/subnets/registration.rs @@ -419,65 +419,67 @@ impl Pallet { } /// Determine which peer to prune from the network by finding the element with the lowest pruning score out of - /// immunity period. If all neurons are in immunity period, return node with lowest prunning score. - /// This function will always return an element to prune. + /// immunity period. If there is a tie for lowest pruning score, the neuron registered earliest is pruned. + /// If all neurons are in immunity period, the neuron with the lowest pruning score is pruned. If there is a tie for + /// the lowest pruning score, the immune neuron registered earliest is pruned. + /// Ties for earliest registration are broken by the neuron with the lowest uid. pub fn get_neuron_to_prune(netuid: u16) -> u16 { let mut min_score: u16 = u16::MAX; - let mut min_score_in_immunity_period = u16::MAX; - let mut uid_with_min_score = 0; - let mut uid_with_min_score_in_immunity_period: u16 = 0; + let mut min_score_in_immunity: u16 = u16::MAX; + let mut earliest_registration: u64 = u64::MAX; + let mut earliest_registration_in_immunity: u64 = u64::MAX; + let mut uid_to_prune: u16 = 0; + let mut uid_to_prune_in_immunity: u16 = 0; + + // This boolean is used instead of checking if min_score == u16::MAX, to avoid the case + // where all non-immune neurons have pruning score u16::MAX + // This may be unlikely in practice. + let mut found_non_immune = false; let neurons_n = Self::get_subnetwork_n(netuid); if neurons_n == 0 { return 0; // If there are no neurons in this network. } - let current_block: u64 = Self::get_current_block_as_u64(); - let immunity_period: u64 = Self::get_immunity_period(netuid) as u64; - for neuron_uid_i in 0..neurons_n { - let pruning_score: u16 = Self::get_pruning_score_for_uid(netuid, neuron_uid_i); + for neuron_uid in 0..neurons_n { + let pruning_score: u16 = Self::get_pruning_score_for_uid(netuid, neuron_uid); let block_at_registration: u64 = - Self::get_neuron_block_at_registration(netuid, neuron_uid_i); - #[allow(clippy::comparison_chain)] - if min_score == pruning_score { - if current_block.saturating_sub(block_at_registration) < immunity_period { - //neuron is in immunity period - if min_score_in_immunity_period > pruning_score { - min_score_in_immunity_period = pruning_score; - uid_with_min_score_in_immunity_period = neuron_uid_i; - } - } else { - uid_with_min_score = neuron_uid_i; + Self::get_neuron_block_at_registration(netuid, neuron_uid); + let is_immune = Self::get_neuron_is_immune(netuid, neuron_uid); + + if is_immune { + // if the immune neuron has a lower pruning score than the minimum for immune neurons, + // or, if the pruning scores are equal and the immune neuron was registered earlier than the current minimum for immune neurons, + // then update the minimum pruning score and the uid to prune for immune neurons + if pruning_score < min_score_in_immunity + || (pruning_score == min_score_in_immunity + && block_at_registration < earliest_registration_in_immunity) + { + min_score_in_immunity = pruning_score; + earliest_registration_in_immunity = block_at_registration; + uid_to_prune_in_immunity = neuron_uid; } - } - // Find min pruning score. - else if min_score > pruning_score { - if current_block.saturating_sub(block_at_registration) < immunity_period { - //neuron is in immunity period - if min_score_in_immunity_period > pruning_score { - min_score_in_immunity_period = pruning_score; - uid_with_min_score_in_immunity_period = neuron_uid_i; - } - } else { + } else { + found_non_immune = true; + // if the non-immune neuron has a lower pruning score than the minimum for non-immune neurons, + // or, if the pruning scores are equal and the non-immune neuron was registered earlier than the current minimum for non-immune neurons, + // then update the minimum pruning score and the uid to prune for non-immune neurons + if pruning_score < min_score + || (pruning_score == min_score && block_at_registration < earliest_registration) + { min_score = pruning_score; - uid_with_min_score = neuron_uid_i; + earliest_registration = block_at_registration; + uid_to_prune = neuron_uid; } } } - if min_score == u16::MAX { - //all neuorns are in immunity period - Self::set_pruning_score_for_uid( - netuid, - uid_with_min_score_in_immunity_period, - u16::MAX, - ); - uid_with_min_score_in_immunity_period + + if found_non_immune { + Self::set_pruning_score_for_uid(netuid, uid_to_prune, u16::MAX); + uid_to_prune } else { - // We replace the pruning score here with u16 max to ensure that all peers always have a - // pruning score. In the event that every peer has been pruned this function will prune - // the last element in the network continually. - Self::set_pruning_score_for_uid(netuid, uid_with_min_score, u16::MAX); - uid_with_min_score + Self::set_pruning_score_for_uid(netuid, uid_to_prune_in_immunity, u16::MAX); + uid_to_prune_in_immunity } } diff --git a/pallets/subtensor/src/utils/misc.rs b/pallets/subtensor/src/utils/misc.rs index 9155357c0..5d321412e 100644 --- a/pallets/subtensor/src/utils/misc.rs +++ b/pallets/subtensor/src/utils/misc.rs @@ -461,6 +461,13 @@ impl Pallet { ImmunityPeriod::::insert(netuid, immunity_period); Self::deposit_event(Event::ImmunityPeriodSet(netuid, immunity_period)); } + /// Check if a neuron is in immunity based on the current block + pub fn get_neuron_is_immune(netuid: u16, uid: u16) -> bool { + let registered_at = Self::get_neuron_block_at_registration(netuid, uid); + let current_block = Self::get_current_block_as_u64(); + let immunity_period = Self::get_immunity_period(netuid); + current_block.saturating_sub(registered_at) < u64::from(immunity_period) + } pub fn get_min_allowed_weights(netuid: u16) -> u16 { MinAllowedWeights::::get(netuid) diff --git a/pallets/subtensor/tests/registration.rs b/pallets/subtensor/tests/registration.rs index 7d6e8ea65..536aa2688 100644 --- a/pallets/subtensor/tests/registration.rs +++ b/pallets/subtensor/tests/registration.rs @@ -538,6 +538,122 @@ fn test_burn_adjustment() { }); } +#[test] +fn test_burn_registration_pruning_scenarios() { + new_test_ext(1).execute_with(|| { + let netuid: u16 = 1; + let tempo: u16 = 13; + let burn_cost = 1000; + let coldkey_account_id = U256::from(667); + let max_allowed_uids = 6; + let immunity_period = 5000; + + const IS_IMMUNE: bool = true; + const NOT_IMMUNE: bool = false; + + // Initial setup + SubtensorModule::set_burn(netuid, burn_cost); + SubtensorModule::set_max_allowed_uids(netuid, max_allowed_uids); + SubtensorModule::set_target_registrations_per_interval(netuid, max_allowed_uids); + SubtensorModule::set_immunity_period(netuid, immunity_period); + + add_network(netuid, tempo, 0); + + let mint_balance = burn_cost * u64::from(max_allowed_uids) + 1_000_000_000; + SubtensorModule::add_balance_to_coldkey_account(&coldkey_account_id, mint_balance); + + // Register first half of neurons + for i in 0..3 { + assert_ok!(SubtensorModule::burned_register( + <::RuntimeOrigin>::signed(coldkey_account_id), + netuid, + U256::from(i) + )); + step_block(1); + } + + // Note: pruning score is set to u16::MAX after getting neuron to prune + + // 1. Test if all immune neurons + assert_eq!(SubtensorModule::get_neuron_is_immune(netuid, 0), IS_IMMUNE); + assert_eq!(SubtensorModule::get_neuron_is_immune(netuid, 1), IS_IMMUNE); + assert_eq!(SubtensorModule::get_neuron_is_immune(netuid, 2), IS_IMMUNE); + + SubtensorModule::set_pruning_score_for_uid(netuid, 0, 100); + SubtensorModule::set_pruning_score_for_uid(netuid, 1, 75); + SubtensorModule::set_pruning_score_for_uid(netuid, 2, 50); + + // The immune neuron with the lowest score should be pruned + assert_eq!(SubtensorModule::get_neuron_to_prune(netuid), 2); + + // 2. Test tie-breaking for immune neurons + SubtensorModule::set_pruning_score_for_uid(netuid, 1, 50); + SubtensorModule::set_pruning_score_for_uid(netuid, 2, 50); + + // Should get the oldest neuron (i.e., neuron that was registered first) + assert_eq!(SubtensorModule::get_neuron_to_prune(netuid), 1); + + // 3. Test if no immune neurons + step_block(immunity_period); + + // ensure all neurons are non-immune + assert_eq!(SubtensorModule::get_neuron_is_immune(netuid, 0), NOT_IMMUNE); + assert_eq!(SubtensorModule::get_neuron_is_immune(netuid, 1), NOT_IMMUNE); + assert_eq!(SubtensorModule::get_neuron_is_immune(netuid, 2), NOT_IMMUNE); + + SubtensorModule::set_pruning_score_for_uid(netuid, 0, 100); + SubtensorModule::set_pruning_score_for_uid(netuid, 1, 50); + SubtensorModule::set_pruning_score_for_uid(netuid, 2, 75); + + // The non-immune neuron with the lowest score should be pruned + assert_eq!(SubtensorModule::get_neuron_to_prune(netuid), 1); + + // 4. Test tie-breaking for non-immune neurons + SubtensorModule::set_pruning_score_for_uid(netuid, 1, 50); + SubtensorModule::set_pruning_score_for_uid(netuid, 2, 50); + + // Should get the oldest non-immune neuron + assert_eq!(SubtensorModule::get_neuron_to_prune(netuid), 1); + + // 5. Test mixed immunity + // Register second batch of neurons (these will be non-immune) + for i in 3..6 { + assert_ok!(SubtensorModule::burned_register( + <::RuntimeOrigin>::signed(coldkey_account_id), + netuid, + U256::from(i) + )); + step_block(1); + } + + // Ensure all new neurons are immune + assert_eq!(SubtensorModule::get_neuron_is_immune(netuid, 3), IS_IMMUNE); + assert_eq!(SubtensorModule::get_neuron_is_immune(netuid, 4), IS_IMMUNE); + assert_eq!(SubtensorModule::get_neuron_is_immune(netuid, 5), IS_IMMUNE); + + // Set pruning scores for all neurons + SubtensorModule::set_pruning_score_for_uid(netuid, 0, 75); // non-immune + SubtensorModule::set_pruning_score_for_uid(netuid, 1, 50); // non-immune + SubtensorModule::set_pruning_score_for_uid(netuid, 2, 60); // non-immune + SubtensorModule::set_pruning_score_for_uid(netuid, 3, 40); // immune + SubtensorModule::set_pruning_score_for_uid(netuid, 4, 55); // immune + SubtensorModule::set_pruning_score_for_uid(netuid, 5, 45); // immune + + // The non-immune neuron with the lowest score should be pruned + assert_eq!(SubtensorModule::get_neuron_to_prune(netuid), 1); + + // If we remove the lowest non-immune neuron, it should choose the next lowest non-immune + SubtensorModule::set_pruning_score_for_uid(netuid, 1, u16::MAX); + assert_eq!(SubtensorModule::get_neuron_to_prune(netuid), 2); + + // If we make all non-immune neurons have high scores, it should choose the oldest non-immune neuron + SubtensorModule::set_pruning_score_for_uid(netuid, 0, u16::MAX); + SubtensorModule::set_pruning_score_for_uid(netuid, 1, u16::MAX); + SubtensorModule::set_pruning_score_for_uid(netuid, 2, u16::MAX); + assert_eq!(SubtensorModule::get_neuron_to_prune(netuid), 0); + }); +} + #[test] fn test_registration_too_many_registrations_per_block() { new_test_ext(1).execute_with(|| {