From b147ed86bf0cf3dc9364fc914b517497f8283abd Mon Sep 17 00:00:00 2001 From: Sid Date: Wed, 14 Aug 2024 09:07:09 -0700 Subject: [PATCH 1/5] Lower retry weight threshold to 0.05. --- constants/__init__.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/constants/__init__.py b/constants/__init__.py index eb321f0..37f9717 100644 --- a/constants/__init__.py +++ b/constants/__init__.py @@ -64,7 +64,7 @@ # Minimum percent of weight on a vali for a miner to be considered a top miner. # Since there can be multiple competitions at different reward percentages we can't just check biggest. -WEIGHT_SYNC_MINER_MIN_PERCENT = 0.10 +WEIGHT_SYNC_MINER_MIN_PERCENT = 0.05 # A mapping of block numbers to the supported model types as of that block. ALLOWED_MODEL_TYPES_1 = { @@ -181,8 +181,7 @@ CompetitionId.B7_MODEL, MODEL_CONSTRAINTS_BY_COMPETITION_ID[CompetitionId.B7_MODEL], 0.57, - ) - + ), ], ), ] @@ -215,7 +214,7 @@ sample_unpack_block = BLOCK_3B_7BSTAR_UNPACK # validators number of pages to eval over miners on each step. -pages_per_eval_unpack = 5 # With sample unpacking +pages_per_eval_unpack = 5 # With sample unpacking pages_per_eval_pack = 18 timestamp_epsilon_experiment_start_block = BLOCK_3B_7BSTAR_UNPACK From 41579a9fd7c682791bc02b5a6b8b22472a51c3e0 Mon Sep 17 00:00:00 2001 From: Sid Date: Wed, 14 Aug 2024 09:11:29 -0700 Subject: [PATCH 2/5] Remove deprecated constant value. --- constants/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/constants/__init__.py b/constants/__init__.py index 37f9717..09b1a1f 100644 --- a/constants/__init__.py +++ b/constants/__init__.py @@ -227,8 +227,6 @@ sample_min = 5 # Max number of uids that can be either pending eval or currently being evaluated. updated_models_limit = 15 -# validator incentive threshold to prioritize updates. All incentives add up to 1. -update_priority_incentive_threshold = 0.01 # time required between updates to the chain. chain_update_cadence = dt.timedelta(minutes=20) # time required between retrying evaluation of a stale model. (First retry will be immediate). From ce622a0977f3116818fbafa6f345882e6b21b817 Mon Sep 17 00:00:00 2001 From: Sid Date: Wed, 14 Aug 2024 15:56:47 -0700 Subject: [PATCH 3/5] Use competition specific weights when prioritizing keeps. --- neurons/validator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/neurons/validator.py b/neurons/validator.py index 8a67024..b01bd76 100644 --- a/neurons/validator.py +++ b/neurons/validator.py @@ -914,8 +914,8 @@ async def run_step(self): model_prioritization = { uid: ( # Add 1 to ensure it is always greater than a win rate. - 1 + self.weights[uid].item() - if self.weights[uid].item() >= 0.001 + 1 + competition_weights[uid].item() + if competition_weights[uid].item() >= 0.001 else wr ) for uid, wr in win_rate.items() From d728604053a9bf1fb6eefa7ce1e48f0404cc2897 Mon Sep 17 00:00:00 2001 From: Sid Date: Wed, 14 Aug 2024 16:03:00 -0700 Subject: [PATCH 4/5] Use tracker competition weights which include moving avg. --- neurons/validator.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/neurons/validator.py b/neurons/validator.py index b01bd76..376ddbe 100644 --- a/neurons/validator.py +++ b/neurons/validator.py @@ -911,11 +911,13 @@ async def run_step(self): # Prioritize models for keeping up to the sample_min for the next eval loop. # If the model has any significant weight, prioritize by weight with greater weights being kept first. # Then for the unweighted models, prioritize by win_rate. + # Use the competition weights from the tracker which also handles moving averages. + tracker_competition_weights = self.competition_tracker.get_competition_weights(competition.id) model_prioritization = { uid: ( # Add 1 to ensure it is always greater than a win rate. - 1 + competition_weights[uid].item() - if competition_weights[uid].item() >= 0.001 + 1 + tracker_competition_weights[uid].item() + if tracker_competition_weights[uid].item() >= 0.001 else wr ) for uid, wr in win_rate.items() From eb42ca3853a19469977f8537e7aad3da5c160cef Mon Sep 17 00:00:00 2001 From: Sid Date: Thu, 15 Aug 2024 08:40:56 -0700 Subject: [PATCH 5/5] Scale model_limit by competition count. --- constants/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/constants/__init__.py b/constants/__init__.py index 09b1a1f..302998c 100644 --- a/constants/__init__.py +++ b/constants/__init__.py @@ -226,7 +226,8 @@ # validator eval batch min to keep for next loop. sample_min = 5 # Max number of uids that can be either pending eval or currently being evaluated. -updated_models_limit = 15 +# We allow the sample_min per competition + 10 additional models to be held at any one time. +updated_models_limit = sample_min * len(MODEL_CONSTRAINTS_BY_COMPETITION_ID) + 10 # time required between updates to the chain. chain_update_cadence = dt.timedelta(minutes=20) # time required between retrying evaluation of a stale model. (First retry will be immediate).