Release 4.1.0

macrocosm-os · Aug 12, 2024 · d017f7b · d017f7b
2 parents 9db1bad + 84a225f
commit d017f7b
Show file tree

Hide file tree

Showing 7 changed files with 421 additions and 202 deletions.
diff --git a/competitions/data.py b/competitions/data.py
@@ -10,6 +10,8 @@ class CompetitionId(IntEnum):
 
     B3_MODEL = 2
 
+    B7_MODEL_LOWER_EPSILON = 3
+
     # Overwrite the default __repr__, which doesn't work with
     # bt.logging for some unknown reason.
     def __repr__(self) -> str:

diff --git a/constants/__init__.py b/constants/__init__.py
@@ -35,7 +35,7 @@
 # ---------------------------------
 
 # Release
-__version__ = "4.0.1"
+__version__ = "4.1.0"
 
 # Validator schema version
 __validator_version__ = "3.0.0"
@@ -59,6 +59,9 @@
 # This corresponded to top-10 validator on july 31st, 2024
 WEIGHT_SYNC_VALI_MIN_STAKE = 200_000
 
+# Starting block for 3B, 7B* (epsilon experiment) and sample unpacking
+BLOCK_3B_7BSTAR_UNPACK = 3_601_190
+
 # Minimum percent of weight on a vali for a miner to be considered a top miner.
 # Since there can be multiple competitions at different reward percentages we can't just check biggest.
 WEIGHT_SYNC_MINER_MIN_PERCENT = 0.10
@@ -92,9 +95,9 @@
 
 # Defined dataset by competition id
 DATASET_BY_COMPETITION_ID: Dict[CompetitionId, str] = {
-    CompetitionId.M772_MODEL : pt.dataset.SubsetFalconLoader,
-    CompetitionId.B3_MODEL : pt.dataset.SubsetFalconLoader,
-    CompetitionId.B7_MODEL : pt.dataset.SubsetFineWebEdu2Loader,
+    CompetitionId.M772_MODEL: pt.dataset.SubsetFalconLoader,
+    CompetitionId.B3_MODEL: pt.dataset.SubsetFalconLoader,
+    CompetitionId.B7_MODEL: pt.dataset.SubsetFineWebEdu2Loader,
 }
 
 # Defined model constraints by competition id to ensure they are constant across blocks.
@@ -130,7 +133,7 @@
             "attn_implementation": "flash_attention_2",
         },
         eval_block_delay=0,
-    )
+    ),
 }
 
 
@@ -144,7 +147,6 @@
                 MODEL_CONSTRAINTS_BY_COMPETITION_ID[CompetitionId.B7_MODEL],
                 1.0,
             )
-
         ],
     ),
     (
@@ -159,6 +161,26 @@
                 CompetitionId.B7_MODEL,
                 MODEL_CONSTRAINTS_BY_COMPETITION_ID[CompetitionId.B7_MODEL],
                 0.65,
+            ),
+        ],
+    ),
+    (
+        BLOCK_3B_7BSTAR_UNPACK,
+        [
+            Competition(
+                CompetitionId.M772_MODEL,
+                MODEL_CONSTRAINTS_BY_COMPETITION_ID[CompetitionId.M772_MODEL],
+                0.14,
+            ),
+            Competition(
+                CompetitionId.B3_MODEL,
+                MODEL_CONSTRAINTS_BY_COMPETITION_ID[CompetitionId.B3_MODEL],
+                0.29,
+            ),
+            Competition(
+                CompetitionId.B7_MODEL,
+                MODEL_CONSTRAINTS_BY_COMPETITION_ID[CompetitionId.B7_MODEL],
+                0.57,
             )
 
         ],
@@ -188,8 +210,18 @@
 temperature = 0.01
 # validator score boosting for earlier models.
 timestamp_epsilon = 0.005
+
+# block to activate sample unpacking
+sample_unpack_block = BLOCK_3B_7BSTAR_UNPACK
+
 # validators number of pages to eval over miners on each step.
-n_eval_pages = 18
+pages_per_eval_unpack = 5 # With sample unpacking
+pages_per_eval_pack = 18
+
+timestamp_epsilon_experiment_start_block = BLOCK_3B_7BSTAR_UNPACK
+timestamp_epsilon_experiment = 0.001
+timestamp_epsilon_experiment_weight_percent = 0.123
+
 # validator eval batch size.
 batch_size = 1
 # validator eval batch min to keep for next loop.

diff --git a/docs/validator.md b/docs/validator.md
@@ -25,7 +25,7 @@ You can view the entire validation system by reading the code in `neurons/valida
             for model_b in models:
                 for i in len( batches )
                     # Determine if better model loss with relative block number boosting.
-                    if iswin( model_losses[ model_a ][ i ], model_losses[ model_b ][ i ], block_a, block_b ):
+                    if iswin( model_losses[ model_a ][ i ], model_losses[ model_b ][ i ], block_a, block_b, epsilon = constants.timestamp_epsilon):
                         model_wins[ model_a ] += 1
 
         # End epoch.
@@ -38,12 +38,12 @@ You can view the entire validation system by reading the code in `neurons/valida
         set_weights( weight )
 ```
 
-The behaviour of `iswin( loss_a, loss_b, block_a, block_b)` function intentionally skews the win function to reward models which have been hosted earlier such that newer models are only better than others iff their loss is `epsilon` percent lower accoring to the following function. Currently `epsilon` is set to 1% and is a hyper parameter of the mechanism
+The behaviour of `iswin( loss_a, loss_b, block_a, block_b, epsilon)` function intentionally skews the win function to reward models which have been hosted earlier such that newer models are only better than others iff their loss is `epsilon` percent lower accoring to the following function. Currently `epsilon` is set to 1% and is a hyper parameter of the mechanism
 
 ```python
-def iswin( loss_a, loss_b, block_a, block_b ):
-    loss_a = (1 - constants.timestamp_epsilon) * loss_a if block_a < block_b else loss_a
-    loss_b = (1 - constants.timestamp_epsilon) * loss_b if block_b < block_a else loss_b
+def iswin( loss_a, loss_b, block_a, block_b, epsilon):
+    loss_a = (1 - epsilon) * loss_a if block_a < block_b else loss_a
+    loss_b = (1 - epsilon) * loss_b if block_b < block_a else loss_b
     return loss_a < loss_b
 ```
 

diff --git a/neurons/config.py b/neurons/config.py
@@ -29,8 +29,8 @@ def validator_config():
     parser.add_argument(
         "--pages_per_eval",
         type=int,
-        default=constants.n_eval_pages,
-        help="Number of pages used to eval each step.",
+        default=None,
+        help="Number of pages used to eval each step. If not specified, it will be automatically set.",
     )
     parser.add_argument(
         "--sample_min",