Merge pull request #169 from macrocosm-os/dev

Release 4.2.0

.gitignore

@@ -2,6 +2,7 @@
 .vscode/
 
 test-models/
+model-store/
 
 # Exclude the Miner's directory for saving the models.
 local-models/

README.md

@@ -12,7 +12,7 @@
 
 ---
 
-[Leaderboard](https://huggingface.co/spaces/macrocosm-os/pretraining-leaderboard) • [Discord](https://discord.gg/bittensor) • [Network](https://taostats.io/subnets/netuid-9/) • [Research](https://bittensor.com/whitepaper) 
+[Leaderboard](https://www.macrocosmos.ai/sn9) • [Discord](https://discord.gg/bittensor) • [Network](https://taostats.io/subnets/netuid-9/) • [Research](https://bittensor.com/whitepaper) 
 </div>
 
 ---
@@ -43,7 +43,7 @@ Miners within this subnet are evaluated based on the number of times the model t
 
 TL;DR:
 1. [Chat](https://discord.gg/bittensor)
-2. [Leaderboard](https://huggingface.co/spaces/macrocosm-os/pretraining-leaderboard)
+2. [Leaderboard](https://www.macrocosmos.ai/sn9)
 
 This repo's main conversation is carried out in the Bittensor [Discord](https://discord.gg/bittensor). Visit the 'pretraining' channel to ask questions and get real time feedback. You can view the ongoing running of the incentive mechanism, the best miners (see 'incentive'), the most in consensus validators (see 'vtrust') using this [taostats link](https://taostats.io/subnets/netuid-9/). The table shows all 256 participant UIDs with corresponding YC stats and earnings. 
 

competitions/data.py

@@ -12,6 +12,8 @@ class CompetitionId(IntEnum):
 
     B7_MODEL_LOWER_EPSILON = 3
 
+    B14_MODEL = 4
+
     # Overwrite the default __repr__, which doesn't work with
     # bt.logging for some unknown reason.
     def __repr__(self) -> str:

constants/__init__.py

@@ -34,10 +34,10 @@
 # ---------------------------------
 
 # Release
-__version__ = "4.1.2"
+__version__ = "4.2.0"
 
 # Validator schema version
-__validator_version__ = "3.0.0"
+__validator_version__ = "3.1.0"
 version_split = __validator_version__.split(".")
 __spec_version__ = (
     (1000 * int(version_split[0]))
@@ -93,6 +93,7 @@
     CompetitionId.M772_MODEL: pt.dataset.SubsetFalconLoader,
     CompetitionId.B3_MODEL: pt.dataset.SubsetFalconLoader,
     CompetitionId.B7_MODEL: pt.dataset.SubsetFineWebEdu2Loader,
+    CompetitionId.B14_MODEL: pt.dataset.SubsetFineWebEdu2Loader,
 }
 
 # Defined model constraints by competition id to ensure they are constant across blocks.
@@ -105,6 +106,7 @@
         tokenizer="distilgpt2",
         eval_block_delay=0,
         epsilon_func=FixedEpsilon(0.005),
+        max_bytes=5 * 1024 * 1024 * 1024,
     ),
     CompetitionId.B7_MODEL: ModelConstraints(
         max_model_parameter_size=6_900_000_000,
@@ -118,6 +120,7 @@
         },
         eval_block_delay=0,
         epsilon_func=FixedEpsilon(0.005),
+        max_bytes=15 * 1024 * 1024 * 1024,
     ),
     CompetitionId.B3_MODEL: ModelConstraints(
         max_model_parameter_size=3_400_000_000,
@@ -131,7 +134,23 @@
         },
         eval_block_delay=0,
         epsilon_func=FixedEpsilon(0.005),
+        max_bytes=15 * 1024 * 1024 * 1024,
     ),
+    CompetitionId.B14_MODEL: ModelConstraints(
+        max_model_parameter_size=13_900_000_000,
+        min_model_parameter_size=13_700_000_000,
+        sequence_length=4096,
+        allowed_architectures=ALLOWED_MODEL_TYPES_2,
+        tokenizer="Xenova/gpt-4",
+        kwargs={
+            "torch_dtype": torch.bfloat16,
+            "attn_implementation": "flash_attention_2",
+        },
+        eval_block_delay=0,
+        epsilon_func=FixedEpsilon(0.005),
+        max_bytes=29 * 1024 * 1024 * 1024,
+    ),
+
 }
 
 
@@ -182,6 +201,33 @@
             ),
         ],
     ),
+    (
+        3_750_683,
+        [
+            Competition(
+                CompetitionId.M772_MODEL,
+                MODEL_CONSTRAINTS_BY_COMPETITION_ID[CompetitionId.M772_MODEL],
+                0.14,
+            ),
+            Competition(
+                CompetitionId.B3_MODEL,
+                MODEL_CONSTRAINTS_BY_COMPETITION_ID[CompetitionId.B3_MODEL],
+                0.29,
+            ),
+            Competition(
+                CompetitionId.B7_MODEL,
+                MODEL_CONSTRAINTS_BY_COMPETITION_ID[CompetitionId.B7_MODEL],
+                0.15,
+            ),
+            Competition(
+                CompetitionId.B14_MODEL,
+                MODEL_CONSTRAINTS_BY_COMPETITION_ID[CompetitionId.B14_MODEL],
+                0.42,
+            )
+
+        ],
+    ),
+
 ]
 
 for block_and_competitions in COMPETITION_SCHEDULE_BY_BLOCK:
@@ -216,6 +262,7 @@
 pages_per_eval_pack = 18
 
 timestamp_epsilon_experiment_start_block = BLOCK_3B_7BSTAR_UNPACK
+timestamp_epsilon_experiment_end_block = 3_750_683
 timestamp_epsilon_experiment = 0.001
 timestamp_epsilon_experiment_weight_percent = 0.123
 

docs/validator.md

@@ -51,7 +51,7 @@ It is important to note that this affects the game theoretics of the incentive l
 
 # System Requirements
 
-Validators will need enough disk space to store the models of miners being evaluated. Each model has a max size by block defined in [constants/__init__.py](https://github.com/macrocosm-os/pretraining/blob/main/constants/__init__.py#L57) and the validator has cleanup logic to remove old models. It is recommended to have at least 1 TB of disk space.
+Validators will need enough disk space to store the models of miners being evaluated. Each model has a max size by block defined in [constants/__init__.py](https://github.com/macrocosm-os/pretraining/blob/main/constants/__init__.py#L57) and the validator has cleanup logic to remove old models. It is recommended to have at least 2 TB of disk space and 80GB of system memory.
 
 Validators will need enough processing power to evaluate their model. As of Apr 1st, 2024 it is required to have a GPU that supports [flash attention 2](https://github.com/Dao-AILab/flash-attention) with atleast 48 GB of VRAM and at least 38 TFLOPs for half precision (bfloat 16) operations.