From b3f9e5f3c9635af17236eafda8338efc8170c50e Mon Sep 17 00:00:00 2001 From: Sid Date: Fri, 23 Aug 2024 10:53:02 -0700 Subject: [PATCH 1/3] Fix table wrapping and bump taoverse. --- neurons/validator.py | 38 +++++++++++++++++++------------------- requirements.txt | 2 +- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/neurons/validator.py b/neurons/validator.py index 923b60e..23ff066 100644 --- a/neurons/validator.py +++ b/neurons/validator.py @@ -707,8 +707,8 @@ async def run_step(self): # Keep track of which block this uid last updated their model. # Default to an infinite block if we can't retrieve the metadata for the miner. uid_to_block = defaultdict(lambda: math.inf) - # Keep track of the hugging face url for this uid. - uid_to_hf_url = defaultdict(lambda: "unknown") + # Keep track of the hugging face repo for this uid. + uid_to_hf = defaultdict(lambda: "unknown") bt.logging.trace(f"Current block: {cur_block}") @@ -790,8 +790,8 @@ async def run_step(self): # Update the block this uid last updated their model. uid_to_block[uid_i] = model_i_metadata.block - # Update the hf url for this model. - uid_to_hf_url[uid_i] = model_utils.get_hf_url(model_i_metadata) + # Update the hf repo for this model. + uid_to_hf[uid_i] = model_utils.get_hf_repo_name(model_i_metadata) # Get the model locally and evaluate its loss. model_i = None @@ -893,7 +893,7 @@ async def run_step(self): CompetitionId.B7_MODEL_LOWER_EPSILON, uids, uid_to_block, - uid_to_hf_url, + uid_to_hf, uids_to_competition_ids_epsilon_experiment, pages, model_weights_epsilon_experiment, @@ -959,7 +959,7 @@ async def run_step(self): competition.id, uids, uid_to_block, - uid_to_hf_url, + uid_to_hf, self._get_uids_to_competition_ids(), pages, model_weights, @@ -978,7 +978,7 @@ def log_step( competition_id: CompetitionId, uids: typing.List[int], uid_to_block: typing.Dict[int, int], - uid_to_hf_url: typing.Dict[int, str], + uid_to_hf: typing.Dict[int, str], uid_to_competition_id: typing.Dict[int, typing.Optional[int]], pages: typing.List[str], model_weights: typing.List[float], @@ -1007,7 +1007,7 @@ def log_step( step_log["uid_data"][str(uid)] = { "uid": uid, "block": uid_to_block[uid], - "hf_url": uid_to_hf_url[uid], + "hf": uid_to_hf[uid], "competition_id": uid_to_competition_id[uid], "average_loss": sum(losses_per_uid[uid]) / len(losses_per_uid[uid]), "win_rate": win_rate[uid], @@ -1015,20 +1015,21 @@ def log_step( "weight": self.weights[uid].item(), "norm_weight": sub_competition_weights[idx].item(), } - table = Table(title="Step") + table = Table(title="Step", expand=True) table.add_column("uid", justify="right", style="cyan", no_wrap=True) - table.add_column("average_loss", style="magenta") - table.add_column("win_rate", style="magenta") - table.add_column("win_total", style="magenta") - table.add_column("weights", style="magenta") - table.add_column("competition_weights", style="magenta") - table.add_column("block", style="magenta") - table.add_column("competition", style="magenta") - table.add_column("hugging_face_url", style="magenta") + table.add_column("hf", style="magenta", overflow="fold") + table.add_column("average_loss", style="magenta", overflow="fold") + table.add_column("win_rate", style="magenta", overflow="fold") + table.add_column("win_total", style="magenta", overflow="fold") + table.add_column("total_weight", style="magenta", overflow="fold") + table.add_column("compe_weight", style="magenta", overflow="fold") + table.add_column("block", style="magenta", overflow="fold") + table.add_column("comp", style="magenta", overflow="fold") for idx, uid in enumerate(uids): try: table.add_row( str(uid), + str(step_log["uid_data"][str(uid)]["hf"]), str(round(step_log["uid_data"][str(uid)]["average_loss"], 4)), str(round(step_log["uid_data"][str(uid)]["win_rate"], 4)), str(step_log["uid_data"][str(uid)]["win_total"]), @@ -1036,7 +1037,6 @@ def log_step( str(round(sub_competition_weights[idx].item(), 4)), str(step_log["uid_data"][str(uid)]["block"]), str(step_log["uid_data"][str(uid)]["competition_id"]), - str(step_log["uid_data"][str(uid)]["hf_url"]), ) except: pass @@ -1092,7 +1092,7 @@ def log_step( str(uid): uid_data[str(uid)]["win_total"] for uid in uids }, "weight_data": {str(uid): self.weights[uid].item() for uid in uids}, - "norm_weight_data": { + "competition_weight_data": { str(uid): sub_competition_weights[i].item() for i, uid in enumerate(uids) }, diff --git a/requirements.txt b/requirements.txt index 08fa165..dcfee97 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,4 +11,4 @@ transformers==4.44.1 wandb datasets flash-attn -taoverse==1.0.1 +taoverse==1.0.2 From 2384ff60b0b7d0e3557edf2d3741d6a64d267dec Mon Sep 17 00:00:00 2001 From: Sid Date: Fri, 23 Aug 2024 11:04:41 -0700 Subject: [PATCH 2/3] Add currently unused epsilon_func for taoverse 1.0.2. --- constants/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/constants/__init__.py b/constants/__init__.py index e0d5ecb..d02abbe 100644 --- a/constants/__init__.py +++ b/constants/__init__.py @@ -24,6 +24,7 @@ ModelConstraints, NormValidationConstraints, ) +from taoverse.model.competition.epsilon import FixedEpsilon from competitions.data import CompetitionId from typing import Dict, List, Tuple @@ -103,6 +104,7 @@ allowed_architectures=ALLOWED_MODEL_TYPES_1, tokenizer="distilgpt2", eval_block_delay=0, + epsilon_func=FixedEpsilon(0.005), ), CompetitionId.B7_MODEL: ModelConstraints( max_model_parameter_size=6_900_000_000, @@ -115,6 +117,7 @@ "attn_implementation": "flash_attention_2", }, eval_block_delay=0, + epsilon_func=FixedEpsilon(0.005), ), CompetitionId.B3_MODEL: ModelConstraints( max_model_parameter_size=3_400_000_000, @@ -127,6 +130,7 @@ "attn_implementation": "flash_attention_2", }, eval_block_delay=0, + epsilon_func=FixedEpsilon(0.005), ), } From 288466b74454acabf9a75b9361c0970a6a81e64a Mon Sep 17 00:00:00 2001 From: Sid Date: Fri, 23 Aug 2024 11:28:13 -0700 Subject: [PATCH 3/3] Bump to 4.1.2 and fix typo. --- constants/__init__.py | 2 +- neurons/validator.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/constants/__init__.py b/constants/__init__.py index d02abbe..2838c56 100644 --- a/constants/__init__.py +++ b/constants/__init__.py @@ -34,7 +34,7 @@ # --------------------------------- # Release -__version__ = "4.1.1" +__version__ = "4.1.2" # Validator schema version __validator_version__ = "3.0.0" diff --git a/neurons/validator.py b/neurons/validator.py index 23ff066..615eb9b 100644 --- a/neurons/validator.py +++ b/neurons/validator.py @@ -1022,7 +1022,7 @@ def log_step( table.add_column("win_rate", style="magenta", overflow="fold") table.add_column("win_total", style="magenta", overflow="fold") table.add_column("total_weight", style="magenta", overflow="fold") - table.add_column("compe_weight", style="magenta", overflow="fold") + table.add_column("comp_weight", style="magenta", overflow="fold") table.add_column("block", style="magenta", overflow="fold") table.add_column("comp", style="magenta", overflow="fold") for idx, uid in enumerate(uids):