Skip to content

Commit

Permalink
Merge pull request #157 from macrocosm-os/fix_wrap_taoverse
Browse files Browse the repository at this point in the history 
Bump pretraining/taoverse versions and improve logging.
  • Loading branch information
@cryptal-mc
cryptal-mc committed Aug 23, 2024
2 parents 4922fe9 + 288466b commit f10dd01
Show file tree
Hide file tree
Showing 3 changed files with 25 additions and 21 deletions.
6 changes: 5 additions & 1 deletion constants/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
ModelConstraints,
NormValidationConstraints,
)
from taoverse.model.competition.epsilon import FixedEpsilon
from competitions.data import CompetitionId

from typing import Dict, List, Tuple
Expand All @@ -33,7 +34,7 @@
# ---------------------------------

# Release
__version__ = "4.1.1"
__version__ = "4.1.2"

# Validator schema version
__validator_version__ = "3.0.0"
Expand Down Expand Up @@ -103,6 +104,7 @@
allowed_architectures=ALLOWED_MODEL_TYPES_1,
tokenizer="distilgpt2",
eval_block_delay=0,
epsilon_func=FixedEpsilon(0.005),
),
CompetitionId.B7_MODEL: ModelConstraints(
max_model_parameter_size=6_900_000_000,
Expand All @@ -115,6 +117,7 @@
"attn_implementation": "flash_attention_2",
},
eval_block_delay=0,
epsilon_func=FixedEpsilon(0.005),
),
CompetitionId.B3_MODEL: ModelConstraints(
max_model_parameter_size=3_400_000_000,
Expand All @@ -127,6 +130,7 @@
"attn_implementation": "flash_attention_2",
},
eval_block_delay=0,
epsilon_func=FixedEpsilon(0.005),
),
}

Expand Down
38 changes: 19 additions & 19 deletions neurons/validator.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -707,8 +707,8 @@ async def run_step(self):
# Keep track of which block this uid last updated their model.
# Default to an infinite block if we can't retrieve the metadata for the miner.
uid_to_block = defaultdict(lambda: math.inf)
# Keep track of the hugging face url for this uid.
uid_to_hf_url = defaultdict(lambda: "unknown")
# Keep track of the hugging face repo for this uid.
uid_to_hf = defaultdict(lambda: "unknown")

bt.logging.trace(f"Current block: {cur_block}")

Expand Down Expand Up @@ -790,8 +790,8 @@ async def run_step(self):

# Update the block this uid last updated their model.
uid_to_block[uid_i] = model_i_metadata.block
# Update the hf url for this model.
uid_to_hf_url[uid_i] = model_utils.get_hf_url(model_i_metadata)
# Update the hf repo for this model.
uid_to_hf[uid_i] = model_utils.get_hf_repo_name(model_i_metadata)

# Get the model locally and evaluate its loss.
model_i = None
Expand Down Expand Up @@ -893,7 +893,7 @@ async def run_step(self):
CompetitionId.B7_MODEL_LOWER_EPSILON,
uids,
uid_to_block,
uid_to_hf_url,
uid_to_hf,
uids_to_competition_ids_epsilon_experiment,
pages,
model_weights_epsilon_experiment,
Expand Down Expand Up @@ -959,7 +959,7 @@ async def run_step(self):
competition.id,
uids,
uid_to_block,
uid_to_hf_url,
uid_to_hf,
self._get_uids_to_competition_ids(),
pages,
model_weights,
Expand All @@ -978,7 +978,7 @@ def log_step(
competition_id: CompetitionId,
uids: typing.List[int],
uid_to_block: typing.Dict[int, int],
uid_to_hf_url: typing.Dict[int, str],
uid_to_hf: typing.Dict[int, str],
uid_to_competition_id: typing.Dict[int, typing.Optional[int]],
pages: typing.List[str],
model_weights: typing.List[float],
Expand Down Expand Up @@ -1007,36 +1007,36 @@ def log_step(
step_log["uid_data"][str(uid)] = {
"uid": uid,
"block": uid_to_block[uid],
"hf_url": uid_to_hf_url[uid],
"hf": uid_to_hf[uid],
"competition_id": uid_to_competition_id[uid],
"average_loss": sum(losses_per_uid[uid]) / len(losses_per_uid[uid]),
"win_rate": win_rate[uid],
"win_total": wins[uid],
"weight": self.weights[uid].item(),
"norm_weight": sub_competition_weights[idx].item(),
}
table = Table(title="Step")
table = Table(title="Step", expand=True)
table.add_column("uid", justify="right", style="cyan", no_wrap=True)
table.add_column("average_loss", style="magenta")
table.add_column("win_rate", style="magenta")
table.add_column("win_total", style="magenta")
table.add_column("weights", style="magenta")
table.add_column("competition_weights", style="magenta")
table.add_column("block", style="magenta")
table.add_column("competition", style="magenta")
table.add_column("hugging_face_url", style="magenta")
table.add_column("hf", style="magenta", overflow="fold")
table.add_column("average_loss", style="magenta", overflow="fold")
table.add_column("win_rate", style="magenta", overflow="fold")
table.add_column("win_total", style="magenta", overflow="fold")
table.add_column("total_weight", style="magenta", overflow="fold")
table.add_column("comp_weight", style="magenta", overflow="fold")
table.add_column("block", style="magenta", overflow="fold")
table.add_column("comp", style="magenta", overflow="fold")
for idx, uid in enumerate(uids):
try:
table.add_row(
str(uid),
str(step_log["uid_data"][str(uid)]["hf"]),
str(round(step_log["uid_data"][str(uid)]["average_loss"], 4)),
str(round(step_log["uid_data"][str(uid)]["win_rate"], 4)),
str(step_log["uid_data"][str(uid)]["win_total"]),
str(round(self.weights[uid].item(), 4)),
str(round(sub_competition_weights[idx].item(), 4)),
str(step_log["uid_data"][str(uid)]["block"]),
str(step_log["uid_data"][str(uid)]["competition_id"]),
str(step_log["uid_data"][str(uid)]["hf_url"]),
)
except:
pass
Expand Down Expand Up @@ -1092,7 +1092,7 @@ def log_step(
str(uid): uid_data[str(uid)]["win_total"] for uid in uids
},
"weight_data": {str(uid): self.weights[uid].item() for uid in uids},
"norm_weight_data": {
"competition_weight_data": {
str(uid): sub_competition_weights[i].item()
for i, uid in enumerate(uids)
},
Expand Down
2 changes: 1 addition & 1 deletion requirements.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,4 +11,4 @@ transformers==4.44.1
wandb
datasets
flash-attn
taoverse==1.0.1
taoverse==1.0.2

0 comments on commit f10dd01

Please sign in to comment.