diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b7867f7..5be5ce19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,16 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.70.1] - 2024-02-15 + +### Fixed + +- Fixed compatibility issues with transformers >= 4.41.0 + +### Changed + +- QoL changes for MonoT5 style models + ## [0.70.0] - 2024-02-15 ### Added diff --git a/setup.py b/setup.py index 657771f1..3ad5b5ef 100755 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name="simpletransformers", - version="0.70.0", + version="0.70.1", author="Thilina Rajapakse", author_email="chaturangarajapakshe@gmail.com", description="An easy-to-use wrapper library for the Transformers library.", diff --git a/simpletransformers/classification/classification_model.py b/simpletransformers/classification/classification_model.py index 6ea168b4..27ba6252 100755 --- a/simpletransformers/classification/classification_model.py +++ b/simpletransformers/classification/classification_model.py @@ -1926,7 +1926,7 @@ def compute_metrics( mcc = matthews_corrcoef(labels, preds) accuracy = accuracy_score(labels, preds) - f1 = f1_score(labels, preds) + f1 = f1_score(labels, preds, average="macro") if self.model.num_labels == 2: tn, fp, fn, tp = confusion_matrix(labels, preds, labels=[0, 1]).ravel() if self.args.sliding_window: diff --git a/simpletransformers/classification/multi_modal_classification_model.py b/simpletransformers/classification/multi_modal_classification_model.py index 9ec83919..2ceeaa79 100644 --- a/simpletransformers/classification/multi_modal_classification_model.py +++ b/simpletransformers/classification/multi_modal_classification_model.py @@ -38,7 +38,6 @@ from torch.optim import AdamW from transformers.optimization import Adafactor from transformers import ( - BERT_PRETRAINED_MODEL_ARCHIVE_LIST, WEIGHTS_NAME, BertConfig, BertModel, @@ -178,24 +177,6 @@ def __init__( self.config, self.transformer, self.img_encoder ) - if model_name not in BERT_PRETRAINED_MODEL_ARCHIVE_LIST: - try: - self.model.load_state_dict( - torch.load(os.path.join(model_name, "pytorch_model.bin")) - ) - except EnvironmentError: - msg = ( - "Model name '{}' was not found in model name list ({}). " - "We assumed '{}' was a path or url to model weight files named one of {} but " - "couldn't find any such file at this path or url.".format( - model_name, - ", ".join(BERT_PRETRAINED_MODEL_ARCHIVE_LIST), - model_name, - "pytorch_model.bin", - ) - ) - raise EnvironmentError(msg) - self.tokenizer = tokenizer_class.from_pretrained( model_name, do_lower_case=self.args.do_lower_case, **kwargs ) diff --git a/simpletransformers/config/model_args.py b/simpletransformers/config/model_args.py index 2bb26f98..2d23ee66 100644 --- a/simpletransformers/config/model_args.py +++ b/simpletransformers/config/model_args.py @@ -224,6 +224,7 @@ class T5Args(ModelArgs): model_class: str = "T5Model" add_prefix: bool = True + as_reranker: bool = False dataset_class: Dataset = None do_sample: bool = False early_stopping: bool = True diff --git a/simpletransformers/custom_models/models.py b/simpletransformers/custom_models/models.py index 75e487d3..03ca4a12 100755 --- a/simpletransformers/custom_models/models.py +++ b/simpletransformers/custom_models/models.py @@ -34,15 +34,8 @@ BigBirdPreTrainedModel, ) from transformers.models.camembert.configuration_camembert import CamembertConfig -from transformers.models.camembert.modeling_camembert import ( - CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_LIST, -) from transformers.models.distilbert.configuration_distilbert import DistilBertConfig -from transformers.models.distilbert.modeling_distilbert import ( - DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST, -) from transformers.models.electra.modeling_electra import ( - ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST, ElectraConfig, ElectraModel, ElectraPreTrainedModel, @@ -59,7 +52,6 @@ from transformers.models.rembert.configuration_rembert import RemBertConfig from transformers.models.roberta.configuration_roberta import RobertaConfig from transformers.models.roberta.modeling_roberta import ( - ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST, RobertaClassificationHead, RobertaForQuestionAnswering, RobertaPreTrainedModel, @@ -67,9 +59,6 @@ MaskedLMOutput, ) from transformers.models.xlm_roberta.configuration_xlm_roberta import XLMRobertaConfig -from transformers.models.xlm_roberta.modeling_xlm_roberta import ( - XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST, -) from simpletransformers.custom_models.retrieval_autoencoder import Autoencoder @@ -237,7 +226,6 @@ class RobertaForMultiLabelSequenceClassification(BertPreTrainedModel): """ config_class = RobertaConfig - pretrained_model_archive_map = ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST base_model_prefix = "roberta" def __init__(self, config, pos_weight=None): @@ -304,7 +292,6 @@ class CamembertForMultiLabelSequenceClassification( """ config_class = CamembertConfig - pretrained_model_archive_map = CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_LIST base_model_prefix = "camembert" @@ -430,7 +417,6 @@ class DistilBertPreTrainedModel(PreTrainedModel): """ config_class = DistilBertConfig - pretrained_model_archive_map = DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST load_tf_weights = None base_model_prefix = "distilbert" @@ -664,7 +650,6 @@ class XLMRobertaForMultiLabelSequenceClassification( RobertaForMultiLabelSequenceClassification ): config_class = XLMRobertaConfig - pretrained_model_archive_map = XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST class ElectraPooler(nn.Module): @@ -773,7 +758,6 @@ class ElectraForSequenceClassification(ElectraPreTrainedModel): Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. """ # noqa config_class = ElectraConfig - pretrained_model_archive_map = ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST base_model_prefix = "electra" def __init__(self, config, weight=None): @@ -829,7 +813,6 @@ class ElectraForMultiLabelSequenceClassification(ElectraPreTrainedModel): """ config_class = ElectraConfig - pretrained_model_archive_map = ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST base_model_prefix = "electra" def __init__(self, config, pos_weight=None): @@ -883,7 +866,6 @@ class ElectraForQuestionAnswering(ElectraPreTrainedModel): """ config_class = ElectraConfig - pretrained_model_archive_map = ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST base_model_prefix = "electra" def __init__(self, config, weight=None): @@ -949,7 +931,6 @@ def forward( class XLMRobertaForQuestionAnswering(RobertaForQuestionAnswering): config_class = XLMRobertaConfig - pretrained_model_archive_map = XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST class BigBirdForMultiLabelSequenceClassification(BigBirdPreTrainedModel): diff --git a/simpletransformers/language_modeling/language_modeling_model.py b/simpletransformers/language_modeling/language_modeling_model.py index b5b65a47..1d852480 100755 --- a/simpletransformers/language_modeling/language_modeling_model.py +++ b/simpletransformers/language_modeling/language_modeling_model.py @@ -910,7 +910,7 @@ def collate(examples: List[torch.Tensor]): if args.fp16: with amp.autocast(): if args.model_type == "longformer": - outputs = model(inputs, attention_mask=None, labels=labels) + outputs = model(inputs, labels=labels) elif args.model_type == "electra": outputs = model( inputs, @@ -933,9 +933,7 @@ def collate(examples: List[torch.Tensor]): loss = outputs[0] else: if args.model_type == "longformer": - outputs = model( - **inputs_dict, attention_mask=None, labels=labels - ) + outputs = model(**inputs_dict, labels=labels) elif args.model_type == "electra": outputs = model( input_ids, diff --git a/simpletransformers/t5/t5_model.py b/simpletransformers/t5/t5_model.py index b6dfeddd..d7f85a26 100644 --- a/simpletransformers/t5/t5_model.py +++ b/simpletransformers/t5/t5_model.py @@ -179,6 +179,8 @@ def train_model( show_running_loss=True, args=None, eval_data=None, + qrels=None, + run_dict=None, verbose=True, **kwargs, ): @@ -194,6 +196,9 @@ def train_model( show_running_loss (optional): Set to False to prevent running loss from being printed to console. Defaults to True. args (optional): Optional changes to the args dict of the model. Any changes made will persist for the model. eval_data (optional): A DataFrame against which evaluation will be performed when evaluate_during_training is enabled. Is required if evaluate_during_training is enabled. + verbose (optional): If verbose, results will be printed to the console on completion of training. Defaults to True. + qrels (optional): Path to qrels file for evaluation. Only used with reranking. + run_dict (optional): Path to run file for evaluation. Only used with reranking. **kwargs: Additional metrics that should be used. Pass in the metrics as keyword arguments (name of metric: function to use). A metric function should take in two parameters. The first parameter will be the true labels, and the second parameter will be the predictions. Both inputs will be lists of strings. Note that this will slow down training significantly as the predicted sequences need to be generated. @@ -240,6 +245,9 @@ def train_model( show_running_loss=show_running_loss, eval_data=eval_data, verbose=verbose, + qrels=qrels, + run_dict=run_dict, + eval_name=None, **kwargs, ) @@ -260,6 +268,9 @@ def train( output_dir, show_running_loss=True, eval_data=None, + qrels=None, + run_dict=None, + eval_name=None, verbose=True, **kwargs, ): @@ -490,7 +501,9 @@ def train( logger.info(" Starting fine-tuning.") if args.evaluate_during_training: - training_progress_scores = self._create_training_progress_scores(**kwargs) + training_progress_scores = self._create_training_progress_scores( + eval_name, **kwargs + ) if args.wandb_project: wandb.init( @@ -610,12 +623,16 @@ def train( and global_step % args.evaluate_during_training_steps == 0 ): # Only evaluate when single GPU otherwise metrics may not average well - results = self.eval_model( + results = self._evaluate_during_training( eval_data, verbose=verbose and args.evaluate_during_training_verbose, silent=args.evaluate_during_training_silent, + qrels=qrels, + run_dict=run_dict, + eval_name=eval_name, **kwargs, ) + for key, value in results.items(): try: tb_writer.add_scalar( @@ -764,10 +781,13 @@ def train( self.save_model(output_dir_current, optimizer, scheduler, model=model) if args.evaluate_during_training and args.evaluate_each_epoch: - results = self.eval_model( + results = self._evaluate_during_training( eval_data, verbose=verbose and args.evaluate_during_training_verbose, silent=args.evaluate_during_training_silent, + qrels=qrels, + run_dict=run_dict, + eval_name=eval_name, **kwargs, ) @@ -1147,6 +1167,11 @@ def rerank(self, eval_data, qrels=None, run_dict=None, beir_format=False): self._move_model_to_device() args = self.args + if not self.args.as_reranker: + warnings.warn( + "rerank() being called on a model without as_reranker set to True. This may not work as expected." + ) + try: import pytrec_eval except ImportError: @@ -1156,7 +1181,10 @@ def rerank(self, eval_data, qrels=None, run_dict=None, beir_format=False): return eval_dataset = self.load_and_cache_examples( - eval_data, evaluate=True, tokenize_targets=False, reranking=True + eval_data, + tokenize_targets=False, + reranking=True, + evaluate=qrels is not None, ) eval_dataloader = DataLoader( eval_dataset, batch_size=self.args.eval_batch_size, shuffle=False @@ -1285,6 +1313,69 @@ def rerank(self, eval_data, qrels=None, run_dict=None, beir_format=False): return result_report + def _evaluate_during_training( + self, + eval_data, + qrels=None, + run_dict=None, + eval_name=None, + verbose=True, + silent=False, + **kwargs, + ): + """ + Utility function to evaluate the model during training. + """ + is_training = self.model.training + self.model.eval() + + if isinstance(eval_data, list): + results = {} + for eval_data_item, name, qrel, run in zip( + eval_data, eval_name, qrels, run_dict + ): + if self.args.as_reranker: + results_default_names = self.rerank( + eval_data_item, + qrels=qrel, + run_dict=run, + eval_name=name, + **kwargs, + ) + else: + results_default_names = self.eval_model( + eval_data_item, + verbose=verbose and self.args.evaluate_during_training_verbose, + silent=self.args.evaluate_during_training_silent, + **kwargs, + ) + + for key in results_default_names: + results[f"{eval_name}_{key}"] = results_default_names[key] + else: + if self.args.as_reranker: + results = self.rerank( + eval_data, + qrels=qrels, + run_dict=run_dict, + **kwargs, + ) + else: + results = self.eval_model( + eval_data, + verbose=verbose and self.args.evaluate_during_training_verbose, + silent=self.args.evaluate_during_training_silent, + **kwargs, + ) + + if is_training: + self.model.train() + + if verbose and self.args.as_reranker: + logger.info(results) + + return results + def _decode(self, output_id): return self.tokenizer.decode( output_id, @@ -1377,6 +1468,7 @@ def load_and_cache_examples( self.args, tokenize_targets=tokenize_targets, reranking=reranking, + evaluate=evaluate, ) return dataset elif args.dataset_class: @@ -1390,7 +1482,7 @@ def load_and_cache_examples( mode, ) - def _create_training_progress_scores(self, **kwargs): + def _create_training_progress_scores(self, eval_name=None, **kwargs): extra_metrics = {key: [] for key in kwargs} training_progress_scores = { "global_step": [], @@ -1399,6 +1491,26 @@ def _create_training_progress_scores(self, **kwargs): **extra_metrics, } + if self.args.as_reranker: + if isinstance(eval_name, list): + for name in eval_name: + training_progress_scores = { + **training_progress_scores, + f"{name}_recip_rank": [], + f"{name}_recall_100": [], + f"{name}_ndcg_cut_10": [], + f"{name}_ndcg": [], + } + training_progress_scores.pop("eval_loss") + else: + training_progress_scores = { + **training_progress_scores, + "recip_rank": [], + "recall_100": [], + "ndcg_cut_10": [], + "ndcg": [], + } + return training_progress_scores def _get_last_metrics(self, metric_values): diff --git a/simpletransformers/t5/t5_utils.py b/simpletransformers/t5/t5_utils.py index b5288004..175739b1 100644 --- a/simpletransformers/t5/t5_utils.py +++ b/simpletransformers/t5/t5_utils.py @@ -77,7 +77,9 @@ def preprocess_batch_for_hf_dataset(dataset, tokenizer, args, tokenize_targets=T return model_inputs -def load_hf_dataset(data, tokenizer, args, tokenize_targets=True, reranking=False): +def load_hf_dataset( + data, tokenizer, args, tokenize_targets=True, reranking=False, evaluate=False +): if isinstance(data, str): if (args.model_type == "eet5" or reranking) and os.path.isdir(data): dataset = load_from_disk(data) @@ -91,11 +93,20 @@ def load_hf_dataset(data, tokenizer, args, tokenize_targets=True, reranking=Fals } ) else: - features = datasets.Features( - { - "input_text": datasets.Value("string"), - } - ) + if evaluate: + features = datasets.Features( + { + "query_id": datasets.Value("string"), + "passage_id": datasets.Value("string"), + "input_text": datasets.Value("string"), + } + ) + else: + features = datasets.Features( + { + "input_text": datasets.Value("string"), + } + ) else: if args.add_prefix: features = datasets.Features( @@ -283,7 +294,9 @@ def __getitem__(self, index): return self.examples[index] -def convert_beir_to_monot5_format(data, run_dict=None, top_k=None, include_title=False, save_path=None): +def convert_beir_to_monot5_format( + data, run_dict=None, top_k=None, include_title=False, save_path=None +): """ Utility function to convert BEIR format to MonoT5 format @@ -306,12 +319,18 @@ def convert_beir_to_monot5_format(data, run_dict=None, top_k=None, include_title run_dict = json.load(f) if top_k: for query_id in run_dict: - run_dict[query_id] = dict(sorted(run_dict[query_id].items(), key=lambda x: x[1], reverse=True)[:top_k]) + run_dict[query_id] = dict( + sorted( + run_dict[query_id].items(), key=lambda x: x[1], reverse=True + )[:top_k] + ) # Make sure both query_id and doc_id are strings updated_dict = {} for query_id in run_dict: - updated_dict[str(query_id)] = {str(k): v for k, v in run_dict[query_id].items()} + updated_dict[str(query_id)] = { + str(k): v for k, v in run_dict[query_id].items() + } run_dict = updated_dict else: @@ -364,7 +383,7 @@ def convert_beir_to_monot5_format(data, run_dict=None, top_k=None, include_title lambda x: f"Query: {x['query']} Document: {x['passage']} Relevant:", axis=1 ) - reranking_df = reranking_df[["query_id", "passage", "input_text"]] + reranking_df = reranking_df[["query_id", "passage_id", "input_text"]] if save_path: os.makedirs(os.path.dirname(save_path), exist_ok=True)