From 33c40a92b10e8bc23513aae57f5a3dd1ecacfbc2 Mon Sep 17 00:00:00 2001 From: Albert Villanova del Moral <8515462+albertvillanova@users.noreply.github.com> Date: Mon, 16 Dec 2024 18:06:04 +0100 Subject: [PATCH 1/6] Support custom results path --- src/lighteval/logging/evaluation_tracker.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/lighteval/logging/evaluation_tracker.py b/src/lighteval/logging/evaluation_tracker.py index 01705534..bcd29981 100644 --- a/src/lighteval/logging/evaluation_tracker.py +++ b/src/lighteval/logging/evaluation_tracker.py @@ -183,7 +183,7 @@ def save(self) -> None: details_datasets[task_name] = dataset # We save results at every case - self.save_results(date_id, results_dict) + self.save_results(date_id=date_id, results_dict=results_dict) if self.should_save_details: self.save_details(date_id, details_datasets) @@ -200,12 +200,19 @@ def save(self) -> None: results=self.metrics_logger.metric_aggregated, details=self.details_logger.compiled_details ) - def save_results(self, date_id: str, results_dict: dict): - output_dir_results = Path(self.output_dir) / "results" / self.general_config_logger.model_name - self.fs.mkdirs(output_dir_results, exist_ok=True) - output_results_file = output_dir_results / f"results_{date_id}.json" - logger.info(f"Saving results to {output_results_file}") - with self.fs.open(output_results_file, "w") as f: + def save_results( + self, date_id: str | None = None, results_dict: dict | None = None, output_path: str | None = None + ): + if output_path: + fs, output_path = url_to_fs(output_path) + else: + output_path = ( + Path(self.output_dir) / "results" / self.general_config_logger.model_name / f"results_{date_id}.json" + ) + fs = self.fs + fs.mkdirs(output_path.parent, exist_ok=True) + logger.info(f"Saving results to {output_path}") + with fs.open(output_path, "w") as f: f.write(json.dumps(results_dict, cls=EnhancedJSONEncoder, indent=2, ensure_ascii=False)) def save_details(self, date_id: str, details_datasets: dict[str, Dataset]): From cec5eabf36ec35243befa4ca036e9d39a5c848b6 Mon Sep 17 00:00:00 2001 From: Albert Villanova del Moral <8515462+albertvillanova@users.noreply.github.com> Date: Tue, 17 Dec 2024 12:51:04 +0100 Subject: [PATCH 2/6] Make custom output_path relative to output_dir --- src/lighteval/logging/evaluation_tracker.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/lighteval/logging/evaluation_tracker.py b/src/lighteval/logging/evaluation_tracker.py index bcd29981..ffa901de 100644 --- a/src/lighteval/logging/evaluation_tracker.py +++ b/src/lighteval/logging/evaluation_tracker.py @@ -204,15 +204,14 @@ def save_results( self, date_id: str | None = None, results_dict: dict | None = None, output_path: str | None = None ): if output_path: - fs, output_path = url_to_fs(output_path) + output_path = Path(self.output_dir) / output_path else: output_path = ( Path(self.output_dir) / "results" / self.general_config_logger.model_name / f"results_{date_id}.json" ) - fs = self.fs - fs.mkdirs(output_path.parent, exist_ok=True) + self.fs.mkdirs(output_path.parent, exist_ok=True) logger.info(f"Saving results to {output_path}") - with fs.open(output_path, "w") as f: + with self.fs.open(output_path, "w") as f: f.write(json.dumps(results_dict, cls=EnhancedJSONEncoder, indent=2, ensure_ascii=False)) def save_details(self, date_id: str, details_datasets: dict[str, Dataset]): From 531eef7d950ca7fa2ab52a345e6338ce6dc4e5aa Mon Sep 17 00:00:00 2001 From: Albert Villanova del Moral <8515462+albertvillanova@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:07:15 +0100 Subject: [PATCH 3/6] Support custom details path --- src/lighteval/logging/evaluation_tracker.py | 23 ++++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/lighteval/logging/evaluation_tracker.py b/src/lighteval/logging/evaluation_tracker.py index ffa901de..c30adb43 100644 --- a/src/lighteval/logging/evaluation_tracker.py +++ b/src/lighteval/logging/evaluation_tracker.py @@ -186,7 +186,7 @@ def save(self) -> None: self.save_results(date_id=date_id, results_dict=results_dict) if self.should_save_details: - self.save_details(date_id, details_datasets) + self.save_details(date_id=date_id, details_datasets=details_datasets) if self.should_push_to_hub: self.push_to_hub( @@ -214,14 +214,21 @@ def save_results( with self.fs.open(output_path, "w") as f: f.write(json.dumps(results_dict, cls=EnhancedJSONEncoder, indent=2, ensure_ascii=False)) - def save_details(self, date_id: str, details_datasets: dict[str, Dataset]): - output_dir_details = Path(self.output_dir) / "details" / self.general_config_logger.model_name - output_dir_details_sub_folder = output_dir_details / date_id - self.fs.mkdirs(output_dir_details_sub_folder, exist_ok=True) - logger.info(f"Saving details to {output_dir_details_sub_folder}") + def save_details(self, date_id: str, details_datasets: dict[str, Dataset], output_path: str | None = None): + if output_path: + output_path = Path(self.output_dir) / output_path + else: + output_path = ( + Path(self.output_dir) + / "details" + / self.general_config_logger.model_name + / date_id + / f"details_{{task_name}}_{date_id}.parquet" + ) + self.fs.mkdirs(output_path.parent, exist_ok=True) + logger.info(f"Saving details to {output_path}") for task_name, dataset in details_datasets.items(): - output_file_details = output_dir_details_sub_folder / f"details_{task_name}_{date_id}.parquet" - with self.fs.open(str(output_file_details), "wb") as f: + with self.fs.open(output_path.as_posix().format(task_name=task_name), "wb") as f: dataset.to_parquet(f) def generate_final_dict(self) -> dict: From 7360b374e536f9cce9cce7d8479416f1eee107ab Mon Sep 17 00:00:00 2001 From: Albert Villanova del Moral <8515462+albertvillanova@users.noreply.github.com> Date: Wed, 18 Dec 2024 08:04:45 +0100 Subject: [PATCH 4/6] Make params optional --- src/lighteval/logging/evaluation_tracker.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/lighteval/logging/evaluation_tracker.py b/src/lighteval/logging/evaluation_tracker.py index c30adb43..717e23f2 100644 --- a/src/lighteval/logging/evaluation_tracker.py +++ b/src/lighteval/logging/evaluation_tracker.py @@ -214,7 +214,9 @@ def save_results( with self.fs.open(output_path, "w") as f: f.write(json.dumps(results_dict, cls=EnhancedJSONEncoder, indent=2, ensure_ascii=False)) - def save_details(self, date_id: str, details_datasets: dict[str, Dataset], output_path: str | None = None): + def save_details( + self, date_id: str | None, details_datasets: dict[str, Dataset] | None = None, output_path: str | None = None + ): if output_path: output_path = Path(self.output_dir) / output_path else: From c35fefa2520118bca608765e70c9932409b90420 Mon Sep 17 00:00:00 2001 From: Albert Villanova del Moral <8515462+albertvillanova@users.noreply.github.com> Date: Wed, 18 Dec 2024 08:53:11 +0100 Subject: [PATCH 5/6] Refactor --- src/lighteval/logging/evaluation_tracker.py | 23 +++++++-------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/src/lighteval/logging/evaluation_tracker.py b/src/lighteval/logging/evaluation_tracker.py index 717e23f2..7d3b0533 100644 --- a/src/lighteval/logging/evaluation_tracker.py +++ b/src/lighteval/logging/evaluation_tracker.py @@ -203,12 +203,8 @@ def save(self) -> None: def save_results( self, date_id: str | None = None, results_dict: dict | None = None, output_path: str | None = None ): - if output_path: - output_path = Path(self.output_dir) / output_path - else: - output_path = ( - Path(self.output_dir) / "results" / self.general_config_logger.model_name / f"results_{date_id}.json" - ) + output_path = output_path or f"results/{self.general_config_logger.model_name}/results_{date_id}.json" + output_path = Path(self.output_dir) / output_path self.fs.mkdirs(output_path.parent, exist_ok=True) logger.info(f"Saving results to {output_path}") with self.fs.open(output_path, "w") as f: @@ -217,16 +213,11 @@ def save_results( def save_details( self, date_id: str | None, details_datasets: dict[str, Dataset] | None = None, output_path: str | None = None ): - if output_path: - output_path = Path(self.output_dir) / output_path - else: - output_path = ( - Path(self.output_dir) - / "details" - / self.general_config_logger.model_name - / date_id - / f"details_{{task_name}}_{date_id}.parquet" - ) + output_path = ( + output_path + or f"details/{self.general_config_logger.model_name}/{date_id}/details_{{task_name}}_{date_id}.parquet" + ) + output_path = Path(self.output_dir) / output_path self.fs.mkdirs(output_path.parent, exist_ok=True) logger.info(f"Saving details to {output_path}") for task_name, dataset in details_datasets.items(): From 2aaafa334e396973beac2571d6a71229f7da0a10 Mon Sep 17 00:00:00 2001 From: Albert Villanova del Moral <8515462+albertvillanova@users.noreply.github.com> Date: Wed, 18 Dec 2024 09:09:02 +0100 Subject: [PATCH 6/6] Make output_dir optional --- src/lighteval/logging/evaluation_tracker.py | 23 ++++++++++++--------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/lighteval/logging/evaluation_tracker.py b/src/lighteval/logging/evaluation_tracker.py index 7d3b0533..445d55f8 100644 --- a/src/lighteval/logging/evaluation_tracker.py +++ b/src/lighteval/logging/evaluation_tracker.py @@ -96,7 +96,7 @@ class EvaluationTracker: def __init__( self, - output_dir: str, + output_dir: str | None = None, save_details: bool = True, push_to_hub: bool = False, push_to_tensorboard: bool = False, @@ -109,7 +109,7 @@ def __init__( Creates all the necessary loggers for evaluation tracking. Args: - output_dir (str): Local folder path where you want results to be saved + output_dir (str, *optional*): Directory where to save results and/or details. save_details (bool): If True, details are saved to the output_dir push_to_hub (bool): If True, details are pushed to the hub. Results are pushed to `{hub_results_org}/details__{sanitized model_name}` for the model `model_name`, a public dataset, @@ -129,7 +129,7 @@ def __init__( self.task_config_logger = TaskConfigLogger() self.api = HfApi() - self.fs, self.output_dir = url_to_fs(output_dir) + self.output_dir = output_dir self.hub_results_org = hub_results_org # will also contain tensorboard results if hub_results_org in ["", None] and any([push_to_hub, push_to_tensorboard]): @@ -204,10 +204,11 @@ def save_results( self, date_id: str | None = None, results_dict: dict | None = None, output_path: str | None = None ): output_path = output_path or f"results/{self.general_config_logger.model_name}/results_{date_id}.json" - output_path = Path(self.output_dir) / output_path - self.fs.mkdirs(output_path.parent, exist_ok=True) + output_path = Path(self.output_dir) / output_path if self.output_dir else output_path + fs, output_path = url_to_fs(output_path) + fs.mkdirs(output_path.parent, exist_ok=True) logger.info(f"Saving results to {output_path}") - with self.fs.open(output_path, "w") as f: + with fs.open(output_path, "w") as f: f.write(json.dumps(results_dict, cls=EnhancedJSONEncoder, indent=2, ensure_ascii=False)) def save_details( @@ -217,11 +218,12 @@ def save_details( output_path or f"details/{self.general_config_logger.model_name}/{date_id}/details_{{task_name}}_{date_id}.parquet" ) - output_path = Path(self.output_dir) / output_path - self.fs.mkdirs(output_path.parent, exist_ok=True) + output_path = Path(self.output_dir) / output_path if self.output_dir else output_path + fs, output_path = url_to_fs(output_path) + fs.mkdirs(output_path.parent, exist_ok=True) logger.info(f"Saving details to {output_path}") for task_name, dataset in details_datasets.items(): - with self.fs.open(output_path.as_posix().format(task_name=task_name), "wb") as f: + with fs.open(output_path.as_posix().format(task_name=task_name), "wb") as f: dataset.to_parquet(f) def generate_final_dict(self) -> dict: @@ -514,7 +516,8 @@ def push_to_tensorboard( # noqa: C901 global_step = 0 run = prefix - output_dir_tb = Path(self.output_dir) / "tb" / run + output_dir_tb = f"tb/{run}" + output_dir_tb = Path(self.output_dir) / output_dir_tb if self.output_dir else output_dir_tb output_dir_tb.mkdir(parents=True, exist_ok=True) tb_context = HFSummaryWriter(