From e8cd39f02f905e8160c3fdb333634485f6fbf91d Mon Sep 17 00:00:00 2001 From: Shane A Date: Fri, 15 Dec 2023 12:50:55 -0800 Subject: [PATCH] Mark folders with config.yaml and wandb/ as runs --- scripts/storage_cleaner.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/storage_cleaner.py b/scripts/storage_cleaner.py index 837c68982..ca2c31f1e 100644 --- a/scripts/storage_cleaner.py +++ b/scripts/storage_cleaner.py @@ -649,10 +649,14 @@ def _is_run(directory: str, run_entries: Optional[List[str]] = None) -> bool: This method is best effort. It may mark run paths as not (false negatives) or mark non-run paths as runs (false positives). We prioritize minimizing false positives. """ + storage = _get_storage_adapter_for_path(directory) if run_entries is None: - storage = _get_storage_adapter_for_path(directory) run_entries = storage.list_entries(directory) + if CONFIG_YAML in run_entries and storage.is_dir(os.path.join(directory, "wandb")): + # A directory with both config.yaml and a wandb subdirectory is most likely a run + return True + return _contains_checkpoint_dir(run_entries)