From 594f174f9dd9987e077702e462047c31d706f457 Mon Sep 17 00:00:00 2001 From: jjallaire Date: Mon, 2 Dec 2024 09:22:19 -0500 Subject: [PATCH] Logging: Only call CreateBucket on Amazon S3 when the bucket does not already exist. (#924) Co-authored-by: jjallaire-aisi --- CHANGELOG.md | 1 + src/inspect_ai/_util/file.py | 17 ++++++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a539a980..03efd75f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Logging: Only call CreateBucket on Amazon S3 when the bucket does not already exist. - Improve cancellation feedback and prevent multiple cancellations when using fullscreen display. ## v0.3.48 (01 December 2024) diff --git a/src/inspect_ai/_util/file.py b/src/inspect_ai/_util/file.py index 6ebf1b093..de721c9eb 100644 --- a/src/inspect_ai/_util/file.py +++ b/src/inspect_ai/_util/file.py @@ -172,7 +172,22 @@ def rm( self.fs.rm(path, recursive=recursive, maxdepth=maxdepth) def mkdir(self, path: str, exist_ok: bool = False) -> None: - self.fs.makedirs(path, exist_ok=exist_ok) + if self.is_s3(): + # try to avoid calling create_bucket on s3 filesystems (as that requires distinct + # privileges from being able to write to an existing bucket). we do this by + # first calling mkdir w/ create_parents=False and then only if that fails + # with FileNotFound do we attempt to create the bucket by calling mkdirs + try: + self.fs.makedir(path, create_parents=False) + except FileExistsError: + if exist_ok: + pass + else: + raise + except FileNotFoundError: + self.fs.makedirs(path, exist_ok=exist_ok) + else: + self.fs.makedirs(path, exist_ok=exist_ok) def info(self, path: str, **kwargs: dict[str, Any]) -> FileInfo: return self._file_info(self.fs.info(path, **kwargs))