From 21ae8e344e32303592c4987b361f0c9848356508 Mon Sep 17 00:00:00 2001 From: Yenda Li Date: Mon, 11 Nov 2024 13:21:48 -0800 Subject: [PATCH] Add support for TTL in directories (#11498) Summary: Support TTL for directories Differential Revision: D65734798 --- velox/common/file/FileSystems.cpp | 6 +++++- velox/common/file/FileSystems.h | 10 +++++++++- velox/common/file/tests/FaultyFileSystem.cpp | 6 ++++-- velox/common/file/tests/FaultyFileSystem.h | 2 +- .../hive/storage_adapters/abfs/AbfsFileSystem.h | 4 +++- .../hive/storage_adapters/gcs/GCSFileSystem.cpp | 4 +++- .../hive/storage_adapters/gcs/GCSFileSystem.h | 3 ++- .../hive/storage_adapters/hdfs/HdfsFileSystem.h | 3 ++- .../hive/storage_adapters/s3fs/S3FileSystem.h | 3 ++- 9 files changed, 31 insertions(+), 10 deletions(-) diff --git a/velox/common/file/FileSystems.cpp b/velox/common/file/FileSystems.cpp index 641ac5c958c60..2292704b42b9a 100644 --- a/velox/common/file/FileSystems.cpp +++ b/velox/common/file/FileSystems.cpp @@ -163,7 +163,11 @@ class LocalFileSystem : public FileSystem { return filePaths; } - void mkdir(std::string_view path) override { + void mkdir(std::string_view path, const DirectoryOptions& options) override { + if (FOLLY_UNLIKELY(options.expirationTimeMs.has_value())) { + VELOX_UNSUPPORTED( + "Expiration time is not supported for LocalFileSystem."); + } std::error_code ec; std::filesystem::create_directories(path, ec); VELOX_CHECK_EQ( diff --git a/velox/common/file/FileSystems.h b/velox/common/file/FileSystems.h index 8fccb5bf6d146..b0ccd50b97326 100644 --- a/velox/common/file/FileSystems.h +++ b/velox/common/file/FileSystems.h @@ -66,6 +66,12 @@ struct FileOptions { bool bufferWrite{true}; }; +/// Dfines operations on a per driectory basis +struct DirectoryOptions : FileOptions { + /// Specifies the expiration time for the directory. + std::optional expirationTimeMs{std::nullopt}; +}; + /// An abstract FileSystem class FileSystem { public: @@ -113,7 +119,9 @@ class FileSystem { virtual std::vector list(std::string_view path) = 0; /// Create a directory (recursively). Throws velox exception on failure. - virtual void mkdir(std::string_view path) = 0; + virtual void mkdir( + std::string_view path, + const DirectoryOptions& options = {}) = 0; /// Remove a directory (all the files and sub-directories underneath /// recursively). Throws velox exception on failure. diff --git a/velox/common/file/tests/FaultyFileSystem.cpp b/velox/common/file/tests/FaultyFileSystem.cpp index 39b4f6b09e158..6e346af61731c 100644 --- a/velox/common/file/tests/FaultyFileSystem.cpp +++ b/velox/common/file/tests/FaultyFileSystem.cpp @@ -112,9 +112,11 @@ std::vector FaultyFileSystem::list(std::string_view path) { return files; } -void FaultyFileSystem::mkdir(std::string_view path) { +void FaultyFileSystem::mkdir( + std::string_view path, + const DirectoryOptions& options) { const auto delegatedDirPath = extractPath(path); - getFileSystem(delegatedDirPath, config_)->mkdir(delegatedDirPath); + getFileSystem(delegatedDirPath, config_)->mkdir(delegatedDirPath, options); } void FaultyFileSystem::rmdir(std::string_view path) { diff --git a/velox/common/file/tests/FaultyFileSystem.h b/velox/common/file/tests/FaultyFileSystem.h index 38451c692e2ae..72d76bc385388 100644 --- a/velox/common/file/tests/FaultyFileSystem.h +++ b/velox/common/file/tests/FaultyFileSystem.h @@ -71,7 +71,7 @@ class FaultyFileSystem : public FileSystem { std::vector list(std::string_view path) override; - void mkdir(std::string_view path) override; + void mkdir(std::string_view path, const DirectoryOptions& options) override; void rmdir(std::string_view path) override; diff --git a/velox/connectors/hive/storage_adapters/abfs/AbfsFileSystem.h b/velox/connectors/hive/storage_adapters/abfs/AbfsFileSystem.h index 319a85b7a382c..903aa80226065 100644 --- a/velox/connectors/hive/storage_adapters/abfs/AbfsFileSystem.h +++ b/velox/connectors/hive/storage_adapters/abfs/AbfsFileSystem.h @@ -66,7 +66,9 @@ class AbfsFileSystem : public FileSystem { VELOX_UNSUPPORTED("list for abfs not implemented"); } - void mkdir(std::string_view path) override { + void mkdir( + std::string_view path, + const filesystems::DirectoryOptions& options = {}) override { VELOX_UNSUPPORTED("mkdir for abfs not implemented"); } diff --git a/velox/connectors/hive/storage_adapters/gcs/GCSFileSystem.cpp b/velox/connectors/hive/storage_adapters/gcs/GCSFileSystem.cpp index 07d85f5746b50..305af2ca3ddf4 100644 --- a/velox/connectors/hive/storage_adapters/gcs/GCSFileSystem.cpp +++ b/velox/connectors/hive/storage_adapters/gcs/GCSFileSystem.cpp @@ -433,7 +433,9 @@ void GCSFileSystem::rename(std::string_view, std::string_view, bool) { VELOX_UNSUPPORTED("rename for GCS not implemented"); } -void GCSFileSystem::mkdir(std::string_view path) { +void GCSFileSystem::mkdir( + std::string_view path, + const DirectoryOptions& options) { VELOX_UNSUPPORTED("mkdir for GCS not implemented"); } diff --git a/velox/connectors/hive/storage_adapters/gcs/GCSFileSystem.h b/velox/connectors/hive/storage_adapters/gcs/GCSFileSystem.h index 0d80cacd9df13..c2352110a54e4 100644 --- a/velox/connectors/hive/storage_adapters/gcs/GCSFileSystem.h +++ b/velox/connectors/hive/storage_adapters/gcs/GCSFileSystem.h @@ -81,7 +81,8 @@ class GCSFileSystem : public FileSystem { void rename(std::string_view, std::string_view, bool) override; /// Unsupported - void mkdir(std::string_view path) override; + void mkdir(std::string_view path, const DirectoryOptions& options = {}) + override; /// Unsupported void rmdir(std::string_view path) override; diff --git a/velox/connectors/hive/storage_adapters/hdfs/HdfsFileSystem.h b/velox/connectors/hive/storage_adapters/hdfs/HdfsFileSystem.h index 1ae7d28b94e5f..a3e9a8efe5d10 100644 --- a/velox/connectors/hive/storage_adapters/hdfs/HdfsFileSystem.h +++ b/velox/connectors/hive/storage_adapters/hdfs/HdfsFileSystem.h @@ -78,7 +78,8 @@ class HdfsFileSystem : public FileSystem { VELOX_UNSUPPORTED("list for HDFS not implemented"); } - void mkdir(std::string_view path) override { + void mkdir(std::string_view path, const DirectoryOptions& options = {}) + override { VELOX_UNSUPPORTED("mkdir for HDFS not implemented"); } diff --git a/velox/connectors/hive/storage_adapters/s3fs/S3FileSystem.h b/velox/connectors/hive/storage_adapters/s3fs/S3FileSystem.h index c624fc6ec5e86..c1e73198d48f1 100644 --- a/velox/connectors/hive/storage_adapters/s3fs/S3FileSystem.h +++ b/velox/connectors/hive/storage_adapters/s3fs/S3FileSystem.h @@ -62,7 +62,8 @@ class S3FileSystem : public FileSystem { VELOX_UNSUPPORTED("list for S3 not implemented"); } - void mkdir(std::string_view path) override { + void mkdir(std::string_view path, const DirectoryOptions& options = {}) + override { VELOX_UNSUPPORTED("mkdir for S3 not implemented"); }