Add support for TTL in directories (facebookincubator#11498)

Summary: Support TTL for directories Differential Revision: D65734798
yuandagits · Nov 11, 2024 · 8d0c98c · 8d0c98c
1 parent 57040fd
commit 8d0c98c
Show file tree

Hide file tree

Showing 9 changed files with 29 additions and 10 deletions.
diff --git a/velox/common/file/FileSystems.cpp b/velox/common/file/FileSystems.cpp
@@ -163,7 +163,11 @@ class LocalFileSystem : public FileSystem {
     return filePaths;
   }
 
-  void mkdir(std::string_view path) override {
+  void mkdir(std::string_view path, const DirectoryOptions& options) override {
+    if (FOLLY_UNLIKELY(options.expirationTimeMs.has_value())) {
+      VELOX_UNSUPPORTED(
+          "Expiration time is not supported for LocalFileSystem.");
+    }
     std::error_code ec;
     std::filesystem::create_directories(path, ec);
     VELOX_CHECK_EQ(

diff --git a/velox/common/file/FileSystems.h b/velox/common/file/FileSystems.h
@@ -66,6 +66,12 @@ struct FileOptions {
   bool bufferWrite{true};
 };
 
+/// Dfines operations on a per driectory basis
+struct DirectoryOptions : FileOptions {
+  /// Specifies the expiration time for the directory.
+  std::optional<int64_t> expirationTimeMs{std::nullopt};
+};
+
 /// An abstract FileSystem
 class FileSystem {
  public:
@@ -113,7 +119,9 @@ class FileSystem {
   virtual std::vector<std::string> list(std::string_view path) = 0;
 
   /// Create a directory (recursively). Throws velox exception on failure.
-  virtual void mkdir(std::string_view path) = 0;
+  virtual void mkdir(
+      std::string_view path,
+      const DirectoryOptions& options = {}) = 0;
 
   /// Remove a directory (all the files and sub-directories underneath
   /// recursively). Throws velox exception on failure.

diff --git a/velox/common/file/tests/FaultyFileSystem.cpp b/velox/common/file/tests/FaultyFileSystem.cpp
@@ -112,9 +112,11 @@ std::vector<std::string> FaultyFileSystem::list(std::string_view path) {
   return files;
 }
 
-void FaultyFileSystem::mkdir(std::string_view path) {
+void FaultyFileSystem::mkdir(
+    std::string_view path,
+    const DirectoryOptions& options) {
   const auto delegatedDirPath = extractPath(path);
-  getFileSystem(delegatedDirPath, config_)->mkdir(delegatedDirPath);
+  getFileSystem(delegatedDirPath, config_)->mkdir(delegatedDirPath, options);
 }
 
 void FaultyFileSystem::rmdir(std::string_view path) {

diff --git a/velox/common/file/tests/FaultyFileSystem.h b/velox/common/file/tests/FaultyFileSystem.h
@@ -71,7 +71,7 @@ class FaultyFileSystem : public FileSystem {
 
   std::vector<std::string> list(std::string_view path) override;
 
-  void mkdir(std::string_view path) override;
+  void mkdir(std::string_view path, const DirectoryOptions& options) override;
 
   void rmdir(std::string_view path) override;
 

diff --git a/velox/connectors/hive/storage_adapters/abfs/AbfsFileSystem.h b/velox/connectors/hive/storage_adapters/abfs/AbfsFileSystem.h
@@ -66,7 +66,8 @@ class AbfsFileSystem : public FileSystem {
     VELOX_UNSUPPORTED("list for abfs not implemented");
   }
 
-  void mkdir(std::string_view path) override {
+  void mkdir(std::string_view path, const filesystems::DirectoryOptions&)
+      override {
     VELOX_UNSUPPORTED("mkdir for abfs not implemented");
   }
 

diff --git a/velox/connectors/hive/storage_adapters/gcs/GCSFileSystem.cpp b/velox/connectors/hive/storage_adapters/gcs/GCSFileSystem.cpp
@@ -433,7 +433,9 @@ void GCSFileSystem::rename(std::string_view, std::string_view, bool) {
   VELOX_UNSUPPORTED("rename for GCS not implemented");
 }
 
-void GCSFileSystem::mkdir(std::string_view path) {
+void GCSFileSystem::mkdir(
+    std::string_view path,
+    const DirectoryOptions& options) {
   VELOX_UNSUPPORTED("mkdir for GCS not implemented");
 }
 

diff --git a/velox/connectors/hive/storage_adapters/gcs/GCSFileSystem.h b/velox/connectors/hive/storage_adapters/gcs/GCSFileSystem.h
@@ -81,7 +81,7 @@ class GCSFileSystem : public FileSystem {
   void rename(std::string_view, std::string_view, bool) override;
 
   /// Unsupported
-  void mkdir(std::string_view path) override;
+  void mkdir(std::string_view path, const DirectoryOptions& options) override;
 
   /// Unsupported
   void rmdir(std::string_view path) override;

diff --git a/velox/connectors/hive/storage_adapters/hdfs/HdfsFileSystem.h b/velox/connectors/hive/storage_adapters/hdfs/HdfsFileSystem.h
@@ -78,7 +78,8 @@ class HdfsFileSystem : public FileSystem {
     VELOX_UNSUPPORTED("list for HDFS not implemented");
   }
 
-  void mkdir(std::string_view path) override {
+  void mkdir(std::string_view path, const DirectoryOptions& /*options*/)
+      override {
     VELOX_UNSUPPORTED("mkdir for HDFS not implemented");
   }
 

diff --git a/velox/connectors/hive/storage_adapters/s3fs/S3FileSystem.h b/velox/connectors/hive/storage_adapters/s3fs/S3FileSystem.h
@@ -62,7 +62,8 @@ class S3FileSystem : public FileSystem {
     VELOX_UNSUPPORTED("list for S3 not implemented");
   }
 
-  void mkdir(std::string_view path) override {
+  void mkdir(std::string_view path, const DirectoryOptions& /*options*/)
+      override {
     VELOX_UNSUPPORTED("mkdir for S3 not implemented");
   }