Add support for TTL in directories (facebookincubator#11498)

Summary:

Support TTL for directories

Differential Revision: D65734798

velox/common/file/FileSystems.cpp

@@ -163,7 +163,11 @@ class LocalFileSystem : public FileSystem {
     return filePaths;
   }
 
-  void mkdir(std::string_view path) override {
+  void mkdir(std::string_view path, const DirectoryOptions& options) override {
+    if (FOLLY_UNLIKELY(options.expirationTimeMs.has_value())) {
+      VELOX_UNSUPPORTED(
+          "Expiration time is not supported for LocalFileSystem.");
+    }
     std::error_code ec;
     std::filesystem::create_directories(path, ec);
     VELOX_CHECK_EQ(

velox/common/file/FileSystems.h

@@ -66,6 +66,12 @@ struct FileOptions {
   bool bufferWrite{true};
 };
 
+/// Dfines operations on a per driectory basis
+struct DirectoryOptions : FileOptions {
+  /// Specifies the expiration time for the directory.
+  std::optional<int64_t> expirationTimeMs{std::nullopt};
+};
+
 /// An abstract FileSystem
 class FileSystem {
  public:
@@ -113,7 +119,9 @@ class FileSystem {
   virtual std::vector<std::string> list(std::string_view path) = 0;
 
   /// Create a directory (recursively). Throws velox exception on failure.
-  virtual void mkdir(std::string_view path) = 0;
+  virtual void mkdir(
+      std::string_view path,
+      const DirectoryOptions& options = {}) = 0;
 
   /// Remove a directory (all the files and sub-directories underneath
   /// recursively). Throws velox exception on failure.

velox/common/file/tests/FaultyFileSystem.cpp

@@ -112,9 +112,11 @@ std::vector<std::string> FaultyFileSystem::list(std::string_view path) {
   return files;
 }
 
-void FaultyFileSystem::mkdir(std::string_view path) {
+void FaultyFileSystem::mkdir(
+    std::string_view path,
+    const DirectoryOptions& options) {
   const auto delegatedDirPath = extractPath(path);
-  getFileSystem(delegatedDirPath, config_)->mkdir(delegatedDirPath);
+  getFileSystem(delegatedDirPath, config_)->mkdir(delegatedDirPath, options);
 }
 
 void FaultyFileSystem::rmdir(std::string_view path) {

velox/common/file/tests/FaultyFileSystem.h

@@ -71,7 +71,7 @@ class FaultyFileSystem : public FileSystem {
 
   std::vector<std::string> list(std::string_view path) override;
 
-  void mkdir(std::string_view path) override;
+  void mkdir(std::string_view path, const DirectoryOptions& options) override;
 
   void rmdir(std::string_view path) override;
 

velox/connectors/hive/storage_adapters/abfs/AbfsFileSystem.h

@@ -66,7 +66,9 @@ class AbfsFileSystem : public FileSystem {
     VELOX_UNSUPPORTED("list for abfs not implemented");
   }
 
-  void mkdir(std::string_view path) override {
+  void mkdir(
+      std::string_view path,
+      const filesystems::DirectoryOptions& options = {}) override {
     VELOX_UNSUPPORTED("mkdir for abfs not implemented");
   }
 

velox/connectors/hive/storage_adapters/gcs/GCSFileSystem.cpp

@@ -433,7 +433,9 @@ void GCSFileSystem::rename(std::string_view, std::string_view, bool) {
   VELOX_UNSUPPORTED("rename for GCS not implemented");
 }
 
-void GCSFileSystem::mkdir(std::string_view path) {
+void GCSFileSystem::mkdir(
+    std::string_view path,
+    const DirectoryOptions& options) {
   VELOX_UNSUPPORTED("mkdir for GCS not implemented");
 }
 

velox/connectors/hive/storage_adapters/gcs/GCSFileSystem.h

@@ -81,7 +81,8 @@ class GCSFileSystem : public FileSystem {
   void rename(std::string_view, std::string_view, bool) override;
 
   /// Unsupported
-  void mkdir(std::string_view path) override;
+  void mkdir(std::string_view path, const DirectoryOptions& options = {})
+      override;
 
   /// Unsupported
   void rmdir(std::string_view path) override;

velox/connectors/hive/storage_adapters/hdfs/HdfsFileSystem.h

@@ -78,7 +78,8 @@ class HdfsFileSystem : public FileSystem {
     VELOX_UNSUPPORTED("list for HDFS not implemented");
   }
 
-  void mkdir(std::string_view path) override {
+  void mkdir(std::string_view path, const DirectoryOptions& options = {})
+      override {
     VELOX_UNSUPPORTED("mkdir for HDFS not implemented");
   }
 

velox/connectors/hive/storage_adapters/s3fs/S3FileSystem.h

@@ -62,7 +62,8 @@ class S3FileSystem : public FileSystem {
     VELOX_UNSUPPORTED("list for S3 not implemented");
   }
 
-  void mkdir(std::string_view path) override {
+  void mkdir(std::string_view path, const DirectoryOptions& options = {})
+      override {
     VELOX_UNSUPPORTED("mkdir for S3 not implemented");
   }