Add support for TTL in directories (#11498)

Summary:
Pull Request resolved: #11498

Support TTL for directories

Reviewed By: xiaoxmeng

Differential Revision: D65734798

fbshipit-source-id: 5082797b1fe82da495e2161514b2bdda118c724c

velox/common/file/FileSystems.cpp

@@ -163,7 +163,8 @@ class LocalFileSystem : public FileSystem {
     return filePaths;
   }
 
-  void mkdir(std::string_view path) override {
+  void mkdir(std::string_view path, const DirectoryOptions& /*options*/)
+      override {
     std::error_code ec;
     std::filesystem::create_directories(path, ec);
     VELOX_CHECK_EQ(

velox/common/file/FileSystems.h

@@ -66,6 +66,13 @@ struct FileOptions {
   bool bufferWrite{true};
 };
 
+/// Defines directory options
+struct DirectoryOptions : FileOptions {
+  /// This is similar to kFileCreateConfig
+  static constexpr folly::StringPiece kMakeDirectoryConfig{
+      "make-directory-config"};
+};
+
 /// An abstract FileSystem
 class FileSystem {
  public:
@@ -113,7 +120,9 @@ class FileSystem {
   virtual std::vector<std::string> list(std::string_view path) = 0;
 
   /// Create a directory (recursively). Throws velox exception on failure.
-  virtual void mkdir(std::string_view path) = 0;
+  virtual void mkdir(
+      std::string_view path,
+      const DirectoryOptions& options = {}) = 0;
 
   /// Remove a directory (all the files and sub-directories underneath
   /// recursively). Throws velox exception on failure.

velox/common/file/tests/FaultyFileSystem.cpp

@@ -112,9 +112,11 @@ std::vector<std::string> FaultyFileSystem::list(std::string_view path) {
   return files;
 }
 
-void FaultyFileSystem::mkdir(std::string_view path) {
+void FaultyFileSystem::mkdir(
+    std::string_view path,
+    const DirectoryOptions& options) {
   const auto delegatedDirPath = extractPath(path);
-  getFileSystem(delegatedDirPath, config_)->mkdir(delegatedDirPath);
+  getFileSystem(delegatedDirPath, config_)->mkdir(delegatedDirPath, options);
 }
 
 void FaultyFileSystem::rmdir(std::string_view path) {

velox/common/file/tests/FaultyFileSystem.h

@@ -71,7 +71,7 @@ class FaultyFileSystem : public FileSystem {
 
   std::vector<std::string> list(std::string_view path) override;
 
-  void mkdir(std::string_view path) override;
+  void mkdir(std::string_view path, const DirectoryOptions& options) override;
 
   void rmdir(std::string_view path) override;
 

velox/connectors/hive/storage_adapters/abfs/AbfsFileSystem.h

@@ -66,7 +66,9 @@ class AbfsFileSystem : public FileSystem {
     VELOX_UNSUPPORTED("list for abfs not implemented");
   }
 
-  void mkdir(std::string_view path) override {
+  void mkdir(
+      std::string_view path,
+      const filesystems::DirectoryOptions& options = {}) override {
     VELOX_UNSUPPORTED("mkdir for abfs not implemented");
   }
 

velox/connectors/hive/storage_adapters/gcs/GcsFileSystem.cpp

@@ -433,7 +433,9 @@ void GcsFileSystem::rename(std::string_view, std::string_view, bool) {
   VELOX_UNSUPPORTED("rename for GCS not implemented");
 }
 
-void GcsFileSystem::mkdir(std::string_view path) {
+void GcsFileSystem::mkdir(
+    std::string_view path,
+    const DirectoryOptions& options) {
   VELOX_UNSUPPORTED("mkdir for GCS not implemented");
 }
 

velox/connectors/hive/storage_adapters/gcs/GcsFileSystem.h

@@ -81,7 +81,8 @@ class GcsFileSystem : public FileSystem {
   void rename(std::string_view, std::string_view, bool) override;
 
   /// Unsupported
-  void mkdir(std::string_view path) override;
+  void mkdir(std::string_view path, const DirectoryOptions& options = {})
+      override;
 
   /// Unsupported
   void rmdir(std::string_view path) override;

velox/connectors/hive/storage_adapters/hdfs/HdfsFileSystem.h

@@ -78,7 +78,8 @@ class HdfsFileSystem : public FileSystem {
     VELOX_UNSUPPORTED("list for HDFS not implemented");
   }
 
-  void mkdir(std::string_view path) override {
+  void mkdir(std::string_view path, const DirectoryOptions& options = {})
+      override {
     VELOX_UNSUPPORTED("mkdir for HDFS not implemented");
   }
 

velox/connectors/hive/storage_adapters/s3fs/S3FileSystem.h

@@ -62,7 +62,8 @@ class S3FileSystem : public FileSystem {
     VELOX_UNSUPPORTED("list for S3 not implemented");
   }
 
-  void mkdir(std::string_view path) override {
+  void mkdir(std::string_view path, const DirectoryOptions& options = {})
+      override {
     VELOX_UNSUPPORTED("mkdir for S3 not implemented");
   }