diff --git a/velox/common/file/FileSystems.cpp b/velox/common/file/FileSystems.cpp index 641ac5c958c6..4f215a7ce5f1 100644 --- a/velox/common/file/FileSystems.cpp +++ b/velox/common/file/FileSystems.cpp @@ -163,7 +163,8 @@ class LocalFileSystem : public FileSystem { return filePaths; } - void mkdir(std::string_view path) override { + void mkdir(std::string_view path, const DirectoryOptions& /*options*/) + override { std::error_code ec; std::filesystem::create_directories(path, ec); VELOX_CHECK_EQ( diff --git a/velox/common/file/FileSystems.h b/velox/common/file/FileSystems.h index 8fccb5bf6d14..189b0b6bf593 100644 --- a/velox/common/file/FileSystems.h +++ b/velox/common/file/FileSystems.h @@ -66,6 +66,13 @@ struct FileOptions { bool bufferWrite{true}; }; +/// Defines directory options +struct DirectoryOptions : FileOptions { + /// This is similar to kFileCreateConfig + static constexpr folly::StringPiece kMakeDirectoryConfig{ + "make-directory-config"}; +}; + /// An abstract FileSystem class FileSystem { public: @@ -113,7 +120,9 @@ class FileSystem { virtual std::vector list(std::string_view path) = 0; /// Create a directory (recursively). Throws velox exception on failure. - virtual void mkdir(std::string_view path) = 0; + virtual void mkdir( + std::string_view path, + const DirectoryOptions& options = {}) = 0; /// Remove a directory (all the files and sub-directories underneath /// recursively). Throws velox exception on failure. diff --git a/velox/common/file/tests/FaultyFileSystem.cpp b/velox/common/file/tests/FaultyFileSystem.cpp index 39b4f6b09e15..6e346af61731 100644 --- a/velox/common/file/tests/FaultyFileSystem.cpp +++ b/velox/common/file/tests/FaultyFileSystem.cpp @@ -112,9 +112,11 @@ std::vector FaultyFileSystem::list(std::string_view path) { return files; } -void FaultyFileSystem::mkdir(std::string_view path) { +void FaultyFileSystem::mkdir( + std::string_view path, + const DirectoryOptions& options) { const auto delegatedDirPath = extractPath(path); - getFileSystem(delegatedDirPath, config_)->mkdir(delegatedDirPath); + getFileSystem(delegatedDirPath, config_)->mkdir(delegatedDirPath, options); } void FaultyFileSystem::rmdir(std::string_view path) { diff --git a/velox/common/file/tests/FaultyFileSystem.h b/velox/common/file/tests/FaultyFileSystem.h index 38451c692e2a..72d76bc38538 100644 --- a/velox/common/file/tests/FaultyFileSystem.h +++ b/velox/common/file/tests/FaultyFileSystem.h @@ -71,7 +71,7 @@ class FaultyFileSystem : public FileSystem { std::vector list(std::string_view path) override; - void mkdir(std::string_view path) override; + void mkdir(std::string_view path, const DirectoryOptions& options) override; void rmdir(std::string_view path) override; diff --git a/velox/connectors/hive/storage_adapters/abfs/AbfsFileSystem.h b/velox/connectors/hive/storage_adapters/abfs/AbfsFileSystem.h index 319a85b7a382..903aa8022606 100644 --- a/velox/connectors/hive/storage_adapters/abfs/AbfsFileSystem.h +++ b/velox/connectors/hive/storage_adapters/abfs/AbfsFileSystem.h @@ -66,7 +66,9 @@ class AbfsFileSystem : public FileSystem { VELOX_UNSUPPORTED("list for abfs not implemented"); } - void mkdir(std::string_view path) override { + void mkdir( + std::string_view path, + const filesystems::DirectoryOptions& options = {}) override { VELOX_UNSUPPORTED("mkdir for abfs not implemented"); } diff --git a/velox/connectors/hive/storage_adapters/gcs/GcsFileSystem.cpp b/velox/connectors/hive/storage_adapters/gcs/GcsFileSystem.cpp index 6bc7aaf0cb82..f5d0830d3de3 100644 --- a/velox/connectors/hive/storage_adapters/gcs/GcsFileSystem.cpp +++ b/velox/connectors/hive/storage_adapters/gcs/GcsFileSystem.cpp @@ -433,7 +433,9 @@ void GcsFileSystem::rename(std::string_view, std::string_view, bool) { VELOX_UNSUPPORTED("rename for GCS not implemented"); } -void GcsFileSystem::mkdir(std::string_view path) { +void GcsFileSystem::mkdir( + std::string_view path, + const DirectoryOptions& options) { VELOX_UNSUPPORTED("mkdir for GCS not implemented"); } diff --git a/velox/connectors/hive/storage_adapters/gcs/GcsFileSystem.h b/velox/connectors/hive/storage_adapters/gcs/GcsFileSystem.h index 99e4f94ab99e..34daff8d6c64 100644 --- a/velox/connectors/hive/storage_adapters/gcs/GcsFileSystem.h +++ b/velox/connectors/hive/storage_adapters/gcs/GcsFileSystem.h @@ -81,7 +81,8 @@ class GcsFileSystem : public FileSystem { void rename(std::string_view, std::string_view, bool) override; /// Unsupported - void mkdir(std::string_view path) override; + void mkdir(std::string_view path, const DirectoryOptions& options = {}) + override; /// Unsupported void rmdir(std::string_view path) override; diff --git a/velox/connectors/hive/storage_adapters/hdfs/HdfsFileSystem.h b/velox/connectors/hive/storage_adapters/hdfs/HdfsFileSystem.h index 1ae7d28b94e5..a3e9a8efe5d1 100644 --- a/velox/connectors/hive/storage_adapters/hdfs/HdfsFileSystem.h +++ b/velox/connectors/hive/storage_adapters/hdfs/HdfsFileSystem.h @@ -78,7 +78,8 @@ class HdfsFileSystem : public FileSystem { VELOX_UNSUPPORTED("list for HDFS not implemented"); } - void mkdir(std::string_view path) override { + void mkdir(std::string_view path, const DirectoryOptions& options = {}) + override { VELOX_UNSUPPORTED("mkdir for HDFS not implemented"); } diff --git a/velox/connectors/hive/storage_adapters/s3fs/S3FileSystem.h b/velox/connectors/hive/storage_adapters/s3fs/S3FileSystem.h index c624fc6ec5e8..c1e73198d48f 100644 --- a/velox/connectors/hive/storage_adapters/s3fs/S3FileSystem.h +++ b/velox/connectors/hive/storage_adapters/s3fs/S3FileSystem.h @@ -62,7 +62,8 @@ class S3FileSystem : public FileSystem { VELOX_UNSUPPORTED("list for S3 not implemented"); } - void mkdir(std::string_view path) override { + void mkdir(std::string_view path, const DirectoryOptions& options = {}) + override { VELOX_UNSUPPORTED("mkdir for S3 not implemented"); }