diff --git a/benchmark/local_infinity/knn/hnsw_benchmark_util.h b/benchmark/local_infinity/knn/hnsw_benchmark_util.h index 7c1ec2ebe9..f4fd30e3d3 100644 --- a/benchmark/local_infinity/knn/hnsw_benchmark_util.h +++ b/benchmark/local_infinity/knn/hnsw_benchmark_util.h @@ -17,7 +17,6 @@ #include "CLI11.hpp" import stl; -import file_system; import virtual_store; import local_file_handle; import infinity_exception; diff --git a/src/common/third_party.cppm b/src/common/third_party.cppm index d43fccfdef..073a3fce37 100644 --- a/src/common/third_party.cppm +++ b/src/common/third_party.cppm @@ -68,7 +68,7 @@ module; #include #include #include - +#include #pragma clang diagnostic pop export module third_party; diff --git a/src/storage/io/buffer_base.cppm b/src/storage/io/buffer_base.cppm deleted file mode 100644 index 7e9047715d..0000000000 --- a/src/storage/io/buffer_base.cppm +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright(C) 2023 InfiniFlow, Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -module; - -export module buffer_base; - -import stl; - -namespace infinity { - -// Base class for ReadBuffer and WriteBuffer. -export class BufferBase { -public: - using Position = char *; - - struct Buffer { - Buffer(Position begin_pos, Position end_pos) : begin_pos_(begin_pos), end_pos_(end_pos) {} - - Position Begin() const { return begin_pos_; } - Position End() const { return end_pos_; } - SizeT Size() const { return SizeT(end_pos_ - begin_pos_); } - void Resize(SizeT size) { end_pos_ = begin_pos_ + size; } - bool Empty() const { return Size() == 0; } - - private: - Position begin_pos_; - Position end_pos_; - }; - - BufferBase(Position ptr, SizeT size, SizeT offset) : pos_(ptr + offset), working_buffer_(ptr, ptr + size), internal_buffer_(ptr, ptr + size) {} - - void Set(Position ptr, SizeT size, SizeT offset) { - internal_buffer_ = Buffer(ptr, ptr + size); - working_buffer_ = Buffer(ptr, ptr + size); - pos_ = ptr + offset; - } - - Buffer &InternalBuffer() { return internal_buffer_; } - - Buffer &GetBuffer() { return working_buffer_; } - - Position &Pos() { return pos_; } - - SizeT Offset() const { return SizeT(pos_ - working_buffer_.Begin()); } - - /// How many bytes are available for read/write - SizeT Available() const { return SizeT(working_buffer_.End() - pos_); } - - /// How many bytes have been read/written - SizeT Count() const { return bytes_ + Offset(); } - - bool HasPendingData() const { return Available() > 0; } - -protected: - void ResetWorkingBuffer() { - working_buffer_.Resize(0); - pos_ = working_buffer_.End(); - } - - /// Read/write position. - Position pos_; - - /// How many bytes have been read/written, not counting those that are now in the buffer. - SizeT bytes_ = 0; - - /// A piece of memory that you can use. - Buffer working_buffer_; - - /// A reference to a piece of memory you are reading. - Buffer internal_buffer_; -}; - -} // namespace infinity diff --git a/src/storage/io/file_system.cpp b/src/storage/io/file_system.cpp deleted file mode 100644 index 562e428011..0000000000 --- a/src/storage/io/file_system.cpp +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright(C) 2023 InfiniFlow, Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -module; - -import stl; - -module file_system; - -namespace infinity { - -i64 FileHandler::Read(void *data, u64 nbytes) { return file_system_.Read(*this, data, nbytes); } - -i64 FileHandler::Write(const void *data, u64 nbytes) { return file_system_.Write(*this, data, nbytes); } - -void FileHandler::Rename(const String &old_name, const String &new_name) { return file_system_.Rename(old_name, new_name); } - -void FileHandler::Sync() { return file_system_.SyncFile(*this); } - -void FileHandler::Close() { return file_system_.Close(*this); } - -} // namespace infinity diff --git a/src/storage/io/file_system.cppm b/src/storage/io/file_system.cppm deleted file mode 100644 index 973169a344..0000000000 --- a/src/storage/io/file_system.cppm +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright(C) 2023 InfiniFlow, Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -module; - -import stl; -import file_system_type; -import status; - -export module file_system; - -namespace infinity { - -export class FileSystem; - -export class FileHandler { -public: - FileHandler(FileSystem &file_system, String path) : file_system_(file_system), path_(std::move(path)) {} - - FileHandler(const FileHandler &) = delete; - - virtual ~FileHandler() = default; - - i64 Read(void *data, u64 nbytes); - - i64 Write(const void *data, u64 nbytes); - - void Rename(const String& old_name, const String& new_name); - - void Sync(); - - void Close(); - -public: - FileSystem &file_system_; - Path path_; -}; - -class FileSystem { -public: - explicit FileSystem(FileSystemType file_system_type) : file_system_type_(file_system_type) {} - - virtual ~FileSystem() = default; - - // File related methods - virtual Pair, Status> OpenFile(const String &path, u8 flags, FileLockType lock_type) = 0; - - virtual i64 Read(FileHandler &file_handler, void *data, u64 nbytes) = 0; - - virtual i64 Write(FileHandler &file_handler, const void *data, u64 nbytes) = 0; - - virtual i64 ReadAt(FileHandler &file_handler, i64 file_offset, void *data, u64 nbytes) = 0; - - virtual i64 WriteAt(FileHandler &file_handler, i64 file_offset, const void *data, u64 nbytes) = 0; - - virtual void Rename(const String &old_path, const String &new_path) = 0; - - virtual void Seek(FileHandler &file_handler, i64 pos) = 0; - - virtual SizeT GetFileSize(FileHandler &file_handler) = 0; - - virtual void DeleteFile(const String &file_name) = 0; - - virtual void SyncFile(FileHandler &file_handler) = 0; - - virtual void Close(FileHandler &file_handler) = 0; - - virtual void AppendFile(const String &dst_path, const String &src_path) = 0; - - // Directory related methods - virtual bool Exists(const String &path) = 0; // if file or directory exists - - virtual void CreateDirectory(const String &path) = 0; - - virtual u64 DeleteDirectory(const String &path) = 0; - - virtual void CleanupDirectory(const String &path) = 0; - - virtual Vector> ListDirectory(const String &path) = 0; - - inline FileSystemType file_system_type() const { - return file_system_type_; - } -private: - FileSystemType file_system_type_{FileSystemType::kPosix}; -}; - -} // namespace infinity diff --git a/src/storage/io/file_system_type.cppm b/src/storage/io/file_system_type.cppm deleted file mode 100644 index 71ec99d8a1..0000000000 --- a/src/storage/io/file_system_type.cppm +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright(C) 2023 InfiniFlow, Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -module; - -export module file_system_type; - -import stl; - -namespace infinity { - -export enum class FileSystemType { - kPosix, - kS3, - kHDFS, - kNFS, -}; - -export enum class FileLockType { kNoLock, kReadLock, kWriteLock }; - -export class FileFlags { -public: - static constexpr u8 READ_FLAG = 1 << 0; - static constexpr u8 WRITE_FLAG = 1 << 1; - static constexpr u8 DIRECT_IO = 1 << 2; - static constexpr u8 CREATE_FLAG = 1 << 3; - static constexpr u8 TRUNCATE_CREATE = 1 << 4; - static constexpr u8 APPEND_FLAG = 1 << 5; -}; - -} // namespace infinity diff --git a/src/storage/io/minio_file.cpp b/src/storage/io/minio_file.cpp deleted file mode 100644 index bdb2b102ec..0000000000 --- a/src/storage/io/minio_file.cpp +++ /dev/null @@ -1,312 +0,0 @@ -// Copyright(C) 2024 InfiniFlow, Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -#if 0 -module; - -#include -#include -#include -#include -#include -#include -#include -#include - -import stl; -import status; -import virtual_storage; -import third_party; -import infinity_exception; -import infinity_context; - -module minio_file; - -namespace infinity { - -namespace fs = std::filesystem; - -std::mutex MinioFile::mtx_{}; -HashMap MinioFile::mapped_files_{}; - -MinioFile::MinioFile(VirtualStorage *storage_system) - : ObjectFile(storage_system, StorageType::kMinio) {} - -MinioFile::~MinioFile() = default; - -Status MinioFile::Open(const String &path, FileAccessMode access_mode) { - if(!path_.empty()) { - return Status::FileIsOpen(path); - } - - if (!std::filesystem::path(path).is_absolute()) { - String error_message = fmt::format("{} isn't absolute path.", path); - return Status::SyntaxError(error_message); - } - - if(access_mode != FileAccessMode::kWrite && !std::filesystem::exists(path)){ - String bucket_name = InfinityContext::instance().config()->ObjectStorageBucket(); - minio::s3::Client * client = storage_system_->GetMinioClient(); - // Create download object arguments. - minio::s3::DownloadObjectArgs args; - args.bucket = bucket_name; - args.object = path; - args.filename = path; - - // Call download object. - minio::s3::DownloadObjectResponse resp = client->DownloadObject(args); - - // Handle response. - if (!resp) { - return Status::IOError(resp.Error().String()); - } - } - - switch (access_mode) - { - case FileAccessMode::kRead: - fd_ = open(path.c_str(), O_RDONLY, 0666); - if (fd_ == -1) { - return Status::IOError(fmt::format("Can't open file: {}: {}", path, strerror(errno))); - } - break; - - case FileAccessMode::kMmapRead: - return std::get<2>(MmapRead(path)); - break; - - case FileAccessMode::kWrite: - fd_ = open(path.c_str(), O_WRONLY | O_CREAT | O_APPEND, 0666); - if (fd_ == -1) { - return Status::IOError(fmt::format("Can't open file: {}: {}", path, strerror(errno))); - } - break; - default: - return Status::IOError("Unsupported access mode for s3 storage."); - break; - } - - open_ = true; - path_ = path; - access_mode_ = access_mode; - return Status::OK(); -} - -Status MinioFile::Close() { - if(access_mode_ == FileAccessMode::kWrite) { - if(!sync_) { - Status status = Sync(); - if(!status.ok()) { - return status; - } - } - } - - if(open_) { - if (close(fd_) != 0) { - String error_message = fmt::format("Can't close file: {}: {}", path_, strerror(errno)); - return Status::IOError(error_message); - } - fd_ = -1; - open_ = false; - } - return Status::OK(); -} - -Status MinioFile::Append(const void *buffer, u64 nbytes) { - if(!open_ or access_mode_ != FileAccessMode::kWrite) { - String error_message = fmt::format("File: {} isn't open.", path_); - UnrecoverableError(error_message); - } - i64 written = 0; - while (written < (i64)nbytes) { - i64 write_count = write(fd_, (char *)buffer + written, nbytes - written); - if (write_count == -1) { - String error_message = fmt::format("Can't write file: {}: {}. fd: {}", path_, strerror(errno), fd_); - return Status::IOError(error_message); - } - written += write_count; - } - - return Status::OK(); -} - -Status MinioFile::Append(const String &buffer, u64 nbytes) { - if(!open_ or access_mode_ != FileAccessMode::kWrite) { - String error_message = fmt::format("File: {} isn't open.", path_); - UnrecoverableError(error_message); - } - i64 written = 0; - while (written < (i64)nbytes) { - i64 write_count = write(fd_, (char *)buffer.c_str() + written, nbytes - written); - if (write_count == -1) { - String error_message = fmt::format("Can't write file: {}: {}. fd: {}", path_, strerror(errno), fd_); - return Status::IOError(error_message); - } - written += write_count; - } - return Status::OK(); -} - -Tuple MinioFile::Read(void *buffer, u64 nbytes) { - if(!open_) { - String error_message = fmt::format("File: {} isn't open.", path_); - UnrecoverableError(error_message); - } - i64 readen = 0; - while (readen < (i64)nbytes) { - SizeT a = nbytes - readen; - i64 read_count = read(fd_, (char*)buffer + readen, a); - if (read_count == 0) { - break; - } - if (read_count == -1) { - String error_message = fmt::format("Can't read file: {}: {}", path_, strerror(errno)); - return {0, Status::IOError(error_message)}; - } - readen += read_count; - } - - return {readen, Status::OK()}; -} - -Tuple MinioFile::Read(String &buffer, u64 nbytes) { - if(!open_) { - String error_message = fmt::format("File: {} isn't open.", path_); - UnrecoverableError(error_message); - } - i64 readen = 0; - while (readen < (i64)nbytes) { - SizeT a = nbytes - readen; - i64 read_count = read(fd_, buffer.data() + readen, a); - if (read_count == 0) { - break; - } - if (read_count == -1) { - String error_message = fmt::format("Can't read file: {}: {}", path_, strerror(errno)); - return {0, Status::IOError(error_message)}; - } - readen += read_count; - } - return {readen, Status::OK()}; -} - -Status MinioFile::Seek(u64 nbytes) { return Status::OK(); } - -Status MinioFile::Download(const String &url, const String &path) { return Status::OK(); } - -Status MinioFile::Upload(const String &path, const String &url) { return Status::OK(); } - - -SizeT MinioFile::FileSize() { - return std::filesystem::file_size(path_); -} - -Tuple MinioFile::MmapRead(const String &file_path) { - if (!std::filesystem::path(file_path).is_absolute()) { - String error_message = fmt::format("{} isn't absolute path.", file_path); - return {nullptr, 0, Status::SyntaxError(error_message)}; - } - - char *data_ptr = nullptr; - SizeT data_len = 0; - - std::lock_guard lock(mtx_); - auto it = mapped_files_.find(file_path); - if (it != mapped_files_.end()) { - auto &mmap_info = it->second; - data_ptr = mmap_info.data_ptr_; - data_len = mmap_info.data_len_; - mmap_info.rc_++; - return {data_ptr, data_len, Status::OK()}; - } - long len_f = fs::file_size(file_path); - if (len_f == 0) - return {nullptr, 0, Status::IOError("MAP_FILE_SIZE_ZERO")}; - int f = open(file_path.c_str(), O_RDONLY); - void *tmpd = mmap(NULL, len_f, PROT_READ, MAP_SHARED, f, 0); - if (tmpd == MAP_FAILED) - return {nullptr, 0, Status::IOError("MAP_FAILED")}; - close(f); - int rc = madvise(tmpd, - len_f, - MADV_NORMAL -#if defined(linux) || defined(__linux) || defined(__linux__) - | MADV_DONTDUMP -#endif - ); - if (rc < 0) - return {nullptr, 0, Status::IOError("MAP_MADVISE_FAILED")}; - data_ptr = (char *)tmpd; - data_len = len_f; - mapped_files_.emplace(file_path, MinioMmapInfo{data_ptr, data_len, 1}); - - path_ = file_path; - access_mode_ = FileAccessMode::kMmapRead; - return {data_ptr, data_len, Status::OK()}; -} - -Status MinioFile::Unmmap(const String &file_path) { - if(!std::filesystem::path(file_path).is_absolute()) { - String error_message = fmt::format("{} isn't absolute path.", file_path); - return Status::SyntaxError(error_message); - } - std::lock_guard lock(mtx_); - auto it = mapped_files_.find(file_path); - if (it == mapped_files_.end()) { - return Status::IOError("MAP_FILE_NOT_FOUND"); - } - auto &mmap_info = it->second; - mmap_info.rc_--; - if (mmap_info.rc_ == 0) { - munmap(mmap_info.data_ptr_, mmap_info.data_len_); - mapped_files_.erase(it); - } - - return Status::OK(); -} - -Status MinioFile::Sync() { - if(access_mode_ != FileAccessMode::kWrite) { - return Status::InvalidCommand("Non-write access mode, shouldn't call Sync()"); - } - - if (!sync_) { - if (fsync(fd_) != 0) { - String error_message = fmt::format("fsync failed: {}, {}", path_, strerror(errno)); - return Status::IOError(error_message); - } - - String bucket_name = InfinityContext::instance().config()->ObjectStorageBucket(); - minio::s3::Client * client = storage_system_->GetMinioClient(); - // Create upload object arguments. - minio::s3::UploadObjectArgs args; - args.bucket = bucket_name; - args.object = path_; - args.filename = path_; - - // Call upload object. - minio::s3::UploadObjectResponse resp = client->UploadObject(args); - - // Handle response. - if (!resp) { - return Status::IOError(resp.Error().String()); - } - - sync_ = true; - } - return Status::OK(); -} - -} -#endif \ No newline at end of file diff --git a/src/storage/io/minio_file.cppm b/src/storage/io/minio_file.cppm deleted file mode 100644 index eb32863b50..0000000000 --- a/src/storage/io/minio_file.cppm +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright(C) 2024 InfiniFlow, Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - - -module; - -export module minio_file; - -#if 0 -import stl; -import status; -import object_file; -import abstract_file_handle; - -namespace infinity { - -class VirtualStorage; -struct MinioMmapInfo { - char *data_ptr_{}; - SizeT data_len_{}; - SizeT rc_{}; -}; - -export class MinioFile final : public ObjectFile { -public: - MinioFile(VirtualStorage *storage_system); - ~MinioFile() final; - Status Open(const String &path, FileAccessMode access_mode) final; - Status Close() final; - Status Append(const void *buffer, u64 nbytes) final; - Status Append(const String &buffer, u64 nbytes) final; - Tuple Read(void *buffer, u64 nbytes) final; - Tuple Read(String &buffer, u64 nbytes) final; - Status Seek(u64 nbytes) final; - Status Download(const String &url, const String &path) final; - Status Upload(const String &path, const String &url) final; - SizeT FileSize() final; - Tuple MmapRead(const String &name) final; - Status Unmmap(const String &name) final; - Status Sync() final; - -private: - i32 fd_{-1}; - static std::mutex mtx_; - static HashMap mapped_files_; -}; - -} // namespace infinity -#endif \ No newline at end of file diff --git a/src/storage/io/s3_client.cppm b/src/storage/io/s3_client.cppm index 83dc70a427..d91fb64461 100644 --- a/src/storage/io/s3_client.cppm +++ b/src/storage/io/s3_client.cppm @@ -1,11 +1,10 @@ module; -#include - export module s3_client; import stl; import status; +import third_party; namespace infinity { diff --git a/src/storage/io/s3_client_minio.cpp b/src/storage/io/s3_client_minio.cpp index ad981dcfed..51adc19f84 100644 --- a/src/storage/io/s3_client_minio.cpp +++ b/src/storage/io/s3_client_minio.cpp @@ -1,6 +1,6 @@ module; -#include +#include module s3_client_minio; diff --git a/src/storage/io/s3_client_minio.cppm b/src/storage/io/s3_client_minio.cppm index 4ac04488ee..17876cf21e 100644 --- a/src/storage/io/s3_client_minio.cppm +++ b/src/storage/io/s3_client_minio.cppm @@ -1,12 +1,11 @@ module; -#include - export module s3_client_minio; import stl; import s3_client; import status; +import third_party; namespace infinity { diff --git a/src/storage/meta/catalog.cpp b/src/storage/meta/catalog.cpp index 660223b15b..5b68f7084a 100644 --- a/src/storage/meta/catalog.cpp +++ b/src/storage/meta/catalog.cpp @@ -1078,14 +1078,13 @@ bool Catalog::SaveDeltaCatalog(TxnTimeStamp last_ckp_ts, TxnTimeStamp &max_commi UnrecoverableError(error_message); } - std::ofstream outfile = std::ofstream(full_path, std::ios::binary); - if (!outfile.is_open()) { + auto [out_file_handle, write_status] = VirtualStore::Open(full_path, FileAccessMode::kWrite); + if (!write_status.ok()) { String error_message = fmt::format("Failed to open delta catalog file: {}", full_path); UnrecoverableError(error_message); } - outfile.write((reinterpret_cast(buf.data())), act_size); - outfile.close(); + out_file_handle->Append((reinterpret_cast(buf.data())), act_size); // { // log for delta op debug diff --git a/src/unit_test/storage/io/file_handle/minio_file.cpp b/src/unit_test/storage/io/file_handle/minio_file.cpp deleted file mode 100644 index 4e9adb405b..0000000000 --- a/src/unit_test/storage/io/file_handle/minio_file.cpp +++ /dev/null @@ -1,194 +0,0 @@ -// Copyright(C) 2024 InfiniFlow, Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -import base_test; -import stl; -import status; -import infinity_context; -import compilation_config; -import virtual_store; - -import minio_file; - -using namespace infinity; - -Map default_config = { - {"url", "http://127.0.0.1:9000"}, - {"access_key", "minioadmin"}, - {"secret_key", "minioadmin"}, - {"enable_https", "false"}, - {"disk_cache_dir", ""}, - {"disk_cache_limit", "0"}, - {"disk_cache_lru_count", "0"}, -}; - -#if 0 -class MinioFileTest : public BaseTest { -protected: - static std::shared_ptr config_path() { - return std::make_shared(std::string(test_data_path()) + "/config/test_minio_s3_storage.toml"); - } - - void SetUp() override { - CleanupDbDirs(); - } - - void TearDown() override { - CleanupDbDirs(); - } -}; - -TEST_F(MinioFileTest, TestFileOpen) { - using namespace infinity; - infinity::InfinityContext::instance().Init(MinioFileTest::config_path()); - - VirtualStorage VS; - VS.Init(StorageType::kMinio, default_config); - auto [minio_file, status] = VS.BuildFileHandle(); - EXPECT_TRUE(status.ok()); - String tmp_dir(GetFullTmpDir()); - tmp_dir += "/minio_open_test.txt"; - Path p{tmp_dir}; - std::filesystem::remove(p); - auto status1 = minio_file->Open(tmp_dir, FileAccessMode::kWrite); - EXPECT_TRUE(status1.ok()); - auto status2 = minio_file->Append("hello world", 11); - EXPECT_TRUE(status2.ok()); - auto status3 = minio_file->Sync(); - EXPECT_TRUE(status3.ok()); - auto status4 = minio_file->Close(); - EXPECT_TRUE(status4.ok()); - - auto [minio_file2, status5] = VS.BuildFileHandle(); - EXPECT_TRUE(status5.ok()); - auto status6 = minio_file2->Open(tmp_dir, FileAccessMode::kRead); - EXPECT_TRUE(status6.ok()); - String buffer; - auto [read_n, status7] = minio_file2->Read(buffer, 11); - EXPECT_TRUE(status7.ok()); - EXPECT_STREQ(buffer.c_str(), "hello world"); - auto status8 = minio_file2->Close(); - EXPECT_TRUE(status8.ok()); - - //open after delete local file - std::filesystem::remove(p); - auto [minio_file3, status10] = VS.BuildFileHandle(); - EXPECT_TRUE(status10.ok()); - auto status11 = minio_file3->Open(tmp_dir, FileAccessMode::kRead); - EXPECT_TRUE(status11.ok()); - buffer.clear(); - auto [read_n2, status12] = minio_file3->Read(buffer, 11); - EXPECT_TRUE(status12.ok()); - EXPECT_STREQ(buffer.c_str(), "hello world"); - auto status13 = minio_file3->Close(); - EXPECT_TRUE(status13.ok()); - - std::filesystem::remove(p); - - infinity::InfinityContext::instance().UnInit(); -} - -TEST_F(MinioFileTest, TestFileAppendAndRead) { - using namespace infinity; - infinity::InfinityContext::instance().Init(MinioFileTest::config_path()); - - VirtualStorage VS; - VS.Init(StorageType::kMinio, default_config); - auto [minio_file, status] = VS.BuildFileHandle(); - EXPECT_TRUE(status.ok()); - String tmp_dir(GetFullTmpDir()); - tmp_dir += "/minio_append_test.txt"; - Path p{tmp_dir}; - std::filesystem::remove(p); - auto status1 = minio_file->Open(tmp_dir, FileAccessMode::kWrite); - EXPECT_TRUE(status1.ok()); - //char * - auto status2 = minio_file->Append("infinity", 8); - EXPECT_TRUE(status2.ok()); - String str("infinity"); - //string - auto status3 = minio_file->Append(str, 8); - EXPECT_TRUE(status3.ok()); - auto status4 = minio_file->Sync(); - EXPECT_TRUE(status4.ok()); - auto status5 = minio_file->Close(); - EXPECT_TRUE(status5.ok()); - - auto [minio_file2, status6] = VS.BuildFileHandle(); - EXPECT_TRUE(status6.ok()); - auto status7 = minio_file2->Open(tmp_dir, FileAccessMode::kRead); - EXPECT_TRUE(status7.ok()); - String buffer; - auto [read_n, status8] = minio_file2->Read(buffer, 16); - EXPECT_TRUE(status8.ok()); - EXPECT_STREQ(buffer.c_str(), "infinityinfinity"); - auto status9 = minio_file2->Close(); - EXPECT_TRUE(status9.ok()); - - auto [minio_file3, status10] = VS.BuildFileHandle(); - EXPECT_TRUE(status10.ok()); - auto status11 = minio_file3->Open(tmp_dir, FileAccessMode::kRead); - EXPECT_TRUE(status11.ok()); - char *buffer1 = new char[17]; - auto [read_n1, status12] = minio_file3->Read(buffer1, 16); - EXPECT_TRUE(status12.ok()); - buffer1[16] = '\0'; - EXPECT_STREQ(buffer1, "infinityinfinity"); - delete[] buffer1; - auto status13 = minio_file3->Close(); - EXPECT_TRUE(status13.ok()); - - std::filesystem::remove(p); - - infinity::InfinityContext::instance().UnInit(); -} - -TEST_F(MinioFileTest, TestFileSize) { - using namespace infinity; - infinity::InfinityContext::instance().Init(MinioFileTest::config_path()); - - VirtualStorage VS; - VS.Init(StorageType::kMinio, default_config); - auto [minio_file, status] = VS.BuildFileHandle(); - EXPECT_TRUE(status.ok()); - String tmp_dir(GetFullTmpDir()); - tmp_dir += "/minio_append_test.txt"; - Path p{tmp_dir}; - std::filesystem::remove(p); - auto status1 = minio_file->Open(tmp_dir, FileAccessMode::kRead); - EXPECT_TRUE(status1.ok()); - SizeT len = minio_file->FileSize(); - EXPECT_EQ(len, 16); - auto status2 = minio_file->Close(); - EXPECT_TRUE(status2.ok()); - std::filesystem::remove(p); - - auto [minio_file1, status3] = VS.BuildFileHandle(); - EXPECT_TRUE(status3.ok()); - String tmp_dir1(GetFullTmpDir()); - tmp_dir1 += "/minio_open_test.txt"; - Path p1{tmp_dir1}; - std::filesystem::remove(p1); - auto status4 = minio_file1->Open(tmp_dir1, FileAccessMode::kRead); - EXPECT_TRUE(status4.ok()); - len = minio_file1->FileSize(); - EXPECT_EQ(len, 11); - auto status5 = minio_file1->Close(); - EXPECT_TRUE(status5.ok()); - std::filesystem::remove(p1); - - infinity::InfinityContext::instance().UnInit(); -} -#endif \ No newline at end of file diff --git a/src/unit_test/storage/io/file_handle/local_file.cpp b/src/unit_test/storage/io/local_file_handle.cpp similarity index 100% rename from src/unit_test/storage/io/file_handle/local_file.cpp rename to src/unit_test/storage/io/local_file_handle.cpp