From d93fa3a758b2124f03ce7fa613921623d5bd4294 Mon Sep 17 00:00:00 2001 From: Hongzhen Luo Date: Mon, 18 Nov 2024 21:29:38 +0800 Subject: [PATCH 01/17] [EROFS]: test: introduce stress testing framework Signed-off-by: Hongzhen Luo --- src/overlaybd/tar/erofs/test/CMakeLists.txt | 29 +- .../erofs/test/{test.cpp => erofs_simple.cpp} | 4 +- src/overlaybd/tar/erofs/test/erofs_stress.cpp | 82 ++++++ .../tar/erofs/test/erofs_stress_base.cpp | 248 ++++++++++++++++++ .../tar/erofs/test/erofs_stress_base.h | 228 ++++++++++++++++ 5 files changed, 583 insertions(+), 8 deletions(-) rename src/overlaybd/tar/erofs/test/{test.cpp => erofs_simple.cpp} (99%) create mode 100644 src/overlaybd/tar/erofs/test/erofs_stress.cpp create mode 100644 src/overlaybd/tar/erofs/test/erofs_stress_base.cpp create mode 100644 src/overlaybd/tar/erofs/test/erofs_stress_base.h diff --git a/src/overlaybd/tar/erofs/test/CMakeLists.txt b/src/overlaybd/tar/erofs/test/CMakeLists.txt index fd82eeb7..db03d8bd 100644 --- a/src/overlaybd/tar/erofs/test/CMakeLists.txt +++ b/src/overlaybd/tar/erofs/test/CMakeLists.txt @@ -4,17 +4,34 @@ link_directories($ENV{GFLAGS}/lib) include_directories($ENV{GTEST}/googletest/include) link_directories($ENV{GTEST}/lib) -add_executable(erofs_test test.cpp) -target_include_directories(erofs_test PUBLIC ${PHOTON_INCLUDE_DIR}) -target_link_libraries(erofs_test gtest gtest_main pthread photon_static +# erofs simple test +add_executable(erofs_simple_test erofs_simple.cpp) +target_include_directories(erofs_simple_test PUBLIC ${PHOTON_INCLUDE_DIR}) +target_link_libraries(erofs_simple_test gtest gtest_main pthread photon_static tar_lib lsmt_lib gzip_lib gzindex_lib checksum_lib overlaybd_image_lib) -target_include_directories(erofs_test PUBLIC +target_include_directories(erofs_simple_test PUBLIC ${PHOTON_INCLUDE_DIR} ${rapidjson_SOURCE_DIR}/include ) add_test( - NAME erofs_test - COMMAND ${EXECUTABLE_OUTPUT_PATH}/erofs_test + NAME erofs_simple_test + COMMAND ${EXECUTABLE_OUTPUT_PATH}/erofs_simple_test +) + +# erofs stress test +add_executable(erofs_stress_test erofs_stress.cpp erofs_stress_base.cpp) +target_include_directories(erofs_stress_test PUBLIC ${PHOTON_INCLUDE_DIR}) +target_link_libraries(erofs_stress_test gtest gtest_main pthread photon_static + tar_lib lsmt_lib gzip_lib gzindex_lib checksum_lib overlaybd_image_lib) + +target_include_directories(erofs_stress_test PUBLIC + ${PHOTON_INCLUDE_DIR} + ${rapidjson_SOURCE_DIR}/include +) + +add_test( + NAME erofs_stress_test + COMMAND ${EXECUTABLE_OUTPUT_PATH}/erofs_stress_test ) diff --git a/src/overlaybd/tar/erofs/test/test.cpp b/src/overlaybd/tar/erofs/test/erofs_simple.cpp similarity index 99% rename from src/overlaybd/tar/erofs/test/test.cpp rename to src/overlaybd/tar/erofs/test/erofs_simple.cpp index 34b33dbb..506b498a 100644 --- a/src/overlaybd/tar/erofs/test/test.cpp +++ b/src/overlaybd/tar/erofs/test/erofs_simple.cpp @@ -32,10 +32,11 @@ #include "../../../gzip/gz.h" #include "../../../../tools/sha256file.h" #include "../../../../tools/comm_func.h" - +#include "../erofs_fs.h" #define FILE_SIZE (2 * 1024 * 1024) #define IMAGE_SIZE 512UL<<20 + class ErofsTest : public ::testing::Test { public: static int inflate(std::string output_file, unsigned char *data, unsigned int size) { @@ -841,7 +842,6 @@ class ErofsTestCleanIncremental: public ::testing::Test { } delete host_fs; } - int traverse_fs(photon::fs::IFileSystem *fs, photon::fs::IFile *out) { std::vector items; diff --git a/src/overlaybd/tar/erofs/test/erofs_stress.cpp b/src/overlaybd/tar/erofs/test/erofs_stress.cpp new file mode 100644 index 00000000..d22d30df --- /dev/null +++ b/src/overlaybd/tar/erofs/test/erofs_stress.cpp @@ -0,0 +1,82 @@ +/* + Copyright The Overlaybd Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include +#include +#include +#include "erofs_stress_base.h" + +#define EROFS_STRESS_UNIMPLEMENTED_FUNC(ret_type, func, ret) \ +ret_type func override { \ + return ret; \ +} + +/* + * TC001 + * + * Create 20 layers, each layer contains a tree with 2 dirs, + * each dir contains 50 empty files. + * + * A simple test for verifying the integrity of the FS tree. + */ +class StressCase001: public StressBase { +public: + StressCase001(std::string path, int layers): StressBase(path, layers) {} + + /* create empty files in build phase*/ + EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, build_gen_mod(StressNode *node, StressHostFile *file), true) + EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, build_gen_own(StressNode *node, StressHostFile *file), true) + EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, build_gen_xattrs(StressNode *node, StressHostFile *file), true) + EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, build_gen_content(StressNode *node, StressHostFile *file), true) + + /* create empty nodes in verify phase */ + EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, verify_gen_mod(StressNode *node, photon::fs::IFile *erofs_file), true) + EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, verify_gen_own(StressNode *node, photon::fs::IFile *erofs_file), true) + EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, verify_gen_xattrs(StressNode *node, photon::fs::IFile *erofs_file), true) + EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, verify_gen_content(StressNode *node, photon::fs::IFile *erofs_file), true) + + /* + * each layer has two dirs: + * dir one contains 50 files, + * dir two contains 50 files + */ + std::vector layer_dirs(int idx) { + std::vector ret; + + ret.emplace_back(50); + ret.emplace_back(50); + return ret; + } +}; + +TEST(ErofsStressTest, TC001) { + std::srand(static_cast(std::time(0))); + StressCase001 *tc001 = new StressCase001("./erofs_stress_001", 20); + + ASSERT_EQ(tc001->run(), true); + delete tc001; +} + +int main(int argc, char **argv) { + + ::testing::InitGoogleTest(&argc, argv); + photon::init(photon::INIT_EVENT_DEFAULT, photon::INIT_IO_DEFAULT); + set_log_output_level(1); + + auto ret = RUN_ALL_TESTS(); + + return ret; +} diff --git a/src/overlaybd/tar/erofs/test/erofs_stress_base.cpp b/src/overlaybd/tar/erofs/test/erofs_stress_base.cpp new file mode 100644 index 00000000..52460a48 --- /dev/null +++ b/src/overlaybd/tar/erofs/test/erofs_stress_base.cpp @@ -0,0 +1,248 @@ +/* + Copyright The Overlaybd Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "erofs_stress_base.h" +#include +#include +#include +#include +#include "../liberofs.h" +#include "../erofs_fs.h" +#include "../../../../tools/comm_func.h" + +std::string get_randomstr(int max_length, bool range) +{ + const char chs[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; + int len = strlen(chs); + int length = range ? rand() % max_length + 1 : max_length; + std::string res; + + for (int i = 0; i < length; i ++) { + res.push_back(chs[std::rand() % len]); + } + return res; +} + +struct LayerNode { + std::string pwd; + std::vector subdirs; + std::vector files; +}; + +static LayerNode *build_layer_tree(std::vector &dirs) { + std::vector nodes; + + for (int i = 0; i < (int)dirs.size(); i ++) { + nodes.emplace_back(new LayerNode); + for (int j = 0; j < dirs[i]; j ++) + nodes[i]->files.emplace_back(get_randomstr(MAX_FILE_NAME, true)); + } + + while (nodes.size() > 1) { + int idx = rand() % nodes.size(); + LayerNode *cur = nodes[idx]; + + nodes.erase(nodes.begin() + idx); + idx = rand() % nodes.size(); + nodes[idx]->subdirs.emplace_back(cur); + } + + return nodes[0]; +} + +bool StressBase::create_layer(int idx) { + std::vector dirs = layer_dirs(idx); + LayerNode *layer_tree = build_layer_tree(dirs); + std::vector q; + std::string root_dirname = get_randomstr(MAX_DIR_NAME, true); + std::string root_path = prefix + "/" + root_dirname; + std::string clean_cmd = "rm -rf " + root_path; + + if (system(clean_cmd.c_str())) + LOG_ERROR_RETURN(-1, false, "fail to prepare clean dir for `", root_path); + + layer_tree->pwd = root_path; + q.emplace_back(layer_tree); + + // traverse the layer tree + while (q.size()) { + bool res; + LayerNode *cur = q.front(); + q.erase(q.begin()); + + StressNode *node = new StressNode(cur->pwd.substr(prefix.length()), NODE_DIR); + if (host_fs->mkdir(cur->pwd.c_str(), 0755) != 0) + LOG_ERROR_RETURN(-1, false, "fail to mkdir `", cur->pwd); + tree->add_node(node); + + for (int i = 0; i < (int)cur->files.size(); i ++) { + std::string filename = cur->pwd.substr(prefix.length()) + "/" + cur->files[i]; + StressNode *node = new StressNode(filename, NODE_REGULAR); + StressHostFile *file_info = new StressHostFile(prefix + filename, host_fs); + + res = build_gen_mod(node, file_info) && + build_gen_own(node, file_info) && + build_gen_xattrs(node, file_info) && + build_gen_content(node, file_info); + if (!res) + LOG_ERROR_RETURN(-1, false, "fail to generate file contents"); + if (!tree->add_node(node)) + LOG_ERROR_RETURN(-1, false, "failt to add node `", filename); + file_info->file->fsync(); + } + + for (int i = 0; i < (int)cur->subdirs.size(); i ++) { + LayerNode *next = cur->subdirs[i]; + next->pwd = cur->pwd + "/" + get_randomstr(MAX_DIR_NAME, true); + q.emplace_back(next); + } + delete cur; + } + + std::string layer_name = prefix + "/layer" + std::to_string(idx); + std::string cmd = std::string(" sudo tar -cf ") + layer_name + ".tar -C " + prefix + " " + root_dirname; + if (system(cmd.c_str())) + LOG_ERROR_RETURN(-1, false, "fail to prepare tar file, cmd: `", cmd); + return true; +} + +LSMT::IFileRW *StressBase::mkfs() +{ + LSMT::IFileRW *lowers = nullptr; + for (int i = 0; i < num_layers; i ++) { + LOG_INFO("processing layer `", i); + std::string src_path = prefix + "/layer" + std::to_string(i) + ".tar"; + std::string idx_path = prefix + "/layer" + std::to_string(i) + ".idx"; + std::string meta_path = prefix + "/layer" + std::to_string(i) + ".meta"; + + /* prepare for idx and meta files */ + auto src_file = host_fs->open(src_path.c_str(), O_RDONLY, 0666); + auto idx_file = host_fs->open(idx_path.c_str(), O_RDWR | O_CREAT | O_TRUNC, S_IRWXU); + auto meta_file = host_fs->open(meta_path.c_str(), O_RDWR | O_CREAT | O_TRUNC, S_IRWXU); + if (!src_file || !idx_file || !meta_file) + LOG_ERROR_RETURN(-1, nullptr, "fail to prepare tar, idx or meta file for layer `", std::to_string(i)); + + LSMT::WarpFileArgs args(idx_file, meta_file, src_file); + args.virtual_size = IMAGE_SIZE; + LSMT::IFileRW *current_layer = create_warpfile(args, false); + if (!current_layer) + LOG_ERROR_RETURN(-1, nullptr, "fail to prepare wrapfile for layer `", std::to_string(i)); + + LSMT::IFileRW *img_file = nullptr; + if (i > 0) + img_file = LSMT::stack_files(current_layer, lowers, false, false); + else + img_file = current_layer; + + /* create erofs fs image */ + auto tar = new LibErofs(img_file, 4096, false); + if (tar->extract_tar(src_file, true, i == 0)) { + delete img_file; + LOG_ERROR_RETURN(-1, nullptr, "fail to extract tar"); + } + delete lowers; + delete tar; + lowers = img_file; + } + + return lowers; +} + +bool StressBase::verify(photon::fs::IFileSystem *erofs_fs) { + std::vector items; + std::string cur; + struct stat st; + bool first = true; + + items.emplace_back(std::string("/")); + do { + StressNode *node; + cur = items.front(); + items.erase(items.begin()); + + if (erofs_fs->stat(cur.c_str(), &st)) + LOG_ERRNO_RETURN(-1, false, "fail to stat file `", cur); + if (S_ISDIR(st.st_mode)) { + node = new StressNode(cur, NODE_DIR); + auto dir = erofs_fs->opendir(cur.c_str()); + do { + dirent *dent = dir->get(); + if (first) + items.emplace_back(cur + std::string(dent->d_name)); + else + items.emplace_back(cur + "/" + std::string(dent->d_name)); + } while (dir->next()); + dir->closedir(); + delete dir; + } else if (S_ISREG(st.st_mode)) { + photon::fs::IFile *file; + bool ret; + + file = erofs_fs->open(cur.c_str(), O_RDONLY); + node = new StressNode(cur, NODE_REGULAR); + if (!file || ! node) + LOG_ERROR_RETURN(0, false, "fail to open file or node `", cur); + ret = verify_gen_mod(node, file) && + verify_gen_own(node, file) && + verify_gen_xattrs(node, file) && + verify_gen_content(node, file); + if (!ret) + LOG_ERROR_RETURN(0, false, "fail to construct StressNode"); + file->close(); + delete file; + } + + if (!tree->query_delete_node(node)) { + delete node; + LOG_ERROR_RETURN(-1, false, "file ` in erofs_fs but not in the in-mem tree", cur); + } + + if (first) + first = false; + } while (!items.empty()); + + return tree->is_emtry(); +} + +bool StressBase::run() +{ + if (workdir_exists) + LOG_ERROR_RETURN(-1, false, "workdir already exists: `", prefix); + + if (!tree->add_node(new StressNode("/", NODE_DIR))) + LOG_ERROR_RETURN(-1, false, "fail to add root node into in-mem tree"); + + for (int i = 0; i < num_layers; i ++) { + if (!create_layer(i)) + LOG_ERRNO_RETURN(-1, false, "fail to crate layer `", std::to_string(i)); + } + + LSMT::IFileRW *lowers = mkfs(); + if (!lowers) + LOG_ERROR_RETURN(-1, false, "failt to mkfs"); + + auto erofs_fs = create_erofs_fs(lowers, 4096); + if (!erofs_fs) + LOG_ERROR_RETURN(-1, false, "failt to crate erofs fs"); + + bool ret = verify(erofs_fs); + + std::string clear_cmd = std::string("rm -rf ") + prefix; + if (system(clear_cmd.c_str())) + LOG_ERROR_RETURN(-1, false, "fail to clear tmp workdir, cmd: `", clear_cmd); + + return ret; +} diff --git a/src/overlaybd/tar/erofs/test/erofs_stress_base.h b/src/overlaybd/tar/erofs/test/erofs_stress_base.h new file mode 100644 index 00000000..d275b240 --- /dev/null +++ b/src/overlaybd/tar/erofs/test/erofs_stress_base.h @@ -0,0 +1,228 @@ +/* + Copyright The Overlaybd Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include "../../../lsmt/file.h" + +#define IMAGE_SIZE 1UL<<30 +#define SECTOR_SIZE 512ULL +#define MAX_DIR_NAME 100 +#define MAX_FILE_NAME 100 + +/* in-mem node type */ +enum NODE_TYPE { + NODE_REGULAR, + NODE_DIR, + NODE_WHITEOUT, + NODE_TYPE_MAX +}; + +class StressNode { +public: + /* meta info for a in-mem node */ + std::string path; + std::string mod; + std::string own; + std::map xattrs; + std::string content; + enum NODE_TYPE type; + + StressNode(std::string _path, NODE_TYPE _type): path(_path), type(_type) {} + + StressNode(StressNode *ano): + path(ano->path), mod(ano->mod), own(ano->own), xattrs(ano->xattrs), + content(ano->content) { } + + bool equal(StressNode *ano) { + if (!ano) + LOG_ERROR_RETURN(-1,false, "invalid ano: nullptr"); + if (xattrs.size() != ano->xattrs.size()) + LOG_ERROR_RETURN(-1,false, "xattrs size not equal: ` != `", xattrs.size(), ano->xattrs.size()); + for (auto it = xattrs.begin(); it != xattrs.end(); it ++) { + auto p = ano->xattrs.find(it->first); + if (p == ano->xattrs.end()) + LOG_ERROR_RETURN(-1, false, "xattr ` not in ano", it->first); + if (p->second.compare(it->second)) + LOG_ERROR_RETURN(-1, false, "xattr ` not equal: ` not equal to `", it->first, p->second, it->second); + if (p == ano->xattrs.end() || p->second.compare(it->second)) + LOG_ERROR_RETURN(-1, false, "xattr ` not equal", it->first); + } + + if (path.compare(ano->path)) + LOG_ERROR_RETURN(-1, false, " path ` not equal to ` (`)", path, ano->path, path); + if (mod.compare(ano->mod)) + LOG_ERROR_RETURN(-1, false, "mode ` not equal to ` (`)", mod, ano->mod, path); + if (own.compare(ano->own)) + LOG_ERROR_RETURN(-1, false, "uid/gid ` not equal to ` (`)", own, ano->own, path); + if (content.compare(ano->content)) + LOG_ERROR_RETURN(-1, false, "content ` not equal to ` (`)", content, ano->content, path); + if (type != ano->type) + LOG_ERROR_RETURN(-1, false, "type ` not equal to ` (`)", type, ano->type, path); + return true; + } +}; + +/* a file in the host fs */ +class StressHostFile { +public: + std::string path; + photon::fs::IFile *file; + StressHostFile() { + file = nullptr; + } + + StressHostFile(std::string _path, photon::fs::IFileSystem *fs): + path(_path) + { + file = fs->open(_path.c_str(), O_RDWR | O_CREAT | O_TRUNC, S_IRWXU); + if (!file) + LOG_ERROR("fail to open file `", _path); + } + + ~StressHostFile() { + file->close(); + delete file; + } +}; + +/* interface to generate corresponding values for in-mem nodes and host-fs files */ +class StressGenInter { +public: + /* for a single file (node) */ + /* generate content for in-memory inodes and host files (prepare layers phase) */ + virtual bool build_gen_mod(StressNode *node /* out */, StressHostFile *file_info /* out */) = 0; + virtual bool build_gen_own(StressNode *node /* out */, StressHostFile *file_info /* out */) = 0; + virtual bool build_gen_xattrs(StressNode *node /* out */, StressHostFile *file_info /* out */) = 0; + virtual bool build_gen_content(StressNode *node /* out */, StressHostFile *file_info /* out */) = 0; + /* generate in-mem inode according to erofs-fs file */ + virtual bool verify_gen_mod(StressNode *node /* out */, photon::fs::IFile *erofs_file /* in */) = 0; + virtual bool verify_gen_own(StressNode *node /* out */, photon::fs::IFile *erofs_file /* in */) = 0; + virtual bool verify_gen_xattrs(StressNode *node /* out */, photon::fs::IFile *erofs_file /* in */) = 0; + virtual bool verify_gen_content(StressNode *node /* out */, photon::fs::IFile *erofs_file /* in */) = 0; + + /* + * construct the structure of a layer, such as how many dirs, + * how many files are in each directory, etc. + */ + virtual std::vector layer_dirs(int idx) = 0; + + virtual ~StressGenInter() {} +}; + +/* + * the in-mem fs tree, which is used + * to do the final verification work. + */ +class StressFsTree { +private: + std::map tree; +public: + StressFsTree() {} + ~StressFsTree() { + for (auto it = tree.begin(); it != tree.end();) { + StressNode *node = it->second; + it = tree.erase(it); + delete node; + } + } + + // build process + bool add_node(StressNode *node) { + if (!node || !node->path.size() || node->type >= NODE_TYPE_MAX) + LOG_ERRNO_RETURN(-1, false, "invalid node"); + + if (node->type != NODE_WHITEOUT) { + tree[node->path] = node; + } else { + auto it = tree.find(node->path); + if (it == tree.end() || it->second->type == NODE_WHITEOUT) + LOG_ERROR_RETURN(-1, false, "whiteout a invalid object"); + if (it->second->type == NODE_REGULAR) + tree.erase(it); + else { + std::string prefix = it->first; + for (auto p = tree.begin(); p != tree.end();) { + if (prefix.compare(0, prefix.size(), p->first) == 0) + p = tree.erase(p); + else + p ++; + } + } + } + return true; + } + + // verify process + bool query_delete_node(StressNode *node) { + auto it = tree.find(node->path); + if (it == tree.end() || !it->second || !it->second->equal(node)) + return false; + tree.erase(it); + return true; + } + + bool is_emtry() { + return tree.empty(); + } +}; + +class StressBase: public StressGenInter { +public: + StressBase(std::string path, int num): prefix(path), num_layers(num) { + host_fs = photon::fs::new_localfs_adaptor(); + if (!host_fs) + LOG_ERROR("fail to create host_fs"); + if (host_fs->access(path.c_str(), 0) == 0) + workdir_exists = true; + else { + workdir_exists = false; + if (host_fs->mkdir(path.c_str(), 0755)) + LOG_ERROR("fail to create dir `", path); + } + tree = new StressFsTree(); + if (!host_fs || !tree) + LOG_ERROR("fail to init StressBase"); + } + + ~StressBase() { + delete host_fs; + delete tree; + } + + bool run(); +private: + std::string prefix; + int num_layers; + photon::fs::IFileSystem *host_fs; + bool workdir_exists; + + StressFsTree *tree; + bool create_layer(int idx); + LSMT::IFileRW *mkfs(); + bool verify(photon::fs::IFileSystem *erofs_fs); +}; + +/* helper functions */ +std::string get_randomstr(int max_length, bool range); From 6722f54efc5f3b7c718092a79ef27bbff6047016 Mon Sep 17 00:00:00 2001 From: Hongzhen Luo Date: Fri, 29 Nov 2024 15:53:45 +0800 Subject: [PATCH 02/17] [EROFS] test: add verification for file contents This adds test case 002. The previous stress test was only for empty files; this patch generates small files with random content. Signed-off-by: Hongzhen Luo --- src/overlaybd/tar/erofs/test/erofs_stress.cpp | 108 ++++++++++++++++++ .../tar/erofs/test/erofs_stress_base.h | 1 + 2 files changed, 109 insertions(+) diff --git a/src/overlaybd/tar/erofs/test/erofs_stress.cpp b/src/overlaybd/tar/erofs/test/erofs_stress.cpp index d22d30df..151cacf1 100644 --- a/src/overlaybd/tar/erofs/test/erofs_stress.cpp +++ b/src/overlaybd/tar/erofs/test/erofs_stress.cpp @@ -17,6 +17,8 @@ #include #include #include +#include +#include #include "erofs_stress_base.h" #define EROFS_STRESS_UNIMPLEMENTED_FUNC(ret_type, func, ret) \ @@ -24,6 +26,62 @@ ret_type func override { \ return ret; \ } +class StressInterImpl: public StressGenInter { +public: + int max_file_size = SECTOR_SIZE * 128; + int min_file_size = SECTOR_SIZE; + int block_size = 4096; + std::hash hash_fn; + + /* generate file content in build phase */ + bool build_gen_content(StressNode *node, StressHostFile *file) override { + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution<> dis(min_file_size, max_file_size); + int size = dis(gen); + int len; + off64_t offset = 0; + std::string hash_val; + + while (size > 0) { + len = std::min(size, block_size); + std::string block_str = get_randomstr(len, false); + if (file->file->pwrite(block_str.c_str(), len, offset) != len) + LOG_ERROR_RETURN(-1, -1, "fail to write to host file `", file->path); + hash_val = std::to_string(hash_fn(hash_val + block_str)); + size -= len; + offset += len; + } + node->content = hash_val; + return true; + } + + /* generate node content in verify phase */ + bool verify_gen_content(StressNode *node, photon::fs::IFile *erofs_file) override { + std::string hash_val; + char buf[block_size]; + struct stat st; + off_t left, offset = 0; + int len; + + if (erofs_file->fstat(&st)) + LOG_ERROR_RETURN(-1, -1, "fail to stat file"); + left = st.st_size; + while (left > 0) { + len = std::min((off_t)block_size, left); + if (len != erofs_file->pread(buf, len, offset)) + LOG_ERROR_RETURN(-1, -1, "fail to pread file"); + + std::string block_str(buf, len); + hash_val = std::to_string(hash_fn(hash_val + block_str)); + left -= len; + offset += len; + } + node->content = hash_val; + return true; + } +}; + /* * TC001 * @@ -62,6 +120,48 @@ class StressCase001: public StressBase { } }; +/* + * TC002 + * + * Create layers, each layer contains 2 dirs, + * each dir contains 10 files. + * + * Testing the integrity of file contents. + */ +class StressCase002: public StressBase, public StressInterImpl { +public: + StressCase002(std::string path, int layers): StressBase(path, layers) {} + + /* leave mod/own/xattr empty */ + EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, build_gen_mod(StressNode *node, StressHostFile *file), true) + EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, build_gen_own(StressNode *node, StressHostFile *file), true) + EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, build_gen_xattrs(StressNode *node, StressHostFile *file), true) + bool build_gen_content(StressNode *node, StressHostFile *file) override { + return StressInterImpl::build_gen_content(node, file); + } + + /* leave mod/own/xattr empty */ + EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, verify_gen_mod(StressNode *node, photon::fs::IFile *erofs_file), true) + EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, verify_gen_own(StressNode *node, photon::fs::IFile *erofs_file), true) + EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, verify_gen_xattrs(StressNode *node, photon::fs::IFile *erofs_file), true) + bool verify_gen_content(StressNode *node, photon::fs::IFile *erofs_file) override { + return StressInterImpl::verify_gen_content(node, erofs_file); + } + + /* + * each layer has two dirs: + * dir one contains 10 files, + * dir two contains 10 files + */ + std::vector layer_dirs(int idx) { + std::vector ret; + + ret.emplace_back(10); + ret.emplace_back(10); + return ret; + } +}; + TEST(ErofsStressTest, TC001) { std::srand(static_cast(std::time(0))); StressCase001 *tc001 = new StressCase001("./erofs_stress_001", 20); @@ -70,6 +170,14 @@ TEST(ErofsStressTest, TC001) { delete tc001; } +TEST(ErofsStressTest, TC002) { + std::srand(static_cast(std::time(0))); + StressCase002 *tc002 = new StressCase002("./erofs_stress_002", 10); + + ASSERT_EQ(tc002->run(), true); + delete tc002; +} + int main(int argc, char **argv) { ::testing::InitGoogleTest(&argc, argv); diff --git a/src/overlaybd/tar/erofs/test/erofs_stress_base.h b/src/overlaybd/tar/erofs/test/erofs_stress_base.h index d275b240..6588e82d 100644 --- a/src/overlaybd/tar/erofs/test/erofs_stress_base.h +++ b/src/overlaybd/tar/erofs/test/erofs_stress_base.h @@ -25,6 +25,7 @@ #include #include #include "../../../lsmt/file.h" +#include "../erofs_fs.h" #define IMAGE_SIZE 1UL<<30 #define SECTOR_SIZE 512ULL From 42ac3f7ee808df8777bf9ac96725f2574ee98b46 Mon Sep 17 00:00:00 2001 From: Hongzhen Luo Date: Sat, 30 Nov 2024 16:44:12 +0800 Subject: [PATCH 03/17] [EROFS] implement {fget,flist}xattr() for ErofsFile Signed-off-by: Hongzhen Luo --- src/overlaybd/tar/erofs/erofs_fs.cpp | 29 ++++++++++++++++++++++++++++ src/overlaybd/tar/erofs/erofs_fs.h | 6 +++++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/src/overlaybd/tar/erofs/erofs_fs.cpp b/src/overlaybd/tar/erofs/erofs_fs.cpp index 81942fcb..3ca1ee07 100644 --- a/src/overlaybd/tar/erofs/erofs_fs.cpp +++ b/src/overlaybd/tar/erofs/erofs_fs.cpp @@ -285,6 +285,35 @@ ssize_t ErofsFile::pread(void *buf, size_t count, off_t offset) return read; } +ssize_t ErofsFile::fgetxattr(const char *name, void *value, size_t size) +{ + ssize_t value_size = 0; + + value_size = erofs_getxattr(&file_private->inode, name, NULL, 0); + if (value_size < 0) + LOG_ERROR_RETURN(-1, size, "[erofs] fail to get xattr `", name); + if ((size_t)value_size > size) + LOG_ERROR_RETURN(-1, -1, "[erofs] buffer is too small to put xattr value of `", name); + return erofs_getxattr(&file_private->inode, name, (char*)value, value_size); +} + +ssize_t ErofsFile::flistxattr(char *list, size_t size) +{ + ssize_t kllen; + + kllen = erofs_listxattr(&file_private->inode, NULL, 0); + if (kllen < 0) + LOG_ERROR_RETURN(-1, kllen, "[erofs] fail to list xattr"); + if ((size_t)kllen > size) + LOG_ERROR_RETURN(-1, -1, "[erofs buffer size is too small to put xattrs"); + if (erofs_listxattr(&file_private->inode, list, kllen) != kllen) + LOG_ERROR_RETURN(-1, -1, "[erofs] fail to list xattr"); + return kllen; +} + +EROFS_UNIMPLEMENTED_FUNC(int, ErofsFile, fsetxattr(const char *name, const void *value, size_t size, int flags), -EROFS_UNIMPLEMENTED) +EROFS_UNIMPLEMENTED_FUNC(int, ErofsFile, fremovexattr(const char *name), -EROFS_UNIMPLEMENTED) + // ErofsFileSystem EROFS_UNIMPLEMENTED_FUNC(photon::fs::IFile*, ErofsFileSystem, open(const char *pathname, int flags, mode_t mode), NULL) EROFS_UNIMPLEMENTED_FUNC(photon::fs::IFile*, ErofsFileSystem, creat(const char *pathname, mode_t mode), NULL) diff --git a/src/overlaybd/tar/erofs/erofs_fs.h b/src/overlaybd/tar/erofs/erofs_fs.h index e20cd9f8..4013ac8b 100644 --- a/src/overlaybd/tar/erofs/erofs_fs.h +++ b/src/overlaybd/tar/erofs/erofs_fs.h @@ -58,7 +58,7 @@ class ErofsFileSystem: public photon::fs::IFileSystem { friend class ErofsFile; }; -class ErofsFile: public photon::fs::VirtualReadOnlyFile { +class ErofsFile: public photon::fs::VirtualReadOnlyFile, public photon::fs::IFileXAttr { public: ErofsFile(ErofsFileSystem *fs); ~ErofsFile(); @@ -66,6 +66,10 @@ class ErofsFile: public photon::fs::VirtualReadOnlyFile { int fstat(struct stat *buf); int fiemap(struct photon::fs::fiemap *map); ssize_t pread(void *buf, size_t count, off_t offset); + ssize_t fgetxattr(const char *name, void *value, size_t size); + ssize_t flistxattr(char *list, size_t size); + int fsetxattr(const char *name, const void *value, size_t size, int flags); + int fremovexattr(const char *name); private: ErofsFileSystem *fs; struct ErofsFileInt; From 85e1df3532d14188613e2c7b64ef4ac762381a8e Mon Sep 17 00:00:00 2001 From: Hongzhen Luo Date: Mon, 2 Dec 2024 11:52:00 +0800 Subject: [PATCH 04/17] [EROFS] test: add test for xattrs Signed-off-by: Hongzhen Luo --- src/overlaybd/tar/erofs/test/erofs_stress.cpp | 93 +++++++++++++++++++ .../tar/erofs/test/erofs_stress_base.cpp | 4 +- .../tar/erofs/test/erofs_stress_base.h | 12 ++- 3 files changed, 107 insertions(+), 2 deletions(-) diff --git a/src/overlaybd/tar/erofs/test/erofs_stress.cpp b/src/overlaybd/tar/erofs/test/erofs_stress.cpp index 151cacf1..70b5dd12 100644 --- a/src/overlaybd/tar/erofs/test/erofs_stress.cpp +++ b/src/overlaybd/tar/erofs/test/erofs_stress.cpp @@ -28,10 +28,19 @@ ret_type func override { \ class StressInterImpl: public StressGenInter { public: + /* file content */ int max_file_size = SECTOR_SIZE * 128; int min_file_size = SECTOR_SIZE; int block_size = 4096; std::hash hash_fn; + /* xattrs */ + int xattrs_max_size = 100; + int xattrs_min_size = 2; + int xattrs_max_count = 10; + int xattrs_min_count = 1; + std::vector xattrs_prefix = {"user."}; + char xattr_key_buffer[8192]; + char xattr_value_buffer[8192]; /* generate file content in build phase */ bool build_gen_content(StressNode *node, StressHostFile *file) override { @@ -80,6 +89,44 @@ class StressInterImpl: public StressGenInter { node->content = hash_val; return true; } + + /* xattrs in build phase */ + bool build_gen_xattrs(StressNode *node, StressHostFile *file) override { + photon::fs::IFileXAttr *xattr_ops = dynamic_cast(file->file); + if (xattr_ops == nullptr) + LOG_ERROR_RETURN(-1, false, "fs does not suppoert xattrs operations!"); + int xattrs_count = get_randomint(xattrs_min_count, xattrs_max_count + 1); + for (int i = 0; i < xattrs_count; i ++) { + int idx = get_randomint(0, xattrs_prefix.size()); + std::string key = xattrs_prefix[idx] + get_randomstr(get_randomint(xattrs_min_size, xattrs_max_size), false); + std::string value = get_randomstr(get_randomint(xattrs_min_size, xattrs_max_size), false); + if (xattr_ops->fsetxattr(key.c_str(), value.c_str(), value.size(), 0)) + LOG_ERROR_RETURN(-1, -1, "fail to set xattr (key: `, value: `) for file `", key, value, file->path); + node->xattrs[key] = value; + } + return true; + } + + /* xattrs in verify phase */ + bool verify_gen_xattrs(StressNode *node, photon::fs::IFile *erofs_file) override { + photon::fs::IFileXAttr *xattr_ops = dynamic_cast(erofs_file); + char *key; + + if (xattr_ops == nullptr) + LOG_ERROR_RETURN(-1, -1, "ErofsFile doest not support xattr operations!"); + ssize_t kllen = xattr_ops->flistxattr(xattr_key_buffer, sizeof(xattr_key_buffer)); + if (kllen < 0) + LOG_ERROR_RETURN(-1, -1, "fail to list xattrs for erofs file"); + for (key = xattr_key_buffer; key < xattr_key_buffer + kllen; key += strlen(key) + 1) { + ssize_t value_len = xattr_ops->fgetxattr(key, xattr_value_buffer, sizeof(xattr_value_buffer)); + if (value_len < 0) + LOG_ERROR_RETURN(-1, -1, "fail to get value for xattr `", key); + std::string str_key = std::string(key, strlen(key)); + std::string str_value = std::string(xattr_value_buffer, value_len); + node->xattrs[str_key] = str_value; + } + return true; + } }; /* @@ -161,6 +208,44 @@ class StressCase002: public StressBase, public StressInterImpl { return ret; } }; +/* + * TC003 + * + * Create layers, each layer contains 10 dirs, + * each dir contains 10 files. + * + * Testing the xattrs of files. + */ +class StressCase003: public StressBase, public StressInterImpl { +public: + StressCase003(std::string path, int layers): StressBase(path, layers) {} + + /* leave mod/own/content empty */ + EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, build_gen_mod(StressNode *node, StressHostFile *file), true) + EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, build_gen_own(StressNode *node, StressHostFile *file), true) + EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, build_gen_content(StressNode *node, StressHostFile *file), true) + bool build_gen_xattrs(StressNode *node, StressHostFile *file) override { + return StressInterImpl::build_gen_xattrs(node, file); + } + + /* leave mod/own/content empty */ + EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, verify_gen_mod(StressNode *node, photon::fs::IFile *erofs_file), true) + EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, verify_gen_own(StressNode *node, photon::fs::IFile *erofs_file), true) + EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, verify_gen_content(StressNode *node, photon::fs::IFile *erofs_file), true) + bool verify_gen_xattrs(StressNode *node, photon::fs::IFile *erofs_file) override { + return StressInterImpl::verify_gen_xattrs(node, erofs_file); + } + + std::vector layer_dirs(int idx) { + std::vector ret; + + /* 10 dirs, each dir contains 10 files */ + for (int i = 0; i < 10; i ++) + ret.emplace_back(10); + return ret; + } + +}; TEST(ErofsStressTest, TC001) { std::srand(static_cast(std::time(0))); @@ -178,6 +263,14 @@ TEST(ErofsStressTest, TC002) { delete tc002; } +TEST(ErofsStressTest, TC003) { + std::srand(static_cast(std::time(0))); + StressCase003 *tc003 = new StressCase003("./erofs_stress_003", 20); + + ASSERT_EQ(tc003->run(), true); + delete tc003; +} + int main(int argc, char **argv) { ::testing::InitGoogleTest(&argc, argv); diff --git a/src/overlaybd/tar/erofs/test/erofs_stress_base.cpp b/src/overlaybd/tar/erofs/test/erofs_stress_base.cpp index 52460a48..f82eee86 100644 --- a/src/overlaybd/tar/erofs/test/erofs_stress_base.cpp +++ b/src/overlaybd/tar/erofs/test/erofs_stress_base.cpp @@ -23,6 +23,8 @@ #include "../erofs_fs.h" #include "../../../../tools/comm_func.h" +#define get_randomint(a, b) ((rand() % (b - a)) + a) + std::string get_randomstr(int max_length, bool range) { const char chs[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; @@ -113,7 +115,7 @@ bool StressBase::create_layer(int idx) { } std::string layer_name = prefix + "/layer" + std::to_string(idx); - std::string cmd = std::string(" sudo tar -cf ") + layer_name + ".tar -C " + prefix + " " + root_dirname; + std::string cmd = std::string(" sudo tar --xattrs --xattrs-include='*' -cf ") + layer_name + ".tar -C " + prefix + " " + root_dirname; if (system(cmd.c_str())) LOG_ERROR_RETURN(-1, false, "fail to prepare tar file, cmd: `", cmd); return true; diff --git a/src/overlaybd/tar/erofs/test/erofs_stress_base.h b/src/overlaybd/tar/erofs/test/erofs_stress_base.h index 6588e82d..512d5794 100644 --- a/src/overlaybd/tar/erofs/test/erofs_stress_base.h +++ b/src/overlaybd/tar/erofs/test/erofs_stress_base.h @@ -59,8 +59,17 @@ class StressNode { bool equal(StressNode *ano) { if (!ano) LOG_ERROR_RETURN(-1,false, "invalid ano: nullptr"); - if (xattrs.size() != ano->xattrs.size()) + if (xattrs.size() != ano->xattrs.size()) { + LOG_INFO("current: `", path); + for (const auto& pair: xattrs) { + LOG_INFO("key: `, value: `", pair.first, pair.second); + } + LOG_INFO("ano: `", ano->path); + for (const auto& pair: ano->xattrs) { + LOG_INFO("key: `, value: `", pair.first, pair.second); + } LOG_ERROR_RETURN(-1,false, "xattrs size not equal: ` != `", xattrs.size(), ano->xattrs.size()); + } for (auto it = xattrs.begin(); it != xattrs.end(); it ++) { auto p = ano->xattrs.find(it->first); if (p == ano->xattrs.end()) @@ -227,3 +236,4 @@ class StressBase: public StressGenInter { /* helper functions */ std::string get_randomstr(int max_length, bool range); +#define get_randomint(a, b) ((rand() % (b - a)) + a) From de5a4e3231afae71942edb936eac66f6ea4d2970 Mon Sep 17 00:00:00 2001 From: Hongzhen Luo Date: Mon, 2 Dec 2024 16:26:45 +0800 Subject: [PATCH 05/17] [EROFS] test: add test for mode Signed-off-by: Hongzhen Luo --- src/overlaybd/tar/erofs/test/erofs_stress.cpp | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/src/overlaybd/tar/erofs/test/erofs_stress.cpp b/src/overlaybd/tar/erofs/test/erofs_stress.cpp index 70b5dd12..1399c5bd 100644 --- a/src/overlaybd/tar/erofs/test/erofs_stress.cpp +++ b/src/overlaybd/tar/erofs/test/erofs_stress.cpp @@ -19,6 +19,8 @@ #include #include #include +#include +#include #include "erofs_stress_base.h" #define EROFS_STRESS_UNIMPLEMENTED_FUNC(ret_type, func, ret) \ @@ -127,6 +129,33 @@ class StressInterImpl: public StressGenInter { } return true; } + + /* mode in build phase */ + bool build_gen_mod(StressNode *node, StressHostFile *file) override { + std::string str_mode; + mode_t mode; + + for (int i = 0; i < 3; i ++) { + str_mode += std::to_string(get_randomint(0, 7)); + } + mode = std::stoi(str_mode, nullptr, 8); + if (file->file->fchmod(mode)) + LOG_ERROR_RETURN(-1, false, "fail to set mode ` for file `", str_mode, file->path); + node->mod = str_mode; + return true; + } + + /* mode in verify phase */ + bool verify_gen_mod(StressNode *node, photon::fs::IFile *erofs_file) override { + struct stat st; + std::ostringstream oss; + + if (erofs_file->fstat(&st)) + LOG_ERROR_RETURN(-1, false, "fail to stat erofs file"); + oss << std::oct << std::setfill('0') << std::setw(3) << (st.st_mode & 0777); + node->mod = oss.str(); + return true; + } }; /* @@ -247,6 +276,42 @@ class StressCase003: public StressBase, public StressInterImpl { }; +/* + * TC004 + * + * Create layers, each layer contains 10 dirs, + * each dir contains 10 files. + * + * Testing the mode of files. + */ +class StressCase004: public StressBase, public StressInterImpl { +public: + StressCase004(std::string path, int layers): StressBase(path, layers) {} + + EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, build_gen_own(StressNode *node, StressHostFile *file), true) + EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, build_gen_xattrs(StressNode *node, StressHostFile *file), true) + EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, build_gen_content(StressNode *node, StressHostFile *file), true) + bool build_gen_mod(StressNode *node, StressHostFile *file) override { + return StressInterImpl::build_gen_mod(node, file); + } + + EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, verify_gen_own(StressNode *node, photon::fs::IFile *erofs_file), true) + EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, verify_gen_xattrs(StressNode *node, photon::fs::IFile *erofs_file), true) + EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, verify_gen_content(StressNode *node, photon::fs::IFile *erofs_file), true) + bool verify_gen_mod(StressNode *node, photon::fs::IFile *erofs_file) override { + return StressInterImpl::verify_gen_mod(node, erofs_file); + } + + std::vector layer_dirs(int idx) { + std::vector ret; + + /* 10 dirs, each dir contains 10 files */ + for (int i = 0; i < 10; i ++) + ret.emplace_back(10); + return ret; + } +}; + TEST(ErofsStressTest, TC001) { std::srand(static_cast(std::time(0))); StressCase001 *tc001 = new StressCase001("./erofs_stress_001", 20); @@ -270,6 +335,13 @@ TEST(ErofsStressTest, TC003) { ASSERT_EQ(tc003->run(), true); delete tc003; } +TEST(ErofsStressTest, TC004) { + std::srand(static_cast(std::time(0))); + StressCase004 *tc004 = new StressCase004("./erofs_stress_004", 10); + + ASSERT_EQ(tc004->run(), true); + delete tc004; +} int main(int argc, char **argv) { From 701f164c27f8e8b45057d9e212f8c0b05b966efb Mon Sep 17 00:00:00 2001 From: Hongzhen Luo Date: Mon, 2 Dec 2024 17:53:15 +0800 Subject: [PATCH 06/17] [EROFS] test: add test for uid/gid Signed-off-by: Hongzhen Luo --- src/overlaybd/tar/erofs/test/erofs_stress.cpp | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/src/overlaybd/tar/erofs/test/erofs_stress.cpp b/src/overlaybd/tar/erofs/test/erofs_stress.cpp index 1399c5bd..beeb87aa 100644 --- a/src/overlaybd/tar/erofs/test/erofs_stress.cpp +++ b/src/overlaybd/tar/erofs/test/erofs_stress.cpp @@ -43,6 +43,9 @@ class StressInterImpl: public StressGenInter { std::vector xattrs_prefix = {"user."}; char xattr_key_buffer[8192]; char xattr_value_buffer[8192]; + /* own */ + int own_id_min = 0; + int own_id_max = UINT32_MAX / 3; /* generate file content in build phase */ bool build_gen_content(StressNode *node, StressHostFile *file) override { @@ -156,6 +159,27 @@ class StressInterImpl: public StressGenInter { node->mod = oss.str(); return true; } + + /* own in build phase */ + bool build_gen_own(StressNode *node, StressHostFile *file) override { + uid_t uid = get_randomint(own_id_min, own_id_max); + gid_t gid = get_randomint(own_id_min, own_id_max); + + if (file->file->fchown(uid, gid)) + LOG_ERROR_RETURN(-1,false, "failt to chown of file `", file->path); + node->own = std::to_string(uid) + std::to_string(gid); + return true; + } + + /* own in verify phase */ + bool verify_gen_own(StressNode *node, photon::fs::IFile *erofs_file) override { + struct stat st; + + if (erofs_file->fstat(&st)) + LOG_ERROR_RETURN(-1, false, "fail to stat erofs file"); + node->own = std::to_string(st.st_uid) + std::to_string(st.st_gid); + return true; + } }; /* @@ -312,6 +336,42 @@ class StressCase004: public StressBase, public StressInterImpl { } }; +/* + * TC005 + * + * Create layers, each layer contains 10 dirs, + * each dir contains 10 files. + * + * Testing the uid/gid of files. + */ +class StressCase005: public StressBase, public StressInterImpl { +public: + StressCase005(std::string path, int layers): StressBase(path, layers) {} + + EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, build_gen_mod(StressNode *node, StressHostFile *file), true) + EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, build_gen_xattrs(StressNode *node, StressHostFile *file), true) + EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, build_gen_content(StressNode *node, StressHostFile *file), true) + bool build_gen_own(StressNode *node, StressHostFile *file) override { + return StressInterImpl::build_gen_own(node, file); + } + + EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, verify_gen_mod(StressNode *node, photon::fs::IFile *erofs_file), true) + EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, verify_gen_xattrs(StressNode *node, photon::fs::IFile *erofs_file), true) + EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, verify_gen_content(StressNode *node, photon::fs::IFile *erofs_file), true) + bool verify_gen_own(StressNode *node, photon::fs::IFile *erofs_file) override { + return StressInterImpl::verify_gen_own(node, erofs_file); + } + + std::vector layer_dirs(int idx) { + std::vector ret; + + /* 10 dirs, each dir contains 10 files */ + for (int i = 0; i < 10; i ++) + ret.emplace_back(10); + return ret; + } +}; + TEST(ErofsStressTest, TC001) { std::srand(static_cast(std::time(0))); StressCase001 *tc001 = new StressCase001("./erofs_stress_001", 20); @@ -343,6 +403,14 @@ TEST(ErofsStressTest, TC004) { delete tc004; } +TEST(ErofsStressTest, TC005) { + std::srand(static_cast(std::time(0))); + StressCase005 *tc005 = new StressCase005("./erofs_stress_005", 2); + + ASSERT_EQ(tc005->run(), true); + delete tc005; +} + int main(int argc, char **argv) { ::testing::InitGoogleTest(&argc, argv); From 9af39f190c2f268147550214dd350b6b96806e5b Mon Sep 17 00:00:00 2001 From: Hongzhen Luo Date: Tue, 3 Dec 2024 11:25:38 +0800 Subject: [PATCH 07/17] [EROFS] test: fix indentation issues Signed-off-by: Hongzhen Luo --- src/overlaybd/tar/erofs/test/erofs_stress.cpp | 52 +++++++++---------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/src/overlaybd/tar/erofs/test/erofs_stress.cpp b/src/overlaybd/tar/erofs/test/erofs_stress.cpp index beeb87aa..74874634 100644 --- a/src/overlaybd/tar/erofs/test/erofs_stress.cpp +++ b/src/overlaybd/tar/erofs/test/erofs_stress.cpp @@ -58,13 +58,13 @@ class StressInterImpl: public StressGenInter { std::string hash_val; while (size > 0) { - len = std::min(size, block_size); - std::string block_str = get_randomstr(len, false); - if (file->file->pwrite(block_str.c_str(), len, offset) != len) - LOG_ERROR_RETURN(-1, -1, "fail to write to host file `", file->path); - hash_val = std::to_string(hash_fn(hash_val + block_str)); - size -= len; - offset += len; + len = std::min(size, block_size); + std::string block_str = get_randomstr(len, false); + if (file->file->pwrite(block_str.c_str(), len, offset) != len) + LOG_ERROR_RETURN(-1, -1, "fail to write to host file `", file->path); + hash_val = std::to_string(hash_fn(hash_val + block_str)); + size -= len; + offset += len; } node->content = hash_val; return true; @@ -82,14 +82,14 @@ class StressInterImpl: public StressGenInter { LOG_ERROR_RETURN(-1, -1, "fail to stat file"); left = st.st_size; while (left > 0) { - len = std::min((off_t)block_size, left); - if (len != erofs_file->pread(buf, len, offset)) - LOG_ERROR_RETURN(-1, -1, "fail to pread file"); - - std::string block_str(buf, len); - hash_val = std::to_string(hash_fn(hash_val + block_str)); - left -= len; - offset += len; + len = std::min((off_t)block_size, left); + if (len != erofs_file->pread(buf, len, offset)) + LOG_ERROR_RETURN(-1, -1, "fail to pread file"); + + std::string block_str(buf, len); + hash_val = std::to_string(hash_fn(hash_val + block_str)); + left -= len; + offset += len; } node->content = hash_val; return true; @@ -290,12 +290,12 @@ class StressCase003: public StressBase, public StressInterImpl { } std::vector layer_dirs(int idx) { - std::vector ret; + std::vector ret; - /* 10 dirs, each dir contains 10 files */ - for (int i = 0; i < 10; i ++) - ret.emplace_back(10); - return ret; + /* 10 dirs, each dir contains 10 files */ + for (int i = 0; i < 10; i ++) + ret.emplace_back(10); + return ret; } }; @@ -327,12 +327,12 @@ class StressCase004: public StressBase, public StressInterImpl { } std::vector layer_dirs(int idx) { - std::vector ret; + std::vector ret; - /* 10 dirs, each dir contains 10 files */ - for (int i = 0; i < 10; i ++) - ret.emplace_back(10); - return ret; + /* 10 dirs, each dir contains 10 files */ + for (int i = 0; i < 10; i ++) + ret.emplace_back(10); + return ret; } }; @@ -405,7 +405,7 @@ TEST(ErofsStressTest, TC004) { TEST(ErofsStressTest, TC005) { std::srand(static_cast(std::time(0))); - StressCase005 *tc005 = new StressCase005("./erofs_stress_005", 2); + StressCase005 *tc005 = new StressCase005("./erofs_stress_005", 10); ASSERT_EQ(tc005->run(), true); delete tc005; From 4a42002c482a8773772ded477b8633245ccfb625 Mon Sep 17 00:00:00 2001 From: Hongzhen Luo Date: Tue, 3 Dec 2024 17:27:51 +0800 Subject: [PATCH 08/17] [EROFS] test: add a stress test for mode, xattrs, content and uid/gid Signed-off-by: Hongzhen Luo --- src/overlaybd/tar/erofs/test/erofs_stress.cpp | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/src/overlaybd/tar/erofs/test/erofs_stress.cpp b/src/overlaybd/tar/erofs/test/erofs_stress.cpp index 74874634..f54c8bf8 100644 --- a/src/overlaybd/tar/erofs/test/erofs_stress.cpp +++ b/src/overlaybd/tar/erofs/test/erofs_stress.cpp @@ -372,6 +372,53 @@ class StressCase005: public StressBase, public StressInterImpl { } }; +/* + * TC006 + * + * Create layers, each layer contains 10 dirs, + * each dir contains 10 files. + * + * Testing the mode, uid/gid, xattrs, content of files. + */ +class StressCase006: public StressBase, public StressInterImpl { +public: + StressCase006(std::string path, int layers): StressBase(path, layers) {} + + bool build_gen_mod(StressNode *node, StressHostFile *file) override { + return StressInterImpl::build_gen_mod(node, file); + } + bool build_gen_own(StressNode *node, StressHostFile *file) override { + return StressInterImpl::build_gen_own(node, file); + } + bool build_gen_xattrs(StressNode *node, StressHostFile *file) override { + return StressInterImpl::build_gen_xattrs(node, file); + } + bool build_gen_content(StressNode *node, StressHostFile *file) override { + return StressInterImpl::build_gen_content(node, file); + } + + bool verify_gen_mod(StressNode *node, photon::fs::IFile *erofs_file) override { + return StressInterImpl::verify_gen_mod(node, erofs_file); + } + bool verify_gen_own(StressNode *node, photon::fs::IFile *erofs_file) override { + return StressInterImpl::verify_gen_own(node, erofs_file); + } + bool verify_gen_xattrs(StressNode *node, photon::fs::IFile *erofs_file) override { + return StressInterImpl::verify_gen_xattrs(node, erofs_file); + } + bool verify_gen_content(StressNode *node, photon::fs::IFile *erofs_file) override { + return StressInterImpl::verify_gen_content(node, erofs_file); + } + + std::vector layer_dirs(int idx) { + std::vector ret; + + for (int i = 0; i < 10; i ++) + ret.emplace_back(10); + return ret; + } +}; + TEST(ErofsStressTest, TC001) { std::srand(static_cast(std::time(0))); StressCase001 *tc001 = new StressCase001("./erofs_stress_001", 20); @@ -411,6 +458,15 @@ TEST(ErofsStressTest, TC005) { delete tc005; } +TEST(ErofsStressTest, TC006) { + std::srand(static_cast(std::time(0))); + /* 30 layers */ + StressCase006 *tc006 = new StressCase006("./erofs_stress_006", 30); + + ASSERT_EQ(tc006->run(), true); + delete tc006; +} + int main(int argc, char **argv) { ::testing::InitGoogleTest(&argc, argv); From ba3a36463c93fc5a16255c6b7aaa5bf1770aa654 Mon Sep 17 00:00:00 2001 From: Hongzhen Luo Date: Thu, 5 Dec 2024 18:32:22 +0800 Subject: [PATCH 09/17] [EROFS] test: add an interface for generating directory and file names To prepare for: 1. ontrol the overlap between the upper layer and lower layer 2. generate .wh.* Relevant tests will be added later. Signed-off-by: Hongzhen Luo --- src/overlaybd/tar/erofs/test/erofs_stress.cpp | 24 +++++++++++++++++++ .../tar/erofs/test/erofs_stress_base.cpp | 18 ++++++++------ .../tar/erofs/test/erofs_stress_base.h | 9 ++++++- 3 files changed, 43 insertions(+), 8 deletions(-) diff --git a/src/overlaybd/tar/erofs/test/erofs_stress.cpp b/src/overlaybd/tar/erofs/test/erofs_stress.cpp index f54c8bf8..c0d5ebbc 100644 --- a/src/overlaybd/tar/erofs/test/erofs_stress.cpp +++ b/src/overlaybd/tar/erofs/test/erofs_stress.cpp @@ -206,6 +206,10 @@ class StressCase001: public StressBase { EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, verify_gen_xattrs(StressNode *node, photon::fs::IFile *erofs_file), true) EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, verify_gen_content(StressNode *node, photon::fs::IFile *erofs_file), true) + /* simplely generate random dir and file names */ + EROFS_STRESS_UNIMPLEMENTED_FUNC(std::string, generate_name(int idx, int depth, std::string _prefix, NODE_TYPE type), \ + get_randomstr(type ? MAX_FILE_NAME : MAX_DIR_NAME, true)) + /* * each layer has two dirs: * dir one contains 50 files, @@ -248,6 +252,10 @@ class StressCase002: public StressBase, public StressInterImpl { return StressInterImpl::verify_gen_content(node, erofs_file); } + /* simplely generate random dir and file names */ + EROFS_STRESS_UNIMPLEMENTED_FUNC(std::string, generate_name(int idx, int depth, std::string _prefix, NODE_TYPE type), \ + get_randomstr(type ? MAX_FILE_NAME : MAX_DIR_NAME, true)) + /* * each layer has two dirs: * dir one contains 10 files, @@ -289,6 +297,10 @@ class StressCase003: public StressBase, public StressInterImpl { return StressInterImpl::verify_gen_xattrs(node, erofs_file); } + /* simplely generate random dir and file names */ + EROFS_STRESS_UNIMPLEMENTED_FUNC(std::string, generate_name(int idx, int depth, std::string _prefix, NODE_TYPE type), \ + get_randomstr(type ? MAX_FILE_NAME : MAX_DIR_NAME, true)) + std::vector layer_dirs(int idx) { std::vector ret; @@ -326,6 +338,10 @@ class StressCase004: public StressBase, public StressInterImpl { return StressInterImpl::verify_gen_mod(node, erofs_file); } + /* simplely generate random dir and file names */ + EROFS_STRESS_UNIMPLEMENTED_FUNC(std::string, generate_name(int idx, int depth, std::string _prefix, NODE_TYPE type), \ + get_randomstr(type ? MAX_FILE_NAME : MAX_DIR_NAME, true)) + std::vector layer_dirs(int idx) { std::vector ret; @@ -362,6 +378,10 @@ class StressCase005: public StressBase, public StressInterImpl { return StressInterImpl::verify_gen_own(node, erofs_file); } + /* simplely generate random dir and file names */ + EROFS_STRESS_UNIMPLEMENTED_FUNC(std::string, generate_name(int idx, int depth, std::string _prefix, NODE_TYPE type), \ + get_randomstr(type ? MAX_FILE_NAME : MAX_DIR_NAME, true)) + std::vector layer_dirs(int idx) { std::vector ret; @@ -410,6 +430,10 @@ class StressCase006: public StressBase, public StressInterImpl { return StressInterImpl::verify_gen_content(node, erofs_file); } + /* simplely generate random dir and file names */ + EROFS_STRESS_UNIMPLEMENTED_FUNC(std::string, generate_name(int idx, int depth, std::string _prefix, NODE_TYPE type), \ + get_randomstr(type ? MAX_FILE_NAME : MAX_DIR_NAME, true)) + std::vector layer_dirs(int idx) { std::vector ret; diff --git a/src/overlaybd/tar/erofs/test/erofs_stress_base.cpp b/src/overlaybd/tar/erofs/test/erofs_stress_base.cpp index f82eee86..bdb0298c 100644 --- a/src/overlaybd/tar/erofs/test/erofs_stress_base.cpp +++ b/src/overlaybd/tar/erofs/test/erofs_stress_base.cpp @@ -41,7 +41,7 @@ std::string get_randomstr(int max_length, bool range) struct LayerNode { std::string pwd; std::vector subdirs; - std::vector files; + int num_files, depth; }; static LayerNode *build_layer_tree(std::vector &dirs) { @@ -49,8 +49,8 @@ static LayerNode *build_layer_tree(std::vector &dirs) { for (int i = 0; i < (int)dirs.size(); i ++) { nodes.emplace_back(new LayerNode); - for (int j = 0; j < dirs[i]; j ++) - nodes[i]->files.emplace_back(get_randomstr(MAX_FILE_NAME, true)); + nodes[i]->depth = 0; + nodes[i]->num_files = dirs[i]; } while (nodes.size() > 1) { @@ -69,7 +69,7 @@ bool StressBase::create_layer(int idx) { std::vector dirs = layer_dirs(idx); LayerNode *layer_tree = build_layer_tree(dirs); std::vector q; - std::string root_dirname = get_randomstr(MAX_DIR_NAME, true); + std::string root_dirname = generate_name(idx, layer_tree->depth, "", NODE_DIR); std::string root_path = prefix + "/" + root_dirname; std::string clean_cmd = "rm -rf " + root_path; @@ -90,8 +90,10 @@ bool StressBase::create_layer(int idx) { LOG_ERROR_RETURN(-1, false, "fail to mkdir `", cur->pwd); tree->add_node(node); - for (int i = 0; i < (int)cur->files.size(); i ++) { - std::string filename = cur->pwd.substr(prefix.length()) + "/" + cur->files[i]; + for (int i = 0; i < (int)cur->num_files; i ++) { + std::string name_prefix = cur->pwd.substr(prefix.length()); + // generate filename for files in the current dir + std::string filename = name_prefix + "/" + generate_name(idx, cur->depth, name_prefix, NODE_REGULAR); StressNode *node = new StressNode(filename, NODE_REGULAR); StressHostFile *file_info = new StressHostFile(prefix + filename, host_fs); @@ -108,7 +110,9 @@ bool StressBase::create_layer(int idx) { for (int i = 0; i < (int)cur->subdirs.size(); i ++) { LayerNode *next = cur->subdirs[i]; - next->pwd = cur->pwd + "/" + get_randomstr(MAX_DIR_NAME, true); + next->depth = cur->depth + 1; + // generate subdir name in the current dir + next->pwd = cur->pwd + "/" + generate_name(idx, cur->depth, cur->pwd.substr(prefix.length()), NODE_DIR); q.emplace_back(next); } delete cur; diff --git a/src/overlaybd/tar/erofs/test/erofs_stress_base.h b/src/overlaybd/tar/erofs/test/erofs_stress_base.h index 512d5794..fed6cd60 100644 --- a/src/overlaybd/tar/erofs/test/erofs_stress_base.h +++ b/src/overlaybd/tar/erofs/test/erofs_stress_base.h @@ -34,8 +34,8 @@ /* in-mem node type */ enum NODE_TYPE { - NODE_REGULAR, NODE_DIR, + NODE_REGULAR, NODE_WHITEOUT, NODE_TYPE_MAX }; @@ -138,6 +138,13 @@ class StressGenInter { */ virtual std::vector layer_dirs(int idx) = 0; + /* + * generate the name for a dir or a file, to: + * 1. control the overlap between the upper layer and lower layer + * 2. generate .wh.* + */ + virtual std::string generate_name(int idx, int depth, std::string root_path, NODE_TYPE type) = 0; + virtual ~StressGenInter() {} }; From fa5d2fb7f2b1dae23a51b3af292723798e5f8b68 Mon Sep 17 00:00:00 2001 From: Hongzhen Luo Date: Thu, 5 Dec 2024 20:23:07 +0800 Subject: [PATCH 10/17] [EROFS] test: add test for same-name files or dirs Signed-off-by: Hongzhen Luo --- src/overlaybd/tar/erofs/test/erofs_stress.cpp | 138 ++++++++++++++++-- .../tar/erofs/test/erofs_stress_base.cpp | 76 ++++++++-- .../tar/erofs/test/erofs_stress_base.h | 5 +- 3 files changed, 193 insertions(+), 26 deletions(-) diff --git a/src/overlaybd/tar/erofs/test/erofs_stress.cpp b/src/overlaybd/tar/erofs/test/erofs_stress.cpp index c0d5ebbc..34ad137c 100644 --- a/src/overlaybd/tar/erofs/test/erofs_stress.cpp +++ b/src/overlaybd/tar/erofs/test/erofs_stress.cpp @@ -46,6 +46,8 @@ class StressInterImpl: public StressGenInter { /* own */ int own_id_min = 0; int own_id_max = UINT32_MAX / 3; + /* dir or file name */ + std::map> name_map; /* generate file content in build phase */ bool build_gen_content(StressNode *node, StressHostFile *file) override { @@ -180,6 +182,25 @@ class StressInterImpl: public StressGenInter { node->own = std::to_string(st.st_uid) + std::to_string(st.st_gid); return true; } + + /* generate a random dir or file name in the current layer */ + std::string generate_name(int idx, int depth, std::string root_path, NODE_TYPE type) override { + std::string res; + int cnt = 0; + + res = get_randomstr(type ? MAX_FILE_NAME : MAX_DIR_NAME, true); + if (name_map.find(idx) ==name_map.end()) + name_map[idx] = std::set(); + while (name_map[idx].find(res) != name_map[idx].end()) { + res = get_randomstr(type ? MAX_FILE_NAME : MAX_DIR_NAME, true); + cnt ++; + /* try up to 1000 times */ + if (cnt > 1000) + LOG_ERROR_RETURN(-1, "", "fail to generate a random name"); + } + name_map[idx].insert(res); + return res; + } }; /* @@ -190,7 +211,7 @@ class StressInterImpl: public StressGenInter { * * A simple test for verifying the integrity of the FS tree. */ -class StressCase001: public StressBase { +class StressCase001: public StressBase, public StressInterImpl { public: StressCase001(std::string path, int layers): StressBase(path, layers) {} @@ -207,8 +228,9 @@ class StressCase001: public StressBase { EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, verify_gen_content(StressNode *node, photon::fs::IFile *erofs_file), true) /* simplely generate random dir and file names */ - EROFS_STRESS_UNIMPLEMENTED_FUNC(std::string, generate_name(int idx, int depth, std::string _prefix, NODE_TYPE type), \ - get_randomstr(type ? MAX_FILE_NAME : MAX_DIR_NAME, true)) + std::string generate_name(int idx, int depth, std::string root_path, NODE_TYPE type) override { + return StressInterImpl::generate_name(idx, depth, root_path, type); + } /* * each layer has two dirs: @@ -253,8 +275,9 @@ class StressCase002: public StressBase, public StressInterImpl { } /* simplely generate random dir and file names */ - EROFS_STRESS_UNIMPLEMENTED_FUNC(std::string, generate_name(int idx, int depth, std::string _prefix, NODE_TYPE type), \ - get_randomstr(type ? MAX_FILE_NAME : MAX_DIR_NAME, true)) + std::string generate_name(int idx, int depth, std::string root_path, NODE_TYPE type) override { + return StressInterImpl::generate_name(idx, depth, root_path, type); + } /* * each layer has two dirs: @@ -298,8 +321,9 @@ class StressCase003: public StressBase, public StressInterImpl { } /* simplely generate random dir and file names */ - EROFS_STRESS_UNIMPLEMENTED_FUNC(std::string, generate_name(int idx, int depth, std::string _prefix, NODE_TYPE type), \ - get_randomstr(type ? MAX_FILE_NAME : MAX_DIR_NAME, true)) + std::string generate_name(int idx, int depth, std::string root_path, NODE_TYPE type) override { + return StressInterImpl::generate_name(idx, depth, root_path, type); + } std::vector layer_dirs(int idx) { std::vector ret; @@ -339,8 +363,9 @@ class StressCase004: public StressBase, public StressInterImpl { } /* simplely generate random dir and file names */ - EROFS_STRESS_UNIMPLEMENTED_FUNC(std::string, generate_name(int idx, int depth, std::string _prefix, NODE_TYPE type), \ - get_randomstr(type ? MAX_FILE_NAME : MAX_DIR_NAME, true)) + std::string generate_name(int idx, int depth, std::string root_path, NODE_TYPE type) override { + return StressInterImpl::generate_name(idx, depth, root_path, type); + } std::vector layer_dirs(int idx) { std::vector ret; @@ -379,8 +404,9 @@ class StressCase005: public StressBase, public StressInterImpl { } /* simplely generate random dir and file names */ - EROFS_STRESS_UNIMPLEMENTED_FUNC(std::string, generate_name(int idx, int depth, std::string _prefix, NODE_TYPE type), \ - get_randomstr(type ? MAX_FILE_NAME : MAX_DIR_NAME, true)) + std::string generate_name(int idx, int depth, std::string root_path, NODE_TYPE type) override { + return StressInterImpl::generate_name(idx, depth, root_path, type); + } std::vector layer_dirs(int idx) { std::vector ret; @@ -431,8 +457,9 @@ class StressCase006: public StressBase, public StressInterImpl { } /* simplely generate random dir and file names */ - EROFS_STRESS_UNIMPLEMENTED_FUNC(std::string, generate_name(int idx, int depth, std::string _prefix, NODE_TYPE type), \ - get_randomstr(type ? MAX_FILE_NAME : MAX_DIR_NAME, true)) + std::string generate_name(int idx, int depth, std::string root_path, NODE_TYPE type) override { + return StressInterImpl::generate_name(idx, depth, root_path, type); + } std::vector layer_dirs(int idx) { std::vector ret; @@ -443,6 +470,82 @@ class StressCase006: public StressBase, public StressInterImpl { } }; +/* + * TC007 + * + * Create layers, each layer contains 10 dirs, + * each dir contains 10 files. + * + * Test the scenario where the upper layer and lower + * layer contain files or directories with the same name. + */ +class StressCase007: public StressBase, public StressInterImpl { +private: + std::map> mp; +public: + StressCase007(std::string path, int layers): StressBase(path, layers) {} + + bool build_gen_mod(StressNode *node, StressHostFile *file) override { + return StressInterImpl::build_gen_mod(node, file); + } + bool build_gen_own(StressNode *node, StressHostFile *file) override { + return StressInterImpl::build_gen_own(node, file); + } + bool build_gen_xattrs(StressNode *node, StressHostFile *file) override { + return StressInterImpl::build_gen_xattrs(node, file); + } + bool build_gen_content(StressNode *node, StressHostFile *file) override { + return StressInterImpl::build_gen_content(node, file); + } + + bool verify_gen_mod(StressNode *node, photon::fs::IFile *erofs_file) override { + return StressInterImpl::verify_gen_mod(node, erofs_file); + } + bool verify_gen_own(StressNode *node, photon::fs::IFile *erofs_file) override { + return StressInterImpl::verify_gen_own(node, erofs_file); + } + bool verify_gen_xattrs(StressNode *node, photon::fs::IFile *erofs_file) override { + return StressInterImpl::verify_gen_xattrs(node, erofs_file); + } + bool verify_gen_content(StressNode *node, photon::fs::IFile *erofs_file) override { + return StressInterImpl::verify_gen_content(node, erofs_file); + } + + std::string generate_name(int idx, int depth, std::string root_path, NODE_TYPE type) override { + std::string res; + int cnt = 0; + + if (idx < 1) { + res = get_randomstr(type ? MAX_FILE_NAME : MAX_DIR_NAME, true); + goto out; + } + res = tree->get_same_name(idx, depth, root_path, type); + /* fall back to a random name */ + if (res.length() == 0) + res = get_randomstr(type ? MAX_FILE_NAME : MAX_DIR_NAME, true); + if (mp.find(idx) == mp.end()) + mp[idx] = std::set(); + /* already used in this layer, fall back to a random name */ + while (mp[idx].find(res) != mp[idx].end()) { + res = get_randomstr(type ? MAX_FILE_NAME : MAX_DIR_NAME, true); + cnt ++; + if (cnt > 1000) + LOG_ERROR_RETURN(-1, "", "fail to gernate name in TC007"); + } + mp[idx].insert(res); + out: + return res; + } + + std::vector layer_dirs(int idx) { + std::vector ret; + + for (int i = 0; i < 10; i ++) + ret.emplace_back(30); + return ret; + } +}; + TEST(ErofsStressTest, TC001) { std::srand(static_cast(std::time(0))); StressCase001 *tc001 = new StressCase001("./erofs_stress_001", 20); @@ -491,6 +594,15 @@ TEST(ErofsStressTest, TC006) { delete tc006; } +TEST(ErofsStressTest, TC007) { + std::srand(static_cast(std::time(0))); + /* 50 layers */ + StressCase007 *tc007 = new StressCase007("./erofs_stress_007", 50); + + ASSERT_EQ(tc007->run(), true); + delete tc007; +} + int main(int argc, char **argv) { ::testing::InitGoogleTest(&argc, argv); diff --git a/src/overlaybd/tar/erofs/test/erofs_stress_base.cpp b/src/overlaybd/tar/erofs/test/erofs_stress_base.cpp index bdb0298c..aa4e6f31 100644 --- a/src/overlaybd/tar/erofs/test/erofs_stress_base.cpp +++ b/src/overlaybd/tar/erofs/test/erofs_stress_base.cpp @@ -19,6 +19,8 @@ #include #include #include +#include +#include #include "../liberofs.h" #include "../erofs_fs.h" #include "../../../../tools/comm_func.h" @@ -37,6 +39,31 @@ std::string get_randomstr(int max_length, bool range) } return res; } +static bool is_substring(const std::string& str, const std::string& substring) { + return str.find(substring) != std::string::npos; +} + +std::string StressFsTree::get_same_name(int idx, int depth, std::string root_path, NODE_TYPE type) { + std::vector vec; + for (const auto& pair : tree) { + if (pair.first == "/" || pair.second->type != type || + !is_substring(pair.first, root_path) || + pair.first.length() == root_path.length()) + continue; + std::string last_component = pair.first.substr(root_path.length() + 1); + if (!is_substring(last_component, "/")) + vec.emplace_back(last_component); + } + + if (vec.empty()) + return get_randomstr(type ? MAX_FILE_NAME : MAX_DIR_NAME, true); + + std::random_device rd; + std::default_random_engine engine(rd()); + std::shuffle(vec.begin(), vec.end(), engine); + + return vec[0]; +} struct LayerNode { std::string pwd; @@ -66,6 +93,15 @@ static LayerNode *build_layer_tree(std::vector &dirs) { } bool StressBase::create_layer(int idx) { + +#define MAX_TRY_TIME 10 + + std::string origin_prefix = prefix; + // add a random prefix to avoid operations in the same dir + prefix = prefix + "/" + get_randomstr(20, false); + if (host_fs->mkdir(prefix.c_str(), 0755) != 0) + LOG_ERROR_RETURN(-1, false, "fail to prepare for the current workdir `", prefix); + std::vector dirs = layer_dirs(idx); LayerNode *layer_tree = build_layer_tree(dirs); std::vector q; @@ -79,17 +115,17 @@ bool StressBase::create_layer(int idx) { layer_tree->pwd = root_path; q.emplace_back(layer_tree); + StressNode *node = new StressNode(layer_tree->pwd.substr(prefix.length()), NODE_DIR); + if (host_fs->mkdir(layer_tree->pwd.c_str(), 0755) != 0) + LOG_ERROR_RETURN(-1, false, "fail to mkdir `", layer_tree->pwd); + tree->add_node(node); + // traverse the layer tree while (q.size()) { bool res; LayerNode *cur = q.front(); q.erase(q.begin()); - StressNode *node = new StressNode(cur->pwd.substr(prefix.length()), NODE_DIR); - if (host_fs->mkdir(cur->pwd.c_str(), 0755) != 0) - LOG_ERROR_RETURN(-1, false, "fail to mkdir `", cur->pwd); - tree->add_node(node); - for (int i = 0; i < (int)cur->num_files; i ++) { std::string name_prefix = cur->pwd.substr(prefix.length()); // generate filename for files in the current dir @@ -106,22 +142,33 @@ bool StressBase::create_layer(int idx) { if (!tree->add_node(node)) LOG_ERROR_RETURN(-1, false, "failt to add node `", filename); file_info->file->fsync(); + delete file_info; } for (int i = 0; i < (int)cur->subdirs.size(); i ++) { LayerNode *next = cur->subdirs[i]; next->depth = cur->depth + 1; // generate subdir name in the current dir - next->pwd = cur->pwd + "/" + generate_name(idx, cur->depth, cur->pwd.substr(prefix.length()), NODE_DIR); - q.emplace_back(next); + for (int try_times = 0; try_times < MAX_TRY_TIME; try_times++) { + next->pwd = cur->pwd + "/" + generate_name(idx, cur->depth, cur->pwd.substr(prefix.length()), NODE_DIR); + if (host_fs->mkdir(next->pwd.c_str(), 0755) == 0) { + StressNode *dir_node = new StressNode(next->pwd.substr(prefix.length()), NODE_DIR); + tree->add_node(dir_node); + q.emplace_back(next); + break; + } + } } delete cur; } - std::string layer_name = prefix + "/layer" + std::to_string(idx); +#undef MAX_TRY_TIME + + std::string layer_name = origin_prefix + "/layer" + std::to_string(idx); std::string cmd = std::string(" sudo tar --xattrs --xattrs-include='*' -cf ") + layer_name + ".tar -C " + prefix + " " + root_dirname; if (system(cmd.c_str())) LOG_ERROR_RETURN(-1, false, "fail to prepare tar file, cmd: `", cmd); + prefix = origin_prefix; return true; } @@ -220,7 +267,9 @@ bool StressBase::verify(photon::fs::IFileSystem *erofs_fs) { first = false; } while (!items.empty()); - return tree->is_emtry(); + if (!tree->is_emtry()) + LOG_ERROR_RETURN(-1, false, "Mismatch: in-mem tree is not empty!"); + return true; } bool StressBase::run() @@ -246,9 +295,12 @@ bool StressBase::run() bool ret = verify(erofs_fs); - std::string clear_cmd = std::string("rm -rf ") + prefix; - if (system(clear_cmd.c_str())) - LOG_ERROR_RETURN(-1, false, "fail to clear tmp workdir, cmd: `", clear_cmd); + if (ret) { + std::string clear_cmd = std::string("rm -rf ") + prefix; + if (system(clear_cmd.c_str())) + LOG_ERROR_RETURN(-1, false, "fail to clear tmp workdir, cmd: `", clear_cmd); + + } return ret; } diff --git a/src/overlaybd/tar/erofs/test/erofs_stress_base.h b/src/overlaybd/tar/erofs/test/erofs_stress_base.h index fed6cd60..f837268c 100644 --- a/src/overlaybd/tar/erofs/test/erofs_stress_base.h +++ b/src/overlaybd/tar/erofs/test/erofs_stress_base.h @@ -203,10 +203,14 @@ class StressFsTree { bool is_emtry() { return tree.empty(); } + + std::string get_same_name(int idx, int depth, std::string root_path, NODE_TYPE type); }; class StressBase: public StressGenInter { public: + StressFsTree *tree; + StressBase(std::string path, int num): prefix(path), num_layers(num) { host_fs = photon::fs::new_localfs_adaptor(); if (!host_fs) @@ -235,7 +239,6 @@ class StressBase: public StressGenInter { photon::fs::IFileSystem *host_fs; bool workdir_exists; - StressFsTree *tree; bool create_layer(int idx); LSMT::IFileRW *mkfs(); bool verify(photon::fs::IFileSystem *erofs_fs); From 3752124ee8eee3ccb79a33424e968953be9b7e4c Mon Sep 17 00:00:00 2001 From: Hongzhen Luo Date: Sun, 8 Dec 2024 16:00:52 +0800 Subject: [PATCH 11/17] [EROFS] test: enhanced testing for same-name file or dir Add tests for cases where file and directory have the same name. Signed-off-by: Hongzhen Luo --- .../tar/erofs/test/erofs_stress_base.cpp | 50 ++++++++++++++++++- .../tar/erofs/test/erofs_stress_base.h | 33 +++--------- 2 files changed, 55 insertions(+), 28 deletions(-) diff --git a/src/overlaybd/tar/erofs/test/erofs_stress_base.cpp b/src/overlaybd/tar/erofs/test/erofs_stress_base.cpp index aa4e6f31..e8f6d68f 100644 --- a/src/overlaybd/tar/erofs/test/erofs_stress_base.cpp +++ b/src/overlaybd/tar/erofs/test/erofs_stress_base.cpp @@ -43,11 +43,57 @@ static bool is_substring(const std::string& str, const std::string& substring) { return str.find(substring) != std::string::npos; } +static bool str_n_equal(std::string s1, std::string s2, long unsigned int n) { + if (s1.length() < n || s2.length() < n) + return false; + return s1.substr(0, n) == s2.substr(0, n); +} + +bool StressFsTree::add_node(StressNode *node) { + if (!node || !node->path.size() || node->type >= NODE_TYPE_MAX) + LOG_ERRNO_RETURN(-1, false, "invalid node"); + + if (node->type != NODE_WHITEOUT) { + /* the upper regular file should remove the lower dir */ + std::map::iterator dir_it; + if (node->type == NODE_REGULAR && (dir_it = tree.find(node->path)) != tree.end() && + dir_it->second->type == NODE_DIR) + { + tree.erase(dir_it); + std::string rm_prefix = node->path + "/"; + for (auto it = tree.begin(); it != tree.end(); ) { + if (str_n_equal(rm_prefix, it->first, rm_prefix.length())) { + it = tree.erase(it); + } else { + ++it; + } + + } + } + tree[node->path] = node; + } else { + auto it = tree.find(node->path); + if (it == tree.end() || it->second->type == NODE_WHITEOUT) + LOG_ERROR_RETURN(-1, false, "whiteout a invalid object"); + if (it->second->type == NODE_REGULAR) + tree.erase(it); + else { + std::string prefix = it->first; + for (auto p = tree.begin(); p != tree.end();) { + if (prefix.compare(0, prefix.size(), p->first) == 0) + p = tree.erase(p); + else + p ++; + } + } + } + return true; +} + std::string StressFsTree::get_same_name(int idx, int depth, std::string root_path, NODE_TYPE type) { std::vector vec; for (const auto& pair : tree) { - if (pair.first == "/" || pair.second->type != type || - !is_substring(pair.first, root_path) || + if (pair.first == "/" || !is_substring(pair.first, root_path) || pair.first.length() == root_path.length()) continue; std::string last_component = pair.first.substr(root_path.length() + 1); diff --git a/src/overlaybd/tar/erofs/test/erofs_stress_base.h b/src/overlaybd/tar/erofs/test/erofs_stress_base.h index f837268c..3d3db17d 100644 --- a/src/overlaybd/tar/erofs/test/erofs_stress_base.h +++ b/src/overlaybd/tar/erofs/test/erofs_stress_base.h @@ -166,36 +166,17 @@ class StressFsTree { } // build process - bool add_node(StressNode *node) { - if (!node || !node->path.size() || node->type >= NODE_TYPE_MAX) - LOG_ERRNO_RETURN(-1, false, "invalid node"); - - if (node->type != NODE_WHITEOUT) { - tree[node->path] = node; - } else { - auto it = tree.find(node->path); - if (it == tree.end() || it->second->type == NODE_WHITEOUT) - LOG_ERROR_RETURN(-1, false, "whiteout a invalid object"); - if (it->second->type == NODE_REGULAR) - tree.erase(it); - else { - std::string prefix = it->first; - for (auto p = tree.begin(); p != tree.end();) { - if (prefix.compare(0, prefix.size(), p->first) == 0) - p = tree.erase(p); - else - p ++; - } - } - } - return true; - } + bool add_node(StressNode *node); // verify process bool query_delete_node(StressNode *node) { auto it = tree.find(node->path); - if (it == tree.end() || !it->second || !it->second->equal(node)) - return false; + if (it == tree.end()) + LOG_ERROR_RETURN(-1,false, "path ` does not exist in in-mem tree", node->path); + if (!it->second) + LOG_ERROR_RETURN(-1, false, "NULL in-mem info (`)", node->path); + if (!it->second->equal(node)) + LOG_ERROR_RETURN(-1, false, "node contents mismatch"); tree.erase(it); return true; } From fd085110428692bb60fd882b21bfd5d1b38c879a Mon Sep 17 00:00:00 2001 From: Hongzhen Luo Date: Mon, 9 Dec 2024 12:36:39 +0800 Subject: [PATCH 12/17] [EROFS] test: add tests for .wh files Add tests for deleting directories and files. Signed-off-by: Hongzhen Luo --- src/overlaybd/tar/erofs/test/erofs_stress.cpp | 95 ++++++++++++++ .../tar/erofs/test/erofs_stress_base.cpp | 124 +++++++++++++----- .../tar/erofs/test/erofs_stress_base.h | 13 +- 3 files changed, 195 insertions(+), 37 deletions(-) diff --git a/src/overlaybd/tar/erofs/test/erofs_stress.cpp b/src/overlaybd/tar/erofs/test/erofs_stress.cpp index 34ad137c..0bcc505c 100644 --- a/src/overlaybd/tar/erofs/test/erofs_stress.cpp +++ b/src/overlaybd/tar/erofs/test/erofs_stress.cpp @@ -546,6 +546,92 @@ class StressCase007: public StressBase, public StressInterImpl { } }; +/* + * TC008 + * + * Create layers, each layer contains 50 dirs, + * each dir contains 2 files. + * + * Test whiteout files. + */ +class StressCase008: public StressBase, public StressInterImpl { +private: + std::map> mp; +public: + StressCase008(std::string path, int layers): StressBase(path, layers) {} + + bool build_gen_mod(StressNode *node, StressHostFile *file) override { + return StressInterImpl::build_gen_mod(node, file); + } + bool build_gen_own(StressNode *node, StressHostFile *file) override { + return StressInterImpl::build_gen_own(node, file); + } + bool build_gen_xattrs(StressNode *node, StressHostFile *file) override { + return StressInterImpl::build_gen_xattrs(node, file); + } + bool build_gen_content(StressNode *node, StressHostFile *file) override { + return StressInterImpl::build_gen_content(node, file); + } + + bool verify_gen_mod(StressNode *node, photon::fs::IFile *erofs_file) override { + return StressInterImpl::verify_gen_mod(node, erofs_file); + } + bool verify_gen_own(StressNode *node, photon::fs::IFile *erofs_file) override { + return StressInterImpl::verify_gen_own(node, erofs_file); + } + bool verify_gen_xattrs(StressNode *node, photon::fs::IFile *erofs_file) override { + return StressInterImpl::verify_gen_xattrs(node, erofs_file); + } + bool verify_gen_content(StressNode *node, photon::fs::IFile *erofs_file) override { + return StressInterImpl::verify_gen_content(node, erofs_file); + } + + std::string generate_name(int idx, int depth, std::string root_path, NODE_TYPE type) override { + std::string res; + int cnt = 0; + + if (idx < 1) { + res = get_randomstr(type ? MAX_FILE_NAME : MAX_DIR_NAME, true); + goto out; + } + + if (idx & 1) { + res = get_randomstr(type ? MAX_FILE_NAME : MAX_DIR_NAME, true); + goto verify_str; + } else { + /* generate whiteout files at even layers */ + res = tree->get_same_name(idx, depth, root_path, type, true); + /* fall back to a random name */ + if (res.length() == 0) + res = get_randomstr(type ? MAX_FILE_NAME : MAX_DIR_NAME, true); + } +verify_str: + if (mp.find(idx) == mp.end()) + mp[idx] = std::set(); + /* already used in this layer, fall back to a random name */ + while (mp[idx].find(res) != mp[idx].end()) { + res = get_randomstr(type ? MAX_FILE_NAME : MAX_DIR_NAME, true); + cnt ++; + if (cnt > 1000) + LOG_ERROR_RETURN(-1, "", "fail to gernate name"); + } + mp[idx].insert(res); + if (tree->get_type(root_path + "/" + res) == type && depth > 0) + res = std::string(EROFS_WHOUT_PREFIX) + res; /* .wh. file */ + out: + return res; + } + + std::vector layer_dirs(int idx) { + std::vector ret; + + /* 50 dirs, each contains 2 files */ + for (int i = 0; i < 50; i ++) + ret.emplace_back(2); + return ret; + } +}; + TEST(ErofsStressTest, TC001) { std::srand(static_cast(std::time(0))); StressCase001 *tc001 = new StressCase001("./erofs_stress_001", 20); @@ -603,6 +689,15 @@ TEST(ErofsStressTest, TC007) { delete tc007; } +TEST(ErofsStressTest, TC008) { + std::srand(static_cast(std::time(0))); + /* 100 layers */ + StressCase008 *tc008 = new StressCase008("./erofs_stress_008", 100); + + ASSERT_EQ(tc008->run(), true); + delete tc008; +} + int main(int argc, char **argv) { ::testing::InitGoogleTest(&argc, argv); diff --git a/src/overlaybd/tar/erofs/test/erofs_stress_base.cpp b/src/overlaybd/tar/erofs/test/erofs_stress_base.cpp index e8f6d68f..d7de7210 100644 --- a/src/overlaybd/tar/erofs/test/erofs_stress_base.cpp +++ b/src/overlaybd/tar/erofs/test/erofs_stress_base.cpp @@ -75,27 +75,33 @@ bool StressFsTree::add_node(StressNode *node) { auto it = tree.find(node->path); if (it == tree.end() || it->second->type == NODE_WHITEOUT) LOG_ERROR_RETURN(-1, false, "whiteout a invalid object"); - if (it->second->type == NODE_REGULAR) + /* whiteout a regular file */ + if (it->second->type == NODE_REGULAR) { tree.erase(it); - else { - std::string prefix = it->first; - for (auto p = tree.begin(); p != tree.end();) { - if (prefix.compare(0, prefix.size(), p->first) == 0) + } else if (it->second->type == NODE_DIR) { + /* whiteout a dir */ + std::string rm_prefix = it->first + "/"; + tree.erase(it); + for (auto p = tree.begin(); p != tree.end(); ) { + if (str_n_equal(rm_prefix, p->first, rm_prefix.length())) p = tree.erase(p); else p ++; } - } + } else + LOG_ERROR_RETURN(-1, false, "invalid object type: `", it->second->type); } return true; } -std::string StressFsTree::get_same_name(int idx, int depth, std::string root_path, NODE_TYPE type) { +std::string StressFsTree::get_same_name(int idx, int depth, std::string root_path, NODE_TYPE type, bool same_type) { std::vector vec; for (const auto& pair : tree) { if (pair.first == "/" || !is_substring(pair.first, root_path) || pair.first.length() == root_path.length()) continue; + if (same_type && pair.second->type != type) + continue; std::string last_component = pair.first.substr(root_path.length() + 1); if (!is_substring(last_component, "/")) vec.emplace_back(last_component); @@ -111,6 +117,15 @@ std::string StressFsTree::get_same_name(int idx, int depth, std::string root_pat return vec[0]; } +NODE_TYPE StressFsTree::get_type(std::string root_path) { + std::map::iterator it; + + it = tree.find(root_path); + if (it == tree.end()) + return NODE_TYPE_MAX; + return it->second->type; +} + struct LayerNode { std::string pwd; std::vector subdirs; @@ -175,20 +190,39 @@ bool StressBase::create_layer(int idx) { for (int i = 0; i < (int)cur->num_files; i ++) { std::string name_prefix = cur->pwd.substr(prefix.length()); // generate filename for files in the current dir - std::string filename = name_prefix + "/" + generate_name(idx, cur->depth, name_prefix, NODE_REGULAR); - StressNode *node = new StressNode(filename, NODE_REGULAR); - StressHostFile *file_info = new StressHostFile(prefix + filename, host_fs); - - res = build_gen_mod(node, file_info) && - build_gen_own(node, file_info) && - build_gen_xattrs(node, file_info) && - build_gen_content(node, file_info); - if (!res) - LOG_ERROR_RETURN(-1, false, "fail to generate file contents"); - if (!tree->add_node(node)) - LOG_ERROR_RETURN(-1, false, "failt to add node `", filename); - file_info->file->fsync(); - delete file_info; + std::string filename = generate_name(idx, cur->depth, name_prefix, NODE_REGULAR); + /* if this is a whout file */ + if (str_n_equal(filename, std::string(EROFS_WHOUT_PREFIX), strlen(EROFS_WHOUT_PREFIX))) { + std::string host_filename = name_prefix + "/" + filename; + filename = name_prefix + "/" + filename.substr(strlen(EROFS_WHOUT_PREFIX)); + if (tree->get_type(filename) != NODE_REGULAR) + LOG_ERROR_RETURN(-1, false, "invalid whiteout filename: `", filename); + + StressHostFile *file_info = new StressHostFile(prefix + host_filename, host_fs); + if (!file_info->file) + LOG_ERROR_RETURN(-1, false, "fail to crate whiteout file in host fs: `", host_filename); + + StressNode *node = new StressNode(filename, NODE_WHITEOUT); + if (!tree->add_node(node)) + LOG_ERROR_RETURN(-1, false, "fail to add WHITEOUT file `", filename); + file_info->file->fsync(); + delete file_info; + } else { + filename = name_prefix + "/" + filename; + StressNode *node = new StressNode(filename, NODE_REGULAR); + StressHostFile *file_info = new StressHostFile(prefix + filename, host_fs); + + res = build_gen_mod(node, file_info) && + build_gen_own(node, file_info) && + build_gen_xattrs(node, file_info) && + build_gen_content(node, file_info); + if (!res) + LOG_ERROR_RETURN(-1, false, "fail to generate file contents"); + if (!tree->add_node(node)) + LOG_ERROR_RETURN(-1, false, "failt to add node `", filename); + file_info->file->fsync(); + delete file_info; + } } for (int i = 0; i < (int)cur->subdirs.size(); i ++) { @@ -196,12 +230,32 @@ bool StressBase::create_layer(int idx) { next->depth = cur->depth + 1; // generate subdir name in the current dir for (int try_times = 0; try_times < MAX_TRY_TIME; try_times++) { - next->pwd = cur->pwd + "/" + generate_name(idx, cur->depth, cur->pwd.substr(prefix.length()), NODE_DIR); - if (host_fs->mkdir(next->pwd.c_str(), 0755) == 0) { - StressNode *dir_node = new StressNode(next->pwd.substr(prefix.length()), NODE_DIR); - tree->add_node(dir_node); - q.emplace_back(next); + std::string dir_name = generate_name(idx, cur->depth, cur->pwd.substr(prefix.length()), NODE_DIR); + + /* if it is a whout dir */ + if (str_n_equal(dir_name, std::string(EROFS_WHOUT_PREFIX), strlen(EROFS_WHOUT_PREFIX))) { + std::string host_filename = cur->pwd + "/" + dir_name; + next->pwd = cur->pwd + "/" + dir_name.substr(strlen(EROFS_WHOUT_PREFIX)); + if (tree->get_type(next->pwd.substr(prefix.length())) != NODE_DIR) + LOG_ERROR_RETURN(-1, false, "invalid whiteout dir name: `", next->pwd.substr(prefix.length())); + StressNode *dir_node = new StressNode(next->pwd.substr(prefix.length()), NODE_WHITEOUT); + if (!tree->add_node(dir_node)) + LOG_ERROR_RETURN(-1, false, "fail to add WHITEOUT dir `", next->pwd.substr(prefix.length())); + /* create a .wh.dirname in the host fs */ + StressHostFile *file_info = new StressHostFile(host_filename, host_fs); + if (!file_info->file) + LOG_ERROR_RETURN(-1, false, "fail to crate whiout dir in host fs: `", host_filename); + file_info->file->fsync(); + delete file_info; break; + } else { + next->pwd = cur->pwd + "/" + dir_name; + if (host_fs->mkdir(next->pwd.c_str(), 0755) == 0) { + StressNode *dir_node = new StressNode(next->pwd.substr(prefix.length()), NODE_DIR); + tree->add_node(dir_node); + q.emplace_back(next); + break; + } } } } @@ -277,13 +331,17 @@ bool StressBase::verify(photon::fs::IFileSystem *erofs_fs) { if (S_ISDIR(st.st_mode)) { node = new StressNode(cur, NODE_DIR); auto dir = erofs_fs->opendir(cur.c_str()); - do { - dirent *dent = dir->get(); - if (first) - items.emplace_back(cur + std::string(dent->d_name)); - else - items.emplace_back(cur + "/" + std::string(dent->d_name)); - } while (dir->next()); + /* the dir may be empty, so check it first */ + if (dir->get() != nullptr) { + do { + dirent *dent = dir->get(); + + if (first) + items.emplace_back(cur + std::string(dent->d_name)); + else + items.emplace_back(cur + "/" + std::string(dent->d_name)); + } while (dir->next()); + } dir->closedir(); delete dir; } else if (S_ISREG(st.st_mode)) { diff --git a/src/overlaybd/tar/erofs/test/erofs_stress_base.h b/src/overlaybd/tar/erofs/test/erofs_stress_base.h index 3d3db17d..459debea 100644 --- a/src/overlaybd/tar/erofs/test/erofs_stress_base.h +++ b/src/overlaybd/tar/erofs/test/erofs_stress_base.h @@ -32,6 +32,8 @@ #define MAX_DIR_NAME 100 #define MAX_FILE_NAME 100 +#define EROFS_WHOUT_PREFIX ".wh." + /* in-mem node type */ enum NODE_TYPE { NODE_DIR, @@ -170,13 +172,15 @@ class StressFsTree { // verify process bool query_delete_node(StressNode *node) { + if (!node) + LOG_ERROR_RETURN(-1, false, "invalid node: nullptr"); auto it = tree.find(node->path); if (it == tree.end()) - LOG_ERROR_RETURN(-1,false, "path ` does not exist in in-mem tree", node->path); + LOG_ERROR_RETURN(-1,false, "path ` does not exist in in-mem tree", node->path); if (!it->second) - LOG_ERROR_RETURN(-1, false, "NULL in-mem info (`)", node->path); + LOG_ERROR_RETURN(-1, false, "NULL in-mem info (`)", node->path); if (!it->second->equal(node)) - LOG_ERROR_RETURN(-1, false, "node contents mismatch"); + LOG_ERROR_RETURN(-1, false, "node contents mismatch"); tree.erase(it); return true; } @@ -185,7 +189,8 @@ class StressFsTree { return tree.empty(); } - std::string get_same_name(int idx, int depth, std::string root_path, NODE_TYPE type); + std::string get_same_name(int idx, int depth, std::string root_path, NODE_TYPE type, bool same_type = false); + NODE_TYPE get_type(std::string root_path); }; class StressBase: public StressGenInter { From 0c16fc7b309433a5dfef9488c878673f5b9946db Mon Sep 17 00:00:00 2001 From: Hongzhen Luo Date: Wed, 11 Dec 2024 15:31:07 +0800 Subject: [PATCH 13/17] [EROFS] test: add test for delete followed by create This test case is used to test the deletion of a directory/file first, followed by the creation of a directory/file with the same name. Signed-off-by: Hongzhen Luo --- src/overlaybd/tar/erofs/test/erofs_stress.cpp | 116 ++++++++++++++++++ .../tar/erofs/test/erofs_stress_base.cpp | 4 +- .../tar/erofs/test/erofs_stress_base.h | 2 + 3 files changed, 120 insertions(+), 2 deletions(-) diff --git a/src/overlaybd/tar/erofs/test/erofs_stress.cpp b/src/overlaybd/tar/erofs/test/erofs_stress.cpp index 0bcc505c..bd503daa 100644 --- a/src/overlaybd/tar/erofs/test/erofs_stress.cpp +++ b/src/overlaybd/tar/erofs/test/erofs_stress.cpp @@ -632,6 +632,114 @@ class StressCase008: public StressBase, public StressInterImpl { } }; +/* + * TC009 + * + * Test the scenario of deleting first and then creating files/dirs. + */ +class StressCase009: public StressBase, public StressInterImpl { +private: + std::map> mp; + std::set deleted_names; +public: + StressCase009(std::string path, int layers): StressBase(path, layers) {} + + bool build_gen_mod(StressNode *node, StressHostFile *file) override { + return StressInterImpl::build_gen_mod(node, file); + } + bool build_gen_own(StressNode *node, StressHostFile *file) override { + return StressInterImpl::build_gen_own(node, file); + } + bool build_gen_xattrs(StressNode *node, StressHostFile *file) override { + return StressInterImpl::build_gen_xattrs(node, file); + } + bool build_gen_content(StressNode *node, StressHostFile *file) override { + return StressInterImpl::build_gen_content(node, file); + } + + bool verify_gen_mod(StressNode *node, photon::fs::IFile *erofs_file) override { + return StressInterImpl::verify_gen_mod(node, erofs_file); + } + bool verify_gen_own(StressNode *node, photon::fs::IFile *erofs_file) override { + return StressInterImpl::verify_gen_own(node, erofs_file); + } + bool verify_gen_xattrs(StressNode *node, photon::fs::IFile *erofs_file) override { + return StressInterImpl::verify_gen_xattrs(node, erofs_file); + } + bool verify_gen_content(StressNode *node, photon::fs::IFile *erofs_file) override { + return StressInterImpl::verify_gen_content(node, erofs_file); + } + + std::string generate_name(int idx, int depth, std::string root_path, NODE_TYPE type) override { + std::string res; + int cnt = 0; + + if (mp.find(idx) == mp.end()) + mp[idx] = std::set(); + + if (idx == 0) + res = get_randomstr(type ? MAX_FILE_NAME : MAX_DIR_NAME, true); + else if (idx == 1) { + res = tree->get_same_name(idx, depth, root_path, type, true); + if (res.length() == 0) + res = get_randomstr(type ? MAX_FILE_NAME : MAX_DIR_NAME, true); + /* if it is already been used, then generate a random name */ + while (mp[idx].find(res) != mp[idx].end()) { + res = get_randomstr(type ? MAX_FILE_NAME : MAX_DIR_NAME, true); + cnt ++; + if (cnt > 1000) + LOG_ERROR_RETURN(-1, "", "fail to generate name"); + } + mp[idx].insert(res); + if (tree->get_type(root_path + "/" + res) == type && depth > 0) { + deleted_names.insert(root_path + "/" + res); + LOG_INFO("delete file/dir: `, type: `", res, tree->get_type(root_path + "/" + res)); + res = std::string(EROFS_WHOUT_PREFIX) + res; + } + } else if (idx == 2) { + if (depth == 0) + res = tree->get_same_name(idx, depth, root_path, type, true); + else { + root_path += "/"; + for (const std::string& name: deleted_names) { + if (str_n_equal(name, root_path, root_path.length())) { + std::string last_component = name.substr(root_path.length()); + if (last_component.length() > 0 && !is_substring(last_component, "/")) { + if (mp[idx].find(last_component) == mp[idx].end()) { + res = last_component; + LOG_INFO("find deleted name: `, reuse it", res); + break; + } + } + } + } + } + if (res.length() == 0) + res = get_randomstr(type ? MAX_FILE_NAME : MAX_DIR_NAME, true); + goto check_res; + } else { + res = get_randomstr(type ? MAX_FILE_NAME : MAX_DIR_NAME, true); + check_res: while (mp[idx].find(res) != mp[idx].end()) { + res = get_randomstr(type ? MAX_FILE_NAME : MAX_DIR_NAME, true); + cnt ++; + if (cnt > 1000) + LOG_ERROR_RETURN(-1, "", "fail to generate name"); + } + mp[idx].insert(res); + } + return res; + } + + std::vector layer_dirs(int idx) { + std::vector ret; + + /* 1000 dirs, each contains 2 files */ + for (int i = 0; i < 1000; i ++) + ret.emplace_back(2); + return ret; + } +}; + TEST(ErofsStressTest, TC001) { std::srand(static_cast(std::time(0))); StressCase001 *tc001 = new StressCase001("./erofs_stress_001", 20); @@ -698,6 +806,14 @@ TEST(ErofsStressTest, TC008) { delete tc008; } +TEST(ErofsStressTest, TC009) { + std::srand(static_cast(std::time(0))); + StressCase009 *tc009 = new StressCase009("./erofs_stress_009", 3); + + ASSERT_EQ(tc009->run(), true); + delete tc009; +} + int main(int argc, char **argv) { ::testing::InitGoogleTest(&argc, argv); diff --git a/src/overlaybd/tar/erofs/test/erofs_stress_base.cpp b/src/overlaybd/tar/erofs/test/erofs_stress_base.cpp index d7de7210..3fa336fd 100644 --- a/src/overlaybd/tar/erofs/test/erofs_stress_base.cpp +++ b/src/overlaybd/tar/erofs/test/erofs_stress_base.cpp @@ -39,11 +39,11 @@ std::string get_randomstr(int max_length, bool range) } return res; } -static bool is_substring(const std::string& str, const std::string& substring) { +bool is_substring(const std::string& str, const std::string& substring) { return str.find(substring) != std::string::npos; } -static bool str_n_equal(std::string s1, std::string s2, long unsigned int n) { +bool str_n_equal(std::string s1, std::string s2, long unsigned int n) { if (s1.length() < n || s2.length() < n) return false; return s1.substr(0, n) == s2.substr(0, n); diff --git a/src/overlaybd/tar/erofs/test/erofs_stress_base.h b/src/overlaybd/tar/erofs/test/erofs_stress_base.h index 459debea..0c1f9077 100644 --- a/src/overlaybd/tar/erofs/test/erofs_stress_base.h +++ b/src/overlaybd/tar/erofs/test/erofs_stress_base.h @@ -233,3 +233,5 @@ class StressBase: public StressGenInter { /* helper functions */ std::string get_randomstr(int max_length, bool range); #define get_randomint(a, b) ((rand() % (b - a)) + a) +bool is_substring(const std::string& str, const std::string& substring); +bool str_n_equal(std::string s1, std::string s2, long unsigned int n); From c43d8f1be5ec8f38da3c53e2d345f476de2b1878 Mon Sep 17 00:00:00 2001 From: Hongzhen Luo Date: Wed, 18 Dec 2024 21:17:53 +0800 Subject: [PATCH 14/17] [EROFS] test: add tests for dirs Signed-off-by: Hongzhen Luo --- src/overlaybd/tar/erofs/test/erofs_stress.cpp | 176 ++++++++++++++++-- .../tar/erofs/test/erofs_stress_base.cpp | 37 +++- .../tar/erofs/test/erofs_stress_base.h | 8 +- 3 files changed, 198 insertions(+), 23 deletions(-) diff --git a/src/overlaybd/tar/erofs/test/erofs_stress.cpp b/src/overlaybd/tar/erofs/test/erofs_stress.cpp index bd503daa..96d6f7df 100644 --- a/src/overlaybd/tar/erofs/test/erofs_stress.cpp +++ b/src/overlaybd/tar/erofs/test/erofs_stress.cpp @@ -201,6 +201,43 @@ class StressInterImpl: public StressGenInter { name_map[idx].insert(res); return res; } + + bool build_dir_mod(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { + std::string str_mode("755"); + mode_t mode; + + mode = std::stoi(str_mode, nullptr, 8); + if (host_fs->chmod(path, mode)) + LOG_ERROR_RETURN(-1, false, "fail to set mode ` for dir `", str_mode, path); + node->mod = str_mode; + return true; + } + + bool build_dir_own(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { + uid_t uid = get_randomint(own_id_min, own_id_max); + gid_t gid = get_randomint(own_id_min, own_id_max); + + if (host_fs->chown(path, uid, gid)) + LOG_ERROR_RETURN(-1,false, "failt to chown of dir `", path); + node->own = std::to_string(uid) + std::to_string(gid); + return true; + } + + bool build_dir_xattrs(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { + photon::fs::IFileSystemXAttr *xattr_ops = dynamic_cast(host_fs); + if (xattr_ops == nullptr) + LOG_ERROR_RETURN(-1, false, "fs does not suppoert xattrs operations!"); + int xattrs_count = get_randomint(xattrs_min_count, xattrs_max_count + 1); + for (int i = 0; i < xattrs_count; i ++) { + int idx = get_randomint(0, xattrs_prefix.size()); + std::string key = xattrs_prefix[idx] + get_randomstr(get_randomint(xattrs_min_size, xattrs_max_size), false); + std::string value = get_randomstr(get_randomint(xattrs_min_size, xattrs_max_size), false); + if (xattr_ops->setxattr(path, key.c_str(), value.c_str(), value.size(), 0)) + LOG_ERROR_RETURN(-1, -1, "fail to set xattr (key: `, value: `) for dir `", key, value, path); + node->xattrs[key] = value; + } + return true; + } }; /* @@ -221,10 +258,27 @@ class StressCase001: public StressBase, public StressInterImpl { EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, build_gen_xattrs(StressNode *node, StressHostFile *file), true) EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, build_gen_content(StressNode *node, StressHostFile *file), true) + bool build_dir_mod(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { + return StressInterImpl::build_dir_mod(node, path, host_fs); + } + bool build_dir_own(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { + return StressInterImpl::build_dir_own(node, path, host_fs); + } + bool build_dir_xattrs(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { + return StressInterImpl::build_dir_xattrs(node, path, host_fs); + } + /* create empty nodes in verify phase */ - EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, verify_gen_mod(StressNode *node, photon::fs::IFile *erofs_file), true) - EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, verify_gen_own(StressNode *node, photon::fs::IFile *erofs_file), true) - EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, verify_gen_xattrs(StressNode *node, photon::fs::IFile *erofs_file), true) + bool verify_gen_mod(StressNode *node, photon::fs::IFile *erofs_file) override { + return node->type ==NODE_DIR ? StressInterImpl::verify_gen_mod(node, erofs_file) : true; + } + bool verify_gen_own(StressNode *node, photon::fs::IFile *erofs_file) override { + return node->type == NODE_DIR ? StressInterImpl::verify_gen_own(node, erofs_file) : true; + } + bool verify_gen_xattrs(StressNode *node, photon::fs::IFile *erofs_file) override { + return node->type == NODE_DIR ? StressInterImpl::verify_gen_xattrs(node, erofs_file): true; + } + EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, verify_gen_content(StressNode *node, photon::fs::IFile *erofs_file), true) /* simplely generate random dir and file names */ @@ -266,10 +320,26 @@ class StressCase002: public StressBase, public StressInterImpl { return StressInterImpl::build_gen_content(node, file); } + bool build_dir_mod(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { + return StressInterImpl::build_dir_mod(node, path, host_fs); + } + bool build_dir_own(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { + return StressInterImpl::build_dir_own(node, path, host_fs); + } + bool build_dir_xattrs(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { + return StressInterImpl::build_dir_xattrs(node, path, host_fs); + } + /* leave mod/own/xattr empty */ - EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, verify_gen_mod(StressNode *node, photon::fs::IFile *erofs_file), true) - EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, verify_gen_own(StressNode *node, photon::fs::IFile *erofs_file), true) - EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, verify_gen_xattrs(StressNode *node, photon::fs::IFile *erofs_file), true) + bool verify_gen_mod(StressNode *node, photon::fs::IFile *erofs_file) override { + return node->type ==NODE_DIR ? StressInterImpl::verify_gen_mod(node, erofs_file) : true; + } + bool verify_gen_own(StressNode *node, photon::fs::IFile *erofs_file) override { + return node->type == NODE_DIR ? StressInterImpl::verify_gen_own(node, erofs_file) : true; + } + bool verify_gen_xattrs(StressNode *node, photon::fs::IFile *erofs_file) override { + return node->type == NODE_DIR ? StressInterImpl::verify_gen_xattrs(node, erofs_file): true; + } bool verify_gen_content(StressNode *node, photon::fs::IFile *erofs_file) override { return StressInterImpl::verify_gen_content(node, erofs_file); } @@ -312,9 +382,23 @@ class StressCase003: public StressBase, public StressInterImpl { return StressInterImpl::build_gen_xattrs(node, file); } + bool build_dir_mod(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { + return StressInterImpl::build_dir_mod(node, path, host_fs); + } + bool build_dir_own(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { + return StressInterImpl::build_dir_own(node, path, host_fs); + } + bool build_dir_xattrs(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { + return StressInterImpl::build_dir_xattrs(node, path, host_fs); + } + /* leave mod/own/content empty */ - EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, verify_gen_mod(StressNode *node, photon::fs::IFile *erofs_file), true) - EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, verify_gen_own(StressNode *node, photon::fs::IFile *erofs_file), true) + bool verify_gen_mod(StressNode *node, photon::fs::IFile *erofs_file) override { + return node->type ==NODE_DIR ? StressInterImpl::verify_gen_mod(node, erofs_file) : true; + } + bool verify_gen_own(StressNode *node, photon::fs::IFile *erofs_file) override { + return node->type == NODE_DIR ? StressInterImpl::verify_gen_own(node, erofs_file) : true; + } EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, verify_gen_content(StressNode *node, photon::fs::IFile *erofs_file), true) bool verify_gen_xattrs(StressNode *node, photon::fs::IFile *erofs_file) override { return StressInterImpl::verify_gen_xattrs(node, erofs_file); @@ -355,12 +439,26 @@ class StressCase004: public StressBase, public StressInterImpl { return StressInterImpl::build_gen_mod(node, file); } - EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, verify_gen_own(StressNode *node, photon::fs::IFile *erofs_file), true) - EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, verify_gen_xattrs(StressNode *node, photon::fs::IFile *erofs_file), true) + bool build_dir_mod(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { + return StressInterImpl::build_dir_mod(node, path, host_fs); + } + bool build_dir_own(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { + return StressInterImpl::build_dir_own(node, path, host_fs); + } + bool build_dir_xattrs(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { + return StressInterImpl::build_dir_xattrs(node, path, host_fs); + } + EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, verify_gen_content(StressNode *node, photon::fs::IFile *erofs_file), true) bool verify_gen_mod(StressNode *node, photon::fs::IFile *erofs_file) override { return StressInterImpl::verify_gen_mod(node, erofs_file); } + bool verify_gen_own(StressNode *node, photon::fs::IFile *erofs_file) override { + return node->type == NODE_DIR ? StressInterImpl::verify_gen_own(node, erofs_file) : true; + } + bool verify_gen_xattrs(StressNode *node, photon::fs::IFile *erofs_file) override { + return node->type == NODE_DIR ? StressInterImpl::verify_gen_xattrs(node, erofs_file): true; + } /* simplely generate random dir and file names */ std::string generate_name(int idx, int depth, std::string root_path, NODE_TYPE type) override { @@ -396,12 +494,26 @@ class StressCase005: public StressBase, public StressInterImpl { return StressInterImpl::build_gen_own(node, file); } - EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, verify_gen_mod(StressNode *node, photon::fs::IFile *erofs_file), true) - EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, verify_gen_xattrs(StressNode *node, photon::fs::IFile *erofs_file), true) + bool build_dir_mod(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { + return StressInterImpl::build_dir_mod(node, path, host_fs); + } + bool build_dir_own(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { + return StressInterImpl::build_dir_own(node, path, host_fs); + } + bool build_dir_xattrs(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { + return StressInterImpl::build_dir_xattrs(node, path, host_fs); + } + + bool verify_gen_mod(StressNode *node, photon::fs::IFile *erofs_file) override { + return node->type ==NODE_DIR ? StressInterImpl::verify_gen_mod(node, erofs_file) : true; + } EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, verify_gen_content(StressNode *node, photon::fs::IFile *erofs_file), true) bool verify_gen_own(StressNode *node, photon::fs::IFile *erofs_file) override { return StressInterImpl::verify_gen_own(node, erofs_file); } + bool verify_gen_xattrs(StressNode *node, photon::fs::IFile *erofs_file) override { + return node->type == NODE_DIR ? StressInterImpl::verify_gen_xattrs(node, erofs_file): true; + } /* simplely generate random dir and file names */ std::string generate_name(int idx, int depth, std::string root_path, NODE_TYPE type) override { @@ -443,6 +555,16 @@ class StressCase006: public StressBase, public StressInterImpl { return StressInterImpl::build_gen_content(node, file); } + bool build_dir_mod(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { + return StressInterImpl::build_dir_mod(node, path, host_fs); + } + bool build_dir_own(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { + return StressInterImpl::build_dir_own(node, path, host_fs); + } + bool build_dir_xattrs(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { + return StressInterImpl::build_dir_xattrs(node, path, host_fs); + } + bool verify_gen_mod(StressNode *node, photon::fs::IFile *erofs_file) override { return StressInterImpl::verify_gen_mod(node, erofs_file); } @@ -498,6 +620,16 @@ class StressCase007: public StressBase, public StressInterImpl { return StressInterImpl::build_gen_content(node, file); } + bool build_dir_mod(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { + return StressInterImpl::build_dir_mod(node, path, host_fs); + } + bool build_dir_own(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { + return StressInterImpl::build_dir_own(node, path, host_fs); + } + bool build_dir_xattrs(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { + return StressInterImpl::build_dir_xattrs(node, path, host_fs); + } + bool verify_gen_mod(StressNode *node, photon::fs::IFile *erofs_file) override { return StressInterImpl::verify_gen_mod(node, erofs_file); } @@ -573,6 +705,16 @@ class StressCase008: public StressBase, public StressInterImpl { return StressInterImpl::build_gen_content(node, file); } + bool build_dir_mod(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { + return StressInterImpl::build_dir_mod(node, path, host_fs); + } + bool build_dir_own(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { + return StressInterImpl::build_dir_own(node, path, host_fs); + } + bool build_dir_xattrs(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { + return StressInterImpl::build_dir_xattrs(node, path, host_fs); + } + bool verify_gen_mod(StressNode *node, photon::fs::IFile *erofs_file) override { return StressInterImpl::verify_gen_mod(node, erofs_file); } @@ -657,6 +799,16 @@ class StressCase009: public StressBase, public StressInterImpl { return StressInterImpl::build_gen_content(node, file); } + bool build_dir_mod(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { + return StressInterImpl::build_dir_mod(node, path, host_fs); + } + bool build_dir_own(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { + return StressInterImpl::build_dir_own(node, path, host_fs); + } + bool build_dir_xattrs(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { + return StressInterImpl::build_dir_xattrs(node, path, host_fs); + } + bool verify_gen_mod(StressNode *node, photon::fs::IFile *erofs_file) override { return StressInterImpl::verify_gen_mod(node, erofs_file); } diff --git a/src/overlaybd/tar/erofs/test/erofs_stress_base.cpp b/src/overlaybd/tar/erofs/test/erofs_stress_base.cpp index 3fa336fd..28ecfb55 100644 --- a/src/overlaybd/tar/erofs/test/erofs_stress_base.cpp +++ b/src/overlaybd/tar/erofs/test/erofs_stress_base.cpp @@ -169,6 +169,7 @@ bool StressBase::create_layer(int idx) { std::string root_dirname = generate_name(idx, layer_tree->depth, "", NODE_DIR); std::string root_path = prefix + "/" + root_dirname; std::string clean_cmd = "rm -rf " + root_path; + bool res; if (system(clean_cmd.c_str())) LOG_ERROR_RETURN(-1, false, "fail to prepare clean dir for `", root_path); @@ -179,11 +180,17 @@ bool StressBase::create_layer(int idx) { StressNode *node = new StressNode(layer_tree->pwd.substr(prefix.length()), NODE_DIR); if (host_fs->mkdir(layer_tree->pwd.c_str(), 0755) != 0) LOG_ERROR_RETURN(-1, false, "fail to mkdir `", layer_tree->pwd); - tree->add_node(node); + + res = build_dir_mod(node, layer_tree->pwd.c_str(), host_fs) && + build_dir_own(node, layer_tree->pwd.c_str(), host_fs) && + build_dir_xattrs(node, layer_tree->pwd.c_str(), host_fs); + if (!res) + LOG_ERROR_RETURN(-1, false, "fail to generate fields for dir `",layer_tree->pwd); + if (!tree->add_node(node)) + LOG_ERROR_RETURN(-1, false, "fail to add node `",layer_tree->pwd); // traverse the layer tree while (q.size()) { - bool res; LayerNode *cur = q.front(); q.erase(q.begin()); @@ -252,7 +259,14 @@ bool StressBase::create_layer(int idx) { next->pwd = cur->pwd + "/" + dir_name; if (host_fs->mkdir(next->pwd.c_str(), 0755) == 0) { StressNode *dir_node = new StressNode(next->pwd.substr(prefix.length()), NODE_DIR); - tree->add_node(dir_node); + + res = build_dir_mod(dir_node, next->pwd.c_str(), host_fs) && + build_dir_own(dir_node, next->pwd.c_str(), host_fs) && + build_dir_xattrs(dir_node, next->pwd.c_str(), host_fs); + if (!res) + LOG_ERROR_RETURN(-1, false, "fail to generate fields for dir `", next->pwd); + if (!tree->add_node(dir_node)) + LOG_ERROR_RETURN(-1, false, "fail to add node `", next->pwd); q.emplace_back(next); break; } @@ -329,7 +343,6 @@ bool StressBase::verify(photon::fs::IFileSystem *erofs_fs) { if (erofs_fs->stat(cur.c_str(), &st)) LOG_ERRNO_RETURN(-1, false, "fail to stat file `", cur); if (S_ISDIR(st.st_mode)) { - node = new StressNode(cur, NODE_DIR); auto dir = erofs_fs->opendir(cur.c_str()); /* the dir may be empty, so check it first */ if (dir->get() != nullptr) { @@ -344,31 +357,35 @@ bool StressBase::verify(photon::fs::IFileSystem *erofs_fs) { } dir->closedir(); delete dir; - } else if (S_ISREG(st.st_mode)) { + } + node = new StressNode(cur, S_ISREG(st.st_mode) ? NODE_REGULAR : NODE_DIR); + + if (!first) { photon::fs::IFile *file; bool ret; file = erofs_fs->open(cur.c_str(), O_RDONLY); - node = new StressNode(cur, NODE_REGULAR); if (!file || ! node) LOG_ERROR_RETURN(0, false, "fail to open file or node `", cur); ret = verify_gen_mod(node, file) && verify_gen_own(node, file) && - verify_gen_xattrs(node, file) && - verify_gen_content(node, file); + verify_gen_xattrs(node, file); + /* do not generate contents for dirs */ + if (S_ISREG(st.st_mode)) + ret += verify_gen_content(node, file); if (!ret) LOG_ERROR_RETURN(0, false, "fail to construct StressNode"); file->close(); delete file; + } if (!tree->query_delete_node(node)) { delete node; LOG_ERROR_RETURN(-1, false, "file ` in erofs_fs but not in the in-mem tree", cur); } + first = false; - if (first) - first = false; } while (!items.empty()); if (!tree->is_emtry()) diff --git a/src/overlaybd/tar/erofs/test/erofs_stress_base.h b/src/overlaybd/tar/erofs/test/erofs_stress_base.h index 0c1f9077..e1c9e86a 100644 --- a/src/overlaybd/tar/erofs/test/erofs_stress_base.h +++ b/src/overlaybd/tar/erofs/test/erofs_stress_base.h @@ -128,7 +128,13 @@ class StressGenInter { virtual bool build_gen_own(StressNode *node /* out */, StressHostFile *file_info /* out */) = 0; virtual bool build_gen_xattrs(StressNode *node /* out */, StressHostFile *file_info /* out */) = 0; virtual bool build_gen_content(StressNode *node /* out */, StressHostFile *file_info /* out */) = 0; - /* generate in-mem inode according to erofs-fs file */ + + /* for a single dir (node) */ + virtual bool build_dir_mod(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) = 0; + virtual bool build_dir_own(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) = 0; + virtual bool build_dir_xattrs(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) = 0; + + /* generate in-mem inode according to erofs-fs file (for both files and dirs) */ virtual bool verify_gen_mod(StressNode *node /* out */, photon::fs::IFile *erofs_file /* in */) = 0; virtual bool verify_gen_own(StressNode *node /* out */, photon::fs::IFile *erofs_file /* in */) = 0; virtual bool verify_gen_xattrs(StressNode *node /* out */, photon::fs::IFile *erofs_file /* in */) = 0; From 8a99dd2f84bc95280e43b167e446177a91fc90b2 Mon Sep 17 00:00:00 2001 From: Hongzhen Luo Date: Thu, 19 Dec 2024 16:29:04 +0800 Subject: [PATCH 15/17] [EROFS] test: add stat check Signed-off-by: Hongzhen Luo --- src/overlaybd/tar/erofs/test/erofs_stress.cpp | 186 ++++++++++-------- .../tar/erofs/test/erofs_stress_base.cpp | 14 +- .../tar/erofs/test/erofs_stress_base.h | 40 ++-- 3 files changed, 138 insertions(+), 102 deletions(-) diff --git a/src/overlaybd/tar/erofs/test/erofs_stress.cpp b/src/overlaybd/tar/erofs/test/erofs_stress.cpp index 96d6f7df..57a6456c 100644 --- a/src/overlaybd/tar/erofs/test/erofs_stress.cpp +++ b/src/overlaybd/tar/erofs/test/erofs_stress.cpp @@ -146,19 +146,6 @@ class StressInterImpl: public StressGenInter { mode = std::stoi(str_mode, nullptr, 8); if (file->file->fchmod(mode)) LOG_ERROR_RETURN(-1, false, "fail to set mode ` for file `", str_mode, file->path); - node->mod = str_mode; - return true; - } - - /* mode in verify phase */ - bool verify_gen_mod(StressNode *node, photon::fs::IFile *erofs_file) override { - struct stat st; - std::ostringstream oss; - - if (erofs_file->fstat(&st)) - LOG_ERROR_RETURN(-1, false, "fail to stat erofs file"); - oss << std::oct << std::setfill('0') << std::setw(3) << (st.st_mode & 0777); - node->mod = oss.str(); return true; } @@ -169,17 +156,6 @@ class StressInterImpl: public StressGenInter { if (file->file->fchown(uid, gid)) LOG_ERROR_RETURN(-1,false, "failt to chown of file `", file->path); - node->own = std::to_string(uid) + std::to_string(gid); - return true; - } - - /* own in verify phase */ - bool verify_gen_own(StressNode *node, photon::fs::IFile *erofs_file) override { - struct stat st; - - if (erofs_file->fstat(&st)) - LOG_ERROR_RETURN(-1, false, "fail to stat erofs file"); - node->own = std::to_string(st.st_uid) + std::to_string(st.st_gid); return true; } @@ -209,7 +185,6 @@ class StressInterImpl: public StressGenInter { mode = std::stoi(str_mode, nullptr, 8); if (host_fs->chmod(path, mode)) LOG_ERROR_RETURN(-1, false, "fail to set mode ` for dir `", str_mode, path); - node->mod = str_mode; return true; } @@ -219,7 +194,6 @@ class StressInterImpl: public StressGenInter { if (host_fs->chown(path, uid, gid)) LOG_ERROR_RETURN(-1,false, "failt to chown of dir `", path); - node->own = std::to_string(uid) + std::to_string(gid); return true; } @@ -238,6 +212,24 @@ class StressInterImpl: public StressGenInter { } return true; } + + bool build_stat_file(StressNode *node, StressHostFile *file_info) override { + if (file_info->file->fstat(&node->node_stat)) + LOG_ERRNO_RETURN(-1, false, "fail to stat file `", file_info->path); + return true; + } + + bool build_stat_dir(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { + if (host_fs->stat(path, &node->node_stat)) + LOG_ERROR_RETURN(-1, false, "fail to stat dir `", path); + return true; + } + + bool verify_stat(StressNode *node, photon::fs::IFile *erofs_file) override { + if (erofs_file->fstat(&node->node_stat)) + LOG_ERRNO_RETURN(-1, false, "fail to stat erofs_file"); + return true; + } }; /* @@ -257,6 +249,9 @@ class StressCase001: public StressBase, public StressInterImpl { EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, build_gen_own(StressNode *node, StressHostFile *file), true) EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, build_gen_xattrs(StressNode *node, StressHostFile *file), true) EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, build_gen_content(StressNode *node, StressHostFile *file), true) + bool build_stat_file(StressNode *node, StressHostFile *file) override { + return StressInterImpl::build_stat_file(node, file); + } bool build_dir_mod(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { return StressInterImpl::build_dir_mod(node, path, host_fs); @@ -267,17 +262,16 @@ class StressCase001: public StressBase, public StressInterImpl { bool build_dir_xattrs(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { return StressInterImpl::build_dir_xattrs(node, path, host_fs); } - - /* create empty nodes in verify phase */ - bool verify_gen_mod(StressNode *node, photon::fs::IFile *erofs_file) override { - return node->type ==NODE_DIR ? StressInterImpl::verify_gen_mod(node, erofs_file) : true; - } - bool verify_gen_own(StressNode *node, photon::fs::IFile *erofs_file) override { - return node->type == NODE_DIR ? StressInterImpl::verify_gen_own(node, erofs_file) : true; + bool build_stat_dir(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { + return StressInterImpl::build_stat_dir(node, path, host_fs); } + bool verify_gen_xattrs(StressNode *node, photon::fs::IFile *erofs_file) override { return node->type == NODE_DIR ? StressInterImpl::verify_gen_xattrs(node, erofs_file): true; } + bool verify_stat(StressNode *node, photon::fs::IFile *erofs_file) override { + return StressInterImpl::verify_stat(node, erofs_file); + } EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, verify_gen_content(StressNode *node, photon::fs::IFile *erofs_file), true) @@ -319,6 +313,9 @@ class StressCase002: public StressBase, public StressInterImpl { bool build_gen_content(StressNode *node, StressHostFile *file) override { return StressInterImpl::build_gen_content(node, file); } + bool build_stat_file(StressNode *node, StressHostFile *file) override { + return StressInterImpl::build_stat_file(node, file); + } bool build_dir_mod(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { return StressInterImpl::build_dir_mod(node, path, host_fs); @@ -329,20 +326,19 @@ class StressCase002: public StressBase, public StressInterImpl { bool build_dir_xattrs(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { return StressInterImpl::build_dir_xattrs(node, path, host_fs); } - - /* leave mod/own/xattr empty */ - bool verify_gen_mod(StressNode *node, photon::fs::IFile *erofs_file) override { - return node->type ==NODE_DIR ? StressInterImpl::verify_gen_mod(node, erofs_file) : true; - } - bool verify_gen_own(StressNode *node, photon::fs::IFile *erofs_file) override { - return node->type == NODE_DIR ? StressInterImpl::verify_gen_own(node, erofs_file) : true; + bool build_stat_dir(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { + return StressInterImpl::build_stat_dir(node, path, host_fs); } + bool verify_gen_xattrs(StressNode *node, photon::fs::IFile *erofs_file) override { return node->type == NODE_DIR ? StressInterImpl::verify_gen_xattrs(node, erofs_file): true; } bool verify_gen_content(StressNode *node, photon::fs::IFile *erofs_file) override { return StressInterImpl::verify_gen_content(node, erofs_file); } + bool verify_stat(StressNode *node, photon::fs::IFile *erofs_file) override { + return StressInterImpl::verify_stat(node, erofs_file); + } /* simplely generate random dir and file names */ std::string generate_name(int idx, int depth, std::string root_path, NODE_TYPE type) override { @@ -381,6 +377,9 @@ class StressCase003: public StressBase, public StressInterImpl { bool build_gen_xattrs(StressNode *node, StressHostFile *file) override { return StressInterImpl::build_gen_xattrs(node, file); } + bool build_stat_file(StressNode *node, StressHostFile *file) override { + return StressInterImpl::build_stat_file(node, file); + } bool build_dir_mod(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { return StressInterImpl::build_dir_mod(node, path, host_fs); @@ -391,18 +390,17 @@ class StressCase003: public StressBase, public StressInterImpl { bool build_dir_xattrs(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { return StressInterImpl::build_dir_xattrs(node, path, host_fs); } - - /* leave mod/own/content empty */ - bool verify_gen_mod(StressNode *node, photon::fs::IFile *erofs_file) override { - return node->type ==NODE_DIR ? StressInterImpl::verify_gen_mod(node, erofs_file) : true; - } - bool verify_gen_own(StressNode *node, photon::fs::IFile *erofs_file) override { - return node->type == NODE_DIR ? StressInterImpl::verify_gen_own(node, erofs_file) : true; + bool build_stat_dir(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { + return StressInterImpl::build_stat_dir(node, path, host_fs); } - EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, verify_gen_content(StressNode *node, photon::fs::IFile *erofs_file), true) + bool verify_gen_xattrs(StressNode *node, photon::fs::IFile *erofs_file) override { return StressInterImpl::verify_gen_xattrs(node, erofs_file); } + bool verify_stat(StressNode *node, photon::fs::IFile *erofs_file) override { + return StressInterImpl::verify_stat(node, erofs_file); + } + EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, verify_gen_content(StressNode *node, photon::fs::IFile *erofs_file), true) /* simplely generate random dir and file names */ std::string generate_name(int idx, int depth, std::string root_path, NODE_TYPE type) override { @@ -438,6 +436,9 @@ class StressCase004: public StressBase, public StressInterImpl { bool build_gen_mod(StressNode *node, StressHostFile *file) override { return StressInterImpl::build_gen_mod(node, file); } + bool build_stat_file(StressNode *node, StressHostFile *file) override { + return StressInterImpl::build_stat_file(node, file); + } bool build_dir_mod(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { return StressInterImpl::build_dir_mod(node, path, host_fs); @@ -448,17 +449,17 @@ class StressCase004: public StressBase, public StressInterImpl { bool build_dir_xattrs(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { return StressInterImpl::build_dir_xattrs(node, path, host_fs); } + bool build_stat_dir(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { + return StressInterImpl::build_stat_dir(node, path, host_fs); + } EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, verify_gen_content(StressNode *node, photon::fs::IFile *erofs_file), true) - bool verify_gen_mod(StressNode *node, photon::fs::IFile *erofs_file) override { - return StressInterImpl::verify_gen_mod(node, erofs_file); - } - bool verify_gen_own(StressNode *node, photon::fs::IFile *erofs_file) override { - return node->type == NODE_DIR ? StressInterImpl::verify_gen_own(node, erofs_file) : true; - } bool verify_gen_xattrs(StressNode *node, photon::fs::IFile *erofs_file) override { return node->type == NODE_DIR ? StressInterImpl::verify_gen_xattrs(node, erofs_file): true; } + bool verify_stat(StressNode *node, photon::fs::IFile *erofs_file) override { + return StressInterImpl::verify_stat(node, erofs_file); + } /* simplely generate random dir and file names */ std::string generate_name(int idx, int depth, std::string root_path, NODE_TYPE type) override { @@ -493,6 +494,9 @@ class StressCase005: public StressBase, public StressInterImpl { bool build_gen_own(StressNode *node, StressHostFile *file) override { return StressInterImpl::build_gen_own(node, file); } + bool build_stat_file(StressNode *node, StressHostFile *file) override { + return StressInterImpl::build_stat_file(node, file); + } bool build_dir_mod(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { return StressInterImpl::build_dir_mod(node, path, host_fs); @@ -503,17 +507,17 @@ class StressCase005: public StressBase, public StressInterImpl { bool build_dir_xattrs(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { return StressInterImpl::build_dir_xattrs(node, path, host_fs); } - - bool verify_gen_mod(StressNode *node, photon::fs::IFile *erofs_file) override { - return node->type ==NODE_DIR ? StressInterImpl::verify_gen_mod(node, erofs_file) : true; - } - EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, verify_gen_content(StressNode *node, photon::fs::IFile *erofs_file), true) - bool verify_gen_own(StressNode *node, photon::fs::IFile *erofs_file) override { - return StressInterImpl::verify_gen_own(node, erofs_file); + bool build_stat_dir(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { + return StressInterImpl::build_stat_dir(node, path, host_fs); } + bool verify_gen_xattrs(StressNode *node, photon::fs::IFile *erofs_file) override { return node->type == NODE_DIR ? StressInterImpl::verify_gen_xattrs(node, erofs_file): true; } + bool verify_stat(StressNode *node, photon::fs::IFile *erofs_file) override { + return StressInterImpl::verify_stat(node, erofs_file); + } + EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, verify_gen_content(StressNode *node, photon::fs::IFile *erofs_file), true) /* simplely generate random dir and file names */ std::string generate_name(int idx, int depth, std::string root_path, NODE_TYPE type) override { @@ -554,6 +558,9 @@ class StressCase006: public StressBase, public StressInterImpl { bool build_gen_content(StressNode *node, StressHostFile *file) override { return StressInterImpl::build_gen_content(node, file); } + bool build_stat_file(StressNode *node, StressHostFile *file) override { + return StressInterImpl::build_stat_file(node, file); + } bool build_dir_mod(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { return StressInterImpl::build_dir_mod(node, path, host_fs); @@ -564,19 +571,19 @@ class StressCase006: public StressBase, public StressInterImpl { bool build_dir_xattrs(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { return StressInterImpl::build_dir_xattrs(node, path, host_fs); } - - bool verify_gen_mod(StressNode *node, photon::fs::IFile *erofs_file) override { - return StressInterImpl::verify_gen_mod(node, erofs_file); - } - bool verify_gen_own(StressNode *node, photon::fs::IFile *erofs_file) override { - return StressInterImpl::verify_gen_own(node, erofs_file); + bool build_stat_dir(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { + return StressInterImpl::build_stat_dir(node, path, host_fs); } + bool verify_gen_xattrs(StressNode *node, photon::fs::IFile *erofs_file) override { return StressInterImpl::verify_gen_xattrs(node, erofs_file); } bool verify_gen_content(StressNode *node, photon::fs::IFile *erofs_file) override { return StressInterImpl::verify_gen_content(node, erofs_file); } + bool verify_stat(StressNode *node, photon::fs::IFile *erofs_file) override { + return StressInterImpl::verify_stat(node, erofs_file); + } /* simplely generate random dir and file names */ std::string generate_name(int idx, int depth, std::string root_path, NODE_TYPE type) override { @@ -619,6 +626,9 @@ class StressCase007: public StressBase, public StressInterImpl { bool build_gen_content(StressNode *node, StressHostFile *file) override { return StressInterImpl::build_gen_content(node, file); } + bool build_stat_file(StressNode *node, StressHostFile *file) override { + return StressInterImpl::build_stat_file(node, file); + } bool build_dir_mod(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { return StressInterImpl::build_dir_mod(node, path, host_fs); @@ -629,19 +639,19 @@ class StressCase007: public StressBase, public StressInterImpl { bool build_dir_xattrs(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { return StressInterImpl::build_dir_xattrs(node, path, host_fs); } - - bool verify_gen_mod(StressNode *node, photon::fs::IFile *erofs_file) override { - return StressInterImpl::verify_gen_mod(node, erofs_file); - } - bool verify_gen_own(StressNode *node, photon::fs::IFile *erofs_file) override { - return StressInterImpl::verify_gen_own(node, erofs_file); + bool build_stat_dir(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { + return StressInterImpl::build_stat_dir(node, path, host_fs); } + bool verify_gen_xattrs(StressNode *node, photon::fs::IFile *erofs_file) override { return StressInterImpl::verify_gen_xattrs(node, erofs_file); } bool verify_gen_content(StressNode *node, photon::fs::IFile *erofs_file) override { return StressInterImpl::verify_gen_content(node, erofs_file); } + bool verify_stat(StressNode *node, photon::fs::IFile *erofs_file) override { + return StressInterImpl::verify_stat(node, erofs_file); + } std::string generate_name(int idx, int depth, std::string root_path, NODE_TYPE type) override { std::string res; @@ -704,6 +714,9 @@ class StressCase008: public StressBase, public StressInterImpl { bool build_gen_content(StressNode *node, StressHostFile *file) override { return StressInterImpl::build_gen_content(node, file); } + bool build_stat_file(StressNode *node, StressHostFile *file) override { + return StressInterImpl::build_stat_file(node, file); + } bool build_dir_mod(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { return StressInterImpl::build_dir_mod(node, path, host_fs); @@ -714,19 +727,19 @@ class StressCase008: public StressBase, public StressInterImpl { bool build_dir_xattrs(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { return StressInterImpl::build_dir_xattrs(node, path, host_fs); } - - bool verify_gen_mod(StressNode *node, photon::fs::IFile *erofs_file) override { - return StressInterImpl::verify_gen_mod(node, erofs_file); - } - bool verify_gen_own(StressNode *node, photon::fs::IFile *erofs_file) override { - return StressInterImpl::verify_gen_own(node, erofs_file); + bool build_stat_dir(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { + return StressInterImpl::build_stat_dir(node, path, host_fs); } + bool verify_gen_xattrs(StressNode *node, photon::fs::IFile *erofs_file) override { return StressInterImpl::verify_gen_xattrs(node, erofs_file); } bool verify_gen_content(StressNode *node, photon::fs::IFile *erofs_file) override { return StressInterImpl::verify_gen_content(node, erofs_file); } + bool verify_stat(StressNode *node, photon::fs::IFile *erofs_file) override { + return StressInterImpl::verify_stat(node, erofs_file); + } std::string generate_name(int idx, int depth, std::string root_path, NODE_TYPE type) override { std::string res; @@ -798,6 +811,9 @@ class StressCase009: public StressBase, public StressInterImpl { bool build_gen_content(StressNode *node, StressHostFile *file) override { return StressInterImpl::build_gen_content(node, file); } + bool build_stat_file(StressNode *node, StressHostFile *file) override { + return StressInterImpl::build_stat_file(node, file); + } bool build_dir_mod(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { return StressInterImpl::build_dir_mod(node, path, host_fs); @@ -808,19 +824,19 @@ class StressCase009: public StressBase, public StressInterImpl { bool build_dir_xattrs(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { return StressInterImpl::build_dir_xattrs(node, path, host_fs); } - - bool verify_gen_mod(StressNode *node, photon::fs::IFile *erofs_file) override { - return StressInterImpl::verify_gen_mod(node, erofs_file); - } - bool verify_gen_own(StressNode *node, photon::fs::IFile *erofs_file) override { - return StressInterImpl::verify_gen_own(node, erofs_file); + bool build_stat_dir(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { + return StressInterImpl::build_stat_dir(node, path, host_fs); } + bool verify_gen_xattrs(StressNode *node, photon::fs::IFile *erofs_file) override { return StressInterImpl::verify_gen_xattrs(node, erofs_file); } bool verify_gen_content(StressNode *node, photon::fs::IFile *erofs_file) override { return StressInterImpl::verify_gen_content(node, erofs_file); } + bool verify_stat(StressNode *node, photon::fs::IFile *erofs_file) override { + return StressInterImpl::verify_stat(node, erofs_file); + } std::string generate_name(int idx, int depth, std::string root_path, NODE_TYPE type) override { std::string res; diff --git a/src/overlaybd/tar/erofs/test/erofs_stress_base.cpp b/src/overlaybd/tar/erofs/test/erofs_stress_base.cpp index 28ecfb55..81fc4a86 100644 --- a/src/overlaybd/tar/erofs/test/erofs_stress_base.cpp +++ b/src/overlaybd/tar/erofs/test/erofs_stress_base.cpp @@ -183,7 +183,8 @@ bool StressBase::create_layer(int idx) { res = build_dir_mod(node, layer_tree->pwd.c_str(), host_fs) && build_dir_own(node, layer_tree->pwd.c_str(), host_fs) && - build_dir_xattrs(node, layer_tree->pwd.c_str(), host_fs); + build_dir_xattrs(node, layer_tree->pwd.c_str(), host_fs) && + build_stat_dir(node, layer_tree->pwd.c_str(), host_fs); if (!res) LOG_ERROR_RETURN(-1, false, "fail to generate fields for dir `",layer_tree->pwd); if (!tree->add_node(node)) @@ -222,7 +223,8 @@ bool StressBase::create_layer(int idx) { res = build_gen_mod(node, file_info) && build_gen_own(node, file_info) && build_gen_xattrs(node, file_info) && - build_gen_content(node, file_info); + build_gen_content(node, file_info) && + build_stat_file(node, file_info); if (!res) LOG_ERROR_RETURN(-1, false, "fail to generate file contents"); if (!tree->add_node(node)) @@ -262,7 +264,8 @@ bool StressBase::create_layer(int idx) { res = build_dir_mod(dir_node, next->pwd.c_str(), host_fs) && build_dir_own(dir_node, next->pwd.c_str(), host_fs) && - build_dir_xattrs(dir_node, next->pwd.c_str(), host_fs); + build_dir_xattrs(dir_node, next->pwd.c_str(), host_fs) && + build_stat_dir(dir_node, next->pwd.c_str(), host_fs); if (!res) LOG_ERROR_RETURN(-1, false, "fail to generate fields for dir `", next->pwd); if (!tree->add_node(dir_node)) @@ -367,9 +370,8 @@ bool StressBase::verify(photon::fs::IFileSystem *erofs_fs) { file = erofs_fs->open(cur.c_str(), O_RDONLY); if (!file || ! node) LOG_ERROR_RETURN(0, false, "fail to open file or node `", cur); - ret = verify_gen_mod(node, file) && - verify_gen_own(node, file) && - verify_gen_xattrs(node, file); + ret = verify_gen_xattrs(node, file) && + verify_stat(node, file); /* do not generate contents for dirs */ if (S_ISREG(st.st_mode)) ret += verify_gen_content(node, file); diff --git a/src/overlaybd/tar/erofs/test/erofs_stress_base.h b/src/overlaybd/tar/erofs/test/erofs_stress_base.h index e1c9e86a..6fb5d724 100644 --- a/src/overlaybd/tar/erofs/test/erofs_stress_base.h +++ b/src/overlaybd/tar/erofs/test/erofs_stress_base.h @@ -46,17 +46,19 @@ class StressNode { public: /* meta info for a in-mem node */ std::string path; - std::string mod; - std::string own; std::map xattrs; std::string content; enum NODE_TYPE type; + struct stat node_stat; - StressNode(std::string _path, NODE_TYPE _type): path(_path), type(_type) {} + StressNode(std::string _path, NODE_TYPE _type): path(_path), type(_type) { + memset(&node_stat, 0, sizeof(node_stat)); + if (type == NODE_DIR) + node_stat.st_nlink = 2; + } StressNode(StressNode *ano): - path(ano->path), mod(ano->mod), own(ano->own), xattrs(ano->xattrs), - content(ano->content) { } + path(ano->path), xattrs(ano->xattrs), content(ano->content) { } bool equal(StressNode *ano) { if (!ano) @@ -84,14 +86,29 @@ class StressNode { if (path.compare(ano->path)) LOG_ERROR_RETURN(-1, false, " path ` not equal to ` (`)", path, ano->path, path); - if (mod.compare(ano->mod)) - LOG_ERROR_RETURN(-1, false, "mode ` not equal to ` (`)", mod, ano->mod, path); - if (own.compare(ano->own)) - LOG_ERROR_RETURN(-1, false, "uid/gid ` not equal to ` (`)", own, ano->own, path); if (content.compare(ano->content)) LOG_ERROR_RETURN(-1, false, "content ` not equal to ` (`)", content, ano->content, path); if (type != ano->type) LOG_ERROR_RETURN(-1, false, "type ` not equal to ` (`)", type, ano->type, path); + if (node_stat.st_mode != ano->node_stat.st_mode) + LOG_ERROR_RETURN(-1, false, "mode ` not equal to ` (`)", node_stat.st_mode, ano->node_stat.st_mode, path); + if (node_stat.st_uid != ano->node_stat.st_uid) + LOG_ERROR_RETURN(-1, false, "uid ` not equal to ` (`)", node_stat.st_uid, ano->node_stat.st_uid, path); + if (node_stat.st_gid != ano->node_stat.st_gid) + LOG_ERROR_RETURN(-1, false, "gid ` not equal to ` (`)", node_stat.st_gid, ano->node_stat.st_gid, path); + /* + * We do not compare the directory's nlink because + * different directories are used when creating the layer. + */ + if (type != NODE_DIR && node_stat.st_nlink != ano->node_stat.st_nlink) + LOG_ERROR_RETURN(-1, false, "nlink ` not equal to ` (`)", node_stat.st_nlink, ano->node_stat.st_nlink, path); + /* + * The host file system (e.g., ext4) has a different + * directory organization than EROFS, so the directory's + * `st_size` is not compared. + */ + if (type != NODE_DIR && node_stat.st_size != ano->node_stat.st_size) + LOG_ERROR_RETURN(-1, false, "file size ` not equal to ` (`)", node_stat.st_size, ano->node_stat.st_size, path); return true; } }; @@ -128,17 +145,18 @@ class StressGenInter { virtual bool build_gen_own(StressNode *node /* out */, StressHostFile *file_info /* out */) = 0; virtual bool build_gen_xattrs(StressNode *node /* out */, StressHostFile *file_info /* out */) = 0; virtual bool build_gen_content(StressNode *node /* out */, StressHostFile *file_info /* out */) = 0; + virtual bool build_stat_file(StressNode *node /* out */, StressHostFile *file_info /* out */) = 0; /* for a single dir (node) */ virtual bool build_dir_mod(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) = 0; virtual bool build_dir_own(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) = 0; virtual bool build_dir_xattrs(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) = 0; + virtual bool build_stat_dir(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) = 0; /* generate in-mem inode according to erofs-fs file (for both files and dirs) */ - virtual bool verify_gen_mod(StressNode *node /* out */, photon::fs::IFile *erofs_file /* in */) = 0; - virtual bool verify_gen_own(StressNode *node /* out */, photon::fs::IFile *erofs_file /* in */) = 0; virtual bool verify_gen_xattrs(StressNode *node /* out */, photon::fs::IFile *erofs_file /* in */) = 0; virtual bool verify_gen_content(StressNode *node /* out */, photon::fs::IFile *erofs_file /* in */) = 0; + virtual bool verify_stat(StressNode *node /* out */, photon::fs::IFile *erofs_file /* in */) = 0; /* * construct the structure of a layer, such as how many dirs, From f4969c16a8ae419a52a1f6dbd82b2adf9a34f87f Mon Sep 17 00:00:00 2001 From: Hongzhen Luo Date: Sat, 21 Dec 2024 17:00:10 +0800 Subject: [PATCH 16/17] [EROFS] test: change the way of creating layers The previous creation of tar files for each layer requires root privileges, and this patch changes that. This patch creates each layer's tar file in a "append" way. Signed-off-by: Hongzhen Luo --- src/overlaybd/tar/erofs/test/erofs_stress.cpp | 175 ++++++++++-------- .../tar/erofs/test/erofs_stress_base.cpp | 56 +++++- .../tar/erofs/test/erofs_stress_base.h | 13 +- 3 files changed, 151 insertions(+), 93 deletions(-) diff --git a/src/overlaybd/tar/erofs/test/erofs_stress.cpp b/src/overlaybd/tar/erofs/test/erofs_stress.cpp index 57a6456c..114b2107 100644 --- a/src/overlaybd/tar/erofs/test/erofs_stress.cpp +++ b/src/overlaybd/tar/erofs/test/erofs_stress.cpp @@ -141,7 +141,8 @@ class StressInterImpl: public StressGenInter { mode_t mode; for (int i = 0; i < 3; i ++) { - str_mode += std::to_string(get_randomint(0, 7)); + /* ensure that the tester can read/write the file */ + str_mode += std::to_string(get_randomint(0, 7) | (i == 0 ? 6 : 0)); } mode = std::stoi(str_mode, nullptr, 8); if (file->file->fchmod(mode)) @@ -150,12 +151,12 @@ class StressInterImpl: public StressGenInter { } /* own in build phase */ - bool build_gen_own(StressNode *node, StressHostFile *file) override { + bool build_gen_own(StressNode *node, struct in_mem_meta *meta) override { uid_t uid = get_randomint(own_id_min, own_id_max); gid_t gid = get_randomint(own_id_min, own_id_max); - if (file->file->fchown(uid, gid)) - LOG_ERROR_RETURN(-1,false, "failt to chown of file `", file->path); + meta->uid = uid; + meta->gid = gid; return true; } @@ -188,12 +189,12 @@ class StressInterImpl: public StressGenInter { return true; } - bool build_dir_own(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { + bool build_dir_own(StressNode *node, struct in_mem_meta *meta) override { uid_t uid = get_randomint(own_id_min, own_id_max); gid_t gid = get_randomint(own_id_min, own_id_max); - if (host_fs->chown(path, uid, gid)) - LOG_ERROR_RETURN(-1,false, "failt to chown of dir `", path); + meta->uid = uid; + meta->gid = gid; return true; } @@ -213,15 +214,20 @@ class StressInterImpl: public StressGenInter { return true; } - bool build_stat_file(StressNode *node, StressHostFile *file_info) override { + bool build_stat_file(StressNode *node, StressHostFile *file_info, struct in_mem_meta *meta) override { if (file_info->file->fstat(&node->node_stat)) LOG_ERRNO_RETURN(-1, false, "fail to stat file `", file_info->path); + node->node_stat.st_uid = meta->uid; + node->node_stat.st_gid = meta->gid; + return true; } - bool build_stat_dir(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { + bool build_stat_dir(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs, struct in_mem_meta *meta) override { if (host_fs->stat(path, &node->node_stat)) LOG_ERROR_RETURN(-1, false, "fail to stat dir `", path); + node->node_stat.st_uid = meta->uid; + node->node_stat.st_gid = meta->gid; return true; } @@ -246,24 +252,26 @@ class StressCase001: public StressBase, public StressInterImpl { /* create empty files in build phase*/ EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, build_gen_mod(StressNode *node, StressHostFile *file), true) - EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, build_gen_own(StressNode *node, StressHostFile *file), true) EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, build_gen_xattrs(StressNode *node, StressHostFile *file), true) EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, build_gen_content(StressNode *node, StressHostFile *file), true) - bool build_stat_file(StressNode *node, StressHostFile *file) override { - return StressInterImpl::build_stat_file(node, file); + bool build_gen_own(StressNode *node, struct in_mem_meta *meta) override { + return StressInterImpl::build_gen_own(node, meta); + } + bool build_stat_file(StressNode *node, StressHostFile *file, struct in_mem_meta *meta) override { + return StressInterImpl::build_stat_file(node, file, meta); } bool build_dir_mod(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { return StressInterImpl::build_dir_mod(node, path, host_fs); } - bool build_dir_own(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { - return StressInterImpl::build_dir_own(node, path, host_fs); + bool build_dir_own(StressNode *node, struct in_mem_meta *meta) override { + return StressInterImpl::build_dir_own(node, meta); } bool build_dir_xattrs(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { return StressInterImpl::build_dir_xattrs(node, path, host_fs); } - bool build_stat_dir(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { - return StressInterImpl::build_stat_dir(node, path, host_fs); + bool build_stat_dir(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs, struct in_mem_meta *meta) override { + return StressInterImpl::build_stat_dir(node, path, host_fs, meta); } bool verify_gen_xattrs(StressNode *node, photon::fs::IFile *erofs_file) override { @@ -308,26 +316,28 @@ class StressCase002: public StressBase, public StressInterImpl { /* leave mod/own/xattr empty */ EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, build_gen_mod(StressNode *node, StressHostFile *file), true) - EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, build_gen_own(StressNode *node, StressHostFile *file), true) EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, build_gen_xattrs(StressNode *node, StressHostFile *file), true) + bool build_gen_own(StressNode *node, struct in_mem_meta *meta) override { + return StressInterImpl::build_gen_own(node, meta); + } bool build_gen_content(StressNode *node, StressHostFile *file) override { return StressInterImpl::build_gen_content(node, file); } - bool build_stat_file(StressNode *node, StressHostFile *file) override { - return StressInterImpl::build_stat_file(node, file); + bool build_stat_file(StressNode *node, StressHostFile *file, struct in_mem_meta *meta) override { + return StressInterImpl::build_stat_file(node, file, meta); } bool build_dir_mod(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { return StressInterImpl::build_dir_mod(node, path, host_fs); } - bool build_dir_own(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { - return StressInterImpl::build_dir_own(node, path, host_fs); + bool build_dir_own(StressNode *node, struct in_mem_meta *meta) override { + return StressInterImpl::build_dir_own(node, meta); } bool build_dir_xattrs(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { return StressInterImpl::build_dir_xattrs(node, path, host_fs); } - bool build_stat_dir(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { - return StressInterImpl::build_stat_dir(node, path, host_fs); + bool build_stat_dir(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs, struct in_mem_meta *meta) override { + return StressInterImpl::build_stat_dir(node, path, host_fs, meta); } bool verify_gen_xattrs(StressNode *node, photon::fs::IFile *erofs_file) override { @@ -358,6 +368,7 @@ class StressCase002: public StressBase, public StressInterImpl { return ret; } }; + /* * TC003 * @@ -370,28 +381,31 @@ class StressCase003: public StressBase, public StressInterImpl { public: StressCase003(std::string path, int layers): StressBase(path, layers) {} + /* leave mod/own/content empty */ EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, build_gen_mod(StressNode *node, StressHostFile *file), true) - EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, build_gen_own(StressNode *node, StressHostFile *file), true) EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, build_gen_content(StressNode *node, StressHostFile *file), true) + bool build_gen_own(StressNode *node, struct in_mem_meta *meta) override { + return StressInterImpl::build_gen_own(node, meta); + } bool build_gen_xattrs(StressNode *node, StressHostFile *file) override { return StressInterImpl::build_gen_xattrs(node, file); } - bool build_stat_file(StressNode *node, StressHostFile *file) override { - return StressInterImpl::build_stat_file(node, file); + bool build_stat_file(StressNode *node, StressHostFile *file, struct in_mem_meta *meta) override { + return StressInterImpl::build_stat_file(node, file, meta); } bool build_dir_mod(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { return StressInterImpl::build_dir_mod(node, path, host_fs); } - bool build_dir_own(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { - return StressInterImpl::build_dir_own(node, path, host_fs); + bool build_dir_own(StressNode *node, struct in_mem_meta *meta) override { + return StressInterImpl::build_dir_own(node, meta); } bool build_dir_xattrs(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { return StressInterImpl::build_dir_xattrs(node, path, host_fs); } - bool build_stat_dir(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { - return StressInterImpl::build_stat_dir(node, path, host_fs); + bool build_stat_dir(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs, struct in_mem_meta *meta) override { + return StressInterImpl::build_stat_dir(node, path, host_fs, meta); } bool verify_gen_xattrs(StressNode *node, photon::fs::IFile *erofs_file) override { @@ -430,27 +444,29 @@ class StressCase004: public StressBase, public StressInterImpl { public: StressCase004(std::string path, int layers): StressBase(path, layers) {} - EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, build_gen_own(StressNode *node, StressHostFile *file), true) EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, build_gen_xattrs(StressNode *node, StressHostFile *file), true) EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, build_gen_content(StressNode *node, StressHostFile *file), true) + bool build_gen_own(StressNode *node, struct in_mem_meta *meta) override { + return StressInterImpl::build_gen_own(node, meta); + } bool build_gen_mod(StressNode *node, StressHostFile *file) override { return StressInterImpl::build_gen_mod(node, file); } - bool build_stat_file(StressNode *node, StressHostFile *file) override { - return StressInterImpl::build_stat_file(node, file); + bool build_stat_file(StressNode *node, StressHostFile *file, struct in_mem_meta *meta) override { + return StressInterImpl::build_stat_file(node, file, meta); } bool build_dir_mod(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { return StressInterImpl::build_dir_mod(node, path, host_fs); } - bool build_dir_own(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { - return StressInterImpl::build_dir_own(node, path, host_fs); + bool build_dir_own(StressNode *node, struct in_mem_meta *meta) override { + return StressInterImpl::build_dir_own(node, meta); } bool build_dir_xattrs(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { return StressInterImpl::build_dir_xattrs(node, path, host_fs); } - bool build_stat_dir(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { - return StressInterImpl::build_stat_dir(node, path, host_fs); + bool build_stat_dir(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs, struct in_mem_meta *meta) override { + return StressInterImpl::build_stat_dir(node, path, host_fs, meta); } EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, verify_gen_content(StressNode *node, photon::fs::IFile *erofs_file), true) @@ -491,24 +507,24 @@ class StressCase005: public StressBase, public StressInterImpl { EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, build_gen_mod(StressNode *node, StressHostFile *file), true) EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, build_gen_xattrs(StressNode *node, StressHostFile *file), true) EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, build_gen_content(StressNode *node, StressHostFile *file), true) - bool build_gen_own(StressNode *node, StressHostFile *file) override { - return StressInterImpl::build_gen_own(node, file); + bool build_gen_own(StressNode *node, struct in_mem_meta *meta) override { + return StressInterImpl::build_gen_own(node, meta); } - bool build_stat_file(StressNode *node, StressHostFile *file) override { - return StressInterImpl::build_stat_file(node, file); + bool build_stat_file(StressNode *node, StressHostFile *file, struct in_mem_meta *meta) override { + return StressInterImpl::build_stat_file(node, file, meta); } bool build_dir_mod(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { return StressInterImpl::build_dir_mod(node, path, host_fs); } - bool build_dir_own(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { - return StressInterImpl::build_dir_own(node, path, host_fs); + bool build_dir_own(StressNode *node, struct in_mem_meta *meta) override { + return StressInterImpl::build_dir_own(node, meta); } bool build_dir_xattrs(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { return StressInterImpl::build_dir_xattrs(node, path, host_fs); } - bool build_stat_dir(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { - return StressInterImpl::build_stat_dir(node, path, host_fs); + bool build_stat_dir(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs, struct in_mem_meta *meta) override { + return StressInterImpl::build_stat_dir(node, path, host_fs, meta); } bool verify_gen_xattrs(StressNode *node, photon::fs::IFile *erofs_file) override { @@ -549,8 +565,8 @@ class StressCase006: public StressBase, public StressInterImpl { bool build_gen_mod(StressNode *node, StressHostFile *file) override { return StressInterImpl::build_gen_mod(node, file); } - bool build_gen_own(StressNode *node, StressHostFile *file) override { - return StressInterImpl::build_gen_own(node, file); + bool build_gen_own(StressNode *node, struct in_mem_meta *meta) override { + return StressInterImpl::build_gen_own(node, meta); } bool build_gen_xattrs(StressNode *node, StressHostFile *file) override { return StressInterImpl::build_gen_xattrs(node, file); @@ -558,21 +574,21 @@ class StressCase006: public StressBase, public StressInterImpl { bool build_gen_content(StressNode *node, StressHostFile *file) override { return StressInterImpl::build_gen_content(node, file); } - bool build_stat_file(StressNode *node, StressHostFile *file) override { - return StressInterImpl::build_stat_file(node, file); + bool build_stat_file(StressNode *node, StressHostFile *file, struct in_mem_meta *meta) override { + return StressInterImpl::build_stat_file(node, file, meta); } bool build_dir_mod(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { return StressInterImpl::build_dir_mod(node, path, host_fs); } - bool build_dir_own(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { - return StressInterImpl::build_dir_own(node, path, host_fs); + bool build_dir_own(StressNode *node, struct in_mem_meta *meta) override { + return StressInterImpl::build_dir_own(node, meta); } bool build_dir_xattrs(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { return StressInterImpl::build_dir_xattrs(node, path, host_fs); } - bool build_stat_dir(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { - return StressInterImpl::build_stat_dir(node, path, host_fs); + bool build_stat_dir(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs, struct in_mem_meta *meta) override { + return StressInterImpl::build_stat_dir(node, path, host_fs, meta); } bool verify_gen_xattrs(StressNode *node, photon::fs::IFile *erofs_file) override { @@ -617,8 +633,8 @@ class StressCase007: public StressBase, public StressInterImpl { bool build_gen_mod(StressNode *node, StressHostFile *file) override { return StressInterImpl::build_gen_mod(node, file); } - bool build_gen_own(StressNode *node, StressHostFile *file) override { - return StressInterImpl::build_gen_own(node, file); + bool build_gen_own(StressNode *node, struct in_mem_meta *meta) override { + return StressInterImpl::build_gen_own(node, meta); } bool build_gen_xattrs(StressNode *node, StressHostFile *file) override { return StressInterImpl::build_gen_xattrs(node, file); @@ -626,21 +642,21 @@ class StressCase007: public StressBase, public StressInterImpl { bool build_gen_content(StressNode *node, StressHostFile *file) override { return StressInterImpl::build_gen_content(node, file); } - bool build_stat_file(StressNode *node, StressHostFile *file) override { - return StressInterImpl::build_stat_file(node, file); + bool build_stat_file(StressNode *node, StressHostFile *file, struct in_mem_meta *meta) override { + return StressInterImpl::build_stat_file(node, file, meta); } bool build_dir_mod(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { return StressInterImpl::build_dir_mod(node, path, host_fs); } - bool build_dir_own(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { - return StressInterImpl::build_dir_own(node, path, host_fs); + bool build_dir_own(StressNode *node, struct in_mem_meta *meta) override { + return StressInterImpl::build_dir_own(node, meta); } bool build_dir_xattrs(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { return StressInterImpl::build_dir_xattrs(node, path, host_fs); } - bool build_stat_dir(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { - return StressInterImpl::build_stat_dir(node, path, host_fs); + bool build_stat_dir(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs, struct in_mem_meta *meta) override { + return StressInterImpl::build_stat_dir(node, path, host_fs, meta); } bool verify_gen_xattrs(StressNode *node, photon::fs::IFile *erofs_file) override { @@ -705,8 +721,8 @@ class StressCase008: public StressBase, public StressInterImpl { bool build_gen_mod(StressNode *node, StressHostFile *file) override { return StressInterImpl::build_gen_mod(node, file); } - bool build_gen_own(StressNode *node, StressHostFile *file) override { - return StressInterImpl::build_gen_own(node, file); + bool build_gen_own(StressNode *node, struct in_mem_meta *meta) override { + return StressInterImpl::build_gen_own(node, meta); } bool build_gen_xattrs(StressNode *node, StressHostFile *file) override { return StressInterImpl::build_gen_xattrs(node, file); @@ -714,21 +730,21 @@ class StressCase008: public StressBase, public StressInterImpl { bool build_gen_content(StressNode *node, StressHostFile *file) override { return StressInterImpl::build_gen_content(node, file); } - bool build_stat_file(StressNode *node, StressHostFile *file) override { - return StressInterImpl::build_stat_file(node, file); + bool build_stat_file(StressNode *node, StressHostFile *file, struct in_mem_meta *meta) override { + return StressInterImpl::build_stat_file(node, file, meta); } bool build_dir_mod(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { return StressInterImpl::build_dir_mod(node, path, host_fs); } - bool build_dir_own(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { - return StressInterImpl::build_dir_own(node, path, host_fs); + bool build_dir_own(StressNode *node, struct in_mem_meta *meta) override { + return StressInterImpl::build_dir_own(node, meta); } bool build_dir_xattrs(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { return StressInterImpl::build_dir_xattrs(node, path, host_fs); } - bool build_stat_dir(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { - return StressInterImpl::build_stat_dir(node, path, host_fs); + bool build_stat_dir(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs, struct in_mem_meta *meta) override { + return StressInterImpl::build_stat_dir(node, path, host_fs, meta); } bool verify_gen_xattrs(StressNode *node, photon::fs::IFile *erofs_file) override { @@ -802,8 +818,8 @@ class StressCase009: public StressBase, public StressInterImpl { bool build_gen_mod(StressNode *node, StressHostFile *file) override { return StressInterImpl::build_gen_mod(node, file); } - bool build_gen_own(StressNode *node, StressHostFile *file) override { - return StressInterImpl::build_gen_own(node, file); + bool build_gen_own(StressNode *node, struct in_mem_meta *meta) override { + return StressInterImpl::build_gen_own(node, meta); } bool build_gen_xattrs(StressNode *node, StressHostFile *file) override { return StressInterImpl::build_gen_xattrs(node, file); @@ -811,21 +827,21 @@ class StressCase009: public StressBase, public StressInterImpl { bool build_gen_content(StressNode *node, StressHostFile *file) override { return StressInterImpl::build_gen_content(node, file); } - bool build_stat_file(StressNode *node, StressHostFile *file) override { - return StressInterImpl::build_stat_file(node, file); + bool build_stat_file(StressNode *node, StressHostFile *file, struct in_mem_meta *meta) override { + return StressInterImpl::build_stat_file(node, file, meta); } bool build_dir_mod(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { return StressInterImpl::build_dir_mod(node, path, host_fs); } - bool build_dir_own(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { - return StressInterImpl::build_dir_own(node, path, host_fs); + bool build_dir_own(StressNode *node, struct in_mem_meta *meta) override { + return StressInterImpl::build_dir_own(node, meta); } bool build_dir_xattrs(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { return StressInterImpl::build_dir_xattrs(node, path, host_fs); } - bool build_stat_dir(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { - return StressInterImpl::build_stat_dir(node, path, host_fs); + bool build_stat_dir(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs, struct in_mem_meta *meta) override { + return StressInterImpl::build_stat_dir(node, path, host_fs, meta); } bool verify_gen_xattrs(StressNode *node, photon::fs::IFile *erofs_file) override { @@ -931,6 +947,7 @@ TEST(ErofsStressTest, TC003) { ASSERT_EQ(tc003->run(), true); delete tc003; } + TEST(ErofsStressTest, TC004) { std::srand(static_cast(std::time(0))); StressCase004 *tc004 = new StressCase004("./erofs_stress_004", 10); @@ -967,8 +984,8 @@ TEST(ErofsStressTest, TC007) { TEST(ErofsStressTest, TC008) { std::srand(static_cast(std::time(0))); - /* 100 layers */ - StressCase008 *tc008 = new StressCase008("./erofs_stress_008", 100); + /* 30 layers */ + StressCase008 *tc008 = new StressCase008("./erofs_stress_008", 30); ASSERT_EQ(tc008->run(), true); delete tc008; diff --git a/src/overlaybd/tar/erofs/test/erofs_stress_base.cpp b/src/overlaybd/tar/erofs/test/erofs_stress_base.cpp index 81fc4a86..3a7c7d52 100644 --- a/src/overlaybd/tar/erofs/test/erofs_stress_base.cpp +++ b/src/overlaybd/tar/erofs/test/erofs_stress_base.cpp @@ -153,6 +153,23 @@ static LayerNode *build_layer_tree(std::vector &dirs) { return nodes[0]; } +static bool append_tar(bool first, std::string tar_name, std::string tmp_tar, std::string prefix, std::string file_name, struct in_mem_meta *meta) +{ + std::string cmd = std::string("tar --create --file=") + (first ? tar_name : tmp_tar) + " --xattrs --xattrs-include='*'"; + if (meta) + cmd = cmd + " --owner=" + std::to_string(meta->uid) + " --group=" + std::to_string(meta->gid); + cmd = cmd + " -C " + prefix + " " + file_name; + + if (system(cmd.c_str())) + LOG_ERROR_RETURN(-1, false, "fail to create tar file for `, cmd: `", prefix + "/" + file_name, cmd); + if (!first) { + cmd = std::string("tar --concatenate --file=") + tar_name + " " + tmp_tar; + if (system(cmd.c_str())) + LOG_ERROR_RETURN(-1, false, "fail to concatenate ` to `, cmd: `", tmp_tar, tar_name, cmd); + } + return true; +} + bool StressBase::create_layer(int idx) { #define MAX_TRY_TIME 10 @@ -170,6 +187,7 @@ bool StressBase::create_layer(int idx) { std::string root_path = prefix + "/" + root_dirname; std::string clean_cmd = "rm -rf " + root_path; bool res; + std::map meta_maps; if (system(clean_cmd.c_str())) LOG_ERROR_RETURN(-1, false, "fail to prepare clean dir for `", root_path); @@ -177,19 +195,26 @@ bool StressBase::create_layer(int idx) { layer_tree->pwd = root_path; q.emplace_back(layer_tree); + std::string layer_name = origin_prefix + "/layer" + std::to_string(idx) + ".tar"; + std::string tmp_tar = origin_prefix + "/layer" + std::to_string(idx) + "_tmp.tar"; + StressNode *node = new StressNode(layer_tree->pwd.substr(prefix.length()), NODE_DIR); if (host_fs->mkdir(layer_tree->pwd.c_str(), 0755) != 0) LOG_ERROR_RETURN(-1, false, "fail to mkdir `", layer_tree->pwd); + meta_maps[layer_tree->pwd] = new struct in_mem_meta(); res = build_dir_mod(node, layer_tree->pwd.c_str(), host_fs) && - build_dir_own(node, layer_tree->pwd.c_str(), host_fs) && + build_dir_own(node, meta_maps[layer_tree->pwd]) && build_dir_xattrs(node, layer_tree->pwd.c_str(), host_fs) && - build_stat_dir(node, layer_tree->pwd.c_str(), host_fs); + build_stat_dir(node, layer_tree->pwd.c_str(), host_fs, meta_maps[layer_tree->pwd]); if (!res) LOG_ERROR_RETURN(-1, false, "fail to generate fields for dir `",layer_tree->pwd); if (!tree->add_node(node)) LOG_ERROR_RETURN(-1, false, "fail to add node `",layer_tree->pwd); + if (!append_tar(true, layer_name, tmp_tar, prefix, root_dirname, meta_maps[layer_tree->pwd])) + LOG_ERROR_RETURN(-1, false, "fail to crate tar for `", layer_tree->pwd); + // traverse the layer tree while (q.size()) { LayerNode *cur = q.front(); @@ -215,22 +240,27 @@ bool StressBase::create_layer(int idx) { LOG_ERROR_RETURN(-1, false, "fail to add WHITEOUT file `", filename); file_info->file->fsync(); delete file_info; + if (!append_tar(false, layer_name, tmp_tar, prefix, host_filename.substr(1), nullptr)) + LOG_ERROR_RETURN(-1, false, "fail to create tar for whiteout file: `", (prefix + host_filename)); } else { filename = name_prefix + "/" + filename; StressNode *node = new StressNode(filename, NODE_REGULAR); StressHostFile *file_info = new StressHostFile(prefix + filename, host_fs); + meta_maps[prefix + filename] = new struct in_mem_meta(); res = build_gen_mod(node, file_info) && - build_gen_own(node, file_info) && + build_gen_own(node, meta_maps[prefix + filename]) && build_gen_xattrs(node, file_info) && build_gen_content(node, file_info) && - build_stat_file(node, file_info); + build_stat_file(node, file_info, meta_maps[prefix + filename]); if (!res) LOG_ERROR_RETURN(-1, false, "fail to generate file contents"); if (!tree->add_node(node)) LOG_ERROR_RETURN(-1, false, "failt to add node `", filename); file_info->file->fsync(); delete file_info; + if (!append_tar(false, layer_name, tmp_tar, prefix, filename.substr(1), meta_maps[prefix + filename])) + LOG_ERROR_RETURN(-1, false, "fail to create tar for file `", (prefix + filename)); } } @@ -256,21 +286,26 @@ bool StressBase::create_layer(int idx) { LOG_ERROR_RETURN(-1, false, "fail to crate whiout dir in host fs: `", host_filename); file_info->file->fsync(); delete file_info; + if (!append_tar(false, layer_name, tmp_tar, prefix, host_filename.substr(prefix.length() + 1), nullptr)) + LOG_ERROR_RETURN(-1, false, "fail to create whitout dir for `", host_filename); break; } else { next->pwd = cur->pwd + "/" + dir_name; if (host_fs->mkdir(next->pwd.c_str(), 0755) == 0) { StressNode *dir_node = new StressNode(next->pwd.substr(prefix.length()), NODE_DIR); + meta_maps[next->pwd] = new struct in_mem_meta(); res = build_dir_mod(dir_node, next->pwd.c_str(), host_fs) && - build_dir_own(dir_node, next->pwd.c_str(), host_fs) && + build_dir_own(dir_node, meta_maps[next->pwd]) && build_dir_xattrs(dir_node, next->pwd.c_str(), host_fs) && - build_stat_dir(dir_node, next->pwd.c_str(), host_fs); + build_stat_dir(dir_node, next->pwd.c_str(), host_fs, meta_maps[next->pwd]); if (!res) LOG_ERROR_RETURN(-1, false, "fail to generate fields for dir `", next->pwd); if (!tree->add_node(dir_node)) LOG_ERROR_RETURN(-1, false, "fail to add node `", next->pwd); q.emplace_back(next); + if (!append_tar(false, layer_name, tmp_tar, prefix, next->pwd.substr(prefix.length() + 1), meta_maps[next->pwd])) + LOG_ERROR_RETURN(-1, false, "fail to create tar for dir `", next->pwd); break; } } @@ -281,10 +316,11 @@ bool StressBase::create_layer(int idx) { #undef MAX_TRY_TIME - std::string layer_name = origin_prefix + "/layer" + std::to_string(idx); - std::string cmd = std::string(" sudo tar --xattrs --xattrs-include='*' -cf ") + layer_name + ".tar -C " + prefix + " " + root_dirname; - if (system(cmd.c_str())) - LOG_ERROR_RETURN(-1, false, "fail to prepare tar file, cmd: `", cmd); + for (auto it = meta_maps.begin(); it != meta_maps.end(); ) { + delete it->second; + it = meta_maps.erase(it); + } + prefix = origin_prefix; return true; } diff --git a/src/overlaybd/tar/erofs/test/erofs_stress_base.h b/src/overlaybd/tar/erofs/test/erofs_stress_base.h index 6fb5d724..ffe1d2a7 100644 --- a/src/overlaybd/tar/erofs/test/erofs_stress_base.h +++ b/src/overlaybd/tar/erofs/test/erofs_stress_base.h @@ -136,22 +136,27 @@ class StressHostFile { } }; +struct in_mem_meta{ + uid_t uid; + gid_t gid; +}; + /* interface to generate corresponding values for in-mem nodes and host-fs files */ class StressGenInter { public: /* for a single file (node) */ /* generate content for in-memory inodes and host files (prepare layers phase) */ virtual bool build_gen_mod(StressNode *node /* out */, StressHostFile *file_info /* out */) = 0; - virtual bool build_gen_own(StressNode *node /* out */, StressHostFile *file_info /* out */) = 0; + virtual bool build_gen_own(StressNode *node /* out */, struct in_mem_meta *meta /* out */) = 0; virtual bool build_gen_xattrs(StressNode *node /* out */, StressHostFile *file_info /* out */) = 0; virtual bool build_gen_content(StressNode *node /* out */, StressHostFile *file_info /* out */) = 0; - virtual bool build_stat_file(StressNode *node /* out */, StressHostFile *file_info /* out */) = 0; + virtual bool build_stat_file(StressNode *node /* out */, StressHostFile *file_info /* out */, struct in_mem_meta *meta) = 0; /* for a single dir (node) */ virtual bool build_dir_mod(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) = 0; - virtual bool build_dir_own(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) = 0; + virtual bool build_dir_own(StressNode *node, struct in_mem_meta *meta) = 0; virtual bool build_dir_xattrs(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) = 0; - virtual bool build_stat_dir(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) = 0; + virtual bool build_stat_dir(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs, struct in_mem_meta *meta) = 0; /* generate in-mem inode according to erofs-fs file (for both files and dirs) */ virtual bool verify_gen_xattrs(StressNode *node /* out */, photon::fs::IFile *erofs_file /* in */) = 0; From 115be8dc520465133512d9ba8125b94b01a86f6c Mon Sep 17 00:00:00 2001 From: Hongzhen Luo Date: Sat, 21 Dec 2024 22:56:04 +0800 Subject: [PATCH 17/17] [EROFS] test: add test for mtime Signed-off-by: Hongzhen Luo --- src/overlaybd/tar/erofs/test/erofs_stress.cpp | 76 ++++++++++++++++++- .../tar/erofs/test/erofs_stress_base.cpp | 5 +- .../tar/erofs/test/erofs_stress_base.h | 9 +++ 3 files changed, 87 insertions(+), 3 deletions(-) diff --git a/src/overlaybd/tar/erofs/test/erofs_stress.cpp b/src/overlaybd/tar/erofs/test/erofs_stress.cpp index 114b2107..db7fea42 100644 --- a/src/overlaybd/tar/erofs/test/erofs_stress.cpp +++ b/src/overlaybd/tar/erofs/test/erofs_stress.cpp @@ -160,6 +160,18 @@ class StressInterImpl: public StressGenInter { return true; } + bool build_gen_mtime(StressNode *node, struct in_mem_meta *meta) override { + size_t now = time(nullptr); + time_t time_sec = get_randomint(now, now + 24 * 60 * 60); + struct tm *time_info = localtime(&time_sec); + char buffer[256]; + strftime(buffer, sizeof(buffer), "%Y-%m-%d %H:%M:%S", time_info); + std::ostringstream oss; + oss << buffer; + meta->mtime_date = oss.str(); + meta->mtime = time_sec; + return true; + } /* generate a random dir or file name in the current layer */ std::string generate_name(int idx, int depth, std::string root_path, NODE_TYPE type) override { std::string res; @@ -198,6 +210,10 @@ class StressInterImpl: public StressGenInter { return true; } + bool build_dir_mtime(StressNode *node, struct in_mem_meta *meta) override { + return build_gen_mtime(node, meta); + } + bool build_dir_xattrs(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { photon::fs::IFileSystemXAttr *xattr_ops = dynamic_cast(host_fs); if (xattr_ops == nullptr) @@ -219,6 +235,7 @@ class StressInterImpl: public StressGenInter { LOG_ERRNO_RETURN(-1, false, "fail to stat file `", file_info->path); node->node_stat.st_uid = meta->uid; node->node_stat.st_gid = meta->gid; + node->node_stat.st_mtime = meta->mtime; return true; } @@ -228,6 +245,7 @@ class StressInterImpl: public StressGenInter { LOG_ERROR_RETURN(-1, false, "fail to stat dir `", path); node->node_stat.st_uid = meta->uid; node->node_stat.st_gid = meta->gid; + node->node_stat.st_mtime = meta->mtime; return true; } @@ -257,6 +275,9 @@ class StressCase001: public StressBase, public StressInterImpl { bool build_gen_own(StressNode *node, struct in_mem_meta *meta) override { return StressInterImpl::build_gen_own(node, meta); } + bool build_gen_mtime(StressNode *node, struct in_mem_meta *meta) override { + return StressInterImpl::build_gen_mtime(node, meta); + } bool build_stat_file(StressNode *node, StressHostFile *file, struct in_mem_meta *meta) override { return StressInterImpl::build_stat_file(node, file, meta); } @@ -267,6 +288,9 @@ class StressCase001: public StressBase, public StressInterImpl { bool build_dir_own(StressNode *node, struct in_mem_meta *meta) override { return StressInterImpl::build_dir_own(node, meta); } + bool build_dir_mtime(StressNode *node, struct in_mem_meta *meta) override { + return StressInterImpl::build_dir_mtime(node, meta); + } bool build_dir_xattrs(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { return StressInterImpl::build_dir_xattrs(node, path, host_fs); } @@ -296,8 +320,8 @@ class StressCase001: public StressBase, public StressInterImpl { std::vector layer_dirs(int idx) { std::vector ret; - ret.emplace_back(50); - ret.emplace_back(50); + ret.emplace_back(3); + ret.emplace_back(3); return ret; } }; @@ -320,6 +344,9 @@ class StressCase002: public StressBase, public StressInterImpl { bool build_gen_own(StressNode *node, struct in_mem_meta *meta) override { return StressInterImpl::build_gen_own(node, meta); } + bool build_gen_mtime(StressNode *node, struct in_mem_meta *meta) override { + return StressInterImpl::build_gen_mtime(node, meta); + } bool build_gen_content(StressNode *node, StressHostFile *file) override { return StressInterImpl::build_gen_content(node, file); } @@ -333,6 +360,9 @@ class StressCase002: public StressBase, public StressInterImpl { bool build_dir_own(StressNode *node, struct in_mem_meta *meta) override { return StressInterImpl::build_dir_own(node, meta); } + bool build_dir_mtime(StressNode *node, struct in_mem_meta *meta) override { + return StressInterImpl::build_dir_mtime(node, meta); + } bool build_dir_xattrs(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { return StressInterImpl::build_dir_xattrs(node, path, host_fs); } @@ -388,6 +418,9 @@ class StressCase003: public StressBase, public StressInterImpl { bool build_gen_own(StressNode *node, struct in_mem_meta *meta) override { return StressInterImpl::build_gen_own(node, meta); } + bool build_gen_mtime(StressNode *node, struct in_mem_meta *meta) override { + return StressInterImpl::build_gen_mtime(node, meta); + } bool build_gen_xattrs(StressNode *node, StressHostFile *file) override { return StressInterImpl::build_gen_xattrs(node, file); } @@ -401,6 +434,9 @@ class StressCase003: public StressBase, public StressInterImpl { bool build_dir_own(StressNode *node, struct in_mem_meta *meta) override { return StressInterImpl::build_dir_own(node, meta); } + bool build_dir_mtime(StressNode *node, struct in_mem_meta *meta) override { + return StressInterImpl::build_dir_mtime(node, meta); + } bool build_dir_xattrs(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { return StressInterImpl::build_dir_xattrs(node, path, host_fs); } @@ -449,6 +485,9 @@ class StressCase004: public StressBase, public StressInterImpl { bool build_gen_own(StressNode *node, struct in_mem_meta *meta) override { return StressInterImpl::build_gen_own(node, meta); } + bool build_gen_mtime(StressNode *node, struct in_mem_meta *meta) override { + return StressInterImpl::build_gen_mtime(node, meta); + } bool build_gen_mod(StressNode *node, StressHostFile *file) override { return StressInterImpl::build_gen_mod(node, file); } @@ -462,6 +501,9 @@ class StressCase004: public StressBase, public StressInterImpl { bool build_dir_own(StressNode *node, struct in_mem_meta *meta) override { return StressInterImpl::build_dir_own(node, meta); } + bool build_dir_mtime(StressNode *node, struct in_mem_meta *meta) override { + return StressInterImpl::build_dir_mtime(node, meta); + } bool build_dir_xattrs(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { return StressInterImpl::build_dir_xattrs(node, path, host_fs); } @@ -510,6 +552,9 @@ class StressCase005: public StressBase, public StressInterImpl { bool build_gen_own(StressNode *node, struct in_mem_meta *meta) override { return StressInterImpl::build_gen_own(node, meta); } + bool build_gen_mtime(StressNode *node, struct in_mem_meta *meta) override { + return StressInterImpl::build_gen_mtime(node, meta); + } bool build_stat_file(StressNode *node, StressHostFile *file, struct in_mem_meta *meta) override { return StressInterImpl::build_stat_file(node, file, meta); } @@ -520,6 +565,9 @@ class StressCase005: public StressBase, public StressInterImpl { bool build_dir_own(StressNode *node, struct in_mem_meta *meta) override { return StressInterImpl::build_dir_own(node, meta); } + bool build_dir_mtime(StressNode *node, struct in_mem_meta *meta) override { + return StressInterImpl::build_dir_mtime(node, meta); + } bool build_dir_xattrs(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { return StressInterImpl::build_dir_xattrs(node, path, host_fs); } @@ -568,6 +616,9 @@ class StressCase006: public StressBase, public StressInterImpl { bool build_gen_own(StressNode *node, struct in_mem_meta *meta) override { return StressInterImpl::build_gen_own(node, meta); } + bool build_gen_mtime(StressNode *node, struct in_mem_meta *meta) override { + return StressInterImpl::build_gen_mtime(node, meta); + } bool build_gen_xattrs(StressNode *node, StressHostFile *file) override { return StressInterImpl::build_gen_xattrs(node, file); } @@ -584,6 +635,9 @@ class StressCase006: public StressBase, public StressInterImpl { bool build_dir_own(StressNode *node, struct in_mem_meta *meta) override { return StressInterImpl::build_dir_own(node, meta); } + bool build_dir_mtime(StressNode *node, struct in_mem_meta *meta) override { + return StressInterImpl::build_dir_mtime(node, meta); + } bool build_dir_xattrs(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { return StressInterImpl::build_dir_xattrs(node, path, host_fs); } @@ -636,6 +690,9 @@ class StressCase007: public StressBase, public StressInterImpl { bool build_gen_own(StressNode *node, struct in_mem_meta *meta) override { return StressInterImpl::build_gen_own(node, meta); } + bool build_gen_mtime(StressNode *node, struct in_mem_meta *meta) override { + return StressInterImpl::build_gen_mtime(node, meta); + } bool build_gen_xattrs(StressNode *node, StressHostFile *file) override { return StressInterImpl::build_gen_xattrs(node, file); } @@ -652,6 +709,9 @@ class StressCase007: public StressBase, public StressInterImpl { bool build_dir_own(StressNode *node, struct in_mem_meta *meta) override { return StressInterImpl::build_dir_own(node, meta); } + bool build_dir_mtime(StressNode *node, struct in_mem_meta *meta) override { + return StressInterImpl::build_dir_mtime(node, meta); + } bool build_dir_xattrs(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { return StressInterImpl::build_dir_xattrs(node, path, host_fs); } @@ -724,6 +784,9 @@ class StressCase008: public StressBase, public StressInterImpl { bool build_gen_own(StressNode *node, struct in_mem_meta *meta) override { return StressInterImpl::build_gen_own(node, meta); } + bool build_gen_mtime(StressNode *node, struct in_mem_meta *meta) override { + return StressInterImpl::build_gen_mtime(node, meta); + } bool build_gen_xattrs(StressNode *node, StressHostFile *file) override { return StressInterImpl::build_gen_xattrs(node, file); } @@ -740,6 +803,9 @@ class StressCase008: public StressBase, public StressInterImpl { bool build_dir_own(StressNode *node, struct in_mem_meta *meta) override { return StressInterImpl::build_dir_own(node, meta); } + bool build_dir_mtime(StressNode *node, struct in_mem_meta *meta) override { + return StressInterImpl::build_dir_mtime(node, meta); + } bool build_dir_xattrs(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { return StressInterImpl::build_dir_xattrs(node, path, host_fs); } @@ -821,6 +887,9 @@ class StressCase009: public StressBase, public StressInterImpl { bool build_gen_own(StressNode *node, struct in_mem_meta *meta) override { return StressInterImpl::build_gen_own(node, meta); } + bool build_gen_mtime(StressNode *node, struct in_mem_meta *meta) override { + return StressInterImpl::build_gen_mtime(node, meta); + } bool build_gen_xattrs(StressNode *node, StressHostFile *file) override { return StressInterImpl::build_gen_xattrs(node, file); } @@ -837,6 +906,9 @@ class StressCase009: public StressBase, public StressInterImpl { bool build_dir_own(StressNode *node, struct in_mem_meta *meta) override { return StressInterImpl::build_dir_own(node, meta); } + bool build_dir_mtime(StressNode *node, struct in_mem_meta *meta) override { + return StressInterImpl::build_dir_mtime(node, meta); + } bool build_dir_xattrs(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) override { return StressInterImpl::build_dir_xattrs(node, path, host_fs); } diff --git a/src/overlaybd/tar/erofs/test/erofs_stress_base.cpp b/src/overlaybd/tar/erofs/test/erofs_stress_base.cpp index 3a7c7d52..6be63c71 100644 --- a/src/overlaybd/tar/erofs/test/erofs_stress_base.cpp +++ b/src/overlaybd/tar/erofs/test/erofs_stress_base.cpp @@ -157,7 +157,7 @@ static bool append_tar(bool first, std::string tar_name, std::string tmp_tar, st { std::string cmd = std::string("tar --create --file=") + (first ? tar_name : tmp_tar) + " --xattrs --xattrs-include='*'"; if (meta) - cmd = cmd + " --owner=" + std::to_string(meta->uid) + " --group=" + std::to_string(meta->gid); + cmd = cmd + " --owner=" + std::to_string(meta->uid) + " --group=" + std::to_string(meta->gid) + " --mtime=\"" + meta->mtime_date + "\""; cmd = cmd + " -C " + prefix + " " + file_name; if (system(cmd.c_str())) @@ -205,6 +205,7 @@ bool StressBase::create_layer(int idx) { meta_maps[layer_tree->pwd] = new struct in_mem_meta(); res = build_dir_mod(node, layer_tree->pwd.c_str(), host_fs) && build_dir_own(node, meta_maps[layer_tree->pwd]) && + build_dir_mtime(node, meta_maps[layer_tree->pwd]) && build_dir_xattrs(node, layer_tree->pwd.c_str(), host_fs) && build_stat_dir(node, layer_tree->pwd.c_str(), host_fs, meta_maps[layer_tree->pwd]); if (!res) @@ -250,6 +251,7 @@ bool StressBase::create_layer(int idx) { meta_maps[prefix + filename] = new struct in_mem_meta(); res = build_gen_mod(node, file_info) && build_gen_own(node, meta_maps[prefix + filename]) && + build_gen_mtime(node, meta_maps[prefix + filename]) && build_gen_xattrs(node, file_info) && build_gen_content(node, file_info) && build_stat_file(node, file_info, meta_maps[prefix + filename]); @@ -297,6 +299,7 @@ bool StressBase::create_layer(int idx) { meta_maps[next->pwd] = new struct in_mem_meta(); res = build_dir_mod(dir_node, next->pwd.c_str(), host_fs) && build_dir_own(dir_node, meta_maps[next->pwd]) && + build_dir_mtime(dir_node, meta_maps[next->pwd]) && build_dir_xattrs(dir_node, next->pwd.c_str(), host_fs) && build_stat_dir(dir_node, next->pwd.c_str(), host_fs, meta_maps[next->pwd]); if (!res) diff --git a/src/overlaybd/tar/erofs/test/erofs_stress_base.h b/src/overlaybd/tar/erofs/test/erofs_stress_base.h index ffe1d2a7..980240c9 100644 --- a/src/overlaybd/tar/erofs/test/erofs_stress_base.h +++ b/src/overlaybd/tar/erofs/test/erofs_stress_base.h @@ -109,6 +109,11 @@ class StressNode { */ if (type != NODE_DIR && node_stat.st_size != ano->node_stat.st_size) LOG_ERROR_RETURN(-1, false, "file size ` not equal to ` (`)", node_stat.st_size, ano->node_stat.st_size, path); + + if (node_stat.st_mtime != ano->node_stat.st_mtime) + LOG_ERRNO_RETURN(-1, false, "mtime ` not equal to ` (`)", node_stat.st_mtime, + ano->node_stat.st_mtime, + path); return true; } }; @@ -139,6 +144,8 @@ class StressHostFile { struct in_mem_meta{ uid_t uid; gid_t gid; + std::string mtime_date; + time_t mtime; }; /* interface to generate corresponding values for in-mem nodes and host-fs files */ @@ -148,6 +155,7 @@ class StressGenInter { /* generate content for in-memory inodes and host files (prepare layers phase) */ virtual bool build_gen_mod(StressNode *node /* out */, StressHostFile *file_info /* out */) = 0; virtual bool build_gen_own(StressNode *node /* out */, struct in_mem_meta *meta /* out */) = 0; + virtual bool build_gen_mtime(StressNode *node, struct in_mem_meta *meta) = 0; virtual bool build_gen_xattrs(StressNode *node /* out */, StressHostFile *file_info /* out */) = 0; virtual bool build_gen_content(StressNode *node /* out */, StressHostFile *file_info /* out */) = 0; virtual bool build_stat_file(StressNode *node /* out */, StressHostFile *file_info /* out */, struct in_mem_meta *meta) = 0; @@ -155,6 +163,7 @@ class StressGenInter { /* for a single dir (node) */ virtual bool build_dir_mod(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) = 0; virtual bool build_dir_own(StressNode *node, struct in_mem_meta *meta) = 0; + virtual bool build_dir_mtime(StressNode *node, struct in_mem_meta *meta) = 0; virtual bool build_dir_xattrs(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs) = 0; virtual bool build_stat_dir(StressNode *node, const char *path, photon::fs::IFileSystem *host_fs, struct in_mem_meta *meta) = 0;