From 72867de6a168558fc08978780ed6c578a6429005 Mon Sep 17 00:00:00 2001 From: Yifan Yuan Date: Mon, 2 Sep 2024 21:46:41 +0800 Subject: [PATCH] [feat.] Support uploading 'output' to registry while merging layers Signed-off-by: Yifan Yuan --- src/overlaybd/lsmt/test/test.cpp | 17 +++++- src/overlaybd/registryfs/registryfs.h | 8 ++- src/overlaybd/registryfs/registryfs_v2.cpp | 22 +++++-- src/tools/CMakeLists.txt | 4 +- src/tools/comm_func.cpp | 21 +++++++ src/tools/comm_func.h | 4 ++ src/tools/overlaybd-commit.cpp | 68 +++++++++++----------- src/tools/overlaybd-merge.cpp | 54 ++++++++++++----- 8 files changed, 139 insertions(+), 59 deletions(-) diff --git a/src/overlaybd/lsmt/test/test.cpp b/src/overlaybd/lsmt/test/test.cpp index 1a107ce8..1197d12f 100644 --- a/src/overlaybd/lsmt/test/test.cpp +++ b/src/overlaybd/lsmt/test/test.cpp @@ -246,6 +246,12 @@ void test_combo(const IMemoryIndex *indexes[], size_t ni, const SegmentMapping s auto ret = memcmp(pm, stdrst, nrst * sizeof(stdrst[0])); EXPECT_EQ(ret, 0); } + LOG_INFO("make RO index of ci"); + auto ro_idx = ci.make_read_only_index(); + if (ro_idx->size() == nrst) { + auto ret = memcmp(ro_idx->buffer(), stdrst, nrst * sizeof(stdrst[0])); + EXPECT_EQ(ret, 0); + } ci.backing_index(); // delete mi; @@ -599,7 +605,10 @@ TEST_F(FileTest3, stack_files) { LOG_INFO("RO valid data: `", stat.valid_data_size); merged = lfs->open(fn_merged, O_RDWR | O_CREAT | O_TRUNC, S_IRWXU); EXPECT_EQ(lower->flatten(merged), 0); - cout << "verifying flattened layer of lowers" << endl; + // auto localfile = lfs->open(fn_merged, O_RDONLY); + struct stat st; + merged->fstat(&st); + cout << "verifying flattened layer of lowers, st_size: "<open(fn_merged, O_RDWR | O_CREAT | O_TRUNC, S_IRWXU); file->flatten(merged); + merged->fstat(&st); + cout << "verifying flattened layer of stacked layers, st_size: "<close(); + verify_file(fn_merged); delete file; + delete merged; } TEST_F(FileTest3, seek_data) { diff --git a/src/overlaybd/registryfs/registryfs.h b/src/overlaybd/registryfs/registryfs.h index 14a71e7c..ce90df01 100644 --- a/src/overlaybd/registryfs/registryfs.h +++ b/src/overlaybd/registryfs/registryfs.h @@ -43,10 +43,14 @@ photon::fs::IFileSystem *new_registryfs_v2(PasswordCB callback, const char *customized_ua = nullptr); photon::fs::IFile* new_registry_uploader(photon::fs::IFile *lfile, - std::string &upload_url, - std::string &username, std::string &password, + const std::string &upload_url, + const std::string &username, + const std::string &password, uint64_t timeout, ssize_t upload_bs = -1, const char *cert_file = nullptr, const char *key_file = nullptr); + +int registry_uploader_fini(photon::fs::IFile *uploader, std::string &digest); + } diff --git a/src/overlaybd/registryfs/registryfs_v2.cpp b/src/overlaybd/registryfs/registryfs_v2.cpp index cac45a9c..92416bc3 100644 --- a/src/overlaybd/registryfs/registryfs_v2.cpp +++ b/src/overlaybd/registryfs/registryfs_v2.cpp @@ -17,6 +17,7 @@ #include "registryfs.h" +#include #include #include #include @@ -561,8 +562,8 @@ class RegistryUploader : public VirtualFile { uint64_t m_timeout = -1; estring m_token; - RegistryUploader(IFile *lfile, std::string &upload_url, std::string &username, - std::string &password, uint64_t timeout, ssize_t upload_bs, + RegistryUploader(IFile *lfile, const std::string &upload_url, const std::string &username, + const std::string &password, uint64_t timeout, ssize_t upload_bs, photon::net::TLSContext *ctx) : m_local_file(lfile), m_origin_upload_url(upload_url), m_username(username), m_password(password), m_timeout(timeout), m_tls_ctx(ctx) { @@ -829,15 +830,15 @@ class RegistryUploader : public VirtualFile { LOG_INFO(VALUE(m_upload_url)); return 0; } - LOG_ERROR_RETURN(0, -1, "failed to get upload url, code=`", op.status_code); + LOG_ERROR_RETURN(0, -1, "failed to get upload url [`], code=`", m_upload_url, op.status_code); } protected: photon::net::TLSContext *m_tls_ctx; }; -IFile *new_registry_uploader(IFile *lfile, std::string &upload_url, std::string &username, - std::string &password, uint64_t timeout, ssize_t upload_bs, +IFile *new_registry_uploader(IFile *lfile, const std::string &upload_url, const std::string &username, + const std::string &password, uint64_t timeout, ssize_t upload_bs, const char *cert_file, const char *key_file) { auto ctx = new_tls_context_from_file(cert_file, key_file); if (!ctx) { @@ -893,3 +894,14 @@ photon::net::TLSContext *new_tls_context_from_file(const char *cert_file, const } return photon::net::new_tls_context(cert_str.c_str(), key_str.c_str(), nullptr); } + +int registry_uploader_fini(photon::fs::IFile *uploader, std::string &digest) { + if (uploader == nullptr) { + LOG_ERRNO_RETURN(0, EINVAL, "invalid pointer"); + } + if (uploader->fsync() < 0) { + LOG_ERRNO_RETURN(0, -1, "failed to upload blob"); + } + digest = ((RegistryUploader *) uploader)->m_sha256sum; + return 0; +} diff --git a/src/tools/CMakeLists.txt b/src/tools/CMakeLists.txt index ea130baa..b551b842 100644 --- a/src/tools/CMakeLists.txt +++ b/src/tools/CMakeLists.txt @@ -16,12 +16,12 @@ add_executable(overlaybd-zfile overlaybd-zfile.cpp) target_include_directories(overlaybd-zfile PUBLIC ${PHOTON_INCLUDE_DIR}) target_link_libraries(overlaybd-zfile photon_static overlaybd_lib) -add_executable(overlaybd-apply overlaybd-apply.cpp comm_func.cpp) +add_executable(overlaybd-apply overlaybd-apply.cpp) target_include_directories(overlaybd-apply PUBLIC ${PHOTON_INCLUDE_DIR} ${rapidjson_SOURCE_DIR}/include) target_link_libraries(overlaybd-apply photon_static overlaybd_lib overlaybd_image_lib checksum_lib) set_target_properties(overlaybd-apply PROPERTIES INSTALL_RPATH "/opt/overlaybd/lib") -add_executable(turboOCI-apply turboOCI-apply.cpp comm_func.cpp) +add_executable(turboOCI-apply turboOCI-apply.cpp) target_include_directories(turboOCI-apply PUBLIC ${PHOTON_INCLUDE_DIR} ${rapidjson_SOURCE_DIR}/include) target_link_libraries(turboOCI-apply photon_static overlaybd_lib overlaybd_image_lib) set_target_properties(turboOCI-apply PROPERTIES INSTALL_RPATH "/opt/overlaybd/lib") diff --git a/src/tools/comm_func.cpp b/src/tools/comm_func.cpp index 1e6e6399..ffa42873 100644 --- a/src/tools/comm_func.cpp +++ b/src/tools/comm_func.cpp @@ -20,6 +20,7 @@ #include #include #include +#include "../overlaybd/registryfs/registryfs.h" #include "../image_service.h" #include "../image_file.h" #include "../overlaybd/tar/erofs/liberofs.h" @@ -91,3 +92,23 @@ photon::fs::IFileSystem *create_erofs_fs(photon::fs::IFile *imgfile, uint64_t bl { return erofs_create_fs(imgfile, blksz); } +IFile *create_uploader(ZFile::CompressArgs *zfile_args, IFile *src, + const string &upload_url, const string &cred_file_path, uint64_t timeout_minute, uint64_t upload_bs_KB, + const string &tls_key_path, const string &tls_cert_path + ){ + + zfile_args->overwrite_header = false; + LOG_INFO("upload to `", upload_url); + std::string username, password; + if (load_cred_from_file(cred_file_path, upload_url, username, password) < 0) { + fprintf(stderr, "failed to read upload cred file\n"); + exit(-1); + } + auto upload_builder = new_registry_uploader(src, upload_url, username, password, + timeout_minute*60*1000*1000, upload_bs_KB*1024, tls_cert_path.c_str(), tls_key_path.c_str()); + if (upload_builder == nullptr) { + fprintf(stderr, "failed to init upload\n"); + exit(-1); + } + return upload_builder; +} diff --git a/src/tools/comm_func.h b/src/tools/comm_func.h index 80458ec9..7c6618f0 100644 --- a/src/tools/comm_func.h +++ b/src/tools/comm_func.h @@ -41,6 +41,10 @@ photon::fs::IFile *open_file(const char *fn, int flags, mode_t mode = 0, photon: int create_overlaybd(const std::string &srv_config, const std::string &dev_config, ImageService *&, photon::fs::IFile *&); +photon::fs::IFile *create_uploader(ZFile::CompressArgs *zfile_args, IFile *src, + const std::string &upload_url, const std::string &cred_file_path, uint64_t timeout_minute, uint64_t upload_bs_KB, + const std::string &tls_key_path, const std::string &tls_cert_path); + photon::fs::IFileSystem *create_ext4fs(photon::fs::IFile *imgfile, bool mkfs, bool enable_buffer, const char* root); diff --git a/src/tools/overlaybd-commit.cpp b/src/tools/overlaybd-commit.cpp index 18ad5c69..61a3a742 100644 --- a/src/tools/overlaybd-commit.cpp +++ b/src/tools/overlaybd-commit.cpp @@ -21,12 +21,9 @@ #include "../overlaybd/lsmt/file.h" #include "../overlaybd/zfile/zfile.h" #include "../overlaybd/tar/tar_file.h" -#include "../overlaybd/registryfs/registryfs.h" -#include "../image_service.h" #include #include #include -#include #include #include #include @@ -34,11 +31,28 @@ #include #include #include "CLI11.hpp" +#include "comm_func.h" +#include "../overlaybd/registryfs/registryfs.h" + using namespace std; using namespace LSMT; using namespace photon::fs; +string commit_msg; +string uuid, parent_uuid; +std::string algorithm; +int block_size = -1; +std::string data_file_path, index_file_path, commit_file_path, remote_mapping_file; +bool compress_zfile = false; +bool build_turboOCI = false; +bool build_fastoci = false; +bool tar = false, rm_old = false, seal = false, commit_sealed = false; +bool verbose = false; +int compress_threads = 1; +std::string upload_url, cred_file_path, tls_key_path, tls_cert_path; +ssize_t upload_bs = 262144; + IFile *open_file(IFileSystem *fs, const char *fn, int flags, mode_t mode = 0) { auto file = fs->open(fn, flags, mode); @@ -50,19 +64,6 @@ IFile *open_file(IFileSystem *fs, const char *fn, int flags, mode_t mode = 0) { } int main(int argc, char **argv) { - string commit_msg; - string uuid, parent_uuid; - std::string algorithm; - int block_size = -1; - std::string data_file_path, index_file_path, commit_file_path, remote_mapping_file; - bool compress_zfile = false; - bool build_turboOCI = false; - bool build_fastoci = false; - bool tar = false, rm_old = false, seal = false, commit_sealed = false; - bool verbose = false; - int compress_threads = 1; - std::string upload_url, cred_file_path; - ssize_t upload_bs = 262144; CLI::App app{"this is overlaybd-commit"}; app.add_option("-m", commit_msg, "add some custom message if needed"); @@ -87,10 +88,18 @@ int main(int argc, char **argv) { app.add_option("--upload", upload_url, "registry upload url"); app.add_option("--upload_bs", upload_bs, "block size for upload, in KB"); app.add_option("--cred_file_path", cred_file_path, "cred file path for registryfs")->type_name("FILEPATH")->check(CLI::ExistingFile); + app.add_option("--tls_key_path", tls_key_path, "TLSKeyPairPath for private Registry")->type_name("FILEPATH")->check(CLI::ExistingFile); + app.add_option("--tls_cert_path", tls_cert_path, "TLSCertPath for private Registry")->type_name("FILEPATH")->check(CLI::ExistingFile); + + CLI11_PARSE(app, argc, argv); build_turboOCI = build_turboOCI || build_fastoci; set_log_output_level(verbose ? 0 : 1); + if (tar && (upload_url.empty() == false)){ + fprintf(stderr, "unsupport option with '-t' and '--upload' at the same time."); + exit(-1); + } photon::init(photon::INIT_EVENT_DEFAULT, photon::INIT_IO_DEFAULT); DEFER({photon::fini();}); @@ -164,21 +173,10 @@ int main(int argc, char **argv) { zfile_args->overwrite_header = true; if (!upload_url.empty()) { - zfile_args->overwrite_header = false; - LOG_INFO("upload to `", upload_url); - std::string username, password; - if (load_cred_from_file(cred_file_path, upload_url, username, password) < 0) { - fprintf(stderr, "failed to read upload cred file\n"); - exit(-1); - } - upload_builder = new_registry_uploader(out, upload_url, username, password, 2UL*60*1000*1000, upload_bs*1024); - if (upload_builder == nullptr) { - fprintf(stderr, "failed to init upload\n"); - exit(-1); - } + LOG_INFO("enable upload. URL: `, upload_bs: `, tls_key_path: `, tls_cert_path: `", upload_url, upload_bs, tls_key_path, tls_cert_path); + upload_builder = create_uploader(zfile_args, out, upload_url, cred_file_path, 2, upload_bs, tls_key_path, tls_cert_path); out = upload_builder; } - zfile_builder = ZFile::new_zfile_builder(out, zfile_args, false); out = zfile_builder; } else { @@ -213,12 +211,12 @@ int main(int argc, char **argv) { if (zfile_args) { delete zfile_args; } - - if (upload_builder != nullptr && upload_builder->fsync() < 0) { - fprintf(stderr, "failed to commit or upload"); - return -1; - } - + string digest = ""; + if (upload_builder != nullptr && registry_uploader_fini(upload_builder, digest) != 0){ + fprintf(stderr, "failed to commit or upload\n"); + exit(-1); + }; + fprintf(stderr, "%s\n", digest.c_str()); delete upload_builder; delete fout; delete fin; diff --git a/src/tools/overlaybd-merge.cpp b/src/tools/overlaybd-merge.cpp index d271b536..d29f544c 100644 --- a/src/tools/overlaybd-merge.cpp +++ b/src/tools/overlaybd-merge.cpp @@ -20,15 +20,9 @@ #include #include #include -#include "../overlaybd/lsmt/file.h" #include "../overlaybd/zfile/zfile.h" -#include "../overlaybd/tar/libtar.h" -#include "../overlaybd/gzindex/gzfile.h" -#include "../overlaybd/gzip/gz.h" -#include #include #include -#include #include #include #include @@ -37,6 +31,8 @@ #include #include "../image_service.h" #include "../image_file.h" +#include "../overlaybd/registryfs/registryfs.h" +#include "../overlaybd/tar/tar_file.h" #include "CLI11.hpp" #include "comm_func.h" #include "sha256file.h" @@ -44,6 +40,14 @@ using namespace std; using namespace photon::fs; +std::string image_config_path, input_path, output, config_path, sha256_checksum; +int upload_bs = 65536; +bool zfile = false, verbose = false, tar = false; +std::string upload_url, cred_file_path, tls_key_path, tls_cert_path; + +IFile *upload_builder = nullptr, *rst = nullptr; + + class FIFOFile : public VirtualReadOnlyFile { public: IFile *m_fifo; @@ -79,15 +83,20 @@ class FIFOFile : public VirtualReadOnlyFile { }; int main(int argc, char **argv) { - std::string image_config_path, input_path, output, config_path, sha256_checksum; - string tarheader; - bool zfile = false, mkfs = false, verbose = false, raw = false; CLI::App app{"this is overlaybd-merge, merge multiple overlaybd layers into a single."}; app.add_flag("--verbose", verbose, "output debug info")->default_val(false); app.add_option("--service_config_path", config_path, "overlaybd image service config path")->type_name("FILEPATH")->check(CLI::ExistingFile)->default_val("/etc/overlaybd/overlaybd.json"); - app.add_flag("--compress", zfile, "do zfile compression for the output layer")->default_val(true); + app.add_flag("--compress", zfile, "do zfile compression for the output layer")->run_callback_for_default()->default_val(true); + app.add_flag("-t", tar, "wrapper with tar")->default_val(false); + + app.add_option("--upload", upload_url, "upload to remote registry URL while generating merged layer."); + app.add_option("--upload_bs", upload_bs, "block size for upload, in KB")->default_val(262144); + app.add_option("--cred_file_path", cred_file_path, "cred file path for registryfs")->type_name("FILEPATH")->check(CLI::ExistingFile); + app.add_option("--tls_key_path", tls_key_path, "TLSKeyPairPath for private Registry")->type_name("FILEPATH")->check(CLI::ExistingFile); + app.add_option("--tls_cert_path", tls_cert_path, "TLSCertPath for private Registry")->type_name("FILEPATH")->check(CLI::ExistingFile); + app.add_option("image_config_path", image_config_path, "overlaybd image config path")->type_name("FILEPATH")->check(CLI::ExistingFile)->required(); app.add_option("output", output, "compacted layer path")->type_name("FILEPATH"); @@ -95,6 +104,10 @@ int main(int argc, char **argv) { CLI11_PARSE(app, argc, argv); set_log_output_level(verbose ? 0 : 1); + + if (tar && (upload_url.empty() == false)) { + rst = new_tar_file_adaptor(rst); + } photon::init(photon::INIT_EVENT_DEFAULT, photon::INIT_IO_DEFAULT); DEFER({photon::fini();}); @@ -110,14 +123,23 @@ int main(int argc, char **argv) { delete imgfile; delete imgservice; }); - auto rst = open_localfile_adaptor(output.c_str(), O_CREAT|O_TRUNC|O_RDWR, 0644); + rst = open_localfile_adaptor(output.c_str(), O_CREAT|O_TRUNC|O_RDWR, 0644); if (rst == nullptr){ fprintf(stderr, "failed to create output file\n"); exit(-1); } DEFER(delete rst); + if (zfile) { - rst = ZFile::new_zfile_builder(rst); + ZFile::CompressOptions opt; + opt.verify = 1; + ZFile::CompressArgs zfile_args(opt); + if (!upload_url.empty()) { + LOG_INFO("enable upload. URL: `, upload_bs: `, tls_key_path: `, tls_cert_path: `", upload_url, upload_bs, tls_key_path, tls_cert_path); + upload_builder = create_uploader(&zfile_args, rst, upload_url, cred_file_path, 2, upload_bs, tls_key_path, tls_cert_path); + rst = upload_builder; + } + rst = ZFile::new_zfile_builder(rst, &zfile_args, true); if (rst == nullptr) { fprintf(stderr, "failed to create zfile\n"); exit(-1); @@ -127,6 +149,12 @@ int main(int argc, char **argv) { fprintf(stderr, "failed to compact\n"); exit(-1); } - + string digest = ""; + if (upload_builder != nullptr && registry_uploader_fini(upload_builder, digest) != 0){ + fprintf(stderr, "failed to upload\n"); + exit(-1); + }; + fprintf(stderr, "%s\n", digest.c_str()); + delete upload_builder; return 0; }