From 1f91ecf09c8da761e9c76569a6aeed5b84d88fec Mon Sep 17 00:00:00 2001 From: Yifan Yuan Date: Mon, 2 Sep 2024 21:46:41 +0800 Subject: [PATCH] [feat.] Support uploading 'output' to registry while merging layers Signed-off-by: Yifan Yuan --- src/overlaybd/lsmt/test/test.cpp | 17 +++++++++-- src/overlaybd/registryfs/registryfs.h | 8 +++-- src/overlaybd/registryfs/registryfs_v2.cpp | 22 ++++++++++---- src/tools/comm_func.cpp | 19 ++++++++++++ src/tools/comm_func.h | 3 ++ src/tools/overlaybd-commit.cpp | 34 +++++++-------------- src/tools/overlaybd-merge.cpp | 35 +++++++++++++++------- 7 files changed, 95 insertions(+), 43 deletions(-) diff --git a/src/overlaybd/lsmt/test/test.cpp b/src/overlaybd/lsmt/test/test.cpp index 1a107ce8..1197d12f 100644 --- a/src/overlaybd/lsmt/test/test.cpp +++ b/src/overlaybd/lsmt/test/test.cpp @@ -246,6 +246,12 @@ void test_combo(const IMemoryIndex *indexes[], size_t ni, const SegmentMapping s auto ret = memcmp(pm, stdrst, nrst * sizeof(stdrst[0])); EXPECT_EQ(ret, 0); } + LOG_INFO("make RO index of ci"); + auto ro_idx = ci.make_read_only_index(); + if (ro_idx->size() == nrst) { + auto ret = memcmp(ro_idx->buffer(), stdrst, nrst * sizeof(stdrst[0])); + EXPECT_EQ(ret, 0); + } ci.backing_index(); // delete mi; @@ -599,7 +605,10 @@ TEST_F(FileTest3, stack_files) { LOG_INFO("RO valid data: `", stat.valid_data_size); merged = lfs->open(fn_merged, O_RDWR | O_CREAT | O_TRUNC, S_IRWXU); EXPECT_EQ(lower->flatten(merged), 0); - cout << "verifying flattened layer of lowers" << endl; + // auto localfile = lfs->open(fn_merged, O_RDONLY); + struct stat st; + merged->fstat(&st); + cout << "verifying flattened layer of lowers, st_size: "<open(fn_merged, O_RDWR | O_CREAT | O_TRUNC, S_IRWXU); file->flatten(merged); + merged->fstat(&st); + cout << "verifying flattened layer of stacked layers, st_size: "<close(); + verify_file(fn_merged); delete file; + delete merged; } TEST_F(FileTest3, seek_data) { diff --git a/src/overlaybd/registryfs/registryfs.h b/src/overlaybd/registryfs/registryfs.h index 14a71e7c..ce90df01 100644 --- a/src/overlaybd/registryfs/registryfs.h +++ b/src/overlaybd/registryfs/registryfs.h @@ -43,10 +43,14 @@ photon::fs::IFileSystem *new_registryfs_v2(PasswordCB callback, const char *customized_ua = nullptr); photon::fs::IFile* new_registry_uploader(photon::fs::IFile *lfile, - std::string &upload_url, - std::string &username, std::string &password, + const std::string &upload_url, + const std::string &username, + const std::string &password, uint64_t timeout, ssize_t upload_bs = -1, const char *cert_file = nullptr, const char *key_file = nullptr); + +int registry_uploader_fini(photon::fs::IFile *uploader, std::string &digest); + } diff --git a/src/overlaybd/registryfs/registryfs_v2.cpp b/src/overlaybd/registryfs/registryfs_v2.cpp index cac45a9c..92416bc3 100644 --- a/src/overlaybd/registryfs/registryfs_v2.cpp +++ b/src/overlaybd/registryfs/registryfs_v2.cpp @@ -17,6 +17,7 @@ #include "registryfs.h" +#include #include #include #include @@ -561,8 +562,8 @@ class RegistryUploader : public VirtualFile { uint64_t m_timeout = -1; estring m_token; - RegistryUploader(IFile *lfile, std::string &upload_url, std::string &username, - std::string &password, uint64_t timeout, ssize_t upload_bs, + RegistryUploader(IFile *lfile, const std::string &upload_url, const std::string &username, + const std::string &password, uint64_t timeout, ssize_t upload_bs, photon::net::TLSContext *ctx) : m_local_file(lfile), m_origin_upload_url(upload_url), m_username(username), m_password(password), m_timeout(timeout), m_tls_ctx(ctx) { @@ -829,15 +830,15 @@ class RegistryUploader : public VirtualFile { LOG_INFO(VALUE(m_upload_url)); return 0; } - LOG_ERROR_RETURN(0, -1, "failed to get upload url, code=`", op.status_code); + LOG_ERROR_RETURN(0, -1, "failed to get upload url [`], code=`", m_upload_url, op.status_code); } protected: photon::net::TLSContext *m_tls_ctx; }; -IFile *new_registry_uploader(IFile *lfile, std::string &upload_url, std::string &username, - std::string &password, uint64_t timeout, ssize_t upload_bs, +IFile *new_registry_uploader(IFile *lfile, const std::string &upload_url, const std::string &username, + const std::string &password, uint64_t timeout, ssize_t upload_bs, const char *cert_file, const char *key_file) { auto ctx = new_tls_context_from_file(cert_file, key_file); if (!ctx) { @@ -893,3 +894,14 @@ photon::net::TLSContext *new_tls_context_from_file(const char *cert_file, const } return photon::net::new_tls_context(cert_str.c_str(), key_str.c_str(), nullptr); } + +int registry_uploader_fini(photon::fs::IFile *uploader, std::string &digest) { + if (uploader == nullptr) { + LOG_ERRNO_RETURN(0, EINVAL, "invalid pointer"); + } + if (uploader->fsync() < 0) { + LOG_ERRNO_RETURN(0, -1, "failed to upload blob"); + } + digest = ((RegistryUploader *) uploader)->m_sha256sum; + return 0; +} diff --git a/src/tools/comm_func.cpp b/src/tools/comm_func.cpp index 1e6e6399..5f1ff55c 100644 --- a/src/tools/comm_func.cpp +++ b/src/tools/comm_func.cpp @@ -20,6 +20,7 @@ #include #include #include +#include "../overlaybd/registryfs/registryfs.h" #include "../image_service.h" #include "../image_file.h" #include "../overlaybd/tar/erofs/liberofs.h" @@ -91,3 +92,21 @@ photon::fs::IFileSystem *create_erofs_fs(photon::fs::IFile *imgfile, uint64_t bl { return erofs_create_fs(imgfile, blksz); } +IFile *create_uploader(ZFile::CompressArgs *zfile_args, IFile *src, + const string &upload_url, const string &cred_file_path, uint64_t timeout, uint64_t upload_bs_KB){ + + zfile_args->overwrite_header = false; + LOG_INFO("upload to `", upload_url); + std::string username, password; + if (load_cred_from_file(cred_file_path, upload_url, username, password) < 0) { + fprintf(stderr, "failed to read upload cred file\n"); + exit(-1); + } + auto upload_builder = new_registry_uploader(src, upload_url, username, password, + timeout*60*1000*1000, upload_bs_KB*1024); + if (upload_builder == nullptr) { + fprintf(stderr, "failed to init upload\n"); + exit(-1); + } + return upload_builder; +} diff --git a/src/tools/comm_func.h b/src/tools/comm_func.h index 80458ec9..716781e6 100644 --- a/src/tools/comm_func.h +++ b/src/tools/comm_func.h @@ -41,6 +41,9 @@ photon::fs::IFile *open_file(const char *fn, int flags, mode_t mode = 0, photon: int create_overlaybd(const std::string &srv_config, const std::string &dev_config, ImageService *&, photon::fs::IFile *&); +photon::fs::IFile *create_uploader(ZFile::CompressArgs *zfile_args, IFile *src, + const std::string &upload_url, const std::string &cred_file_path, uint64_t timeout_minute, uint64_t upload_bs_KB); + photon::fs::IFileSystem *create_ext4fs(photon::fs::IFile *imgfile, bool mkfs, bool enable_buffer, const char* root); diff --git a/src/tools/overlaybd-commit.cpp b/src/tools/overlaybd-commit.cpp index 18ad5c69..0b15beb5 100644 --- a/src/tools/overlaybd-commit.cpp +++ b/src/tools/overlaybd-commit.cpp @@ -21,12 +21,9 @@ #include "../overlaybd/lsmt/file.h" #include "../overlaybd/zfile/zfile.h" #include "../overlaybd/tar/tar_file.h" -#include "../overlaybd/registryfs/registryfs.h" -#include "../image_service.h" #include #include #include -#include #include #include #include @@ -34,6 +31,9 @@ #include #include #include "CLI11.hpp" +#include "comm_func.h" +#include "../overlaybd/registryfs/registryfs.h" + using namespace std; using namespace LSMT; @@ -85,7 +85,7 @@ int main(int argc, char **argv) { app.add_option("--compress_threads", compress_threads, "compress threads")->default_val(1); app.add_flag("--verbose", verbose, "output debug info")->default_val(false); app.add_option("--upload", upload_url, "registry upload url"); - app.add_option("--upload_bs", upload_bs, "block size for upload, in KB"); + app.add_option("--upload_bs", upload_bs, "block size for upload, in KB")->default_val(1024); app.add_option("--cred_file_path", cred_file_path, "cred file path for registryfs")->type_name("FILEPATH")->check(CLI::ExistingFile); CLI11_PARSE(app, argc, argv); @@ -164,21 +164,9 @@ int main(int argc, char **argv) { zfile_args->overwrite_header = true; if (!upload_url.empty()) { - zfile_args->overwrite_header = false; - LOG_INFO("upload to `", upload_url); - std::string username, password; - if (load_cred_from_file(cred_file_path, upload_url, username, password) < 0) { - fprintf(stderr, "failed to read upload cred file\n"); - exit(-1); - } - upload_builder = new_registry_uploader(out, upload_url, username, password, 2UL*60*1000*1000, upload_bs*1024); - if (upload_builder == nullptr) { - fprintf(stderr, "failed to init upload\n"); - exit(-1); - } + upload_builder = create_uploader(zfile_args, out, upload_url, cred_file_path, 2, upload_bs); out = upload_builder; } - zfile_builder = ZFile::new_zfile_builder(out, zfile_args, false); out = zfile_builder; } else { @@ -213,12 +201,12 @@ int main(int argc, char **argv) { if (zfile_args) { delete zfile_args; } - - if (upload_builder != nullptr && upload_builder->fsync() < 0) { - fprintf(stderr, "failed to commit or upload"); - return -1; - } - + string digest = ""; + if (upload_builder != nullptr && registry_uploader_fini(upload_builder, digest) != 0){ + fprintf(stderr, "failed to commit or upload\n"); + exit(-1); + }; + fprintf(stderr, "%s\n", digest.c_str()); delete upload_builder; delete fout; delete fin; diff --git a/src/tools/overlaybd-merge.cpp b/src/tools/overlaybd-merge.cpp index d271b536..bf1c15e0 100644 --- a/src/tools/overlaybd-merge.cpp +++ b/src/tools/overlaybd-merge.cpp @@ -20,15 +20,9 @@ #include #include #include -#include "../overlaybd/lsmt/file.h" #include "../overlaybd/zfile/zfile.h" -#include "../overlaybd/tar/libtar.h" -#include "../overlaybd/gzindex/gzfile.h" -#include "../overlaybd/gzip/gz.h" -#include #include #include -#include #include #include #include @@ -37,6 +31,7 @@ #include #include "../image_service.h" #include "../image_file.h" +#include "../overlaybd/registryfs/registryfs.h" #include "CLI11.hpp" #include "comm_func.h" #include "sha256file.h" @@ -79,15 +74,19 @@ class FIFOFile : public VirtualReadOnlyFile { }; int main(int argc, char **argv) { - std::string image_config_path, input_path, output, config_path, sha256_checksum; - string tarheader; - bool zfile = false, mkfs = false, verbose = false, raw = false; + std::string image_config_path, input_path, output, config_path, sha256_checksum, upload_url, cred_file_path; + int upload_bs = 0; + bool zfile = false, verbose = false; + IFile *upload_builder = nullptr; CLI::App app{"this is overlaybd-merge, merge multiple overlaybd layers into a single."}; app.add_flag("--verbose", verbose, "output debug info")->default_val(false); app.add_option("--service_config_path", config_path, "overlaybd image service config path")->type_name("FILEPATH")->check(CLI::ExistingFile)->default_val("/etc/overlaybd/overlaybd.json"); app.add_flag("--compress", zfile, "do zfile compression for the output layer")->default_val(true); + app.add_option("--upload", upload_url, "upload to remote registry URL while generating merged layer."); + app.add_option("--upload_bs", upload_bs, "block size for upload, in KB")->default_val(65536); + app.add_option("--cred_file_path", cred_file_path, "cred file path for registryfs")->type_name("FILEPATH")->check(CLI::ExistingFile); app.add_option("image_config_path", image_config_path, "overlaybd image config path")->type_name("FILEPATH")->check(CLI::ExistingFile)->required(); app.add_option("output", output, "compacted layer path")->type_name("FILEPATH"); @@ -117,7 +116,15 @@ int main(int argc, char **argv) { } DEFER(delete rst); if (zfile) { - rst = ZFile::new_zfile_builder(rst); + ZFile::CompressOptions opt; + opt.verify = 1; + ZFile::CompressArgs zfile_args(opt); + if (!upload_url.empty()) { + LOG_INFO("upload url detected, create uploader"); + upload_builder = create_uploader(&zfile_args, rst, upload_url, cred_file_path, 2 /* 2min timeout foreach block */, upload_bs); + rst = upload_builder; + } + rst = ZFile::new_zfile_builder(rst, &zfile_args, true); if (rst == nullptr) { fprintf(stderr, "failed to create zfile\n"); exit(-1); @@ -127,6 +134,12 @@ int main(int argc, char **argv) { fprintf(stderr, "failed to compact\n"); exit(-1); } - + string digest = ""; + if (upload_builder != nullptr && registry_uploader_fini(upload_builder, digest) != 0){ + fprintf(stderr, "failed to upload\n"); + exit(-1); + }; + fprintf(stderr, "%s\n", digest.c_str()); + delete upload_builder; return 0; }