From 25e47ef3e94389a23ff90e954331524edc9bf690 Mon Sep 17 00:00:00 2001 From: Hongzhen Luo Date: Tue, 26 Mar 2024 16:17:56 +0800 Subject: [PATCH] Introduce tarerofs for TurboOCI-apply Currently erofs-utils is integrated into overlaybd by using `mkfs.erofs` executable file. It's not optimial since raw data needs to be dumped first and output data needs to be write into overlaybd then. Later API integration may be a better form but it needs more work. Signed-off-by: Hongzhen Luo --- src/overlaybd/tar/tarerofs.cpp | 135 +++++++++++++++++++++++++++++++++ src/overlaybd/tar/tarerofs.h | 36 +++++++++ src/tools/turboOCI-apply.cpp | 39 ++++++---- 3 files changed, 197 insertions(+), 13 deletions(-) create mode 100644 src/overlaybd/tar/tarerofs.cpp create mode 100644 src/overlaybd/tar/tarerofs.h diff --git a/src/overlaybd/tar/tarerofs.cpp b/src/overlaybd/tar/tarerofs.cpp new file mode 100644 index 00000000..7293689a --- /dev/null +++ b/src/overlaybd/tar/tarerofs.cpp @@ -0,0 +1,135 @@ +#include "tarerofs.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../lsmt/file.h" +#include "../lsmt/index.h" + +#define TAREROFS_BLOCK_SIZE 4096 +#define __stringify_1(x...) #x +#define __stringify(x...) __stringify_1(x) + +int TarErofs::extract_all() { + ssize_t read; + struct stat st; + char buf[128*1024]; + char base_path[64] = "/tmp/tarerofs_base_XXXXXX"; + char command_line[256] = "mkfs.erofs --tar=0,upper.map,1073741824 -b" __stringify(TAREROFS_BLOCK_SIZE) " --aufs"; + const char command_line2[] = " upper.erofs"; + + FILE *fp; + photon::fs::IFile *rawfs = nullptr; + int status; + uint64_t blkaddr, toff, metasize; + uint32_t nblocks; + + if (!meta_only) { + LOG_ERROR("currently EROFS supports fastoci mode only", strerror(errno)); + return -1; + } + + if (fs_base_file->fstat(&st)>= 0 && st.st_blocks > 0) { + int fd = mkstemp(base_path); + if (fd < 0) { + LOG_ERROR("cannot generate a temporary file to dump overlaybd disk"); + return -1; + } + std::strcat(command_line, " --base "); + std::strcat(command_line, base_path); + + if (fs_base_file->pread(&metasize, sizeof(metasize), 0) != + sizeof(metasize)) { + LOG_ERROR("failed to read EROFS metadata size"); + return -1; + } + + while (metasize) { + int count = std::min(sizeof(buf), metasize); + read = fs_base_file->read(buf, count); + if (read != count || + write(fd, buf, read) != read) { + read = -1; + break; + } + metasize -= read; + } + close(fd); + if (read < 0) { + return -1; + } + } + std::strcat(command_line, command_line2); + fp = popen(command_line, "w"); + if (fp == NULL) { + LOG_ERROR("failed to execute mkfs.erofs", strerror(errno)); + return -1; + } + + while ((read = file->read(buf, sizeof(buf))) > 0) { + if (fwrite(buf, read, 1, fp) != 1) { + read = -1; + break; + } + } + status = pclose(fp); + + if (read < 0 || status) { + return -1; + } + + rawfs = photon::fs::open_localfile_adaptor("upper.erofs", O_RDONLY, 0644); + DEFER({ delete rawfs; }); + + /* write to LSMT */ + metasize = rawfs->lseek(0, SEEK_END); + rawfs->lseek(0, 0); + fout->lseek(0, 0); + while ((read = rawfs->read(buf, sizeof(buf))) > 0) { + if (metasize) { // since pwrite < 512 is unsupported. + *(uint64_t *)buf = metasize; + metasize = 0; + } + + if (fout->write(buf, read) != read) { + read = -1; + break; + } + metasize += read; + } + + + /* write mapfile */ + fp = fopen("upper.map", "r"); + if (fp == NULL) { + LOG_ERROR("unable to get upper.map, ignored"); + return -1; + } + while (fscanf(fp, "%" PRIx64" %x %" PRIx64 "\n", &blkaddr, &nblocks, &toff) >= 3) { + LSMT::RemoteMapping lba; + lba.offset = blkaddr * TAREROFS_BLOCK_SIZE; + lba.count = nblocks * TAREROFS_BLOCK_SIZE; + lba.roffset = toff; + int nwrite = fout->ioctl(LSMT::IFileRW::RemoteData, lba); + if ((unsigned) nwrite != lba.count) { + LOG_ERRNO_RETURN(0, -1, "failed to write lba"); + } + } + fclose(fp); + return 0; +} diff --git a/src/overlaybd/tar/tarerofs.h b/src/overlaybd/tar/tarerofs.h new file mode 100644 index 00000000..32bae5a6 --- /dev/null +++ b/src/overlaybd/tar/tarerofs.h @@ -0,0 +1,36 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +class TarErofs { +public: + TarErofs(photon::fs::IFile *file, photon::fs::IFile *target, uint64_t fs_blocksize = 4096, + photon::fs::IFile *bf = nullptr, bool meta_only = true) + : file(file), fout(target), fs_base_file(bf), meta_only(meta_only) {} + + int extract_all(); + +private: + photon::fs::IFile *file = nullptr; // source + photon::fs::IFile *fout = nullptr; // target + photon::fs::IFile *fs_base_file = nullptr; + bool meta_only; + std::set unpackedPaths; + std::list> dirs; // +}; diff --git a/src/tools/turboOCI-apply.cpp b/src/tools/turboOCI-apply.cpp index e56a99ff..aef53802 100644 --- a/src/tools/turboOCI-apply.cpp +++ b/src/tools/turboOCI-apply.cpp @@ -23,6 +23,7 @@ #include "../overlaybd/lsmt/file.h" #include "../overlaybd/zfile/zfile.h" #include "../overlaybd/tar/libtar.h" +#include "../overlaybd/tar/tarerofs.h" #include "../overlaybd/gzindex/gzfile.h" #include "../overlaybd/gzip/gz.h" #include @@ -60,12 +61,13 @@ int dump_tar_headers(IFile *src_file, const string &out) { } int main(int argc, char **argv) { - std::string image_config_path, input_path, gz_index_path, config_path; + std::string image_config_path, input_path, gz_index_path, config_path, fstype; bool raw = false, mkfs = false, verbose = false; bool export_tar_headers = false, import_tar_headers = false; CLI::App app{"this is turboOCI-apply, apply OCIv1 tar layer to 'Overlaybd-TurboOCI v1' format"}; app.add_flag("--mkfs", mkfs, "mkfs before apply")->default_val(false); + app.add_option("--fstype", fstype, "filesystem type")->default_val("ext4"); app.add_flag("--verbose", verbose, "output debug info")->default_val(false); app.add_option("--service_config_path", config_path, "overlaybd image service config path") ->type_name("FILEPATH") @@ -124,18 +126,29 @@ int main(int argc, char **argv) { }); // for now, buffer_file can't be used with turboOCI - auto target = create_ext4fs(imgfile, mkfs, false, "/"); - DEFER({ delete target; }); - - photon::fs::IFile *base_file = raw ? nullptr : ((ImageFile *)imgfile)->get_base(); - bool gen_turboOCI = true; - int option = (import_tar_headers ? TAR_IGNORE_CRC : 0); - auto tar = - new UnTar(src_file, target, option, 4096, base_file, gen_turboOCI, import_tar_headers); - - if (tar->extract_all() < 0) { - fprintf(stderr, "failed to extract\n"); - exit(-1); + if (fstype == "erofs") { + photon::fs::IFile* base_file = raw ? nullptr : ((ImageFile *)imgfile)->get_base(); + + auto tar = + new TarErofs(src_file, imgfile, 4096, base_file, true); + + if (tar->extract_all() < 0) { + fprintf(stderr, "failed to extract\n"); + exit(-1); + } + } else { + auto target = create_ext4fs(imgfile, mkfs, false, "/"); + DEFER({ delete target; }); + + photon::fs::IFile *base_file = raw ? nullptr : ((ImageFile *)imgfile)->get_base(); + bool gen_turboOCI = true; + int option = (import_tar_headers ? TAR_IGNORE_CRC : 0); + auto tar = + new UnTar(src_file, target, option, 4096, base_file, gen_turboOCI, import_tar_headers); + if (tar->extract_all() < 0) { + fprintf(stderr, "failed to extract\n"); + exit(-1); + } } fprintf(stdout, "turboOCI-apply done\n"); return 0;