Skip to content

Commit

Permalink
Merge pull request #296 from WaberZhuang/main
Browse files Browse the repository at this point in the history 
Fix registry_v2 and gzip cache
  • Loading branch information
@liulanzheng
liulanzheng authored Nov 29, 2023
2 parents 1a50073 + 69f9d1d commit d272e1c
Show file tree
Hide file tree
Showing 3 changed files with 33 additions and 45 deletions.
7 changes: 6 additions & 1 deletion src/overlaybd/cache/gzip_cache/cached_fs.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
#include "cached_fs.h"
#include "../full_file_cache/cache_pool.h"
#include "../cache.h"
#include <photon/common/estring.h>

namespace Cache {

Expand All @@ -34,7 +35,11 @@ class GzipCachedFsImpl : public GzipCachedFs {
if (!file) {
LOG_ERRNO_RETURN(0, nullptr, "Open source gzfile failed");
}
auto cache_store = pool_->open(file_name, O_RDWR | O_CREAT, 0644);
estring fn = file_name;
if (fn[0] != '/') {
fn = estring().appends("/", fn);
}
auto cache_store = pool_->open(fn, O_RDWR | O_CREAT, 0644);
if (cache_store == nullptr) {
delete file;
LOG_ERRNO_RETURN(0, nullptr, "file cache pool open file failed, name : `", file_name);
Expand Down
65 changes: 23 additions & 42 deletions src/overlaybd/gzindex/test/test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,7 @@
*/

#include "../gzfile.h"
#include "../../cache/pool_store.h"
#include "../../cache/full_file_cache/cache_pool.h"
#include "../../cache/gzip_cache/cached_fs.h"
#include "../../cache/cache.h"
#include <photon/photon.h>
#include <photon/common/io-alloc.h>
Expand Down Expand Up @@ -251,13 +250,11 @@ class GzCacheTest : public ::testing::Test {
system(cmd.c_str());
cmd = std::string("mkdir -p /tmp/gzip_src");
system(cmd.c_str());
cmd = std::string("mkdir -p /tmp/gzip_cache");
cmd = std::string("mkdir -p /tmp/gzip_cache_compress");
system(cmd.c_str());
cmd = std::string("mkdir -p /tmp/gzip_cache_decompress");
system(cmd.c_str());
lfs = photon::fs::new_localfs_adaptor("/tmp/gzip_src");
cfs = photon::fs::new_localfs_adaptor("/tmp/gzip_cache");
cfs1 = photon::fs::new_localfs_adaptor("/tmp/gzip_cache");
pool = new Cache::FileCachePool(cfs1, 4, 10000000, (uint64_t)1048576 * 4096, 1024 * 1024);

if (buildDataFile() != 0) {
LOG_ERROR("failed to build ` and `", fn_defile, fn_gzdata);
exit(-1);
Expand All @@ -266,34 +263,27 @@ class GzCacheTest : public ::testing::Test {
LOG_ERROR("failed to build gz index: `", fn_gzindex);
exit(-1);
}

auto mediafs = photon::fs::new_localfs_adaptor("/tmp/gzip_cache_compress");
lfs = FileSystem::new_full_file_cached_fs(
lfs, cfs, 1024 * 1024, 1, 10000000,
lfs, mediafs, 1024 * 1024, 1, 10000000,
(uint64_t)1048576 * 4096, nullptr, 0, nullptr);
gzdata = lfs->open(fn_gzdata, O_CREAT | O_TRUNC | O_RDWR, 0644);
delete gzdata;
gzdata = lfs->open(fn_gzdata, O_RDONLY, 0644);
if (gzdata == nullptr) {
LOG_ERROR("gzdata create failed");
exit(-1);
}
gzfile = new_gzfile(gzdata, gzindex);
struct stat st = {};
if (gzfile) {
auto ok = gzfile->fstat(&st);
if (ok == -1) {
exit(-1);
}
}
auto store = pool->open(fn_defile, O_RDWR | O_CREAT, 0644);
if (store == nullptr) {
delete gzfile;
if (gzfile == nullptr) {
LOG_ERROR("gzfile create failed");
exit(-1);
}
auto io_alloc = new IOAlloc;
store->set_src_file(gzfile);
store->set_page_size(4096);
store->set_allocator(io_alloc);
store->set_actual_size(st.st_size);
gzfile = new_cached_file(store, 4096, nullptr);


mediafs = photon::fs::new_localfs_adaptor("/tmp/gzip_cache_decompress");
cfs = Cache::new_gzip_cached_fs(mediafs, 1024 * 1024, 4, 10000000, (uint64_t)1048576 * 4096, nullptr);
gzfile = cfs->open_cached_gzip_file(gzfile, fn_defile);
if (gzfile == nullptr) {
LOG_ERROR("failed create new cached gzip file");
exit(-1);
Expand All @@ -306,17 +296,12 @@ class GzCacheTest : public ::testing::Test {
delete defile;
delete gzfile;

if (lfs->access(fn_defile, 0) == 0) {
lfs->unlink(fn_defile);
}
if (lfs->access(fn_gzdata, 0) == 0) {
lfs->unlink(fn_gzdata);
}
if (lfs->access(fn_gzindex, 0) == 0) {
lfs->unlink(fn_gzindex);
}
delete pool;
lfs->unlink(fn_defile);
lfs->unlink(fn_gzdata);
lfs->unlink(fn_gzindex);

delete lfs;
delete cfs;
}

void test_pread(PreadTestCase t) {
Expand All @@ -343,9 +328,7 @@ class GzCacheTest : public ::testing::Test {
}
private:
static photon::fs::IFileSystem *lfs;
static photon::fs::IFileSystem *cfs;
static photon::fs::IFileSystem *cfs1;
static FileSystem::ICachePool *pool;
static Cache::GzipCachedFs *cfs;
static photon::fs::IFile *gzdata;
static photon::fs::IFile *gzindex;

Expand Down Expand Up @@ -419,9 +402,7 @@ class GzCacheTest : public ::testing::Test {
};

photon::fs::IFileSystem *GzCacheTest::lfs = nullptr;
photon::fs::IFileSystem *GzCacheTest::cfs = nullptr;
photon::fs::IFileSystem *GzCacheTest::cfs1 = nullptr;
FileSystem::ICachePool *GzCacheTest::pool = nullptr;
Cache::GzipCachedFs *GzCacheTest::cfs = nullptr;
photon::fs::IFile *GzCacheTest::defile = nullptr;
photon::fs::IFile *GzCacheTest::gzfile = nullptr;
photon::fs::IFile *GzCacheTest::gzdata = nullptr;
Expand All @@ -447,7 +428,7 @@ TEST_F(GzCacheTest, cache_store) {
DEFER(delete []cbuf1);
DEFER(delete []cbuf2);
auto fp1 = fopen("/tmp/gzip_src/fdata", "r");
auto fp2 = fopen("/tmp/gzip_cache/fdata", "r");
auto fp2 = fopen("/tmp/gzip_cache_decompress/fdata", "r");
DEFER(fclose(fp1));
DEFER(fclose(fp2));
fread(cbuf1, 1, vsize, fp1);
Expand Down
6 changes: 4 additions & 2 deletions src/overlaybd/registryfs/registryfs_v2.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -208,8 +208,10 @@ class RegistryFSImpl_v2 : public RegistryFS {
HTTP_OP op(m_client, Verb::GET, url);
op.follow = 0;
op.retry = 0;
op.req.headers.insert(kAuthHeaderKey, "Bearer ");
op.req.headers.value_append(*token);
if (token && !token->empty()) {
op.req.headers.insert(kAuthHeaderKey, "Bearer ");
op.req.headers.value_append(*token);
}
op.timeout = tmo.timeout();
op.call();
code = op.status_code;
Expand Down

0 comments on commit d272e1c

Please sign in to comment.