new cache implementation

Co-authored-by: zhuangbowei.zbw <[email protected]>
Signed-off-by: Lanzheng Liu <[email protected]>

src/image_file.cpp

@@ -128,57 +128,59 @@ IFile *ImageFile::__open_ro_target_remote(const std::string &dir, const std::str
     return remote_file;
 }
 
+void get_error_msg(int eno, std::string &err_msg) {
+    if (eno == EPERM || eno == EACCES) {
+        err_msg = "Authentication failed";
+    } else if (eno == ENOTCONN) {
+        err_msg = "Connection failed";
+    } else if (eno == ETIMEDOUT) {
+        err_msg = "Get meta timedout";
+    } else if (eno == ENOENT) {
+        err_msg = "No such file or directory";
+    } else if (eno == EBUSY) {
+        err_msg = "Too many requests";
+    } else if (eno == EIO) {
+        err_msg = "Unexpected response";
+    } else {
+        err_msg = std::string(strerror(eno));
+    }
+}
+
 IFile *ImageFile::__open_ro_remote(const std::string &dir, const std::string &digest,
                                    const uint64_t size, int layer_index) {
-    std::string url;
-
     if (conf.repoBlobUrl() == "") {
         set_failed("empty repoBlobUrl");
         LOG_ERROR_RETURN(0, nullptr, "empty repoBlobUrl for remote layer");
     }
-    url = conf.repoBlobUrl();
-
-    if (url[url.length() - 1] != '/')
-        url += "/";
-    url += digest;
+    estring url = estring().appends("/", conf.repoBlobUrl(),
+                                    (conf.repoBlobUrl().back() != '/') ? "/" : "",
+                                    digest);
 
     LOG_INFO("open file from remotefs: `, size: `", url, size);
     IFile *remote_file = image_service.global_fs.remote_fs->open(url.c_str(), O_RDONLY);
     if (!remote_file) {
-        std::string err_msg = "failed to open remote file " + url + ": ";
-        if (errno == EPERM || errno == EACCES) {
-            err_msg += "Authentication failed";
-        } else if (errno == ENOTCONN) {
-            err_msg += "Connection failed";
-        } else if (errno == ETIMEDOUT) {
-            err_msg += "Get meta timedout";
-        } else if (errno == ENOENT) {
-            err_msg += "No such file or directory";
-        } else if (errno == EBUSY) {
-            err_msg += "Too many requests";
-        } else if (errno == EIO) {
-            err_msg += "Unexpected response";
-        } else {
-            err_msg += std::string(strerror(errno));
-        }
-        set_failed(err_msg);
-        LOG_ERRNO_RETURN(0, nullptr, err_msg);
+        std::string err_msg;
+        get_error_msg(errno, err_msg);
+        set_failed("failed to open remote file ", url, ": ", err_msg);
+        LOG_ERRNO_RETURN(0, nullptr, "failed to open remote file `: `", url, err_msg);
     }
     remote_file->ioctl(SET_SIZE, size);
     remote_file->ioctl(SET_LOCAL_DIR, dir);
 
     IFile *tar_file = new_tar_file_adaptor(remote_file);
     if (!tar_file) {
-        set_failed("failed to open remote file as tar file " + url);
+        std::string err_msg;
+        get_error_msg(errno, err_msg);
+        set_failed("failed to open remote file as tar file ", url, ": ", err_msg);
         delete remote_file;
-        LOG_ERROR_RETURN(0, nullptr, "failed to open remote file as tar file `", url);
+        LOG_ERRNO_RETURN(0, nullptr, "failed to open remote file as tar file `: `", url, err_msg);
     }
 
     ISwitchFile *switch_file = new_switch_file(tar_file, false, url.c_str());
     if (!switch_file) {
-        set_failed("failed to open switch file " + url);
+        set_failed("failed to open switch file ", url);
         delete tar_file;
-        LOG_ERROR_RETURN(0, nullptr, "failed to open switch file `", url);
+        LOG_ERRNO_RETURN(0, nullptr, "failed to open switch file `", url);
     }
 
     if (conf.HasMember("download") && conf.download().enable() == 1) {
@@ -517,10 +519,11 @@ void ImageFile::set_auth_failed() {
     }
 }
 
-void ImageFile::set_failed(std::string reason) {
+template<typename...Ts>
+void ImageFile::set_failed(const Ts&...xs) {
     if (m_status == 0) // only set exit in image boot phase
     {
         m_status = -1;
-        m_exception = reason;
+        m_exception = estring().appends(xs...);
     }
 }

src/image_file.h

@@ -116,7 +116,7 @@ class ImageFile : public photon::fs::ForwardFile {
     ImageService &image_service;
 
     int init_image_file();
-    void set_failed(std::string reason);
+    template<typename...Ts> void set_failed(const Ts&...xs);
     LSMT::IFileRO *open_lowers(std::vector<ImageConfigNS::LayerConfig> &, bool &);
     LSMT::IFileRW *open_upper(ImageConfigNS::UpperConfig &);
     IFile *__open_ro_file(const std::string &);

src/image_service.cpp

@@ -274,8 +274,16 @@ void ImageService::set_result_file(std::string &filename, std::string &data) {
               data.c_str());
 }
 
-static std::string cache_fn_trans_sha256(std::string_view path) {
-    return std::string(photon::fs::Path(path).basename());
+size_t cache_fn_trans_sha256(void *, std::string_view origin, char *name, size_t namesize) {
+    auto target = photon::fs::Path(origin).basename();
+    if (target.size()+2 > namesize) {
+        // return 0, no name trans, use origin name for cache
+        LOG_ERROR_RETURN(ERANGE, 0, "name out of range");
+    }
+    name[0] = '/';
+    strncpy(name + 1, target.data(), target.size());
+    name[target.size()+1] = 0;
+    return target.size()+1;
 }
 
 bool check_accelerate_url(std::string_view a_url) {
@@ -369,7 +377,7 @@ int ImageService::init() {
             // file cache will delete its src_fs automatically when destructed
             global_fs.cached_fs = FileSystem::new_full_file_cached_fs(
                 global_fs.srcfs, registry_cache_fs, refill_size, cache_size_GB, 10000000,
-                (uint64_t)1048576 * 4096, global_fs.io_alloc, cache_fn_trans_sha256);
+                (uint64_t)1048576 * 1024, global_fs.io_alloc, 0, {nullptr, &cache_fn_trans_sha256});
 
         } else if (cache_type == "ocf") {
             auto namespace_dir = std::string(cache_dir + "/namespace");

src/overlaybd/cache/CMakeLists.txt

@@ -1,4 +1,3 @@
-add_subdirectory(frontend)
 add_subdirectory(full_file_cache)
 add_subdirectory(ocf_cache)
 add_subdirectory(download_cache)
@@ -9,7 +8,6 @@ file(GLOB SRC_CACHE "*.cpp")
 add_library(cache_lib STATIC ${SRC_CACHE})
 target_link_libraries(cache_lib
     photon_static
-    cache_frontend_lib
     full_file_cache_lib
     ocf_cache_lib
     download_cache_lib
@@ -18,3 +16,7 @@ target_link_libraries(cache_lib
 target_include_directories(cache_lib PUBLIC
     ${PHOTON_INCLUDE_DIR}
 )
+
+if(BUILD_TESTING)
+  add_subdirectory(test)
+endif()

src/overlaybd/cache/cache.cpp

@@ -15,95 +15,99 @@
 */
 #include "cache.h"
 #include <photon/common/alog.h>
+#include <photon/common/alog-stdstring.h>
 #include <photon/common/io-alloc.h>
 #include <photon/common/iovector.h>
+#include <photon/common/expirecontainer.h>
+#include <photon/thread/thread-pool.h>
 
-#include "frontend/cached_file.h"
-#include "pool_store.h"
 #include "full_file_cache/cache_pool.h"
 
 namespace FileSystem {
 using namespace photon::fs;
-
-ICacheStore::try_preadv_result ICacheStore::try_preadv(const struct iovec *iov, int iovcnt,
-                                                       off_t offset) {
-    try_preadv_result rst;
-    iovector_view view((iovec *)iov, iovcnt);
-    rst.iov_sum = view.sum();
-    auto q = queryRefillRange(offset, rst.iov_sum);
-    if (q.second == 0) { // no need to refill
-        rst.refill_size = 0;
-        rst.size = this->preadv(iov, iovcnt, offset);
-    } else {
-        rst.refill_size = q.second;
-        rst.refill_offset = q.first;
-    }
-    return rst;
-}
-ICacheStore::try_preadv_result ICacheStore::try_preadv_mutable(struct iovec *iov, int iovcnt,
-                                                               off_t offset) {
-    return try_preadv(iov, iovcnt, offset);
-}
-ssize_t ICacheStore::preadv(const struct iovec *iov, int iovcnt, off_t offset) {
-    SmartCloneIOV<32> ciov(iov, iovcnt);
-    return preadv_mutable(ciov.iov, iovcnt, offset);
-}
-ssize_t ICacheStore::preadv_mutable(struct iovec *iov, int iovcnt, off_t offset) {
-    return preadv(iov, iovcnt, offset);
-}
-ssize_t ICacheStore::pwritev(const struct iovec *iov, int iovcnt, off_t offset) {
-    SmartCloneIOV<32> ciov(iov, iovcnt);
-    return pwritev_mutable(ciov.iov, iovcnt, offset);
-}
-ssize_t ICacheStore::pwritev_mutable(struct iovec *iov, int iovcnt, off_t offset) {
-    return pwritev(iov, iovcnt, offset);
-}
-
 ICachedFileSystem *new_full_file_cached_fs(IFileSystem *srcFs, IFileSystem *mediaFs,
                                            uint64_t refillUnit, uint64_t capacityInGB,
                                            uint64_t periodInUs, uint64_t diskAvailInBytes,
-                                           IOAlloc *allocator, Fn_trans_func name_trans) {
-    if (refillUnit % 4096 != 0) {
-        LOG_ERROR_RETURN(EINVAL, nullptr, "refill Unit need to be aligned to 4KB")
+                                           IOAlloc *allocator, int quotaDirLevel,
+                                           CacheFnTransFunc fn_trans_func) {
+    if (refillUnit % 4096 != 0 || !is_power_of_2(refillUnit)) {
+        LOG_ERROR_RETURN(EINVAL, nullptr, "refill Unit need to be aligned to 4KB and power of 2")
     }
     if (!allocator) {
         allocator = new IOAlloc;
     }
     Cache::FileCachePool *pool = nullptr;
     pool =
-        new ::Cache::FileCachePool(mediaFs, capacityInGB, periodInUs, diskAvailInBytes, refillUnit, name_trans);
+        new ::Cache::FileCachePool(mediaFs, capacityInGB, periodInUs, diskAvailInBytes, refillUnit);
     pool->Init();
-    return new_cached_fs(srcFs, pool, 4096, refillUnit, allocator);
+    return new_cached_fs(srcFs, pool, 4096, allocator, fn_trans_func);
+}
+
+using OC = ObjectCache<std::string, ICacheStore *>;
+ICachePool::ICachePool(uint32_t pool_size, uint32_t max_refilling, uint32_t refilling_threshold)
+    : m_stores(new OC(10UL * 1000 * 1000)), m_max_refilling(max_refilling),
+      m_refilling_threshold(refilling_threshold) {
+    if (pool_size != 0) {
+        m_thread_pool = photon::new_thread_pool(pool_size, 128 * 1024UL);
+        m_vcpu = photon::get_vcpu();
+    };
+}
+
+#define cast(x) static_cast<OC *>(x)
+ICachePool::~ICachePool() {
+    stores_clear();
+    delete cast(m_stores);
+}
+
+void ICachePool::stores_clear() {
+    if (m_thread_pool) {
+        auto pool = static_cast<photon::ThreadPoolBase *>(m_thread_pool);
+        m_thread_pool = nullptr;
+        photon::delete_thread_pool(pool);
+    }
+    cast(m_stores)->clear();
 }
 
 ICacheStore *ICachePool::open(std::string_view filename, int flags, mode_t mode) {
-    ICacheStore *cache_store = nullptr;
-    auto it = m_stores.find(filename);
-    if (it != m_stores.end())
-        cache_store = it->second;
-    if (cache_store == nullptr) {
-        cache_store = this->do_open(filename, flags, mode);
+    char store_name[4096];
+    std::string x(filename);
+    auto len = this->fn_trans_func(filename, store_name, sizeof(store_name));
+    std::string_view store_sv = len ? std::string_view(store_name, len) : filename;
+    auto ctor = [&]() -> ICacheStore * {
+        auto cache_store = this->do_open(store_sv, flags, mode);
         if (nullptr == cache_store) {
             LOG_ERRNO_RETURN(0, nullptr, "fileCachePool_ open file failed, name : `",
                              filename.data());
         }
-        m_stores.emplace(filename, cache_store);
-        auto it = m_stores.find(filename);
-        std::string_view map_key = it->first;
-        cache_store->set_pathname(map_key);
+        auto it = cast(m_stores)->find(store_sv);
+        std::string_view map_key = (*it)->key();
+        cache_store->set_store_key(map_key);
+        cache_store->set_src_name(filename);
         cache_store->set_pool(this);
+        struct stat st;
+        SET_STRUCT_STAT(&st);
+        st.st_size = -1;
+        if (cache_store->fstat(&st) == 0) {
+            cache_store->set_cached_size(st.st_size);
+            cache_store->set_actual_size(st.st_size);
+        }
+        cache_store->set_open_flags(flags);
+        return cache_store;
+    };
+    auto store = cast(m_stores)->acquire(store_sv, ctor);
+    if (store) {
+        auto cnt = store->ref_.fetch_add(1, std::memory_order_relaxed);
+        if (cnt)
+            cast(m_stores)->release(store_sv);
     }
-    cache_store->add_ref();
-    return cache_store;
+    return store;
+}
+
+void ICachePool::set_trans_func(CacheFnTransFunc fn_trans_func) {
+    this->fn_trans_func = fn_trans_func;
 }
 
 int ICachePool::store_release(ICacheStore *store) {
-    auto iter = m_stores.find(store->get_pathname());
-    if (iter == m_stores.end()) {
-        LOG_ERROR_RETURN(0, -1, "try to erase an unexist store from map m_stores , name : `",
-                         store->get_pathname().data());
-    }
-    m_stores.erase(iter);
-    return 0;
+    return cast(m_stores)->release(store->get_store_key());
 }
 } // namespace FileSystem