Merge pull request #13 from argonne-lcf/size_t_large_files_read

Size t large files read

src/cache/path_status_cache.cpp

@@ -12,7 +12,7 @@ bool PathStatusCache::check_and_put_force(const Key& key) {
     return true;
 }
 
-void PathStatusCache::update_cache_status(const Key& key, const int status) {
+void PathStatusCache::update_cache_status(const Key& key, const ssize_t status) {
     std::lock_guard guard(mtx);
 
     auto entry = cache.find(key);

src/cache/path_status_cache.h

@@ -4,6 +4,7 @@
 #include <mutex>
 #include <string>
 #include <thallium.hpp>
+#include <stddef.h>
 
 namespace tl = thallium;
 
@@ -16,7 +17,7 @@ using PathStatusMDCacheTableVal = std::optional<int>;
 class PathStatusCache final : public TLCache<PathStatusCacheTableKey, PathStatusMDCacheTableVal> {
     public:
     bool check_and_put_force(const Key& key);
-    void update_cache_status(const Key&, int status);
+    void update_cache_status(const Key&, ssize_t status);
     int wait_on_cache_status(const Key& key);
 
     tl::condition_variable cv;

src/copper/server_local_cache_provider.h

@@ -86,8 +86,8 @@ class ServerLocalCacheProvider : public tl::provider<ServerLocalCacheProvider> {
     using lstat_final_return_type = int;
     void rpcLstat(const tl::request& req, bool dest, const std::string& path_string) const;
 
-    using read_return_type = std::pair<int, std::vector<std::byte>>;
-    using read_final_return_type = int;
+    using read_return_type = std::pair<ssize_t, std::vector<std::byte>>;
+    using read_final_return_type = ssize_t;
     void rpcRead(const tl::request& req, bool dest, const std::string& path_string) const;
 
     using readdir_return_type = std::pair<int, std::vector<std::string>>;

src/fs/cu_fuse.cpp

@@ -69,13 +69,13 @@ static int cu_fuse_read(const char* path_, char* buf, const size_t size, const o
     LOG(DEBUG) << "requested size: " << size << std::endl;
 
     auto md_entry = CacheTables::md_cache_table.get(path_string);
-    if(md_entry.has_value()) {
+    /*if(md_entry.has_value()) {
         struct stat* md_st = (struct stat*)md_entry.value()->get_vec().data();
         if(md_st->st_size >= Constants::max_cacheable_byte_size) {
             LOG(INFO) << "file larger than max cacheable size... going to lustre" << std::endl;
 
             int fd = open(path_string.c_str(), O_RDONLY);
-            int res = pread(fd, buf, size, offset);
+            ssize_t res = pread(fd, buf, size, offset);
             if(res == -1) {
                 res = -errno;
             }
@@ -84,7 +84,7 @@ static int cu_fuse_read(const char* path_, char* buf, const size_t size, const o
             return Metric::stop_cache_operation(
             OperationFunction::read, OperationResult::neg, CacheEvent::data_cache_event_table, path_string, start, res);
         }
-    }
+    }*/
 
     const auto entry_opt = CacheTables::data_cache_table.get(path_string);
     std::vector<std::byte>* bytes = nullptr;
@@ -114,14 +114,14 @@ static int cu_fuse_read(const char* path_, char* buf, const size_t size, const o
         bytes = entry_opt.value();
     }
 
-    int read_size = 0;
+    size_t read_size = 0;
     if(offset < static_cast<off_t>(bytes->size())) {
         // Calculate the amount to copy, ensuring not to exceed the bounds of the vector
-        size_t copy_size = std::min(static_cast<size_t>(size), bytes->size() - static_cast<size_t>(offset));
+        ssize_t copy_size = std::min(static_cast<size_t>(size), bytes->size() - static_cast<size_t>(offset));
 
         // Copy data from bytes to buf
         std::memcpy(buf, bytes->data() + offset, copy_size);
-        read_size = static_cast<int>(copy_size);
+        read_size = static_cast<size_t>(copy_size);
     }
 
     if(cache) {

src/fs/util.cpp

@@ -151,17 +151,31 @@ std::vector<std::string> Util::process_args(const int argc, char* argv[]) {
 }
 
 std::vector<std::byte> Util::read_ent_file(const std::string& path) {
-    if(std::ifstream source_file{path, std::ios::binary}) {
-        std::streamsize file_size{};
-        file_size = static_cast<std::streamsize>(std::filesystem::file_size(path));
-        std::vector<std::byte> bytes(file_size);
-        source_file.read(reinterpret_cast<char*>(bytes.data()), file_size);
-        source_file.close();
-
-        return bytes;
+    int fd = open(path.c_str(), O_RDONLY);
+    if (fd == -1) {
+	LOG(WARNING) << "read file failed with errno: " << std::strerror(errno) << std::endl; 
+        throw std::system_error(errno, std::generic_category(), "failed to open file: " + path);
     }
 
-    throw std::runtime_error("failed to open path: " + path);
+    struct stat file_stat;
+    if (fstat(fd, &file_stat) == -1) {
+        close(fd);
+	LOG(WARNING) << "read file failed with errno: " << std::strerror(errno) << std::endl; 
+        throw std::system_error(errno, std::generic_category(), "failed to get file size for: " + path);
+    }
+
+    std::vector<std::byte> bytes(file_stat.st_size);
+
+    ssize_t bytes_read = read(fd, bytes.data(), bytes.size());
+    if (bytes_read == -1) {
+        close(fd);
+	LOG(WARNING) << "read file failed with errno: " << std::strerror(errno) << std::endl; 
+        throw std::system_error(errno, std::generic_category(), "failed to read file: " + path);
+    }
+
+    close(fd);
+
+    return bytes;
 }
 
 std::optional<std::ofstream> Util::try_get_fstream_from_path(const char* path) {