From 680d24a4bd959b572271741577e043b4091c075b Mon Sep 17 00:00:00 2001 From: Syoyo Fujita Date: Mon, 14 Oct 2024 03:33:13 +0900 Subject: [PATCH 1/3] Use mmap to load USD file for faster loading. --- src/io-util.cc | 143 +++++++++++++++++++++++---- src/io-util.hh | 10 +- src/tinyusdz.cc | 250 ++++++++++++++++++++++++++++++++++++++---------- 3 files changed, 332 insertions(+), 71 deletions(-) diff --git a/src/io-util.cc b/src/io-util.cc index 6ce44b3a..4f9e2681 100644 --- a/src/io-util.cc +++ b/src/io-util.cc @@ -90,6 +90,42 @@ namespace tinyusdz { namespace io { +#if defined(_WIN32) +namespace { + +// from llama.cpp ---- +// MIT license +std::string GetErrorMessageWin32(DWORD error_code) { + std::string ret; + LPSTR lpMsgBuf = NULL; + DWORD bufLen = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, error_code, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&lpMsgBuf, 0, NULL); + if (!bufLen) { + ret = "Win32 error code: " + std::to_string(error_code); + } else { + ret = lpMsgBuf; + LocalFree(lpMsgBuf); + } + + return ret; +} + +static std::string llama_format_win_err(DWORD err) { + LPSTR buf; + size_t size = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, err, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&buf, 0, NULL); + if (!size) { + return "FormatMessageA failed"; + } + std::string ret(buf, size); + LocalFree(buf); + return ret; +} +// ---- + +} +#endif + #ifdef TINYUSDZ_ANDROID_LOAD_FROM_ASSETS AAssetManager *asset_manager = nullptr; #endif @@ -103,28 +139,87 @@ bool IsMMapSupported() { #endif } -bool MMapFile(const std::string &filepath, MMapFileHandle *handle, bool writable) { - (void)filepath; - (void)handle; - (void)writable; +bool MMapFile(const std::string &filepath, MMapFileHandle *handle, bool writable, std::string *err) { #if TINYUSDZ_MMAP_SUPPORTED #if defined(_WIN32) -#if 0 // TODO - int fd = open(filepath.c_str(), writable ? O_RDWR : O_RDONLY); - HANDLE hFile = _get_ofhandle(fd); - HANDLE hMapping = CreateFileMapping(hFile, nullptr, PAGE_READWRITE, 0, 0, nullptr); + //int fd = open(filepath.c_str(), writable ? O_RDWR : O_RDONLY); + HANDLE hFile = CreateFile(filepath.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); + if (hFile == INVALID_HANDLE_VALUE) { + if (err) { + (*err) += "Failed to open file."; + } + return false; + } + + uint64_t size{0}; + { + LARGE_INTEGER sz{}; + if (!GetFileSizeEx(hFile, &sz)) { + if (err) { + (*err) += "GetFileSizeEx failed: " + llama_format_win_err(GetLastError()); + } + return false; + } + + size = sz.QuadPart; + } + + HANDLE hMapping = CreateFileMapping(hFile, nullptr, writable ? PAGE_READWRITE : PAGE_READONLY, 0, 0, nullptr); if (hMapping == nullptr) { + if (err) { + (*err) += "CreateFileMapping failed: " + llama_format_win_err(GetLastError()); + } return false; } - void *data = MapViewOfFile(hMapping, FILE_MAP_READ, 0, 0, 0); - if (!data) { + void *addr = MapViewOfFile(hMapping, FILE_MAP_READ, 0, 0, 0); + DWORD lastError = GetLastError(); + CloseHandle(hMapping); + if (!addr) { + if (err) { + (*err) += "MapViewOfFile failed: " + llama_format_win_err(lastError); + } return false; } - CloseHandle(hMapping); -#else - return false; -#endif + + size_t prefetch = 0; // TODO + if (prefetch > 0) { + #if _WIN32_WINNT >= 0x602 + // PrefetchVirtualMemory is only present on Windows 8 and above, so we dynamically load it + BOOL (WINAPI *pPrefetchVirtualMemory) (HANDLE, ULONG_PTR, PWIN32_MEMORY_RANGE_ENTRY, ULONG); + HMODULE hKernel32 = GetModuleHandleW(L"kernel32.dll"); + + // may fail on pre-Windows 8 systems + pPrefetchVirtualMemory = reinterpret_cast (GetProcAddress(hKernel32, "PrefetchVirtualMemory")); + + if (pPrefetchVirtualMemory) { + // advise the kernel to preload the mapped memory + WIN32_MEMORY_RANGE_ENTRY range; + range.VirtualAddress = addr; + range.NumberOfBytes = static_cast( (std::min)(size, prefetch) ); + if (!pPrefetchVirtualMemory(GetCurrentProcess(), 1, &range, 0)) { + // warn + if (err) { + (*err) += "warning: PrefetchVirtualMemory failed: " + llama_format_win_err(GetLastError()); + } + } + } + #else + throw std::runtime_error("PrefetchVirtualMemory unavailable"); + if (err) { + (*err) += "PrefetchVirtualMemory unavailable"; + } + return false; + #endif + } + + handle->addr = reinterpret_cast(addr); + handle->size = size; + handle->writable = writable; + handle->filename = filepath; + + return true; + #else // !WIN32 // assume posix FILE *fp = fopen(filepath.c_str(), writable ? "rw" : "r"); @@ -150,7 +245,7 @@ bool MMapFile(const std::string &filepath, MMapFileHandle *handle, bool writable } handle->addr = reinterpret_cast(addr); - handle->size = size; + handle->size = uint64_t(size); handle->writable = writable; handle->filename = filepath; close(fd); @@ -158,15 +253,26 @@ bool MMapFile(const std::string &filepath, MMapFileHandle *handle, bool writable return true; #endif // !WIN32 #else // !TINYUSDZ_MMAP_SUPPORTED + (void)filepath; + (void)handle; + (void)writable; return false; #endif } -bool UnmapFile(const MMapFileHandle &handle) { - (void)handle; +bool UnmapFile(const MMapFileHandle &handle, std::string *err) { #if TINYUSDZ_MMAP_SUPPORTED #if defined(_WIN32) - // TODO + if (handle.addr && handle.size) { + if (!UnmapViewOfFile(handle.addr)) { + if (err) { + (*err) += "warning: UnmapViewOfFile failed: " + llama_format_win_err(GetLastError()); + } + // May ok for now + return true; + } + } + return false; #else // !WIN32 if (handle.addr && handle.size) { @@ -178,6 +284,7 @@ bool UnmapFile(const MMapFileHandle &handle) { return false; #endif #else // !TINYUSDZ_MMAP_SUPPORTED + (void)handle; return false; #endif } diff --git a/src/io-util.hh b/src/io-util.hh index 18a7e98a..54529c0d 100644 --- a/src/io-util.hh +++ b/src/io-util.hh @@ -133,16 +133,20 @@ struct MMapFileHandle std::string filename; bool writable{false}; uint8_t *addr{nullptr}; - size_t size{0}; + uint64_t size{0}; }; /// /// memory-map file. /// Returns false when file is not found, invalid, or mmap feature is not available. +/// err = warning message when the API returns true. /// -bool MMapFile(const std::string &filepath, MMapFileHandle *handle, bool writable = false); +bool MMapFile(const std::string &filepath, MMapFileHandle *handle, bool writable, std::string *err); -bool UnmapFile(const MMapFileHandle &handle); +/// +/// err = warning message when the API returns true. +/// +bool UnmapFile(const MMapFileHandle &handle, std::string *err); /// /// Filepath is treated as WideChar(UNICODE) on Windows. diff --git a/src/tinyusdz.cc b/src/tinyusdz.cc index 1c186996..bbb7f978 100644 --- a/src/tinyusdz.cc +++ b/src/tinyusdz.cc @@ -176,30 +176,68 @@ bool LoadUSDCFromFile(const std::string &_filename, Stage *stage, const USDLoadOptions &options) { std::string filepath = io::ExpandFilePath(_filename, /* userdata */ nullptr); - std::vector data; - size_t max_bytes = 1024 * 1024 * size_t(options.max_memory_limit_in_mb); - if (!io::ReadWholeFile(&data, err, filepath, max_bytes, - /* userdata */ nullptr)) { - if (err) { - (*err) += "File not found or failed to read : \"" + filepath + "\"\n"; + if (io::IsMMapSupported()) { + io::MMapFileHandle handle; + + { + std::string _err; + if (!io::MMapFile(filepath, &handle, /* writable */false, &_err)) { + if (err) { + (*err) += _err + "\n"; + } + return false; + } + + if (_err.size()) { + if (warn) { + (*warn) += _err + "\n"; + } + } } - return false; - } + bool ret = LoadUSDCFromMemory(handle.addr, size_t(handle.size), filepath, stage, warn, + err, options); - DCOUT("File size: " + std::to_string(data.size()) + " bytes."); + { + std::string _err; + // Ignore unmap result for now. + io::UnmapFile(handle, &_err); - if (data.size() < (11 * 8)) { - // ??? - if (err) { - (*err) += "File size too short. Looks like this file is not a USDC : \"" + - filepath + "\"\n"; + if (_err.size()) { + if (warn) { + (*warn) += _err + "\n"; + } + } + } + + return ret; + + } else { + std::vector data; + size_t max_bytes = 1024 * 1024 * size_t(options.max_memory_limit_in_mb); + if (!io::ReadWholeFile(&data, err, filepath, max_bytes, + /* userdata */ nullptr)) { + if (err) { + (*err) += "File not found or failed to read : \"" + filepath + "\"\n"; + } + + return false; + } + + DCOUT("File size: " + std::to_string(data.size()) + " bytes."); + + if (data.size() < (11 * 8)) { + // ??? + if (err) { + (*err) += "File size too short. Looks like this file is not a USDC : \"" + + filepath + "\"\n"; + } + return false; } - return false; - } - return LoadUSDCFromMemory(data.data(), data.size(), filepath, stage, warn, - err, options); + return LoadUSDCFromMemory(data.data(), data.size(), filepath, stage, warn, + err, options); + } } namespace { @@ -599,24 +637,62 @@ bool LoadUSDZFromFile(const std::string &_filename, Stage *stage, std::string filepath = io::ExpandFilePath(_filename, /* userdata */ nullptr); - std::vector data; - size_t max_bytes = 1024 * 1024 * size_t(options.max_memory_limit_in_mb); - if (!io::ReadWholeFile(&data, err, filepath, max_bytes, - /* userdata */ nullptr)) { - return false; - } - if (data.size() < (11 * 8) + 30) { // 88 for USDC header, 30 for ZIP header - // ??? - if (err) { - (*err) += "File size too short. Looks like this file is not a USDZ : \"" + - filepath + "\"\n"; + if (io::IsMMapSupported()) { + io::MMapFileHandle handle; + + { + std::string _err; + if (!io::MMapFile(filepath, &handle, /* writable */false, &_err)) { + if (err) { + (*err) += _err + "\n"; + } + return false; + } + + if (_err.size()) { + if (warn) { + (*warn) += _err + "\n"; + } + } } - return false; - } - return LoadUSDZFromMemory(data.data(), data.size(), filepath, stage, warn, - err, options); + bool ret = LoadUSDZFromMemory(handle.addr, size_t(handle.size), filepath, stage, warn, + err, options); + + { + std::string _err; + // Ignore unmap result for now. + io::UnmapFile(handle, &_err); + + if (_err.size()) { + if (warn) { + (*warn) += _err + "\n"; + } + } + } + + return ret; + } else { + std::vector data; + size_t max_bytes = 1024 * 1024 * size_t(options.max_memory_limit_in_mb); + if (!io::ReadWholeFile(&data, err, filepath, max_bytes, + /* userdata */ nullptr)) { + return false; + } + + if (data.size() < (11 * 8) + 30) { // 88 for USDC header, 30 for ZIP header + // ??? + if (err) { + (*err) += "File size too short. Looks like this file is not a USDZ : \"" + + filepath + "\"\n"; + } + return false; + } + + return LoadUSDZFromMemory(data.data(), data.size(), filepath, stage, warn, + err, options); + } } #ifdef _WIN32 @@ -695,17 +771,54 @@ bool LoadUSDAFromFile(const std::string &_filename, Stage *stage, std::string filepath = io::ExpandFilePath(_filename, /* userdata */ nullptr); std::string base_dir = io::GetBaseDir(_filename); - std::vector data; - size_t max_bytes = 1024 * 1024 * size_t(options.max_memory_limit_in_mb); - if (!io::ReadWholeFile(&data, err, filepath, max_bytes, - /* userdata */ nullptr)) { - if (err) { - (*err) += "File not found or failed to read : \"" + filepath + "\"\n"; + if (io::IsMMapSupported()) { + io::MMapFileHandle handle; + + { + std::string _err; + if (!io::MMapFile(filepath, &handle, /* writable */false, &_err)) { + if (err) { + (*err) += _err + "\n"; + } + return false; + } + + if (_err.size()) { + if (warn) { + (*warn) += _err + "\n"; + } + } } - } - return LoadUSDAFromMemory(data.data(), data.size(), base_dir, stage, warn, - err, options); + bool ret = LoadUSDAFromMemory(handle.addr, size_t(handle.size), filepath, stage, warn, + err, options); + + { + std::string _err; + // Ignore unmap result for now. + io::UnmapFile(handle, &_err); + + if (_err.size()) { + if (warn) { + (*warn) += _err + "\n"; + } + } + } + + return ret; + } else { + std::vector data; + size_t max_bytes = 1024 * 1024 * size_t(options.max_memory_limit_in_mb); + if (!io::ReadWholeFile(&data, err, filepath, max_bytes, + /* userdata */ nullptr)) { + if (err) { + (*err) += "File not found or failed to read : \"" + filepath + "\"\n"; + } + } + + return LoadUSDAFromMemory(data.data(), data.size(), base_dir, stage, warn, + err, options); + } } bool LoadUSDFromFile(const std::string &_filename, Stage *stage, @@ -714,15 +827,52 @@ bool LoadUSDFromFile(const std::string &_filename, Stage *stage, std::string filepath = io::ExpandFilePath(_filename, /* userdata */ nullptr); std::string base_dir = io::GetBaseDir(_filename); - std::vector data; - size_t max_bytes = 1024 * 1024 * size_t(options.max_memory_limit_in_mb); - if (!io::ReadWholeFile(&data, err, filepath, max_bytes, - /* userdata */ nullptr)) { - return false; - } + if (io::IsMMapSupported()) { + io::MMapFileHandle handle; + + { + std::string _err; + if (!io::MMapFile(filepath, &handle, /* writable */false, &_err)) { + if (err) { + (*err) += _err + "\n"; + } + return false; + } - return LoadUSDFromMemory(data.data(), data.size(), base_dir, stage, warn, err, - options); + if (_err.size()) { + if (warn) { + (*warn) += _err + "\n"; + } + } + } + + bool ret = LoadUSDFromMemory(handle.addr, size_t(handle.size), filepath, stage, warn, + err, options); + + { + std::string _err; + // Ignore unmap result for now. + io::UnmapFile(handle, &_err); + + if (_err.size()) { + if (warn) { + (*warn) += _err + "\n"; + } + } + } + + return ret; + } else { + std::vector data; + size_t max_bytes = 1024 * 1024 * size_t(options.max_memory_limit_in_mb); + if (!io::ReadWholeFile(&data, err, filepath, max_bytes, + /* userdata */ nullptr)) { + return false; + } + + return LoadUSDFromMemory(data.data(), data.size(), base_dir, stage, warn, err, + options); + } } bool LoadUSDFromMemory(const uint8_t *addr, const size_t length, From 53dbc2bbf119e16ef013d78b7797814eb3488478 Mon Sep 17 00:00:00 2001 From: Syoyo Fujita Date: Mon, 14 Oct 2024 04:23:16 +0900 Subject: [PATCH 2/3] Fix build on non-win32 environment. Apply clang-format. --- src/io-util.cc | 252 +++++++++++++++++++++++++++++-------------------- 1 file changed, 149 insertions(+), 103 deletions(-) diff --git a/src/io-util.cc b/src/io-util.cc index 4f9e2681..47019217 100644 --- a/src/io-util.cc +++ b/src/io-util.cc @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache 2.0 // Copyright 2022 - 2023, Syoyo Fujita. // Copyright 2023 - Present, Light Transport Entertainment Inc. -// +// #include #include @@ -17,14 +17,13 @@ #define WIN32_LEAN_AND_MEAN #endif -#include // include API for expanding a file path #include +#include // include API for expanding a file path #ifndef TINYUSDZ_MMAP_SUPPORTED #define TINYUSDZ_MMAP_SUPPORTED (1) #endif - #ifdef _MSC_VER #undef NOMINMAX #endif @@ -52,16 +51,14 @@ #else // Assume Posix -#include - -#include #include +#include +#include #ifndef TINYUSDZ_MMAP_SUPPORTED #define TINYUSDZ_MMAP_SUPPORTED (1) #endif - #endif #endif // _WIN32 @@ -96,41 +93,46 @@ namespace { // from llama.cpp ---- // MIT license std::string GetErrorMessageWin32(DWORD error_code) { - std::string ret; - LPSTR lpMsgBuf = NULL; - DWORD bufLen = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, - NULL, error_code, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&lpMsgBuf, 0, NULL); - if (!bufLen) { - ret = "Win32 error code: " + std::to_string(error_code); - } else { - ret = lpMsgBuf; - LocalFree(lpMsgBuf); - } + std::string ret; + LPSTR lpMsgBuf = NULL; + DWORD bufLen = FormatMessageA( + FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, error_code, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + (LPSTR)&lpMsgBuf, 0, NULL); + if (!bufLen) { + ret = "Win32 error code: " + std::to_string(error_code); + } else { + ret = lpMsgBuf; + LocalFree(lpMsgBuf); + } - return ret; + return ret; } static std::string llama_format_win_err(DWORD err) { - LPSTR buf; - size_t size = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, - NULL, err, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&buf, 0, NULL); - if (!size) { - return "FormatMessageA failed"; - } - std::string ret(buf, size); - LocalFree(buf); - return ret; + LPSTR buf; + size_t size = FormatMessageA( + FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, err, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&buf, 0, + NULL); + if (!size) { + return "FormatMessageA failed"; + } + std::string ret(buf, size); + LocalFree(buf); + return ret; } // ---- -} +} // namespace #endif #ifdef TINYUSDZ_ANDROID_LOAD_FROM_ASSETS AAssetManager *asset_manager = nullptr; #endif - bool IsMMapSupported() { #if TINYUSDZ_MMAP_SUPPORTED return true; @@ -139,12 +141,22 @@ bool IsMMapSupported() { #endif } -bool MMapFile(const std::string &filepath, MMapFileHandle *handle, bool writable, std::string *err) { +bool MMapFile(const std::string &filepath, MMapFileHandle *handle, + bool writable, std::string *err) { + + if (!FileExists(filepath, /* userdata */nullptr)) { + if (err) { + (*err) += "File not found: " + filepath + "\n"; + } + return false; + } #if TINYUSDZ_MMAP_SUPPORTED #if defined(_WIN32) - //int fd = open(filepath.c_str(), writable ? O_RDWR : O_RDONLY); - HANDLE hFile = CreateFile(filepath.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); + // int fd = open(filepath.c_str(), writable ? O_RDWR : O_RDONLY); + HANDLE hFile = + CreateFile(filepath.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, + OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); if (hFile == INVALID_HANDLE_VALUE) { if (err) { (*err) += "Failed to open file."; @@ -157,90 +169,115 @@ bool MMapFile(const std::string &filepath, MMapFileHandle *handle, bool writable LARGE_INTEGER sz{}; if (!GetFileSizeEx(hFile, &sz)) { if (err) { - (*err) += "GetFileSizeEx failed: " + llama_format_win_err(GetLastError()); + (*err) += + "GetFileSizeEx failed: " + llama_format_win_err(GetLastError()); } return false; } size = sz.QuadPart; } - - HANDLE hMapping = CreateFileMapping(hFile, nullptr, writable ? PAGE_READWRITE : PAGE_READONLY, 0, 0, nullptr); - if (hMapping == nullptr) { - if (err) { - (*err) += "CreateFileMapping failed: " + llama_format_win_err(GetLastError()); - } - return false; - } - void *addr = MapViewOfFile(hMapping, FILE_MAP_READ, 0, 0, 0); - DWORD lastError = GetLastError(); - CloseHandle(hMapping); - if (!addr) { + + HANDLE hMapping = CreateFileMapping( + hFile, nullptr, writable ? PAGE_READWRITE : PAGE_READONLY, 0, 0, nullptr); + if (hMapping == nullptr) { + if (err) { + (*err) += + "CreateFileMapping failed: " + llama_format_win_err(GetLastError()); + } + return false; + } + void *addr = MapViewOfFile(hMapping, FILE_MAP_READ, 0, 0, 0); + DWORD lastError = GetLastError(); + CloseHandle(hMapping); + if (!addr) { + if (err) { + (*err) += "MapViewOfFile failed: " + llama_format_win_err(lastError); + } + return false; + } + + size_t prefetch = 0; // TODO + if (prefetch > 0) { +#if _WIN32_WINNT >= 0x602 + // PrefetchVirtualMemory is only present on Windows 8 and above, so we + // dynamically load it + BOOL(WINAPI * pPrefetchVirtualMemory) + (HANDLE, ULONG_PTR, PWIN32_MEMORY_RANGE_ENTRY, ULONG); + HMODULE hKernel32 = GetModuleHandleW(L"kernel32.dll"); + + // may fail on pre-Windows 8 systems + pPrefetchVirtualMemory = reinterpret_cast( + GetProcAddress(hKernel32, "PrefetchVirtualMemory")); + + if (pPrefetchVirtualMemory) { + // advise the kernel to preload the mapped memory + WIN32_MEMORY_RANGE_ENTRY range; + range.VirtualAddress = addr; + range.NumberOfBytes = static_cast((std::min)(size, prefetch)); + if (!pPrefetchVirtualMemory(GetCurrentProcess(), 1, &range, 0)) { + // warn if (err) { - (*err) += "MapViewOfFile failed: " + llama_format_win_err(lastError); + (*err) += "warning: PrefetchVirtualMemory failed: " + + llama_format_win_err(GetLastError()); } - return false; } + } +#else + throw std::runtime_error("PrefetchVirtualMemory unavailable"); + if (err) { + (*err) += "PrefetchVirtualMemory unavailable"; + } + return false; +#endif + } - size_t prefetch = 0; // TODO - if (prefetch > 0) { - #if _WIN32_WINNT >= 0x602 - // PrefetchVirtualMemory is only present on Windows 8 and above, so we dynamically load it - BOOL (WINAPI *pPrefetchVirtualMemory) (HANDLE, ULONG_PTR, PWIN32_MEMORY_RANGE_ENTRY, ULONG); - HMODULE hKernel32 = GetModuleHandleW(L"kernel32.dll"); - - // may fail on pre-Windows 8 systems - pPrefetchVirtualMemory = reinterpret_cast (GetProcAddress(hKernel32, "PrefetchVirtualMemory")); - - if (pPrefetchVirtualMemory) { - // advise the kernel to preload the mapped memory - WIN32_MEMORY_RANGE_ENTRY range; - range.VirtualAddress = addr; - range.NumberOfBytes = static_cast( (std::min)(size, prefetch) ); - if (!pPrefetchVirtualMemory(GetCurrentProcess(), 1, &range, 0)) { - // warn - if (err) { - (*err) += "warning: PrefetchVirtualMemory failed: " + llama_format_win_err(GetLastError()); - } - } - } - #else - throw std::runtime_error("PrefetchVirtualMemory unavailable"); - if (err) { - (*err) += "PrefetchVirtualMemory unavailable"; - } - return false; - #endif - } - - handle->addr = reinterpret_cast(addr); - handle->size = size; - handle->writable = writable; - handle->filename = filepath; + handle->addr = reinterpret_cast(addr); + handle->size = size; + handle->writable = writable; + handle->filename = filepath; return true; - -#else // !WIN32 + +#else // !WIN32 // assume posix FILE *fp = fopen(filepath.c_str(), writable ? "rw" : "r"); + if (!fp) { + if (err) { + (*err) += "fopen failed."; + } + return false; + } + int ret = std::fseek(fp, 0, SEEK_END); if (ret != 0) { + if (err) { + (*err) += "Failed to fseek."; + } fclose(fp); return false; - } + } size_t size = size_t(std::ftell(fp)); std::fseek(fp, 0, SEEK_SET); if (size == 0) { + if (err) { + (*err) += "File size is zero."; + } return false; } - + int fd = fileno(fp); - - int flags = MAP_PRIVATE; // delayed access - void *addr = mmap(nullptr, size, writable ? PROT_READ|PROT_WRITE : PROT_READ, flags, fd, 0); + + int flags = MAP_PRIVATE; // delayed access + void *addr = + mmap(nullptr, size, writable ? PROT_READ | PROT_WRITE : PROT_READ, flags, + fd, 0); if (addr == MAP_FAILED) { + if (err) { + (*err) += "mmap failed."; + } return false; } @@ -251,11 +288,12 @@ bool MMapFile(const std::string &filepath, MMapFileHandle *handle, bool writable close(fd); return true; -#endif // !WIN32 -#else // !TINYUSDZ_MMAP_SUPPORTED +#endif // !WIN32 +#else // !TINYUSDZ_MMAP_SUPPORTED (void)filepath; (void)handle; (void)writable; + (void)err; return false; #endif } @@ -266,7 +304,8 @@ bool UnmapFile(const MMapFileHandle &handle, std::string *err) { if (handle.addr && handle.size) { if (!UnmapViewOfFile(handle.addr)) { if (err) { - (*err) += "warning: UnmapViewOfFile failed: " + llama_format_win_err(GetLastError()); + (*err) += "warning: UnmapViewOfFile failed: " + + llama_format_win_err(GetLastError()); } // May ok for now return true; @@ -274,22 +313,26 @@ bool UnmapFile(const MMapFileHandle &handle, std::string *err) { } return false; -#else // !WIN32 +#else // !WIN32 if (handle.addr && handle.size) { - int ret = munmap(reinterpret_cast(handle.addr), handle.size); + int ret = munmap(reinterpret_cast(handle.addr), handle.size); + if (!ret) { + if (err) { + (*err) += "warning: munmap failed."; + } + } // ignore return code for now - (void)ret; return true; } return false; #endif -#else // !TINYUSDZ_MMAP_SUPPORTED +#else // !TINYUSDZ_MMAP_SUPPORTED (void)handle; + (void)err; return false; #endif } - std::string ExpandFilePath(const std::string &_filepath, void *) { std::string filepath = _filepath; if (filepath.size() > 2048) { @@ -396,10 +439,10 @@ bool ReadWholeFile(std::vector *out, std::string *err, size_t size = size_t(len); if (size >= filesize_max) { - (*err) += "File size exceeds filesize_max : " + filepath + - " (filesize_max " + std::to_string(filesize_max) + ")"; + (*err) += "File size exceeds filesize_max : " + filepath + + " (filesize_max " + std::to_string(filesize_max) + ")"; - return false; + return false; } out->resize(size); @@ -443,7 +486,9 @@ bool ReadWholeFile(std::vector *out, std::string *err, (void)buf; if (!f) { if (err) { - (*err) += "File read error. Maybe empty file or invalid file : " + filepath + "\n"; + (*err) += + "File read error. Maybe empty file or invalid file : " + filepath + + "\n"; } return false; } @@ -569,12 +614,13 @@ bool ReadFileHeader(std::vector *out, std::string *err, (void)buf; if (!f) { if (err) { - (*err) += "File read error. Maybe empty file or invalid file : " + filepath + "\n"; + (*err) += + "File read error. Maybe empty file or invalid file : " + filepath + + "\n"; } return false; } - f.seekg(0, f.end); size_t sz = static_cast(f.tellg()); f.seekg(0, f.beg); From a11ab34e87645776b810dade575b2a60257525f8 Mon Sep 17 00:00:00 2001 From: Syoyo Fujita Date: Mon, 14 Oct 2024 04:53:49 +0900 Subject: [PATCH 3/3] fix build on 32bit linux & windows. --- src/io-util.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/io-util.cc b/src/io-util.cc index 47019217..243a8295 100644 --- a/src/io-util.cc +++ b/src/io-util.cc @@ -214,7 +214,7 @@ bool MMapFile(const std::string &filepath, MMapFileHandle *handle, // advise the kernel to preload the mapped memory WIN32_MEMORY_RANGE_ENTRY range; range.VirtualAddress = addr; - range.NumberOfBytes = static_cast((std::min)(size, prefetch)); + range.NumberOfBytes = static_cast((std::min)(size_t(size), prefetch)); if (!pPrefetchVirtualMemory(GetCurrentProcess(), 1, &range, 0)) { // warn if (err) { @@ -315,7 +315,7 @@ bool UnmapFile(const MMapFileHandle &handle, std::string *err) { return false; #else // !WIN32 if (handle.addr && handle.size) { - int ret = munmap(reinterpret_cast(handle.addr), handle.size); + int ret = munmap(reinterpret_cast(handle.addr), size_t(handle.size)); if (!ret) { if (err) { (*err) += "warning: munmap failed.";