core: Add C++ classes for read-only memory-mapped files and UNIX file…

… descriptors; Remove Zstandard's dependency on Boost. (y-scope#445)

Co-authored-by: kirkrodrigues <[email protected]>

components/core/CMakeLists.txt

@@ -324,6 +324,8 @@ set(SOURCE_FILES_unitTest
         src/clp/ffi/search/Subquery.hpp
         src/clp/ffi/search/WildcardToken.cpp
         src/clp/ffi/search/WildcardToken.hpp
+        src/clp/FileDescriptor.cpp
+        src/clp/FileDescriptor.hpp
         src/clp/FileReader.cpp
         src/clp/FileReader.hpp
         src/clp/FileWriter.cpp
@@ -381,6 +383,8 @@ set(SOURCE_FILES_unitTest
         src/clp/Query.hpp
         src/clp/ReaderInterface.cpp
         src/clp/ReaderInterface.hpp
+        src/clp/ReadOnlyMemoryMappedFile.cpp
+        src/clp/ReadOnlyMemoryMappedFile.hpp
         src/clp/spdlog_with_specializations.hpp
         src/clp/SQLiteDB.cpp
         src/clp/SQLiteDB.hpp
@@ -456,6 +460,7 @@ set(SOURCE_FILES_unitTest
         tests/test-kql.cpp
         tests/test-main.cpp
         tests/test-math_utils.cpp
+        tests/test-MemoryMappedFile.cpp
         tests/test-NetworkReader.cpp
         tests/test-ParserWithUserSchema.cpp
         tests/test-query_methods.cpp

components/core/src/clp/FileDescriptor.cpp

@@ -0,0 +1,63 @@
+#include "FileDescriptor.hpp"
+
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include <cerrno>
+#include <cstddef>
+#include <string_view>
+
+#include "ErrorCode.hpp"
+#include "type_utils.hpp"
+
+namespace clp {
+FileDescriptor::FileDescriptor(
+        std::string_view path,
+        OpenMode open_mode,
+        CloseFailureCallback close_failure_callback
+)
+        : m_open_mode{open_mode},
+          m_close_failure_callback{close_failure_callback} {
+    // For newly created files, we enable writing for the owner and reading for everyone.
+    // Callers can change the created file's permissions as necessary.
+    constexpr auto cNewFilePermission{S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH};
+    auto const flag{enum_to_underlying_type(open_mode)};
+    if (0 != (flag & O_CREAT)) {
+        m_fd = open(path.data(), flag, cNewFilePermission);
+    } else {
+        m_fd = open(path.data(), flag);
+    }
+    if (-1 == m_fd) {
+        throw OperationFailed(
+                ErrorCode_errno,
+                __FILE__,
+                __LINE__,
+                "Failed to open file descriptor for path: " + std::string{path}
+        );
+    }
+}
+
+FileDescriptor::~FileDescriptor() {
+    if (-1 == m_fd) {
+        return;
+    }
+    if (0 != close(m_fd) && nullptr != m_close_failure_callback) {
+        m_close_failure_callback(errno);
+    }
+}
+
+auto FileDescriptor::get_size() const -> size_t {
+    struct stat stat_result {};
+
+    if (0 != fstat(m_fd, &stat_result)) {
+        throw OperationFailed(
+                ErrorCode_errno,
+                __FILE__,
+                __LINE__,
+                "Failed to stat file using file descriptor."
+        );
+    }
+    return static_cast<size_t>(stat_result.st_size);
+}
+}  // namespace clp

components/core/src/clp/FileDescriptor.hpp

@@ -0,0 +1,93 @@
+#ifndef CLP_FILEDESCRIPTOR_HPP
+#define CLP_FILEDESCRIPTOR_HPP
+
+#include <fcntl.h>
+
+#include <cstddef>
+#include <string>
+#include <string_view>
+#include <utility>
+
+#include "ErrorCode.hpp"
+#include "TraceableException.hpp"
+
+namespace clp {
+/**
+ * Wrapper for a UNIX file descriptor.
+ */
+class FileDescriptor {
+public:
+    // Types
+    /**
+     * `close` is called in the destructor to close the file descriptor. However, `close` may return
+     * an error indicated by `errno`. This type alias defines a callback to handle the `close`
+     * failure in the destructor.
+     * The signature of the callback: void close_failure_callback(int errno)
+     */
+    using CloseFailureCallback = void (*)(int);
+
+    class OperationFailed : public TraceableException {
+    public:
+        OperationFailed(
+                ErrorCode error_code,
+                char const* const filename,
+                int line_number,
+                std::string msg
+        )
+                : TraceableException{error_code, filename, line_number},
+                  m_msg{std::move(msg)} {}
+
+        [[nodiscard]] auto what() const noexcept -> char const* override { return m_msg.c_str(); }
+
+    private:
+        std::string m_msg;
+    };
+
+    /**
+     * A C++ wrapper for Unix oflag that describes the open mode.
+     */
+    // NOLINTNEXTLINE(performance-enum-size)
+    enum class OpenMode : int {
+        ReadOnly = O_RDONLY,
+        CreateForWrite = O_WRONLY | O_CREAT | O_TRUNC,
+    };
+
+    // Constructors
+    FileDescriptor(
+            std::string_view path,
+            OpenMode open_mode,
+            CloseFailureCallback close_failure_callback = nullptr
+    );
+
+    // Destructor
+    ~FileDescriptor();
+
+    // Disable copy/move constructors/assignment operators
+    FileDescriptor(FileDescriptor const&) = delete;
+    FileDescriptor(FileDescriptor&&) = delete;
+    auto operator=(FileDescriptor const&) -> FileDescriptor& = delete;
+    auto operator=(FileDescriptor&&) -> FileDescriptor& = delete;
+
+    /**
+     * @return The raw fd.
+     */
+    [[nodiscard]] auto get_raw_fd() const -> int { return m_fd; }
+
+    /**
+     * @return The size of the file.
+     */
+    [[nodiscard]] auto get_size() const -> size_t;
+
+    /**
+     * @return The open mode.
+     */
+    [[nodiscard]] auto get_open_mode() const -> OpenMode { return m_open_mode; }
+
+private:
+    int m_fd{-1};
+    OpenMode m_open_mode;
+    CloseFailureCallback m_close_failure_callback{nullptr};
+};
+}  // namespace clp
+
+#endif

components/core/src/clp/ReadOnlyMemoryMappedFile.cpp

@@ -0,0 +1,41 @@
+#include "ReadOnlyMemoryMappedFile.hpp"
+
+#include <sys/mman.h>
+
+#include <string>
+#include <string_view>
+
+#include "ErrorCode.hpp"
+#include "FileDescriptor.hpp"
+
+namespace clp {
+ReadOnlyMemoryMappedFile::ReadOnlyMemoryMappedFile(std::string_view path) {
+    FileDescriptor const fd{path, FileDescriptor::OpenMode::ReadOnly};
+    auto const file_size{fd.get_size()};
+    if (0 == file_size) {
+        // `mmap` doesn't allow mapping an empty file, so we don't need to call it.
+        return;
+    }
+    auto* mmap_ptr{mmap(nullptr, file_size, PROT_READ, MAP_PRIVATE, fd.get_raw_fd(), 0)};
+    if (MAP_FAILED == mmap_ptr) {
+        throw OperationFailed(
+                ErrorCode_errno,
+                __FILE__,
+                __LINE__,
+                "`mmap` failed to map path: " + std::string{path}
+        );
+    }
+    m_data = mmap_ptr;
+    m_buf_size = file_size;
+}
+
+ReadOnlyMemoryMappedFile::~ReadOnlyMemoryMappedFile() {
+    if (0 == m_buf_size) {
+        // We don't call `mmap` for empty files, so we don't need to call `munmap`.
+        return;
+    }
+    // We skip error checking since the only likely reason for `munmap` to fail is if we give it
+    // invalid arguments.
+    munmap(m_data, m_buf_size);
+}
+}  // namespace clp

components/core/src/clp/ReadOnlyMemoryMappedFile.hpp

@@ -0,0 +1,66 @@
+#ifndef CLP_READONLYMEMORYMAPPEDFILE_HPP
+#define CLP_READONLYMEMORYMAPPEDFILE_HPP
+
+#include <cstddef>
+#include <span>
+#include <string>
+#include <string_view>
+#include <utility>
+
+#include "ErrorCode.hpp"
+#include "TraceableException.hpp"
+
+namespace clp {
+/**
+ * A class for mapping a read-only file into memory. It maintains the memory buffer created by the
+ * underlying `mmap` system call and provides methods to get a view of the memory buffer.
+ */
+class ReadOnlyMemoryMappedFile {
+public:
+    // Types
+    class OperationFailed : public TraceableException {
+    public:
+        OperationFailed(
+                ErrorCode error_code,
+                char const* const filename,
+                int line_number,
+                std::string msg
+        )
+                : TraceableException{error_code, filename, line_number},
+                  m_msg{std::move(msg)} {}
+
+        [[nodiscard]] auto what() const noexcept -> char const* override { return m_msg.c_str(); }
+
+    private:
+        std::string m_msg;
+    };
+
+    // Constructors
+    /**
+     * @param path The path of the file to map.
+     */
+    explicit ReadOnlyMemoryMappedFile(std::string_view path);
+
+    // Destructor
+    ~ReadOnlyMemoryMappedFile();
+
+    // Disable copy/move constructors/assignment operators
+    ReadOnlyMemoryMappedFile(ReadOnlyMemoryMappedFile const&) = delete;
+    ReadOnlyMemoryMappedFile(ReadOnlyMemoryMappedFile&&) = delete;
+    auto operator=(ReadOnlyMemoryMappedFile const&) -> ReadOnlyMemoryMappedFile& = delete;
+    auto operator=(ReadOnlyMemoryMappedFile&&) -> ReadOnlyMemoryMappedFile& = delete;
+
+    /**
+     * @return A view of the mapped file in memory.
+     */
+    [[nodiscard]] auto get_view() const -> std::span<char> {
+        return std::span<char>{static_cast<char*>(m_data), m_buf_size};
+    }
+
+private:
+    void* m_data{nullptr};
+    size_t m_buf_size{0};
+};
+}  // namespace clp
+
+#endif

components/core/src/clp/clg/CMakeLists.txt

@@ -18,6 +18,8 @@ set(
         ../ffi/ir_stream/decoding_methods.cpp
         ../ffi/ir_stream/decoding_methods.hpp
         ../ffi/ir_stream/decoding_methods.inc
+        ../FileDescriptor.cpp
+        ../FileDescriptor.hpp
         ../FileReader.cpp
         ../FileReader.hpp
         ../FileWriter.cpp
@@ -57,6 +59,8 @@ set(
         ../Query.hpp
         ../ReaderInterface.cpp
         ../ReaderInterface.hpp
+        ../ReadOnlyMemoryMappedFile.cpp
+        ../ReadOnlyMemoryMappedFile.hpp
         ../spdlog_with_specializations.hpp
         ../SQLiteDB.cpp
         ../SQLiteDB.hpp

components/core/src/clp/clo/CMakeLists.txt

@@ -20,6 +20,8 @@ set(
         ../ffi/ir_stream/decoding_methods.cpp
         ../ffi/ir_stream/decoding_methods.hpp
         ../ffi/ir_stream/decoding_methods.inc
+        ../FileDescriptor.cpp
+        ../FileDescriptor.hpp
         ../FileReader.cpp
         ../FileReader.hpp
         ../FileWriter.cpp
@@ -49,6 +51,8 @@ set(
         ../Query.hpp
         ../ReaderInterface.cpp
         ../ReaderInterface.hpp
+        ../ReadOnlyMemoryMappedFile.cpp
+        ../ReadOnlyMemoryMappedFile.hpp
         ../spdlog_with_specializations.hpp
         ../SQLiteDB.cpp
         ../SQLiteDB.hpp

components/core/src/clp/clp/CMakeLists.txt

@@ -27,6 +27,8 @@ set(
         ../ffi/ir_stream/encoding_methods.hpp
         ../ffi/ir_stream/utils.cpp
         ../ffi/ir_stream/utils.hpp
+        ../FileDescriptor.cpp
+        ../FileDescriptor.hpp
         ../FileReader.cpp
         ../FileReader.hpp
         ../FileWriter.cpp
@@ -80,6 +82,8 @@ set(
         ../Query.hpp
         ../ReaderInterface.cpp
         ../ReaderInterface.hpp
+        ../ReadOnlyMemoryMappedFile.cpp
+        ../ReadOnlyMemoryMappedFile.hpp
         ../spdlog_with_specializations.hpp
         ../SQLiteDB.cpp
         ../SQLiteDB.hpp

components/core/src/clp/make_dictionaries_readable/CMakeLists.txt

@@ -4,6 +4,8 @@ set(
         ../dictionary_utils.hpp
         ../DictionaryEntry.hpp
         ../DictionaryReader.hpp
+        ../FileDescriptor.cpp
+        ../FileDescriptor.hpp
         ../FileReader.cpp
         ../FileReader.hpp
         ../FileWriter.cpp
@@ -17,6 +19,8 @@ set(
         ../ParsedMessage.hpp
         ../ReaderInterface.cpp
         ../ReaderInterface.hpp
+        ../ReadOnlyMemoryMappedFile.cpp
+        ../ReadOnlyMemoryMappedFile.hpp
         ../spdlog_with_specializations.hpp
         ../streaming_compression/Decompressor.hpp
         ../streaming_compression/passthrough/Decompressor.cpp