Temp draft 5

include/sisl/cache/simple_hashmap.hpp

@@ -95,7 +95,7 @@ class SimpleHashMap {
 
 private:
     SimpleHashBucket< K, V >& get_bucket(const K& key) const;
-    SimpleHashBucket< K, V >& get_bucket(size_t hash_code) const;
+    SimpleHashBucket< K, V >& get_bucket_from_hash(size_t hash_code) const;
 };
 
 ///////////////////////////////////////////// MultiEntryHashNode Definitions ///////////////////////////////////
@@ -127,6 +127,8 @@ class SimpleHashBucket {
         auto it{m_list.begin()};
         while (it != m_list.end()) {
             SingleEntryHashNode< V >* n = &*it;
+            const K k = SimpleHashMap< K, V >::extractor_cb()(it->m_value);
+            access_cb(*n, k, hash_op_t::DELETE);
             it = m_list.erase(it);
             delete n;
         }
@@ -278,6 +280,7 @@ SimpleHashMap< K, V >::SimpleHashMap(uint32_t nBuckets, const key_extractor_cb_t
 
 template < typename K, typename V >
 SimpleHashMap< K, V >::~SimpleHashMap() {
+    set_current_instance(this);
     delete[] m_buckets;
 }
 
@@ -353,7 +356,7 @@ SimpleHashBucket< K, V >& SimpleHashMap< K, V >::get_bucket(const K& key) const
 }
 
 template < typename K, typename V >
-SimpleHashBucket< K, V >& SimpleHashMap< K, V >::get_bucket(size_t hash_code) const {
+SimpleHashBucket< K, V >& SimpleHashMap< K, V >::get_bucket_from_hash(size_t hash_code) const {
     return (m_buckets[hash_code % m_nbuckets]);
 }
 

include/sisl/fds/buffer.hpp

@@ -509,4 +509,34 @@ struct byte_view {
     byte_array m_base_buf;
     blob m_view;
 };
+
+struct buf_builder {
+public:
+    buf_builder(uint32_t sz, uint32_t alignment = 0, buftag tag = buftag::common) : alignment_{alignment} {
+        buf_ = make_byte_array(sz, alignment, tag);
+        cur_ptr_ = buf_->bytes();
+    }
+
+    void append(sisl::blob const& incoming_buf) {
+        if (available_space() < incoming_buf.size()) {
+            auto const increase_size = std::max(incoming_buf.size(), uint32_cast(buf_->size() * 1.5));
+            auto const cur_offset = occupied_space();
+            buf_->buf_realloc(buf_->size() + increase_size, alignment_, buf_->m_tag);
+            cur_ptr_ = buf_->bytes() + cur_offset;
+        }
+        std::memcpy(cur_ptr_, incoming_buf.cbytes(), incoming_buf.size());
+        cur_ptr_ += incoming_buf.size();
+    }
+
+    sisl::byte_view view() const { return sisl::byte_view{buf_, 0, occupied_space()}; }
+    uint8_t* bytes() const { return buf_->bytes(); }
+    uint32_t occupied_space() const { return cur_ptr_ - buf_->bytes(); }
+    uint32_t available_space() const { return buf_->size() - occupied_space(); }
+
+private:
+    byte_array buf_;
+    uint32_t alignment_{0};
+    uint8_t* cur_ptr_{nullptr};
+};
+
 } // namespace sisl

include/sisl/fds/concurrent_insert_vector.hpp

@@ -60,6 +60,23 @@ class ConcurrentInsertVector {
             }
         }
 
+        void operator+=(int64_t count) {
+            while ((count > 0) && (next_thread < vec->per_thread_vec_ptrs_.size())) {
+                // Determine how many steps we can take in the current thread
+                size_t remaining_in_thread = vec->per_thread_vec_ptrs_[next_thread]->size() - next_id_in_thread;
+
+                if (count < remaining_in_thread) {
+                    next_id_in_thread += count;
+                    break;
+                } else {
+                    // Move to the next thread
+                    count -= remaining_in_thread;
+                    ++next_thread;
+                    next_id_in_thread = 0;
+                }
+            }
+        }
+
         bool operator==(iterator const& other) const = default;
         bool operator!=(iterator const& other) const = default;
 
@@ -117,6 +134,13 @@ class ConcurrentInsertVector {
             });
         return sz;
     }
+
+    void clear() {
+        tvector_.access_all_threads([this](std::vector< T >* tvec, bool, bool) {
+            if (tvec) { tvec->clear(); }
+            return false;
+        });
+    }
 };
 
 } // namespace sisl

include/sisl/flip/flip.hpp

@@ -39,8 +39,6 @@
 #include "flip_rpc_server.hpp"
 #include <sisl/logging/logging.h>
 
-SISL_LOGGING_DECL(flip)
-
 namespace flip {
 
 template < size_t Index = 0,                                                     // start iteration at 0 index
@@ -435,7 +433,8 @@ static constexpr int DELAYED_RETURN = 3;
 
 class Flip {
 public:
-    Flip() : m_flip_enabled(false) {}
+    Flip() : m_flip_enabled(false) { REGISTER_LOG_MODS(flip) }
+
     ~Flip() {
         if (m_flip_server) { stop_rpc_server(); }
     }

include/sisl/logging/logging.h

@@ -82,8 +82,6 @@ constexpr const char* file_name(const char* const str) { return str_slant(str) ?
 #endif
 #endif
 
-#define LEVELCHECK(mod, lvl) (module_level_##mod <= (lvl))
-
 #define LINEOUTPUTFORMAT "[{}:{}:{}] "
 #define LINEOUTPUTARGS file_name(__FILE__), __LINE__, __FUNCTION__
 
@@ -453,8 +451,153 @@ MODLEVELDEC(_, _, base)
     sisl::logging::InitModules s_init_enabled_mods{                                                                    \
         BOOST_PP_SEQ_FOR_EACH(MOD_LEVEL_STRING, , BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__))};
 
+#ifdef LOG_MODS_V2_SUPPORT
+#define MOD_DECLTYPE(name) decltype(BOOST_PP_CAT(BOOST_PP_STRINGIZE(name), _tstr))
+
+#define REGISTER_LOG_MOD(name)                                                                                         \
+    {                                                                                                                  \
+        using namespace sisl::logging;                                                                                 \
+        auto& mod = ModuleName< MOD_DECLTYPE(name) >::instance();                                                      \
+        sisl::logging::LogModulesV2::instance().register_module(&mod);                                                 \
+    }
+
+#define _REGISTER_LOG_MOD_MACRO(r, _, module) REGISTER_LOG_MOD(module)
+#define REGISTER_LOG_MODS(...) BOOST_PP_SEQ_FOR_EACH(_REGISTER_LOG_MOD_MACRO, , BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__))
+#define LEVELCHECK(mod, lvl) (ModuleName< MOD_DECLTYPE(mod) >::instance().get_level() <= (lvl))
+
 namespace sisl {
 namespace logging {
+
+class ModuleBase {
+public:
+    ModuleBase() = default;
+    virtual std::string get_name() const = 0;
+    virtual void set_level(spdlog::level::level_enum level) = 0;
+    virtual spdlog::level::level_enum get_level() const = 0;
+    virtual ~ModuleBase() = default;
+};
+
+class LogModulesV2 {
+public:
+    static constexpr spdlog::level::level_enum k_default_level{spdlog::level::level_enum::err};
+
+    static LogModulesV2& instance() {
+        static LogModulesV2 s_inst{};
+        return s_inst;
+    }
+
+    void register_module(ModuleBase* mod) {
+        std::unique_lock lock{m_mutex};
+        m_registered_modules.emplace(mod->get_name(), mod);
+        auto const it = m_requested_modules.find(mod->get_name());
+        mod->set_level(it != m_requested_modules.end() ? it->second : k_default_level);
+    }
+
+    void set_module_level(const std::string& name, spdlog::level::level_enum level) {
+        std::unique_lock lock{m_mutex};
+        m_requested_modules[name] = level;
+
+        auto it = m_registered_modules.find(name);
+        if (it != m_registered_modules.end()) { it->second->set_level(level); }
+    }
+
+    spdlog::level::level_enum get_module_level(const std::string& module_name) {
+        std::unique_lock lock{m_mutex};
+        if (auto it = m_registered_modules.find(module_name); it != m_registered_modules.end()) {
+            return it->second->get_level();
+        } else if (auto it2 = m_requested_modules.find(module_name); (it2 != m_requested_modules.end())) {
+            return it2->second;
+        }
+
+        return k_default_level;
+    }
+
+    std::unordered_map< std::string, spdlog::level::level_enum > get_all_module_levels() {
+        std::unique_lock lock{m_mutex};
+        std::unordered_map< std::string, spdlog::level::level_enum > ret;
+        ret = m_requested_modules;
+        for (auto& [name, mod] : m_registered_modules) {
+            ret[name] = mod->get_level();
+        }
+        return ret;
+    }
+
+    void set_all_module_levels(spdlog::level::level_enum level) {
+        std::unique_lock lock{m_mutex};
+        for (auto& [name, mod] : m_registered_modules) {
+            mod->set_level(level);
+        }
+
+        for (auto& [name, _] : m_requested_modules) {
+            m_requested_modules[name] = level;
+        }
+    }
+
+private:
+    LogModulesV2() = default;
+    std::mutex m_mutex;
+    std::unordered_map< std::string, ModuleBase* > m_registered_modules;
+    std::unordered_map< std::string, spdlog::level::level_enum > m_requested_modules;
+};
+
+} // namespace logging
+} // namespace sisl
+
+template < char... chars >
+using tstring = std::integer_sequence< char, chars... >;
+
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wpedantic"
+#ifdef __clang__
+#pragma GCC diagnostic ignored "-Wgnu-string-literal-operator-template"
+#endif
+template < typename T, T... chars >
+constexpr tstring< chars... > operator""_tstr() {
+    return {};
+}
+#pragma GCC diagnostic pop
+
+template < typename >
+class ModuleName;
+
+template < char... elements >
+class ModuleName< tstring< elements... > > : public sisl::logging::ModuleBase {
+public:
+    ModuleName(const ModuleName&) = delete;
+    ModuleName(ModuleName&&) noexcept = delete;
+    ModuleName& operator=(const ModuleName&) = delete;
+    ModuleName& operator=(ModuleName&&) noexcept = delete;
+
+    static ModuleName& instance() {
+        static ModuleName inst{};
+        return inst;
+    }
+
+    bool is_registered() const { return m_registered; }
+    spdlog::level::level_enum get_level() const override { return m_level; }
+    void set_level(spdlog::level::level_enum level) override { m_level = level; }
+    std::string get_name() const override { return std::string(s_name); }
+
+private:
+    ModuleName() = default;
+
+private:
+    static constexpr char s_name[sizeof...(elements) + 1] = {elements..., '\0'};
+    spdlog::level::level_enum m_level{spdlog::level::level_enum::off};
+    bool m_registered{false};
+};
+
+#else
+#define REGISTER_LOG_MODS(...)                                                                                         \
+    {}
+#define REGISTER_LOG_MOD(name)                                                                                         \
+    {}
+#define LEVELCHECK(mod, lvl) (module_level_##mod <= (lvl))
+#endif
+
+namespace sisl {
+namespace logging {
+
 typedef int SignalType;
 typedef void (*sig_handler_t)(SignalType);
 

include/sisl/metrics/metrics.hpp

@@ -40,9 +40,6 @@
 #include "metrics_rcu.hpp"
 #include "metrics_tlocal.hpp"
 
-// TODO: Commenting out this tempoarily till the SISL_OPTIONS and SISL_LOGGING issue is resolved
-// SISL_LOGGING_DECL(vmod_metrics_framework)
-
 namespace sisl {
 
 class MetricsGroupStaticInfo;

include/sisl/utility/enum.hpp

@@ -105,6 +105,8 @@ class EnumSupportBase {
             return static_cast< enum_type >(itr->second);
     }
 
+    [[nodiscard]] size_t get_count() const { return m_value_to_tokens.size(); }
+
 private:
     std::unordered_map< underlying_type, std::string > m_value_to_tokens;
     std::unordered_map< std::string, underlying_type > m_token_to_value;
@@ -171,6 +173,12 @@ class EnumSupportBase {
     [[nodiscard]] inline const std::string& enum_name(const FQEnumName##Support::enum_type es) {                       \
         return FQEnumName##Support::instance().get_name(es);                                                           \
     }                                                                                                                  \
+                                                                                                                       \
+    template < typename ET = FQEnumName##Support::enum_type >                                                          \
+    typename std::enable_if< std::is_same< ET, FQEnumName##Support::enum_type >::value, size_t >::type enum_count() {  \
+        return FQEnumName##Support::instance().get_count();                                                            \
+    }                                                                                                                  \
+                                                                                                                       \
     [[nodiscard]] inline FQEnumName##Support::underlying_type enum_value(const FQEnumName##Support::enum_type es) {    \
         return static_cast< FQEnumName##Support::underlying_type >(es);                                                \
     }

src/cache/tests/test_range_cache.cpp

@@ -33,8 +33,6 @@
 #include <sisl/cache/lru_evictor.hpp>
 
 using namespace sisl;
-SISL_LOGGING_INIT(test_rangecache)
-
 static uint32_t g_num_chunks;
 static int64_t g_chunk_size;
 static constexpr uint32_t g_blk_size{4096};
@@ -244,6 +242,7 @@ SISL_OPTION_GROUP(test_rangecache,
                    ::cxxopts::value< uint32_t >()->default_value("65536"), "number"))
 
 int main(int argc, char* argv[]) {
+    REGISTER_LOG_MOD(test_rangecache);
     ::testing::InitGoogleTest(&argc, argv);
     SISL_OPTIONS_LOAD(argc, argv, logging, test_rangecache)
     sisl::logging::SetLogger("test_rangecache");

src/cache/tests/test_range_hashmap.cpp

@@ -26,7 +26,7 @@
 #include <sisl/cache/range_hashmap.hpp>
 
 using namespace sisl;
-SISL_LOGGING_INIT(test_hashmap)
+REGISTER_LOG_MOD(test_hashmap)
 
 static uint32_t g_max_offset;
 static constexpr uint32_t per_val_size = 128;

src/cache/tests/test_simple_cache.cpp

@@ -33,7 +33,7 @@
 #include <sisl/cache/lru_evictor.hpp>
 
 using namespace sisl;
-SISL_LOGGING_INIT(test_simplecache)
+REGISTER_LOG_MOD(test_simplecache)
 
 static constexpr uint32_t g_val_size{512};
 static thread_local std::random_device g_rd{};

src/logging/CMakeLists.txt

@@ -18,6 +18,7 @@ if (${CMAKE_BUILD_TYPE} STREQUAL Debug)
 endif()
 
 add_library(sisl_logging)
+target_compile_definitions (sisl_logging PRIVATE LOG_MODS_V2_SUPPORT)
 target_sources(sisl_logging PRIVATE
   ${LOGGING_SOURCE_FILES}
 )
@@ -28,3 +29,10 @@ target_sources(logging_example PRIVATE
   test/example.cpp
   )
 target_link_libraries(logging_example sisl_logging)
+
+add_executable(logging_example_v2)
+target_compile_definitions (logging_example_v2 PRIVATE LOG_MODS_V2_SUPPORT)
+target_sources(logging_example_v2 PRIVATE
+  test/example_v2.cpp
+  )
+target_link_libraries(logging_example_v2 ${LOGGING_DEPS} sisl_logging)

src/logging/logging.cpp

@@ -234,20 +234,24 @@ void set_global_logger(N const& name, S const& sinks, S const& crit_sinks) {
 static spdlog::level::level_enum* to_mod_log_level_ptr(const std::string& module_name) {
     const auto sym = std::string{"module_level_"} + module_name;
     auto* mod_level = static_cast< spdlog::level::level_enum* >(::dlsym(RTLD_DEFAULT, sym.c_str()));
-    if (mod_level == nullptr) {
-        std::cout << fmt::format("Unable to locate the module [{}] in registered modules, error: {}\n", module_name,
-                                 dlerror());
-    }
     return mod_level;
 }
 
 static void set_module_log_level(const std::string& module_name, const spdlog::level::level_enum level) {
     auto* mod_level = to_mod_log_level_ptr(module_name);
     if (mod_level != nullptr) { *mod_level = level; }
+
+#ifdef LOG_MODS_V2_SUPPORT
+    sisl::logging::LogModulesV2::instance().set_module_level(module_name, level);
+#else
+    std::cout << fmt::format("Unable to locate the module [{}] in registered modules, error: {}\n", module_name,
+                             dlerror());
+#endif
 }
 
 static std::string setup_modules() {
     std::string out_str;
+    REGISTER_LOG_MOD(base);
 
     if (SISL_OPTIONS.count("verbosity")) {
         auto lvl_str{SISL_OPTIONS["verbosity"].as< std::string >()};
@@ -263,8 +267,9 @@ static std::string setup_modules() {
             fmt::vformat_to(std::back_inserter(out_str), fmt::string_view{"{}={}, "},
                             fmt::make_format_args(mod_name, lvl_str));
         }
-    } else
+    } else {
         set_module_log_level("base", spdlog::level::level_enum::info);
+    }
 
     if (SISL_OPTIONS.count("log_mods")) {
         std::regex re{"[\\s,]+"};
@@ -275,18 +280,13 @@ static std::string setup_modules() {
             auto mod_stream{std::istringstream(it->str())};
             std::string module_name, module_level;
             std::getline(mod_stream, module_name, ':');
-            if (module_name.empty()) continue;
-            const auto sym{std::string{"module_level_"} + module_name};
-            if (auto* const mod_level{static_cast< spdlog::level::level_enum* >(::dlsym(RTLD_DEFAULT, sym.c_str()))};
-                nullptr != mod_level) {
-                if (std::getline(mod_stream, module_level, ':')) {
-                    *mod_level = (1 == module_level.size())
-                        ? static_cast< spdlog::level::level_enum >(std::strtol(module_level.data(), nullptr, 0))
-                        : spdlog::level::from_str(module_level.data());
-                }
-            } else {
-                std::cout << fmt::format("Unable to setup the module [{}] in registered modules, error: {}\n",
-                                         module_name, dlerror());
+            if (module_name.empty()) { continue; }
+
+            if (std::getline(mod_stream, module_level, ':')) {
+                set_module_log_level(module_name,
+                                     (1 == module_level.size()) ? static_cast< spdlog::level::level_enum >(
+                                                                      std::strtol(module_level.data(), nullptr, 0))
+                                                                : spdlog::level::from_str(module_level.data()));
             }
         }
     }
@@ -298,6 +298,15 @@ static std::string setup_modules() {
             fmt::make_format_args(mod_name, spdlog::level::to_string_view(GetModuleLogLevel(mod_name)).data()));
     }
 
+#ifdef LOG_MODS_V2_SUPPORT
+    fmt::format_to(std::back_inserter(out_str), fmt::string_view("\nV2 Modules: "));
+    auto m = sisl::logging::LogModulesV2::instance().get_all_module_levels();
+    for (auto const& [mod_name, lvl] : m) {
+        fmt::vformat_to(std::back_inserter(out_str), fmt::string_view{"{}={}, "},
+                        fmt::make_format_args(mod_name, spdlog::level::to_string_view(lvl).data()));
+    }
+#endif
+
     return out_str;
 }
 
@@ -326,7 +335,7 @@ void SetLogger(std::string const& name, std::string const& pkg, std::string cons
         std::exit(0);
     }
 
-    const auto log_details{setup_modules()};
+    const auto log_details = setup_modules();
     LOGINFO("Logging initialized: {}/{}, [logmods: {}]", pkg, ver, log_details);
 }
 
@@ -370,7 +379,15 @@ void SetModuleLogLevel(const std::string& module_name, const spdlog::level::leve
 
 spdlog::level::level_enum GetModuleLogLevel(const std::string& module_name) {
     auto* mod_level = to_mod_log_level_ptr(module_name);
-    return mod_level ? *mod_level : spdlog::level::level_enum::off;
+    if (mod_level != nullptr) { return *mod_level; }
+
+#ifdef LOG_MODS_V2_SUPPORT
+    return sisl::logging::LogModulesV2::instance().get_module_level(module_name);
+#else
+    std::cout << fmt::format("Unable to locate the module [{}] in registered modules, error: {}\n", module_name,
+                             dlerror());
+    return spdlog::level::level_enum::off;
+#endif
 }
 
 nlohmann::json GetAllModuleLogLevel() {
@@ -379,14 +396,24 @@ nlohmann::json GetAllModuleLogLevel() {
         const std::string& mod_name{glob_enabled_mods[mod_num]};
         j[mod_name] = spdlog::level::to_string_view(GetModuleLogLevel(mod_name)).data();
     }
+
+#ifdef LOG_MODS_V2_SUPPORT
+    auto m = sisl::logging::LogModulesV2::instance().get_all_module_levels();
+    for (auto const& [mod_name, lvl] : m) {
+        j[mod_name] = spdlog::level::to_string_view(lvl).data();
+    }
+#endif
     return j;
 }
 
 void SetAllModuleLogLevel(const spdlog::level::level_enum level) {
     for (size_t mod_num{0}; mod_num < glob_num_mods; ++mod_num) {
-        const std::string& mod_name{glob_enabled_mods[mod_num]};
-        SetModuleLogLevel(mod_name, level);
+        const std::string& mod_name = glob_enabled_mods[mod_num];
+        set_module_log_level(mod_name, level);
     }
+#ifdef LOG_MODS_V2_SUPPORT
+    sisl::logging::LogModulesV2::instance().set_all_module_levels(level);
+#endif
 }
 
 std::string format_log_msg() { return std::string{}; }

src/logging/test/example_v2.cpp

@@ -0,0 +1,67 @@
+/*********************************************************************************
+ * Modifications Copyright 2017-2019 eBay Inc.
+ *
+ * Author/Developer(s): Brian Szmyd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *    https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed
+ * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ * CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ *********************************************************************************/
+#include <chrono>
+#include <csignal>
+#include <cstdint>
+#include <iostream>
+#include <thread>
+
+#include <sisl/logging/logging.h>
+#include <sisl/options/options.h>
+
+SISL_OPTIONS_ENABLE(logging)
+
+static void log_messages() {
+    LOGINFOMOD(module1, "Module1 Info or lower enabled");
+    LOGINFOMOD(module2, "Module2 Info or lower enabled");
+    LOGINFOMOD(module3, "Module3 Info or lower enabled");
+    LOGINFOMOD(module4, "Module4 Info or lower enabled");
+    LOGINFOMOD(module5, "Module5 Info or lower enabled");
+    LOGINFOMOD(module6, "Module6 Info or lower enabled");
+    LOGTRACEMOD(module3, "Module3 Trace or lower enabled");
+}
+
+int main(int argc, char** argv) {
+    SISL_OPTIONS_LOAD(argc, argv, logging)
+    sisl::logging::SetLogger(std::string{argv[0]});
+    spdlog::set_pattern("[%D %T%z] [%^%l%$] [%n] [%t] %v");
+
+    sisl::logging::install_crash_handler();
+
+    REGISTER_LOG_MOD(module1);
+    REGISTER_LOG_MOD(module2);
+    REGISTER_LOG_MODS(module3, module4, module5);
+
+    sisl::logging::SetModuleLogLevel("module1", spdlog::level::level_enum::info);
+    sisl::logging::SetModuleLogLevel("module2", spdlog::level::level_enum::debug);
+    sisl::logging::SetModuleLogLevel("module3", spdlog::level::level_enum::trace);
+    sisl::logging::SetModuleLogLevel("module4", spdlog::level::level_enum::critical);
+    sisl::logging::SetModuleLogLevel("module5", spdlog::level::level_enum::err);
+    REGISTER_LOG_MODS(module6);
+    sisl::logging::SetModuleLogLevel("module6", spdlog::level::level_enum::warn);
+
+    auto j = sisl::logging::GetAllModuleLogLevel();
+    LOGINFO("Modules and levels default: {}", j.dump(2));
+    log_messages();
+
+    sisl::logging::SetAllModuleLogLevel(spdlog::level::level_enum::debug);
+    j = sisl::logging::GetAllModuleLogLevel();
+    LOGINFO("Modules and levels after set all module log level: {}", j.dump(2));
+    log_messages();
+
+    return 0;
+}