From 1a3da518f6c1b7c9c9787f71a7ac9484cb4c52c3 Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Sat, 9 Nov 2024 21:46:29 +0000 Subject: [PATCH 01/52] add new class --- src/clp_ffi_js/ir/LogEventWithFilterData.hpp | 41 +++++++++++++++++++ src/clp_ffi_js/ir/LogEventWithLevel.hpp | 41 ------------------- .../ir/UnstructuredIrStreamReader.cpp | 28 +++++++------ .../ir/UnstructuredIrStreamReader.hpp | 5 ++- 4 files changed, 59 insertions(+), 56 deletions(-) create mode 100644 src/clp_ffi_js/ir/LogEventWithFilterData.hpp delete mode 100644 src/clp_ffi_js/ir/LogEventWithLevel.hpp diff --git a/src/clp_ffi_js/ir/LogEventWithFilterData.hpp b/src/clp_ffi_js/ir/LogEventWithFilterData.hpp new file mode 100644 index 00000000..b38437b5 --- /dev/null +++ b/src/clp_ffi_js/ir/LogEventWithFilterData.hpp @@ -0,0 +1,41 @@ +#ifndef CLP_FFI_JS_IR_LOGEVENTWITHFILTERDATA_HPP +#define CLP_FFI_JS_IR_LOGEVENTWITHFILTERDATA_HPP + +#include + +#include +#include +#include +#include + +#include + +namespace clp_ffi_js::ir { +/** + * A class which accepts a log event type as a template parameter and provides additional members + * for the log level and timestamp. The additional members facilitate log level filtering. + * @tparam LogEvent The type of the log event. + */ +template +class LogEventWithFilterData { +public: + // Constructor + explicit LogEventWithFilterData(LogEvent log_event, LogLevel log_level, clp::ir::epoch_time_ms_t timestamp) + : m_log_event{std::move(log_event)}, + m_log_level{log_level}, + m_timestamp{timestamp} {} + + [[nodiscard]] auto get_log_event() const -> LogEvent const& { return m_log_event; } + + [[nodiscard]] auto get_log_level() const -> LogLevel { return m_log_level; } + + [[nodiscard]] auto get_timestamp() const -> clp::ir::epoch_time_ms_t { return m_timestamp; } + +private: + LogEvent m_log_event; + LogLevel m_log_level; + clp::ir::epoch_time_ms_t m_timestamp; +}; +} // namespace clp_ffi_js::ir + +#endif // CLP_FFI_JS_IR_LOGEVENTWITHFILTERDATA_HPP \ No newline at end of file diff --git a/src/clp_ffi_js/ir/LogEventWithLevel.hpp b/src/clp_ffi_js/ir/LogEventWithLevel.hpp deleted file mode 100644 index 22a00af3..00000000 --- a/src/clp_ffi_js/ir/LogEventWithLevel.hpp +++ /dev/null @@ -1,41 +0,0 @@ -#ifndef CLP_FFI_JS_IR_LOGEVENTWITHLEVEL_HPP -#define CLP_FFI_JS_IR_LOGEVENTWITHLEVEL_HPP - -#include - -#include -#include -#include -#include - -#include - -namespace clp_ffi_js::ir { -/** - * A class derived from `clp::ir::LogEvent` with an additional member for the log level. - * - * NOTE: Once we move to the next IR format, this class should no longer be necessary since each - * IR log event will contain a set of key-value pairs, one of which should be the log level. - * @tparam encoded_variable_t The type of encoded variables in the event - */ -template -class LogEventWithLevel : public clp::ir::LogEvent { -public: - // Constructors - LogEventWithLevel( - clp::ir::epoch_time_ms_t timestamp, - clp::UtcOffset utc_offset, - clp::ir::EncodedTextAst message, - LogLevel log_level - ) - : clp::ir::LogEvent{timestamp, utc_offset, std::move(message)}, - m_log_level{log_level} {} - - [[nodiscard]] auto get_log_level() const -> LogLevel { return m_log_level; } - -private: - LogLevel m_log_level; -}; -} // namespace clp_ffi_js::ir - -#endif // CLP_FFI_JS_IR_LOGEVENTWITHLEVEL_HPP diff --git a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp index 9a5a8f95..2cc908be 100644 --- a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp @@ -25,8 +25,7 @@ #include #include -#include -#include +#include "clp_ffi_js/ir/LogEventWithFilterData.hpp" #include #include @@ -147,13 +146,12 @@ auto UnstructuredIrStreamReader::deserialize_stream() -> size_t { } } - auto log_viewer_event{LogEventWithLevel( - log_event.get_timestamp(), - log_event.get_utc_offset(), - message, - log_level + auto log_event_with_filter_data{LogEventWithFilterData( + log_event, + log_level, + log_event.get_timestamp() )}; - m_encoded_log_events.emplace_back(std::move(log_viewer_event)); + m_encoded_log_events.emplace_back(std::move(log_event_with_filter_data)); } m_stream_reader_data_context.reset(nullptr); return m_encoded_log_events.size(); @@ -187,9 +185,13 @@ auto UnstructuredIrStreamReader::decode_range(size_t begin_idx, size_t end_idx, } else { log_event_idx = i; } - auto const& log_event{m_encoded_log_events[log_event_idx]}; + auto const& log_event_with_filter_data{m_encoded_log_events[log_event_idx]}; + auto const& log_level = log_event_with_filter_data.get_log_level(); + auto const& timestamp = log_event_with_filter_data.get_timestamp(); - auto const parsed{log_event.get_message().decode_and_unparse()}; + auto const& unstructured_log_event = log_event_with_filter_data.get_log_event(); + + auto const parsed{unstructured_log_event.get_message().decode_and_unparse()}; if (false == parsed.has_value()) { throw ClpFfiJsException{ clp::ErrorCode::ErrorCode_Failure, @@ -200,14 +202,14 @@ auto UnstructuredIrStreamReader::decode_range(size_t begin_idx, size_t end_idx, } message = parsed.value(); - m_ts_pattern.insert_formatted_timestamp(log_event.get_timestamp(), message); + m_ts_pattern.insert_formatted_timestamp(timestamp, message); EM_ASM( { Emval.toValue($0).push([UTF8ToString($1), $2, $3, $4]); }, results.as_handle(), message.c_str(), - log_event.get_timestamp(), - log_event.get_log_level(), + timestamp, + log_level, log_event_idx + 1 ); } diff --git a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp index b4f22107..700d3b41 100644 --- a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp +++ b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp @@ -12,13 +12,14 @@ #include #include -#include +#include #include #include namespace clp_ffi_js::ir { using clp::ir::four_byte_encoded_variable_t; using UnstructuredIrDeserializer = clp::ir::LogEventDeserializer; +using UnstructuredLogEvent = clp::ir::LogEvent; /** * Mapping between an index in the filtered log events collection to an index in the unfiltered @@ -85,7 +86,7 @@ class UnstructuredIrStreamReader : public StreamReader { ); // Variables - std::vector> m_encoded_log_events; + std::vector> m_encoded_log_events; std::unique_ptr> m_stream_reader_data_context; FilteredLogEventsMap m_filtered_log_event_map; From 29a87d224cdead7d1a38ceb07a0f509ebde44fcb Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Sat, 9 Nov 2024 22:41:27 +0000 Subject: [PATCH 02/52] fix-lint --- src/clp_ffi_js/ir/LogEventWithFilterData.hpp | 17 +++++++++-------- .../ir/UnstructuredIrStreamReader.cpp | 5 ++++- .../ir/UnstructuredIrStreamReader.hpp | 1 + 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/clp_ffi_js/ir/LogEventWithFilterData.hpp b/src/clp_ffi_js/ir/LogEventWithFilterData.hpp index b38437b5..15538a50 100644 --- a/src/clp_ffi_js/ir/LogEventWithFilterData.hpp +++ b/src/clp_ffi_js/ir/LogEventWithFilterData.hpp @@ -3,10 +3,7 @@ #include -#include -#include #include -#include #include @@ -20,10 +17,14 @@ template class LogEventWithFilterData { public: // Constructor - explicit LogEventWithFilterData(LogEvent log_event, LogLevel log_level, clp::ir::epoch_time_ms_t timestamp) - : m_log_event{std::move(log_event)}, - m_log_level{log_level}, - m_timestamp{timestamp} {} + explicit LogEventWithFilterData( + LogEvent log_event, + LogLevel log_level, + clp::ir::epoch_time_ms_t timestamp + ) + : m_log_event{std::move(log_event)}, + m_log_level{log_level}, + m_timestamp{timestamp} {} [[nodiscard]] auto get_log_event() const -> LogEvent const& { return m_log_event; } @@ -38,4 +39,4 @@ class LogEventWithFilterData { }; } // namespace clp_ffi_js::ir -#endif // CLP_FFI_JS_IR_LOGEVENTWITHFILTERDATA_HPP \ No newline at end of file +#endif // CLP_FFI_JS_IR_LOGEVENTWITHFILTERDATA_HPP diff --git a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp index 2cc908be..34ed1c10 100644 --- a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp @@ -19,16 +19,19 @@ #include #include #include + #include #include #include #include +#include #include -#include "clp_ffi_js/ir/LogEventWithFilterData.hpp" #include #include +#include "clp_ffi_js/ir/LogEventWithFilterData.hpp" + namespace clp_ffi_js::ir { using namespace std::literals::string_literals; diff --git a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp index 700d3b41..af1c9a31 100644 --- a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp +++ b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp @@ -12,6 +12,7 @@ #include #include +#include #include #include #include From e5a8e4c0435e0c46c000df4db31c4687ec0075ad Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Sat, 9 Nov 2024 22:41:54 +0000 Subject: [PATCH 03/52] fix lint --- src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp index 34ed1c10..869bb80e 100644 --- a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp @@ -19,14 +19,13 @@ #include #include #include - #include #include #include #include -#include #include +#include #include #include From 921c32271ed003bdf13d78be09c7c1ec7bb864c5 Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Sat, 9 Nov 2024 23:21:59 +0000 Subject: [PATCH 04/52] small change --- src/clp_ffi_js/ir/LogEventWithFilterData.hpp | 11 +++++++++++ src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/clp_ffi_js/ir/LogEventWithFilterData.hpp b/src/clp_ffi_js/ir/LogEventWithFilterData.hpp index 15538a50..7ee77703 100644 --- a/src/clp_ffi_js/ir/LogEventWithFilterData.hpp +++ b/src/clp_ffi_js/ir/LogEventWithFilterData.hpp @@ -26,6 +26,17 @@ class LogEventWithFilterData { m_log_level{log_level}, m_timestamp{timestamp} {} + // Disable copy constructor and assignment operator + LogEventWithFilterData(LogEventWithFilterData const&) = delete; + auto operator=(LogEventWithFilterData const&) -> LogEventWithFilterData& = delete; + + // Default move constructor and assignment operator + LogEventWithFilterData(LogEventWithFilterData&&) = default; + auto operator=(LogEventWithFilterData&&) -> LogEventWithFilterData& = default; + + // Destructor + ~LogEventWithFilterData() = default; + [[nodiscard]] auto get_log_event() const -> LogEvent const& { return m_log_event; } [[nodiscard]] auto get_log_level() const -> LogLevel { return m_log_level; } diff --git a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp index 869bb80e..b6bb6420 100644 --- a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp @@ -26,10 +26,10 @@ #include #include +#include "clp_ffi_js/ir/LogEventWithFilterData.hpp" #include #include -#include "clp_ffi_js/ir/LogEventWithFilterData.hpp" namespace clp_ffi_js::ir { From 865e41d33b6e91a9a380eb28cb83f1d1f5df4019 Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Sat, 9 Nov 2024 23:26:27 +0000 Subject: [PATCH 05/52] small change --- src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp index b6bb6420..cf3e3c33 100644 --- a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp @@ -26,7 +26,7 @@ #include #include -#include "clp_ffi_js/ir/LogEventWithFilterData.hpp" +#include #include #include From 0234365078c849358f2bae57e1388c9e743f0638 Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Sat, 9 Nov 2024 23:37:25 +0000 Subject: [PATCH 06/52] fix lint --- src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp index cf3e3c33..9f027def 100644 --- a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp @@ -30,7 +30,6 @@ #include #include - namespace clp_ffi_js::ir { using namespace std::literals::string_literals; From 0c86cf46bd73032c9e18e5f033689b65ff42e25e Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Sat, 9 Nov 2024 18:50:37 -0500 Subject: [PATCH 07/52] Update src/clp_ffi_js/ir/LogEventWithFilterData.hpp Co-authored-by: Junhao Liao --- src/clp_ffi_js/ir/LogEventWithFilterData.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/clp_ffi_js/ir/LogEventWithFilterData.hpp b/src/clp_ffi_js/ir/LogEventWithFilterData.hpp index 7ee77703..e1007b59 100644 --- a/src/clp_ffi_js/ir/LogEventWithFilterData.hpp +++ b/src/clp_ffi_js/ir/LogEventWithFilterData.hpp @@ -17,7 +17,7 @@ template class LogEventWithFilterData { public: // Constructor - explicit LogEventWithFilterData( + LogEventWithFilterData( LogEvent log_event, LogLevel log_level, clp::ir::epoch_time_ms_t timestamp From 84037cf1816bcae24dfe9a0c96a600117a1e4063 Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Sun, 10 Nov 2024 00:17:13 +0000 Subject: [PATCH 08/52] fix lint --- src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp index af1c9a31..55adf2b9 100644 --- a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp +++ b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp @@ -8,6 +8,7 @@ #include #include +#include #include #include #include From ef1ff5db8c9de64c764a36cc1cb4222ab9e4f23b Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Sun, 10 Nov 2024 00:19:36 +0000 Subject: [PATCH 09/52] fix lint --- src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp index 55adf2b9..77cce568 100644 --- a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp +++ b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp @@ -7,8 +7,8 @@ #include #include -#include #include +#include #include #include #include From cdcad949c28c8970157389907892428af32780ba Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Sun, 10 Nov 2024 00:30:22 +0000 Subject: [PATCH 10/52] fix lint --- src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp index 9f027def..d96661ed 100644 --- a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp @@ -25,7 +25,6 @@ #include #include -#include #include #include #include From a1e91428f3beac677a1d8b42e456128c655d6136 Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Sun, 10 Nov 2024 01:08:34 +0000 Subject: [PATCH 11/52] fix lint --- src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp index 77cce568..108ca157 100644 --- a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp +++ b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp @@ -13,7 +13,6 @@ #include #include -#include #include #include #include From 2738dd56356c36145261f8d53bb853d3b908d22a Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Sun, 10 Nov 2024 02:53:22 +0000 Subject: [PATCH 12/52] first draft --- .../ir/StructuredIrStreamReader.cpp | 116 ++++++++++++++---- .../ir/StructuredIrStreamReader.hpp | 46 ++++--- 2 files changed, 116 insertions(+), 46 deletions(-) diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp index 799da91c..951d1947 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp @@ -22,6 +22,7 @@ #include #include +#include #include #include @@ -31,21 +32,109 @@ constexpr std::string_view cEmptyJsonStr{"{}"}; constexpr std::string_view cLogLevelFilteringNotSupportedErrorMsg{ "Log level filtering is not yet supported in this reader." }; +constexpr std::string_view cReaderOptionsLogLevelKey{"logLevelKey"}; constexpr std::string_view cReaderOptionsTimestampKey{"timestampKey"}; + +/** + * Gets the `LogLevel` from an input string. + * @param str + * @return + */ +auto get_log_level(std::string_view str) -> LogLevel; + +auto get_log_level(std::string_view str) -> LogLevel { + LogLevel log_level{LogLevel::NONE}; + + // Convert the string to uppercase, + std::string log_level_name_upper_case{str}; + std::transform( + log_level_name_upper_case.begin(), + log_level_name_upper_case.end(), + log_level_name_upper_case.begin(), + [](unsigned char c) { return std::toupper(c); } + ); + + // Do not accept "None" when checking if input string is in `cLogLevelNames`. + const auto* it = std::ranges::find(cLogLevelNames.begin() + clp::enum_to_underlying_type(cValidLogLevelsBeginIdx), cLogLevelNames.end(), log_level_name_upper_case); + + if (it == cLogLevelNames.end()) { + return log_level; + } + + return static_cast(std::distance(cLogLevelNames.begin(), it));; +} + } // namespace using clp::ir::four_byte_encoded_variable_t; +auto IrUnitHandler::handle_schema_tree_node_insertion( + clp::ffi::SchemaTree::NodeLocator schema_tree_node_locator +) -> clp::ffi::ir_stream::IRErrorCode { + ++m_current_node_id; + + auto const& key_name{schema_tree_node_locator.get_key_name()}; + if (m_log_level_key == key_name) { + m_log_level_node_id.emplace(m_current_node_id); + } else if (m_timestamp_key == key_name) { + m_timestamp_node_id.emplace(m_current_node_id); + } + + return clp::ffi::ir_stream::IRErrorCode::IRErrorCode_Success; +} + +auto IrUnitHandler::handle_log_event(StructuredLogEvent&& log_event +) -> clp::ffi::ir_stream::IRErrorCode { + + auto const& id_value_pairs{log_event.get_node_id_value_pairs()}; + clp::ffi::value_int_t timestamp{0}; + + if (m_timestamp_node_id.has_value()) { + auto const& timestamp_pair{id_value_pairs.at(m_timestamp_node_id.value())}; + if (timestamp_pair.has_value()) { + if (timestamp_pair->is()) { + timestamp = timestamp_pair.value().get_immutable_view(); + } else { + // TODO: Add support for parsing timestamp values of string type. + SPDLOG_ERROR("Timestamp type is not int"); + } + } + } + + LogLevel log_level{LogLevel::NONE}; + if (m_log_level_node_id.has_value()) { + auto const& log_level_pair{id_value_pairs.at(m_log_level_node_id.value())}; + if (log_level_pair.has_value()) { + if (log_level_pair->is()) { + auto const& log_level_name = log_level_pair.value().get_immutable_view(); + log_level = get_log_level(log_level_name); + } else { + SPDLOG_ERROR("Log level type is not string"); + } + } + } + + auto log_event_with_filter_data{ + LogEventWithFilterData(std::move(log_event), log_level, timestamp) + }; + + m_deserialized_log_events->emplace_back(std::move(log_event_with_filter_data)); + + return clp::ffi::ir_stream::IRErrorCode::IRErrorCode_Success; +} + + auto StructuredIrStreamReader::create( std::unique_ptr&& zstd_decompressor, clp::Array data_array, ReaderOptions const& reader_options ) -> StructuredIrStreamReader { - auto deserialized_log_events{std::make_shared>()}; + auto deserialized_log_events{std::make_shared>>()}; auto result{StructuredIrDeserializer::create( *zstd_decompressor, IrUnitHandler{ deserialized_log_events, + reader_options[cReaderOptionsLogLevelKey.data()].as(), reader_options[cReaderOptionsTimestampKey.data()].as() } )}; @@ -119,9 +208,6 @@ auto StructuredIrStreamReader::deserialize_stream() -> size_t { ) }; } - m_timestamp_node_id = m_stream_reader_data_context->get_deserializer() - .get_ir_unit_handler() - .get_timestamp_node_id(); m_stream_reader_data_context.reset(nullptr); return m_deserialized_log_events->size(); } @@ -140,9 +226,10 @@ auto StructuredIrStreamReader::decode_range(size_t begin_idx, size_t end_idx, bo auto const results{emscripten::val::array()}; for (size_t log_event_idx = begin_idx; log_event_idx < end_idx; ++log_event_idx) { - auto const& log_event{m_deserialized_log_events->at(log_event_idx)}; + auto const& log_event_with_filter_data{m_deserialized_log_events->at(log_event_idx)}; + auto const& structured_log_event = log_event_with_filter_data.get_log_event(); - auto const json_result{log_event.serialize_to_json()}; + auto const json_result{structured_log_event.serialize_to_json()}; std::string json_str{cEmptyJsonStr}; if (false == json_result.has_value()) { auto error_code{json_result.error()}; @@ -155,26 +242,13 @@ auto StructuredIrStreamReader::decode_range(size_t begin_idx, size_t end_idx, bo json_str = json_result.value().dump(); } - auto const& id_value_pairs{log_event.get_node_id_value_pairs()}; - clp::ffi::value_int_t timestamp{0}; - if (m_timestamp_node_id.has_value()) { - auto const& timestamp_pair{id_value_pairs.at(m_timestamp_node_id.value())}; - if (timestamp_pair.has_value()) { - if (timestamp_pair->is()) { - timestamp = timestamp_pair.value().get_immutable_view(); - } else { - // TODO: Add support for parsing timestamp values of string type. - SPDLOG_ERROR("Unable to parse timestamp for log_event_idx={}", log_event_idx); - } - } - } EM_ASM( { Emval.toValue($0).push([UTF8ToString($1), $2, $3, $4]); }, results.as_handle(), json_str.c_str(), - timestamp, - LogLevel::NONE, + log_event_with_filter_data.get_timestamp(), + log_event_with_filter_data.get_log_level(), log_event_idx + 1 ); } diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp index e93dee03..b227cc3a 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp @@ -17,11 +17,13 @@ #include #include +#include #include #include namespace clp_ffi_js::ir { using schema_tree_node_id_t = std::optional; +using StructuredLogEvent = clp::ffi::KeyValuePairLogEvent; /** * Class that implements the `clp::ffi::ir_stream::IrUnitHandlerInterface` to buffer log events and @@ -31,14 +33,16 @@ class IrUnitHandler { public: /** * @param deserialized_log_events The vector in which to store deserialized log events. - * @param timestamp_key Key name of schema-tree node that contains the authoritative timestamp - * for events. + * @param log_level_key Key name of schema-tree node that contains the authoritative log level. + * @param timestamp_key Key name of schema-tree node that contains the authoritative timestamp. */ IrUnitHandler( - std::shared_ptr> deserialized_log_events, + std::shared_ptr>> deserialized_log_events, + std::string log_level_key, std::string timestamp_key ) - : m_timestamp_key{std::move(timestamp_key)}, + : m_log_level_key{std::move(log_level_key)}, + m_timestamp_key{std::move(timestamp_key)}, m_deserialized_log_events{std::move(deserialized_log_events)} {} // Methods implementing `clp::ffi::ir_stream::IrUnitHandlerInterface`. @@ -47,12 +51,8 @@ class IrUnitHandler { * @param log_event * @return IRErrorCode::IRErrorCode_Success */ - [[nodiscard]] auto handle_log_event(clp::ffi::KeyValuePairLogEvent&& log_event - ) -> clp::ffi::ir_stream::IRErrorCode { - m_deserialized_log_events->emplace_back(std::move(log_event)); - - return clp::ffi::ir_stream::IRErrorCode::IRErrorCode_Success; - } + [[nodiscard]] auto handle_log_event(StructuredLogEvent&& log_event + ) -> clp::ffi::ir_stream::IRErrorCode; /** * @param utc_offset_old @@ -75,16 +75,7 @@ class IrUnitHandler { */ [[nodiscard]] auto handle_schema_tree_node_insertion( clp::ffi::SchemaTree::NodeLocator schema_tree_node_locator - ) -> clp::ffi::ir_stream::IRErrorCode { - ++m_current_node_id; - - auto const& key_name{schema_tree_node_locator.get_key_name()}; - if (m_timestamp_key == key_name) { - m_timestamp_node_id.emplace(m_current_node_id); - } - - return clp::ffi::ir_stream::IRErrorCode::IRErrorCode_Success; - } + ) -> clp::ffi::ir_stream::IRErrorCode; /** * @return IRErrorCode::IRErrorCode_Success @@ -94,6 +85,11 @@ class IrUnitHandler { } // Methods + + + + + /** * @return The schema-tree node ID associated with events' authoritative timestamp key. */ @@ -103,16 +99,18 @@ class IrUnitHandler { private: // Variables + std::string m_log_level_key; std::string m_timestamp_key; clp::ffi::SchemaTree::Node::id_t m_current_node_id{clp::ffi::SchemaTree::cRootId}; schema_tree_node_id_t m_timestamp_node_id; + schema_tree_node_id_t m_log_level_node_id; // TODO: Technically, we don't need to use a `shared_ptr` since the parent stream reader will // have a longer lifetime than this class. Instead, we could use `gsl::not_null` once we add // `gsl` into the project. - std::shared_ptr> m_deserialized_log_events; + std::shared_ptr>> m_deserialized_log_events; }; using StructuredIrDeserializer = clp::ffi::ir_stream::Deserializer; @@ -175,14 +173,12 @@ class StructuredIrStreamReader : public StreamReader { // Constructor explicit StructuredIrStreamReader( StreamReaderDataContext&& stream_reader_data_context, - std::shared_ptr> deserialized_log_events + std::shared_ptr>> deserialized_log_events ); // Variables - std::shared_ptr> m_deserialized_log_events; + std::shared_ptr>> m_deserialized_log_events; std::unique_ptr> m_stream_reader_data_context; - - schema_tree_node_id_t m_timestamp_node_id; }; } // namespace clp_ffi_js::ir From d4afd351212e1b948d8f9a47ee5cf3fd1957808a Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Sun, 10 Nov 2024 03:01:24 +0000 Subject: [PATCH 13/52] add missing filtering --- src/clp_ffi_js/ir/StreamReader.hpp | 6 +++++ .../ir/StructuredIrStreamReader.cpp | 27 +++++++++++++++---- .../ir/StructuredIrStreamReader.hpp | 1 + .../ir/UnstructuredIrStreamReader.hpp | 5 ---- 4 files changed, 29 insertions(+), 10 deletions(-) diff --git a/src/clp_ffi_js/ir/StreamReader.hpp b/src/clp_ffi_js/ir/StreamReader.hpp index 06e7c094..c45443c6 100644 --- a/src/clp_ffi_js/ir/StreamReader.hpp +++ b/src/clp_ffi_js/ir/StreamReader.hpp @@ -23,6 +23,12 @@ enum class StreamType : uint8_t { Unstructured, }; +/** + * Mapping between an index in the filtered log events collection to an index in the unfiltered + * log events collection. + */ +using FilteredLogEventsMap = std::optional>; + /** * Class to deserialize and decode Zstandard-compressed CLP IR streams as well as format decoded * log events. diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp index 951d1947..5fa2cf8b 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp @@ -123,7 +123,6 @@ auto IrUnitHandler::handle_log_event(StructuredLogEvent&& log_event return clp::ffi::ir_stream::IRErrorCode::IRErrorCode_Success; } - auto StructuredIrStreamReader::create( std::unique_ptr&& zstd_decompressor, clp::Array data_array, @@ -164,15 +163,33 @@ auto StructuredIrStreamReader::get_num_events_buffered() const -> size_t { } auto StructuredIrStreamReader::get_filtered_log_event_map() const -> FilteredLogEventMapTsType { - SPDLOG_ERROR(cLogLevelFilteringNotSupportedErrorMsg); - return FilteredLogEventMapTsType{emscripten::val::null()}; + if (false == m_filtered_log_event_map.has_value()) { + return FilteredLogEventMapTsType{emscripten::val::null()}; + } + + return FilteredLogEventMapTsType{emscripten::val::array(m_filtered_log_event_map.value())}; } void StructuredIrStreamReader::filter_log_events(LogLevelFilterTsType const& log_level_filter) { if (log_level_filter.isNull()) { + m_filtered_log_event_map.reset(); return; } - SPDLOG_ERROR(cLogLevelFilteringNotSupportedErrorMsg); + + m_filtered_log_event_map.emplace(); + auto filter_levels{emscripten::vecFromJSArray>(log_level_filter + )}; + for (size_t log_event_idx = 0; log_event_idx < m_deserialized_log_events->size(); ++log_event_idx) { + auto const& log_event = m_deserialized_log_events->at(log_event_idx); + if (std::ranges::find( + filter_levels, + clp::enum_to_underlying_type(log_event.get_log_level()) + ) + != filter_levels.end()) + { + m_filtered_log_event_map->emplace_back(log_event_idx); + } + } } auto StructuredIrStreamReader::deserialize_stream() -> size_t { @@ -258,7 +275,7 @@ auto StructuredIrStreamReader::decode_range(size_t begin_idx, size_t end_idx, bo StructuredIrStreamReader::StructuredIrStreamReader( StreamReaderDataContext&& stream_reader_data_context, - std::shared_ptr> deserialized_log_events + std::shared_ptr>> deserialized_log_events ) : m_deserialized_log_events{std::move(deserialized_log_events)}, m_stream_reader_data_context{ diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp index b227cc3a..f6120275 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp @@ -179,6 +179,7 @@ class StructuredIrStreamReader : public StreamReader { // Variables std::shared_ptr>> m_deserialized_log_events; std::unique_ptr> m_stream_reader_data_context; + FilteredLogEventsMap m_filtered_log_event_map; }; } // namespace clp_ffi_js::ir diff --git a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp index 108ca157..c4c9ce67 100644 --- a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp +++ b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp @@ -22,11 +22,6 @@ using clp::ir::four_byte_encoded_variable_t; using UnstructuredIrDeserializer = clp::ir::LogEventDeserializer; using UnstructuredLogEvent = clp::ir::LogEvent; -/** - * Mapping between an index in the filtered log events collection to an index in the unfiltered - * log events collection. - */ -using FilteredLogEventsMap = std::optional>; /** * Class to deserialize and decode Zstd-compressed CLP unstructured IR streams, as well as format From 3f32f91497d99c41024485d6e141bde9ed5bfc08 Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Sun, 10 Nov 2024 03:05:37 +0000 Subject: [PATCH 14/52] fix lint --- .../ir/StructuredIrStreamReader.cpp | 25 +++++++++++++------ .../ir/StructuredIrStreamReader.hpp | 16 ++++++------ .../ir/UnstructuredIrStreamReader.hpp | 1 - 3 files changed, 25 insertions(+), 17 deletions(-) diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp index 5fa2cf8b..21b8132a 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp @@ -55,13 +55,18 @@ auto get_log_level(std::string_view str) -> LogLevel { ); // Do not accept "None" when checking if input string is in `cLogLevelNames`. - const auto* it = std::ranges::find(cLogLevelNames.begin() + clp::enum_to_underlying_type(cValidLogLevelsBeginIdx), cLogLevelNames.end(), log_level_name_upper_case); + auto const* it = std::ranges::find( + cLogLevelNames.begin() + clp::enum_to_underlying_type(cValidLogLevelsBeginIdx), + cLogLevelNames.end(), + log_level_name_upper_case + ); if (it == cLogLevelNames.end()) { return log_level; } - return static_cast(std::distance(cLogLevelNames.begin(), it));; + return static_cast(std::distance(cLogLevelNames.begin(), it)); + ; } } // namespace @@ -85,7 +90,6 @@ auto IrUnitHandler::handle_schema_tree_node_insertion( auto IrUnitHandler::handle_log_event(StructuredLogEvent&& log_event ) -> clp::ffi::ir_stream::IRErrorCode { - auto const& id_value_pairs{log_event.get_node_id_value_pairs()}; clp::ffi::value_int_t timestamp{0}; @@ -106,7 +110,8 @@ auto IrUnitHandler::handle_log_event(StructuredLogEvent&& log_event auto const& log_level_pair{id_value_pairs.at(m_log_level_node_id.value())}; if (log_level_pair.has_value()) { if (log_level_pair->is()) { - auto const& log_level_name = log_level_pair.value().get_immutable_view(); + auto const& log_level_name + = log_level_pair.value().get_immutable_view(); log_level = get_log_level(log_level_name); } else { SPDLOG_ERROR("Log level type is not string"); @@ -128,7 +133,9 @@ auto StructuredIrStreamReader::create( clp::Array data_array, ReaderOptions const& reader_options ) -> StructuredIrStreamReader { - auto deserialized_log_events{std::make_shared>>()}; + auto deserialized_log_events{ + std::make_shared>>() + }; auto result{StructuredIrDeserializer::create( *zstd_decompressor, IrUnitHandler{ @@ -179,7 +186,9 @@ void StructuredIrStreamReader::filter_log_events(LogLevelFilterTsType const& log m_filtered_log_event_map.emplace(); auto filter_levels{emscripten::vecFromJSArray>(log_level_filter )}; - for (size_t log_event_idx = 0; log_event_idx < m_deserialized_log_events->size(); ++log_event_idx) { + for (size_t log_event_idx = 0; log_event_idx < m_deserialized_log_events->size(); + ++log_event_idx) + { auto const& log_event = m_deserialized_log_events->at(log_event_idx); if (std::ranges::find( filter_levels, @@ -259,7 +268,6 @@ auto StructuredIrStreamReader::decode_range(size_t begin_idx, size_t end_idx, bo json_str = json_result.value().dump(); } - EM_ASM( { Emval.toValue($0).push([UTF8ToString($1), $2, $3, $4]); }, results.as_handle(), @@ -275,7 +283,8 @@ auto StructuredIrStreamReader::decode_range(size_t begin_idx, size_t end_idx, bo StructuredIrStreamReader::StructuredIrStreamReader( StreamReaderDataContext&& stream_reader_data_context, - std::shared_ptr>> deserialized_log_events + std::shared_ptr>> + deserialized_log_events ) : m_deserialized_log_events{std::move(deserialized_log_events)}, m_stream_reader_data_context{ diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp index f6120275..397cd7bf 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp @@ -37,7 +37,8 @@ class IrUnitHandler { * @param timestamp_key Key name of schema-tree node that contains the authoritative timestamp. */ IrUnitHandler( - std::shared_ptr>> deserialized_log_events, + std::shared_ptr>> + deserialized_log_events, std::string log_level_key, std::string timestamp_key ) @@ -86,10 +87,6 @@ class IrUnitHandler { // Methods - - - - /** * @return The schema-tree node ID associated with events' authoritative timestamp key. */ @@ -110,7 +107,8 @@ class IrUnitHandler { // TODO: Technically, we don't need to use a `shared_ptr` since the parent stream reader will // have a longer lifetime than this class. Instead, we could use `gsl::not_null` once we add // `gsl` into the project. - std::shared_ptr>> m_deserialized_log_events; + std::shared_ptr>> + m_deserialized_log_events; }; using StructuredIrDeserializer = clp::ffi::ir_stream::Deserializer; @@ -173,11 +171,13 @@ class StructuredIrStreamReader : public StreamReader { // Constructor explicit StructuredIrStreamReader( StreamReaderDataContext&& stream_reader_data_context, - std::shared_ptr>> deserialized_log_events + std::shared_ptr>> + deserialized_log_events ); // Variables - std::shared_ptr>> m_deserialized_log_events; + std::shared_ptr>> + m_deserialized_log_events; std::unique_ptr> m_stream_reader_data_context; FilteredLogEventsMap m_filtered_log_event_map; }; diff --git a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp index c4c9ce67..8e5dc1ee 100644 --- a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp +++ b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp @@ -22,7 +22,6 @@ using clp::ir::four_byte_encoded_variable_t; using UnstructuredIrDeserializer = clp::ir::LogEventDeserializer; using UnstructuredLogEvent = clp::ir::LogEvent; - /** * Class to deserialize and decode Zstd-compressed CLP unstructured IR streams, as well as format * decoded log events. From 01ef4154c19369ebe7683833f025711bcd0508b1 Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Sun, 10 Nov 2024 22:56:09 +0000 Subject: [PATCH 15/52] add support for int --- src/clp_ffi_js/constants.hpp | 1 + src/clp_ffi_js/ir/StructuredIrStreamReader.cpp | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/src/clp_ffi_js/constants.hpp b/src/clp_ffi_js/constants.hpp index 045227da..5cd0ab1d 100644 --- a/src/clp_ffi_js/constants.hpp +++ b/src/clp_ffi_js/constants.hpp @@ -19,6 +19,7 @@ enum class LogLevel : std::uint8_t { FATAL, }; constexpr LogLevel cValidLogLevelsBeginIdx{LogLevel::TRACE}; +constexpr LogLevel cValidLogLevelsEndIdx{LogLevel::FATAL}; /** * Strings corresponding to `LogLevel`. diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp index 21b8132a..af409bdd 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -113,6 +114,11 @@ auto IrUnitHandler::handle_log_event(StructuredLogEvent&& log_event auto const& log_level_name = log_level_pair.value().get_immutable_view(); log_level = get_log_level(log_level_name); + } else if (log_level_pair->is()) { + auto const& value = (log_level_pair.value().get_immutable_view()); + if (value <= (clp::enum_to_underlying_type(cValidLogLevelsEndIdx))) { + log_level = static_cast(value); + } } else { SPDLOG_ERROR("Log level type is not string"); } From de5aaf596185e3ca3bdb5bde5f1c58f41f0783f8 Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Mon, 11 Nov 2024 00:06:37 +0000 Subject: [PATCH 16/52] add new functions --- src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp | 148 ++++++++++++++++++ src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp | 130 +++++++++++++++ 2 files changed, 278 insertions(+) create mode 100644 src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp create mode 100644 src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp diff --git a/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp b/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp new file mode 100644 index 00000000..197143e9 --- /dev/null +++ b/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp @@ -0,0 +1,148 @@ +#include "StructuredIrUnitHandler.hpp" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace clp_ffi_js::ir { + +namespace { +/** + * Parses the `LogLevel` from an input string. + * @param str + * @return + */ +auto parse_log_level(std::string_view str) -> LogLevel; + +auto parse_log_level(std::string_view str) -> LogLevel { + LogLevel log_level{LogLevel::NONE}; + + // Convert the string to uppercase, + std::string log_level_name_upper_case{str}; + std::transform( + log_level_name_upper_case.begin(), + log_level_name_upper_case.end(), + log_level_name_upper_case.begin(), + [](unsigned char c) { return std::toupper(c); } + ); + + // Do not accept "None" when checking if input string is in `cLogLevelNames`. + auto const* it = std::ranges::find( + cLogLevelNames.begin() + clp::enum_to_underlying_type(cValidLogLevelsBeginIdx), + cLogLevelNames.end(), + log_level_name_upper_case + ); + + if (it == cLogLevelNames.end()) { + return log_level; + } + + return static_cast(std::distance(cLogLevelNames.begin(), it)); +} +} // namespace + +using clp::ir::four_byte_encoded_variable_t; +auto IrUnitHandler::handle_schema_tree_node_insertion( + clp::ffi::SchemaTree::NodeLocator schema_tree_node_locator +) -> clp::ffi::ir_stream::IRErrorCode { + ++m_current_node_id; + + auto const& key_name{schema_tree_node_locator.get_key_name()}; + if (m_log_level_key == key_name) { + m_log_level_node_id.emplace(m_current_node_id); + } else if (m_timestamp_key == key_name) { + m_timestamp_node_id.emplace(m_current_node_id); + } + + return clp::ffi::ir_stream::IRErrorCode::IRErrorCode_Success; +} + +auto IrUnitHandler::handle_log_event(StructuredLogEvent&& log_event +) -> clp::ffi::ir_stream::IRErrorCode { + auto const& id_value_pairs{log_event.get_node_id_value_pairs()}; + clp::ffi::value_int_t timestamp = get_timestamp(id_value_pairs); + LogLevel log_level = get_log_level(id_value_pairs); + + auto log_event_with_filter_data{ + LogEventWithFilterData(std::move(log_event), log_level, timestamp) + }; + + m_deserialized_log_events->emplace_back(std::move(log_event_with_filter_data)); + + return clp::ffi::ir_stream::IRErrorCode::IRErrorCode_Success; +} + +auto IrUnitHandler::get_log_level(StructuredLogEvent::NodeIdValuePairs const& id_value_pairs) const -> LogLevel { + LogLevel log_level{LogLevel::NONE}; + + if (false == m_log_level_node_id.has_value()) { + return log_level; + } + + auto const& log_level_pair{id_value_pairs.at(m_log_level_node_id.value())}; + + if (false == log_level_pair.has_value()) { + return log_level; + } + + if (log_level_pair->is()) { + auto const& log_level_node_value + = log_level_pair.value().get_immutable_view(); + log_level = parse_log_level(log_level_node_value); + } else if (log_level_pair->is()) { + auto const& log_level_node_value = (log_level_pair.value().get_immutable_view()); + if (log_level_node_value <= (clp::enum_to_underlying_type(cValidLogLevelsEndIdx))) { + log_level = static_cast(log_level_node_value); + } + } else { + auto log_event_idx = m_deserialized_log_events->size(); + SPDLOG_ERROR("Log level type is not int or string for log event index {}", log_event_idx); + } + + return log_level; + +} + +auto IrUnitHandler::get_timestamp(StructuredLogEvent::NodeIdValuePairs const& id_value_pairs) const -> clp::ffi::value_int_t { + clp::ffi::value_int_t timestamp{0}; + + if (false == m_timestamp_node_id.has_value()) { + return timestamp; + } + + auto const& timestamp_pair{id_value_pairs.at(m_timestamp_node_id.value())}; + + if (false == timestamp_pair.has_value()) { + return timestamp; + } + + if (timestamp_pair->is()) { + timestamp = timestamp_pair.value().get_immutable_view(); + } else { + // TODO: Add support for parsing timestamp values of string type. + auto log_event_idx = m_deserialized_log_events->size(); + } + + return timestamp; +} +} // namespace clp_ffi_js::ir \ No newline at end of file diff --git a/src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp b/src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp new file mode 100644 index 00000000..a8bd569f --- /dev/null +++ b/src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp @@ -0,0 +1,130 @@ +#ifndef CLP_FFI_JS_IR_STRUCTUREDIRUNITHANDLER_HPP +#define CLP_FFI_JS_IR_STRUCTUREDIRUNITHANDLER_HPP + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace clp_ffi_js::ir { +using schema_tree_node_id_t = std::optional; +using StructuredLogEvent = clp::ffi::KeyValuePairLogEvent; + +/** + * Class that implements the `clp::ffi::ir_stream::IrUnitHandlerInterface` to buffer log events and + * determine the schema-tree node ID of the timestamp kv-pair. + */ +class IrUnitHandler { +public: + /** + * @param deserialized_log_events The vector in which to store deserialized log events. + * @param log_level_key Key name of schema-tree node that contains the authoritative log level. + * @param timestamp_key Key name of schema-tree node that contains the authoritative timestamp. + */ + IrUnitHandler( + std::shared_ptr>> + deserialized_log_events, + std::string log_level_key, + std::string timestamp_key + ) + : m_log_level_key{std::move(log_level_key)}, + m_timestamp_key{std::move(timestamp_key)}, + m_deserialized_log_events{std::move(deserialized_log_events)} {} + + // Methods implementing `clp::ffi::ir_stream::IrUnitHandlerInterface`. + /** + * Buffers the log event. + * @param log_event + * @return IRErrorCode::IRErrorCode_Success + */ + [[nodiscard]] auto handle_log_event(StructuredLogEvent&& log_event + ) -> clp::ffi::ir_stream::IRErrorCode; + + /** + * @param utc_offset_old + * @param utc_offset_new + * @return IRErrorCode::IRErrorCode_Success + */ + [[nodiscard]] static auto handle_utc_offset_change( + [[maybe_unused]] clp::UtcOffset utc_offset_old, + [[maybe_unused]] clp::UtcOffset utc_offset_new + ) -> clp::ffi::ir_stream::IRErrorCode { + SPDLOG_WARN("UTC offset change packets aren't handled currently."); + + return clp::ffi::ir_stream::IRErrorCode::IRErrorCode_Success; + } + + /** + * Saves the node's ID if it corresponds to events' authoritative timestamp kv-pair. + * @param schema_tree_node_locator + * @return IRErrorCode::IRErrorCode_Success + */ + [[nodiscard]] auto handle_schema_tree_node_insertion( + clp::ffi::SchemaTree::NodeLocator schema_tree_node_locator + ) -> clp::ffi::ir_stream::IRErrorCode; + + /** + * @return IRErrorCode::IRErrorCode_Success + */ + [[nodiscard]] static auto handle_end_of_stream() -> clp::ffi::ir_stream::IRErrorCode { + return clp::ffi::ir_stream::IRErrorCode::IRErrorCode_Success; + } + + // Methods + + /** + * @return The schema-tree node ID associated with events' authoritative timestamp key. + */ + [[nodiscard]] auto get_timestamp_node_id() const -> schema_tree_node_id_t { + return m_timestamp_node_id; + } + +private: + // Methods + + /** + * @param id_value_pairs + * @return Timestamp from `StructuredLogEvent` + */ + [[nodiscard]] auto get_log_level(StructuredLogEvent::NodeIdValuePairs const& id_value_pairs) const -> LogLevel; + + /** + * @param id_value_pairs + * @return Timestamp from `StructuredLogEvent` + */ + [[nodiscard]] auto get_timestamp(StructuredLogEvent::NodeIdValuePairs const& id_value_pairs) const -> clp::ffi::value_int_t; + + + // Variables + std::string m_log_level_key; + std::string m_timestamp_key; + + clp::ffi::SchemaTree::Node::id_t m_current_node_id{clp::ffi::SchemaTree::cRootId}; + + schema_tree_node_id_t m_timestamp_node_id; + schema_tree_node_id_t m_log_level_node_id; + + // TODO: Technically, we don't need to use a `shared_ptr` since the parent stream reader will + // have a longer lifetime than this class. Instead, we could use `gsl::not_null` once we add + // `gsl` into the project. + std::shared_ptr>> + m_deserialized_log_events; +}; +} // namespace clp_ffi_js::ir + +#endif // CLP_FFI_JS_IR_STRUCTUREDIRSTREAMREADER_HPP \ No newline at end of file From 67f43f097e0a4d71d9574ce9f2cc3a606c70a741 Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Mon, 11 Nov 2024 02:16:51 +0000 Subject: [PATCH 17/52] cleanup --- .../ir/StructuredIrStreamReader.cpp | 102 +----------------- .../ir/StructuredIrStreamReader.hpp | 91 +--------------- src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp | 47 ++++---- src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp | 23 ++-- .../ir/UnstructuredIrStreamReader.hpp | 1 - 5 files changed, 40 insertions(+), 224 deletions(-) diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp index af409bdd..9bcf905f 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp @@ -14,8 +14,6 @@ #include #include #include -#include -#include #include #include #include @@ -25,6 +23,7 @@ #include #include #include +#include #include namespace clp_ffi_js::ir { @@ -35,105 +34,8 @@ constexpr std::string_view cLogLevelFilteringNotSupportedErrorMsg{ }; constexpr std::string_view cReaderOptionsLogLevelKey{"logLevelKey"}; constexpr std::string_view cReaderOptionsTimestampKey{"timestampKey"}; - -/** - * Gets the `LogLevel` from an input string. - * @param str - * @return - */ -auto get_log_level(std::string_view str) -> LogLevel; - -auto get_log_level(std::string_view str) -> LogLevel { - LogLevel log_level{LogLevel::NONE}; - - // Convert the string to uppercase, - std::string log_level_name_upper_case{str}; - std::transform( - log_level_name_upper_case.begin(), - log_level_name_upper_case.end(), - log_level_name_upper_case.begin(), - [](unsigned char c) { return std::toupper(c); } - ); - - // Do not accept "None" when checking if input string is in `cLogLevelNames`. - auto const* it = std::ranges::find( - cLogLevelNames.begin() + clp::enum_to_underlying_type(cValidLogLevelsBeginIdx), - cLogLevelNames.end(), - log_level_name_upper_case - ); - - if (it == cLogLevelNames.end()) { - return log_level; - } - - return static_cast(std::distance(cLogLevelNames.begin(), it)); - ; -} - } // namespace -using clp::ir::four_byte_encoded_variable_t; - -auto IrUnitHandler::handle_schema_tree_node_insertion( - clp::ffi::SchemaTree::NodeLocator schema_tree_node_locator -) -> clp::ffi::ir_stream::IRErrorCode { - ++m_current_node_id; - - auto const& key_name{schema_tree_node_locator.get_key_name()}; - if (m_log_level_key == key_name) { - m_log_level_node_id.emplace(m_current_node_id); - } else if (m_timestamp_key == key_name) { - m_timestamp_node_id.emplace(m_current_node_id); - } - - return clp::ffi::ir_stream::IRErrorCode::IRErrorCode_Success; -} - -auto IrUnitHandler::handle_log_event(StructuredLogEvent&& log_event -) -> clp::ffi::ir_stream::IRErrorCode { - auto const& id_value_pairs{log_event.get_node_id_value_pairs()}; - clp::ffi::value_int_t timestamp{0}; - - if (m_timestamp_node_id.has_value()) { - auto const& timestamp_pair{id_value_pairs.at(m_timestamp_node_id.value())}; - if (timestamp_pair.has_value()) { - if (timestamp_pair->is()) { - timestamp = timestamp_pair.value().get_immutable_view(); - } else { - // TODO: Add support for parsing timestamp values of string type. - SPDLOG_ERROR("Timestamp type is not int"); - } - } - } - - LogLevel log_level{LogLevel::NONE}; - if (m_log_level_node_id.has_value()) { - auto const& log_level_pair{id_value_pairs.at(m_log_level_node_id.value())}; - if (log_level_pair.has_value()) { - if (log_level_pair->is()) { - auto const& log_level_name - = log_level_pair.value().get_immutable_view(); - log_level = get_log_level(log_level_name); - } else if (log_level_pair->is()) { - auto const& value = (log_level_pair.value().get_immutable_view()); - if (value <= (clp::enum_to_underlying_type(cValidLogLevelsEndIdx))) { - log_level = static_cast(value); - } - } else { - SPDLOG_ERROR("Log level type is not string"); - } - } - } - - auto log_event_with_filter_data{ - LogEventWithFilterData(std::move(log_event), log_level, timestamp) - }; - - m_deserialized_log_events->emplace_back(std::move(log_event_with_filter_data)); - - return clp::ffi::ir_stream::IRErrorCode::IRErrorCode_Success; -} - auto StructuredIrStreamReader::create( std::unique_ptr&& zstd_decompressor, clp::Array data_array, @@ -144,7 +46,7 @@ auto StructuredIrStreamReader::create( }; auto result{StructuredIrDeserializer::create( *zstd_decompressor, - IrUnitHandler{ + StructuredIrUnitHandler{ deserialized_log_events, reader_options[cReaderOptionsLogLevelKey.data()].as(), reader_options[cReaderOptionsTimestampKey.data()].as() diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp index 397cd7bf..5a372fbf 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp @@ -1,6 +1,7 @@ #ifndef CLP_FFI_JS_IR_STRUCTUREDIRSTREAMREADER_HPP #define CLP_FFI_JS_IR_STRUCTUREDIRSTREAMREADER_HPP +#include "clp_ffi_js/ir/StructuredIrUnitHandler.hpp" #include #include #include @@ -19,99 +20,13 @@ #include #include +#include #include namespace clp_ffi_js::ir { using schema_tree_node_id_t = std::optional; using StructuredLogEvent = clp::ffi::KeyValuePairLogEvent; - -/** - * Class that implements the `clp::ffi::ir_stream::IrUnitHandlerInterface` to buffer log events and - * determine the schema-tree node ID of the timestamp kv-pair. - */ -class IrUnitHandler { -public: - /** - * @param deserialized_log_events The vector in which to store deserialized log events. - * @param log_level_key Key name of schema-tree node that contains the authoritative log level. - * @param timestamp_key Key name of schema-tree node that contains the authoritative timestamp. - */ - IrUnitHandler( - std::shared_ptr>> - deserialized_log_events, - std::string log_level_key, - std::string timestamp_key - ) - : m_log_level_key{std::move(log_level_key)}, - m_timestamp_key{std::move(timestamp_key)}, - m_deserialized_log_events{std::move(deserialized_log_events)} {} - - // Methods implementing `clp::ffi::ir_stream::IrUnitHandlerInterface`. - /** - * Buffers the log event. - * @param log_event - * @return IRErrorCode::IRErrorCode_Success - */ - [[nodiscard]] auto handle_log_event(StructuredLogEvent&& log_event - ) -> clp::ffi::ir_stream::IRErrorCode; - - /** - * @param utc_offset_old - * @param utc_offset_new - * @return IRErrorCode::IRErrorCode_Success - */ - [[nodiscard]] static auto handle_utc_offset_change( - [[maybe_unused]] clp::UtcOffset utc_offset_old, - [[maybe_unused]] clp::UtcOffset utc_offset_new - ) -> clp::ffi::ir_stream::IRErrorCode { - SPDLOG_WARN("UTC offset change packets aren't handled currently."); - - return clp::ffi::ir_stream::IRErrorCode::IRErrorCode_Success; - } - - /** - * Saves the node's ID if it corresponds to events' authoritative timestamp kv-pair. - * @param schema_tree_node_locator - * @return IRErrorCode::IRErrorCode_Success - */ - [[nodiscard]] auto handle_schema_tree_node_insertion( - clp::ffi::SchemaTree::NodeLocator schema_tree_node_locator - ) -> clp::ffi::ir_stream::IRErrorCode; - - /** - * @return IRErrorCode::IRErrorCode_Success - */ - [[nodiscard]] static auto handle_end_of_stream() -> clp::ffi::ir_stream::IRErrorCode { - return clp::ffi::ir_stream::IRErrorCode::IRErrorCode_Success; - } - - // Methods - - /** - * @return The schema-tree node ID associated with events' authoritative timestamp key. - */ - [[nodiscard]] auto get_timestamp_node_id() const -> schema_tree_node_id_t { - return m_timestamp_node_id; - } - -private: - // Variables - std::string m_log_level_key; - std::string m_timestamp_key; - - clp::ffi::SchemaTree::Node::id_t m_current_node_id{clp::ffi::SchemaTree::cRootId}; - - schema_tree_node_id_t m_timestamp_node_id; - schema_tree_node_id_t m_log_level_node_id; - - // TODO: Technically, we don't need to use a `shared_ptr` since the parent stream reader will - // have a longer lifetime than this class. Instead, we could use `gsl::not_null` once we add - // `gsl` into the project. - std::shared_ptr>> - m_deserialized_log_events; -}; - -using StructuredIrDeserializer = clp::ffi::ir_stream::Deserializer; +using StructuredIrDeserializer = clp::ffi::ir_stream::Deserializer; /** * Class to deserialize and decode Zstd-compressed CLP structured IR streams, as well as format diff --git a/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp b/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp index 197143e9..f7075cc0 100644 --- a/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp +++ b/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp @@ -1,28 +1,25 @@ #include "StructuredIrUnitHandler.hpp" +#include #include -#include +#include +#include #include #include #include #include +#include + -#include -#include -#include #include +#include #include #include -#include -#include #include #include -#include #include #include -#include -#include namespace clp_ffi_js::ir { @@ -39,7 +36,7 @@ auto parse_log_level(std::string_view str) -> LogLevel { // Convert the string to uppercase, std::string log_level_name_upper_case{str}; - std::transform( + std::ranges::transform( log_level_name_upper_case.begin(), log_level_name_upper_case.end(), log_level_name_upper_case.begin(), @@ -57,12 +54,14 @@ auto parse_log_level(std::string_view str) -> LogLevel { return log_level; } - return static_cast(std::distance(cLogLevelNames.begin(), it)); + log_level = static_cast(std::distance(cLogLevelNames.begin(), it)); + return log_level; } -} // namespace +} // namespace using clp::ir::four_byte_encoded_variable_t; -auto IrUnitHandler::handle_schema_tree_node_insertion( + +auto StructuredIrUnitHandler::handle_schema_tree_node_insertion( clp::ffi::SchemaTree::NodeLocator schema_tree_node_locator ) -> clp::ffi::ir_stream::IRErrorCode { ++m_current_node_id; @@ -77,11 +76,11 @@ auto IrUnitHandler::handle_schema_tree_node_insertion( return clp::ffi::ir_stream::IRErrorCode::IRErrorCode_Success; } -auto IrUnitHandler::handle_log_event(StructuredLogEvent&& log_event +auto StructuredIrUnitHandler::handle_log_event(StructuredLogEvent&& log_event ) -> clp::ffi::ir_stream::IRErrorCode { auto const& id_value_pairs{log_event.get_node_id_value_pairs()}; - clp::ffi::value_int_t timestamp = get_timestamp(id_value_pairs); - LogLevel log_level = get_log_level(id_value_pairs); + auto const timestamp = get_timestamp(id_value_pairs); + auto const log_level = get_log_level(id_value_pairs); auto log_event_with_filter_data{ LogEventWithFilterData(std::move(log_event), log_level, timestamp) @@ -92,7 +91,8 @@ auto IrUnitHandler::handle_log_event(StructuredLogEvent&& log_event return clp::ffi::ir_stream::IRErrorCode::IRErrorCode_Success; } -auto IrUnitHandler::get_log_level(StructuredLogEvent::NodeIdValuePairs const& id_value_pairs) const -> LogLevel { +auto StructuredIrUnitHandler::get_log_level(StructuredLogEvent::NodeIdValuePairs const& id_value_pairs +) const -> LogLevel { LogLevel log_level{LogLevel::NONE}; if (false == m_log_level_node_id.has_value()) { @@ -106,11 +106,11 @@ auto IrUnitHandler::get_log_level(StructuredLogEvent::NodeIdValuePairs const& id } if (log_level_pair->is()) { - auto const& log_level_node_value - = log_level_pair.value().get_immutable_view(); + auto const& log_level_node_value = log_level_pair.value().get_immutable_view(); log_level = parse_log_level(log_level_node_value); } else if (log_level_pair->is()) { - auto const& log_level_node_value = (log_level_pair.value().get_immutable_view()); + auto const& log_level_node_value + = (log_level_pair.value().get_immutable_view()); if (log_level_node_value <= (clp::enum_to_underlying_type(cValidLogLevelsEndIdx))) { log_level = static_cast(log_level_node_value); } @@ -120,10 +120,10 @@ auto IrUnitHandler::get_log_level(StructuredLogEvent::NodeIdValuePairs const& id } return log_level; - } -auto IrUnitHandler::get_timestamp(StructuredLogEvent::NodeIdValuePairs const& id_value_pairs) const -> clp::ffi::value_int_t { +auto StructuredIrUnitHandler::get_timestamp(StructuredLogEvent::NodeIdValuePairs const& id_value_pairs +) const -> clp::ffi::value_int_t { clp::ffi::value_int_t timestamp{0}; if (false == m_timestamp_node_id.has_value()) { @@ -141,8 +141,9 @@ auto IrUnitHandler::get_timestamp(StructuredLogEvent::NodeIdValuePairs const& id } else { // TODO: Add support for parsing timestamp values of string type. auto log_event_idx = m_deserialized_log_events->size(); + SPDLOG_ERROR("Timestamp type is not int for log event index {}", log_event_idx); } return timestamp; } -} // namespace clp_ffi_js::ir \ No newline at end of file +} // namespace clp_ffi_js::ir diff --git a/src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp b/src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp index a8bd569f..8d9df353 100644 --- a/src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp +++ b/src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp @@ -8,18 +8,16 @@ #include #include -#include + #include -#include #include +#include #include #include -#include #include +#include #include -#include -#include namespace clp_ffi_js::ir { using schema_tree_node_id_t = std::optional; @@ -29,14 +27,14 @@ using StructuredLogEvent = clp::ffi::KeyValuePairLogEvent; * Class that implements the `clp::ffi::ir_stream::IrUnitHandlerInterface` to buffer log events and * determine the schema-tree node ID of the timestamp kv-pair. */ -class IrUnitHandler { +class StructuredIrUnitHandler { public: /** * @param deserialized_log_events The vector in which to store deserialized log events. * @param log_level_key Key name of schema-tree node that contains the authoritative log level. * @param timestamp_key Key name of schema-tree node that contains the authoritative timestamp. */ - IrUnitHandler( + StructuredIrUnitHandler( std::shared_ptr>> deserialized_log_events, std::string log_level_key, @@ -97,18 +95,19 @@ class IrUnitHandler { private: // Methods - /** + /** * @param id_value_pairs * @return Timestamp from `StructuredLogEvent` */ - [[nodiscard]] auto get_log_level(StructuredLogEvent::NodeIdValuePairs const& id_value_pairs) const -> LogLevel; + [[nodiscard]] auto get_log_level(StructuredLogEvent::NodeIdValuePairs const& id_value_pairs + ) const -> LogLevel; /** * @param id_value_pairs * @return Timestamp from `StructuredLogEvent` */ - [[nodiscard]] auto get_timestamp(StructuredLogEvent::NodeIdValuePairs const& id_value_pairs) const -> clp::ffi::value_int_t; - + [[nodiscard]] auto get_timestamp(StructuredLogEvent::NodeIdValuePairs const& id_value_pairs + ) const -> clp::ffi::value_int_t; // Variables std::string m_log_level_key; @@ -127,4 +126,4 @@ class IrUnitHandler { }; } // namespace clp_ffi_js::ir -#endif // CLP_FFI_JS_IR_STRUCTUREDIRSTREAMREADER_HPP \ No newline at end of file +#endif // CLP_FFI_JS_IR_STRUCTUREDIRSTREAMREADER_HPP diff --git a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp index 8e5dc1ee..3ba49b71 100644 --- a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp +++ b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp @@ -4,7 +4,6 @@ #include #include #include -#include #include #include From be0e5bacf7336bd324988f75e5a45114b1e13898 Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Mon, 11 Nov 2024 03:37:05 +0000 Subject: [PATCH 18/52] lint --- src/clp_ffi_js/ir/LogEventWithFilterData.hpp | 21 +++++++++++-- .../ir/StructuredIrStreamReader.cpp | 30 +++++-------------- .../ir/StructuredIrStreamReader.hpp | 10 +------ src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp | 10 ++++--- src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp | 3 +- .../ir/UnstructuredIrStreamReader.cpp | 23 ++------------ 6 files changed, 36 insertions(+), 61 deletions(-) diff --git a/src/clp_ffi_js/ir/LogEventWithFilterData.hpp b/src/clp_ffi_js/ir/LogEventWithFilterData.hpp index e1007b59..6d1fde72 100644 --- a/src/clp_ffi_js/ir/LogEventWithFilterData.hpp +++ b/src/clp_ffi_js/ir/LogEventWithFilterData.hpp @@ -1,19 +1,35 @@ #ifndef CLP_FFI_JS_IR_LOGEVENTWITHFILTERDATA_HPP #define CLP_FFI_JS_IR_LOGEVENTWITHFILTERDATA_HPP +#include #include +#include +#include #include #include namespace clp_ffi_js::ir { +using clp::ir::four_byte_encoded_variable_t; +using UnstructuredLogEvent = clp::ir::LogEvent; +using StructuredLogEvent = clp::ffi::KeyValuePairLogEvent; + +// Concept defining valid log event types. +// TODO: Extend valid log event types when filtering support is added for structured logs. +template +concept ValidLogEventTypes + = std::same_as || std::same_as; + /** - * A class which accepts a log event type as a template parameter and provides additional members - * for the log level and timestamp. The additional members facilitate log level filtering. + * A templated class that extends a log event type with processed versions of some of its fields, + * specifically the fields that are used for filtering in the `StreamReader` classes and their + * callers. + * * @tparam LogEvent The type of the log event. */ template +requires ValidLogEventTypes class LogEventWithFilterData { public: // Constructor @@ -48,6 +64,7 @@ class LogEventWithFilterData { LogLevel m_log_level; clp::ir::epoch_time_ms_t m_timestamp; }; + } // namespace clp_ffi_js::ir #endif // CLP_FFI_JS_IR_LOGEVENTWITHFILTERDATA_HPP diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp index 9bcf905f..96c6be85 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp @@ -6,7 +6,6 @@ #include #include #include -#include #include #include @@ -23,8 +22,9 @@ #include #include #include -#include #include +#include +#include namespace clp_ffi_js::ir { namespace { @@ -86,27 +86,11 @@ auto StructuredIrStreamReader::get_filtered_log_event_map() const -> FilteredLog } void StructuredIrStreamReader::filter_log_events(LogLevelFilterTsType const& log_level_filter) { - if (log_level_filter.isNull()) { - m_filtered_log_event_map.reset(); - return; - } - - m_filtered_log_event_map.emplace(); - auto filter_levels{emscripten::vecFromJSArray>(log_level_filter - )}; - for (size_t log_event_idx = 0; log_event_idx < m_deserialized_log_events->size(); - ++log_event_idx) - { - auto const& log_event = m_deserialized_log_events->at(log_event_idx); - if (std::ranges::find( - filter_levels, - clp::enum_to_underlying_type(log_event.get_log_level()) - ) - != filter_levels.end()) - { - m_filtered_log_event_map->emplace_back(log_event_idx); - } - } + filter_deserialized_events( + log_level_filter, + m_filtered_log_event_map, + *m_deserialized_log_events + ); } auto StructuredIrStreamReader::deserialize_stream() -> size_t { diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp index 5a372fbf..6792aa62 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp @@ -1,31 +1,23 @@ #ifndef CLP_FFI_JS_IR_STRUCTUREDIRSTREAMREADER_HPP #define CLP_FFI_JS_IR_STRUCTUREDIRSTREAMREADER_HPP -#include "clp_ffi_js/ir/StructuredIrUnitHandler.hpp" #include #include #include -#include -#include #include #include -#include #include -#include #include -#include #include -#include #include #include -#include #include +#include namespace clp_ffi_js::ir { using schema_tree_node_id_t = std::optional; -using StructuredLogEvent = clp::ffi::KeyValuePairLogEvent; using StructuredIrDeserializer = clp::ffi::ir_stream::Deserializer; /** diff --git a/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp b/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp index f7075cc0..9060faae 100644 --- a/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp +++ b/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp @@ -1,8 +1,8 @@ #include "StructuredIrUnitHandler.hpp" #include -#include #include +#include #include #include #include @@ -10,7 +10,7 @@ #include #include - +#include #include #include #include @@ -91,7 +91,8 @@ auto StructuredIrUnitHandler::handle_log_event(StructuredLogEvent&& log_event return clp::ffi::ir_stream::IRErrorCode::IRErrorCode_Success; } -auto StructuredIrUnitHandler::get_log_level(StructuredLogEvent::NodeIdValuePairs const& id_value_pairs +auto StructuredIrUnitHandler::get_log_level( + StructuredLogEvent::NodeIdValuePairs const& id_value_pairs ) const -> LogLevel { LogLevel log_level{LogLevel::NONE}; @@ -122,7 +123,8 @@ auto StructuredIrUnitHandler::get_log_level(StructuredLogEvent::NodeIdValuePairs return log_level; } -auto StructuredIrUnitHandler::get_timestamp(StructuredLogEvent::NodeIdValuePairs const& id_value_pairs +auto StructuredIrUnitHandler::get_timestamp( + StructuredLogEvent::NodeIdValuePairs const& id_value_pairs ) const -> clp::ffi::value_int_t { clp::ffi::value_int_t timestamp{0}; diff --git a/src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp b/src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp index 8d9df353..fe53f40d 100644 --- a/src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp +++ b/src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp @@ -8,11 +8,10 @@ #include #include - #include #include -#include #include +#include #include #include diff --git a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp index d96661ed..aab9ffbc 100644 --- a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp @@ -9,7 +9,6 @@ #include #include #include -#include #include #include @@ -18,7 +17,6 @@ #include #include #include -#include #include #include #include @@ -28,6 +26,7 @@ #include #include #include +#include namespace clp_ffi_js::ir { @@ -73,25 +72,7 @@ auto UnstructuredIrStreamReader::get_filtered_log_event_map() const -> FilteredL } void UnstructuredIrStreamReader::filter_log_events(LogLevelFilterTsType const& log_level_filter) { - if (log_level_filter.isNull()) { - m_filtered_log_event_map.reset(); - return; - } - - m_filtered_log_event_map.emplace(); - auto filter_levels{emscripten::vecFromJSArray>(log_level_filter - )}; - for (size_t log_event_idx = 0; log_event_idx < m_encoded_log_events.size(); ++log_event_idx) { - auto const& log_event = m_encoded_log_events[log_event_idx]; - if (std::ranges::find( - filter_levels, - clp::enum_to_underlying_type(log_event.get_log_level()) - ) - != filter_levels.end()) - { - m_filtered_log_event_map->emplace_back(log_event_idx); - } - } + filter_deserialized_events(log_level_filter, m_filtered_log_event_map, m_encoded_log_events); } auto UnstructuredIrStreamReader::deserialize_stream() -> size_t { From 8e8c64816fa40b6558e0fa038de6db1c8b3d3df5 Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Mon, 11 Nov 2024 04:02:04 +0000 Subject: [PATCH 19/52] add files --- src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp | 11 +---- src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp | 20 +++------ src/clp_ffi_js/ir/utils.cpp | 42 +++++++++++++++++++ src/clp_ffi_js/ir/utils.hpp | 18 ++++++++ 4 files changed, 67 insertions(+), 24 deletions(-) create mode 100644 src/clp_ffi_js/ir/utils.cpp create mode 100644 src/clp_ffi_js/ir/utils.hpp diff --git a/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp b/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp index 9060faae..91083a59 100644 --- a/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp +++ b/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp @@ -34,7 +34,7 @@ auto parse_log_level(std::string_view str) -> LogLevel; auto parse_log_level(std::string_view str) -> LogLevel { LogLevel log_level{LogLevel::NONE}; - // Convert the string to uppercase, + // Convert the string to uppercase. std::string log_level_name_upper_case{str}; std::ranges::transform( log_level_name_upper_case.begin(), @@ -43,7 +43,6 @@ auto parse_log_level(std::string_view str) -> LogLevel { [](unsigned char c) { return std::toupper(c); } ); - // Do not accept "None" when checking if input string is in `cLogLevelNames`. auto const* it = std::ranges::find( cLogLevelNames.begin() + clp::enum_to_underlying_type(cValidLogLevelsBeginIdx), cLogLevelNames.end(), @@ -59,8 +58,6 @@ auto parse_log_level(std::string_view str) -> LogLevel { } } // namespace -using clp::ir::four_byte_encoded_variable_t; - auto StructuredIrUnitHandler::handle_schema_tree_node_insertion( clp::ffi::SchemaTree::NodeLocator schema_tree_node_locator ) -> clp::ffi::ir_stream::IRErrorCode { @@ -82,11 +79,7 @@ auto StructuredIrUnitHandler::handle_log_event(StructuredLogEvent&& log_event auto const timestamp = get_timestamp(id_value_pairs); auto const log_level = get_log_level(id_value_pairs); - auto log_event_with_filter_data{ - LogEventWithFilterData(std::move(log_event), log_level, timestamp) - }; - - m_deserialized_log_events->emplace_back(std::move(log_event_with_filter_data)); + m_deserialized_log_events->emplace_back(std::move(log_event), log_level, timestamp); return clp::ffi::ir_stream::IRErrorCode::IRErrorCode_Success; } diff --git a/src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp b/src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp index fe53f40d..ac1a8665 100644 --- a/src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp +++ b/src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp @@ -67,7 +67,7 @@ class StructuredIrUnitHandler { } /** - * Saves the node's ID if it corresponds to events' authoritative timestamp kv-pair. + * Saves the node's ID if it corresponds to events' authoritative log level or timestamp kv-pair. * @param schema_tree_node_locator * @return IRErrorCode::IRErrorCode_Success */ @@ -81,29 +81,19 @@ class StructuredIrUnitHandler { [[nodiscard]] static auto handle_end_of_stream() -> clp::ffi::ir_stream::IRErrorCode { return clp::ffi::ir_stream::IRErrorCode::IRErrorCode_Success; } - - // Methods - - /** - * @return The schema-tree node ID associated with events' authoritative timestamp key. - */ - [[nodiscard]] auto get_timestamp_node_id() const -> schema_tree_node_id_t { - return m_timestamp_node_id; - } - private: - // Methods + // Methods /** * @param id_value_pairs - * @return Timestamp from `StructuredLogEvent` + * @return `LogLevel` from node with id `m_log_level_node_id` */ [[nodiscard]] auto get_log_level(StructuredLogEvent::NodeIdValuePairs const& id_value_pairs ) const -> LogLevel; /** * @param id_value_pairs - * @return Timestamp from `StructuredLogEvent` + * @return Timestamp from node with id `m_timestamp_node_id;` */ [[nodiscard]] auto get_timestamp(StructuredLogEvent::NodeIdValuePairs const& id_value_pairs ) const -> clp::ffi::value_int_t; @@ -114,8 +104,8 @@ class StructuredIrUnitHandler { clp::ffi::SchemaTree::Node::id_t m_current_node_id{clp::ffi::SchemaTree::cRootId}; - schema_tree_node_id_t m_timestamp_node_id; schema_tree_node_id_t m_log_level_node_id; + schema_tree_node_id_t m_timestamp_node_id; // TODO: Technically, we don't need to use a `shared_ptr` since the parent stream reader will // have a longer lifetime than this class. Instead, we could use `gsl::not_null` once we add diff --git a/src/clp_ffi_js/ir/utils.cpp b/src/clp_ffi_js/ir/utils.cpp new file mode 100644 index 00000000..b26f7393 --- /dev/null +++ b/src/clp_ffi_js/ir/utils.cpp @@ -0,0 +1,42 @@ +#include "utils.hpp" + +#include +#include +#include +#include + +#include +#include + +#include +#include + +namespace clp_ffi_js::ir { +template +auto filter_deserialized_events( + LogLevelFilterTsType const& log_level_filter, + FilteredLogEventsMap& filtered_log_event_map, + LogEvents const& log_events +) -> void { + if (log_level_filter.isNull()) { + filtered_log_event_map.reset(); + return; + } + + filtered_log_event_map.emplace(); + auto filter_levels + = emscripten::vecFromJSArray>(log_level_filter); + + for (size_t log_event_idx = 0; log_event_idx < log_events.size(); ++log_event_idx) { + auto const& log_event = log_events[log_event_idx]; + if (std::ranges::find( + filter_levels, + clp::enum_to_underlying_type(log_event.get_log_level()) + ) + != filter_levels.end()) + { + filtered_log_event_map->emplace_back(log_event_idx); + } + } +} +} // namespace clp_ffi_js::ir diff --git a/src/clp_ffi_js/ir/utils.hpp b/src/clp_ffi_js/ir/utils.hpp new file mode 100644 index 00000000..996bff51 --- /dev/null +++ b/src/clp_ffi_js/ir/utils.hpp @@ -0,0 +1,18 @@ +#ifndef CLP_FFI_JS_IR_UTILS_HPP +#define CLP_FFI_JS_IR_UTILS_HPP + +#include + +#include "clp_ffi_js/ir/StreamReader.hpp" + +namespace clp_ffi_js::ir { + +template +auto filter_deserialized_events( + LogLevelFilterTsType log_level_filter, + FilteredLogEventsMap& filtered_log_event_map, + LogEvents const& log_events +) -> void; +} // namespace clp_ffi_js::ir + +#endif // CLP_FFI_JS_IR_UTILS_HPP From f2907f40f7147fc52c61b0d8920f37518db5883b Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Mon, 11 Nov 2024 18:13:22 +0000 Subject: [PATCH 20/52] changes --- CMakeLists.txt | 1 + src/clp_ffi_js/ir/LogEventWithFilterData.hpp | 12 +----- .../ir/StructuredIrStreamReader.cpp | 6 +-- src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp | 4 +- .../ir/UnstructuredIrStreamReader.cpp | 2 +- src/clp_ffi_js/ir/utils.cpp | 42 ------------------- src/clp_ffi_js/ir/utils.hpp | 42 +++++++++++++++++-- 7 files changed, 46 insertions(+), 63 deletions(-) delete mode 100644 src/clp_ffi_js/ir/utils.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index a0f20b76..b501712d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -84,6 +84,7 @@ endif() set(CLP_FFI_JS_SRC_MAIN src/clp_ffi_js/ir/StreamReader.cpp src/clp_ffi_js/ir/StructuredIrStreamReader.cpp + src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp ) diff --git a/src/clp_ffi_js/ir/LogEventWithFilterData.hpp b/src/clp_ffi_js/ir/LogEventWithFilterData.hpp index 6d1fde72..012cec2d 100644 --- a/src/clp_ffi_js/ir/LogEventWithFilterData.hpp +++ b/src/clp_ffi_js/ir/LogEventWithFilterData.hpp @@ -1,7 +1,6 @@ #ifndef CLP_FFI_JS_IR_LOGEVENTWITHFILTERDATA_HPP #define CLP_FFI_JS_IR_LOGEVENTWITHFILTERDATA_HPP -#include #include #include @@ -15,12 +14,6 @@ using clp::ir::four_byte_encoded_variable_t; using UnstructuredLogEvent = clp::ir::LogEvent; using StructuredLogEvent = clp::ffi::KeyValuePairLogEvent; -// Concept defining valid log event types. -// TODO: Extend valid log event types when filtering support is added for structured logs. -template -concept ValidLogEventTypes - = std::same_as || std::same_as; - /** * A templated class that extends a log event type with processed versions of some of its fields, * specifically the fields that are used for filtering in the `StreamReader` classes and their @@ -29,7 +22,7 @@ concept ValidLogEventTypes * @tparam LogEvent The type of the log event. */ template -requires ValidLogEventTypes +requires std::same_as || std::same_as class LogEventWithFilterData { public: // Constructor @@ -64,7 +57,6 @@ class LogEventWithFilterData { LogLevel m_log_level; clp::ir::epoch_time_ms_t m_timestamp; }; - } // namespace clp_ffi_js::ir -#endif // CLP_FFI_JS_IR_LOGEVENTWITHFILTERDATA_HPP +#endif // CLP_FFI_JS_IR_LOGEVENTWITHFILTERDATA_HPP \ No newline at end of file diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp index 96c6be85..ef1b763b 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp @@ -86,11 +86,7 @@ auto StructuredIrStreamReader::get_filtered_log_event_map() const -> FilteredLog } void StructuredIrStreamReader::filter_log_events(LogLevelFilterTsType const& log_level_filter) { - filter_deserialized_events( - log_level_filter, - m_filtered_log_event_map, - *m_deserialized_log_events - ); + filter_deserialized_events(m_filtered_log_event_map, log_level_filter, *m_deserialized_log_events); } auto StructuredIrStreamReader::deserialize_stream() -> size_t { diff --git a/src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp b/src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp index ac1a8665..2f05e30a 100644 --- a/src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp +++ b/src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp @@ -86,14 +86,14 @@ class StructuredIrUnitHandler { // Methods /** * @param id_value_pairs - * @return `LogLevel` from node with id `m_log_level_node_id` + * @return `LogLevel` from node with id `m_log_level_node_id`. */ [[nodiscard]] auto get_log_level(StructuredLogEvent::NodeIdValuePairs const& id_value_pairs ) const -> LogLevel; /** * @param id_value_pairs - * @return Timestamp from node with id `m_timestamp_node_id;` + * @return Timestamp from node with id `m_timestamp_node_id`. */ [[nodiscard]] auto get_timestamp(StructuredLogEvent::NodeIdValuePairs const& id_value_pairs ) const -> clp::ffi::value_int_t; diff --git a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp index aab9ffbc..9ec44fdd 100644 --- a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp @@ -72,7 +72,7 @@ auto UnstructuredIrStreamReader::get_filtered_log_event_map() const -> FilteredL } void UnstructuredIrStreamReader::filter_log_events(LogLevelFilterTsType const& log_level_filter) { - filter_deserialized_events(log_level_filter, m_filtered_log_event_map, m_encoded_log_events); + filter_deserialized_events(m_filtered_log_event_map, log_level_filter, m_encoded_log_events); } auto UnstructuredIrStreamReader::deserialize_stream() -> size_t { diff --git a/src/clp_ffi_js/ir/utils.cpp b/src/clp_ffi_js/ir/utils.cpp deleted file mode 100644 index b26f7393..00000000 --- a/src/clp_ffi_js/ir/utils.cpp +++ /dev/null @@ -1,42 +0,0 @@ -#include "utils.hpp" - -#include -#include -#include -#include - -#include -#include - -#include -#include - -namespace clp_ffi_js::ir { -template -auto filter_deserialized_events( - LogLevelFilterTsType const& log_level_filter, - FilteredLogEventsMap& filtered_log_event_map, - LogEvents const& log_events -) -> void { - if (log_level_filter.isNull()) { - filtered_log_event_map.reset(); - return; - } - - filtered_log_event_map.emplace(); - auto filter_levels - = emscripten::vecFromJSArray>(log_level_filter); - - for (size_t log_event_idx = 0; log_event_idx < log_events.size(); ++log_event_idx) { - auto const& log_event = log_events[log_event_idx]; - if (std::ranges::find( - filter_levels, - clp::enum_to_underlying_type(log_event.get_log_level()) - ) - != filter_levels.end()) - { - filtered_log_event_map->emplace_back(log_event_idx); - } - } -} -} // namespace clp_ffi_js::ir diff --git a/src/clp_ffi_js/ir/utils.hpp b/src/clp_ffi_js/ir/utils.hpp index 996bff51..3f3d8e1a 100644 --- a/src/clp_ffi_js/ir/utils.hpp +++ b/src/clp_ffi_js/ir/utils.hpp @@ -1,18 +1,54 @@ #ifndef CLP_FFI_JS_IR_UTILS_HPP #define CLP_FFI_JS_IR_UTILS_HPP +#include +#include +#include +#include + +#include #include -#include "clp_ffi_js/ir/StreamReader.hpp" +#include +#include namespace clp_ffi_js::ir { +/** +* Generates a filtered collection from all log events. +* +* @param log_level_filter Array of selected log levels. +* @param filtered_log_event_map A reference to `FilteredLogEventsMap` that stores filtered result. +* @param log_events +*/ template auto filter_deserialized_events( - LogLevelFilterTsType log_level_filter, FilteredLogEventsMap& filtered_log_event_map, + LogLevelFilterTsType const& log_level_filter, LogEvents const& log_events -) -> void; +) -> void { + if (log_level_filter.isNull()) { + filtered_log_event_map.reset(); + return; + } + + filtered_log_event_map.emplace(); + auto filter_levels + = emscripten::vecFromJSArray>(log_level_filter); + + for (size_t log_event_idx = 0; log_event_idx < log_events.size(); ++log_event_idx) { + auto const& log_event = log_events[log_event_idx]; + if (std::ranges::find( + filter_levels, + clp::enum_to_underlying_type(log_event.get_log_level()) + ) + != filter_levels.end()) + { + filtered_log_event_map->emplace_back(log_event_idx); + } + } +} } // namespace clp_ffi_js::ir + #endif // CLP_FFI_JS_IR_UTILS_HPP From a1b776fbda4daec6e096f4e30330c4fe61c5e6e6 Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Mon, 11 Nov 2024 18:47:42 +0000 Subject: [PATCH 21/52] fix filter --- src/clp_ffi_js/ir/StreamReader.cpp | 2 +- src/clp_ffi_js/ir/StructuredIrStreamReader.cpp | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/clp_ffi_js/ir/StreamReader.cpp b/src/clp_ffi_js/ir/StreamReader.cpp index e7d43d19..d00035cc 100644 --- a/src/clp_ffi_js/ir/StreamReader.cpp +++ b/src/clp_ffi_js/ir/StreamReader.cpp @@ -117,7 +117,7 @@ EMSCRIPTEN_BINDINGS(ClpStreamReader) { // JS types used as inputs emscripten::register_type("Uint8Array"); emscripten::register_type("number[] | null"); - emscripten::register_type("{timestampKey: string} | null"); + emscripten::register_type("{logLevelKey: string, timestampKey: string} | null"); // JS types used as outputs emscripten::enum_("IrStreamType") diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp index ef1b763b..f211d84d 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp @@ -128,8 +128,18 @@ auto StructuredIrStreamReader::deserialize_stream() -> size_t { auto StructuredIrStreamReader::decode_range(size_t begin_idx, size_t end_idx, bool use_filter) const -> DecodedResultsTsType { + + if (use_filter && false == m_filtered_log_event_map.has_value()) { + return DecodedResultsTsType{emscripten::val::null()}; + } + + size_t length{0}; if (use_filter) { - SPDLOG_ERROR(cLogLevelFilteringNotSupportedErrorMsg); + length = m_filtered_log_event_map->size(); + } else { + length = m_deserialized_log_events->size(); + } + if (length < end_idx || begin_idx > end_idx) { return DecodedResultsTsType{emscripten::val::null()}; } From d00d0a2c5d4676cbdee37d126e49a1ad93fb4886 Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Mon, 11 Nov 2024 18:58:34 +0000 Subject: [PATCH 22/52] change version --- CMakeLists.txt | 2 +- package.json | 2 +- src/clp_ffi_js/ir/LogEventWithFilterData.hpp | 5 +---- src/clp_ffi_js/ir/StreamReader.cpp | 4 +++- src/clp_ffi_js/ir/StructuredIrStreamReader.cpp | 7 +++++-- src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp | 5 +++-- src/clp_ffi_js/ir/utils.hpp | 13 ++++++------- 7 files changed, 20 insertions(+), 18 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b501712d..8f31295f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,7 +14,7 @@ project( LANGUAGES C CXX - VERSION 0.3.0 + VERSION 0.4.0 ) # Enable exporting compile commands diff --git a/package.json b/package.json index fac4f786..bcc2df79 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "clp-ffi-js", - "version": "0.3.0", + "version": "0.4.0", "description": "clp-ffi-js is a JavaScript FFI library for CLP.", "author": "YScope Inc. ", "license": "Apache-2.0", diff --git a/src/clp_ffi_js/ir/LogEventWithFilterData.hpp b/src/clp_ffi_js/ir/LogEventWithFilterData.hpp index ea37ad3b..542c0e30 100644 --- a/src/clp_ffi_js/ir/LogEventWithFilterData.hpp +++ b/src/clp_ffi_js/ir/LogEventWithFilterData.hpp @@ -1,14 +1,13 @@ #ifndef CLP_FFI_JS_IR_LOGEVENTWITHFILTERDATA_HPP #define CLP_FFI_JS_IR_LOGEVENTWITHFILTERDATA_HPP - #include #include +#include #include #include -#include #include namespace clp_ffi_js::ir { @@ -16,7 +15,6 @@ using clp::ir::four_byte_encoded_variable_t; using UnstructuredLogEvent = clp::ir::LogEvent; using StructuredLogEvent = clp::ffi::KeyValuePairLogEvent; - /** * A templated class that extends a log event type with processed versions of some of its fields, * specifically the fields that are used for filtering in the `StreamReader` classes and their @@ -64,4 +62,3 @@ class LogEventWithFilterData { } // namespace clp_ffi_js::ir #endif // CLP_FFI_JS_IR_LOGEVENTWITHFILTERDATA_HPP - diff --git a/src/clp_ffi_js/ir/StreamReader.cpp b/src/clp_ffi_js/ir/StreamReader.cpp index d00035cc..bcd8b952 100644 --- a/src/clp_ffi_js/ir/StreamReader.cpp +++ b/src/clp_ffi_js/ir/StreamReader.cpp @@ -117,7 +117,9 @@ EMSCRIPTEN_BINDINGS(ClpStreamReader) { // JS types used as inputs emscripten::register_type("Uint8Array"); emscripten::register_type("number[] | null"); - emscripten::register_type("{logLevelKey: string, timestampKey: string} | null"); + emscripten::register_type( + "{logLevelKey: string, timestampKey: string} | null" + ); // JS types used as outputs emscripten::enum_("IrStreamType") diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp index f211d84d..de8af41d 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp @@ -86,7 +86,11 @@ auto StructuredIrStreamReader::get_filtered_log_event_map() const -> FilteredLog } void StructuredIrStreamReader::filter_log_events(LogLevelFilterTsType const& log_level_filter) { - filter_deserialized_events(m_filtered_log_event_map, log_level_filter, *m_deserialized_log_events); + filter_deserialized_events( + m_filtered_log_event_map, + log_level_filter, + *m_deserialized_log_events + ); } auto StructuredIrStreamReader::deserialize_stream() -> size_t { @@ -128,7 +132,6 @@ auto StructuredIrStreamReader::deserialize_stream() -> size_t { auto StructuredIrStreamReader::decode_range(size_t begin_idx, size_t end_idx, bool use_filter) const -> DecodedResultsTsType { - if (use_filter && false == m_filtered_log_event_map.has_value()) { return DecodedResultsTsType{emscripten::val::null()}; } diff --git a/src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp b/src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp index 2f05e30a..ef0367bb 100644 --- a/src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp +++ b/src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp @@ -67,7 +67,8 @@ class StructuredIrUnitHandler { } /** - * Saves the node's ID if it corresponds to events' authoritative log level or timestamp kv-pair. + * Saves the node's ID if it corresponds to events' authoritative log level or timestamp + * kv-pair. * @param schema_tree_node_locator * @return IRErrorCode::IRErrorCode_Success */ @@ -81,8 +82,8 @@ class StructuredIrUnitHandler { [[nodiscard]] static auto handle_end_of_stream() -> clp::ffi::ir_stream::IRErrorCode { return clp::ffi::ir_stream::IRErrorCode::IRErrorCode_Success; } -private: +private: // Methods /** * @param id_value_pairs diff --git a/src/clp_ffi_js/ir/utils.hpp b/src/clp_ffi_js/ir/utils.hpp index 3f3d8e1a..674f5cd4 100644 --- a/src/clp_ffi_js/ir/utils.hpp +++ b/src/clp_ffi_js/ir/utils.hpp @@ -15,12 +15,12 @@ namespace clp_ffi_js::ir { /** -* Generates a filtered collection from all log events. -* -* @param log_level_filter Array of selected log levels. -* @param filtered_log_event_map A reference to `FilteredLogEventsMap` that stores filtered result. -* @param log_events -*/ + * Generates a filtered collection from all log events. + * + * @param log_level_filter Array of selected log levels. + * @param filtered_log_event_map A reference to `FilteredLogEventsMap` that stores filtered result. + * @param log_events + */ template auto filter_deserialized_events( FilteredLogEventsMap& filtered_log_event_map, @@ -50,5 +50,4 @@ auto filter_deserialized_events( } } // namespace clp_ffi_js::ir - #endif // CLP_FFI_JS_IR_UTILS_HPP From e9ed45db560b5cf61a8e227cc1f91d5cd5f1a7bf Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Mon, 11 Nov 2024 19:18:32 +0000 Subject: [PATCH 23/52] fix lint --- src/clp_ffi_js/ir/StreamReader.hpp | 2 ++ src/clp_ffi_js/ir/StructuredIrStreamReader.cpp | 1 - src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp | 5 +++-- src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/clp_ffi_js/ir/StreamReader.hpp b/src/clp_ffi_js/ir/StreamReader.hpp index c45443c6..ba87964a 100644 --- a/src/clp_ffi_js/ir/StreamReader.hpp +++ b/src/clp_ffi_js/ir/StreamReader.hpp @@ -4,6 +4,8 @@ #include #include #include +#include +#include #include #include diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp index de8af41d..386c515d 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp @@ -19,7 +19,6 @@ #include #include -#include #include #include #include diff --git a/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp b/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp index 91083a59..3a632cb7 100644 --- a/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp +++ b/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp @@ -14,7 +14,6 @@ #include #include #include -#include #include #include @@ -105,7 +104,9 @@ auto StructuredIrUnitHandler::get_log_level( } else if (log_level_pair->is()) { auto const& log_level_node_value = (log_level_pair.value().get_immutable_view()); - if (log_level_node_value <= (clp::enum_to_underlying_type(cValidLogLevelsEndIdx))) { + if (log_level_node_value >= clp::enum_to_underlying_type(cValidLogLevelsBeginIdx) + && log_level_node_value <= clp::enum_to_underlying_type(cValidLogLevelsEndIdx)) + { log_level = static_cast(log_level_node_value); } } else { diff --git a/src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp b/src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp index ef0367bb..1ab53cce 100644 --- a/src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp +++ b/src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp @@ -116,4 +116,4 @@ class StructuredIrUnitHandler { }; } // namespace clp_ffi_js::ir -#endif // CLP_FFI_JS_IR_STRUCTUREDIRSTREAMREADER_HPP +#endif // CLP_FFI_JS_IR_STRUCTUREDIRUNITHANDLER_HPP From 72029f1f9df5f3bda0d7594eb3cdc2f573991fa9 Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Mon, 11 Nov 2024 19:25:27 +0000 Subject: [PATCH 24/52] small change --- src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp | 3 +-- src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp | 2 -- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp b/src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp index 1ab53cce..d12b1900 100644 --- a/src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp +++ b/src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp @@ -20,11 +20,10 @@ namespace clp_ffi_js::ir { using schema_tree_node_id_t = std::optional; -using StructuredLogEvent = clp::ffi::KeyValuePairLogEvent; /** * Class that implements the `clp::ffi::ir_stream::IrUnitHandlerInterface` to buffer log events and - * determine the schema-tree node ID of the timestamp kv-pair. + * determine the schema-tree node ID of the log level and timestamp kv-pair. */ class StructuredIrUnitHandler { public: diff --git a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp index 3ba49b71..d2ba4d20 100644 --- a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp +++ b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp @@ -6,7 +6,6 @@ #include #include -#include #include #include #include @@ -19,7 +18,6 @@ namespace clp_ffi_js::ir { using clp::ir::four_byte_encoded_variable_t; using UnstructuredIrDeserializer = clp::ir::LogEventDeserializer; -using UnstructuredLogEvent = clp::ir::LogEvent; /** * Class to deserialize and decode Zstd-compressed CLP unstructured IR streams, as well as format From c75757104dc5819564fdd2dc64063040d3035400 Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Fri, 15 Nov 2024 16:15:02 +0000 Subject: [PATCH 25/52] junhao review --- CMakeLists.txt | 2 +- package.json | 2 +- src/clp_ffi_js/constants.hpp | 4 +++- src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp | 22 ++++++++++++++----- src/clp_ffi_js/ir/utils.hpp | 2 +- 5 files changed, 22 insertions(+), 10 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8f31295f..b501712d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,7 +14,7 @@ project( LANGUAGES C CXX - VERSION 0.4.0 + VERSION 0.3.0 ) # Enable exporting compile commands diff --git a/package.json b/package.json index bcc2df79..fac4f786 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "clp-ffi-js", - "version": "0.4.0", + "version": "0.3.0", "description": "clp-ffi-js is a JavaScript FFI library for CLP.", "author": "YScope Inc. ", "license": "Apache-2.0", diff --git a/src/clp_ffi_js/constants.hpp b/src/clp_ffi_js/constants.hpp index 5cd0ab1d..d96220d4 100644 --- a/src/clp_ffi_js/constants.hpp +++ b/src/clp_ffi_js/constants.hpp @@ -17,9 +17,11 @@ enum class LogLevel : std::uint8_t { WARN, ERROR, FATAL, + SIZE, // This isn't a valid log level. }; + constexpr LogLevel cValidLogLevelsBeginIdx{LogLevel::TRACE}; -constexpr LogLevel cValidLogLevelsEndIdx{LogLevel::FATAL}; +constexpr LogLevel cValidLogLevelsEndIdx{LogLevel::SIZE}; /** * Strings corresponding to `LogLevel`. diff --git a/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp b/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp index 3a632cb7..a91b7f19 100644 --- a/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp +++ b/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp @@ -24,12 +24,23 @@ namespace clp_ffi_js::ir { namespace { /** - * Parses the `LogLevel` from an input string. + * Parses a string to determine the corresponding `LogLevel` enum value. * @param str - * @return + * @return `LogLevel` enum corresponding to a string in `cLogLevelNames`. If input string + * is not in `cLogLevelNames`, returns `LogLevel::NONE`. + * */ auto parse_log_level(std::string_view str) -> LogLevel; + +/** + * Parses a string to determine the corresponding `LogLevel` enumeration value. + * + * @param str It should match one of the valid log level names in `cLogLevelNames`. + * @return The parsed value if the input is valid. + * If invalid, returns `LogLevel::NONE`. + */ + auto parse_log_level(std::string_view str) -> LogLevel { LogLevel log_level{LogLevel::NONE}; @@ -49,11 +60,10 @@ auto parse_log_level(std::string_view str) -> LogLevel { ); if (it == cLogLevelNames.end()) { - return log_level; + return LogLevel::NONE; } - log_level = static_cast(std::distance(cLogLevelNames.begin(), it)); - return log_level; + return static_cast(std::distance(cLogLevelNames.begin(), it)); } } // namespace @@ -105,7 +115,7 @@ auto StructuredIrUnitHandler::get_log_level( auto const& log_level_node_value = (log_level_pair.value().get_immutable_view()); if (log_level_node_value >= clp::enum_to_underlying_type(cValidLogLevelsBeginIdx) - && log_level_node_value <= clp::enum_to_underlying_type(cValidLogLevelsEndIdx)) + && log_level_node_value < clp::enum_to_underlying_type(cValidLogLevelsEndIdx)) { log_level = static_cast(log_level_node_value); } diff --git a/src/clp_ffi_js/ir/utils.hpp b/src/clp_ffi_js/ir/utils.hpp index 674f5cd4..bac8e525 100644 --- a/src/clp_ffi_js/ir/utils.hpp +++ b/src/clp_ffi_js/ir/utils.hpp @@ -18,8 +18,8 @@ namespace clp_ffi_js::ir { * Generates a filtered collection from all log events. * * @param log_level_filter Array of selected log levels. - * @param filtered_log_event_map A reference to `FilteredLogEventsMap` that stores filtered result. * @param log_events + * @param[out] filtered_log_event_map A reference to `FilteredLogEventsMap` that stores filtered result. */ template auto filter_deserialized_events( From 4975f791af0397096eb699fa805e2928dd616011 Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Mon, 25 Nov 2024 15:22:03 +0000 Subject: [PATCH 26/52] fix lint --- src/clp_ffi_js/constants.hpp | 2 +- src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp | 13 ++++++------- src/clp_ffi_js/ir/utils.hpp | 3 ++- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/clp_ffi_js/constants.hpp b/src/clp_ffi_js/constants.hpp index d96220d4..8c003b7f 100644 --- a/src/clp_ffi_js/constants.hpp +++ b/src/clp_ffi_js/constants.hpp @@ -17,7 +17,7 @@ enum class LogLevel : std::uint8_t { WARN, ERROR, FATAL, - SIZE, // This isn't a valid log level. + SIZE, // This isn't a valid log level. }; constexpr LogLevel cValidLogLevelsBeginIdx{LogLevel::TRACE}; diff --git a/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp b/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp index a91b7f19..3da4b3f9 100644 --- a/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp +++ b/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp @@ -32,14 +32,13 @@ namespace { */ auto parse_log_level(std::string_view str) -> LogLevel; - /** - * Parses a string to determine the corresponding `LogLevel` enumeration value. - * - * @param str It should match one of the valid log level names in `cLogLevelNames`. - * @return The parsed value if the input is valid. - * If invalid, returns `LogLevel::NONE`. - */ + * Parses a string to determine the corresponding `LogLevel` enumeration value. + * + * @param str It should match one of the valid log level names in `cLogLevelNames`. + * @return The parsed value if the input is valid. + * If invalid, returns `LogLevel::NONE`. + */ auto parse_log_level(std::string_view str) -> LogLevel { LogLevel log_level{LogLevel::NONE}; diff --git a/src/clp_ffi_js/ir/utils.hpp b/src/clp_ffi_js/ir/utils.hpp index bac8e525..a23040c2 100644 --- a/src/clp_ffi_js/ir/utils.hpp +++ b/src/clp_ffi_js/ir/utils.hpp @@ -19,7 +19,8 @@ namespace clp_ffi_js::ir { * * @param log_level_filter Array of selected log levels. * @param log_events - * @param[out] filtered_log_event_map A reference to `FilteredLogEventsMap` that stores filtered result. + * @param[out] filtered_log_event_map A reference to `FilteredLogEventsMap` that stores filtered + * result. */ template auto filter_deserialized_events( From 7346715991b241c9b362f32e551b2a16a828fe72 Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Mon, 25 Nov 2024 15:37:15 +0000 Subject: [PATCH 27/52] fix lint --- src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp b/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp index 3da4b3f9..2a498871 100644 --- a/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp +++ b/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp @@ -41,8 +41,6 @@ auto parse_log_level(std::string_view str) -> LogLevel; */ auto parse_log_level(std::string_view str) -> LogLevel { - LogLevel log_level{LogLevel::NONE}; - // Convert the string to uppercase. std::string log_level_name_upper_case{str}; std::ranges::transform( From 04a8d1f79c669c3c1a4b77a0306c681fcdfbcc44 Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Mon, 2 Dec 2024 19:18:10 +0000 Subject: [PATCH 28/52] junhao changes --- src/clp_ffi_js/constants.hpp | 23 ++++++----- .../ir/StructuredIrStreamReader.cpp | 10 ++--- src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp | 40 ++++++++++--------- src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp | 13 ++---- 4 files changed, 42 insertions(+), 44 deletions(-) diff --git a/src/clp_ffi_js/constants.hpp b/src/clp_ffi_js/constants.hpp index 8c003b7f..dd9ba2ad 100644 --- a/src/clp_ffi_js/constants.hpp +++ b/src/clp_ffi_js/constants.hpp @@ -4,6 +4,7 @@ #include #include #include +#include namespace clp_ffi_js { /** @@ -17,26 +18,26 @@ enum class LogLevel : std::uint8_t { WARN, ERROR, FATAL, - SIZE, // This isn't a valid log level. + LENGTH, // This isn't a valid log level. }; constexpr LogLevel cValidLogLevelsBeginIdx{LogLevel::TRACE}; -constexpr LogLevel cValidLogLevelsEndIdx{LogLevel::SIZE}; /** * Strings corresponding to `LogLevel`. * * NOTE: These must be kept in sync manually. */ -constexpr std::array cLogLevelNames{ - "NONE", // This isn't a valid log level. - "TRACE", - "DEBUG", - "INFO", - "WARN", - "ERROR", - "FATAL", -}; +constexpr std::array + cLogLevelNames{ + "NONE", // This isn't a valid log level. + "TRACE", + "DEBUG", + "INFO", + "WARN", + "ERROR", + "FATAL", + }; } // namespace clp_ffi_js #endif // CLP_FFI_JS_CONSTANTS_HPP diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp index 386c515d..3072dabc 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp @@ -100,16 +100,14 @@ auto StructuredIrStreamReader::deserialize_stream() -> size_t { constexpr size_t cDefaultNumReservedLogEvents{500'000}; m_deserialized_log_events->reserve(cDefaultNumReservedLogEvents); auto& reader{m_stream_reader_data_context->get_reader()}; - while (true) { - auto result{m_stream_reader_data_context->get_deserializer().deserialize_next_ir_unit(reader - )}; + auto& deserializer = m_stream_reader_data_context->get_deserializer(); + + while (false == deserializer.is_stream_completed()) { + auto result{deserializer.deserialize_next_ir_unit(reader)}; if (false == result.has_error()) { continue; } auto const error{result.error()}; - if (std::errc::operation_not_permitted == error) { - break; - } if (std::errc::result_out_of_range == error) { SPDLOG_ERROR("File contains an incomplete IR stream"); break; diff --git a/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp b/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp index 2a498871..1c838f17 100644 --- a/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp +++ b/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -28,18 +29,9 @@ namespace { * @param str * @return `LogLevel` enum corresponding to a string in `cLogLevelNames`. If input string * is not in `cLogLevelNames`, returns `LogLevel::NONE`. - * */ auto parse_log_level(std::string_view str) -> LogLevel; -/** - * Parses a string to determine the corresponding `LogLevel` enumeration value. - * - * @param str It should match one of the valid log level names in `cLogLevelNames`. - * @return The parsed value if the input is valid. - * If invalid, returns `LogLevel::NONE`. - */ - auto parse_log_level(std::string_view str) -> LogLevel { // Convert the string to uppercase. std::string log_level_name_upper_case{str}; @@ -64,6 +56,25 @@ auto parse_log_level(std::string_view str) -> LogLevel { } } // namespace +auto StructuredIrUnitHandler::handle_log_event(StructuredLogEvent&& log_event +) -> clp::ffi::ir_stream::IRErrorCode { + auto const& id_value_pairs{log_event.get_node_id_value_pairs()}; + auto const timestamp = get_timestamp(id_value_pairs); + auto const log_level = get_log_level(id_value_pairs); + + m_deserialized_log_events->emplace_back(std::move(log_event), log_level, timestamp); + + return clp::ffi::ir_stream::IRErrorCode::IRErrorCode_Success; +} + +auto StructuredIrUnitHandler::handle_utc_offset_change( + [[maybe_unused]] clp::UtcOffset utc_offset_old, + [[maybe_unused]] clp::UtcOffset utc_offset_new +) -> clp::ffi::ir_stream::IRErrorCode { + SPDLOG_WARN("UTC offset change packets aren't handled currently."); + return clp::ffi::ir_stream::IRErrorCode::IRErrorCode_Success; +} + auto StructuredIrUnitHandler::handle_schema_tree_node_insertion( clp::ffi::SchemaTree::NodeLocator schema_tree_node_locator ) -> clp::ffi::ir_stream::IRErrorCode { @@ -79,14 +90,7 @@ auto StructuredIrUnitHandler::handle_schema_tree_node_insertion( return clp::ffi::ir_stream::IRErrorCode::IRErrorCode_Success; } -auto StructuredIrUnitHandler::handle_log_event(StructuredLogEvent&& log_event -) -> clp::ffi::ir_stream::IRErrorCode { - auto const& id_value_pairs{log_event.get_node_id_value_pairs()}; - auto const timestamp = get_timestamp(id_value_pairs); - auto const log_level = get_log_level(id_value_pairs); - - m_deserialized_log_events->emplace_back(std::move(log_event), log_level, timestamp); - +auto StructuredIrUnitHandler::handle_end_of_stream() -> clp::ffi::ir_stream::IRErrorCode { return clp::ffi::ir_stream::IRErrorCode::IRErrorCode_Success; } @@ -112,7 +116,7 @@ auto StructuredIrUnitHandler::get_log_level( auto const& log_level_node_value = (log_level_pair.value().get_immutable_view()); if (log_level_node_value >= clp::enum_to_underlying_type(cValidLogLevelsBeginIdx) - && log_level_node_value < clp::enum_to_underlying_type(cValidLogLevelsEndIdx)) + && log_level_node_value < clp::enum_to_underlying_type(LogLevel::LENGTH)) { log_level = static_cast(log_level_node_value); } diff --git a/src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp b/src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp index d12b1900..fccf2569 100644 --- a/src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp +++ b/src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp @@ -13,7 +13,6 @@ #include #include #include -#include #include #include @@ -52,6 +51,7 @@ class StructuredIrUnitHandler { ) -> clp::ffi::ir_stream::IRErrorCode; /** + * Dummy implementation that does nothing but conforms to the interface. * @param utc_offset_old * @param utc_offset_new * @return IRErrorCode::IRErrorCode_Success @@ -59,11 +59,7 @@ class StructuredIrUnitHandler { [[nodiscard]] static auto handle_utc_offset_change( [[maybe_unused]] clp::UtcOffset utc_offset_old, [[maybe_unused]] clp::UtcOffset utc_offset_new - ) -> clp::ffi::ir_stream::IRErrorCode { - SPDLOG_WARN("UTC offset change packets aren't handled currently."); - - return clp::ffi::ir_stream::IRErrorCode::IRErrorCode_Success; - } + ) -> clp::ffi::ir_stream::IRErrorCode; /** * Saves the node's ID if it corresponds to events' authoritative log level or timestamp @@ -76,11 +72,10 @@ class StructuredIrUnitHandler { ) -> clp::ffi::ir_stream::IRErrorCode; /** + * Dummy implementation that does nothing but conforms to the interface. * @return IRErrorCode::IRErrorCode_Success */ - [[nodiscard]] static auto handle_end_of_stream() -> clp::ffi::ir_stream::IRErrorCode { - return clp::ffi::ir_stream::IRErrorCode::IRErrorCode_Success; - } + [[nodiscard]] static auto handle_end_of_stream() -> clp::ffi::ir_stream::IRErrorCode; private: // Methods From 2a202b8d35ed29edc75dd5e15950fff14ad6863f Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Thu, 5 Dec 2024 20:21:03 +0000 Subject: [PATCH 29/52] add missing code in structured reader --- src/clp_ffi_js/ir/StructuredIrStreamReader.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp index 3072dabc..818bcd27 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp @@ -149,7 +149,14 @@ auto StructuredIrStreamReader::decode_range(size_t begin_idx, size_t end_idx, bo auto const results{emscripten::val::array()}; - for (size_t log_event_idx = begin_idx; log_event_idx < end_idx; ++log_event_idx) { + for (size_t i = begin_idx; i < end_idx; ++i) { + size_t log_event_idx{0}; + if (use_filter) { + log_event_idx = m_filtered_log_event_map->at(i); + } else { + log_event_idx = i; + } + auto const& log_event_with_filter_data{m_deserialized_log_events->at(log_event_idx)}; auto const& structured_log_event = log_event_with_filter_data.get_log_event(); From 5332e9b64d1b71b298c10321fa6386cdc243a4df Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Fri, 6 Dec 2024 02:46:55 +0000 Subject: [PATCH 30/52] add template for decode --- .../ir/StructuredIrStreamReader.cpp | 80 ++---------- .../ir/StructuredIrStreamReader.hpp | 7 +- .../ir/UnstructuredIrStreamReader.cpp | 69 ++--------- .../ir/UnstructuredIrStreamReader.hpp | 3 +- src/clp_ffi_js/ir/utils.hpp | 114 +++++++++++++++++- 5 files changed, 136 insertions(+), 137 deletions(-) diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp index 818bcd27..8cb131e8 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp @@ -7,19 +7,15 @@ #include #include #include -#include #include #include #include -#include #include -#include #include #include #include -#include #include #include #include @@ -27,10 +23,6 @@ namespace clp_ffi_js::ir { namespace { -constexpr std::string_view cEmptyJsonStr{"{}"}; -constexpr std::string_view cLogLevelFilteringNotSupportedErrorMsg{ - "Log level filtering is not yet supported in this reader." -}; constexpr std::string_view cReaderOptionsLogLevelKey{"logLevelKey"}; constexpr std::string_view cReaderOptionsTimestampKey{"timestampKey"}; } // namespace @@ -40,9 +32,7 @@ auto StructuredIrStreamReader::create( clp::Array data_array, ReaderOptions const& reader_options ) -> StructuredIrStreamReader { - auto deserialized_log_events{ - std::make_shared>>() - }; + auto deserialized_log_events{std::make_shared()}; auto result{StructuredIrDeserializer::create( *zstd_decompressor, StructuredIrUnitHandler{ @@ -85,7 +75,7 @@ auto StructuredIrStreamReader::get_filtered_log_event_map() const -> FilteredLog } void StructuredIrStreamReader::filter_log_events(LogLevelFilterTsType const& log_level_filter) { - filter_deserialized_events( + generic_filter_log_events( m_filtered_log_event_map, log_level_filter, *m_deserialized_log_events @@ -129,67 +119,19 @@ auto StructuredIrStreamReader::deserialize_stream() -> size_t { auto StructuredIrStreamReader::decode_range(size_t begin_idx, size_t end_idx, bool use_filter) const -> DecodedResultsTsType { - if (use_filter && false == m_filtered_log_event_map.has_value()) { - return DecodedResultsTsType{emscripten::val::null()}; - } - - size_t length{0}; - if (use_filter) { - length = m_filtered_log_event_map->size(); - } else { - length = m_deserialized_log_events->size(); - } - if (length < end_idx || begin_idx > end_idx) { - return DecodedResultsTsType{emscripten::val::null()}; - } - - if (m_deserialized_log_events->size() < end_idx || begin_idx > end_idx) { - return DecodedResultsTsType{emscripten::val::null()}; - } - - auto const results{emscripten::val::array()}; - - for (size_t i = begin_idx; i < end_idx; ++i) { - size_t log_event_idx{0}; - if (use_filter) { - log_event_idx = m_filtered_log_event_map->at(i); - } else { - log_event_idx = i; - } - - auto const& log_event_with_filter_data{m_deserialized_log_events->at(log_event_idx)}; - auto const& structured_log_event = log_event_with_filter_data.get_log_event(); - - auto const json_result{structured_log_event.serialize_to_json()}; - std::string json_str{cEmptyJsonStr}; - if (false == json_result.has_value()) { - auto error_code{json_result.error()}; - SPDLOG_ERROR( - "Failed to deserialize log event to JSON: {}:{}", - error_code.category().name(), - error_code.message() - ); - } else { - json_str = json_result.value().dump(); - } - - EM_ASM( - { Emval.toValue($0).push([UTF8ToString($1), $2, $3, $4]); }, - results.as_handle(), - json_str.c_str(), - log_event_with_filter_data.get_timestamp(), - log_event_with_filter_data.get_log_level(), - log_event_idx + 1 - ); - } - - return DecodedResultsTsType(results); + return generic_decode_range( + begin_idx, + end_idx, + m_filtered_log_event_map, + *m_deserialized_log_events, + use_filter, + clp::TimestampPattern() + ); } StructuredIrStreamReader::StructuredIrStreamReader( StreamReaderDataContext&& stream_reader_data_context, - std::shared_ptr>> - deserialized_log_events + std::shared_ptr deserialized_log_events ) : m_deserialized_log_events{std::move(deserialized_log_events)}, m_stream_reader_data_context{ diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp index 6792aa62..fc0dcbe3 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp @@ -19,6 +19,7 @@ namespace clp_ffi_js::ir { using schema_tree_node_id_t = std::optional; using StructuredIrDeserializer = clp::ffi::ir_stream::Deserializer; +using StructuredLogEvents = std::vector>; /** * Class to deserialize and decode Zstd-compressed CLP structured IR streams, as well as format @@ -78,13 +79,11 @@ class StructuredIrStreamReader : public StreamReader { // Constructor explicit StructuredIrStreamReader( StreamReaderDataContext&& stream_reader_data_context, - std::shared_ptr>> - deserialized_log_events + std::shared_ptr deserialized_log_events ); // Variables - std::shared_ptr>> - m_deserialized_log_events; + std::shared_ptr m_deserialized_log_events; std::unique_ptr> m_stream_reader_data_context; FilteredLogEventsMap m_filtered_log_event_map; }; diff --git a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp index f46cdac9..cc826742 100644 --- a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp @@ -5,12 +5,10 @@ #include #include #include -#include #include #include #include #include -#include #include #include @@ -18,13 +16,11 @@ #include #include #include -#include #include #include #include #include -#include #include #include #include @@ -73,7 +69,7 @@ auto UnstructuredIrStreamReader::get_filtered_log_event_map() const -> FilteredL } void UnstructuredIrStreamReader::filter_log_events(LogLevelFilterTsType const& log_level_filter) { - filter_deserialized_events(m_filtered_log_event_map, log_level_filter, m_encoded_log_events); + generic_filter_log_events(m_filtered_log_event_map, log_level_filter, m_encoded_log_events); } auto UnstructuredIrStreamReader::deserialize_stream() -> size_t { @@ -136,61 +132,14 @@ auto UnstructuredIrStreamReader::deserialize_stream() -> size_t { auto UnstructuredIrStreamReader::decode_range(size_t begin_idx, size_t end_idx, bool use_filter) const -> DecodedResultsTsType { - if (use_filter && false == m_filtered_log_event_map.has_value()) { - return DecodedResultsTsType{emscripten::val::null()}; - } - - size_t length{0}; - if (use_filter) { - length = m_filtered_log_event_map->size(); - } else { - length = m_encoded_log_events.size(); - } - if (length < end_idx || begin_idx > end_idx) { - return DecodedResultsTsType{emscripten::val::null()}; - } - - std::string message; - constexpr size_t cDefaultReservedMessageLength{512}; - message.reserve(cDefaultReservedMessageLength); - auto const results{emscripten::val::array()}; - - for (size_t i = begin_idx; i < end_idx; ++i) { - size_t log_event_idx{0}; - if (use_filter) { - log_event_idx = m_filtered_log_event_map->at(i); - } else { - log_event_idx = i; - } - auto const& log_event_with_filter_data{m_encoded_log_events[log_event_idx]}; - auto const& unstructured_log_event = log_event_with_filter_data.get_log_event(); - auto const& log_level = log_event_with_filter_data.get_log_level(); - auto const& timestamp = log_event_with_filter_data.get_timestamp(); - - auto const parsed{unstructured_log_event.get_message().decode_and_unparse()}; - if (false == parsed.has_value()) { - throw ClpFfiJsException{ - clp::ErrorCode::ErrorCode_Failure, - __FILENAME__, - __LINE__, - "Failed to decode message" - }; - } - message = parsed.value(); - - m_ts_pattern.insert_formatted_timestamp(timestamp, message); - - EM_ASM( - { Emval.toValue($0).push([UTF8ToString($1), $2, $3, $4]); }, - results.as_handle(), - message.c_str(), - timestamp, - log_level, - log_event_idx + 1 - ); - } - - return DecodedResultsTsType(results); + return generic_decode_range( + begin_idx, + end_idx, + m_filtered_log_event_map, + m_encoded_log_events, + use_filter, + m_ts_pattern + ); } UnstructuredIrStreamReader::UnstructuredIrStreamReader( diff --git a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp index d2ba4d20..e6fdf1ba 100644 --- a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp +++ b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp @@ -18,6 +18,7 @@ namespace clp_ffi_js::ir { using clp::ir::four_byte_encoded_variable_t; using UnstructuredIrDeserializer = clp::ir::LogEventDeserializer; +using UnstructuredLogEvents = std::vector>; /** * Class to deserialize and decode Zstd-compressed CLP unstructured IR streams, as well as format @@ -78,7 +79,7 @@ class UnstructuredIrStreamReader : public StreamReader { ); // Variables - std::vector> m_encoded_log_events; + UnstructuredLogEvents m_encoded_log_events; std::unique_ptr> m_stream_reader_data_context; FilteredLogEventsMap m_filtered_log_event_map; diff --git a/src/clp_ffi_js/ir/utils.hpp b/src/clp_ffi_js/ir/utils.hpp index a23040c2..157709a3 100644 --- a/src/clp_ffi_js/ir/utils.hpp +++ b/src/clp_ffi_js/ir/utils.hpp @@ -3,27 +3,38 @@ #include #include +#include #include #include +#include +#include #include +#include #include +#include +#include #include #include +#include +#include namespace clp_ffi_js::ir { +constexpr std::string_view cEmptyJsonStr{"{}"}; /** - * Generates a filtered collection from all log events. + * A templated function that implements `StreamReader::filter_log_events` for both + * `UnstructuredIrStreamReader` and `StructuredIrStreamReader`. Additional + * arguments are private members of the derived `StreamReader` classes. * - * @param log_level_filter Array of selected log levels. + * @param log_level_filter * @param log_events * @param[out] filtered_log_event_map A reference to `FilteredLogEventsMap` that stores filtered * result. */ template -auto filter_deserialized_events( +auto generic_filter_log_events( FilteredLogEventsMap& filtered_log_event_map, LogLevelFilterTsType const& log_level_filter, LogEvents const& log_events @@ -49,6 +60,103 @@ auto filter_deserialized_events( } } } + +/** + * A templated function that implements `StreamReader::decode_range` for both + * `UnstructuredIrStreamReader` and `StructuredIrStreamReader`. Additional + * arguments are private members of the derived `StreamReader` classes. + * + * @param begin_idx + * @param end_idx + * @param filtered_log_event_map + * @param log_events + * @param use_filter + * @param ts_pattern Pattern for formatting unstructured log event timestamps as date strings. + * Structured log event timestamps are not formatted in clp-ffi-js, so the pattern has no effect. + * @return + */ +template +auto generic_decode_range( + size_t begin_idx, + size_t end_idx, + FilteredLogEventsMap const& filtered_log_event_map, + LogEvents const& log_events, + bool use_filter, + clp::TimestampPattern const& ts_pattern +) -> DecodedResultsTsType { + if (use_filter && false == filtered_log_event_map.has_value()) { + return DecodedResultsTsType{emscripten::val::null()}; + } + + size_t length{0}; + if (use_filter) { + length = filtered_log_event_map->size(); + } else { + length = log_events.size(); + } + if (length < end_idx || begin_idx > end_idx) { + return DecodedResultsTsType{emscripten::val::null()}; + } + + auto const results{emscripten::val::array()}; + + for (size_t i = begin_idx; i < end_idx; ++i) { + size_t log_event_idx{0}; + if (use_filter) { + log_event_idx = filtered_log_event_map->at(i); + } else { + log_event_idx = i; + } + + auto const& log_event_with_filter_data{log_events.at(log_event_idx)}; + auto const& log_event = log_event_with_filter_data.get_log_event(); + auto const& timestamp = log_event_with_filter_data.get_timestamp(); + auto const& log_level = log_event_with_filter_data.get_log_level(); + + std::string generic_event_string; + + if constexpr (std::is_same_v) { + constexpr size_t cDefaultReservedLength{512}; + generic_event_string.reserve(cDefaultReservedLength); + + auto const parsed{log_event.get_message().decode_and_unparse()}; + if (false == parsed.has_value()) { + throw ClpFfiJsException{ + clp::ErrorCode::ErrorCode_Failure, + __FILENAME__, + __LINE__, + "Failed to decode message" + }; + } + generic_event_string = parsed.value(); + ts_pattern.insert_formatted_timestamp(timestamp, generic_event_string); + } else if constexpr (std::is_same_v) { + auto const json_result{log_event.serialize_to_json()}; + if (false == json_result.has_value()) { + auto error_code{json_result.error()}; + SPDLOG_ERROR( + "Failed to deserialize log event to JSON: {}:{}", + error_code.category().name(), + error_code.message() + ); + generic_event_string = std::string(cEmptyJsonStr); + } else { + generic_event_string = json_result.value().dump(); + } + } + + EM_ASM( + { Emval.toValue($0).push([UTF8ToString($1), $2, $3, $4]); }, + results.as_handle(), + generic_event_string.c_str(), + timestamp, + log_level, + log_event_idx + 1 + ); + } + + return DecodedResultsTsType(results); +} } // namespace clp_ffi_js::ir #endif // CLP_FFI_JS_IR_UTILS_HPP From da6350f87a7b0bb79ea2cc36158804ec8442de3a Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Fri, 6 Dec 2024 02:49:56 +0000 Subject: [PATCH 31/52] lint --- src/clp_ffi_js/ir/utils.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/clp_ffi_js/ir/utils.hpp b/src/clp_ffi_js/ir/utils.hpp index 157709a3..e2403c6b 100644 --- a/src/clp_ffi_js/ir/utils.hpp +++ b/src/clp_ffi_js/ir/utils.hpp @@ -3,17 +3,17 @@ #include #include +#include #include #include #include -#include +#include #include #include #include #include -#include #include #include #include From ee2eb429e277b1d6fc759924d52a81ec7b75e5e7 Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Fri, 6 Dec 2024 16:31:38 +0000 Subject: [PATCH 32/52] fix lint --- src/clp_ffi_js/ir/utils.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/clp_ffi_js/ir/utils.hpp b/src/clp_ffi_js/ir/utils.hpp index e2403c6b..c76c5177 100644 --- a/src/clp_ffi_js/ir/utils.hpp +++ b/src/clp_ffi_js/ir/utils.hpp @@ -10,6 +10,7 @@ #include #include +#include #include #include #include From 90b977fbe3028dcfb9fc3a696efeacb8843d49e6 Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Fri, 6 Dec 2024 17:20:56 +0000 Subject: [PATCH 33/52] reword some comments --- src/clp_ffi_js/constants.hpp | 1 - src/clp_ffi_js/ir/StructuredIrStreamReader.cpp | 2 ++ src/clp_ffi_js/ir/utils.hpp | 14 ++++++++------ 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/clp_ffi_js/constants.hpp b/src/clp_ffi_js/constants.hpp index dd9ba2ad..c49829c5 100644 --- a/src/clp_ffi_js/constants.hpp +++ b/src/clp_ffi_js/constants.hpp @@ -20,7 +20,6 @@ enum class LogLevel : std::uint8_t { FATAL, LENGTH, // This isn't a valid log level. }; - constexpr LogLevel cValidLogLevelsBeginIdx{LogLevel::TRACE}; /** diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp index 8cb131e8..ea1fb3e5 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp @@ -125,6 +125,8 @@ auto StructuredIrStreamReader::decode_range(size_t begin_idx, size_t end_idx, bo m_filtered_log_event_map, *m_deserialized_log_events, use_filter, + // `ts_pattern` argument is unused in `generic_decode_range`. + // Default timestamp pattern is used to conform to `generic_decode_range` interface. clp::TimestampPattern() ); } diff --git a/src/clp_ffi_js/ir/utils.hpp b/src/clp_ffi_js/ir/utils.hpp index c76c5177..5945ef0b 100644 --- a/src/clp_ffi_js/ir/utils.hpp +++ b/src/clp_ffi_js/ir/utils.hpp @@ -26,8 +26,9 @@ constexpr std::string_view cEmptyJsonStr{"{}"}; /** * A templated function that implements `StreamReader::filter_log_events` for both - * `UnstructuredIrStreamReader` and `StructuredIrStreamReader`. Additional - * arguments are private members of the derived `StreamReader` classes. + * `UnstructuredIrStreamReader` and `StructuredIrStreamReader`. The additional arguments, + * which are not part of `filter_log_events`, are private members of the derived + * `StreamReader` classes. * * @param log_level_filter * @param log_events @@ -64,16 +65,17 @@ auto generic_filter_log_events( /** * A templated function that implements `StreamReader::decode_range` for both - * `UnstructuredIrStreamReader` and `StructuredIrStreamReader`. Additional - * arguments are private members of the derived `StreamReader` classes. + * `UnstructuredIrStreamReader` and `StructuredIrStreamReader`. The additional + * arguments, which are not part of `decode_range`, are private members of + * the derived `StreamReader` classes. * * @param begin_idx * @param end_idx * @param filtered_log_event_map * @param log_events * @param use_filter - * @param ts_pattern Pattern for formatting unstructured log event timestamps as date strings. - * Structured log event timestamps are not formatted in clp-ffi-js, so the pattern has no effect. + * @param ts_pattern A pattern for formatting unstructured log event timestamps as date strings. + * The pattern is unused for structured log events, as they are not formatted in `clp-ffi-js`. * @return */ template From 8cc73d0588d6c6c4c47a339aff8bb8fc58b1a6b6 Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Mon, 9 Dec 2024 21:51:38 +0000 Subject: [PATCH 34/52] junhao review --- src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp | 15 ++++++--------- src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp | 5 +++-- src/clp_ffi_js/ir/utils.hpp | 1 + 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp b/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp index 1c838f17..7430c389 100644 --- a/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp +++ b/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -102,13 +103,10 @@ auto StructuredIrUnitHandler::get_log_level( if (false == m_log_level_node_id.has_value()) { return log_level; } - auto const& log_level_pair{id_value_pairs.at(m_log_level_node_id.value())}; - if (false == log_level_pair.has_value()) { return log_level; } - if (log_level_pair->is()) { auto const& log_level_node_value = log_level_pair.value().get_immutable_view(); log_level = parse_log_level(log_level_node_value); @@ -130,21 +128,20 @@ auto StructuredIrUnitHandler::get_log_level( auto StructuredIrUnitHandler::get_timestamp( StructuredLogEvent::NodeIdValuePairs const& id_value_pairs -) const -> clp::ffi::value_int_t { - clp::ffi::value_int_t timestamp{0}; +) const -> clp::ir::epoch_time_ms_t { + clp::ir::epoch_time_ms_t timestamp{0}; if (false == m_timestamp_node_id.has_value()) { return timestamp; } - auto const& timestamp_pair{id_value_pairs.at(m_timestamp_node_id.value())}; - if (false == timestamp_pair.has_value()) { return timestamp; } - if (timestamp_pair->is()) { - timestamp = timestamp_pair.value().get_immutable_view(); + timestamp = static_cast( + timestamp_pair.value().get_immutable_view() + ); } else { // TODO: Add support for parsing timestamp values of string type. auto log_event_idx = m_deserialized_log_events->size(); diff --git a/src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp b/src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp index fccf2569..caaaf8f7 100644 --- a/src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp +++ b/src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -43,7 +44,7 @@ class StructuredIrUnitHandler { // Methods implementing `clp::ffi::ir_stream::IrUnitHandlerInterface`. /** - * Buffers the log event. + * Buffers the log event with filter data extracted. * @param log_event * @return IRErrorCode::IRErrorCode_Success */ @@ -91,7 +92,7 @@ class StructuredIrUnitHandler { * @return Timestamp from node with id `m_timestamp_node_id`. */ [[nodiscard]] auto get_timestamp(StructuredLogEvent::NodeIdValuePairs const& id_value_pairs - ) const -> clp::ffi::value_int_t; + ) const -> clp::ir::epoch_time_ms_t; // Variables std::string m_log_level_key; diff --git a/src/clp_ffi_js/ir/utils.hpp b/src/clp_ffi_js/ir/utils.hpp index 5945ef0b..0048996b 100644 --- a/src/clp_ffi_js/ir/utils.hpp +++ b/src/clp_ffi_js/ir/utils.hpp @@ -102,6 +102,7 @@ auto generic_decode_range( } auto const results{emscripten::val::array()}; + std::string generic_event_string; for (size_t i = begin_idx; i < end_idx; ++i) { size_t log_event_idx{0}; From 4d905e8311d85e0811aba639df9702dd370a34d9 Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Mon, 9 Dec 2024 21:53:04 +0000 Subject: [PATCH 35/52] remove double declaration --- src/clp_ffi_js/ir/utils.hpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/clp_ffi_js/ir/utils.hpp b/src/clp_ffi_js/ir/utils.hpp index 0048996b..58ec3cc3 100644 --- a/src/clp_ffi_js/ir/utils.hpp +++ b/src/clp_ffi_js/ir/utils.hpp @@ -117,8 +117,6 @@ auto generic_decode_range( auto const& timestamp = log_event_with_filter_data.get_timestamp(); auto const& log_level = log_event_with_filter_data.get_log_level(); - std::string generic_event_string; - if constexpr (std::is_same_v) { constexpr size_t cDefaultReservedLength{512}; generic_event_string.reserve(cDefaultReservedLength); From 5854e36f4bdb2203de5dc02e84dc2461fc86e092 Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Tue, 10 Dec 2024 02:30:03 +0000 Subject: [PATCH 36/52] fix lint --- src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp b/src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp index caaaf8f7..6d4bb1a2 100644 --- a/src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp +++ b/src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp @@ -11,7 +11,6 @@ #include #include #include -#include #include #include From f3f6e76d0ee8613a33d6370e05cc267573e88fe1 Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Tue, 10 Dec 2024 15:41:33 +0000 Subject: [PATCH 37/52] move reserve up --- src/clp_ffi_js/ir/utils.hpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/clp_ffi_js/ir/utils.hpp b/src/clp_ffi_js/ir/utils.hpp index 58ec3cc3..44af72c4 100644 --- a/src/clp_ffi_js/ir/utils.hpp +++ b/src/clp_ffi_js/ir/utils.hpp @@ -103,6 +103,8 @@ auto generic_decode_range( auto const results{emscripten::val::array()}; std::string generic_event_string; + constexpr size_t cDefaultReservedLength{512}; + generic_event_string.reserve(cDefaultReservedLength); for (size_t i = begin_idx; i < end_idx; ++i) { size_t log_event_idx{0}; @@ -118,9 +120,6 @@ auto generic_decode_range( auto const& log_level = log_event_with_filter_data.get_log_level(); if constexpr (std::is_same_v) { - constexpr size_t cDefaultReservedLength{512}; - generic_event_string.reserve(cDefaultReservedLength); - auto const parsed{log_event.get_message().decode_and_unparse()}; if (false == parsed.has_value()) { throw ClpFfiJsException{ From d4467ead77eab13ce33232f1e0c1565754a1e11b Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Thu, 12 Dec 2024 10:49:49 -0500 Subject: [PATCH 38/52] Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- src/clp_ffi_js/ir/LogEventWithFilterData.hpp | 1 - src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp | 22 +++++++++++-------- src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp | 11 +++++++--- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/src/clp_ffi_js/ir/LogEventWithFilterData.hpp b/src/clp_ffi_js/ir/LogEventWithFilterData.hpp index 542c0e30..96710a03 100644 --- a/src/clp_ffi_js/ir/LogEventWithFilterData.hpp +++ b/src/clp_ffi_js/ir/LogEventWithFilterData.hpp @@ -24,7 +24,6 @@ using StructuredLogEvent = clp::ffi::KeyValuePairLogEvent; */ template requires std::same_as || std::same_as - class LogEventWithFilterData { public: // Constructor diff --git a/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp b/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp index 7430c389..68f992b5 100644 --- a/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp +++ b/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp @@ -23,13 +23,12 @@ #include namespace clp_ffi_js::ir { - namespace { /** * Parses a string to determine the corresponding `LogLevel` enum value. * @param str - * @return `LogLevel` enum corresponding to a string in `cLogLevelNames`. If input string - * is not in `cLogLevelNames`, returns `LogLevel::NONE`. + * @return `LogLevel` enum corresponding to `str` if `str` matches a string in `cLogLevelNames`. + * @return `LogLevel::NONE` otherwise. */ auto parse_log_level(std::string_view str) -> LogLevel; @@ -48,7 +47,6 @@ auto parse_log_level(std::string_view str) -> LogLevel { cLogLevelNames.end(), log_level_name_upper_case ); - if (it == cLogLevelNames.end()) { return LogLevel::NONE; } @@ -82,9 +80,9 @@ auto StructuredIrUnitHandler::handle_schema_tree_node_insertion( ++m_current_node_id; auto const& key_name{schema_tree_node_locator.get_key_name()}; - if (m_log_level_key == key_name) { + if (key_name == m_log_level_key) { m_log_level_node_id.emplace(m_current_node_id); - } else if (m_timestamp_key == key_name) { + } else if (key_name == m_timestamp_key) { m_timestamp_node_id.emplace(m_current_node_id); } @@ -120,7 +118,10 @@ auto StructuredIrUnitHandler::get_log_level( } } else { auto log_event_idx = m_deserialized_log_events->size(); - SPDLOG_ERROR("Log level type is not int or string for log event index {}", log_event_idx); + SPDLOG_ERROR( + "Authoritative log level's value is not an int or string for log event index {}", + log_event_idx + ); } return log_level; @@ -143,9 +144,12 @@ auto StructuredIrUnitHandler::get_timestamp( timestamp_pair.value().get_immutable_view() ); } else { - // TODO: Add support for parsing timestamp values of string type. + // TODO: Add support for parsing string-type timestamp values. auto log_event_idx = m_deserialized_log_events->size(); - SPDLOG_ERROR("Timestamp type is not int for log event index {}", log_event_idx); + SPDLOG_ERROR( + "Authoritative timestamp's value is not an int for log event index {}", + log_event_idx + ); } return timestamp; diff --git a/src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp b/src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp index 6d4bb1a2..bd8b6400 100644 --- a/src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp +++ b/src/clp_ffi_js/ir/StructuredIrUnitHandler.hpp @@ -22,10 +22,11 @@ using schema_tree_node_id_t = std::optional; /** * Class that implements the `clp::ffi::ir_stream::IrUnitHandlerInterface` to buffer log events and - * determine the schema-tree node ID of the log level and timestamp kv-pair. + * determine the schema-tree node IDs of the log level and timestamp kv-pairs. */ class StructuredIrUnitHandler { public: + // Constructors /** * @param deserialized_log_events The vector in which to store deserialized log events. * @param log_level_key Key name of schema-tree node that contains the authoritative log level. @@ -81,14 +82,18 @@ class StructuredIrUnitHandler { // Methods /** * @param id_value_pairs - * @return `LogLevel` from node with id `m_log_level_node_id`. + * @return `LogLevel::NONE` if `m_log_level_node_id` is unset, the node has no value, or the + * node's value is not an integer or string. + * @return `LogLevel` from node with id `m_log_level_node_id` otherwise. */ [[nodiscard]] auto get_log_level(StructuredLogEvent::NodeIdValuePairs const& id_value_pairs ) const -> LogLevel; /** * @param id_value_pairs - * @return Timestamp from node with id `m_timestamp_node_id`. + * @return 0 if `m_timestamp_node_id` is unset, the node has no value, or the node's value is + * not an integer. + * @return Timestamp from node with ID `m_timestamp_node_id` otherwise. */ [[nodiscard]] auto get_timestamp(StructuredLogEvent::NodeIdValuePairs const& id_value_pairs ) const -> clp::ir::epoch_time_ms_t; From 6660763574b15ac92939f97ed20d81af11823f93 Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Thu, 12 Dec 2024 16:17:15 +0000 Subject: [PATCH 39/52] add reader option param --- src/clp_ffi_js/ir/StreamReader.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/clp_ffi_js/ir/StreamReader.hpp b/src/clp_ffi_js/ir/StreamReader.hpp index ba87964a..6cf4bd45 100644 --- a/src/clp_ffi_js/ir/StreamReader.hpp +++ b/src/clp_ffi_js/ir/StreamReader.hpp @@ -43,6 +43,7 @@ class StreamReader { * Creates a `StreamReader` to read from the given array. * * @param data_array An array containing a Zstandard-compressed IR stream. + * @param reader_options * @return The created instance. * @throw ClpFfiJsException if any error occurs. */ From 5b684ab88b1e3af6d377e138ce3e74de67f50179 Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Thu, 12 Dec 2024 21:58:06 +0000 Subject: [PATCH 40/52] refactor using lambda --- src/clp_ffi_js/ir/StreamReader.hpp | 112 ++++++++++++ .../ir/StructuredIrStreamReader.cpp | 30 +++- .../ir/UnstructuredIrStreamReader.cpp | 30 +++- src/clp_ffi_js/ir/utils.hpp | 163 ------------------ 4 files changed, 160 insertions(+), 175 deletions(-) delete mode 100644 src/clp_ffi_js/ir/utils.hpp diff --git a/src/clp_ffi_js/ir/StreamReader.hpp b/src/clp_ffi_js/ir/StreamReader.hpp index 6cf4bd45..533b3d7e 100644 --- a/src/clp_ffi_js/ir/StreamReader.hpp +++ b/src/clp_ffi_js/ir/StreamReader.hpp @@ -1,15 +1,22 @@ #ifndef CLP_FFI_JS_IR_STREAMREADER_HPP #define CLP_FFI_JS_IR_STREAMREADER_HPP +#include #include #include #include #include +#include +#include #include #include +#include +#include #include +#include + namespace clp_ffi_js::ir { // JS types used as inputs EMSCRIPTEN_DECLARE_VAL_TYPE(DataArrayTsType); @@ -109,6 +116,111 @@ class StreamReader { [[nodiscard]] virtual auto decode_range(size_t begin_idx, size_t end_idx, bool use_filter) const -> DecodedResultsTsType = 0; + /** + * A templated function that implements `filter_log_events` for both + * `UnstructuredIrStreamReader` and `StructuredIrStreamReader`. + * + * @param log_level_filter + * @param log_events Derived class's log events. + * @param[out] filtered_log_event_map A reference to derived class's `FilteredLogEventsMap` + * that stores filtered result. + */ + template + static auto generic_filter_log_events( + FilteredLogEventsMap& filtered_log_event_map, + LogLevelFilterTsType const& log_level_filter, + LogEvents const& log_events + ) -> void { + if (log_level_filter.isNull()) { + filtered_log_event_map.reset(); + return; + } + + filtered_log_event_map.emplace(); + auto filter_levels + = emscripten::vecFromJSArray>(log_level_filter); + + for (size_t log_event_idx = 0; log_event_idx < log_events.size(); ++log_event_idx) { + auto const& log_event = log_events[log_event_idx]; + if (std::ranges::find( + filter_levels, + clp::enum_to_underlying_type(log_event.get_log_level()) + ) + != filter_levels.end()) + { + filtered_log_event_map->emplace_back(log_event_idx); + } + } + } + + /** + * A templated function that implements `decode_range` for both + * `UnstructuredIrStreamReader` and `StructuredIrStreamReader`. + * + * @param begin_idx + * @param end_idx + * @param filtered_log_event_map Derived class's filtered log event map. + * @param log_events Derived class's log events. + * @param use_filter + * @param log_event_to_string Lambda function which retrieves a string from a single log event. + * The derived class must implement lambda since `Unstructured` and `Structured` + * log events have different interfaces. + * @return + */ + template + static auto generic_decode_range( + size_t begin_idx, + size_t end_idx, + FilteredLogEventsMap const& filtered_log_event_map, + LogEvents const& log_events, + ToStringFunc log_event_to_string, + bool use_filter + ) -> DecodedResultsTsType { + if (use_filter && false == filtered_log_event_map.has_value()) { + return DecodedResultsTsType{emscripten::val::null()}; + } + + size_t length{0}; + if (use_filter) { + length = filtered_log_event_map->size(); + } else { + length = log_events.size(); + } + if (length < end_idx || begin_idx > end_idx) { + return DecodedResultsTsType{emscripten::val::null()}; + } + + auto const results{emscripten::val::array()}; + std::string generic_event_string; + + for (size_t i = begin_idx; i < end_idx; ++i) { + size_t log_event_idx{0}; + if (use_filter) { + log_event_idx = filtered_log_event_map->at(i); + } else { + log_event_idx = i; + } + + auto const& log_event_with_filter_data{log_events.at(log_event_idx)}; + auto const& log_event = log_event_with_filter_data.get_log_event(); + auto const& timestamp = log_event_with_filter_data.get_timestamp(); + auto const& log_level = log_event_with_filter_data.get_log_level(); + + generic_event_string = log_event_to_string(log_event); + + EM_ASM( + { Emval.toValue($0).push([UTF8ToString($1), $2, $3, $4]); }, + results.as_handle(), + generic_event_string.c_str(), + timestamp, + log_level, + log_event_idx + 1 + ); + } + + return DecodedResultsTsType(results); + } + protected: explicit StreamReader() = default; }; diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp index ea1fb3e5..ae142e29 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp @@ -16,13 +16,14 @@ #include #include +#include #include #include #include -#include namespace clp_ffi_js::ir { namespace { +constexpr std::string_view cEmptyJsonStr{"{}"}; constexpr std::string_view cReaderOptionsLogLevelKey{"logLevelKey"}; constexpr std::string_view cReaderOptionsTimestampKey{"timestampKey"}; } // namespace @@ -75,7 +76,7 @@ auto StructuredIrStreamReader::get_filtered_log_event_map() const -> FilteredLog } void StructuredIrStreamReader::filter_log_events(LogLevelFilterTsType const& log_level_filter) { - generic_filter_log_events( + StreamReader::generic_filter_log_events( m_filtered_log_event_map, log_level_filter, *m_deserialized_log_events @@ -119,15 +120,30 @@ auto StructuredIrStreamReader::deserialize_stream() -> size_t { auto StructuredIrStreamReader::decode_range(size_t begin_idx, size_t end_idx, bool use_filter) const -> DecodedResultsTsType { - return generic_decode_range( + auto log_event_to_string = [](StructuredLogEvent const& log_event) -> std::string { + std::string json_str; + auto const json_result{log_event.serialize_to_json()}; + if (false == json_result.has_value()) { + auto error_code{json_result.error()}; + SPDLOG_ERROR( + "Failed to deserialize log event to JSON: {}:{}", + error_code.category().name(), + error_code.message() + ); + json_str = std::string(cEmptyJsonStr); + } else { + json_str = json_result.value().dump(); + } + return json_str; + }; + + return StreamReader::generic_decode_range( begin_idx, end_idx, m_filtered_log_event_map, *m_deserialized_log_events, - use_filter, - // `ts_pattern` argument is unused in `generic_decode_range`. - // Default timestamp pattern is used to conform to `generic_decode_range` interface. - clp::TimestampPattern() + log_event_to_string, + use_filter ); } diff --git a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp index cc826742..a2e93059 100644 --- a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp @@ -21,9 +21,9 @@ #include #include +#include #include #include -#include namespace clp_ffi_js::ir { @@ -69,7 +69,11 @@ auto UnstructuredIrStreamReader::get_filtered_log_event_map() const -> FilteredL } void UnstructuredIrStreamReader::filter_log_events(LogLevelFilterTsType const& log_level_filter) { - generic_filter_log_events(m_filtered_log_event_map, log_level_filter, m_encoded_log_events); + StreamReader::generic_filter_log_events( + m_filtered_log_event_map, + log_level_filter, + m_encoded_log_events + ); } auto UnstructuredIrStreamReader::deserialize_stream() -> size_t { @@ -132,13 +136,29 @@ auto UnstructuredIrStreamReader::deserialize_stream() -> size_t { auto UnstructuredIrStreamReader::decode_range(size_t begin_idx, size_t end_idx, bool use_filter) const -> DecodedResultsTsType { - return generic_decode_range( + auto log_event_to_string = [this](UnstructuredLogEvent const& log_event) -> std::string { + std::string message; + auto const parsed{log_event.get_message().decode_and_unparse()}; + if (false == parsed.has_value()) { + throw ClpFfiJsException{ + clp::ErrorCode::ErrorCode_Failure, + __FILENAME__, + __LINE__, + "Failed to decode message" + }; + } + message = parsed.value(); + m_ts_pattern.insert_formatted_timestamp(log_event.get_timestamp(), message); + return message; + }; + + return StreamReader::generic_decode_range( begin_idx, end_idx, m_filtered_log_event_map, m_encoded_log_events, - use_filter, - m_ts_pattern + log_event_to_string, + use_filter ); } diff --git a/src/clp_ffi_js/ir/utils.hpp b/src/clp_ffi_js/ir/utils.hpp deleted file mode 100644 index 44af72c4..00000000 --- a/src/clp_ffi_js/ir/utils.hpp +++ /dev/null @@ -1,163 +0,0 @@ -#ifndef CLP_FFI_JS_IR_UTILS_HPP -#define CLP_FFI_JS_IR_UTILS_HPP - -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -namespace clp_ffi_js::ir { -constexpr std::string_view cEmptyJsonStr{"{}"}; - -/** - * A templated function that implements `StreamReader::filter_log_events` for both - * `UnstructuredIrStreamReader` and `StructuredIrStreamReader`. The additional arguments, - * which are not part of `filter_log_events`, are private members of the derived - * `StreamReader` classes. - * - * @param log_level_filter - * @param log_events - * @param[out] filtered_log_event_map A reference to `FilteredLogEventsMap` that stores filtered - * result. - */ -template -auto generic_filter_log_events( - FilteredLogEventsMap& filtered_log_event_map, - LogLevelFilterTsType const& log_level_filter, - LogEvents const& log_events -) -> void { - if (log_level_filter.isNull()) { - filtered_log_event_map.reset(); - return; - } - - filtered_log_event_map.emplace(); - auto filter_levels - = emscripten::vecFromJSArray>(log_level_filter); - - for (size_t log_event_idx = 0; log_event_idx < log_events.size(); ++log_event_idx) { - auto const& log_event = log_events[log_event_idx]; - if (std::ranges::find( - filter_levels, - clp::enum_to_underlying_type(log_event.get_log_level()) - ) - != filter_levels.end()) - { - filtered_log_event_map->emplace_back(log_event_idx); - } - } -} - -/** - * A templated function that implements `StreamReader::decode_range` for both - * `UnstructuredIrStreamReader` and `StructuredIrStreamReader`. The additional - * arguments, which are not part of `decode_range`, are private members of - * the derived `StreamReader` classes. - * - * @param begin_idx - * @param end_idx - * @param filtered_log_event_map - * @param log_events - * @param use_filter - * @param ts_pattern A pattern for formatting unstructured log event timestamps as date strings. - * The pattern is unused for structured log events, as they are not formatted in `clp-ffi-js`. - * @return - */ -template -auto generic_decode_range( - size_t begin_idx, - size_t end_idx, - FilteredLogEventsMap const& filtered_log_event_map, - LogEvents const& log_events, - bool use_filter, - clp::TimestampPattern const& ts_pattern -) -> DecodedResultsTsType { - if (use_filter && false == filtered_log_event_map.has_value()) { - return DecodedResultsTsType{emscripten::val::null()}; - } - - size_t length{0}; - if (use_filter) { - length = filtered_log_event_map->size(); - } else { - length = log_events.size(); - } - if (length < end_idx || begin_idx > end_idx) { - return DecodedResultsTsType{emscripten::val::null()}; - } - - auto const results{emscripten::val::array()}; - std::string generic_event_string; - constexpr size_t cDefaultReservedLength{512}; - generic_event_string.reserve(cDefaultReservedLength); - - for (size_t i = begin_idx; i < end_idx; ++i) { - size_t log_event_idx{0}; - if (use_filter) { - log_event_idx = filtered_log_event_map->at(i); - } else { - log_event_idx = i; - } - - auto const& log_event_with_filter_data{log_events.at(log_event_idx)}; - auto const& log_event = log_event_with_filter_data.get_log_event(); - auto const& timestamp = log_event_with_filter_data.get_timestamp(); - auto const& log_level = log_event_with_filter_data.get_log_level(); - - if constexpr (std::is_same_v) { - auto const parsed{log_event.get_message().decode_and_unparse()}; - if (false == parsed.has_value()) { - throw ClpFfiJsException{ - clp::ErrorCode::ErrorCode_Failure, - __FILENAME__, - __LINE__, - "Failed to decode message" - }; - } - generic_event_string = parsed.value(); - ts_pattern.insert_formatted_timestamp(timestamp, generic_event_string); - } else if constexpr (std::is_same_v) { - auto const json_result{log_event.serialize_to_json()}; - if (false == json_result.has_value()) { - auto error_code{json_result.error()}; - SPDLOG_ERROR( - "Failed to deserialize log event to JSON: {}:{}", - error_code.category().name(), - error_code.message() - ); - generic_event_string = std::string(cEmptyJsonStr); - } else { - generic_event_string = json_result.value().dump(); - } - } - - EM_ASM( - { Emval.toValue($0).push([UTF8ToString($1), $2, $3, $4]); }, - results.as_handle(), - generic_event_string.c_str(), - timestamp, - log_level, - log_event_idx + 1 - ); - } - - return DecodedResultsTsType(results); -} -} // namespace clp_ffi_js::ir - -#endif // CLP_FFI_JS_IR_UTILS_HPP From 9a5a5024d3d0d2fd58fdb31f2e582582ffaaf145 Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Thu, 12 Dec 2024 22:14:50 +0000 Subject: [PATCH 41/52] small nits --- src/clp_ffi_js/ir/StreamReader.hpp | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/clp_ffi_js/ir/StreamReader.hpp b/src/clp_ffi_js/ir/StreamReader.hpp index 533b3d7e..2c4739af 100644 --- a/src/clp_ffi_js/ir/StreamReader.hpp +++ b/src/clp_ffi_js/ir/StreamReader.hpp @@ -159,7 +159,7 @@ class StreamReader { * * @param begin_idx * @param end_idx - * @param filtered_log_event_map Derived class's filtered log event map. + * @param filtered_log_event_map Derived class's `FilteredLogEventsMap`. * @param log_events Derived class's log events. * @param use_filter * @param log_event_to_string Lambda function which retrieves a string from a single log event. @@ -191,7 +191,6 @@ class StreamReader { } auto const results{emscripten::val::array()}; - std::string generic_event_string; for (size_t i = begin_idx; i < end_idx; ++i) { size_t log_event_idx{0}; @@ -206,12 +205,10 @@ class StreamReader { auto const& timestamp = log_event_with_filter_data.get_timestamp(); auto const& log_level = log_event_with_filter_data.get_log_level(); - generic_event_string = log_event_to_string(log_event); - EM_ASM( { Emval.toValue($0).push([UTF8ToString($1), $2, $3, $4]); }, results.as_handle(), - generic_event_string.c_str(), + log_event_to_string(log_event).c_str(), timestamp, log_level, log_event_idx + 1 From cce798088cc09229ee6cb731be7d637bf0d7a8d0 Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Thu, 12 Dec 2024 22:27:50 +0000 Subject: [PATCH 42/52] fix lint --- src/clp_ffi_js/ir/StreamReader.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/clp_ffi_js/ir/StreamReader.hpp b/src/clp_ffi_js/ir/StreamReader.hpp index 2c4739af..290f7878 100644 --- a/src/clp_ffi_js/ir/StreamReader.hpp +++ b/src/clp_ffi_js/ir/StreamReader.hpp @@ -6,7 +6,6 @@ #include #include #include -#include #include #include From feec5efd3ec927d409263bf8c47b5361bc47f57b Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Fri, 13 Dec 2024 08:57:24 -0500 Subject: [PATCH 43/52] Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- src/clp_ffi_js/ir/StreamReader.hpp | 37 +++++++++++-------- .../ir/StructuredIrStreamReader.cpp | 4 +- .../ir/UnstructuredIrStreamReader.cpp | 4 +- 3 files changed, 25 insertions(+), 20 deletions(-) diff --git a/src/clp_ffi_js/ir/StreamReader.hpp b/src/clp_ffi_js/ir/StreamReader.hpp index 290f7878..c391ca13 100644 --- a/src/clp_ffi_js/ir/StreamReader.hpp +++ b/src/clp_ffi_js/ir/StreamReader.hpp @@ -116,19 +116,19 @@ class StreamReader { -> DecodedResultsTsType = 0; /** - * A templated function that implements `filter_log_events` for both - * `UnstructuredIrStreamReader` and `StructuredIrStreamReader`. + * Templated implementation of `filter_log_events`. * + * @tparam LogEvent * @param log_level_filter * @param log_events Derived class's log events. - * @param[out] filtered_log_event_map A reference to derived class's `FilteredLogEventsMap` - * that stores filtered result. + * @param log_events + * @param[out] filtered_log_event_map Returns the filtered log events. */ - template + template static auto generic_filter_log_events( FilteredLogEventsMap& filtered_log_event_map, LogLevelFilterTsType const& log_level_filter, - LogEvents const& log_events + std::vector> const& log_events ) -> void { if (log_level_filter.isNull()) { filtered_log_event_map.reset(); @@ -153,25 +153,30 @@ class StreamReader { } /** - * A templated function that implements `decode_range` for both - * `UnstructuredIrStreamReader` and `StructuredIrStreamReader`. + * Templated implementation of `decode_range` that uses `log_event_to_string` to convert + * `log_event` to a string for the returned result. * + * @tparam LogEvent + * @tparam ToStringFunc Function to convert a log event into a string. * @param begin_idx * @param end_idx - * @param filtered_log_event_map Derived class's `FilteredLogEventsMap`. - * @param log_events Derived class's log events. + * @param filtered_log_event_map + * @param log_events * @param use_filter - * @param log_event_to_string Lambda function which retrieves a string from a single log event. - * The derived class must implement lambda since `Unstructured` and `Structured` - * log events have different interfaces. - * @return + * @param log_event_to_string + * @return See `decode_range`. */ - template + template + requires requires(ToStringFunc func, LogEvent const& log_event) { + { + func(log_event) + } -> std::convertible_to; + } static auto generic_decode_range( size_t begin_idx, size_t end_idx, FilteredLogEventsMap const& filtered_log_event_map, - LogEvents const& log_events, + std::vector> const& log_events, ToStringFunc log_event_to_string, bool use_filter ) -> DecodedResultsTsType { diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp index ae142e29..f12b66b4 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp @@ -76,7 +76,7 @@ auto StructuredIrStreamReader::get_filtered_log_event_map() const -> FilteredLog } void StructuredIrStreamReader::filter_log_events(LogLevelFilterTsType const& log_level_filter) { - StreamReader::generic_filter_log_events( + generic_filter_log_events( m_filtered_log_event_map, log_level_filter, *m_deserialized_log_events @@ -137,7 +137,7 @@ auto StructuredIrStreamReader::decode_range(size_t begin_idx, size_t end_idx, bo return json_str; }; - return StreamReader::generic_decode_range( + return generic_decode_range( begin_idx, end_idx, m_filtered_log_event_map, diff --git a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp index a2e93059..36638b15 100644 --- a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp @@ -69,7 +69,7 @@ auto UnstructuredIrStreamReader::get_filtered_log_event_map() const -> FilteredL } void UnstructuredIrStreamReader::filter_log_events(LogLevelFilterTsType const& log_level_filter) { - StreamReader::generic_filter_log_events( + generic_filter_log_events( m_filtered_log_event_map, log_level_filter, m_encoded_log_events @@ -152,7 +152,7 @@ auto UnstructuredIrStreamReader::decode_range(size_t begin_idx, size_t end_idx, return message; }; - return StreamReader::generic_decode_range( + return generic_decode_range( begin_idx, end_idx, m_filtered_log_event_map, From 61e1e83e8d80616261a34f85a7a4f26a33448bcc Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Fri, 13 Dec 2024 15:24:32 +0000 Subject: [PATCH 44/52] kirk review --- src/clp_ffi_js/ir/StreamReader.hpp | 176 ++++++++++-------- .../ir/StructuredIrStreamReader.hpp | 2 +- src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp | 34 ++-- .../ir/UnstructuredIrStreamReader.cpp | 6 +- .../ir/UnstructuredIrStreamReader.hpp | 2 +- 5 files changed, 123 insertions(+), 97 deletions(-) diff --git a/src/clp_ffi_js/ir/StreamReader.hpp b/src/clp_ffi_js/ir/StreamReader.hpp index c391ca13..0aae6748 100644 --- a/src/clp_ffi_js/ir/StreamReader.hpp +++ b/src/clp_ffi_js/ir/StreamReader.hpp @@ -15,6 +15,7 @@ #include #include +#include namespace clp_ffi_js::ir { // JS types used as inputs @@ -31,6 +32,9 @@ enum class StreamType : uint8_t { Unstructured, }; +template +using LogEvents = std::vector>; + /** * Mapping between an index in the filtered log events collection to an index in the unfiltered * log events collection. @@ -115,42 +119,8 @@ class StreamReader { [[nodiscard]] virtual auto decode_range(size_t begin_idx, size_t end_idx, bool use_filter) const -> DecodedResultsTsType = 0; - /** - * Templated implementation of `filter_log_events`. - * - * @tparam LogEvent - * @param log_level_filter - * @param log_events Derived class's log events. - * @param log_events - * @param[out] filtered_log_event_map Returns the filtered log events. - */ - template - static auto generic_filter_log_events( - FilteredLogEventsMap& filtered_log_event_map, - LogLevelFilterTsType const& log_level_filter, - std::vector> const& log_events - ) -> void { - if (log_level_filter.isNull()) { - filtered_log_event_map.reset(); - return; - } - - filtered_log_event_map.emplace(); - auto filter_levels - = emscripten::vecFromJSArray>(log_level_filter); - - for (size_t log_event_idx = 0; log_event_idx < log_events.size(); ++log_event_idx) { - auto const& log_event = log_events[log_event_idx]; - if (std::ranges::find( - filter_levels, - clp::enum_to_underlying_type(log_event.get_log_level()) - ) - != filter_levels.end()) - { - filtered_log_event_map->emplace_back(log_event_idx); - } - } - } +protected: + explicit StreamReader() = default; /** * Templated implementation of `decode_range` that uses `log_event_to_string` to convert @@ -176,55 +146,111 @@ class StreamReader { size_t begin_idx, size_t end_idx, FilteredLogEventsMap const& filtered_log_event_map, - std::vector> const& log_events, + LogEvents const& log_events, ToStringFunc log_event_to_string, bool use_filter - ) -> DecodedResultsTsType { - if (use_filter && false == filtered_log_event_map.has_value()) { - return DecodedResultsTsType{emscripten::val::null()}; - } + ) -> DecodedResultsTsType; + + /** + * Templated implementation of `filter_log_events`. + * + * @tparam LogEvent + * @param log_level_filter + * @param log_events Derived class's log events. + * @param log_events + * @param[out] filtered_log_event_map Returns the filtered log events. + */ + template + static auto generic_filter_log_events( + FilteredLogEventsMap& filtered_log_event_map, + LogLevelFilterTsType const& log_level_filter, + LogEvents const& log_events + ) -> void; +}; + +template +requires requires(ToStringFunc func, LogEvent const& log_event) { + { + func(log_event) + } -> std::convertible_to; +} +auto StreamReader::generic_decode_range( + size_t begin_idx, + size_t end_idx, + FilteredLogEventsMap const& filtered_log_event_map, + LogEvents const& log_events, + ToStringFunc log_event_to_string, + bool use_filter +) -> DecodedResultsTsType { + if (use_filter && false == filtered_log_event_map.has_value()) { + return DecodedResultsTsType{emscripten::val::null()}; + } + + size_t length{0}; + if (use_filter) { + length = filtered_log_event_map->size(); + } else { + length = log_events.size(); + } + if (length < end_idx || begin_idx > end_idx) { + return DecodedResultsTsType{emscripten::val::null()}; + } - size_t length{0}; + auto const results{emscripten::val::array()}; + + for (size_t i = begin_idx; i < end_idx; ++i) { + size_t log_event_idx{0}; if (use_filter) { - length = filtered_log_event_map->size(); + log_event_idx = filtered_log_event_map->at(i); } else { - length = log_events.size(); - } - if (length < end_idx || begin_idx > end_idx) { - return DecodedResultsTsType{emscripten::val::null()}; + log_event_idx = i; } - auto const results{emscripten::val::array()}; - - for (size_t i = begin_idx; i < end_idx; ++i) { - size_t log_event_idx{0}; - if (use_filter) { - log_event_idx = filtered_log_event_map->at(i); - } else { - log_event_idx = i; - } - - auto const& log_event_with_filter_data{log_events.at(log_event_idx)}; - auto const& log_event = log_event_with_filter_data.get_log_event(); - auto const& timestamp = log_event_with_filter_data.get_timestamp(); - auto const& log_level = log_event_with_filter_data.get_log_level(); - - EM_ASM( - { Emval.toValue($0).push([UTF8ToString($1), $2, $3, $4]); }, - results.as_handle(), - log_event_to_string(log_event).c_str(), - timestamp, - log_level, - log_event_idx + 1 - ); - } + auto const& log_event_with_filter_data{log_events.at(log_event_idx)}; + auto const& log_event = log_event_with_filter_data.get_log_event(); + auto const& timestamp = log_event_with_filter_data.get_timestamp(); + auto const& log_level = log_event_with_filter_data.get_log_level(); - return DecodedResultsTsType(results); + EM_ASM( + { Emval.toValue($0).push([UTF8ToString($1), $2, $3, $4]); }, + results.as_handle(), + log_event_to_string(log_event).c_str(), + timestamp, + log_level, + log_event_idx + 1 + ); } -protected: - explicit StreamReader() = default; -}; + return DecodedResultsTsType(results); +} + +template +auto StreamReader::generic_filter_log_events( + FilteredLogEventsMap& filtered_log_event_map, + LogLevelFilterTsType const& log_level_filter, + LogEvents const& log_events +) -> void { + if (log_level_filter.isNull()) { + filtered_log_event_map.reset(); + return; + } + + filtered_log_event_map.emplace(); + auto filter_levels + = emscripten::vecFromJSArray>(log_level_filter); + + for (size_t log_event_idx = 0; log_event_idx < log_events.size(); ++log_event_idx) { + auto const& log_event = log_events[log_event_idx]; + if (std::ranges::find( + filter_levels, + clp::enum_to_underlying_type(log_event.get_log_level()) + ) + != filter_levels.end()) + { + filtered_log_event_map->emplace_back(log_event_idx); + } + } +} } // namespace clp_ffi_js::ir #endif // CLP_FFI_JS_IR_STREAMREADER_HPP diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp index fc0dcbe3..640fe9d0 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp @@ -19,7 +19,7 @@ namespace clp_ffi_js::ir { using schema_tree_node_id_t = std::optional; using StructuredIrDeserializer = clp::ffi::ir_stream::Deserializer; -using StructuredLogEvents = std::vector>; +using StructuredLogEvents = LogEvents; /** * Class to deserialize and decode Zstd-compressed CLP structured IR streams, as well as format diff --git a/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp b/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp index 68f992b5..cc7bfcd6 100644 --- a/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp +++ b/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp @@ -101,20 +101,22 @@ auto StructuredIrUnitHandler::get_log_level( if (false == m_log_level_node_id.has_value()) { return log_level; } - auto const& log_level_pair{id_value_pairs.at(m_log_level_node_id.value())}; - if (false == log_level_pair.has_value()) { + auto const& log_level_optional_value{id_value_pairs.at(m_log_level_node_id.value())}; + if (false == log_level_optional_value.has_value()) { return log_level; } - if (log_level_pair->is()) { - auto const& log_level_node_value = log_level_pair.value().get_immutable_view(); - log_level = parse_log_level(log_level_node_value); - } else if (log_level_pair->is()) { - auto const& log_level_node_value - = (log_level_pair.value().get_immutable_view()); - if (log_level_node_value >= clp::enum_to_underlying_type(cValidLogLevelsBeginIdx) - && log_level_node_value < clp::enum_to_underlying_type(LogLevel::LENGTH)) + + auto const log_level_value = log_level_optional_value.value(); + + if (log_level_value.is()) { + auto const& log_level_str = log_level_value.get_immutable_view(); + log_level = parse_log_level(log_level_str); + } else if (log_level_value.is()) { + auto const& log_level_int = (log_level_value.get_immutable_view()); + if (log_level_int >= clp::enum_to_underlying_type(cValidLogLevelsBeginIdx) + && log_level_int < clp::enum_to_underlying_type(LogLevel::LENGTH)) { - log_level = static_cast(log_level_node_value); + log_level = static_cast(log_level_int); } } else { auto log_event_idx = m_deserialized_log_events->size(); @@ -135,13 +137,15 @@ auto StructuredIrUnitHandler::get_timestamp( if (false == m_timestamp_node_id.has_value()) { return timestamp; } - auto const& timestamp_pair{id_value_pairs.at(m_timestamp_node_id.value())}; - if (false == timestamp_pair.has_value()) { + auto const& timestamp_optional_value{id_value_pairs.at(m_timestamp_node_id.value())}; + if (false == timestamp_optional_value.has_value()) { return timestamp; } - if (timestamp_pair->is()) { + auto const timestamp_value = timestamp_optional_value.value(); + + if (timestamp_value.is()) { timestamp = static_cast( - timestamp_pair.value().get_immutable_view() + timestamp_value.get_immutable_view() ); } else { // TODO: Add support for parsing string-type timestamp values. diff --git a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp index 36638b15..22085e02 100644 --- a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp @@ -69,11 +69,7 @@ auto UnstructuredIrStreamReader::get_filtered_log_event_map() const -> FilteredL } void UnstructuredIrStreamReader::filter_log_events(LogLevelFilterTsType const& log_level_filter) { - generic_filter_log_events( - m_filtered_log_event_map, - log_level_filter, - m_encoded_log_events - ); + generic_filter_log_events(m_filtered_log_event_map, log_level_filter, m_encoded_log_events); } auto UnstructuredIrStreamReader::deserialize_stream() -> size_t { diff --git a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp index e6fdf1ba..915724af 100644 --- a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp +++ b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp @@ -18,7 +18,7 @@ namespace clp_ffi_js::ir { using clp::ir::four_byte_encoded_variable_t; using UnstructuredIrDeserializer = clp::ir::LogEventDeserializer; -using UnstructuredLogEvents = std::vector>; +using UnstructuredLogEvents = LogEvents; /** * Class to deserialize and decode Zstd-compressed CLP unstructured IR streams, as well as format From 907c327149dfa1c254d170355fc3f23a4ae47bc4 Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Fri, 13 Dec 2024 15:39:36 +0000 Subject: [PATCH 45/52] fix lint --- src/clp_ffi_js/ir/StreamReader.hpp | 1 + src/clp_ffi_js/ir/StructuredIrStreamReader.hpp | 1 - src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp | 1 - 3 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/clp_ffi_js/ir/StreamReader.hpp b/src/clp_ffi_js/ir/StreamReader.hpp index 0aae6748..2a117a33 100644 --- a/src/clp_ffi_js/ir/StreamReader.hpp +++ b/src/clp_ffi_js/ir/StreamReader.hpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp index 640fe9d0..6ccd5255 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp @@ -4,7 +4,6 @@ #include #include #include -#include #include #include diff --git a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp index 915724af..60e515f3 100644 --- a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp +++ b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp @@ -4,7 +4,6 @@ #include #include #include -#include #include #include From 1da4c7d0fda42385152853b05d753a772699a0a5 Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Fri, 13 Dec 2024 15:55:43 +0000 Subject: [PATCH 46/52] fix lint --- src/clp_ffi_js/ir/StreamReader.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/clp_ffi_js/ir/StreamReader.hpp b/src/clp_ffi_js/ir/StreamReader.hpp index 2a117a33..f7332dc6 100644 --- a/src/clp_ffi_js/ir/StreamReader.hpp +++ b/src/clp_ffi_js/ir/StreamReader.hpp @@ -2,6 +2,7 @@ #define CLP_FFI_JS_IR_STREAMREADER_HPP #include +#include #include #include #include From f08e07f8875e3cce058f5b01bcaff10dd7769772 Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Fri, 13 Dec 2024 14:02:44 -0500 Subject: [PATCH 47/52] Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp b/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp index cc7bfcd6..ee5a2a15 100644 --- a/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp +++ b/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp @@ -105,14 +105,13 @@ auto StructuredIrUnitHandler::get_log_level( if (false == log_level_optional_value.has_value()) { return log_level; } - auto const log_level_value = log_level_optional_value.value(); if (log_level_value.is()) { auto const& log_level_str = log_level_value.get_immutable_view(); log_level = parse_log_level(log_level_str); } else if (log_level_value.is()) { - auto const& log_level_int = (log_level_value.get_immutable_view()); + auto const& log_level_int = log_level_value.get_immutable_view(); if (log_level_int >= clp::enum_to_underlying_type(cValidLogLevelsBeginIdx) && log_level_int < clp::enum_to_underlying_type(LogLevel::LENGTH)) { From aee6422df23ee689ffefa427f39e48b9fb1a15bd Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Fri, 13 Dec 2024 19:08:15 +0000 Subject: [PATCH 48/52] kirk review --- src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp b/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp index ee5a2a15..0bff407c 100644 --- a/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp +++ b/src/clp_ffi_js/ir/StructuredIrUnitHandler.cpp @@ -101,11 +101,11 @@ auto StructuredIrUnitHandler::get_log_level( if (false == m_log_level_node_id.has_value()) { return log_level; } - auto const& log_level_optional_value{id_value_pairs.at(m_log_level_node_id.value())}; - if (false == log_level_optional_value.has_value()) { + auto const& optional_log_level_value{id_value_pairs.at(m_log_level_node_id.value())}; + if (false == optional_log_level_value.has_value()) { return log_level; } - auto const log_level_value = log_level_optional_value.value(); + auto const log_level_value = optional_log_level_value.value(); if (log_level_value.is()) { auto const& log_level_str = log_level_value.get_immutable_view(); @@ -136,11 +136,11 @@ auto StructuredIrUnitHandler::get_timestamp( if (false == m_timestamp_node_id.has_value()) { return timestamp; } - auto const& timestamp_optional_value{id_value_pairs.at(m_timestamp_node_id.value())}; - if (false == timestamp_optional_value.has_value()) { + auto const& optional_timestamp_value{id_value_pairs.at(m_timestamp_node_id.value())}; + if (false == optional_timestamp_value.has_value()) { return timestamp; } - auto const timestamp_value = timestamp_optional_value.value(); + auto const timestamp_value = optional_timestamp_value.value(); if (timestamp_value.is()) { timestamp = static_cast( From 0581310b6cc8ed565f325e822cbe94d0450cae3c Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Sat, 14 Dec 2024 01:53:37 +0000 Subject: [PATCH 49/52] added rerror for invalid range --- src/clp_ffi_js/ir/StreamReader.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/clp_ffi_js/ir/StreamReader.hpp b/src/clp_ffi_js/ir/StreamReader.hpp index f7332dc6..4623315a 100644 --- a/src/clp_ffi_js/ir/StreamReader.hpp +++ b/src/clp_ffi_js/ir/StreamReader.hpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -195,6 +196,7 @@ auto StreamReader::generic_decode_range( length = log_events.size(); } if (length < end_idx || begin_idx > end_idx) { + SPDLOG_ERROR("Invalid log event index range: {}-{}", begin_idx, end_idx); return DecodedResultsTsType{emscripten::val::null()}; } From 2aabf7b6469f6b9d3adf3d4938344904d99db010 Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Wed, 18 Dec 2024 09:18:15 -0500 Subject: [PATCH 50/52] Update src/clp_ffi_js/ir/StreamReader.hpp Co-authored-by: Junhao Liao --- src/clp_ffi_js/ir/StreamReader.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/clp_ffi_js/ir/StreamReader.hpp b/src/clp_ffi_js/ir/StreamReader.hpp index 4623315a..f8314ff0 100644 --- a/src/clp_ffi_js/ir/StreamReader.hpp +++ b/src/clp_ffi_js/ir/StreamReader.hpp @@ -138,6 +138,7 @@ class StreamReader { * @param use_filter * @param log_event_to_string * @return See `decode_range`. + * @throw Propagates `ToStringFunc`'s exception. */ template requires requires(ToStringFunc func, LogEvent const& log_event) { From 2d7e02ec78ae1a9284c21a136e858776bf406597 Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Wed, 18 Dec 2024 14:24:26 +0000 Subject: [PATCH 51/52] add throws doc strings --- src/clp_ffi_js/ir/StreamReader.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/clp_ffi_js/ir/StreamReader.hpp b/src/clp_ffi_js/ir/StreamReader.hpp index f8314ff0..f78937d2 100644 --- a/src/clp_ffi_js/ir/StreamReader.hpp +++ b/src/clp_ffi_js/ir/StreamReader.hpp @@ -138,7 +138,7 @@ class StreamReader { * @param use_filter * @param log_event_to_string * @return See `decode_range`. - * @throw Propagates `ToStringFunc`'s exception. + * @throw Propagates exceptions from `ToStringFunc`. */ template requires requires(ToStringFunc func, LogEvent const& log_event) { From 7d0a2c8325fdaf39152bd4365fca1cf81420114b Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Wed, 18 Dec 2024 14:29:43 +0000 Subject: [PATCH 52/52] fix --- src/clp_ffi_js/ir/StreamReader.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/clp_ffi_js/ir/StreamReader.hpp b/src/clp_ffi_js/ir/StreamReader.hpp index f78937d2..6cc231a4 100644 --- a/src/clp_ffi_js/ir/StreamReader.hpp +++ b/src/clp_ffi_js/ir/StreamReader.hpp @@ -138,7 +138,7 @@ class StreamReader { * @param use_filter * @param log_event_to_string * @return See `decode_range`. - * @throw Propagates exceptions from `ToStringFunc`. + * @throws Propagates `ToStringFunc`'s exceptions. */ template requires requires(ToStringFunc func, LogEvent const& log_event) {