From fc08027c396a8779471326013aa6aa1afef72746 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 6 Nov 2024 23:02:30 -0500 Subject: [PATCH 01/43] Add StructuredIrStreamReader. --- CMakeLists.txt | 5 + src/clp_ffi_js/ir/StreamReader.cpp | 52 ++++-- src/clp_ffi_js/ir/StreamReader.hpp | 4 +- src/clp_ffi_js/ir/StreamReaderDataContext.hpp | 4 + .../ir/StructuredIrStreamReader.hpp | 169 ++++++++++++++++++ src/submodules/clp | 2 +- 6 files changed, 221 insertions(+), 15 deletions(-) create mode 100644 src/clp_ffi_js/ir/StructuredIrStreamReader.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 204352ce..73ad4966 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -83,11 +83,16 @@ endif() set(CLP_FFI_JS_SRC_MAIN src/clp_ffi_js/ir/StreamReader.cpp + src/clp_ffi_js/ir/StructuredIrStreamReader.cpp src/clp_ffi_js/ir/UnstructuredIrStreamReader.cpp ) set(CLP_FFI_JS_SRC_CLP_CORE src/submodules/clp/components/core/src/clp/ffi/ir_stream/decoding_methods.cpp + src/submodules/clp/components/core/src/clp/ffi/ir_stream/ir_unit_deserialization_methods.cpp + src/submodules/clp/components/core/src/clp/ffi/ir_stream/utils.cpp + src/submodules/clp/components/core/src/clp/ffi/KeyValuePairLogEvent.cpp + src/submodules/clp/components/core/src/clp/ffi/SchemaTree.cpp src/submodules/clp/components/core/src/clp/ir/EncodedTextAst.cpp src/submodules/clp/components/core/src/clp/ir/LogEventDeserializer.cpp src/submodules/clp/components/core/src/clp/ReadOnlyMemoryMappedFile.cpp diff --git a/src/clp_ffi_js/ir/StreamReader.cpp b/src/clp_ffi_js/ir/StreamReader.cpp index c6b097cb..c84d9d19 100644 --- a/src/clp_ffi_js/ir/StreamReader.cpp +++ b/src/clp_ffi_js/ir/StreamReader.cpp @@ -24,6 +24,7 @@ #include #include +#include namespace { using ClpFfiJsException = clp_ffi_js::ClpFfiJsException; @@ -117,6 +118,9 @@ EMSCRIPTEN_BINDINGS(ClpStreamReader) { // JS types used as inputs emscripten::register_type("Uint8Array"); emscripten::register_type("number[] | null"); + emscripten::register_type( + "interface{logLevelKey: string, timestampKey: string} | null" + ); // JS types used as outputs emscripten::register_type( @@ -143,7 +147,8 @@ EMSCRIPTEN_BINDINGS(ClpStreamReader) { } // namespace namespace clp_ffi_js::ir { -auto StreamReader::create(DataArrayTsType const& data_array) -> std::unique_ptr { +auto StreamReader::create(DataArrayTsType const& data_array, ReaderOptions const& reader_options) + -> std::unique_ptr { auto const length{data_array["length"].as()}; SPDLOG_INFO("StreamReader::create: got buffer of length={}", length); @@ -162,16 +167,28 @@ auto StreamReader::create(DataArrayTsType const& data_array) -> std::unique_ptr< // Validate the stream's version auto pos = zstd_decompressor->get_pos(); auto const version{get_version(*zstd_decompressor)}; - if (std::ranges::find(cUnstructuredIrVersions, version) == cUnstructuredIrVersions.end()) { - throw ClpFfiJsException{ - clp::ErrorCode::ErrorCode_Unsupported, - __FILENAME__, - __LINE__, - std::format("Unable to create reader for IR stream with version {}.", version) - }; + if (std::ranges::find(cUnstructuredIrVersions, version) != cUnstructuredIrVersions.end()) { + try { + zstd_decompressor->seek_from_begin(pos); + } catch (ZstdDecompressor::OperationFailed& e) { + throw ClpFfiJsException{ + clp::ErrorCode::ErrorCode_Failure, + __FILENAME__, + __LINE__, + std::format("Unable to rewind zstd decompressor: {}", e.what()) + }; + } + return std::make_unique(UnstructuredIrStreamReader::create( + std::move(zstd_decompressor), + std::move(data_buffer) + )); } +// if (clp::ffi::ir_stream::IRProtocolErrorCode_Supported +// == clp::ffi::ir_stream::validate_protocol_version(version)) +// { + // FIXME: wait for https://github.com/y-scope/clp/pull/573 try { - zstd_decompressor->seek_from_begin(pos); + zstd_decompressor->seek_from_begin(0); } catch (ZstdDecompressor::OperationFailed& e) { throw ClpFfiJsException{ clp::ErrorCode::ErrorCode_Failure, @@ -180,9 +197,18 @@ auto StreamReader::create(DataArrayTsType const& data_array) -> std::unique_ptr< std::format("Unable to rewind zstd decompressor: {}", e.what()) }; } - - return std::make_unique( - UnstructuredIrStreamReader::create(std::move(zstd_decompressor), std::move(data_buffer)) - ); + return std::make_unique(StructuredIrStreamReader::create( + std::move(zstd_decompressor), + std::move(data_buffer), + reader_options + )); +// } + +// throw ClpFfiJsException{ +// clp::ErrorCode::ErrorCode_Unsupported, +// __FILENAME__, +// __LINE__, +// std::format("Unable to create reader for IR stream with version {}.", version) +// }; } } // namespace clp_ffi_js::ir diff --git a/src/clp_ffi_js/ir/StreamReader.hpp b/src/clp_ffi_js/ir/StreamReader.hpp index 5f298674..a06cd24b 100644 --- a/src/clp_ffi_js/ir/StreamReader.hpp +++ b/src/clp_ffi_js/ir/StreamReader.hpp @@ -13,6 +13,7 @@ namespace clp_ffi_js::ir { // JS types used as inputs EMSCRIPTEN_DECLARE_VAL_TYPE(DataArrayTsType); EMSCRIPTEN_DECLARE_VAL_TYPE(LogLevelFilterTsType); +EMSCRIPTEN_DECLARE_VAL_TYPE(ReaderOptions); // JS types used as outputs EMSCRIPTEN_DECLARE_VAL_TYPE(DecodedResultsTsType); @@ -36,7 +37,8 @@ class StreamReader { * @return The created instance. * @throw ClpFfiJsException if any error occurs. */ - [[nodiscard]] static auto create(DataArrayTsType const& data_array + [[nodiscard]] static auto create(DataArrayTsType const& data_array, + ReaderOptions const& reader_options ) -> std::unique_ptr; // Destructor diff --git a/src/clp_ffi_js/ir/StreamReaderDataContext.hpp b/src/clp_ffi_js/ir/StreamReaderDataContext.hpp index c32ad35c..0697d591 100644 --- a/src/clp_ffi_js/ir/StreamReaderDataContext.hpp +++ b/src/clp_ffi_js/ir/StreamReaderDataContext.hpp @@ -44,6 +44,10 @@ class StreamReaderDataContext { */ [[nodiscard]] auto get_deserializer() -> Deserializer& { return m_deserializer; } + /** + * @return A reference to the reader. + */ + [[nodiscard]] auto get_reader() -> clp::ReaderInterface& { return *m_reader; } private: clp::Array m_data_buffer; std::unique_ptr m_reader; diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp new file mode 100644 index 00000000..64c8b1c3 --- /dev/null +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp @@ -0,0 +1,169 @@ +#ifndef CLP_FFI_JS_IR_STRUCTUREDIRSTREAMREADER_HPP +#define CLP_FFI_JS_IR_STRUCTUREDIRSTREAMREADER_HPP + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace clp_ffi_js::ir { +using parsed_tree_node_id_t = std::optional; + +/** + * Class to handle deserialized IR units. + */ +class IrUnitHandler { +public: + IrUnitHandler( + std::vector& deserialized_log_events, + std::string log_level_key, + std::string timestamp_key + ) + : m_deserialized_log_events{deserialized_log_events}, + m_log_level_key{std::move(log_level_key)}, + m_timestamp_key{std::move(timestamp_key)} {} + + // Implements `clp::ffi::ir_stream::IrUnitHandlerInterface` interface + [[nodiscard]] auto handle_log_event(clp::ffi::KeyValuePairLogEvent&& log_event + ) -> clp::ffi::ir_stream::IRErrorCode { + m_deserialized_log_events.emplace_back(std::move(log_event)); + + return clp::ffi::ir_stream::IRErrorCode::IRErrorCode_Success; + } + + [[nodiscard]] static auto handle_utc_offset_change( + [[maybe_unused]] clp::UtcOffset utc_offset_old, + [[maybe_unused]] clp::UtcOffset utc_offset_new + ) -> clp::ffi::ir_stream::IRErrorCode { + SPDLOG_WARN("UTC offset change packets are currently not handled."); + + return clp::ffi::ir_stream::IRErrorCode::IRErrorCode_Success; + } + + [[nodiscard]] auto handle_schema_tree_node_insertion( + [[maybe_unused]] clp::ffi::SchemaTree::NodeLocator schema_tree_node_locator + ) -> clp::ffi::ir_stream::IRErrorCode { + ++m_current_node_id; + auto const& key_name{schema_tree_node_locator.get_key_name()}; + + if (m_log_level_key == key_name) { + m_level_node_id.emplace(m_current_node_id); + } else if (m_timestamp_key == key_name) { + m_timestamp_node_id.emplace(m_current_node_id); + } + + return clp::ffi::ir_stream::IRErrorCode::IRErrorCode_Success; + } + + // FIXME: do i need this? + [[nodiscard]] static auto handle_end_of_stream() -> clp::ffi::ir_stream::IRErrorCode { + return clp::ffi::ir_stream::IRErrorCode::IRErrorCode_Success; + } + + // Methods + [[nodiscard]] auto get_deserialized_log_events( + ) const -> std::vector const& { + return m_deserialized_log_events; + } + + [[nodiscard]] auto get_level_node_id() const -> parsed_tree_node_id_t { + return m_level_node_id; + } + + [[nodiscard]] auto get_timestamp_node_id() const -> parsed_tree_node_id_t { + return m_timestamp_node_id; + } + +private: + std::string m_log_level_key; + std::string m_timestamp_key; + + // the root node has id=0 + clp::ffi::SchemaTree::Node::id_t m_current_node_id; + parsed_tree_node_id_t m_level_node_id; + parsed_tree_node_id_t m_timestamp_node_id; + + std::vector& m_deserialized_log_events; + bool m_is_complete{false}; +}; + +using StructuredIrDeserializer = clp::ffi::ir_stream::Deserializer; + +/** + * Class to deserialize and decode Zstd-compressed CLP structured IR streams, as well as format + * decoded log events. + */ +class StructuredIrStreamReader : public StreamReader { +public: + // Destructor + ~StructuredIrStreamReader() override = default; + + // Disable copy constructor and assignment operator + StructuredIrStreamReader(StructuredIrStreamReader const&) = delete; + auto operator=(StructuredIrStreamReader const&) -> StructuredIrStreamReader& = delete; + + // Define default move constructor + StructuredIrStreamReader(StructuredIrStreamReader&&) = default; + // Delete move assignment operator since it's also disabled in `clp::ir::LogEventDeserializer`. + auto operator=(StructuredIrStreamReader&&) -> StructuredIrStreamReader& = delete; + + /** + * @param zstd_decompressor A decompressor for an IR stream, where the read head of the stream + * is just after the stream's encoding type. + * @param data_array The array backing `zstd_decompressor`. + * @return The created instance. + * @throw ClpFfiJsException if any error occurs. + */ + [[nodiscard]] static auto create( + std::unique_ptr&& zstd_decompressor, + clp::Array data_array, + ReaderOptions const& reader_options + ) -> StructuredIrStreamReader; + + [[nodiscard]] auto get_num_events_buffered() const -> size_t override; + + [[nodiscard]] auto get_filtered_log_event_map() const -> FilteredLogEventMapTsType override; + + void filter_log_events(LogLevelFilterTsType const& log_level_filter) override; + + /** + * @see StreamReader::deserialize_stream + * + * After the stream has been exhausted, it will be deallocated. + * + * @return @see StreamReader::deserialize_stream + */ + [[nodiscard]] auto deserialize_stream() -> size_t override; + + [[nodiscard]] auto decode_range(size_t begin_idx, size_t end_idx, bool use_filter) const + -> DecodedResultsTsType override; + +private: + // Constructor + explicit StructuredIrStreamReader( + StreamReaderDataContext&& stream_reader_data_context, + std::shared_ptr> deserialized_log_events + ); + + // Variables + std::shared_ptr> m_deserialized_log_events; + std::unique_ptr> m_stream_reader_data_context; + + parsed_tree_node_id_t m_level_node_id; + parsed_tree_node_id_t m_timestamp_node_id; +}; +} // namespace clp_ffi_js::ir + +#endif // CLP_FFI_JS_IR_STRUCTUREDIRSTREAMREADER_HPP diff --git a/src/submodules/clp b/src/submodules/clp index e1f3f2ab..ee0adef9 160000 --- a/src/submodules/clp +++ b/src/submodules/clp @@ -1 +1 @@ -Subproject commit e1f3f2abe3473324b19d66e22c182ec3ac0d408f +Subproject commit ee0adef9985b28f4d543cfe6ba1676d8912bb3eb From 8d205830995d5c85526f238a02bf7a6f74e6ea98 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 6 Nov 2024 23:08:20 -0500 Subject: [PATCH 02/43] Reformat; Add missing files to VCS. --- src/clp_ffi_js/ir/StreamReader.cpp | 34 +-- src/clp_ffi_js/ir/StreamReader.hpp | 5 +- src/clp_ffi_js/ir/StreamReaderDataContext.hpp | 1 + .../ir/StructuredIrStreamReader.cpp | 195 ++++++++++++++++++ .../ir/StructuredIrStreamReader.hpp | 29 ++- 5 files changed, 228 insertions(+), 36 deletions(-) create mode 100644 src/clp_ffi_js/ir/StructuredIrStreamReader.cpp diff --git a/src/clp_ffi_js/ir/StreamReader.cpp b/src/clp_ffi_js/ir/StreamReader.cpp index c84d9d19..98e5cb84 100644 --- a/src/clp_ffi_js/ir/StreamReader.cpp +++ b/src/clp_ffi_js/ir/StreamReader.cpp @@ -23,8 +23,8 @@ #include #include -#include #include +#include namespace { using ClpFfiJsException = clp_ffi_js::ClpFfiJsException; @@ -183,9 +183,9 @@ auto StreamReader::create(DataArrayTsType const& data_array, ReaderOptions const std::move(data_buffer) )); } -// if (clp::ffi::ir_stream::IRProtocolErrorCode_Supported -// == clp::ffi::ir_stream::validate_protocol_version(version)) -// { + // if (clp::ffi::ir_stream::IRProtocolErrorCode_Supported + // == clp::ffi::ir_stream::validate_protocol_version(version)) + // { // FIXME: wait for https://github.com/y-scope/clp/pull/573 try { zstd_decompressor->seek_from_begin(0); @@ -197,18 +197,18 @@ auto StreamReader::create(DataArrayTsType const& data_array, ReaderOptions const std::format("Unable to rewind zstd decompressor: {}", e.what()) }; } - return std::make_unique(StructuredIrStreamReader::create( - std::move(zstd_decompressor), - std::move(data_buffer), - reader_options - )); -// } - -// throw ClpFfiJsException{ -// clp::ErrorCode::ErrorCode_Unsupported, -// __FILENAME__, -// __LINE__, -// std::format("Unable to create reader for IR stream with version {}.", version) -// }; + return std::make_unique(StructuredIrStreamReader::create( + std::move(zstd_decompressor), + std::move(data_buffer), + reader_options + )); + // } + + // throw ClpFfiJsException{ + // clp::ErrorCode::ErrorCode_Unsupported, + // __FILENAME__, + // __LINE__, + // std::format("Unable to create reader for IR stream with version {}.", version) + // }; } } // namespace clp_ffi_js::ir diff --git a/src/clp_ffi_js/ir/StreamReader.hpp b/src/clp_ffi_js/ir/StreamReader.hpp index a06cd24b..111b110f 100644 --- a/src/clp_ffi_js/ir/StreamReader.hpp +++ b/src/clp_ffi_js/ir/StreamReader.hpp @@ -37,8 +37,9 @@ class StreamReader { * @return The created instance. * @throw ClpFfiJsException if any error occurs. */ - [[nodiscard]] static auto create(DataArrayTsType const& data_array, - ReaderOptions const& reader_options + [[nodiscard]] static auto create( + DataArrayTsType const& data_array, + ReaderOptions const& reader_options ) -> std::unique_ptr; // Destructor diff --git a/src/clp_ffi_js/ir/StreamReaderDataContext.hpp b/src/clp_ffi_js/ir/StreamReaderDataContext.hpp index 0697d591..b28cead4 100644 --- a/src/clp_ffi_js/ir/StreamReaderDataContext.hpp +++ b/src/clp_ffi_js/ir/StreamReaderDataContext.hpp @@ -48,6 +48,7 @@ class StreamReaderDataContext { * @return A reference to the reader. */ [[nodiscard]] auto get_reader() -> clp::ReaderInterface& { return *m_reader; } + private: clp::Array m_data_buffer; std::unique_ptr m_reader; diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp new file mode 100644 index 00000000..3d52d48a --- /dev/null +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp @@ -0,0 +1,195 @@ +#include "StructuredIrStreamReader.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace clp_ffi_js::ir { + +using namespace std::literals::string_literals; +using clp::ir::four_byte_encoded_variable_t; + +constexpr std::string_view cLogLevelFilteringNotSupportedPrompt{ + "Log level filtering is not yet supported in this reader." +}; + +auto StructuredIrStreamReader::create( + std::unique_ptr&& zstd_decompressor, + clp::Array data_array, + ReaderOptions const& reader_options +) -> StructuredIrStreamReader { + auto deserialized_log_events{std::make_shared>()}; + auto result{StructuredIrDeserializer::create( + *zstd_decompressor, + IrUnitHandler( + deserialized_log_events, + reader_options["logLevelKey"].as(), + reader_options["timestampKey"].as() + ) + )}; + if (result.has_error()) { + auto const error_code{result.error()}; + throw ClpFfiJsException{ + clp::ErrorCode::ErrorCode_Failure, + __FILENAME__, + __LINE__, + std::format( + "Failed to create deserializer: {} {}", + error_code.category().name(), + error_code.message() + ) + }; + } + auto data_context = StreamReaderDataContext( + std::move(data_array), + std::move(zstd_decompressor), + std::move(result.value()) + ); + return StructuredIrStreamReader(std::move(data_context), std::move(deserialized_log_events)); +} + +auto StructuredIrStreamReader::get_num_events_buffered() const -> size_t { + return m_deserialized_log_events->size(); +} + +auto StructuredIrStreamReader::get_filtered_log_event_map() const -> FilteredLogEventMapTsType { + SPDLOG_ERROR(cLogLevelFilteringNotSupportedPrompt); + return FilteredLogEventMapTsType{emscripten::val::null()}; +} + +void StructuredIrStreamReader::filter_log_events(LogLevelFilterTsType const& log_level_filter) { + if (log_level_filter.isNull()) { + return; + } + SPDLOG_ERROR(cLogLevelFilteringNotSupportedPrompt); +} + +auto StructuredIrStreamReader::deserialize_stream() -> size_t { + if (nullptr == m_stream_reader_data_context) { + return m_deserialized_log_events->size(); + } + + constexpr size_t cDefaultNumReservedLogEvents{500'000}; + m_deserialized_log_events->reserve(cDefaultNumReservedLogEvents); + auto& reader{m_stream_reader_data_context->get_reader()}; + while (true) { + auto result{m_stream_reader_data_context->get_deserializer().deserialize_next_ir_unit(reader + )}; + if (false == result.has_error()) { + continue; + } + auto const error{result.error()}; + if (std::errc::no_message_available == error || std::errc::operation_not_permitted == error) + { + break; + } + if (std::errc::result_out_of_range == error) { + SPDLOG_ERROR("File contains an incomplete IR stream"); + break; + } + throw ClpFfiJsException{ + clp::ErrorCode::ErrorCode_Corrupt, + __FILENAME__, + __LINE__, + std::format( + "Failed to deserialize: {}:{}", + error.category().name(), + error.message() + ) + }; + } + m_level_node_id = m_stream_reader_data_context->get_deserializer() + .get_ir_unit_handler() + .get_level_node_id(); + m_timestamp_node_id = m_stream_reader_data_context->get_deserializer() + .get_ir_unit_handler() + .get_timestamp_node_id(); + m_stream_reader_data_context.reset(nullptr); + return m_deserialized_log_events->size(); +} + +auto StructuredIrStreamReader::decode_range(size_t begin_idx, size_t end_idx, bool use_filter) const + -> DecodedResultsTsType { + if (use_filter) { + SPDLOG_ERROR(cLogLevelFilteringNotSupportedPrompt); + return DecodedResultsTsType{emscripten::val::null()}; + } + + if (m_deserialized_log_events->size() < end_idx || begin_idx > end_idx) { + return DecodedResultsTsType{emscripten::val::null()}; + } + + std::string message; + constexpr size_t cDefaultReservedMessageLength{512}; + message.reserve(cDefaultReservedMessageLength); + auto const results{emscripten::val::array()}; + + for (size_t log_event_idx = begin_idx; log_event_idx < end_idx; ++log_event_idx) { + auto const& log_event{m_deserialized_log_events->at(log_event_idx)}; + + auto const json{log_event.serialize_to_json()}; + if (false == json.has_value()) { + SPDLOG_ERROR("Failed to decode message."); + break; + } + + auto const& id_value_pairs{log_event.get_node_id_value_pairs()}; + clp::ffi::value_int_t log_level{static_cast(LogLevel::NONE)}; + if (m_level_node_id.has_value()) { + auto const& log_level_pair{id_value_pairs.at(m_level_node_id.value())}; + log_level = log_level_pair.has_value() + ? log_level_pair.value().get_immutable_view() + : static_cast(LogLevel::NONE); + } + clp::ffi::value_int_t timestamp{0}; + if (m_timestamp_node_id.has_value()) { + auto const& timestamp_pair{id_value_pairs.at(m_timestamp_node_id.value())}; + timestamp = timestamp_pair.has_value() + ? timestamp_pair.value().get_immutable_view() + : 0; + } + + EM_ASM( + { Emval.toValue($0).push([UTF8ToString($1), $2, $3, $4]); }, + results.as_handle(), + json.value().dump().c_str(), + log_level, + timestamp, + log_event_idx + 1 + ); + } + + return DecodedResultsTsType(results); +} + +StructuredIrStreamReader::StructuredIrStreamReader( + StreamReaderDataContext&& stream_reader_data_context, + std::shared_ptr> deserialized_log_events +) + : m_stream_reader_data_context{std::make_unique< + StreamReaderDataContext>( + std::move(stream_reader_data_context) + )}, + m_deserialized_log_events{std::move(deserialized_log_events)} {} +} // namespace clp_ffi_js::ir diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp index 64c8b1c3..cd0df16d 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp @@ -3,18 +3,20 @@ #include #include +#include +#include +#include #include #include +#include +#include +#include #include #include -#include -#include -#include #include #include -#include #include #include @@ -27,18 +29,18 @@ using parsed_tree_node_id_t = std::optional; class IrUnitHandler { public: IrUnitHandler( - std::vector& deserialized_log_events, + std::shared_ptr> deserialized_log_events, std::string log_level_key, std::string timestamp_key ) - : m_deserialized_log_events{deserialized_log_events}, + : m_deserialized_log_events{std::move(deserialized_log_events)}, m_log_level_key{std::move(log_level_key)}, m_timestamp_key{std::move(timestamp_key)} {} // Implements `clp::ffi::ir_stream::IrUnitHandlerInterface` interface [[nodiscard]] auto handle_log_event(clp::ffi::KeyValuePairLogEvent&& log_event ) -> clp::ffi::ir_stream::IRErrorCode { - m_deserialized_log_events.emplace_back(std::move(log_event)); + m_deserialized_log_events->emplace_back(std::move(log_event)); return clp::ffi::ir_stream::IRErrorCode::IRErrorCode_Success; } @@ -56,8 +58,8 @@ class IrUnitHandler { [[maybe_unused]] clp::ffi::SchemaTree::NodeLocator schema_tree_node_locator ) -> clp::ffi::ir_stream::IRErrorCode { ++m_current_node_id; - auto const& key_name{schema_tree_node_locator.get_key_name()}; + auto const& key_name{schema_tree_node_locator.get_key_name()}; if (m_log_level_key == key_name) { m_level_node_id.emplace(m_current_node_id); } else if (m_timestamp_key == key_name) { @@ -67,17 +69,11 @@ class IrUnitHandler { return clp::ffi::ir_stream::IRErrorCode::IRErrorCode_Success; } - // FIXME: do i need this? [[nodiscard]] static auto handle_end_of_stream() -> clp::ffi::ir_stream::IRErrorCode { return clp::ffi::ir_stream::IRErrorCode::IRErrorCode_Success; } // Methods - [[nodiscard]] auto get_deserialized_log_events( - ) const -> std::vector const& { - return m_deserialized_log_events; - } - [[nodiscard]] auto get_level_node_id() const -> parsed_tree_node_id_t { return m_level_node_id; } @@ -90,12 +86,11 @@ class IrUnitHandler { std::string m_log_level_key; std::string m_timestamp_key; - // the root node has id=0 - clp::ffi::SchemaTree::Node::id_t m_current_node_id; + clp::ffi::SchemaTree::Node::id_t m_current_node_id{clp::ffi::SchemaTree::cRootId}; parsed_tree_node_id_t m_level_node_id; parsed_tree_node_id_t m_timestamp_node_id; - std::vector& m_deserialized_log_events; + std::shared_ptr> m_deserialized_log_events; bool m_is_complete{false}; }; From 4757bd4d310a84c876dbd15780116ce0b03e31f6 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 6 Nov 2024 23:38:30 -0500 Subject: [PATCH 03/43] Fix clp submodule commit id. --- src/submodules/clp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/submodules/clp b/src/submodules/clp index ee0adef9..bf3d1973 160000 --- a/src/submodules/clp +++ b/src/submodules/clp @@ -1 +1 @@ -Subproject commit ee0adef9985b28f4d543cfe6ba1676d8912bb3eb +Subproject commit bf3d1973750fbd6f21f1d7d7bc292278dc4a8c8a From 12eeca640ccf87407b3dbf161897b774a800a895 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Nov 2024 00:19:07 -0500 Subject: [PATCH 04/43] Change log level return type int64_t -> uint8_t. --- src/clp_ffi_js/ir/StructuredIrStreamReader.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp index 3d52d48a..5a4ca21c 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp @@ -155,19 +155,22 @@ auto StructuredIrStreamReader::decode_range(size_t begin_idx, size_t end_idx, bo } auto const& id_value_pairs{log_event.get_node_id_value_pairs()}; - clp::ffi::value_int_t log_level{static_cast(LogLevel::NONE)}; + LogLevel log_level{LogLevel::NONE}; if (m_level_node_id.has_value()) { auto const& log_level_pair{id_value_pairs.at(m_level_node_id.value())}; log_level = log_level_pair.has_value() - ? log_level_pair.value().get_immutable_view() - : static_cast(LogLevel::NONE); + ? static_cast( + log_level_pair.value() + .get_immutable_view() + ) + : log_level; } clp::ffi::value_int_t timestamp{0}; if (m_timestamp_node_id.has_value()) { auto const& timestamp_pair{id_value_pairs.at(m_timestamp_node_id.value())}; timestamp = timestamp_pair.has_value() ? timestamp_pair.value().get_immutable_view() - : 0; + : timestamp; } EM_ASM( From d870b1966a679ca4485028625d0930c2cdb2573a Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Nov 2024 00:26:05 -0500 Subject: [PATCH 05/43] update clp commit id. --- src/submodules/clp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/submodules/clp b/src/submodules/clp index bf3d1973..426cc3d6 160000 --- a/src/submodules/clp +++ b/src/submodules/clp @@ -1 +1 @@ -Subproject commit bf3d1973750fbd6f21f1d7d7bc292278dc4a8c8a +Subproject commit 426cc3d657c67e9fdffe6681e670cba617f4154f From edda4b38132bc96b00519b92f1abc07d60e054d2 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Nov 2024 00:56:19 -0500 Subject: [PATCH 06/43] Use version checking utility from CLP FFI. --- src/clp_ffi_js/ir/StreamReader.cpp | 61 +++++++++++++----------------- src/clp_ffi_js/ir/StreamReader.hpp | 5 --- 2 files changed, 26 insertions(+), 40 deletions(-) diff --git a/src/clp_ffi_js/ir/StreamReader.cpp b/src/clp_ffi_js/ir/StreamReader.cpp index 98e5cb84..f41d79ff 100644 --- a/src/clp_ffi_js/ir/StreamReader.cpp +++ b/src/clp_ffi_js/ir/StreamReader.cpp @@ -1,6 +1,5 @@ #include "StreamReader.hpp" -#include #include #include #include @@ -167,29 +166,27 @@ auto StreamReader::create(DataArrayTsType const& data_array, ReaderOptions const // Validate the stream's version auto pos = zstd_decompressor->get_pos(); auto const version{get_version(*zstd_decompressor)}; - if (std::ranges::find(cUnstructuredIrVersions, version) != cUnstructuredIrVersions.end()) { - try { + auto const version_validation_result{clp::ffi::ir_stream::validate_protocol_version(version)}; + + try { + if (clp::ffi::ir_stream::IRProtocolErrorCode::Supported == version_validation_result) { + zstd_decompressor->seek_from_begin(0); + return std::make_unique(StructuredIrStreamReader::create( + std::move(zstd_decompressor), + std::move(data_buffer), + reader_options + )); + } + if (clp::ffi::ir_stream::IRProtocolErrorCode::BackwardCompatible + == version_validation_result) + { zstd_decompressor->seek_from_begin(pos); - } catch (ZstdDecompressor::OperationFailed& e) { - throw ClpFfiJsException{ - clp::ErrorCode::ErrorCode_Failure, - __FILENAME__, - __LINE__, - std::format("Unable to rewind zstd decompressor: {}", e.what()) - }; + return std::make_unique(UnstructuredIrStreamReader::create( + std::move(zstd_decompressor), + std::move(data_buffer) + )); } - return std::make_unique(UnstructuredIrStreamReader::create( - std::move(zstd_decompressor), - std::move(data_buffer) - )); - } - // if (clp::ffi::ir_stream::IRProtocolErrorCode_Supported - // == clp::ffi::ir_stream::validate_protocol_version(version)) - // { - // FIXME: wait for https://github.com/y-scope/clp/pull/573 - try { - zstd_decompressor->seek_from_begin(0); - } catch (ZstdDecompressor::OperationFailed& e) { + } catch (ZstdDecompressor::OperationFailed const& e) { throw ClpFfiJsException{ clp::ErrorCode::ErrorCode_Failure, __FILENAME__, @@ -197,18 +194,12 @@ auto StreamReader::create(DataArrayTsType const& data_array, ReaderOptions const std::format("Unable to rewind zstd decompressor: {}", e.what()) }; } - return std::make_unique(StructuredIrStreamReader::create( - std::move(zstd_decompressor), - std::move(data_buffer), - reader_options - )); - // } - - // throw ClpFfiJsException{ - // clp::ErrorCode::ErrorCode_Unsupported, - // __FILENAME__, - // __LINE__, - // std::format("Unable to create reader for IR stream with version {}.", version) - // }; + + throw ClpFfiJsException{ + clp::ErrorCode::ErrorCode_Unsupported, + __FILENAME__, + __LINE__, + std::format("Unable to create reader for IR stream with version {}.", version) + }; } } // namespace clp_ffi_js::ir diff --git a/src/clp_ffi_js/ir/StreamReader.hpp b/src/clp_ffi_js/ir/StreamReader.hpp index 111b110f..51b1e23a 100644 --- a/src/clp_ffi_js/ir/StreamReader.hpp +++ b/src/clp_ffi_js/ir/StreamReader.hpp @@ -1,10 +1,8 @@ #ifndef CLP_FFI_JS_IR_STREAMREADER_HPP #define CLP_FFI_JS_IR_STREAMREADER_HPP -#include #include #include -#include #include #include @@ -19,9 +17,6 @@ EMSCRIPTEN_DECLARE_VAL_TYPE(ReaderOptions); EMSCRIPTEN_DECLARE_VAL_TYPE(DecodedResultsTsType); EMSCRIPTEN_DECLARE_VAL_TYPE(FilteredLogEventMapTsType); -constexpr std::array cUnstructuredIrVersions - = {"v0.0.2", "v0.0.1", "v0.0.0", "0.0.2", "0.0.1", "0.0.0"}; - /** * Class to deserialize and decode Zstandard-compressed CLP IR streams as well as format decoded * log events. From 8a1e9a68da0f1affb21b1de16c71266dbb261be1 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Nov 2024 01:07:13 -0500 Subject: [PATCH 07/43] Add docstrings for class IrUnitHandler. --- .../ir/StructuredIrStreamReader.hpp | 39 +++++++++++++++++-- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp index cd0df16d..eb6f4d89 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp @@ -24,10 +24,17 @@ namespace clp_ffi_js::ir { using parsed_tree_node_id_t = std::optional; /** - * Class to handle deserialized IR units. + * Class to handle deserialized IR units, managing log events, UTC offset changes, schema node + * insertions and End of Stream in a stream. */ class IrUnitHandler { public: + /** + * @param deserialized_log_events Shared pointer to a vector that stores deserialized log + * events. + * @param log_level_key Key name of Key-Value Pair node that contains log level information. + * @param timestamp_key Key name of Key-Value Pair node that contains timestamp information. + */ IrUnitHandler( std::shared_ptr> deserialized_log_events, std::string log_level_key, @@ -37,7 +44,11 @@ class IrUnitHandler { m_log_level_key{std::move(log_level_key)}, m_timestamp_key{std::move(timestamp_key)} {} - // Implements `clp::ffi::ir_stream::IrUnitHandlerInterface` interface + /** + * Handles a log event by inserting it into the deserialized log events vector. + * @param log_event + * @return IRErrorCode::IRErrorCode_Success + */ [[nodiscard]] auto handle_log_event(clp::ffi::KeyValuePairLogEvent&& log_event ) -> clp::ffi::ir_stream::IRErrorCode { m_deserialized_log_events->emplace_back(std::move(log_event)); @@ -45,6 +56,11 @@ class IrUnitHandler { return clp::ffi::ir_stream::IRErrorCode::IRErrorCode_Success; } + /** + * @param utc_offset_old + * @param utc_offset_new + * @return IRErrorCode::IRErrorCode_Success + */ [[nodiscard]] static auto handle_utc_offset_change( [[maybe_unused]] clp::UtcOffset utc_offset_old, [[maybe_unused]] clp::UtcOffset utc_offset_new @@ -54,6 +70,12 @@ class IrUnitHandler { return clp::ffi::ir_stream::IRErrorCode::IRErrorCode_Success; } + /** + * Handles the insertion of a schema tree node by finding node IDs for log level and + * timestamp keys. + * @param schema_tree_node_locator + * @return IRErrorCode::IRErrorCode_Success + */ [[nodiscard]] auto handle_schema_tree_node_insertion( [[maybe_unused]] clp::ffi::SchemaTree::NodeLocator schema_tree_node_locator ) -> clp::ffi::ir_stream::IRErrorCode { @@ -69,29 +91,38 @@ class IrUnitHandler { return clp::ffi::ir_stream::IRErrorCode::IRErrorCode_Success; } + /** + * @return IRErrorCode::IRErrorCode_Success + */ [[nodiscard]] static auto handle_end_of_stream() -> clp::ffi::ir_stream::IRErrorCode { return clp::ffi::ir_stream::IRErrorCode::IRErrorCode_Success; } // Methods + /** + * @return The node ID associated with the log level key. + */ [[nodiscard]] auto get_level_node_id() const -> parsed_tree_node_id_t { return m_level_node_id; } + /** + * @return The node ID associated with the timestamp key. + */ [[nodiscard]] auto get_timestamp_node_id() const -> parsed_tree_node_id_t { return m_timestamp_node_id; } private: + // Variables std::string m_log_level_key; std::string m_timestamp_key; clp::ffi::SchemaTree::Node::id_t m_current_node_id{clp::ffi::SchemaTree::cRootId}; + parsed_tree_node_id_t m_level_node_id; parsed_tree_node_id_t m_timestamp_node_id; - std::shared_ptr> m_deserialized_log_events; - bool m_is_complete{false}; }; using StructuredIrDeserializer = clp::ffi::ir_stream::Deserializer; From 5f005e7c4b9e5a26016d9aafdae20b5e185394db Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Nov 2024 01:47:32 -0500 Subject: [PATCH 08/43] Correct ReaderOptions TS type. --- src/clp_ffi_js/ir/StreamReader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/clp_ffi_js/ir/StreamReader.cpp b/src/clp_ffi_js/ir/StreamReader.cpp index f41d79ff..0d200a36 100644 --- a/src/clp_ffi_js/ir/StreamReader.cpp +++ b/src/clp_ffi_js/ir/StreamReader.cpp @@ -118,7 +118,7 @@ EMSCRIPTEN_BINDINGS(ClpStreamReader) { emscripten::register_type("Uint8Array"); emscripten::register_type("number[] | null"); emscripten::register_type( - "interface{logLevelKey: string, timestampKey: string} | null" + "{logLevelKey: string, timestampKey: string} | null" ); // JS types used as outputs From 958157802047e9aa81b6c2a5ae64ba4b00f792ca Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Nov 2024 02:58:03 -0500 Subject: [PATCH 09/43] Swap timestamp & log_level return order. --- src/clp_ffi_js/ir/StructuredIrStreamReader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp index 5a4ca21c..238baad9 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp @@ -177,8 +177,8 @@ auto StructuredIrStreamReader::decode_range(size_t begin_idx, size_t end_idx, bo { Emval.toValue($0).push([UTF8ToString($1), $2, $3, $4]); }, results.as_handle(), json.value().dump().c_str(), - log_level, timestamp, + log_level, log_event_idx + 1 ); } From dc652ad62617226e14c8395dca63d64c3d7fa1d6 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Nov 2024 03:21:06 -0500 Subject: [PATCH 10/43] Return IR version via `get_ir_protocol_error_code`. --- src/clp_ffi_js/ir/StreamReader.cpp | 10 ++++++++++ src/clp_ffi_js/ir/StreamReader.hpp | 4 ++++ src/clp_ffi_js/ir/StructuredIrStreamReader.hpp | 5 +++++ src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp | 5 +++++ 4 files changed, 24 insertions(+) diff --git a/src/clp_ffi_js/ir/StreamReader.cpp b/src/clp_ffi_js/ir/StreamReader.cpp index 0d200a36..eaaf9c36 100644 --- a/src/clp_ffi_js/ir/StreamReader.cpp +++ b/src/clp_ffi_js/ir/StreamReader.cpp @@ -122,6 +122,12 @@ EMSCRIPTEN_BINDINGS(ClpStreamReader) { ); // JS types used as outputs + emscripten::enum_("IRProtocolErrorCode") + .value("SUPPORTED", clp::ffi::ir_stream::IRProtocolErrorCode::Supported) + .value("BACKWARD_COMPATIBLE", + clp::ffi::ir_stream::IRProtocolErrorCode::BackwardCompatible) + .value("UNSUPPORTED", clp::ffi::ir_stream::IRProtocolErrorCode::Unsupported) + .value("INVALID", clp::ffi::ir_stream::IRProtocolErrorCode::Invalid); emscripten::register_type( "Array<[string, number, number, number]>" ); @@ -131,6 +137,10 @@ EMSCRIPTEN_BINDINGS(ClpStreamReader) { &clp_ffi_js::ir::StreamReader::create, emscripten::return_value_policy::take_ownership() ) + .function( + "getIrProtocolErrorCode", + &clp_ffi_js::ir::StreamReader::get_ir_protocol_error_code + ) .function( "getNumEventsBuffered", &clp_ffi_js::ir::StreamReader::get_num_events_buffered diff --git a/src/clp_ffi_js/ir/StreamReader.hpp b/src/clp_ffi_js/ir/StreamReader.hpp index 51b1e23a..c8aa1268 100644 --- a/src/clp_ffi_js/ir/StreamReader.hpp +++ b/src/clp_ffi_js/ir/StreamReader.hpp @@ -4,6 +4,7 @@ #include #include +#include #include #include @@ -50,6 +51,9 @@ class StreamReader { auto operator=(StreamReader&&) -> StreamReader& = delete; // Methods + [[nodiscard]] virtual auto get_ir_protocol_error_code( + ) const -> clp::ffi::ir_stream::IRProtocolErrorCode = 0; + /** * @return The number of events buffered. */ diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp index eb6f4d89..1e36c1a4 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp @@ -158,6 +158,11 @@ class StructuredIrStreamReader : public StreamReader { ReaderOptions const& reader_options ) -> StructuredIrStreamReader; + [[nodiscard]] auto get_ir_protocol_error_code( + ) const -> clp::ffi::ir_stream::IRProtocolErrorCode override { + return clp::ffi::ir_stream::IRProtocolErrorCode::Supported; + } + [[nodiscard]] auto get_num_events_buffered() const -> size_t override; [[nodiscard]] auto get_filtered_log_event_map() const -> FilteredLogEventMapTsType override; diff --git a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp index 709e9a26..0504a61b 100644 --- a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp +++ b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp @@ -56,6 +56,11 @@ class UnstructuredIrStreamReader : public StreamReader { clp::Array data_array ) -> UnstructuredIrStreamReader; + [[nodiscard]] auto get_ir_protocol_error_code( + ) const -> clp::ffi::ir_stream::IRProtocolErrorCode override { + return clp::ffi::ir_stream::IRProtocolErrorCode::BackwardCompatible; + } + [[nodiscard]] auto get_num_events_buffered() const -> size_t override; [[nodiscard]] auto get_filtered_log_event_map() const -> FilteredLogEventMapTsType override; From 7d1fa0e56f7b48ea853067eb3cdb9f519b713d3e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Nov 2024 03:31:58 -0500 Subject: [PATCH 11/43] Fix lint. --- src/clp_ffi_js/ir/StructuredIrStreamReader.cpp | 2 +- src/clp_ffi_js/ir/StructuredIrStreamReader.hpp | 7 ++++--- src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp | 1 + 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp index 238baad9..7d875d11 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp @@ -30,7 +30,7 @@ namespace clp_ffi_js::ir { using namespace std::literals::string_literals; using clp::ir::four_byte_encoded_variable_t; -constexpr std::string_view cLogLevelFilteringNotSupportedPrompt{ +static constexpr std::string_view cLogLevelFilteringNotSupportedPrompt{ "Log level filtering is not yet supported in this reader." }; diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp index 1e36c1a4..45ca7435 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp @@ -3,9 +3,7 @@ #include #include -#include -#include -#include + #include #include #include @@ -13,7 +11,10 @@ #include #include +#include #include +#include +#include #include #include diff --git a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp index 0504a61b..a444366b 100644 --- a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp +++ b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp @@ -7,6 +7,7 @@ #include #include +#include #include #include #include From c675c61d5f84e98d386f901975a2e70b708b22a5 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Nov 2024 03:35:35 -0500 Subject: [PATCH 12/43] Do not mark schema_tree_node_locator [[maybe_unused]] - Apply suggestions from code review Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- src/clp_ffi_js/ir/StructuredIrStreamReader.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp index 45ca7435..d6e85e4b 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp @@ -78,7 +78,7 @@ class IrUnitHandler { * @return IRErrorCode::IRErrorCode_Success */ [[nodiscard]] auto handle_schema_tree_node_insertion( - [[maybe_unused]] clp::ffi::SchemaTree::NodeLocator schema_tree_node_locator + clp::ffi::SchemaTree::NodeLocator schema_tree_node_locator ) -> clp::ffi::ir_stream::IRErrorCode { ++m_current_node_id; From 5c231822cae2081abade8630019b155102984526 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Nov 2024 17:12:08 -0500 Subject: [PATCH 13/43] Remove unused using string_literals - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- src/clp_ffi_js/ir/StructuredIrStreamReader.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp index 7d875d11..66ed9fe0 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp @@ -26,8 +26,6 @@ #include namespace clp_ffi_js::ir { - -using namespace std::literals::string_literals; using clp::ir::four_byte_encoded_variable_t; static constexpr std::string_view cLogLevelFilteringNotSupportedPrompt{ From a4c2c0746e879d618f1662010163274f3fd3bf4a Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Nov 2024 17:13:46 -0500 Subject: [PATCH 14/43] Use braces for declarations - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- src/clp_ffi_js/ir/StructuredIrStreamReader.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp index 66ed9fe0..d9c4167e 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp @@ -59,12 +59,12 @@ auto StructuredIrStreamReader::create( ) }; } - auto data_context = StreamReaderDataContext( + StreamReaderDataContext data_context{ std::move(data_array), std::move(zstd_decompressor), std::move(result.value()) - ); - return StructuredIrStreamReader(std::move(data_context), std::move(deserialized_log_events)); + }; + return StructuredIrStreamReader{std::move(data_context), std::move(deserialized_log_events)}; } auto StructuredIrStreamReader::get_num_events_buffered() const -> size_t { From b940296b93d8620667b9b0d4991c8412c9201a99 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Nov 2024 17:14:25 -0500 Subject: [PATCH 15/43] Use braces for declarations - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- src/clp_ffi_js/ir/StructuredIrStreamReader.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp index d9c4167e..ee80acb4 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp @@ -40,11 +40,11 @@ auto StructuredIrStreamReader::create( auto deserialized_log_events{std::make_shared>()}; auto result{StructuredIrDeserializer::create( *zstd_decompressor, - IrUnitHandler( + IrUnitHandler{ deserialized_log_events, reader_options["logLevelKey"].as(), reader_options["timestampKey"].as() - ) + } )}; if (result.has_error()) { auto const error_code{result.error()}; From 1087f9457973b6c1e09cefdca685dd1e6f3d3dc1 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Nov 2024 17:15:24 -0500 Subject: [PATCH 16/43] Error messages - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- src/clp_ffi_js/ir/StructuredIrStreamReader.cpp | 4 ++-- src/clp_ffi_js/ir/StructuredIrStreamReader.hpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp index ee80acb4..457e12be 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp @@ -111,7 +111,7 @@ auto StructuredIrStreamReader::deserialize_stream() -> size_t { __FILENAME__, __LINE__, std::format( - "Failed to deserialize: {}:{}", + "Failed to deserialize IR unit: {}:{}", error.category().name(), error.message() ) @@ -148,7 +148,7 @@ auto StructuredIrStreamReader::decode_range(size_t begin_idx, size_t end_idx, bo auto const json{log_event.serialize_to_json()}; if (false == json.has_value()) { - SPDLOG_ERROR("Failed to decode message."); + SPDLOG_ERROR("Failed to decode log event."); break; } diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp index d6e85e4b..f2693659 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp @@ -66,7 +66,7 @@ class IrUnitHandler { [[maybe_unused]] clp::UtcOffset utc_offset_old, [[maybe_unused]] clp::UtcOffset utc_offset_new ) -> clp::ffi::ir_stream::IRErrorCode { - SPDLOG_WARN("UTC offset change packets are currently not handled."); + SPDLOG_WARN("UTC offset change packets aren't handled currently."); return clp::ffi::ir_stream::IRErrorCode::IRErrorCode_Success; } From 5e8d42a1350eee6e111f02c9ec08a6cb5581a026 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Nov 2024 17:15:48 -0500 Subject: [PATCH 17/43] Remove dead code - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- src/clp_ffi_js/ir/StructuredIrStreamReader.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp index 457e12be..4ddcaf42 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp @@ -138,9 +138,6 @@ auto StructuredIrStreamReader::decode_range(size_t begin_idx, size_t end_idx, bo return DecodedResultsTsType{emscripten::val::null()}; } - std::string message; - constexpr size_t cDefaultReservedMessageLength{512}; - message.reserve(cDefaultReservedMessageLength); auto const results{emscripten::val::array()}; for (size_t log_event_idx = begin_idx; log_event_idx < end_idx; ++log_event_idx) { From 570b8eb2556762c7153fc41170738f60b34124e5 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Nov 2024 17:16:31 -0500 Subject: [PATCH 18/43] Add clp/ prefix in includes - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- src/clp_ffi_js/ir/StructuredIrStreamReader.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp index f2693659..fabb7f39 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp @@ -1,13 +1,13 @@ #ifndef CLP_FFI_JS_IR_STRUCTUREDIRSTREAMREADER_HPP #define CLP_FFI_JS_IR_STRUCTUREDIRSTREAMREADER_HPP -#include +#include #include #include #include #include -#include +#include #include #include From 59b163bc8241e182900e1cb965558cf88b0ae82c Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Nov 2024 17:18:37 -0500 Subject: [PATCH 19/43] Docs - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- .../ir/StructuredIrStreamReader.hpp | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp index fabb7f39..a5065078 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp @@ -25,16 +25,17 @@ namespace clp_ffi_js::ir { using parsed_tree_node_id_t = std::optional; /** - * Class to handle deserialized IR units, managing log events, UTC offset changes, schema node - * insertions and End of Stream in a stream. + * Class that implements the `clp::ffi::ir_stream::IrUnitHandlerInterface` to buffer log events and + * determine the schema-tree node IDs of the timestamp and log level kv-pairs. */ class IrUnitHandler { public: /** - * @param deserialized_log_events Shared pointer to a vector that stores deserialized log - * events. - * @param log_level_key Key name of Key-Value Pair node that contains log level information. - * @param timestamp_key Key name of Key-Value Pair node that contains timestamp information. + * @param deserialized_log_events The vector in which to store deserialized log events. + * @param log_level_key Key name of the schema-tree node that contains the authoritative log + * level for events. + * @param timestamp_key Key name of schema-tree node that contains the authoritative timestamp + * for events. */ IrUnitHandler( std::shared_ptr> deserialized_log_events, @@ -45,8 +46,9 @@ class IrUnitHandler { m_log_level_key{std::move(log_level_key)}, m_timestamp_key{std::move(timestamp_key)} {} + // Methods implementing `clp::ffi::ir_stream::IrUnitHandlerInterface`. /** - * Handles a log event by inserting it into the deserialized log events vector. + * Buffers the log event. * @param log_event * @return IRErrorCode::IRErrorCode_Success */ @@ -72,8 +74,8 @@ class IrUnitHandler { } /** - * Handles the insertion of a schema tree node by finding node IDs for log level and - * timestamp keys. + * Saves the node's ID if it corresponds to events' authoritative log level or timestamp + * kv-pairs. * @param schema_tree_node_locator * @return IRErrorCode::IRErrorCode_Success */ @@ -101,14 +103,14 @@ class IrUnitHandler { // Methods /** - * @return The node ID associated with the log level key. + * @return The schema-tree node ID associated with events' authoritative log-level key. */ [[nodiscard]] auto get_level_node_id() const -> parsed_tree_node_id_t { return m_level_node_id; } /** - * @return The node ID associated with the timestamp key. + * @return The schema-tree node ID associated with events' authoritative timestamp key. */ [[nodiscard]] auto get_timestamp_node_id() const -> parsed_tree_node_id_t { return m_timestamp_node_id; @@ -150,6 +152,7 @@ class StructuredIrStreamReader : public StreamReader { * @param zstd_decompressor A decompressor for an IR stream, where the read head of the stream * is just after the stream's encoding type. * @param data_array The array backing `zstd_decompressor`. + * @param reader_options * @return The created instance. * @throw ClpFfiJsException if any error occurs. */ From 92e44b3bd01345e604a085f633ef73409294e94d Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Nov 2024 17:19:47 -0500 Subject: [PATCH 20/43] Add docs about potential shared_ptr replacement with gsl::not_null - Apply suggestions from code review --- src/clp_ffi_js/ir/StructuredIrStreamReader.hpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp index a5065078..1d1cfac6 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp @@ -125,6 +125,10 @@ class IrUnitHandler { parsed_tree_node_id_t m_level_node_id; parsed_tree_node_id_t m_timestamp_node_id; + + // TODO: Technically, we don't need to use a `shared_ptr` since the parent stream reader will + // have a longer lifetime than this class. Instead, we could use `gsl::not_null` once we add + // `gsl` into the project. std::shared_ptr> m_deserialized_log_events; }; From d1704345373cf823e139754dcbd62cf98dca444e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Nov 2024 17:10:09 -0500 Subject: [PATCH 21/43] Rename get_ir_protocol_error_code -> get_ir_stream_type; return IrStreamType instead of IRProtocolErrorCode. --- src/clp_ffi_js/ir/StreamReader.cpp | 14 ++++---------- src/clp_ffi_js/ir/StreamReader.hpp | 9 +++++++-- src/clp_ffi_js/ir/StructuredIrStreamReader.hpp | 6 ++---- src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp | 6 ++---- 4 files changed, 15 insertions(+), 20 deletions(-) diff --git a/src/clp_ffi_js/ir/StreamReader.cpp b/src/clp_ffi_js/ir/StreamReader.cpp index eaaf9c36..220aa902 100644 --- a/src/clp_ffi_js/ir/StreamReader.cpp +++ b/src/clp_ffi_js/ir/StreamReader.cpp @@ -122,12 +122,9 @@ EMSCRIPTEN_BINDINGS(ClpStreamReader) { ); // JS types used as outputs - emscripten::enum_("IRProtocolErrorCode") - .value("SUPPORTED", clp::ffi::ir_stream::IRProtocolErrorCode::Supported) - .value("BACKWARD_COMPATIBLE", - clp::ffi::ir_stream::IRProtocolErrorCode::BackwardCompatible) - .value("UNSUPPORTED", clp::ffi::ir_stream::IRProtocolErrorCode::Unsupported) - .value("INVALID", clp::ffi::ir_stream::IRProtocolErrorCode::Invalid); + emscripten::enum_("IrStreamType") + .value("STRUCTURED", clp_ffi_js::ir::IrStreamType::Structured) + .value("UNSTRUCTURED", clp_ffi_js::ir::IrStreamType::Unstructured); emscripten::register_type( "Array<[string, number, number, number]>" ); @@ -137,10 +134,7 @@ EMSCRIPTEN_BINDINGS(ClpStreamReader) { &clp_ffi_js::ir::StreamReader::create, emscripten::return_value_policy::take_ownership() ) - .function( - "getIrProtocolErrorCode", - &clp_ffi_js::ir::StreamReader::get_ir_protocol_error_code - ) + .function("getIrStreamType", &clp_ffi_js::ir::StreamReader::get_ir_stream_type) .function( "getNumEventsBuffered", &clp_ffi_js::ir::StreamReader::get_num_events_buffered diff --git a/src/clp_ffi_js/ir/StreamReader.hpp b/src/clp_ffi_js/ir/StreamReader.hpp index c8aa1268..4c80629d 100644 --- a/src/clp_ffi_js/ir/StreamReader.hpp +++ b/src/clp_ffi_js/ir/StreamReader.hpp @@ -2,6 +2,7 @@ #define CLP_FFI_JS_IR_STREAMREADER_HPP #include +#include #include #include @@ -18,6 +19,11 @@ EMSCRIPTEN_DECLARE_VAL_TYPE(ReaderOptions); EMSCRIPTEN_DECLARE_VAL_TYPE(DecodedResultsTsType); EMSCRIPTEN_DECLARE_VAL_TYPE(FilteredLogEventMapTsType); +enum class IrStreamType : uint8_t { + Structured, + Unstructured, +}; + /** * Class to deserialize and decode Zstandard-compressed CLP IR streams as well as format decoded * log events. @@ -51,8 +57,7 @@ class StreamReader { auto operator=(StreamReader&&) -> StreamReader& = delete; // Methods - [[nodiscard]] virtual auto get_ir_protocol_error_code( - ) const -> clp::ffi::ir_stream::IRProtocolErrorCode = 0; + [[nodiscard]] virtual auto get_ir_stream_type() const -> IrStreamType = 0; /** * @return The number of events buffered. diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp index 1d1cfac6..ef944807 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp @@ -3,7 +3,6 @@ #include #include - #include #include #include @@ -166,9 +165,8 @@ class StructuredIrStreamReader : public StreamReader { ReaderOptions const& reader_options ) -> StructuredIrStreamReader; - [[nodiscard]] auto get_ir_protocol_error_code( - ) const -> clp::ffi::ir_stream::IRProtocolErrorCode override { - return clp::ffi::ir_stream::IRProtocolErrorCode::Supported; + [[nodiscard]] auto get_ir_stream_type() const -> IrStreamType override { + return IrStreamType::Structured; } [[nodiscard]] auto get_num_events_buffered() const -> size_t override; diff --git a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp index a444366b..9f921df4 100644 --- a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp +++ b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp @@ -7,7 +7,6 @@ #include #include -#include #include #include #include @@ -57,9 +56,8 @@ class UnstructuredIrStreamReader : public StreamReader { clp::Array data_array ) -> UnstructuredIrStreamReader; - [[nodiscard]] auto get_ir_protocol_error_code( - ) const -> clp::ffi::ir_stream::IRProtocolErrorCode override { - return clp::ffi::ir_stream::IRProtocolErrorCode::BackwardCompatible; + [[nodiscard]] auto get_ir_stream_type() const -> IrStreamType override { + return IrStreamType::Unstructured; } [[nodiscard]] auto get_num_events_buffered() const -> size_t override; From b1104798c7a3a7e3afb5ab5ff39bd6827fe2a849 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Nov 2024 17:40:35 -0500 Subject: [PATCH 22/43] Move version_validation_result into try {}; update docs. --- src/clp_ffi_js/ir/StreamReader.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/clp_ffi_js/ir/StreamReader.cpp b/src/clp_ffi_js/ir/StreamReader.cpp index 220aa902..306b08a7 100644 --- a/src/clp_ffi_js/ir/StreamReader.cpp +++ b/src/clp_ffi_js/ir/StreamReader.cpp @@ -167,12 +167,11 @@ auto StreamReader::create(DataArrayTsType const& data_array, ReaderOptions const rewind_reader_and_validate_encoding_type(*zstd_decompressor); - // Validate the stream's version + // Validate the stream's version and decide which type of IR stream reader to create. auto pos = zstd_decompressor->get_pos(); auto const version{get_version(*zstd_decompressor)}; - auto const version_validation_result{clp::ffi::ir_stream::validate_protocol_version(version)}; - try { + auto const version_validation_result{clp::ffi::ir_stream::validate_protocol_version(version)}; if (clp::ffi::ir_stream::IRProtocolErrorCode::Supported == version_validation_result) { zstd_decompressor->seek_from_begin(0); return std::make_unique(StructuredIrStreamReader::create( From 181206e966071caefea1bb2654a4e5d6ecc85ea3 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Nov 2024 17:41:14 -0500 Subject: [PATCH 23/43] Remove redundant docs for get_deserializer and get_reader. --- src/clp_ffi_js/ir/StreamReaderDataContext.hpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/clp_ffi_js/ir/StreamReaderDataContext.hpp b/src/clp_ffi_js/ir/StreamReaderDataContext.hpp index b28cead4..536c6b2b 100644 --- a/src/clp_ffi_js/ir/StreamReaderDataContext.hpp +++ b/src/clp_ffi_js/ir/StreamReaderDataContext.hpp @@ -39,14 +39,8 @@ class StreamReaderDataContext { ~StreamReaderDataContext() = default; // Methods - /** - * @return A reference to the deserializer. - */ [[nodiscard]] auto get_deserializer() -> Deserializer& { return m_deserializer; } - /** - * @return A reference to the reader. - */ [[nodiscard]] auto get_reader() -> clp::ReaderInterface& { return *m_reader; } private: From cac21ee83653edb36a2aa2a5f72f1a9a2f5afda3 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Nov 2024 17:42:04 -0500 Subject: [PATCH 24/43] Rename cLogLevelFilteringNotSupportedPrompt -> cLogLevelFilteringNotSupportedErrorMsg. --- src/clp_ffi_js/ir/StructuredIrStreamReader.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp index 4ddcaf42..14dddf83 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp @@ -28,7 +28,7 @@ namespace clp_ffi_js::ir { using clp::ir::four_byte_encoded_variable_t; -static constexpr std::string_view cLogLevelFilteringNotSupportedPrompt{ +static constexpr std::string_view cLogLevelFilteringNotSupportedErrorMsg{ "Log level filtering is not yet supported in this reader." }; @@ -72,7 +72,7 @@ auto StructuredIrStreamReader::get_num_events_buffered() const -> size_t { } auto StructuredIrStreamReader::get_filtered_log_event_map() const -> FilteredLogEventMapTsType { - SPDLOG_ERROR(cLogLevelFilteringNotSupportedPrompt); + SPDLOG_ERROR(cLogLevelFilteringNotSupportedErrorMsg); return FilteredLogEventMapTsType{emscripten::val::null()}; } @@ -80,7 +80,7 @@ void StructuredIrStreamReader::filter_log_events(LogLevelFilterTsType const& log if (log_level_filter.isNull()) { return; } - SPDLOG_ERROR(cLogLevelFilteringNotSupportedPrompt); + SPDLOG_ERROR(cLogLevelFilteringNotSupportedErrorMsg); } auto StructuredIrStreamReader::deserialize_stream() -> size_t { @@ -130,7 +130,7 @@ auto StructuredIrStreamReader::deserialize_stream() -> size_t { auto StructuredIrStreamReader::decode_range(size_t begin_idx, size_t end_idx, bool use_filter) const -> DecodedResultsTsType { if (use_filter) { - SPDLOG_ERROR(cLogLevelFilteringNotSupportedPrompt); + SPDLOG_ERROR(cLogLevelFilteringNotSupportedErrorMsg); return DecodedResultsTsType{emscripten::val::null()}; } From b6d837cc37acd97260edbd220a0dc827b1668b05 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Nov 2024 17:44:48 -0500 Subject: [PATCH 25/43] Add const for cReaderOptionsLogLevelKey and cReaderOptionsTimestampKey. --- src/clp_ffi_js/ir/StructuredIrStreamReader.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp index 14dddf83..bacfb3fd 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp @@ -31,6 +31,8 @@ using clp::ir::four_byte_encoded_variable_t; static constexpr std::string_view cLogLevelFilteringNotSupportedErrorMsg{ "Log level filtering is not yet supported in this reader." }; +static constexpr std::string_view cReaderOptionsLogLevelKey{"logLevelKey"}; +static constexpr std::string_view cReaderOptionsTimestampKey{"timestampKey"}; auto StructuredIrStreamReader::create( std::unique_ptr&& zstd_decompressor, @@ -42,8 +44,8 @@ auto StructuredIrStreamReader::create( *zstd_decompressor, IrUnitHandler{ deserialized_log_events, - reader_options["logLevelKey"].as(), - reader_options["timestampKey"].as() + reader_options[cReaderOptionsLogLevelKey].as(), + reader_options[cReaderOptionsTimestampKey].as() } )}; if (result.has_error()) { From 11b347bac20635d77593164a858629ce05f69846 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Nov 2024 18:23:13 -0500 Subject: [PATCH 26/43] Use const instead of magic strings. --- src/clp_ffi_js/ir/StructuredIrStreamReader.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp index bacfb3fd..2097946a 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp @@ -44,8 +44,8 @@ auto StructuredIrStreamReader::create( *zstd_decompressor, IrUnitHandler{ deserialized_log_events, - reader_options[cReaderOptionsLogLevelKey].as(), - reader_options[cReaderOptionsTimestampKey].as() + reader_options[cReaderOptionsLogLevelKey.data()].as(), + reader_options[cReaderOptionsTimestampKey.data()].as() } )}; if (result.has_error()) { From 6588d566eea6a6675d14477cbaee7eb0a71be178 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Nov 2024 18:23:20 -0500 Subject: [PATCH 27/43] Reformat code. --- src/clp_ffi_js/ir/StreamReader.cpp | 3 ++- src/clp_ffi_js/ir/StructuredIrStreamReader.hpp | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/clp_ffi_js/ir/StreamReader.cpp b/src/clp_ffi_js/ir/StreamReader.cpp index 306b08a7..73749ec9 100644 --- a/src/clp_ffi_js/ir/StreamReader.cpp +++ b/src/clp_ffi_js/ir/StreamReader.cpp @@ -171,7 +171,8 @@ auto StreamReader::create(DataArrayTsType const& data_array, ReaderOptions const auto pos = zstd_decompressor->get_pos(); auto const version{get_version(*zstd_decompressor)}; try { - auto const version_validation_result{clp::ffi::ir_stream::validate_protocol_version(version)}; + auto const version_validation_result{clp::ffi::ir_stream::validate_protocol_version(version) + }; if (clp::ffi::ir_stream::IRProtocolErrorCode::Supported == version_validation_result) { zstd_decompressor->seek_from_begin(0); return std::make_unique(StructuredIrStreamReader::create( diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp index ef944807..eeaa9327 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp @@ -1,19 +1,19 @@ #ifndef CLP_FFI_JS_IR_STRUCTUREDIRSTREAMREADER_HPP #define CLP_FFI_JS_IR_STRUCTUREDIRSTREAMREADER_HPP -#include #include #include #include #include -#include #include #include +#include #include #include #include #include +#include #include #include From 2bca1cc34a5b84ae110ee1ca8191760b06d70b0f Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Nov 2024 18:26:23 -0500 Subject: [PATCH 28/43] Rename `json` -> `json_result`. --- src/clp_ffi_js/ir/StructuredIrStreamReader.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp index 2097946a..bed56a7c 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp @@ -145,8 +145,8 @@ auto StructuredIrStreamReader::decode_range(size_t begin_idx, size_t end_idx, bo for (size_t log_event_idx = begin_idx; log_event_idx < end_idx; ++log_event_idx) { auto const& log_event{m_deserialized_log_events->at(log_event_idx)}; - auto const json{log_event.serialize_to_json()}; - if (false == json.has_value()) { + auto const json_result{log_event.serialize_to_json()}; + if (false == json_result.has_value()) { SPDLOG_ERROR("Failed to decode log event."); break; } @@ -173,7 +173,7 @@ auto StructuredIrStreamReader::decode_range(size_t begin_idx, size_t end_idx, bo EM_ASM( { Emval.toValue($0).push([UTF8ToString($1), $2, $3, $4]); }, results.as_handle(), - json.value().dump().c_str(), + json_result.value().dump().c_str(), timestamp, log_level, log_event_idx + 1 From 94f31af809f3a9a4f31c9090987e4e9e7b7246a5 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Nov 2024 18:30:30 -0500 Subject: [PATCH 29/43] Reorder member variable initializations to match their declaration order. --- src/clp_ffi_js/ir/StructuredIrStreamReader.cpp | 6 +++--- src/clp_ffi_js/ir/StructuredIrStreamReader.hpp | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp index bed56a7c..ee3d3c04 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp @@ -187,9 +187,9 @@ StructuredIrStreamReader::StructuredIrStreamReader( StreamReaderDataContext&& stream_reader_data_context, std::shared_ptr> deserialized_log_events ) - : m_stream_reader_data_context{std::make_unique< + : m_deserialized_log_events{std::move(deserialized_log_events)}, + m_stream_reader_data_context{std::make_unique< StreamReaderDataContext>( std::move(stream_reader_data_context) - )}, - m_deserialized_log_events{std::move(deserialized_log_events)} {} + )} {} } // namespace clp_ffi_js::ir diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp index eeaa9327..56b337da 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp @@ -41,9 +41,9 @@ class IrUnitHandler { std::string log_level_key, std::string timestamp_key ) - : m_deserialized_log_events{std::move(deserialized_log_events)}, - m_log_level_key{std::move(log_level_key)}, - m_timestamp_key{std::move(timestamp_key)} {} + : m_log_level_key{std::move(log_level_key)}, + m_timestamp_key{std::move(timestamp_key)}, + m_deserialized_log_events{std::move(deserialized_log_events)} {} // Methods implementing `clp::ffi::ir_stream::IrUnitHandlerInterface`. /** From 072c7bc2398657c96c3460f08f87a29b674eb1ee Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Nov 2024 18:31:20 -0500 Subject: [PATCH 30/43] Rename parsed_tree_node_id_t -> schema_tree_node_id_t. --- src/clp_ffi_js/ir/StructuredIrStreamReader.hpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp index 56b337da..764e8663 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp @@ -21,7 +21,7 @@ #include namespace clp_ffi_js::ir { -using parsed_tree_node_id_t = std::optional; +using schema_tree_node_id_t = std::optional; /** * Class that implements the `clp::ffi::ir_stream::IrUnitHandlerInterface` to buffer log events and @@ -104,14 +104,14 @@ class IrUnitHandler { /** * @return The schema-tree node ID associated with events' authoritative log-level key. */ - [[nodiscard]] auto get_level_node_id() const -> parsed_tree_node_id_t { + [[nodiscard]] auto get_level_node_id() const -> schema_tree_node_id_t { return m_level_node_id; } /** * @return The schema-tree node ID associated with events' authoritative timestamp key. */ - [[nodiscard]] auto get_timestamp_node_id() const -> parsed_tree_node_id_t { + [[nodiscard]] auto get_timestamp_node_id() const -> schema_tree_node_id_t { return m_timestamp_node_id; } @@ -122,8 +122,8 @@ class IrUnitHandler { clp::ffi::SchemaTree::Node::id_t m_current_node_id{clp::ffi::SchemaTree::cRootId}; - parsed_tree_node_id_t m_level_node_id; - parsed_tree_node_id_t m_timestamp_node_id; + schema_tree_node_id_t m_level_node_id; + schema_tree_node_id_t m_timestamp_node_id; // TODO: Technically, we don't need to use a `shared_ptr` since the parent stream reader will // have a longer lifetime than this class. Instead, we could use `gsl::not_null` once we add @@ -198,8 +198,8 @@ class StructuredIrStreamReader : public StreamReader { std::shared_ptr> m_deserialized_log_events; std::unique_ptr> m_stream_reader_data_context; - parsed_tree_node_id_t m_level_node_id; - parsed_tree_node_id_t m_timestamp_node_id; + schema_tree_node_id_t m_level_node_id; + schema_tree_node_id_t m_timestamp_node_id; }; } // namespace clp_ffi_js::ir From 08e0bddb24ba294026a7b12335b4bb73318284d0 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Nov 2024 18:38:26 -0500 Subject: [PATCH 31/43] Rename level_node_id -> log_level_node_id. --- src/clp_ffi_js/ir/StructuredIrStreamReader.cpp | 2 +- src/clp_ffi_js/ir/StructuredIrStreamReader.hpp | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp index ee3d3c04..ad63c09e 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp @@ -121,7 +121,7 @@ auto StructuredIrStreamReader::deserialize_stream() -> size_t { } m_level_node_id = m_stream_reader_data_context->get_deserializer() .get_ir_unit_handler() - .get_level_node_id(); + .get_log_level_node_id(); m_timestamp_node_id = m_stream_reader_data_context->get_deserializer() .get_ir_unit_handler() .get_timestamp_node_id(); diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp index 764e8663..5cceab65 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp @@ -85,7 +85,7 @@ class IrUnitHandler { auto const& key_name{schema_tree_node_locator.get_key_name()}; if (m_log_level_key == key_name) { - m_level_node_id.emplace(m_current_node_id); + m_log_level_node_id.emplace(m_current_node_id); } else if (m_timestamp_key == key_name) { m_timestamp_node_id.emplace(m_current_node_id); } @@ -104,8 +104,8 @@ class IrUnitHandler { /** * @return The schema-tree node ID associated with events' authoritative log-level key. */ - [[nodiscard]] auto get_level_node_id() const -> schema_tree_node_id_t { - return m_level_node_id; + [[nodiscard]] auto get_log_level_node_id() const -> schema_tree_node_id_t { + return m_log_level_node_id; } /** @@ -122,7 +122,7 @@ class IrUnitHandler { clp::ffi::SchemaTree::Node::id_t m_current_node_id{clp::ffi::SchemaTree::cRootId}; - schema_tree_node_id_t m_level_node_id; + schema_tree_node_id_t m_log_level_node_id; schema_tree_node_id_t m_timestamp_node_id; // TODO: Technically, we don't need to use a `shared_ptr` since the parent stream reader will From 95b81023abae32c7b2208c40784ed8bc01058cff Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Nov 2024 18:40:27 -0500 Subject: [PATCH 32/43] Move StructuredIrStreamReader::create declaration to the top of the class since this is static. --- .../ir/StructuredIrStreamReader.hpp | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp index 5cceab65..5c64f020 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp @@ -139,18 +139,6 @@ using StructuredIrDeserializer = clp::ffi::ir_stream::Deserializer StructuredIrStreamReader& = delete; - - // Define default move constructor - StructuredIrStreamReader(StructuredIrStreamReader&&) = default; - // Delete move assignment operator since it's also disabled in `clp::ir::LogEventDeserializer`. - auto operator=(StructuredIrStreamReader&&) -> StructuredIrStreamReader& = delete; - /** * @param zstd_decompressor A decompressor for an IR stream, where the read head of the stream * is just after the stream's encoding type. @@ -165,6 +153,18 @@ class StructuredIrStreamReader : public StreamReader { ReaderOptions const& reader_options ) -> StructuredIrStreamReader; + // Destructor + ~StructuredIrStreamReader() override = default; + + // Disable copy constructor and assignment operator + StructuredIrStreamReader(StructuredIrStreamReader const&) = delete; + auto operator=(StructuredIrStreamReader const&) -> StructuredIrStreamReader& = delete; + + // Define default move constructor + StructuredIrStreamReader(StructuredIrStreamReader&&) = default; + // Delete move assignment operator since it's also disabled in `clp::ir::LogEventDeserializer`. + auto operator=(StructuredIrStreamReader&&) -> StructuredIrStreamReader& = delete; + [[nodiscard]] auto get_ir_stream_type() const -> IrStreamType override { return IrStreamType::Structured; } From 37e6f3073bab27b37a6b64ead56b125b84d2bc30 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Nov 2024 18:47:36 -0500 Subject: [PATCH 33/43] Remove all log level handling in StructuredIrStreamReader. --- src/clp_ffi_js/ir/StreamReader.cpp | 2 +- .../ir/StructuredIrStreamReader.cpp | 17 +------------ .../ir/StructuredIrStreamReader.hpp | 25 +++---------------- 3 files changed, 6 insertions(+), 38 deletions(-) diff --git a/src/clp_ffi_js/ir/StreamReader.cpp b/src/clp_ffi_js/ir/StreamReader.cpp index 73749ec9..81d3eee3 100644 --- a/src/clp_ffi_js/ir/StreamReader.cpp +++ b/src/clp_ffi_js/ir/StreamReader.cpp @@ -118,7 +118,7 @@ EMSCRIPTEN_BINDINGS(ClpStreamReader) { emscripten::register_type("Uint8Array"); emscripten::register_type("number[] | null"); emscripten::register_type( - "{logLevelKey: string, timestampKey: string} | null" + "{timestampKey: string} | null" ); // JS types used as outputs diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp index ad63c09e..1f4676c4 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp @@ -31,7 +31,6 @@ using clp::ir::four_byte_encoded_variable_t; static constexpr std::string_view cLogLevelFilteringNotSupportedErrorMsg{ "Log level filtering is not yet supported in this reader." }; -static constexpr std::string_view cReaderOptionsLogLevelKey{"logLevelKey"}; static constexpr std::string_view cReaderOptionsTimestampKey{"timestampKey"}; auto StructuredIrStreamReader::create( @@ -44,7 +43,6 @@ auto StructuredIrStreamReader::create( *zstd_decompressor, IrUnitHandler{ deserialized_log_events, - reader_options[cReaderOptionsLogLevelKey.data()].as(), reader_options[cReaderOptionsTimestampKey.data()].as() } )}; @@ -119,9 +117,6 @@ auto StructuredIrStreamReader::deserialize_stream() -> size_t { ) }; } - m_level_node_id = m_stream_reader_data_context->get_deserializer() - .get_ir_unit_handler() - .get_log_level_node_id(); m_timestamp_node_id = m_stream_reader_data_context->get_deserializer() .get_ir_unit_handler() .get_timestamp_node_id(); @@ -152,16 +147,6 @@ auto StructuredIrStreamReader::decode_range(size_t begin_idx, size_t end_idx, bo } auto const& id_value_pairs{log_event.get_node_id_value_pairs()}; - LogLevel log_level{LogLevel::NONE}; - if (m_level_node_id.has_value()) { - auto const& log_level_pair{id_value_pairs.at(m_level_node_id.value())}; - log_level = log_level_pair.has_value() - ? static_cast( - log_level_pair.value() - .get_immutable_view() - ) - : log_level; - } clp::ffi::value_int_t timestamp{0}; if (m_timestamp_node_id.has_value()) { auto const& timestamp_pair{id_value_pairs.at(m_timestamp_node_id.value())}; @@ -175,7 +160,7 @@ auto StructuredIrStreamReader::decode_range(size_t begin_idx, size_t end_idx, bo results.as_handle(), json_result.value().dump().c_str(), timestamp, - log_level, + LogLevel::NONE, log_event_idx + 1 ); } diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp index 5c64f020..c728c1e9 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp @@ -25,24 +25,20 @@ using schema_tree_node_id_t = std::optional; /** * Class that implements the `clp::ffi::ir_stream::IrUnitHandlerInterface` to buffer log events and - * determine the schema-tree node IDs of the timestamp and log level kv-pairs. + * determine the schema-tree node ID of the timestamp kv-pair. */ class IrUnitHandler { public: /** * @param deserialized_log_events The vector in which to store deserialized log events. - * @param log_level_key Key name of the schema-tree node that contains the authoritative log - * level for events. * @param timestamp_key Key name of schema-tree node that contains the authoritative timestamp * for events. */ IrUnitHandler( std::shared_ptr> deserialized_log_events, - std::string log_level_key, std::string timestamp_key ) - : m_log_level_key{std::move(log_level_key)}, - m_timestamp_key{std::move(timestamp_key)}, + : m_timestamp_key{std::move(timestamp_key)}, m_deserialized_log_events{std::move(deserialized_log_events)} {} // Methods implementing `clp::ffi::ir_stream::IrUnitHandlerInterface`. @@ -73,8 +69,7 @@ class IrUnitHandler { } /** - * Saves the node's ID if it corresponds to events' authoritative log level or timestamp - * kv-pairs. + * Saves the node's ID if it corresponds to events' authoritative timestamp kv-pair. * @param schema_tree_node_locator * @return IRErrorCode::IRErrorCode_Success */ @@ -84,9 +79,7 @@ class IrUnitHandler { ++m_current_node_id; auto const& key_name{schema_tree_node_locator.get_key_name()}; - if (m_log_level_key == key_name) { - m_log_level_node_id.emplace(m_current_node_id); - } else if (m_timestamp_key == key_name) { + if (m_timestamp_key == key_name) { m_timestamp_node_id.emplace(m_current_node_id); } @@ -101,13 +94,6 @@ class IrUnitHandler { } // Methods - /** - * @return The schema-tree node ID associated with events' authoritative log-level key. - */ - [[nodiscard]] auto get_log_level_node_id() const -> schema_tree_node_id_t { - return m_log_level_node_id; - } - /** * @return The schema-tree node ID associated with events' authoritative timestamp key. */ @@ -117,12 +103,10 @@ class IrUnitHandler { private: // Variables - std::string m_log_level_key; std::string m_timestamp_key; clp::ffi::SchemaTree::Node::id_t m_current_node_id{clp::ffi::SchemaTree::cRootId}; - schema_tree_node_id_t m_log_level_node_id; schema_tree_node_id_t m_timestamp_node_id; // TODO: Technically, we don't need to use a `shared_ptr` since the parent stream reader will @@ -198,7 +182,6 @@ class StructuredIrStreamReader : public StreamReader { std::shared_ptr> m_deserialized_log_events; std::unique_ptr> m_stream_reader_data_context; - schema_tree_node_id_t m_level_node_id; schema_tree_node_id_t m_timestamp_node_id; }; } // namespace clp_ffi_js::ir From 058ec7805c524bbf31902146c2a3583dd41efb3f Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Nov 2024 18:57:42 -0500 Subject: [PATCH 34/43] Type check timestamp value before calling get_immutable_view. --- src/clp_ffi_js/ir/StructuredIrStreamReader.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp index 1f4676c4..16c3074e 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp @@ -150,9 +150,14 @@ auto StructuredIrStreamReader::decode_range(size_t begin_idx, size_t end_idx, bo clp::ffi::value_int_t timestamp{0}; if (m_timestamp_node_id.has_value()) { auto const& timestamp_pair{id_value_pairs.at(m_timestamp_node_id.value())}; - timestamp = timestamp_pair.has_value() - ? timestamp_pair.value().get_immutable_view() - : timestamp; + if (timestamp_pair.has_value()) { + if (timestamp_pair->is()) { + timestamp = timestamp_pair.value().get_immutable_view(); + } else { + // TODO: add support for parsing timestamp values of string type. + SPDLOG_ERROR("Unable to parse timestamp for log_event_idx={}", log_event_idx); + } + } } EM_ASM( From 4d2c0760d1660c57a3d40cf2afd699d6b459380d Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Nov 2024 19:05:29 -0500 Subject: [PATCH 35/43] Put empty json string when unable to decode. --- src/clp_ffi_js/ir/StructuredIrStreamReader.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp index 16c3074e..6a2a627f 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp @@ -28,6 +28,7 @@ namespace clp_ffi_js::ir { using clp::ir::four_byte_encoded_variable_t; +static constexpr std::string_view cEmptyJsonStr{"{}"}; static constexpr std::string_view cLogLevelFilteringNotSupportedErrorMsg{ "Log level filtering is not yet supported in this reader." }; @@ -141,9 +142,11 @@ auto StructuredIrStreamReader::decode_range(size_t begin_idx, size_t end_idx, bo auto const& log_event{m_deserialized_log_events->at(log_event_idx)}; auto const json_result{log_event.serialize_to_json()}; + std::string json_str{cEmptyJsonStr}; if (false == json_result.has_value()) { SPDLOG_ERROR("Failed to decode log event."); - break; + } else { + json_str = json_result.value().dump(); } auto const& id_value_pairs{log_event.get_node_id_value_pairs()}; @@ -163,7 +166,7 @@ auto StructuredIrStreamReader::decode_range(size_t begin_idx, size_t end_idx, bo EM_ASM( { Emval.toValue($0).push([UTF8ToString($1), $2, $3, $4]); }, results.as_handle(), - json_result.value().dump().c_str(), + json_str.c_str(), timestamp, LogLevel::NONE, log_event_idx + 1 From 49a72c393894b89c87ed1b68539de4299e8a448e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Nov 2024 19:11:24 -0500 Subject: [PATCH 36/43] Lint. --- src/clp_ffi_js/ir/StreamReader.cpp | 5 +---- src/clp_ffi_js/ir/StructuredIrStreamReader.cpp | 9 +++++---- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/clp_ffi_js/ir/StreamReader.cpp b/src/clp_ffi_js/ir/StreamReader.cpp index 81d3eee3..331bd330 100644 --- a/src/clp_ffi_js/ir/StreamReader.cpp +++ b/src/clp_ffi_js/ir/StreamReader.cpp @@ -11,7 +11,6 @@ #include #include -#include #include #include #include @@ -117,9 +116,7 @@ EMSCRIPTEN_BINDINGS(ClpStreamReader) { // JS types used as inputs emscripten::register_type("Uint8Array"); emscripten::register_type("number[] | null"); - emscripten::register_type( - "{timestampKey: string} | null" - ); + emscripten::register_type("{timestampKey: string} | null"); // JS types used as outputs emscripten::enum_("IrStreamType") diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp index 6a2a627f..6fe2e7a0 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp @@ -181,8 +181,9 @@ StructuredIrStreamReader::StructuredIrStreamReader( std::shared_ptr> deserialized_log_events ) : m_deserialized_log_events{std::move(deserialized_log_events)}, - m_stream_reader_data_context{std::make_unique< - StreamReaderDataContext>( - std::move(stream_reader_data_context) - )} {} + m_stream_reader_data_context{ + std::make_unique>( + std::move(stream_reader_data_context) + ) + } {} } // namespace clp_ffi_js::ir From 12dfb9ee5dc6948ae05f5ce9f7899f52a2b818cd Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 8 Nov 2024 09:19:27 -0500 Subject: [PATCH 37/43] Rename `IrStreamType` -> `StreamType` in the `ir` namespace. --- src/clp_ffi_js/ir/StreamReader.cpp | 6 +++--- src/clp_ffi_js/ir/StreamReader.hpp | 4 ++-- src/clp_ffi_js/ir/StructuredIrStreamReader.hpp | 4 ++-- src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/clp_ffi_js/ir/StreamReader.cpp b/src/clp_ffi_js/ir/StreamReader.cpp index 331bd330..d78b7e75 100644 --- a/src/clp_ffi_js/ir/StreamReader.cpp +++ b/src/clp_ffi_js/ir/StreamReader.cpp @@ -119,9 +119,9 @@ EMSCRIPTEN_BINDINGS(ClpStreamReader) { emscripten::register_type("{timestampKey: string} | null"); // JS types used as outputs - emscripten::enum_("IrStreamType") - .value("STRUCTURED", clp_ffi_js::ir::IrStreamType::Structured) - .value("UNSTRUCTURED", clp_ffi_js::ir::IrStreamType::Unstructured); + emscripten::enum_("IrStreamType") + .value("STRUCTURED", clp_ffi_js::ir::StreamType::Structured) + .value("UNSTRUCTURED", clp_ffi_js::ir::StreamType::Unstructured); emscripten::register_type( "Array<[string, number, number, number]>" ); diff --git a/src/clp_ffi_js/ir/StreamReader.hpp b/src/clp_ffi_js/ir/StreamReader.hpp index 4c80629d..4af40a4f 100644 --- a/src/clp_ffi_js/ir/StreamReader.hpp +++ b/src/clp_ffi_js/ir/StreamReader.hpp @@ -19,7 +19,7 @@ EMSCRIPTEN_DECLARE_VAL_TYPE(ReaderOptions); EMSCRIPTEN_DECLARE_VAL_TYPE(DecodedResultsTsType); EMSCRIPTEN_DECLARE_VAL_TYPE(FilteredLogEventMapTsType); -enum class IrStreamType : uint8_t { +enum class StreamType : uint8_t { Structured, Unstructured, }; @@ -57,7 +57,7 @@ class StreamReader { auto operator=(StreamReader&&) -> StreamReader& = delete; // Methods - [[nodiscard]] virtual auto get_ir_stream_type() const -> IrStreamType = 0; + [[nodiscard]] virtual auto get_ir_stream_type() const -> StreamType = 0; /** * @return The number of events buffered. diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp index c728c1e9..e93dee03 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.hpp @@ -149,8 +149,8 @@ class StructuredIrStreamReader : public StreamReader { // Delete move assignment operator since it's also disabled in `clp::ir::LogEventDeserializer`. auto operator=(StructuredIrStreamReader&&) -> StructuredIrStreamReader& = delete; - [[nodiscard]] auto get_ir_stream_type() const -> IrStreamType override { - return IrStreamType::Structured; + [[nodiscard]] auto get_ir_stream_type() const -> StreamType override { + return StreamType::Structured; } [[nodiscard]] auto get_num_events_buffered() const -> size_t override; diff --git a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp index 9f921df4..b4f22107 100644 --- a/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp +++ b/src/clp_ffi_js/ir/UnstructuredIrStreamReader.hpp @@ -56,8 +56,8 @@ class UnstructuredIrStreamReader : public StreamReader { clp::Array data_array ) -> UnstructuredIrStreamReader; - [[nodiscard]] auto get_ir_stream_type() const -> IrStreamType override { - return IrStreamType::Unstructured; + [[nodiscard]] auto get_ir_stream_type() const -> StreamType override { + return StreamType::Unstructured; } [[nodiscard]] auto get_num_events_buffered() const -> size_t override; From 2df4df826d5f4c19a0a1720bf21544863bc53b00 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 8 Nov 2024 09:24:27 -0500 Subject: [PATCH 38/43] Use anonymous namespace for internal linkages for const strings. --- src/clp_ffi_js/ir/StructuredIrStreamReader.cpp | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp index 6fe2e7a0..972abbb6 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp @@ -25,14 +25,16 @@ #include #include -namespace clp_ffi_js::ir { -using clp::ir::four_byte_encoded_variable_t; - -static constexpr std::string_view cEmptyJsonStr{"{}"}; -static constexpr std::string_view cLogLevelFilteringNotSupportedErrorMsg{ +namespace { +constexpr std::string_view cEmptyJsonStr{"{}"}; +constexpr std::string_view cLogLevelFilteringNotSupportedErrorMsg{ "Log level filtering is not yet supported in this reader." }; -static constexpr std::string_view cReaderOptionsTimestampKey{"timestampKey"}; +constexpr std::string_view cReaderOptionsTimestampKey{"timestampKey"}; +} // namespace + +namespace clp_ffi_js::ir { +using clp::ir::four_byte_encoded_variable_t; auto StructuredIrStreamReader::create( std::unique_ptr&& zstd_decompressor, From 7edb19e1887ba733f50822def366e950b626b0bc Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 8 Nov 2024 09:26:50 -0500 Subject: [PATCH 39/43] Improve error message when serialize_to_json fails - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- src/clp_ffi_js/ir/StructuredIrStreamReader.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp index 972abbb6..a660d924 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp @@ -146,7 +146,12 @@ auto StructuredIrStreamReader::decode_range(size_t begin_idx, size_t end_idx, bo auto const json_result{log_event.serialize_to_json()}; std::string json_str{cEmptyJsonStr}; if (false == json_result.has_value()) { - SPDLOG_ERROR("Failed to decode log event."); + auto error_code{json_result.error()}; + SPDLOG_ERROR( + "Failed to deserialize log event to JSON: {}:{}", + error_code.category().name(), + error_code.message() + ); } else { json_str = json_result.value().dump(); } From 77e2b2ea4fe2f03ee794cfb398196ccdb9880cf0 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 8 Nov 2024 09:27:27 -0500 Subject: [PATCH 40/43] Fix sentence case in docs - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- src/clp_ffi_js/ir/StructuredIrStreamReader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp index a660d924..61b138a2 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp @@ -164,7 +164,7 @@ auto StructuredIrStreamReader::decode_range(size_t begin_idx, size_t end_idx, bo if (timestamp_pair->is()) { timestamp = timestamp_pair.value().get_immutable_view(); } else { - // TODO: add support for parsing timestamp values of string type. + // TODO: Add support for parsing timestamp values of string type. SPDLOG_ERROR("Unable to parse timestamp for log_event_idx={}", log_event_idx); } } From 320f1ea5b76f5b8e101cc4f47402e4a772ae1195 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 8 Nov 2024 09:32:11 -0500 Subject: [PATCH 41/43] Update clp commit id which has https://github.com/y-scope/clp/pull/579 fixed; remove `no_message_available` handling in deserialize_stream accordingly. --- src/clp_ffi_js/ir/StructuredIrStreamReader.cpp | 2 +- src/submodules/clp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp index 61b138a2..98a81fc5 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp @@ -101,7 +101,7 @@ auto StructuredIrStreamReader::deserialize_stream() -> size_t { continue; } auto const error{result.error()}; - if (std::errc::no_message_available == error || std::errc::operation_not_permitted == error) + if (std::errc::operation_not_permitted == error) { break; } diff --git a/src/submodules/clp b/src/submodules/clp index 426cc3d6..31de766e 160000 --- a/src/submodules/clp +++ b/src/submodules/clp @@ -1 +1 @@ -Subproject commit 426cc3d657c67e9fdffe6681e670cba617f4154f +Subproject commit 31de766ecc3175b1fa472d12881587e3673294de From 81135fa95a520f1305e8b53dfa18dc399b81f20c Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 8 Nov 2024 09:39:06 -0500 Subject: [PATCH 42/43] lint. --- src/clp_ffi_js/ir/StreamReader.cpp | 1 + src/clp_ffi_js/ir/StreamReader.hpp | 1 - src/clp_ffi_js/ir/StructuredIrStreamReader.cpp | 3 +-- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/clp_ffi_js/ir/StreamReader.cpp b/src/clp_ffi_js/ir/StreamReader.cpp index d78b7e75..e7d43d19 100644 --- a/src/clp_ffi_js/ir/StreamReader.cpp +++ b/src/clp_ffi_js/ir/StreamReader.cpp @@ -11,6 +11,7 @@ #include #include +#include #include #include #include diff --git a/src/clp_ffi_js/ir/StreamReader.hpp b/src/clp_ffi_js/ir/StreamReader.hpp index 4af40a4f..06e7c094 100644 --- a/src/clp_ffi_js/ir/StreamReader.hpp +++ b/src/clp_ffi_js/ir/StreamReader.hpp @@ -5,7 +5,6 @@ #include #include -#include #include #include diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp index 98a81fc5..6bd2a3e0 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp @@ -101,8 +101,7 @@ auto StructuredIrStreamReader::deserialize_stream() -> size_t { continue; } auto const error{result.error()}; - if (std::errc::operation_not_permitted == error) - { + if (std::errc::operation_not_permitted == error) { break; } if (std::errc::result_out_of_range == error) { From cca920a40068b5ec9338facdfdeba3a49ed85bd2 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 8 Nov 2024 18:10:58 -0500 Subject: [PATCH 43/43] Nest anonymous namespace into clp_ffi_js::ir to reduce scope of const strings with internal linkage - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- src/clp_ffi_js/ir/StructuredIrStreamReader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp index 6bd2a3e0..799da91c 100644 --- a/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp +++ b/src/clp_ffi_js/ir/StructuredIrStreamReader.cpp @@ -25,6 +25,7 @@ #include #include +namespace clp_ffi_js::ir { namespace { constexpr std::string_view cEmptyJsonStr{"{}"}; constexpr std::string_view cLogLevelFilteringNotSupportedErrorMsg{ @@ -33,7 +34,6 @@ constexpr std::string_view cLogLevelFilteringNotSupportedErrorMsg{ constexpr std::string_view cReaderOptionsTimestampKey{"timestampKey"}; } // namespace -namespace clp_ffi_js::ir { using clp::ir::four_byte_encoded_variable_t; auto StructuredIrStreamReader::create(