Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Prototype: Add initial code for IRv2 decoder. #10

Closed
wants to merge 20 commits into from
Closed
Changes from 1 commit
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
acd4563
Add initial code for IRv2 decoder.
junhaoliao Oct 1, 2024
d883966
Revert changes to enable exception catching linker flags - Update CMa…
junhaoliao Oct 8, 2024
3301fb6
Merge branch 'main' into irv2
junhaoliao Oct 8, 2024
322b60a
Switch clp to OSS' main.
junhaoliao Oct 8, 2024
ab37228
Add code to parse and validate the CLP IR version.
junhaoliao Oct 8, 2024
f2ed47e
Templatize StreamReaderDataContext.
junhaoliao Oct 8, 2024
9062027
Rename IRv2 specialized StreamReader -> KVPairIRStreamReader.
junhaoliao Oct 8, 2024
0f25962
Create a new StreamReader base class and refactor KVPairIRStreamReade…
junhaoliao Oct 10, 2024
80f059b
Revert KVPairIRStreamReader(StreamReaderDataContext<deserializer_t>&&…
junhaoliao Oct 10, 2024
9be08c4
Reformat code.
junhaoliao Oct 10, 2024
7842d29
Optimize imports.
junhaoliao Oct 10, 2024
541b84e
Merge branch 'main' into irv2
junhaoliao Oct 10, 2024
5b49ec0
Rename the original StreamReader -> IrStreamReader and adapt it to th…
junhaoliao Oct 10, 2024
ff7321b
Add log level filtering to KVPairIRStreamReader.
junhaoliao Oct 11, 2024
98b0b70
Rename IrStreamReader -> IRStreamReader.
junhaoliao Oct 11, 2024
f080ece
Update to latest CLP commit.
junhaoliao Nov 1, 2024
7c529d8
Upgrade clp submodule commit to 9f6a6ced4da504f6ba3c131efa26fd5b30c6f533
junhaoliao Nov 6, 2024
6c0ecde
Add tree node id parsing for timestampKey and logLevelKey.
junhaoliao Nov 6, 2024
40908f0
Complete log level filtering.
junhaoliao Nov 6, 2024
e411420
Update yscope-dev-utils version.
junhaoliao Nov 6, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Rename IrStreamReader -> IRStreamReader.
junhaoliao committed Oct 11, 2024
commit 98b0b704e17c1b9f831c7a2d79d0e6c5aff2e16a
2 changes: 1 addition & 1 deletion CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -115,7 +115,7 @@ target_include_directories(${CLP_FFI_JS_BIN_NAME} PRIVATE src/)
set(CLP_FFI_JS_SRC_MAIN
src/clp_ffi_js/bindings.cpp
src/clp_ffi_js/ir/KVPairIRStreamReader.cpp
src/clp_ffi_js/ir/IrStreamReader.cpp
src/clp_ffi_js/ir/IRStreamReader.cpp
src/clp_ffi_js/ir/StreamReader.cpp
)

16 changes: 8 additions & 8 deletions src/clp_ffi_js/bindings.cpp
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#include <emscripten/bind.h>
#include "clp_ffi_js/ir/StreamReader.hpp"
#include "clp_ffi_js/ir/KVPairIRStreamReader.hpp"
#include "clp_ffi_js/ir/IrStreamReader.hpp"
#include "clp_ffi_js/ir/IRStreamReader.hpp"

namespace {
EMSCRIPTEN_BINDINGS(ClpIrStreamReader) {
@@ -12,23 +12,23 @@ namespace {
emscripten::register_type<clp_ffi_js::ir::FilteredLogEventMapTsType>("number[] | null");
emscripten::register_type<clp_ffi_js::ir::ReaderOptions>("interface{logLevelKey?: string, timestampKey?: string}");

emscripten::class_<clp_ffi_js::ir::IrStreamReader,
emscripten::class_<clp_ffi_js::ir::IRStreamReader,
emscripten::base<clp_ffi_js::ir::StreamReader>>("ClpIrStreamReader")
.constructor(
&clp_ffi_js::ir::IrStreamReader::create,
&clp_ffi_js::ir::IRStreamReader::create,
emscripten::return_value_policy::take_ownership()
)
.function(
"getNumEventsBuffered",
&clp_ffi_js::ir::IrStreamReader::get_num_events_buffered
&clp_ffi_js::ir::IRStreamReader::get_num_events_buffered
)
.function(
"getFilteredLogEventMap",
&clp_ffi_js::ir::IrStreamReader::get_filtered_log_event_map
&clp_ffi_js::ir::IRStreamReader::get_filtered_log_event_map
)
.function("filterLogEvents", &clp_ffi_js::ir::IrStreamReader::filter_log_events)
.function("deserializeStream", &clp_ffi_js::ir::IrStreamReader::deserialize_stream)
.function("decodeRange", &clp_ffi_js::ir::IrStreamReader::decode_range);
.function("filterLogEvents", &clp_ffi_js::ir::IRStreamReader::filter_log_events)
.function("deserializeStream", &clp_ffi_js::ir::IRStreamReader::deserialize_stream)
.function("decodeRange", &clp_ffi_js::ir::IRStreamReader::decode_range);

emscripten::class_<clp_ffi_js::ir::KVPairIRStreamReader,
emscripten::base<clp_ffi_js::ir::StreamReader>>("ClpKVPairIRStreamReader")
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#include "IrStreamReader.hpp"
#include "IRStreamReader.hpp"

#include <algorithm>
#include <cstddef>
@@ -34,9 +34,9 @@ using namespace std::literals::string_literals;
using clp::ir::four_byte_encoded_variable_t;

namespace clp_ffi_js::ir {
auto IrStreamReader::create(DataArrayTsType const& data_array, ReaderOptions const& reader_options) -> IrStreamReader {
auto IRStreamReader::create(DataArrayTsType const& data_array, ReaderOptions const& reader_options) -> IRStreamReader {
auto const length{data_array["length"].as<size_t>()};
SPDLOG_INFO("IrStreamReader::create: got buffer of length={}", length);
SPDLOG_INFO("IRStreamReader::create: got buffer of length={}", length);

// Copy array from JavaScript to C++
clp::Array<char> data_buffer{length};
@@ -94,22 +94,22 @@ auto IrStreamReader::create(DataArrayTsType const& data_array, ReaderOptions con
std::move(zstd_decompressor),
std::move(result.value())
};
return IrStreamReader{std::move(stream_reader_data_context), std::move(reader_options)};
return IRStreamReader{std::move(stream_reader_data_context), std::move(reader_options)};
}

auto IrStreamReader::get_num_events_buffered() const -> size_t {
auto IRStreamReader::get_num_events_buffered() const -> size_t {
return m_encoded_log_events.size();
}

auto IrStreamReader::get_filtered_log_event_map() const -> FilteredLogEventMapTsType {
auto IRStreamReader::get_filtered_log_event_map() const -> FilteredLogEventMapTsType {
if (false == m_filtered_log_event_map.has_value()) {
return FilteredLogEventMapTsType{emscripten::val::null()};
}

return FilteredLogEventMapTsType{emscripten::val::array(m_filtered_log_event_map.value())};
}

void IrStreamReader::filter_log_events(emscripten::val const& log_level_filter) {
void IRStreamReader::filter_log_events(emscripten::val const& log_level_filter) {
if (log_level_filter.isNull()) {
m_filtered_log_event_map.reset();
return;
@@ -131,7 +131,7 @@ void IrStreamReader::filter_log_events(emscripten::val const& log_level_filter)
}
}

auto IrStreamReader::deserialize_stream() -> size_t {
auto IRStreamReader::deserialize_stream() -> size_t {
if (nullptr == m_stream_reader_data_context) {
return m_encoded_log_events.size();
}
@@ -189,7 +189,7 @@ auto IrStreamReader::deserialize_stream() -> size_t {
return m_encoded_log_events.size();
}

auto IrStreamReader::decode_range(size_t begin_idx, size_t end_idx, bool use_filter) const
auto IRStreamReader::decode_range(size_t begin_idx, size_t end_idx, bool use_filter) const
-> DecodedResultsTsType {
if (use_filter && false == m_filtered_log_event_map.has_value()) {
return DecodedResultsTsType{emscripten::val::null()};
@@ -242,7 +242,7 @@ auto IrStreamReader::decode_range(size_t begin_idx, size_t end_idx, bool use_fil
return DecodedResultsTsType(results);
}

IrStreamReader::IrStreamReader(
IRStreamReader::IRStreamReader(
StreamReaderDataContext<clp::ir::LogEventDeserializer<clp::ir::four_byte_encoded_variable_t>>&& stream_reader_data_context, ReaderOptions const& reader_options
)
: m_stream_reader_data_context{std::make_unique<
Original file line number Diff line number Diff line change
@@ -21,28 +21,28 @@ namespace clp_ffi_js::ir {
* Class to deserialize and decode Zstandard-compressed CLP IR streams as well as format decoded
* log events.
*/
class IrStreamReader: public StreamReader {
class IRStreamReader: public StreamReader {
public:
/**
* Creates a IrStreamReader to read from the given array.
* Creates a IRStreamReader to read from the given array.
*
* @param data_array An array containing a Zstandard-compressed IR stream.
* @return The created instance.
* @throw ClpFfiJsException if any error occurs.
*/
[[nodiscard]] static auto create(DataArrayTsType const& data_array, ReaderOptions const& reader_options) -> IrStreamReader;
[[nodiscard]] static auto create(DataArrayTsType const& data_array, ReaderOptions const& reader_options) -> IRStreamReader;

// Destructor
~IrStreamReader() = default;
~IRStreamReader() = default;

// Disable copy constructor and assignment operator
IrStreamReader(IrStreamReader const&) = delete;
auto operator=(IrStreamReader const&) -> IrStreamReader& = delete;
IRStreamReader(IRStreamReader const&) = delete;
auto operator=(IRStreamReader const&) -> IRStreamReader& = delete;

// Define default move constructor
IrStreamReader(IrStreamReader&&) = default;
IRStreamReader(IRStreamReader&&) = default;
// Delete move assignment operator since it's also disabled in `clp::ir::LogEventDeserializer`.
auto operator=(IrStreamReader&&) -> IrStreamReader& = delete;
auto operator=(IRStreamReader&&) -> IRStreamReader& = delete;

/**
* @return The number of events buffered.
@@ -89,7 +89,7 @@ class IrStreamReader: public StreamReader {

private:
// Constructor
explicit IrStreamReader(StreamReaderDataContext<clp::ir::LogEventDeserializer<clp::ir::four_byte_encoded_variable_t>>&&
explicit IRStreamReader(StreamReaderDataContext<clp::ir::LogEventDeserializer<clp::ir::four_byte_encoded_variable_t>>&&
stream_reader_data_context, ReaderOptions const& reader_options);

// Variables
4 changes: 2 additions & 2 deletions src/clp_ffi_js/ir/StreamReader.cpp
Original file line number Diff line number Diff line change
@@ -20,7 +20,7 @@
#include <spdlog/spdlog.h>

#include <clp_ffi_js/ClpFfiJsException.hpp>
#include <clp_ffi_js/ir/IrStreamReader.hpp>
#include <clp_ffi_js/ir/IRStreamReader.hpp>
#include <clp_ffi_js/ir/KVPairIRStreamReader.hpp>

namespace clp_ffi_js::ir {
@@ -78,7 +78,7 @@ auto StreamReader::create(DataArrayTsType const& data_array, ReaderOptions const
auto const& version{metadata.at(clp::ffi::ir_stream::cProtocol::Metadata::VersionKey)};
SPDLOG_INFO("The version is {}", version);
if (version == "v0.0.0") {
return std::make_unique<IrStreamReader>(IrStreamReader::create(data_array, std::move(reader_options)));
return std::make_unique<IRStreamReader>(IRStreamReader::create(data_array, std::move(reader_options)));
}

return std::make_unique<KVPairIRStreamReader>(KVPairIRStreamReader::create(data_array, std::move(reader_options)));