From 9fd124a1f66399bcb461b1fe66349936b6b9d7f0 Mon Sep 17 00:00:00 2001 From: Thomas Applencourt Date: Fri, 15 Jul 2022 00:03:27 +0000 Subject: [PATCH 01/24] proff of concept --- utils/xprof_utils.cpp | 15 +++++++-- utils/xprof_utils.hpp | 5 +-- xprof/interval.c.erb | 4 +++ xprof/interval.h.erb | 3 +- xprof/interval_model.yaml | 30 ++++++++++++++++++ xprof/perfetto_prunned.proto | 2 ++ xprof/tally.cpp | 4 +-- xprof/timeline.cpp | 34 ++++++++++++++++---- ze/gen_zeinterval_callbacks.rb | 1 + ze/zeinterval_callbacks.cpp.erb | 55 +++++++++++++++++++++++---------- ze/zeinterval_callbacks.hpp | 11 ++++--- 11 files changed, 129 insertions(+), 35 deletions(-) diff --git a/utils/xprof_utils.cpp b/utils/xprof_utils.cpp index 1ed1d8d0..af76dbd1 100644 --- a/utils/xprof_utils.cpp +++ b/utils/xprof_utils.cpp @@ -22,7 +22,7 @@ thread_id_t borrow_thread_id(const bt_event *event){ bt_message* create_host_message(const char* hostname, const process_id_t process_id, const thread_id_t thread_id, const char* name, const uint64_t ts, const uint64_t duration, const bool err, - bt_event_class *event_class, bt_self_message_iterator *message_iterator, bt_stream *stream, backend_t backend) { + bt_event_class *event_class, bt_self_message_iterator *message_iterator, bt_stream *stream, backend_t backend, const flow_id_t flow_id) { /* Message creation */ bt_message *message = bt_message_event_create( @@ -65,6 +65,11 @@ bt_message* create_host_message(const char* hostname, const process_id_t process bt_field *err_field = bt_field_structure_borrow_member_field_by_index(payload_field, 2); bt_field_integer_unsigned_set_value(err_field, err); + // flow id + if (flow_id != 0) { + bt_field *flow_id_field = bt_field_structure_borrow_member_field_by_index(payload_field, 3); + bt_field_integer_unsigned_set_value(flow_id_field, flow_id); + } return message; } @@ -73,7 +78,7 @@ bt_message* create_device_message(const char* hostname, const process_id_t proce const thapi_device_id device_id, const thapi_device_id subdevice_id, const char* name, const uint64_t ts, const uint64_t duration, const bool err, const char* metadata, - bt_event_class *event_class, bt_self_message_iterator *message_iterator, bt_stream *stream) { + bt_event_class *event_class, bt_self_message_iterator *message_iterator, bt_stream *stream, const flow_id_t flow_id) { /* Message creation */ bt_message *message = bt_message_event_create( @@ -126,6 +131,12 @@ bt_message* create_device_message(const char* hostname, const process_id_t proce bt_field *metadata_field = bt_field_structure_borrow_member_field_by_index(payload_field, 5); bt_field_string_set_value(metadata_field, metadata); + // flow id + if (flow_id != 0) { + bt_field *flow_id_field = bt_field_structure_borrow_member_field_by_index(payload_field, 6); + bt_field_integer_unsigned_set_value(flow_id_field, flow_id); + } + return message; } diff --git a/utils/xprof_utils.hpp b/utils/xprof_utils.hpp index c68a1138..adfc827f 100644 --- a/utils/xprof_utils.hpp +++ b/utils/xprof_utils.hpp @@ -28,6 +28,7 @@ typedef uintptr_t thread_id_t; typedef std::string hostname_t; typedef std::string thapi_function_name; typedef uintptr_t thapi_device_id; +typedef uint64_t flow_id_t; // Represent a device and a sub device typedef std::tuple dsd_t; @@ -120,11 +121,11 @@ thread_id_t borrow_thread_id(const bt_event*); bt_message* create_host_message(const char *hostname, const process_id_t, const thread_id_t, const char *name, const uint64_t ts, const uint64_t duration, const bool err, - bt_event_class*, bt_self_message_iterator*, bt_stream*, backend_t = BACKEND_UNKNOWN); + bt_event_class*, bt_self_message_iterator*, bt_stream*, backend_t = BACKEND_UNKNOWN, flow_id_t flow_id = 0); bt_message* create_device_message(const char *hostname, const process_id_t, const thread_id_t, const thapi_device_id, const thapi_device_id, const char *name, const uint64_t ts, const uint64_t duration, const bool err, const char* metadata, - bt_event_class*, bt_self_message_iterator*, bt_stream*); + bt_event_class*, bt_self_message_iterator*, bt_stream*, flow_id_t flow_id = 0); bt_message* create_device_name_message(const char* hostname, const process_id_t process_id, const thapi_device_id device_id, const char* name, diff --git a/xprof/interval.c.erb b/xprof/interval.c.erb index abc96b67..c00a559d 100644 --- a/xprof/interval.c.erb +++ b/xprof/interval.c.erb @@ -92,8 +92,12 @@ bt_component_class_initialize_method_status <%= namespace %>_dispatch_initialize // We don't create a clock, because we ensure monotonic order for the downstream messages. populate_ze_stream_class_common_context(trace_class, stream_class); + // This should be generated! dispatch->host_event_class = create_lttng_host_event_class_message(trace_class,stream_class); + dispatch->host_flow_event_class = create_lttng_host_flow_event_class_message(trace_class,stream_class); + dispatch->device_event_class = create_lttng_device_event_class_message(trace_class, stream_class); + dispatch->device_flow_event_class = create_lttng_device_flow_event_class_message(trace_class, stream_class); dispatch->traffic_event_class = create_lttng_traffic_event_class_message(trace_class, stream_class); dispatch->device_name_event_class = create_lttng_device_name_event_class_message(trace_class, stream_class); diff --git a/xprof/interval.h.erb b/xprof/interval.h.erb index ce94241f..58bdba82 100644 --- a/xprof/interval.h.erb +++ b/xprof/interval.h.erb @@ -53,10 +53,11 @@ struct <%= namespace %>_dispatch { /* Downstream message */ bt_stream *stream; bt_event_class *host_event_class; + bt_event_class *host_flow_event_class; bt_event_class *device_event_class; + bt_event_class *device_flow_event_class; bt_event_class *traffic_event_class; bt_event_class *device_name_event_class; - /* Component's input port (weak) */ bt_self_component_port_input *in_port; }; diff --git a/xprof/interval_model.yaml b/xprof/interval_model.yaml index 098bb85e..f020a9e8 100644 --- a/xprof/interval_model.yaml +++ b/xprof/interval_model.yaml @@ -24,6 +24,16 @@ :class: unsigned - :name: err :class: bool +- :name: lttng:host_flow + :payload: + - :name: name + :class: string + - :name: dur + :class: unsigned + - :name: err + :class: bool + - :name: flow_id + :class: unsigned - :name: lttng:device :payload: - :name: name @@ -42,6 +52,26 @@ :class: bool - :name: metadata :class: string +- :name: lttng:device_flow + :payload: + - :name: name + :class: string + - :name: dur + :class: unsigned + - :name: did + :class: unsigned + :class_properties: + :preferred_display_base: 16 + - :name: sdid + :class: unsigned + :class_properties: + :preferred_display_base: 16 + - :name: err + :class: bool + - :name: metadata + :class: string + - :name: flow_id + :class: unsigned - :name: lttng:device_name :payload: - :name: name diff --git a/xprof/perfetto_prunned.proto b/xprof/perfetto_prunned.proto index 3fdc0023..2a43834e 100644 --- a/xprof/perfetto_prunned.proto +++ b/xprof/perfetto_prunned.proto @@ -51,6 +51,8 @@ message TrackEvent { string name = 23; } + repeated uint64 flow_ids = 36; + repeated uint64 terminating_flow_ids = 42; } message TracePacket { diff --git a/xprof/tally.cpp b/xprof/tally.cpp index 19188925..fc8d3a9c 100644 --- a/xprof/tally.cpp +++ b/xprof/tally.cpp @@ -101,7 +101,7 @@ tally_dispatch_consume(bt_self_component_sink *self_component_sink) { const bt_field *payload_field = bt_event_borrow_payload_field_const(event); - if (strcmp(class_name, "lttng:host") == 0) { + if ( (strcmp(class_name, "lttng:host") == 0) || (strcmp(class_name, "lttng:host_flow") == 0) ) { auto dur_tuple0 = std::make_tuple( std::make_tuple(0, bt_field_string_get_value, (std::string) ""), std::make_tuple(1, &bt_field_integer_unsigned_get_value, @@ -119,7 +119,7 @@ tally_dispatch_consume(bt_self_component_sink *self_component_sink) { dispatch->host_backend_name[level].insert(backend_name[backend_id]); dispatch->host[level][hpt_function_name_t(hostname, process_id, thread_id, name)] += a; - } else if (strcmp(class_name, "lttng:device") == 0) { + } else if ( (strcmp(class_name, "lttng:device") == 0) || (strcmp(class_name, "lttng:device_flow") == 0) ) { auto dur_tuple0 = std::make_tuple( std::make_tuple(0, bt_field_string_get_value, (std::string) ""), std::make_tuple(1, &bt_field_integer_unsigned_get_value, diff --git a/xprof/timeline.cpp b/xprof/timeline.cpp index 6c1ceafc..bcbf36b8 100644 --- a/xprof/timeline.cpp +++ b/xprof/timeline.cpp @@ -15,7 +15,7 @@ static perfetto_uuid_t gen_perfetto_uuid() { } static void add_event_begin(struct timeline_dispatch *dispatch, perfetto_uuid_t uuid, - timestamp_t begin, std::string name) { + timestamp_t begin, std::string name, flow_id_t flow_id) { auto *packet = dispatch->trace.add_packet(); packet->set_timestamp(begin); packet->set_trusted_packet_sequence_id(10); @@ -23,6 +23,8 @@ static void add_event_begin(struct timeline_dispatch *dispatch, perfetto_uuid_t track_event->set_type(perfetto_pruned::TrackEvent::TYPE_SLICE_BEGIN); track_event->set_name(name); track_event->set_track_uuid(uuid); + if (flow_id != 0) + track_event->add_flow_ids(flow_id); } static void add_event_end(struct timeline_dispatch *dispatch, perfetto_uuid_t uuid, uint64_t end) { @@ -114,13 +116,13 @@ static perfetto_uuid_t get_parent_uuid(struct timeline_dispatch *dispatch, std:: static void add_event_cpu(struct timeline_dispatch *dispatch, std::string hostname, uint64_t process_id, uint64_t thread_id, std::string name, uint64_t begin, - uint64_t dur) { + uint64_t dur, uint64_t flow_id = 0) { // Assume perfecly nessted const uint64_t end = begin + dur; perfetto_uuid_t parent_uuid = get_parent_uuid(dispatch, hostname, process_id, thread_id); // Handling perfecly nested event - add_event_begin(dispatch, parent_uuid, begin, name); + add_event_begin(dispatch, parent_uuid, begin, name, flow_id); std::stack &s = dispatch->uuid2stack[parent_uuid]; while ((!s.empty()) && (s.top() <= begin)) { add_event_end(dispatch, parent_uuid, s.top()); @@ -131,7 +133,7 @@ static void add_event_cpu(struct timeline_dispatch *dispatch, std::string hostna static void add_event_gpu(struct timeline_dispatch *dispatch, std::string hostname, uint64_t process_id, uint64_t thread_id, thapi_device_id did, - thapi_device_id sdid, std::string name, uint64_t begin, uint64_t dur) { + thapi_device_id sdid, std::string name, uint64_t begin, uint64_t dur, uint64_t flow_id = 0) { // This function Assume non perfecly nested const uint64_t end = begin + dur; perfetto_uuid_t parent_uuid = get_parent_uuid(dispatch, hostname, process_id, thread_id, did, sdid); @@ -166,7 +168,7 @@ static void add_event_gpu(struct timeline_dispatch *dispatch, std::string hostna // Update the table m[end] = uuid; // Add event - add_event_begin(dispatch, uuid, begin, name); + add_event_begin(dispatch, uuid, begin, name, flow_id); add_event_end(dispatch, uuid, end); } @@ -240,6 +242,12 @@ timeline_dispatch_consume(bt_self_component_sink *self_component_sink) { if (std::string(class_name) == "lttng:host") { add_event_cpu(dispatch, hostname, process_id, thread_id, name, ts, dur); + } else if (std::string(class_name) == "lttng:host_flow") { + + const bt_field *flow_id_field = + bt_field_structure_borrow_member_field_by_index_const(payload_field, 3); + const long flow_id = bt_field_integer_unsigned_get_value(flow_id_field); + add_event_cpu(dispatch, hostname, process_id, thread_id, name, ts, dur, flow_id); } else if (std::string(class_name) == "lttng:device") { const bt_field *did_field = @@ -250,7 +258,22 @@ timeline_dispatch_consume(bt_self_component_sink *self_component_sink) { bt_field_structure_borrow_member_field_by_index_const(payload_field, 3); const thapi_device_id sdid = bt_field_integer_unsigned_get_value(sdid_field); add_event_gpu(dispatch, hostname, process_id, thread_id, did, sdid, name, ts, dur); + } else if (std::string(class_name) == "lttng:device_flow") { + + const bt_field *did_field = + bt_field_structure_borrow_member_field_by_index_const(payload_field, 2); + const thapi_device_id did = bt_field_integer_unsigned_get_value(did_field); + + const bt_field *sdid_field = + bt_field_structure_borrow_member_field_by_index_const(payload_field, 3); + const thapi_device_id sdid = bt_field_integer_unsigned_get_value(sdid_field); + + const bt_field *flow_id_field = + bt_field_structure_borrow_member_field_by_index_const(payload_field, 6); + const long flow_id = bt_field_integer_unsigned_get_value(flow_id_field); + add_event_gpu(dispatch, hostname, process_id, thread_id, did, sdid, name, ts, dur, flow_id); } + } bt_message_put_ref(message); } @@ -291,7 +314,6 @@ timeline_dispatch_initialize(bt_self_component_sink *self_component_sink, trace_packet_defaults->set_timestamp_clock_id(perfetto_pruned::BUILTIN_CLOCK_BOOTTIME); packet->set_previous_packet_dropped(true); } - return BT_COMPONENT_CLASS_INITIALIZE_METHOD_STATUS_OK; } diff --git a/ze/gen_zeinterval_callbacks.rb b/ze/gen_zeinterval_callbacks.rb index 84311ae1..5af730a1 100644 --- a/ze/gen_zeinterval_callbacks.rb +++ b/ze/gen_zeinterval_callbacks.rb @@ -58,6 +58,7 @@ def callback_signature DBT_event.new(klass) } +$dbt_events_who_signal = $dbt_events.filter_map { |dbt_event| dbt_event.name_striped if dbt_event.start? and dbt_event.fields_name.include?('hSignalEvent') } $profiling_apis = Set.new template = File.read(File.join(SRC_DIR, "zeinterval_callbacks.cpp.erb")) diff --git a/ze/zeinterval_callbacks.cpp.erb b/ze/zeinterval_callbacks.cpp.erb index d1aea864..7f3f78c2 100644 --- a/ze/zeinterval_callbacks.cpp.erb +++ b/ze/zeinterval_callbacks.cpp.erb @@ -91,10 +91,24 @@ static void create_and_enqueue_host_message(const char* hostname, const process_ state->downstream_message_queue.push(message); } +static void create_and_enqueue_host_flow_message(const char* hostname, const process_id_t process_id, const thread_id_t thread_id, const char* name, + const uint64_t ts, const uint64_t duration, const bool err, const flow_id_t flow_id) { + + /* Message creation */ + bt_message *message = create_host_message(hostname, process_id, thread_id, name, ts, duration, err, + zeinterval_iter_g->dispatch->host_flow_event_class, + zeinterval_self_message_iterator_g, + zeinterval_iter_g->dispatch->stream, BACKEND_ZE, flow_id); + + zeinterval_callbacks_state* state = (zeinterval_callbacks_state*) zeinterval_iter_g->callbacks_state; + state->downstream_message_queue.push(message); +} + static void create_and_enqueue_device_message( const char* hostname, const process_id_t process_id, const thread_id_t thread_id, thapi_device_id device, const char* commandname, const char* metadata, bool err, const uint64_t globalStart, const uint64_t globalEnd, const uint64_t lltng_min, + const flow_id_t flow_id, const clock_lttng_device_t ×tamp_pair_ref, const std::unordered_map &device_to_properties, const std::unordered_map &subdevice_parent) { @@ -123,13 +137,12 @@ static void create_and_enqueue_device_message( if (it2 != device_to_properties.cend()) device_hash = hash_device(it2->second); } - /* Message creation */ bt_message *message = create_device_message(hostname, process_id, thread_id, device_hash, subdevice_hash, commandname, start, delta, err, metadata, - zeinterval_iter_g->dispatch->device_event_class, + zeinterval_iter_g->dispatch->device_flow_event_class, zeinterval_self_message_iterator_g, - zeinterval_iter_g->dispatch->stream); + zeinterval_iter_g->dispatch->stream, flow_id); /* Set message */ zeinterval_callbacks_state* state = (zeinterval_callbacks_state*) zeinterval_iter_g->callbacks_state; @@ -277,12 +290,16 @@ static void zeinterval_<%= dbt_event.name %>_callback( int64_t ns_from_origin; bt_clock_snapshot_get_ns_from_origin(bt_clock, &ns_from_origin); <% if dbt_event.start? %> - state->host_start[hpt_function_name_t(hostname,process_id, thread_id, "<%= dbt_event.name_striped %>")] = ns_from_origin; + state->host_start[ {hostname,process_id, thread_id, "<%= dbt_event.name_striped %>"} ] = {ns_from_origin, state->flow_id}; <% elsif dbt_event.stop? %> - auto it_start = state->host_start.find(hpt_function_name_t(hostname,process_id, thread_id, "<%= dbt_event.name_striped %>")); - const uint64_t start_g = it_start->second; + auto it_start = state->host_start.find( { hostname,process_id, thread_id, "<%= dbt_event.name_striped %>"} ); + const auto& [start_g, flow_id] = it_start->second; state->host_start.erase(it_start); - <% if dbt_event.fields_name.include?('zeResult') %> + <% if $dbt_events_who_signal.include?(dbt_event.name_striped) %> + create_and_enqueue_host_flow_message(hostname.c_str(), process_id, thread_id, "<%= dbt_event.name_striped %>", + start_g, ns_from_origin-start_g, zeResult, flow_id); + <% elsif dbt_event.fields_name.include?('zeResult') %> + (void) flow_id; <%# In level0 1.0-rev-3 only those use `ZE_RESULT_NOT_READY` %> <% if dbt_event.name_striped.include?('Synchronize') or dbt_event.name_striped.include?('Query') %> create_and_enqueue_host_message(hostname.c_str(), process_id, thread_id, "<%= dbt_event.name_striped %>", @@ -507,8 +524,10 @@ Case B: <% else %> const std::string name{"<%= dbt_event.name_striped %>"}; <% end %> - const auto device = state->command_list_to_device[hp_command_list_t(hostname,process_id,hCommandList)]; - state->command_partial_payload[hpt_t(hostname,process_id,thread_id)] = l_tfnm_m_d_ts_t(hCommandList, name, metadata.str(), device, ns_from_origin); + const flow_id_t flow_id = state->flow_id; + state->flow_id++; + const auto device = state->command_list_to_device[ { hostname,process_id,hCommandList } ]; + state->command_partial_payload[ { hostname,process_id,thread_id } ] = { hCommandList, name, metadata.str(), device, ns_from_origin, flow_id }; <% end %> <%# @@ -524,13 +543,13 @@ Case B: const hpt_t hpt(hostname, process_id, thread_id); auto it_pp = state->command_partial_payload.find(hpt); - // We didn't find the command who initiated this even_profiling, + // We didn't find the command who initiated this event_profiling, // Best we can do is to bailout if (it_pp == state->command_partial_payload.end()) return; - const auto [hCommandList, commandname, metadata, device, lltng_min] = it_pp->second; + const auto [hCommandList, commandname, metadata, device, lltng_min, flow_id] = it_pp->second; state->command_partial_payload.erase(it_pp); - state->command_list_to_events[hp_command_list_t(hostname, process_id, hCommandList)].insert(hEvent); + state->command_list_to_events[ { hostname, process_id, hCommandList } ].insert(hEvent); // Got the timestamp pair referance clock_lttng_device_t timestamp_pair_ref; @@ -543,7 +562,7 @@ Case B: auto it = state->event_to_profiling_result.find(hpe); if (it == state->event_to_profiling_result.end()){ //Creeate the payload who will be used by event_profiling_results - state->event_payload[hpe] = t_tfnm_m_d_ts_cld_t(thread_id, commandname, metadata, device, lltng_min, timestamp_pair_ref); + state->event_payload[hpe] = { thread_id, commandname, metadata, device, lltng_min, timestamp_pair_ref, flow_id }; } else { // Handle the callbacks case, where `event_profiling_results` was called before `event_profiling`. const auto& [err, globalStart, globalEnd] = it->second; @@ -551,7 +570,8 @@ Case B: const auto& subdevice_parent = state->subdevice_parent; create_and_enqueue_device_message(hostname.c_str(), process_id, thread_id, device, commandname.c_str(), metadata.c_str(), - err, globalStart, globalEnd, lltng_min, timestamp_pair_ref, + err, globalStart, globalEnd, lltng_min, flow_id, + timestamp_pair_ref, device_to_properties, subdevice_parent); } @@ -562,15 +582,16 @@ Case B: auto it = state->event_payload.find(hpe); if (it != state->event_payload.end()) { - const auto& [thread_id, commandname, metadata, device, lltng_min, timestamp_pair_ref] = it->second; + const auto& [thread_id, commandname, metadata, device, lltng_min, timestamp_pair_ref, flow_id] = it->second; const auto& device_to_properties = state->device_to_properties; const auto& subdevice_parent = state->subdevice_parent; create_and_enqueue_device_message(hostname.c_str(), process_id, thread_id, device, commandname.c_str(), metadata.c_str(), - err, globalStart, globalEnd, lltng_min, timestamp_pair_ref, + err, globalStart, globalEnd, lltng_min, flow_id, + timestamp_pair_ref, device_to_properties, subdevice_parent); } else { - state->event_to_profiling_result[hpe] = event_profiling_result_t(err, globalStart,globalEnd); + state->event_to_profiling_result[hpe] = { err, globalStart, globalEnd }; } <% end %> <%# Cleanup state for profiling %> diff --git a/ze/zeinterval_callbacks.hpp b/ze/zeinterval_callbacks.hpp index 47775510..195f53c3 100644 --- a/ze/zeinterval_callbacks.hpp +++ b/ze/zeinterval_callbacks.hpp @@ -21,8 +21,8 @@ typedef hp_event_t hpe_t; typedef hp_kernel_t hpk_t; typedef std::tuple clock_lttng_device_t; -typedef std::tuple t_tfnm_m_d_ts_cld_t; -typedef std::tuple l_tfnm_m_d_ts_t; +typedef std::tuple t_tfnm_m_d_ts_cld_tp; +typedef std::tuple l_tfnm_m_d_ts_tp; typedef std::tuple event_profiling_result_t; @@ -36,7 +36,7 @@ struct zeinterval_callbacks_state { std::unordered_map kernel_to_name; std::unordered_map kernel_to_groupsize_str; std::unordered_map kernel_to_simdsize_str; - std::unordered_map host_start; + std::unordered_map> host_start; std::unordered_map profiled_function_name; /* Handle memory copy */ @@ -48,9 +48,10 @@ struct zeinterval_callbacks_state { /* Handle variable */ std::unordered_map event_to_profiling_result; std::unordered_map device_timestamps_pair_ref; - std::unordered_map command_partial_payload; - std::unordered_map event_payload; + std::unordered_map command_partial_payload; + std::unordered_map event_payload; std::unordered_map subdevice_parent; + flow_id_t flow_id = 1; /* Stack to get begin end */ std::unordered_map> last_command; From a2e4c635651fbd2e56b2af90f8591553378f3246 Mon Sep 17 00:00:00 2001 From: Thomas Applencourt Date: Fri, 15 Jul 2022 22:37:21 +0000 Subject: [PATCH 02/24] Ugly working --- utils/xprof_utils.cpp | 75 +++++++++++++++++++-- utils/xprof_utils.hpp | 8 ++- xprof/interval_model.yaml | 8 ++- xprof/timeline.cpp | 57 ++++++++++++++-- ze/zeinterval_callbacks.cpp.erb | 113 ++++++++++++++++++++++---------- ze/zeinterval_callbacks.hpp | 12 ++-- 6 files changed, 219 insertions(+), 54 deletions(-) diff --git a/utils/xprof_utils.cpp b/utils/xprof_utils.cpp index af76dbd1..65b8aa98 100644 --- a/utils/xprof_utils.cpp +++ b/utils/xprof_utils.cpp @@ -78,7 +78,7 @@ bt_message* create_device_message(const char* hostname, const process_id_t proce const thapi_device_id device_id, const thapi_device_id subdevice_id, const char* name, const uint64_t ts, const uint64_t duration, const bool err, const char* metadata, - bt_event_class *event_class, bt_self_message_iterator *message_iterator, bt_stream *stream, const flow_id_t flow_id) { + bt_event_class *event_class, bt_self_message_iterator *message_iterator, bt_stream *stream) { /* Message creation */ bt_message *message = bt_message_event_create( @@ -130,14 +130,75 @@ bt_message* create_device_message(const char* hostname, const process_id_t proce //Metadata bt_field *metadata_field = bt_field_structure_borrow_member_field_by_index(payload_field, 5); bt_field_string_set_value(metadata_field, metadata); + return message; +} - // flow id - if (flow_id != 0) { - bt_field *flow_id_field = bt_field_structure_borrow_member_field_by_index(payload_field, 6); - bt_field_integer_unsigned_set_value(flow_id_field, flow_id); - } +bt_message* create_device_flow_message(const char* hostname, const process_id_t process_id, const uint64_t uuid, + const thapi_device_id device_id, const thapi_device_id subdevice_id, + const char* name, const uint64_t ts, const uint64_t duration, const bool err, + const char* metadata, const char* queue_name, + bt_event_class *event_class, bt_self_message_iterator *message_iterator, bt_stream *stream) { - return message; + /* Message creation */ + bt_message *message = bt_message_event_create( + message_iterator, event_class, stream); + + + /* event */ + bt_event *downstream_event = bt_message_event_borrow_event(message); + + /* Common context */ + bt_field *context_field = bt_event_borrow_common_context_field(downstream_event); + + // Hostname + bt_field *hostname_msg_field = bt_field_structure_borrow_member_field_by_index(context_field,0); + bt_field_string_set_value(hostname_msg_field, hostname); + // pid + bt_field *vpid_field = bt_field_structure_borrow_member_field_by_index(context_field,1); + bt_field_integer_signed_set_value(vpid_field, process_id); + // vid + bt_field *vtid_field = bt_field_structure_borrow_member_field_by_index(context_field,2); + bt_field_integer_signed_set_value(vtid_field, 0); + // ts + bt_field *ts_field = bt_field_structure_borrow_member_field_by_index(context_field,3); + bt_field_integer_signed_set_value(ts_field, ts); + + /* Payload */ + bt_field *payload_field = bt_event_borrow_payload_field(downstream_event); + + // name + bt_field *name_field = bt_field_structure_borrow_member_field_by_index(payload_field, 0); + bt_field_string_set_value(name_field, name); + + // dur + bt_field *dur_field = bt_field_structure_borrow_member_field_by_index(payload_field, 1); + bt_field_integer_unsigned_set_value(dur_field, duration); + + // did + bt_field *device_id_field = bt_field_structure_borrow_member_field_by_index(payload_field,2); + bt_field_integer_unsigned_set_value(device_id_field, device_id); + + // sdid + bt_field *subdevice_id_field = bt_field_structure_borrow_member_field_by_index(payload_field,3); + bt_field_integer_unsigned_set_value(subdevice_id_field, subdevice_id); + + // err + bt_field *err_field = bt_field_structure_borrow_member_field_by_index(payload_field, 4); + bt_field_integer_unsigned_set_value(err_field, err); + + //Metadata + bt_field *metadata_field = bt_field_structure_borrow_member_field_by_index(payload_field, 5); + bt_field_string_set_value(metadata_field, metadata); + + // uuid + bt_field *uuid_field = bt_field_structure_borrow_member_field_by_index(payload_field, 6); + bt_field_integer_unsigned_set_value(uuid_field, uuid); + + //Queue_name + bt_field *queue_name_field = bt_field_structure_borrow_member_field_by_index(payload_field, 7); + bt_field_string_set_value(queue_name_field, queue_name); + + return message; } bt_message* create_device_name_message(const char* hostname, const process_id_t process_id, diff --git a/utils/xprof_utils.hpp b/utils/xprof_utils.hpp index adfc827f..4af16174 100644 --- a/utils/xprof_utils.hpp +++ b/utils/xprof_utils.hpp @@ -125,7 +125,13 @@ bt_message* create_host_message(const char *hostname, const process_id_t, const bt_message* create_device_message(const char *hostname, const process_id_t, const thread_id_t, const thapi_device_id, const thapi_device_id, const char *name, const uint64_t ts, const uint64_t duration, const bool err, const char* metadata, - bt_event_class*, bt_self_message_iterator*, bt_stream*, flow_id_t flow_id = 0); + bt_event_class*, bt_self_message_iterator*, bt_stream*); + +bt_message* create_device_flow_message(const char* hostname, const process_id_t process_id, const uint64_t uuid, + const thapi_device_id device_id, const thapi_device_id subdevice_id, + const char* name, const uint64_t ts, const uint64_t duration, const bool err, + const char* metadata, const char* queue_name, + bt_event_class *event_class, bt_self_message_iterator *message_iterator, bt_stream *stream); bt_message* create_device_name_message(const char* hostname, const process_id_t process_id, const thapi_device_id device_id, const char* name, diff --git a/xprof/interval_model.yaml b/xprof/interval_model.yaml index f020a9e8..7e88dc4c 100644 --- a/xprof/interval_model.yaml +++ b/xprof/interval_model.yaml @@ -70,8 +70,14 @@ :class: bool - :name: metadata :class: string - - :name: flow_id + - :name: uuid + :cast_type: void * :class: unsigned + :class_properties: + :field_value_range: 64 + :preferred_display_base: 16 + - :name: queue_name + :class: string - :name: lttng:device_name :payload: - :name: name diff --git a/xprof/timeline.cpp b/xprof/timeline.cpp index bcbf36b8..681615d4 100644 --- a/xprof/timeline.cpp +++ b/xprof/timeline.cpp @@ -131,7 +131,7 @@ static void add_event_cpu(struct timeline_dispatch *dispatch, std::string hostna s.push(end); } -static void add_event_gpu(struct timeline_dispatch *dispatch, std::string hostname, +static void add_event_gpu_old(struct timeline_dispatch *dispatch, std::string hostname, uint64_t process_id, uint64_t thread_id, thapi_device_id did, thapi_device_id sdid, std::string name, uint64_t begin, uint64_t dur, uint64_t flow_id = 0) { // This function Assume non perfecly nested @@ -172,6 +172,48 @@ static void add_event_gpu(struct timeline_dispatch *dispatch, std::string hostna add_event_end(dispatch, uuid, end); } +static void add_event_gpu(struct timeline_dispatch *dispatch, std::string hostname, + uint64_t process_id, uint64_t c_uuid, std::string queue_name, + thapi_device_id did, thapi_device_id sdid, + std::string name, uint64_t begin, uint64_t dur, uint64_t flow_id = 0) { + // This function Assume non perfecly nested + const uint64_t end = begin + dur; + perfetto_uuid_t parent_uuid = get_parent_uuid(dispatch, hostname, process_id, c_uuid, did, sdid); + // Now see if we need a to generate a new children + std::map &m = dispatch->parents2tracks[parent_uuid]; + perfetto_uuid_t uuid; + + // Pre-historical event + if (m.empty() || begin < m.begin()->first) { + uuid = gen_perfetto_uuid(); + // Generate a new children track + { + auto *packet = dispatch->trace.add_packet(); + packet->set_trusted_packet_sequence_id(10); + packet->set_timestamp(0); + + auto *track_descriptor = packet->mutable_track_descriptor(); + track_descriptor->set_uuid(uuid); + track_descriptor->set_parent_uuid(parent_uuid); + + std::ostringstream oss; + oss << queue_name << " " << c_uuid; + track_descriptor->set_name(oss.str()); + } + } else { + // Find the uuid who finished just before this one + auto it_ub = std::prev(m.upper_bound(begin)); + uuid = it_ub->second; + // Erase the old timestamps + m.erase(it_ub); + } + // Update the table + m[end] = uuid; + // Add event + add_event_begin(dispatch, uuid, begin, name, flow_id); + add_event_end(dispatch, uuid, end); +} + bt_component_class_sink_consume_method_status timeline_dispatch_consume(bt_self_component_sink *self_component_sink) { bt_component_class_sink_consume_method_status status = @@ -257,7 +299,7 @@ timeline_dispatch_consume(bt_self_component_sink *self_component_sink) { const bt_field *sdid_field = bt_field_structure_borrow_member_field_by_index_const(payload_field, 3); const thapi_device_id sdid = bt_field_integer_unsigned_get_value(sdid_field); - add_event_gpu(dispatch, hostname, process_id, thread_id, did, sdid, name, ts, dur); + add_event_gpu_old(dispatch, hostname, process_id, thread_id, did, sdid, name, ts, dur); } else if (std::string(class_name) == "lttng:device_flow") { const bt_field *did_field = @@ -268,10 +310,15 @@ timeline_dispatch_consume(bt_self_component_sink *self_component_sink) { bt_field_structure_borrow_member_field_by_index_const(payload_field, 3); const thapi_device_id sdid = bt_field_integer_unsigned_get_value(sdid_field); - const bt_field *flow_id_field = + const bt_field *uuid_field = bt_field_structure_borrow_member_field_by_index_const(payload_field, 6); - const long flow_id = bt_field_integer_unsigned_get_value(flow_id_field); - add_event_gpu(dispatch, hostname, process_id, thread_id, did, sdid, name, ts, dur, flow_id); + const long uuid = bt_field_integer_unsigned_get_value(uuid_field); + + const bt_field *queue_name_field = + bt_field_structure_borrow_member_field_by_index_const(payload_field, 7); + const std::string queue_name = bt_field_string_get_value(queue_name_field); + + add_event_gpu(dispatch, hostname, process_id, uuid, queue_name, did, sdid, name, ts, dur); } } diff --git a/ze/zeinterval_callbacks.cpp.erb b/ze/zeinterval_callbacks.cpp.erb index 7f3f78c2..e95c59d0 100644 --- a/ze/zeinterval_callbacks.cpp.erb +++ b/ze/zeinterval_callbacks.cpp.erb @@ -105,7 +105,10 @@ static void create_and_enqueue_host_flow_message(const char* hostname, const pro } static void create_and_enqueue_device_message( - const char* hostname, const process_id_t process_id, const thread_id_t thread_id, thapi_device_id device, + const char* hostname, const process_id_t process_id, + const uint64_t uuid, + const char* queue_name, + thapi_device_id device, const char* commandname, const char* metadata, bool err, const uint64_t globalStart, const uint64_t globalEnd, const uint64_t lltng_min, const flow_id_t flow_id, @@ -138,11 +141,11 @@ static void create_and_enqueue_device_message( device_hash = hash_device(it2->second); } /* Message creation */ - bt_message *message = create_device_message(hostname, process_id, thread_id, device_hash, subdevice_hash, - commandname, start, delta, err, metadata, - zeinterval_iter_g->dispatch->device_flow_event_class, - zeinterval_self_message_iterator_g, - zeinterval_iter_g->dispatch->stream, flow_id); + bt_message *message = create_device_flow_message(hostname, process_id, uuid, device_hash, subdevice_hash, + commandname, start, delta, err, metadata, queue_name, + zeinterval_iter_g->dispatch->device_flow_event_class, + zeinterval_self_message_iterator_g, + zeinterval_iter_g->dispatch->stream); /* Set message */ zeinterval_callbacks_state* state = (zeinterval_callbacks_state*) zeinterval_iter_g->callbacks_state; @@ -328,19 +331,19 @@ static void zeinterval_<%= dbt_event.name %>_callback( <% $profiling_apis.add("#{dbt_event.name_prefix}_#{STOP}") %> <% else dbt_event.stop? %> if ( zeResult == ZE_RESULT_SUCCESS ) { - const auto hModule = retrieve_start(state, hpt_t(hostname, process_id,thread_id)); - state->module_to_module_globals[hp_module_t(hostname, process_id, hModule)].insert((uintptr_t)pptr_val); - state->rangeset_memory_shared[hp_t(hostname, process_id)][(uintptr_t) pptr_val] = (uintptr_t)pptr_val + pSize_val; + const auto hModule = retrieve_start(state, {hostname, process_id,thread_id} ); + state->module_to_module_globals[ {hostname, process_id, hModule} ].insert((uintptr_t)pptr_val); + state->rangeset_memory_shared[ {hostname, process_id} ][(uintptr_t) pptr_val] = (uintptr_t)pptr_val + pSize_val; } <% end %> <% elsif dbt_event.name_striped == "zeModuleDestroy" and dbt_event.start? %> <% $profiling_apis.add("#{dbt_event.name_prefix}_#{START}") %> <% $profiling_apis.add("#{dbt_event.name_prefix}_#{STOP}") %> auto& s = state->module_to_module_globals; - auto it = s.find(hp_module_t(hostname, process_id, hModule)); + auto it = s.find( {hostname, process_id, hModule} ); if (it != s.end()) { for (auto &s2: it->second) - state->rangeset_memory_shared[hp_t(hostname, process_id)].erase(s2); + state->rangeset_memory_shared[ {hostname, process_id} ].erase(s2); s.erase(it); } <% elsif dbt_event.name_striped == "zeModuleCreate" and dbt_event.start? %> @@ -354,7 +357,7 @@ static void zeinterval_<%= dbt_event.name %>_callback( <% $profiling_apis.add("#{dbt_event.name_prefix}_#{STOP}") %> <% elsif ['zeMemAllocHost','zeMemAllocDevice','zeMemAllocShared'].include?(dbt_event.name_striped) and dbt_event.stop? %> if ( zeResult == ZE_RESULT_SUCCESS ) { - const auto size = retrieve_start(state, hpt_t(hostname, process_id, thread_id)); + const auto size = retrieve_start(state, {hostname, process_id, thread_id} ); <% if dbt_event.name_striped == "zeMemAllocHost" %> auto &s = state->rangeset_memory_host; <% elsif dbt_event.name_striped == "zeMemAllocDevice" %> @@ -371,8 +374,8 @@ static void zeinterval_<%= dbt_event.name %>_callback( <% $profiling_apis.add("#{dbt_event.name_prefix}_#{STOP}") %> <% elsif dbt_event.name_striped == "zeMemFree" and dbt_event.stop? %> if ( zeResult == ZE_RESULT_SUCCESS ) { - const auto ptr = retrieve_start(state, hpt_t(hostname, process_id, thread_id)); - const auto size = remove_memory(state, hp_t(hostname, process_id), ptr); + const auto ptr = retrieve_start(state, {hostname, process_id, thread_id} ); + const auto size = remove_memory(state, {hostname, process_id}, ptr); (void) size; //create_and_enqueue_traffic_message(hostname.c_str(), process_id, thread_id, "<%= dbt_event.name_striped %>", size); } @@ -455,10 +458,10 @@ Case B: <% if ['zeCommandListCreate','zeCommandListCreateImmediate'].include?(dbt_event.name_striped) %> <% $profiling_apis.add(dbt_event.name_unsanitized) %> <% if dbt_event.start? %> - save_start(state, hpt_t(hostname,process_id,thread_id), hDevice); + save_start(state, {hostname,process_id,thread_id} , hDevice); <% elsif dbt_event.stop? %> - const auto device = retrieve_start(state, hpt_t(hostname, process_id,thread_id)); - state->command_list_to_device[hp_command_list_t(hostname,process_id,phCommandList_val)] = device; + const auto device = retrieve_start(state, {hostname, process_id,thread_id} ); + state->command_list_to_device[ {hostname,process_id,phCommandList_val} ] = device; <% end %> <% end %> @@ -474,18 +477,18 @@ Case B: save_start(state, hpt_t(hostname,process_id,thread_id), std::string{desc__pKernelName_val}); <% elsif dbt_event.name_striped == "zeKernelCreate" and dbt_event.stop? %> <% $profiling_apis.add(dbt_event.name_unsanitized) %> - state->kernel_to_name[hpk_t(hostname,process_id,phKernel_val)] = retrieve_start(state, hpt_t(hostname,process_id,thread_id)); + state->kernel_to_name[ { hostname,process_id,phKernel_val} ] = retrieve_start(state, { hostname,process_id,thread_id} ); <% elsif dbt_event.name_striped == "zeKernelSetGroupSize" and dbt_event.start? %> std::stringstream groupsize; groupsize << "{" << groupSizeX << "," << groupSizeY << "," << groupSizeZ << "}"; - state->kernel_to_groupsize_str[hp_kernel_t(hostname,process_id,hKernel)] = groupsize.str(); + state->kernel_to_groupsize_str[ {hostname,process_id,hKernel} ] = groupsize.str(); <% end %> <% if dbt_event.name_unsanitized == "lttng_ust_ze_properties:kernel" %> //Not sure why SIMD size == maxSubgroupSize, but this is what Intel Tool are doing - state->kernel_to_simdsize_str[hp_kernel_t(hostname,process_id,hKernel)] = "SIMD" + std::to_string(pKernelProperties_val->maxSubgroupSize); + state->kernel_to_simdsize_str[ {hostname,process_id,hKernel} ] = "SIMD" + std::to_string(pKernelProperties_val->maxSubgroupSize); <% end %> <%# @@ -512,22 +515,26 @@ Case B: << state->kernel_to_groupsize_str[hpk]; <% elsif dbt_event.name_unsanitized.include?("Memory") and dbt_event.fields_name.any?{ |n| n.include?('ptr') } %> - const hp_t hp(hostname, process_id); - std::stringstream name_; - name_ << "<%= dbt_event.name_striped.sub("CommandListAppend",'') %>" - << "(" + const hp_t hp(hostname, process_id); + std::stringstream name_; + name_ << "<%= dbt_event.name_striped.sub("CommandListAppend",'') %>" + << "(" <% dbt_event.fields_name.filter { |n| n.include?('ptr') }.reverse.each do |n| %> << memory_location(state, hp, (uintptr_t) <%= n %>) <% end %> - << ")"; + << ")"; std::string name = name_.str(); <% else %> const std::string name{"<%= dbt_event.name_striped %>"}; <% end %> const flow_id_t flow_id = state->flow_id; state->flow_id++; - const auto device = state->command_list_to_device[ { hostname,process_id,hCommandList } ]; - state->command_partial_payload[ { hostname,process_id,thread_id } ] = { hCommandList, name, metadata.str(), device, ns_from_origin, flow_id }; + state->command_partial_payload[ { hostname,process_id,thread_id } ] = { hCommandList, name, metadata.str(), ns_from_origin, flow_id }; + <% end %> + + <% if dbt_event.start? and dbt_event.name_striped == "zeCommandQueueExecuteCommandLists" %> + for (uint32_t i=0; i < numCommandLists; i++) + state->command_list_to_command_queue[{hostname, process_id, phCommandLists_vals[i]} ] = hCommandQueue; <% end %> <%# @@ -547,10 +554,12 @@ Case B: // Best we can do is to bailout if (it_pp == state->command_partial_payload.end()) return; - const auto [hCommandList, commandname, metadata, device, lltng_min, flow_id] = it_pp->second; + const auto [hCommandList, commandname, metadata, lltng_min, flow_id] = it_pp->second; state->command_partial_payload.erase(it_pp); state->command_list_to_events[ { hostname, process_id, hCommandList } ].insert(hEvent); + const auto device = state->command_list_to_device[ { hostname,process_id,hCommandList } ]; + // Got the timestamp pair referance clock_lttng_device_t timestamp_pair_ref; const auto& m0 = state->device_timestamps_pair_ref; @@ -562,13 +571,27 @@ Case B: auto it = state->event_to_profiling_result.find(hpe); if (it == state->event_to_profiling_result.end()){ //Creeate the payload who will be used by event_profiling_results - state->event_payload[hpe] = { thread_id, commandname, metadata, device, lltng_min, timestamp_pair_ref, flow_id }; + state->event_payload[hpe] = { hCommandList, commandname, metadata, lltng_min, timestamp_pair_ref, flow_id }; } else { + + // Can we have a race condition with the command list destroy? + uint64_t uuid; + std::string queue_name; + const auto it1 = state->command_list_to_command_queue.find( { hostname,process_id, hCommandList } ); + if (it1 != state->command_list_to_command_queue.end() ) { + uuid = reinterpret_cast(it1->second); + queue_name = "zeCommandQueue"; + } else { + uuid = reinterpret_cast(hCommandList); + queue_name = "zeCommandListImmediate"; + } + // Handle the callbacks case, where `event_profiling_results` was called before `event_profiling`. const auto& [err, globalStart, globalEnd] = it->second; const auto& device_to_properties = state->device_to_properties; const auto& subdevice_parent = state->subdevice_parent; - create_and_enqueue_device_message(hostname.c_str(), process_id, thread_id, device, + + create_and_enqueue_device_message(hostname.c_str(), process_id, uuid, queue_name.c_str(), device, commandname.c_str(), metadata.c_str(), err, globalStart, globalEnd, lltng_min, flow_id, timestamp_pair_ref, @@ -582,10 +605,24 @@ Case B: auto it = state->event_payload.find(hpe); if (it != state->event_payload.end()) { - const auto& [thread_id, commandname, metadata, device, lltng_min, timestamp_pair_ref, flow_id] = it->second; + const auto& [hCommandList, commandname, metadata, lltng_min, timestamp_pair_ref, flow_id] = it->second; const auto& device_to_properties = state->device_to_properties; const auto& subdevice_parent = state->subdevice_parent; - create_and_enqueue_device_message(hostname.c_str(), process_id, thread_id, device, + + // Can we have a race condition with the command list destroy? + const auto device = state->command_list_to_device[ { hostname,process_id,hCommandList } ]; + uint64_t uuid; + std::string queue_name; + const auto it1 = state->command_list_to_command_queue.find( { hostname,process_id, hCommandList } ); + if (it1 != state->command_list_to_command_queue.end() ) { + uuid = reinterpret_cast(it1->second); + queue_name = "zeCommandQueue"; + } else { + uuid = reinterpret_cast(hCommandList); + queue_name = "zeCommandListImmediate"; + } + + create_and_enqueue_device_message(hostname.c_str(), process_id, uuid, queue_name.c_str(), device, commandname.c_str(), metadata.c_str(), err, globalStart, globalEnd, lltng_min, flow_id, timestamp_pair_ref, @@ -601,8 +638,11 @@ Case B: save_start(state, hpt_t(hostname, process_id, thread_id), hCommandList); <% else %> if (zeResult == ZE_RESULT_SUCCESS) { - const auto hCommandList = retrieve_start(state, hpt_t(hostname, process_id, thread_id)); + const auto hCommandList = retrieve_start(state, {hostname, process_id, thread_id} ); const hp_command_list_t hpl(hostname, process_id, hCommandList); + + state->command_list_to_command_queue.erase(hpl); + auto it_e = state->command_list_to_events.find(hpl); if (it_e != state->command_list_to_events.end()) { for (auto& hEvent: it_e->second) { @@ -616,7 +656,7 @@ Case B: <% end %> <% elsif ["zeEventDestroy", "zeEventHostReset"].include?(dbt_event.name_striped)%> <%if dbt_event.start? %> - save_start(state, hpt_t(hostname, process_id, thread_id), hEvent); + save_start(state, {hostname, process_id, thread_id}, hEvent); <% else %> if (zeResult == ZE_RESULT_SUCCESS) { const auto hEvent = retrieve_start(state, hpt_t(hostname, process_id, thread_id)); @@ -625,6 +665,7 @@ Case B: state->event_payload.erase(hpe); } <% end %> + <% end %> <%# @@ -633,13 +674,13 @@ Case B: | | (/_ | | | (_) | \/ | | (_| | | | (_ / %> - //Todo this should check for ZE_RESULT! + <%# Todo this should check for ZE_RESULT! #%> <% if dbt_event.start? and dbt_event.name_striped.include?("Memory") and dbt_event.fields_name.include?("size") %> <% if !dbt_event.fields_name.include?('hSignalEvent') %> //Name is set by the profiling, but some function transfert memory but cannot be profiled - //ie, prefetch, maxing memory resident... + //ie, prefetch, make memory resident... const std::string name{"<%= dbt_event.name_striped %>"}; <% else %> <% $profiling_apis.add(dbt_event.name_unsanitized) %> diff --git a/ze/zeinterval_callbacks.hpp b/ze/zeinterval_callbacks.hpp index 195f53c3..8525685d 100644 --- a/ze/zeinterval_callbacks.hpp +++ b/ze/zeinterval_callbacks.hpp @@ -21,8 +21,8 @@ typedef hp_event_t hpe_t; typedef hp_kernel_t hpk_t; typedef std::tuple clock_lttng_device_t; -typedef std::tuple t_tfnm_m_d_ts_cld_tp; -typedef std::tuple l_tfnm_m_d_ts_tp; +typedef std::tuple t_tfnm_m_d_ts_cld_tp; +typedef std::tuple l_tfnm_m_ts_tp; typedef std::tuple event_profiling_result_t; @@ -30,6 +30,8 @@ struct zeinterval_callbacks_state { std::queue downstream_message_queue; // https://spec.oneapi.io/level-zero/latest/core/api.html#_CPPv4N16ze_device_uuid_t2idE std::unordered_map command_list_to_device; + std::unordered_map command_list_to_command_queue; + std::unordered_map> command_list_to_events; std::unordered_map device_to_properties; // Kernel name & metadata @@ -48,9 +50,11 @@ struct zeinterval_callbacks_state { /* Handle variable */ std::unordered_map event_to_profiling_result; std::unordered_map device_timestamps_pair_ref; - std::unordered_map command_partial_payload; - std::unordered_map event_payload; + std::unordered_map command_partial_payload; + std::unordered_map event_payload; std::unordered_map subdevice_parent; + + flow_id_t flow_id = 1; /* Stack to get begin end */ From 443d1153d88979deef2bd9cb19a77dd9186faa1e Mon Sep 17 00:00:00 2001 From: Thomas Applencourt Date: Mon, 18 Jul 2022 19:40:24 +0000 Subject: [PATCH 03/24] Add support for dynamic array --- utils/gen_babeltrace_emitter.rb | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/utils/gen_babeltrace_emitter.rb b/utils/gen_babeltrace_emitter.rb index 008d4fee..7e26d4a2 100644 --- a/utils/gen_babeltrace_emitter.rb +++ b/utils/gen_babeltrace_emitter.rb @@ -8,7 +8,7 @@ #include EOF - +# We should refractor the class_create to take a parameter declare_signed = lambda { |_, name| puts < declare_unsigned, 'string' => declare_string, 'bool' => declare_bool, - 'structure' => declare_structure + 'structure' => declare_structure, + 'array_dynamic' => declare_array_dynamic }) def declare_group(type, method, group_name, content) @@ -65,6 +73,10 @@ def declare_group(type, method, group_name, content) content.each { |field| name = field[:name] klass = field[:class] + if klass == "array_dynamic" + element_klass = field[:field][:class] + $print_declarators[element_klass].call(element_klass, "#{name}_element") + end $print_declarators[klass].call(klass, name) append_member(name, group_name) } @@ -73,6 +85,11 @@ def declare_group(type, method, group_name, content) bt_field_class_put_ref(#{group_name}_field_class); EOF content.each { |field| + if field[:class] == "array_dynamic" + puts < Date: Mon, 18 Jul 2022 22:58:09 +0000 Subject: [PATCH 04/24] Command list flow --- utils/xprof_utils.cpp | 25 +++++++++++++---- utils/xprof_utils.hpp | 6 ++--- xprof/interval_model.yaml | 12 ++++++--- xprof/timeline.cpp | 41 +++++++++++++++++----------- ze/zeinterval_callbacks.cpp.erb | 48 ++++++++++++++++++++------------- ze/zeinterval_callbacks.hpp | 17 +++++++----- 6 files changed, 98 insertions(+), 51 deletions(-) diff --git a/utils/xprof_utils.cpp b/utils/xprof_utils.cpp index 65b8aa98..822c35d5 100644 --- a/utils/xprof_utils.cpp +++ b/utils/xprof_utils.cpp @@ -1,4 +1,5 @@ #include "xprof_utils.hpp" +#include const char* borrow_hostname(const bt_event *event){ const bt_stream *stream = bt_event_borrow_stream_const(event); @@ -22,7 +23,7 @@ thread_id_t borrow_thread_id(const bt_event *event){ bt_message* create_host_message(const char* hostname, const process_id_t process_id, const thread_id_t thread_id, const char* name, const uint64_t ts, const uint64_t duration, const bool err, - bt_event_class *event_class, bt_self_message_iterator *message_iterator, bt_stream *stream, backend_t backend, const flow_id_t flow_id) { + bt_event_class *event_class, bt_self_message_iterator *message_iterator, bt_stream *stream, backend_t backend, std::set flow_ids) { /* Message creation */ bt_message *message = bt_message_event_create( @@ -66,9 +67,19 @@ bt_message* create_host_message(const char* hostname, const process_id_t process bt_field_integer_unsigned_set_value(err_field, err); // flow id - if (flow_id != 0) { - bt_field *flow_id_field = bt_field_structure_borrow_member_field_by_index(payload_field, 3); - bt_field_integer_unsigned_set_value(flow_id_field, flow_id); + if (!flow_ids.empty() ) { + const auto size = flow_ids.size(); + bt_field *flow_id_field_length = bt_field_structure_borrow_member_field_by_index(payload_field, 3); + bt_field_integer_unsigned_set_value(flow_id_field_length, size); + + bt_field *flow_id_field = bt_field_structure_borrow_member_field_by_index(payload_field, 4); + bt_field_array_dynamic_set_length(flow_id_field, size); + + int i=0; + for (auto element: flow_ids) { + bt_field *element_field = bt_field_structure_borrow_member_field_by_index(flow_id_field, i++); + bt_field_integer_unsigned_set_value(element_field, element); + } } return message; } @@ -136,7 +147,7 @@ bt_message* create_device_message(const char* hostname, const process_id_t proce bt_message* create_device_flow_message(const char* hostname, const process_id_t process_id, const uint64_t uuid, const thapi_device_id device_id, const thapi_device_id subdevice_id, const char* name, const uint64_t ts, const uint64_t duration, const bool err, - const char* metadata, const char* queue_name, + const char* metadata, const char* queue_name, const flow_id_t flow_id, bt_event_class *event_class, bt_self_message_iterator *message_iterator, bt_stream *stream) { /* Message creation */ @@ -198,6 +209,10 @@ bt_message* create_device_flow_message(const char* hostname, const process_id_t bt_field *queue_name_field = bt_field_structure_borrow_member_field_by_index(payload_field, 7); bt_field_string_set_value(queue_name_field, queue_name); + // flow_id + bt_field *flow_field = bt_field_structure_borrow_member_field_by_index(payload_field, 8); + bt_field_integer_unsigned_set_value(flow_field, flow_id); + return message; } diff --git a/utils/xprof_utils.hpp b/utils/xprof_utils.hpp index 4af16174..4dc62b61 100644 --- a/utils/xprof_utils.hpp +++ b/utils/xprof_utils.hpp @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include #include "babeltrace2/babeltrace.h" @@ -121,7 +121,7 @@ thread_id_t borrow_thread_id(const bt_event*); bt_message* create_host_message(const char *hostname, const process_id_t, const thread_id_t, const char *name, const uint64_t ts, const uint64_t duration, const bool err, - bt_event_class*, bt_self_message_iterator*, bt_stream*, backend_t = BACKEND_UNKNOWN, flow_id_t flow_id = 0); + bt_event_class*, bt_self_message_iterator*, bt_stream*, backend_t = BACKEND_UNKNOWN, std::set flow_ids = {}); bt_message* create_device_message(const char *hostname, const process_id_t, const thread_id_t, const thapi_device_id, const thapi_device_id, const char *name, const uint64_t ts, const uint64_t duration, const bool err, const char* metadata, @@ -130,7 +130,7 @@ bt_message* create_device_message(const char *hostname, const process_id_t, cons bt_message* create_device_flow_message(const char* hostname, const process_id_t process_id, const uint64_t uuid, const thapi_device_id device_id, const thapi_device_id subdevice_id, const char* name, const uint64_t ts, const uint64_t duration, const bool err, - const char* metadata, const char* queue_name, + const char* metadata, const char* queue_name, const uint64_t flow_id, bt_event_class *event_class, bt_self_message_iterator *message_iterator, bt_stream *stream); bt_message* create_device_name_message(const char* hostname, const process_id_t process_id, diff --git a/xprof/interval_model.yaml b/xprof/interval_model.yaml index 7e88dc4c..9ed62ea7 100644 --- a/xprof/interval_model.yaml +++ b/xprof/interval_model.yaml @@ -32,8 +32,12 @@ :class: unsigned - :name: err :class: bool - - :name: flow_id + - :name: flow_ids_length :class: unsigned + - :name: flow_ids + :class: array_dynamic + :field: + :class: unsigned - :name: lttng:device :payload: - :name: name @@ -45,7 +49,7 @@ :class_properties: :preferred_display_base: 16 - :name: sdid - :class: unsigned + :class: unsigned :class_properties: :preferred_display_base: 16 - :name: err @@ -78,6 +82,8 @@ :preferred_display_base: 16 - :name: queue_name :class: string + - :name: flow_id + :class: unsigned - :name: lttng:device_name :payload: - :name: name @@ -89,4 +95,4 @@ - :name: name :class: string - :name: size - :class: unsigned + :class: unsigned diff --git a/xprof/timeline.cpp b/xprof/timeline.cpp index 681615d4..b681bb7f 100644 --- a/xprof/timeline.cpp +++ b/xprof/timeline.cpp @@ -15,7 +15,7 @@ static perfetto_uuid_t gen_perfetto_uuid() { } static void add_event_begin(struct timeline_dispatch *dispatch, perfetto_uuid_t uuid, - timestamp_t begin, std::string name, flow_id_t flow_id) { + timestamp_t begin, std::string name, std::set flow_ids = {} ) { auto *packet = dispatch->trace.add_packet(); packet->set_timestamp(begin); packet->set_trusted_packet_sequence_id(10); @@ -23,10 +23,11 @@ static void add_event_begin(struct timeline_dispatch *dispatch, perfetto_uuid_t track_event->set_type(perfetto_pruned::TrackEvent::TYPE_SLICE_BEGIN); track_event->set_name(name); track_event->set_track_uuid(uuid); - if (flow_id != 0) + for (const auto& flow_id: flow_ids) track_event->add_flow_ids(flow_id); } + static void add_event_end(struct timeline_dispatch *dispatch, perfetto_uuid_t uuid, uint64_t end) { auto *packet = dispatch->trace.add_packet(); packet->set_trusted_packet_sequence_id(10); @@ -116,13 +117,13 @@ static perfetto_uuid_t get_parent_uuid(struct timeline_dispatch *dispatch, std:: static void add_event_cpu(struct timeline_dispatch *dispatch, std::string hostname, uint64_t process_id, uint64_t thread_id, std::string name, uint64_t begin, - uint64_t dur, uint64_t flow_id = 0) { + uint64_t dur, std::set flow_ids = {}) { // Assume perfecly nessted const uint64_t end = begin + dur; perfetto_uuid_t parent_uuid = get_parent_uuid(dispatch, hostname, process_id, thread_id); // Handling perfecly nested event - add_event_begin(dispatch, parent_uuid, begin, name, flow_id); + add_event_begin(dispatch, parent_uuid, begin, name, flow_ids); std::stack &s = dispatch->uuid2stack[parent_uuid]; while ((!s.empty()) && (s.top() <= begin)) { add_event_end(dispatch, parent_uuid, s.top()); @@ -133,7 +134,7 @@ static void add_event_cpu(struct timeline_dispatch *dispatch, std::string hostna static void add_event_gpu_old(struct timeline_dispatch *dispatch, std::string hostname, uint64_t process_id, uint64_t thread_id, thapi_device_id did, - thapi_device_id sdid, std::string name, uint64_t begin, uint64_t dur, uint64_t flow_id = 0) { + thapi_device_id sdid, std::string name, uint64_t begin, uint64_t dur) { // This function Assume non perfecly nested const uint64_t end = begin + dur; perfetto_uuid_t parent_uuid = get_parent_uuid(dispatch, hostname, process_id, thread_id, did, sdid); @@ -168,14 +169,14 @@ static void add_event_gpu_old(struct timeline_dispatch *dispatch, std::string ho // Update the table m[end] = uuid; // Add event - add_event_begin(dispatch, uuid, begin, name, flow_id); + add_event_begin(dispatch, uuid, begin, name); add_event_end(dispatch, uuid, end); } static void add_event_gpu(struct timeline_dispatch *dispatch, std::string hostname, uint64_t process_id, uint64_t c_uuid, std::string queue_name, - thapi_device_id did, thapi_device_id sdid, - std::string name, uint64_t begin, uint64_t dur, uint64_t flow_id = 0) { + thapi_device_id did, thapi_device_id sdid, + std::string name, uint64_t begin, uint64_t dur, std::set flow_ids) { // This function Assume non perfecly nested const uint64_t end = begin + dur; perfetto_uuid_t parent_uuid = get_parent_uuid(dispatch, hostname, process_id, c_uuid, did, sdid); @@ -210,7 +211,7 @@ static void add_event_gpu(struct timeline_dispatch *dispatch, std::string hostna // Update the table m[end] = uuid; // Add event - add_event_begin(dispatch, uuid, begin, name, flow_id); + add_event_begin(dispatch, uuid, begin, name, flow_ids); add_event_end(dispatch, uuid, end); } @@ -285,11 +286,16 @@ timeline_dispatch_consume(bt_self_component_sink *self_component_sink) { if (std::string(class_name) == "lttng:host") { add_event_cpu(dispatch, hostname, process_id, thread_id, name, ts, dur); } else if (std::string(class_name) == "lttng:host_flow") { - - const bt_field *flow_id_field = - bt_field_structure_borrow_member_field_by_index_const(payload_field, 3); - const long flow_id = bt_field_integer_unsigned_get_value(flow_id_field); - add_event_cpu(dispatch, hostname, process_id, thread_id, name, ts, dur, flow_id); + + const bt_field *flow_ids_field = + bt_field_structure_borrow_member_field_by_index_const(payload_field, 4); + + std::set flow_ids; + for (unsigned i=0; i < bt_field_array_get_length(flow_ids_field); i++) { + const bt_field *flow_id_field = bt_field_array_borrow_element_field_by_index_const(flow_ids_field, i); + flow_ids.insert(bt_field_integer_unsigned_get_value(flow_id_field)); + } + add_event_cpu(dispatch, hostname, process_id, thread_id, name, ts, dur, flow_ids); } else if (std::string(class_name) == "lttng:device") { const bt_field *did_field = @@ -318,7 +324,12 @@ timeline_dispatch_consume(bt_self_component_sink *self_component_sink) { bt_field_structure_borrow_member_field_by_index_const(payload_field, 7); const std::string queue_name = bt_field_string_get_value(queue_name_field); - add_event_gpu(dispatch, hostname, process_id, uuid, queue_name, did, sdid, name, ts, dur); + const bt_field *flow_id_field = + bt_field_structure_borrow_member_field_by_index_const(payload_field, 8); + const uint64_t flow_id = bt_field_integer_unsigned_get_value(flow_id_field); + + + add_event_gpu(dispatch, hostname, process_id, uuid, queue_name, did, sdid, name, ts, dur, {flow_id} ); } } diff --git a/ze/zeinterval_callbacks.cpp.erb b/ze/zeinterval_callbacks.cpp.erb index e95c59d0..9ce0b926 100644 --- a/ze/zeinterval_callbacks.cpp.erb +++ b/ze/zeinterval_callbacks.cpp.erb @@ -11,6 +11,7 @@ #include #include +#include #include #include #include @@ -92,21 +93,21 @@ static void create_and_enqueue_host_message(const char* hostname, const process_ } static void create_and_enqueue_host_flow_message(const char* hostname, const process_id_t process_id, const thread_id_t thread_id, const char* name, - const uint64_t ts, const uint64_t duration, const bool err, const flow_id_t flow_id) { + const uint64_t ts, const uint64_t duration, const bool err, const std::set flow_ids) { /* Message creation */ bt_message *message = create_host_message(hostname, process_id, thread_id, name, ts, duration, err, zeinterval_iter_g->dispatch->host_flow_event_class, zeinterval_self_message_iterator_g, - zeinterval_iter_g->dispatch->stream, BACKEND_ZE, flow_id); + zeinterval_iter_g->dispatch->stream, BACKEND_ZE, flow_ids); zeinterval_callbacks_state* state = (zeinterval_callbacks_state*) zeinterval_iter_g->callbacks_state; state->downstream_message_queue.push(message); } static void create_and_enqueue_device_message( - const char* hostname, const process_id_t process_id, - const uint64_t uuid, + const char* hostname, const process_id_t process_id, + const uint64_t uuid, const char* queue_name, thapi_device_id device, const char* commandname, const char* metadata, @@ -142,7 +143,7 @@ static void create_and_enqueue_device_message( } /* Message creation */ bt_message *message = create_device_flow_message(hostname, process_id, uuid, device_hash, subdevice_hash, - commandname, start, delta, err, metadata, queue_name, + commandname, start, delta, err, metadata, queue_name, flow_id, zeinterval_iter_g->dispatch->device_flow_event_class, zeinterval_self_message_iterator_g, zeinterval_iter_g->dispatch->stream); @@ -293,16 +294,24 @@ static void zeinterval_<%= dbt_event.name %>_callback( int64_t ns_from_origin; bt_clock_snapshot_get_ns_from_origin(bt_clock, &ns_from_origin); <% if dbt_event.start? %> - state->host_start[ {hostname,process_id, thread_id, "<%= dbt_event.name_striped %>"} ] = {ns_from_origin, state->flow_id}; + state->host_start[ {hostname,process_id, thread_id, "<%= dbt_event.name_striped %>"} ] = ns_from_origin; + <% if $dbt_events_who_signal.include?(dbt_event.name_striped) %> + state->host_flow_start [ { hostname,process_id, thread_id } ] = {state->flow_id}; + <% elsif dbt_event.name_striped == "zeCommandQueueExecuteCommandLists" %> + for (uint32_t i=0; i < numCommandLists; i++) { + const auto flow_ids = state->command_list_to_flow_id[ {hostname,process_id,phCommandLists_vals[i]} ]; + state->host_flow_start[ {hostname,process_id, thread_id} ].insert(flow_ids); + } + <% end %> <% elsif dbt_event.stop? %> auto it_start = state->host_start.find( { hostname,process_id, thread_id, "<%= dbt_event.name_striped %>"} ); - const auto& [start_g, flow_id] = it_start->second; + const auto& start_g = it_start->second; state->host_start.erase(it_start); - <% if $dbt_events_who_signal.include?(dbt_event.name_striped) %> + <% if $dbt_events_who_signal.include?(dbt_event.name_striped) or dbt_event.name_striped == "zeCommandQueueExecuteCommandLists" %> + const auto& flow_ids = state->host_flow_start [ { hostname,process_id, thread_id } ]; create_and_enqueue_host_flow_message(hostname.c_str(), process_id, thread_id, "<%= dbt_event.name_striped %>", - start_g, ns_from_origin-start_g, zeResult, flow_id); + start_g, ns_from_origin-start_g, zeResult, flow_ids); <% elsif dbt_event.fields_name.include?('zeResult') %> - (void) flow_id; <%# In level0 1.0-rev-3 only those use `ZE_RESULT_NOT_READY` %> <% if dbt_event.name_striped.include?('Synchronize') or dbt_event.name_striped.include?('Query') %> create_and_enqueue_host_message(hostname.c_str(), process_id, thread_id, "<%= dbt_event.name_striped %>", @@ -497,6 +506,7 @@ Case B: | | (_) | | | | | | (_| o \_ (_) | | | | | | (_| | | (_| _| %> + <%# Map last command partial payload %> <% if dbt_event.start? and dbt_event.fields_name.include?('hSignalEvent') %> <% $profiling_apis.add("#{dbt_event.name_prefix}_#{START}") %> @@ -529,7 +539,8 @@ Case B: <% end %> const flow_id_t flow_id = state->flow_id; state->flow_id++; - state->command_partial_payload[ { hostname,process_id,thread_id } ] = { hCommandList, name, metadata.str(), ns_from_origin, flow_id }; + state->command_list_to_flow_id[ { hostname,process_id, hCommandList } ] = flow_id; + state->command_partial_payload[ { hostname,process_id,thread_id } ] = { hCommandList, name, metadata.str(), ns_from_origin}; <% end %> <% if dbt_event.start? and dbt_event.name_striped == "zeCommandQueueExecuteCommandLists" %> @@ -554,7 +565,7 @@ Case B: // Best we can do is to bailout if (it_pp == state->command_partial_payload.end()) return; - const auto [hCommandList, commandname, metadata, lltng_min, flow_id] = it_pp->second; + const auto [hCommandList, commandname, metadata, lltng_min ] = it_pp->second; state->command_partial_payload.erase(it_pp); state->command_list_to_events[ { hostname, process_id, hCommandList } ].insert(hEvent); @@ -571,14 +582,14 @@ Case B: auto it = state->event_to_profiling_result.find(hpe); if (it == state->event_to_profiling_result.end()){ //Creeate the payload who will be used by event_profiling_results - state->event_payload[hpe] = { hCommandList, commandname, metadata, lltng_min, timestamp_pair_ref, flow_id }; + state->event_payload[hpe] = { hCommandList, commandname, metadata, lltng_min, timestamp_pair_ref }; } else { // Can we have a race condition with the command list destroy? uint64_t uuid; std::string queue_name; const auto it1 = state->command_list_to_command_queue.find( { hostname,process_id, hCommandList } ); - if (it1 != state->command_list_to_command_queue.end() ) { + if (it1 != state->command_list_to_command_queue.end() ) { uuid = reinterpret_cast(it1->second); queue_name = "zeCommandQueue"; } else { @@ -590,10 +601,10 @@ Case B: const auto& [err, globalStart, globalEnd] = it->second; const auto& device_to_properties = state->device_to_properties; const auto& subdevice_parent = state->subdevice_parent; - + const auto flow_id = state-> command_list_to_flow_id[ { hostname,process_id, hCommandList } ]; create_and_enqueue_device_message(hostname.c_str(), process_id, uuid, queue_name.c_str(), device, commandname.c_str(), metadata.c_str(), - err, globalStart, globalEnd, lltng_min, flow_id, + err, globalStart, globalEnd, lltng_min, flow_id, timestamp_pair_ref, device_to_properties, subdevice_parent); } @@ -605,9 +616,10 @@ Case B: auto it = state->event_payload.find(hpe); if (it != state->event_payload.end()) { - const auto& [hCommandList, commandname, metadata, lltng_min, timestamp_pair_ref, flow_id] = it->second; + const auto& [hCommandList, commandname, metadata, lltng_min, timestamp_pair_ref] = it->second; const auto& device_to_properties = state->device_to_properties; const auto& subdevice_parent = state->subdevice_parent; + const auto flow_id = state-> command_list_to_flow_id[ { hostname,process_id, hCommandList } ]; // Can we have a race condition with the command list destroy? const auto device = state->command_list_to_device[ { hostname,process_id,hCommandList } ]; @@ -624,7 +636,7 @@ Case B: create_and_enqueue_device_message(hostname.c_str(), process_id, uuid, queue_name.c_str(), device, commandname.c_str(), metadata.c_str(), - err, globalStart, globalEnd, lltng_min, flow_id, + err, globalStart, globalEnd, lltng_min, flow_id, timestamp_pair_ref, device_to_properties, subdevice_parent); } else { diff --git a/ze/zeinterval_callbacks.hpp b/ze/zeinterval_callbacks.hpp index 8525685d..5c7cfb22 100644 --- a/ze/zeinterval_callbacks.hpp +++ b/ze/zeinterval_callbacks.hpp @@ -7,7 +7,8 @@ #include #include #include - +#include + typedef std::map MemoryInterval; typedef std::tuple hp_event_t; @@ -21,24 +22,26 @@ typedef hp_event_t hpe_t; typedef hp_kernel_t hpk_t; typedef std::tuple clock_lttng_device_t; -typedef std::tuple t_tfnm_m_d_ts_cld_tp; -typedef std::tuple l_tfnm_m_ts_tp; +typedef std::tuple t_tfnm_m_d_ts_cld_tp; +typedef std::tuple l_tfnm_m_ts_tp; typedef std::tuple event_profiling_result_t; struct zeinterval_callbacks_state { std::queue downstream_message_queue; // https://spec.oneapi.io/level-zero/latest/core/api.html#_CPPv4N16ze_device_uuid_t2idE - std::unordered_map command_list_to_device; + std::unordered_map command_list_to_device; std::unordered_map command_list_to_command_queue; - + std::unordered_map command_list_to_flow_id; std::unordered_map> command_list_to_events; + std::unordered_map device_to_properties; // Kernel name & metadata std::unordered_map kernel_to_name; std::unordered_map kernel_to_groupsize_str; std::unordered_map kernel_to_simdsize_str; - std::unordered_map> host_start; + std::unordered_map host_start; + std::unordered_map> host_flow_start; std::unordered_map profiled_function_name; /* Handle memory copy */ @@ -50,7 +53,7 @@ struct zeinterval_callbacks_state { /* Handle variable */ std::unordered_map event_to_profiling_result; std::unordered_map device_timestamps_pair_ref; - std::unordered_map command_partial_payload; + std::unordered_map command_partial_payload; std::unordered_map event_payload; std::unordered_map subdevice_parent; From f3f4a5e6ced6c5f7d0821cd9149c81060624feb4 Mon Sep 17 00:00:00 2001 From: Thomas Applencourt Date: Tue, 19 Jul 2022 23:28:41 +0000 Subject: [PATCH 05/24] Using flow as uuid --- utils/xprof_utils.cpp | 4 +- xprof/interval_model.yaml | 1 - xprof/timeline.cpp | 18 ++++++-- ze/zeinterval_callbacks.cpp.erb | 80 ++++++++++++++++----------------- ze/zeinterval_callbacks.hpp | 11 +++-- 5 files changed, 62 insertions(+), 52 deletions(-) diff --git a/utils/xprof_utils.cpp b/utils/xprof_utils.cpp index 822c35d5..f4eada52 100644 --- a/utils/xprof_utils.cpp +++ b/utils/xprof_utils.cpp @@ -69,8 +69,8 @@ bt_message* create_host_message(const char* hostname, const process_id_t process // flow id if (!flow_ids.empty() ) { const auto size = flow_ids.size(); - bt_field *flow_id_field_length = bt_field_structure_borrow_member_field_by_index(payload_field, 3); - bt_field_integer_unsigned_set_value(flow_id_field_length, size); + //bt_field *flow_id_field_length = bt_field_structure_borrow_member_field_by_index(payload_field, 3); + //bt_field_integer_unsigned_set_value(flow_id_field_length, size); bt_field *flow_id_field = bt_field_structure_borrow_member_field_by_index(payload_field, 4); bt_field_array_dynamic_set_length(flow_id_field, size); diff --git a/xprof/interval_model.yaml b/xprof/interval_model.yaml index 9ed62ea7..82a472dc 100644 --- a/xprof/interval_model.yaml +++ b/xprof/interval_model.yaml @@ -75,7 +75,6 @@ - :name: metadata :class: string - :name: uuid - :cast_type: void * :class: unsigned :class_properties: :field_value_range: 64 diff --git a/xprof/timeline.cpp b/xprof/timeline.cpp index b681bb7f..6751d987 100644 --- a/xprof/timeline.cpp +++ b/xprof/timeline.cpp @@ -14,6 +14,17 @@ static perfetto_uuid_t gen_perfetto_uuid() { return uuid++; } +static uint64_t rescale_queue_uuid(uint64_t uuid) { + static uint64_t current_queue_uuid = 0; + static std::unordered_map m; + auto ret = m.insert( { uuid, current_queue_uuid } ); + if (!ret.second) { + return ret.first->second; + } else { + return current_queue_uuid++; + } +} + static void add_event_begin(struct timeline_dispatch *dispatch, perfetto_uuid_t uuid, timestamp_t begin, std::string name, std::set flow_ids = {} ) { auto *packet = dispatch->trace.add_packet(); @@ -316,9 +327,9 @@ timeline_dispatch_consume(bt_self_component_sink *self_component_sink) { bt_field_structure_borrow_member_field_by_index_const(payload_field, 3); const thapi_device_id sdid = bt_field_integer_unsigned_get_value(sdid_field); - const bt_field *uuid_field = + const bt_field *queue_uuid_field = bt_field_structure_borrow_member_field_by_index_const(payload_field, 6); - const long uuid = bt_field_integer_unsigned_get_value(uuid_field); + const long queue_uuid = bt_field_integer_unsigned_get_value(queue_uuid_field); const bt_field *queue_name_field = bt_field_structure_borrow_member_field_by_index_const(payload_field, 7); @@ -328,8 +339,7 @@ timeline_dispatch_consume(bt_self_component_sink *self_component_sink) { bt_field_structure_borrow_member_field_by_index_const(payload_field, 8); const uint64_t flow_id = bt_field_integer_unsigned_get_value(flow_id_field); - - add_event_gpu(dispatch, hostname, process_id, uuid, queue_name, did, sdid, name, ts, dur, {flow_id} ); + add_event_gpu(dispatch, hostname, process_id, rescale_queue_uuid(queue_uuid), queue_name, did, sdid, name, ts, dur, {flow_id} ); } } diff --git a/ze/zeinterval_callbacks.cpp.erb b/ze/zeinterval_callbacks.cpp.erb index 9ce0b926..d622c63f 100644 --- a/ze/zeinterval_callbacks.cpp.erb +++ b/ze/zeinterval_callbacks.cpp.erb @@ -299,9 +299,14 @@ static void zeinterval_<%= dbt_event.name %>_callback( state->host_flow_start [ { hostname,process_id, thread_id } ] = {state->flow_id}; <% elsif dbt_event.name_striped == "zeCommandQueueExecuteCommandLists" %> for (uint32_t i=0; i < numCommandLists; i++) { - const auto flow_ids = state->command_list_to_flow_id[ {hostname,process_id,phCommandLists_vals[i]} ]; - state->host_flow_start[ {hostname,process_id, thread_id} ].insert(flow_ids); + const auto& flow_ids = state->zeCommandListAppend2flowIds[ {hostname,process_id,phCommandLists_vals[i]} ]; + state->host_flow_start[ {hostname, process_id, thread_id} ].insert(flow_ids.begin(), flow_ids.end() ); } + <% elsif dbt_event.name_striped == "zeCommandQueueSynchronize" %> + for (const auto& [hp_hCommandList, hCommandQueue2] : state->zeCommandList2zeCommandQueue) { + if (hCommandQueue2 == hCommandQueue) + state->zeCommandListAppend2flowIds.erase(hp_hCommandList); + } <% end %> <% elsif dbt_event.stop? %> auto it_start = state->host_start.find( { hostname,process_id, thread_id, "<%= dbt_event.name_striped %>"} ); @@ -507,6 +512,19 @@ Case B: _| %> + <%# Map last command partial payload %> + <% if dbt_event.stop? and dbt_event.name_striped == "zeCommandQueueCreate" %> + const flow_id_t flow_id = state->flow_id; + state->zeCommandQueueCreate2flowId[ { hostname, process_id, phCommandQueue_val } ] = flow_id; + state->flow_id++; + <% end %> + + <% if dbt_event.stop? and dbt_event.name_striped.include?("zeCommandListCreate") %> + const flow_id_t flow_id = state->flow_id; + state->zeCommandListCreate2flowId[ { hostname, process_id, phCommandList_val } ] = flow_id; + state->flow_id++; + <% end %> + <%# Map last command partial payload %> <% if dbt_event.start? and dbt_event.fields_name.include?('hSignalEvent') %> <% $profiling_apis.add("#{dbt_event.name_prefix}_#{START}") %> @@ -538,14 +556,15 @@ Case B: const std::string name{"<%= dbt_event.name_striped %>"}; <% end %> const flow_id_t flow_id = state->flow_id; + state->zeCommandListAppend2flowIds[ { hostname,process_id, hCommandList } ].insert(flow_id); + state->command_partial_payload[ { hostname,process_id,thread_id } ] = + { flow_id, name, metadata.str(), ns_from_origin, hCommandList}; state->flow_id++; - state->command_list_to_flow_id[ { hostname,process_id, hCommandList } ] = flow_id; - state->command_partial_payload[ { hostname,process_id,thread_id } ] = { hCommandList, name, metadata.str(), ns_from_origin}; <% end %> <% if dbt_event.start? and dbt_event.name_striped == "zeCommandQueueExecuteCommandLists" %> for (uint32_t i=0; i < numCommandLists; i++) - state->command_list_to_command_queue[{hostname, process_id, phCommandLists_vals[i]} ] = hCommandQueue; + state->zeCommandList2zeCommandQueue[{hostname, process_id, phCommandLists_vals[i]} ] = hCommandQueue; <% end %> <%# @@ -565,7 +584,7 @@ Case B: // Best we can do is to bailout if (it_pp == state->command_partial_payload.end()) return; - const auto [hCommandList, commandname, metadata, lltng_min ] = it_pp->second; + const auto [flow_id_kernel, commandname, metadata, lltng_min, hCommandList ] = it_pp->second; state->command_partial_payload.erase(it_pp); state->command_list_to_events[ { hostname, process_id, hCommandList } ].insert(hEvent); @@ -581,32 +600,9 @@ Case B: auto it = state->event_to_profiling_result.find(hpe); if (it == state->event_to_profiling_result.end()){ - //Creeate the payload who will be used by event_profiling_results - state->event_payload[hpe] = { hCommandList, commandname, metadata, lltng_min, timestamp_pair_ref }; + state->event_payload[hpe] = { flow_id_kernel, commandname, metadata, lltng_min, timestamp_pair_ref, hCommandList}; } else { - - // Can we have a race condition with the command list destroy? - uint64_t uuid; - std::string queue_name; - const auto it1 = state->command_list_to_command_queue.find( { hostname,process_id, hCommandList } ); - if (it1 != state->command_list_to_command_queue.end() ) { - uuid = reinterpret_cast(it1->second); - queue_name = "zeCommandQueue"; - } else { - uuid = reinterpret_cast(hCommandList); - queue_name = "zeCommandListImmediate"; - } - - // Handle the callbacks case, where `event_profiling_results` was called before `event_profiling`. - const auto& [err, globalStart, globalEnd] = it->second; - const auto& device_to_properties = state->device_to_properties; - const auto& subdevice_parent = state->subdevice_parent; - const auto flow_id = state-> command_list_to_flow_id[ { hostname,process_id, hCommandList } ]; - create_and_enqueue_device_message(hostname.c_str(), process_id, uuid, queue_name.c_str(), device, - commandname.c_str(), metadata.c_str(), - err, globalStart, globalEnd, lltng_min, flow_id, - timestamp_pair_ref, - device_to_properties, subdevice_parent); + //exit("Not implemented Error"); } <% elsif dbt_event.name_unsanitized == "lttng_ust_ze_profiling:event_profiling_results" %> @@ -616,27 +612,29 @@ Case B: auto it = state->event_payload.find(hpe); if (it != state->event_payload.end()) { - const auto& [hCommandList, commandname, metadata, lltng_min, timestamp_pair_ref] = it->second; + const auto& [flow_id_kernel, commandname, metadata, lltng_min, timestamp_pair_ref, hCommandList] = it->second; const auto& device_to_properties = state->device_to_properties; const auto& subdevice_parent = state->subdevice_parent; - const auto flow_id = state-> command_list_to_flow_id[ { hostname,process_id, hCommandList } ]; // Can we have a race condition with the command list destroy? const auto device = state->command_list_to_device[ { hostname,process_id,hCommandList } ]; - uint64_t uuid; + + // We use use the flow as a UUID + uint64_t queue_uuid; std::string queue_name; - const auto it1 = state->command_list_to_command_queue.find( { hostname,process_id, hCommandList } ); - if (it1 != state->command_list_to_command_queue.end() ) { - uuid = reinterpret_cast(it1->second); + const auto it1 = state->zeCommandList2zeCommandQueue.find( { hostname,process_id, hCommandList } ); + if (it1 != state->zeCommandList2zeCommandQueue.end() ) { + const auto& hCommandQueue = it1->second; + queue_uuid = state->zeCommandQueueCreate2flowId[ {hostname, process_id, hCommandQueue} ]; queue_name = "zeCommandQueue"; } else { - uuid = reinterpret_cast(hCommandList); + queue_uuid = state->zeCommandListCreate2flowId[ {hostname, process_id, hCommandList} ]; queue_name = "zeCommandListImmediate"; } - create_and_enqueue_device_message(hostname.c_str(), process_id, uuid, queue_name.c_str(), device, + create_and_enqueue_device_message(hostname.c_str(), process_id, queue_uuid, queue_name.c_str(), device, commandname.c_str(), metadata.c_str(), - err, globalStart, globalEnd, lltng_min, flow_id, + err, globalStart, globalEnd, lltng_min, flow_id_kernel, timestamp_pair_ref, device_to_properties, subdevice_parent); } else { @@ -653,7 +651,7 @@ Case B: const auto hCommandList = retrieve_start(state, {hostname, process_id, thread_id} ); const hp_command_list_t hpl(hostname, process_id, hCommandList); - state->command_list_to_command_queue.erase(hpl); + state->zeCommandList2zeCommandQueue.erase(hpl); auto it_e = state->command_list_to_events.find(hpl); if (it_e != state->command_list_to_events.end()) { diff --git a/ze/zeinterval_callbacks.hpp b/ze/zeinterval_callbacks.hpp index 5c7cfb22..8d514720 100644 --- a/ze/zeinterval_callbacks.hpp +++ b/ze/zeinterval_callbacks.hpp @@ -22,8 +22,8 @@ typedef hp_event_t hpe_t; typedef hp_kernel_t hpk_t; typedef std::tuple clock_lttng_device_t; -typedef std::tuple t_tfnm_m_d_ts_cld_tp; -typedef std::tuple l_tfnm_m_ts_tp; +typedef std::tuple t_tfnm_m_d_ts_cld_tp; +typedef std::tuple l_tfnm_m_ts_tp; typedef std::tuple event_profiling_result_t; @@ -31,10 +31,13 @@ struct zeinterval_callbacks_state { std::queue downstream_message_queue; // https://spec.oneapi.io/level-zero/latest/core/api.html#_CPPv4N16ze_device_uuid_t2idE std::unordered_map command_list_to_device; - std::unordered_map command_list_to_command_queue; - std::unordered_map command_list_to_flow_id; std::unordered_map> command_list_to_events; + std::unordered_map zeCommandList2zeCommandQueue; + std::unordered_map> zeCommandListAppend2flowIds; + std::unordered_map zeCommandQueueCreate2flowId; + std::unordered_map zeCommandListCreate2flowId; + std::unordered_map device_to_properties; // Kernel name & metadata std::unordered_map kernel_to_name; From b7bcb52312d8fd371621e5065e05d713f7409353 Mon Sep 17 00:00:00 2001 From: Thomas Applencourt Date: Tue, 19 Jul 2022 23:32:36 +0000 Subject: [PATCH 06/24] Add event_getter setter --- .../gen_babeltrace_event_getter_and_setter.rb | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 utils/gen_babeltrace_event_getter_and_setter.rb diff --git a/utils/gen_babeltrace_event_getter_and_setter.rb b/utils/gen_babeltrace_event_getter_and_setter.rb new file mode 100644 index 00000000..00c3de80 --- /dev/null +++ b/utils/gen_babeltrace_event_getter_and_setter.rb @@ -0,0 +1,55 @@ +require 'yaml' +model = ARGV[0] +babeltrace_model = YAML::load_file(model) + +puts < +EOF + +def set_scalar(field, index, bt2_class, name) + puts < Date: Wed, 20 Jul 2022 20:22:26 +0000 Subject: [PATCH 07/24] Fix reset flow --- .../gen_babeltrace_event_getter_and_setter.rb | 70 +++++++++++++++---- xprof/interval_model.yaml | 2 + ze/zeinterval_callbacks.cpp.erb | 7 +- 3 files changed, 62 insertions(+), 17 deletions(-) diff --git a/utils/gen_babeltrace_event_getter_and_setter.rb b/utils/gen_babeltrace_event_getter_and_setter.rb index 00c3de80..ba5f92f2 100644 --- a/utils/gen_babeltrace_event_getter_and_setter.rb +++ b/utils/gen_babeltrace_event_getter_and_setter.rb @@ -7,25 +7,71 @@ #include EOF -def set_scalar(field, index, bt2_class, name) +def set_scalar(field_name, index, bt2_class, name) puts <_callback( const auto& flow_ids = state->zeCommandListAppend2flowIds[ {hostname,process_id,phCommandLists_vals[i]} ]; state->host_flow_start[ {hostname, process_id, thread_id} ].insert(flow_ids.begin(), flow_ids.end() ); } - <% elsif dbt_event.name_striped == "zeCommandQueueSynchronize" %> - for (const auto& [hp_hCommandList, hCommandQueue2] : state->zeCommandList2zeCommandQueue) { - if (hCommandQueue2 == hCommandQueue) - state->zeCommandListAppend2flowIds.erase(hp_hCommandList); - } + <% elsif dbt_event.name_striped == "zeCommandListReset" %> + state->zeCommandListAppend2flowIds.erase( {hostname, process_id, hCommandList} ); <% end %> <% elsif dbt_event.stop? %> auto it_start = state->host_start.find( { hostname,process_id, thread_id, "<%= dbt_event.name_striped %>"} ); From 71b43d3c375984ecaaa53ec3f2b8732ef5e7d343 Mon Sep 17 00:00:00 2001 From: Thomas Applencourt Date: Thu, 21 Jul 2022 20:58:05 +0000 Subject: [PATCH 08/24] Remove comment --- xprof/Makefile.am | 4 ---- 1 file changed, 4 deletions(-) diff --git a/xprof/Makefile.am b/xprof/Makefile.am index 8d88e27b..d119f89c 100644 --- a/xprof/Makefile.am +++ b/xprof/Makefile.am @@ -48,10 +48,6 @@ libXProf_la_SOURCES = \ my_demangle.h \ $(top_srcdir)/utils/include/json.hpp -# Right now we harcode it. We should use pkgtools at some point https://github.com/nlohmann/json/blob/develop/single_include/nlohmann/json.hpp - -# -I/home/tapplencourt/tmp/proto/protobuf-3.20.1/ici/include/ -L/home/tapplencourt/tmp/proto/protobuf-3.20.1/ici/lib/ -lprotobuf - # Compiler flags libXProf_la_CPPFLAGS = -I$(top_srcdir)/utils/include -I$(srcdir)/include -I./ -I/home/tapplencourt/tmp/proto/protobuf-3.20.1/ici/include/ libXProf_la_CFLAGS = -Wall -Wextra -Wno-unused-parameter $(WERROR) $(BABELTRACE2_CFLAGS) From e0c5e3280ad01515d30bbd91aad2acb3456c42d2 Mon Sep 17 00:00:00 2001 From: Thomas Applencourt Date: Fri, 22 Jul 2022 16:53:39 +0000 Subject: [PATCH 09/24] include babeltrace_zeinterval_event_getter_and_setter --- .../gen_babeltrace_event_getter_and_setter.rb | 62 +++++++++++++------ xprof/interval_model.yaml | 6 ++ xprof/tally_utils.hpp | 2 +- ze/Makefile.am | 6 ++ ze/zeinterval_callbacks.cpp.erb | 17 +++-- 5 files changed, 66 insertions(+), 27 deletions(-) diff --git a/utils/gen_babeltrace_event_getter_and_setter.rb b/utils/gen_babeltrace_event_getter_and_setter.rb index ba5f92f2..b61414cf 100644 --- a/utils/gen_babeltrace_event_getter_and_setter.rb +++ b/utils/gen_babeltrace_event_getter_and_setter.rb @@ -9,24 +9,24 @@ def set_scalar(field_name, index, bt2_class, name) puts < to_string() = 0; - const auto to_string_size() { + auto to_string_size() { std::vector v; for (auto &e : to_string()) v.push_back(static_cast(e.size())); diff --git a/ze/Makefile.am b/ze/Makefile.am index 1b830141..8146b5a2 100644 --- a/ze/Makefile.am +++ b/ze/Makefile.am @@ -272,6 +272,9 @@ babeltrace_zeinterval_dispatchers.c: $(top_srcdir)/utils/gen_babeltrace_dispatch babeltrace_zeinterval_emitter.h: $(top_srcdir)/utils/gen_babeltrace_emitter.rb $(top_srcdir)/xprof/interval_model.yaml SRC_DIR=$(srcdir) $(RUBY) $^ zeinterval > $@ +babeltrace_zeinterval_event_getter_and_setter.hpp: $(top_srcdir)/utils/gen_babeltrace_event_getter_and_setter.rb $(top_srcdir)/xprof/interval_model.yaml + SRC_DIR=$(srcdir) $(RUBY) $^ zeinterval > $@ + babeltrace_zeinterval.h: $(top_srcdir)/xprof/interval.h.erb $(ERB) namespace="zeinterval" $< > $@ @@ -294,6 +297,7 @@ CLEANFILES += \ babeltrace_zeinterval_callbacks.h \ babeltrace_zeinterval_dispatchers.c \ babeltrace_zeinterval_emitter.h \ + babeltrace_zeinterval_event_getter_and_setter.hpp \ babeltrace_zeinterval.h \ babeltrace_zeinterval.c \ xprof_utils.hpp \ @@ -303,6 +307,7 @@ BUILT_SOURCES += \ babeltrace_zeinterval_callbacks.h \ babeltrace_zeinterval.h \ babeltrace_zeinterval_emitter.h \ + babeltrace_zeinterval_event_getter_and_setter.hpp \ xprof_utils.hpp nodist_libZEInterval_la_SOURCES = \ @@ -312,6 +317,7 @@ nodist_libZEInterval_la_SOURCES = \ babeltrace_zeinterval_dispatchers.c \ babeltrace_zeinterval_callbacks.cpp \ babeltrace_zeinterval_emitter.h \ + babeltrace_zeinterval_event_getter_and_setter.hpp \ xprof_utils.hpp \ xprof_utils.cpp diff --git a/ze/zeinterval_callbacks.cpp.erb b/ze/zeinterval_callbacks.cpp.erb index 67c17b28..3ed3915e 100644 --- a/ze/zeinterval_callbacks.cpp.erb +++ b/ze/zeinterval_callbacks.cpp.erb @@ -1,6 +1,7 @@ #include "babeltrace_zeinterval.h" #include "zeinterval_callbacks.hpp" #include "xprof_utils.hpp" +#include "babeltrace_zeinterval_event_getter_and_setter.hpp" #include #include @@ -83,10 +84,11 @@ static void create_and_enqueue_host_message(const char* hostname, const process_ const uint64_t ts, const uint64_t duration, const bool err) { /* Message creation */ - bt_message *message = create_host_message(hostname, process_id, thread_id, name, ts, duration, err, + bt_message *message = bt_message_create_lttng_host( zeinterval_iter_g->dispatch->host_event_class, zeinterval_self_message_iterator_g, - zeinterval_iter_g->dispatch->stream, BACKEND_ZE); + zeinterval_iter_g->dispatch->stream, + hostname, process_id, thread_id, ts, BACKEND_ZE, name, duration, err); zeinterval_callbacks_state* state = (zeinterval_callbacks_state*) zeinterval_iter_g->callbacks_state; state->downstream_message_queue.push(message); @@ -95,11 +97,13 @@ static void create_and_enqueue_host_message(const char* hostname, const process_ static void create_and_enqueue_host_flow_message(const char* hostname, const process_id_t process_id, const thread_id_t thread_id, const char* name, const uint64_t ts, const uint64_t duration, const bool err, const std::set flow_ids) { + std::vector flow_ids_vec(flow_ids.begin(), flow_ids.end()); /* Message creation */ - bt_message *message = create_host_message(hostname, process_id, thread_id, name, ts, duration, err, + bt_message *message = bt_message_create_lttng_host_flow( zeinterval_iter_g->dispatch->host_flow_event_class, zeinterval_self_message_iterator_g, - zeinterval_iter_g->dispatch->stream, BACKEND_ZE, flow_ids); + zeinterval_iter_g->dispatch->stream, + hostname, process_id, thread_id, ts, BACKEND_ZE, name, duration, err, flow_ids.size(), flow_ids_vec.data()); zeinterval_callbacks_state* state = (zeinterval_callbacks_state*) zeinterval_iter_g->callbacks_state; state->downstream_message_queue.push(message); @@ -157,10 +161,11 @@ static void create_and_enqueue_traffic_message(const char* hostname, const proce const uint64_t size) { /* Message creation */ - bt_message *message = create_traffic_message(hostname, process_id, thread_id, name, size, + bt_message *message = bt_message_create_lttng_traffic( zeinterval_iter_g->dispatch->traffic_event_class, zeinterval_self_message_iterator_g, - zeinterval_iter_g->dispatch->stream, BACKEND_ZE); + zeinterval_iter_g->dispatch->stream, + hostname, process_id, thread_id, 0, BACKEND_ZE, name, size); zeinterval_callbacks_state* state = (zeinterval_callbacks_state*) zeinterval_iter_g->callbacks_state; state->downstream_message_queue.push(message); From 6e003a1af2edcc0fddbf964c1bf7ca286e9e68a0 Mon Sep 17 00:00:00 2001 From: Thomas Applencourt Date: Mon, 25 Jul 2022 20:21:16 +0000 Subject: [PATCH 10/24] More uuid --- .../gen_babeltrace_event_getter_and_setter.rb | 15 +++++++++ utils/xprof_utils.hpp | 4 +++ xprof/interval_model.yaml | 11 ++++++- xprof/timeline.cpp | 28 ++++++++-------- xprof/timeline.hpp | 32 +++++++++++++++++++ 5 files changed, 74 insertions(+), 16 deletions(-) diff --git a/utils/gen_babeltrace_event_getter_and_setter.rb b/utils/gen_babeltrace_event_getter_and_setter.rb index b61414cf..c18c0efe 100644 --- a/utils/gen_babeltrace_event_getter_and_setter.rb +++ b/utils/gen_babeltrace_event_getter_and_setter.rb @@ -25,10 +25,25 @@ def set_array_dynamic(field_name, index, bt2_class, name, field) EOF end +def set_option(field_name, index, bt2_class, name, field) + puts < dsd_t; typedef std::tuple h_device_t; + +typedef std::tuple h_d_t; +typedef std::tuple h_dsd_t; + typedef std::tuple hp_t; typedef std::tuple hpt_t; typedef std::tuple hpt_function_name_t; diff --git a/xprof/interval_model.yaml b/xprof/interval_model.yaml index cc476805..7b66a651 100644 --- a/xprof/interval_model.yaml +++ b/xprof/interval_model.yaml @@ -93,8 +93,17 @@ :payload: - :name: name :class: string - - :name: did + - :name: ptr :class: unsigned + :class_properties: + :field_value_range: 64 +# Should be an option on a struct of 2 64 bits UUID +# - :name: uuid +# :class: option +# :field: +# :class: unsigned +# :class_properties: +# :field_value_range: 64 - :name: lttng:traffic :payload: - :name: name diff --git a/xprof/timeline.cpp b/xprof/timeline.cpp index 6751d987..922958f2 100644 --- a/xprof/timeline.cpp +++ b/xprof/timeline.cpp @@ -14,17 +14,6 @@ static perfetto_uuid_t gen_perfetto_uuid() { return uuid++; } -static uint64_t rescale_queue_uuid(uint64_t uuid) { - static uint64_t current_queue_uuid = 0; - static std::unordered_map m; - auto ret = m.insert( { uuid, current_queue_uuid } ); - if (!ret.second) { - return ret.first->second; - } else { - return current_queue_uuid++; - } -} - static void add_event_begin(struct timeline_dispatch *dispatch, perfetto_uuid_t uuid, timestamp_t begin, std::string name, std::set flow_ids = {} ) { auto *packet = dispatch->trace.add_packet(); @@ -81,7 +70,9 @@ static perfetto_uuid_t get_parent_uuid(struct timeline_dispatch *dispatch, std:: auto *process = track_descriptor->mutable_process(); process->set_pid(hp_uuid); std::ostringstream oss; - oss << hostname << " | Process " << process_id ; + oss << hostname; + if (process_id!=0) + oss << " | Process " << process_id ; if (did !=0) { oss << " | Device " << did; if (sdid !=0) @@ -209,7 +200,7 @@ static void add_event_gpu(struct timeline_dispatch *dispatch, std::string hostna track_descriptor->set_parent_uuid(parent_uuid); std::ostringstream oss; - oss << queue_name << " " << c_uuid; + oss << queue_name << " " << rescale_uuid(dispatch->m4[hostname][did][sdid], c_uuid); track_descriptor->set_name(oss.str()); } } else { @@ -329,7 +320,7 @@ timeline_dispatch_consume(bt_self_component_sink *self_component_sink) { const bt_field *queue_uuid_field = bt_field_structure_borrow_member_field_by_index_const(payload_field, 6); - const long queue_uuid = bt_field_integer_unsigned_get_value(queue_uuid_field); + const uint64_t queue_uuid = bt_field_integer_unsigned_get_value(queue_uuid_field); const bt_field *queue_name_field = bt_field_structure_borrow_member_field_by_index_const(payload_field, 7); @@ -339,7 +330,14 @@ timeline_dispatch_consume(bt_self_component_sink *self_component_sink) { bt_field_structure_borrow_member_field_by_index_const(payload_field, 8); const uint64_t flow_id = bt_field_integer_unsigned_get_value(flow_id_field); - add_event_gpu(dispatch, hostname, process_id, rescale_queue_uuid(queue_uuid), queue_name, did, sdid, name, ts, dur, {flow_id} ); + // Hack to put Process at 0, meaning the pointer are uniq per hostname. + const uint64_t resclated_queue_uuid = rescale_uuid(dispatch->resclate_queue_uuid, queue_uuid); + const uint64_t rescaled_did = rescale_uuid(dispatch->m2[hostname], did); + uint64_t rescaled_sdid = 0; + if (sdid != 0) + rescaled_sdid = rescale_uuid(dispatch->m3[hostname][did], sdid); + + add_event_gpu(dispatch, hostname, 0, resclated_queue_uuid, queue_name, rescaled_did, rescaled_sdid, name, ts, dur, {flow_id} ); } } diff --git a/xprof/timeline.hpp b/xprof/timeline.hpp index 90b3faa6..87019457 100644 --- a/xprof/timeline.hpp +++ b/xprof/timeline.hpp @@ -20,4 +20,36 @@ struct timeline_dispatch { std::map> uuid2stack; perfetto_pruned::Trace trace; + + std::unordered_map resclate_queue_uuid; + std::unordered_map resclate_device_uuid; + std::unordered_map resclate_subdevice_uuid; + + std::unordered_map > m2; + + + std::unordered_map > > m3; + + + std::unordered_map> > > m4; }; + + +// This only work because we never remove a value in the map +template +uint64_t rescale_uuid(std::unordered_map &m, T uuid) { + const uint64_t current_uuid = m.size() + 1 ; // Start at 1, 0 is special + auto ret = m.insert( {uuid, current_uuid } ); + if (!ret.second) { + return ret.first->second; + } else { + return current_uuid; + } +} + From d1615129bf0d1ebfb8a51967cd01182a93b8c04a Mon Sep 17 00:00:00 2001 From: Brice Videau Date: Wed, 3 Aug 2022 18:16:17 -0500 Subject: [PATCH 11/24] Prototype unified babeltrace 2 generator. --- utils/gen_babeltrace_base.rb | 322 +++++++++++++++++++++++++++++++++++ 1 file changed, 322 insertions(+) create mode 100644 utils/gen_babeltrace_base.rb diff --git a/utils/gen_babeltrace_base.rb b/utils/gen_babeltrace_base.rb new file mode 100644 index 00000000..8a3b4c46 --- /dev/null +++ b/utils/gen_babeltrace_base.rb @@ -0,0 +1,322 @@ +module Babeltrac2Gen + class BTFieldClass + def self.from_h(model) + FIELD_CLASS_NAME_MAP[model[:class]].from_h(model) + end + end + + class BTFieldClass::Bool < BTFieldClass + def self.from_h(model) + self.new + end + end + BTFieldClassBool = BTFieldClass::Bool + + class BTFieldClass::BitArray < BTFieldClass + attr_reader :length + + def intialize(length:) + @length = length + end + + def self.from_h(model) + self.new(length: model[:length]) + end + end + BTFieldClassBitArray = BTFieldClass::BitArray + + class BTFieldClass::Integer < BTFieldClass + attr_reader :field_value_range, :preferred_display_base + + def initialize(field_value_range:, preferred_display_base: 10) + @field_value_range = field_value_range + @preferred_display_base = preferred_display_base + end + + def self.from_h(model) + self.new(**model[:class_properties]) + end + end + BTFieldClassInteger = BTFieldClass::Integer + + class BTFieldClass::Integer::Unsigned < BTFieldClassInteger + end + BTFieldClass::IntegerUnsigned = BTFieldClass::Integer::Unsigned + BTFieldClassIntegerUnsigned = BTFieldClass::Integer::Unsigned + + class BTFieldClass::Integer::Signed < BTFieldClassInteger + end + BTFieldClass::IntegerSigned = BTFieldClass::Integer::Signed + BTFieldClassIntegerSigned = BTFieldClass::Integer::Signed + + class BTFieldClass::Real < BTFieldClass + def self.from_h(model) + self.new + end + end + BTFieldClassReal = BTFieldClass::Real + + class BTFieldClass::Real::SinglePrecision < BTFieldClassReal + end + BTFieldClass::RealSinglePrecision = BTFieldClass::Real::SinglePrecision + BTFieldClassRealSinglePrecision = BTFieldClass::Real::SinglePrecision + + class BTFieldClass::Real::DoublePrecision < BTFieldClassReal + end + BTFieldClass::RealDoublePrecision = BTFieldClass::Real::DoublePrecision + BTFieldClassRealDoublePrecision = BTFieldClass::Real::DoublePrecision + + module BTFieldClass::Enumeration + attr_reader :mappings + class Mapping + end + end + BTFieldClassEnumeration = BTFieldClass::Enumeration + BTFieldClassEnumerationMapping = BTFieldClass::Enumeration::Mapping + + class BTFieldClass::Enumeration::Unsigned < BTFieldClass::Integer::Unsigned + include BTFieldClass::Enumeration + class Mapping < BTFieldClass::Enumeration::Mapping + end + + def initialize(field_value_range:, preferred_display_base: 10, mappings:) + @mappings = mappings #TODO init Mapping + super(field_value_range: field_value_range, preferred_display_base: preferred_display_base) + end + + def self.from_h(model) + self.new(**model) + end + end + BTFieldClassEnumerationUnsigned = BTFieldClass::Enumeration::Unsigned + BTFieldClassEnumerationUnsignedMapping = BTFieldClass::Enumeration::Unsigned::Mapping + + class BTFieldClass::Enumeration::Signed < BTFieldClass::Integer::Signed + include BTFieldClass::Enumeration + class Mapping < BTFieldClass::Enumeration::Mapping + end + + def initialize(field_value_range:, preferred_display_base: 10, mappings:) + @mappings = mappings #TODO init Mapping + super(field_value_range: field_value_range, preferred_display_base: preferred_display_base) + end + + def self.from_h(model) + self.new(**model) + end + end + BTFieldClassEnumerationSigned = BTFieldClass::Enumeration::Signed + BTFieldClassEnumerationSignedMapping = BTFieldClass::Enumeration::Signed::Mapping + + class BTFieldClass::String < BTFieldClass + def self.from_h(model) + self.new + end + end + BTFieldClassString = BTFieldClass::String + + class BTFieldClass::Array < BTFieldClass + attr_reader :element_field_class + def initialize(element_field_class:) + @element_field_class = BTFieldClass.from_h(element_field_class) + end + end + BTFieldClassArray = BTFieldClass::Array + + class BTFieldClass::Array::Static < BTFieldClass::Array + attr_reader :length + + def initialize(element_field_class:, length:) + @length = length + super(element_field_class: element_field_class) + end + + def self.from_h(model) + self.new(element_field_class: model[:field], length: model[:length]) + end + end + BTFieldClassArrayStatic = BTFieldClass::Array::Static + + class BTFieldClass::Array::Dynamic < BTFieldClass::Array + module WithLengthField + attr_reader :length_field_path + end + + def initialize(element_field_class:, length_field_path: nil) + super(element_field_class: element_field_class) + if length_field_path + self.extend(WithLengthField) + @length_field_path = length_field_path + end + end + + def self.from_h(model) + self.new(element_field_class: model[:field], length_field_path: model[:length_field_path]) + end + end + BTFieldClassArrayDynamic = BTFieldClass::Array::Dynamic + BTFieldClassArrayDynamicWithLengthField = BTFieldClass::Array::Dynamic::WithLengthField + + class BTFieldClass::Structure < BTFieldClass + attr_reader :members + + class Member + attr_reader :name, :field_class + + def initialize(name:, field_class:) + @name = name + @field_class = BTFieldClass.from_h(field_class) + end + end + + def initialize(members:) + @members = members.collect { |m| Member.new(name: m[:name], field_class: m) } + end + + def self.from_h(model) + self.new(members: model[:members]) + end + end + BTFieldClassStructure = BTFieldClass::Structure + BTFieldClassStructureMember = BTFieldClass::Structure::Member + + class BTFieldClass::Option < BTFieldClass + attr_reader :field_class + + def initialize(field_class:) + @field_class = BTFieldClass.from_h(field_class) + end + end + BTFieldClassOption = BTFieldClass::Option + + class BTFieldClass::Option::WithoutSelectorField < BTFieldClass::Option + def self.from_h(model) + self.new(field_class: model[:field]) + end + end + BTFieldClassOptionWithoutSelectorField = BTFieldClass::Option::WithoutSelectorField + + class BTFieldClass::Option::WithSelectorField < BTFieldClass::Option + attr_reader :selector_field_path + + def initialize(field_class:, selector_field_path:) + @selector_field_path = selector_field_path + super(field_class: field_class) + end + end + BTFieldClassOptionWithSelectorField = BTFieldClass::Option::WithSelectorField + + class BTFieldClass::Option::WithSelectorField::Bool < BTFieldClass::Option::WithSelectorField + attr_reader :selector_is_reversed + + def initialize(field_class:, selector_field_path:, selector_is_reversed: nil) + @selector_is_reversed = selector_is_reversed + super(field_class: field_class, selector_field_path: selector_field_path) + end + + def self.from_h(model) + self.new(field_class: model[:field], selector_field_path: model[:selector_field_path], selector_is_reversed: [selector_field_path]) + end + end + BTFieldClassOptionWithSelectorFieldBool = BTFieldClass::Option::WithSelectorField::Bool + + class BTFieldClass::Option::WithSelectorField::IntegerUnsigned < BTFieldClass::Option::WithSelectorField + attr_reader :selector_ranges + + def initialize(field_class:, selector_field_path:, selector_ranges:) + @selector_ranges = selector_ranges + super(field_class: field_class, selector_field_path: selector_field_path) + end + + def self.from_h(model) + self.new(field_class: model[:field], selector_field_path: model[:selector_field_path], selector_ranges: model[:selector_ranges]) + end + end + BTFieldClassOptionWithSelectorFieldIntegerUnsigned = BTFieldClass::Option::WithSelectorField::IntegerUnsigned + + class BTFieldClass::Option::WithSelectorField::IntegerSigned < BTFieldClass::Option::WithSelectorField + attr_reader :selector_ranges + + def initialize(field_class:, selector_field_path:, selector_ranges:) + @selector_ranges = selector_ranges + super(field_class: field_class, selector_field_path: selector_field_path) + end + + def self.from_h(model) + self.new(field_class: model[:field], selector_field_path: model[:selector_field_path], selector_ranges: model[:selector_ranges]) + end + end + BTFieldClassOptionWithSelectorFieldIntegerSigned = BTFieldClass::Option::WithSelectorField::IntegerSigned + + class BTFieldClass::Variant < BTFieldClass + attr_reader :options + + class Option + attr_reader :name, :field_class + def initialize(name:, field_class:) + @name = name + @field_class = BTFieldClass.from_h(field_class) + end + end + + end + BTFieldClassVariant = BTFieldClass::Variant + BTFieldClassVariantOption = BTFieldClass::Variant::Option + + class BTFieldClass::Variant + module WithoutSelectorField + end + module WithSelectorField + attr_reader :selector_field_class + class Option < BTFieldClassVariantOption + attr_reader :ranges + def initialize(name:, field_class:, ranges:) + @ranges = ranges + super(name: name, field_class: field_class) + end + end + end + + def initialize(options:, selector_field_class: nil) + if selector_field_class + self.extend(WithSelectorField) + @selector_field_class = selector_field_class + @options = options.collect { |o| + BTFieldClass::Variant::WithSelectorField::Option.new(name: o[:name], field_class: o[:field], range: o[:range]) + } + else + self.extend(WithoutSelectorField) + @options = options.collect { |o| + BTFieldClass::Variant::Option.new(name: o[:name], field_class: o[:field]) + } + end + end + end + BTFieldClassVariantWithoutSelectorField = + BTFieldClass::Variant::WithoutSelectorField + BTFieldClassVariantWithSelectorField = + BTFieldClass::Variant::WithSelectorField + BTFieldClassVariantWithSelectorFieldOption = + BTFieldClass::Variant::WithSelectorField::Option + + FIELD_CLASS_NAME_MAP = { + "bool" => BTFieldClass::Bool, + "bit_array" => BTFieldClass::BitArray, + "unsigned" => BTFieldClass::Integer::Unsigned, + "signed" => BTFieldClass::Integer::Signed, + "single" => BTFieldClass::Real::SinglePrecision, + "double" => BTFieldClass::Real::DoublePrecision, + "enumeration_unsigned" => BTFieldClass::Enumeration::Unsigned, + "enumeration_signed" => BTFieldClass::Enumeration::Signed, + "string" => BTFieldClass::String, + "array_static" => BTFieldClass::Array::Static, + "array_dynamic" => BTFieldClass::Array::Dynamic, + "structure" => BTFieldClass::Structure, + "option_without_selector_field" => BTFieldClass::Option::WithoutSelectorField, + "option_with_selector_field_bool" => BTFieldClass::Option::WithSelectorField::Bool, + "option_with_selector_field_unsigned" => BTFieldClass::Option::WithSelectorField::IntegerUnsigned, + "option_with_selector_field_signed" => BTFieldClass::Option::WithSelectorField::IntegerSigned, + "variant" => BTFieldClass::Variant + } + +end From bdfa8c51b5e84f91ff7a6cfb4dfc9c1cdaa52775 Mon Sep 17 00:00:00 2001 From: Thomas Applencourt Date: Mon, 29 Aug 2022 23:41:39 +0000 Subject: [PATCH 12/24] Started working on get_declarator --- utils/gen_babeltrace_base.rb | 51 ++++++++++++++++++++++++++++++++++-- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/utils/gen_babeltrace_base.rb b/utils/gen_babeltrace_base.rb index 8a3b4c46..49db7e55 100644 --- a/utils/gen_babeltrace_base.rb +++ b/utils/gen_babeltrace_base.rb @@ -1,14 +1,23 @@ -module Babeltrac2Gen +module Babeltrace2Gen + class BTFieldClass def self.from_h(model) FIELD_CLASS_NAME_MAP[model[:class]].from_h(model) end + + def get_declarator + raise NotImplementedError + end end class BTFieldClass::Bool < BTFieldClass def self.from_h(model) self.new end + + def get_declarator(trace_class:, variable:, indent: 0) + [ "#{' '*indent}#{variable} = bt_field_class_bool_create(#{trace_class});" ] + end end BTFieldClassBool = BTFieldClass::Bool @@ -22,13 +31,18 @@ def intialize(length:) def self.from_h(model) self.new(length: model[:length]) end + + def get_declarator(trace_class:, variable:, indent: 0) + [ "#{' '*indent}#{variable} = bt_field_class_bit_array_create(#{trace_class}, #{@length});" ] + end + end BTFieldClassBitArray = BTFieldClass::BitArray class BTFieldClass::Integer < BTFieldClass attr_reader :field_value_range, :preferred_display_base - def initialize(field_value_range:, preferred_display_base: 10) + def initialize(field_value_range: 64, preferred_display_base: 10) @field_value_range = field_value_range @preferred_display_base = preferred_display_base end @@ -36,15 +50,28 @@ def initialize(field_value_range:, preferred_display_base: 10) def self.from_h(model) self.new(**model[:class_properties]) end + + def get_declarator(trace_class: nil, variable:, indent: 0) + indent_str = ' '*indent + a = [] + a << "#{indent_str}bt_field_class_integer_set_field_value_range(#{variable}, #{@field_value_range});" if @field_value_range != 64 + a << "#{indent_str}bt_field_class_integer_set_preferred_display_base(#{variable}, #{@preferred_display_base});" if @preferred_display_base != 10 + end end BTFieldClassInteger = BTFieldClass::Integer class BTFieldClass::Integer::Unsigned < BTFieldClassInteger + def get_declarator(trace_class:, variable:, indent: 0) + [ "#{' '*indent}#{variable} = bt_field_class_integer_unsigned_create(#{trace_class});" ] + super + end end BTFieldClass::IntegerUnsigned = BTFieldClass::Integer::Unsigned BTFieldClassIntegerUnsigned = BTFieldClass::Integer::Unsigned class BTFieldClass::Integer::Signed < BTFieldClassInteger + def get_declarator(trace_class:, variable:, indent: 0) + [ "#{' '*indent}#{variable} = bt_field_class_integer_signed_create(#{trace_class});" ] + super + end end BTFieldClass::IntegerSigned = BTFieldClass::Integer::Signed BTFieldClassIntegerSigned = BTFieldClass::Integer::Signed @@ -57,11 +84,17 @@ def self.from_h(model) BTFieldClassReal = BTFieldClass::Real class BTFieldClass::Real::SinglePrecision < BTFieldClassReal + def get_declarator(trace_class:, variable:, indent: 0) + [ "#{' '*indent}#{variable} = bt_field_class_real_single_precision_create(#{trace_class});" ] + end end BTFieldClass::RealSinglePrecision = BTFieldClass::Real::SinglePrecision BTFieldClassRealSinglePrecision = BTFieldClass::Real::SinglePrecision class BTFieldClass::Real::DoublePrecision < BTFieldClassReal + def get_declarator(trace_class:, variable:, indent: 0) + [ "#{' '*indent}#{variable} = bt_field_class_real_double_precision_create(#{trace_class});" ] + end end BTFieldClass::RealDoublePrecision = BTFieldClass::Real::DoublePrecision BTFieldClassRealDoublePrecision = BTFieldClass::Real::DoublePrecision @@ -134,6 +167,20 @@ def initialize(element_field_class:, length:) def self.from_h(model) self.new(element_field_class: model[:field], length: model[:length]) end + + def get_declarator(trace_class:, variable:, indent: 0) + a = ["#{' '*(indent)}{" ] + indent += 2 + + indent_str = ' '*(indent); + element_field_class_variable= "#{variable}_field_class"; + a << "#{indent_str}bt_field_class *#{element_field_class_variable};" + a += @element_field_class.get_declarator(trace_class: trace_class, variable: element_field_class_variable, indent: indent) + a << "#{indent_str}#{variable} = bt_field_class_array_static_create(#{trace_class}, #{element_field_class_variable}, #{@length});" + + indent -= 2 + a << "#{' '*(indent)}}" + end end BTFieldClassArrayStatic = BTFieldClass::Array::Static From eb86f50d34191eeeeebaf5827fbc7a0765532d78 Mon Sep 17 00:00:00 2001 From: Thomas Applencourt Date: Fri, 16 Sep 2022 22:53:25 +0000 Subject: [PATCH 13/24] generator refractor --- utils/gen_babeltrace_base.rb | 385 ++++++++++++++++++++++++++--------- 1 file changed, 291 insertions(+), 94 deletions(-) diff --git a/utils/gen_babeltrace_base.rb b/utils/gen_babeltrace_base.rb index 49db7e55..09d3de22 100644 --- a/utils/gen_babeltrace_base.rb +++ b/utils/gen_babeltrace_base.rb @@ -1,22 +1,179 @@ module Babeltrace2Gen + module BTPrinter + @@output = "" + @@indent = 0 + INDENT_INCREMENT = " " + + def pr(str) + @@output << INDENT_INCREMENT * @@indent << str << "\n" + end + module_function :pr + + def scope(&block) + pr "{" + @@indent += 1 + block.call + @@indent -= 1 + pr "}" + end + module_function :scope + end + + module BTLocator + attr_reader :parent + attr_reader :variable + def root + @parent ? @parent.root : self + end + + def stream_class + @parent ? @parent.stream_class : nil + end + + def trace_class + @parent ? @parent.trace_class : nil + end + + def event_class + @parent ? @parent.event_class : nil + end + + def find_field_class_path(path, variable) + path.scan(/\["(\w+)"\]|\[(\d+)\]/).each { |m| + # String + if m.first + pr "#{variable} = bt_field_class_structure_borrow_member_by_name(#{variable}, \"#{m.first}\");" + else + pr "#{variable} = bt_field_class_structure_borrow_member_by_index(#{variable}, #{m.last});" + end + } + end + + def find_field_class(path, variable) + m = path.match(/\A(PACKET_CONTEXT|EVENT_COMMON_CONTEXT|EVENT_SPECIFIC_CONTEXT|EVENT_PAYLOAD)(.*)/) + case m[1] + when "PACKET_CONTEXT" + pr "#{variable} = #{stream_class.packet_context_field_class.variable};" + when "EVENT_COMMON_CONTEXT" + pr "#{variable} = #{stream_class.event_common_context_field_class.variable};" + when "EVENT_SPECIFIC_CONTEXT" + pr "#{variable} = #{event_class.specific_context_field_class.varible};" + when "EVENT_PAYLOAD" + pr "#{variable} = #{event_class.payload_field_class.variable};" + else + raise "invalid path #{path}" + end + find_field_class_path(m[2], variable) + end + + end + + class BTTraceClass + include BTLocator + attr_reader :stream_classes, :assigns_automatic_stream_class_id + + def initialize(parent:, stream_classes:, assigns_automatic_stream_class_id: true) + raise if parent + @parent = nil + @stream_classes = stream_classes.collect{ |m| BTStreamClass.new(self, **m) } + end + + def self.from_h(parent, model) + self.new(parent: parent, **model) + end + + def trace_class + self + end + + end + + class BTStreamClass + include BTLocator + attr_reader :packet_context_field_class, :event_common_context_field_class, :event_classes + def initialize(parent:, packet_context_field_class: nil, event_common_context_field_class: nil, event_classes: []) + @parent = parent + @packet_context_field_class = BTFieldClass.from_h(self, **packet_context_field_class) if packet_context_field_class + @event_common_context_field_class = BTFieldClass.from_h(self, **event_common_context_field_class) if event_common_context_field_class + @event_classes = event_classes.collect { |ec| BTEventClass.from_h(self, **ec) } + end + + def self.from_h(parent, model) + self.new(parent: parent, **model) + end + + def stream_class + self + end + + end + + class BTEventClass + include BTLocator + include BTPrinter + attr_reader :name, :specific_context_field_class, :payload_field_class + def initialize(parent:, name:, specific_context_field_class: nil, payload_field_class: nil) + @parent = parent + @name = name + @specific_context_field_class = BTFieldClass.from_h(self, **specific_context_field_class) if specific_context_field_class + @payload_field_class = BTFieldClass.from_h(self, **payload_field_class) if payload_field_class + end + + def self.from_h(parent, model) + self.new(parent: parent, **model) + end + + def get_declarator(trace_class:, variable:, stream_class:) + @variable = variable + scope { + pr "#{variable} = bt_event_class_create(#{stream_class});" + pr "bt_event_class_set_name(#{variable}, \"#{@name}\");" + if @specific_context_field_class + var_name = "#{variable}_sc_fc" + pr "bt_field_class *#{var_name};" + @specific_context_field_class.get_declarator(trace_class: trace_class, variable: var_name) + pr "bt_event_class_set_specific_context_field_class(#{variable}, #{var_name});" + end + + if @payload_field_class + var_name = "#{variable}_p_fc" + pr "bt_field_class *#{var_name};" + @payload_field_class.get_declarator(trace_class: trace_class, variable: var_name) + pr "bt_event_class_set_payload_field_class(#{variable}, #{var_name});" + end + } + end + + def event_class + self + end + end class BTFieldClass - def self.from_h(model) - FIELD_CLASS_NAME_MAP[model[:class]].from_h(model) + include BTLocator + include BTPrinter + + def initialize(parent:) + @parent = parent + end + + def self.from_h(parent, model) + FIELD_CLASS_NAME_MAP[model[:class]].from_h(parent, model) end def get_declarator raise NotImplementedError end + end class BTFieldClass::Bool < BTFieldClass - def self.from_h(model) - self.new + def self.from_h(parent, model) + self.new(parent: parent) end - def get_declarator(trace_class:, variable:, indent: 0) - [ "#{' '*indent}#{variable} = bt_field_class_bool_create(#{trace_class});" ] + def get_declarator(trace_class:, variable:) + pr "#{variable} = bt_field_class_bool_create(#{trace_class});" end end BTFieldClassBool = BTFieldClass::Bool @@ -24,16 +181,17 @@ def get_declarator(trace_class:, variable:, indent: 0) class BTFieldClass::BitArray < BTFieldClass attr_reader :length - def intialize(length:) + def initialize(parent:, length:) + @parent = parent @length = length end - def self.from_h(model) - self.new(length: model[:length]) + def self.from_h(parent, model) + self.new(parent: parent, length: model[:length]) end - def get_declarator(trace_class:, variable:, indent: 0) - [ "#{' '*indent}#{variable} = bt_field_class_bit_array_create(#{trace_class}, #{@length});" ] + def get_declarator(trace_class:, variable:) + pr "#{variable} = bt_field_class_bit_array_create(#{trace_class}, #{@length});" end end @@ -42,58 +200,59 @@ def get_declarator(trace_class:, variable:, indent: 0) class BTFieldClass::Integer < BTFieldClass attr_reader :field_value_range, :preferred_display_base - def initialize(field_value_range: 64, preferred_display_base: 10) + def initialize(parent:, field_value_range: 64, preferred_display_base: 10) + @parent = parent @field_value_range = field_value_range @preferred_display_base = preferred_display_base end - def self.from_h(model) - self.new(**model[:class_properties]) + def self.from_h(parent, model) + self.new(parent: parent, **model[:class_properties]) end - def get_declarator(trace_class: nil, variable:, indent: 0) - indent_str = ' '*indent - a = [] - a << "#{indent_str}bt_field_class_integer_set_field_value_range(#{variable}, #{@field_value_range});" if @field_value_range != 64 - a << "#{indent_str}bt_field_class_integer_set_preferred_display_base(#{variable}, #{@preferred_display_base});" if @preferred_display_base != 10 + def get_declarator(trace_class: nil, variable:) + pr "bt_field_class_integer_set_field_value_range(#{variable}, #{@field_value_range});" if @field_value_range != 64 + pr "bt_field_class_integer_set_preferred_display_base(#{variable}, #{@preferred_display_base});" if @preferred_display_base != 10 end end BTFieldClassInteger = BTFieldClass::Integer class BTFieldClass::Integer::Unsigned < BTFieldClassInteger - def get_declarator(trace_class:, variable:, indent: 0) - [ "#{' '*indent}#{variable} = bt_field_class_integer_unsigned_create(#{trace_class});" ] + super + def get_declarator(trace_class:, variable:) + pr "#{variable} = bt_field_class_integer_unsigned_create(#{trace_class});" + super end end BTFieldClass::IntegerUnsigned = BTFieldClass::Integer::Unsigned BTFieldClassIntegerUnsigned = BTFieldClass::Integer::Unsigned class BTFieldClass::Integer::Signed < BTFieldClassInteger - def get_declarator(trace_class:, variable:, indent: 0) - [ "#{' '*indent}#{variable} = bt_field_class_integer_signed_create(#{trace_class});" ] + super + def get_declarator(trace_class:, variable:) + pr "#{variable} = bt_field_class_integer_signed_create(#{trace_class});" + super end end BTFieldClass::IntegerSigned = BTFieldClass::Integer::Signed BTFieldClassIntegerSigned = BTFieldClass::Integer::Signed class BTFieldClass::Real < BTFieldClass - def self.from_h(model) - self.new + def self.from_h(parent, model) + self.new(parent: parent) end end BTFieldClassReal = BTFieldClass::Real class BTFieldClass::Real::SinglePrecision < BTFieldClassReal - def get_declarator(trace_class:, variable:, indent: 0) - [ "#{' '*indent}#{variable} = bt_field_class_real_single_precision_create(#{trace_class});" ] + def get_declarator(trace_class:, variable:) + pr "#{variable} = bt_field_class_real_single_precision_create(#{trace_class});" end end BTFieldClass::RealSinglePrecision = BTFieldClass::Real::SinglePrecision BTFieldClassRealSinglePrecision = BTFieldClass::Real::SinglePrecision class BTFieldClass::Real::DoublePrecision < BTFieldClassReal - def get_declarator(trace_class:, variable:, indent: 0) - [ "#{' '*indent}#{variable} = bt_field_class_real_double_precision_create(#{trace_class});" ] + def get_declarator(trace_class:, variable:) + pr "#{variable} = bt_field_class_real_double_precision_create(#{trace_class});" end end BTFieldClass::RealDoublePrecision = BTFieldClass::Real::DoublePrecision @@ -112,13 +271,13 @@ class BTFieldClass::Enumeration::Unsigned < BTFieldClass::Integer::Unsigned class Mapping < BTFieldClass::Enumeration::Mapping end - def initialize(field_value_range:, preferred_display_base: 10, mappings:) + def initialize(parent:, field_value_range:, preferred_display_base: 10, mappings:) @mappings = mappings #TODO init Mapping - super(field_value_range: field_value_range, preferred_display_base: preferred_display_base) + super(parent: parent, field_value_range: field_value_range, preferred_display_base: preferred_display_base) end - def self.from_h(model) - self.new(**model) + def self.from_h(parent, model) + self.new(parent: parent, **model) end end BTFieldClassEnumerationUnsigned = BTFieldClass::Enumeration::Unsigned @@ -129,29 +288,30 @@ class BTFieldClass::Enumeration::Signed < BTFieldClass::Integer::Signed class Mapping < BTFieldClass::Enumeration::Mapping end - def initialize(field_value_range:, preferred_display_base: 10, mappings:) + def initialize(parent:, field_value_range:, preferred_display_base: 10, mappings:) @mappings = mappings #TODO init Mapping - super(field_value_range: field_value_range, preferred_display_base: preferred_display_base) + super(parent: parent, field_value_range: field_value_range, preferred_display_base: preferred_display_base) end - def self.from_h(model) - self.new(**model) + def self.from_h(parent, model) + self.new(parent: parent, **model) end end BTFieldClassEnumerationSigned = BTFieldClass::Enumeration::Signed BTFieldClassEnumerationSignedMapping = BTFieldClass::Enumeration::Signed::Mapping class BTFieldClass::String < BTFieldClass - def self.from_h(model) - self.new + def self.from_h(parent, model) + self.new(parent: parent) end end BTFieldClassString = BTFieldClass::String class BTFieldClass::Array < BTFieldClass attr_reader :element_field_class - def initialize(element_field_class:) - @element_field_class = BTFieldClass.from_h(element_field_class) + def initialize(parent:, element_field_class:) + @parent = parent + @element_field_class = BTFieldClass.from_h(self, element_field_class) end end BTFieldClassArray = BTFieldClass::Array @@ -159,27 +319,22 @@ def initialize(element_field_class:) class BTFieldClass::Array::Static < BTFieldClass::Array attr_reader :length - def initialize(element_field_class:, length:) + def initialize(parent:, element_field_class:, length:) @length = length - super(element_field_class: element_field_class) + super(parent: parent, element_field_class: element_field_class) end - def self.from_h(model) - self.new(element_field_class: model[:field], length: model[:length]) + def self.from_h(parent, model) + self.new(parent: parent, element_field_class: model[:field], length: model[:length]) end - def get_declarator(trace_class:, variable:, indent: 0) - a = ["#{' '*(indent)}{" ] - indent += 2 - - indent_str = ' '*(indent); - element_field_class_variable= "#{variable}_field_class"; - a << "#{indent_str}bt_field_class *#{element_field_class_variable};" - a += @element_field_class.get_declarator(trace_class: trace_class, variable: element_field_class_variable, indent: indent) - a << "#{indent_str}#{variable} = bt_field_class_array_static_create(#{trace_class}, #{element_field_class_variable}, #{@length});" - - indent -= 2 - a << "#{' '*(indent)}}" + def get_declarator(trace_class:, variable:) + scope { + element_field_class_variable = "#{variable}_field_class" + pr "bt_field_class *#{element_field_class_variable};" + @element_field_class.get_declarator(trace_class: trace_class, variable: element_field_class_variable) + pr "#{variable} = bt_field_class_array_static_create(#{trace_class}, #{element_field_class_variable}, #{@length});" + } end end BTFieldClassArrayStatic = BTFieldClass::Array::Static @@ -189,56 +344,96 @@ module WithLengthField attr_reader :length_field_path end - def initialize(element_field_class:, length_field_path: nil) - super(element_field_class: element_field_class) + def initialize(parent:, element_field_class:, length_field_path: nil) + super(parent: parent, element_field_class: element_field_class) if length_field_path self.extend(WithLengthField) @length_field_path = length_field_path end end - def self.from_h(model) - self.new(element_field_class: model[:field], length_field_path: model[:length_field_path]) + def self.from_h(parent, model) + self.new(parent: parent, element_field_class: model[:field], length_field_path: model[:length_field_path]) + end + + def get_declarator(trace_class:, variable:) + scope { + element_field_class_variable= "#{variable}_field_class" + pr "bt_field_class *#{element_field_class_variable};" + @element_field_class.get_declarator(trace_class: trace_class, variable: element_field_class_variable) + if @length_field_path + element_field_class_variable_length = "#{element_field_class_variable}_length" + pr "bt_field_class *#{element_field_class_variable_length};" + find_field_class(@length_field_path, element_field_class_variable_length) + pr "#{variable} = bt_field_class_array_dynamic_create(#{trace_class}, #{element_field_class_variable}, #{element_field_class_variable_length});" + else + pr "#{variable} = bt_field_class_array_dynamic_create(#{trace_class}, #{element_field_class_variable}, NULL);" + end + } end + end BTFieldClassArrayDynamic = BTFieldClass::Array::Dynamic BTFieldClassArrayDynamicWithLengthField = BTFieldClass::Array::Dynamic::WithLengthField - class BTFieldClass::Structure < BTFieldClass attr_reader :members - class Member + include BTLocator + attr_reader :parent attr_reader :name, :field_class - def initialize(name:, field_class:) + def initialize(parent:, name:, field_class:) + @parent = parent @name = name - @field_class = BTFieldClass.from_h(field_class) + @field_class = BTFieldClass.from_h(self, field_class) end end - - def initialize(members:) - @members = members.collect { |m| Member.new(name: m[:name], field_class: m) } + + def initialize(parent:, members:) + @parent = parent + @members = members.collect { |m| Member.new(parent: self, name: m[:name], field_class: m) } + end + + def [](index) + case index + when Integer + @members[index] + when String + @members.find { |m| m.name == index } + end end - def self.from_h(model) - self.new(members: model[:members]) + def self.from_h(parent, model) + self.new(parent: parent, members: model[:members]) + end + + def get_declarator(trace_class:, variable:) + @variable = variable + scope { + pr "#{variable} = bt_field_class_structure_create(#{trace_class});" + @members.each { |m| + pr "bt_field_class *#{variable}_#{m.name};" + m.field_class.get_declarator(trace_class: trace_class, variable: "#{variable}_#{m.name}") + pr "bt_field_class_structure_append_member(#{variable}, \"#{m.name}\", #{variable}_#{m.name});" + } + } end end BTFieldClassStructure = BTFieldClass::Structure BTFieldClassStructureMember = BTFieldClass::Structure::Member - class BTFieldClass::Option < BTFieldClass attr_reader :field_class - def initialize(field_class:) - @field_class = BTFieldClass.from_h(field_class) + def initialize(parent:, field_class:) + @parent = parent + @field_class = BTFieldClass.from_h(self, field_class) end end BTFieldClassOption = BTFieldClass::Option class BTFieldClass::Option::WithoutSelectorField < BTFieldClass::Option - def self.from_h(model) - self.new(field_class: model[:field]) + def self.from_h(parent, model) + self.new(parent: parent, field_class: model[:field]) end end BTFieldClassOptionWithoutSelectorField = BTFieldClass::Option::WithoutSelectorField @@ -246,9 +441,9 @@ def self.from_h(model) class BTFieldClass::Option::WithSelectorField < BTFieldClass::Option attr_reader :selector_field_path - def initialize(field_class:, selector_field_path:) + def initialize(parent:, field_class:, selector_field_path:) @selector_field_path = selector_field_path - super(field_class: field_class) + super(parent: parent, field_class: field_class) end end BTFieldClassOptionWithSelectorField = BTFieldClass::Option::WithSelectorField @@ -256,13 +451,13 @@ def initialize(field_class:, selector_field_path:) class BTFieldClass::Option::WithSelectorField::Bool < BTFieldClass::Option::WithSelectorField attr_reader :selector_is_reversed - def initialize(field_class:, selector_field_path:, selector_is_reversed: nil) + def initialize(parent:, field_class:, selector_field_path:, selector_is_reversed: nil) @selector_is_reversed = selector_is_reversed - super(field_class: field_class, selector_field_path: selector_field_path) + super(parent: parent, field_class: field_class, selector_field_path: selector_field_path) end - def self.from_h(model) - self.new(field_class: model[:field], selector_field_path: model[:selector_field_path], selector_is_reversed: [selector_field_path]) + def self.from_h(parent, model) + self.new(parent: parent, field_class: model[:field], selector_field_path: model[:selector_field_path], selector_is_reversed: [selector_field_path]) end end BTFieldClassOptionWithSelectorFieldBool = BTFieldClass::Option::WithSelectorField::Bool @@ -270,13 +465,13 @@ def self.from_h(model) class BTFieldClass::Option::WithSelectorField::IntegerUnsigned < BTFieldClass::Option::WithSelectorField attr_reader :selector_ranges - def initialize(field_class:, selector_field_path:, selector_ranges:) + def initialize(parent:, field_class:, selector_field_path:, selector_ranges:) @selector_ranges = selector_ranges - super(field_class: field_class, selector_field_path: selector_field_path) + super(parent: parent, field_class: field_class, selector_field_path: selector_field_path) end - def self.from_h(model) - self.new(field_class: model[:field], selector_field_path: model[:selector_field_path], selector_ranges: model[:selector_ranges]) + def self.from_h(parent, model) + self.new(parent: parent, field_class: model[:field], selector_field_path: model[:selector_field_path], selector_ranges: model[:selector_ranges]) end end BTFieldClassOptionWithSelectorFieldIntegerUnsigned = BTFieldClass::Option::WithSelectorField::IntegerUnsigned @@ -284,13 +479,13 @@ def self.from_h(model) class BTFieldClass::Option::WithSelectorField::IntegerSigned < BTFieldClass::Option::WithSelectorField attr_reader :selector_ranges - def initialize(field_class:, selector_field_path:, selector_ranges:) + def initialize(parent:, field_class:, selector_field_path:, selector_ranges:) @selector_ranges = selector_ranges - super(field_class: field_class, selector_field_path: selector_field_path) + super(parent: parent, field_class: field_class, selector_field_path: selector_field_path) end - def self.from_h(model) - self.new(field_class: model[:field], selector_field_path: model[:selector_field_path], selector_ranges: model[:selector_ranges]) + def self.from_h(parent, model) + self.new(parent: parent, field_class: model[:field], selector_field_path: model[:selector_field_path], selector_ranges: model[:selector_ranges]) end end BTFieldClassOptionWithSelectorFieldIntegerSigned = BTFieldClass::Option::WithSelectorField::IntegerSigned @@ -300,9 +495,10 @@ class BTFieldClass::Variant < BTFieldClass class Option attr_reader :name, :field_class - def initialize(name:, field_class:) + def initialize(parent:, name:, field_class:) + @parent = parent @name = name - @field_class = BTFieldClass.from_h(field_class) + @field_class = BTFieldClass.from_h(self, field_class) end end @@ -317,14 +513,15 @@ module WithSelectorField attr_reader :selector_field_class class Option < BTFieldClassVariantOption attr_reader :ranges - def initialize(name:, field_class:, ranges:) + def initialize(parent:, name:, field_class:, ranges:) @ranges = ranges - super(name: name, field_class: field_class) + super(parent: parent, name: name, field_class: field_class) end end end - def initialize(options:, selector_field_class: nil) + def initialize(parent:, options:, selector_field_class: nil) + @parent = parent if selector_field_class self.extend(WithSelectorField) @selector_field_class = selector_field_class From 29f480d6f44c1bf47ed942ab87628e415faf7532 Mon Sep 17 00:00:00 2001 From: Thomas Applencourt Date: Tue, 20 Sep 2022 22:57:37 +0000 Subject: [PATCH 14/24] Add shortcut --- utils/gen_babeltrace_base.rb | 53 ++++++++++++++++++++++++++++++------ 1 file changed, 45 insertions(+), 8 deletions(-) diff --git a/utils/gen_babeltrace_base.rb b/utils/gen_babeltrace_base.rb index 09d3de22..f7694875 100644 --- a/utils/gen_babeltrace_base.rb +++ b/utils/gen_babeltrace_base.rb @@ -70,12 +70,14 @@ def find_field_class(path, variable) class BTTraceClass include BTLocator + include BTPrinter attr_reader :stream_classes, :assigns_automatic_stream_class_id def initialize(parent:, stream_classes:, assigns_automatic_stream_class_id: true) raise if parent @parent = nil - @stream_classes = stream_classes.collect{ |m| BTStreamClass.new(self, **m) } + @assigns_automatic_stream_class_id = assigns_automatic_stream_class_id + @stream_classes = stream_classes.collect{ |m| BTStreamClass.from_h(self, m) } end def self.from_h(parent, model) @@ -86,15 +88,42 @@ def trace_class self end + def get_declarator(self_component:, variable:) + scope { + pr "bt_trace_class *#{variable} = bt_trace_class_create(#{self_component});" + @stream_classes.each_with_index { |m,i| + stream_class_name = "bt_stream_class_#{i}" + pr "bt_stream_class *#{stream_class_name};" + auto_id = @assigns_automatic_stream_class_id ? "true" : "false" + pr "bt_stream_class_set_assigns_automatic_stream_id(#{stream_class_name}, #{auto_id});" + m.get_declarator(trace_class: variable, variable: stream_class_name) + } + } + end + end class BTStreamClass include BTLocator - attr_reader :packet_context_field_class, :event_common_context_field_class, :event_classes - def initialize(parent:, packet_context_field_class: nil, event_common_context_field_class: nil, event_classes: []) + include BTPrinter + attr_reader :packet_context_field_class, :packet_context + attr_reader :event_common_context_field_class, :event_common_context, :event_classes, :id + def initialize(parent:, packet_context_field_class: nil, packet_context: nil, + event_common_context_field_class: nil, event_common_context: nil, event_classes: [], id: nil) @parent = parent - @packet_context_field_class = BTFieldClass.from_h(self, **packet_context_field_class) if packet_context_field_class - @event_common_context_field_class = BTFieldClass.from_h(self, **event_common_context_field_class) if event_common_context_field_class + raise "Incoherent scheme for IDs" if (id.nil? != @parent.assigns_automatic_stream_class_id) + @id = id + + raise "Two packet_context" if packet_context_field_class and packet_context + # Should put assert to check for struct + @packet_context_field_class = BTFieldClass.from_h(self, packet_context_field_class) if packet_context_field_class + @packet_context_field_class = BTFieldClass.from_h(self, {:class => "structure", :members => packet_context} ) if packet_context + + raise "Two event_common_context" if event_common_context_field_class and event_common_context + # Should put assert to check for struct + @event_common_context_field_class = BTFieldClass.from_h(self, event_common_context_field_class) if event_common_context_field_class + @event_common_context_field_class = BTFieldClass.from_h(self, {:class => "structure", :members => event_common_context}) if event_common_context + @event_classes = event_classes.collect { |ec| BTEventClass.from_h(self, **ec) } end @@ -106,6 +135,13 @@ def stream_class self end + def get_declarator(trace_class:, variable:) + if id.nil? + pr "#{variable} = bt_stream_class_create(#{trace_class});" + else + pr "#{variable} = bt_stream_class_create_with_id(#{trace_class}, #{@id});" + end + end end class BTEventClass @@ -158,7 +194,8 @@ def initialize(parent:) end def self.from_h(parent, model) - FIELD_CLASS_NAME_MAP[model[:class]].from_h(parent, model) + key = model.delete(:class) + FIELD_CLASS_NAME_MAP[key].from_h(parent, model) end def get_declarator @@ -389,7 +426,7 @@ def initialize(parent:, name:, field_class:) end end - def initialize(parent:, members:) + def initialize(parent:, members: []) @parent = parent @members = members.collect { |m| Member.new(parent: self, name: m[:name], field_class: m) } end @@ -404,7 +441,7 @@ def [](index) end def self.from_h(parent, model) - self.new(parent: parent, members: model[:members]) + self.new(parent: parent, **model) end def get_declarator(trace_class:, variable:) From 40b79d568cfe3c1033850480f1337b523f928601 Mon Sep 17 00:00:00 2001 From: Thomas Applencourt Date: Tue, 20 Sep 2022 23:11:08 +0000 Subject: [PATCH 15/24] Remove some useless code --- utils/gen_babeltrace_base.rb | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/utils/gen_babeltrace_base.rb b/utils/gen_babeltrace_base.rb index f7694875..4287d283 100644 --- a/utils/gen_babeltrace_base.rb +++ b/utils/gen_babeltrace_base.rb @@ -1,4 +1,11 @@ module Babeltrace2Gen + + module BTWrap + def wrapt_in_structure(d) + {:class => "structure", :members => d} + end + end + module BTPrinter @@output = "" @@indent = 0 @@ -106,8 +113,8 @@ def get_declarator(self_component:, variable:) class BTStreamClass include BTLocator include BTPrinter - attr_reader :packet_context_field_class, :packet_context - attr_reader :event_common_context_field_class, :event_common_context, :event_classes, :id + include BTWrap + attr_reader :packet_context_field_class, :event_common_context_field_class, :event_classes, :id def initialize(parent:, packet_context_field_class: nil, packet_context: nil, event_common_context_field_class: nil, event_common_context: nil, event_classes: [], id: nil) @parent = parent @@ -117,12 +124,12 @@ def initialize(parent:, packet_context_field_class: nil, packet_context: nil, raise "Two packet_context" if packet_context_field_class and packet_context # Should put assert to check for struct @packet_context_field_class = BTFieldClass.from_h(self, packet_context_field_class) if packet_context_field_class - @packet_context_field_class = BTFieldClass.from_h(self, {:class => "structure", :members => packet_context} ) if packet_context + @packet_context_field_class = BTFieldClass.from_h(self, wrapt_in_structure(packet_context) ) if packet_context raise "Two event_common_context" if event_common_context_field_class and event_common_context # Should put assert to check for struct @event_common_context_field_class = BTFieldClass.from_h(self, event_common_context_field_class) if event_common_context_field_class - @event_common_context_field_class = BTFieldClass.from_h(self, {:class => "structure", :members => event_common_context}) if event_common_context + @event_common_context_field_class = BTFieldClass.from_h(self, wrapt_in_structure(event_common_context)) if event_common_context @event_classes = event_classes.collect { |ec| BTEventClass.from_h(self, **ec) } end @@ -147,12 +154,18 @@ def get_declarator(trace_class:, variable:) class BTEventClass include BTLocator include BTPrinter + include BTWrap attr_reader :name, :specific_context_field_class, :payload_field_class - def initialize(parent:, name:, specific_context_field_class: nil, payload_field_class: nil) + def initialize(parent:, name:, specific_context_field_class: nil, specific_context: nil, payload_field_class: nil, payload: nil) @parent = parent @name = name - @specific_context_field_class = BTFieldClass.from_h(self, **specific_context_field_class) if specific_context_field_class - @payload_field_class = BTFieldClass.from_h(self, **payload_field_class) if payload_field_class + raise "Two specific_context" if specific_context_field_class and specific_context + @specific_context_field_class = BTFieldClass.from_h(self, specific_context_field_class) if specific_context_field_class + @specific_context_field_class = BTFieldClass.from_h(self, wrapt_in_structure(specific_context_field_class)) if specific_context + + raise "Two payload" if payload_field_class and payload + @payload_field_class = BTFieldClass.from_h(self, payload_field_class) if payload_field_class + @payload_field_class = BTFieldClass.from_h(self, wrapt_in_structure(payload_field_class)) if payload_field end def self.from_h(parent, model) From 229f77b619f4a2f58350af6546649ffcace41539 Mon Sep 17 00:00:00 2001 From: Thomas Applencourt Date: Tue, 20 Sep 2022 23:15:16 +0000 Subject: [PATCH 16/24] Move logic to correct place --- utils/gen_babeltrace_base.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/utils/gen_babeltrace_base.rb b/utils/gen_babeltrace_base.rb index 4287d283..5c1708b1 100644 --- a/utils/gen_babeltrace_base.rb +++ b/utils/gen_babeltrace_base.rb @@ -84,7 +84,10 @@ def initialize(parent:, stream_classes:, assigns_automatic_stream_class_id: true raise if parent @parent = nil @assigns_automatic_stream_class_id = assigns_automatic_stream_class_id - @stream_classes = stream_classes.collect{ |m| BTStreamClass.from_h(self, m) } + @stream_classes = stream_classes.collect{ |m| + raise "Incoherent ID scheme" if (m[:id].nil? != @assigns_automatic_stream_class_id) + BTStreamClass.from_h(self, m) + } end def self.from_h(parent, model) @@ -118,7 +121,6 @@ class BTStreamClass def initialize(parent:, packet_context_field_class: nil, packet_context: nil, event_common_context_field_class: nil, event_common_context: nil, event_classes: [], id: nil) @parent = parent - raise "Incoherent scheme for IDs" if (id.nil? != @parent.assigns_automatic_stream_class_id) @id = id raise "Two packet_context" if packet_context_field_class and packet_context From 4905d6d17e58c28ce62031d51243747d70c62395 Mon Sep 17 00:00:00 2001 From: Thomas Applencourt Date: Thu, 22 Sep 2022 00:36:53 +0000 Subject: [PATCH 17/24] closer and closer --- utils/gen_babeltrace_base.rb | 180 +++++++++++++++++++++++------------ 1 file changed, 117 insertions(+), 63 deletions(-) diff --git a/utils/gen_babeltrace_base.rb b/utils/gen_babeltrace_base.rb index 5c1708b1..8287c7c9 100644 --- a/utils/gen_babeltrace_base.rb +++ b/utils/gen_babeltrace_base.rb @@ -1,11 +1,5 @@ module Babeltrace2Gen - module BTWrap - def wrapt_in_structure(d) - {:class => "structure", :members => d} - end - end - module BTPrinter @@output = "" @@indent = 0 @@ -80,12 +74,12 @@ class BTTraceClass include BTPrinter attr_reader :stream_classes, :assigns_automatic_stream_class_id - def initialize(parent:, stream_classes:, assigns_automatic_stream_class_id: true) + def initialize(parent:, stream_classes:, assigns_automatic_stream_class_id: nil) raise if parent @parent = nil @assigns_automatic_stream_class_id = assigns_automatic_stream_class_id @stream_classes = stream_classes.collect{ |m| - raise "Incoherent ID scheme" if (m[:id].nil? != @assigns_automatic_stream_class_id) + raise "Incoherent ID scheme" if (m[:id].nil? != (@assigns_automatic_stream_class_id.nil? || @assigns_automatic_stream_class_id)) BTStreamClass.from_h(self, m) } end @@ -99,16 +93,19 @@ def trace_class end def get_declarator(self_component:, variable:) - scope { pr "bt_trace_class *#{variable} = bt_trace_class_create(#{self_component});" + unless @assigns_automatic_stream_class_id.nil? + auto_sc_id = @assigns_automatic_stream_class_id ? "BT_TRUE": "BT_FALSE" + pr "bt_trace_class_set_assigns_automatic_stream_class_id(#{variable}, #{auto_sc_id});" + end + @stream_classes.each_with_index { |m,i| - stream_class_name = "bt_stream_class_#{i}" - pr "bt_stream_class *#{stream_class_name};" - auto_id = @assigns_automatic_stream_class_id ? "true" : "false" - pr "bt_stream_class_set_assigns_automatic_stream_id(#{stream_class_name}, #{auto_id});" - m.get_declarator(trace_class: variable, variable: stream_class_name) + stream_class_name = "#{variable}_sc_#{i}" + scope { + pr "bt_stream_class *#{stream_class_name};" + m.get_declarator(trace_class: variable, variable: stream_class_name) + } } - } end end @@ -116,24 +113,26 @@ def get_declarator(self_component:, variable:) class BTStreamClass include BTLocator include BTPrinter - include BTWrap - attr_reader :packet_context_field_class, :event_common_context_field_class, :event_classes, :id - def initialize(parent:, packet_context_field_class: nil, packet_context: nil, - event_common_context_field_class: nil, event_common_context: nil, event_classes: [], id: nil) + attr_reader :packet_context_field_class, :event_common_context_field_class, :event_classes, :id, :name + def initialize(parent:, name: nil, packet_context_field_class: nil, event_common_context_field_class: nil, + event_classes: [], id: nil, assigns_automatic_event_class_id: nil, assigns_automatic_stream_id: nil) @parent = parent - @id = id + @name = name raise "Two packet_context" if packet_context_field_class and packet_context # Should put assert to check for struct @packet_context_field_class = BTFieldClass.from_h(self, packet_context_field_class) if packet_context_field_class - @packet_context_field_class = BTFieldClass.from_h(self, wrapt_in_structure(packet_context) ) if packet_context - raise "Two event_common_context" if event_common_context_field_class and event_common_context # Should put assert to check for struct @event_common_context_field_class = BTFieldClass.from_h(self, event_common_context_field_class) if event_common_context_field_class - @event_common_context_field_class = BTFieldClass.from_h(self, wrapt_in_structure(event_common_context)) if event_common_context - @event_classes = event_classes.collect { |ec| BTEventClass.from_h(self, **ec) } + @assigns_automatic_event_class_id = assigns_automatic_event_class_id + @event_classes = event_classes.collect { |ec| + raise "Incorect id scheme" if ec[:id].nil? != (@assigns_automatic_event_class_id.nil? || @assigns_automatic_event_class_id.nil) + BTEventClass.from_h(self, ec) } + @assigns_automatic_stream_id = assigns_automatic_stream_id + @id = id + end def self.from_h(parent, model) @@ -145,10 +144,47 @@ def stream_class end def get_declarator(trace_class:, variable:) - if id.nil? - pr "#{variable} = bt_stream_class_create(#{trace_class});" - else + if @id pr "#{variable} = bt_stream_class_create_with_id(#{trace_class}, #{@id});" + else + pr "#{variable} = bt_stream_class_create(#{trace_class});" + end + pr "bt_stream_class_set_name(#{variable}, \"#{name}\");" if @name + + if @packet_context_field_class + var_pc = "#{variable}_pc_fc" + scope { + pr "bt_field_class *#{var_pc};" + @packet_context_field_class.get_declarator(trace_class: trace_class, variable: var_pc) + pr "bt_stream_class_set_packet_context_field_class(#{variable}, #{var_pc});" + } + end + + if @event_common_context_field_class + var_ecc = "#{variable}_ecc_fc" + scope { + pr "bt_field_class *#{var_ecc};" + @event_common_context_field_class.get_declarator(trace_class: trace_class, variable: var_ecc) + pr "bt_stream_class_set_event_common_context_field_class(#{variable}, #{var_ecc});" + } + end + # Need to do is avec packet and event_common_context because it can refer members to those + unless @assigns_automatic_event_class_id.nil? + auto_id_event_class = @assigns_automatic_event_class_id ? "BT_TRUE" : "BT_FALSE" + pr "bt_stream_class_set_assigns_automatic_event_class_id(#{variable}, #{auto_id_event_class});" + end + + @event_classes.each_with_index { |ec,i| + var_name = "#{variable}_ec_#{i}" + scope { + pr "bt_event_class *#{var_name};" + ec.get_declarator(trace_class: trace_class, variable: var_name, stream_class: variable) + } + } + + unless @assigns_automatic_stream_id.nil? + auto_id_stream = @assigns_automatic_stream_id ? "BT_TRUE" : "BT_FALSE" + pr "bt_stream_class_set_assigns_automatic_stream_id(#{variable}, #{auto_id_stream});" end end end @@ -156,18 +192,14 @@ def get_declarator(trace_class:, variable:) class BTEventClass include BTLocator include BTPrinter - include BTWrap attr_reader :name, :specific_context_field_class, :payload_field_class - def initialize(parent:, name:, specific_context_field_class: nil, specific_context: nil, payload_field_class: nil, payload: nil) + def initialize(parent:, name: nil, specific_context_field_class: nil, payload_field_class: nil, id: nil) @parent = parent @name = name - raise "Two specific_context" if specific_context_field_class and specific_context @specific_context_field_class = BTFieldClass.from_h(self, specific_context_field_class) if specific_context_field_class - @specific_context_field_class = BTFieldClass.from_h(self, wrapt_in_structure(specific_context_field_class)) if specific_context - - raise "Two payload" if payload_field_class and payload @payload_field_class = BTFieldClass.from_h(self, payload_field_class) if payload_field_class - @payload_field_class = BTFieldClass.from_h(self, wrapt_in_structure(payload_field_class)) if payload_field + + @id = id end def self.from_h(parent, model) @@ -175,24 +207,33 @@ def self.from_h(parent, model) end def get_declarator(trace_class:, variable:, stream_class:) + # Store the variable name for instrocption purpose for LOCATION_PATH @variable = variable - scope { + if @id + pr "#{variable} = bt_event_class_create_with_id(#{stream_class}, #{@id});" + else pr "#{variable} = bt_event_class_create(#{stream_class});" - pr "bt_event_class_set_name(#{variable}, \"#{@name}\");" - if @specific_context_field_class - var_name = "#{variable}_sc_fc" + end + + pr "bt_event_class_set_name(#{variable}, \"#{@name}\");" if @name + + if @specific_context_field_class + var_name = "#{variable}_sc_fc" + scope { pr "bt_field_class *#{var_name};" @specific_context_field_class.get_declarator(trace_class: trace_class, variable: var_name) pr "bt_event_class_set_specific_context_field_class(#{variable}, #{var_name});" - end + } + end - if @payload_field_class - var_name = "#{variable}_p_fc" + if @payload_field_class + var_name = "#{variable}_p_fc" + scope { pr "bt_field_class *#{var_name};" @payload_field_class.get_declarator(trace_class: trace_class, variable: var_name) pr "bt_event_class_set_payload_field_class(#{variable}, #{var_name});" - end - } + } + end end def event_class @@ -204,17 +245,24 @@ class BTFieldClass include BTLocator include BTPrinter + attr_accessor :cast_type def initialize(parent:) @parent = parent end def self.from_h(parent, model) - key = model.delete(:class) - FIELD_CLASS_NAME_MAP[key].from_h(parent, model) + key = model.delete(:type) + raise "No type in #{model}" unless key + raise "No #{key} in FIELD_CLASS_NAME_MAP" unless FIELD_CLASS_NAME_MAP.include?(key) + + cast_type = model.delete(:cast_type) + fc = FIELD_CLASS_NAME_MAP[key].from_h(parent, model) + fc.cast_type = cast_type if cast_type + fc end - def get_declarator - raise NotImplementedError + def get_declarator(*args, **dict) + raise NotImplementedError, self.class end end @@ -252,19 +300,19 @@ def get_declarator(trace_class:, variable:) class BTFieldClass::Integer < BTFieldClass attr_reader :field_value_range, :preferred_display_base - def initialize(parent:, field_value_range: 64, preferred_display_base: 10) + def initialize(parent:, field_value_range: nil, preferred_display_base: nil) @parent = parent @field_value_range = field_value_range @preferred_display_base = preferred_display_base end def self.from_h(parent, model) - self.new(parent: parent, **model[:class_properties]) + self.new(parent: parent, **model) end def get_declarator(trace_class: nil, variable:) - pr "bt_field_class_integer_set_field_value_range(#{variable}, #{@field_value_range});" if @field_value_range != 64 - pr "bt_field_class_integer_set_preferred_display_base(#{variable}, #{@preferred_display_base});" if @preferred_display_base != 10 + pr "bt_field_class_integer_set_field_value_range(#{variable}, #{@field_value_range});" if @field_value_range + pr "bt_field_class_integer_set_preferred_display_base(#{variable}, #{@preferred_display_base});" if @preferred_display_base end end BTFieldClassInteger = BTFieldClass::Integer @@ -356,6 +404,11 @@ class BTFieldClass::String < BTFieldClass def self.from_h(parent, model) self.new(parent: parent) end + + def get_declarator(trace_class:, variable:) + pr "#{variable} = bt_field_class_string_create(#{trace_class});" + end + end BTFieldClassString = BTFieldClass::String @@ -381,8 +434,8 @@ def self.from_h(parent, model) end def get_declarator(trace_class:, variable:) + element_field_class_variable = "#{variable}_field_class" scope { - element_field_class_variable = "#{variable}_field_class" pr "bt_field_class *#{element_field_class_variable};" @element_field_class.get_declarator(trace_class: trace_class, variable: element_field_class_variable) pr "#{variable} = bt_field_class_array_static_create(#{trace_class}, #{element_field_class_variable}, #{@length});" @@ -409,8 +462,8 @@ def self.from_h(parent, model) end def get_declarator(trace_class:, variable:) + element_field_class_variable= "#{variable}_field_class" scope { - element_field_class_variable= "#{variable}_field_class" pr "bt_field_class *#{element_field_class_variable};" @element_field_class.get_declarator(trace_class: trace_class, variable: element_field_class_variable) if @length_field_path @@ -443,7 +496,7 @@ def initialize(parent:, name:, field_class:) def initialize(parent:, members: []) @parent = parent - @members = members.collect { |m| Member.new(parent: self, name: m[:name], field_class: m) } + @members = members.collect { |m| Member.new(parent: self, **m) } end def [](index) @@ -460,15 +513,16 @@ def self.from_h(parent, model) end def get_declarator(trace_class:, variable:) - @variable = variable - scope { + @variable = variable pr "#{variable} = bt_field_class_structure_create(#{trace_class});" - @members.each { |m| - pr "bt_field_class *#{variable}_#{m.name};" - m.field_class.get_declarator(trace_class: trace_class, variable: "#{variable}_#{m.name}") - pr "bt_field_class_structure_append_member(#{variable}, \"#{m.name}\", #{variable}_#{m.name});" + @members.each_with_index { |m, i| + var_name = "#{variable}_m_#{i}" + scope { + pr "bt_field_class *#{var_name};" + m.field_class.get_declarator(trace_class: trace_class, variable: var_name) + pr "bt_field_class_structure_append_member(#{variable}, \"#{m.name}\", #{var_name});" + } } - } end end BTFieldClassStructure = BTFieldClass::Structure @@ -598,8 +652,8 @@ def initialize(parent:, options:, selector_field_class: nil) FIELD_CLASS_NAME_MAP = { "bool" => BTFieldClass::Bool, "bit_array" => BTFieldClass::BitArray, - "unsigned" => BTFieldClass::Integer::Unsigned, - "signed" => BTFieldClass::Integer::Signed, + "integer_unsigned" => BTFieldClass::Integer::Unsigned, + "integer_signed" => BTFieldClass::Integer::Signed, "single" => BTFieldClass::Real::SinglePrecision, "double" => BTFieldClass::Real::DoublePrecision, "enumeration_unsigned" => BTFieldClass::Enumeration::Unsigned, From 07d1e855b5794d12e32a2f6fda4e9e07248fe4cc Mon Sep 17 00:00:00 2001 From: Thomas Applencourt Date: Thu, 22 Sep 2022 00:54:54 +0000 Subject: [PATCH 18/24] master of block --- utils/gen_babeltrace_base.rb | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/utils/gen_babeltrace_base.rb b/utils/gen_babeltrace_base.rb index 8287c7c9..526329ab 100644 --- a/utils/gen_babeltrace_base.rb +++ b/utils/gen_babeltrace_base.rb @@ -17,7 +17,10 @@ def scope(&block) @@indent -= 1 pr "}" end - module_function :scope + + def bt_set_conditionally(guard) + yield guard ? "BT_TRUE": "BT_FALSE" unless guard.nil? + end end module BTLocator @@ -94,10 +97,10 @@ def trace_class def get_declarator(self_component:, variable:) pr "bt_trace_class *#{variable} = bt_trace_class_create(#{self_component});" - unless @assigns_automatic_stream_class_id.nil? - auto_sc_id = @assigns_automatic_stream_class_id ? "BT_TRUE": "BT_FALSE" - pr "bt_trace_class_set_assigns_automatic_stream_class_id(#{variable}, #{auto_sc_id});" - end + + bt_set_conditionally ( @assigns_automatic_stream_class_id) { |v| + pr "bt_trace_class_set_assigns_automatic_stream_class_id(#{variable}, #{v});" + } @stream_classes.each_with_index { |m,i| stream_class_name = "#{variable}_sc_#{i}" @@ -168,11 +171,10 @@ def get_declarator(trace_class:, variable:) pr "bt_stream_class_set_event_common_context_field_class(#{variable}, #{var_ecc});" } end - # Need to do is avec packet and event_common_context because it can refer members to those - unless @assigns_automatic_event_class_id.nil? - auto_id_event_class = @assigns_automatic_event_class_id ? "BT_TRUE" : "BT_FALSE" - pr "bt_stream_class_set_assigns_automatic_event_class_id(#{variable}, #{auto_id_event_class});" - end + # Need to do is afert packet an devent_common_context because it can refer members to those + bt_set_conditionally( @assigns_automatic_event_class_id ) { |v| + pr "bt_stream_class_set_assigns_automatic_event_class_id(#{variable}, #{v});" + } @event_classes.each_with_index { |ec,i| var_name = "#{variable}_ec_#{i}" @@ -182,10 +184,9 @@ def get_declarator(trace_class:, variable:) } } - unless @assigns_automatic_stream_id.nil? - auto_id_stream = @assigns_automatic_stream_id ? "BT_TRUE" : "BT_FALSE" - pr "bt_stream_class_set_assigns_automatic_stream_id(#{variable}, #{auto_id_stream});" - end + bt_set_conditionally( @assigns_automatic_stream_id ) { |v| + pr "bt_stream_class_set_assigns_automatic_stream_id(#{variable}, #{v});" + } end end From f4158a2239ff02fb81c3d549a28c49716edba096 Mon Sep 17 00:00:00 2001 From: Thomas Applencourt Date: Thu, 22 Sep 2022 16:45:37 +0000 Subject: [PATCH 19/24] Add context wrapper --- utils/gen_babeltrace_base.rb | 44 ++++++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/utils/gen_babeltrace_base.rb b/utils/gen_babeltrace_base.rb index 526329ab..37c0028e 100644 --- a/utils/gen_babeltrace_base.rb +++ b/utils/gen_babeltrace_base.rb @@ -1,5 +1,11 @@ module Babeltrace2Gen + module BTUtils + def bt_set_conditionally(guard) + yield guard ? "BT_TRUE": "BT_FALSE" unless guard.nil? + end + end + module BTPrinter @@output = "" @@indent = 0 @@ -10,17 +16,25 @@ def pr(str) end module_function :pr - def scope(&block) + def scope pr "{" @@indent += 1 - block.call + yield @@indent -= 1 pr "}" end - def bt_set_conditionally(guard) - yield guard ? "BT_TRUE": "BT_FALSE" unless guard.nil? + def self.context(output: "", indent: 0) + @@output = output + @@indent = indent + yield + @@output end + + end + + def self.context(**args, &block) + BTPrinter::context(**args, &block) end module BTLocator @@ -75,6 +89,7 @@ def find_field_class(path, variable) class BTTraceClass include BTLocator include BTPrinter + include BTUtils attr_reader :stream_classes, :assigns_automatic_stream_class_id def initialize(parent:, stream_classes:, assigns_automatic_stream_class_id: nil) @@ -96,19 +111,19 @@ def trace_class end def get_declarator(self_component:, variable:) - pr "bt_trace_class *#{variable} = bt_trace_class_create(#{self_component});" + pr "bt_trace_class *#{variable} = bt_trace_class_create(#{self_component});" - bt_set_conditionally ( @assigns_automatic_stream_class_id) { |v| - pr "bt_trace_class_set_assigns_automatic_stream_class_id(#{variable}, #{v});" - } + bt_set_conditionally ( @assigns_automatic_stream_class_id) { |v| + pr "bt_trace_class_set_assigns_automatic_stream_class_id(#{variable}, #{v});" + } - @stream_classes.each_with_index { |m,i| - stream_class_name = "#{variable}_sc_#{i}" - scope { - pr "bt_stream_class *#{stream_class_name};" - m.get_declarator(trace_class: variable, variable: stream_class_name) - } + @stream_classes.each_with_index { |m,i| + stream_class_name = "#{variable}_sc_#{i}" + scope { + pr "bt_stream_class *#{stream_class_name};" + m.get_declarator(trace_class: variable, variable: stream_class_name) } + } end end @@ -116,6 +131,7 @@ def get_declarator(self_component:, variable:) class BTStreamClass include BTLocator include BTPrinter + include BTUtils attr_reader :packet_context_field_class, :event_common_context_field_class, :event_classes, :id, :name def initialize(parent:, name: nil, packet_context_field_class: nil, event_common_context_field_class: nil, event_classes: [], id: nil, assigns_automatic_event_class_id: nil, assigns_automatic_stream_id: nil) From db24fb9511787a17bfe7e0f9da8c16d33dcff27d Mon Sep 17 00:00:00 2001 From: Thomas Applencourt Date: Mon, 26 Sep 2022 18:07:35 +0000 Subject: [PATCH 20/24] Remove stuff who should be automatic --- utils/gen_babeltrace_base.rb | 51 +++++------------------------------- 1 file changed, 7 insertions(+), 44 deletions(-) diff --git a/utils/gen_babeltrace_base.rb b/utils/gen_babeltrace_base.rb index 37c0028e..a6d955ba 100644 --- a/utils/gen_babeltrace_base.rb +++ b/utils/gen_babeltrace_base.rb @@ -112,7 +112,6 @@ def trace_class def get_declarator(self_component:, variable:) pr "bt_trace_class *#{variable} = bt_trace_class_create(#{self_component});" - bt_set_conditionally ( @assigns_automatic_stream_class_id) { |v| pr "bt_trace_class_set_assigns_automatic_stream_class_id(#{variable}, #{v});" } @@ -293,7 +292,6 @@ def get_declarator(trace_class:, variable:) pr "#{variable} = bt_field_class_bool_create(#{trace_class});" end end - BTFieldClassBool = BTFieldClass::Bool class BTFieldClass::BitArray < BTFieldClass attr_reader :length @@ -312,7 +310,6 @@ def get_declarator(trace_class:, variable:) end end - BTFieldClassBitArray = BTFieldClass::BitArray class BTFieldClass::Integer < BTFieldClass attr_reader :field_value_range, :preferred_display_base @@ -334,54 +331,43 @@ def get_declarator(trace_class: nil, variable:) end BTFieldClassInteger = BTFieldClass::Integer - class BTFieldClass::Integer::Unsigned < BTFieldClassInteger + class BTFieldClass::Integer::Unsigned < BTFieldClass::Integer def get_declarator(trace_class:, variable:) pr "#{variable} = bt_field_class_integer_unsigned_create(#{trace_class});" super end end - BTFieldClass::IntegerUnsigned = BTFieldClass::Integer::Unsigned - BTFieldClassIntegerUnsigned = BTFieldClass::Integer::Unsigned - class BTFieldClass::Integer::Signed < BTFieldClassInteger + class BTFieldClass::Integer::Signed < BTFieldClass::Integer def get_declarator(trace_class:, variable:) pr "#{variable} = bt_field_class_integer_signed_create(#{trace_class});" super end end - BTFieldClass::IntegerSigned = BTFieldClass::Integer::Signed - BTFieldClassIntegerSigned = BTFieldClass::Integer::Signed class BTFieldClass::Real < BTFieldClass def self.from_h(parent, model) self.new(parent: parent) end end - BTFieldClassReal = BTFieldClass::Real - class BTFieldClass::Real::SinglePrecision < BTFieldClassReal + class BTFieldClass::Real::SinglePrecision < BTFieldClass::Real def get_declarator(trace_class:, variable:) pr "#{variable} = bt_field_class_real_single_precision_create(#{trace_class});" end end - BTFieldClass::RealSinglePrecision = BTFieldClass::Real::SinglePrecision - BTFieldClassRealSinglePrecision = BTFieldClass::Real::SinglePrecision - class BTFieldClass::Real::DoublePrecision < BTFieldClassReal + class BTFieldClass::Real::DoublePrecision < BTFieldClass::Real def get_declarator(trace_class:, variable:) pr "#{variable} = bt_field_class_real_double_precision_create(#{trace_class});" end end - BTFieldClass::RealDoublePrecision = BTFieldClass::Real::DoublePrecision - BTFieldClassRealDoublePrecision = BTFieldClass::Real::DoublePrecision module BTFieldClass::Enumeration attr_reader :mappings class Mapping end end - BTFieldClassEnumeration = BTFieldClass::Enumeration - BTFieldClassEnumerationMapping = BTFieldClass::Enumeration::Mapping class BTFieldClass::Enumeration::Unsigned < BTFieldClass::Integer::Unsigned include BTFieldClass::Enumeration @@ -397,8 +383,6 @@ def self.from_h(parent, model) self.new(parent: parent, **model) end end - BTFieldClassEnumerationUnsigned = BTFieldClass::Enumeration::Unsigned - BTFieldClassEnumerationUnsignedMapping = BTFieldClass::Enumeration::Unsigned::Mapping class BTFieldClass::Enumeration::Signed < BTFieldClass::Integer::Signed include BTFieldClass::Enumeration @@ -414,8 +398,6 @@ def self.from_h(parent, model) self.new(parent: parent, **model) end end - BTFieldClassEnumerationSigned = BTFieldClass::Enumeration::Signed - BTFieldClassEnumerationSignedMapping = BTFieldClass::Enumeration::Signed::Mapping class BTFieldClass::String < BTFieldClass def self.from_h(parent, model) @@ -427,7 +409,6 @@ def get_declarator(trace_class:, variable:) end end - BTFieldClassString = BTFieldClass::String class BTFieldClass::Array < BTFieldClass attr_reader :element_field_class @@ -436,7 +417,6 @@ def initialize(parent:, element_field_class:) @element_field_class = BTFieldClass.from_h(self, element_field_class) end end - BTFieldClassArray = BTFieldClass::Array class BTFieldClass::Array::Static < BTFieldClass::Array attr_reader :length @@ -459,7 +439,6 @@ def get_declarator(trace_class:, variable:) } end end - BTFieldClassArrayStatic = BTFieldClass::Array::Static class BTFieldClass::Array::Dynamic < BTFieldClass::Array module WithLengthField @@ -493,10 +472,8 @@ def get_declarator(trace_class:, variable:) end } end - end - BTFieldClassArrayDynamic = BTFieldClass::Array::Dynamic - BTFieldClassArrayDynamicWithLengthField = BTFieldClass::Array::Dynamic::WithLengthField + class BTFieldClass::Structure < BTFieldClass attr_reader :members class Member @@ -542,8 +519,7 @@ def get_declarator(trace_class:, variable:) } end end - BTFieldClassStructure = BTFieldClass::Structure - BTFieldClassStructureMember = BTFieldClass::Structure::Member + class BTFieldClass::Option < BTFieldClass attr_reader :field_class @@ -559,7 +535,6 @@ def self.from_h(parent, model) self.new(parent: parent, field_class: model[:field]) end end - BTFieldClassOptionWithoutSelectorField = BTFieldClass::Option::WithoutSelectorField class BTFieldClass::Option::WithSelectorField < BTFieldClass::Option attr_reader :selector_field_path @@ -569,7 +544,6 @@ def initialize(parent:, field_class:, selector_field_path:) super(parent: parent, field_class: field_class) end end - BTFieldClassOptionWithSelectorField = BTFieldClass::Option::WithSelectorField class BTFieldClass::Option::WithSelectorField::Bool < BTFieldClass::Option::WithSelectorField attr_reader :selector_is_reversed @@ -583,7 +557,6 @@ def self.from_h(parent, model) self.new(parent: parent, field_class: model[:field], selector_field_path: model[:selector_field_path], selector_is_reversed: [selector_field_path]) end end - BTFieldClassOptionWithSelectorFieldBool = BTFieldClass::Option::WithSelectorField::Bool class BTFieldClass::Option::WithSelectorField::IntegerUnsigned < BTFieldClass::Option::WithSelectorField attr_reader :selector_ranges @@ -597,7 +570,6 @@ def self.from_h(parent, model) self.new(parent: parent, field_class: model[:field], selector_field_path: model[:selector_field_path], selector_ranges: model[:selector_ranges]) end end - BTFieldClassOptionWithSelectorFieldIntegerUnsigned = BTFieldClass::Option::WithSelectorField::IntegerUnsigned class BTFieldClass::Option::WithSelectorField::IntegerSigned < BTFieldClass::Option::WithSelectorField attr_reader :selector_ranges @@ -611,7 +583,6 @@ def self.from_h(parent, model) self.new(parent: parent, field_class: model[:field], selector_field_path: model[:selector_field_path], selector_ranges: model[:selector_ranges]) end end - BTFieldClassOptionWithSelectorFieldIntegerSigned = BTFieldClass::Option::WithSelectorField::IntegerSigned class BTFieldClass::Variant < BTFieldClass attr_reader :options @@ -626,15 +597,13 @@ def initialize(parent:, name:, field_class:) end end - BTFieldClassVariant = BTFieldClass::Variant - BTFieldClassVariantOption = BTFieldClass::Variant::Option class BTFieldClass::Variant module WithoutSelectorField end module WithSelectorField attr_reader :selector_field_class - class Option < BTFieldClassVariantOption + class Option < BTFieldClass::Variant::Option attr_reader :ranges def initialize(parent:, name:, field_class:, ranges:) @ranges = ranges @@ -659,12 +628,6 @@ def initialize(parent:, options:, selector_field_class: nil) end end end - BTFieldClassVariantWithoutSelectorField = - BTFieldClass::Variant::WithoutSelectorField - BTFieldClassVariantWithSelectorField = - BTFieldClass::Variant::WithSelectorField - BTFieldClassVariantWithSelectorFieldOption = - BTFieldClass::Variant::WithSelectorField::Option FIELD_CLASS_NAME_MAP = { "bool" => BTFieldClass::Bool, From 8ea41182978c74fd30e4cb50af2f7fc24c609484 Mon Sep 17 00:00:00 2001 From: Thomas Applencourt Date: Mon, 26 Sep 2022 18:54:39 +0000 Subject: [PATCH 21/24] Refractor --- utils/gen_babeltrace_base.rb | 98 +++++++++++------------------------- 1 file changed, 28 insertions(+), 70 deletions(-) diff --git a/utils/gen_babeltrace_base.rb b/utils/gen_babeltrace_base.rb index a6d955ba..a272e091 100644 --- a/utils/gen_babeltrace_base.rb +++ b/utils/gen_babeltrace_base.rb @@ -1,5 +1,11 @@ module Babeltrace2Gen + module BTFromH + def from_h(parent, model) + new(parent: parent, **model) + end + end + module BTUtils def bt_set_conditionally(guard) yield guard ? "BT_TRUE": "BT_FALSE" unless guard.nil? @@ -90,6 +96,8 @@ class BTTraceClass include BTLocator include BTPrinter include BTUtils + extend BTFromH + attr_reader :stream_classes, :assigns_automatic_stream_class_id def initialize(parent:, stream_classes:, assigns_automatic_stream_class_id: nil) @@ -102,10 +110,6 @@ def initialize(parent:, stream_classes:, assigns_automatic_stream_class_id: nil) } end - def self.from_h(parent, model) - self.new(parent: parent, **model) - end - def trace_class self end @@ -131,6 +135,7 @@ class BTStreamClass include BTLocator include BTPrinter include BTUtils + extend BTFromH attr_reader :packet_context_field_class, :event_common_context_field_class, :event_classes, :id, :name def initialize(parent:, name: nil, packet_context_field_class: nil, event_common_context_field_class: nil, event_classes: [], id: nil, assigns_automatic_event_class_id: nil, assigns_automatic_stream_id: nil) @@ -153,10 +158,6 @@ def initialize(parent:, name: nil, packet_context_field_class: nil, event_common end - def self.from_h(parent, model) - self.new(parent: parent, **model) - end - def stream_class self end @@ -208,6 +209,7 @@ def get_declarator(trace_class:, variable:) class BTEventClass include BTLocator include BTPrinter + extend BTFromH attr_reader :name, :specific_context_field_class, :payload_field_class def initialize(parent:, name: nil, specific_context_field_class: nil, payload_field_class: nil, id: nil) @parent = parent @@ -218,10 +220,6 @@ def initialize(parent:, name: nil, specific_context_field_class: nil, payload_fi @id = id end - def self.from_h(parent, model) - self.new(parent: parent, **model) - end - def get_declarator(trace_class:, variable:, stream_class:) # Store the variable name for instrocption purpose for LOCATION_PATH @variable = variable @@ -284,16 +282,14 @@ def get_declarator(*args, **dict) end class BTFieldClass::Bool < BTFieldClass - def self.from_h(parent, model) - self.new(parent: parent) - end - + extend BTFromH def get_declarator(trace_class:, variable:) pr "#{variable} = bt_field_class_bool_create(#{trace_class});" end end class BTFieldClass::BitArray < BTFieldClass + extend BTFromH attr_reader :length def initialize(parent:, length:) @@ -301,10 +297,6 @@ def initialize(parent:, length:) @length = length end - def self.from_h(parent, model) - self.new(parent: parent, length: model[:length]) - end - def get_declarator(trace_class:, variable:) pr "#{variable} = bt_field_class_bit_array_create(#{trace_class}, #{@length});" end @@ -320,10 +312,6 @@ def initialize(parent:, field_value_range: nil, preferred_display_base: nil) @preferred_display_base = preferred_display_base end - def self.from_h(parent, model) - self.new(parent: parent, **model) - end - def get_declarator(trace_class: nil, variable:) pr "bt_field_class_integer_set_field_value_range(#{variable}, #{@field_value_range});" if @field_value_range pr "bt_field_class_integer_set_preferred_display_base(#{variable}, #{@preferred_display_base});" if @preferred_display_base @@ -332,6 +320,7 @@ def get_declarator(trace_class: nil, variable:) BTFieldClassInteger = BTFieldClass::Integer class BTFieldClass::Integer::Unsigned < BTFieldClass::Integer + extend BTFromH def get_declarator(trace_class:, variable:) pr "#{variable} = bt_field_class_integer_unsigned_create(#{trace_class});" super @@ -339,6 +328,7 @@ def get_declarator(trace_class:, variable:) end class BTFieldClass::Integer::Signed < BTFieldClass::Integer + extend BTFromH def get_declarator(trace_class:, variable:) pr "#{variable} = bt_field_class_integer_signed_create(#{trace_class});" super @@ -346,18 +336,17 @@ def get_declarator(trace_class:, variable:) end class BTFieldClass::Real < BTFieldClass - def self.from_h(parent, model) - self.new(parent: parent) - end end class BTFieldClass::Real::SinglePrecision < BTFieldClass::Real + extend BTFromH def get_declarator(trace_class:, variable:) pr "#{variable} = bt_field_class_real_single_precision_create(#{trace_class});" end end class BTFieldClass::Real::DoublePrecision < BTFieldClass::Real + extend BTFromH def get_declarator(trace_class:, variable:) pr "#{variable} = bt_field_class_real_double_precision_create(#{trace_class});" end @@ -378,10 +367,6 @@ def initialize(parent:, field_value_range:, preferred_display_base: 10, mappings @mappings = mappings #TODO init Mapping super(parent: parent, field_value_range: field_value_range, preferred_display_base: preferred_display_base) end - - def self.from_h(parent, model) - self.new(parent: parent, **model) - end end class BTFieldClass::Enumeration::Signed < BTFieldClass::Integer::Signed @@ -393,21 +378,14 @@ def initialize(parent:, field_value_range:, preferred_display_base: 10, mappings @mappings = mappings #TODO init Mapping super(parent: parent, field_value_range: field_value_range, preferred_display_base: preferred_display_base) end - - def self.from_h(parent, model) - self.new(parent: parent, **model) - end end class BTFieldClass::String < BTFieldClass - def self.from_h(parent, model) - self.new(parent: parent) - end + extend BTFromH def get_declarator(trace_class:, variable:) pr "#{variable} = bt_field_class_string_create(#{trace_class});" end - end class BTFieldClass::Array < BTFieldClass @@ -419,6 +397,7 @@ def initialize(parent:, element_field_class:) end class BTFieldClass::Array::Static < BTFieldClass::Array + extend BTFromH attr_reader :length def initialize(parent:, element_field_class:, length:) @@ -426,10 +405,6 @@ def initialize(parent:, element_field_class:, length:) super(parent: parent, element_field_class: element_field_class) end - def self.from_h(parent, model) - self.new(parent: parent, element_field_class: model[:field], length: model[:length]) - end - def get_declarator(trace_class:, variable:) element_field_class_variable = "#{variable}_field_class" scope { @@ -441,6 +416,7 @@ def get_declarator(trace_class:, variable:) end class BTFieldClass::Array::Dynamic < BTFieldClass::Array + extend BTFromH module WithLengthField attr_reader :length_field_path end @@ -453,10 +429,6 @@ def initialize(parent:, element_field_class:, length_field_path: nil) end end - def self.from_h(parent, model) - self.new(parent: parent, element_field_class: model[:field], length_field_path: model[:length_field_path]) - end - def get_declarator(trace_class:, variable:) element_field_class_variable= "#{variable}_field_class" scope { @@ -475,6 +447,8 @@ def get_declarator(trace_class:, variable:) end class BTFieldClass::Structure < BTFieldClass + extend BTFromH + attr_reader :members class Member include BTLocator @@ -502,10 +476,6 @@ def [](index) end end - def self.from_h(parent, model) - self.new(parent: parent, **model) - end - def get_declarator(trace_class:, variable:) @variable = variable pr "#{variable} = bt_field_class_structure_create(#{trace_class});" @@ -519,7 +489,7 @@ def get_declarator(trace_class:, variable:) } end end - + class BTFieldClass::Option < BTFieldClass attr_reader :field_class @@ -531,9 +501,7 @@ def initialize(parent:, field_class:) BTFieldClassOption = BTFieldClass::Option class BTFieldClass::Option::WithoutSelectorField < BTFieldClass::Option - def self.from_h(parent, model) - self.new(parent: parent, field_class: model[:field]) - end + extend BTFromH end class BTFieldClass::Option::WithSelectorField < BTFieldClass::Option @@ -546,45 +514,37 @@ def initialize(parent:, field_class:, selector_field_path:) end class BTFieldClass::Option::WithSelectorField::Bool < BTFieldClass::Option::WithSelectorField + extend BTFromH attr_reader :selector_is_reversed def initialize(parent:, field_class:, selector_field_path:, selector_is_reversed: nil) @selector_is_reversed = selector_is_reversed super(parent: parent, field_class: field_class, selector_field_path: selector_field_path) end - - def self.from_h(parent, model) - self.new(parent: parent, field_class: model[:field], selector_field_path: model[:selector_field_path], selector_is_reversed: [selector_field_path]) - end end class BTFieldClass::Option::WithSelectorField::IntegerUnsigned < BTFieldClass::Option::WithSelectorField + extend BTFromH attr_reader :selector_ranges def initialize(parent:, field_class:, selector_field_path:, selector_ranges:) @selector_ranges = selector_ranges super(parent: parent, field_class: field_class, selector_field_path: selector_field_path) end - - def self.from_h(parent, model) - self.new(parent: parent, field_class: model[:field], selector_field_path: model[:selector_field_path], selector_ranges: model[:selector_ranges]) - end end class BTFieldClass::Option::WithSelectorField::IntegerSigned < BTFieldClass::Option::WithSelectorField + extend BTFromH attr_reader :selector_ranges def initialize(parent:, field_class:, selector_field_path:, selector_ranges:) @selector_ranges = selector_ranges super(parent: parent, field_class: field_class, selector_field_path: selector_field_path) end - - def self.from_h(parent, model) - self.new(parent: parent, field_class: model[:field], selector_field_path: model[:selector_field_path], selector_ranges: model[:selector_ranges]) - end end class BTFieldClass::Variant < BTFieldClass + extend BTFromH attr_reader :options class Option @@ -596,11 +556,9 @@ def initialize(parent:, name:, field_class:) end end - end - - class BTFieldClass::Variant module WithoutSelectorField end + module WithSelectorField attr_reader :selector_field_class class Option < BTFieldClass::Variant::Option From 69e1bebf992277ff8f883cdd658f11c486b096d8 Mon Sep 17 00:00:00 2001 From: Thomas Applencourt Date: Mon, 26 Sep 2022 19:26:00 +0000 Subject: [PATCH 22/24] rubocopt --- utils/gen_babeltrace_base.rb | 223 +++++++++++++++++++---------------- 1 file changed, 120 insertions(+), 103 deletions(-) diff --git a/utils/gen_babeltrace_base.rb b/utils/gen_babeltrace_base.rb index a272e091..d6b87402 100644 --- a/utils/gen_babeltrace_base.rb +++ b/utils/gen_babeltrace_base.rb @@ -1,5 +1,4 @@ module Babeltrace2Gen - module BTFromH def from_h(parent, model) new(parent: parent, **model) @@ -8,14 +7,14 @@ def from_h(parent, model) module BTUtils def bt_set_conditionally(guard) - yield guard ? "BT_TRUE": "BT_FALSE" unless guard.nil? + yield guard ? 'BT_TRUE' : 'BT_FALSE' unless guard.nil? end end module BTPrinter - @@output = "" + @@output = '' @@indent = 0 - INDENT_INCREMENT = " " + INDENT_INCREMENT = ' ' def pr(str) @@output << INDENT_INCREMENT * @@indent << str << "\n" @@ -23,29 +22,28 @@ def pr(str) module_function :pr def scope - pr "{" + pr '{' @@indent += 1 yield @@indent -= 1 - pr "}" + pr '}' end - def self.context(output: "", indent: 0) + def self.context(output: '', indent: 0) @@output = output @@indent = indent yield @@output end - end def self.context(**args, &block) - BTPrinter::context(**args, &block) + BTPrinter.context(**args, &block) end module BTLocator - attr_reader :parent - attr_reader :variable + attr_reader :parent, :variable + def root @parent ? @parent.root : self end @@ -63,33 +61,32 @@ def event_class end def find_field_class_path(path, variable) - path.scan(/\["(\w+)"\]|\[(\d+)\]/).each { |m| + path.scan(/\["(\w+)"\]|\[(\d+)\]/).each do |m| # String if m.first pr "#{variable} = bt_field_class_structure_borrow_member_by_name(#{variable}, \"#{m.first}\");" else pr "#{variable} = bt_field_class_structure_borrow_member_by_index(#{variable}, #{m.last});" end - } + end end def find_field_class(path, variable) m = path.match(/\A(PACKET_CONTEXT|EVENT_COMMON_CONTEXT|EVENT_SPECIFIC_CONTEXT|EVENT_PAYLOAD)(.*)/) case m[1] - when "PACKET_CONTEXT" + when 'PACKET_CONTEXT' pr "#{variable} = #{stream_class.packet_context_field_class.variable};" - when "EVENT_COMMON_CONTEXT" + when 'EVENT_COMMON_CONTEXT' pr "#{variable} = #{stream_class.event_common_context_field_class.variable};" - when "EVENT_SPECIFIC_CONTEXT" + when 'EVENT_SPECIFIC_CONTEXT' pr "#{variable} = #{event_class.specific_context_field_class.varible};" - when "EVENT_PAYLOAD" + when 'EVENT_PAYLOAD' pr "#{variable} = #{event_class.payload_field_class.variable};" else raise "invalid path #{path}" end find_field_class_path(m[2], variable) end - end class BTTraceClass @@ -102,12 +99,16 @@ class BTTraceClass def initialize(parent:, stream_classes:, assigns_automatic_stream_class_id: nil) raise if parent + @parent = nil @assigns_automatic_stream_class_id = assigns_automatic_stream_class_id - @stream_classes = stream_classes.collect{ |m| - raise "Incoherent ID scheme" if (m[:id].nil? != (@assigns_automatic_stream_class_id.nil? || @assigns_automatic_stream_class_id)) - BTStreamClass.from_h(self, m) - } + @stream_classes = stream_classes.collect do |m| + if m[:id].nil? != (@assigns_automatic_stream_class_id.nil? || @assigns_automatic_stream_class_id) + raise 'Incoherent ID scheme' + end + + BTStreamClass.from_h(self, m) + end end def trace_class @@ -116,19 +117,18 @@ def trace_class def get_declarator(self_component:, variable:) pr "bt_trace_class *#{variable} = bt_trace_class_create(#{self_component});" - bt_set_conditionally ( @assigns_automatic_stream_class_id) { |v| + bt_set_conditionally(@assigns_automatic_stream_class_id) do |v| pr "bt_trace_class_set_assigns_automatic_stream_class_id(#{variable}, #{v});" - } + end - @stream_classes.each_with_index { |m,i| + @stream_classes.each_with_index do |m, i| stream_class_name = "#{variable}_sc_#{i}" - scope { + scope do pr "bt_stream_class *#{stream_class_name};" - m.get_declarator(trace_class: variable, variable: stream_class_name) - } - } + m.get_declarator(trace_class: variable, variable: stream_class_name) + end + end end - end class BTStreamClass @@ -137,25 +137,33 @@ class BTStreamClass include BTUtils extend BTFromH attr_reader :packet_context_field_class, :event_common_context_field_class, :event_classes, :id, :name + def initialize(parent:, name: nil, packet_context_field_class: nil, event_common_context_field_class: nil, event_classes: [], id: nil, assigns_automatic_event_class_id: nil, assigns_automatic_stream_id: nil) @parent = parent @name = name - raise "Two packet_context" if packet_context_field_class and packet_context + raise 'Two packet_context' if packet_context_field_class and packet_context + # Should put assert to check for struct @packet_context_field_class = BTFieldClass.from_h(self, packet_context_field_class) if packet_context_field_class # Should put assert to check for struct - @event_common_context_field_class = BTFieldClass.from_h(self, event_common_context_field_class) if event_common_context_field_class + if event_common_context_field_class + @event_common_context_field_class = BTFieldClass.from_h(self, + event_common_context_field_class) + end @assigns_automatic_event_class_id = assigns_automatic_event_class_id - @event_classes = event_classes.collect { |ec| - raise "Incorect id scheme" if ec[:id].nil? != (@assigns_automatic_event_class_id.nil? || @assigns_automatic_event_class_id.nil) - BTEventClass.from_h(self, ec) } + @event_classes = event_classes.collect do |ec| + if ec[:id].nil? != (@assigns_automatic_event_class_id.nil? || @assigns_automatic_event_class_id.nil) + raise 'Incorect id scheme' + end + + BTEventClass.from_h(self, ec) + end @assigns_automatic_stream_id = assigns_automatic_stream_id @id = id - end def stream_class @@ -172,37 +180,37 @@ def get_declarator(trace_class:, variable:) if @packet_context_field_class var_pc = "#{variable}_pc_fc" - scope { + scope do pr "bt_field_class *#{var_pc};" @packet_context_field_class.get_declarator(trace_class: trace_class, variable: var_pc) pr "bt_stream_class_set_packet_context_field_class(#{variable}, #{var_pc});" - } + end end if @event_common_context_field_class var_ecc = "#{variable}_ecc_fc" - scope { + scope do pr "bt_field_class *#{var_ecc};" @event_common_context_field_class.get_declarator(trace_class: trace_class, variable: var_ecc) pr "bt_stream_class_set_event_common_context_field_class(#{variable}, #{var_ecc});" - } + end end # Need to do is afert packet an devent_common_context because it can refer members to those - bt_set_conditionally( @assigns_automatic_event_class_id ) { |v| + bt_set_conditionally(@assigns_automatic_event_class_id) do |v| pr "bt_stream_class_set_assigns_automatic_event_class_id(#{variable}, #{v});" - } + end - @event_classes.each_with_index { |ec,i| + @event_classes.each_with_index do |ec, i| var_name = "#{variable}_ec_#{i}" - scope { + scope do pr "bt_event_class *#{var_name};" ec.get_declarator(trace_class: trace_class, variable: var_name, stream_class: variable) - } - } + end + end - bt_set_conditionally( @assigns_automatic_stream_id ) { |v| + bt_set_conditionally(@assigns_automatic_stream_id) do |v| pr "bt_stream_class_set_assigns_automatic_stream_id(#{variable}, #{v});" - } + end end end @@ -211,10 +219,14 @@ class BTEventClass include BTPrinter extend BTFromH attr_reader :name, :specific_context_field_class, :payload_field_class + def initialize(parent:, name: nil, specific_context_field_class: nil, payload_field_class: nil, id: nil) @parent = parent @name = name - @specific_context_field_class = BTFieldClass.from_h(self, specific_context_field_class) if specific_context_field_class + if specific_context_field_class + @specific_context_field_class = BTFieldClass.from_h(self, + specific_context_field_class) + end @payload_field_class = BTFieldClass.from_h(self, payload_field_class) if payload_field_class @id = id @@ -233,20 +245,20 @@ def get_declarator(trace_class:, variable:, stream_class:) if @specific_context_field_class var_name = "#{variable}_sc_fc" - scope { + scope do pr "bt_field_class *#{var_name};" @specific_context_field_class.get_declarator(trace_class: trace_class, variable: var_name) pr "bt_event_class_set_specific_context_field_class(#{variable}, #{var_name});" - } + end end if @payload_field_class var_name = "#{variable}_p_fc" - scope { + scope do pr "bt_field_class *#{var_name};" @payload_field_class.get_declarator(trace_class: trace_class, variable: var_name) pr "bt_event_class_set_payload_field_class(#{variable}, #{var_name});" - } + end end end @@ -260,6 +272,7 @@ class BTFieldClass include BTPrinter attr_accessor :cast_type + def initialize(parent:) @parent = parent end @@ -278,7 +291,6 @@ def self.from_h(parent, model) def get_declarator(*args, **dict) raise NotImplementedError, self.class end - end class BTFieldClass::Bool < BTFieldClass @@ -300,7 +312,6 @@ def initialize(parent:, length:) def get_declarator(trace_class:, variable:) pr "#{variable} = bt_field_class_bit_array_create(#{trace_class}, #{@length});" end - end class BTFieldClass::Integer < BTFieldClass @@ -312,9 +323,11 @@ def initialize(parent:, field_value_range: nil, preferred_display_base: nil) @preferred_display_base = preferred_display_base end - def get_declarator(trace_class: nil, variable:) + def get_declarator(variable:, trace_class: nil) pr "bt_field_class_integer_set_field_value_range(#{variable}, #{@field_value_range});" if @field_value_range - pr "bt_field_class_integer_set_preferred_display_base(#{variable}, #{@preferred_display_base});" if @preferred_display_base + if @preferred_display_base + pr "bt_field_class_integer_set_preferred_display_base(#{variable}, #{@preferred_display_base});" + end end end BTFieldClassInteger = BTFieldClass::Integer @@ -354,6 +367,7 @@ def get_declarator(trace_class:, variable:) module BTFieldClass::Enumeration attr_reader :mappings + class Mapping end end @@ -363,8 +377,8 @@ class BTFieldClass::Enumeration::Unsigned < BTFieldClass::Integer::Unsigned class Mapping < BTFieldClass::Enumeration::Mapping end - def initialize(parent:, field_value_range:, preferred_display_base: 10, mappings:) - @mappings = mappings #TODO init Mapping + def initialize(parent:, field_value_range:, mappings:, preferred_display_base: 10) + @mappings = mappings # TODO: init Mapping super(parent: parent, field_value_range: field_value_range, preferred_display_base: preferred_display_base) end end @@ -374,8 +388,8 @@ class BTFieldClass::Enumeration::Signed < BTFieldClass::Integer::Signed class Mapping < BTFieldClass::Enumeration::Mapping end - def initialize(parent:, field_value_range:, preferred_display_base: 10, mappings:) - @mappings = mappings #TODO init Mapping + def initialize(parent:, field_value_range:, mappings:, preferred_display_base: 10) + @mappings = mappings # TODO: init Mapping super(parent: parent, field_value_range: field_value_range, preferred_display_base: preferred_display_base) end end @@ -390,6 +404,7 @@ def get_declarator(trace_class:, variable:) class BTFieldClass::Array < BTFieldClass attr_reader :element_field_class + def initialize(parent:, element_field_class:) @parent = parent @element_field_class = BTFieldClass.from_h(self, element_field_class) @@ -407,11 +422,11 @@ def initialize(parent:, element_field_class:, length:) def get_declarator(trace_class:, variable:) element_field_class_variable = "#{variable}_field_class" - scope { + scope do pr "bt_field_class *#{element_field_class_variable};" @element_field_class.get_declarator(trace_class: trace_class, variable: element_field_class_variable) pr "#{variable} = bt_field_class_array_static_create(#{trace_class}, #{element_field_class_variable}, #{@length});" - } + end end end @@ -424,14 +439,14 @@ module WithLengthField def initialize(parent:, element_field_class:, length_field_path: nil) super(parent: parent, element_field_class: element_field_class) if length_field_path - self.extend(WithLengthField) + extend(WithLengthField) @length_field_path = length_field_path end end def get_declarator(trace_class:, variable:) - element_field_class_variable= "#{variable}_field_class" - scope { + element_field_class_variable = "#{variable}_field_class" + scope do pr "bt_field_class *#{element_field_class_variable};" @element_field_class.get_declarator(trace_class: trace_class, variable: element_field_class_variable) if @length_field_path @@ -442,7 +457,7 @@ def get_declarator(trace_class:, variable:) else pr "#{variable} = bt_field_class_array_dynamic_create(#{trace_class}, #{element_field_class_variable}, NULL);" end - } + end end end @@ -450,10 +465,10 @@ class BTFieldClass::Structure < BTFieldClass extend BTFromH attr_reader :members + class Member include BTLocator - attr_reader :parent - attr_reader :name, :field_class + attr_reader :parent, :name, :field_class def initialize(parent:, name:, field_class:) @parent = parent @@ -477,16 +492,16 @@ def [](index) end def get_declarator(trace_class:, variable:) - @variable = variable - pr "#{variable} = bt_field_class_structure_create(#{trace_class});" - @members.each_with_index { |m, i| - var_name = "#{variable}_m_#{i}" - scope { - pr "bt_field_class *#{var_name};" - m.field_class.get_declarator(trace_class: trace_class, variable: var_name) - pr "bt_field_class_structure_append_member(#{variable}, \"#{m.name}\", #{var_name});" - } - } + @variable = variable + pr "#{variable} = bt_field_class_structure_create(#{trace_class});" + @members.each_with_index do |m, i| + var_name = "#{variable}_m_#{i}" + scope do + pr "bt_field_class *#{var_name};" + m.field_class.get_declarator(trace_class: trace_class, variable: var_name) + pr "bt_field_class_structure_append_member(#{variable}, \"#{m.name}\", #{var_name});" + end + end end end @@ -549,6 +564,7 @@ class BTFieldClass::Variant < BTFieldClass class Option attr_reader :name, :field_class + def initialize(parent:, name:, field_class:) @parent = parent @name = name @@ -561,8 +577,10 @@ module WithoutSelectorField module WithSelectorField attr_reader :selector_field_class + class Option < BTFieldClass::Variant::Option attr_reader :ranges + def initialize(parent:, name:, field_class:, ranges:) @ranges = ranges super(parent: parent, name: name, field_class: field_class) @@ -573,38 +591,37 @@ def initialize(parent:, name:, field_class:, ranges:) def initialize(parent:, options:, selector_field_class: nil) @parent = parent if selector_field_class - self.extend(WithSelectorField) + extend(WithSelectorField) @selector_field_class = selector_field_class - @options = options.collect { |o| + @options = options.collect do |o| BTFieldClass::Variant::WithSelectorField::Option.new(name: o[:name], field_class: o[:field], range: o[:range]) - } + end else - self.extend(WithoutSelectorField) - @options = options.collect { |o| + extend(WithoutSelectorField) + @options = options.collect do |o| BTFieldClass::Variant::Option.new(name: o[:name], field_class: o[:field]) - } + end end end end FIELD_CLASS_NAME_MAP = { - "bool" => BTFieldClass::Bool, - "bit_array" => BTFieldClass::BitArray, - "integer_unsigned" => BTFieldClass::Integer::Unsigned, - "integer_signed" => BTFieldClass::Integer::Signed, - "single" => BTFieldClass::Real::SinglePrecision, - "double" => BTFieldClass::Real::DoublePrecision, - "enumeration_unsigned" => BTFieldClass::Enumeration::Unsigned, - "enumeration_signed" => BTFieldClass::Enumeration::Signed, - "string" => BTFieldClass::String, - "array_static" => BTFieldClass::Array::Static, - "array_dynamic" => BTFieldClass::Array::Dynamic, - "structure" => BTFieldClass::Structure, - "option_without_selector_field" => BTFieldClass::Option::WithoutSelectorField, - "option_with_selector_field_bool" => BTFieldClass::Option::WithSelectorField::Bool, - "option_with_selector_field_unsigned" => BTFieldClass::Option::WithSelectorField::IntegerUnsigned, - "option_with_selector_field_signed" => BTFieldClass::Option::WithSelectorField::IntegerSigned, - "variant" => BTFieldClass::Variant + 'bool' => BTFieldClass::Bool, + 'bit_array' => BTFieldClass::BitArray, + 'integer_unsigned' => BTFieldClass::Integer::Unsigned, + 'integer_signed' => BTFieldClass::Integer::Signed, + 'single' => BTFieldClass::Real::SinglePrecision, + 'double' => BTFieldClass::Real::DoublePrecision, + 'enumeration_unsigned' => BTFieldClass::Enumeration::Unsigned, + 'enumeration_signed' => BTFieldClass::Enumeration::Signed, + 'string' => BTFieldClass::String, + 'array_static' => BTFieldClass::Array::Static, + 'array_dynamic' => BTFieldClass::Array::Dynamic, + 'structure' => BTFieldClass::Structure, + 'option_without_selector_field' => BTFieldClass::Option::WithoutSelectorField, + 'option_with_selector_field_bool' => BTFieldClass::Option::WithSelectorField::Bool, + 'option_with_selector_field_unsigned' => BTFieldClass::Option::WithSelectorField::IntegerUnsigned, + 'option_with_selector_field_signed' => BTFieldClass::Option::WithSelectorField::IntegerSigned, + 'variant' => BTFieldClass::Variant } - end From 60ae16b4c87a7f8367fce59c2e91f8fe5ebc2fc2 Mon Sep 17 00:00:00 2001 From: Thomas Applencourt Date: Mon, 26 Sep 2022 19:58:37 +0000 Subject: [PATCH 23/24] rubocop --- utils/gen_babeltrace_base.rb | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/utils/gen_babeltrace_base.rb b/utils/gen_babeltrace_base.rb index d6b87402..7a771397 100644 --- a/utils/gen_babeltrace_base.rb +++ b/utils/gen_babeltrace_base.rb @@ -14,7 +14,7 @@ def bt_set_conditionally(guard) module BTPrinter @@output = '' @@indent = 0 - INDENT_INCREMENT = ' ' + INDENT_INCREMENT = ' '.freeze def pr(str) @@output << INDENT_INCREMENT * @@indent << str << "\n" @@ -132,9 +132,9 @@ def get_declarator(self_component:, variable:) end class BTStreamClass - include BTLocator - include BTPrinter include BTUtils + include BTPrinter + include BTLocator extend BTFromH attr_reader :packet_context_field_class, :event_common_context_field_class, :event_classes, :id, :name @@ -143,7 +143,7 @@ def initialize(parent:, name: nil, packet_context_field_class: nil, event_common @parent = parent @name = name - raise 'Two packet_context' if packet_context_field_class and packet_context + raise 'Two packet_context' if packet_context_field_class && packet_context # Should put assert to check for struct @packet_context_field_class = BTFieldClass.from_h(self, packet_context_field_class) if packet_context_field_class @@ -215,8 +215,8 @@ def get_declarator(trace_class:, variable:) end class BTEventClass - include BTLocator include BTPrinter + include BTLocator extend BTFromH attr_reader :name, :specific_context_field_class, :payload_field_class @@ -323,7 +323,7 @@ def initialize(parent:, field_value_range: nil, preferred_display_base: nil) @preferred_display_base = preferred_display_base end - def get_declarator(variable:, trace_class: nil) + def get_declarator(variable:) pr "bt_field_class_integer_set_field_value_range(#{variable}, #{@field_value_range});" if @field_value_range if @preferred_display_base pr "bt_field_class_integer_set_preferred_display_base(#{variable}, #{@preferred_display_base});" @@ -336,7 +336,7 @@ class BTFieldClass::Integer::Unsigned < BTFieldClass::Integer extend BTFromH def get_declarator(trace_class:, variable:) pr "#{variable} = bt_field_class_integer_unsigned_create(#{trace_class});" - super + super(variable: variable) end end @@ -344,7 +344,7 @@ class BTFieldClass::Integer::Signed < BTFieldClass::Integer extend BTFromH def get_declarator(trace_class:, variable:) pr "#{variable} = bt_field_class_integer_signed_create(#{trace_class});" - super + super(variable: variable) end end @@ -623,5 +623,5 @@ def initialize(parent:, options:, selector_field_class: nil) 'option_with_selector_field_unsigned' => BTFieldClass::Option::WithSelectorField::IntegerUnsigned, 'option_with_selector_field_signed' => BTFieldClass::Option::WithSelectorField::IntegerSigned, 'variant' => BTFieldClass::Variant - } + }.freeze end From 422b9b5e589c44373d84605af1766b762c24800d Mon Sep 17 00:00:00 2001 From: Thomas Applencourt Date: Fri, 7 Oct 2022 21:50:46 +0000 Subject: [PATCH 24/24] Setter kind of working --- utils/gen_babeltrace_base.rb | 153 ++++++++++++++++++++++++++++------- 1 file changed, 126 insertions(+), 27 deletions(-) diff --git a/utils/gen_babeltrace_base.rb b/utils/gen_babeltrace_base.rb index 7a771397..820f6644 100644 --- a/utils/gen_babeltrace_base.rb +++ b/utils/gen_babeltrace_base.rb @@ -1,4 +1,7 @@ module Babeltrace2Gen + class GeneratedArg < Struct.new(:type, :name) + end + module BTFromH def from_h(parent, model) new(parent: parent, **model) @@ -9,6 +12,16 @@ module BTUtils def bt_set_conditionally(guard) yield guard ? 'BT_TRUE' : 'BT_FALSE' unless guard.nil? end + + def bt_get_variable(field, type, arg_variables) + if arg_variables.empty? || arg_variables.first.is_a?(GeneratedArg) + variable = "usr_#{field}" + arg_variables << GeneratedArg.new(type, variable) + variable + else + arg_variables.shift + end + end end module BTPrinter @@ -89,11 +102,18 @@ def find_field_class(path, variable) end end + # ______ _____ _____ _ + # | ___ \_ _| / __ \ | + # | |_/ / | | | / \/ | __ _ ___ ___ ___ ___ + # | ___ \ | | | | | |/ _` / __/ __|/ _ \/ __| + # | |_/ / | | | \__/\ | (_| \__ \__ \ __/\__ \ + # \____/ \_/ \____/_|\__,_|___/___/\___||___/ + # class BTTraceClass include BTLocator include BTPrinter include BTUtils - extend BTFromH + # extend BTFromH attr_reader :stream_classes, :assigns_automatic_stream_class_id @@ -102,9 +122,9 @@ def initialize(parent:, stream_classes:, assigns_automatic_stream_class_id: nil) @parent = nil @assigns_automatic_stream_class_id = assigns_automatic_stream_class_id - @stream_classes = stream_classes.collect do |m| + @stream_classes = stream_classes.collect.with_index do |m, i| if m[:id].nil? != (@assigns_automatic_stream_class_id.nil? || @assigns_automatic_stream_class_id) - raise 'Incoherent ID scheme' + raise "Incoherence between trace::assigns_automatic_stream_class_id and stream_class[#{i}]::id" end BTStreamClass.from_h(self, m) @@ -115,6 +135,11 @@ def trace_class self end + # Remove when the model is nice + def self.from_h(parent, model) + new(parent: parent, stream_classes: model[:stream_classes]) + end + def get_declarator(self_component:, variable:) pr "bt_trace_class *#{variable} = bt_trace_class_create(#{self_component});" bt_set_conditionally(@assigns_automatic_stream_class_id) do |v| @@ -262,6 +287,37 @@ def get_declarator(trace_class:, variable:, stream_class:) end end + def get_setter(message:, arg_variables:) + event = "#{message}_e" + scope do + pr "bt_event* #{event} = bt_message_event_borrow_event(message);" + + if stream_class.event_common_context_field_class + field = "#{event}_cc_f" + scope do + pr "bt_field *#{field} = bt_event_borrow_common_context_field(#{event});" + stream_class.event_common_context_field_class.get_setter(variable: field, arg_variables: arg_variables) + end + end + + if @specific_context_field_class + field = "#{event}_sc_f" + scope do + pr "bt_field *#{field} = bt_event_borrow_specific_context_field(#{event});" + @specific_context_field_class.get_setter(variable: field, arg_variables: arg_variables) + end + end + + if @payload_field_class + field = "#{event}_p_f" + scope do + pr "bt_field *#{field} = bt_event_borrow_payload_field(#{event});" + @payload_field_class.get_setter(variable: field, arg_variables: arg_variables) + end + end + end + end + def event_class self end @@ -280,10 +336,31 @@ def initialize(parent:) def self.from_h(parent, model) key = model.delete(:type) raise "No type in #{model}" unless key - raise "No #{key} in FIELD_CLASS_NAME_MAP" unless FIELD_CLASS_NAME_MAP.include?(key) + + h = { + 'bool' => BTFieldClass::Bool, + 'bit_array' => BTFieldClass::BitArray, + 'integer_unsigned' => BTFieldClass::Integer::Unsigned, + 'integer_signed' => BTFieldClass::Integer::Signed, + 'single' => BTFieldClass::Real::SinglePrecision, + 'double' => BTFieldClass::Real::DoublePrecision, + 'enumeration_unsigned' => BTFieldClass::Enumeration::Unsigned, + 'enumeration_signed' => BTFieldClass::Enumeration::Signed, + 'string' => BTFieldClass::String, + 'array_static' => BTFieldClass::Array::Static, + 'array_dynamic' => BTFieldClass::Array::Dynamic, + 'structure' => BTFieldClass::Structure, + 'option_without_selector_field' => BTFieldClass::Option::WithoutSelectorField, + 'option_with_selector_field_bool' => BTFieldClass::Option::WithSelectorField::Bool, + 'option_with_selector_field_unsigned' => BTFieldClass::Option::WithSelectorField::IntegerUnsigned, + 'option_with_selector_field_signed' => BTFieldClass::Option::WithSelectorField::IntegerSigned, + 'variant' => BTFieldClass::Variant + }.freeze + + raise "No #{key} in FIELD_CLASS_NAME_MAP" unless h.include?(key) cast_type = model.delete(:cast_type) - fc = FIELD_CLASS_NAME_MAP[key].from_h(parent, model) + fc = h[key].from_h(parent, model) fc.cast_type = cast_type if cast_type fc end @@ -295,13 +372,21 @@ def get_declarator(*args, **dict) class BTFieldClass::Bool < BTFieldClass extend BTFromH + include BTUtils + def get_declarator(trace_class:, variable:) pr "#{variable} = bt_field_class_bool_create(#{trace_class});" end + + def get_setter(field:, arg_variables:) + variable = bt_get_variable(field, 'bt_bool', arg_variables) + pr "bt_field_bit_array_set_value_as_integer(#{field}, #{variable});" + end end class BTFieldClass::BitArray < BTFieldClass extend BTFromH + include BTUtils attr_reader :length def initialize(parent:, length:) @@ -312,11 +397,18 @@ def initialize(parent:, length:) def get_declarator(trace_class:, variable:) pr "#{variable} = bt_field_class_bit_array_create(#{trace_class}, #{@length});" end + + def get_setter(field:, arg_variables:) + variable = bt_get_variable(field, 'uint64_t', arg_variables) + pr "bt_field_bit_array_set_value_as_integer(#{field}, #{variable});" + end end class BTFieldClass::Integer < BTFieldClass attr_reader :field_value_range, :preferred_display_base + include BTUtils + def initialize(parent:, field_value_range: nil, preferred_display_base: nil) @parent = parent @field_value_range = field_value_range @@ -338,6 +430,11 @@ def get_declarator(trace_class:, variable:) pr "#{variable} = bt_field_class_integer_unsigned_create(#{trace_class});" super(variable: variable) end + + def get_setter(field:, arg_variables:) + variable = bt_get_variable(field, 'uint64_t', arg_variables) + pr "bt_field_integer_unsigned_get_value(#{field}, #{variable});" + end end class BTFieldClass::Integer::Signed < BTFieldClass::Integer @@ -346,6 +443,11 @@ def get_declarator(trace_class:, variable:) pr "#{variable} = bt_field_class_integer_signed_create(#{trace_class});" super(variable: variable) end + + def get_setter(field:, arg_variables:) + variable = bt_get_variable(field, 'int64_t', arg_variables) + pr "bt_field_integer_signed_get_value(#{field}, #{variable});" + end end class BTFieldClass::Real < BTFieldClass @@ -396,10 +498,16 @@ def initialize(parent:, field_value_range:, mappings:, preferred_display_base: 1 class BTFieldClass::String < BTFieldClass extend BTFromH + include BTUtils def get_declarator(trace_class:, variable:) pr "#{variable} = bt_field_class_string_create(#{trace_class});" end + + def get_setter(field:, arg_variables:) + variable = bt_get_variable(field, 'const char*', arg_variables) + pr "bt_field_string_set_value(#{field}, #{variable});" + end end class BTFieldClass::Array < BTFieldClass @@ -503,6 +611,16 @@ def get_declarator(trace_class:, variable:) end end end + + def get_setter(variable:, arg_variables:) + @members.each_with_index do |m, i| + field = "#{variable}_m_#{i}" + scope do + pr "bt_field *#{field} = bt_field_structure_borrow_member_field_by_index(#{variable}, #{i});" + m.field_class.get_setter(field: field, arg_variables: arg_variables) + end + end + end end class BTFieldClass::Option < BTFieldClass @@ -594,34 +712,15 @@ def initialize(parent:, options:, selector_field_class: nil) extend(WithSelectorField) @selector_field_class = selector_field_class @options = options.collect do |o| - BTFieldClass::Variant::WithSelectorField::Option.new(name: o[:name], field_class: o[:field], range: o[:range]) + BTFieldClass::Variant::WithSelectorField::Option.new(name: o[:name], field_class: o[:field_class], + range: o[:range]) end else extend(WithoutSelectorField) @options = options.collect do |o| - BTFieldClass::Variant::Option.new(name: o[:name], field_class: o[:field]) + BTFieldClass::Variant::Option.new(name: o[:name], field_class: o[:field_class]) end end end end - - FIELD_CLASS_NAME_MAP = { - 'bool' => BTFieldClass::Bool, - 'bit_array' => BTFieldClass::BitArray, - 'integer_unsigned' => BTFieldClass::Integer::Unsigned, - 'integer_signed' => BTFieldClass::Integer::Signed, - 'single' => BTFieldClass::Real::SinglePrecision, - 'double' => BTFieldClass::Real::DoublePrecision, - 'enumeration_unsigned' => BTFieldClass::Enumeration::Unsigned, - 'enumeration_signed' => BTFieldClass::Enumeration::Signed, - 'string' => BTFieldClass::String, - 'array_static' => BTFieldClass::Array::Static, - 'array_dynamic' => BTFieldClass::Array::Dynamic, - 'structure' => BTFieldClass::Structure, - 'option_without_selector_field' => BTFieldClass::Option::WithoutSelectorField, - 'option_with_selector_field_bool' => BTFieldClass::Option::WithSelectorField::Bool, - 'option_with_selector_field_unsigned' => BTFieldClass::Option::WithSelectorField::IntegerUnsigned, - 'option_with_selector_field_signed' => BTFieldClass::Option::WithSelectorField::IntegerSigned, - 'variant' => BTFieldClass::Variant - }.freeze end