diff --git a/utils/gen_babeltrace_base.rb b/utils/gen_babeltrace_base.rb new file mode 100644 index 00000000..820f6644 --- /dev/null +++ b/utils/gen_babeltrace_base.rb @@ -0,0 +1,726 @@ +module Babeltrace2Gen + class GeneratedArg < Struct.new(:type, :name) + end + + module BTFromH + def from_h(parent, model) + new(parent: parent, **model) + end + end + + module BTUtils + def bt_set_conditionally(guard) + yield guard ? 'BT_TRUE' : 'BT_FALSE' unless guard.nil? + end + + def bt_get_variable(field, type, arg_variables) + if arg_variables.empty? || arg_variables.first.is_a?(GeneratedArg) + variable = "usr_#{field}" + arg_variables << GeneratedArg.new(type, variable) + variable + else + arg_variables.shift + end + end + end + + module BTPrinter + @@output = '' + @@indent = 0 + INDENT_INCREMENT = ' '.freeze + + def pr(str) + @@output << INDENT_INCREMENT * @@indent << str << "\n" + end + module_function :pr + + def scope + pr '{' + @@indent += 1 + yield + @@indent -= 1 + pr '}' + end + + def self.context(output: '', indent: 0) + @@output = output + @@indent = indent + yield + @@output + end + end + + def self.context(**args, &block) + BTPrinter.context(**args, &block) + end + + module BTLocator + attr_reader :parent, :variable + + def root + @parent ? @parent.root : self + end + + def stream_class + @parent ? @parent.stream_class : nil + end + + def trace_class + @parent ? @parent.trace_class : nil + end + + def event_class + @parent ? @parent.event_class : nil + end + + def find_field_class_path(path, variable) + path.scan(/\["(\w+)"\]|\[(\d+)\]/).each do |m| + # String + if m.first + pr "#{variable} = bt_field_class_structure_borrow_member_by_name(#{variable}, \"#{m.first}\");" + else + pr "#{variable} = bt_field_class_structure_borrow_member_by_index(#{variable}, #{m.last});" + end + end + end + + def find_field_class(path, variable) + m = path.match(/\A(PACKET_CONTEXT|EVENT_COMMON_CONTEXT|EVENT_SPECIFIC_CONTEXT|EVENT_PAYLOAD)(.*)/) + case m[1] + when 'PACKET_CONTEXT' + pr "#{variable} = #{stream_class.packet_context_field_class.variable};" + when 'EVENT_COMMON_CONTEXT' + pr "#{variable} = #{stream_class.event_common_context_field_class.variable};" + when 'EVENT_SPECIFIC_CONTEXT' + pr "#{variable} = #{event_class.specific_context_field_class.varible};" + when 'EVENT_PAYLOAD' + pr "#{variable} = #{event_class.payload_field_class.variable};" + else + raise "invalid path #{path}" + end + find_field_class_path(m[2], variable) + end + end + + # ______ _____ _____ _ + # | ___ \_ _| / __ \ | + # | |_/ / | | | / \/ | __ _ ___ ___ ___ ___ + # | ___ \ | | | | | |/ _` / __/ __|/ _ \/ __| + # | |_/ / | | | \__/\ | (_| \__ \__ \ __/\__ \ + # \____/ \_/ \____/_|\__,_|___/___/\___||___/ + # + class BTTraceClass + include BTLocator + include BTPrinter + include BTUtils + # extend BTFromH + + attr_reader :stream_classes, :assigns_automatic_stream_class_id + + def initialize(parent:, stream_classes:, assigns_automatic_stream_class_id: nil) + raise if parent + + @parent = nil + @assigns_automatic_stream_class_id = assigns_automatic_stream_class_id + @stream_classes = stream_classes.collect.with_index do |m, i| + if m[:id].nil? != (@assigns_automatic_stream_class_id.nil? || @assigns_automatic_stream_class_id) + raise "Incoherence between trace::assigns_automatic_stream_class_id and stream_class[#{i}]::id" + end + + BTStreamClass.from_h(self, m) + end + end + + def trace_class + self + end + + # Remove when the model is nice + def self.from_h(parent, model) + new(parent: parent, stream_classes: model[:stream_classes]) + end + + def get_declarator(self_component:, variable:) + pr "bt_trace_class *#{variable} = bt_trace_class_create(#{self_component});" + bt_set_conditionally(@assigns_automatic_stream_class_id) do |v| + pr "bt_trace_class_set_assigns_automatic_stream_class_id(#{variable}, #{v});" + end + + @stream_classes.each_with_index do |m, i| + stream_class_name = "#{variable}_sc_#{i}" + scope do + pr "bt_stream_class *#{stream_class_name};" + m.get_declarator(trace_class: variable, variable: stream_class_name) + end + end + end + end + + class BTStreamClass + include BTUtils + include BTPrinter + include BTLocator + extend BTFromH + attr_reader :packet_context_field_class, :event_common_context_field_class, :event_classes, :id, :name + + def initialize(parent:, name: nil, packet_context_field_class: nil, event_common_context_field_class: nil, + event_classes: [], id: nil, assigns_automatic_event_class_id: nil, assigns_automatic_stream_id: nil) + @parent = parent + @name = name + + raise 'Two packet_context' if packet_context_field_class && packet_context + + # Should put assert to check for struct + @packet_context_field_class = BTFieldClass.from_h(self, packet_context_field_class) if packet_context_field_class + + # Should put assert to check for struct + if event_common_context_field_class + @event_common_context_field_class = BTFieldClass.from_h(self, + event_common_context_field_class) + end + + @assigns_automatic_event_class_id = assigns_automatic_event_class_id + @event_classes = event_classes.collect do |ec| + if ec[:id].nil? != (@assigns_automatic_event_class_id.nil? || @assigns_automatic_event_class_id.nil) + raise 'Incorect id scheme' + end + + BTEventClass.from_h(self, ec) + end + @assigns_automatic_stream_id = assigns_automatic_stream_id + @id = id + end + + def stream_class + self + end + + def get_declarator(trace_class:, variable:) + if @id + pr "#{variable} = bt_stream_class_create_with_id(#{trace_class}, #{@id});" + else + pr "#{variable} = bt_stream_class_create(#{trace_class});" + end + pr "bt_stream_class_set_name(#{variable}, \"#{name}\");" if @name + + if @packet_context_field_class + var_pc = "#{variable}_pc_fc" + scope do + pr "bt_field_class *#{var_pc};" + @packet_context_field_class.get_declarator(trace_class: trace_class, variable: var_pc) + pr "bt_stream_class_set_packet_context_field_class(#{variable}, #{var_pc});" + end + end + + if @event_common_context_field_class + var_ecc = "#{variable}_ecc_fc" + scope do + pr "bt_field_class *#{var_ecc};" + @event_common_context_field_class.get_declarator(trace_class: trace_class, variable: var_ecc) + pr "bt_stream_class_set_event_common_context_field_class(#{variable}, #{var_ecc});" + end + end + # Need to do is afert packet an devent_common_context because it can refer members to those + bt_set_conditionally(@assigns_automatic_event_class_id) do |v| + pr "bt_stream_class_set_assigns_automatic_event_class_id(#{variable}, #{v});" + end + + @event_classes.each_with_index do |ec, i| + var_name = "#{variable}_ec_#{i}" + scope do + pr "bt_event_class *#{var_name};" + ec.get_declarator(trace_class: trace_class, variable: var_name, stream_class: variable) + end + end + + bt_set_conditionally(@assigns_automatic_stream_id) do |v| + pr "bt_stream_class_set_assigns_automatic_stream_id(#{variable}, #{v});" + end + end + end + + class BTEventClass + include BTPrinter + include BTLocator + extend BTFromH + attr_reader :name, :specific_context_field_class, :payload_field_class + + def initialize(parent:, name: nil, specific_context_field_class: nil, payload_field_class: nil, id: nil) + @parent = parent + @name = name + if specific_context_field_class + @specific_context_field_class = BTFieldClass.from_h(self, + specific_context_field_class) + end + @payload_field_class = BTFieldClass.from_h(self, payload_field_class) if payload_field_class + + @id = id + end + + def get_declarator(trace_class:, variable:, stream_class:) + # Store the variable name for instrocption purpose for LOCATION_PATH + @variable = variable + if @id + pr "#{variable} = bt_event_class_create_with_id(#{stream_class}, #{@id});" + else + pr "#{variable} = bt_event_class_create(#{stream_class});" + end + + pr "bt_event_class_set_name(#{variable}, \"#{@name}\");" if @name + + if @specific_context_field_class + var_name = "#{variable}_sc_fc" + scope do + pr "bt_field_class *#{var_name};" + @specific_context_field_class.get_declarator(trace_class: trace_class, variable: var_name) + pr "bt_event_class_set_specific_context_field_class(#{variable}, #{var_name});" + end + end + + if @payload_field_class + var_name = "#{variable}_p_fc" + scope do + pr "bt_field_class *#{var_name};" + @payload_field_class.get_declarator(trace_class: trace_class, variable: var_name) + pr "bt_event_class_set_payload_field_class(#{variable}, #{var_name});" + end + end + end + + def get_setter(message:, arg_variables:) + event = "#{message}_e" + scope do + pr "bt_event* #{event} = bt_message_event_borrow_event(message);" + + if stream_class.event_common_context_field_class + field = "#{event}_cc_f" + scope do + pr "bt_field *#{field} = bt_event_borrow_common_context_field(#{event});" + stream_class.event_common_context_field_class.get_setter(variable: field, arg_variables: arg_variables) + end + end + + if @specific_context_field_class + field = "#{event}_sc_f" + scope do + pr "bt_field *#{field} = bt_event_borrow_specific_context_field(#{event});" + @specific_context_field_class.get_setter(variable: field, arg_variables: arg_variables) + end + end + + if @payload_field_class + field = "#{event}_p_f" + scope do + pr "bt_field *#{field} = bt_event_borrow_payload_field(#{event});" + @payload_field_class.get_setter(variable: field, arg_variables: arg_variables) + end + end + end + end + + def event_class + self + end + end + + class BTFieldClass + include BTLocator + include BTPrinter + + attr_accessor :cast_type + + def initialize(parent:) + @parent = parent + end + + def self.from_h(parent, model) + key = model.delete(:type) + raise "No type in #{model}" unless key + + h = { + 'bool' => BTFieldClass::Bool, + 'bit_array' => BTFieldClass::BitArray, + 'integer_unsigned' => BTFieldClass::Integer::Unsigned, + 'integer_signed' => BTFieldClass::Integer::Signed, + 'single' => BTFieldClass::Real::SinglePrecision, + 'double' => BTFieldClass::Real::DoublePrecision, + 'enumeration_unsigned' => BTFieldClass::Enumeration::Unsigned, + 'enumeration_signed' => BTFieldClass::Enumeration::Signed, + 'string' => BTFieldClass::String, + 'array_static' => BTFieldClass::Array::Static, + 'array_dynamic' => BTFieldClass::Array::Dynamic, + 'structure' => BTFieldClass::Structure, + 'option_without_selector_field' => BTFieldClass::Option::WithoutSelectorField, + 'option_with_selector_field_bool' => BTFieldClass::Option::WithSelectorField::Bool, + 'option_with_selector_field_unsigned' => BTFieldClass::Option::WithSelectorField::IntegerUnsigned, + 'option_with_selector_field_signed' => BTFieldClass::Option::WithSelectorField::IntegerSigned, + 'variant' => BTFieldClass::Variant + }.freeze + + raise "No #{key} in FIELD_CLASS_NAME_MAP" unless h.include?(key) + + cast_type = model.delete(:cast_type) + fc = h[key].from_h(parent, model) + fc.cast_type = cast_type if cast_type + fc + end + + def get_declarator(*args, **dict) + raise NotImplementedError, self.class + end + end + + class BTFieldClass::Bool < BTFieldClass + extend BTFromH + include BTUtils + + def get_declarator(trace_class:, variable:) + pr "#{variable} = bt_field_class_bool_create(#{trace_class});" + end + + def get_setter(field:, arg_variables:) + variable = bt_get_variable(field, 'bt_bool', arg_variables) + pr "bt_field_bit_array_set_value_as_integer(#{field}, #{variable});" + end + end + + class BTFieldClass::BitArray < BTFieldClass + extend BTFromH + include BTUtils + attr_reader :length + + def initialize(parent:, length:) + @parent = parent + @length = length + end + + def get_declarator(trace_class:, variable:) + pr "#{variable} = bt_field_class_bit_array_create(#{trace_class}, #{@length});" + end + + def get_setter(field:, arg_variables:) + variable = bt_get_variable(field, 'uint64_t', arg_variables) + pr "bt_field_bit_array_set_value_as_integer(#{field}, #{variable});" + end + end + + class BTFieldClass::Integer < BTFieldClass + attr_reader :field_value_range, :preferred_display_base + + include BTUtils + + def initialize(parent:, field_value_range: nil, preferred_display_base: nil) + @parent = parent + @field_value_range = field_value_range + @preferred_display_base = preferred_display_base + end + + def get_declarator(variable:) + pr "bt_field_class_integer_set_field_value_range(#{variable}, #{@field_value_range});" if @field_value_range + if @preferred_display_base + pr "bt_field_class_integer_set_preferred_display_base(#{variable}, #{@preferred_display_base});" + end + end + end + BTFieldClassInteger = BTFieldClass::Integer + + class BTFieldClass::Integer::Unsigned < BTFieldClass::Integer + extend BTFromH + def get_declarator(trace_class:, variable:) + pr "#{variable} = bt_field_class_integer_unsigned_create(#{trace_class});" + super(variable: variable) + end + + def get_setter(field:, arg_variables:) + variable = bt_get_variable(field, 'uint64_t', arg_variables) + pr "bt_field_integer_unsigned_get_value(#{field}, #{variable});" + end + end + + class BTFieldClass::Integer::Signed < BTFieldClass::Integer + extend BTFromH + def get_declarator(trace_class:, variable:) + pr "#{variable} = bt_field_class_integer_signed_create(#{trace_class});" + super(variable: variable) + end + + def get_setter(field:, arg_variables:) + variable = bt_get_variable(field, 'int64_t', arg_variables) + pr "bt_field_integer_signed_get_value(#{field}, #{variable});" + end + end + + class BTFieldClass::Real < BTFieldClass + end + + class BTFieldClass::Real::SinglePrecision < BTFieldClass::Real + extend BTFromH + def get_declarator(trace_class:, variable:) + pr "#{variable} = bt_field_class_real_single_precision_create(#{trace_class});" + end + end + + class BTFieldClass::Real::DoublePrecision < BTFieldClass::Real + extend BTFromH + def get_declarator(trace_class:, variable:) + pr "#{variable} = bt_field_class_real_double_precision_create(#{trace_class});" + end + end + + module BTFieldClass::Enumeration + attr_reader :mappings + + class Mapping + end + end + + class BTFieldClass::Enumeration::Unsigned < BTFieldClass::Integer::Unsigned + include BTFieldClass::Enumeration + class Mapping < BTFieldClass::Enumeration::Mapping + end + + def initialize(parent:, field_value_range:, mappings:, preferred_display_base: 10) + @mappings = mappings # TODO: init Mapping + super(parent: parent, field_value_range: field_value_range, preferred_display_base: preferred_display_base) + end + end + + class BTFieldClass::Enumeration::Signed < BTFieldClass::Integer::Signed + include BTFieldClass::Enumeration + class Mapping < BTFieldClass::Enumeration::Mapping + end + + def initialize(parent:, field_value_range:, mappings:, preferred_display_base: 10) + @mappings = mappings # TODO: init Mapping + super(parent: parent, field_value_range: field_value_range, preferred_display_base: preferred_display_base) + end + end + + class BTFieldClass::String < BTFieldClass + extend BTFromH + include BTUtils + + def get_declarator(trace_class:, variable:) + pr "#{variable} = bt_field_class_string_create(#{trace_class});" + end + + def get_setter(field:, arg_variables:) + variable = bt_get_variable(field, 'const char*', arg_variables) + pr "bt_field_string_set_value(#{field}, #{variable});" + end + end + + class BTFieldClass::Array < BTFieldClass + attr_reader :element_field_class + + def initialize(parent:, element_field_class:) + @parent = parent + @element_field_class = BTFieldClass.from_h(self, element_field_class) + end + end + + class BTFieldClass::Array::Static < BTFieldClass::Array + extend BTFromH + attr_reader :length + + def initialize(parent:, element_field_class:, length:) + @length = length + super(parent: parent, element_field_class: element_field_class) + end + + def get_declarator(trace_class:, variable:) + element_field_class_variable = "#{variable}_field_class" + scope do + pr "bt_field_class *#{element_field_class_variable};" + @element_field_class.get_declarator(trace_class: trace_class, variable: element_field_class_variable) + pr "#{variable} = bt_field_class_array_static_create(#{trace_class}, #{element_field_class_variable}, #{@length});" + end + end + end + + class BTFieldClass::Array::Dynamic < BTFieldClass::Array + extend BTFromH + module WithLengthField + attr_reader :length_field_path + end + + def initialize(parent:, element_field_class:, length_field_path: nil) + super(parent: parent, element_field_class: element_field_class) + if length_field_path + extend(WithLengthField) + @length_field_path = length_field_path + end + end + + def get_declarator(trace_class:, variable:) + element_field_class_variable = "#{variable}_field_class" + scope do + pr "bt_field_class *#{element_field_class_variable};" + @element_field_class.get_declarator(trace_class: trace_class, variable: element_field_class_variable) + if @length_field_path + element_field_class_variable_length = "#{element_field_class_variable}_length" + pr "bt_field_class *#{element_field_class_variable_length};" + find_field_class(@length_field_path, element_field_class_variable_length) + pr "#{variable} = bt_field_class_array_dynamic_create(#{trace_class}, #{element_field_class_variable}, #{element_field_class_variable_length});" + else + pr "#{variable} = bt_field_class_array_dynamic_create(#{trace_class}, #{element_field_class_variable}, NULL);" + end + end + end + end + + class BTFieldClass::Structure < BTFieldClass + extend BTFromH + + attr_reader :members + + class Member + include BTLocator + attr_reader :parent, :name, :field_class + + def initialize(parent:, name:, field_class:) + @parent = parent + @name = name + @field_class = BTFieldClass.from_h(self, field_class) + end + end + + def initialize(parent:, members: []) + @parent = parent + @members = members.collect { |m| Member.new(parent: self, **m) } + end + + def [](index) + case index + when Integer + @members[index] + when String + @members.find { |m| m.name == index } + end + end + + def get_declarator(trace_class:, variable:) + @variable = variable + pr "#{variable} = bt_field_class_structure_create(#{trace_class});" + @members.each_with_index do |m, i| + var_name = "#{variable}_m_#{i}" + scope do + pr "bt_field_class *#{var_name};" + m.field_class.get_declarator(trace_class: trace_class, variable: var_name) + pr "bt_field_class_structure_append_member(#{variable}, \"#{m.name}\", #{var_name});" + end + end + end + + def get_setter(variable:, arg_variables:) + @members.each_with_index do |m, i| + field = "#{variable}_m_#{i}" + scope do + pr "bt_field *#{field} = bt_field_structure_borrow_member_field_by_index(#{variable}, #{i});" + m.field_class.get_setter(field: field, arg_variables: arg_variables) + end + end + end + end + + class BTFieldClass::Option < BTFieldClass + attr_reader :field_class + + def initialize(parent:, field_class:) + @parent = parent + @field_class = BTFieldClass.from_h(self, field_class) + end + end + BTFieldClassOption = BTFieldClass::Option + + class BTFieldClass::Option::WithoutSelectorField < BTFieldClass::Option + extend BTFromH + end + + class BTFieldClass::Option::WithSelectorField < BTFieldClass::Option + attr_reader :selector_field_path + + def initialize(parent:, field_class:, selector_field_path:) + @selector_field_path = selector_field_path + super(parent: parent, field_class: field_class) + end + end + + class BTFieldClass::Option::WithSelectorField::Bool < BTFieldClass::Option::WithSelectorField + extend BTFromH + attr_reader :selector_is_reversed + + def initialize(parent:, field_class:, selector_field_path:, selector_is_reversed: nil) + @selector_is_reversed = selector_is_reversed + super(parent: parent, field_class: field_class, selector_field_path: selector_field_path) + end + end + + class BTFieldClass::Option::WithSelectorField::IntegerUnsigned < BTFieldClass::Option::WithSelectorField + extend BTFromH + attr_reader :selector_ranges + + def initialize(parent:, field_class:, selector_field_path:, selector_ranges:) + @selector_ranges = selector_ranges + super(parent: parent, field_class: field_class, selector_field_path: selector_field_path) + end + end + + class BTFieldClass::Option::WithSelectorField::IntegerSigned < BTFieldClass::Option::WithSelectorField + extend BTFromH + attr_reader :selector_ranges + + def initialize(parent:, field_class:, selector_field_path:, selector_ranges:) + @selector_ranges = selector_ranges + super(parent: parent, field_class: field_class, selector_field_path: selector_field_path) + end + end + + class BTFieldClass::Variant < BTFieldClass + extend BTFromH + attr_reader :options + + class Option + attr_reader :name, :field_class + + def initialize(parent:, name:, field_class:) + @parent = parent + @name = name + @field_class = BTFieldClass.from_h(self, field_class) + end + end + + module WithoutSelectorField + end + + module WithSelectorField + attr_reader :selector_field_class + + class Option < BTFieldClass::Variant::Option + attr_reader :ranges + + def initialize(parent:, name:, field_class:, ranges:) + @ranges = ranges + super(parent: parent, name: name, field_class: field_class) + end + end + end + + def initialize(parent:, options:, selector_field_class: nil) + @parent = parent + if selector_field_class + extend(WithSelectorField) + @selector_field_class = selector_field_class + @options = options.collect do |o| + BTFieldClass::Variant::WithSelectorField::Option.new(name: o[:name], field_class: o[:field_class], + range: o[:range]) + end + else + extend(WithoutSelectorField) + @options = options.collect do |o| + BTFieldClass::Variant::Option.new(name: o[:name], field_class: o[:field_class]) + end + end + end + end +end diff --git a/utils/gen_babeltrace_emitter.rb b/utils/gen_babeltrace_emitter.rb index 008d4fee..7e26d4a2 100644 --- a/utils/gen_babeltrace_emitter.rb +++ b/utils/gen_babeltrace_emitter.rb @@ -8,7 +8,7 @@ #include EOF - +# We should refractor the class_create to take a parameter declare_signed = lambda { |_, name| puts < declare_unsigned, 'string' => declare_string, 'bool' => declare_bool, - 'structure' => declare_structure + 'structure' => declare_structure, + 'array_dynamic' => declare_array_dynamic }) def declare_group(type, method, group_name, content) @@ -65,6 +73,10 @@ def declare_group(type, method, group_name, content) content.each { |field| name = field[:name] klass = field[:class] + if klass == "array_dynamic" + element_klass = field[:field][:class] + $print_declarators[element_klass].call(element_klass, "#{name}_element") + end $print_declarators[klass].call(klass, name) append_member(name, group_name) } @@ -73,6 +85,11 @@ def declare_group(type, method, group_name, content) bt_field_class_put_ref(#{group_name}_field_class); EOF content.each { |field| + if field[:class] == "array_dynamic" + puts < +EOF + +def set_scalar(field_name, index, bt2_class, name) + puts < const char* borrow_hostname(const bt_event *event){ const bt_stream *stream = bt_event_borrow_stream_const(event); @@ -22,7 +23,7 @@ thread_id_t borrow_thread_id(const bt_event *event){ bt_message* create_host_message(const char* hostname, const process_id_t process_id, const thread_id_t thread_id, const char* name, const uint64_t ts, const uint64_t duration, const bool err, - bt_event_class *event_class, bt_self_message_iterator *message_iterator, bt_stream *stream, backend_t backend) { + bt_event_class *event_class, bt_self_message_iterator *message_iterator, bt_stream *stream, backend_t backend, std::set flow_ids) { /* Message creation */ bt_message *message = bt_message_event_create( @@ -65,6 +66,21 @@ bt_message* create_host_message(const char* hostname, const process_id_t process bt_field *err_field = bt_field_structure_borrow_member_field_by_index(payload_field, 2); bt_field_integer_unsigned_set_value(err_field, err); + // flow id + if (!flow_ids.empty() ) { + const auto size = flow_ids.size(); + //bt_field *flow_id_field_length = bt_field_structure_borrow_member_field_by_index(payload_field, 3); + //bt_field_integer_unsigned_set_value(flow_id_field_length, size); + + bt_field *flow_id_field = bt_field_structure_borrow_member_field_by_index(payload_field, 4); + bt_field_array_dynamic_set_length(flow_id_field, size); + + int i=0; + for (auto element: flow_ids) { + bt_field *element_field = bt_field_structure_borrow_member_field_by_index(flow_id_field, i++); + bt_field_integer_unsigned_set_value(element_field, element); + } + } return message; } @@ -125,8 +141,79 @@ bt_message* create_device_message(const char* hostname, const process_id_t proce //Metadata bt_field *metadata_field = bt_field_structure_borrow_member_field_by_index(payload_field, 5); bt_field_string_set_value(metadata_field, metadata); + return message; +} - return message; +bt_message* create_device_flow_message(const char* hostname, const process_id_t process_id, const uint64_t uuid, + const thapi_device_id device_id, const thapi_device_id subdevice_id, + const char* name, const uint64_t ts, const uint64_t duration, const bool err, + const char* metadata, const char* queue_name, const flow_id_t flow_id, + bt_event_class *event_class, bt_self_message_iterator *message_iterator, bt_stream *stream) { + + /* Message creation */ + bt_message *message = bt_message_event_create( + message_iterator, event_class, stream); + + + /* event */ + bt_event *downstream_event = bt_message_event_borrow_event(message); + + /* Common context */ + bt_field *context_field = bt_event_borrow_common_context_field(downstream_event); + + // Hostname + bt_field *hostname_msg_field = bt_field_structure_borrow_member_field_by_index(context_field,0); + bt_field_string_set_value(hostname_msg_field, hostname); + // pid + bt_field *vpid_field = bt_field_structure_borrow_member_field_by_index(context_field,1); + bt_field_integer_signed_set_value(vpid_field, process_id); + // vid + bt_field *vtid_field = bt_field_structure_borrow_member_field_by_index(context_field,2); + bt_field_integer_signed_set_value(vtid_field, 0); + // ts + bt_field *ts_field = bt_field_structure_borrow_member_field_by_index(context_field,3); + bt_field_integer_signed_set_value(ts_field, ts); + + /* Payload */ + bt_field *payload_field = bt_event_borrow_payload_field(downstream_event); + + // name + bt_field *name_field = bt_field_structure_borrow_member_field_by_index(payload_field, 0); + bt_field_string_set_value(name_field, name); + + // dur + bt_field *dur_field = bt_field_structure_borrow_member_field_by_index(payload_field, 1); + bt_field_integer_unsigned_set_value(dur_field, duration); + + // did + bt_field *device_id_field = bt_field_structure_borrow_member_field_by_index(payload_field,2); + bt_field_integer_unsigned_set_value(device_id_field, device_id); + + // sdid + bt_field *subdevice_id_field = bt_field_structure_borrow_member_field_by_index(payload_field,3); + bt_field_integer_unsigned_set_value(subdevice_id_field, subdevice_id); + + // err + bt_field *err_field = bt_field_structure_borrow_member_field_by_index(payload_field, 4); + bt_field_integer_unsigned_set_value(err_field, err); + + //Metadata + bt_field *metadata_field = bt_field_structure_borrow_member_field_by_index(payload_field, 5); + bt_field_string_set_value(metadata_field, metadata); + + // uuid + bt_field *uuid_field = bt_field_structure_borrow_member_field_by_index(payload_field, 6); + bt_field_integer_unsigned_set_value(uuid_field, uuid); + + //Queue_name + bt_field *queue_name_field = bt_field_structure_borrow_member_field_by_index(payload_field, 7); + bt_field_string_set_value(queue_name_field, queue_name); + + // flow_id + bt_field *flow_field = bt_field_structure_borrow_member_field_by_index(payload_field, 8); + bt_field_integer_unsigned_set_value(flow_field, flow_id); + + return message; } bt_message* create_device_name_message(const char* hostname, const process_id_t process_id, diff --git a/utils/xprof_utils.hpp b/utils/xprof_utils.hpp index c68a1138..16f939cd 100644 --- a/utils/xprof_utils.hpp +++ b/utils/xprof_utils.hpp @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include #include "babeltrace2/babeltrace.h" @@ -28,10 +28,15 @@ typedef uintptr_t thread_id_t; typedef std::string hostname_t; typedef std::string thapi_function_name; typedef uintptr_t thapi_device_id; +typedef uint64_t flow_id_t; // Represent a device and a sub device typedef std::tuple dsd_t; typedef std::tuple h_device_t; + +typedef std::tuple h_d_t; +typedef std::tuple h_dsd_t; + typedef std::tuple hp_t; typedef std::tuple hpt_t; typedef std::tuple hpt_function_name_t; @@ -120,12 +125,18 @@ thread_id_t borrow_thread_id(const bt_event*); bt_message* create_host_message(const char *hostname, const process_id_t, const thread_id_t, const char *name, const uint64_t ts, const uint64_t duration, const bool err, - bt_event_class*, bt_self_message_iterator*, bt_stream*, backend_t = BACKEND_UNKNOWN); + bt_event_class*, bt_self_message_iterator*, bt_stream*, backend_t = BACKEND_UNKNOWN, std::set flow_ids = {}); bt_message* create_device_message(const char *hostname, const process_id_t, const thread_id_t, const thapi_device_id, const thapi_device_id, const char *name, const uint64_t ts, const uint64_t duration, const bool err, const char* metadata, bt_event_class*, bt_self_message_iterator*, bt_stream*); +bt_message* create_device_flow_message(const char* hostname, const process_id_t process_id, const uint64_t uuid, + const thapi_device_id device_id, const thapi_device_id subdevice_id, + const char* name, const uint64_t ts, const uint64_t duration, const bool err, + const char* metadata, const char* queue_name, const uint64_t flow_id, + bt_event_class *event_class, bt_self_message_iterator *message_iterator, bt_stream *stream); + bt_message* create_device_name_message(const char* hostname, const process_id_t process_id, const thapi_device_id device_id, const char* name, bt_event_class *event_class, bt_self_message_iterator *message_iterator, bt_stream *stream); diff --git a/xprof/Makefile.am b/xprof/Makefile.am index 8d88e27b..d119f89c 100644 --- a/xprof/Makefile.am +++ b/xprof/Makefile.am @@ -48,10 +48,6 @@ libXProf_la_SOURCES = \ my_demangle.h \ $(top_srcdir)/utils/include/json.hpp -# Right now we harcode it. We should use pkgtools at some point https://github.com/nlohmann/json/blob/develop/single_include/nlohmann/json.hpp - -# -I/home/tapplencourt/tmp/proto/protobuf-3.20.1/ici/include/ -L/home/tapplencourt/tmp/proto/protobuf-3.20.1/ici/lib/ -lprotobuf - # Compiler flags libXProf_la_CPPFLAGS = -I$(top_srcdir)/utils/include -I$(srcdir)/include -I./ -I/home/tapplencourt/tmp/proto/protobuf-3.20.1/ici/include/ libXProf_la_CFLAGS = -Wall -Wextra -Wno-unused-parameter $(WERROR) $(BABELTRACE2_CFLAGS) diff --git a/xprof/interval.c.erb b/xprof/interval.c.erb index abc96b67..c00a559d 100644 --- a/xprof/interval.c.erb +++ b/xprof/interval.c.erb @@ -92,8 +92,12 @@ bt_component_class_initialize_method_status <%= namespace %>_dispatch_initialize // We don't create a clock, because we ensure monotonic order for the downstream messages. populate_ze_stream_class_common_context(trace_class, stream_class); + // This should be generated! dispatch->host_event_class = create_lttng_host_event_class_message(trace_class,stream_class); + dispatch->host_flow_event_class = create_lttng_host_flow_event_class_message(trace_class,stream_class); + dispatch->device_event_class = create_lttng_device_event_class_message(trace_class, stream_class); + dispatch->device_flow_event_class = create_lttng_device_flow_event_class_message(trace_class, stream_class); dispatch->traffic_event_class = create_lttng_traffic_event_class_message(trace_class, stream_class); dispatch->device_name_event_class = create_lttng_device_name_event_class_message(trace_class, stream_class); diff --git a/xprof/interval.h.erb b/xprof/interval.h.erb index ce94241f..58bdba82 100644 --- a/xprof/interval.h.erb +++ b/xprof/interval.h.erb @@ -53,10 +53,11 @@ struct <%= namespace %>_dispatch { /* Downstream message */ bt_stream *stream; bt_event_class *host_event_class; + bt_event_class *host_flow_event_class; bt_event_class *device_event_class; + bt_event_class *device_flow_event_class; bt_event_class *traffic_event_class; bt_event_class *device_name_event_class; - /* Component's input port (weak) */ bt_self_component_port_input *in_port; }; diff --git a/xprof/interval_model.yaml b/xprof/interval_model.yaml index 098bb85e..7b66a651 100644 --- a/xprof/interval_model.yaml +++ b/xprof/interval_model.yaml @@ -12,6 +12,8 @@ - :name: vtid :class: unsigned - :name: ts + :class_properties: + :field_value_range: 64 :class: signed - :name: backend :class: signed @@ -24,6 +26,22 @@ :class: unsigned - :name: err :class: bool +- :name: lttng:host_flow + :payload: + - :name: name + :class: string + - :name: dur + :class: unsigned + - :name: err + :class: bool + - :name: flow_ids_length + :class: unsigned + - :name: flow_ids + :class: array_dynamic + :field: + :class: unsigned + :class_properties: + :field_value_range: 64 - :name: lttng:device :payload: - :name: name @@ -35,22 +53,62 @@ :class_properties: :preferred_display_base: 16 - :name: sdid - :class: unsigned + :class: unsigned :class_properties: :preferred_display_base: 16 - :name: err :class: bool - :name: metadata :class: string -- :name: lttng:device_name +- :name: lttng:device_flow :payload: - :name: name :class: string + - :name: dur + :class: unsigned - :name: did :class: unsigned + :class_properties: + :preferred_display_base: 16 + - :name: sdid + :class: unsigned + :class_properties: + :preferred_display_base: 16 + - :name: err + :class: bool + - :name: metadata + :class: string + - :name: uuid + :class: unsigned + :class_properties: + :field_value_range: 64 + :preferred_display_base: 16 + - :name: queue_name + :class: string + - :name: flow_id + :class: unsigned + :class_properties: + :field_value_range: 64 +- :name: lttng:device_name + :payload: + - :name: name + :class: string + - :name: ptr + :class: unsigned + :class_properties: + :field_value_range: 64 +# Should be an option on a struct of 2 64 bits UUID +# - :name: uuid +# :class: option +# :field: +# :class: unsigned +# :class_properties: +# :field_value_range: 64 - :name: lttng:traffic :payload: - :name: name :class: string - :name: size - :class: unsigned + :class: unsigned + :class_properties: + :field_value_range: 64 diff --git a/xprof/perfetto_prunned.proto b/xprof/perfetto_prunned.proto index 3fdc0023..2a43834e 100644 --- a/xprof/perfetto_prunned.proto +++ b/xprof/perfetto_prunned.proto @@ -51,6 +51,8 @@ message TrackEvent { string name = 23; } + repeated uint64 flow_ids = 36; + repeated uint64 terminating_flow_ids = 42; } message TracePacket { diff --git a/xprof/tally.cpp b/xprof/tally.cpp index 19188925..fc8d3a9c 100644 --- a/xprof/tally.cpp +++ b/xprof/tally.cpp @@ -101,7 +101,7 @@ tally_dispatch_consume(bt_self_component_sink *self_component_sink) { const bt_field *payload_field = bt_event_borrow_payload_field_const(event); - if (strcmp(class_name, "lttng:host") == 0) { + if ( (strcmp(class_name, "lttng:host") == 0) || (strcmp(class_name, "lttng:host_flow") == 0) ) { auto dur_tuple0 = std::make_tuple( std::make_tuple(0, bt_field_string_get_value, (std::string) ""), std::make_tuple(1, &bt_field_integer_unsigned_get_value, @@ -119,7 +119,7 @@ tally_dispatch_consume(bt_self_component_sink *self_component_sink) { dispatch->host_backend_name[level].insert(backend_name[backend_id]); dispatch->host[level][hpt_function_name_t(hostname, process_id, thread_id, name)] += a; - } else if (strcmp(class_name, "lttng:device") == 0) { + } else if ( (strcmp(class_name, "lttng:device") == 0) || (strcmp(class_name, "lttng:device_flow") == 0) ) { auto dur_tuple0 = std::make_tuple( std::make_tuple(0, bt_field_string_get_value, (std::string) ""), std::make_tuple(1, &bt_field_integer_unsigned_get_value, diff --git a/xprof/tally_utils.hpp b/xprof/tally_utils.hpp index 686e1dca..0352c689 100644 --- a/xprof/tally_utils.hpp +++ b/xprof/tally_utils.hpp @@ -85,7 +85,7 @@ class TallyCoreBase { // ` symbol lookup error: ./ici/lib/libXProf.so: undefined symbol: _ZTI13TallyCoreBase` virtual const std::vector to_string() = 0; - const auto to_string_size() { + auto to_string_size() { std::vector v; for (auto &e : to_string()) v.push_back(static_cast(e.size())); diff --git a/xprof/timeline.cpp b/xprof/timeline.cpp index 6c1ceafc..922958f2 100644 --- a/xprof/timeline.cpp +++ b/xprof/timeline.cpp @@ -15,7 +15,7 @@ static perfetto_uuid_t gen_perfetto_uuid() { } static void add_event_begin(struct timeline_dispatch *dispatch, perfetto_uuid_t uuid, - timestamp_t begin, std::string name) { + timestamp_t begin, std::string name, std::set flow_ids = {} ) { auto *packet = dispatch->trace.add_packet(); packet->set_timestamp(begin); packet->set_trusted_packet_sequence_id(10); @@ -23,8 +23,11 @@ static void add_event_begin(struct timeline_dispatch *dispatch, perfetto_uuid_t track_event->set_type(perfetto_pruned::TrackEvent::TYPE_SLICE_BEGIN); track_event->set_name(name); track_event->set_track_uuid(uuid); + for (const auto& flow_id: flow_ids) + track_event->add_flow_ids(flow_id); } + static void add_event_end(struct timeline_dispatch *dispatch, perfetto_uuid_t uuid, uint64_t end) { auto *packet = dispatch->trace.add_packet(); packet->set_trusted_packet_sequence_id(10); @@ -67,7 +70,9 @@ static perfetto_uuid_t get_parent_uuid(struct timeline_dispatch *dispatch, std:: auto *process = track_descriptor->mutable_process(); process->set_pid(hp_uuid); std::ostringstream oss; - oss << hostname << " | Process " << process_id ; + oss << hostname; + if (process_id!=0) + oss << " | Process " << process_id ; if (did !=0) { oss << " | Device " << did; if (sdid !=0) @@ -114,13 +119,13 @@ static perfetto_uuid_t get_parent_uuid(struct timeline_dispatch *dispatch, std:: static void add_event_cpu(struct timeline_dispatch *dispatch, std::string hostname, uint64_t process_id, uint64_t thread_id, std::string name, uint64_t begin, - uint64_t dur) { + uint64_t dur, std::set flow_ids = {}) { // Assume perfecly nessted const uint64_t end = begin + dur; perfetto_uuid_t parent_uuid = get_parent_uuid(dispatch, hostname, process_id, thread_id); // Handling perfecly nested event - add_event_begin(dispatch, parent_uuid, begin, name); + add_event_begin(dispatch, parent_uuid, begin, name, flow_ids); std::stack &s = dispatch->uuid2stack[parent_uuid]; while ((!s.empty()) && (s.top() <= begin)) { add_event_end(dispatch, parent_uuid, s.top()); @@ -129,7 +134,7 @@ static void add_event_cpu(struct timeline_dispatch *dispatch, std::string hostna s.push(end); } -static void add_event_gpu(struct timeline_dispatch *dispatch, std::string hostname, +static void add_event_gpu_old(struct timeline_dispatch *dispatch, std::string hostname, uint64_t process_id, uint64_t thread_id, thapi_device_id did, thapi_device_id sdid, std::string name, uint64_t begin, uint64_t dur) { // This function Assume non perfecly nested @@ -170,6 +175,48 @@ static void add_event_gpu(struct timeline_dispatch *dispatch, std::string hostna add_event_end(dispatch, uuid, end); } +static void add_event_gpu(struct timeline_dispatch *dispatch, std::string hostname, + uint64_t process_id, uint64_t c_uuid, std::string queue_name, + thapi_device_id did, thapi_device_id sdid, + std::string name, uint64_t begin, uint64_t dur, std::set flow_ids) { + // This function Assume non perfecly nested + const uint64_t end = begin + dur; + perfetto_uuid_t parent_uuid = get_parent_uuid(dispatch, hostname, process_id, c_uuid, did, sdid); + // Now see if we need a to generate a new children + std::map &m = dispatch->parents2tracks[parent_uuid]; + perfetto_uuid_t uuid; + + // Pre-historical event + if (m.empty() || begin < m.begin()->first) { + uuid = gen_perfetto_uuid(); + // Generate a new children track + { + auto *packet = dispatch->trace.add_packet(); + packet->set_trusted_packet_sequence_id(10); + packet->set_timestamp(0); + + auto *track_descriptor = packet->mutable_track_descriptor(); + track_descriptor->set_uuid(uuid); + track_descriptor->set_parent_uuid(parent_uuid); + + std::ostringstream oss; + oss << queue_name << " " << rescale_uuid(dispatch->m4[hostname][did][sdid], c_uuid); + track_descriptor->set_name(oss.str()); + } + } else { + // Find the uuid who finished just before this one + auto it_ub = std::prev(m.upper_bound(begin)); + uuid = it_ub->second; + // Erase the old timestamps + m.erase(it_ub); + } + // Update the table + m[end] = uuid; + // Add event + add_event_begin(dispatch, uuid, begin, name, flow_ids); + add_event_end(dispatch, uuid, end); +} + bt_component_class_sink_consume_method_status timeline_dispatch_consume(bt_self_component_sink *self_component_sink) { bt_component_class_sink_consume_method_status status = @@ -240,6 +287,17 @@ timeline_dispatch_consume(bt_self_component_sink *self_component_sink) { if (std::string(class_name) == "lttng:host") { add_event_cpu(dispatch, hostname, process_id, thread_id, name, ts, dur); + } else if (std::string(class_name) == "lttng:host_flow") { + + const bt_field *flow_ids_field = + bt_field_structure_borrow_member_field_by_index_const(payload_field, 4); + + std::set flow_ids; + for (unsigned i=0; i < bt_field_array_get_length(flow_ids_field); i++) { + const bt_field *flow_id_field = bt_field_array_borrow_element_field_by_index_const(flow_ids_field, i); + flow_ids.insert(bt_field_integer_unsigned_get_value(flow_id_field)); + } + add_event_cpu(dispatch, hostname, process_id, thread_id, name, ts, dur, flow_ids); } else if (std::string(class_name) == "lttng:device") { const bt_field *did_field = @@ -249,8 +307,39 @@ timeline_dispatch_consume(bt_self_component_sink *self_component_sink) { const bt_field *sdid_field = bt_field_structure_borrow_member_field_by_index_const(payload_field, 3); const thapi_device_id sdid = bt_field_integer_unsigned_get_value(sdid_field); - add_event_gpu(dispatch, hostname, process_id, thread_id, did, sdid, name, ts, dur); + add_event_gpu_old(dispatch, hostname, process_id, thread_id, did, sdid, name, ts, dur); + } else if (std::string(class_name) == "lttng:device_flow") { + + const bt_field *did_field = + bt_field_structure_borrow_member_field_by_index_const(payload_field, 2); + const thapi_device_id did = bt_field_integer_unsigned_get_value(did_field); + + const bt_field *sdid_field = + bt_field_structure_borrow_member_field_by_index_const(payload_field, 3); + const thapi_device_id sdid = bt_field_integer_unsigned_get_value(sdid_field); + + const bt_field *queue_uuid_field = + bt_field_structure_borrow_member_field_by_index_const(payload_field, 6); + const uint64_t queue_uuid = bt_field_integer_unsigned_get_value(queue_uuid_field); + + const bt_field *queue_name_field = + bt_field_structure_borrow_member_field_by_index_const(payload_field, 7); + const std::string queue_name = bt_field_string_get_value(queue_name_field); + + const bt_field *flow_id_field = + bt_field_structure_borrow_member_field_by_index_const(payload_field, 8); + const uint64_t flow_id = bt_field_integer_unsigned_get_value(flow_id_field); + + // Hack to put Process at 0, meaning the pointer are uniq per hostname. + const uint64_t resclated_queue_uuid = rescale_uuid(dispatch->resclate_queue_uuid, queue_uuid); + const uint64_t rescaled_did = rescale_uuid(dispatch->m2[hostname], did); + uint64_t rescaled_sdid = 0; + if (sdid != 0) + rescaled_sdid = rescale_uuid(dispatch->m3[hostname][did], sdid); + + add_event_gpu(dispatch, hostname, 0, resclated_queue_uuid, queue_name, rescaled_did, rescaled_sdid, name, ts, dur, {flow_id} ); } + } bt_message_put_ref(message); } @@ -291,7 +380,6 @@ timeline_dispatch_initialize(bt_self_component_sink *self_component_sink, trace_packet_defaults->set_timestamp_clock_id(perfetto_pruned::BUILTIN_CLOCK_BOOTTIME); packet->set_previous_packet_dropped(true); } - return BT_COMPONENT_CLASS_INITIALIZE_METHOD_STATUS_OK; } diff --git a/xprof/timeline.hpp b/xprof/timeline.hpp index 90b3faa6..87019457 100644 --- a/xprof/timeline.hpp +++ b/xprof/timeline.hpp @@ -20,4 +20,36 @@ struct timeline_dispatch { std::map> uuid2stack; perfetto_pruned::Trace trace; + + std::unordered_map resclate_queue_uuid; + std::unordered_map resclate_device_uuid; + std::unordered_map resclate_subdevice_uuid; + + std::unordered_map > m2; + + + std::unordered_map > > m3; + + + std::unordered_map> > > m4; }; + + +// This only work because we never remove a value in the map +template +uint64_t rescale_uuid(std::unordered_map &m, T uuid) { + const uint64_t current_uuid = m.size() + 1 ; // Start at 1, 0 is special + auto ret = m.insert( {uuid, current_uuid } ); + if (!ret.second) { + return ret.first->second; + } else { + return current_uuid; + } +} + diff --git a/ze/Makefile.am b/ze/Makefile.am index 1b830141..8146b5a2 100644 --- a/ze/Makefile.am +++ b/ze/Makefile.am @@ -272,6 +272,9 @@ babeltrace_zeinterval_dispatchers.c: $(top_srcdir)/utils/gen_babeltrace_dispatch babeltrace_zeinterval_emitter.h: $(top_srcdir)/utils/gen_babeltrace_emitter.rb $(top_srcdir)/xprof/interval_model.yaml SRC_DIR=$(srcdir) $(RUBY) $^ zeinterval > $@ +babeltrace_zeinterval_event_getter_and_setter.hpp: $(top_srcdir)/utils/gen_babeltrace_event_getter_and_setter.rb $(top_srcdir)/xprof/interval_model.yaml + SRC_DIR=$(srcdir) $(RUBY) $^ zeinterval > $@ + babeltrace_zeinterval.h: $(top_srcdir)/xprof/interval.h.erb $(ERB) namespace="zeinterval" $< > $@ @@ -294,6 +297,7 @@ CLEANFILES += \ babeltrace_zeinterval_callbacks.h \ babeltrace_zeinterval_dispatchers.c \ babeltrace_zeinterval_emitter.h \ + babeltrace_zeinterval_event_getter_and_setter.hpp \ babeltrace_zeinterval.h \ babeltrace_zeinterval.c \ xprof_utils.hpp \ @@ -303,6 +307,7 @@ BUILT_SOURCES += \ babeltrace_zeinterval_callbacks.h \ babeltrace_zeinterval.h \ babeltrace_zeinterval_emitter.h \ + babeltrace_zeinterval_event_getter_and_setter.hpp \ xprof_utils.hpp nodist_libZEInterval_la_SOURCES = \ @@ -312,6 +317,7 @@ nodist_libZEInterval_la_SOURCES = \ babeltrace_zeinterval_dispatchers.c \ babeltrace_zeinterval_callbacks.cpp \ babeltrace_zeinterval_emitter.h \ + babeltrace_zeinterval_event_getter_and_setter.hpp \ xprof_utils.hpp \ xprof_utils.cpp diff --git a/ze/gen_zeinterval_callbacks.rb b/ze/gen_zeinterval_callbacks.rb index 84311ae1..5af730a1 100644 --- a/ze/gen_zeinterval_callbacks.rb +++ b/ze/gen_zeinterval_callbacks.rb @@ -58,6 +58,7 @@ def callback_signature DBT_event.new(klass) } +$dbt_events_who_signal = $dbt_events.filter_map { |dbt_event| dbt_event.name_striped if dbt_event.start? and dbt_event.fields_name.include?('hSignalEvent') } $profiling_apis = Set.new template = File.read(File.join(SRC_DIR, "zeinterval_callbacks.cpp.erb")) diff --git a/ze/zeinterval_callbacks.cpp.erb b/ze/zeinterval_callbacks.cpp.erb index d1aea864..3ed3915e 100644 --- a/ze/zeinterval_callbacks.cpp.erb +++ b/ze/zeinterval_callbacks.cpp.erb @@ -1,6 +1,7 @@ #include "babeltrace_zeinterval.h" #include "zeinterval_callbacks.hpp" #include "xprof_utils.hpp" +#include "babeltrace_zeinterval_event_getter_and_setter.hpp" #include #include @@ -11,6 +12,7 @@ #include #include +#include #include #include #include @@ -82,19 +84,39 @@ static void create_and_enqueue_host_message(const char* hostname, const process_ const uint64_t ts, const uint64_t duration, const bool err) { /* Message creation */ - bt_message *message = create_host_message(hostname, process_id, thread_id, name, ts, duration, err, + bt_message *message = bt_message_create_lttng_host( zeinterval_iter_g->dispatch->host_event_class, zeinterval_self_message_iterator_g, - zeinterval_iter_g->dispatch->stream, BACKEND_ZE); + zeinterval_iter_g->dispatch->stream, + hostname, process_id, thread_id, ts, BACKEND_ZE, name, duration, err); + + zeinterval_callbacks_state* state = (zeinterval_callbacks_state*) zeinterval_iter_g->callbacks_state; + state->downstream_message_queue.push(message); +} + +static void create_and_enqueue_host_flow_message(const char* hostname, const process_id_t process_id, const thread_id_t thread_id, const char* name, + const uint64_t ts, const uint64_t duration, const bool err, const std::set flow_ids) { + + std::vector flow_ids_vec(flow_ids.begin(), flow_ids.end()); + /* Message creation */ + bt_message *message = bt_message_create_lttng_host_flow( + zeinterval_iter_g->dispatch->host_flow_event_class, + zeinterval_self_message_iterator_g, + zeinterval_iter_g->dispatch->stream, + hostname, process_id, thread_id, ts, BACKEND_ZE, name, duration, err, flow_ids.size(), flow_ids_vec.data()); zeinterval_callbacks_state* state = (zeinterval_callbacks_state*) zeinterval_iter_g->callbacks_state; state->downstream_message_queue.push(message); } static void create_and_enqueue_device_message( - const char* hostname, const process_id_t process_id, const thread_id_t thread_id, thapi_device_id device, + const char* hostname, const process_id_t process_id, + const uint64_t uuid, + const char* queue_name, + thapi_device_id device, const char* commandname, const char* metadata, bool err, const uint64_t globalStart, const uint64_t globalEnd, const uint64_t lltng_min, + const flow_id_t flow_id, const clock_lttng_device_t ×tamp_pair_ref, const std::unordered_map &device_to_properties, const std::unordered_map &subdevice_parent) { @@ -123,13 +145,12 @@ static void create_and_enqueue_device_message( if (it2 != device_to_properties.cend()) device_hash = hash_device(it2->second); } - /* Message creation */ - bt_message *message = create_device_message(hostname, process_id, thread_id, device_hash, subdevice_hash, - commandname, start, delta, err, metadata, - zeinterval_iter_g->dispatch->device_event_class, - zeinterval_self_message_iterator_g, - zeinterval_iter_g->dispatch->stream); + bt_message *message = create_device_flow_message(hostname, process_id, uuid, device_hash, subdevice_hash, + commandname, start, delta, err, metadata, queue_name, flow_id, + zeinterval_iter_g->dispatch->device_flow_event_class, + zeinterval_self_message_iterator_g, + zeinterval_iter_g->dispatch->stream); /* Set message */ zeinterval_callbacks_state* state = (zeinterval_callbacks_state*) zeinterval_iter_g->callbacks_state; @@ -140,10 +161,11 @@ static void create_and_enqueue_traffic_message(const char* hostname, const proce const uint64_t size) { /* Message creation */ - bt_message *message = create_traffic_message(hostname, process_id, thread_id, name, size, + bt_message *message = bt_message_create_lttng_traffic( zeinterval_iter_g->dispatch->traffic_event_class, zeinterval_self_message_iterator_g, - zeinterval_iter_g->dispatch->stream, BACKEND_ZE); + zeinterval_iter_g->dispatch->stream, + hostname, process_id, thread_id, 0, BACKEND_ZE, name, size); zeinterval_callbacks_state* state = (zeinterval_callbacks_state*) zeinterval_iter_g->callbacks_state; state->downstream_message_queue.push(message); @@ -277,12 +299,26 @@ static void zeinterval_<%= dbt_event.name %>_callback( int64_t ns_from_origin; bt_clock_snapshot_get_ns_from_origin(bt_clock, &ns_from_origin); <% if dbt_event.start? %> - state->host_start[hpt_function_name_t(hostname,process_id, thread_id, "<%= dbt_event.name_striped %>")] = ns_from_origin; + state->host_start[ {hostname,process_id, thread_id, "<%= dbt_event.name_striped %>"} ] = ns_from_origin; + <% if $dbt_events_who_signal.include?(dbt_event.name_striped) %> + state->host_flow_start [ { hostname,process_id, thread_id } ] = {state->flow_id}; + <% elsif dbt_event.name_striped == "zeCommandQueueExecuteCommandLists" %> + for (uint32_t i=0; i < numCommandLists; i++) { + const auto& flow_ids = state->zeCommandListAppend2flowIds[ {hostname,process_id,phCommandLists_vals[i]} ]; + state->host_flow_start[ {hostname, process_id, thread_id} ].insert(flow_ids.begin(), flow_ids.end() ); + } + <% elsif dbt_event.name_striped == "zeCommandListReset" %> + state->zeCommandListAppend2flowIds.erase( {hostname, process_id, hCommandList} ); + <% end %> <% elsif dbt_event.stop? %> - auto it_start = state->host_start.find(hpt_function_name_t(hostname,process_id, thread_id, "<%= dbt_event.name_striped %>")); - const uint64_t start_g = it_start->second; + auto it_start = state->host_start.find( { hostname,process_id, thread_id, "<%= dbt_event.name_striped %>"} ); + const auto& start_g = it_start->second; state->host_start.erase(it_start); - <% if dbt_event.fields_name.include?('zeResult') %> + <% if $dbt_events_who_signal.include?(dbt_event.name_striped) or dbt_event.name_striped == "zeCommandQueueExecuteCommandLists" %> + const auto& flow_ids = state->host_flow_start [ { hostname,process_id, thread_id } ]; + create_and_enqueue_host_flow_message(hostname.c_str(), process_id, thread_id, "<%= dbt_event.name_striped %>", + start_g, ns_from_origin-start_g, zeResult, flow_ids); + <% elsif dbt_event.fields_name.include?('zeResult') %> <%# In level0 1.0-rev-3 only those use `ZE_RESULT_NOT_READY` %> <% if dbt_event.name_striped.include?('Synchronize') or dbt_event.name_striped.include?('Query') %> create_and_enqueue_host_message(hostname.c_str(), process_id, thread_id, "<%= dbt_event.name_striped %>", @@ -311,19 +347,19 @@ static void zeinterval_<%= dbt_event.name %>_callback( <% $profiling_apis.add("#{dbt_event.name_prefix}_#{STOP}") %> <% else dbt_event.stop? %> if ( zeResult == ZE_RESULT_SUCCESS ) { - const auto hModule = retrieve_start(state, hpt_t(hostname, process_id,thread_id)); - state->module_to_module_globals[hp_module_t(hostname, process_id, hModule)].insert((uintptr_t)pptr_val); - state->rangeset_memory_shared[hp_t(hostname, process_id)][(uintptr_t) pptr_val] = (uintptr_t)pptr_val + pSize_val; + const auto hModule = retrieve_start(state, {hostname, process_id,thread_id} ); + state->module_to_module_globals[ {hostname, process_id, hModule} ].insert((uintptr_t)pptr_val); + state->rangeset_memory_shared[ {hostname, process_id} ][(uintptr_t) pptr_val] = (uintptr_t)pptr_val + pSize_val; } <% end %> <% elsif dbt_event.name_striped == "zeModuleDestroy" and dbt_event.start? %> <% $profiling_apis.add("#{dbt_event.name_prefix}_#{START}") %> <% $profiling_apis.add("#{dbt_event.name_prefix}_#{STOP}") %> auto& s = state->module_to_module_globals; - auto it = s.find(hp_module_t(hostname, process_id, hModule)); + auto it = s.find( {hostname, process_id, hModule} ); if (it != s.end()) { for (auto &s2: it->second) - state->rangeset_memory_shared[hp_t(hostname, process_id)].erase(s2); + state->rangeset_memory_shared[ {hostname, process_id} ].erase(s2); s.erase(it); } <% elsif dbt_event.name_striped == "zeModuleCreate" and dbt_event.start? %> @@ -337,7 +373,7 @@ static void zeinterval_<%= dbt_event.name %>_callback( <% $profiling_apis.add("#{dbt_event.name_prefix}_#{STOP}") %> <% elsif ['zeMemAllocHost','zeMemAllocDevice','zeMemAllocShared'].include?(dbt_event.name_striped) and dbt_event.stop? %> if ( zeResult == ZE_RESULT_SUCCESS ) { - const auto size = retrieve_start(state, hpt_t(hostname, process_id, thread_id)); + const auto size = retrieve_start(state, {hostname, process_id, thread_id} ); <% if dbt_event.name_striped == "zeMemAllocHost" %> auto &s = state->rangeset_memory_host; <% elsif dbt_event.name_striped == "zeMemAllocDevice" %> @@ -354,8 +390,8 @@ static void zeinterval_<%= dbt_event.name %>_callback( <% $profiling_apis.add("#{dbt_event.name_prefix}_#{STOP}") %> <% elsif dbt_event.name_striped == "zeMemFree" and dbt_event.stop? %> if ( zeResult == ZE_RESULT_SUCCESS ) { - const auto ptr = retrieve_start(state, hpt_t(hostname, process_id, thread_id)); - const auto size = remove_memory(state, hp_t(hostname, process_id), ptr); + const auto ptr = retrieve_start(state, {hostname, process_id, thread_id} ); + const auto size = remove_memory(state, {hostname, process_id}, ptr); (void) size; //create_and_enqueue_traffic_message(hostname.c_str(), process_id, thread_id, "<%= dbt_event.name_striped %>", size); } @@ -438,10 +474,10 @@ Case B: <% if ['zeCommandListCreate','zeCommandListCreateImmediate'].include?(dbt_event.name_striped) %> <% $profiling_apis.add(dbt_event.name_unsanitized) %> <% if dbt_event.start? %> - save_start(state, hpt_t(hostname,process_id,thread_id), hDevice); + save_start(state, {hostname,process_id,thread_id} , hDevice); <% elsif dbt_event.stop? %> - const auto device = retrieve_start(state, hpt_t(hostname, process_id,thread_id)); - state->command_list_to_device[hp_command_list_t(hostname,process_id,phCommandList_val)] = device; + const auto device = retrieve_start(state, {hostname, process_id,thread_id} ); + state->command_list_to_device[ {hostname,process_id,phCommandList_val} ] = device; <% end %> <% end %> @@ -457,18 +493,18 @@ Case B: save_start(state, hpt_t(hostname,process_id,thread_id), std::string{desc__pKernelName_val}); <% elsif dbt_event.name_striped == "zeKernelCreate" and dbt_event.stop? %> <% $profiling_apis.add(dbt_event.name_unsanitized) %> - state->kernel_to_name[hpk_t(hostname,process_id,phKernel_val)] = retrieve_start(state, hpt_t(hostname,process_id,thread_id)); + state->kernel_to_name[ { hostname,process_id,phKernel_val} ] = retrieve_start(state, { hostname,process_id,thread_id} ); <% elsif dbt_event.name_striped == "zeKernelSetGroupSize" and dbt_event.start? %> std::stringstream groupsize; groupsize << "{" << groupSizeX << "," << groupSizeY << "," << groupSizeZ << "}"; - state->kernel_to_groupsize_str[hp_kernel_t(hostname,process_id,hKernel)] = groupsize.str(); + state->kernel_to_groupsize_str[ {hostname,process_id,hKernel} ] = groupsize.str(); <% end %> <% if dbt_event.name_unsanitized == "lttng_ust_ze_properties:kernel" %> //Not sure why SIMD size == maxSubgroupSize, but this is what Intel Tool are doing - state->kernel_to_simdsize_str[hp_kernel_t(hostname,process_id,hKernel)] = "SIMD" + std::to_string(pKernelProperties_val->maxSubgroupSize); + state->kernel_to_simdsize_str[ {hostname,process_id,hKernel} ] = "SIMD" + std::to_string(pKernelProperties_val->maxSubgroupSize); <% end %> <%# @@ -477,6 +513,20 @@ Case B: | | (_) | | | | | | (_| o \_ (_) | | | | | | (_| | | (_| _| %> + + <%# Map last command partial payload %> + <% if dbt_event.stop? and dbt_event.name_striped == "zeCommandQueueCreate" %> + const flow_id_t flow_id = state->flow_id; + state->zeCommandQueueCreate2flowId[ { hostname, process_id, phCommandQueue_val } ] = flow_id; + state->flow_id++; + <% end %> + + <% if dbt_event.stop? and dbt_event.name_striped.include?("zeCommandListCreate") %> + const flow_id_t flow_id = state->flow_id; + state->zeCommandListCreate2flowId[ { hostname, process_id, phCommandList_val } ] = flow_id; + state->flow_id++; + <% end %> + <%# Map last command partial payload %> <% if dbt_event.start? and dbt_event.fields_name.include?('hSignalEvent') %> <% $profiling_apis.add("#{dbt_event.name_prefix}_#{START}") %> @@ -495,20 +545,28 @@ Case B: << state->kernel_to_groupsize_str[hpk]; <% elsif dbt_event.name_unsanitized.include?("Memory") and dbt_event.fields_name.any?{ |n| n.include?('ptr') } %> - const hp_t hp(hostname, process_id); - std::stringstream name_; - name_ << "<%= dbt_event.name_striped.sub("CommandListAppend",'') %>" - << "(" + const hp_t hp(hostname, process_id); + std::stringstream name_; + name_ << "<%= dbt_event.name_striped.sub("CommandListAppend",'') %>" + << "(" <% dbt_event.fields_name.filter { |n| n.include?('ptr') }.reverse.each do |n| %> << memory_location(state, hp, (uintptr_t) <%= n %>) <% end %> - << ")"; + << ")"; std::string name = name_.str(); <% else %> const std::string name{"<%= dbt_event.name_striped %>"}; <% end %> - const auto device = state->command_list_to_device[hp_command_list_t(hostname,process_id,hCommandList)]; - state->command_partial_payload[hpt_t(hostname,process_id,thread_id)] = l_tfnm_m_d_ts_t(hCommandList, name, metadata.str(), device, ns_from_origin); + const flow_id_t flow_id = state->flow_id; + state->zeCommandListAppend2flowIds[ { hostname,process_id, hCommandList } ].insert(flow_id); + state->command_partial_payload[ { hostname,process_id,thread_id } ] = + { flow_id, name, metadata.str(), ns_from_origin, hCommandList}; + state->flow_id++; + <% end %> + + <% if dbt_event.start? and dbt_event.name_striped == "zeCommandQueueExecuteCommandLists" %> + for (uint32_t i=0; i < numCommandLists; i++) + state->zeCommandList2zeCommandQueue[{hostname, process_id, phCommandLists_vals[i]} ] = hCommandQueue; <% end %> <%# @@ -524,13 +582,15 @@ Case B: const hpt_t hpt(hostname, process_id, thread_id); auto it_pp = state->command_partial_payload.find(hpt); - // We didn't find the command who initiated this even_profiling, + // We didn't find the command who initiated this event_profiling, // Best we can do is to bailout if (it_pp == state->command_partial_payload.end()) return; - const auto [hCommandList, commandname, metadata, device, lltng_min] = it_pp->second; + const auto [flow_id_kernel, commandname, metadata, lltng_min, hCommandList ] = it_pp->second; state->command_partial_payload.erase(it_pp); - state->command_list_to_events[hp_command_list_t(hostname, process_id, hCommandList)].insert(hEvent); + state->command_list_to_events[ { hostname, process_id, hCommandList } ].insert(hEvent); + + const auto device = state->command_list_to_device[ { hostname,process_id,hCommandList } ]; // Got the timestamp pair referance clock_lttng_device_t timestamp_pair_ref; @@ -542,17 +602,9 @@ Case B: auto it = state->event_to_profiling_result.find(hpe); if (it == state->event_to_profiling_result.end()){ - //Creeate the payload who will be used by event_profiling_results - state->event_payload[hpe] = t_tfnm_m_d_ts_cld_t(thread_id, commandname, metadata, device, lltng_min, timestamp_pair_ref); + state->event_payload[hpe] = { flow_id_kernel, commandname, metadata, lltng_min, timestamp_pair_ref, hCommandList}; } else { - // Handle the callbacks case, where `event_profiling_results` was called before `event_profiling`. - const auto& [err, globalStart, globalEnd] = it->second; - const auto& device_to_properties = state->device_to_properties; - const auto& subdevice_parent = state->subdevice_parent; - create_and_enqueue_device_message(hostname.c_str(), process_id, thread_id, device, - commandname.c_str(), metadata.c_str(), - err, globalStart, globalEnd, lltng_min, timestamp_pair_ref, - device_to_properties, subdevice_parent); + //exit("Not implemented Error"); } <% elsif dbt_event.name_unsanitized == "lttng_ust_ze_profiling:event_profiling_results" %> @@ -562,15 +614,33 @@ Case B: auto it = state->event_payload.find(hpe); if (it != state->event_payload.end()) { - const auto& [thread_id, commandname, metadata, device, lltng_min, timestamp_pair_ref] = it->second; + const auto& [flow_id_kernel, commandname, metadata, lltng_min, timestamp_pair_ref, hCommandList] = it->second; const auto& device_to_properties = state->device_to_properties; const auto& subdevice_parent = state->subdevice_parent; - create_and_enqueue_device_message(hostname.c_str(), process_id, thread_id, device, + + // Can we have a race condition with the command list destroy? + const auto device = state->command_list_to_device[ { hostname,process_id,hCommandList } ]; + + // We use use the flow as a UUID + uint64_t queue_uuid; + std::string queue_name; + const auto it1 = state->zeCommandList2zeCommandQueue.find( { hostname,process_id, hCommandList } ); + if (it1 != state->zeCommandList2zeCommandQueue.end() ) { + const auto& hCommandQueue = it1->second; + queue_uuid = state->zeCommandQueueCreate2flowId[ {hostname, process_id, hCommandQueue} ]; + queue_name = "zeCommandQueue"; + } else { + queue_uuid = state->zeCommandListCreate2flowId[ {hostname, process_id, hCommandList} ]; + queue_name = "zeCommandListImmediate"; + } + + create_and_enqueue_device_message(hostname.c_str(), process_id, queue_uuid, queue_name.c_str(), device, commandname.c_str(), metadata.c_str(), - err, globalStart, globalEnd, lltng_min, timestamp_pair_ref, + err, globalStart, globalEnd, lltng_min, flow_id_kernel, + timestamp_pair_ref, device_to_properties, subdevice_parent); } else { - state->event_to_profiling_result[hpe] = event_profiling_result_t(err, globalStart,globalEnd); + state->event_to_profiling_result[hpe] = { err, globalStart, globalEnd }; } <% end %> <%# Cleanup state for profiling %> @@ -580,8 +650,11 @@ Case B: save_start(state, hpt_t(hostname, process_id, thread_id), hCommandList); <% else %> if (zeResult == ZE_RESULT_SUCCESS) { - const auto hCommandList = retrieve_start(state, hpt_t(hostname, process_id, thread_id)); + const auto hCommandList = retrieve_start(state, {hostname, process_id, thread_id} ); const hp_command_list_t hpl(hostname, process_id, hCommandList); + + state->zeCommandList2zeCommandQueue.erase(hpl); + auto it_e = state->command_list_to_events.find(hpl); if (it_e != state->command_list_to_events.end()) { for (auto& hEvent: it_e->second) { @@ -595,7 +668,7 @@ Case B: <% end %> <% elsif ["zeEventDestroy", "zeEventHostReset"].include?(dbt_event.name_striped)%> <%if dbt_event.start? %> - save_start(state, hpt_t(hostname, process_id, thread_id), hEvent); + save_start(state, {hostname, process_id, thread_id}, hEvent); <% else %> if (zeResult == ZE_RESULT_SUCCESS) { const auto hEvent = retrieve_start(state, hpt_t(hostname, process_id, thread_id)); @@ -604,6 +677,7 @@ Case B: state->event_payload.erase(hpe); } <% end %> + <% end %> <%# @@ -612,13 +686,13 @@ Case B: | | (/_ | | | (_) | \/ | | (_| | | | (_ / %> - //Todo this should check for ZE_RESULT! + <%# Todo this should check for ZE_RESULT! #%> <% if dbt_event.start? and dbt_event.name_striped.include?("Memory") and dbt_event.fields_name.include?("size") %> <% if !dbt_event.fields_name.include?('hSignalEvent') %> //Name is set by the profiling, but some function transfert memory but cannot be profiled - //ie, prefetch, maxing memory resident... + //ie, prefetch, make memory resident... const std::string name{"<%= dbt_event.name_striped %>"}; <% else %> <% $profiling_apis.add(dbt_event.name_unsanitized) %> diff --git a/ze/zeinterval_callbacks.hpp b/ze/zeinterval_callbacks.hpp index 47775510..8d514720 100644 --- a/ze/zeinterval_callbacks.hpp +++ b/ze/zeinterval_callbacks.hpp @@ -7,7 +7,8 @@ #include #include #include - +#include + typedef std::map MemoryInterval; typedef std::tuple hp_event_t; @@ -21,22 +22,29 @@ typedef hp_event_t hpe_t; typedef hp_kernel_t hpk_t; typedef std::tuple clock_lttng_device_t; -typedef std::tuple t_tfnm_m_d_ts_cld_t; -typedef std::tuple l_tfnm_m_d_ts_t; +typedef std::tuple t_tfnm_m_d_ts_cld_tp; +typedef std::tuple l_tfnm_m_ts_tp; typedef std::tuple event_profiling_result_t; struct zeinterval_callbacks_state { std::queue downstream_message_queue; // https://spec.oneapi.io/level-zero/latest/core/api.html#_CPPv4N16ze_device_uuid_t2idE - std::unordered_map command_list_to_device; + std::unordered_map command_list_to_device; std::unordered_map> command_list_to_events; + + std::unordered_map zeCommandList2zeCommandQueue; + std::unordered_map> zeCommandListAppend2flowIds; + std::unordered_map zeCommandQueueCreate2flowId; + std::unordered_map zeCommandListCreate2flowId; + std::unordered_map device_to_properties; // Kernel name & metadata std::unordered_map kernel_to_name; std::unordered_map kernel_to_groupsize_str; std::unordered_map kernel_to_simdsize_str; std::unordered_map host_start; + std::unordered_map> host_flow_start; std::unordered_map profiled_function_name; /* Handle memory copy */ @@ -48,10 +56,13 @@ struct zeinterval_callbacks_state { /* Handle variable */ std::unordered_map event_to_profiling_result; std::unordered_map device_timestamps_pair_ref; - std::unordered_map command_partial_payload; - std::unordered_map event_payload; + std::unordered_map command_partial_payload; + std::unordered_map event_payload; std::unordered_map subdevice_parent; + + flow_id_t flow_id = 1; + /* Stack to get begin end */ std::unordered_map> last_command; };