From ca4e9e86b2af805f56cb098117f664b3f3930312 Mon Sep 17 00:00:00 2001 From: Ayaka Yorihiro Date: Tue, 19 Nov 2024 13:50:54 -0500 Subject: [PATCH] Starting incorporation into fud2 --- tools/profiler/create-visuals.py | 404 ------------------- tools/profiler/new-parse-vcd.py | 503 ------------------------ tools/profiler/new-profiler-approach.sh | 2 +- tools/profiler/parse-vcd.py | 392 ------------------ tools/profiler/run-up-to-tdcc.sh | 12 +- 5 files changed, 6 insertions(+), 1307 deletions(-) delete mode 100644 tools/profiler/create-visuals.py delete mode 100644 tools/profiler/new-parse-vcd.py delete mode 100644 tools/profiler/parse-vcd.py diff --git a/tools/profiler/create-visuals.py b/tools/profiler/create-visuals.py deleted file mode 100644 index 1c279380d..000000000 --- a/tools/profiler/create-visuals.py +++ /dev/null @@ -1,404 +0,0 @@ -""" -Takes in a dump file created by parse-vcd.py and creates files for visualization: -- *.folded files for producing flame graphs -- *.json files displaying the timeline (to be read by Perfetto UI) -""" -import json -import sys - -class CallStackElement: - """ - A component on the stack that is active at a given snapshot in time. Contains cell name and any active groups. - (The element may not have any active groups when control is taking extra cycles for FSMs, etc.) - starting_group is the first group that shows up. When starting_group is None, the stack simply contains the cell. - """ - def __init__(self, component, starting_group): - self.component = component - if starting_group is None: - self.active_groups = [] - else: - self.active_groups = [starting_group] - self.cell_fullname = ".".join(starting_group.split(".")[:-1]) # remove group name - self.cell_id = None - - def __repr__(self): - return f"([{self.component}] Active groups: {self.active_groups})" - - """ - Registers a new group active on the stack. - """ - def add_group(self, group_name): - if group_name not in self.active_groups: - self.active_groups.append(group_name) - - """ - Returns the active group if the component is sequential (has only one active group), or None if the cell has no active groups. - Throws exception if there are multiple groups active at the same time, since we are assuming sequential programs right now. - """ - def get_active_group(self): - if len(self.active_groups) == 0: - return None - elif len(self.active_groups) == 1: - return self.active_groups[0] - else: - # concatenate all parallel active groups - sorted_active_groups = list(sorted(self.active_groups)) - acc = sorted_active_groups[0] - for group in sorted_active_groups[1:]: - suffix = group.split(".")[-1] - acc += "/" + suffix - return acc - - """ - Returns the identifier of this stack: either the full name of the active group, or the full name of the cell if no groups are active. - """ - def get_fullname(self): - active_group = self.get_active_group() - if active_group is None: - return self.cell_fullname - else: - return active_group - - """ - Returns the representation of the current stack for *.folded flame graph files. - """ - def flame_stack_string(self, main_component): - main_shortname = main_component.split("TOP.toplevel.")[1] - if self.component == main_shortname: - prefix = main_component - else: - prefix = self.cell_id - - active_group = self.get_active_group() - if active_group is not None: - return prefix + ";" + active_group.split(".")[-1] - else: - return prefix - - """ - Returns the name of the stack's component. - """ - def component_flame_stack_string(self, main_component): - main_shortname = main_component.split("TOP.toplevel.")[1] - if self.component == main_shortname: - return main_component - else: - return self.component - - # other utility functions - - def get_stack_depth(self): - return self.get_fullname().count(".") - - def add_flame_cell_name(self, parent_component, cell_name): - self.cell_id = f"{self.component}[{parent_component}.{cell_name}]" - self.parent_component = parent_component - self.cell_name = cell_name - - def get_active_groups(self): - return self.active_groups - - def add_cell_fullname(self, cell_fullname): - self.cell_fullname = cell_fullname - -""" -Computes which groups were tracked by FSMs. -""" -def get_fsm_groups(profiled_info): - fsm_groups = set() - all_groups = set() - for group_info in profiled_info: - if group_info["name"] == "TOTAL" or group_info["component"] is None: - continue - all_groups.add(group_info["name"]) - if group_info["fsm_name"] is not None: - fsm_groups.add(group_info["name"]) - return fsm_groups, all_groups - -""" -Adds a new CallStackElement object for `component` to `timeline_map` -""" -def add_elem_to_callstack(timeline_map, component, name, is_cell): - if component not in timeline_map: - if is_cell: - timeline_map[component] = CallStackElement(component, None) - timeline_map[component].add_cell_fullname(name) - else: - timeline_map[component] = CallStackElement(component, name) - else: - timeline_map[component].add_group(name) - -""" -Returns the stack element that is deepest in the simultaneously running sets of components -""" -def get_deepest_stack_element(stack_elems): - elems_sorted = sorted(stack_elems, key=lambda k : stack_elems[k].get_stack_depth(), reverse=True) - # NOTE: assuming sequential for now, which means (with n as the "deepest stack size"): - # (1) There is only one element with stack depth n (this should be a group) - # (2) There are two elements with stack depth n (a group and a cell). We want to return the cell in this case. - if len(elems_sorted) == 1 or stack_elems[elems_sorted[0]].get_stack_depth() > stack_elems[elems_sorted[1]].get_stack_depth(): - return elems_sorted[0] - elif stack_elems[elems_sorted[0]].get_active_groups() is None: # 0th element is a cell - return elems_sorted[0] - else: # 1th element has to be a cell, assuming sequential programs. - return elems_sorted[1] - -""" -Helper method to put trace stack elements in calling order -""" -def order_trace(main_component, cells_map, timeline): - main_shortname = main_component.split("TOP.toplevel.")[1] - # timeline_map has an *unordered* - processed_trace = {} - for i in timeline: - if main_shortname not in timeline[i]: - continue - stack = [timeline[i][main_shortname]] - # get the element that is deepest within the stack, then reconstruct from there - component = get_deepest_stack_element(timeline[i]) - if component != main_shortname: - cell_full_name = timeline[i][component].cell_fullname - after_main = cell_full_name.split(f"{main_component}.")[1] - after_main_split = after_main.split(".") - prev_component = main_shortname - for cell_name in after_main_split: - cell_component = cells_map[prev_component][cell_name] - timeline[i][cell_component].add_flame_cell_name(prev_component, cell_name) - stack.append(timeline[i][cell_component]) - prev_component = cell_component - processed_trace[i] = stack - return processed_trace - -""" -Constructs traces: for every cycle, what was the stack of active cells/groups? -""" -def create_trace(profiled_info, main_component, cells_map, fsm_groups, all_groups): - summary = list(filter(lambda x : x["name"] == "TOTAL", profiled_info))[0] - total_cycles = summary["total_cycles"] - only_gt_groups = all_groups.difference(fsm_groups) - timeline_map = {i : {} for i in range(total_cycles)} - fsm_timeline_map = {i : {} for i in range(total_cycles)} - group_to_gt_segments = {} # we need segment info for frequency checking - # first iterate through all of the cells - for cell_info in filter(lambda x : "is_cell" in x and x["is_cell"], profiled_info): - for segment in cell_info["closed_segments"]: - for i in range(segment["start"], segment["end"]): - add_elem_to_callstack(fsm_timeline_map[i], cell_info["component"], cell_info["name"], True) - add_elem_to_callstack(timeline_map[i], cell_info["component"], cell_info["name"], True) - # next iterate through everything else - for group_info in profiled_info: - group_name = group_info["name"] - if group_name == "TOTAL" or group_info["is_cell"]: # only care about actual groups - continue - group_component = group_info["component"] - for segment in group_info["closed_segments"]: - if group_info["fsm_name"] is None: - if group_name not in group_to_gt_segments: - group_to_gt_segments[group_name] = {} # segment start cycle to segment end cycle - group_to_gt_segments[group_name][segment["start"]] = segment["end"] - for i in range(segment["start"], segment["end"]): # really janky, I wonder if there's a better way to do this? - if group_info["fsm_name"] is not None: # FSM version - add_elem_to_callstack(fsm_timeline_map[i], group_component, group_name, False) - elif group_name in only_gt_groups: # A group that isn't managed by an FSM. In which case it has to be in both FSM and GT - add_elem_to_callstack(fsm_timeline_map[i], group_component, group_name, False) - add_elem_to_callstack(timeline_map[i], group_component, group_name, False) - else: # The ground truth info about a group managed by an FSM. - add_elem_to_callstack(timeline_map[i], group_component, group_name, False) - - trace = order_trace(main_component, cells_map, timeline_map) - fsm_trace = order_trace(main_component, cells_map, fsm_timeline_map) - - return trace, fsm_trace, len(timeline_map) - -""" -Writes a flame graph counting the number of times a group was active to `frequency_flame_out`. -""" -def create_frequency_flame_graph(main_component, trace, total_cycles, frequency_flame_out): - frequency_stacks = {} - stack_last_cycle = "" - for i in range(total_cycles): - current_stack = "" - if i in trace and len(trace[i]) != 0: - current_stack = ";".join(map(lambda x : x.flame_stack_string(main_component), trace[i])) - if stack_last_cycle != current_stack and stack_last_cycle.count(";") <= current_stack.count(";"): # We activated a different group, or invoked a different component! - if current_stack not in frequency_stacks: - frequency_stacks[current_stack] = 0 - frequency_stacks[current_stack] += 1 - stack_last_cycle = current_stack - - write_flame_graph(frequency_flame_out, frequency_stacks) - -""" -Returns a representation of how many cycles each stack combination was active for. -""" -def compute_flame_stacks(trace, main_component, total_cycles): - stacks = {} - component_stacks = {} - for i in trace: - stack = ";".join(map(lambda x : x.flame_stack_string(main_component), trace[i])) - # FIXME: really should separate out component stack - component_stack = ";".join(map(lambda x : x.component_flame_stack_string(main_component), trace[i])) - if stack not in stacks: - stacks[stack] = 0 - if component_stack not in component_stacks: - component_stacks[component_stack] = 0 - stacks[stack] += 1 - component_stacks[component_stack] += 1 - component_stacks[main_component] = total_cycles - len(trace) - return stacks, component_stacks - -""" -Constructs and writes flame graphs. -""" -def create_flame_graph(main_component, trace, fsm_trace, num_cycles, flame_out, fsm_flame_out, component_out, fsm_component_out): - stacks, component_stacks = compute_flame_stacks(trace, main_component, num_cycles) - write_flame_graph(flame_out, stacks) - write_flame_graph(component_out, component_stacks) - fsm_stacks, fsm_component_stacks = compute_flame_stacks(fsm_trace, main_component, num_cycles) - write_flame_graph(fsm_flame_out, fsm_stacks) - write_flame_graph(fsm_component_out, fsm_component_stacks) - -""" -Creates the JSON timeline representation. -""" -def create_timeline_stacks(trace, main_component): - events = [] - currently_active = {} # group name to beginning traceEvent entry (so end event can copy) - ts_multiplier = 100 # some arbitrary number to multiply by so that it's easier to see in the viewer - cell_to_stackframe_info = {main_component : (2, 1)} # (stack_number, parent_stack_number) - stack_number_acc = 3 # To guarantee that we get unique stack numbers when we need a new one - - cell_to_stackframe_info["MAIN"] = (1, None) - cell_to_stackframe_info["TOP.toplevel"] = (2, 1) - - for i in trace: - active_this_cycle = set() - # Start from the bottom up. Parent is the previous stack! - parent = "MAIN" - for elem in trace[i]: - # we want an entry for the cell and the group, if the stack entry has a group in it. - # cell - cell_name = elem.cell_fullname - active_this_cycle.add(cell_name) - if cell_name not in currently_active: # first cycle of the cell. - # get the stackframe - if cell_name not in cell_to_stackframe_info: - (parent_stackframe, _) = cell_to_stackframe_info[parent] # the parent better have been registered by now - cell_to_stackframe_info[cell_name] = (stack_number_acc, parent_stackframe) - stack_number_acc += 1 - (cell_stackframe, _) = cell_to_stackframe_info[cell_name] - # add start event of this cell - start_event = {"name": cell_name.split(".")[-1], "cat": "cell", "ph": "B", "pid" : 1, "tid": 1, "ts": i * ts_multiplier, "sf" : cell_stackframe} - events.append(start_event) - currently_active[cell_name] = start_event - # add a group if one is active - group_name = elem.get_active_group() - if group_name is not None: - active_this_cycle.add(group_name) - if group_name not in currently_active: - # get the stackframe - if group_name not in cell_to_stackframe_info: - cell_to_stackframe_info[group_name] = (stack_number_acc, cell_stackframe) # parent of the group is the cell - stack_number_acc += 1 - (group_stackframe, _) = cell_to_stackframe_info[group_name] - # add start event of this group - start_event = {"name": group_name.split(".")[-1], "cat": "group", "ph": "B", "pid" : 1, "tid": 1, "ts": i * ts_multiplier, "sf" : group_stackframe} - events.append(start_event) - currently_active[group_name] = start_event - parent = group_name # the next cell's parent will be this group. - # Any element that was previously active but not active this cycle need to end - for non_active_group in set(currently_active.keys()).difference(active_this_cycle): - end_event = currently_active[non_active_group].copy() - del currently_active[non_active_group] - end_event["ts"] = (i) * ts_multiplier - 1 - end_event["ph"] = "E" - events.append(end_event) - # postprocess - add end events for all events still active by the end - for event in currently_active: - end_event = currently_active[event].copy() - end_event["ts"] = (len(trace)) * ts_multiplier - 1 # only difference w the above - end_event["ph"] = "E" - events.append(end_event) - - # "stackFrames" field of the Trace Format JSON - stacks = {} - stack_category = "C" - for cell in cell_to_stackframe_info: - stack_id, parent_stack_id = cell_to_stackframe_info[cell] - if parent_stack_id is None: - stacks[stack_id] = {"name" : "MAIN", "category": stack_category} - else: - stacks[stack_id] = {"name" : cell, "parent": parent_stack_id, "category" : stack_category} - - return { "traceEvents": events, "stackFrames": stacks } - -""" -Wrapper function for constructing and writing a timeline representation -""" -def create_timeline_json(trace, fsm_trace, main_component, timeline_out, fsm_timeline_out): - timeline_json_data = create_timeline_stacks(trace, main_component) - with open(timeline_out, "w", encoding="utf-8") as timeline_file: - timeline_file.write(json.dumps(timeline_json_data, indent=4)) - fsm_timeline_json_data = create_timeline_stacks(fsm_trace, main_component) - with open(fsm_timeline_out, "w", encoding="utf-8") as fsm_timeline_file: - fsm_timeline_file.write(json.dumps(fsm_timeline_json_data, indent=4)) - -""" -Helper function to output the *.folded file for flame graphs. -""" -def write_flame_graph(flame_out, stacks): - with open(flame_out, "w") as f: - for stack in sorted(stacks, key=lambda k : len(k)): # main needs to come first for flame graph script to not make two boxes for main? - f.write(f"{stack} {stacks[stack]}\n") - -""" -Helper function to process the cells_json input. -""" -def build_cells_map(json_file): - cell_json = json.load(open(json_file)) - cells_map = {} - for component_entry in cell_json: - inner_cells_map = {} - for cell_entry in component_entry["cell_info"]: - inner_cells_map[cell_entry["cell_name"]] = cell_entry["component_name"] - cells_map[component_entry["component"]] = inner_cells_map - return cells_map - -def main(profiler_dump_file, cells_json, timeline_out, fsm_timeline_out, flame_out, fsm_flame_out, frequency_flame_out, component_out, fsm_component_out): - profiled_info = json.load(open(profiler_dump_file, "r")) - fsm_groups, all_groups = get_fsm_groups(profiled_info) - # This cells_map is different from the one in parse-vcd.py - cells_map = build_cells_map(cells_json) - summary = list(filter(lambda x : x["name"] == "TOTAL", profiled_info))[0] - main_component = summary["main_full_path"] - trace, fsm_trace, num_cycles = create_trace(profiled_info, main_component, cells_map, fsm_groups, all_groups) - create_flame_graph(main_component, trace, fsm_trace, num_cycles, flame_out, fsm_flame_out, component_out, fsm_component_out) - create_timeline_json(trace, fsm_trace, main_component, timeline_out, fsm_timeline_out) - create_frequency_flame_graph(main_component, trace, num_cycles, frequency_flame_out) - -if __name__ == "__main__": - if len(sys.argv) > 9: - profiler_dump_json = sys.argv[1] - cells_json = sys.argv[2] - timeline_out = sys.argv[3] - fsm_timeline_out = sys.argv[4] - flame_out = sys.argv[5] - fsm_flame_out = sys.argv[6] - frequency_flame_out = sys.argv[7] - component_flame_out = sys.argv[8] - fsm_component_flame_out = sys.argv[9] - main(profiler_dump_json, cells_json, timeline_out, fsm_timeline_out, flame_out, fsm_flame_out, frequency_flame_out, component_flame_out, fsm_component_flame_out) - else: - args_desc = [ - "PROFILER_JSON", - "CELLS_JSON", - "TIMELINE_VIEW_JSON", - "FSM_TIMELINE_VIEW_JSON", - "FLAME_GRAPH_FOLDED", - "FSM_FLAME_GRAPH_FOLDED", - "FREQUENCY_FLAME_GRAPH_FOLDED", - "COMPONENT_FOLDED", - "FSM_COMPONENT_FOLDED" - ] - print(f"Usage: {sys.argv[0]} {' '.join(args_desc)}") - sys.exit(-1) diff --git a/tools/profiler/new-parse-vcd.py b/tools/profiler/new-parse-vcd.py deleted file mode 100644 index 63be71bf5..000000000 --- a/tools/profiler/new-parse-vcd.py +++ /dev/null @@ -1,503 +0,0 @@ -import csv -import json -import os -import re -import shutil -import sys -import vcdvcd - -DELIMITER = "__" -INVISIBLE = "gray" -TREE_PICTURE_LIMIT=300 - -def remove_size_from_name(name: str) -> str: - """ changes e.g. "state[2:0]" to "state" """ - return name.split('[')[0] - -class ProfilingInfo: - def __init__(self, name, callsite=None, component=None, is_cell=False): - self.name = name - self.callsite = callsite # "official" call site. Should only be used for probes? - self.component = component - self.shortname = self.name.split(".")[-1] - self.closed_segments = [] # Segments will be (start_time, end_time) - self.current_segment = None - self.total_cycles = 0 - self.is_cell = is_cell - - def flame_repr(self): - if self.is_cell: - return self.name - else: - return self.shortname - - def __repr__ (self): - if self.is_cell: - header = f"[Cell][{self.callsite}] {self.name}" # FIXME: fix this later - else: - header = f"[{self.component}][{self.callsite}] {self.name}" - return header - - def start_new_segment(self, curr_clock_cycle): - if self.current_segment is None: - self.current_segment = {"start": curr_clock_cycle, "end": -1, "callsite": self.callsite} # NOTE: see if this backfires - else: - print(f"Error! The group {self.name} is starting a new segment while the current segment is not closed.") - print(f"Current segment: {self.current_segment}") - sys.exit(1) - - def end_current_segment(self, curr_clock_cycle): - if self.current_segment is not None and self.current_segment["end"] == -1: # ignore cases where done is high forever - self.current_segment["end"] = curr_clock_cycle - self.closed_segments.append(self.current_segment) - self.total_cycles += curr_clock_cycle - self.current_segment["start"] - self.current_segment = None # Reset current segment - - def get_segment_active_at_cycle(self, i): # get the segment that contains cycle i if one exists. Otherwise return None - for segment in self.closed_segments: - if segment["start"] <= i and i < segment["end"]: - return segment - return None - - def is_active_at_cycle(self, i): # is the probe active at cycle i? - return self.get_segment_active_at_cycle(i) != None - - def id(self): - return f"{self.name}{DELIMITER}{self.component}" - -class VCDConverter(vcdvcd.StreamParserCallbacks): - def __init__(self, main_component, cells_to_components): - # NOTE: assuming single-component programs for now - super().__init__() - self.main_component = main_component - self.probes = set() - # Map from timestamps [ns] to value change events that happened on that timestamp - self.timestamps_to_events = {} - self.cells = cells_to_components - self.active_elements_info = {} # for every group/cell, maps to a corresponding ProfilingInfo object signalling when group/cell was active - self.call_stack_probe_info = {} # group --> {parent group --> ProfilingInfo object}. This is for tracking structural enables within the same component. - self.cell_invoke_probe_info = {} # {cell}_{component the cell was called from} --> {parent group --> ProfilingInfo Object} (FIXME: maybe we want this the other way around?) - self.cell_invoke_caller_probe_info = {} # {group}_{component} --> {cell --> ProfilingInfo object} - for cell in self.cells: - self.active_elements_info[cell] = ProfilingInfo(cell, is_cell=True) - - def enddefinitions(self, vcd, signals, cur_sig_vals): - # convert references to list and sort by name - refs = [(k, v) for k, v in vcd.references_to_ids.items()] - refs = sorted(refs, key=lambda e: e[0]) - names = [remove_size_from_name(e[0]) for e in refs] - signal_id_dict = {sid : [] for sid in vcd.references_to_ids.values()} # one id can map to multiple signal names since wires are connected - - clock_name = f"{self.main_component}.clk" - if clock_name not in names: - print("Can't find the clock? Exiting...") - sys.exit(1) - signal_id_dict[vcd.references_to_ids[clock_name]] = [clock_name] - - # get go and done for cells (the signals are exactly {cell}.go and {cell}.done) - for cell in self.cells: - cell_go = cell + ".go" - cell_done = cell + ".done" - if cell_go not in vcd.references_to_ids: - print(f"Not accounting for cell {cell} (probably combinational)") - continue - signal_id_dict[vcd.references_to_ids[cell_go]].append(cell_go) - signal_id_dict[vcd.references_to_ids[cell_done]].append(cell_done) - - for name, sid in refs: - # check if we have a probe. instrumentation probes are "_____probe". - # callsite is either the name of another group or "instrumentation_wrapper[0-9]*" in the case it is invoked from control. - if "group_probe_out" in name: # this signal is a probe for group activation. - encoded_info = name.split("_group_probe_out")[0] - probe_info_split = encoded_info.split("__") - group_name = probe_info_split[0] - group_component = probe_info_split[1] - self.active_elements_info[encoded_info] = ProfilingInfo(group_name, component=group_component) - signal_id_dict[sid].append(name) - - elif "se_probe_out" in name: # this signal is a probe for structural enables. - encoded_info = name.split("_se_probe_out")[0] - probe_info_split = encoded_info.split("__") - group_name = probe_info_split[0] - group_parent = probe_info_split[1] - group_component = probe_info_split[2] - group_id = group_name + DELIMITER + group_component - if group_id not in self.call_stack_probe_info: - self.call_stack_probe_info[group_id] = {} - self.call_stack_probe_info[group_id][group_parent] = ProfilingInfo(group_name, callsite=group_parent, component=group_component) - signal_id_dict[sid].append(name) - - elif "cell_probe_out" in name: - encoded_info = name.split("_cell_probe_out")[0] - probe_info_split = encoded_info.split("__") - cell_name = probe_info_split[0] - invoker_group = probe_info_split[1] - component = probe_info_split[2] - probe_info_obj = ProfilingInfo(cell_name, callsite=invoker_group, component=component, is_cell=True) - cell_id = cell_name + DELIMITER + component - if cell_id not in self.cell_invoke_probe_info: - self.cell_invoke_probe_info[cell_id] = {invoker_group: probe_info_obj} - else: - self.cell_invoke_probe_info[cell_id][invoker_group] = probe_info_obj - caller_id = invoker_group + DELIMITER + component - if caller_id not in self.cell_invoke_caller_probe_info: - self.cell_invoke_caller_probe_info[caller_id] = {cell_name : probe_info_obj} - else: - self.cell_invoke_caller_probe_info[caller_id][cell_name] = probe_info_obj - signal_id_dict[sid].append(name) - - # don't need to check for signal ids that don't pertain to signals we're interested in - self.signal_id_to_names = {k:v for k,v in signal_id_dict.items() if len(v) > 0} - - def value(self, vcd, time, value, identifier_code, cur_sig_vals): - # ignore all signals we don't care about - if identifier_code not in self.signal_id_to_names: - return - - signal_names = self.signal_id_to_names[identifier_code] - int_value = int(value, 2) - - if time not in self.timestamps_to_events: - self.timestamps_to_events[time] = [] - - for signal_name in signal_names: - event = {"signal": signal_name, "value": int_value} - self.timestamps_to_events[time].append(event) - - # Postprocess data mapping timestamps to events (signal changes) - # We have to postprocess instead of processing signals in a stream because - # signal changes that happen at the same time as a clock tick might be recorded - # *before* or *after* the clock change on the VCD file (hence why we can't process - # everything within a stream if we wanted to be precise) - def postprocess(self): - clock_name = f"{self.main_component}.clk" - clock_cycles = -1 - started = False - currently_active = set() - se_currently_active = set() # structural group enables - ci_currently_active = set() # cell invokes - for ts in self.timestamps_to_events: - events = self.timestamps_to_events[ts] - started = started or [x for x in events if x["signal"] == f"{self.main_component}.go" and x["value"] == 1] - if not started: # only start counting when main component is on. - continue - # checking whether the timestamp has a rising edge - if {"signal": clock_name, "value": 1} in events: - clock_cycles += 1 - for event in events: - signal_name = event["signal"] - value = event["value"] - if signal_name.endswith(".go") and value == 1: # cells have .go and .done - cell = signal_name.split(".go")[0] - self.active_elements_info[cell].start_new_segment(clock_cycles) - currently_active.add(cell) - if signal_name.endswith(".done") and value == 1: # cells have .go and .done - cell = signal_name.split(".done")[0] - self.active_elements_info[cell].end_current_segment(clock_cycles) - currently_active.remove(cell) - if "group_probe_out" in signal_name and value == 1: # instrumented group started being active - encoded_info = signal_name.split("_group_probe_out")[0] - self.active_elements_info[encoded_info].start_new_segment(clock_cycles) - currently_active.add(encoded_info) - elif "group_probe_out" in signal_name and value == 0: # instrumented group stopped being active - encoded_info = signal_name.split("_group_probe_out")[0] - self.active_elements_info[encoded_info].end_current_segment(clock_cycles) - currently_active.remove(encoded_info) - elif "se_probe_out" in signal_name and value == 1: - encoded_info_split = signal_name.split("_se_probe_out")[0].split("__") - child_group_name = encoded_info_split[0] - parent = encoded_info_split[1] - child_group_component = encoded_info_split[2] - group_id = child_group_name + DELIMITER + child_group_component - self.call_stack_probe_info[group_id][parent].start_new_segment(clock_cycles) - se_currently_active.add((group_id, parent)) - elif "se_probe_out" in signal_name and value == 0: - encoded_info_split = signal_name.split("_se_probe_out")[0].split("__") - child_group_name = encoded_info_split[0] - parent = encoded_info_split[1] - child_group_component = encoded_info_split[2] - group_id = child_group_name + DELIMITER + child_group_component - self.call_stack_probe_info[group_id][parent].end_current_segment(clock_cycles) - se_currently_active.remove((group_id, parent)) - elif "cell_probe_out" in signal_name and value == 1: - encoded_info_split = signal_name.split("_cell_probe_out")[0].split("__") - cell_name = encoded_info_split[0] - parent = encoded_info_split[1] - parent_component = encoded_info_split[2] - caller_id = parent + DELIMITER + parent_component - self.cell_invoke_caller_probe_info[caller_id][cell_name].start_new_segment(clock_cycles) - ci_currently_active.add((caller_id, cell_name)) - elif "cell_probe_out" in signal_name and value == 0: - encoded_info_split = signal_name.split("_cell_probe_out")[0].split("__") - cell_name = encoded_info_split[0] - parent = encoded_info_split[1] - parent_component = encoded_info_split[2] - caller_id = parent + DELIMITER + parent_component - self.cell_invoke_caller_probe_info[caller_id][cell_name].end_current_segment(clock_cycles) - ci_currently_active.remove((caller_id, cell_name)) - for active in currently_active: # end any group/cell activitations that are still around... - self.active_elements_info[active].end_current_segment(clock_cycles) - for (group_id, parent) in se_currently_active: # end any structural enables that are still around... - self.call_stack_probe_info[group_id][parent].end_current_segment(clock_cycles) - for (caller_id, cell_name) in ci_currently_active: - self.cell_invoke_caller_probe_info[caller_id][cell_name].end_current_segment(clock_cycles) - - self.clock_cycles = clock_cycles - -# Generates a list of all of the components to potential cell names -# `prefix` is the cell's "path" (ex. for a cell "my_cell" defined in "main", the prefix would be "TOP.toplevel.main") -# The initial value of curr_component should be the top level/main component -def build_components_to_cells(prefix, curr_component, cells_to_components, components_to_cells): - for (cell, cell_component) in cells_to_components[curr_component].items(): - if cell_component not in components_to_cells: - components_to_cells[cell_component] = [f"{prefix}.{cell}"] - else: - components_to_cells[cell_component].append(f"{prefix}.{cell}") - build_components_to_cells(prefix + f".{cell}", cell_component, cells_to_components, components_to_cells) - -# Reads json generated by component-cells backend to produce a mapping from all components -# to cell names they could have. -def read_component_cell_names_json(json_file): - cell_json = json.load(open(json_file)) - # For each component, contains a map from each cell name to its corresponding component - # component name --> { cell name --> component name } - cells_to_components = {} - main_component = "" - for curr_component_entry in cell_json: - cell_map = {} # mapping cell names to component names for all cells in the current component - if curr_component_entry["is_main_component"]: - main_component = curr_component_entry["component"] - for cell_info in curr_component_entry["cell_info"]: - cell_map[cell_info["cell_name"]] = cell_info["component_name"] - cells_to_components[curr_component_entry["component"]] = cell_map - full_main_component = f"TOP.toplevel.{main_component}" - components_to_cells = {main_component : [full_main_component]} # come up with a better name for this - build_components_to_cells(full_main_component, main_component, cells_to_components, components_to_cells) - full_cell_names_to_components = {} - for component in components_to_cells: - for cell in components_to_cells[component]: - full_cell_names_to_components[cell] = component - - return full_main_component, full_cell_names_to_components - -def create_traces(active_element_probes_info, call_stack_probes_info, cell_caller_probes_info, total_cycles, cells_to_components, main_component): - - timeline_map = {i : set() for i in range(total_cycles)} - # first iterate through all of the profiled info - for unit_name in active_element_probes_info: - unit = active_element_probes_info[unit_name] - for segment in unit.closed_segments: - for i in range(segment["start"], segment["end"]): - timeline_map[i].add(unit) # maybe too memory intensive? - - new_timeline_map = {i : [] for i in range(total_cycles)} - # now, we need to figure out the sets of traces - for i in timeline_map: - parents = set() # keeping track of entities that are parents of other entities - i_mapping = {} # each unique group inv mapping to its stack. the "group" should be the last item on each stack - i_mapping[main_component] = [main_component.split(".")[-1]] - - cell_worklist = [main_component] # FIXME: maybe remove the hardcoding? - while len(cell_worklist) > 0: - current_cell = cell_worklist.pop() - current_component = cells_to_components[current_cell] - covered_units_in_component = set() # collect all of the units we've covered. - # catch all active units that are groups in this component. - units_to_cover = set(filter(lambda unit: not unit.is_cell and unit.component == current_component, timeline_map[i])) - # find all enables from control. these are all units that either (1) don't have any maps in call_stack_probes_info, or (2) have no active parent calls in call_stack_probes_info - for active_unit in units_to_cover: - if active_unit.is_cell: # skip cells for now as we're considering only single component programs - continue - if active_unit.id() not in call_stack_probes_info: # no maps in call_stack_probes_info - i_mapping[active_unit.name] = i_mapping[current_cell] + [active_unit.shortname] - parents.add(current_cell) - covered_units_in_component.add(active_unit.name) - else: - # loop through all parents and see if any of them are active - contains_active_parent = False - for parent, call_probe_info in call_stack_probes_info[active_unit.id()].items(): - if call_probe_info.is_active_at_cycle(i): - contains_active_parent = True - break - if not contains_active_parent: - i_mapping[active_unit.name] = i_mapping[current_cell] + [active_unit.shortname] - parents.add(current_cell) - covered_units_in_component.add(active_unit.name) - while len(covered_units_in_component) < len(units_to_cover): - # loop through all other elements to figure out parent child info - for active_unit in units_to_cover: - if active_unit.is_cell or active_unit.name in i_mapping: - continue - for parent, call_probe_info in call_stack_probes_info[active_unit.id()].items(): - if f"{main_component}.{parent}" in i_mapping: # we can directly build on top of the parent - i_mapping[active_unit.name] = i_mapping[f"{current_cell}.{parent}"] + [active_unit.shortname] - covered_units_in_component.add(active_unit.name) - parents.add(f"{current_cell}.{parent}") - # by this point, we should have covered all groups in the same component... - # now we need to construct stacks for any cells that are called from a group in the current component. - # collect caller ids in cell_caller_probes_info that belong to our component - cell_invoker_ids = list(filter(lambda x : x.split(DELIMITER)[1] == current_component, cell_caller_probes_info)) - for cell_invoker_id in cell_invoker_ids: - cell_invoker = cell_invoker_id.split(DELIMITER)[0] - # iterate through all of the cells that the group invokes - for invoked_cell_name in cell_caller_probes_info[cell_invoker_id]: - cell_calling_probe = cell_caller_probes_info[cell_invoker_id][invoked_cell_name] - cell_active_probe = active_element_probes_info[invoked_cell_name] - if cell_calling_probe.is_active_at_cycle(i) and cell_active_probe.is_active_at_cycle(i): - cell_worklist.append(cell_active_probe.name) - # invoker group is the parent of the cell. - cell_component = cells_to_components[cell_active_probe.name] - i_mapping[cell_active_probe.name] = i_mapping[f"{current_cell}.{cell_invoker}"] + [f"{cell_active_probe.shortname} [{cell_component}]"] - parents.add(f"{current_cell}.{cell_invoker}") - - # Only retain paths that lead to leaf nodes. - for elem in i_mapping: - if elem not in parents: - new_timeline_map[i].append(i_mapping[elem]) - - for i in new_timeline_map: - print(i) - for stack in new_timeline_map[i]: - print(f"\t{stack}") - - return new_timeline_map - -""" -Creates a tree that encapsulates all stacks that occur within the program. -""" -def create_tree(timeline_map): - node_id_acc = 0 - tree_dict = {} # node id --> node name - path_dict = {} # stack list string --> list of node ids - path_prefixes_dict = {} # stack list string --> list of node ids - stack_list = [] - # collect all of the stacks from the list. (i.e. "flatten" the timeline map values.) - for sl in timeline_map.values(): - for s in sl: - if s not in stack_list: - stack_list.append(s) - stack_list.sort(key=len) - for stack in stack_list: - stack_len = len(stack) - id_path_list = [] - prefix = "" - # obtain the longest prefix of the current stack. Everything after the prefix is a new stack element. - for i in range(1, stack_len+1): - attempted_prefix = ";".join(stack[0:stack_len-i]) - if attempted_prefix in path_prefixes_dict: - prefix = attempted_prefix - id_path_list = list(path_prefixes_dict[prefix]) - break - # create nodes - if prefix != "": - new_nodes = stack[stack_len - i:] - new_prefix = prefix - else: - new_nodes = stack - new_prefix = "" - for elem in new_nodes: - if new_prefix == "": - new_prefix = elem - else: - new_prefix += f";{elem}" - tree_dict[node_id_acc] = elem - id_path_list.append(node_id_acc) - path_prefixes_dict[new_prefix] = list(id_path_list) - node_id_acc += 1 - path_dict[new_prefix] = id_path_list - - return tree_dict, path_dict - -def create_path_dot_str_dict(path_dict): - path_to_dot_str = {} # stack list string --> stack path representation on dot file. - - for path_id in path_dict: - path = path_dict[path_id] - path_acc = "" - for node_id in path[0:-1]: - path_acc += f'{node_id} -> ' - path_acc += f'{path[-1]}' - path_to_dot_str[path_id] = path_acc - - return path_to_dot_str - -def create_output(timeline_map, out_dir): - - os.mkdir(out_dir) - - # make flame graph folded file - stacks = {} # stack to number of cycles - for i in timeline_map: - for stack_list in timeline_map[i]: - stack_id = ";".join(stack_list) - if stack_id not in stacks: - stacks[stack_id] = 1 - else: - stacks[stack_id] += 1 - - with open(os.path.join(out_dir, "flame.folded"), "w") as flame_out: - for stack in stacks: - flame_out.write(f"{stack} {stacks[stack]}\n") - - # probably wise to not have a billion dot files. - if len(timeline_map) > TREE_PICTURE_LIMIT: - print(f"Simulation exceeds {TREE_PICTURE_LIMIT} cycles, skipping trees...") - return - tree_dict, path_dict = create_tree(timeline_map) - path_to_dot_str = create_path_dot_str_dict(path_dict) - all_paths_ordered = sorted(path_dict.keys()) - for i in timeline_map: - used_paths = set() - used_nodes = set() - all_nodes = set(tree_dict.keys()) - # figure out what nodes are used and what nodes aren't used - for stack in timeline_map[i]: - stack_id = ";".join(stack) - used_paths.add(stack_id) - for node_id in path_dict[stack_id]: - used_nodes.add(node_id) - - fpath = os.path.join(out_dir, f"cycle{i}.dot") - with open(fpath, "w") as f: - f.write("digraph cycle" + str(i) + " {\n") - # declare nodes. - # used nodes should simply be declared - for used_node in used_nodes: - f.write(f'\t{used_node} [label="{tree_dict[used_node]}"];\n') - # unused nodes should be declared with gray - for unused_node in all_nodes.difference(used_nodes): - f.write(f'\t{unused_node} [label="{tree_dict[unused_node]}",color="{INVISIBLE}",fontcolor="{INVISIBLE}"];\n') - # write all paths. - for path_id in all_paths_ordered: - if ";" not in path_id or path_id in used_paths: - f.write(f'\t{path_to_dot_str[path_id]} ;\n') - else: - f.write(f'\t{path_to_dot_str[path_id]} [color="{INVISIBLE}"];\n') - f.write("}") - -def main(vcd_filename, cells_json_file, out_dir): - main_component, cells_to_components = read_component_cell_names_json(cells_json_file) - converter = VCDConverter(main_component, cells_to_components) - vcdvcd.VCDVCD(vcd_filename, callbacks=converter) - converter.postprocess() - - new_timeline_map = create_traces(converter.active_elements_info, converter.call_stack_probe_info, converter.cell_invoke_caller_probe_info, converter.clock_cycles, cells_to_components, main_component) - - create_output(new_timeline_map, out_dir) - - -if __name__ == "__main__": - if len(sys.argv) > 3: - vcd_filename = sys.argv[1] - cells_json = sys.argv[2] - out_dir = sys.argv[3] - main(vcd_filename, cells_json, out_dir) - else: - args_desc = [ - "VCD_FILE", - "CELLS_JSON", - "OUT_DIR" - ] - print(f"Usage: {sys.argv[0]} {' '.join(args_desc)}") - print("CELLS_JSON: Run the `component_cells` tool") - sys.exit(-1) diff --git a/tools/profiler/new-profiler-approach.sh b/tools/profiler/new-profiler-approach.sh index 8fb3e89fb..b6f6398e3 100644 --- a/tools/profiler/new-profiler-approach.sh +++ b/tools/profiler/new-profiler-approach.sh @@ -79,7 +79,7 @@ fi echo "[${SCRIPT_NAME}] Using FSM info and VCD file to obtain cycle level counts" ( set -o xtrace - python3 ${SCRIPT_DIR}/new-parse-vcd.py ${VCD_FILE} ${CELLS_JSON} ${OUT_DIR} + python3 ${SCRIPT_DIR}/profiler-process.py ${VCD_FILE} ${CELLS_JSON} ${OUT_DIR} set +o xtrace ) &> ${LOGS_DIR}/gol-process diff --git a/tools/profiler/parse-vcd.py b/tools/profiler/parse-vcd.py deleted file mode 100644 index 8c2d79d7a..000000000 --- a/tools/profiler/parse-vcd.py +++ /dev/null @@ -1,392 +0,0 @@ -import csv -import sys -import json -import vcdvcd - -def remove_size_from_name(name: str) -> str: - """ changes e.g. "state[2:0]" to "state" """ - return name.split('[')[0] - -class ProfilingInfo: - def __init__(self, name, component, fsm_name=None, fsm_values=None, tdcc_group_name=None, is_cell=False): - self.name = name - self.fsm_name = fsm_name - self.fsm_values = list(sorted(fsm_values)) if fsm_values is not None else None - self.total_cycles = 0 - self.closed_segments = [] # Segments will be (start_time, end_time) - self.current_segment = None - self.tdcc_group = tdcc_group_name - self.component = component - self.is_cell = is_cell - - def __repr__ (self): - segments_str = "" - for segment in self.closed_segments: - if (segments_str != ""): - segments_str += ", " - segments_str += f"[{segment['start']}, {segment['end']})" - if self.fsm_name is not None: - header = (f"[FSM] Group {self.name}:\n" + - f"\tFSM name: {self.fsm_name}\n" + - f"\tFSM state ids: {self.fsm_values}\n" - ) - elif self.component is None: - header = f"[CMP] Group {self.name}:\n" - else: - header = f"[GT] Group {self.name}:\n" - - return (header + - f"\tTotal cycles: {self.total_cycles}\n" + - f"\t# of times active: {len(self.closed_segments)}\n" + - f"\tSegments: {segments_str}\n" - ) - - def is_active(self): - return self.current_segment is not None - - def start_clock_cycle(self): - if self.current_segment is None: - return -1 - else: - return self.current_segment["start"] - - def compute_average_cycles(self): - if len(self.closed_segments) == 0: - return 0 - else: - return round(self.total_cycles / len(self.closed_segments), 2) - - def emit_csv_data(self): - name = self.name - if self.fsm_name is not None: - name += "[FSM]" - if self.component is None: - name += "[CMP]" - return {"name": name, - "total-cycles" : self.total_cycles, - "times-active" : len(self.closed_segments), - "avg" : self.compute_average_cycles()} - - def summary(self): - if self.fsm_name is None: - header = "[GT] " - else: - header = "[FSM]" - return (f"{header} Group {self.name} Summary:\n" + - f"\tTotal cycles: {self.total_cycles}\n" + - f"\t# of times active: {len(self.closed_segments)}\n" + - f"\tAvg runtime: {self.compute_average_cycles()}\n" - ) - - def start_new_segment(self, curr_clock_cycle): - if self.current_segment is None: - self.current_segment = {"start": curr_clock_cycle, "end": -1} - else: - print(f"Error! The group {self.name} is starting a new segment while the current segment is not closed.") - print(f"Current segment: {self.current_segment}") - sys.exit(1) - - def end_current_segment(self, curr_clock_cycle): - if self.current_segment is not None and self.current_segment["end"] == -1: # ignore cases where done is high forever - self.current_segment["end"] = curr_clock_cycle - self.closed_segments.append(self.current_segment) - self.total_cycles += curr_clock_cycle - self.current_segment["start"] - self.current_segment = None # Reset current segment - -class VCDConverter(vcdvcd.StreamParserCallbacks): - - def __init__(self, fsms, tdcc_groups, fsm_group_maps, main_component, cells): - super().__init__() - self.main_component = main_component - self.fsms = fsms - self.single_enable_names = set() - # Recording the first cycle when the TDCC group became active - self.tdcc_group_active_cycle = {tdcc_group_name : -1 for tdcc_group_name in tdcc_groups} - # Map from a TDCC group to all FSMs that depend on it. maybe a 1:1 mapping - self.tdcc_group_to_dep_fsms = tdcc_groups - # Group name --> ProfilingInfo object - self.profiling_info = {} - for group in fsm_group_maps: - # Differentiate FSM versions from ground truth versions - self.profiling_info[f"{group}FSM"] = ProfilingInfo(group, fsm_group_maps[group]["component"], fsm_group_maps[group]["fsm"], fsm_group_maps[group]["ids"], fsm_group_maps[group]["tdcc-group-name"]) - self.cells = set(cells.keys()) - for cell in cells: - self.profiling_info[cell] = ProfilingInfo(cell, cells[cell], is_cell=True) - # Map from timestamps [ns] to value change events that happened on that timestamp - self.timestamps_to_events = {} - - def enddefinitions(self, vcd, signals, cur_sig_vals): - # convert references to list and sort by name - refs = [(k, v) for k, v in vcd.references_to_ids.items()] - refs = sorted(refs, key=lambda e: e[0]) - names = [remove_size_from_name(e[0]) for e in refs] - signal_id_dict = {sid : [] for sid in vcd.references_to_ids.values()} # one id can map to multiple signal names since wires are connected - - clock_name = f"{self.main_component}.clk" - if clock_name not in names: - print("Can't find the clock? Exiting...") - sys.exit(1) - signal_id_dict[vcd.references_to_ids[clock_name]] = [clock_name] - - # get go and done for cells (the signals are exactly {cell}.go and {cell}.done) - for cell in self.cells: - cell_go = cell + ".go" - cell_done = cell + ".done" - if cell_go not in vcd.references_to_ids: - print(f"Not accounting for cell {cell} (probably combinational)") - continue - signal_id_dict[vcd.references_to_ids[cell_go]].append(cell_go) - signal_id_dict[vcd.references_to_ids[cell_done]].append(cell_done) - - for name, sid in refs: - # FIXME: We may want to optimize these nested for loops - for tdcc_group in self.tdcc_group_to_dep_fsms: - if name.startswith(f"{tdcc_group}_go.out["): - signal_id_dict[sid].append(name) - for fsm in self.fsms: - if name.startswith(f"{fsm}.out["): - signal_id_dict[sid].append(name) - if "_probe_out" in name: # instrumentation probes are "___probe" - group_component_split = name.split("_probe_out")[0].split("__") - group_name = group_component_split[0] - self.single_enable_names.add(group_name) - self.profiling_info[group_name] = ProfilingInfo(group_name, group_component_split[1]) - signal_id_dict[sid].append(name) - - # don't need to check for signal ids that don't pertain to signals we're interested in - self.signal_id_to_names = {k:v for k,v in signal_id_dict.items() if len(v) > 0} - - # Stream processes the events recorded in the VCD and stores them in self.timestamps_to_events - # NOTE: Stream processing doesn't work because value changes that happen in the same timestamp - # are not processed at the same time. - # NOTE: when we reimplement this script, we probably want to separate this part from the - # clock cycle processing - def value( - self, - vcd, - time, - value, - identifier_code, - cur_sig_vals, - ): - # ignore all signals we don't care about - if identifier_code not in self.signal_id_to_names: - return - - signal_names = self.signal_id_to_names[identifier_code] - int_value = int(value, 2) - - if time not in self.timestamps_to_events: - self.timestamps_to_events[time] = [] - - for signal_name in signal_names: - event = {"signal": signal_name, "value": int_value} - self.timestamps_to_events[time].append(event) - - # Postprocess data mapping timestamps to events (signal changes) - # We have to postprocess instead of processing signals in a stream because - # signal changes that happen at the same time as a clock tick might be recorded - # *before* or *after* the clock change on the VCD file (hence why we can't process - # everything within a stream if we wanted to be precise) - def postprocess(self): - clock_name = f"{self.main_component}.clk" - clock_cycles = -1 - fsm_to_active_group = {fsm : None for fsm in self.fsms} - # current values of FSM registers. This is different from fsm_to_active_group since the TDCC group for the FSM - # may not be active (which means that no group managed by the FSM is active) - fsm_to_curr_value = {fsm: -1 for fsm in self.fsms} - started = False - for ts in self.timestamps_to_events: - events = self.timestamps_to_events[ts] - started = started or [x for x in events if x["signal"] == f"{self.main_component}.go" and x["value"] == 1] - if not started: - # Update fsm_to_curr_value for any FSM signals that got updated. We will start the events corresponding - # to those values once the TDCC group for the FSM starts. - # Realistically this will most likely only happen on the 0th cycle just to set the FSM value to 0, - # but trying to be extra safe here. - for event in filter(lambda e : "fsm" in e["signal"], events): - fsm = ".".join(event["signal"].split(".")[0:-1]) - if event["value"] in self.fsms[fsm]: - fsm_to_curr_value[fsm] = event["value"] - continue - # checking whether the timestamp has a rising edge (hacky) - if {"signal": clock_name, "value": 1} in events: - clock_cycles += 1 - # TDCC groups need to be recorded (before FSMs) for tracking FSM values - # (ex. if the FSM has value 0 but the TDCC group isn't active, then the group represented by the - # FSM's 0 value should not be considered as active) - for tdcc_event in filter(lambda e : "tdcc" in e["signal"] and "go" in e["signal"], events): - tdcc_group = "_".join(tdcc_event["signal"].split("_")[0:-1]) - if self.tdcc_group_active_cycle[tdcc_group] == -1 and tdcc_event["value"] == 1: # value changed to 1 - self.tdcc_group_active_cycle[tdcc_group] = clock_cycles - for fsm in self.tdcc_group_to_dep_fsms[tdcc_group]: - value = fsm_to_curr_value[fsm] - if value != -1: - if value not in self.fsms[fsm]: - continue - next_group = f"{self.fsms[fsm][value]}FSM" - fsm_to_active_group[fsm] = next_group - self.profiling_info[next_group].start_new_segment(clock_cycles) - elif self.tdcc_group_active_cycle[tdcc_group] > -1 and tdcc_event["value"] == 0: # tdcc group that was active's signal turned to 0 - self.tdcc_group_active_cycle[tdcc_group] = -1 - for event in events: - signal_name = event["signal"] - value = event["value"] - if "tdcc" in signal_name and "go" in signal_name: # skip all tdcc events since we've already processed them - continue - if signal_name.endswith(".go") and value == 1: # cells have .go and .done - cell = signal_name.split(".go")[0] - self.profiling_info[cell].start_new_segment(clock_cycles) - if signal_name.endswith(".done") and value == 1: # cells have .go and .done - cell = signal_name.split(".done")[0] - self.profiling_info[cell].end_current_segment(clock_cycles) - if "_probe_out" in signal_name and value == 1: # instrumented group started being active - group = signal_name.split("_probe_out")[0].split("__")[0] - self.profiling_info[group].start_new_segment(clock_cycles) - elif "_probe_out" in signal_name and value == 0: # instrumented group stopped being active - group = signal_name.split("_probe_out")[0].split("__")[0] - self.profiling_info[group].end_current_segment(clock_cycles) - elif "fsm" in signal_name: - fsm = ".".join(signal_name.split(".")[0:-1]) - fsm_to_curr_value[fsm] = value - # Workarounds because the value 0 may not correspond to a group - if fsm_to_active_group[fsm] is not None: - prev_group = fsm_to_active_group[fsm] # getting the "FSM" variant of the group - self.profiling_info[prev_group].end_current_segment(clock_cycles) - if value in self.fsms[fsm]: - next_group = f"{self.fsms[fsm][value]}FSM" # getting the "FSM" variant of the group - tdcc_group_active_cycle = self.tdcc_group_active_cycle[self.profiling_info[next_group].tdcc_group] - if tdcc_group_active_cycle == -1: # If the TDCC group is not active, then no segments should start - continue - fsm_to_active_group[fsm] = next_group - self.profiling_info[next_group].start_new_segment(clock_cycles) - - self.clock_cycles = clock_cycles - -# Generates a list of all of the components to potential cell names -# `prefix` is the cell's "path" (ex. for a cell "my_cell" defined in "main", the prefix would be "TOP.toplevel.main") -# The initial value of curr_component should be the top level/main component -def build_components_to_cells(prefix, curr_component, cells_to_components, components_to_cells): - for (cell, cell_component) in cells_to_components[curr_component].items(): - if cell_component not in components_to_cells: - components_to_cells[cell_component] = [f"{prefix}.{cell}"] - else: - components_to_cells[cell_component].append(f"{prefix}.{cell}") - build_components_to_cells(prefix + f".{cell}", cell_component, cells_to_components, components_to_cells) - -# Reads json generated by component-cells backend to produce a mapping from all components -# to cell names they could have. -def read_component_cell_names_json(json_file): - cell_json = json.load(open(json_file)) - # For each component, contains a map from each cell name to its corresponding component - # component name --> { cell name --> component name} - cells_to_components = {} - main_component = "" - for curr_component_entry in cell_json: - cell_map = {} # mapping cell names to component names for all cells in the current component - if curr_component_entry["is_main_component"]: - main_component = curr_component_entry["component"] - for cell_info in curr_component_entry["cell_info"]: - cell_map[cell_info["cell_name"]] = cell_info["component_name"] - cells_to_components[curr_component_entry["component"]] = cell_map - full_main_component = f"TOP.toplevel.{main_component}" - components_to_cells = {main_component : [full_main_component]} # come up with a better name for this - build_components_to_cells(full_main_component, main_component, cells_to_components, components_to_cells) - return full_main_component, components_to_cells - -# Reads json generated by TDCC (via dump-fsm-json option) to produce initial group information -def remap_tdcc_json(tdcc_json_file, components_to_cells): - profiling_infos = json.load(open(tdcc_json_file)) - cells_to_components = {} # go and done info are needed for cells. cell --> component name - tdcc_groups = {} # TDCC-generated groups that manage control flow using FSMs. maps to all fsms that map to the tdcc group - fsm_group_maps = {} # fsm-managed groups info (fsm register, TDCC group that manages fsm, id of group within fsm) - fsms = {} # Remapping of JSON data for easy access - for profiling_info in profiling_infos: - if "Fsm" in profiling_info: - fsm = profiling_info["Fsm"] - # create entries for all possible cells of component - for cell in components_to_cells[fsm["component"]]: - fsm_name = cell + "." + fsm["fsm"] - fsms[fsm_name] = {} - for state in fsm["states"]: - group_name = cell + "." + state["group"] - fsms[fsm_name][state["id"]] = group_name - tdcc_group = cell + "." + fsm["group"] - if group_name not in fsm_group_maps: - fsm_group_maps[group_name] = {"fsm": fsm_name, "tdcc-group-name": tdcc_group, "ids": [state["id"]], "component": fsm["component"]} - if tdcc_group not in tdcc_groups: # Keep track of the TDCC group to figure out when first group starts - tdcc_groups[tdcc_group] = set() - tdcc_groups[tdcc_group].add(fsm_name) - else: - fsm_group_maps[group_name]["ids"].append(state["id"]) - for component in components_to_cells: - for cell in components_to_cells[component]: - cells_to_components[cell] = component - - return fsms, tdcc_groups, fsm_group_maps, cells_to_components - -def output_result(out_csv, dump_out_json, converter): - print(f"Total clock cycles: {converter.clock_cycles}") - print("=====SUMMARY=====") - print() - groups_to_emit = list(filter(lambda group : not group.name.startswith("tdcc") and not group.name.endswith("END"), converter.profiling_info.values())) - groups_to_emit.sort(key=lambda x : x.name) # to preserve stability - groups_to_emit.sort(key=lambda x : x.total_cycles, reverse=True) - csv_acc = [] - dump_json_acc = [] - for group_info in groups_to_emit: - csv_acc.append(group_info.emit_csv_data()) - dump_json_acc.append(group_info.__dict__) - print(group_info.summary()) - print("=====DUMP=====") - print() - for group_info in groups_to_emit: - print(group_info) - # Add total cycles for visualizer script (probably want to do this in a neater fashion in the future) - dump_json_acc.append({"name": "TOTAL", "total_cycles": converter.clock_cycles, "main_full_path": converter.main_component}) - # emit a json for visualizer script - print(f"Writing dump JSON to {dump_out_json}") - with open(dump_out_json, "w", encoding="utf-8") as dump_file: - dump_file.write(json.dumps(dump_json_acc, indent=4)) - # emitting a CSV file for easier eyeballing - print(f"Writing summary to {out_csv}") - csv_keys = ["name", "total-cycles", "times-active", "avg"] - csv_acc.append({ "name": "TOTAL", "total-cycles": converter.clock_cycles, "times-active": "-", "avg": "-"}) - if (out_csv == "STDOUT"): - writer = csv.DictWriter(sys.stdout, csv_keys, lineterminator="\n") - else: - writer = csv.DictWriter(open(out_csv, "w"), csv_keys, lineterminator="\n") - writer.writeheader() - writer.writerows(csv_acc) - -def main(vcd_filename, tdcc_json_file, cells_json_file, out_csv, dump_out_json): - main_component, components_to_cells = read_component_cell_names_json(cells_json_file) - fsms, tdcc_group_names, fsm_group_maps, cells = remap_tdcc_json(tdcc_json_file, components_to_cells) - converter = VCDConverter(fsms, tdcc_group_names, fsm_group_maps, main_component, cells) - vcdvcd.VCDVCD(vcd_filename, callbacks=converter, store_tvs=False) - converter.postprocess() - output_result(out_csv, dump_out_json, converter) - -if __name__ == "__main__": - if len(sys.argv) > 5: - vcd_filename = sys.argv[1] - fsm_json = sys.argv[2] - cells_json = sys.argv[3] - out_csv = sys.argv[4] - dump_out_json = sys.argv[5] - main(vcd_filename, fsm_json, cells_json, out_csv, dump_out_json) - else: - args_desc = [ - "VCD_FILE", - "TDCC_JSON", - "CELLS_JSON", - "SUMMARY_OUT_CSV", - "DUMP_OUT_JSON" - ] - print(f"Usage: {sys.argv[0]} {' '.join(args_desc)}") - print("TDCC_JSON: Run Calyx with `tdcc:dump-fsm-json` option") - print("CELLS_JSON: Run the `component_cells` tool") - print("GROUPS_JSON: Run the `component_groups` tool") - print("If SUMMARY_OUT_CSV is STDOUT, then summary CSV will be printed to stdout") - print("DUMP_OUT_JSON: output json file for group-specific") - sys.exit(-1) diff --git a/tools/profiler/run-up-to-tdcc.sh b/tools/profiler/run-up-to-tdcc.sh index 2ae2f7055..44373c166 100644 --- a/tools/profiler/run-up-to-tdcc.sh +++ b/tools/profiler/run-up-to-tdcc.sh @@ -9,16 +9,14 @@ fi SCRIPT_DIR=$( cd $( dirname $0 ) && pwd ) CALYX_DIR=$( dirname $( dirname ${SCRIPT_DIR} ) ) -if [ "$2" == "-o" ]; then -( +if [ "$2" == "-no" ]; then + ( cd ${CALYX_DIR} - cargo run $1 -p compile-repeat -p well-formed -p papercut -p canonicalize -p infer-data-path -p collapse-control -p compile-sync-without-sync-reg -p group2seq -p dead-assign-removal -p group2invoke -p infer-share -p inline -p comb-prop -p dead-cell-removal -p cell-share -p simplify-with-control -p compile-invoke -p static-inference -p static-promotion -p compile-repeat -p dead-group-removal -p collapse-control -p static-inline -p merge-assigns -p dead-group-removal -p simplify-static-guards -p add-guard -p static-fsm-opts -p compile-static -p dead-group-removal -p tdcc -) + cargo run $1 -p compile-repeat -p well-formed -p papercut -p canonicalize -p compile-sync -p simplify-with-control -p compile-invoke -p static-inline -p merge-assigns -p dead-group-removal -p simplify-static-guards -p add-guard -p static-fsm-opts -p compile-static -p dead-group-removal -p tdcc + ) else - ( cd ${CALYX_DIR} - cargo run $1 -p compile-repeat -p well-formed -p papercut -p canonicalize -p compile-sync -p simplify-with-control -p compile-invoke -p static-inline -p merge-assigns -p dead-group-removal -p simplify-static-guards -p add-guard -p static-fsm-opts -p compile-static -p dead-group-removal -p tdcc + cargo run $1 -p compile-repeat -p well-formed -p papercut -p canonicalize -p infer-data-path -p collapse-control -p compile-sync-without-sync-reg -p group2seq -p dead-assign-removal -p group2invoke -p infer-share -p inline -p comb-prop -p dead-cell-removal -p cell-share -p simplify-with-control -p compile-invoke -p static-inference -p static-promotion -p compile-repeat -p dead-group-removal -p collapse-control -p static-inline -p merge-assigns -p dead-group-removal -p simplify-static-guards -p add-guard -p static-fsm-opts -p compile-static -p dead-group-removal # -p tdcc ) - fi