Merge pull request #189 from Perfexionists/traces-and-views

Add incremental sankey diff view

.github/workflows/ubuntu.yml

@@ -36,6 +36,7 @@ jobs:
         uses: codecov/codecov-action@v3
         with:
           flags: coverage-${{ matrix.python-version }}
+          token: ${{ secrets.CODECOV_TOKEN }}
           verbose: true
 
   # Tests that perun is buildable from distribution packages (this is precursor for pypi install).

perun/cli.py

@@ -1293,3 +1293,7 @@ def launch_cli() -> None:
         launch_cli_in_dev_mode()
     else:
         launch_cli_safely()
+
+
+if __name__ == "__main__":
+    launch_cli()

perun/logic/store.py

@@ -3,6 +3,7 @@
 Store is a collection of helper functions that can be used to pack content, compute checksums,
 or load and store into the directories or filenames.
 """
+
 from __future__ import annotations
 
 # Standard Imports
@@ -347,7 +348,8 @@ def load_profile_from_handle(
 
     # Try to load the json, if there is issue with the profile
     try:
-        return Profile(json.loads(body))
+        with common_kit.disposable_resources(json.loads(body)) as json_profile:
+            return Profile(json_profile)
     except ValueError:
         raise IncorrectProfileFormatException(
             file_name, f"profile '{file_name}' is not in profile format"

perun/profile/convert.py

@@ -12,6 +12,7 @@
 `pandas`_ library one can obtain efficient interpreter for executing more
 complex queries and statistical tests over the profiles.
 """
+
 from __future__ import annotations
 
 # Standard Imports
@@ -71,7 +72,7 @@ def resources_to_pandas_dataframe(profile: Profile) -> pandas.DataFrame:
     values["snapshots"] = array.array("I")
 
     # All resources at this point should be flat
-    for snapshot, resource in profile.all_resources(True):
+    for snapshot, resource in profile.all_resources(flatten_values=True):
         values["snapshots"].append(snapshot)
         for resource_key in resource_keys:
             values[resource_key].append(resource.get(resource_key, numpy.nan))
@@ -102,7 +103,7 @@ def models_to_pandas_dataframe(profile: Profile) -> pandas.DataFrame:
     return pandas.DataFrame(values)
 
 
-def to_flame_graph_format(profile: Profile) -> list[str]:
+def to_flame_graph_format(profile: Profile, profile_key: str = "amount") -> list[str]:
     """Transforms the **memory** profile w.r.t. :ref:`profile-spec` into the
     format supported by perl script of Brendan Gregg.
 
@@ -127,20 +128,23 @@ def to_flame_graph_format(profile: Profile) -> list[str]:
     identifiers joined using ``;`` character).
 
     :param Profile profile: the memory profile
+    :param profile_key: key that is used to obtain the data
     :returns: list of lines, each representing one allocation call stack
     """
     stacks = []
     for _, snapshot in profile.all_snapshots():
         for alloc in snapshot:
             if "subtype" not in alloc.keys() or alloc["subtype"] != "free":
-                stack_str = to_uid(alloc["uid"]) + ";"
-                for frame in alloc["trace"][::-1]:
-                    line = to_string_line(frame)
-                    stack_str += line + ";"
-                if stack_str and stack_str.endswith(";"):
-                    final = stack_str[:-1]
-                    final += " " + str(alloc["amount"]) + "\n"
-                    stacks.append(final)
+                # Workaround for total time used in some collectors, so it is not outputted
+                if alloc["uid"] != "%TOTAL_TIME%":
+                    stack_str = to_uid(alloc["uid"]) + ";"
+                    for frame in alloc["trace"][::-1]:
+                        line = to_string_line(frame)
+                        stack_str += line + ";"
+                    if stack_str and stack_str.endswith(";"):
+                        final = stack_str[:-1]
+                        final += " " + str(alloc[profile_key]) + "\n"
+                        stacks.append(final)
 
     return stacks
 
@@ -157,13 +161,15 @@ def to_uid(record: dict[str, Any] | str) -> str:
         return to_string_line(record)
 
 
-def to_string_line(frame: dict[str, Any]) -> str:
+def to_string_line(frame: dict[str, Any] | str) -> str:
     """Create string representing call stack's frame
 
     :param dict frame: call stack's frame
     :returns str: line representing call stack's frame
     """
-    if "function" in frame.keys() and "source" in frame.keys() and "line" in frame.keys():
+    if isinstance(frame, str):
+        return frame
+    elif "function" in frame.keys() and "source" in frame.keys() and "line" in frame.keys():
         return f"{frame['function']}()~{frame['source']}~{frame['line']}"
     else:
         assert "func" in frame.keys()

perun/templates/__init__.py

@@ -0,0 +1 @@
+"""Templates and utilities for jinja2 templating"""

perun/templates/diff_view_flamegraph.html.jinja2

@@ -77,6 +77,29 @@
             margin: 0;
             padding: 0;
         }
+
+        div.tools table {
+            border-collapse: collapse;
+            border-bottom: 1px solid #ddd;
+            margin: 0 auto 2em auto;
+        }
+
+        div.tools td.value {
+            padding: 1em 0.5em;
+            text-align: left;
+        }
+
+        div.tools td.key {
+            padding: 1em 0.5em;
+            text-align: right;
+            font-weight: bold;
+        }
+
+        div.flamegraphs h2 {
+            text-align: center;
+            border-bottom: 1px solid #ddd;
+        }
+        
         {{ profile_overview.css_style() }}
         {{ accordion.css_style() }}
     </style>
@@ -86,46 +109,90 @@
 <div class="left column">
     <h2 class="column-head">{{ lhs_tag }}</h2>
     {{ profile_overview.overview_table('toggleLeftCollapse', 'left-info', lhs_header, rhs_header) }}
-    {{ profile_overview.top_n_table('toggleLeftTopCollapse', 'left-top', lhs_top, rhs_uids, "tgt") }}
-    <div class='svg-container'>
-    {{ lhs_flamegraph }}
-    </div>
 </div>
 
 <div class="right column">
     <h2 class="column-head">{{ rhs_tag }}</h2>
     {{ profile_overview.overview_table('toggleRightCollapse', 'right-info', rhs_header, lhs_header) }}
-    {{ profile_overview.top_n_table('toggleRightTopCollapse', 'right-top', rhs_top, lhs_uids, "base") }}
-    <div class='svg-container'>
-        {{ rhs_flamegraph }}
-    </div>
 </div>
 
-<div class="middle">
-    <h2 class="column-head">Difference of profiles</h2>
-    <div class='svg-container'>
-        {{ diff_flamegraph }}
+<div class="tools">
+    <table>
+        <tbody>
+        <tr>
+            <td class="key" title="Displays resources for different type.">Resource Type:</td>
+            <td class="value" title="Displays resources for different type.">
+                <select class="filter" id="type_selector">
+                    {% for data_type in data_types %}
+                        <option value="{{ data_type }}">{{ data_type }}</option>
+                    {%- endfor %}
+                </select>
+            </td>
+        </tr>
+        </tbody>
+    </table>
+</div>
+
+{% for (_, lhs_flamegraph, rhs_flamegraph, diff_flamegraph) in flamegraphs %}
+<div class="flamegraphs" id="{{ data_types[loop.index0] }}">
+    <h2>{{ data_types[loop.index0] }}</h2>
+    <div class="left column">
+        <div class='svg-container'>
+            {{ lhs_flamegraph}}
+        </div>
+    </div>
+    <div class="right column">
+        <div class='svg-container'>
+            {{ rhs_flamegraph }}
+        </div>
     </div>
+    <div class="middle">
+        <h2 class="column-head">Difference of profiles</h2>
+        <div class='svg-container'>
+            {{ diff_flamegraph }}
+        </div>
 
-    <div class="help">
-        <h2>Help</h2>
-        <ul>
-            <li>> Click on the square to nested into selected trace.</li>
-            <li>> The size of the rectangle represents relative consumption with respect to parent.</li>
-            <li>> The color of the rectangle represents nothing.</li>
-            <li>> Use <it>reset zoom</it> (top left) to return to original view.</li>
-            <li>> Use <it>search</it> (top right) to highlight selected functions.</li>
-        </ul>
+        <div class="help">
+            <h2>Help</h2>
+            <ul>
+                <li>> Click on the square to nested into selected trace.</li>
+                <li>> The size of the rectangle represents relative consumption with respect to parent.</li>
+                <li>> The color of the rectangle represents nothing.</li>
+                <li>> Use <it>reset zoom</it> (top left) to return to original view.</li>
+                <li>> Use <it>search</it> (top right) to highlight selected functions.</li>
+            </ul>
+        </div>
     </div>
 </div>
+{%- endfor %}
 
 
 <script>
 {{ profile_overview.toggle_script('toggleLeftCollapse', 'left-info') }}
-{{ profile_overview.toggle_script('toggleLeftTopCollapse', 'left-top') }}
 {{ profile_overview.toggle_script('toggleRightCollapse', 'right-info') }}
-{{ profile_overview.toggle_script('toggleRightTopCollapse', 'right-top') }}
 {{ accordion.script("table-row") }}
+
+function switch_type() {
+    const selected_type = document.querySelector("#type_selector").value
+
+    var selectedDiv = document.getElementById(selected_type);
+    var allDivs = document.getElementsByClassName("flamegraphs");
+
+    for (var i = 0; i < allDivs.length; i++) {
+        allDivs.item(i).style.visibility = "collapse";
+        allDivs.item(i).style.display = "none";
+    }
+    selectedDiv.style.visibility = "visible";
+    selectedDiv.style.display = "block";
+}
+document.querySelectorAll('select.filter').forEach((el) => {
+    el.addEventListener('change', () => {
+        switch_type()
+    })
+});
+
+switch_type()
+
 </script>
 
 </body>