From cfdfe28f52ce718e48098cc55e74b94a3c7469b2 Mon Sep 17 00:00:00 2001 From: Morgan Schwartz Date: Thu, 19 Dec 2024 14:34:08 -0500 Subject: [PATCH 1/7] Add written documentation for the ctc error types with example graphs --- .gitignore | 1 + docs/source/conf.py | 1 + docs/source/index.rst | 1 + docs/source/track_errors/ctc.rst | 193 +++++++++++++++++++++++++++++++ pyproject.toml | 1 + 5 files changed, 197 insertions(+) create mode 100644 docs/source/track_errors/ctc.rst diff --git a/.gitignore b/.gitignore index ff95ce2b..e316cd1f 100644 --- a/.gitignore +++ b/.gitignore @@ -68,6 +68,7 @@ instance/ # Sphinx documentation docs/_build/ +docs/jupyter_execute # PyBuilder target/ diff --git a/docs/source/conf.py b/docs/source/conf.py index 42bf6a73..32cff540 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -53,6 +53,7 @@ "nbsphinx", # add notebooks to docs "nbsphinx_link", # add notebooks to docs "sphinx_click", # auto document cli + "jupyter_sphinx", # executable code blocks in rst ] napoleon_google_docstring = True diff --git a/docs/source/index.rst b/docs/source/index.rst index 7ee4f43a..bad280e1 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -25,6 +25,7 @@ track_errors/nodes track_errors/edges track_errors/divisions + track_errors/ctc .. toctree:: :maxdepth: 2 diff --git a/docs/source/track_errors/ctc.rst b/docs/source/track_errors/ctc.rst new file mode 100644 index 00000000..8a11f3ae --- /dev/null +++ b/docs/source/track_errors/ctc.rst @@ -0,0 +1,193 @@ +CTC Errors +========== + +These node and edge error annotations are used to calculate the CTC metrics TRA and DET as well as the basic AOGM metric as described in :doc:`../metrics/ctc`. + +.. jupyter-execute:: + :hide-code: + + import sys + + import matplotlib.pyplot as plt + import networkx as nx + import numpy as np + from matplotlib.colors import ListedColormap + from matplotlib.patches import Patch + + from traccuracy._tracking_graph import TrackingGraph + from traccuracy.matchers import Matched + + sys.path.append("../tests") + + import examples.graphs as ex_graphs + import examples.segs as ex_segs + + def get_loc(graph, node): + return graph.graph.nodes[node]["t"], graph.graph.nodes[node]["y"] + + + def plot_graph(ax, graph: TrackingGraph, color="black", annotations={}, ann_color="red"): + if graph.graph.number_of_nodes() == 0: + return [0, 0], [0, 0] + ids = list(graph.graph.nodes) + x = [graph.graph.nodes[node]["t"] for node in ids] + y = [graph.graph.nodes[node]["y"] for node in ids] + ax.scatter(x, y, color=color) + for _x, _y, _id in zip(x, y, ids): + ax.text(_x + 0.05, _y + 0.05, str(_id)) + # Plot annotation if available + ann = annotations.get(_id) + if ann: + ax.text(_x + 0.05, _y - 0.25, ann, color='purple') + + for u, v in graph.graph.edges(): + xs = [graph.graph.nodes[u]["t"], graph.graph.nodes[v]["t"]] + ys = [graph.graph.nodes[u]["y"], graph.graph.nodes[v]["y"]] + ax.plot(xs, ys, color=color) + # Plot edge annotation if available + ann = annotations.get((u, v)) + if ann: + xx = sum(xs) / 2 + yy = sum(ys) / 2 + ax.text(xx + 0.1, yy, ann, color='orange', horizontalalignment='center') + + return [max(x), min(x)], [max(y), min(y)] + + + def plot_matching(ax, matched, color="grey"): + for u, v in matched.mapping: + xs = [ + matched.gt_graph.graph.nodes[u]["t"], + matched.pred_graph.graph.nodes[v]["t"], + ] + ys = [ + matched.gt_graph.graph.nodes[u]["y"], + matched.pred_graph.graph.nodes[v]["y"], + ] + ax.plot(xs, ys, color=color, linestyle="dashed") + + + def plot_matched(examples, annotations, title): + gt_color = "black" + pred_color = "blue" + mapping_color = "grey" + fig, ax = plt.subplots(1, len(examples) + 1, figsize=(3 * len(examples) + 1, 2)) + for i, (matched, anns) in enumerate(zip(examples, annotations)): + axis = ax[i] + xbounds, ybounds = plot_graph(axis, matched.gt_graph, color=gt_color, annotations=anns) + bounds = plot_graph(axis, matched.pred_graph, color=pred_color, annotations=anns) + xbounds.extend(bounds[0]) + ybounds.extend(bounds[1]) + plot_matching(axis, matched, color=mapping_color) + axis.set_ybound(min(ybounds) - 0.5, max(ybounds) + 0.5) + axis.set_xbound(min(xbounds) - 0.5, max(xbounds) + 0.5) + axis.set_ylabel("Y Value") + axis.set_xlabel("Time") + + handles = [ + Patch(color=gt_color), + Patch(color=pred_color), + Patch(color=mapping_color), + Patch(color='orange'), + Patch(color='purple') + ] + labels = ["Ground Truth", "Prediction", "Mapping", "Edge Annotations", "Node Annotations"] + ax[-1].legend(handles=handles, labels=labels, loc="center") + ax[-1].set_axis_off() + plt.tight_layout() + fig.suptitle(title, y=1.05) + +Nodes +----- + +True Positives +^^^^^^^^^^^^^^ + +A true positive node is one that is matched to only one node in the predicted graph. Additionally, the predicted node is not matched to any other node in the ground truth. True positive nodes are annotated on both the ground truth and the predicted graph. + +False Positives +^^^^^^^^^^^^^^^ + +False positive nodes are annotated on the predicted graph and correspond to a predicted node without a match in the ground truth graph. + +False Negatives +^^^^^^^^^^^^^^^ + +False negative nodes are annotated on the ground truth graph and correspond to a ground truth node without a match in the predicted graph. + +Non-Split +^^^^^^^^^ + +Non-split nodes are annotated on the predicted graph and correspond to a node in the prediction that has been matched to two nodes in the ground truth graph. + +.. jupyter-execute:: + :hide-code: + + plot_matched([ex_graphs.two_to_one(t) for t in [0, 1]], [{4: "NS"}, {5: "NS"}], "Non-Split Nodes") + + +Edges +----- + +False Positives +^^^^^^^^^^^^^^^ + +False positive edges are annotated on the predicted graph. An edge is considered a false positive if both nodes are true positive nodes, but the edge does not match to any edge in the ground truth graph. In the example below, edge (4, 8) is a false positive. + +.. jupyter-execute:: + :hide-code: + + # plot_matched([ex_graphs.crossover_edge()], [{}], "") + +False Negatives +^^^^^^^^^^^^^^^ + +False negative edges are annotated on the ground truth graph. An edge is considered a false negative if: + +1. Either node is annotated as false negative nodes + +.. jupyter-execute:: + :hide-code: + + plot_matched([ex_graphs.fn_node_matched(0)], [{1: "FN", (1, 2): "FN"}], "") + +2. The corresponding edge in the predicted graph does not exist between two true positive nodes + +.. jupyter-execute:: + :hide-code: + + plot_matched([ex_graphs.fn_edge_matched(0)], [{(1, 2): "FN"}], "") + +3. Either node matches to a non-split node in the predicted graph + +.. jupyter-execute:: + :hide-code: + + plot_matched([ex_graphs.two_to_one(t) for t in [0, 1]], [{4: "NS", (1, 2): "FN"}, {5: "NS", (1, 7): "FN", (2, 3): "FN"}], "") + +Intertrack +^^^^^^^^^^ + +Intertrack edges connect parent cells to daughter cells. + +.. jupyter-execute:: + :hide-code: + + div_graph = ex_graphs.basic_division(1) + matched = Matched(div_graph, TrackingGraph(nx.DiGraph()), [], {}) + plot_matched([matched], [{(2, 3): "IT", (2, 4): "IT"}], "") + + +Wrong Semantic +^^^^^^^^^^^^^^ + +After identifying a matched pair of edges from the ground truth and predicted graphs, the predicted edge is annotated as wrong semantic if the ground truth and predicted edge have different intertrack edge annotations. + +.. jupyter-execute:: + :hide-code: + + plot_matched( + [ex_graphs.fp_div(1), ex_graphs.one_child(1)], + [{(6, 7): "WS"}, {(2, 3): "WS"}], + "" + ) diff --git a/pyproject.toml b/pyproject.toml index 314c9867..4bf17380 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,6 +71,7 @@ docs = [ "sphinx-click==5.0.1", "sphinx-rtd-theme==1.3.0", "Sphinx==7.2.6", + "jupyter-sphinx" ] [project.urls] From 2eb257233df94ab1e6dff7b16f230be54a27627a Mon Sep 17 00:00:00 2001 From: Morgan Schwartz Date: Thu, 19 Dec 2024 15:58:40 -0500 Subject: [PATCH 2/7] Add matplotlib as dependency for new plots in the docs --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 4bf17380..ee4e3e2f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,7 +71,8 @@ docs = [ "sphinx-click==5.0.1", "sphinx-rtd-theme==1.3.0", "Sphinx==7.2.6", - "jupyter-sphinx" + "jupyter-sphinx", + "matplotlib" ] [project.urls] From 218ec7a92c9aa29ee192a509dc1ac1fc838d5c6e Mon Sep 17 00:00:00 2001 From: Morgan Schwartz Date: Thu, 19 Dec 2024 16:02:14 -0500 Subject: [PATCH 3/7] Fix path to import of example graphs --- docs/source/track_errors/ctc.rst | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/docs/source/track_errors/ctc.rst b/docs/source/track_errors/ctc.rst index 8a11f3ae..949212d0 100644 --- a/docs/source/track_errors/ctc.rst +++ b/docs/source/track_errors/ctc.rst @@ -6,8 +6,6 @@ These node and edge error annotations are used to calculate the CTC metrics TRA .. jupyter-execute:: :hide-code: - import sys - import matplotlib.pyplot as plt import networkx as nx import numpy as np @@ -17,10 +15,8 @@ These node and edge error annotations are used to calculate the CTC metrics TRA from traccuracy._tracking_graph import TrackingGraph from traccuracy.matchers import Matched - sys.path.append("../tests") - - import examples.graphs as ex_graphs - import examples.segs as ex_segs + import tests.examples.graphs as ex_graphs + import tests.examples.segs as ex_segs def get_loc(graph, node): return graph.graph.nodes[node]["t"], graph.graph.nodes[node]["y"] From 568c37530dd66d436d6f0be577eee051ab989c83 Mon Sep 17 00:00:00 2001 From: Morgan Schwartz Date: Thu, 19 Dec 2024 16:13:55 -0500 Subject: [PATCH 4/7] Try different path configuration for rtd build --- docs/source/track_errors/ctc.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/source/track_errors/ctc.rst b/docs/source/track_errors/ctc.rst index 949212d0..a4a77a99 100644 --- a/docs/source/track_errors/ctc.rst +++ b/docs/source/track_errors/ctc.rst @@ -6,6 +6,9 @@ These node and edge error annotations are used to calculate the CTC metrics TRA .. jupyter-execute:: :hide-code: + import sys + sys.path.append('../../../tests') + import matplotlib.pyplot as plt import networkx as nx import numpy as np @@ -15,8 +18,8 @@ These node and edge error annotations are used to calculate the CTC metrics TRA from traccuracy._tracking_graph import TrackingGraph from traccuracy.matchers import Matched - import tests.examples.graphs as ex_graphs - import tests.examples.segs as ex_segs + import examples.graphs as ex_graphs + import examples.segs as ex_segs def get_loc(graph, node): return graph.graph.nodes[node]["t"], graph.graph.nodes[node]["y"] From 74db891baae407f5fbde800363a2d249619ebb80 Mon Sep 17 00:00:00 2001 From: Morgan Schwartz Date: Thu, 19 Dec 2024 16:27:22 -0500 Subject: [PATCH 5/7] Try path with two ../ --- docs/source/track_errors/ctc.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/track_errors/ctc.rst b/docs/source/track_errors/ctc.rst index a4a77a99..de5e2cd0 100644 --- a/docs/source/track_errors/ctc.rst +++ b/docs/source/track_errors/ctc.rst @@ -7,7 +7,7 @@ These node and edge error annotations are used to calculate the CTC metrics TRA :hide-code: import sys - sys.path.append('../../../tests') + sys.path.append('../../tests') import matplotlib.pyplot as plt import networkx as nx @@ -18,8 +18,8 @@ These node and edge error annotations are used to calculate the CTC metrics TRA from traccuracy._tracking_graph import TrackingGraph from traccuracy.matchers import Matched - import examples.graphs as ex_graphs - import examples.segs as ex_segs + import tests.examples.graphs as ex_graphs + import tests.examples.segs as ex_segs def get_loc(graph, node): return graph.graph.nodes[node]["t"], graph.graph.nodes[node]["y"] From 939cbe8cf4537a3ae0d0be7f96a0bb1ab44fdb81 Mon Sep 17 00:00:00 2001 From: Morgan Schwartz Date: Thu, 19 Dec 2024 16:30:09 -0500 Subject: [PATCH 6/7] Correct imports with two ../ --- docs/source/track_errors/ctc.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/track_errors/ctc.rst b/docs/source/track_errors/ctc.rst index de5e2cd0..0cfcf751 100644 --- a/docs/source/track_errors/ctc.rst +++ b/docs/source/track_errors/ctc.rst @@ -18,8 +18,8 @@ These node and edge error annotations are used to calculate the CTC metrics TRA from traccuracy._tracking_graph import TrackingGraph from traccuracy.matchers import Matched - import tests.examples.graphs as ex_graphs - import tests.examples.segs as ex_segs + import examples.graphs as ex_graphs + import examples.segs as ex_segs def get_loc(graph, node): return graph.graph.nodes[node]["t"], graph.graph.nodes[node]["y"] From c056d98cabe423e0201ac9ea91f3ed3b3456db4f Mon Sep 17 00:00:00 2001 From: Morgan Schwartz Date: Mon, 6 Jan 2025 11:12:29 -0500 Subject: [PATCH 7/7] Draga's edits from code review Co-authored-by: Draga Doncila Pop <17995243+DragaDoncila@users.noreply.github.com> --- docs/source/track_errors/ctc.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/track_errors/ctc.rst b/docs/source/track_errors/ctc.rst index 0cfcf751..5c2e306f 100644 --- a/docs/source/track_errors/ctc.rst +++ b/docs/source/track_errors/ctc.rst @@ -117,7 +117,7 @@ False negative nodes are annotated on the ground truth graph and correspond to a Non-Split ^^^^^^^^^ -Non-split nodes are annotated on the predicted graph and correspond to a node in the prediction that has been matched to two nodes in the ground truth graph. +Non-split nodes are annotated on the predicted graph and correspond to a node in the prediction that has been matched to more than one node in the ground truth graph. .. jupyter-execute:: :hide-code: @@ -143,7 +143,7 @@ False Negatives False negative edges are annotated on the ground truth graph. An edge is considered a false negative if: -1. Either node is annotated as false negative nodes +1. Either node is annotated as a false negative node .. jupyter-execute:: :hide-code: @@ -167,7 +167,7 @@ False negative edges are annotated on the ground truth graph. An edge is conside Intertrack ^^^^^^^^^^ -Intertrack edges connect parent cells to daughter cells. +Intertrack edges connect two nodes with different track IDs, most commonly parent cells to daughter cells, or two cells in non-consecutive frames. Note that intertrack edges are not errors, but support the annotation of Wrong Semantic edges, as detailed below. .. jupyter-execute:: :hide-code: