From 5ab879df377083852c979b555c5ab9543cd0ddb6 Mon Sep 17 00:00:00 2001 From: Shion Date: Sat, 30 Sep 2023 18:42:10 +0900 Subject: [PATCH 01/29] add docs --- docs/source/links.rst | 1 + .../segmentation/hausdorff_distance.rst | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 docs/source/segmentation/hausdorff_distance.rst diff --git a/docs/source/links.rst b/docs/source/links.rst index 0b0de53f325..2cc3dae5746 100644 --- a/docs/source/links.rst +++ b/docs/source/links.rst @@ -171,3 +171,4 @@ .. _averaging curve objects: https://scikit-learn.org/stable/auto_examples/model_selection/plot_roc.html .. _SCC: https://www.ingentaconnect.com/content/tandf/tres/1998/00000019/00000004/art00013 .. _Generalized Dice Score: https://arxiv.org/abs/1707.03237 +.. _Hausdorff Distance: https://en.wikipedia.org/wiki/Hausdorff_distance diff --git a/docs/source/segmentation/hausdorff_distance.rst b/docs/source/segmentation/hausdorff_distance.rst new file mode 100644 index 00000000000..182e547c4b3 --- /dev/null +++ b/docs/source/segmentation/hausdorff_distance.rst @@ -0,0 +1,21 @@ +.. customcarditem:: + :header: Hausdorff Distance + :image: https://pl-flash-data.s3.amazonaws.com/assets/thumbnails/text_classification.svg + :tags: Retrieval + +.. include:: ../links.rst + +################## +Hausdorff Distance +################## + +Module Interface +________________ + +.. autoclass:: torchmetrics.segmentation.HausdorffDistance + :exclude-members: update, compute + +Functional Interface +____________________ + +.. autofunction:: torchmetrics.functional.segmentation.hausdorff_distance From b6519c35bb92ad6b0ae41236fdc1f2da94b81d44 Mon Sep 17 00:00:00 2001 From: Shion Date: Sun, 1 Oct 2023 13:58:48 +0900 Subject: [PATCH 02/29] initial commit --- .../functional/segmentation/__init__.py | 3 +- .../segmentation/hausdorff_distance.py | 113 +++++++++++++++++ .../functional/segmentation/utils.py | 19 +-- src/torchmetrics/segmentation/__init__.py | 3 +- .../segmentation/hausdorff_distance.py | 115 ++++++++++++++++++ tests/unittests/segmentation/inputs.py | 47 +++++++ .../segmentation/test_hausdorff_distance.py | 71 +++++++++++ 7 files changed, 360 insertions(+), 11 deletions(-) create mode 100644 src/torchmetrics/functional/segmentation/hausdorff_distance.py create mode 100644 src/torchmetrics/segmentation/hausdorff_distance.py create mode 100644 tests/unittests/segmentation/inputs.py create mode 100644 tests/unittests/segmentation/test_hausdorff_distance.py diff --git a/src/torchmetrics/functional/segmentation/__init__.py b/src/torchmetrics/functional/segmentation/__init__.py index 3d23192a36a..e0887da3942 100644 --- a/src/torchmetrics/functional/segmentation/__init__.py +++ b/src/torchmetrics/functional/segmentation/__init__.py @@ -13,5 +13,6 @@ # limitations under the License. from torchmetrics.functional.segmentation.generalized_dice import generalized_dice_score from torchmetrics.functional.segmentation.mean_iou import mean_iou +from torchmetrics.functional.segmentation.hausdorff_distance import hausdorff_distance -__all__ = ["generalized_dice_score", "mean_iou"] +__all__ = ["generalized_dice_score", "mean_iou", "hausdorff_distance"] diff --git a/src/torchmetrics/functional/segmentation/hausdorff_distance.py b/src/torchmetrics/functional/segmentation/hausdorff_distance.py new file mode 100644 index 00000000000..158daf1b806 --- /dev/null +++ b/src/torchmetrics/functional/segmentation/hausdorff_distance.py @@ -0,0 +1,113 @@ +# Copyright The Lightning team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Literal, Optional, Tuple, Union + +import torch +from torch import Tensor + +from torchmetrics.functional.segmentation.utils import _check_if_binarized, surface_distance +from torchmetrics.utilities.checks import _check_same_shape + + +def _hausdorff_distance_update(preds: Tensor, target: Tensor) -> Tuple[Tensor, Tensor]: + """Update and returns variables required to compute `Hausdorff Distance`_. + + Args: + preds: predicted binarized segmentation map + target: target binarized segmentation map + + Returns: + preds: predicted binarized segmentation map + target: target binarized segmentation map + + """ + _check_if_binarized(preds) + _check_if_binarized(target) + _check_same_shape(preds, target) + return preds, target + + +def _hausdorff_distance_compute( + preds: Tensor, + target: Tensor, + distance_metric: Literal["euclidean", "chessboard", "taxicab"] = "euclidean", + spacing: Optional[Union[Tensor, list[float]]] = None, +) -> Tensor: + """Compute `Hausdorff Distance`_. + + Args: + preds: predicted binarized segmentation map + target: target binarized segmentation map + distance_metric: distance metric to calculate surface distance. One of `["euclidean", "chessboard", "taxicab"]`. + spacing: spacing between pixels along each spatial dimension + + Returns: + Hausdorff distance + + Example: + >>> preds = torch.tensor([[1, 1, 1, 1, 1], + ... [1, 0, 0, 0, 1], + ... [1, 0, 0, 0, 1], + ... [1, 0, 0, 0, 1], + ... [1, 1, 1, 1, 1]], dtype=torch.bool) + >>> target = torch.tensor([[1, 1, 1, 1, 0], + ... [1, 0, 0, 1, 0], + ... [1, 0, 0, 1, 0], + ... [1, 0, 0, 1, 0], + ... [1, 1, 1, 1, 0]], dtype=torch.bool) + >>> hausdorff_distance(preds, target, distance_metric="euclidean") + tensor(1.0) + + """ + fwd = surface_distance(preds, target, distance_metric=distance_metric, spacing=spacing) + bwd = surface_distance(target, preds, distance_metric=distance_metric, spacing=spacing) + return torch.max(torch.tensor([fwd.max(), bwd.max()])) + + +def hausdorff_distance( + preds: Tensor, + target: Tensor, + distance_metric: Literal["euclidean", "chessboard", "taxicab"] = "euclidean", + spacing: Optional[Union[Tensor, list[float]]] = None, +) -> Tensor: + """Calculate `Hausdorff Distance`_. + + Args: + preds: predicted binarized segmentation map + target: target binarized segmentation map + distance_metric: distance metric to calculate surface distance. One of `["euclidean", "chessboard", "taxicab"]`. + spacing: spacing between pixels along each spatial dimension + + Returns: + Hausdorff Distance + + Example: + >>> from torchmetrics.functional.segmentation import hausdorff_distance + >>> preds = torch.tensor([[1, 1, 1, 1, 1], + ... [1, 0, 0, 0, 1], + ... [1, 0, 0, 0, 1], + ... [1, 0, 0, 0, 1], + ... [1, 1, 1, 1, 1]], dtype=torch.bool) + >>> target = torch.tensor([[1, 1, 1, 1, 0], + ... [1, 0, 0, 1, 0], + ... [1, 0, 0, 1, 0], + ... [1, 0, 0, 1, 0], + ... [1, 1, 1, 1, 0]], dtype=torch.bool) + >>> hausdorff_distance(preds, target, distance_metric="euclidean") + tensor(1.0) + + """ + preds, target = _hausdorff_distance_update(preds, target) + return _hausdorff_distance_compute(preds, target, distance_metric=distance_metric, spacing=spacing) diff --git a/src/torchmetrics/functional/segmentation/utils.py b/src/torchmetrics/functional/segmentation/utils.py index 6c2fed92df2..ad51968da75 100644 --- a/src/torchmetrics/functional/segmentation/utils.py +++ b/src/torchmetrics/functional/segmentation/utils.py @@ -24,6 +24,7 @@ from torchmetrics.utilities.imports import _SCIPY_AVAILABLE +def _check_if_binarized(x: Tensor) -> None: def _ignore_background(preds: Tensor, target: Tensor) -> Tuple[Tensor, Tensor]: """Ignore the background class in the computation assuming it is the first, index 0.""" preds = preds[:, 1:] if preds.shape[1] > 1 else preds @@ -249,25 +250,25 @@ def distance_transform( raise ValueError(f"Expected argument `sampling` to have length 2 but got length `{len(sampling)}`.") if engine == "pytorch": + x = x.float() # calculate distance from every foreground pixel to every background pixel i0, j0 = torch.where(x == 0) i1, j1 = torch.where(x == 1) - dis_row = (i1.unsqueeze(1) - i0.unsqueeze(0)).abs_().mul_(sampling[0]) - dis_col = (j1.unsqueeze(1) - j0.unsqueeze(0)).abs_().mul_(sampling[1]) + dis_row = (i1.view(-1, 1) - i0.view(1, -1)).abs() + dis_col = (j1.view(-1, 1) - j0.view(1, -1)).abs() # # calculate distance h, _ = x.shape if metric == "euclidean": - dis_row = dis_row.float() - dis_row.pow_(2).add_(dis_col.pow_(2)).sqrt_() + dis = ((sampling[0] * dis_row) ** 2 + (sampling[1] * dis_col) ** 2).sqrt() if metric == "chessboard": - dis_row = dis_row.max(dis_col) + dis = torch.max(sampling[0] * dis_row, sampling[1] * dis_col).float() if metric == "taxicab": - dis_row.add_(dis_col) + dis = (sampling[0] * dis_row + sampling[1] * dis_col).float() # select only the closest distance - mindis, _ = torch.min(dis_row, dim=1) - z = torch.zeros_like(x, dtype=mindis.dtype).view(-1) + mindis, _ = torch.min(dis, dim=1) + z = torch.zeros_like(x).view(-1) z[i1 * h + j1] = mindis return z.view(x.shape) @@ -279,7 +280,7 @@ def distance_transform( if metric == "euclidean": return ndimage.distance_transform_edt(x.cpu().numpy(), sampling) - return ndimage.distance_transform_cdt(x.cpu().numpy(), metric=metric) + return ndimage.distance_transform_cdt(x.cpu().numpy(), sampling, metric=metric) def mask_edges( diff --git a/src/torchmetrics/segmentation/__init__.py b/src/torchmetrics/segmentation/__init__.py index 5b609c2c738..4c6f8bce528 100644 --- a/src/torchmetrics/segmentation/__init__.py +++ b/src/torchmetrics/segmentation/__init__.py @@ -13,5 +13,6 @@ # limitations under the License. from torchmetrics.segmentation.generalized_dice import GeneralizedDiceScore from torchmetrics.segmentation.mean_iou import MeanIoU +from torchmetrics.segmentation.hausdorff_distance import HausdorffDistance -__all__ = ["GeneralizedDiceScore", "MeanIoU"] +__all__ = ["GeneralizedDiceScore", "MeanIoU", "HausdorffDistance"] diff --git a/src/torchmetrics/segmentation/hausdorff_distance.py b/src/torchmetrics/segmentation/hausdorff_distance.py new file mode 100644 index 00000000000..ad8f0b51160 --- /dev/null +++ b/src/torchmetrics/segmentation/hausdorff_distance.py @@ -0,0 +1,115 @@ +# Copyright The Lightning team. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Any, Optional, Sequence, Union + +from torch import Tensor + +from torchmetrics.functional.segmentation.hausdorff_distance import hausdorff_distance +from torchmetrics.metric import Metric +from torchmetrics.utilities.imports import _MATPLOTLIB_AVAILABLE +from torchmetrics.utilities.plot import _AX_TYPE, _PLOT_OUT_TYPE + +if not _MATPLOTLIB_AVAILABLE: + __doctest_skip__ = ["HausdorffDistance.plot"] + + +class HausdorffDistance(Metric): + r"""Compute the Hausdorff distance between two subsets of a metric space. + + .. math:: + d_{\Pi}(X,Y) = \max{/sup_{x\in X} {d(x,Y)}, /sup_{y\in Y} {d(X,y)}} + + where :math:`\X, \Y` are ________________, :math:`\X, \Y` ______. + + As input to ``forward`` and ``update`` the metric accepts the following input: + + - ``preds`` (:class:`~torch.Tensor`): + - ``target`` (:class:`~torch.Tensor`): + + As output of ``forward`` and ``compute`` the metric returns the following output: + + - ``hausdorff_distance`` (:class:`~torch.Tensor`): A scalar float tensor with the Hausdorff distance. + + Args: + p: p-norm used for distance metric + kwargs: Additional keyword arguments, see :ref:`Metric kwargs` for more info. + + Example: + >>> preds = torch.tensor([[1, 1, 1, 1, 1], + ... [1, 0, 0, 0, 1], + ... [1, 0, 0, 0, 1], + ... [1, 0, 0, 0, 1], + ... [1, 1, 1, 1, 1]], dtype=torch.bool) + >>> target = torch.tensor([[1, 1, 1, 1, 0], + ... [1, 0, 0, 1, 0], + ... [1, 0, 0, 1, 0], + ... [1, 0, 0, 1, 0], + ... [1, 1, 1, 1, 0]], dtype=torch.bool) + >>> hausdorff_distance = HausdorffDistance(p=2) + >>> hausdorff_distance.update(preds, target) + >>> hausdorff_distance.compute() + tensor(1.0) + + """ + is_differentiable: bool = True + higher_is_better: bool = True + full_state_update: bool = True + plot_lower_bound: float = 0.0 + plot_upper_bound: float = 1.0 + preds: list[Tensor] + target: list[Tensor] + + def __init__(self, p: float = 2, **kwargs: Any) -> None: + super().__init__(**kwargs) + self.p = p + + self.add_state("preds", default=[], dist_reduce_fx="cat") + self.add_state("target", default=[], dist_reduce_fx="cat") + + def update(self, preds: Tensor, target: Tensor) -> None: + """Update state with predictions and targets.""" + self.preds.append(preds) + self.target.append(target) + + def compute(self) -> Tensor: + """Compute final Hausdorff distance over states.""" + return hausdorff_distance(self.preds, self.target, self.p) + + def plot( + self, val: Optional[Union[Tensor, Sequence[Tensor]]] = None, ax: Optional[_AX_TYPE] = None + ) -> _PLOT_OUT_TYPE: + """Plot a single or multiple values from the metric. + + Args: + val: Either a single result from calling `metric.forward` or `metric.compute` or a list of these results. + If no value is provided, will automatically call `metric.compute` and plot that result. + ax: An matplotlib axis object. If provided will add plot to that axis + + Returns: + Figure and Axes object + + Raises: + ModuleNotFoundError: + If `matplotlib` is not installed + + .. plot:: + :scale: 75 + + >>> from torch import randn + >>> from torchmetrics.regression import HausdorffDistance + >>> metric = HausdorffDistance() + >>> metric.update(randn(10,), randn(10,)) + >>> fig_, ax_ = metric.plot() + + """ + return self._plot(val, ax) diff --git a/tests/unittests/segmentation/inputs.py b/tests/unittests/segmentation/inputs.py new file mode 100644 index 00000000000..a276876a06d --- /dev/null +++ b/tests/unittests/segmentation/inputs.py @@ -0,0 +1,47 @@ +# Copyright The Lightning team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from collections import namedtuple + +import torch + +from unittests.helpers import seed_all + +seed_all(42) + + +# extrinsic input for clustering metrics that requires predicted clustering labels and target clustering labels +Input = namedtuple("Input", ["preds", "target"]) + + +preds = torch.tensor( + [ + [[1, 1, 1, 1, 1], [1, 0, 0, 0, 1], [1, 0, 0, 0, 1], [1, 0, 0, 0, 1], [1, 1, 1, 1, 1]], + [[1, 1, 1, 1, 1], [1, 0, 0, 0, 1], [1, 0, 0, 0, 1], [1, 0, 0, 0, 1], [1, 1, 1, 1, 1]], + [[1, 1, 1, 1, 1], [1, 0, 0, 0, 1], [1, 0, 0, 0, 1], [1, 0, 0, 0, 1], [1, 1, 1, 1, 1]], + [[1, 1, 1, 1, 1], [1, 0, 0, 0, 1], [1, 0, 0, 0, 1], [1, 0, 0, 0, 1], [1, 1, 1, 1, 1]], + ], + dtype=torch.bool, +) + +target = torch.tensor( + [ + [[1, 1, 1, 1, 0], [1, 0, 0, 1, 0], [1, 0, 0, 1, 0], [1, 0, 0, 1, 0], [1, 1, 1, 1, 0]], + [[1, 1, 1, 1, 0], [1, 0, 0, 1, 0], [1, 0, 0, 1, 0], [1, 0, 0, 1, 0], [1, 1, 1, 1, 0]], + [[1, 1, 1, 1, 0], [1, 0, 0, 1, 0], [1, 0, 0, 1, 0], [1, 0, 0, 1, 0], [1, 1, 1, 1, 0]], + [[1, 1, 1, 1, 0], [1, 0, 0, 1, 0], [1, 0, 0, 1, 0], [1, 0, 0, 1, 0], [1, 1, 1, 1, 0]], + ], + dtype=torch.bool, +) + +_inputs = Input(preds=preds, target=target) diff --git a/tests/unittests/segmentation/test_hausdorff_distance.py b/tests/unittests/segmentation/test_hausdorff_distance.py new file mode 100644 index 00000000000..03eb6b20517 --- /dev/null +++ b/tests/unittests/segmentation/test_hausdorff_distance.py @@ -0,0 +1,71 @@ +# Copyright The Lightning team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from functools import partial + +import pytest +from skimage.metrics import hausdorff_distance as skimage_hausdorff_distance +from torchmetrics.functional.segmentation.hausdorff_distance import hausdorff_distance +from torchmetrics.segmentation.hausdorff_distance import HausdorffDistance + +from unittests.helpers import seed_all +from unittests.helpers.testers import MetricTester +from unittests.segmentation.inputs import _inputs + +seed_all(42) + + +@pytest.mark.parametrize( + "preds, target", + [ + (_inputs.preds, _inputs.target), + ], +) +@pytest.mark.parametrize( + "distance_metric", + ["euclidean", "chessboard", "taxicab"], +) +class TestHausdorffDistance(MetricTester): + """Test class for `HausdorffDistance` metric.""" + + atol = 1e-5 + + @pytest.mark.parametrize("ddp", [True, False]) + def test_hausdorff_distance(self, preds, target, distance_metric, ddp): + """Test class implementation of metric.""" + self.run_class_metric_test( + ddp=ddp, + preds=preds, + target=target, + metric_class=HausdorffDistance, + reference_metric=partial(skimage_hausdorff_distance, method="standard"), + metric_args={"distance_metric": distance_metric, "spacing": None}, + ) + + def test_hausdorff_distance_functional(self, preds, target, distance_metric): + """Test functional implementation of metric.""" + self.run_functional_metric_test( + preds=preds, + target=target, + metric_functional=hausdorff_distance, + reference_metric=partial(skimage_hausdorff_distance, method="standard"), + distance_metric=distance_metric, + spacing=None, + ) + + +def test_hausdorff_distance_functional_raises_invalid_task(): + """Check that metric rejects continuous-valued inputs.""" + preds, target = _inputs + with pytest.raises(ValueError, match=r"Expected *"): + hausdorff_distance(preds, target) From 4f5d6068bdab7eb7dd166a52620903a152c992b2 Mon Sep 17 00:00:00 2001 From: Shion Date: Wed, 4 Oct 2023 20:58:35 +0900 Subject: [PATCH 03/29] fix hausdorff metric args --- .../segmentation/hausdorff_distance.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/torchmetrics/segmentation/hausdorff_distance.py b/src/torchmetrics/segmentation/hausdorff_distance.py index ad8f0b51160..8b52d4bcb8c 100644 --- a/src/torchmetrics/segmentation/hausdorff_distance.py +++ b/src/torchmetrics/segmentation/hausdorff_distance.py @@ -10,11 +10,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Optional, Sequence, Union +from typing import Any, Literal, Optional, Sequence, Union from torch import Tensor -from torchmetrics.functional.segmentation.hausdorff_distance import hausdorff_distance +from torchmetrics.functional.segmentation import hausdorff_distance from torchmetrics.metric import Metric from torchmetrics.utilities.imports import _MATPLOTLIB_AVAILABLE from torchmetrics.utilities.plot import _AX_TYPE, _PLOT_OUT_TYPE @@ -55,7 +55,7 @@ class HausdorffDistance(Metric): ... [1, 0, 0, 1, 0], ... [1, 0, 0, 1, 0], ... [1, 1, 1, 1, 0]], dtype=torch.bool) - >>> hausdorff_distance = HausdorffDistance(p=2) + >>> hausdorff_distance = HausdorffDistance(distance_metric="euclidean") >>> hausdorff_distance.update(preds, target) >>> hausdorff_distance.compute() tensor(1.0) @@ -69,9 +69,15 @@ class HausdorffDistance(Metric): preds: list[Tensor] target: list[Tensor] - def __init__(self, p: float = 2, **kwargs: Any) -> None: + def __init__( + self, + distance_metric: Literal["euclidean", "chessboard", "taxicab"] = "euclidean", + spacing: Optional[Union[Tensor, list[float]]] = None, + **kwargs: Any + ) -> None: super().__init__(**kwargs) - self.p = p + self.distance_metric = distance_metric + self.spacing = spacing self.add_state("preds", default=[], dist_reduce_fx="cat") self.add_state("target", default=[], dist_reduce_fx="cat") @@ -83,7 +89,7 @@ def update(self, preds: Tensor, target: Tensor) -> None: def compute(self) -> Tensor: """Compute final Hausdorff distance over states.""" - return hausdorff_distance(self.preds, self.target, self.p) + return hausdorff_distance(self.preds, self.target, self.distance_metric, self.spacing) def plot( self, val: Optional[Union[Tensor, Sequence[Tensor]]] = None, ax: Optional[_AX_TYPE] = None From 80cbb1abbac26206ea2c97cf4f67197e3751e2f4 Mon Sep 17 00:00:00 2001 From: Shion Date: Sat, 14 Oct 2023 15:41:32 +0900 Subject: [PATCH 04/29] ci: switch to custom docker images (#2123) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- requirements/integrate.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 requirements/integrate.txt diff --git a/requirements/integrate.txt b/requirements/integrate.txt new file mode 100644 index 00000000000..e69de29bb2d From 05c154aa1f8038a4890b63f7756d684612d62dd7 Mon Sep 17 00:00:00 2001 From: Shion Date: Sat, 14 Oct 2023 15:43:49 +0900 Subject: [PATCH 05/29] Add `average` to curve metrics (#2084) Co-authored-by: Jirka Borovec <6035284+Borda@users.noreply.github.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- docs/source/links.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/links.rst b/docs/source/links.rst index 2cc3dae5746..2bbafae88e5 100644 --- a/docs/source/links.rst +++ b/docs/source/links.rst @@ -172,3 +172,4 @@ .. _SCC: https://www.ingentaconnect.com/content/tandf/tres/1998/00000019/00000004/art00013 .. _Generalized Dice Score: https://arxiv.org/abs/1707.03237 .. _Hausdorff Distance: https://en.wikipedia.org/wiki/Hausdorff_distance +.. _averaging curve objects: https://scikit-learn.org/stable/auto_examples/model_selection/plot_roc.html From ea58776bc5385ecc4cbdd18f9fe6c197e02138f4 Mon Sep 17 00:00:00 2001 From: Shion Date: Sat, 14 Oct 2023 16:47:45 +0900 Subject: [PATCH 06/29] symmetric test --- .../segmentation/hausdorff_distance.py | 5 +++- tests/unittests/segmentation/inputs.py | 1 - .../segmentation/test_hausdorff_distance.py | 23 ++++++++++++++----- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/src/torchmetrics/segmentation/hausdorff_distance.py b/src/torchmetrics/segmentation/hausdorff_distance.py index 8b52d4bcb8c..d38b5c3b35f 100644 --- a/src/torchmetrics/segmentation/hausdorff_distance.py +++ b/src/torchmetrics/segmentation/hausdorff_distance.py @@ -16,6 +16,7 @@ from torchmetrics.functional.segmentation import hausdorff_distance from torchmetrics.metric import Metric +from torchmetrics.utilities.data import dim_zero_cat from torchmetrics.utilities.imports import _MATPLOTLIB_AVAILABLE from torchmetrics.utilities.plot import _AX_TYPE, _PLOT_OUT_TYPE @@ -89,7 +90,9 @@ def update(self, preds: Tensor, target: Tensor) -> None: def compute(self) -> Tensor: """Compute final Hausdorff distance over states.""" - return hausdorff_distance(self.preds, self.target, self.distance_metric, self.spacing) + return hausdorff_distance( + dim_zero_cat(self.preds), dim_zero_cat(self.target), self.distance_metric, self.spacing + ) def plot( self, val: Optional[Union[Tensor, Sequence[Tensor]]] = None, ax: Optional[_AX_TYPE] = None diff --git a/tests/unittests/segmentation/inputs.py b/tests/unittests/segmentation/inputs.py index a276876a06d..80d0a240d29 100644 --- a/tests/unittests/segmentation/inputs.py +++ b/tests/unittests/segmentation/inputs.py @@ -23,7 +23,6 @@ # extrinsic input for clustering metrics that requires predicted clustering labels and target clustering labels Input = namedtuple("Input", ["preds", "target"]) - preds = torch.tensor( [ [[1, 1, 1, 1, 1], [1, 0, 0, 0, 1], [1, 0, 0, 0, 1], [1, 0, 0, 0, 1], [1, 1, 1, 1, 1]], diff --git a/tests/unittests/segmentation/test_hausdorff_distance.py b/tests/unittests/segmentation/test_hausdorff_distance.py index 03eb6b20517..ee390a95c57 100644 --- a/tests/unittests/segmentation/test_hausdorff_distance.py +++ b/tests/unittests/segmentation/test_hausdorff_distance.py @@ -11,9 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from functools import partial - import pytest +import torch from skimage.metrics import hausdorff_distance as skimage_hausdorff_distance from torchmetrics.functional.segmentation.hausdorff_distance import hausdorff_distance from torchmetrics.segmentation.hausdorff_distance import HausdorffDistance @@ -48,7 +47,7 @@ def test_hausdorff_distance(self, preds, target, distance_metric, ddp): preds=preds, target=target, metric_class=HausdorffDistance, - reference_metric=partial(skimage_hausdorff_distance, method="standard"), + reference_metric=skimage_hausdorff_distance, metric_args={"distance_metric": distance_metric, "spacing": None}, ) @@ -58,9 +57,8 @@ def test_hausdorff_distance_functional(self, preds, target, distance_metric): preds=preds, target=target, metric_functional=hausdorff_distance, - reference_metric=partial(skimage_hausdorff_distance, method="standard"), - distance_metric=distance_metric, - spacing=None, + reference_metric=skimage_hausdorff_distance, + metric_args={"distance_metric": distance_metric, "spacing": None}, ) @@ -69,3 +67,16 @@ def test_hausdorff_distance_functional_raises_invalid_task(): preds, target = _inputs with pytest.raises(ValueError, match=r"Expected *"): hausdorff_distance(preds, target) + + +@pytest.mark.parametrize( + "distance_metric", + ["euclidean", "chessboard", "taxicab"], +) +def test_hausdorff_distance_is_symmetric(distance_metric): + """Check that the metric functional is symmetric.""" + for p, t in zip(_inputs.preds, _inputs.target): + assert torch.allclose( + hausdorff_distance(p, t, distance_metric), + hausdorff_distance(t, p, distance_metric), + ) From efc972fd351ef0401b04577056939b71a3b774bf Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 21 May 2024 09:21:38 +0000 Subject: [PATCH 07/29] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/torchmetrics/functional/segmentation/__init__.py | 2 +- src/torchmetrics/segmentation/__init__.py | 2 +- src/torchmetrics/segmentation/hausdorff_distance.py | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/torchmetrics/functional/segmentation/__init__.py b/src/torchmetrics/functional/segmentation/__init__.py index e0887da3942..068bf77d775 100644 --- a/src/torchmetrics/functional/segmentation/__init__.py +++ b/src/torchmetrics/functional/segmentation/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. from torchmetrics.functional.segmentation.generalized_dice import generalized_dice_score -from torchmetrics.functional.segmentation.mean_iou import mean_iou from torchmetrics.functional.segmentation.hausdorff_distance import hausdorff_distance +from torchmetrics.functional.segmentation.mean_iou import mean_iou __all__ = ["generalized_dice_score", "mean_iou", "hausdorff_distance"] diff --git a/src/torchmetrics/segmentation/__init__.py b/src/torchmetrics/segmentation/__init__.py index 4c6f8bce528..6e9b1c63313 100644 --- a/src/torchmetrics/segmentation/__init__.py +++ b/src/torchmetrics/segmentation/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. from torchmetrics.segmentation.generalized_dice import GeneralizedDiceScore -from torchmetrics.segmentation.mean_iou import MeanIoU from torchmetrics.segmentation.hausdorff_distance import HausdorffDistance +from torchmetrics.segmentation.mean_iou import MeanIoU __all__ = ["GeneralizedDiceScore", "MeanIoU", "HausdorffDistance"] diff --git a/src/torchmetrics/segmentation/hausdorff_distance.py b/src/torchmetrics/segmentation/hausdorff_distance.py index d38b5c3b35f..11919e21ae6 100644 --- a/src/torchmetrics/segmentation/hausdorff_distance.py +++ b/src/torchmetrics/segmentation/hausdorff_distance.py @@ -62,6 +62,7 @@ class HausdorffDistance(Metric): tensor(1.0) """ + is_differentiable: bool = True higher_is_better: bool = True full_state_update: bool = True @@ -74,7 +75,7 @@ def __init__( self, distance_metric: Literal["euclidean", "chessboard", "taxicab"] = "euclidean", spacing: Optional[Union[Tensor, list[float]]] = None, - **kwargs: Any + **kwargs: Any, ) -> None: super().__init__(**kwargs) self.distance_metric = distance_metric From 0c9b1a8d10ab14cff3bac457774a10f7bdac0376 Mon Sep 17 00:00:00 2001 From: Shion Date: Mon, 27 May 2024 20:52:56 +0900 Subject: [PATCH 08/29] fix merge error --- src/torchmetrics/functional/segmentation/utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/torchmetrics/functional/segmentation/utils.py b/src/torchmetrics/functional/segmentation/utils.py index ad51968da75..e1c80831a7a 100644 --- a/src/torchmetrics/functional/segmentation/utils.py +++ b/src/torchmetrics/functional/segmentation/utils.py @@ -24,7 +24,6 @@ from torchmetrics.utilities.imports import _SCIPY_AVAILABLE -def _check_if_binarized(x: Tensor) -> None: def _ignore_background(preds: Tensor, target: Tensor) -> Tuple[Tensor, Tensor]: """Ignore the background class in the computation assuming it is the first, index 0.""" preds = preds[:, 1:] if preds.shape[1] > 1 else preds @@ -33,7 +32,7 @@ def _ignore_background(preds: Tensor, target: Tensor) -> Tuple[Tensor, Tensor]: def check_if_binarized(x: Tensor) -> None: - """Check if the input is binarized. + """Check if tensor is binarized. Example: >>> from torchmetrics.functional.segmentation.utils import check_if_binarized From a3dcc86048d43ae3092d3a5d9ca3fdc74880f9f9 Mon Sep 17 00:00:00 2001 From: Shion Date: Mon, 27 May 2024 20:54:17 +0900 Subject: [PATCH 09/29] fix imports --- .../functional/segmentation/hausdorff_distance.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/torchmetrics/functional/segmentation/hausdorff_distance.py b/src/torchmetrics/functional/segmentation/hausdorff_distance.py index 158daf1b806..9b342dbf78f 100644 --- a/src/torchmetrics/functional/segmentation/hausdorff_distance.py +++ b/src/torchmetrics/functional/segmentation/hausdorff_distance.py @@ -17,7 +17,7 @@ import torch from torch import Tensor -from torchmetrics.functional.segmentation.utils import _check_if_binarized, surface_distance +from torchmetrics.functional.segmentation.utils import check_if_binarized, surface_distance from torchmetrics.utilities.checks import _check_same_shape @@ -33,8 +33,8 @@ def _hausdorff_distance_update(preds: Tensor, target: Tensor) -> Tuple[Tensor, T target: target binarized segmentation map """ - _check_if_binarized(preds) - _check_if_binarized(target) + check_if_binarized(preds) + check_if_binarized(target) _check_same_shape(preds, target) return preds, target From bfe0a3be89ea9df4ccabeade011aa0e96544b582 Mon Sep 17 00:00:00 2001 From: Shion Date: Mon, 27 May 2024 21:03:32 +0900 Subject: [PATCH 10/29] tests running --- tests/unittests/segmentation/inputs.py | 34 +++++-------------- .../segmentation/test_hausdorff_distance.py | 31 ++++++++++++++--- 2 files changed, 35 insertions(+), 30 deletions(-) diff --git a/tests/unittests/segmentation/inputs.py b/tests/unittests/segmentation/inputs.py index 80d0a240d29..996b8364e9c 100644 --- a/tests/unittests/segmentation/inputs.py +++ b/tests/unittests/segmentation/inputs.py @@ -11,36 +11,18 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from collections import namedtuple +__all__ = ["_Input"] -import torch +from typing import NamedTuple -from unittests.helpers import seed_all +from torch import Tensor + +from unittests._helpers import seed_all seed_all(42) # extrinsic input for clustering metrics that requires predicted clustering labels and target clustering labels -Input = namedtuple("Input", ["preds", "target"]) - -preds = torch.tensor( - [ - [[1, 1, 1, 1, 1], [1, 0, 0, 0, 1], [1, 0, 0, 0, 1], [1, 0, 0, 0, 1], [1, 1, 1, 1, 1]], - [[1, 1, 1, 1, 1], [1, 0, 0, 0, 1], [1, 0, 0, 0, 1], [1, 0, 0, 0, 1], [1, 1, 1, 1, 1]], - [[1, 1, 1, 1, 1], [1, 0, 0, 0, 1], [1, 0, 0, 0, 1], [1, 0, 0, 0, 1], [1, 1, 1, 1, 1]], - [[1, 1, 1, 1, 1], [1, 0, 0, 0, 1], [1, 0, 0, 0, 1], [1, 0, 0, 0, 1], [1, 1, 1, 1, 1]], - ], - dtype=torch.bool, -) - -target = torch.tensor( - [ - [[1, 1, 1, 1, 0], [1, 0, 0, 1, 0], [1, 0, 0, 1, 0], [1, 0, 0, 1, 0], [1, 1, 1, 1, 0]], - [[1, 1, 1, 1, 0], [1, 0, 0, 1, 0], [1, 0, 0, 1, 0], [1, 0, 0, 1, 0], [1, 1, 1, 1, 0]], - [[1, 1, 1, 1, 0], [1, 0, 0, 1, 0], [1, 0, 0, 1, 0], [1, 0, 0, 1, 0], [1, 1, 1, 1, 0]], - [[1, 1, 1, 1, 0], [1, 0, 0, 1, 0], [1, 0, 0, 1, 0], [1, 0, 0, 1, 0], [1, 1, 1, 1, 0]], - ], - dtype=torch.bool, -) - -_inputs = Input(preds=preds, target=target) +class _Input(NamedTuple): + preds: Tensor + target: Tensor diff --git a/tests/unittests/segmentation/test_hausdorff_distance.py b/tests/unittests/segmentation/test_hausdorff_distance.py index ee390a95c57..15e8e5ca721 100644 --- a/tests/unittests/segmentation/test_hausdorff_distance.py +++ b/tests/unittests/segmentation/test_hausdorff_distance.py @@ -17,13 +17,36 @@ from torchmetrics.functional.segmentation.hausdorff_distance import hausdorff_distance from torchmetrics.segmentation.hausdorff_distance import HausdorffDistance -from unittests.helpers import seed_all -from unittests.helpers.testers import MetricTester -from unittests.segmentation.inputs import _inputs +from unittests._helpers import seed_all +from unittests._helpers.testers import MetricTester +from unittests.segmentation.inputs import _Input seed_all(42) +preds = torch.tensor( + [ + [[1, 1, 1, 1, 1], [1, 0, 0, 0, 1], [1, 0, 0, 0, 1], [1, 0, 0, 0, 1], [1, 1, 1, 1, 1]], + [[1, 1, 1, 1, 1], [1, 0, 0, 0, 1], [1, 0, 0, 0, 1], [1, 0, 0, 0, 1], [1, 1, 1, 1, 1]], + [[1, 1, 1, 1, 1], [1, 0, 0, 0, 1], [1, 0, 0, 0, 1], [1, 0, 0, 0, 1], [1, 1, 1, 1, 1]], + [[1, 1, 1, 1, 1], [1, 0, 0, 0, 1], [1, 0, 0, 0, 1], [1, 0, 0, 0, 1], [1, 1, 1, 1, 1]], + ], + dtype=torch.bool, +) + +target = torch.tensor( + [ + [[1, 1, 1, 1, 0], [1, 0, 0, 1, 0], [1, 0, 0, 1, 0], [1, 0, 0, 1, 0], [1, 1, 1, 1, 0]], + [[1, 1, 1, 1, 0], [1, 0, 0, 1, 0], [1, 0, 0, 1, 0], [1, 0, 0, 1, 0], [1, 1, 1, 1, 0]], + [[1, 1, 1, 1, 0], [1, 0, 0, 1, 0], [1, 0, 0, 1, 0], [1, 0, 0, 1, 0], [1, 1, 1, 1, 0]], + [[1, 1, 1, 1, 0], [1, 0, 0, 1, 0], [1, 0, 0, 1, 0], [1, 0, 0, 1, 0], [1, 1, 1, 1, 0]], + ], + dtype=torch.bool, +) + +_inputs = _Input(preds=preds, target=target) + + @pytest.mark.parametrize( "preds, target", [ @@ -40,7 +63,7 @@ class TestHausdorffDistance(MetricTester): atol = 1e-5 @pytest.mark.parametrize("ddp", [True, False]) - def test_hausdorff_distance(self, preds, target, distance_metric, ddp): + def test_hausdorff_distance_class(self, preds, target, distance_metric, ddp): """Test class implementation of metric.""" self.run_class_metric_test( ddp=ddp, From 550770b68f5aa7d83d4a8b6ce943e892c7ab4ae7 Mon Sep 17 00:00:00 2001 From: baskrahmer Date: Sat, 10 Aug 2024 09:31:40 +0200 Subject: [PATCH 11/29] Add Torch-to-numpy wrapper for skimage metric --- tests/unittests/segmentation/test_hausdorff_distance.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/unittests/segmentation/test_hausdorff_distance.py b/tests/unittests/segmentation/test_hausdorff_distance.py index 15e8e5ca721..5936d1c6ef8 100644 --- a/tests/unittests/segmentation/test_hausdorff_distance.py +++ b/tests/unittests/segmentation/test_hausdorff_distance.py @@ -46,6 +46,9 @@ _inputs = _Input(preds=preds, target=target) +# Wrapper that converts to numpy to avoid Torch-to-numpy functional issues +torch_skimage_hausdorff_distance = lambda p, t: skimage_hausdorff_distance(p.numpy(), t.numpy()) + @pytest.mark.parametrize( "preds, target", @@ -70,7 +73,7 @@ def test_hausdorff_distance_class(self, preds, target, distance_metric, ddp): preds=preds, target=target, metric_class=HausdorffDistance, - reference_metric=skimage_hausdorff_distance, + reference_metric=torch_skimage_hausdorff_distance, metric_args={"distance_metric": distance_metric, "spacing": None}, ) @@ -80,7 +83,7 @@ def test_hausdorff_distance_functional(self, preds, target, distance_metric): preds=preds, target=target, metric_functional=hausdorff_distance, - reference_metric=skimage_hausdorff_distance, + reference_metric=torch_skimage_hausdorff_distance, metric_args={"distance_metric": distance_metric, "spacing": None}, ) From 78da6608be2602cf8bcd95ea7c214df169a54e12 Mon Sep 17 00:00:00 2001 From: baskrahmer Date: Sat, 10 Aug 2024 10:27:13 +0200 Subject: [PATCH 12/29] Return average over states --- src/torchmetrics/segmentation/hausdorff_distance.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/torchmetrics/segmentation/hausdorff_distance.py b/src/torchmetrics/segmentation/hausdorff_distance.py index 11919e21ae6..fb79fcd795c 100644 --- a/src/torchmetrics/segmentation/hausdorff_distance.py +++ b/src/torchmetrics/segmentation/hausdorff_distance.py @@ -16,7 +16,7 @@ from torchmetrics.functional.segmentation import hausdorff_distance from torchmetrics.metric import Metric -from torchmetrics.utilities.data import dim_zero_cat +from torchmetrics.utilities.data import dim_zero_mean, dim_zero_cat from torchmetrics.utilities.imports import _MATPLOTLIB_AVAILABLE from torchmetrics.utilities.plot import _AX_TYPE, _PLOT_OUT_TYPE @@ -91,9 +91,9 @@ def update(self, preds: Tensor, target: Tensor) -> None: def compute(self) -> Tensor: """Compute final Hausdorff distance over states.""" - return hausdorff_distance( - dim_zero_cat(self.preds), dim_zero_cat(self.target), self.distance_metric, self.spacing - ) + return dim_zero_mean(dim_zero_cat([ + hausdorff_distance(p, t, self.distance_metric, self.spacing) for p, t in zip(self.preds, self.target) + ])) def plot( self, val: Optional[Union[Tensor, Sequence[Tensor]]] = None, ax: Optional[_AX_TYPE] = None From 011722d60194e138c7d816692c04d5c2ccd26fad Mon Sep 17 00:00:00 2001 From: baskrahmer Date: Fri, 16 Aug 2024 12:03:44 +0200 Subject: [PATCH 13/29] Fix docs for doctests --- .../functional/segmentation/hausdorff_distance.py | 5 +++-- src/torchmetrics/segmentation/hausdorff_distance.py | 9 +++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/torchmetrics/functional/segmentation/hausdorff_distance.py b/src/torchmetrics/functional/segmentation/hausdorff_distance.py index 9b342dbf78f..4655b76aeab 100644 --- a/src/torchmetrics/functional/segmentation/hausdorff_distance.py +++ b/src/torchmetrics/functional/segmentation/hausdorff_distance.py @@ -68,7 +68,7 @@ def _hausdorff_distance_compute( ... [1, 0, 0, 1, 0], ... [1, 1, 1, 1, 0]], dtype=torch.bool) >>> hausdorff_distance(preds, target, distance_metric="euclidean") - tensor(1.0) + tensor(1.) """ fwd = surface_distance(preds, target, distance_metric=distance_metric, spacing=spacing) @@ -94,6 +94,7 @@ def hausdorff_distance( Hausdorff Distance Example: + >>> import torch >>> from torchmetrics.functional.segmentation import hausdorff_distance >>> preds = torch.tensor([[1, 1, 1, 1, 1], ... [1, 0, 0, 0, 1], @@ -106,7 +107,7 @@ def hausdorff_distance( ... [1, 0, 0, 1, 0], ... [1, 1, 1, 1, 0]], dtype=torch.bool) >>> hausdorff_distance(preds, target, distance_metric="euclidean") - tensor(1.0) + tensor(1.) """ preds, target = _hausdorff_distance_update(preds, target) diff --git a/src/torchmetrics/segmentation/hausdorff_distance.py b/src/torchmetrics/segmentation/hausdorff_distance.py index fb79fcd795c..9d93d4e0f21 100644 --- a/src/torchmetrics/segmentation/hausdorff_distance.py +++ b/src/torchmetrics/segmentation/hausdorff_distance.py @@ -46,6 +46,7 @@ class HausdorffDistance(Metric): kwargs: Additional keyword arguments, see :ref:`Metric kwargs` for more info. Example: + >>> import torch >>> preds = torch.tensor([[1, 1, 1, 1, 1], ... [1, 0, 0, 0, 1], ... [1, 0, 0, 0, 1], @@ -59,7 +60,7 @@ class HausdorffDistance(Metric): >>> hausdorff_distance = HausdorffDistance(distance_metric="euclidean") >>> hausdorff_distance.update(preds, target) >>> hausdorff_distance.compute() - tensor(1.0) + tensor(1.) """ @@ -115,10 +116,10 @@ def plot( .. plot:: :scale: 75 - >>> from torch import randn - >>> from torchmetrics.regression import HausdorffDistance + >>> from torch import randint >>> metric = HausdorffDistance() - >>> metric.update(randn(10,), randn(10,)) + >>> data1, data2 = randint(0, 2, (1,10,)).bool(), randint(0, 2, (1,10,)).bool() + >>> metric.update(data1, data2) >>> fig_, ax_ = metric.plot() """ From c0091e10740efde1d034883bd291ddd981c8af26 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 16 Aug 2024 10:11:54 +0000 Subject: [PATCH 14/29] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/torchmetrics/segmentation/hausdorff_distance.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/torchmetrics/segmentation/hausdorff_distance.py b/src/torchmetrics/segmentation/hausdorff_distance.py index 9d93d4e0f21..4dc26469ec7 100644 --- a/src/torchmetrics/segmentation/hausdorff_distance.py +++ b/src/torchmetrics/segmentation/hausdorff_distance.py @@ -16,7 +16,7 @@ from torchmetrics.functional.segmentation import hausdorff_distance from torchmetrics.metric import Metric -from torchmetrics.utilities.data import dim_zero_mean, dim_zero_cat +from torchmetrics.utilities.data import dim_zero_cat, dim_zero_mean from torchmetrics.utilities.imports import _MATPLOTLIB_AVAILABLE from torchmetrics.utilities.plot import _AX_TYPE, _PLOT_OUT_TYPE @@ -92,9 +92,11 @@ def update(self, preds: Tensor, target: Tensor) -> None: def compute(self) -> Tensor: """Compute final Hausdorff distance over states.""" - return dim_zero_mean(dim_zero_cat([ - hausdorff_distance(p, t, self.distance_metric, self.spacing) for p, t in zip(self.preds, self.target) - ])) + return dim_zero_mean( + dim_zero_cat([ + hausdorff_distance(p, t, self.distance_metric, self.spacing) for p, t in zip(self.preds, self.target) + ]) + ) def plot( self, val: Optional[Union[Tensor, Sequence[Tensor]]] = None, ax: Optional[_AX_TYPE] = None From 80829e812e6b2ac41d1980f26f1f103b74538e2c Mon Sep 17 00:00:00 2001 From: baskrahmer Date: Tue, 24 Sep 2024 09:30:09 +0200 Subject: [PATCH 15/29] Add pytest param for ddp --- tests/unittests/segmentation/test_hausdorff_distance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unittests/segmentation/test_hausdorff_distance.py b/tests/unittests/segmentation/test_hausdorff_distance.py index 5936d1c6ef8..a4c6bec2cce 100644 --- a/tests/unittests/segmentation/test_hausdorff_distance.py +++ b/tests/unittests/segmentation/test_hausdorff_distance.py @@ -65,7 +65,7 @@ class TestHausdorffDistance(MetricTester): atol = 1e-5 - @pytest.mark.parametrize("ddp", [True, False]) + @pytest.mark.parametrize("ddp", [pytest.param(True, marks=pytest.mark.DDP), False]) def test_hausdorff_distance_class(self, preds, target, distance_metric, ddp): """Test class implementation of metric.""" self.run_class_metric_test( From 0e96276c09dbdf1af077d65a865d49946a21153c Mon Sep 17 00:00:00 2001 From: baskrahmer Date: Tue, 24 Sep 2024 10:30:44 +0200 Subject: [PATCH 16/29] Fix type hints --- src/torchmetrics/segmentation/hausdorff_distance.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/torchmetrics/segmentation/hausdorff_distance.py b/src/torchmetrics/segmentation/hausdorff_distance.py index 4dc26469ec7..bf512f01cc3 100644 --- a/src/torchmetrics/segmentation/hausdorff_distance.py +++ b/src/torchmetrics/segmentation/hausdorff_distance.py @@ -10,7 +10,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Literal, Optional, Sequence, Union +from typing import Any, Literal, Optional, Sequence, Union, List from torch import Tensor @@ -69,13 +69,13 @@ class HausdorffDistance(Metric): full_state_update: bool = True plot_lower_bound: float = 0.0 plot_upper_bound: float = 1.0 - preds: list[Tensor] - target: list[Tensor] + preds: List[Tensor] + target: List[Tensor] def __init__( self, distance_metric: Literal["euclidean", "chessboard", "taxicab"] = "euclidean", - spacing: Optional[Union[Tensor, list[float]]] = None, + spacing: Optional[Union[Tensor, List[float]]] = None, **kwargs: Any, ) -> None: super().__init__(**kwargs) From 7b84f03a9ff2ab660c6712015f2777b7fed0a165 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 24 Sep 2024 08:37:03 +0000 Subject: [PATCH 17/29] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/torchmetrics/segmentation/hausdorff_distance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/torchmetrics/segmentation/hausdorff_distance.py b/src/torchmetrics/segmentation/hausdorff_distance.py index bf512f01cc3..55987f74bdc 100644 --- a/src/torchmetrics/segmentation/hausdorff_distance.py +++ b/src/torchmetrics/segmentation/hausdorff_distance.py @@ -10,7 +10,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Literal, Optional, Sequence, Union, List +from typing import Any, List, Literal, Optional, Sequence, Union from torch import Tensor From cc0d239948c384d3f2a5dae9a88f13a7da90f093 Mon Sep 17 00:00:00 2001 From: baskrahmer Date: Tue, 24 Sep 2024 11:07:07 +0200 Subject: [PATCH 18/29] Refactor lambda to function definition --- tests/unittests/segmentation/test_hausdorff_distance.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unittests/segmentation/test_hausdorff_distance.py b/tests/unittests/segmentation/test_hausdorff_distance.py index a4c6bec2cce..0766ccc9b04 100644 --- a/tests/unittests/segmentation/test_hausdorff_distance.py +++ b/tests/unittests/segmentation/test_hausdorff_distance.py @@ -47,8 +47,8 @@ _inputs = _Input(preds=preds, target=target) # Wrapper that converts to numpy to avoid Torch-to-numpy functional issues -torch_skimage_hausdorff_distance = lambda p, t: skimage_hausdorff_distance(p.numpy(), t.numpy()) - +def torch_skimage_hausdorff_distance(p: torch.Tensor, t: torch.Tensor) -> float: + return skimage_hausdorff_distance(p.numpy(), t.numpy()) @pytest.mark.parametrize( "preds, target", From a07e021bc9059cb304aa6e209c8be457b6dcd454 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 24 Sep 2024 09:08:54 +0000 Subject: [PATCH 19/29] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/unittests/segmentation/test_hausdorff_distance.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unittests/segmentation/test_hausdorff_distance.py b/tests/unittests/segmentation/test_hausdorff_distance.py index 0766ccc9b04..34b720b5fec 100644 --- a/tests/unittests/segmentation/test_hausdorff_distance.py +++ b/tests/unittests/segmentation/test_hausdorff_distance.py @@ -46,10 +46,12 @@ _inputs = _Input(preds=preds, target=target) + # Wrapper that converts to numpy to avoid Torch-to-numpy functional issues def torch_skimage_hausdorff_distance(p: torch.Tensor, t: torch.Tensor) -> float: return skimage_hausdorff_distance(p.numpy(), t.numpy()) + @pytest.mark.parametrize( "preds, target", [ From c7bdce9db51ed167a2f3f4d1a88fd8b0c6d78978 Mon Sep 17 00:00:00 2001 From: baskrahmer Date: Tue, 24 Sep 2024 11:22:54 +0200 Subject: [PATCH 20/29] Fix docs --- docs/source/segmentation/hausdorff_distance.rst | 4 +--- src/torchmetrics/segmentation/hausdorff_distance.py | 1 + 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/source/segmentation/hausdorff_distance.rst b/docs/source/segmentation/hausdorff_distance.rst index 182e547c4b3..ba1eb35a6b4 100644 --- a/docs/source/segmentation/hausdorff_distance.rst +++ b/docs/source/segmentation/hausdorff_distance.rst @@ -1,9 +1,7 @@ .. customcarditem:: :header: Hausdorff Distance :image: https://pl-flash-data.s3.amazonaws.com/assets/thumbnails/text_classification.svg - :tags: Retrieval - -.. include:: ../links.rst + :tags: segmentation ################## Hausdorff Distance diff --git a/src/torchmetrics/segmentation/hausdorff_distance.py b/src/torchmetrics/segmentation/hausdorff_distance.py index 55987f74bdc..b7542845a0a 100644 --- a/src/torchmetrics/segmentation/hausdorff_distance.py +++ b/src/torchmetrics/segmentation/hausdorff_distance.py @@ -47,6 +47,7 @@ class HausdorffDistance(Metric): Example: >>> import torch + >>> from torchmetrics.segmentation import HausdorffDistance >>> preds = torch.tensor([[1, 1, 1, 1, 1], ... [1, 0, 0, 0, 1], ... [1, 0, 0, 0, 1], From 8541d97fd15f079c56f2cef30b99519c32f4de06 Mon Sep 17 00:00:00 2001 From: baskrahmer Date: Tue, 24 Sep 2024 12:58:59 +0200 Subject: [PATCH 21/29] Output a tensor for reference metric --- tests/unittests/segmentation/test_hausdorff_distance.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/unittests/segmentation/test_hausdorff_distance.py b/tests/unittests/segmentation/test_hausdorff_distance.py index 34b720b5fec..db37a71e6ea 100644 --- a/tests/unittests/segmentation/test_hausdorff_distance.py +++ b/tests/unittests/segmentation/test_hausdorff_distance.py @@ -48,8 +48,9 @@ # Wrapper that converts to numpy to avoid Torch-to-numpy functional issues -def torch_skimage_hausdorff_distance(p: torch.Tensor, t: torch.Tensor) -> float: - return skimage_hausdorff_distance(p.numpy(), t.numpy()) +def torch_skimage_hausdorff_distance(p: torch.Tensor, t: torch.Tensor) -> torch.Tensor: + out = skimage_hausdorff_distance(p.numpy(), t.numpy()) + return torch.tensor([out]) @pytest.mark.parametrize( From 90ad41438556120fd09b9b381790b001f3e0da7d Mon Sep 17 00:00:00 2001 From: baskrahmer Date: Tue, 24 Sep 2024 13:45:45 +0200 Subject: [PATCH 22/29] Set dtype to float32 in reference metric --- tests/unittests/segmentation/test_hausdorff_distance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unittests/segmentation/test_hausdorff_distance.py b/tests/unittests/segmentation/test_hausdorff_distance.py index db37a71e6ea..e9c2976c7ea 100644 --- a/tests/unittests/segmentation/test_hausdorff_distance.py +++ b/tests/unittests/segmentation/test_hausdorff_distance.py @@ -50,7 +50,7 @@ # Wrapper that converts to numpy to avoid Torch-to-numpy functional issues def torch_skimage_hausdorff_distance(p: torch.Tensor, t: torch.Tensor) -> torch.Tensor: out = skimage_hausdorff_distance(p.numpy(), t.numpy()) - return torch.tensor([out]) + return torch.tensor([out], dtype=torch.float32) @pytest.mark.parametrize( From f61e7ac3ca0aeb3546483b02ee78fdf5f65b7d82 Mon Sep 17 00:00:00 2001 From: baskrahmer Date: Tue, 24 Sep 2024 17:11:15 +0200 Subject: [PATCH 23/29] Add links back --- docs/source/segmentation/hausdorff_distance.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/segmentation/hausdorff_distance.rst b/docs/source/segmentation/hausdorff_distance.rst index ba1eb35a6b4..cfe1d3fdb5b 100644 --- a/docs/source/segmentation/hausdorff_distance.rst +++ b/docs/source/segmentation/hausdorff_distance.rst @@ -3,6 +3,8 @@ :image: https://pl-flash-data.s3.amazonaws.com/assets/thumbnails/text_classification.svg :tags: segmentation +.. include:: ../links.rst + ################## Hausdorff Distance ################## From 33453938aa8ec571725086b63c13403e19eb58c7 Mon Sep 17 00:00:00 2001 From: Nicki Skafte Date: Sat, 12 Oct 2024 12:29:02 +0200 Subject: [PATCH 24/29] changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 50be7dd1f53..f432c7afa26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added `truncation` argument to `BERTScore` ([#2776](https://github.com/Lightning-AI/torchmetrics/pull/2776)) +- Added `HausdorffDistance` to segmentation package ([#2122](https://github.com/Lightning-AI/torchmetrics/pull/2122)) + + ### Changed - Tracker higher is better integration ([#2649](https://github.com/Lightning-AI/torchmetrics/pull/2649)) From a4f129f7d7f832134fa4cc6900dfa076c3669865 Mon Sep 17 00:00:00 2001 From: Nicki Skafte Date: Sat, 12 Oct 2024 15:57:14 +0200 Subject: [PATCH 25/29] fix docstring + add input validation --- .../segmentation/hausdorff_distance.py | 13 +++++++++ .../segmentation/hausdorff_distance.py | 29 +++++++++++++------ 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/src/torchmetrics/functional/segmentation/hausdorff_distance.py b/src/torchmetrics/functional/segmentation/hausdorff_distance.py index 4655b76aeab..171bf15e248 100644 --- a/src/torchmetrics/functional/segmentation/hausdorff_distance.py +++ b/src/torchmetrics/functional/segmentation/hausdorff_distance.py @@ -21,6 +21,19 @@ from torchmetrics.utilities.checks import _check_same_shape +def _hausdorff_distance_validate_args( + distance_metric: Literal["euclidean", "chessboard", "taxicab"] = "euclidean", + spacing: Optional[Union[Tensor, list[float]]] = None, +) -> None: + """Validate the arguments of `hausdorff_distance` function.""" + if distance_metric not in ["euclidean", "chessboard", "taxicab"]: + raise ValueError( + f"Arg `distance_metric` must be one of 'euclidean', 'chessboard', 'taxicab', but got {distance_metric}." + ) + if spacing is not None and not isinstance(spacing, (list, Tensor)): + raise ValueError(f"Arg `spacing` must be a list or tensor, but got {type(spacing)}.") + + def _hausdorff_distance_update(preds: Tensor, target: Tensor) -> Tuple[Tensor, Tensor]: """Update and returns variables required to compute `Hausdorff Distance`_. diff --git a/src/torchmetrics/segmentation/hausdorff_distance.py b/src/torchmetrics/segmentation/hausdorff_distance.py index b7542845a0a..d4c01e3808a 100644 --- a/src/torchmetrics/segmentation/hausdorff_distance.py +++ b/src/torchmetrics/segmentation/hausdorff_distance.py @@ -14,7 +14,10 @@ from torch import Tensor -from torchmetrics.functional.segmentation import hausdorff_distance +from torchmetrics.functional.segmentation.hausdorff_distance import ( + _hausdorff_distance_validate_args, + hausdorff_distance, +) from torchmetrics.metric import Metric from torchmetrics.utilities.data import dim_zero_cat, dim_zero_mean from torchmetrics.utilities.imports import _MATPLOTLIB_AVAILABLE @@ -25,24 +28,33 @@ class HausdorffDistance(Metric): - r"""Compute the Hausdorff distance between two subsets of a metric space. + r"""Compute the `Hausdorff Distance`_ between two subsets of a metric space for semantic segmentation. .. math:: d_{\Pi}(X,Y) = \max{/sup_{x\in X} {d(x,Y)}, /sup_{y\in Y} {d(X,y)}} - where :math:`\X, \Y` are ________________, :math:`\X, \Y` ______. + where :math:`\X, \Y` are two subsets of a metric space with distance metric :math:`d`. The Hausdorff distance is + the maximum distance from a point in one set to the closest point in the other set. The Hausdorff distance is a + measure of the degree of mismatch between two sets. As input to ``forward`` and ``update`` the metric accepts the following input: - - ``preds`` (:class:`~torch.Tensor`): - - ``target`` (:class:`~torch.Tensor`): + - ``preds`` (:class:`~torch.Tensor`): An one-hot boolean tensor of shape ``(N, C, ...)`` with ``N`` being + the number of samples and ``C`` the number of classes. Alternatively, an integer tensor of shape ``(N, ...)`` + can be provided, where the integer values correspond to the class index. The input type can be controlled + with the ``input_format`` argument. + - ``target`` (:class:`~torch.Tensor`): An one-hot boolean tensor of shape ``(N, C, ...)`` with ``N`` being + the number of samples and ``C`` the number of classes. Alternatively, an integer tensor of shape ``(N, ...)`` + can be provided, where the integer values correspond to the class index. The input type can be controlled + with the ``input_format`` argument. As output of ``forward`` and ``compute`` the metric returns the following output: - ``hausdorff_distance`` (:class:`~torch.Tensor`): A scalar float tensor with the Hausdorff distance. Args: - p: p-norm used for distance metric + distance_metric: distance metric to calculate surface distance. Choose between "euclidean", "chessboard" or + "taxicab". kwargs: Additional keyword arguments, see :ref:`Metric kwargs` for more info. Example: @@ -59,8 +71,7 @@ class HausdorffDistance(Metric): ... [1, 0, 0, 1, 0], ... [1, 1, 1, 1, 0]], dtype=torch.bool) >>> hausdorff_distance = HausdorffDistance(distance_metric="euclidean") - >>> hausdorff_distance.update(preds, target) - >>> hausdorff_distance.compute() + >>> hausdorff_distance(preds, target) tensor(1.) """ @@ -80,9 +91,9 @@ def __init__( **kwargs: Any, ) -> None: super().__init__(**kwargs) + _hausdorff_distance_validate_args(distance_metric, spacing) self.distance_metric = distance_metric self.spacing = spacing - self.add_state("preds", default=[], dist_reduce_fx="cat") self.add_state("target", default=[], dist_reduce_fx="cat") From 3f4b68ef37531c97cce4df1fa5c7840ca7e3f7ec Mon Sep 17 00:00:00 2001 From: Nicki Skafte Date: Mon, 14 Oct 2024 10:53:49 +0200 Subject: [PATCH 26/29] add edge_surface_distance utility --- .../segmentation/hausdorff_distance.py | 1 + .../functional/segmentation/utils.py | 34 ++++++++++++- tests/unittests/segmentation/test_utils.py | 49 +++++++++++++++++++ 3 files changed, 83 insertions(+), 1 deletion(-) diff --git a/src/torchmetrics/functional/segmentation/hausdorff_distance.py b/src/torchmetrics/functional/segmentation/hausdorff_distance.py index 171bf15e248..fc9e7dae22f 100644 --- a/src/torchmetrics/functional/segmentation/hausdorff_distance.py +++ b/src/torchmetrics/functional/segmentation/hausdorff_distance.py @@ -123,5 +123,6 @@ def hausdorff_distance( tensor(1.) """ + _hausdorff_distance_validate_args(distance_metric, spacing) preds, target = _hausdorff_distance_update(preds, target) return _hausdorff_distance_compute(preds, target, distance_metric=distance_metric, spacing=spacing) diff --git a/src/torchmetrics/functional/segmentation/utils.py b/src/torchmetrics/functional/segmentation/utils.py index e1c80831a7a..39984614fa8 100644 --- a/src/torchmetrics/functional/segmentation/utils.py +++ b/src/torchmetrics/functional/segmentation/utils.py @@ -345,7 +345,7 @@ def surface_distance( target: Tensor, distance_metric: Literal["euclidean", "chessboard", "taxicab"] = "euclidean", spacing: Optional[Union[Tensor, List[float]]] = None, -) -> Tensor: +) -> Union[Tensor, Tuple[Tensor, Tensor]]: """Calculate the surface distance between two binary edge masks. May return infinity if the predicted mask is empty and the target mask is not, or vice versa. @@ -390,6 +390,38 @@ def surface_distance( return dis[preds] +def edge_surface_distance( + preds: Tensor, + target: Tensor, + distance_metric: Literal["euclidean", "chessboard", "taxicab"] = "euclidean", + spacing: Optional[Union[Tensor, List[float]]] = None, + symmetric: bool = False, +) -> Tensor: + """Extracts the edges from the input masks and calculates the surface distance between them. + + Args: + preds: The predicted binary edge mask. + target: The target binary edge mask. + distance_metric: The distance metric to use. One of `["euclidean", "chessboard", "taxicab"]`. + spacing: The spacing between pixels along each spatial dimension. + symmetric: Whether to calculate the symmetric distance between the edges. + + Returns: + A tensor with length equal to the number of edges in predictions e.g. `preds.sum()`. Each element is the + distance from the corresponding edge in `preds` to the closest edge in `target`. If `symmetric` is `True`, the + function returns a tuple containing the distances from the predicted edges to the target edges and vice versa. + + """ + output = mask_edges(preds, target) + edges_preds, edges_target = output[0].bool(), output[1].bool() + if symmetric: + return ( + surface_distance(edges_preds, edges_target, distance_metric=distance_metric, spacing=spacing), + surface_distance(edges_target, edges_preds, distance_metric=distance_metric, spacing=spacing), + ) + return surface_distance(edges_preds, edges_target, distance_metric=distance_metric, spacing=spacing) + + @functools.lru_cache def get_neighbour_tables( spacing: Union[Tuple[int, int], Tuple[int, int, int]], device: Optional[torch.device] = None diff --git a/tests/unittests/segmentation/test_utils.py b/tests/unittests/segmentation/test_utils.py index d37941a6ff3..39cff09a2dd 100644 --- a/tests/unittests/segmentation/test_utils.py +++ b/tests/unittests/segmentation/test_utils.py @@ -14,6 +14,7 @@ import pytest import torch from monai.metrics.utils import get_code_to_measure_table +from monai.metrics.utils import get_edge_surface_distance as monai_get_edge_surface_distance from monai.metrics.utils import get_mask_edges as monai_get_mask_edges from monai.metrics.utils import get_surface_distance as monai_get_surface_distance from scipy.ndimage import binary_erosion as scibinary_erosion @@ -23,6 +24,7 @@ from torchmetrics.functional.segmentation.utils import ( binary_erosion, distance_transform, + edge_surface_distance, generate_binary_structure, get_neighbour_tables, mask_edges, @@ -231,3 +233,50 @@ def test_mask_edges(cases, spacing, crop, device): for r1, r2 in zip(res, reference_res): assert torch.allclose(r1.cpu().float(), torch.from_numpy(r2).float()) + + +@pytest.mark.parametrize( + "cases", + [ + ( + torch.tensor( + [[1, 1, 1, 1, 1], [1, 0, 0, 0, 1], [1, 0, 0, 0, 1], [1, 0, 0, 0, 1], [1, 1, 1, 1, 1]], dtype=torch.bool + ), + torch.tensor( + [[1, 1, 1, 1, 0], [1, 0, 0, 1, 0], [1, 0, 0, 1, 0], [1, 0, 0, 1, 0], [1, 1, 1, 1, 0]], dtype=torch.bool + ), + ), + (torch.randint(0, 2, (5, 5), dtype=torch.bool), torch.randint(0, 2, (5, 5), dtype=torch.bool)), + (torch.randint(0, 2, (50, 50), dtype=torch.bool), torch.randint(0, 2, (50, 50), dtype=torch.bool)), + ], +) +@pytest.mark.parametrize("distance_metric", ["euclidean", "chessboard", "taxicab"]) +@pytest.mark.parametrize("symmetric", [False, True]) +@pytest.mark.parametrize("spacing", [None, 1, 2]) +@pytest.mark.parametrize("device", ["cpu", "cuda"]) +def test_edge_surface_distance(cases, distance_metric, symmetric, spacing, device): + """Test the edge surface distance function.""" + if device == "cuda" and not torch.cuda.is_available(): + pytest.skip("CUDA device not available.") + if spacing == 2 and distance_metric != "euclidean": + pytest.skip("Only euclidean distance is supported for spacing != 1 in reference") + preds, target = cases + if spacing is not None: + spacing = preds.ndim * [spacing] + + res = edge_surface_distance( + preds.to(device), target.to(device), spacing=spacing, distance_metric=distance_metric, symmetric=symmetric + ) + _, reference_res, _ = monai_get_edge_surface_distance( + preds, + target, + spacing=tuple(spacing) if spacing is not None else spacing, + distance_metric=distance_metric, + symmetric=symmetric, + ) + + if symmetric: + assert torch.allclose(res[0].cpu(), reference_res[0].to(res[0].dtype)) + assert torch.allclose(res[1].cpu(), reference_res[1].to(res[1].dtype)) + else: + assert torch.allclose(res.cpu(), reference_res[0].to(res.dtype)) From 6c8b5b644ea3862e05cdf8d0a23e227fe2ecf1e1 Mon Sep 17 00:00:00 2001 From: Nicki Skafte Date: Mon, 14 Oct 2024 14:47:52 +0200 Subject: [PATCH 27/29] fix functional implementation --- .../segmentation/hausdorff_distance.py | 140 ++++++++---------- .../functional/segmentation/utils.py | 5 +- 2 files changed, 65 insertions(+), 80 deletions(-) diff --git a/src/torchmetrics/functional/segmentation/hausdorff_distance.py b/src/torchmetrics/functional/segmentation/hausdorff_distance.py index fc9e7dae22f..bc5d5ac419a 100644 --- a/src/torchmetrics/functional/segmentation/hausdorff_distance.py +++ b/src/torchmetrics/functional/segmentation/hausdorff_distance.py @@ -12,117 +12,103 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Literal, Optional, Tuple, Union +from typing import Literal, Optional, Union import torch from torch import Tensor -from torchmetrics.functional.segmentation.utils import check_if_binarized, surface_distance +from torchmetrics.functional.segmentation.utils import ( + _ignore_background, + edge_surface_distance, +) from torchmetrics.utilities.checks import _check_same_shape def _hausdorff_distance_validate_args( + num_classes: int, + include_background: bool, distance_metric: Literal["euclidean", "chessboard", "taxicab"] = "euclidean", spacing: Optional[Union[Tensor, list[float]]] = None, + directed: bool = False, + input_format: Literal["one-hot", "index"] = "one-hot", ) -> None: """Validate the arguments of `hausdorff_distance` function.""" + if num_classes <= 0: + raise ValueError(f"Expected argument `num_classes` must be a positive integer, but got {num_classes}.") + if not isinstance(include_background, bool): + raise ValueError(f"Expected argument `include_background` must be a boolean, but got {include_background}.") if distance_metric not in ["euclidean", "chessboard", "taxicab"]: raise ValueError( f"Arg `distance_metric` must be one of 'euclidean', 'chessboard', 'taxicab', but got {distance_metric}." ) if spacing is not None and not isinstance(spacing, (list, Tensor)): raise ValueError(f"Arg `spacing` must be a list or tensor, but got {type(spacing)}.") - - -def _hausdorff_distance_update(preds: Tensor, target: Tensor) -> Tuple[Tensor, Tensor]: - """Update and returns variables required to compute `Hausdorff Distance`_. - - Args: - preds: predicted binarized segmentation map - target: target binarized segmentation map - - Returns: - preds: predicted binarized segmentation map - target: target binarized segmentation map - - """ - check_if_binarized(preds) - check_if_binarized(target) - _check_same_shape(preds, target) - return preds, target - - -def _hausdorff_distance_compute( - preds: Tensor, - target: Tensor, - distance_metric: Literal["euclidean", "chessboard", "taxicab"] = "euclidean", - spacing: Optional[Union[Tensor, list[float]]] = None, -) -> Tensor: - """Compute `Hausdorff Distance`_. - - Args: - preds: predicted binarized segmentation map - target: target binarized segmentation map - distance_metric: distance metric to calculate surface distance. One of `["euclidean", "chessboard", "taxicab"]`. - spacing: spacing between pixels along each spatial dimension - - Returns: - Hausdorff distance - - Example: - >>> preds = torch.tensor([[1, 1, 1, 1, 1], - ... [1, 0, 0, 0, 1], - ... [1, 0, 0, 0, 1], - ... [1, 0, 0, 0, 1], - ... [1, 1, 1, 1, 1]], dtype=torch.bool) - >>> target = torch.tensor([[1, 1, 1, 1, 0], - ... [1, 0, 0, 1, 0], - ... [1, 0, 0, 1, 0], - ... [1, 0, 0, 1, 0], - ... [1, 1, 1, 1, 0]], dtype=torch.bool) - >>> hausdorff_distance(preds, target, distance_metric="euclidean") - tensor(1.) - - """ - fwd = surface_distance(preds, target, distance_metric=distance_metric, spacing=spacing) - bwd = surface_distance(target, preds, distance_metric=distance_metric, spacing=spacing) - return torch.max(torch.tensor([fwd.max(), bwd.max()])) + if not isinstance(directed, bool): + raise ValueError(f"Expected argument `directed` must be a boolean, but got {directed}.") + if input_format not in ["one-hot", "index"]: + raise ValueError(f"Expected argument `input_format` to be one of 'one-hot', 'index', but got {input_format}.") def hausdorff_distance( preds: Tensor, target: Tensor, + num_classes: int, + include_background: bool = False, distance_metric: Literal["euclidean", "chessboard", "taxicab"] = "euclidean", spacing: Optional[Union[Tensor, list[float]]] = None, + directed: bool = False, + input_format: Literal["one-hot", "index"] = "one-hot", ) -> Tensor: - """Calculate `Hausdorff Distance`_. + """Calculate `Hausdorff Distance`_ for semantic segmentation. Args: preds: predicted binarized segmentation map target: target binarized segmentation map - distance_metric: distance metric to calculate surface distance. One of `["euclidean", "chessboard", "taxicab"]`. - spacing: spacing between pixels along each spatial dimension + num_classes: number of classes + include_background: whether to include background class in calculation + distance_metric: distance metric to calculate surface distance. Choose one of `"euclidean"`, + `"chessboard"` or `"taxicab"` + spacing: spacing between pixels along each spatial dimension. If not provided the spacing is assumed to be 1 + directed: whether to calculate directed or undirected Hausdorff distance + input_format: What kind of input the function receives. Choose between ``"one-hot"`` for one-hot encoded tensors + or ``"index"`` for index tensors Returns: - Hausdorff Distance + Hausdorff Distance for each class and batch element Example: - >>> import torch + >>> from torch import randint >>> from torchmetrics.functional.segmentation import hausdorff_distance - >>> preds = torch.tensor([[1, 1, 1, 1, 1], - ... [1, 0, 0, 0, 1], - ... [1, 0, 0, 0, 1], - ... [1, 0, 0, 0, 1], - ... [1, 1, 1, 1, 1]], dtype=torch.bool) - >>> target = torch.tensor([[1, 1, 1, 1, 0], - ... [1, 0, 0, 1, 0], - ... [1, 0, 0, 1, 0], - ... [1, 0, 0, 1, 0], - ... [1, 1, 1, 1, 0]], dtype=torch.bool) - >>> hausdorff_distance(preds, target, distance_metric="euclidean") - tensor(1.) + >>> preds = randint(0, 2, (4, 5, 16, 16)) # 4 samples, 5 classes, 16x16 prediction + >>> target = randint(0, 2, (4, 5, 16, 16)) # 4 samples, 5 classes, 16x16 target + >>> hausdorff_distance(preds, target, num_classes=5) + tensor([[2.0000, 1.4142, 2.0000, 2.0000], + [1.4142, 2.0000, 2.0000, 2.0000], + [2.0000, 2.0000, 1.4142, 2.0000], + [2.0000, 2.8284, 2.0000, 2.2361]]) """ - _hausdorff_distance_validate_args(distance_metric, spacing) - preds, target = _hausdorff_distance_update(preds, target) - return _hausdorff_distance_compute(preds, target, distance_metric=distance_metric, spacing=spacing) + _hausdorff_distance_validate_args(num_classes, include_background, distance_metric, spacing, directed, input_format) + _check_same_shape(preds, target) + + if input_format == "index": + preds = torch.nn.functional.one_hot(preds, num_classes=num_classes).movedim(-1, 1) + target = torch.nn.functional.one_hot(target, num_classes=num_classes).movedim(-1, 1) + + if not include_background: + preds, target = _ignore_background(preds, target) + + distances = torch.zeros(preds.shape[0], preds.shape[1], device=preds.device) + + # TODO: add support for batched inputs + for b in range(preds.shape[0]): + for c in range(preds.shape[1]): + dist = edge_surface_distance( + preds=preds[b, c], + target=target[b, c], + distance_metric=distance_metric, + spacing=spacing, + symmetric=not directed, + ) + distances[b, c] = torch.max(dist) if directed else torch.max(dist[0].max(), dist[1].max()) + return distances diff --git a/src/torchmetrics/functional/segmentation/utils.py b/src/torchmetrics/functional/segmentation/utils.py index 39984614fa8..c5f0e4424e6 100644 --- a/src/torchmetrics/functional/segmentation/utils.py +++ b/src/torchmetrics/functional/segmentation/utils.py @@ -200,9 +200,8 @@ def distance_transform( Args: x: The binary tensor to calculate the distance transform of. - sampling: Only relevant when distance is calculated using the euclidean distance. The sampling refers to the - pixel spacing in the image, i.e. the distance between two adjacent pixels. If not provided, the pixel - spacing is assumed to be 1. + sampling: The sampling refers to the pixel spacing in the image, i.e. the distance between two adjacent pixels. + If not provided, the pixel spacing is assumed to be 1. metric: The distance to use for the distance transform. Can be one of ``"euclidean"``, ``"chessboard"`` or ``"taxicab"``. engine: The engine to use for the distance transform. Can be one of ``["pytorch", "scipy"]``. In general, From d2723c5748a572ceb4efa5e23a970634920f639d Mon Sep 17 00:00:00 2001 From: Nicki Skafte Date: Mon, 14 Oct 2024 14:48:49 +0200 Subject: [PATCH 28/29] fix modular implementation --- .../segmentation/hausdorff_distance.py | 90 +++++++++++-------- 1 file changed, 53 insertions(+), 37 deletions(-) diff --git a/src/torchmetrics/segmentation/hausdorff_distance.py b/src/torchmetrics/segmentation/hausdorff_distance.py index d4c01e3808a..5c3100be387 100644 --- a/src/torchmetrics/segmentation/hausdorff_distance.py +++ b/src/torchmetrics/segmentation/hausdorff_distance.py @@ -10,8 +10,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, List, Literal, Optional, Sequence, Union +from typing import Any, Literal, Optional, Sequence, Union +import torch from torch import Tensor from torchmetrics.functional.segmentation.hausdorff_distance import ( @@ -19,7 +20,6 @@ hausdorff_distance, ) from torchmetrics.metric import Metric -from torchmetrics.utilities.data import dim_zero_cat, dim_zero_mean from torchmetrics.utilities.imports import _MATPLOTLIB_AVAILABLE from torchmetrics.utilities.plot import _AX_TYPE, _PLOT_OUT_TYPE @@ -50,65 +50,80 @@ class HausdorffDistance(Metric): As output of ``forward`` and ``compute`` the metric returns the following output: - - ``hausdorff_distance`` (:class:`~torch.Tensor`): A scalar float tensor with the Hausdorff distance. + - ``hausdorff_distance`` (:class:`~torch.Tensor`): A scalar float tensor with the Hausdorff distance averaged over + classes and samples Args: - distance_metric: distance metric to calculate surface distance. Choose between "euclidean", "chessboard" or - "taxicab". + num_classes: number of classes + include_background: whether to include background class in calculation + distance_metric: distance metric to calculate surface distance. Choose one of `"euclidean"`, + `"chessboard"` or `"taxicab"` + spacing: spacing between pixels along each spatial dimension. If not provided the spacing is assumed to be 1 + directed: whether to calculate directed or undirected Hausdorff distance + input_format: What kind of input the function receives. Choose between ``"one-hot"`` for one-hot encoded tensors + or ``"index"`` for index tensors kwargs: Additional keyword arguments, see :ref:`Metric kwargs` for more info. Example: - >>> import torch + >>> from torch import randint >>> from torchmetrics.segmentation import HausdorffDistance - >>> preds = torch.tensor([[1, 1, 1, 1, 1], - ... [1, 0, 0, 0, 1], - ... [1, 0, 0, 0, 1], - ... [1, 0, 0, 0, 1], - ... [1, 1, 1, 1, 1]], dtype=torch.bool) - >>> target = torch.tensor([[1, 1, 1, 1, 0], - ... [1, 0, 0, 1, 0], - ... [1, 0, 0, 1, 0], - ... [1, 0, 0, 1, 0], - ... [1, 1, 1, 1, 0]], dtype=torch.bool) - >>> hausdorff_distance = HausdorffDistance(distance_metric="euclidean") + >>> preds = randint(0, 2, (4, 5, 16, 16)) # 4 samples, 5 classes, 16x16 prediction + >>> target = randint(0, 2, (4, 5, 16, 16)) # 4 samples, 5 classes, 16x16 target + >>> hausdorff_distance = HausdorffDistance(distance_metric="euclidean", num_classes=5) >>> hausdorff_distance(preds, target) - tensor(1.) + tensor(1.9567) """ is_differentiable: bool = True - higher_is_better: bool = True - full_state_update: bool = True + higher_is_better: bool = False + full_state_update: bool = False plot_lower_bound: float = 0.0 - plot_upper_bound: float = 1.0 - preds: List[Tensor] - target: List[Tensor] + + score: Tensor + total: Tensor def __init__( self, + num_classes: int, + include_background: bool = False, distance_metric: Literal["euclidean", "chessboard", "taxicab"] = "euclidean", - spacing: Optional[Union[Tensor, List[float]]] = None, + spacing: Optional[Union[Tensor, list[float]]] = None, + directed: bool = False, + input_format: Literal["one-hot", "index"] = "one-hot", **kwargs: Any, ) -> None: super().__init__(**kwargs) - _hausdorff_distance_validate_args(distance_metric, spacing) + _hausdorff_distance_validate_args( + num_classes, include_background, distance_metric, spacing, directed, input_format + ) + self.num_classes = num_classes + self.include_background = include_background self.distance_metric = distance_metric self.spacing = spacing - self.add_state("preds", default=[], dist_reduce_fx="cat") - self.add_state("target", default=[], dist_reduce_fx="cat") + self.directed = directed + self.input_format = input_format + self.add_state("score", default=torch.tensor(0.0), dist_reduce_fx="sum") + self.add_state("total", default=torch.tensor(0), dist_reduce_fx="sum") def update(self, preds: Tensor, target: Tensor) -> None: """Update state with predictions and targets.""" - self.preds.append(preds) - self.target.append(target) + score = hausdorff_distance( + preds, + target, + self.num_classes, + include_background=self.include_background, + distance_metric=self.distance_metric, + spacing=self.spacing, + directed=self.directed, + input_format=self.input_format, + ) + self.score += score.sum() + self.total += score.numel() def compute(self) -> Tensor: """Compute final Hausdorff distance over states.""" - return dim_zero_mean( - dim_zero_cat([ - hausdorff_distance(p, t, self.distance_metric, self.spacing) for p, t in zip(self.preds, self.target) - ]) - ) + return self.score / self.total def plot( self, val: Optional[Union[Tensor, Sequence[Tensor]]] = None, ax: Optional[_AX_TYPE] = None @@ -131,9 +146,10 @@ def plot( :scale: 75 >>> from torch import randint - >>> metric = HausdorffDistance() - >>> data1, data2 = randint(0, 2, (1,10,)).bool(), randint(0, 2, (1,10,)).bool() - >>> metric.update(data1, data2) + >>> preds = randint(0, 2, (4, 5, 16, 16)) # 4 samples, 5 classes, 16x16 prediction + >>> target = randint(0, 2, (4, 5, 16, 16)) # 4 samples, 5 classes, 16x16 target + >>> metric = HausdorffDistance(num_classes=5) + >>> metric.update(preds, target) >>> fig_, ax_ = metric.plot() """ From cc7294d2dfe94bf5a552e9c0f5db139deae2769c Mon Sep 17 00:00:00 2001 From: Nicki Skafte Date: Mon, 14 Oct 2024 14:49:54 +0200 Subject: [PATCH 29/29] tests --- .../segmentation/test_hausdorff_distance.py | 125 +++++++++--------- 1 file changed, 65 insertions(+), 60 deletions(-) diff --git a/tests/unittests/segmentation/test_hausdorff_distance.py b/tests/unittests/segmentation/test_hausdorff_distance.py index e9c2976c7ea..afd77c1f4b2 100644 --- a/tests/unittests/segmentation/test_hausdorff_distance.py +++ b/tests/unittests/segmentation/test_hausdorff_distance.py @@ -11,101 +11,106 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from functools import partial +from typing import Any + import pytest import torch -from skimage.metrics import hausdorff_distance as skimage_hausdorff_distance +from monai.metrics.hausdorff_distance import compute_hausdorff_distance as monai_hausdorff_distance from torchmetrics.functional.segmentation.hausdorff_distance import hausdorff_distance from torchmetrics.segmentation.hausdorff_distance import HausdorffDistance +from unittests import NUM_BATCHES, _Input from unittests._helpers import seed_all from unittests._helpers.testers import MetricTester -from unittests.segmentation.inputs import _Input seed_all(42) +BATCH_SIZE = 4 # use smaller than normal batch size to reduce test time +NUM_CLASSES = 3 # use smaller than normal class size to reduce test time - -preds = torch.tensor( - [ - [[1, 1, 1, 1, 1], [1, 0, 0, 0, 1], [1, 0, 0, 0, 1], [1, 0, 0, 0, 1], [1, 1, 1, 1, 1]], - [[1, 1, 1, 1, 1], [1, 0, 0, 0, 1], [1, 0, 0, 0, 1], [1, 0, 0, 0, 1], [1, 1, 1, 1, 1]], - [[1, 1, 1, 1, 1], [1, 0, 0, 0, 1], [1, 0, 0, 0, 1], [1, 0, 0, 0, 1], [1, 1, 1, 1, 1]], - [[1, 1, 1, 1, 1], [1, 0, 0, 0, 1], [1, 0, 0, 0, 1], [1, 0, 0, 0, 1], [1, 1, 1, 1, 1]], - ], - dtype=torch.bool, +_inputs1 = _Input( + preds=torch.randint(0, 2, (NUM_BATCHES, BATCH_SIZE, NUM_CLASSES, 16, 16)), + target=torch.randint(0, 2, (NUM_BATCHES, BATCH_SIZE, NUM_CLASSES, 16, 16)), ) - -target = torch.tensor( - [ - [[1, 1, 1, 1, 0], [1, 0, 0, 1, 0], [1, 0, 0, 1, 0], [1, 0, 0, 1, 0], [1, 1, 1, 1, 0]], - [[1, 1, 1, 1, 0], [1, 0, 0, 1, 0], [1, 0, 0, 1, 0], [1, 0, 0, 1, 0], [1, 1, 1, 1, 0]], - [[1, 1, 1, 1, 0], [1, 0, 0, 1, 0], [1, 0, 0, 1, 0], [1, 0, 0, 1, 0], [1, 1, 1, 1, 0]], - [[1, 1, 1, 1, 0], [1, 0, 0, 1, 0], [1, 0, 0, 1, 0], [1, 0, 0, 1, 0], [1, 1, 1, 1, 0]], - ], - dtype=torch.bool, +_inputs2 = _Input( + preds=torch.randint(0, NUM_CLASSES, (NUM_BATCHES, BATCH_SIZE, 32, 32)), + target=torch.randint(0, NUM_CLASSES, (NUM_BATCHES, BATCH_SIZE, 32, 32)), ) -_inputs = _Input(preds=preds, target=target) +def reference_metric(preds, target, input_format, reduce, **kwargs: Any): + """Reference implementation of metric.""" + if input_format == "index": + preds = torch.nn.functional.one_hot(preds, num_classes=NUM_CLASSES).movedim(-1, 1) + target = torch.nn.functional.one_hot(target, num_classes=NUM_CLASSES).movedim(-1, 1) + score = monai_hausdorff_distance(preds, target, **kwargs) + return score.mean() if reduce else score -# Wrapper that converts to numpy to avoid Torch-to-numpy functional issues -def torch_skimage_hausdorff_distance(p: torch.Tensor, t: torch.Tensor) -> torch.Tensor: - out = skimage_hausdorff_distance(p.numpy(), t.numpy()) - return torch.tensor([out], dtype=torch.float32) - -@pytest.mark.parametrize( - "preds, target", - [ - (_inputs.preds, _inputs.target), - ], -) -@pytest.mark.parametrize( - "distance_metric", - ["euclidean", "chessboard", "taxicab"], -) +@pytest.mark.parametrize("inputs, input_format", [(_inputs1, "one-hot"), (_inputs2, "index")]) +@pytest.mark.parametrize("distance_metric", ["euclidean", "chessboard", "taxicab"]) +@pytest.mark.parametrize("directed", [True, False]) +@pytest.mark.parametrize("spacing", [None, [2, 2]]) class TestHausdorffDistance(MetricTester): """Test class for `HausdorffDistance` metric.""" atol = 1e-5 @pytest.mark.parametrize("ddp", [pytest.param(True, marks=pytest.mark.DDP), False]) - def test_hausdorff_distance_class(self, preds, target, distance_metric, ddp): + def test_hausdorff_distance_class(self, inputs, input_format, distance_metric, directed, spacing, ddp): """Test class implementation of metric.""" + if spacing is not None and distance_metric != "euclidean": + pytest.skip("Spacing is only supported for Euclidean distance metric.") + preds, target = inputs self.run_class_metric_test( ddp=ddp, preds=preds, target=target, metric_class=HausdorffDistance, - reference_metric=torch_skimage_hausdorff_distance, - metric_args={"distance_metric": distance_metric, "spacing": None}, + reference_metric=partial( + reference_metric, + input_format=input_format, + distance_metric=distance_metric, + directed=directed, + spacing=spacing, + reduce=True, + ), + metric_args={ + "num_classes": NUM_CLASSES, + "distance_metric": distance_metric, + "directed": directed, + "spacing": spacing, + "input_format": input_format, + }, ) - def test_hausdorff_distance_functional(self, preds, target, distance_metric): + def test_hausdorff_distance_functional(self, inputs, input_format, distance_metric, directed, spacing): """Test functional implementation of metric.""" + if spacing is not None and distance_metric != "euclidean": + pytest.skip("Spacing is only supported for Euclidean distance metric.") + preds, target = inputs self.run_functional_metric_test( preds=preds, target=target, metric_functional=hausdorff_distance, - reference_metric=torch_skimage_hausdorff_distance, - metric_args={"distance_metric": distance_metric, "spacing": None}, + reference_metric=partial( + reference_metric, + input_format=input_format, + distance_metric=distance_metric, + directed=directed, + spacing=spacing, + reduce=False, + ), + metric_args={ + "num_classes": NUM_CLASSES, + "distance_metric": distance_metric, + "directed": directed, + "spacing": spacing, + "input_format": input_format, + }, ) -def test_hausdorff_distance_functional_raises_invalid_task(): - """Check that metric rejects continuous-valued inputs.""" - preds, target = _inputs - with pytest.raises(ValueError, match=r"Expected *"): - hausdorff_distance(preds, target) - - -@pytest.mark.parametrize( - "distance_metric", - ["euclidean", "chessboard", "taxicab"], -) -def test_hausdorff_distance_is_symmetric(distance_metric): - """Check that the metric functional is symmetric.""" - for p, t in zip(_inputs.preds, _inputs.target): - assert torch.allclose( - hausdorff_distance(p, t, distance_metric), - hausdorff_distance(t, p, distance_metric), - ) +def test_hausdorff_distance_raises_error(): + """Check that metric raises appropriate errors.""" + preds, target = _inputs1