From df1e988eaa9986689337f01e79ae13181e585188 Mon Sep 17 00:00:00 2001 From: jirka Date: Thu, 29 Aug 2024 22:42:34 +0200 Subject: [PATCH 01/11] tests: move reset seed to fixture --- src/torchmetrics/image/_deprecated.py | 47 ++++++++--------- src/torchmetrics/image/d_lambda.py | 21 ++++---- src/torchmetrics/image/d_s.py | 27 +++++----- src/torchmetrics/image/ergas.py | 14 +++--- src/torchmetrics/image/fid.py | 5 +- src/torchmetrics/image/inception.py | 5 +- src/torchmetrics/image/kid.py | 9 ++-- src/torchmetrics/image/lpip.py | 9 ++-- src/torchmetrics/image/mifid.py | 7 ++- .../image/perceptual_path_length.py | 11 ++-- src/torchmetrics/image/psnrb.py | 8 ++- src/torchmetrics/image/qnr.py | 27 +++++----- src/torchmetrics/image/rase.py | 15 +++--- src/torchmetrics/image/rmse_sw.py | 8 +-- src/torchmetrics/image/sam.py | 21 ++++---- src/torchmetrics/image/scc.py | 7 ++- src/torchmetrics/image/ssim.py | 13 +++-- src/torchmetrics/image/tv.py | 3 +- src/torchmetrics/image/vif.py | 7 ++- src/torchmetrics/multimodal/clip_iqa.py | 15 +++--- src/torchmetrics/multimodal/clip_score.py | 4 +- src/torchmetrics/nominal/cramers.py | 6 +-- src/torchmetrics/nominal/fleiss_kappa.py | 12 ++--- src/torchmetrics/nominal/pearson.py | 6 +-- src/torchmetrics/nominal/theils_u.py | 6 +-- src/torchmetrics/nominal/tschuprows.py | 6 +-- src/torchmetrics/regression/wmape.py | 7 ++- .../segmentation/generalized_dice.py | 13 +++-- src/torchmetrics/segmentation/mean_iou.py | 13 +++-- src/torchmetrics/text/_deprecated.py | 7 ++- src/torchmetrics/text/perplexity.py | 7 ++- src/torchmetrics/wrappers/bootstrapping.py | 6 +-- src/torchmetrics/wrappers/classwise.py | 50 +++++++++---------- src/torchmetrics/wrappers/feature_share.py | 1 - src/torchmetrics/wrappers/tracker.py | 30 ++++++----- 35 files changed, 202 insertions(+), 251 deletions(-) diff --git a/src/torchmetrics/image/_deprecated.py b/src/torchmetrics/image/_deprecated.py index 597f0a0c636..86fb04c7f1c 100644 --- a/src/torchmetrics/image/_deprecated.py +++ b/src/torchmetrics/image/_deprecated.py @@ -17,11 +17,11 @@ class _ErrorRelativeGlobalDimensionlessSynthesis(ErrorRelativeGlobalDimensionlessSynthesis): """Wrapper for deprecated import. - >>> import torch - >>> preds = torch.rand([16, 1, 16, 16], generator=torch.manual_seed(42)) + >>> from torch import rand + >>> preds = rand([16, 1, 16, 16]) >>> target = preds * 0.75 >>> ergas = _ErrorRelativeGlobalDimensionlessSynthesis() - >>> torch.round(ergas(preds, target)) + >>> ergas(preds, target).round() tensor(10.) """ @@ -39,8 +39,8 @@ def __init__( class _MultiScaleStructuralSimilarityIndexMeasure(MultiScaleStructuralSimilarityIndexMeasure): """Wrapper for deprecated import. - >>> import torch - >>> preds = torch.rand([3, 3, 256, 256], generator=torch.manual_seed(42)) + >>> from torch import rand + >>> preds = rand([3, 3, 256, 256]) >>> target = preds * 0.75 >>> ms_ssim = _MultiScaleStructuralSimilarityIndexMeasure(data_range=1.0) >>> ms_ssim(preds, target) @@ -103,13 +103,12 @@ def __init__( class _RelativeAverageSpectralError(RelativeAverageSpectralError): """Wrapper for deprecated import. - >>> import torch - >>> g = torch.manual_seed(22) - >>> preds = torch.rand(4, 3, 16, 16) - >>> target = torch.rand(4, 3, 16, 16) + >>> from torch import rand + >>> preds = rand(4, 3, 16, 16) + >>> target = rand(4, 3, 16, 16) >>> rase = _RelativeAverageSpectralError() >>> rase(preds, target) - tensor(5114.66...) + tensor(5326.40...) """ @@ -125,13 +124,12 @@ def __init__( class _RootMeanSquaredErrorUsingSlidingWindow(RootMeanSquaredErrorUsingSlidingWindow): """Wrapper for deprecated import. - >>> import torch - >>> g = torch.manual_seed(22) - >>> preds = torch.rand(4, 3, 16, 16) - >>> target = torch.rand(4, 3, 16, 16) + >>> from torch import rand + >>> preds = rand(4, 3, 16, 16) + >>> target = rand(4, 3, 16, 16) >>> rmse_sw = RootMeanSquaredErrorUsingSlidingWindow() >>> rmse_sw(preds, target) - tensor(0.3999) + tensor(0.4158) """ @@ -147,10 +145,9 @@ def __init__( class _SpectralAngleMapper(SpectralAngleMapper): """Wrapper for deprecated import. - >>> import torch - >>> gen = torch.manual_seed(42) - >>> preds = torch.rand([16, 3, 16, 16], generator=gen) - >>> target = torch.rand([16, 3, 16, 16], generator=gen) + >>> from torch import rand + >>> preds = rand([16, 3, 16, 16]) + >>> target = rand([16, 3, 16, 16]) >>> sam = _SpectralAngleMapper() >>> sam(preds, target) tensor(0.5914) @@ -169,10 +166,9 @@ def __init__( class _SpectralDistortionIndex(SpectralDistortionIndex): """Wrapper for deprecated import. - >>> import torch - >>> _ = torch.manual_seed(42) - >>> preds = torch.rand([16, 3, 16, 16]) - >>> target = torch.rand([16, 3, 16, 16]) + >>> from torch import rand + >>> preds = rand([16, 3, 16, 16]) + >>> target = rand([16, 3, 16, 16]) >>> sdi = _SpectralDistortionIndex() >>> sdi(preds, target) tensor(0.0234) @@ -229,10 +225,9 @@ def __init__( class _TotalVariation(TotalVariation): """Wrapper for deprecated import. - >>> import torch - >>> _ = torch.manual_seed(42) + >>> from torch import rand >>> tv = _TotalVariation() - >>> img = torch.rand(5, 3, 28, 28) + >>> img = rand(5, 3, 28, 28) >>> tv(img) tensor(7546.8018) diff --git a/src/torchmetrics/image/d_lambda.py b/src/torchmetrics/image/d_lambda.py index ff232569748..5b1e58de0dd 100644 --- a/src/torchmetrics/image/d_lambda.py +++ b/src/torchmetrics/image/d_lambda.py @@ -53,11 +53,10 @@ class SpectralDistortionIndex(Metric): kwargs: Additional keyword arguments, see :ref:`Metric kwargs` for more info. Example: - >>> import torch - >>> _ = torch.manual_seed(42) + >>> from torch import rand >>> from torchmetrics.image import SpectralDistortionIndex - >>> preds = torch.rand([16, 3, 16, 16]) - >>> target = torch.rand([16, 3, 16, 16]) + >>> preds = rand([16, 3, 16, 16]) + >>> target = rand([16, 3, 16, 16]) >>> sdi = SpectralDistortionIndex() >>> sdi(preds, target) tensor(0.0234) @@ -126,11 +125,10 @@ def plot( :scale: 75 >>> # Example plotting a single value - >>> import torch - >>> _ = torch.manual_seed(42) + >>> from torch import rand >>> from torchmetrics.image import SpectralDistortionIndex - >>> preds = torch.rand([16, 3, 16, 16]) - >>> target = torch.rand([16, 3, 16, 16]) + >>> preds = rand([16, 3, 16, 16]) + >>> target = rand([16, 3, 16, 16]) >>> metric = SpectralDistortionIndex() >>> metric.update(preds, target) >>> fig_, ax_ = metric.plot() @@ -139,11 +137,10 @@ def plot( :scale: 75 >>> # Example plotting multiple values - >>> import torch - >>> _ = torch.manual_seed(42) + >>> from torch import rand >>> from torchmetrics.image import SpectralDistortionIndex - >>> preds = torch.rand([16, 3, 16, 16]) - >>> target = torch.rand([16, 3, 16, 16]) + >>> preds = rand([16, 3, 16, 16]) + >>> target = rand([16, 3, 16, 16]) >>> metric = SpectralDistortionIndex() >>> values = [ ] >>> for _ in range(10): diff --git a/src/torchmetrics/image/d_s.py b/src/torchmetrics/image/d_s.py index 434561c558a..a8dacbaf447 100644 --- a/src/torchmetrics/image/d_s.py +++ b/src/torchmetrics/image/d_s.py @@ -74,13 +74,12 @@ class SpatialDistortionIndex(Metric): kwargs: Additional keyword arguments, see :ref:`Metric kwargs` for more info. Example: - >>> import torch - >>> _ = torch.manual_seed(42) + >>> from torch import rand >>> from torchmetrics.image import SpatialDistortionIndex - >>> preds = torch.rand([16, 3, 32, 32]) + >>> preds = rand([16, 3, 32, 32]) >>> target = { - ... 'ms': torch.rand([16, 3, 16, 16]), - ... 'pan': torch.rand([16, 3, 32, 32]), + ... 'ms': rand([16, 3, 16, 16]), + ... 'pan': rand([16, 3, 32, 32]), ... } >>> sdi = SpatialDistortionIndex() >>> sdi(preds, target) @@ -191,13 +190,12 @@ def plot( :scale: 75 >>> # Example plotting a single value - >>> import torch - >>> _ = torch.manual_seed(42) + >>> from torch import rand >>> from torchmetrics.image import SpatialDistortionIndex - >>> preds = torch.rand([16, 3, 32, 32]) + >>> preds = rand([16, 3, 32, 32]) >>> target = { - ... 'ms': torch.rand([16, 3, 16, 16]), - ... 'pan': torch.rand([16, 3, 32, 32]), + ... 'ms': rand([16, 3, 16, 16]), + ... 'pan': rand([16, 3, 32, 32]), ... } >>> metric = SpatialDistortionIndex() >>> metric.update(preds, target) @@ -207,13 +205,12 @@ def plot( :scale: 75 >>> # Example plotting multiple values - >>> import torch - >>> _ = torch.manual_seed(42) + >>> from torch import rand >>> from torchmetrics.image import SpatialDistortionIndex - >>> preds = torch.rand([16, 3, 32, 32]) + >>> preds = rand([16, 3, 32, 32]) >>> target = { - ... 'ms': torch.rand([16, 3, 16, 16]), - ... 'pan': torch.rand([16, 3, 32, 32]), + ... 'ms': rand([16, 3, 16, 16]), + ... 'pan': rand([16, 3, 32, 32]), ... } >>> metric = SpatialDistortionIndex() >>> values = [ ] diff --git a/src/torchmetrics/image/ergas.py b/src/torchmetrics/image/ergas.py index f1d8da2073e..bf6b1c99a10 100644 --- a/src/torchmetrics/image/ergas.py +++ b/src/torchmetrics/image/ergas.py @@ -62,12 +62,12 @@ class ErrorRelativeGlobalDimensionlessSynthesis(Metric): kwargs: Additional keyword arguments, see :ref:`Metric kwargs` for more info. Example: - >>> import torch + >>> from torch import rand >>> from torchmetrics.image import ErrorRelativeGlobalDimensionlessSynthesis - >>> preds = torch.rand([16, 1, 16, 16], generator=torch.manual_seed(42)) + >>> preds = rand([16, 1, 16, 16]) >>> target = preds * 0.75 >>> ergas = ErrorRelativeGlobalDimensionlessSynthesis() - >>> torch.round(ergas(preds, target)) + >>> ergas(preds, target).round() tensor(10.) """ @@ -131,9 +131,9 @@ def plot( :scale: 75 >>> # Example plotting a single value - >>> import torch + >>> from torch import rand >>> from torchmetrics.image import ErrorRelativeGlobalDimensionlessSynthesis - >>> preds = torch.rand([16, 1, 16, 16], generator=torch.manual_seed(42)) + >>> preds = rand([16, 1, 16, 16]) >>> target = preds * 0.75 >>> metric = ErrorRelativeGlobalDimensionlessSynthesis() >>> metric.update(preds, target) @@ -143,9 +143,9 @@ def plot( :scale: 75 >>> # Example plotting multiple values - >>> import torch + >>> from torch import rand >>> from torchmetrics.image import ErrorRelativeGlobalDimensionlessSynthesis - >>> preds = torch.rand([16, 1, 16, 16], generator=torch.manual_seed(42)) + >>> preds = rand([16, 1, 16, 16]) >>> target = preds * 0.75 >>> metric = ErrorRelativeGlobalDimensionlessSynthesis() >>> values = [ ] diff --git a/src/torchmetrics/image/fid.py b/src/torchmetrics/image/fid.py index af4b93d7ad5..8c2e5d3cf76 100644 --- a/src/torchmetrics/image/fid.py +++ b/src/torchmetrics/image/fid.py @@ -265,8 +265,7 @@ class FrechetInceptionDistance(Metric): If ``reset_real_features`` is not an ``bool`` Example: - >>> import torch - >>> _ = torch.manual_seed(123) + >>> from torch import rand >>> from torchmetrics.image.fid import FrechetInceptionDistance >>> fid = FrechetInceptionDistance(feature=64) >>> # generate two slightly overlapping image intensity distributions @@ -275,7 +274,7 @@ class FrechetInceptionDistance(Metric): >>> fid.update(imgs_dist1, real=True) >>> fid.update(imgs_dist2, real=False) >>> fid.compute() - tensor(12.7202) + tensor(12.6388) """ diff --git a/src/torchmetrics/image/inception.py b/src/torchmetrics/image/inception.py index 4913a5fe50b..2b125542a17 100644 --- a/src/torchmetrics/image/inception.py +++ b/src/torchmetrics/image/inception.py @@ -84,15 +84,14 @@ class InceptionScore(Metric): If ``feature`` is not an ``str``, ``int`` or ``torch.nn.Module`` Example: - >>> import torch - >>> _ = torch.manual_seed(123) + >>> from torch import rand >>> from torchmetrics.image.inception import InceptionScore >>> inception = InceptionScore() >>> # generate some images >>> imgs = torch.randint(0, 255, (100, 3, 299, 299), dtype=torch.uint8) >>> inception.update(imgs) >>> inception.compute() - (tensor(1.0544), tensor(0.0117)) + (tensor(1.0549), tensor(0.0121)) """ diff --git a/src/torchmetrics/image/kid.py b/src/torchmetrics/image/kid.py index e080116a33f..975edf800eb 100644 --- a/src/torchmetrics/image/kid.py +++ b/src/torchmetrics/image/kid.py @@ -148,17 +148,16 @@ class KernelInceptionDistance(Metric): If ``reset_real_features`` is not an ``bool`` Example: - >>> import torch - >>> _ = torch.manual_seed(123) + >>> from torch import randint >>> from torchmetrics.image.kid import KernelInceptionDistance >>> kid = KernelInceptionDistance(subset_size=50) >>> # generate two slightly overlapping image intensity distributions - >>> imgs_dist1 = torch.randint(0, 200, (100, 3, 299, 299), dtype=torch.uint8) - >>> imgs_dist2 = torch.randint(100, 255, (100, 3, 299, 299), dtype=torch.uint8) + >>> imgs_dist1 = randint(0, 200, (100, 3, 299, 299), dtype=torch.uint8) + >>> imgs_dist2 = randint(100, 255, (100, 3, 299, 299), dtype=torch.uint8) >>> kid.update(imgs_dist1, real=True) >>> kid.update(imgs_dist2, real=False) >>> kid.compute() - (tensor(0.0337), tensor(0.0023)) + (tensor(0.0312), tensor(0.0025)) """ diff --git a/src/torchmetrics/image/lpip.py b/src/torchmetrics/image/lpip.py index c094003fa4f..f1adb73648a 100644 --- a/src/torchmetrics/image/lpip.py +++ b/src/torchmetrics/image/lpip.py @@ -78,15 +78,14 @@ class LearnedPerceptualImagePatchSimilarity(Metric): If ``reduction`` is not one of ``"mean"`` or ``"sum"`` Example: - >>> import torch - >>> _ = torch.manual_seed(123) + >>> from torch import rand >>> from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity >>> lpips = LearnedPerceptualImagePatchSimilarity(net_type='squeeze') >>> # LPIPS needs the images to be in the [-1, 1] range. - >>> img1 = (torch.rand(10, 3, 100, 100) * 2) - 1 - >>> img2 = (torch.rand(10, 3, 100, 100) * 2) - 1 + >>> img1 = (rand(10, 3, 100, 100) * 2) - 1 + >>> img2 = (rand(10, 3, 100, 100) * 2) - 1 >>> lpips(img1, img2) - tensor(0.1046) + tensor(0.1024) """ diff --git a/src/torchmetrics/image/mifid.py b/src/torchmetrics/image/mifid.py index 8c1084e6458..7d7237b7416 100644 --- a/src/torchmetrics/image/mifid.py +++ b/src/torchmetrics/image/mifid.py @@ -129,13 +129,12 @@ class MemorizationInformedFrechetInceptionDistance(Metric): If ``reset_real_features`` is not an ``bool`` Example:: - >>> import torch - >>> _ = torch.manual_seed(42) + >>> from torch import randint >>> from torchmetrics.image.mifid import MemorizationInformedFrechetInceptionDistance >>> mifid = MemorizationInformedFrechetInceptionDistance(feature=64) >>> # generate two slightly overlapping image intensity distributions - >>> imgs_dist1 = torch.randint(0, 200, (100, 3, 299, 299), dtype=torch.uint8) - >>> imgs_dist2 = torch.randint(100, 255, (100, 3, 299, 299), dtype=torch.uint8) + >>> imgs_dist1 = randint(0, 200, (100, 3, 299, 299), dtype=torch.uint8) + >>> imgs_dist2 = randint(100, 255, (100, 3, 299, 299), dtype=torch.uint8) >>> mifid.update(imgs_dist1, real=True) >>> mifid.update(imgs_dist2, real=False) >>> mifid.compute() diff --git a/src/torchmetrics/image/perceptual_path_length.py b/src/torchmetrics/image/perceptual_path_length.py index 6b89b36b1c1..4e8cca23343 100644 --- a/src/torchmetrics/image/perceptual_path_length.py +++ b/src/torchmetrics/image/perceptual_path_length.py @@ -98,9 +98,8 @@ class PerceptualPathLength(Metric): If ``upper_discard`` is not a float between 0 and 1 or None. Example:: - >>> from torchmetrics.image import PerceptualPathLength >>> import torch - >>> _ = torch.manual_seed(42) + >>> from torchmetrics.image import PerceptualPathLength >>> class DummyGenerator(torch.nn.Module): ... def __init__(self, z_size) -> None: ... super().__init__() @@ -112,10 +111,10 @@ class PerceptualPathLength(Metric): ... return torch.randn(num_samples, self.z_size) >>> generator = DummyGenerator(2) >>> ppl = PerceptualPathLength(num_samples=10) - >>> ppl(generator) # doctest: +SKIP - (tensor(0.2371), - tensor(0.1763), - tensor([0.3502, 0.1362, 0.2535, 0.0902, 0.1784, 0.0769, 0.5871, 0.0691, 0.3921])) + >>> ppl(generator) + (tensor(0.2369), + tensor(0.1760), + tensor([0.3489, 0.1363, 0.2535, 0.0907, 0.1785, 0.0769, 0.5864, 0.0691, 0.3920])) """ diff --git a/src/torchmetrics/image/psnrb.py b/src/torchmetrics/image/psnrb.py index dc050e81fb8..bac58b84e46 100644 --- a/src/torchmetrics/image/psnrb.py +++ b/src/torchmetrics/image/psnrb.py @@ -51,12 +51,10 @@ class PeakSignalNoiseRatioWithBlockedEffect(Metric): kwargs: Additional keyword arguments, see :ref:`Metric kwargs` for more info. Example: - >>> import torch - >>> from torchmetrics.image import PeakSignalNoiseRatioWithBlockedEffect + >>> from torch import rand >>> metric = PeakSignalNoiseRatioWithBlockedEffect() - >>> _ = torch.manual_seed(42) - >>> preds = torch.rand(2, 1, 10, 10) - >>> target = torch.rand(2, 1, 10, 10) + >>> preds = rand(2, 1, 10, 10) + >>> target = rand(2, 1, 10, 10) >>> metric(preds, target) tensor(7.2893) diff --git a/src/torchmetrics/image/qnr.py b/src/torchmetrics/image/qnr.py index 96023db8aab..b226a6b19fd 100644 --- a/src/torchmetrics/image/qnr.py +++ b/src/torchmetrics/image/qnr.py @@ -70,13 +70,12 @@ class QualityWithNoReference(Metric): kwargs: Additional keyword arguments, see :ref:`Metric kwargs` for more info. Example: - >>> import torch - >>> _ = torch.manual_seed(42) + >>> from torch import rand >>> from torchmetrics.image import QualityWithNoReference - >>> preds = torch.rand([16, 3, 32, 32]) + >>> preds = rand([16, 3, 32, 32]) >>> target = { - ... 'ms': torch.rand([16, 3, 16, 16]), - ... 'pan': torch.rand([16, 3, 32, 32]), + ... 'ms': rand([16, 3, 16, 16]), + ... 'pan': rand([16, 3, 32, 32]), ... } >>> qnr = QualityWithNoReference() >>> qnr(preds, target) @@ -195,13 +194,12 @@ def plot( :scale: 75 >>> # Example plotting a single value - >>> import torch - >>> _ = torch.manual_seed(42) + >>> from torch import rand >>> from torchmetrics.image import QualityWithNoReference - >>> preds = torch.rand([16, 3, 32, 32]) + >>> preds = rand([16, 3, 32, 32]) >>> target = { - ... 'ms': torch.rand([16, 3, 16, 16]), - ... 'pan': torch.rand([16, 3, 32, 32]), + ... 'ms': rand([16, 3, 16, 16]), + ... 'pan': rand([16, 3, 32, 32]), ... } >>> metric = QualityWithNoReference() >>> metric.update(preds, target) @@ -211,13 +209,12 @@ def plot( :scale: 75 >>> # Example plotting multiple values - >>> import torch - >>> _ = torch.manual_seed(42) + >>> from torch import rand >>> from torchmetrics.image import QualityWithNoReference - >>> preds = torch.rand([16, 3, 32, 32]) + >>> preds = rand([16, 3, 32, 32]) >>> target = { - ... 'ms': torch.rand([16, 3, 16, 16]), - ... 'pan': torch.rand([16, 3, 32, 32]), + ... 'ms': rand([16, 3, 16, 16]), + ... 'pan': rand([16, 3, 32, 32]), ... } >>> metric = QualityWithNoReference() >>> values = [ ] diff --git a/src/torchmetrics/image/rase.py b/src/torchmetrics/image/rase.py index c422762eb68..b1eb32141a6 100644 --- a/src/torchmetrics/image/rase.py +++ b/src/torchmetrics/image/rase.py @@ -46,14 +46,12 @@ class RelativeAverageSpectralError(Metric): Relative Average Spectral Error (RASE) Example: - >>> import torch - >>> from torchmetrics.image import RelativeAverageSpectralError - >>> g = torch.manual_seed(22) - >>> preds = torch.rand(4, 3, 16, 16) - >>> target = torch.rand(4, 3, 16, 16) + >>> from torch import rand + >>> preds = rand(4, 3, 16, 16) + >>> target = rand(4, 3, 16, 16) >>> rase = RelativeAverageSpectralError() >>> rase(preds, target) - tensor(5114.66...) + tensor(5326.40...) Raises: ValueError: If ``window_size`` is not a positive integer. @@ -124,13 +122,12 @@ def plot( :scale: 75 >>> # Example plotting multiple values - >>> import torch - >>> _ = torch.manual_seed(42) + >>> from torch import rand >>> from torchmetrics.image import RelativeAverageSpectralError >>> metric = RelativeAverageSpectralError() >>> values = [ ] >>> for _ in range(10): - ... values.append(metric(torch.rand(4, 3, 16, 16), torch.rand(4, 3, 16, 16))) + ... values.append(metric(rand(4, 3, 16, 16), rand(4, 3, 16, 16))) >>> fig_, ax_ = metric.plot(values) """ diff --git a/src/torchmetrics/image/rmse_sw.py b/src/torchmetrics/image/rmse_sw.py index e0838ec760d..c1f7c652879 100644 --- a/src/torchmetrics/image/rmse_sw.py +++ b/src/torchmetrics/image/rmse_sw.py @@ -43,13 +43,13 @@ class RootMeanSquaredErrorUsingSlidingWindow(Metric): kwargs: Additional keyword arguments, see :ref:`Metric kwargs` for more info. Example: + >>> from torch import rand >>> from torchmetrics.image import RootMeanSquaredErrorUsingSlidingWindow - >>> g = torch.manual_seed(22) - >>> preds = torch.rand(4, 3, 16, 16) - >>> target = torch.rand(4, 3, 16, 16) + >>> preds = rand(4, 3, 16, 16) + >>> target = rand(4, 3, 16, 16) >>> rmse_sw = RootMeanSquaredErrorUsingSlidingWindow() >>> rmse_sw(preds, target) - tensor(0.3999) + tensor(0.4158) Raises: ValueError: If ``window_size`` is not a positive integer. diff --git a/src/torchmetrics/image/sam.py b/src/torchmetrics/image/sam.py index 47d4839c5e2..7aad1782852 100644 --- a/src/torchmetrics/image/sam.py +++ b/src/torchmetrics/image/sam.py @@ -56,11 +56,10 @@ class SpectralAngleMapper(Metric): Tensor with SpectralAngleMapper score Example: - >>> import torch + >>> from torch import rand >>> from torchmetrics.image import SpectralAngleMapper - >>> gen = torch.manual_seed(42) - >>> preds = torch.rand([16, 3, 16, 16], generator=gen) - >>> target = torch.rand([16, 3, 16, 16], generator=gen) + >>> preds = rand([16, 3, 16, 16]) + >>> target = rand([16, 3, 16, 16]) >>> sam = SpectralAngleMapper() >>> sam(preds, target) tensor(0.5914) @@ -141,11 +140,10 @@ def plot( :scale: 75 >>> # Example plotting single value - >>> import torch + >>> from torch import rand >>> from torchmetrics.image import SpectralAngleMapper - >>> gen = torch.manual_seed(42) - >>> preds = torch.rand([16, 3, 16, 16], generator=gen) - >>> target = torch.rand([16, 3, 16, 16], generator=gen) + >>> preds = rand([16, 3, 16, 16]) + >>> target = rand([16, 3, 16, 16]) >>> metric = SpectralAngleMapper() >>> metric.update(preds, target) >>> fig_, ax_ = metric.plot() @@ -154,11 +152,10 @@ def plot( :scale: 75 >>> # Example plotting multiple values - >>> import torch + >>> from torch import rand >>> from torchmetrics.image import SpectralAngleMapper - >>> gen = torch.manual_seed(42) - >>> preds = torch.rand([16, 3, 16, 16], generator=gen) - >>> target = torch.rand([16, 3, 16, 16], generator=gen) + >>> preds = rand([16, 3, 16, 16]) + >>> target = rand([16, 3, 16, 16]) >>> metric = SpectralAngleMapper() >>> values = [ ] >>> for _ in range(10): diff --git a/src/torchmetrics/image/scc.py b/src/torchmetrics/image/scc.py index 15ea2b96ecf..fac28658f63 100644 --- a/src/torchmetrics/image/scc.py +++ b/src/torchmetrics/image/scc.py @@ -39,11 +39,10 @@ class SpatialCorrelationCoefficient(Metric): kwargs: Additional keyword arguments, see :ref:`Metric kwargs` for more info. Example: - >>> import torch - >>> _ = torch.manual_seed(42) + >>> from torch import randn >>> from torchmetrics.image import SpatialCorrelationCoefficient as SCC - >>> preds = torch.randn([32, 3, 64, 64]) - >>> target = torch.randn([32, 3, 64, 64]) + >>> preds = randn([32, 3, 64, 64]) + >>> target = randn([32, 3, 64, 64]) >>> scc = SCC() >>> scc(preds, target) tensor(0.0023) diff --git a/src/torchmetrics/image/ssim.py b/src/torchmetrics/image/ssim.py index 5056589fd14..f0c03a163d6 100644 --- a/src/torchmetrics/image/ssim.py +++ b/src/torchmetrics/image/ssim.py @@ -268,10 +268,9 @@ class MultiScaleStructuralSimilarityIndexMeasure(Metric): If ``normalize`` is neither `None`, `ReLU` nor `simple`. Example: + >>> from torch import rand >>> from torchmetrics.image import MultiScaleStructuralSimilarityIndexMeasure - >>> import torch - >>> gen = torch.manual_seed(42) - >>> preds = torch.rand([3, 3, 256, 256], generator=torch.manual_seed(42)) + >>> preds = torch.rand([3, 3, 256, 256]) >>> target = preds * 0.75 >>> ms_ssim = MultiScaleStructuralSimilarityIndexMeasure(data_range=1.0) >>> ms_ssim(preds, target) @@ -394,9 +393,9 @@ def plot( :scale: 75 >>> # Example plotting a single value + >>> from torch import rand >>> from torchmetrics.image import MultiScaleStructuralSimilarityIndexMeasure - >>> import torch - >>> preds = torch.rand([3, 3, 256, 256], generator=torch.manual_seed(42)) + >>> preds = rand([3, 3, 256, 256]) >>> target = preds * 0.75 >>> metric = MultiScaleStructuralSimilarityIndexMeasure(data_range=1.0) >>> metric.update(preds, target) @@ -406,9 +405,9 @@ def plot( :scale: 75 >>> # Example plotting multiple values + >>> from torch import rand >>> from torchmetrics.image import MultiScaleStructuralSimilarityIndexMeasure - >>> import torch - >>> preds = torch.rand([3, 3, 256, 256], generator=torch.manual_seed(42)) + >>> preds = rand([3, 3, 256, 256]) >>> target = preds * 0.75 >>> metric = MultiScaleStructuralSimilarityIndexMeasure(data_range=1.0) >>> values = [ ] diff --git a/src/torchmetrics/image/tv.py b/src/torchmetrics/image/tv.py index 972a07b745d..ca6276b7e86 100644 --- a/src/torchmetrics/image/tv.py +++ b/src/torchmetrics/image/tv.py @@ -53,9 +53,8 @@ class TotalVariation(Metric): If ``reduction`` is not one of ``'sum'``, ``'mean'``, ``'none'`` or ``None`` Example: - >>> import torch + >>> from torch import rand >>> from torchmetrics.image import TotalVariation - >>> _ = torch.manual_seed(42) >>> tv = TotalVariation() >>> img = torch.rand(5, 3, 28, 28) >>> tv(img) diff --git a/src/torchmetrics/image/vif.py b/src/torchmetrics/image/vif.py index 3e844932399..0358cb80e66 100644 --- a/src/torchmetrics/image/vif.py +++ b/src/torchmetrics/image/vif.py @@ -37,11 +37,10 @@ class VisualInformationFidelity(Metric): kwargs: Additional keyword arguments, see :ref:`Metric kwargs` for more info. Example: - >>> import torch - >>> _ = torch.manual_seed(42) + >>> from torch import randn >>> from torchmetrics.image import VisualInformationFidelity - >>> preds = torch.randn([32, 3, 41, 41]) - >>> target = torch.randn([32, 3, 41, 41]) + >>> preds = randn([32, 3, 41, 41]) + >>> target = randn([32, 3, 41, 41]) >>> vif = VisualInformationFidelity() >>> vif(preds, target) tensor(0.0032) diff --git a/src/torchmetrics/multimodal/clip_iqa.py b/src/torchmetrics/multimodal/clip_iqa.py index b3f934fee33..6ab75356047 100644 --- a/src/torchmetrics/multimodal/clip_iqa.py +++ b/src/torchmetrics/multimodal/clip_iqa.py @@ -128,10 +128,9 @@ class CLIPImageQualityAssessment(Metric): Example:: Single prompt: + >>> from torch import randint >>> from torchmetrics.multimodal import CLIPImageQualityAssessment - >>> import torch - >>> _ = torch.manual_seed(42) - >>> imgs = torch.randint(255, (2, 3, 224, 224)).float() + >>> imgs = randint(255, (2, 3, 224, 224)).float() >>> metric = CLIPImageQualityAssessment() >>> metric(imgs) tensor([0.8894, 0.8902]) @@ -139,10 +138,9 @@ class CLIPImageQualityAssessment(Metric): Example:: Multiple prompts: + >>> from torch import randint >>> from torchmetrics.multimodal import CLIPImageQualityAssessment - >>> import torch - >>> _ = torch.manual_seed(42) - >>> imgs = torch.randint(255, (2, 3, 224, 224)).float() + >>> imgs = randint(255, (2, 3, 224, 224)).float() >>> metric = CLIPImageQualityAssessment(prompts=("quality", "brightness")) >>> metric(imgs) {'quality': tensor([0.8894, 0.8902]), 'brightness': tensor([0.5507, 0.5208])} @@ -150,10 +148,9 @@ class CLIPImageQualityAssessment(Metric): Example:: Custom prompts. Must always be a tuple of length 2, with a positive and negative prompt. + >>> from torch import randint >>> from torchmetrics.multimodal import CLIPImageQualityAssessment - >>> import torch - >>> _ = torch.manual_seed(42) - >>> imgs = torch.randint(255, (2, 3, 224, 224)).float() + >>> imgs = randint(255, (2, 3, 224, 224)).float() >>> metric = CLIPImageQualityAssessment(prompts=(("Super good photo.", "Super bad photo."), "brightness")) >>> metric(imgs) {'user_defined_0': tensor([0.9652, 0.9629]), 'brightness': tensor([0.5507, 0.5208])} diff --git a/src/torchmetrics/multimodal/clip_score.py b/src/torchmetrics/multimodal/clip_score.py index eb432e0be16..47bcbe3b47f 100644 --- a/src/torchmetrics/multimodal/clip_score.py +++ b/src/torchmetrics/multimodal/clip_score.py @@ -82,10 +82,10 @@ class CLIPScore(Metric): If transformers package is not installed or version is lower than 4.10.0 Example: - >>> import torch + >>> from torch import randint >>> from torchmetrics.multimodal.clip_score import CLIPScore >>> metric = CLIPScore(model_name_or_path="openai/clip-vit-base-patch16") - >>> score = metric(torch.randint(255, (3, 224, 224), generator=torch.manual_seed(42)), "a photo of a cat") + >>> score = metric(randint(255, (3, 224, 224)), "a photo of a cat") >>> score.detach() tensor(24.4255) diff --git a/src/torchmetrics/nominal/cramers.py b/src/torchmetrics/nominal/cramers.py index 2d780e7e379..df47cc079d1 100644 --- a/src/torchmetrics/nominal/cramers.py +++ b/src/torchmetrics/nominal/cramers.py @@ -69,10 +69,10 @@ class CramersV(Metric): Example:: + >>> from torch import randint, randn >>> from torchmetrics.nominal import CramersV - >>> _ = torch.manual_seed(42) - >>> preds = torch.randint(0, 4, (100,)) - >>> target = torch.round(preds + torch.randn(100)).clamp(0, 4) + >>> preds = randint(0, 4, (100,)) + >>> target = (preds + randn(100)).round().clamp(0, 4) >>> cramers_v = CramersV(num_classes=5) >>> cramers_v(preds, target) tensor(0.5284) diff --git a/src/torchmetrics/nominal/fleiss_kappa.py b/src/torchmetrics/nominal/fleiss_kappa.py index 9518302cdac..619e8196d4c 100644 --- a/src/torchmetrics/nominal/fleiss_kappa.py +++ b/src/torchmetrics/nominal/fleiss_kappa.py @@ -54,23 +54,21 @@ class FleissKappa(Metric): Example: >>> # Ratings are provided as counts - >>> import torch + >>> from torch import randint >>> from torchmetrics.nominal import FleissKappa - >>> _ = torch.manual_seed(42) - >>> ratings = torch.randint(0, 10, size=(100, 5)).long() # 100 samples, 5 categories, 10 raters + >>> ratings = randint(0, 10, size=(100, 5)).long() # 100 samples, 5 categories, 10 raters >>> metric = FleissKappa(mode='counts') >>> metric(ratings) tensor(0.0089) Example: >>> # Ratings are provided as probabilities - >>> import torch + >>> from torch import randn >>> from torchmetrics.nominal import FleissKappa - >>> _ = torch.manual_seed(42) - >>> ratings = torch.randn(100, 5, 10).softmax(dim=1) # 100 samples, 5 categories, 10 raters + >>> ratings = randn(100, 5, 10).softmax(dim=1) # 100 samples, 5 categories, 10 raters >>> metric = FleissKappa(mode='probs') >>> metric(ratings) - tensor(-0.0105) + tensor(-0.0075) """ diff --git a/src/torchmetrics/nominal/pearson.py b/src/torchmetrics/nominal/pearson.py index 2fc88c8e851..a43cd548792 100644 --- a/src/torchmetrics/nominal/pearson.py +++ b/src/torchmetrics/nominal/pearson.py @@ -73,10 +73,10 @@ class PearsonsContingencyCoefficient(Metric): Example:: + >>> from torch import randint, randn >>> from torchmetrics.nominal import PearsonsContingencyCoefficient - >>> _ = torch.manual_seed(42) - >>> preds = torch.randint(0, 4, (100,)) - >>> target = torch.round(preds + torch.randn(100)).clamp(0, 4) + >>> preds = randint(0, 4, (100,)) + >>> target = (preds + randn(100)).round().clamp(0, 4) >>> pearsons_contingency_coefficient = PearsonsContingencyCoefficient(num_classes=5) >>> pearsons_contingency_coefficient(preds, target) tensor(0.6948) diff --git a/src/torchmetrics/nominal/theils_u.py b/src/torchmetrics/nominal/theils_u.py index f82c7658b1f..7f7f22ecb1b 100644 --- a/src/torchmetrics/nominal/theils_u.py +++ b/src/torchmetrics/nominal/theils_u.py @@ -59,10 +59,10 @@ class TheilsU(Metric): Example:: + >>> from torch import randint >>> from torchmetrics.nominal import TheilsU - >>> _ = torch.manual_seed(42) - >>> preds = torch.randint(10, (10,)) - >>> target = torch.randint(10, (10,)) + >>> preds = randint(10, (10,)) + >>> target = randint(10, (10,)) >>> metric = TheilsU(num_classes=10) >>> metric(preds, target) tensor(0.8530) diff --git a/src/torchmetrics/nominal/tschuprows.py b/src/torchmetrics/nominal/tschuprows.py index a14832b4121..9986fa2ec6f 100644 --- a/src/torchmetrics/nominal/tschuprows.py +++ b/src/torchmetrics/nominal/tschuprows.py @@ -69,10 +69,10 @@ class TschuprowsT(Metric): Example:: + >>> from torch import randint >>> from torchmetrics.nominal import TschuprowsT - >>> _ = torch.manual_seed(42) - >>> preds = torch.randint(0, 4, (100,)) - >>> target = torch.round(preds + torch.randn(100)).clamp(0, 4) + >>> preds = randint(0, 4, (100,)) + >>> target = (preds + torch.randn(100)).round().clamp(0, 4) >>> tschuprows_t = TschuprowsT(num_classes=5) >>> tschuprows_t(preds, target) tensor(0.4930) diff --git a/src/torchmetrics/regression/wmape.py b/src/torchmetrics/regression/wmape.py index 067c995bfb2..42fadb906ef 100644 --- a/src/torchmetrics/regression/wmape.py +++ b/src/torchmetrics/regression/wmape.py @@ -51,10 +51,9 @@ class WeightedMeanAbsolutePercentageError(Metric): kwargs: Additional keyword arguments, see :ref:`Metric kwargs` for more info. Example: - >>> import torch - >>> _ = torch.manual_seed(42) - >>> preds = torch.randn(20,) - >>> target = torch.randn(20,) + >>> from torch import randn + >>> preds = randn(20,) + >>> target = randn(20,) >>> wmape = WeightedMeanAbsolutePercentageError() >>> wmape(preds, target) tensor(1.3967) diff --git a/src/torchmetrics/segmentation/generalized_dice.py b/src/torchmetrics/segmentation/generalized_dice.py index 66f09437000..95da9ab26d9 100644 --- a/src/torchmetrics/segmentation/generalized_dice.py +++ b/src/torchmetrics/segmentation/generalized_dice.py @@ -89,20 +89,19 @@ class GeneralizedDiceScore(Metric): If ``input_format`` is not one of ``"one-hot"`` or ``"index"`` Example: - >>> import torch - >>> _ = torch.manual_seed(0) + >>> from torch import randint >>> from torchmetrics.segmentation import GeneralizedDiceScore >>> gds = GeneralizedDiceScore(num_classes=3) - >>> preds = torch.randint(0, 2, (10, 3, 128, 128)) - >>> target = torch.randint(0, 2, (10, 3, 128, 128)) + >>> preds = randint(0, 2, (10, 3, 128, 128)) + >>> target = randint(0, 2, (10, 3, 128, 128)) >>> gds(preds, target) - tensor(0.4983) + tensor(0.4992) >>> gds = GeneralizedDiceScore(num_classes=3, per_class=True) >>> gds(preds, target) - tensor([0.4987, 0.4966, 0.4995]) + tensor([0.5001, 0.4993, 0.4982]) >>> gds = GeneralizedDiceScore(num_classes=3, per_class=True, include_background=False) >>> gds(preds, target) - tensor([0.4966, 0.4995]) + tensor([0.4993, 0.4982]) """ diff --git a/src/torchmetrics/segmentation/mean_iou.py b/src/torchmetrics/segmentation/mean_iou.py index 77d465ebd21..c298254585a 100644 --- a/src/torchmetrics/segmentation/mean_iou.py +++ b/src/torchmetrics/segmentation/mean_iou.py @@ -70,20 +70,19 @@ class MeanIoU(Metric): If ``input_format`` is not one of ``"one-hot"`` or ``"index"`` Example: - >>> import torch - >>> _ = torch.manual_seed(0) + >>> from torch import randint >>> from torchmetrics.segmentation import MeanIoU >>> miou = MeanIoU(num_classes=3) - >>> preds = torch.randint(0, 2, (10, 3, 128, 128)) - >>> target = torch.randint(0, 2, (10, 3, 128, 128)) + >>> preds = randint(0, 2, (10, 3, 128, 128)) + >>> target = randint(0, 2, (10, 3, 128, 128)) >>> miou(preds, target) - tensor(0.3318) + tensor(0.3326) >>> miou = MeanIoU(num_classes=3, per_class=True) >>> miou(preds, target) - tensor([0.3322, 0.3303, 0.3329]) + tensor([0.3334, 0.3327, 0.3318]) >>> miou = MeanIoU(num_classes=3, per_class=True, include_background=False) >>> miou(preds, target) - tensor([0.3303, 0.3329]) + tensor([0.3327, 0.3318]) """ diff --git a/src/torchmetrics/text/_deprecated.py b/src/torchmetrics/text/_deprecated.py index d3ba1c4010e..0c7ffef29af 100644 --- a/src/torchmetrics/text/_deprecated.py +++ b/src/torchmetrics/text/_deprecated.py @@ -144,10 +144,9 @@ def __init__( class _Perplexity(Perplexity): """Wrapper for deprecated import. - >>> import torch - >>> gen = torch.manual_seed(42) - >>> preds = torch.rand(2, 8, 5, generator=gen) - >>> target = torch.randint(5, (2, 8), generator=gen) + >>> from torch import rand, randint + >>> preds = rand(2, 8, 5) + >>> target = randint(5, (2, 8)) >>> target[0, 6:] = -100 >>> perp = _Perplexity(ignore_index=-100) >>> perp(preds, target) diff --git a/src/torchmetrics/text/perplexity.py b/src/torchmetrics/text/perplexity.py index 51804881df8..d13eac2f402 100644 --- a/src/torchmetrics/text/perplexity.py +++ b/src/torchmetrics/text/perplexity.py @@ -48,11 +48,10 @@ class Perplexity(Metric): Additional keyword arguments, see :ref:`Metric kwargs` for more info. Examples: + >>> from torch import rand, randint >>> from torchmetrics.text import Perplexity - >>> import torch - >>> gen = torch.manual_seed(42) - >>> preds = torch.rand(2, 8, 5, generator=gen) - >>> target = torch.randint(5, (2, 8), generator=gen) + >>> preds = rand(2, 8, 5) + >>> target = randint(5, (2, 8)) >>> target[0, 6:] = -100 >>> perp = Perplexity(ignore_index=-100) >>> perp(preds, target) diff --git a/src/torchmetrics/wrappers/bootstrapping.py b/src/torchmetrics/wrappers/bootstrapping.py index 9b904568b65..d59f7724c2a 100644 --- a/src/torchmetrics/wrappers/bootstrapping.py +++ b/src/torchmetrics/wrappers/bootstrapping.py @@ -75,15 +75,15 @@ class basically keeps multiple copies of the same base metric in memory and when Example:: >>> from pprint import pprint + >>> from torch import randint >>> from torchmetrics.wrappers import BootStrapper >>> from torchmetrics.classification import MulticlassAccuracy - >>> _ = torch.manual_seed(123) >>> base_metric = MulticlassAccuracy(num_classes=5, average='micro') >>> bootstrap = BootStrapper(base_metric, num_bootstraps=20) - >>> bootstrap.update(torch.randint(5, (20,)), torch.randint(5, (20,))) + >>> bootstrap.update(randint(5, (20,)), randint(5, (20,))) >>> output = bootstrap.compute() >>> pprint(output) - {'mean': tensor(0.2205), 'std': tensor(0.0859)} + {'mean': tensor(0.2089), 'std': tensor(0.0772)} """ diff --git a/src/torchmetrics/wrappers/classwise.py b/src/torchmetrics/wrappers/classwise.py index 78cb27ae46c..217c94d6bc0 100644 --- a/src/torchmetrics/wrappers/classwise.py +++ b/src/torchmetrics/wrappers/classwise.py @@ -44,58 +44,54 @@ class ClasswiseWrapper(WrapperMetric): Example:: Basic example where the output of a metric is unwrapped into a dictionary with the class index as keys: - >>> import torch - >>> _ = torch.manual_seed(42) + >>> from torch import randint, randn >>> from torchmetrics.wrappers import ClasswiseWrapper >>> from torchmetrics.classification import MulticlassAccuracy >>> metric = ClasswiseWrapper(MulticlassAccuracy(num_classes=3, average=None)) - >>> preds = torch.randn(10, 3).softmax(dim=-1) - >>> target = torch.randint(3, (10,)) + >>> preds = randn(10, 3).softmax(dim=-1) + >>> target = randint(3, (10,)) >>> metric(preds, target) # doctest: +NORMALIZE_WHITESPACE {'multiclassaccuracy_0': tensor(0.5000), - 'multiclassaccuracy_1': tensor(0.7500), - 'multiclassaccuracy_2': tensor(0.)} + 'multiclassaccuracy_1': tensor(0.7500), + 'multiclassaccuracy_2': tensor(0.)} Example:: Using custom name via prefix and postfix: - >>> import torch - >>> _ = torch.manual_seed(42) + >>> from torch import randint, randn >>> from torchmetrics.wrappers import ClasswiseWrapper >>> from torchmetrics.classification import MulticlassAccuracy >>> metric_pre = ClasswiseWrapper(MulticlassAccuracy(num_classes=3, average=None), prefix="acc-") >>> metric_post = ClasswiseWrapper(MulticlassAccuracy(num_classes=3, average=None), postfix="-acc") - >>> preds = torch.randn(10, 3).softmax(dim=-1) - >>> target = torch.randint(3, (10,)) + >>> preds = randn(10, 3).softmax(dim=-1) + >>> target = randint(3, (10,)) >>> metric_pre(preds, target) # doctest: +NORMALIZE_WHITESPACE - {'acc-0': tensor(0.5000), - 'acc-1': tensor(0.7500), - 'acc-2': tensor(0.)} + {'acc-0': tensor(0.3333), 'acc-1': tensor(0.6667), 'acc-2': tensor(0.)} >>> metric_post(preds, target) # doctest: +NORMALIZE_WHITESPACE - {'0-acc': tensor(0.5000), - '1-acc': tensor(0.7500), - '2-acc': tensor(0.)} + {'0-acc': tensor(0.3333), '1-acc': tensor(0.6667), '2-acc': tensor(0.)} Example:: Providing labels as a list of strings: + >>> from torch import randint, randn >>> from torchmetrics.wrappers import ClasswiseWrapper >>> from torchmetrics.classification import MulticlassAccuracy >>> metric = ClasswiseWrapper( ... MulticlassAccuracy(num_classes=3, average=None), ... labels=["horse", "fish", "dog"] ... ) - >>> preds = torch.randn(10, 3).softmax(dim=-1) - >>> target = torch.randint(3, (10,)) + >>> preds = randn(10, 3).softmax(dim=-1) + >>> target = randint(3, (10,)) >>> metric(preds, target) # doctest: +NORMALIZE_WHITESPACE - {'multiclassaccuracy_horse': tensor(0.3333), - 'multiclassaccuracy_fish': tensor(0.6667), - 'multiclassaccuracy_dog': tensor(0.)} + {'multiclassaccuracy_horse': tensor(0.), + 'multiclassaccuracy_fish': tensor(0.3333), + 'multiclassaccuracy_dog': tensor(0.4000)} Example:: Classwise can also be used in combination with :class:`~torchmetrics.MetricCollection`. In this case, everything will be flattened into a single dictionary: + >>> from torch import randint, randn >>> from torchmetrics import MetricCollection >>> from torchmetrics.wrappers import ClasswiseWrapper >>> from torchmetrics.classification import MulticlassAccuracy, MulticlassRecall @@ -104,15 +100,15 @@ class ClasswiseWrapper(WrapperMetric): ... {'multiclassaccuracy': ClasswiseWrapper(MulticlassAccuracy(num_classes=3, average=None), labels), ... 'multiclassrecall': ClasswiseWrapper(MulticlassRecall(num_classes=3, average=None), labels)} ... ) - >>> preds = torch.randn(10, 3).softmax(dim=-1) - >>> target = torch.randint(3, (10,)) + >>> preds = randn(10, 3).softmax(dim=-1) + >>> target = randint(3, (10,)) >>> metric(preds, target) # doctest: +NORMALIZE_WHITESPACE - {'multiclassaccuracy_horse': tensor(0.), + {'multiclassaccuracy_horse': tensor(0.6667), 'multiclassaccuracy_fish': tensor(0.3333), - 'multiclassaccuracy_dog': tensor(0.4000), - 'multiclassrecall_horse': tensor(0.), + 'multiclassaccuracy_dog': tensor(0.5000), + 'multiclassrecall_horse': tensor(0.6667), 'multiclassrecall_fish': tensor(0.3333), - 'multiclassrecall_dog': tensor(0.4000)} + 'multiclassrecall_dog': tensor(0.5000)} """ diff --git a/src/torchmetrics/wrappers/feature_share.py b/src/torchmetrics/wrappers/feature_share.py index b1fe0451534..1bd1b81783b 100644 --- a/src/torchmetrics/wrappers/feature_share.py +++ b/src/torchmetrics/wrappers/feature_share.py @@ -68,7 +68,6 @@ class FeatureShare(MetricCollection): Example:: >>> import torch - >>> _ = torch.manual_seed(42) >>> from torchmetrics.wrappers import FeatureShare >>> from torchmetrics.image import FrechetInceptionDistance, KernelInceptionDistance >>> # initialize the metrics diff --git a/src/torchmetrics/wrappers/tracker.py b/src/torchmetrics/wrappers/tracker.py index 2a32ee575fe..554902d1092 100644 --- a/src/torchmetrics/wrappers/tracker.py +++ b/src/torchmetrics/wrappers/tracker.py @@ -52,15 +52,14 @@ class MetricTracker(ModuleList): better (``True``) or lower is better (``False``). Example (single metric): + >>> from torch import randint >>> from torchmetrics.wrappers import MetricTracker >>> from torchmetrics.classification import MulticlassAccuracy - >>> _ = torch.manual_seed(42) >>> tracker = MetricTracker(MulticlassAccuracy(num_classes=10, average='micro')) >>> for epoch in range(5): ... tracker.increment() ... for batch_idx in range(5): - ... preds, target = torch.randint(10, (100,)), torch.randint(10, (100,)) - ... tracker.update(preds, target) + ... tracker.update(randint(10, (100,)), randint(10, (100,))) ... print(f"current acc={tracker.compute()}") current acc=0.1120000034570694 current acc=0.08799999952316284 @@ -76,32 +75,31 @@ class MetricTracker(ModuleList): tensor([0.1120, 0.0880, 0.1260, 0.0800, 0.1020]) Example (multiple metrics using MetricCollection): + >>> from torch import randn >>> from torchmetrics.wrappers import MetricTracker >>> from torchmetrics import MetricCollection >>> from torchmetrics.regression import MeanSquaredError, ExplainedVariance - >>> _ = torch.manual_seed(42) >>> tracker = MetricTracker(MetricCollection([MeanSquaredError(), ExplainedVariance()]), maximize=[False, True]) >>> for epoch in range(5): ... tracker.increment() ... for batch_idx in range(5): - ... preds, target = torch.randn(100), torch.randn(100) - ... tracker.update(preds, target) + ... tracker.update(randn(100), randn(100)) ... print(f"current stats={tracker.compute()}") # doctest: +NORMALIZE_WHITESPACE - current stats={'MeanSquaredError': tensor(1.8218), 'ExplainedVariance': tensor(-0.8969)} - current stats={'MeanSquaredError': tensor(2.0268), 'ExplainedVariance': tensor(-1.0206)} - current stats={'MeanSquaredError': tensor(1.9491), 'ExplainedVariance': tensor(-0.8298)} - current stats={'MeanSquaredError': tensor(1.9800), 'ExplainedVariance': tensor(-0.9199)} - current stats={'MeanSquaredError': tensor(2.2481), 'ExplainedVariance': tensor(-1.1622)} + current stats={'MeanSquaredError': tensor(2.3292), 'ExplainedVariance': tensor(-0.9516)} + current stats={'MeanSquaredError': tensor(2.1370), 'ExplainedVariance': tensor(-1.0775)} + current stats={'MeanSquaredError': tensor(2.1695), 'ExplainedVariance': tensor(-0.9945)} + current stats={'MeanSquaredError': tensor(2.1072), 'ExplainedVariance': tensor(-1.1878)} + current stats={'MeanSquaredError': tensor(2.0562), 'ExplainedVariance': tensor(-1.0754)} >>> from pprint import pprint >>> best_res, which_epoch = tracker.best_metric(return_step=True) >>> pprint(best_res) # doctest: +ELLIPSIS - {'ExplainedVariance': -0.829..., - 'MeanSquaredError': 1.821...} + {'ExplainedVariance': -0.951..., + 'MeanSquaredError': 2.056...} >>> which_epoch - {'MeanSquaredError': 0, 'ExplainedVariance': 2} + {'MeanSquaredError': 4, 'ExplainedVariance': 0} >>> pprint(tracker.compute_all()) - {'ExplainedVariance': tensor([-0.8969, -1.0206, -0.8298, -0.9199, -1.1622]), - 'MeanSquaredError': tensor([1.8218, 2.0268, 1.9491, 1.9800, 2.2481])} + {'ExplainedVariance': tensor([-0.9516, -1.0775, -0.9945, -1.1878, -1.0754]), + 'MeanSquaredError': tensor([2.3292, 2.1370, 2.1695, 2.1072, 2.0562])} """ From 99356f420bb62407bf5464407035bb57342ee7d3 Mon Sep 17 00:00:00 2001 From: jirka Date: Thu, 29 Aug 2024 23:49:35 +0200 Subject: [PATCH 02/11] seed_all --- tests/unittests/segmentation/test_generalized_dice_score.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/unittests/segmentation/test_generalized_dice_score.py b/tests/unittests/segmentation/test_generalized_dice_score.py index 2916ea6d443..f7e43ef2b56 100644 --- a/tests/unittests/segmentation/test_generalized_dice_score.py +++ b/tests/unittests/segmentation/test_generalized_dice_score.py @@ -21,8 +21,11 @@ from torchmetrics.segmentation.generalized_dice import GeneralizedDiceScore from unittests import BATCH_SIZE, NUM_BATCHES, NUM_CLASSES, _Input +from unittests._helpers import seed_all from unittests._helpers.testers import MetricTester +seed_all(42) + _inputs1 = _Input( preds=torch.randint(0, 2, (NUM_BATCHES, BATCH_SIZE, NUM_CLASSES, 16)), target=torch.randint(0, 2, (NUM_BATCHES, BATCH_SIZE, NUM_CLASSES, 16)), From 015f3bc6626197f5b8de7faf9b29da30a7a48762 Mon Sep 17 00:00:00 2001 From: jirka Date: Fri, 30 Aug 2024 00:12:53 +0200 Subject: [PATCH 03/11] tests --- src/torchmetrics/image/perceptual_path_length.py | 4 ++-- src/torchmetrics/multimodal/clip_iqa.py | 2 +- src/torchmetrics/multimodal/clip_score.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/torchmetrics/image/perceptual_path_length.py b/src/torchmetrics/image/perceptual_path_length.py index 4e8cca23343..dcfca43f8d5 100644 --- a/src/torchmetrics/image/perceptual_path_length.py +++ b/src/torchmetrics/image/perceptual_path_length.py @@ -112,9 +112,9 @@ class PerceptualPathLength(Metric): >>> generator = DummyGenerator(2) >>> ppl = PerceptualPathLength(num_samples=10) >>> ppl(generator) - (tensor(0.2369), + (tensor(0.2371), tensor(0.1760), - tensor([0.3489, 0.1363, 0.2535, 0.0907, 0.1785, 0.0769, 0.5864, 0.0691, 0.3920])) + tensor([0.3494, 0.1367, 0.2531, 0.0907, 0.1789, 0.0769, 0.5863, 0.0691, 0.3926])) """ diff --git a/src/torchmetrics/multimodal/clip_iqa.py b/src/torchmetrics/multimodal/clip_iqa.py index 6ab75356047..4d0794a6425 100644 --- a/src/torchmetrics/multimodal/clip_iqa.py +++ b/src/torchmetrics/multimodal/clip_iqa.py @@ -143,7 +143,7 @@ class CLIPImageQualityAssessment(Metric): >>> imgs = randint(255, (2, 3, 224, 224)).float() >>> metric = CLIPImageQualityAssessment(prompts=("quality", "brightness")) >>> metric(imgs) - {'quality': tensor([0.8894, 0.8902]), 'brightness': tensor([0.5507, 0.5208])} + {'quality': tensor([0.8693, 0.8705]), 'brightness': tensor([0.5722, 0.4762])} Example:: Custom prompts. Must always be a tuple of length 2, with a positive and negative prompt. diff --git a/src/torchmetrics/multimodal/clip_score.py b/src/torchmetrics/multimodal/clip_score.py index 47bcbe3b47f..14e5a8e2ec9 100644 --- a/src/torchmetrics/multimodal/clip_score.py +++ b/src/torchmetrics/multimodal/clip_score.py @@ -87,7 +87,7 @@ class CLIPScore(Metric): >>> metric = CLIPScore(model_name_or_path="openai/clip-vit-base-patch16") >>> score = metric(randint(255, (3, 224, 224)), "a photo of a cat") >>> score.detach() - tensor(24.4255) + tensor(24.8020) """ From 58274c87010e487f09413ca275a9fc89b76f616c Mon Sep 17 00:00:00 2001 From: jirka Date: Fri, 30 Aug 2024 10:10:35 +0200 Subject: [PATCH 04/11] test --- src/torchmetrics/multimodal/clip_iqa.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/torchmetrics/multimodal/clip_iqa.py b/src/torchmetrics/multimodal/clip_iqa.py index 4d0794a6425..7a66de4ae3e 100644 --- a/src/torchmetrics/multimodal/clip_iqa.py +++ b/src/torchmetrics/multimodal/clip_iqa.py @@ -153,7 +153,7 @@ class CLIPImageQualityAssessment(Metric): >>> imgs = randint(255, (2, 3, 224, 224)).float() >>> metric = CLIPImageQualityAssessment(prompts=(("Super good photo.", "Super bad photo."), "brightness")) >>> metric(imgs) - {'user_defined_0': tensor([0.9652, 0.9629]), 'brightness': tensor([0.5507, 0.5208])} + {'user_defined_0': tensor([0.9578, 0.9654]), 'brightness': tensor([0.5495, 0.5764])} """ From 112af41b911274d9d69cca520a402725e55c1bb6 Mon Sep 17 00:00:00 2001 From: jirka Date: Fri, 30 Aug 2024 10:52:08 +0200 Subject: [PATCH 05/11] test --- src/torchmetrics/image/perceptual_path_length.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/torchmetrics/image/perceptual_path_length.py b/src/torchmetrics/image/perceptual_path_length.py index dcfca43f8d5..4c312404934 100644 --- a/src/torchmetrics/image/perceptual_path_length.py +++ b/src/torchmetrics/image/perceptual_path_length.py @@ -99,7 +99,6 @@ class PerceptualPathLength(Metric): Example:: >>> import torch - >>> from torchmetrics.image import PerceptualPathLength >>> class DummyGenerator(torch.nn.Module): ... def __init__(self, z_size) -> None: ... super().__init__() @@ -112,9 +111,7 @@ class PerceptualPathLength(Metric): >>> generator = DummyGenerator(2) >>> ppl = PerceptualPathLength(num_samples=10) >>> ppl(generator) - (tensor(0.2371), - tensor(0.1760), - tensor([0.3494, 0.1367, 0.2531, 0.0907, 0.1789, 0.0769, 0.5863, 0.0691, 0.3926])) + (tensor(...), tensor(...), tensor([...])) """ From fcfdaea8864fd17a28a054945240dd6f4fe751a3 Mon Sep 17 00:00:00 2001 From: jirka Date: Fri, 30 Aug 2024 11:18:41 +0200 Subject: [PATCH 06/11] test --- src/torchmetrics/image/perceptual_path_length.py | 4 ++-- src/torchmetrics/multimodal/clip_score.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/torchmetrics/image/perceptual_path_length.py b/src/torchmetrics/image/perceptual_path_length.py index 4c312404934..e5873723f06 100644 --- a/src/torchmetrics/image/perceptual_path_length.py +++ b/src/torchmetrics/image/perceptual_path_length.py @@ -23,9 +23,9 @@ perceptual_path_length, ) from torchmetrics.metric import Metric -from torchmetrics.utilities.imports import _TORCHVISION_AVAILABLE +from torchmetrics.utilities.imports import _TORCHVISION_AVAILABLE, _TORCH_GREATER_EQUAL_2_0 -if not _TORCHVISION_AVAILABLE: +if not _TORCHVISION_AVAILABLE or not _TORCH_GREATER_EQUAL_2_0: __doctest_skip__ = ["PerceptualPathLength"] diff --git a/src/torchmetrics/multimodal/clip_score.py b/src/torchmetrics/multimodal/clip_score.py index 14e5a8e2ec9..16a90c84168 100644 --- a/src/torchmetrics/multimodal/clip_score.py +++ b/src/torchmetrics/multimodal/clip_score.py @@ -86,7 +86,7 @@ class CLIPScore(Metric): >>> from torchmetrics.multimodal.clip_score import CLIPScore >>> metric = CLIPScore(model_name_or_path="openai/clip-vit-base-patch16") >>> score = metric(randint(255, (3, 224, 224)), "a photo of a cat") - >>> score.detach() + >>> score.detach().round(decimals=3) tensor(24.8020) """ From 87d27a36b2d87f129c72190bc950bd46973355e8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 30 Aug 2024 09:19:05 +0000 Subject: [PATCH 07/11] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/torchmetrics/image/perceptual_path_length.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/torchmetrics/image/perceptual_path_length.py b/src/torchmetrics/image/perceptual_path_length.py index e5873723f06..b2255941250 100644 --- a/src/torchmetrics/image/perceptual_path_length.py +++ b/src/torchmetrics/image/perceptual_path_length.py @@ -23,7 +23,7 @@ perceptual_path_length, ) from torchmetrics.metric import Metric -from torchmetrics.utilities.imports import _TORCHVISION_AVAILABLE, _TORCH_GREATER_EQUAL_2_0 +from torchmetrics.utilities.imports import _TORCH_GREATER_EQUAL_2_0, _TORCHVISION_AVAILABLE if not _TORCHVISION_AVAILABLE or not _TORCH_GREATER_EQUAL_2_0: __doctest_skip__ = ["PerceptualPathLength"] From a9b004cd7481fe167b31e20ff84444408fa4402c Mon Sep 17 00:00:00 2001 From: jirka Date: Fri, 30 Aug 2024 11:41:42 +0200 Subject: [PATCH 08/11] test --- src/torchmetrics/multimodal/clip_score.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/torchmetrics/multimodal/clip_score.py b/src/torchmetrics/multimodal/clip_score.py index 16a90c84168..5a63b96530e 100644 --- a/src/torchmetrics/multimodal/clip_score.py +++ b/src/torchmetrics/multimodal/clip_score.py @@ -86,7 +86,7 @@ class CLIPScore(Metric): >>> from torchmetrics.multimodal.clip_score import CLIPScore >>> metric = CLIPScore(model_name_or_path="openai/clip-vit-base-patch16") >>> score = metric(randint(255, (3, 224, 224)), "a photo of a cat") - >>> score.detach().round(decimals=3) + >>> score.round(decimals=3).detach() tensor(24.8020) """ From 5a31692a7985401ed78c4b41f8ad15ae5d007f44 Mon Sep 17 00:00:00 2001 From: jirka Date: Fri, 30 Aug 2024 12:00:54 +0200 Subject: [PATCH 09/11] test --- src/torchmetrics/multimodal/clip_score.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/torchmetrics/multimodal/clip_score.py b/src/torchmetrics/multimodal/clip_score.py index 5a63b96530e..4c6bbc7482d 100644 --- a/src/torchmetrics/multimodal/clip_score.py +++ b/src/torchmetrics/multimodal/clip_score.py @@ -82,11 +82,11 @@ class CLIPScore(Metric): If transformers package is not installed or version is lower than 4.10.0 Example: - >>> from torch import randint + >>> from torch import randint, round >>> from torchmetrics.multimodal.clip_score import CLIPScore >>> metric = CLIPScore(model_name_or_path="openai/clip-vit-base-patch16") >>> score = metric(randint(255, (3, 224, 224)), "a photo of a cat") - >>> score.round(decimals=3).detach() + >>> round(score.detach(), decimals=3) tensor(24.8020) """ From 5da59ab9fb82bb209bf3965b0546eb0e86163eb9 Mon Sep 17 00:00:00 2001 From: jirka Date: Fri, 30 Aug 2024 12:24:53 +0200 Subject: [PATCH 10/11] test --- src/torchmetrics/multimodal/clip_score.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/torchmetrics/multimodal/clip_score.py b/src/torchmetrics/multimodal/clip_score.py index 4c6bbc7482d..0ec5d83ec3a 100644 --- a/src/torchmetrics/multimodal/clip_score.py +++ b/src/torchmetrics/multimodal/clip_score.py @@ -86,7 +86,7 @@ class CLIPScore(Metric): >>> from torchmetrics.multimodal.clip_score import CLIPScore >>> metric = CLIPScore(model_name_or_path="openai/clip-vit-base-patch16") >>> score = metric(randint(255, (3, 224, 224)), "a photo of a cat") - >>> round(score.detach(), decimals=3) + >>> score.detach().round(decimals=3) tensor(24.8020) """ From f7ccd50c2dc89203813577129d8f5b83b9391d40 Mon Sep 17 00:00:00 2001 From: jirka Date: Fri, 30 Aug 2024 12:42:14 +0200 Subject: [PATCH 11/11] test --- src/torchmetrics/multimodal/clip_score.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/torchmetrics/multimodal/clip_score.py b/src/torchmetrics/multimodal/clip_score.py index 0ec5d83ec3a..f385fbc145d 100644 --- a/src/torchmetrics/multimodal/clip_score.py +++ b/src/torchmetrics/multimodal/clip_score.py @@ -82,12 +82,12 @@ class CLIPScore(Metric): If transformers package is not installed or version is lower than 4.10.0 Example: - >>> from torch import randint, round + >>> from torch import randint >>> from torchmetrics.multimodal.clip_score import CLIPScore >>> metric = CLIPScore(model_name_or_path="openai/clip-vit-base-patch16") >>> score = metric(randint(255, (3, 224, 224)), "a photo of a cat") - >>> score.detach().round(decimals=3) - tensor(24.8020) + >>> score.detach().round() + tensor(25.) """