Update changelog

CHANGELOG.md

@@ -27,6 +27,8 @@ All notable changes to this project will be documented in this file.
   (<https://github.com/openvinotoolkit/training_extensions/pull/4073>)
 - Upgrade OpenVINO to 2024.5 and NNCF to 2.14.0
   (<https://github.com/openvinotoolkit/training_extensions/pull/4123>)
+- Improve FMetric computation
+  (<https://github.com/openvinotoolkit/training_extensions/pull/4130>)
 
 ### Bug fixes
 

src/otx/core/metrics/fmeasure.py

@@ -25,31 +25,6 @@
 ALL_CLASSES_NAME = "All Classes"
 
 
-def _get_n_false_negatives(iou_matrix: np.ndarray, iou_threshold: float) -> int:
-    """Get the number of false negatives inside the IoU matrix for a given threshold.
-
-    The first loop accounts for all the ground truth boxes which do not have a high enough iou with any predicted
-    box (they go undetected)
-    The second loop accounts for the much rarer case where two ground truth boxes are detected by the same predicted
-    box. The principle is that each ground truth box requires a unique prediction box
-
-    Args:
-        iou_matrix (np.ndarray): IoU matrix of shape [ground_truth_boxes, predicted_boxes]
-        iou_threshold (float): IoU threshold to use for the false negatives.
-
-    Returns:
-        int: Number of false negatives
-    """
-    n_false_negatives = 0
-    for row in iou_matrix:
-        if max(row) < iou_threshold:
-            n_false_negatives += 1
-    for column in np.rot90(iou_matrix):
-        indices = np.where(column > iou_threshold)
-        n_false_negatives += max(len(indices[0]) - 1, 0)
-    return n_false_negatives
-
-
 def get_n_false_negatives(iou_matrix: Tensor, iou_threshold: float) -> Tensor:
     """Get the number of false negatives inside the IoU matrix for a given threshold.
 
@@ -59,11 +34,11 @@ def get_n_false_negatives(iou_matrix: Tensor, iou_threshold: float) -> Tensor:
     box. The principle is that each ground truth box requires a unique prediction box
 
     Args:
-        iou_matrix (np.ndarray): IoU matrix of shape [ground_truth_boxes, predicted_boxes]
+        iou_matrix (torch.Tensor): IoU matrix of shape [ground_truth_boxes, predicted_boxes]
         iou_threshold (float): IoU threshold to use for the false negatives.
 
     Returns:
-        int: Number of false negatives
+        Tensor: Number of false negatives
     """
     # First loop
     n_false_negatives = 0
@@ -175,8 +150,7 @@ class _FMeasureCalculator:
     """This class contains the functions to calculate FMeasure.
 
     Args:
-        gt_entities (list[DetDataEntity]): list of ground truth detection entities.
-        pred_entities (list[DetPredEntity]): list of predicted detection entities.
+       classes (list[str]): List of classes.
     """
 
     def __init__(self, classes: list[str]):
@@ -201,7 +175,8 @@ def evaluate_detections(
         used to achieve them.
 
         Args:
-            classes (list[str]): Names of classes to be evaluated.
+            gt_entities (list[DetDataEntity]): List of ground truth entities.
+            pred_entities (list[DetPredEntity]): List of predicted entities.
             iou_threshold (float): IOU threshold. Defaults to 0.5.
 
         Returns:
@@ -243,6 +218,8 @@ def _get_results_per_confidence(
 
         Args:
             classes (list[str]): Names of classes to be evaluated.
+            gt_entities (list[DetDataEntity]): List of ground truth entities.
+            pred_entities (list[DetPredEntity]): List of predicted entities.
             confidence_range (list[float]): list of confidence thresholds to be evaluated.
             iou_threshold (float): IoU threshold to use for false negatives.
 
@@ -329,7 +306,7 @@ def get_f_measure_for_class(
         all boxes are filtered at this stage by class and predicted boxes are filtered by confidence threshold
 
         Args:
-            gt_entites (list[DetDataEntity]): List of ground truth entities.
+            gt_entities (list[DetDataEntity]): List of ground truth entities.
             pred_entities (list[DetPredEntity]): List of predicted entities.
             label_idx (int): Index of the class for which the boxes are filtered.
             iou_threshold (float): IoU threshold

tests/unit/core/metrics/test_fmeasure.py

@@ -5,9 +5,10 @@
 
 from __future__ import annotations
 
+import numpy as np
 import pytest
 import torch
-from otx.core.metrics.fmeasure import FMeasure
+from otx.core.metrics.fmeasure import FMeasure, get_n_false_negatives
 from otx.core.types.label import LabelInfo
 
 
@@ -65,3 +66,22 @@ def test_fmeasure_with_fixed_threshold(self, fxt_preds, fxt_targets) -> None:
         metric.update(fxt_preds, fxt_targets)
         result = metric.compute(best_confidence_threshold=0.85)
         assert result["f1-score"] == 0.3333333432674408
+
+    def test_get_fn(self):
+        def _get_n_false_negatives_numpy(iou_matrix: np.ndarray, iou_threshold: float) -> int:
+            n_false_negatives = 0
+            for row in iou_matrix:
+                if max(row) < iou_threshold:
+                    n_false_negatives += 1
+            for column in np.rot90(iou_matrix):
+                indices = np.where(column > iou_threshold)
+                n_false_negatives += max(len(indices[0]) - 1, 0)
+            return n_false_negatives
+
+        iou_matrix = torch.rand((10, 20))
+        iou_threshold = np.random.rand()
+
+        assert get_n_false_negatives(iou_matrix, iou_threshold) == _get_n_false_negatives_numpy(
+            iou_matrix.numpy(),
+            iou_threshold,
+        )