From f84713487fcb9334ba921883195b1ddbc9bb70f4 Mon Sep 17 00:00:00 2001
From: Nicolas Mowen <nickmowen213@gmail.com>
Date: Wed, 25 Dec 2024 20:52:35 -0600
Subject: [PATCH] Face recognition logic improvements (#15679)

* Always initialize face model on startup

* Add ability to save face images for debugging

* Implement better face recognition reasonability
---
 frigate/config/semantic_search.py |  3 +++
 frigate/embeddings/maintainer.py  | 21 +++++++++++++++++----
 frigate/util/model.py             |  5 +++++
 3 files changed, 25 insertions(+), 4 deletions(-)

diff --git a/frigate/config/semantic_search.py b/frigate/config/semantic_search.py
index f0eb5d98c8..aa509910ec 100644
--- a/frigate/config/semantic_search.py
+++ b/frigate/config/semantic_search.py
@@ -32,6 +32,9 @@ class FaceRecognitionConfig(FrigateBaseModel):
     min_area: int = Field(
         default=500, title="Min area of face box to consider running face recognition."
     )
+    debug_save_images: bool = Field(
+        default=False, title="Save images of face detections for debugging."
+    )
 
 
 class LicensePlateRecognitionConfig(FrigateBaseModel):
diff --git a/frigate/embeddings/maintainer.py b/frigate/embeddings/maintainer.py
index d3df385cc2..0d796c4888 100644
--- a/frigate/embeddings/maintainer.py
+++ b/frigate/embeddings/maintainer.py
@@ -505,13 +505,26 @@ def _process_face(self, obj_data: dict[str, any], frame: np.ndarray) -> None:
 
         sub_label, score = res
 
+        # calculate the overall face score as the probability * area of face
+        # this will help to reduce false positives from small side-angle faces
+        # if a large front-on face image may have scored slightly lower but
+        # is more likely to be accurate due to the larger face area
+        face_score = round(score * face_frame.shape[0] * face_frame.shape[1], 2)
+
         logger.debug(
-            f"Detected best face for person as: {sub_label} with score {score}"
+            f"Detected best face for person as: {sub_label} with probability {score} and overall face score {face_score}"
         )
 
-        if id in self.detected_faces and score <= self.detected_faces[id]:
+        if self.config.face_recognition.debug_save_images:
+            # write face to library
+            folder = os.path.join(FACE_DIR, "debug")
+            file = os.path.join(folder, f"{id}-{sub_label}-{score}-{face_score}.webp")
+            os.makedirs(folder, exist_ok=True)
+            cv2.imwrite(file, face_frame)
+
+        if id in self.detected_faces and face_score <= self.detected_faces[id]:
             logger.debug(
-                f"Recognized face distance {score} is less than previous face distance ({self.detected_faces.get(id)})."
+                f"Recognized face distance {score} and overall score {face_score} is less than previous overall face score ({self.detected_faces.get(id)})."
             )
             return
 
@@ -525,7 +538,7 @@ def _process_face(self, obj_data: dict[str, any], frame: np.ndarray) -> None:
         )
 
         if resp.status_code == 200:
-            self.detected_faces[id] = score
+            self.detected_faces[id] = face_score
 
     def _detect_license_plate(self, input: np.ndarray) -> tuple[int, int, int, int]:
         """Return the dimensions of the input image as [x, y, width, height]."""
diff --git a/frigate/util/model.py b/frigate/util/model.py
index b3e3102253..2709b4594a 100644
--- a/frigate/util/model.py
+++ b/frigate/util/model.py
@@ -170,6 +170,7 @@ def __init__(self, config: FaceRecognitionConfig, db: SqliteQueueDatabase):
             )
         )
         self.label_map: dict[int, str] = {}
+        self.__build_classifier()
 
     def __build_classifier(self) -> None:
         labels = []
@@ -177,6 +178,9 @@ def __build_classifier(self) -> None:
 
         dir = "/media/frigate/clips/faces"
         for idx, name in enumerate(os.listdir(dir)):
+            if name == "debug":
+                continue
+
             self.label_map[idx] = name
             face_folder = os.path.join(dir, name)
             for image in os.listdir(face_folder):
@@ -248,6 +252,7 @@ def __align_face(
     def clear_classifier(self) -> None:
         self.classifier = None
         self.labeler = None
+        self.label_map = {}
 
     def classify_face(self, face_image: np.ndarray) -> Optional[tuple[str, float]]:
         if not self.label_map: