diff --git a/.github/workflows/auto_deploy_on_release.yml b/.github/workflows/auto_deploy_on_release.yml index f0f1d49f..187178d6 100644 --- a/.github/workflows/auto_deploy_on_release.yml +++ b/.github/workflows/auto_deploy_on_release.yml @@ -16,7 +16,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v2 with: - python-version: "3.12" + python-version: "3.11" - name: Pypa build run: | @@ -40,7 +40,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v2 with: - python-version: "3.12" + python-version: "3.11" - name: Upgrade pip run: | diff --git a/.github/workflows/manual_docs.yml b/.github/workflows/manual_docs.yml index 339227ce..ce32103b 100644 --- a/.github/workflows/manual_docs.yml +++ b/.github/workflows/manual_docs.yml @@ -14,7 +14,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v2 with: - python-version: "3.12" + python-version: "3.11" - name: Upgrade pip run: | diff --git a/.github/workflows/tests_and_docs.yml b/.github/workflows/tests_and_docs.yml index 1bfbf50b..d0057850 100644 --- a/.github/workflows/tests_and_docs.yml +++ b/.github/workflows/tests_and_docs.yml @@ -26,11 +26,17 @@ jobs: # OSs to test os: [ubuntu-latest, macos-latest] # Python versions to test - python-version: ["3.10", "3.11", "3.12"] + python-version: ["3.10", "3.11"] # By default everything should pass for the workflow to pass experimental: [false] include: # Windows sometimes fails to install due to dependency changes, but eventually sort themselves out. So let these tests fail. Also issue on macos 3.11 with joblib so let that fail + - os: ubuntu-latest + python-version: "3.12" + experimental: true + - os: macos-latest + python-version: "3.12" + experimental: true - os: windows-latest python-version: "3.10" experimental: true diff --git a/.github/workflows/tests_on_pr.yml b/.github/workflows/tests_on_pr.yml index 0422bce9..41375500 100644 --- a/.github/workflows/tests_on_pr.yml +++ b/.github/workflows/tests_on_pr.yml @@ -24,11 +24,17 @@ jobs: # OSs to test os: [ubuntu-latest, macos-latest] # Python versions to test - python-version: ["3.10", "3.11", "3.12"] + python-version: ["3.10", "3.11"] # By default everything should pass for the workflow to pass experimental: [false] include: # Windows sometimes fails to install due to dependency changes, but eventually sort themselves out. So let these tests fail. Also issue on macos 3.11 with joblib so let that fail + - os: ubuntu-latest + python-version: "3.12" + experimental: true + - os: macos-latest + python-version: "3.12" + experimental: true - os: windows-latest python-version: "3.10" experimental: true diff --git a/feat/FastDetector.py b/feat/FastDetector.py index b734e106..95be7069 100644 --- a/feat/FastDetector.py +++ b/feat/FastDetector.py @@ -24,16 +24,12 @@ openface_2d_landmark_columns, FEAT_EMOTION_COLUMNS, FEAT_FACEBOX_COLUMNS, - # FEAT_FACEPOSE_COLUMNS_3D, FEAT_FACEPOSE_COLUMNS_6D, FEAT_IDENTITY_COLUMNS, ) -from feat.utils.mp_plotting import FaceLandmarksConnections from feat.utils.io import get_resource_path from feat.utils.image_operations import ( convert_image_to_tensor, - align_face, - mask_image, extract_face_from_bbox_torch, inverse_transform_landmarks_torch, extract_hog_features, @@ -47,341 +43,14 @@ import torch.nn as nn from torch.utils.data import DataLoader from torchvision.models.detection.backbone_utils import resnet_fpn_backbone -from torchvision.utils import draw_keypoints, draw_bounding_boxes, make_grid -import torchvision.transforms as transforms from torchvision.transforms import Compose, Normalize -from scipy.spatial import ConvexHull -from skimage.morphology.convex_hull import grid_points_in_poly import sys -from PIL import Image -import matplotlib.pyplot as plt sys.modules["__main__"].__dict__["XGBClassifier"] = XGBClassifier sys.modules["__main__"].__dict__["SVMClassifier"] = SVMClassifier sys.modules["__main__"].__dict__["EmoSVMClassifier"] = EmoSVMClassifier -def plot_frame( - frame, - boxes=None, - landmarks=None, - boxes_width=2, - boxes_colors="cyan", - landmarks_radius=2, - landmarks_width=2, - landmarks_colors="white", -): - """ - Plot Torch Frames and py-feat output. If multiple frames will create a grid of images - - Args: - frame (torch.Tensor): Tensor of shape (B, C, H, W) or (C, H, W) - boxes (torch.Tensor): Tensor of shape (N, 4) containing bounding boxes - landmarks (torch.Tensor): Tensor of shape (N, 136) containing flattened 68 point landmark keystones - - Returns: - PILImage - """ - - if len(frame.shape) == 4: - B, C, H, W = frame.shape - elif len(frame.shape) == 3: - C, H, W = frame.shape - else: - raise ValueError("Can only plot (B,C,H,W) or (C,H,W)") - if B == 1: - if boxes is not None: - new_frame = draw_bounding_boxes( - frame.squeeze(0), boxes, width=boxes_width, colors=boxes_colors - ) - - if landmarks is not None: - new_frame = draw_keypoints( - new_frame, - landmarks.reshape(landmarks.shape[0], -1, 2), - radius=landmarks_radius, - width=landmarks_width, - colors=landmarks_colors, - ) - else: - if landmarks is not None: - new_frame = draw_keypoints( - frame.squeeze(0), - landmarks.reshape(landmarks.shape[0], -1, 2), - radius=landmarks_radius, - width=landmarks_width, - colors=landmarks_colors, - ) - else: - new_frame = frame.squeeze(0) - return transforms.ToPILImage()(new_frame.squeeze(0)) - else: - if (boxes is not None) & (landmarks is None): - new_frame = make_grid( - torch.stack( - [ - draw_bounding_boxes( - f, b.unsqueeze(0), width=boxes_width, colors=boxes_colors - ) - for f, b in zip(frame.unbind(dim=0), boxes.unbind(dim=0)) - ], - dim=0, - ) - ) - elif (landmarks is not None) & (boxes is None): - new_frame = make_grid( - torch.stack( - [ - draw_keypoints( - f, - l.unsqueeze(0), - radius=landmarks_radius, - width=landmarks_width, - colors=landmarks_colors, - ) - for f, l in zip( - frame.unbind(dim=0), - landmarks.reshape(landmarks.shape[0], -1, 2).unbind(dim=0), - ) - ], - dim=0, - ) - ) - elif (boxes is not None) & (landmarks is not None): - new_frame = make_grid( - torch.stack( - [ - draw_keypoints( - fr, - l.unsqueeze(0), - radius=landmarks_radius, - width=landmarks_width, - colors=landmarks_colors, - ) - for fr, l in zip( - [ - draw_bounding_boxes( - f, - b.unsqueeze(0), - width=boxes_width, - colors=boxes_colors, - ) - for f, b in zip(frame.unbind(dim=0), boxes.unbind(dim=0)) - ], - landmarks.reshape(landmarks.shape[0], -1, 2).unbind(dim=0), - ) - ] - ) - ) - else: - new_frame = make_grid(frame) - return transforms.ToPILImage()(new_frame) - - -def extract_face_from_landmarks(frame, landmarks, face_size=112): - """Extract a face in a frame with a convex hull of landmarks. - - This function extracts the faces of the frame with convex hulls and masks out the rest. - - Args: - frame (array): The original image] - detected_faces (list): face bounding box - landmarks (list): the landmark information] - align (bool): align face to standard position - size_output (int, optional): [description]. Defaults to 112. - - Returns: - resized_face_np: resized face as a numpy array - new_landmarks: landmarks of aligned face - """ - - if not isinstance(frame, torch.Tensor): - raise ValueError(f"image must be a tensor not {type(frame)}") - - if len(frame.shape) != 4: - frame = frame.unsqueeze(0) - - landmarks = landmarks.cpu().detach().numpy() - - aligned_img, new_landmarks = align_face( - frame, - landmarks.flatten(), - landmark_type=68, - box_enlarge=2.5, - img_size=face_size, - ) - - hull = ConvexHull(new_landmarks) - mask = grid_points_in_poly( - shape=aligned_img.shape[-2:], - # for some reason verts need to be flipped - verts=list( - zip( - new_landmarks[hull.vertices][:, 1], - new_landmarks[hull.vertices][:, 0], - ) - ), - ) - mask[ - 0 : np.min([new_landmarks[0][1], new_landmarks[16][1]]), - new_landmarks[0][0] : new_landmarks[16][0], - ] = True - masked_image = mask_image(aligned_img, mask) - - return (masked_image, new_landmarks) - - -def plot_face_landmarks( - fex, - frame_idx, - ax=None, - oval_color="white", - oval_linestyle="-", - oval_linewidth=3, - tesselation_color="gray", - tesselation_linestyle="-", - tesselation_linewidth=1, - mouth_color="white", - mouth_linestyle="-", - mouth_linewidth=3, - eye_color="navy", - eye_linestyle="-", - eye_linewidth=2, - iris_color="skyblue", - iris_linestyle="-", - iris_linewidth=2, -): - """Plots face landmarks on the given frame using specified styles for each part. - - Args: - fex: DataFrame containing face landmarks (x, y coordinates). - frame_idx: Index of the frame to plot. - ax: Matplotlib axis to draw on. If None, a new axis is created. - oval_color, tesselation_color, mouth_color, eye_color, iris_color: Colors for each face part. - oval_linestyle, tesselation_linestyle, mouth_linestyle, eye_linestyle, iris_linestyle: Linestyle for each face part. - oval_linewidth, tesselation_linewidth, mouth_linewidth, eye_linewidth, iris_linewidth: Linewidth for each face part. - n_faces: Number of faces in the frame. If None, will be determined from fex. - """ - if ax is None: - fig, ax = plt.subplots(figsize=(10, 10)) - - # Get frame data - fex_frame = fex.query("frame == @frame_idx") - n_faces_frame = fex_frame.shape[0] - - # Add the frame image - ax.imshow(Image.open(fex_frame["input"].unique()[0])) - - # Helper function to draw lines for a set of connections - def draw_connections(face_idx, connections, color, linestyle, linewidth): - for connection in connections: - start = connection.start - end = connection.end - line = plt.Line2D( - [fex.loc[face_idx, f"x_{start}"], fex.loc[face_idx, f"x_{end}"]], - [fex.loc[face_idx, f"y_{start}"], fex.loc[face_idx, f"y_{end}"]], - color=color, - linestyle=linestyle, - linewidth=linewidth, - ) - ax.add_line(line) - - # Face tessellation - for face in range(n_faces_frame): - draw_connections( - face, - FaceLandmarksConnections.FACE_LANDMARKS_TESSELATION, - tesselation_color, - tesselation_linestyle, - tesselation_linewidth, - ) - - # Mouth - for face in range(n_faces_frame): - draw_connections( - face, - FaceLandmarksConnections.FACE_LANDMARKS_LIPS, - mouth_color, - mouth_linestyle, - mouth_linewidth, - ) - - # Left iris - for face in range(n_faces_frame): - draw_connections( - face, - FaceLandmarksConnections.FACE_LANDMARKS_LEFT_IRIS, - iris_color, - iris_linestyle, - iris_linewidth, - ) - - # Left eye - for face in range(n_faces_frame): - draw_connections( - face, - FaceLandmarksConnections.FACE_LANDMARKS_LEFT_EYE, - eye_color, - eye_linestyle, - eye_linewidth, - ) - - # Left eyebrow - for face in range(n_faces_frame): - draw_connections( - face, - FaceLandmarksConnections.FACE_LANDMARKS_LEFT_EYEBROW, - eye_color, - eye_linestyle, - eye_linewidth, - ) - - # Right iris - for face in range(n_faces_frame): - draw_connections( - face, - FaceLandmarksConnections.FACE_LANDMARKS_RIGHT_IRIS, - iris_color, - iris_linestyle, - iris_linewidth, - ) - - # Right eye - for face in range(n_faces_frame): - draw_connections( - face, - FaceLandmarksConnections.FACE_LANDMARKS_RIGHT_EYE, - eye_color, - eye_linestyle, - eye_linewidth, - ) - - # Right eyebrow - for face in range(n_faces_frame): - draw_connections( - face, - FaceLandmarksConnections.FACE_LANDMARKS_RIGHT_EYEBROW, - eye_color, - eye_linestyle, - eye_linewidth, - ) - - # Face oval - for face in range(n_faces_frame): - draw_connections( - face, - FaceLandmarksConnections.FACE_LANDMARKS_FACE_OVAL, - oval_color, - oval_linestyle, - oval_linewidth, - ) - - # Optionally turn off axis for a clean plot - ax.axis("off") - - return ax - - class FastDetector(nn.Module, PyTorchModelHubMixin): def __init__( self, diff --git a/feat/plotting.py b/feat/plotting.py index 15fec0ea..cc8fba6c 100644 --- a/feat/plotting.py +++ b/feat/plotting.py @@ -5,12 +5,14 @@ import os import sys import h5py +import torch import numpy as np from sklearn.cross_decomposition import PLSRegression from sklearn import __version__ as skversion import matplotlib.pyplot as plt from feat.pretrained import AU_LANDMARK_MAP -from feat.utils.io import get_resource_path +from feat.utils.io import get_resource_path, download_url +from feat.utils.image_operations import align_face, mask_image from math import sin, cos import warnings import seaborn as sns @@ -20,8 +22,12 @@ from PIL import Image from textwrap import wrap from joblib import load -from feat.utils.io import download_url import json +from skimage.morphology.convex_hull import grid_points_in_poly +from scipy.spatial import ConvexHull +import torchvision.transforms as transforms +from torchvision.utils import draw_keypoints, draw_bounding_boxes, make_grid +from feat.utils.mp_plotting import FaceLandmarksConnections __all__ = [ "draw_lineface", @@ -1384,3 +1390,324 @@ def load_viz_model( except Exception as e: raise IOError(f"Unable to load data: {e}") return model + + +def plot_frame( + frame, + boxes=None, + landmarks=None, + boxes_width=2, + boxes_colors="cyan", + landmarks_radius=2, + landmarks_width=2, + landmarks_colors="white", +): + """ + Plot Torch Frames and py-feat output. If multiple frames will create a grid of images + + Args: + frame (torch.Tensor): Tensor of shape (B, C, H, W) or (C, H, W) + boxes (torch.Tensor): Tensor of shape (N, 4) containing bounding boxes + landmarks (torch.Tensor): Tensor of shape (N, 136) containing flattened 68 point landmark keystones + + Returns: + PILImage + """ + + if len(frame.shape) == 4: + B, C, H, W = frame.shape + elif len(frame.shape) == 3: + C, H, W = frame.shape + else: + raise ValueError("Can only plot (B,C,H,W) or (C,H,W)") + if B == 1: + if boxes is not None: + new_frame = draw_bounding_boxes( + frame.squeeze(0), boxes, width=boxes_width, colors=boxes_colors + ) + + if landmarks is not None: + new_frame = draw_keypoints( + new_frame, + landmarks.reshape(landmarks.shape[0], -1, 2), + radius=landmarks_radius, + width=landmarks_width, + colors=landmarks_colors, + ) + else: + if landmarks is not None: + new_frame = draw_keypoints( + frame.squeeze(0), + landmarks.reshape(landmarks.shape[0], -1, 2), + radius=landmarks_radius, + width=landmarks_width, + colors=landmarks_colors, + ) + else: + new_frame = frame.squeeze(0) + return transforms.ToPILImage()(new_frame.squeeze(0)) + else: + if (boxes is not None) & (landmarks is None): + new_frame = make_grid( + torch.stack( + [ + draw_bounding_boxes( + f, b.unsqueeze(0), width=boxes_width, colors=boxes_colors + ) + for f, b in zip(frame.unbind(dim=0), boxes.unbind(dim=0)) + ], + dim=0, + ) + ) + elif (landmarks is not None) & (boxes is None): + new_frame = make_grid( + torch.stack( + [ + draw_keypoints( + f, + l.unsqueeze(0), + radius=landmarks_radius, + width=landmarks_width, + colors=landmarks_colors, + ) + for f, l in zip( + frame.unbind(dim=0), + landmarks.reshape(landmarks.shape[0], -1, 2).unbind(dim=0), + ) + ], + dim=0, + ) + ) + elif (boxes is not None) & (landmarks is not None): + new_frame = make_grid( + torch.stack( + [ + draw_keypoints( + fr, + l.unsqueeze(0), + radius=landmarks_radius, + width=landmarks_width, + colors=landmarks_colors, + ) + for fr, l in zip( + [ + draw_bounding_boxes( + f, + b.unsqueeze(0), + width=boxes_width, + colors=boxes_colors, + ) + for f, b in zip(frame.unbind(dim=0), boxes.unbind(dim=0)) + ], + landmarks.reshape(landmarks.shape[0], -1, 2).unbind(dim=0), + ) + ] + ) + ) + else: + new_frame = make_grid(frame) + return transforms.ToPILImage()(new_frame) + + +def extract_face_from_landmarks(frame, landmarks, face_size=112): + """Extract a face in a frame with a convex hull of landmarks. + + This function extracts the faces of the frame with convex hulls and masks out the rest. + + Args: + frame (array): The original image] + detected_faces (list): face bounding box + landmarks (list): the landmark information] + align (bool): align face to standard position + size_output (int, optional): [description]. Defaults to 112. + + Returns: + resized_face_np: resized face as a numpy array + new_landmarks: landmarks of aligned face + """ + + if not isinstance(frame, torch.Tensor): + raise ValueError(f"image must be a tensor not {type(frame)}") + + if len(frame.shape) != 4: + frame = frame.unsqueeze(0) + + landmarks = landmarks.cpu().detach().numpy() + + aligned_img, new_landmarks = align_face( + frame, + landmarks.flatten(), + landmark_type=68, + box_enlarge=2.5, + img_size=face_size, + ) + + hull = ConvexHull(new_landmarks) + mask = grid_points_in_poly( + shape=aligned_img.shape[-2:], + # for some reason verts need to be flipped + verts=list( + zip( + new_landmarks[hull.vertices][:, 1], + new_landmarks[hull.vertices][:, 0], + ) + ), + ) + mask[ + 0 : np.min([new_landmarks[0][1], new_landmarks[16][1]]), + new_landmarks[0][0] : new_landmarks[16][0], + ] = True + masked_image = mask_image(aligned_img, mask) + + return (masked_image, new_landmarks) + + +def plot_face_landmarks( + fex, + frame_idx, + ax=None, + oval_color="white", + oval_linestyle="-", + oval_linewidth=3, + tesselation_color="gray", + tesselation_linestyle="-", + tesselation_linewidth=1, + mouth_color="white", + mouth_linestyle="-", + mouth_linewidth=3, + eye_color="navy", + eye_linestyle="-", + eye_linewidth=2, + iris_color="skyblue", + iris_linestyle="-", + iris_linewidth=2, +): + """Plots face landmarks on the given frame using specified styles for each part. + + Args: + fex: DataFrame containing face landmarks (x, y coordinates). + frame_idx: Index of the frame to plot. + ax: Matplotlib axis to draw on. If None, a new axis is created. + oval_color, tesselation_color, mouth_color, eye_color, iris_color: Colors for each face part. + oval_linestyle, tesselation_linestyle, mouth_linestyle, eye_linestyle, iris_linestyle: Linestyle for each face part. + oval_linewidth, tesselation_linewidth, mouth_linewidth, eye_linewidth, iris_linewidth: Linewidth for each face part. + n_faces: Number of faces in the frame. If None, will be determined from fex. + """ + if ax is None: + fig, ax = plt.subplots(figsize=(10, 10)) + + # Get frame data + fex_frame = fex.query("frame == @frame_idx") + n_faces_frame = fex_frame.shape[0] + + # Add the frame image + ax.imshow(Image.open(fex_frame["input"].unique()[0])) + + # Helper function to draw lines for a set of connections + def draw_connections(face_idx, connections, color, linestyle, linewidth): + for connection in connections: + start = connection.start + end = connection.end + line = plt.Line2D( + [fex.loc[face_idx, f"x_{start}"], fex.loc[face_idx, f"x_{end}"]], + [fex.loc[face_idx, f"y_{start}"], fex.loc[face_idx, f"y_{end}"]], + color=color, + linestyle=linestyle, + linewidth=linewidth, + ) + ax.add_line(line) + + # Face tessellation + for face in range(n_faces_frame): + draw_connections( + face, + FaceLandmarksConnections.FACE_LANDMARKS_TESSELATION, + tesselation_color, + tesselation_linestyle, + tesselation_linewidth, + ) + + # Mouth + for face in range(n_faces_frame): + draw_connections( + face, + FaceLandmarksConnections.FACE_LANDMARKS_LIPS, + mouth_color, + mouth_linestyle, + mouth_linewidth, + ) + + # Left iris + for face in range(n_faces_frame): + draw_connections( + face, + FaceLandmarksConnections.FACE_LANDMARKS_LEFT_IRIS, + iris_color, + iris_linestyle, + iris_linewidth, + ) + + # Left eye + for face in range(n_faces_frame): + draw_connections( + face, + FaceLandmarksConnections.FACE_LANDMARKS_LEFT_EYE, + eye_color, + eye_linestyle, + eye_linewidth, + ) + + # Left eyebrow + for face in range(n_faces_frame): + draw_connections( + face, + FaceLandmarksConnections.FACE_LANDMARKS_LEFT_EYEBROW, + eye_color, + eye_linestyle, + eye_linewidth, + ) + + # Right iris + for face in range(n_faces_frame): + draw_connections( + face, + FaceLandmarksConnections.FACE_LANDMARKS_RIGHT_IRIS, + iris_color, + iris_linestyle, + iris_linewidth, + ) + + # Right eye + for face in range(n_faces_frame): + draw_connections( + face, + FaceLandmarksConnections.FACE_LANDMARKS_RIGHT_EYE, + eye_color, + eye_linestyle, + eye_linewidth, + ) + + # Right eyebrow + for face in range(n_faces_frame): + draw_connections( + face, + FaceLandmarksConnections.FACE_LANDMARKS_RIGHT_EYEBROW, + eye_color, + eye_linestyle, + eye_linewidth, + ) + + # Face oval + for face in range(n_faces_frame): + draw_connections( + face, + FaceLandmarksConnections.FACE_LANDMARKS_FACE_OVAL, + oval_color, + oval_linestyle, + oval_linewidth, + ) + + # Optionally turn off axis for a clean plot + ax.axis("off") + + return ax