apputils.py

import math
from typing import List, Tuple
import PIL

import cv2
import pathlib
import numpy as np
from imutils.object_detection import non_max_suppression
from mmif import Mmif, View, DocumentTypes, AnnotationTypes

BOX_MIN_CONF = 0.1
SAMPLE_RATIO = 30
net = cv2.dnn.readNet(str(pathlib.Path(__file__).parent / 'cv_data' / 'frozen_east_text_detection.pb'))


def process_image(f):
    return f


def decode_predictions(scores, geometry, box_min_conf=BOX_MIN_CONF):
    """
    Taken from pyimagesearch, convert results to rectangles and confidences
    """

    # grab the number of rows and columns from the scores volume, then
    # initialize our set of bounding box rectangles and corresponding
    # confidence scores
    (numRows, numCols) = scores.shape[2:4]
    rects = []
    confidences = []

    # loop over the number of rows
    for y in range(0, numRows):
        # extract the scores (probabilities), followed by the
        # geometrical data used to derive potential bounding box
        # coordinates that surround text
        scoresData = scores[0, 0, y]
        xData0 = geometry[0, 0, y]
        xData1 = geometry[0, 1, y]
        xData2 = geometry[0, 2, y]
        xData3 = geometry[0, 3, y]
        anglesData = geometry[0, 4, y]

        # loop over the number of columns
        for x in range(0, numCols):
            # if our score does not have sufficient probability,
            # ignore it
            if scoresData[x] < box_min_conf:
                continue

            # compute the offset factor as our resulting feature
            # maps will be 4x smaller than the input image
            (offsetX, offsetY) = (x * 4.0, y * 4.0)

            # extract the rotation angle for the prediction and
            # then compute the sin and cosine
            angle = anglesData[x]
            cos = np.cos(angle)
            sin = np.sin(angle)

            # use the geometry volume to derive the width and height
            # of the bounding box
            h = xData0[x] + xData2[x]
            w = xData1[x] + xData3[x]

            # compute both the starting and ending (x, y)-coordinates
            # for the text prediction bounding box
            endX = int(offsetX + (cos * xData1[x]) + (sin * xData2[x]))
            endY = int(offsetY - (sin * xData1[x]) + (cos * xData2[x]))
            startX = int(endX - w)
            startY = int(endY - h)

            # add the bounding box coordinates and probability score
            # to our respective lists
            rects.append((startX, startY, endX, endY))
            confidences.append(scoresData[x])

    # return a tuple of the bounding boxes and associated confidences
    return rects, confidences


def image_to_east_boxes(image: np.array) -> List[Tuple[int, int, int, int]]:
    (newW, newH) = (320, 320)  # newH and newW must a multiple of 32.
    (H, W) = image.shape[:2]
    rW = W / float(newW)
    rH = H / float(newH)

    # resize the frame, this time ignoring aspect ratio
    image = cv2.resize(image, (newW, newH))

    # construct a blob from the frame and then perform a forward pass
    # of the model to obtain the two output layer sets
    blob = cv2.dnn.blobFromImage(
        image, 1.0, (newW, newH), (123.68, 116.78, 103.94), swapRB=True, crop=False
    )
    net.setInput(blob)
    layerNames = ["feature_fusion/Conv_7/Sigmoid", "feature_fusion/concat_3"]
    (scores, geometry) = net.forward(layerNames)
    (rects, confidences) = decode_predictions(scores, geometry)
    boxes = non_max_suppression(np.array(rects), probs=confidences)
    box_list = []
    for (startX, startY, endX, endY) in boxes:
        # scale the bounding box coordinates based on the respective
        # ratios
        startX = int(startX * rW)
        startY = int(startY * rH)
        endX = int(endX * rW)
        endY = int(endY * rH)
        box_list.append((startX, startY, endX, endY))
    return box_list


def get_chyron(frame, threshold=.03):
    boxes = image_to_east_boxes(frame)
    text_box_mask = np.zeros(frame.shape)
    for box in boxes:  # box is (startX, startY, endX, endY)
        text_box_mask[box[1]:box[3], box[0]:box[2]] = 1
    bottom_third = text_box_mask[math.floor(.6 * frame.shape[0]):, :]
    top = text_box_mask[:math.floor(.6 * frame.shape[0]), :]
    if np.sum(top) / (top.shape[0] * top.shape[1]) > .5:
        return None
    if np.sum(bottom_third) / (bottom_third.shape[0] * bottom_third.shape[1]) > threshold:
        bottom_third_boxes = [box for box in boxes if box[1] > (math.floor(.4 * frame.shape[0]))]
        return max(bottom_third_boxes, key=lambda x: (x[3] - x[1]) * (x[2] - x[0]), default=None)
    return None