Add support to visualmetrics to identify key frames matching the give…

…n colors.

This patch updates the visualmetrics.py and visualmetrics-portable.py
scripts to allow it to identify and record timestamps for frames that
match the given color configuration. This consists of an RGB value with
fuzz (anything +/- fuzz matches) and a fraction between 0 and 1 of the
required percentage of each channel from the histogram that must match it.

lib/chrome/webdriver/setupChromiumOptions.js

@@ -171,6 +171,10 @@ export function setupChromiumOptions(
     }
   }
 
+  if (browserOptions.enableVideoAutoplay) {
+    seleniumOptions.addArguments("--autoplay-policy=no-user-gesture-required");
+  }
+
   // It's a new splash screen introduced in Chrome 98
   // for new profiles
   // disable it with ChromeWhatsNewUI

lib/core/engine/iteration.js

@@ -16,7 +16,7 @@ import {
   addConnectivity,
   removeConnectivity
 } from '../../connectivity/index.js';
-import { jsonifyVisualProgress } from '../../support/util.js';
+import { jsonifyVisualProgress, jsonifyKeyColorFrames } from '../../support/util.js';
 import { flushDNS } from '../../support/dns.js';
 
 import { getNumberOfRunningProcesses } from '../../support/processes.js';
@@ -235,6 +235,11 @@ export class Iteration {
                 );
               }
             }
+            if (videoMetrics.visualMetrics['KeyColorFrames']) {
+                videoMetrics.visualMetrics['KeyColorFrames'] = jsonifyKeyColorFrames(
+                  videoMetrics.visualMetrics['KeyColorFrames']
+                );
+            }
             result[index_].videoRecordingStart =
               videoMetrics.videoRecordingStart;
             result[index_].visualMetrics = videoMetrics.visualMetrics;

lib/support/cli.js

@@ -274,6 +274,11 @@ export function parseCommandLine() {
       type: 'boolean',
       group: 'chrome'
     })
+    .option('chrome.enableVideoAutoplay', {
+      describe: 'Allow videos to autoplay.',
+      type: 'boolean',
+      group: 'chrome'
+    })
     .option('chrome.timeline', {
       alias: 'chrome.trace',
       describe:
@@ -720,6 +725,11 @@ export function parseCommandLine() {
       describe:
         'Use the portable visual-metrics processing script (no ImageMagick dependencies).'
     })
+    .option('visualMetricsKeyColor', {
+      type: 'array',
+      nargs: 6,
+      describe: 'Collect Key Color frame metrics when you run --visualMetrics. Each --visualMetricsKeyColor supplied must have 6 arguments: key name, red channel (0-255), green channel (0-255), blue channel (0-255), channel fuzz (0-255), fraction (0.0-1.0) of pixels that must match each channel.',
+    })
     .option('scriptInput.visualElements', {
       describe:
         'Include specific elements in visual elements. Give the element a name and select it with document.body.querySelector. Use like this: --scriptInput.visualElements name:domSelector see https://developer.mozilla.org/en-US/docs/Web/CSS/CSS_Selectors. Add multiple instances to measure multiple elements. Visual Metrics will use these elements and calculate when they are visible and fully rendered.'

lib/support/har/index.js

@@ -70,7 +70,8 @@ function addExtrasToHAR(
       'ContentfulSpeedIndex',
       'VisualProgress',
       'ContentfulSpeedIndexProgress',
-      'PerceptualSpeedIndexProgress'
+      'PerceptualSpeedIndexProgress',
+      'KeyColorFrames'
     ]);
 
     for (let key of Object.keys(visualMetricsData)) {

lib/support/util.js

@@ -230,6 +230,29 @@ export function jsonifyVisualProgress(visualProgress) {
   }
   return visualProgress;
 }
+export function jsonifyKeyColorFrames(keyColorFrames) {
+  // Original data looks like
+  //  "FrameName1=[0-133 255-300], FrameName2=[133-255] FrameName3=[]"
+  if (typeof keyColorFrames === 'string') {
+    const keyColorFramesObject = {};
+    for (const keyColorPair of keyColorFrames.split(', ')) {
+      const [name, values] = keyColorPair.split('=');
+      keyColorFramesObject[name] = [];
+      const rangePairs = values.replace('[', '').replace(']', '');
+      if (rangePairs) {
+        for (const rangePair of rangePairs.split(' ')) {
+          const [start, end] = rangePair.split('-');
+          keyColorFramesObject[name].push({
+            startTimestamp: Number.parseInt(start, 10),
+            endTimestamp: Number.parseInt(end, 10),
+          });
+        }
+      }
+    }
+    return keyColorFramesObject;
+  }
+  return keyColorFrames;
+}
 export function adjustVisualProgressTimestamps(
   visualProgress,
   profilerStartTime,

lib/video/postprocessing/visualmetrics/visualMetrics.js

@@ -84,6 +84,15 @@ export async function run(
     scriptArguments.push('--contentful');
   }
 
+  if (options.visualMetricsKeyColor) {
+    for (let i = 0; i < options.visualMetricsKeyColor.length; ++i) {
+      if (i % 6 == 0) {
+        scriptArguments.push('--keycolor')
+      }
+      scriptArguments.push(options.visualMetricsKeyColor[i]);
+    }
+  }
+
   // There seems to be a bug with --startwhite that makes VM bail out
   // 11:20:14.950 - Calculating image histograms
   // 11:20:14.951 - No video frames found in /private/var/folders/27/xpnvcsbs0nlfbb4qq397z3rh0000gn/T/vis-cn_JMf

visualmetrics/visualmetrics-portable.py

@@ -1212,6 +1212,8 @@ def calculate_visual_metrics(
     dirs,
     progress_file,
     hero_elements_file,
+    key_colors,
+    key_colors_file,
 ):
     metrics = None
     histograms = load_histograms(histograms_file, start, end)
@@ -1225,6 +1227,15 @@ def calculate_visual_metrics(
                 f = open(progress_file, "w")
             json.dump(progress, f)
             f.close()
+        key_color_frames = calculate_key_color_frames(histograms, key_colors)
+        if key_color_frames and key_colors_file is not None:
+            file_name, ext = os.path.splitext(key_colors_file)
+            if ext.lower() == ".gz":
+                f = gzip.open(key_colors_file, GZIP_TEXT, 7)
+            else:
+                f = open(key_colors_file, "w")
+            json.dump(key_color_frames, f)
+            f.close()
         if len(histograms) > 1:
             metrics = [
                 {"name": "First Visual Change", "value": histograms[1]["time"]},
@@ -1315,6 +1326,20 @@ def calculate_visual_metrics(
                 metrics.append({"name": "Perceptual Speed Index", "value": 0})
             if contentful:
                 metrics.append({"name": "Contentful Speed Index", "value": 0})
+        if key_color_frames:
+            keysum = ""
+            for key in key_color_frames:
+                if len(keysum):
+                    keysum += ", "
+                framesum = ""
+                for frame in key_color_frames[key]:
+                    if len(framesum):
+                        framesum += " "
+                    framesum += "{0:d}-{1:d}".format(
+                        frame["start_time"], frame["end_time"]
+                    )
+                keysum += "{0}=[{1}]".format(key, framesum)
+            metrics.append({"name": "Key Color Frames", "value": keysum})
         prog = ""
         for p in progress:
             if len(prog):
@@ -1346,6 +1371,90 @@ def load_histograms(histograms_file, start, end):
     return histograms
 
 
+def calculate_key_color_frames(histograms, key_colors):
+    if not key_colors:
+        return {}
+
+    key_color_frames = {}
+    for key in key_colors:
+        key_color_frames[key] = []
+
+    current = None
+    current_key = None
+    total = 0
+    matched = 0
+    channels = ["r", "g", "b"]
+    for index, histogram in enumerate(histograms):
+        buckets = 256
+        available = [0 for i in range(buckets)]
+
+        # Determine how many samples there are for each histogram channel
+        total = {}
+        for channel in channels:
+            total[channel] = 0
+            for i in range(buckets):
+                total[channel] += histogram["histogram"][channel][i]
+
+        matching_key = None
+        for key in key_colors:
+            fuzz = key_colors[key]["fuzz"]
+            fraction = key_colors[key]["fraction"]
+
+            channel_matches = True
+            for channel in channels:
+                # Find the acceptable range around the target channel value
+                target = key_colors[key][channel]
+                low = max(0, target - fuzz)
+                high = min(buckets, target + fuzz)
+
+                target_total = 0
+                for j in range(low, high):
+                    target_total += histogram["histogram"][channel][j]
+
+                if target_total < total[channel] * fraction:
+                    channel_matches = False
+                    break
+
+            if channel_matches:
+                matching_key = key
+                break
+
+        if matching_key is not None:
+            if current_key is not None:
+                if current_key != key:
+                    current["end_time"] = histogram["time"]
+                    key_color_frames[current_key].append(current)
+                    current_key = matching_key
+                    current = {
+                        "frame_count": 1,
+                        "start_time": histogram["time"],
+                    }
+                else:
+                    current["frame_count"] += 1
+            else:
+                current_key = matching_key
+                current = {
+                    "frame_count": 1,
+                    "start_time": histogram["time"],
+                }
+
+            logging.debug(
+                "{0:d}ms - Matched key color frame {1}".format(
+                    histogram["time"], matching_key
+                )
+            )
+        elif current_key is not None:
+            current["end_time"] = histogram["time"]
+            key_color_frames[current_key].append(current)
+            current_key = None
+            current = None
+
+    if current_key is not None:
+        current["end_time"] = histograms[-1]["time"]
+        key_color_frames[current_key].append(current)
+    return key_color_frames
+
+
 def calculate_visual_progress(histograms):
     progress = []
     first = histograms[0]["histogram"]
@@ -1760,6 +1869,16 @@ def main():
         default=False,
         help="Remove orange-colored frames from the beginning of the video.",
     )
+    parser.add_argument(
+        "--keycolor",
+        action="append",
+        nargs=6,
+        metavar=("key", "red", "green", "blue", "fuzz", "fraction"),
+        help="Identify frames that match the given channels (0-255) plus or "
+        "minus the given per channel fuzz. Fraction is the percentage of the "
+        "pixels per channel that must be in the given range (0-1).",
+    )
+    parser.add_argument("--keycolors", help="Key color frames output file.")
     parser.add_argument(
         "-p",
         "--viewport",
@@ -1916,6 +2035,17 @@ def main():
                     options.full,
                 )
 
+            key_colors = {}
+            if options.keycolor:
+                for key_params in options.keycolor:
+                    key_colors[key_params[0]] = {
+                        "r": int(key_params[1]),
+                        "g": int(key_params[2]),
+                        "b": int(key_params[3]),
+                        "fuzz": int(key_params[4]),
+                        "fraction": float(key_params[5]),
+                    }
+
             # Calculate the histograms and visual metrics
             calculate_histograms(directory, histogram_file, options.force)
             metrics = calculate_visual_metrics(
@@ -1927,6 +2057,8 @@ def main():
                 directory,
                 options.progress,
                 options.herodata,
+                key_colors,
+                options.keycolors,
             )
 
             if options.screenshot is not None: