From 51a0aad94a2b7a46505d385e846860f4696b8a15 Mon Sep 17 00:00:00 2001 From: Andrew Osmond Date: Thu, 25 Apr 2024 17:32:06 -0400 Subject: [PATCH] Add support to visualmetrics to identify key frames matching the given colors. This patch updates the visualmetrics.py and visualmetrics-portable.py scripts to allow it to identify and record timestamps for frames that match the given color configuration. This consists of an RGB value with fuzz (anything +/- fuzz matches) and a fraction between 0 and 1 of the required percentage of each channel from the histogram that must match it. --- lib/chrome/webdriver/setupChromiumOptions.js | 4 + lib/core/engine/iteration.js | 7 +- lib/support/cli.js | 10 ++ lib/support/har/index.js | 3 +- lib/support/util.js | 23 +++ .../visualmetrics/visualMetrics.js | 9 ++ visualmetrics/visualmetrics-portable.py | 132 ++++++++++++++++++ visualmetrics/visualmetrics.py | 131 +++++++++++++++++ 8 files changed, 317 insertions(+), 2 deletions(-) diff --git a/lib/chrome/webdriver/setupChromiumOptions.js b/lib/chrome/webdriver/setupChromiumOptions.js index be1d029c3..66ec4b6f7 100644 --- a/lib/chrome/webdriver/setupChromiumOptions.js +++ b/lib/chrome/webdriver/setupChromiumOptions.js @@ -171,6 +171,10 @@ export function setupChromiumOptions( } } + if (browserOptions.enableVideoAutoplay) { + seleniumOptions.addArguments("--autoplay-policy=no-user-gesture-required"); + } + // It's a new splash screen introduced in Chrome 98 // for new profiles // disable it with ChromeWhatsNewUI diff --git a/lib/core/engine/iteration.js b/lib/core/engine/iteration.js index 053dbf439..ff26bbd8a 100644 --- a/lib/core/engine/iteration.js +++ b/lib/core/engine/iteration.js @@ -16,7 +16,7 @@ import { addConnectivity, removeConnectivity } from '../../connectivity/index.js'; -import { jsonifyVisualProgress } from '../../support/util.js'; +import { jsonifyVisualProgress, jsonifyKeyColorFrames } from '../../support/util.js'; import { flushDNS } from '../../support/dns.js'; import { getNumberOfRunningProcesses } from '../../support/processes.js'; @@ -235,6 +235,11 @@ export class Iteration { ); } } + if (videoMetrics.visualMetrics['KeyColorFrames']) { + videoMetrics.visualMetrics['KeyColorFrames'] = jsonifyKeyColorFrames( + videoMetrics.visualMetrics['KeyColorFrames'] + ); + } result[index_].videoRecordingStart = videoMetrics.videoRecordingStart; result[index_].visualMetrics = videoMetrics.visualMetrics; diff --git a/lib/support/cli.js b/lib/support/cli.js index cfaa8a432..835b3184f 100644 --- a/lib/support/cli.js +++ b/lib/support/cli.js @@ -274,6 +274,11 @@ export function parseCommandLine() { type: 'boolean', group: 'chrome' }) + .option('chrome.enableVideoAutoplay', { + describe: 'Allow videos to autoplay.', + type: 'boolean', + group: 'chrome' + }) .option('chrome.timeline', { alias: 'chrome.trace', describe: @@ -720,6 +725,11 @@ export function parseCommandLine() { describe: 'Use the portable visual-metrics processing script (no ImageMagick dependencies).' }) + .option('visualMetricsKeyColor', { + type: 'array', + nargs: 6, + describe: 'Collect Key Color frame metrics when you run --visualMetrics. Each --visualMetricsKeyColor supplied must have 6 arguments: key name, red channel (0-255), green channel (0-255), blue channel (0-255), channel fuzz (0-255), fraction (0.0-1.0) of pixels that must match each channel.', + }) .option('scriptInput.visualElements', { describe: 'Include specific elements in visual elements. Give the element a name and select it with document.body.querySelector. Use like this: --scriptInput.visualElements name:domSelector see https://developer.mozilla.org/en-US/docs/Web/CSS/CSS_Selectors. Add multiple instances to measure multiple elements. Visual Metrics will use these elements and calculate when they are visible and fully rendered.' diff --git a/lib/support/har/index.js b/lib/support/har/index.js index c68acde37..ac41f0d61 100644 --- a/lib/support/har/index.js +++ b/lib/support/har/index.js @@ -70,7 +70,8 @@ function addExtrasToHAR( 'ContentfulSpeedIndex', 'VisualProgress', 'ContentfulSpeedIndexProgress', - 'PerceptualSpeedIndexProgress' + 'PerceptualSpeedIndexProgress', + 'KeyColorFrames' ]); for (let key of Object.keys(visualMetricsData)) { diff --git a/lib/support/util.js b/lib/support/util.js index 058032e8a..ad46492f1 100644 --- a/lib/support/util.js +++ b/lib/support/util.js @@ -230,6 +230,29 @@ export function jsonifyVisualProgress(visualProgress) { } return visualProgress; } +export function jsonifyKeyColorFrames(keyColorFrames) { + // Original data looks like + // "FrameName1=[0-133 255-300], FrameName2=[133-255] FrameName3=[]" + if (typeof keyColorFrames === 'string') { + const keyColorFramesObject = {}; + for (const keyColorPair of keyColorFrames.split(', ')) { + const [name, values] = keyColorPair.split('='); + keyColorFramesObject[name] = []; + const rangePairs = values.replace('[', '').replace(']', ''); + if (rangePairs) { + for (const rangePair of rangePairs.split(' ')) { + const [start, end] = rangePair.split('-'); + keyColorFramesObject[name].push({ + startTimestamp: Number.parseInt(start, 10), + endTimestamp: Number.parseInt(end, 10), + }); + } + } + } + return keyColorFramesObject; + } + return keyColorFrames; +} export function adjustVisualProgressTimestamps( visualProgress, profilerStartTime, diff --git a/lib/video/postprocessing/visualmetrics/visualMetrics.js b/lib/video/postprocessing/visualmetrics/visualMetrics.js index c821ba480..469b1ea1a 100644 --- a/lib/video/postprocessing/visualmetrics/visualMetrics.js +++ b/lib/video/postprocessing/visualmetrics/visualMetrics.js @@ -84,6 +84,15 @@ export async function run( scriptArguments.push('--contentful'); } + if (options.visualMetricsKeyColor) { + for (let i = 0; i < options.visualMetricsKeyColor.length; ++i) { + if (i % 6 == 0) { + scriptArguments.push('--keycolor') + } + scriptArguments.push(options.visualMetricsKeyColor[i]); + } + } + // There seems to be a bug with --startwhite that makes VM bail out // 11:20:14.950 - Calculating image histograms // 11:20:14.951 - No video frames found in /private/var/folders/27/xpnvcsbs0nlfbb4qq397z3rh0000gn/T/vis-cn_JMf diff --git a/visualmetrics/visualmetrics-portable.py b/visualmetrics/visualmetrics-portable.py index bd044bcd1..4e13fa297 100755 --- a/visualmetrics/visualmetrics-portable.py +++ b/visualmetrics/visualmetrics-portable.py @@ -1212,6 +1212,8 @@ def calculate_visual_metrics( dirs, progress_file, hero_elements_file, + key_colors, + key_colors_file, ): metrics = None histograms = load_histograms(histograms_file, start, end) @@ -1225,6 +1227,15 @@ def calculate_visual_metrics( f = open(progress_file, "w") json.dump(progress, f) f.close() + key_color_frames = calculate_key_color_frames(histograms, key_colors) + if key_color_frames and key_colors_file is not None: + file_name, ext = os.path.splitext(key_colors_file) + if ext.lower() == ".gz": + f = gzip.open(key_colors_file, GZIP_TEXT, 7) + else: + f = open(key_colors_file, "w") + json.dump(key_color_frames, f) + f.close() if len(histograms) > 1: metrics = [ {"name": "First Visual Change", "value": histograms[1]["time"]}, @@ -1315,6 +1326,20 @@ def calculate_visual_metrics( metrics.append({"name": "Perceptual Speed Index", "value": 0}) if contentful: metrics.append({"name": "Contentful Speed Index", "value": 0}) + if key_color_frames: + keysum = "" + for key in key_color_frames: + if len(keysum): + keysum += ", " + framesum = "" + for frame in key_color_frames[key]: + if len(framesum): + framesum += " " + framesum += "{0:d}-{1:d}".format( + frame["start_time"], frame["end_time"] + ) + keysum += "{0}=[{1}]".format(key, framesum) + metrics.append({"name": "Key Color Frames", "value": keysum}) prog = "" for p in progress: if len(prog): @@ -1346,6 +1371,90 @@ def load_histograms(histograms_file, start, end): return histograms +def calculate_key_color_frames(histograms, key_colors): + if not key_colors: + return {} + + key_color_frames = {} + for key in key_colors: + key_color_frames[key] = [] + + current = None + current_key = None + total = 0 + matched = 0 + channels = ["r", "g", "b"] + for index, histogram in enumerate(histograms): + buckets = 256 + available = [0 for i in range(buckets)] + + # Determine how many samples there are for each histogram channel + total = {} + for channel in channels: + total[channel] = 0 + for i in range(buckets): + total[channel] += histogram["histogram"][channel][i] + + matching_key = None + for key in key_colors: + fuzz = key_colors[key]["fuzz"] + fraction = key_colors[key]["fraction"] + + channel_matches = True + for channel in channels: + # Find the acceptable range around the target channel value + target = key_colors[key][channel] + low = max(0, target - fuzz) + high = min(buckets, target + fuzz) + + target_total = 0 + for j in range(low, high): + target_total += histogram["histogram"][channel][j] + + if target_total < total[channel] * fraction: + channel_matches = False + break + + if channel_matches: + matching_key = key + break + + if matching_key is not None: + if current_key is not None: + if current_key != key: + current["end_time"] = histogram["time"] + key_color_frames[current_key].append(current) + current_key = matching_key + current = { + "frame_count": 1, + "start_time": histogram["time"], + } + else: + current["frame_count"] += 1 + else: + current_key = matching_key + current = { + "frame_count": 1, + "start_time": histogram["time"], + } + + logging.debug( + "{0:d}ms - Matched key color frame {1}".format( + histogram["time"], matching_key + ) + ) + elif current_key is not None: + current["end_time"] = histogram["time"] + key_color_frames[current_key].append(current) + current_key = None + current = None + + if current_key is not None: + current["end_time"] = histograms[-1]["time"] + key_color_frames[current_key].append(current) + return key_color_frames + + def calculate_visual_progress(histograms): progress = [] first = histograms[0]["histogram"] @@ -1760,6 +1869,16 @@ def main(): default=False, help="Remove orange-colored frames from the beginning of the video.", ) + parser.add_argument( + "--keycolor", + action="append", + nargs=6, + metavar=("key", "red", "green", "blue", "fuzz", "fraction"), + help="Identify frames that match the given channels (0-255) plus or " + "minus the given per channel fuzz. Fraction is the percentage of the " + "pixels per channel that must be in the given range (0-1).", + ) + parser.add_argument("--keycolors", help="Key color frames output file.") parser.add_argument( "-p", "--viewport", @@ -1916,6 +2035,17 @@ def main(): options.full, ) + key_colors = {} + if options.keycolor: + for key_params in options.keycolor: + key_colors[key_params[0]] = { + "r": int(key_params[1]), + "g": int(key_params[2]), + "b": int(key_params[3]), + "fuzz": int(key_params[4]), + "fraction": float(key_params[5]), + } + # Calculate the histograms and visual metrics calculate_histograms(directory, histogram_file, options.force) metrics = calculate_visual_metrics( @@ -1927,6 +2057,8 @@ def main(): directory, options.progress, options.herodata, + key_colors, + options.keycolors, ) if options.screenshot is not None: diff --git a/visualmetrics/visualmetrics.py b/visualmetrics/visualmetrics.py index 4c9709a27..5ffab38bb 100755 --- a/visualmetrics/visualmetrics.py +++ b/visualmetrics/visualmetrics.py @@ -1618,6 +1618,8 @@ def calculate_visual_metrics( dirs, progress_file, hero_elements_file, + key_colors, + key_colors_file, ): metrics = None histograms = load_histograms(histograms_file, start, end) @@ -1631,6 +1633,15 @@ def calculate_visual_metrics( f = open(progress_file, "w") json.dump(progress, f) f.close() + key_color_frames = calculate_key_color_frames(histograms, key_colors) + if key_color_frames and key_colors_file is not None: + file_name, ext = os.path.splitext(key_colors_file) + if ext.lower() == ".gz": + f = gzip.open(key_colors_file, GZIP_TEXT, 7) + else: + f = open(key_colors_file, "w") + json.dump(key_color_frames, f) + f.close() if len(histograms) > 1: metrics = [ {"name": "First Visual Change", "value": histograms[1]["time"]}, @@ -1720,6 +1731,20 @@ def calculate_visual_metrics( metrics.append({"name": "Perceptual Speed Index", "value": 0}) if contentful: metrics.append({"name": "Contentful Speed Index", "value": 0}) + if key_color_frames: + keysum = "" + for key in key_color_frames: + if len(keysum): + keysum += ", " + framesum = "" + for frame in key_color_frames[key]: + if len(framesum): + framesum += " " + framesum += "{0:d}-{1:d}".format( + frame["start_time"], frame["end_time"] + ) + keysum += "{0}=[{1}]".format(key, framesum) + metrics.append({"name": "Key Color Frames", "value": keysum}) prog = "" for p in progress: if len(prog): @@ -1751,6 +1776,90 @@ def load_histograms(histograms_file, start, end): return histograms +def calculate_key_color_frames(histograms, key_colors): + if not key_colors: + return {} + + key_color_frames = {} + for key in key_colors: + key_color_frames[key] = [] + + current = None + current_key = None + total = 0 + matched = 0 + channels = ["r", "g", "b"] + for index, histogram in enumerate(histograms): + buckets = 256 + available = [0 for i in range(buckets)] + + # Determine how many samples there are for each histogram channel + total = {} + for channel in channels: + total[channel] = 0 + for i in range(buckets): + total[channel] += histogram["histogram"][channel][i] + + matching_key = None + for key in key_colors: + fuzz = key_colors[key]["fuzz"] + fraction = key_colors[key]["fraction"] + + channel_matches = True + for channel in channels: + # Find the acceptable range around the target channel value + target = key_colors[key][channel] + low = max(0, target - fuzz) + high = min(buckets, target + fuzz) + + target_total = 0 + for j in range(low, high): + target_total += histogram["histogram"][channel][j] + + if target_total < total[channel] * fraction: + channel_matches = False + break + + if channel_matches: + matching_key = key + break + + if matching_key is not None: + if current_key is not None: + if current_key != key: + current["end_time"] = histogram["time"] + key_color_frames[current_key].append(current) + current_key = matching_key + current = { + "frame_count": 1, + "start_time": histogram["time"], + } + else: + current["frame_count"] += 1 + else: + current_key = matching_key + current = { + "frame_count": 1, + "start_time": histogram["time"], + } + + logging.debug( + "{0:d}ms - Matched key color frame {1}".format( + histogram["time"], matching_key + ) + ) + elif current_key is not None: + current["end_time"] = histogram["time"] + key_color_frames[current_key].append(current) + current_key = None + current = None + + if current_key is not None: + current["end_time"] = histograms[-1]["time"] + key_color_frames[current_key].append(current) + return key_color_frames + + def calculate_visual_progress(histograms): progress = [] first = histograms[0]["histogram"] @@ -2200,6 +2309,16 @@ def main(): help="Wait for a full white frame after a non-white frame " "at the beginning of the video.", ) + parser.add_argument( + "--keycolor", + action="append", + nargs=6, + metavar=("key", "red", "green", "blue", "fuzz", "fraction"), + help="Identify frames that match the given channels (0-255) plus or " + "minus the given per channel fuzz. Fraction is the percentage of the " + "pixels per channel that must be in the given range (0-1).", + ) + parser.add_argument("--keycolors", help="Key color frames output file.") parser.add_argument( "--multiple", action="store_true", @@ -2464,6 +2583,16 @@ def main(): if not options.multiple: if options.render is not None: render_video(directory, options.render) + key_colors = {} + if options.keycolor: + for key_params in options.keycolor: + key_colors[key_params[0]] = { + "r": int(key_params[1]), + "g": int(key_params[2]), + "b": int(key_params[3]), + "fuzz": int(key_params[4]), + "fraction": float(key_params[5]), + } # Calculate the histograms and visual metrics calculate_histograms(directory, histogram_file, options.force) @@ -2476,6 +2605,8 @@ def main(): directory, options.progress, options.herodata, + key_colors, + options.keycolors, ) if options.screenshot is not None: