Log custom metrics for webalmanac (#132)

* check changes with js

* formatting

* ads custom metric present

* all metrics for unit tests

* format

* fix

* logs on

* stdout log

* filter review

* cleanup

* check metricsList scope

* commenting with unit tests

* async callback return

* no callback

* sync exec and comment formatting

* cleanup

* yml cleanup

.github/workflows/wpt-test.yml

@@ -25,20 +25,13 @@ jobs:
           npm install jest
 
       - name: Run WebPageTest with unit tests
-        run: |
-          echo "::group::Unit tests"
-            npm test
-          echo "::endgroup::"
+        run: npm test
         env:
           WPT_SERVER: "webpagetest.httparchive.org"
           WPT_API_KEY: ${{ secrets.HA_API_KEY }}
 
       - name: Run WebPageTest for more websites
         run: |
-          # Get the list of files that changed in the PR
-          METRICS=$(git diff --name-only --diff-filter=ACMRT origin/main | \
-            grep -E "^dist/.*\.js$" | xargs -I {} basename {} | cut -d. -f1 | sort | uniq)
-
           # Get the PR body
           PR_BODY="$(cat <<'EOF'
           ${{ github.event.pull_request.body }}
@@ -75,7 +68,7 @@ jobs:
             # Run WebPageTest for each URL
             for TEST_WEBSITE in "${URLS[@]}"; do
               echo "::group::Custom metrics for $TEST_WEBSITE"
-              node tests/wpt.js "$TEST_WEBSITE" "$METRICS"
+              node tests/wpt.js "$TEST_WEBSITE"
               echo "::endgroup::"
             done
           else

dist/privacy.js

@@ -367,38 +367,38 @@ return JSON.stringify({
       'shadingLanguageVersion',
       'WEBGL_debug_renderer_info',
       'getShaderPrecisionFormat'
-  ].map(api => api.toLowerCase())
+    ].map(api => api.toLowerCase())
 
-  const response_bodies = $WPT_BODIES.filter(body => (body.response_body && (body.type === 'Document' || body.type === 'Script')))
+    const response_bodies = $WPT_BODIES.filter(body => (body.response_body && (body.type === 'Document' || body.type === 'Script')))
 
-  let fingerprintingUsageCounts = {}
-  let likelyFingerprintingScripts = []
+    let fingerprintingUsageCounts = {}
+    let likelyFingerprintingScripts = []
 
-  response_bodies.forEach(req => {
-    let total_occurrences = 0
+    response_bodies.forEach(req => {
+      let total_occurrences = 0
 
-    let body = req.response_body.toLowerCase()
+      let body = req.response_body.toLowerCase()
 
-    fingerprintingAPIs.forEach(api => {
-      let api_occurrences = 0
-      let index = body.indexOf(api)
-      while (index !== -1) {
-        api_occurrences++
-        index = body.indexOf(api, index + 1)
-      }
+      fingerprintingAPIs.forEach(api => {
+        let api_occurrences = 0
+        let index = body.indexOf(api)
+        while (index !== -1) {
+          api_occurrences++
+          index = body.indexOf(api, index + 1)
+        }
 
-      if (api_occurrences > 0) {
-        fingerprintingUsageCounts[api] = (fingerprintingUsageCounts[api] || 0) + api_occurrences
+        if (api_occurrences > 0) {
+          fingerprintingUsageCounts[api] = (fingerprintingUsageCounts[api] || 0) + api_occurrences
+        }
+        total_occurrences += api_occurrences
+      })
+
+      if (total_occurrences >= 5) { //TODO what should this threshold be?
+        likelyFingerprintingScripts.push(req.url)
       }
-      total_occurrences += api_occurrences
     })
 
-    if (total_occurrences >= 5) { //TODO what should this threshold be?
-      likelyFingerprintingScripts.push(req.url)
-    }
-  })
-
-  return {counts: fingerprintingUsageCounts, likelyFingerprintingScripts}
+    return { counts: fingerprintingUsageCounts, likelyFingerprintingScripts }
   })(),
 
   /**
@@ -419,7 +419,7 @@ return JSON.stringify({
             results[dns_hostname] = dns_info.results.canonical_names;
           }
         }
-      } catch {}
+      } catch { }
     }
 
     return results;

tests/unit-tests.test.js

@@ -8,6 +8,7 @@ beforeAll(async () => {
 }, 400000);
 
 test('_ads parsing', () => {
+  assert.ok(wpt_data["_ads"])
   assert.ok(wpt_data["_ads"].ads);
   assert.ok(wpt_data["_ads"].app_ads);
   assert.ok(wpt_data["_ads"].sellers);

tests/wpt.js

@@ -2,34 +2,55 @@ const WebPageTest = require('webpagetest');
 const fs = require('fs');
 const path = require('path');
 const { argv } = require('node:process');
+const { execSync } = require('child_process');
 
 const is_direct_run = require.main === module;
 
 const wptServer = process.env.WPT_SERVER;
 const wptApiKey = process.env.WPT_API_KEY;
 const wpt = new WebPageTest(wptServer, wptApiKey);
 
+
 /**
  * Retrieves the names of all JavaScript files in the 'dist/' directory.
  *
  * @returns {string[]} An array of the base names of the JavaScript files without the '.js' extension.
  */
 function getCustomMetrics() {
   return fs.readdirSync('dist/', { withFileTypes: true })
-    .filter((file) => path.extname(file.name) === '.js')
-    .map((file) => path.basename(file.name, '.js'));
+    .filter(file => path.extname(file.name) === '.js')
+    .map(file => path.basename(file.name, '.js'));
 }
 
+
+/**
+ * Retrieves the names of all JavaScript files in the 'dist/' directory that have changed in the current branch.
+ *
+ * @returns {string[]} An array of the base names of the JavaScript files without the '.js' extension.
+ */
+function getChangedCustomMetrics() {
+  const stdout = execSync('git diff --name-only --diff-filter=ACMRT origin/main', { encoding: 'utf-8' })
+
+  metricsList = stdout.split('\n')
+    .filter(file => RegExp('^dist\/.*\.js$', 'g').test(file))
+    .map(file => path.basename(file, '.js'))
+
+  metricsList = Array.from(new Set(metricsList)).sort()
+
+  return metricsList;
+}
+
+
 /**
  * Runs a WebPageTest (WPT) test for a given URL.
  *
  * @param {string} url - The URL to run the test on.
  * @returns {Promise<object>} A promise that resolves with an object containing the custom metrics.
  * @throws {Error} If the test run fails or the response status code is not 200.
  */
-function runWPTTest(url, metrics_to_log = []) {
-  const custom_metrics = getCustomMetrics();
-  metrics_to_log = metrics_to_log.length > 0 ? metrics_to_log : custom_metrics;
+function runWPTTest(url) {
+  const custom_metrics = getCustomMetrics()
+  const metrics_to_log = getChangedCustomMetrics()
 
   let options = { key: wptApiKey, custom: '' };
   for (const metric_name of custom_metrics) {
@@ -47,32 +68,42 @@ function runWPTTest(url, metrics_to_log = []) {
       } else {
         console.log(`WPT test run for ${url} completed`);
         let wpt_custom_metrics = {}
-        for (const metric_name of metrics_to_log) {
+        let wpt_custom_metrics_to_log = {}
+
+        for (const metric_name of custom_metrics) {
           wpt_custom_metric = response.data.runs['1'].firstView[`_${metric_name}`];
           try {
             wpt_custom_metrics[`_${metric_name}`] = JSON.parse(wpt_custom_metric);
+            if (metrics_to_log.includes(metric_name)) {
+              wpt_custom_metrics_to_log[`_${metric_name}`] = JSON.parse(wpt_custom_metric);
+            }
+
           } catch (e) {
             wpt_custom_metrics[`_${metric_name}`] = wpt_custom_metric;
           }
         }
 
-        fs.appendFileSync('test-results.md', '<details>\n' +
-          `<summary><strong>Custom metrics for ${url}</strong></summary>\n\n` +
-          `WPT test run results: ${response.data.summary}\n` +
-          (is_direct_run ? 'Changed custom metrics values:\n' +
-            `\`\`\`json\n${JSON.stringify(wpt_custom_metrics, null, 4)}\n\`\`\`\n` : '') +
-          '</details>\n');
+        fs.appendFileSync('test-results.md', `<details>
+<summary><strong>Custom metrics for ${url}</strong></summary>
+
+WPT test run results: ${response.data.summary}
+Changed custom metrics values:
+\`\`\`json
+${JSON.stringify(wpt_custom_metrics_to_log, null, 4)}
+\`\`\`
+</details>\n\n`);
 
         resolve(wpt_custom_metrics);
       }
     });
   });
 }
 
+
 if (is_direct_run) {
   const url = argv[2];
-  const metrics_to_log = argv[3].split('\n');
-  runWPTTest(url, metrics_to_log);
+
+  runWPTTest(url);
 }
 
 module.exports = { runWPTTest };