Precompile regexes (#382)

* Precompile regexes Signed-off-by: Emanuele Feliziani <[email protected]> * Commit compiled files Signed-off-by: Emanuele Feliziani <[email protected]> * Mark generated file as such in git Signed-off-by: Emanuele Feliziani <[email protected]> * Fix unintended changes Signed-off-by: Emanuele Feliziani <[email protected]> * Fix spacing Signed-off-by: Emanuele Feliziani <[email protected]> * Remove safeRegex Signed-off-by: Emanuele Feliziani <[email protected]> * Remove compiled matching config from gitattributes Signed-off-by: Emanuele Feliziani <[email protected]> --------- Signed-off-by: Emanuele Feliziani <[email protected]>
duckduckgo · Sep 21, 2023 · 4a0fd69 · 4a0fd69
1 parent f3eccad
commit 4a0fd69
Show file tree

Hide file tree

Showing 18 changed files with 1,428 additions and 2,764 deletions.
diff --git a/.eslintrc b/.eslintrc
@@ -30,6 +30,7 @@
         "integration-test/extension/autofill.js",
         "integration-test/extension/autofill-debug.js",
         "src/deviceApiCalls/__generated__/*",
+        "src/Form/matching-config/__generated__/*",
         "playwright-report/*"
     ]
 }
diff --git a/Gruntfile.js b/Gruntfile.js
@@ -78,7 +78,8 @@ module.exports = function (grunt) {
         },
         exec: {
             copyAssets: 'npm run copy-assets',
-            schemaCompile: 'npm run schema:generate'
+            schemaCompile: 'npm run schema:generate',
+            precompileRegexes: 'npm run precompile-regexes'
         },
         /**
          * Run predefined tasks whenever watched files are added,
@@ -89,6 +90,10 @@ module.exports = function (grunt) {
                 files: ['src/deviceApiCalls/**/*.{json,js}', 'packages/device-api/**/*.{json,js}'],
                 tasks: ['exec:schemaCompile']
             },
+            precompileRegexes: {
+                files: ['src/Form/matching-config/*'],
+                tasks: ['exec:precompileRegexes']
+            },
             scripts: {
                 files: ['src/**/*.{json,js}', 'packages/password/**/*.{json,js}', 'packages/device-api/**/*.{json,js}'],
                 tasks: ['browserify:dist', 'browserify:debug', 'exec:copyAssets']
@@ -105,6 +110,7 @@ module.exports = function (grunt) {
     })
 
     grunt.registerTask('default', [
+        'exec:precompileRegexes',
         'exec:schemaCompile',
         'browserify:dist',
         'browserify:debug',

diff --git a/dist/autofill-debug.js b/dist/autofill-debug.js
diff --git a/dist/autofill.js b/dist/autofill.js
diff --git a/package.json b/package.json
@@ -13,6 +13,7 @@
     "lint": "eslint .",
     "lint:fix": "npm run lint -- --fix",
     "copy-assets": "node scripts/copy-assets.js",
+    "precompile-regexes": "node scripts/precompile-regexes.js",
     "open-test-extension": "npx web-ext run -t chromium -u https://privacy-test-pages.site/ -s integration-test/extension",
     "schema:generate": "node scripts/api-call-generator.js",
     "test": "npm run test:unit && npm run lint && tsc",

diff --git a/scripts/precompile-regexes.js b/scripts/precompile-regexes.js
@@ -0,0 +1,72 @@
+const {matchingConfiguration} = require('../src/Form/matching-config/matching-config-source.js')
+const {writeFileSync} = require('fs')
+const {join} = require('path')
+const {inspect} = require('util')
+
+/**
+ * DDGRegexes are stored as strings so we can annotate them with comments, here we transform them into RegExp
+ */
+
+/**
+ * Loop through Object.entries and transform all values to RegExp
+ * @param {Object} obj
+ */
+function convertAllValuesToRegex (obj) {
+    for (const [key, value] of Object.entries(obj)) {
+        const source = String(value).normalize('NFKC')
+        obj[key] = new RegExp(source, 'ui')
+    }
+    return obj
+}
+for (const [key, value] of Object.entries(matchingConfiguration.strategies.ddgMatcher.matchers)) {
+    matchingConfiguration.strategies.ddgMatcher.matchers[key] = convertAllValuesToRegex(value)
+}
+
+/**
+ * Prepare CSS rules by concatenating arrays and removing whitespace
+ */
+Object.entries(matchingConfiguration.strategies.cssSelector.selectors).forEach(([name, selector]) => {
+    if (Array.isArray(selector)) {
+        selector = selector.join(',')
+    }
+    matchingConfiguration.strategies.cssSelector.selectors[name] = selector.replace(/\n/g, ' ').replace(/\s{2,}/g, ' ').trim()
+})
+
+/**
+ * VendorRules come from different providers, here we merge them all together in one RegEx per inputType
+ */
+
+/**
+ * Merge our vendor rules into a single RegEx
+ * @param {keyof VendorRegexRules} ruleName
+ * @param {VendorRegexConfiguration["ruleSets"]} ruleSets
+ * @return {{RULES: Record<keyof VendorRegexRules, RegExp | undefined>}}
+ */
+function mergeVendorRules (ruleName, ruleSets) {
+    let rules = []
+    ruleSets.forEach(set => {
+        if (set[ruleName]) {
+            rules.push(`(${set[ruleName]?.toLowerCase()})`.normalize('NFKC'))
+        }
+    })
+    return new RegExp(rules.join('|'), 'iu')
+}
+const ruleSets = matchingConfiguration.strategies.vendorRegex.ruleSets
+for (const ruleName of Object.keys(matchingConfiguration.strategies.vendorRegex.rules)) {
+    matchingConfiguration.strategies.vendorRegex.rules[ruleName] = mergeVendorRules(ruleName, ruleSets)
+}
+
+/**
+ * Build the file contents
+ */
+const fileContents = [
+    `/* DO NOT EDIT, this file was generated by scripts/precompile-regexes.js */\n\n`,
+    `/** @type {MatchingConfiguration} */\n`,
+    'const matchingConfiguration = ',
+    inspect(matchingConfiguration, {maxArrayLength: Infinity, depth: Infinity, maxStringLength: Infinity}),
+    '\n\nexport { matchingConfiguration }\n'
+].join('')
+
+// Write to file
+const outputPath = join(__dirname, '../src/Form/matching-config/__generated__', '/compiled-matching-config.js')
+writeFileSync(outputPath, fileContents)
diff --git a/src/Form/Form.js b/src/Form/Form.js
@@ -10,7 +10,7 @@ import {
     safeExecute, getTextShallow, wasAutofilledByChrome, shouldLog
 } from '../autofill-utils.js'
 
-import {getInputSubtype, getInputMainType, createMatching, safeRegex} from './matching.js'
+import {getInputSubtype, getInputMainType, createMatching} from './matching.js'
 import { getIconStylesAutofilled, getIconStylesBase, getIconStylesAlternate } from './inputStyles.js'
 import {canBeInteractedWith, getInputConfig, isFieldDecorated} from './inputTypeConfig.js'
 
@@ -205,7 +205,7 @@ class Form {
             // If we have a password but no username, let's search further
             const hiddenFields = /** @type [HTMLInputElement] */([...this.form.querySelectorAll('input[type=hidden]')])
             const probableField = hiddenFields.find((field) => {
-                const regex = safeRegex('email|' + this.matching.ddgMatcher('username')?.match)
+                const regex = new RegExp('email|' + this.matching.getDDGMatcherRegex('username')?.source)
                 const attributeText = field.id + ' ' + field.name
                 return regex?.test(attributeText)
             })

diff --git a/src/Form/FormAnalyzer.js b/src/Form/FormAnalyzer.js
@@ -1,6 +1,6 @@
 import { removeExcessWhitespace, Matching } from './matching.js'
 import { constants } from '../constants.js'
-import { matchingConfiguration } from './matching-configuration.js'
+import { matchingConfiguration } from './matching-config/__generated__/compiled-matching-config.js'
 import { getTextShallow, isLikelyASubmitButton } from '../autofill-utils.js'
 
 class FormAnalyzer {