Skip to content

Commit

Permalink
Support cosmetic filterlists (#466)
Browse files Browse the repository at this point in the history
* easylist experiment

* update easylist

* Tweak easylist hack

* delay easylist detection

* increase the easylist timeout

* update easylist

* Use fanboy-cookiemonster in the test extension

* Initialize and apply cosmetic filters together with prehide rules

* Fall back to filterist when no pop-up is found

* Remove easylist rule

* WIP

* WIP: add perf metrics

* add performance metrics

* Allow disabling autoconsent per site (in the test extension)

* Use uBO version of easylist cookie

* remove the perf metrics code

* Use constructed stylesheets for cosmetic styles

* minify the extension content script

* filterlist experiment

* Add filterlist overrides

* Update bundled filterlist

* minor filterlist changes

* Logging tweaks

* Tweak logs

* Add a rule for wise.com

* add a rule for nike

* Add a rule for dan.com

* Remove generic-cosmetic rule (too many false positives)

* add rule for medium.com

* Add a rule for abc.net.au

* Lint fix

* removed unused file

* support mobile aliexpress

* Update rule for temu

* Add rule for american airlines

* Add rule for tesla

* Wrap filterlist code in try catch

* Add a rule for admiral GDPR popups

* Tweak ensighten rule for britishairways.com

* Address minor PR comments

* comment overrides

* update the adblocker library and use the provided style override

* Bundle filterlist in JS

* Update filterlist on every release

* Tweak watch command

* Remove outdated comments

* Disable filterlist by default

* Move log to a more appropriate place

* remove unused ignore

* Lint fix in auto-generated file

* Update filterlist

* Remove autogenerated filterlist from git index

* Do not commit changes to filterlist

* Update the build scripts to update filterlist when necessary

* Regenerate package-lock (see npm/cli#4828)

* remove performance marks

* Produce separate builds with and without filterlist

* Avoid generating filterlist twice in ci

* Export json rules

* Do not commit filterlist file during release

* Build filterlist from easylist source

* Update readme

* add link to DDG download page

* cliqz/adblocker is renamed to ghostery/adblocker

* Address minor PR comments

* Track the resulting filterlist in git

* Add a ci job to update EasyList
  • Loading branch information
muodov authored Nov 7, 2024
1 parent 6acdffa commit 23fc31e
Show file tree
Hide file tree
Showing 21 changed files with 31,350 additions and 1,581 deletions.
8 changes: 7 additions & 1 deletion .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,16 @@ jobs:
with:
node-version: 18.x

- name: Install dependencies
run: npm ci

- name: Compile filterlist
run: |
npm run compile-filterlist
- name: Create Release
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
run: |
npm ci
npm run release
39 changes: 39 additions & 0 deletions .github/workflows/update-filterlist.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
name: Update filterlist

on:
workflow_dispatch:

schedule:
- cron: '0 3 * * SUN' # run every Sunday at 3:00

jobs:
update_filterlist:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4

- name: Use Node.js 18.x
uses: actions/setup-node@v3
with:
node-version: 18.x

- name: Install dependencies
run: npm ci

- name: Update EasyList
run: |
npm run update-easylist
- name: Read EasyList revision
id: read-revision
run: echo "revision=`cat rules/filterlists/easylist_revision.txt`"" >> "$GITHUB_OUTPUT"

- name: Create Pull Request
uses: peter-evans/create-pull-request@v7
with:
title: Update filterlist to ${{ steps.read-revision.outputs.revision }}
commit-message: Update filterlist to ${{ steps.read-revision.outputs.revision }}
labels: |
minor
dependencies
reviewers: muodov,sammacbeth
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -11,3 +11,5 @@ addon/*.js
.vscode/
.env
.DS_Store
rules/filterlists/easylist_*.txt
lib/filterlist-engine.ts
3 changes: 1 addition & 2 deletions addon/background.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,8 @@ import { initConfig, isEnabledForDomain, showOptOutStatus } from "./utils";
const openDevToolsPanels = new Map<number, chrome.runtime.Port>();

async function loadRules() {
const res = await fetch("./rules.json");
storageSet({
rules: await res.json(),
rules: await (await fetch("./rules.json")).json(),
});
}

Expand Down
4 changes: 4 additions & 0 deletions addon/popup.html
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,10 @@
</div>
<button id="reload">Reload rules</button>
<button id="reset">Reset settings</button>
<div>
<input type="checkbox" id="filterlist" name="filterlist" checked>
<label for="filterlist">Enable cosmetic filterlist</label>
</div>
</fieldset>
<script src="popup.bundle.js"></script>
</body>
Expand Down
7 changes: 7 additions & 0 deletions addon/popup.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ async function init() {
const logsMessagesCheckbox = document.querySelector('input#logs-messages') as HTMLInputElement;
const ruleReloadButton = document.querySelector('#reload') as HTMLButtonElement;
const resetButton = document.querySelector('#reset') as HTMLButtonElement;
const filterlistCheckbox = document.querySelector('input#filterlist') as HTMLInputElement;

// enable proceed button when necessary

Expand Down Expand Up @@ -60,6 +61,7 @@ async function init() {
logsErrorsCheckbox.checked = autoconsentConfig.logs.errors;
logsMessagesCheckbox.checked = autoconsentConfig.logs.messages;
retriesInput.value = autoconsentConfig.detectRetries.toString();
filterlistCheckbox.checked = autoconsentConfig.enableFilterList;
if (autoconsentConfig.autoAction === 'optIn') {
optInRadio.checked = true;
} else if (autoconsentConfig.autoAction === 'optOut') {
Expand Down Expand Up @@ -146,6 +148,11 @@ async function init() {
updateLogsConfig();
});

filterlistCheckbox.addEventListener('change', () => {
autoconsentConfig.enableFilterList = filterlistCheckbox.checked;
storageSet({ config: autoconsentConfig });
});

ruleReloadButton.addEventListener('click', async () => {
const res = await fetch("./rules.json");
storageSet({
Expand Down
16 changes: 9 additions & 7 deletions build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,17 @@ set -ex

ESBUILD="node_modules/.bin/esbuild --bundle"

$ESBUILD --format=iife --target=es2021 --minify playwright/content.ts --outfile=dist/autoconsent.playwright.js
$ESBUILD --format=esm --target=es2021 lib/web.ts --outfile=dist/autoconsent.esm.js
$ESBUILD --format=cjs --target=es2021 --platform=node lib/web.ts --outfile=dist/autoconsent.cjs.js
$ESBUILD --format=iife --define:BUNDLE_FILTERLIST=true --target=es2021 playwright/content.ts --outfile=dist/autoconsent.playwright.js
$ESBUILD --format=esm --define:BUNDLE_FILTERLIST=true --target=es2021 lib/web.ts --outfile=dist/autoconsent.extra.esm.js
$ESBUILD --format=cjs --define:BUNDLE_FILTERLIST=true --target=es2021 --platform=node lib/web.ts --outfile=dist/autoconsent.extra.cjs.js
$ESBUILD --format=esm --define:BUNDLE_FILTERLIST=false --target=es2021 lib/web.ts --outfile=dist/autoconsent.esm.js
$ESBUILD --format=cjs --define:BUNDLE_FILTERLIST=false --target=es2021 --platform=node lib/web.ts --outfile=dist/autoconsent.cjs.js

# Extension
$ESBUILD addon/background.ts --outfile=dist/addon-mv3/background.bundle.js
$ESBUILD addon/content.ts --outfile=dist/addon-mv3/content.bundle.js
$ESBUILD addon/popup.ts --outfile=dist/addon-mv3/popup.bundle.js
$ESBUILD addon/devtools/panel.ts --outfile=dist/addon-mv3/devtools/panel.js
$ESBUILD addon/background.ts --define:BUNDLE_FILTERLIST=true --outfile=dist/addon-mv3/background.bundle.js
$ESBUILD addon/content.ts --define:BUNDLE_FILTERLIST=true --outfile=dist/addon-mv3/content.bundle.js
$ESBUILD addon/popup.ts --define:BUNDLE_FILTERLIST=true --outfile=dist/addon-mv3/popup.bundle.js
$ESBUILD addon/devtools/panel.ts --define:BUNDLE_FILTERLIST=true --outfile=dist/addon-mv3/devtools/panel.js

## Copy extension files into place
mkdir -p dist/addon-firefox
Expand Down
2 changes: 1 addition & 1 deletion lib/cmps/airbnb.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { waitFor } from "../utils";
import AutoConsentCMPBase from "./base";

export default class Airbnb extends AutoConsentCMPBase {
name: "airbnb";
name = "airbnb";

runContext: RunContext = {
urlPattern: '^https://(www\\.)?airbnb\\.[^/]+/'
Expand Down
16 changes: 16 additions & 0 deletions lib/dom-actions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,22 @@ export class DomActions implements DomActionsProvider {
return !!existingElement;
}

async createOrUpdateStyleSheet(cssText: string, styleSheet?: CSSStyleSheet) {
if (!styleSheet) {
styleSheet = new CSSStyleSheet();
}
styleSheet = await styleSheet.replace(cssText);
return styleSheet;
}

removeStyleSheet(styleSheet?: CSSStyleSheet): boolean {
if (styleSheet) {
styleSheet.replace('');
return true;
}
return false;
}

querySingleReplySelector(selector: string, parent: any = document): HTMLElement[] {
if (selector.startsWith('aria/')) {
return []
Expand Down
45 changes: 45 additions & 0 deletions lib/filterlist-utils.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
import { FiltersEngine } from '@ghostery/adblocker';
import { extractFeaturesFromDOM } from '@ghostery/adblocker-content';
import { parse as tldtsParse } from 'tldts-experimental';
import { getHidingStyle } from './utils';

export function deserializeFilterList(serializedEngine: Uint8Array) {
return FiltersEngine.deserialize(serializedEngine)
}

export function getCosmeticStylesheet(engine: FiltersEngine): string {
try {
const parsed = tldtsParse(location.href);
const hostname = parsed.hostname || '';
const domain = parsed.domain || '';

const cosmetics = engine.getCosmeticsFilters({
url: location.href,
hostname,
domain,

// this extracts current ids, classes and attributes (depends on the current DOM state)
...extractFeaturesFromDOM([document.documentElement]),

getBaseRules: true,
getInjectionRules: false, // we don't inject scripts atm
getExtendedRules: true,
getRulesFromDOM: true,
getRulesFromHostname: true,

hidingStyle: getHidingStyle('opacity'),
});
return cosmetics.styles;
} catch (e) {
console.error('Error getting cosmetic rules', e);
return '';
}
}

export function getFilterlistSelectors(styles: string): string {
if (styles) {
const selectorsOnly = styles.replace(/\s*{[^\\}]*}\s*/g, ',').replace(/,$/, '');
return selectorsOnly;
}
return '';
}
5 changes: 4 additions & 1 deletion lib/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ export interface DomActionsProvider {

export type RuleBundle = {
autoconsent: AutoConsentCMPRule[];
consentomatic: { [name: string]: ConsentOMaticConfig };
consentomatic?: { [name: string]: ConsentOMaticConfig };
};

export type AutoAction = 'optOut' | 'optIn' | null;
Expand All @@ -52,6 +52,7 @@ export type Config = {
detectRetries: number;
isMainWorld: boolean;
prehideTimeout: number;
enableFilterList: boolean;
logs: {
lifecycle: boolean;
rulesteps: boolean;
Expand All @@ -66,6 +67,7 @@ export type LifecycleState = 'loading' |
'waitingForInitResponse' |
'started' |
'nothingDetected' |
'cosmeticFiltersDetected' |
'cmpDetected' |
'openPopupDetected' |
'runningOptOut' |
Expand All @@ -77,6 +79,7 @@ export type LifecycleState = 'loading' |
'done';

export type ConsentState = {
cosmeticFiltersOn: boolean; // true if cosmetic filter rules are currently applied.
lifecycle: LifecycleState; // What point in the autoconsent lifecycle this script is at.
prehideOn: boolean; // If the script is currently hiding preHide elements.
findCmpAttempts: number; // Number of times we tried to find CMPs in this frame.
Expand Down
9 changes: 7 additions & 2 deletions lib/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,18 @@ export function getStyleElement(styleOverrideElementId = "autoconsent-css-rules"
}
}

export function getHidingStyle(method: HideMethod) {
const hidingSnippet = method === "opacity" ? `opacity: 0` : `display: none`; // use display by default
return `${hidingSnippet} !important; z-index: -1 !important; pointer-events: none !important;`
}

// hide elements with a CSS rule
export function hideElements(
styleEl: HTMLStyleElement,
selector: string,
method: HideMethod = 'display',
): boolean {
const hidingSnippet = method === "opacity" ? `opacity: 0` : `display: none`; // use display by default
const rule = `${selector} { ${hidingSnippet} !important; z-index: -1 !important; pointer-events: none !important; } `;
const rule = `${selector} { ${getHidingStyle(method)} } `;

if (styleEl instanceof HTMLStyleElement) {
styleEl.innerText += rule;
Expand Down Expand Up @@ -79,6 +83,7 @@ export function normalizeConfig(providedConfig: any): Config {
detectRetries: 20,
isMainWorld: false,
prehideTimeout: 2000,
enableFilterList: false,
logs: {
lifecycle: false,
rulesteps: false,
Expand Down
Loading

0 comments on commit 23fc31e

Please sign in to comment.