From e7f60dd440c14005d800a0bb6a5d7c229ca29596 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Tue, 29 May 2018 20:07:54 -0700 Subject: [PATCH 01/12] Add initial reasons/referer.js --- src/js/reasons/referer.js | 53 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 src/js/reasons/referer.js diff --git a/src/js/reasons/referer.js b/src/js/reasons/referer.js new file mode 100644 index 0000000..c9f25e5 --- /dev/null +++ b/src/js/reasons/referer.js @@ -0,0 +1,53 @@ +"use strict"; + +[(function(exports) { + +const {LruMap} = require('../utils'); + +function markHeadersMutated(details) { + details[details.headerPropName].mutated = true; +} + +/* request thing: + * + * remove referer, store requestId, & referer value + * + * If this returns a 4xx, then redirect with the referer added back in. + * If this still fails, give up. + */ +function refererHeader({requestIdCache}, details, header) { + header.value = details.url; + markHeadersMutated(details); + return false; +} + +class Referer { + constructor() { + this.requestIdCache = new LruMap(1000); + this.badRedirects = new Set(); + } + + removeRefererFailed({statusCode, requestId}) { + return ((400 <= statusCode) && (statusCode < 500)) && this.requestId.has(requestId) && !this.badRedirects.has(requestId); + } + + onBeforeSendHeaders(details, header) { + if (this.failedAlready(details)) { + return false; + } + return true; + } + + onHeadersReceived(details) { + if (this.removeRefererFailedOnce(details)) { + this.badRedirects.add(details.requestId); + return {redirectUrl: details.url} + } + } + onBeforeRedirect(details) { + } +} + +Object.assign(exports, {Referer}); + +})].map(func => typeof exports == 'undefined' ? define('/reasons/referer', func) : func(exports)); From b3db5c7a7473f19a722df10e7e439f8bb650ff21 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Tue, 29 May 2018 20:08:17 -0700 Subject: [PATCH 02/12] Add test/reasons/referer_test.js --- src/js/test/reasons/referer_test.js | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 src/js/test/reasons/referer_test.js diff --git a/src/js/test/reasons/referer_test.js b/src/js/test/reasons/referer_test.js new file mode 100644 index 0000000..7adfe6e --- /dev/null +++ b/src/js/test/reasons/referer_test.js @@ -0,0 +1,10 @@ +"use strict"; + +const {assert} = require('chai'), + {Referer} = require('../../reasons/referer'); + +describe('referer.js', function() { + it('Referer', async function() { + const referer = new Referer(); + }); +}); From c114e3af1e1bcfac2e9e03f5d26cfcdfd7f5a07f Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Thu, 31 May 2018 13:45:38 -0700 Subject: [PATCH 03/12] Continue building out Referer class --- src/js/reasons/referer.js | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/src/js/reasons/referer.js b/src/js/reasons/referer.js index c9f25e5..8e8251d 100644 --- a/src/js/reasons/referer.js +++ b/src/js/reasons/referer.js @@ -21,31 +21,42 @@ function refererHeader({requestIdCache}, details, header) { return false; } +function is4xx(statusCode) { + return (400 <= statusCode) && (statusCode < 500); +} + class Referer { constructor() { this.requestIdCache = new LruMap(1000); - this.badRedirects = new Set(); + this.badRedirects = new LruMap(1000); + } + + removeRefererFailedOnce({statusCode, requestId}) { + return (is4xx(statusCode) && this.requestIdCache.has(requestId)) && !this.badRedirects.has(requestId); } - removeRefererFailed({statusCode, requestId}) { - return ((400 <= statusCode) && (statusCode < 500)) && this.requestId.has(requestId) && !this.badRedirects.has(requestId); + failedAlready({requestId}) { + return this.badRedirects.has(requestId); } - onBeforeSendHeaders(details, header) { + shouldRemoveHeader(details, header) { + if (!this.requestIdCache.has(details.requestId)) { + this.requestIdCache.set(details.requestId, header.value); + } + if (this.failedAlready(details)) { return false; } + return true; } onHeadersReceived(details) { if (this.removeRefererFailedOnce(details)) { - this.badRedirects.add(details.requestId); - return {redirectUrl: details.url} + this.badRedirects.set(details.requestId); + return details.response = {redirectUrl: details.url}; } } - onBeforeRedirect(details) { - } } Object.assign(exports, {Referer}); From 12816b600c1fad335c2cf019a58c9f81ce7d9101 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Thu, 31 May 2018 13:45:55 -0700 Subject: [PATCH 04/12] Build out tests for Referer class --- src/js/test/reasons/referer_test.js | 38 +++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/src/js/test/reasons/referer_test.js b/src/js/test/reasons/referer_test.js index 7adfe6e..076f406 100644 --- a/src/js/test/reasons/referer_test.js +++ b/src/js/test/reasons/referer_test.js @@ -4,7 +4,41 @@ const {assert} = require('chai'), {Referer} = require('../../reasons/referer'); describe('referer.js', function() { - it('Referer', async function() { - const referer = new Referer(); + let requestId = 1, + url = 'https://whatever.com/nope'; + beforeEach(function() { + this.details = {requestId, url}; + this.header = {name: 'Referer', value: 'https://foo.com/stuff'}; + this.referer = new Referer(); + }); + describe('#shouldRemoveHeader', function() { + it('removes first', function() { + assert.isTrue(this.referer.shouldRemoveHeader(this.details, this.header)); + }); + it('does not remove when failedAlready', function() { + this.referer.badRedirects.set(requestId); + assert.isFalse(this.referer.shouldRemoveHeader(this.details, this.header)); + }); + }); + + describe('#onHeadersReceived', function() { + it('no action for non-400 responses', function() { + assert.isUndefined(this.referer.onHeadersReceived({statusCode: 200})); + }); + describe('sent', function() { + beforeEach(function() { + this.referer.shouldRemoveHeader(this.details, this.header); + }); + it('no actionn for already failed but 400 again responses', function() { + this.referer.badRedirects.set(requestId); + assert.isUndefined(this.referer.onHeadersReceived({requestId, statusCode: 403})); + }); + it('redirects on first 400', function() { + let {details} = this, + statusCode = 403; + Object.assign(details, {statusCode}); + assert.deepEqual(this.referer.onHeadersReceived(details), {redirectUrl: details.url}); + }); + }); }); }); From eda04f382042f0b541bf85298b2cf5e57bb53b3d Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Thu, 31 May 2018 13:46:57 -0700 Subject: [PATCH 05/12] Add a headers.mutated check in headers func --- src/js/webrequest.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/js/webrequest.js b/src/js/webrequest.js index 3ddce3b..49da58b 100644 --- a/src/js/webrequest.js +++ b/src/js/webrequest.js @@ -109,7 +109,7 @@ class WebRequest { let headers = details[details.headerPropName], removed = this.removeHeaders(details, headers); this.checkAllRequestActions(details); - if (!details.shortCircuit && removed.length) { + if (!details.shortCircuit && (removed.length || headers.mutated)) { details.response = {[details.headerPropName]: headers}; this.markHeaders(removed, details); } From 1cc9aeeafa25f6060eab56647e9fc09da2f36f1d Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Fri, 1 Jun 2018 15:50:13 -0700 Subject: [PATCH 06/12] Add reasons/referer.js to manifest & popoup --- src/manifest.json | 1 + src/skin/popup.html | 1 + 2 files changed, 2 insertions(+) diff --git a/src/manifest.json b/src/manifest.json index d2967c3..9b3be13 100644 --- a/src/manifest.json +++ b/src/manifest.json @@ -55,6 +55,7 @@ "js/reasons/reasons.js", "js/reasons/fingerprinting.js", "js/reasons/user_url_deactivate.js", + "js/reasons/referer.js", "js/reasons/headers.js", "js/reasons/etag.js", "js/reasons/utils.js", diff --git a/src/skin/popup.html b/src/skin/popup.html index b7a81b5..167aa34 100644 --- a/src/skin/popup.html +++ b/src/skin/popup.html @@ -14,6 +14,7 @@ + From 0436d058c8becb1a4d7bbf986b0c37b9cf1c84c4 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Fri, 1 Jun 2018 15:50:45 -0700 Subject: [PATCH 07/12] Add logging to Referer, remove cruft --- src/js/reasons/referer.js | 22 +++------------------- 1 file changed, 3 insertions(+), 19 deletions(-) diff --git a/src/js/reasons/referer.js b/src/js/reasons/referer.js index 8e8251d..c7eaa7b 100644 --- a/src/js/reasons/referer.js +++ b/src/js/reasons/referer.js @@ -2,24 +2,7 @@ [(function(exports) { -const {LruMap} = require('../utils'); - -function markHeadersMutated(details) { - details[details.headerPropName].mutated = true; -} - -/* request thing: - * - * remove referer, store requestId, & referer value - * - * If this returns a 4xx, then redirect with the referer added back in. - * If this still fails, give up. - */ -function refererHeader({requestIdCache}, details, header) { - header.value = details.url; - markHeadersMutated(details); - return false; -} +const {LruMap, log} = require('../utils'); function is4xx(statusCode) { return (400 <= statusCode) && (statusCode < 500); @@ -45,15 +28,16 @@ class Referer { } if (this.failedAlready(details)) { + log('failed referer already'); return false; } - return true; } onHeadersReceived(details) { if (this.removeRefererFailedOnce(details)) { this.badRedirects.set(details.requestId); + log(`failed referer removal, redirecting ${details.url}`); return details.response = {redirectUrl: details.url}; } } From 614000cebaa04ee82b6b7d8a32df29d9fbde16bc Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Fri, 1 Jun 2018 15:51:15 -0700 Subject: [PATCH 08/12] Use Referer in reasons/headers.js --- src/js/reasons/headers.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/js/reasons/headers.js b/src/js/reasons/headers.js index b565c94..0c2b3ef 100644 --- a/src/js/reasons/headers.js +++ b/src/js/reasons/headers.js @@ -4,18 +4,20 @@ const {Action} = require('../schemes'), {URL} = require('../shim'), - {LruMap, hasAction} = require('../utils'), + {hasAction} = require('../utils'), {newEtagHeaderFunc} = require('./etag'), + {Referer} = require('./referer'), {HEADER_DEACTIVATE_ON_HOST, header_methods, NO_ACTION, TAB_DEACTIVATE_HEADERS} = require('../constants'); const alwaysTrue = () => true; class HeaderHandler { constructor(store) { + this.referer = new Referer(); this.badHeaders = new Map([ ['cookie', alwaysTrue], ['set-cookie', alwaysTrue], - ['referer', alwaysTrue], + ['referer', this.referer.shouldRemoveHeader.bind(this.referer)], ['etag', newEtagHeaderFunc(store)], ['if-none-match', alwaysTrue] ]); From 427696d972144ea378fb73c1eddd832c521b855c Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Fri, 1 Jun 2018 15:51:44 -0700 Subject: [PATCH 09/12] Add checkOnHeadersReceived to webrequests.js --- src/js/webrequest.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/js/webrequest.js b/src/js/webrequest.js index 49da58b..26ebfd4 100644 --- a/src/js/webrequest.js +++ b/src/js/webrequest.js @@ -104,11 +104,16 @@ class WebRequest { return details.response; } + checkOnHeadersReceived(details) { + return this.handler.headerHandler.referer.onHeadersReceived(details); + } + headerHandler(details) { if (this.isThirdParty(details)) { let headers = details[details.headerPropName], removed = this.removeHeaders(details, headers); this.checkAllRequestActions(details); + this.checkOnHeadersReceived(details); if (!details.shortCircuit && (removed.length || headers.mutated)) { details.response = {[details.headerPropName]: headers}; this.markHeaders(removed, details); From ba781391ae710bf92cd09d14120f4eb69b136222 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Mon, 4 Jun 2018 17:08:06 -0700 Subject: [PATCH 10/12] Add shortCircuit to referer redirect --- src/js/reasons/referer.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/js/reasons/referer.js b/src/js/reasons/referer.js index c7eaa7b..a33616c 100644 --- a/src/js/reasons/referer.js +++ b/src/js/reasons/referer.js @@ -38,6 +38,7 @@ class Referer { if (this.removeRefererFailedOnce(details)) { this.badRedirects.set(details.requestId); log(`failed referer removal, redirecting ${details.url}`); + details.shortCircuit = true; return details.response = {redirectUrl: details.url}; } } From 945e900f5520029fdc19ea64c68a38d752b5475d Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Mon, 4 Jun 2018 17:08:49 -0700 Subject: [PATCH 11/12] refactoring constants usage --- src/js/webrequest.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/js/webrequest.js b/src/js/webrequest.js index 26ebfd4..29a86a8 100644 --- a/src/js/webrequest.js +++ b/src/js/webrequest.js @@ -5,6 +5,7 @@ const shim = require('./shim'), {URL} = shim, constants = require('./constants'), {header_methods, request_methods} = constants, + {ON_BEFORE_REQUEST, ON_BEFORE_SEND_HEADERS, ON_HEADERS_RECEIVED} = request_methods, {Handler} = require('./reasons/handlers'); function annotateDetails(details, requestType) { @@ -85,20 +86,20 @@ class WebRequest { } onBeforeRequest(details) { - annotateDetails(details, request_methods.ON_BEFORE_REQUEST); + annotateDetails(details, ON_BEFORE_REQUEST); this.recordRequest(details); return this.commitRequest(details); } onBeforeSendHeaders(details) { - annotateDetails(details, request_methods.ON_BEFORE_SEND_HEADERS); + annotateDetails(details, ON_BEFORE_SEND_HEADERS); this.headerHandler(details); this.markAction(details); return details.response; } onHeadersReceived(details) { - annotateDetails(details, request_methods.ON_HEADERS_RECEIVED); + annotateDetails(details, ON_HEADERS_RECEIVED); this.headerHandler(details); this.markAction(details); return details.response; From a07f16ec3b053ad7a92dbc3b56f6b4fb15db20c4 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Mon, 4 Jun 2018 17:09:39 -0700 Subject: [PATCH 12/12] Update header check with referer Only applies to onBeforeSendHeaders --- src/js/webrequest.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/js/webrequest.js b/src/js/webrequest.js index 29a86a8..3fd7931 100644 --- a/src/js/webrequest.js +++ b/src/js/webrequest.js @@ -105,8 +105,12 @@ class WebRequest { return details.response; } - checkOnHeadersReceived(details) { - return this.handler.headerHandler.referer.onHeadersReceived(details); + requestOrResponseAction(details) { + if (!details.shortCircuit) { + if (details.requestType == ON_HEADERS_RECEIVED) { + return this.handler.headerHandler.referer.onHeadersReceived(details); + } + } } headerHandler(details) { @@ -114,7 +118,7 @@ class WebRequest { let headers = details[details.headerPropName], removed = this.removeHeaders(details, headers); this.checkAllRequestActions(details); - this.checkOnHeadersReceived(details); + this.requestOrResponseAction(details); if (!details.shortCircuit && (removed.length || headers.mutated)) { details.response = {[details.headerPropName]: headers}; this.markHeaders(removed, details);