From f4ae631e052cb8cd48eb4492c0981726665f7f78 Mon Sep 17 00:00:00 2001 From: Maxim Tsoy Date: Wed, 4 Sep 2024 11:22:15 -0400 Subject: [PATCH] Fix some unit tests --- post-processing/clickhouse.js | 2 +- tests/collectors/APICallCollector.mocha.js | 2 +- tests/collectors/CMPCollector.mocha.js | 19 +++++------- tests/collectors/CookieCollector.test.js | 2 +- tests/collectors/RequestCollector.test.js | 35 +++++++++++----------- tests/collectors/TargetCollector.test.js | 4 +-- 6 files changed, 30 insertions(+), 34 deletions(-) diff --git a/post-processing/clickhouse.js b/post-processing/clickhouse.js index 402e2b24..4b22bf6c 100644 --- a/post-processing/clickhouse.js +++ b/post-processing/clickhouse.js @@ -23,7 +23,7 @@ Examples: .option('--delete', 'Delete data for the given crawlid') .parse(process.argv); -const opts = program.opts() +const opts = program.opts(); const ch = new Clickhouse(); const crawlName = opts.crawlname; diff --git a/tests/collectors/APICallCollector.mocha.js b/tests/collectors/APICallCollector.mocha.js index f4e99b61..5cbf22db 100644 --- a/tests/collectors/APICallCollector.mocha.js +++ b/tests/collectors/APICallCollector.mocha.js @@ -65,7 +65,7 @@ describe('APICallCollector', () => { }); // @ts-ignore not a real CDP client - await collector.addTarget({cdpClient: fakeCDPClient, type: 'page', url: 'https://example.com'}); + await collector.addTarget({session: fakeCDPClient, type: 'page', url: 'https://example.com'}); const executionContextCreated = listeners.find(a => a.name === 'Runtime.executionContextCreated'); assert(executionContextCreated, 'executionContextCreated listener was set'); diff --git a/tests/collectors/CMPCollector.mocha.js b/tests/collectors/CMPCollector.mocha.js index 104e4a96..29871b04 100644 --- a/tests/collectors/CMPCollector.mocha.js +++ b/tests/collectors/CMPCollector.mocha.js @@ -66,7 +66,7 @@ describe('CMPCollector', () => { } }); // @ts-ignore not a real CDP client - await collector.addTarget({cdpClient: fakeCDPClient, type: 'page', url: 'https://example.com'}); + await collector.addTarget({session: fakeCDPClient, type: 'page', url: 'https://example.com'}); }); describe('handleMessage', () => { @@ -284,18 +284,15 @@ describe('CMPCollector', () => { const contentScriptEval = commands.find(cmd => cmd[0] === 'Runtime.evaluate')[1]; assert.strictEqual(contentScriptEval.contextId, 31337); - // @ts-ignore no need to provide all params - collector.context.pages = () => Promise.resolve([ - { - frames: () => [ - { - evaluate: () => Promise.resolve('This website is using cookies. We are using cookies! To reiterate, you consent to the use of cookies on this website. In fact, there is nothing you can possibly do.') - } - ] + const origSessionSend = collector._cdpClient.send; + // @ts-expect-error monkey patching the method + collector._cdpClient.send = () => Promise.resolve({ + result: { + value: 'This website is using cookies. We are using cookies! To reiterate, you consent to the use of cookies on this website. In fact, there is nothing you can possibly do.' } - ]); - + }); await collector.postLoad(); + collector._cdpClient.send = origSessionSend; const results = await collector.getData(); assert.deepStrictEqual(results, [{ name: '', diff --git a/tests/collectors/CookieCollector.test.js b/tests/collectors/CookieCollector.test.js index 06b7ea0b..4919ab3c 100644 --- a/tests/collectors/CookieCollector.test.js +++ b/tests/collectors/CookieCollector.test.js @@ -51,7 +51,7 @@ const fakeCDPClient = { collector.init({}); // @ts-ignore not a real CDP client -collector.addTarget({cdpClient: fakeCDPClient, type: 'page', url: 'http://example.com'}); +collector.addTarget({session: fakeCDPClient, type: 'page', url: 'http://example.com'}); collector.getData() .then(data => { diff --git a/tests/collectors/RequestCollector.test.js b/tests/collectors/RequestCollector.test.js index bc4e7031..2a675ee9 100644 --- a/tests/collectors/RequestCollector.test.js +++ b/tests/collectors/RequestCollector.test.js @@ -8,7 +8,7 @@ function createFakeCDP() { */ const listeners = []; - const cdpClient = { + const cdpSession = { send: () => Promise.resolve(), on: (/** @type {string} **/name, /** @type {function(object)} **/callback) => { listeners.push({name, callback}); @@ -18,7 +18,7 @@ function createFakeCDP() { return { listeners, - cdpClient + cdpSession }; } @@ -29,7 +29,7 @@ async function testDefaultSettings() { /** * getData */ - const {listeners, cdpClient: fakeCDPClient} = createFakeCDP(); + const {listeners, cdpSession: fakeCDPClient} = createFakeCDP(); // @ts-ignore no need to provide all params collector.init({ @@ -37,7 +37,7 @@ async function testDefaultSettings() { }); // @ts-ignore not a real CDP client - await collector.addTarget({cdpClient: fakeCDPClient, type: 'page', url: 'http://example.com'}); + await collector.addTarget({session: fakeCDPClient, type: 'page', url: 'http://example.com'}); /** * Regular request - success @@ -273,7 +273,16 @@ async function testResponseHashSetting() { /** * getData */ - const {listeners, cdpClient: fakeCDPClient} = createFakeCDP(); + const {listeners, cdpSession: fakeCDPClient} = createFakeCDP(); + // @ts-expect-error monkeypatching a method + fakeCDPClient.send = command => { + if (command === 'Network.getResponseBody') { + // eslint-disable-next-line no-console + return Promise.resolve({body: 'dGVzdA==', base64Encoded: true});//btoa('test') + } + + return Promise.resolve(); + }; // @ts-ignore no need to provide all params collector.init({ @@ -281,7 +290,7 @@ async function testResponseHashSetting() { }); // @ts-ignore not a real CDP client - await collector.addTarget({cdpClient: fakeCDPClient, type: 'page', url: 'http://example.com'}); + await collector.addTarget({session: fakeCDPClient, type: 'page', url: 'http://example.com'}); const requestWillBeSent = listeners.find(a => a.name === 'Network.requestWillBeSent'); const responseReceived = listeners.find(a => a.name === 'Network.responseReceived'); @@ -311,16 +320,6 @@ async function testResponseHashSetting() { } }); - //@ts-ignore - fakeCDPClient.send = command => { - if (command === 'Network.getResponseBody') { - // eslint-disable-next-line no-console - return Promise.resolve({body: 'dGVzdA==', base64Encoded: true});//btoa('test') - } - - return Promise.resolve(); - }; - await loadingFinished.callback({requestId: 100, encodedDataLength: 666, timestamp: 223456}); const data1 = collector.getData({finalUrl: 'https://example.com/'}); @@ -350,7 +349,7 @@ async function testCustomHeadersSetting() { /** * getData */ - const {listeners, cdpClient: fakeCDPClient} = createFakeCDP(); + const {listeners, cdpSession: fakeCDPClient} = createFakeCDP(); // @ts-ignore no need to provide all params collector.init({ @@ -358,7 +357,7 @@ async function testCustomHeadersSetting() { }); // @ts-ignore not a real CDP client - await collector.addTarget({cdpClient: fakeCDPClient, type: 'page', url: 'http://example.com'}); + await collector.addTarget({session: fakeCDPClient, type: 'page', url: 'http://example.com'}); const requestWillBeSent = listeners.find(a => a.name === 'Network.requestWillBeSent'); const responseReceived = listeners.find(a => a.name === 'Network.responseReceived'); diff --git a/tests/collectors/TargetCollector.test.js b/tests/collectors/TargetCollector.test.js index e1043856..0d78c031 100644 --- a/tests/collectors/TargetCollector.test.js +++ b/tests/collectors/TargetCollector.test.js @@ -11,9 +11,9 @@ const fakeCDPClient = {}; collector.init(); // @ts-ignore not a real CDP client -collector.addTarget({cdpClient: fakeCDPClient, type: 'page', url: 'http://example.com'}); +collector.addTarget({session: fakeCDPClient, type: 'page', url: 'http://example.com'}); // @ts-ignore not a real CDP client -collector.addTarget({cdpClient: fakeCDPClient, type: 'service_worker', url: 'http://example.com/sw.js'}); +collector.addTarget({session: fakeCDPClient, type: 'service_worker', url: 'http://example.com/sw.js'}); const targets = collector.getData();