removing unused var

the-markup · Aug 28, 2024 · 6df8aff · 6df8aff
1 parent e5e130f
commit 6df8aff
Showing 1 changed file with 6 additions and 16 deletions.
diff --git a/src/collector.ts b/src/collector.ts
@@ -5,7 +5,6 @@ import { join } from 'path';
 import puppeteer, { Browser, Page, PuppeteerLifeCycleEvent, KnownDevices, PuppeteerLaunchOptions } from 'puppeteer';
 import PuppeteerHar from 'puppeteer-har';
 import { getDomain, getSubdomain, parse } from 'tldts';
-import url from 'url';
 import { captureBrowserCookies, clearCookiesCache, setupHttpCookieCapture } from './cookie-collector';
 import { setupBlacklightInspector } from './inspector';
 import { setupKeyLoggingInspector } from './key-logging';
@@ -64,7 +63,7 @@ export const collect = async (inUrl: string, args: CollectorOptions) => {
         uri_dest: null,
         uri_redirects: null,
         secure_connection: {},
-        host: url.parse(inUrl).hostname,
+        host: new URL(inUrl).hostname,
         config: {
             cleareCache: args.clearCache,
             captureHar: args.captureHar,
@@ -102,7 +101,6 @@ export const collect = async (inUrl: string, args: CollectorOptions) => {
     let pageIndex = 1;
     let har = {} as any;
     let page_response = null;
-    let loadError = false;
     const userDataDir = args.saveBrowserProfile ? join(args.outDir, 'browser-profile') : undefined;
     let didBrowserDisconnect = false;
 
@@ -184,6 +182,7 @@ export const collect = async (inUrl: string, args: CollectorOptions) => {
         // Function to navigate to a page with a timeout guard
         const navigateWithTimeout = async (page: Page, url: string, timeout: number, waitUntil: PuppeteerLifeCycleEvent) => {
             try {
+                console.log(page);
                 page_response = await Promise.race([
                     page.goto(url, {
                         timeout: timeout,
@@ -198,7 +197,7 @@ export const collect = async (inUrl: string, args: CollectorOptions) => {
                 ]);
             } catch (error) {
                 console.log('First attempt failed, trying with domcontentloaded');
-
+                console.log(page);
                 page_response = await page.goto(url, {
                     timeout: timeout,
                     waitUntil: 'domcontentloaded' as PuppeteerLifeCycleEvent
@@ -208,7 +207,7 @@ export const collect = async (inUrl: string, args: CollectorOptions) => {
         };
 
         // Go to the first url
-        console.log('Going to the first url');
+        console.log('Going to the first url', inUrl);
         await navigateWithTimeout(page, inUrl, args.defaultTimeout, args.defaultWaitUntil as PuppeteerLifeCycleEvent);
 
         pageIndex++;
@@ -220,17 +219,6 @@ export const collect = async (inUrl: string, args: CollectorOptions) => {
             third_party: []
         };
 
-        // Return if the page doesnt load
-        if (loadError) {
-            await closeBrowser(browser);
-            if (typeof userDataDir !== 'undefined') {
-                clearDir(userDataDir, false);
-            }
-            if (args.outDir.includes('bl-tmp')) {
-                clearDir(args.outDir, false);
-            }
-            return { status: 'failed', page_response };
-        }
         output.uri_redirects = page_response
             .request()
             .redirectChain()
@@ -272,6 +260,8 @@ export const collect = async (inUrl: string, args: CollectorOptions) => {
         console.log('About to browse more links');
 
         // try {
+        console.log(output.browsing_history);
+        // console.log(output.browsing_history.slice(1));
         for (const link of output.browsing_history.slice(1)) {
             logger.log('info', `browsing now to ${link}`, { type: 'Browser' });
             if (didBrowserDisconnect) {