From b3dd6478f65eaf377fcbeb9d8c0346075027198d Mon Sep 17 00:00:00 2001 From: kkoooqq <> Date: Tue, 9 Nov 2021 14:33:22 -0500 Subject: [PATCH] remove dependence on express --- README.md | 2 +- demo/demo6-connect-to-instance.js | 74 ++++++++++++++++++-- package.json | 5 +- src/core/BrowserLauncher.ts | 111 +++++++++++++++++------------- src/core/PptrPatcher.ts | 25 +++++++ 5 files changed, 159 insertions(+), 58 deletions(-) diff --git a/README.md b/README.md index d36e1f1..375415d 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ FakeBrowser automatic login demo: To use FakeBrowser in your project, run: ```bash -yarn add puppeteer axios express fakebrowser +yarn add puppeteer axios fakebrowser ``` --- diff --git a/demo/demo6-connect-to-instance.js b/demo/demo6-connect-to-instance.js index 33f1899..b68d80b 100644 --- a/demo/demo6-connect-to-instance.js +++ b/demo/demo6-connect-to-instance.js @@ -1,24 +1,86 @@ const {FakeBrowser} = require('../dist/cjs/FakeBrowser'); +const {spawn, execSync} = require('child_process'); +const path = require('path'); process.on('unhandledRejection', (error, p) => { console.log('=== UNHANDLED REJECTION ==='); console.dir(error.stack); }); -!(async () => { - const dd = require('../device-hub/Windows.json'); +const userDataDir = path.resolve(__dirname, './fakeBrowserUserData'); + +execSync(`rm -rf ${userDataDir}`); +execSync(`mkdir ${userDataDir}`); + +const dd = require('../device-hub/macOS.json'); +const child = spawn('/Applications/Google Chrome 93.0.4577.82.app/Contents/MacOS/Google Chrome', + [ + '--no-default-browser-check', + '--no-first-run', + `--window-position=0,0`, + `--window-size=${dd.window.outerWidth},${dd.window.outerHeight}`, + `--user-data-dir=${userDataDir}`, + '--remote-debugging-port=9222', + ], +); + +child.stdout.on('data', (data) => { +}); + +child.stderr.on('data', (data) => { + const dataStr = data.toString(); + if (dataStr.includes('DevTools listening on ')) { + const wsEndPoint = dataStr.substr(dataStr.indexOf('ws://')); + console.log(wsEndPoint); + + launchFBPure(wsEndPoint).then(r => r); + } +}); + +const launchFBPure = async (wsEndPoint) => { + const {addExtra, PuppeteerExtra} = require('puppeteer-extra'); + const pptr = addExtra(require('puppeteer')); + const browser = await pptr.connect({browserWSEndpoint: wsEndPoint}); + const page = await browser.newPage(); + await page.goto('https://nike.com/ca/'); +}; + +const launchFB = async (wsEndPoint) => { const builder = new FakeBrowser.Builder() .deviceDescriptor(dd) + .displayUserActionLayer(false) .doNotHook(false) .vanillaConnectOptions({ - browserWSEndpoint: 'ws://127.0.0.1:9222/devtools/browser/a060d8a6-82ad-4035-8e24-2fbf35ee4de9', + browserWSEndpoint: wsEndPoint, }) - .userDataDir('./fakeBrowserUserData'); + .userDataDir(userDataDir); const fakeBrowser = await builder.connect(); - const page = await fakeBrowser.vanillaBrowser.newPage(); await page.goto('https://nike.com/ca'); + console.log('wait #hf_title_signin_membership appeared'); + await page.waitForSelector('#hf_title_signin_membership'); + + await page.waitForTimeout(5 * 1000); + + console.log('click login button'); + const signInButton = await page.$('#hf_title_signin_membership'); + await fakeBrowser.userAction.simClickElement(signInButton); + + const emailTextField = await page.$('input[name="emailAddress"]'); + const passwordField = await page.$('input[name="password"]'); + const loginButton = await page.$('.loginSubmit'); + + await fakeBrowser.userAction.simClickElement(emailTextField); + await fakeBrowser.userAction.simKeyboardType('sdfioioiuwerlklj88@gmail.com'); + + await fakeBrowser.userAction.simClickElement(passwordField); + await fakeBrowser.userAction.simKeyboardType('passowdd+34.3710-'); + + await fakeBrowser.userAction.simClickElement(loginButton); + + // await fakeBrowser.shutdown(); -})(); +}; + diff --git a/package.json b/package.json index 7aff7ea..2e4bf14 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "fakebrowser", - "version": "0.0.51", + "version": "0.0.52", "description": "🤖 Fake fingerprints to bypass anti-bot systems. Simulate mouse and keyboard operations to make behavior like a real person.", "repository": { "type": "git", @@ -56,18 +56,15 @@ }, "peerDependencies": { "axios": "*", - "express": "*", "puppeteer": "*" }, "devDependencies": { "@types/axios": "*", - "@types/express": "*", "@types/fs-extra": "^9.0.13", "@types/puppeteer": "*", "axios": "^0.24.0", "cpy-cli": "^3.1.1", "cross-env": "^7.0.3", - "express": "^4.17.1", "jest": "^27.3.1", "puppeteer": "^11.0.0", "rimraf": "^3.0.2", diff --git a/src/core/BrowserLauncher.ts b/src/core/BrowserLauncher.ts index dcf46bc..2aaf4f7 100644 --- a/src/core/BrowserLauncher.ts +++ b/src/core/BrowserLauncher.ts @@ -2,9 +2,10 @@ import * as fs from "fs-extra"; import * as path from "path"; import * as URLToolkit from 'url-toolkit' import * as http from "http"; +import {IncomingMessage, ServerResponse} from "http"; +import * as url from "url"; import axios from "axios"; -import express, {Application} from "express"; import {Agent} from "https"; import {strict as assert} from 'assert'; @@ -20,8 +21,7 @@ export class BrowserLauncher { static _fakeBrowserInstances: FakeBrowser[] = [] static _checkerIntervalId: NodeJS.Timer | null = null - static _app: Application | null = null - static _appServer: http.Server | null = null + static _httpServer: http.Server | null = null private static checkLaunchOptionsLegal(options?: VanillaLaunchOptions) { if (!options || !options.args || !options.args.length) { @@ -161,69 +161,80 @@ export class BrowserLauncher { } private static async bootInternalHTTPServer() { - if (!this._app) { - this._app = express() + if (!this._httpServer) { + this._httpServer = http.createServer() - this._app.get('/hb', async (req, res) => { - res.send(kInternalHttpServerHeartbeatMagic) - }) + this._httpServer.on('request', async (req: IncomingMessage, res: ServerResponse) => { + assert(req.url) + const {query, pathname} = url.parse(req.url, true) + + if (pathname === '/hb') { + res.write(kInternalHttpServerHeartbeatMagic) + res.end() + } - this._app.get('/patchWorker', async (req, res) => { - const relUrl = req.query['relUrl'] as string - const workerUrl = req.query['workerUrl'] as string - const uuid = req.query['uuid'] as string + if (pathname === '/patchWorker') { + const relUrl = query['relUrl'] as string + const workerUrl = query['workerUrl'] as string + const uuid = query['uuid'] as string - const fullUrl = URLToolkit.buildAbsoluteURL(relUrl, workerUrl) + const fullUrl = URLToolkit.buildAbsoluteURL(relUrl, workerUrl) - console.log('request worker content from: ', fullUrl) + console.log('request worker content from: ', fullUrl) - // Object.fromEntries ES2019 - const reqHeaders = Object.fromEntries( - Object.entries( - req.headers - ).map( - e => ([e[0], e[1]![0]]) + // Object.fromEntries ES2019 + const reqHeaders = Object.fromEntries( + Object.entries( + req.headers + ).map( + e => ([e[0], e[1]![0]]) + ) ) - ) - delete reqHeaders.host + delete reqHeaders.host - const jsResp = await axios.get( - fullUrl, { - headers: reqHeaders, - httpsAgent: new Agent({ - rejectUnauthorized: false - }) - } - ) - - let jsContent = jsResp.data - const browser = BrowserLauncher.getBrowserWithUUID(uuid) + // TODO: get through proxy + const jsResp = await axios.get( + fullUrl, { + headers: reqHeaders, + httpsAgent: new Agent({ + rejectUnauthorized: false + }) + } + ) - if (browser) { - jsContent = await PptrPatcher.patchWorkerJsContent(browser, jsContent) - } + let jsContent = jsResp.data + const browser = BrowserLauncher.getBrowserWithUUID(uuid) - for (const {name, value} of Object.entries(jsResp.headers).map(e => ({ - name: e[0], - value: e[1] as string - }))) { - if (name.toLowerCase() != 'content-length') { - res.header(name, value) + if (browser) { + jsContent = await PptrPatcher.patchWorkerJsContent(browser, jsContent) } - } - res.send(jsContent) + res.writeHead( + jsResp.status, + jsResp.statusText, + jsResp.headers as NodeJS.Dict + ) + + res.write(jsContent) + res.end() + } }) // If the port listens to errors, determine if the heartbeat interface is successful try { - this._appServer = this._app.listen(FakeBrowser.globalConfig.internalHttpServerPort) + this._httpServer.listen(FakeBrowser.globalConfig.internalHttpServerPort) } catch (ex: any) { const hbUrl = `http://127.0.0.1:${FakeBrowser.globalConfig.internalHttpServerPort}/hb` try { const hbData = (await axios.get(hbUrl)).data if (hbData === kInternalHttpServerHeartbeatMagic) { + try { + this._httpServer.close() + } finally { + this._httpServer = null + } + return } } catch (ignore: any) { @@ -268,8 +279,14 @@ export class BrowserLauncher { // If all browsers have exited, close internal http service if (this._fakeBrowserInstances.length === 0) { // console.log('close appserver') - this._appServer!.close() - this._app = null + + if (this._httpServer) { + try { + this._httpServer.close() + } finally { + this._httpServer = null + } + } } } } diff --git a/src/core/PptrPatcher.ts b/src/core/PptrPatcher.ts index 4d9ec39..a8515ad 100644 --- a/src/core/PptrPatcher.ts +++ b/src/core/PptrPatcher.ts @@ -48,6 +48,7 @@ export class PptrPatcher { await this.patchLast(uuid, pptr, params) } + // @ts-ignore private static patchTaskEnv( uuid: string, pptr: PuppeteerExtra, @@ -64,6 +65,7 @@ export class PptrPatcher { pptr.use(plugin) } + // @ts-ignore private static patchUserActionLayer( uuid: string, pptr: PuppeteerExtra, @@ -76,6 +78,7 @@ export class PptrPatcher { } } + // @ts-ignore private static pathChrome( uuid: string, pptr: PuppeteerExtra, @@ -95,6 +98,7 @@ export class PptrPatcher { } } + // @ts-ignore private static patchWindowHistoryLength( uuid: string, pptr: PuppeteerExtra, @@ -108,6 +112,7 @@ export class PptrPatcher { } + // @ts-ignore private static patchWindowMatchMedia( uuid: string, pptr: PuppeteerExtra, @@ -118,6 +123,7 @@ export class PptrPatcher { pptr.use(plugin) } + // @ts-ignore private static pathWebdriver( uuid: string, pptr: PuppeteerExtra, @@ -128,6 +134,7 @@ export class PptrPatcher { pptr.use(plugin) } + // @ts-ignore private static pathSourceUrl( uuid: string, pptr: PuppeteerExtra, @@ -138,6 +145,7 @@ export class PptrPatcher { pptr.use(plugin) } + // @ts-ignore private static patchPluginsMineTypes( uuid: string, pptr: PuppeteerExtra, @@ -154,6 +162,7 @@ export class PptrPatcher { pptr.use(plugin) } + // @ts-ignore private static patchWebGL( uuid: string, pptr: PuppeteerExtra, @@ -170,6 +179,7 @@ export class PptrPatcher { pptr.use(plugin) } + // @ts-ignore private static patchMimeTypes( uuid: string, pptr: PuppeteerExtra, @@ -185,6 +195,7 @@ export class PptrPatcher { pptr.use(plugin) } + // @ts-ignore private static patchMediaDevices( uuid: string, pptr: PuppeteerExtra, @@ -200,6 +211,7 @@ export class PptrPatcher { pptr.use(plugin) } + // @ts-ignore private static patchBluetooth( uuid: string, pptr: PuppeteerExtra, @@ -211,6 +223,7 @@ export class PptrPatcher { pptr.use(plugin) } + // @ts-ignore private static patchPermissions( uuid: string, pptr: PuppeteerExtra, @@ -226,6 +239,7 @@ export class PptrPatcher { pptr.use(plugin) } + // @ts-ignore private static patchBatteryManager( uuid: string, pptr: PuppeteerExtra, @@ -241,6 +255,7 @@ export class PptrPatcher { pptr.use(plugin) } + // @ts-ignore private static async patchWebRtc( uuid: string, pptr: PuppeteerExtra, @@ -269,6 +284,7 @@ export class PptrPatcher { } } + // @ts-ignore private static patchCanvas2DFingerprint( uuid: string, pptr: PuppeteerExtra, @@ -285,6 +301,7 @@ export class PptrPatcher { pptr.use(plugin) } + // @ts-ignore private static patchFonts( uuid: string, pptr: PuppeteerExtra, @@ -300,6 +317,7 @@ export class PptrPatcher { pptr.use(plugin) } + // @ts-ignore private static patchEmojis( uuid: string, pptr: PuppeteerExtra, @@ -315,6 +333,7 @@ export class PptrPatcher { pptr.use(plugin) } + // @ts-ignore private static patchSpeechSynthesis( uuid: string, pptr: PuppeteerExtra, @@ -330,6 +349,7 @@ export class PptrPatcher { pptr.use(plugin) } + // @ts-ignore private static patchWorkers( uuid: string, pptr: PuppeteerExtra, @@ -346,6 +366,7 @@ export class PptrPatcher { pptr.use(plugin) } + // @ts-ignore private static patchKeyboard( uuid: string, pptr: PuppeteerExtra, @@ -361,6 +382,7 @@ export class PptrPatcher { pptr.use(plugin) } + // @ts-ignore private static patchUserAgent( uuid: string, pptr: PuppeteerExtra, @@ -378,6 +400,7 @@ export class PptrPatcher { pptr.use(plugin) } + // @ts-ignore private static patchIFrame( uuid: string, pptr: PuppeteerExtra, @@ -392,6 +415,7 @@ export class PptrPatcher { pptr.use(plugin) } + // @ts-ignore private static patchPropertiesGetters( uuid: string, pptr: PuppeteerExtra, @@ -410,6 +434,7 @@ export class PptrPatcher { pptr.use(plugin) } + // @ts-ignore private static patchLast( uuid: string, pptr: PuppeteerExtra,