From cbf3730565af45fcd7963dd69a2cc1b1cb64790b Mon Sep 17 00:00:00 2001 From: RutamBhagat Date: Sat, 28 Dec 2024 18:16:05 +0530 Subject: [PATCH 1/5] fix(js-sdk): Add CJS-compatible WebSocket loading --- apps/js-sdk/firecrawl/src/index.ts | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.ts index 29fabf5d6..bad7ed7e2 100644 --- a/apps/js-sdk/firecrawl/src/index.ts +++ b/apps/js-sdk/firecrawl/src/index.ts @@ -1,9 +1,9 @@ import axios, { type AxiosResponse, type AxiosRequestHeaders, AxiosError } from "axios"; import type * as zt from "zod"; import { zodToJsonSchema } from "zod-to-json-schema"; -import { WebSocket } from "isows"; import { TypedEventTarget } from "typescript-event-target"; + /** * Configuration interface for FirecrawlApp. * @param apiKey - Optional API key for authentication. @@ -947,10 +947,23 @@ export class CrawlWatcher extends TypedEventTarget { public status: CrawlStatusResponse["status"]; public id: string; + private static loadWebSocket() { + try { + return require('isows').WebSocket; + } catch { + try { + return import('isows').then(m => m.WebSocket); + } catch { + throw new FirecrawlError("WebSocket module failed to load", 500); + } + } + } + constructor(id: string, app: FirecrawlApp) { super(); + const WS = CrawlWatcher.loadWebSocket(); this.id = id; - this.ws = new WebSocket(`${app.apiUrl}/v1/crawl/${id}`, app.apiKey); + this.ws = new WS(`${app.apiUrl}/v1/crawl/${id}`, app.apiKey); this.status = "scraping"; this.data = []; From 15398cc80182bc946d8814e034e558d26d71e06e Mon Sep 17 00:00:00 2001 From: RutamBhagat Date: Mon, 30 Dec 2024 20:11:20 +0530 Subject: [PATCH 2/5] fix(js-sdk): WebSocket initialization handles promises correctly --- apps/js-sdk/firecrawl/src/index.ts | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.ts index bad7ed7e2..5770d8414 100644 --- a/apps/js-sdk/firecrawl/src/index.ts +++ b/apps/js-sdk/firecrawl/src/index.ts @@ -942,7 +942,7 @@ interface CrawlWatcherEvents { } export class CrawlWatcher extends TypedEventTarget { - private ws: WebSocket; + private ws!: WebSocket; public data: FirecrawlDocument[]; public status: CrawlStatusResponse["status"]; public id: string; @@ -963,10 +963,22 @@ export class CrawlWatcher extends TypedEventTarget { super(); const WS = CrawlWatcher.loadWebSocket(); this.id = id; - this.ws = new WS(`${app.apiUrl}/v1/crawl/${id}`, app.apiKey); + + if (WS instanceof Promise) { + WS.then(WebSocketClass => { + this.ws = new WebSocketClass(`${app.apiUrl}/v1/crawl/${id}`, app.apiKey); + this.initializeWebSocket(); + }); + } else { + this.ws = new WS(`${app.apiUrl}/v1/crawl/${id}`, app.apiKey); + this.initializeWebSocket(); + } + this.status = "scraping"; this.data = []; + } + private initializeWebSocket() { type ErrorMessage = { type: "error", error: string, From 4238270561817c54d823969d4246e1f60ccd8e35 Mon Sep 17 00:00:00 2001 From: RutamBhagat Date: Tue, 31 Dec 2024 13:11:02 +0530 Subject: [PATCH 3/5] fix(js-sdk): fix loadWebSocket race condition --- apps/js-sdk/firecrawl/src/index.ts | 38 +++++++++++++----------------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.ts index 5770d8414..a1689b462 100644 --- a/apps/js-sdk/firecrawl/src/index.ts +++ b/apps/js-sdk/firecrawl/src/index.ts @@ -3,7 +3,6 @@ import type * as zt from "zod"; import { zodToJsonSchema } from "zod-to-json-schema"; import { TypedEventTarget } from "typescript-event-target"; - /** * Configuration interface for FirecrawlApp. * @param apiKey - Optional API key for authentication. @@ -943,39 +942,35 @@ interface CrawlWatcherEvents { export class CrawlWatcher extends TypedEventTarget { private ws!: WebSocket; + private initialized: Promise; public data: FirecrawlDocument[]; public status: CrawlStatusResponse["status"]; public id: string; - private static loadWebSocket() { + private static async loadWebSocket() { try { - return require('isows').WebSocket; - } catch { - try { - return import('isows').then(m => m.WebSocket); - } catch { - throw new FirecrawlError("WebSocket module failed to load", 500); + if (typeof require !== 'undefined') { + return require('isows').WebSocket; + } else { + const module = await import('isows'); + return module.WebSocket; } + } catch (e) { + throw new FirecrawlError('Failed to load WebSocket implementation', 500); } } constructor(id: string, app: FirecrawlApp) { super(); - const WS = CrawlWatcher.loadWebSocket(); this.id = id; - - if (WS instanceof Promise) { - WS.then(WebSocketClass => { - this.ws = new WebSocketClass(`${app.apiUrl}/v1/crawl/${id}`, app.apiKey); - this.initializeWebSocket(); - }); - } else { - this.ws = new WS(`${app.apiUrl}/v1/crawl/${id}`, app.apiKey); - this.initializeWebSocket(); - } - this.status = "scraping"; this.data = []; + + this.initialized = CrawlWatcher.loadWebSocket() + .then((WebSocket) => { + this.ws = new WebSocket(`${app.apiUrl}/v1/crawl/${id}`, app.apiKey); + this.initializeWebSocket(); + }); } private initializeWebSocket() { @@ -1074,7 +1069,8 @@ export class CrawlWatcher extends TypedEventTarget { }).bind(this); } - close() { + async close() { + await this.initialized; this.ws.close(); } } From 950ceb99221eb464fc5853e95c770c334945ca41 Mon Sep 17 00:00:00 2001 From: RutamBhagat Date: Tue, 31 Dec 2024 15:36:06 +0530 Subject: [PATCH 4/5] fix(js-sdk): ensure WebSocket closes after initialization completes --- apps/js-sdk/firecrawl/src/index.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.ts index a1689b462..cd405c5ec 100644 --- a/apps/js-sdk/firecrawl/src/index.ts +++ b/apps/js-sdk/firecrawl/src/index.ts @@ -1069,8 +1069,9 @@ export class CrawlWatcher extends TypedEventTarget { }).bind(this); } - async close() { - await this.initialized; - this.ws.close(); + close() { + this.initialized.then(() => { + this.ws.close(); + }); } } From 67db6a15181eb10e5bd2e23cdf215d40390cad4f Mon Sep 17 00:00:00 2001 From: RutamBhagat Date: Mon, 6 Jan 2025 14:44:26 +0530 Subject: [PATCH 5/5] fix(build): resolve CJS/ESM WebSocket import with build-time code injection --- apps/js-sdk/firecrawl/src/global.d.ts | 5 ++++ apps/js-sdk/firecrawl/src/index.ts | 42 ++++++++++----------------- apps/js-sdk/firecrawl/tsup.config.ts | 9 ++++++ 3 files changed, 30 insertions(+), 26 deletions(-) create mode 100644 apps/js-sdk/firecrawl/src/global.d.ts diff --git a/apps/js-sdk/firecrawl/src/global.d.ts b/apps/js-sdk/firecrawl/src/global.d.ts new file mode 100644 index 000000000..bf9177ded --- /dev/null +++ b/apps/js-sdk/firecrawl/src/global.d.ts @@ -0,0 +1,5 @@ +export {}; + +declare global { + const __WEBSOCKET_LOADER__: string; +} \ No newline at end of file diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.ts index cd405c5ec..c1cc53955 100644 --- a/apps/js-sdk/firecrawl/src/index.ts +++ b/apps/js-sdk/firecrawl/src/index.ts @@ -1,8 +1,16 @@ import axios, { type AxiosResponse, type AxiosRequestHeaders, AxiosError } from "axios"; import type * as zt from "zod"; import { zodToJsonSchema } from "zod-to-json-schema"; +import { WebSocket as IsowsWebSocket } from "isows"; import { TypedEventTarget } from "typescript-event-target"; + +let WebSocket: typeof IsowsWebSocket; + +if (typeof __WEBSOCKET_LOADER__ === 'string') { + WebSocket = new Function('return ' + __WEBSOCKET_LOADER__)(); +} + /** * Configuration interface for FirecrawlApp. * @param apiKey - Optional API key for authentication. @@ -941,39 +949,23 @@ interface CrawlWatcherEvents { } export class CrawlWatcher extends TypedEventTarget { - private ws!: WebSocket; - private initialized: Promise; + private ws: WebSocket; public data: FirecrawlDocument[]; public status: CrawlStatusResponse["status"]; public id: string; - private static async loadWebSocket() { - try { - if (typeof require !== 'undefined') { - return require('isows').WebSocket; - } else { - const module = await import('isows'); - return module.WebSocket; - } - } catch (e) { - throw new FirecrawlError('Failed to load WebSocket implementation', 500); - } - } - constructor(id: string, app: FirecrawlApp) { super(); this.id = id; + console.log('typeof WebSocket', typeof WebSocket) + // @ts-ignore + if (typeof WebSocket === "function" && WebSocket.prototype.then) { + throw new Error("WebSocket behaves like a Promise, which is unexpected."); + } + this.ws = new WebSocket(`${app.apiUrl}/v1/crawl/${id}`, app.apiKey); this.status = "scraping"; this.data = []; - this.initialized = CrawlWatcher.loadWebSocket() - .then((WebSocket) => { - this.ws = new WebSocket(`${app.apiUrl}/v1/crawl/${id}`, app.apiKey); - this.initializeWebSocket(); - }); - } - - private initializeWebSocket() { type ErrorMessage = { type: "error", error: string, @@ -1070,8 +1062,6 @@ export class CrawlWatcher extends TypedEventTarget { } close() { - this.initialized.then(() => { - this.ws.close(); - }); + this.ws.close(); } } diff --git a/apps/js-sdk/firecrawl/tsup.config.ts b/apps/js-sdk/firecrawl/tsup.config.ts index b3b7e42d5..4d4c80aca 100644 --- a/apps/js-sdk/firecrawl/tsup.config.ts +++ b/apps/js-sdk/firecrawl/tsup.config.ts @@ -6,4 +6,13 @@ export default defineConfig({ dts: true, outDir: "dist", clean: true, + esbuildOptions(options, context) { + options.define = { + __WEBSOCKET_LOADER__: JSON.stringify( + context.format === 'cjs' + ? `const { WebSocket } = require('isows'); WebSocket` + : `import('isows').then(m => m.WebSocket)` + ) + }; + }, }); \ No newline at end of file