fix(repeater): introduce max-body-size to avoid breaking socket conne…

…ction on large responses (#585) Introduce a maxBodySize limit to manage large response bodies, truncating or omitting them as necessary to prevent socket connection disruptions
NeuraLegion · Aug 13, 2024 · 94ca28d · 94ca28d
1 parent 5fcd3fc
commit 94ca28d
Show file tree

Hide file tree

Showing 7 changed files with 170 additions and 56 deletions.
diff --git a/src/Commands/RunRepeater.ts b/src/Commands/RunRepeater.ts
@@ -173,24 +173,28 @@ export class RunRepeater implements CommandModule {
               timeout: args.timeout as number,
               proxyUrl: (args.proxyTarget ?? args.proxy) as string,
               certs: args.cert as Cert[],
+              maxBodySize: Infinity,
               maxContentLength: 100,
               reuseConnection:
                 !!args.ntlm || !!args.experimentalConnectionReuse,
               whitelistMimes: [
-                'text/html',
-                'text/plain',
-                'text/css',
-                'text/javascript',
-                'text/markdown',
-                'text/xml',
-                'application/javascript',
-                'application/x-javascript',
-                'application/json',
-                'application/xml',
-                'application/x-www-form-urlencoded',
-                'application/msgpack',
-                'application/ld+json',
-                'application/graphql'
+                { type: 'text/html', allowTruncation: false },
+                { type: 'text/plain', allowTruncation: true },
+                { type: 'text/css', allowTruncation: false },
+                { type: 'text/javascript', allowTruncation: false },
+                { type: 'text/markdown', allowTruncation: true },
+                { type: 'text/xml', allowTruncation: false },
+                { type: 'application/javascript', allowTruncation: false },
+                { type: 'application/x-javascript', allowTruncation: false },
+                { type: 'application/json', allowTruncation: false },
+                { type: 'application/xml', allowTruncation: false },
+                {
+                  type: 'application/x-www-form-urlencoded',
+                  allowTruncation: false
+                },
+                { type: 'application/msgpack', allowTruncation: false },
+                { type: 'application/ld+json', allowTruncation: false },
+                { type: 'application/graphql', allowTruncation: false }
               ],
               proxyDomains: args.proxyDomains as string[]
             }

diff --git a/src/Repeater/DefaultRepeaterServer.ts b/src/Repeater/DefaultRepeaterServer.ts
@@ -16,7 +16,8 @@ import {
   RepeaterServerRequestEvent,
   RepeaterServerRequestResponse,
   RepeaterServerScriptsUpdatedEvent,
-  RepeaterUpgradeAvailableEvent
+  RepeaterUpgradeAvailableEvent,
+  RepeaterLimitsEvent
 } from './RepeaterServer';
 import { inject, injectable } from 'tsyringe';
 import io, { Socket } from 'socket.io-client';
@@ -52,7 +53,8 @@ const enum SocketEvents {
   UPDATE_AVAILABLE = 'update-available',
   SCRIPT_UPDATED = 'scripts-updated',
   PING = 'ping',
-  REQUEST = 'request'
+  REQUEST = 'request',
+  LIMITS = 'limits'
 }
 
 interface SocketListeningEventMap {
@@ -73,6 +75,7 @@ interface SocketListeningEventMap {
     request: RepeaterServerRequestEvent,
     callback: CallbackFunction<RepeaterServerRequestResponse>
   ) => void;
+  [SocketEvents.LIMITS]: (request: RepeaterLimitsEvent) => void;
 }
 
 interface SocketEmitEventMap {
@@ -226,6 +229,9 @@ export class DefaultRepeaterServer implements RepeaterServer {
     this.socket.on(SocketEvents.SCRIPT_UPDATED, (event) =>
       this.events.emit(RepeaterServerEvents.SCRIPTS_UPDATED, event)
     );
+    this.socket.on(SocketEvents.LIMITS, (event) =>
+      this.events.emit(RepeaterServerEvents.LIMITS, event)
+    );
   }
 
   private listenToReservedEvents() {

diff --git a/src/Repeater/RepeaterServer.ts b/src/Repeater/RepeaterServer.ts
@@ -80,6 +80,10 @@ export interface RepeaterUpgradeAvailableEvent {
   version: string;
 }
 
+export interface RepeaterLimitsEvent {
+  maxBodySize: number;
+}
+
 export interface RepeaterServerScriptsUpdatedEvent {
   script: string | Record<string, string>;
 }
@@ -107,6 +111,7 @@ export const enum RepeaterServerEvents {
   REQUEST = 'request',
   TEST_NETWORK = 'test_network',
   UPDATE_AVAILABLE = 'update_available',
+  LIMITS = 'limits',
   SCRIPTS_UPDATED = 'scripts_updated',
   RECONNECTION_FAILED = 'reconnection_failed',
   RECONNECT_ATTEMPT = 'reconnect_attempt',
@@ -123,6 +128,7 @@ export interface RepeaterServerEventsMap {
   [RepeaterServerEvents.REQUEST]: RepeaterServerRequestEvent;
   [RepeaterServerEvents.TEST_NETWORK]: RepeaterServerNetworkTestEvent;
   [RepeaterServerEvents.UPDATE_AVAILABLE]: RepeaterUpgradeAvailableEvent;
+  [RepeaterServerEvents.LIMITS]: RepeaterLimitsEvent;
   [RepeaterServerEvents.SCRIPTS_UPDATED]: RepeaterServerScriptsUpdatedEvent;
   [RepeaterServerEvents.RECONNECTION_FAILED]: RepeaterServerReconnectionFailedEvent;
   [RepeaterServerEvents.RECONNECT_ATTEMPT]: RepeaterServerReconnectionAttemptedEvent;

diff --git a/src/Repeater/ServerRepeaterLauncher.ts b/src/Repeater/ServerRepeaterLauncher.ts
@@ -1,6 +1,7 @@
 import { RepeaterLauncher } from './RepeaterLauncher';
 import {
   DeploymentRuntime,
+  RepeaterLimitsEvent,
   RepeaterErrorCodes,
   RepeaterServer,
   RepeaterServerErrorEvent,
@@ -12,7 +13,11 @@ import {
 import { RuntimeDetector } from './RuntimeDetector';
 import { ScriptLoader, VirtualScripts } from '../Scripts';
 import { StartupManager } from '../StartupScripts';
-import { Certificates, Request } from '../RequestExecutor';
+import {
+  Certificates,
+  Request,
+  RequestExecutorOptions
+} from '../RequestExecutor';
 import { Helpers, logger } from '../Utils';
 import { CliInfo } from '../Config';
 import { RepeaterCommandHub } from './RepeaterCommandHub';
@@ -36,6 +41,8 @@ export class ServerRepeaterLauncher implements RepeaterLauncher {
     private readonly commandHub: RepeaterCommandHub,
     @inject(Certificates) private readonly certificates: Certificates,
     @inject(ScriptLoader) private readonly scriptLoader: ScriptLoader,
+    @inject(RequestExecutorOptions)
+    private readonly requestExecutorOptions: RequestExecutorOptions,
     @inject(delay(() => CliInfo)) private readonly info: CliInfo
   ) {}
 
@@ -130,6 +137,7 @@ export class ServerRepeaterLauncher implements RepeaterLauncher {
       this.reconnectionFailed
     );
     this.repeaterServer.on(RepeaterServerEvents.REQUEST, this.requestReceived);
+    this.repeaterServer.on(RepeaterServerEvents.LIMITS, this.limitsReceived);
     this.repeaterServer.on(
       RepeaterServerEvents.TEST_NETWORK,
       this.testingNetwork
@@ -230,6 +238,11 @@ export class ServerRepeaterLauncher implements RepeaterLauncher {
     }
   };
 
+  private limitsReceived = (event: RepeaterLimitsEvent) => {
+    logger.debug('Limits received: %i', event.maxBodySize);
+    this.requestExecutorOptions.maxBodySize = event.maxBodySize;
+  };
+
   private requestReceived = async (event: RepeaterServerRequestEvent) => {
     const response = await this.commandHub.sendRequest(
       new Request({ ...event })

diff --git a/src/RequestExecutor/HttpRequestExecutor.spec.ts b/src/RequestExecutor/HttpRequestExecutor.spec.ts
@@ -211,10 +211,10 @@ describe('HttpRequestExecutor', () => {
       expect(response.body).toEqual(bigBody.slice(0, 1024));
     });
 
-    it('should not truncate response body if it is in allowed mime types', async () => {
-      when(spiedExecutorOptions.maxContentLength).thenReturn(1);
+    it('should not truncate response body if its smaller than limit and it is in allowed mime types', async () => {
+      when(spiedExecutorOptions.maxBodySize).thenReturn(1025);
       when(spiedExecutorOptions.whitelistMimes).thenReturn([
-        'application/x-custom'
+        { type: 'application/x-custom', allowTruncation: false }
       ]);
       const { request, requestOptions } = createRequest();
       const bigBody = 'x'.repeat(1025);
@@ -227,9 +227,44 @@ describe('HttpRequestExecutor', () => {
       expect(response.body).toEqual(bigBody);
     });
 
+    it('should truncate response body if its larger than limit and it is in allowed mime types that require truncation', async () => {
+      when(spiedExecutorOptions.maxBodySize).thenReturn(1024);
+      when(spiedExecutorOptions.whitelistMimes).thenReturn([
+        { type: 'text/plain', allowTruncation: true }
+      ]);
+      const { request, requestOptions } = createRequest();
+      const bigBody = 'x'.repeat(1025);
+      const expected = bigBody.slice(0, 1024);
+      nock(requestOptions.url).get('/').reply(200, bigBody, {
+        'content-type': 'text/plain'
+      });
+
+      const response = await executor.execute(request);
+
+      expect(response.body).toEqual(expected);
+    });
+
+    it('should omit response body if its larger than limit and it is in allowed mime types that require omission', async () => {
+      when(spiedExecutorOptions.maxBodySize).thenReturn(1024);
+      when(spiedExecutorOptions.whitelistMimes).thenReturn([
+        { type: 'application/json', allowTruncation: false }
+      ]);
+      const { request, requestOptions } = createRequest();
+      const bigBody = 'x'.repeat(1025);
+      nock(requestOptions.url).get('/').reply(200, bigBody, {
+        'content-type': 'application/json'
+      });
+
+      const response = await executor.execute(request);
+
+      expect(response.body).toEqual('');
+    });
+
     it('should decode response body if content-encoding is brotli', async () => {
-      when(spiedExecutorOptions.maxContentLength).thenReturn(1);
-      when(spiedExecutorOptions.whitelistMimes).thenReturn(['text/plain']);
+      when(spiedExecutorOptions.maxBodySize).thenReturn(1025);
+      when(spiedExecutorOptions.whitelistMimes).thenReturn([
+        { type: 'text/plain', allowTruncation: true }
+      ]);
       const { request, requestOptions } = createRequest();
       const expected = 'x'.repeat(1025);
       const bigBody = await promisify(brotliCompress)(expected);
@@ -244,8 +279,10 @@ describe('HttpRequestExecutor', () => {
     });
 
     it('should prevent decoding response body if decompress option is disabled', async () => {
-      when(spiedExecutorOptions.maxContentLength).thenReturn(1);
-      when(spiedExecutorOptions.whitelistMimes).thenReturn(['text/plain']);
+      when(spiedExecutorOptions.maxBodySize).thenReturn(1024);
+      when(spiedExecutorOptions.whitelistMimes).thenReturn([
+        { type: 'text/plain', allowTruncation: true }
+      ]);
       const { request, requestOptions } = createRequest({
         decompress: false,
         encoding: 'base64'
@@ -267,8 +304,10 @@ describe('HttpRequestExecutor', () => {
     });
 
     it('should decode response body if content-encoding is gzip', async () => {
-      when(spiedExecutorOptions.maxContentLength).thenReturn(1);
-      when(spiedExecutorOptions.whitelistMimes).thenReturn(['text/plain']);
+      when(spiedExecutorOptions.maxBodySize).thenReturn(1025);
+      when(spiedExecutorOptions.whitelistMimes).thenReturn([
+        { type: 'text/plain', allowTruncation: true }
+      ]);
       const { request, requestOptions } = createRequest();
       const expected = 'x'.repeat(1025);
       const bigBody = await promisify(gzip)(expected, {
@@ -286,8 +325,10 @@ describe('HttpRequestExecutor', () => {
     });
 
     it('should decode response body if content-encoding is deflate', async () => {
-      when(spiedExecutorOptions.maxContentLength).thenReturn(1);
-      when(spiedExecutorOptions.whitelistMimes).thenReturn(['text/plain']);
+      when(spiedExecutorOptions.maxBodySize).thenReturn(1025);
+      when(spiedExecutorOptions.whitelistMimes).thenReturn([
+        { type: 'text/plain', allowTruncation: true }
+      ]);
       const { request, requestOptions } = createRequest();
       const expected = 'x'.repeat(1025);
       const bigBody = await promisify(deflate)(expected, {
@@ -305,8 +346,10 @@ describe('HttpRequestExecutor', () => {
     });
 
     it('should decode response body if content-encoding is deflate and content does not have zlib headers', async () => {
-      when(spiedExecutorOptions.maxContentLength).thenReturn(1);
-      when(spiedExecutorOptions.whitelistMimes).thenReturn(['text/plain']);
+      when(spiedExecutorOptions.maxBodySize).thenReturn(1025);
+      when(spiedExecutorOptions.whitelistMimes).thenReturn([
+        { type: 'text/plain', allowTruncation: true }
+      ]);
       const { request, requestOptions } = createRequest();
       const expected = 'x'.repeat(1025);
       const bigBody = await promisify(deflateRaw)(expected, {
@@ -325,7 +368,9 @@ describe('HttpRequestExecutor', () => {
 
     it('should decode and truncate gzipped response body if content-type is not in allowed list', async () => {
       when(spiedExecutorOptions.maxContentLength).thenReturn(1);
-      when(spiedExecutorOptions.whitelistMimes).thenReturn(['text/plain']);
+      when(spiedExecutorOptions.whitelistMimes).thenReturn([
+        { type: 'text/plain', allowTruncation: true }
+      ]);
       const { request, requestOptions } = createRequest();
       const bigBody = 'x'.repeat(1025);
       const expected = bigBody.slice(0, 1024);
@@ -344,9 +389,9 @@ describe('HttpRequestExecutor', () => {
     });
 
     it('should not truncate response body if allowed mime type starts with actual one', async () => {
-      when(spiedExecutorOptions.maxContentLength).thenReturn(1);
+      when(spiedExecutorOptions.maxBodySize).thenReturn(1025);
       when(spiedExecutorOptions.whitelistMimes).thenReturn([
-        'application/x-custom'
+        { type: 'application/x-custom', allowTruncation: false }
       ]);
       const { request, requestOptions } = createRequest();
       const bigBody = 'x'.repeat(1025);