added support and tests for max header and payload sizes

src/main/scala/ai/diffy/Settings.scala

@@ -22,6 +22,7 @@ class Settings(
                 @Value("${rootUrl:}") val rootUrl: String = "",
                 @Value("${allowHttpSideEffects:false}") val allowHttpSideEffects: Boolean = false,
                 @Value("${excludeHttpHeadersComparison:false}") val excludeHttpHeadersComparison: Boolean = false,
+                @Value("${maxHeaderSize:8192}") val maxHeaderSize: Int = 8192,
                 @Value("${resource.mapping:}") resourceMappings: String = "",
                 @Value("${responseMode:primary}") mode: String = ResponseMode.primary.name(),
                 @Value("${dockerComposeLocal:false}") val dockerComposeLocal: Boolean = false)

src/main/scala/ai/diffy/interpreter/http/HttpLambdaServer.java

@@ -28,6 +28,10 @@ public HttpLambdaServer(int port, Source httpLambda) {
         lambda = new Lambda<HttpRequest, HttpResponse>(HttpResponse.class, httpLambda).suppressThrowable();
         server = HttpServer.create()
                 .port(port)
+                .httpRequestDecoder(httpRequestDecoderSpec ->
+                        httpRequestDecoderSpec
+                                .maxChunkSize(32*1024*1024)
+                                .maxHeaderSize(32*1024*1024))
                 .handle((req, res) ->
                     Mono.fromFuture(
                         HttpEndpoint.RequestBuffer

src/main/scala/ai/diffy/proxy/HttpEndpoint.java

@@ -9,7 +9,6 @@
 import ai.diffy.transformations.TransformationEdge;
 import io.netty.handler.codec.http.HttpHeaderNames;
 import io.netty.handler.codec.http.HttpHeaderValues;
-import io.netty.handler.codec.http.HttpHeaders;
 import io.netty.handler.codec.http.HttpMethod;
 import reactor.core.publisher.Mono;
 import reactor.netty.ByteBufMono;
@@ -59,22 +58,28 @@ public HttpEndpoint(String name, HttpClient client) {
     public Endpoint<HttpServerRequest, CompletableFuture<HttpResponse>> withSeverRequestBuffer(){
         return Endpoint.from(this.getName(), () -> (serverRequest -> requestBuffer.apply(serverRequest).thenApply(this::apply)));
     }
-    public static HttpEndpoint from(String name, String host, int port) {
+    private static HttpEndpoint from(String name, String host, int port, int maxHeader) {
         final HttpClient client = HttpClient
-                .create().host(host).port(port);
+                .create().host(host).port(port)
+                .httpResponseDecoder(httpResponseDecoderSpec ->
+                        httpResponseDecoderSpec
+                                .maxHeaderSize(maxHeader));
         return new HttpEndpoint(name, client);
     }
-    public static HttpEndpoint from(String name, String baseUrl) {
+    private static HttpEndpoint from(String name, String baseUrl, int maxHeader) {
         final HttpClient client = HttpClient
-                .create().baseUrl(baseUrl);
+                .create().baseUrl(baseUrl)
+                .httpResponseDecoder(httpResponseDecoderSpec ->
+                        httpResponseDecoderSpec
+                                .maxHeaderSize(maxHeader));
         return new HttpEndpoint(name, client);
     }
 
-    public static HttpEndpoint from(String name, Downstream downstream) {
+    public static HttpEndpoint from(String name, Downstream downstream, int maxHeader) {
         if(downstream instanceof BaseUrl){
-            return from(name, ((BaseUrl) downstream).baseUrl());
+            return from(name, ((BaseUrl) downstream).baseUrl(), maxHeader);
         }
         HostPort hostport = (HostPort)downstream;
-        return from(name, hostport.host(), hostport.port());
+        return from(name, hostport.host(), hostport.port(), maxHeader);
     }
 }

src/main/scala/ai/diffy/proxy/ReactorHttpDifferenceProxy.java

@@ -93,14 +93,17 @@ public ReactorHttpDifferenceProxy(
          */
         primary = Async.common(HttpEndpoint.from(
                 "primary",
-                settings.primary()
+                settings.primary(),
+                settings.maxHeaderSize()
         ));
         secondary = Async.common(HttpEndpoint.from(
                 "secondary",
-                settings.secondary()
+                settings.secondary(),
+                settings.maxHeaderSize()
         ));
         candidate = Async.common(HttpEndpoint.from(
-                "candidate", settings.candidate()
+                "candidate", settings.candidate(),
+                settings.maxHeaderSize()
         ));
         analyzer = Async.common(Endpoint.from(
             "analyzerWithRepo",
@@ -140,6 +143,8 @@ public ReactorHttpDifferenceProxy(
          */
         server = HttpServer.create()
                 .port(settings.servicePort())
+                .httpRequestDecoder(httpRequestDecoderSpec ->
+                        httpRequestDecoderSpec.maxHeaderSize(settings.maxHeaderSize()))
                 .handle(this::selectHandler)
                 .bindNow();
     }

src/test/resources/application.yml

@@ -17,6 +17,7 @@ service:
 serviceName : "Default Sample Service"
 
 allowHttpSideEffects: true
+maxHeaderSize : 33554432
 
 spring:
   mongodb:

src/test/resources/echo.js

@@ -0,0 +1,17 @@
+(request) => {
+    const {
+        uri,
+        method,
+        path,
+        params,
+        headers,
+        body
+    } = request;
+
+    delete headers['Content-Length']
+    return {
+        status: '200 OK',
+        headers,
+        body
+    }
+}

src/test/scala/ai/diffy/IntegrationTest.java

@@ -2,6 +2,7 @@
 
 import ai.diffy.interpreter.http.HttpLambdaServer;
 import ai.diffy.proxy.ReactorHttpDifferenceProxy;
+import lombok.extern.slf4j.Slf4j;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
@@ -12,6 +13,7 @@
 import org.springframework.http.HttpEntity;
 import org.springframework.http.HttpHeaders;
 import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
 import org.springframework.util.FileCopyUtils;
 import org.springframework.web.client.RestTemplate;
 import static org.junit.jupiter.api.Assertions.*;
@@ -20,6 +22,7 @@
 import java.io.IOException;
 import java.io.InputStreamReader;
 
+@Slf4j
 @SpringBootTest
 @TestInstance(TestInstance.Lifecycle.PER_CLASS)
 public class IntegrationTest {
@@ -34,11 +37,13 @@ private Integer extractPort(Downstream downstream) {
         return hostPort.port();
     }
     HttpLambdaServer primary, secondary, candidate;
+    String proxyUrl;
     @BeforeAll
     public void setup() throws IOException {
-        primary = new HttpLambdaServer(extractPort(settings.primary()), new File("src/test/resources/lambda.js"));
-        secondary = new HttpLambdaServer(extractPort(settings.secondary()), new File("src/test/resources/lambda.js"));
-        candidate = new HttpLambdaServer(extractPort(settings.candidate()), new File("src/test/resources/lambda.js"));
+        primary = new HttpLambdaServer(extractPort(settings.primary()), new File("src/test/resources/echo.js"));
+        secondary = new HttpLambdaServer(extractPort(settings.secondary()), new File("src/test/resources/echo.js"));
+        candidate = new HttpLambdaServer(extractPort(settings.candidate()), new File("src/test/resources/echo.js"));
+        proxyUrl = "http://localhost:"+settings.servicePort()+"/base";
     }
 
     @AfterAll
@@ -50,13 +55,39 @@ public void shutdown() {
 
     @Test
     public void warmup() throws Exception {
-        String proxyUrl = "http://localhost:"+settings.servicePort()+"/base";
         HttpHeaders headers = new HttpHeaders();
         headers.setContentType(MediaType.APPLICATION_JSON);
 
         FileSystemResource payload = new FileSystemResource("src/test/resources/payload.json");
         String json = FileCopyUtils.copyToString(new InputStreamReader(payload.getInputStream()));
-        String response = restTemplate.postForObject(proxyUrl, new HttpEntity<String>(json, headers), String.class);
+        String response = restTemplate.postForObject(proxyUrl, new HttpEntity<>(json, headers), String.class);
         assertEquals(json, response);
     }
+
+    @Test
+    public void largeRequestBody() {
+        int largeSize = 16*1024*1024; // 16 MB
+        String json = "{\"a\":\""+new String(new char[largeSize])+"\"}";
+        log.info("Testing request body of {} MB", json.getBytes().length/1024/1024);
+        HttpHeaders headers = new HttpHeaders();
+        headers.setContentType(MediaType.APPLICATION_JSON);
+
+        String response = restTemplate.postForObject(proxyUrl, new HttpEntity<>(json, headers), String.class);
+        assertEquals(json, response);
+    }
+
+    @Test
+    public void largeRequestHeaders() {
+        int largeSize = 16*1024*1024; // 16 MB
+        String json = "{\"a\":\"\"}";
+        String header = new String(new char[largeSize]).replaceAll(".", "0");
+        log.info("Testing request header of {} MB", header.getBytes().length/1024/1024);
+        HttpHeaders headers = new HttpHeaders();
+        headers.setContentType(MediaType.APPLICATION_JSON);
+        headers.set("a", header);
+
+        ResponseEntity<String> response = restTemplate.postForEntity(proxyUrl, new HttpEntity<>(json, headers), String.class);
+        assertEquals(json, response.getBody());
+        assertEquals(header, response.getHeaders().getFirst("a"));
+    }
 }