From 61b9f70e12ca6412677c65bf32d90659d74f1795 Mon Sep 17 00:00:00 2001 From: Puneet Khanduri Date: Mon, 11 Sep 2023 23:51:54 +0530 Subject: [PATCH] added support and tests for max header and payload sizes --- src/main/scala/ai/diffy/Settings.scala | 1 + .../interpreter/http/HttpLambdaServer.java | 4 ++ .../scala/ai/diffy/proxy/HttpEndpoint.java | 21 ++++++---- .../proxy/ReactorHttpDifferenceProxy.java | 11 +++-- src/test/resources/application.yml | 1 + src/test/resources/echo.js | 17 ++++++++ src/test/scala/ai/diffy/IntegrationTest.java | 41 ++++++++++++++++--- 7 files changed, 80 insertions(+), 16 deletions(-) create mode 100644 src/test/resources/echo.js diff --git a/src/main/scala/ai/diffy/Settings.scala b/src/main/scala/ai/diffy/Settings.scala index 660bd56..75a2821 100644 --- a/src/main/scala/ai/diffy/Settings.scala +++ b/src/main/scala/ai/diffy/Settings.scala @@ -22,6 +22,7 @@ class Settings( @Value("${rootUrl:}") val rootUrl: String = "", @Value("${allowHttpSideEffects:false}") val allowHttpSideEffects: Boolean = false, @Value("${excludeHttpHeadersComparison:false}") val excludeHttpHeadersComparison: Boolean = false, + @Value("${maxHeaderSize:8192}") val maxHeaderSize: Int = 8192, @Value("${resource.mapping:}") resourceMappings: String = "", @Value("${responseMode:primary}") mode: String = ResponseMode.primary.name(), @Value("${dockerComposeLocal:false}") val dockerComposeLocal: Boolean = false) diff --git a/src/main/scala/ai/diffy/interpreter/http/HttpLambdaServer.java b/src/main/scala/ai/diffy/interpreter/http/HttpLambdaServer.java index 7c8c6e3..281c3a7 100644 --- a/src/main/scala/ai/diffy/interpreter/http/HttpLambdaServer.java +++ b/src/main/scala/ai/diffy/interpreter/http/HttpLambdaServer.java @@ -28,6 +28,10 @@ public HttpLambdaServer(int port, Source httpLambda) { lambda = new Lambda(HttpResponse.class, httpLambda).suppressThrowable(); server = HttpServer.create() .port(port) + .httpRequestDecoder(httpRequestDecoderSpec -> + httpRequestDecoderSpec + .maxChunkSize(32*1024*1024) + .maxHeaderSize(32*1024*1024)) .handle((req, res) -> Mono.fromFuture( HttpEndpoint.RequestBuffer diff --git a/src/main/scala/ai/diffy/proxy/HttpEndpoint.java b/src/main/scala/ai/diffy/proxy/HttpEndpoint.java index 59812fa..52cbd6f 100644 --- a/src/main/scala/ai/diffy/proxy/HttpEndpoint.java +++ b/src/main/scala/ai/diffy/proxy/HttpEndpoint.java @@ -9,7 +9,6 @@ import ai.diffy.transformations.TransformationEdge; import io.netty.handler.codec.http.HttpHeaderNames; import io.netty.handler.codec.http.HttpHeaderValues; -import io.netty.handler.codec.http.HttpHeaders; import io.netty.handler.codec.http.HttpMethod; import reactor.core.publisher.Mono; import reactor.netty.ByteBufMono; @@ -59,22 +58,28 @@ public HttpEndpoint(String name, HttpClient client) { public Endpoint> withSeverRequestBuffer(){ return Endpoint.from(this.getName(), () -> (serverRequest -> requestBuffer.apply(serverRequest).thenApply(this::apply))); } - public static HttpEndpoint from(String name, String host, int port) { + private static HttpEndpoint from(String name, String host, int port, int maxHeader) { final HttpClient client = HttpClient - .create().host(host).port(port); + .create().host(host).port(port) + .httpResponseDecoder(httpResponseDecoderSpec -> + httpResponseDecoderSpec + .maxHeaderSize(maxHeader)); return new HttpEndpoint(name, client); } - public static HttpEndpoint from(String name, String baseUrl) { + private static HttpEndpoint from(String name, String baseUrl, int maxHeader) { final HttpClient client = HttpClient - .create().baseUrl(baseUrl); + .create().baseUrl(baseUrl) + .httpResponseDecoder(httpResponseDecoderSpec -> + httpResponseDecoderSpec + .maxHeaderSize(maxHeader)); return new HttpEndpoint(name, client); } - public static HttpEndpoint from(String name, Downstream downstream) { + public static HttpEndpoint from(String name, Downstream downstream, int maxHeader) { if(downstream instanceof BaseUrl){ - return from(name, ((BaseUrl) downstream).baseUrl()); + return from(name, ((BaseUrl) downstream).baseUrl(), maxHeader); } HostPort hostport = (HostPort)downstream; - return from(name, hostport.host(), hostport.port()); + return from(name, hostport.host(), hostport.port(), maxHeader); } } diff --git a/src/main/scala/ai/diffy/proxy/ReactorHttpDifferenceProxy.java b/src/main/scala/ai/diffy/proxy/ReactorHttpDifferenceProxy.java index b901fe3..7d75120 100644 --- a/src/main/scala/ai/diffy/proxy/ReactorHttpDifferenceProxy.java +++ b/src/main/scala/ai/diffy/proxy/ReactorHttpDifferenceProxy.java @@ -93,14 +93,17 @@ public ReactorHttpDifferenceProxy( */ primary = Async.common(HttpEndpoint.from( "primary", - settings.primary() + settings.primary(), + settings.maxHeaderSize() )); secondary = Async.common(HttpEndpoint.from( "secondary", - settings.secondary() + settings.secondary(), + settings.maxHeaderSize() )); candidate = Async.common(HttpEndpoint.from( - "candidate", settings.candidate() + "candidate", settings.candidate(), + settings.maxHeaderSize() )); analyzer = Async.common(Endpoint.from( "analyzerWithRepo", @@ -140,6 +143,8 @@ public ReactorHttpDifferenceProxy( */ server = HttpServer.create() .port(settings.servicePort()) + .httpRequestDecoder(httpRequestDecoderSpec -> + httpRequestDecoderSpec.maxHeaderSize(settings.maxHeaderSize())) .handle(this::selectHandler) .bindNow(); } diff --git a/src/test/resources/application.yml b/src/test/resources/application.yml index 562e00c..df56c73 100644 --- a/src/test/resources/application.yml +++ b/src/test/resources/application.yml @@ -17,6 +17,7 @@ service: serviceName : "Default Sample Service" allowHttpSideEffects: true +maxHeaderSize : 33554432 spring: mongodb: diff --git a/src/test/resources/echo.js b/src/test/resources/echo.js new file mode 100644 index 0000000..e3ca9ed --- /dev/null +++ b/src/test/resources/echo.js @@ -0,0 +1,17 @@ +(request) => { + const { + uri, + method, + path, + params, + headers, + body + } = request; + + delete headers['Content-Length'] + return { + status: '200 OK', + headers, + body + } +} \ No newline at end of file diff --git a/src/test/scala/ai/diffy/IntegrationTest.java b/src/test/scala/ai/diffy/IntegrationTest.java index 043c837..cc69620 100644 --- a/src/test/scala/ai/diffy/IntegrationTest.java +++ b/src/test/scala/ai/diffy/IntegrationTest.java @@ -2,6 +2,7 @@ import ai.diffy.interpreter.http.HttpLambdaServer; import ai.diffy.proxy.ReactorHttpDifferenceProxy; +import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -12,6 +13,7 @@ import org.springframework.http.HttpEntity; import org.springframework.http.HttpHeaders; import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; import org.springframework.util.FileCopyUtils; import org.springframework.web.client.RestTemplate; import static org.junit.jupiter.api.Assertions.*; @@ -20,6 +22,7 @@ import java.io.IOException; import java.io.InputStreamReader; +@Slf4j @SpringBootTest @TestInstance(TestInstance.Lifecycle.PER_CLASS) public class IntegrationTest { @@ -34,11 +37,13 @@ private Integer extractPort(Downstream downstream) { return hostPort.port(); } HttpLambdaServer primary, secondary, candidate; + String proxyUrl; @BeforeAll public void setup() throws IOException { - primary = new HttpLambdaServer(extractPort(settings.primary()), new File("src/test/resources/lambda.js")); - secondary = new HttpLambdaServer(extractPort(settings.secondary()), new File("src/test/resources/lambda.js")); - candidate = new HttpLambdaServer(extractPort(settings.candidate()), new File("src/test/resources/lambda.js")); + primary = new HttpLambdaServer(extractPort(settings.primary()), new File("src/test/resources/echo.js")); + secondary = new HttpLambdaServer(extractPort(settings.secondary()), new File("src/test/resources/echo.js")); + candidate = new HttpLambdaServer(extractPort(settings.candidate()), new File("src/test/resources/echo.js")); + proxyUrl = "http://localhost:"+settings.servicePort()+"/base"; } @AfterAll @@ -50,13 +55,39 @@ public void shutdown() { @Test public void warmup() throws Exception { - String proxyUrl = "http://localhost:"+settings.servicePort()+"/base"; HttpHeaders headers = new HttpHeaders(); headers.setContentType(MediaType.APPLICATION_JSON); FileSystemResource payload = new FileSystemResource("src/test/resources/payload.json"); String json = FileCopyUtils.copyToString(new InputStreamReader(payload.getInputStream())); - String response = restTemplate.postForObject(proxyUrl, new HttpEntity(json, headers), String.class); + String response = restTemplate.postForObject(proxyUrl, new HttpEntity<>(json, headers), String.class); assertEquals(json, response); } + + @Test + public void largeRequestBody() { + int largeSize = 16*1024*1024; // 16 MB + String json = "{\"a\":\""+new String(new char[largeSize])+"\"}"; + log.info("Testing request body of {} MB", json.getBytes().length/1024/1024); + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + + String response = restTemplate.postForObject(proxyUrl, new HttpEntity<>(json, headers), String.class); + assertEquals(json, response); + } + + @Test + public void largeRequestHeaders() { + int largeSize = 16*1024*1024; // 16 MB + String json = "{\"a\":\"\"}"; + String header = new String(new char[largeSize]).replaceAll(".", "0"); + log.info("Testing request header of {} MB", header.getBytes().length/1024/1024); + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + headers.set("a", header); + + ResponseEntity response = restTemplate.postForEntity(proxyUrl, new HttpEntity<>(json, headers), String.class); + assertEquals(json, response.getBody()); + assertEquals(header, response.getHeaders().getFirst("a")); + } }