Fix 'Transfer-Encoding: chunked' issue when sending request via proxy

gateway/src/apicast/http_proxy.lua

@@ -1,10 +1,12 @@
 local format = string.format
+local string_lower = string.lower
 
 local resty_url = require "resty.url"
 local resty_resolver = require 'resty.resolver'
 local round_robin = require 'resty.balancer.round_robin'
 local http_proxy = require 'resty.http.proxy'
 local file_reader = require("resty.file").file_reader
+local file_size = require("resty.file").file_size
 
 local _M = { }
 
@@ -81,6 +83,27 @@ local function absolute_url(uri)
     )
 end
 
+-- Sets a header with the given value. Any existing header with the same name will be overidden
+-- This also taking care of lowercase header when running on Openshift
+local function set_header(headers, header, value)
+    local lowercase_header = string_lower(header)
+    if headers[header] then
+        headers[header] = value
+    else
+        headers[lowercase_header] = value
+    end
+end
+
+local function modify_chunked_request_headers(req, file)
+    local contentLength, err = file_size(file)
+     if err then
+        ngx.log(ngx.ERR, "HTTPS proxy: Failed to set content length, err: ", err)
+        ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR)
+    end
+    set_header(req.headers, "Content-Length", contentLength)
+end
+
+
 local function forward_https_request(proxy_uri, uri, skip_https_connect)
     -- This is needed to call ngx.req.get_body_data() below.
     ngx.req.read_body()
@@ -114,10 +137,22 @@ local function forward_https_request(proxy_uri, uri, skip_https_connect)
             ngx.log(ngx.ERR, "HTTPS proxy: Failed to read temp body file, err: ", err)
             ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR)
           end
+          if ngx.var.http_transfer_encoding == "chunked" then
+            -- If the body is smaller than "client_boby_buffer_size" the Content-Length header is
+            -- set based on the size of the buffer. However, when the body is rendered to a file,
+            -- we will need to calculate and manually set the Content-Length header based on the
+            -- file size
+            modify_chunked_request_headers(request, temp_file_path)
+          end
           request.body = body
         end
     end
 
+    -- The whole request is buffered in the memory so remove the Transfer-Encoding: chunked
+    if ngx.var.http_transfer_encoding == "chunked" then
+        set_header(request.headers, "Transfer-Encoding", nil)
+    end
+
     local httpc, err = http_proxy.new(request, skip_https_connect)
 
     if not httpc then

gateway/src/resty/file.lua

@@ -28,4 +28,16 @@ function _M.file_reader(filename)
     end)
 end
 
+function _M.file_size(filename)
+  local handle, err = open(filename)
+  if err then
+    return nil, err
+  end
+  local current = handle:seek()
+  local size = handle:seek("end")
+  handle:seek("set", current)
+  handle:close()
+  return size
+end
+
 return _M

t/apicast-policy-camel.t

@@ -315,3 +315,223 @@ ETag: foobar
 <<EOF
 using proxy: http://127.0.0.1:$Test::Nginx::Util::PROXY_SSL_PORT,
 EOF
+
+
+=== TEST 5: API backend connection uses http proxy with chunked request
+--- configuration
+{
+  "services": [
+    {
+      "id": 42,
+      "backend_version":  1,
+      "backend_authentication_type": "service_token",
+      "backend_authentication_value": "token-value",
+      "proxy": {
+        "api_backend": "http://test-upstream.lvh.me:$TEST_NGINX_SERVER_PORT/",
+        "proxy_rules": [
+          { "pattern": "/", "http_method": "POST", "metric_system_name": "hits", "delta": 2 }
+        ],
+        "policy_chain": [
+          {
+            "name": "apicast.policy.apicast"
+          },
+          {
+            "name": "apicast.policy.camel",
+            "configuration": {
+                "http_proxy": "$TEST_NGINX_HTTP_PROXY"
+            }
+          }
+        ]
+      }
+    }
+  ]
+}
+--- backend
+  location /transactions/authrep.xml {
+    content_by_lua_block {
+      ngx.exit(ngx.OK)
+    }
+  }
+--- upstream
+  server_name test-upstream.lvh.me;
+  location / {
+    access_by_lua_block {
+      assert = require('luassert')
+      local content_length = ngx.req.get_headers()["Content-Length"]
+      local encoding = ngx.req.get_headers()["Transfer-Encoding"]
+      assert.equal('12', content_length)
+      assert.falsy(encoding)
+      ngx.say("yay, api backend")
+    }
+  }
+--- more_headers
+Transfer-Encoding: chunked
+--- request eval
+"POST /?user_key=value
+7\r
+hello, \r
+5\r
+world\r
+0\r
+\r
+"
+--- error_code: 200
+--- error_log env
+using proxy: $TEST_NGINX_HTTP_PROXY
+
+
+=== TEST 6: API backend using all_proxy with chunked request
+--- configuration
+{
+  "services": [
+    {
+      "id": 42,
+      "backend_version":  1,
+      "backend_authentication_type": "service_token",
+      "backend_authentication_value": "token-value",
+      "proxy": {
+        "api_backend": "http://test-upstream.lvh.me:$TEST_NGINX_SERVER_PORT/",
+        "proxy_rules": [
+          { "pattern": "/", "http_method": "POST", "metric_system_name": "hits", "delta": 2 }
+        ],
+        "policy_chain": [
+          {
+            "name": "apicast.policy.apicast"
+          },
+          {
+            "name": "apicast.policy.http_proxy",
+            "configuration": {
+                "all_proxy": "$TEST_NGINX_HTTP_PROXY"
+            }
+          }
+        ]
+      }
+    }
+  ]
+}
+--- backend
+  location /transactions/authrep.xml {
+    content_by_lua_block {
+      local expected = "service_token=token-value&service_id=42&usage%5Bhits%5D=2&user_key=value"
+      require('luassert').same(ngx.decode_args(expected), ngx.req.get_uri_args(0))
+    }
+  }
+--- upstream
+  server_name test-upstream.lvh.me;
+  location / {
+    access_by_lua_block {
+      assert = require('luassert')
+      local content_length = ngx.req.get_headers()["Content-Length"]
+      local encoding = ngx.req.get_headers()["Transfer-Encoding"]
+      assert.equal('12', content_length)
+      assert.falsy(encoding)
+      ngx.say("yay, api backend")
+    }
+  }
+--- more_headers
+Transfer-Encoding: chunked
+--- request eval
+"POST /?user_key=value
+7\r
+hello, \r
+5\r
+world\r
+0\r
+\r
+"
+--- error_code: 200
+--- error_log env
+using proxy: $TEST_NGINX_HTTP_PROXY
+
+
+=== TEST 7: using HTTPS proxy for backend with chunked request
+--- ONLY
+--- init eval
+$Test::Nginx::Util::PROXY_SSL_PORT = Test::APIcast::get_random_port();
+$Test::Nginx::Util::ENDPOINT_SSL_PORT = Test::APIcast::get_random_port();
+--- configuration random_port env eval
+<<EOF
+{
+  "services": [
+    {
+      "backend_version":  1,
+      "proxy": {
+        "api_backend": "https://localhost:$Test::Nginx::Util::ENDPOINT_SSL_PORT",
+        "proxy_rules": [
+          { "pattern": "/", "http_method": "POST", "metric_system_name": "hits", "delta": 2 }
+        ],
+        "policy_chain": [
+          {
+            "name": "apicast.policy.apicast"
+          },
+          {
+            "name": "apicast.policy.camel",
+            "configuration": {
+                "https_proxy": "http://127.0.0.1:$Test::Nginx::Util::PROXY_SSL_PORT"
+            }
+          }
+        ]
+      }
+    }
+  ]
+}
+EOF
+--- backend
+  location /transactions/authrep.xml {
+    content_by_lua_block {
+      ngx.exit(ngx.OK)
+    }
+  }
+--- upstream eval
+<<EOF
+  # Endpoint config
+  listen $Test::Nginx::Util::ENDPOINT_SSL_PORT ssl;
+
+  ssl_certificate $Test::Nginx::Util::ServRoot/html/server.crt;
+  ssl_certificate_key $Test::Nginx::Util::ServRoot/html/server.key;
+
+  server_name _ default_server;
+
+  location / {
+    access_by_lua_block {
+      assert = require('luassert')
+      local content_length = ngx.req.get_headers()["Content-Length"]
+      local encoding = ngx.req.get_headers()["Transfer-Encoding"]
+      assert.equal('12', content_length)
+      assert.falsy(encoding)
+      ngx.say("yay, api backend")
+    }
+  }
+}
+server {
+  # Proxy config
+  listen $Test::Nginx::Util::PROXY_SSL_PORT ssl;
+
+  ssl_certificate $Test::Nginx::Util::ServRoot/html/server.crt;
+  ssl_certificate_key $Test::Nginx::Util::ServRoot/html/server.key;
+
+
+  server_name _ default_server;
+
+  location ~ /.* {
+    proxy_http_version 1.1;
+    proxy_pass https://\$http_host;
+  }
+EOF
+--- more_headers
+Transfer-Encoding: chunked
+--- request eval
+"POST /?user_key=value
+7\r
+hello, \r
+5\r
+world\r
+0\r
+\r
+"
+--- error_code: 200
+--- user_files fixture=tls.pl eval
+--- error_log eval
+<<EOF
+using proxy: http://127.0.0.1:$Test::Nginx::Util::PROXY_SSL_PORT,
+EOF