Fetch: added forward proxy support with HTTPS tunneling.

Supports Basic authentication via Proxy-Authorization header.

- js_fetch_proxy - configures forward proxy URL

example.conf:
...
  http {
      js_import main.js;

      server {
          listen 8080;

          resolver   127.0.0.1:5353;

          location /api {
              js_fetch_proxy http://user:[email protected]:3128;
              js_content main.fetch_handler;
          }
      }
  }

This implements #956 feature request on Github.

Author: xeioex

nginx/ngx_http_js_module.c

@@ -593,6 +593,13 @@ static ngx_command_t  ngx_http_js_commands[] = {
       offsetof(ngx_http_js_loc_conf_t, fetch_keepalive_timeout),
       NULL },
 
+    { ngx_string("js_fetch_proxy"),
+      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1,
+      ngx_js_fetch_proxy,
+      NGX_HTTP_LOC_CONF_OFFSET,
+      0,
+      NULL },
+
       ngx_null_command
 };
 
@@ -818,6 +825,16 @@ static njs_external_t  ngx_http_js_ext_request[] = {
         }
     },
 
+    {
+        .flags = NJS_EXTERN_PROPERTY,
+        .name.string = njs_str("requestLine"),
+        .enumerable = 1,
+        .u.property = {
+            .handler = ngx_js_ext_string,
+            .magic32 = offsetof(ngx_http_request_t, request_line),
+        }
+    },
+
     {
         .flags = NJS_EXTERN_PROPERTY,
         .name.string = njs_str("requestText"),
@@ -1080,6 +1097,8 @@ static const JSCFunctionListEntry ngx_http_qjs_ext_request[] = {
     JS_CGETSET_DEF("remoteAddress", ngx_http_qjs_ext_remote_address, NULL),
     JS_CGETSET_MAGIC_DEF("requestBuffer", ngx_http_qjs_ext_request_body, NULL,
                          NGX_JS_BUFFER),
+    JS_CGETSET_MAGIC_DEF("requestLine", ngx_http_qjs_ext_string, NULL,
+                         offsetof(ngx_http_request_t, request_line)),
     JS_CGETSET_MAGIC_DEF("requestText", ngx_http_qjs_ext_request_body, NULL,
                          NGX_JS_STRING),
     JS_CGETSET_MAGIC_DEF("responseBuffer", ngx_http_qjs_ext_response_body, NULL,

nginx/ngx_js.c

@@ -3305,6 +3305,162 @@ ngx_js_preload_object(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
 }
 
 
+static ngx_int_t
+ngx_js_build_proxy_auth_header(ngx_pool_t *pool, ngx_js_loc_conf_t *jscf,
+    ngx_str_t *user, ngx_str_t *pass)
+{
+    u_char     *p;
+    size_t      len;
+    ngx_str_t   userpass, b64;
+
+    userpass.len = user->len + 1 + pass->len;
+    userpass.data = ngx_pnalloc(pool, userpass.len);
+    if (userpass.data == NULL) {
+        return NGX_ERROR;
+    }
+
+    p = ngx_cpymem(userpass.data, user->data, user->len);
+    *p++ = ':';
+    ngx_memcpy(p, pass->data, pass->len);
+
+    b64.len = ngx_base64_encoded_length(userpass.len);
+    b64.data = ngx_pnalloc(pool, b64.len);
+    if (b64.data == NULL) {
+        return NGX_ERROR;
+    }
+
+    ngx_encode_base64(&b64, &userpass);
+
+    len = sizeof("Proxy-Authorization: Basic \r\n") - 1 + b64.len;
+    p = ngx_pnalloc(pool, len);
+    if (p == NULL) {
+        return NGX_ERROR;
+    }
+
+    ngx_sprintf(p, "Proxy-Authorization: Basic %V\r\n", &b64);
+    jscf->fetch_proxy_auth_header.data = p;
+    jscf->fetch_proxy_auth_header.len = len;
+
+    return NGX_OK;
+}
+
+
+char *
+ngx_js_fetch_proxy(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
+{
+    u_char             *p, *at, *colon, *host_start, *user_start, *pass_start;
+    u_char             *decoded_user, *decoded_pass, *decoded_end;
+    size_t              user_len, pass_len;
+    ngx_url_t          *u;
+    ngx_str_t          *value;
+    ngx_str_t           user, pass;
+    ngx_js_loc_conf_t  *jscf;
+
+    jscf = conf;
+
+    value = cf->args->elts;
+
+    if (ngx_strncmp(value[1].data, "http://", sizeof("http://") - 1) != 0) {
+        ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
+                           "js_fetch_proxy URL must use http:// scheme");
+        return NGX_CONF_ERROR;
+    }
+
+    host_start = value[1].data + (sizeof("http://") - 1);
+    at = ngx_strlchr(host_start, value[1].data + value[1].len, '@');
+
+    if (at != NULL) {
+        colon = NULL;
+
+        for (p = at - 1; p > host_start; p--) {
+            if (*p == ':') {
+                colon = p;
+                break;
+            }
+        }
+
+        if (colon == NULL) {
+            ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
+                               "js_fetch_proxy URL credentials must be in "
+                               "user:password format");
+            return NGX_CONF_ERROR;
+        }
+
+        user_start = host_start;
+        user_len = colon - host_start;
+        pass_start = colon + 1;
+        pass_len = at - pass_start;
+
+        decoded_user = ngx_pnalloc(cf->pool, 128);
+        if (decoded_user == NULL) {
+            return NGX_CONF_ERROR;
+        }
+
+        decoded_pass = ngx_pnalloc(cf->pool, 128);
+        if (decoded_pass == NULL) {
+            return NGX_CONF_ERROR;
+        }
+
+        p = user_start;
+        decoded_end = decoded_user;
+        ngx_unescape_uri(&decoded_end, &p, user_len, NGX_UNESCAPE_URI);
+
+        user_len = decoded_end - decoded_user;
+        if (user_len == 0 || user_len > 127) {
+            ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
+                               "js_fetch_proxy username invalid or too long "
+                               "(max 127 bytes after decoding)");
+            return NGX_CONF_ERROR;
+        }
+
+        p = pass_start;
+        decoded_end = decoded_pass;
+        ngx_unescape_uri(&decoded_end, &p, pass_len, NGX_UNESCAPE_URI);
+
+        pass_len = decoded_end - decoded_pass;
+        if (pass_len == 0 || pass_len > 127) {
+            ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
+                               "js_fetch_proxy password invalid or too long "
+                               "(max 127 bytes after decoding)");
+            return NGX_CONF_ERROR;
+        }
+
+        user.data = decoded_user;
+        user.len = user_len;
+        pass.data = decoded_pass;
+        pass.len = pass_len;
+
+        if (ngx_js_build_proxy_auth_header(cf->pool, jscf, &user, &pass)
+            != NGX_OK)
+        {
+            return NGX_CONF_ERROR;
+        }
+
+        host_start = at + 1;
+    }
+
+    u = ngx_pcalloc(cf->pool, sizeof(ngx_url_t));
+    if (u == NULL) {
+        return NGX_CONF_ERROR;
+    }
+
+    u->url.data = host_start;
+    u->url.len = value[1].data + value[1].len - host_start;
+    u->default_port = 3128;
+    u->no_resolve = 1;
+
+    if (ngx_parse_url(cf->pool, u) != NGX_OK) {
+        ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
+                           "invalid proxy URL: %V", &value[1]);
+        return NGX_CONF_ERROR;
+    }
+
+    jscf->fetch_proxy_url = u;
+
+    return NGX_CONF_OK;
+}
+
+
 static ngx_int_t
 ngx_js_init_preload_vm(njs_vm_t *vm, ngx_js_loc_conf_t *conf)
 {
@@ -4121,6 +4277,13 @@ ngx_js_merge_conf(ngx_conf_t *cf, void *parent, void *child,
     ngx_conf_merge_msec_value(conf->fetch_keepalive_timeout,
                               prev->fetch_keepalive_timeout, 60000);
 
+    ngx_conf_merge_str_value(conf->fetch_proxy_auth_header,
+                             prev->fetch_proxy_auth_header, "");
+
+    if (conf->fetch_proxy_url == NULL && prev->fetch_proxy_url != NULL) {
+        conf->fetch_proxy_url = prev->fetch_proxy_url;
+    }
+
     ngx_queue_init(&conf->fetch_keepalive_cache);
     ngx_queue_init(&conf->fetch_keepalive_free);
 

nginx/ngx_js.h

@@ -141,7 +141,15 @@ typedef struct {
     ngx_msec_t             fetch_keepalive_time;                              \
     ngx_msec_t             fetch_keepalive_timeout;                           \
     ngx_queue_t            fetch_keepalive_cache;                             \
-    ngx_queue_t            fetch_keepalive_free
+    ngx_queue_t            fetch_keepalive_free;                              \
+                                                                              \
+    ngx_url_t              *fetch_proxy_url;                                  \
+    ngx_str_t              fetch_proxy_auth_header
+
+
+#define ngx_js_proxy(conf)                                                    \
+    ((conf)->fetch_proxy_url != NULL                                          \
+     && (conf)->fetch_proxy_url->host.len > 0)
 
 
 #if (NGX_SSL)
@@ -423,6 +431,7 @@ void ngx_js_logger(ngx_connection_t *c, ngx_uint_t level,
 char * ngx_js_import(ngx_conf_t *cf, ngx_command_t *cmd, void *conf);
 char * ngx_js_engine(ngx_conf_t *cf, ngx_command_t *cmd, void *conf);
 char * ngx_js_preload_object(ngx_conf_t *cf, ngx_command_t *cmd, void *conf);
+char * ngx_js_fetch_proxy(ngx_conf_t *cf, ngx_command_t *cmd, void *conf);
 ngx_int_t ngx_js_merge_vm(ngx_conf_t *cf, ngx_js_loc_conf_t *conf,
     ngx_js_loc_conf_t *prev,
     ngx_int_t (*init_vm)(ngx_conf_t *cf, ngx_js_loc_conf_t *conf));

nginx/ngx_js_fetch.c

@@ -508,6 +508,7 @@ ngx_js_ext_fetch(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs,
 {
     njs_int_t            ret;
     ngx_url_t            u;
+    ngx_str_t           *resolve_host;
     ngx_pool_t          *pool;
     njs_value_t         *init, *value;
     ngx_js_http_t       *http;
@@ -595,12 +596,30 @@ ngx_js_ext_fetch(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs,
     NJS_CHB_MP_INIT(&http->chain, njs_vm_memory_pool(vm));
     NJS_CHB_MP_INIT(&http->response.chain, njs_vm_memory_pool(vm));
 
-    ngx_js_fetch_build_request(http, &request, &u.uri, &u);
+    ngx_js_fetch_build_request(http, &request, &u.uri, &u,
+                               ngx_js_proxy(http->conf) && !ngx_js_https(&u));
+
+    resolve_host = NULL;
+    http->connect_port = http->port;
+
+    if (ngx_js_proxy(http->conf)) {
+        if (http->conf->fetch_proxy_url->addrs == NULL) {
+            resolve_host = &http->conf->fetch_proxy_url->host;
+            http->connect_port = http->conf->fetch_proxy_url->port;
+        }
+
+    } else if (u.addrs == NULL) {
+        resolve_host = &u.host;
+    }
+
+    if (resolve_host != NULL) {
+        ngx_log_debug0(NGX_LOG_DEBUG_EVENT, http->log, 0,
+                       "js http fetch: resolving");
 
-    if (u.addrs == NULL) {
         ctx = ngx_js_http_resolve(http, ngx_external_resolver(vm, external),
-                                  &u.host,
+                                  resolve_host,
                                   ngx_external_resolver_timeout(vm, external));
+
         if (ctx == NULL) {
             njs_vm_memory_error(vm);
             return NJS_ERROR;
@@ -617,9 +636,16 @@ ngx_js_ext_fetch(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs,
     }
 
     http->naddrs = 1;
-    ngx_memcpy(&http->addr, &u.addrs[0], sizeof(ngx_addr_t));
     http->addrs = &http->addr;
 
+    if (ngx_js_proxy(http->conf)) {
+        ngx_memcpy(&http->addr, &http->conf->fetch_proxy_url->addrs[0],
+                   sizeof(ngx_addr_t));
+
+    } else {
+        ngx_memcpy(&http->addr, &u.addrs[0], sizeof(ngx_addr_t));
+    }
+
     ngx_js_http_connect(http);
 
     njs_value_assign(retval, njs_value_arg(&fetch->promise));
@@ -1031,6 +1057,12 @@ ngx_js_fetch_alloc(njs_vm_t *vm, ngx_pool_t *pool, ngx_log_t *log,
         goto failed;
     }
 
+    /*
+     * set by ngx_pcalloc():
+     *
+     * fetch->http.proxy.state = HTTP_STATE_DIRECT;
+     */
+
     http = &fetch->http;
 
     http->pool = pool;