Add no_proxy_hosts configuration to proxy options/config. (#532)

Co-authored-by: Dengke Tang <[email protected]>

Author: alextwoods

include/aws/http/private/proxy_impl.h

@@ -57,6 +57,8 @@ struct aws_http_proxy_config {
     struct aws_tls_connection_options *tls_options;
 
     struct aws_http_proxy_strategy *proxy_strategy;
+
+    struct aws_byte_buf no_proxy_hosts;
 };
 
 /*

include/aws/http/proxy.h

@@ -152,6 +152,12 @@ struct aws_http_proxy_options {
      * Replaced by instantiating a proxy_strategy via aws_http_proxy_strategy_new_basic_auth()
      */
     struct aws_byte_cursor auth_password;
+
+    /**
+     * Optional
+     * No proxy hosts - Comma seperated list of hosts for which not to use a proxy, if one is specified.
+     */
+    struct aws_byte_cursor no_proxy_hosts;
 };
 
 /**

source/proxy_connection.c

@@ -1163,8 +1163,9 @@ static int s_proxy_uri_init_from_env_variable(
         if (aws_http_host_matches_no_proxy(allocator, host_cursor, no_proxy_str)) {
             AWS_LOGF_DEBUG(
                 AWS_LS_HTTP_CONNECTION,
-                "Host \"" PRInSTR "\" found in NO_PROXY, bypassing proxy",
-                AWS_BYTE_CURSOR_PRI(host_cursor));
+                "Host \"" PRInSTR "\" found in no_proxy_hosts: \" %s \", bypassing proxy",
+                AWS_BYTE_CURSOR_PRI(options->host_name),
+                aws_string_c_str(no_proxy_str));
             aws_string_destroy(no_proxy_str);
             return AWS_OP_SUCCESS;
         }
@@ -1204,6 +1205,25 @@ static int s_connect_proxy(const struct aws_http_client_connection_options *opti
         return AWS_OP_ERR;
     }
 
+    if (options->proxy_options->no_proxy_hosts.len > 0) {
+        struct aws_string *no_proxy_host_str =
+            aws_string_new_from_cursor(options->allocator, &options->proxy_options->no_proxy_hosts);
+        if (aws_http_host_matches_no_proxy(options->allocator, options->host_name, no_proxy_host_str)) {
+            AWS_LOGF_DEBUG(
+                AWS_LS_HTTP_CONNECTION,
+                "Host \"" PRInSTR "\" found in NO_PROXY, bypassing proxy",
+                AWS_BYTE_CURSOR_PRI(options->host_name));
+            aws_string_destroy(no_proxy_host_str);
+
+            /* host matched no_proxy, connect without a proxy.: Fill in a new connection options with NULL proxy_options
+             */
+            struct aws_http_client_connection_options options_copy = *options;
+            options_copy.proxy_options = NULL;
+            return aws_http_client_connect_internal(&options_copy, NULL);
+        }
+        aws_string_destroy(no_proxy_host_str);
+    }
+
     enum aws_http_proxy_connection_type proxy_connection_type =
         s_determine_proxy_connection_type(options->proxy_options->connection_type, options->tls_options != NULL);
 
@@ -1360,6 +1380,10 @@ static struct aws_http_proxy_config *s_aws_http_proxy_config_new(
         goto on_error;
     }
 
+    if (aws_byte_buf_init_copy_from_cursor(&config->no_proxy_hosts, allocator, proxy_options->no_proxy_hosts)) {
+        goto on_error;
+    }
+
     if (proxy_options->tls_options) {
         config->tls_options = aws_mem_calloc(allocator, 1, sizeof(struct aws_tls_connection_options));
         if (aws_tls_connection_options_copy(config->tls_options, proxy_options->tls_options)) {
@@ -1480,6 +1504,11 @@ struct aws_http_proxy_config *aws_http_proxy_config_new_clone(
         goto on_error;
     }
 
+    if (aws_byte_buf_init_copy_from_cursor(
+            &config->no_proxy_hosts, allocator, aws_byte_cursor_from_buf(&proxy_config->no_proxy_hosts))) {
+        goto on_error;
+    }
+
     if (proxy_config->tls_options) {
         config->tls_options = aws_mem_calloc(allocator, 1, sizeof(struct aws_tls_connection_options));
         if (aws_tls_connection_options_copy(config->tls_options, proxy_config->tls_options)) {
@@ -1506,6 +1535,7 @@ void aws_http_proxy_config_destroy(struct aws_http_proxy_config *config) {
     }
 
     aws_byte_buf_clean_up(&config->host);
+    aws_byte_buf_clean_up(&config->no_proxy_hosts);
 
     if (config->tls_options) {
         aws_tls_connection_options_clean_up(config->tls_options);
@@ -1527,6 +1557,7 @@ void aws_http_proxy_options_init_from_config(
     options->port = config->port;
     options->tls_options = config->tls_options;
     options->proxy_strategy = config->proxy_strategy;
+    options->no_proxy_hosts = aws_byte_cursor_from_buf(&config->no_proxy_hosts);
 }
 
 int aws_http_options_validate_proxy_configuration(const struct aws_http_client_connection_options *options) {

tests/CMakeLists.txt

@@ -636,6 +636,8 @@ add_test_case(http_proxy_adaptive_ntlm_success)
 add_test_case(http_proxy_adaptive_failure)
 add_test_case(http_forwarding_proxy_uri_rewrite)
 add_test_case(http_forwarding_proxy_uri_rewrite_options_star)
+add_test_case(http_proxy_no_proxy_hosts_match)
+add_test_case(http_proxy_no_proxy_hosts_no_match)
 
 # NO_PROXY tests
 add_test_case(test_no_proxy_subdomain_matching)
@@ -693,11 +695,11 @@ add_net_test_case(h2_sm_connection_ping)
 add_net_test_case(h2_sm_acquire_stream)
 add_net_test_case(h2_sm_acquire_stream_multiple_connections)
 add_net_test_case(h2_sm_closing_before_connection_acquired)
-add_net_test_case(h2_sm_close_connection_on_server_error)
 
 # Tests against local server
 if(ENABLE_LOCALHOST_INTEGRATION_TESTS)
     # Tests should be named with localhost_integ_*
+    add_net_test_case(localhost_integ_h2_sm_close_connection_on_server_error)
     add_net_test_case(localhost_integ_h2_sm_prior_knowledge)
     add_net_test_case(localhost_integ_h2_sm_acquire_stream_stress)
     add_net_test_case(localhost_integ_h2_sm_acquire_stream_stress_with_body)

tests/proxy_test_helper.c

@@ -489,6 +489,24 @@ int proxy_tester_verify_connection_attempt_was_to_proxy(
     return AWS_OP_SUCCESS;
 }
 
+int proxy_tester_verify_connection_attempt_was_to_target(
+    struct proxy_tester *tester,
+    struct aws_byte_cursor expected_host,
+    uint32_t expected_port) {
+    ASSERT_BIN_ARRAYS_EQUALS(
+        tester->connection_host_name.buffer,
+        tester->connection_host_name.len,
+        expected_host.ptr,
+        expected_host.len,
+        "Connection host should have been \"" PRInSTR "\", but was \"" PRInSTR "\".",
+        AWS_BYTE_CURSOR_PRI(expected_host),
+        AWS_BYTE_BUF_PRI(tester->connection_host_name));
+
+    ASSERT_TRUE(tester->connection_port == expected_port);
+
+    return AWS_OP_SUCCESS;
+}
+
 struct testing_channel *proxy_tester_get_current_channel(struct proxy_tester *tester) {
     struct testing_channel_bootstrap_wrapper *wrapper = s_get_current_channel_bootstrap_wrapper(tester);
     if (wrapper == NULL) {

tests/proxy_test_helper.h

@@ -23,6 +23,7 @@ enum proxy_tester_test_mode {
     PTTM_HTTP_FORWARD = 0,
     PTTM_HTTP_TUNNEL,
     PTTM_HTTPS_TUNNEL,
+    PTTM_NO_PROXY,
 };
 
 enum proxy_tester_failure_type {
@@ -122,6 +123,11 @@ int proxy_tester_verify_connection_attempt_was_to_proxy(
     struct aws_byte_cursor expected_host,
     uint32_t expected_port);
 
+int proxy_tester_verify_connection_attempt_was_to_target(
+    struct proxy_tester *tester,
+    struct aws_byte_cursor expected_host,
+    uint32_t expected_port);
+
 struct testing_channel *proxy_tester_get_current_channel(struct proxy_tester *tester);
 
 #endif /* AWS_HTTP_PROXY_TEST_HELPER_H */

tests/py_localhost/README.md

@@ -6,35 +6,39 @@ Local server based on [python-hyper/h2](https://github.com/python-hyper/h2).
 
 Python 3.5+ required.
 
-- Install hyper h2 python module. `python3 -m pip install h2`
+* Install hyper h2 python module. `python3 -m pip install h2`
 
 ### TLS server
 
-- The code is based the [example](https://github.com/python-hyper/h2/blob/master/examples/asyncio/asyncio-server.py) from hyper h2 server.
-- Have the cert/key ready. The script now using `../resources/unittests.crt`, you can either just run the script within this directory, which will find the certificates and key from the related path, or you can use your own and change the code coordinately.
-- Run python. `python3 ./server.py`.
+* The code is based the [example](https://github.com/python-hyper/h2/blob/master/examples/asyncio/asyncio-server.py) from hyper h2 server.
+* Have the cert/key ready. The script now using `../resources/unittests.crt`, you can either just run the script within this directory, which will find the certificates and key from the related path, or you can use your own and change the code coordinately.
+* Run python. `python3 ./server.py`.
 
 #### Echo
 
-- Minor changed based on the example to response the headers of requests back within the headers from `/echo`.
-- To test the server runs correctly, you can do `curl -k -v -H "foo:bar" https://localhost:3443/echo` and check the result.
+* Minor changed based on the example to response the headers of requests back within the headers from `/echo`.
+* To test the server runs correctly, you can do `curl -k -v -H "foo:bar" https://localhost:3443/echo` and check the result.
 
 #### Download test
 
-- To test download, when `:path` is `/downloadTest`, server will response a repeated string with length `self.download_test_length`, which is 2,500,000,000 now. It will be repeats of sting "This is CRT HTTP test."
-- To test the server runs correctly, you can do `curl -k -v -H "foo:bar" https://localhost:3443/downloadTest` and check the result.
+* To test download, when `:path` is `/downloadTest`, server will response a repeated string with length `self.download_test_length`, which is 2,500,000,000 now. It will be repeats of sting "This is CRT HTTP test."
+* To test the server runs correctly, you can do `curl -k -v -H "foo:bar" https://localhost:3443/downloadTest` and check the result.
 
 #### Slow Connection Test
 
-- Simulate a slow connection when `:path` is `/slowConnTest`. The speed is controlled by `out_bytes_per_second`. Default speed is 900 B/s, which will send 900 bytes of data and wait a sec to send new 900 bytes of data.
+* Simulate a slow connection when `:path` is `/slowConnTest`. The speed is controlled by `out_bytes_per_second`. Default speed is 900 B/s, which will send 900 bytes of data and wait a sec to send new 900 bytes of data.
 
 #### Upload test
 
-- To test upload, when `:method` is `POST` or `PUT`, server will response the length received from response body
-- To test the server runs correctly, you can do `curl -k -X POST -F'data=@upload_test.txt' https://localhost:3443/upload_test` where `upload_test.txt` is file to upload.
+* To test upload, when `:method` is `POST` or `PUT`, server will response the length received from response body
+* To test the server runs correctly, you can do `curl -k -X POST -F'data=@upload_test.txt' https://localhost:3443/upload_test` where `upload_test.txt` is file to upload.
+
+#### expect500
+
+* The server will always return `500` for `:status`, when the `:path` is `/expect500`
 
 ### Non-TLS server
 
-- The code is based the non-tls [example](http://python-hyper.org/projects/h2/en/stable/basic-usage.html) from hyper h2 server.
-- Run python. `python3 ./non_tls_server.py`.
-- To test the server runs correctly, you can do `curl -v --http2-prior-knowledge http://localhost:3280` and check the result.
+* The code is based the non-tls [example](http://python-hyper.org/projects/h2/en/stable/basic-usage.html) from hyper h2 server.
+* Run python. `python3 ./non_tls_server.py`.
+* To test the server runs correctly, you can do `curl -v --http2-prior-knowledge http://localhost:3280` and check the result.

tests/py_localhost/server.py

@@ -130,7 +130,10 @@ def stream_complete(self, stream_id: int):
 
         path = request_data.headers[':path']
         method = request_data.headers[':method']
-        if method == "PUT" or method == "POST":
+        if path == '/expect500':
+            self.conn.send_headers(stream_id, [(':status', '500')])
+            asyncio.ensure_future(self.send_data(b"Internal Server Error", stream_id))
+        elif method == "PUT" or method == "POST":
             self.conn.send_headers(stream_id, [(':status', '200')])
             asyncio.ensure_future(self.send_data(
                 str(self.num_sentence_received[stream_id]).encode(), stream_id))

tests/test_proxy.c

@@ -195,6 +195,10 @@ static int s_test_aws_proxy_new_socket_channel(struct aws_socket_channel_bootstr
         testing_channel_run_currently_queued_tasks(channel);
     }
 
+    if (tester.test_mode == PTTM_NO_PROXY) {
+        return AWS_OP_SUCCESS;
+    }
+
     if (tester.failure_type == PTFT_NONE || tester.failure_type == PTFT_CONNECT_REQUEST ||
         tester.failure_type == PTFT_TLS_NEGOTIATION) {
         if (tester.proxy_options.connection_type == AWS_HPCT_HTTP_TUNNEL) {
@@ -1142,3 +1146,77 @@ static int s_test_http_forwarding_proxy_uri_rewrite_options_star(struct aws_allo
     return AWS_OP_SUCCESS;
 }
 AWS_TEST_CASE(http_forwarding_proxy_uri_rewrite_options_star, s_test_http_forwarding_proxy_uri_rewrite_options_star);
+
+/*
+ * Test no_proxy_hosts functionality - host matches no_proxy pattern, should bypass proxy
+ */
+static int s_test_http_proxy_no_proxy_hosts_match(struct aws_allocator *allocator, void *ctx) {
+    (void)ctx;
+
+    aws_http_connection_set_system_vtable(&s_proxy_connection_system_vtable);
+    aws_http_proxy_system_set_vtable(&s_proxy_table_for_tls);
+
+    struct aws_http_proxy_options proxy_options = {
+        .connection_type = AWS_HPCT_HTTP_TUNNEL,
+        .host = aws_byte_cursor_from_c_str(s_proxy_host_name),
+        .port = s_proxy_port,
+        .no_proxy_hosts = aws_byte_cursor_from_c_str("aws.amazon.com"),
+    };
+
+    struct proxy_tester_options options = {
+        .alloc = allocator,
+        .proxy_options = &proxy_options,
+        .host = aws_byte_cursor_from_c_str(s_host_name), /* aws.amazon.com */
+        .port = s_port,
+        .test_mode = PTTM_NO_PROXY,
+        .failure_type = PTFT_NONE,
+    };
+
+    ASSERT_SUCCESS(proxy_tester_init(&tester, &options));
+    proxy_tester_wait(&tester, proxy_tester_connection_setup_pred);
+
+    /* Should connect directly to target host, not proxy */
+    ASSERT_SUCCESS(
+        proxy_tester_verify_connection_attempt_was_to_target(&tester, aws_byte_cursor_from_c_str(s_host_name), s_port));
+
+    ASSERT_SUCCESS(proxy_tester_clean_up(&tester));
+    return AWS_OP_SUCCESS;
+}
+AWS_TEST_CASE(http_proxy_no_proxy_hosts_match, s_test_http_proxy_no_proxy_hosts_match);
+
+/*
+ * Test no_proxy_hosts functionality - host does not match no_proxy pattern, should use proxy
+ */
+static int s_test_http_proxy_no_proxy_hosts_no_match(struct aws_allocator *allocator, void *ctx) {
+    (void)ctx;
+
+    aws_http_connection_set_system_vtable(&s_proxy_connection_system_vtable);
+    aws_http_proxy_system_set_vtable(&s_proxy_table_for_tls);
+
+    struct aws_http_proxy_options proxy_options = {
+        .connection_type = AWS_HPCT_HTTP_TUNNEL,
+        .host = aws_byte_cursor_from_c_str(s_proxy_host_name),
+        .port = s_proxy_port,
+        .no_proxy_hosts = aws_byte_cursor_from_c_str("example.com"),
+    };
+
+    struct proxy_tester_options options = {
+        .alloc = allocator,
+        .proxy_options = &proxy_options,
+        .host = aws_byte_cursor_from_c_str(s_host_name), /* aws.amazon.com */
+        .port = s_port,
+        .test_mode = PTTM_HTTP_TUNNEL,
+        .failure_type = PTFT_NONE,
+    };
+
+    ASSERT_SUCCESS(proxy_tester_init(&tester, &options));
+    proxy_tester_wait(&tester, proxy_tester_connection_setup_pred);
+
+    /* Should connect to proxy since host doesn't match no_proxy pattern */
+    ASSERT_SUCCESS(proxy_tester_verify_connection_attempt_was_to_proxy(
+        &tester, aws_byte_cursor_from_c_str(s_proxy_host_name), s_proxy_port));
+
+    ASSERT_SUCCESS(proxy_tester_clean_up(&tester));
+    return AWS_OP_SUCCESS;
+}
+AWS_TEST_CASE(http_proxy_no_proxy_hosts_no_match, s_test_http_proxy_no_proxy_hosts_no_match);

tests/test_stream_manager.c

@@ -1228,28 +1228,6 @@ TEST_CASE(h2_sm_acquire_stream_multiple_connections) {
     return s_tester_clean_up();
 }
 
-/* Test that makes tons of real streams against real world */
-TEST_CASE(h2_sm_close_connection_on_server_error) {
-    (void)ctx;
-    /* page not exist. */
-    struct aws_byte_cursor uri_cursor = aws_byte_cursor_from_c_str("https://www.amazon.com/non-exists");
-    struct sm_tester_options options = {
-        .max_connections = 1,
-        .max_concurrent_streams_per_connection = 10,
-        .alloc = allocator,
-        .uri_cursor = &uri_cursor,
-        .close_connection_on_server_error = true,
-    };
-    ASSERT_SUCCESS(s_tester_init(&options));
-    int num_to_acquire = 50;
-    ASSERT_SUCCESS(s_sm_stream_acquiring(num_to_acquire));
-    ASSERT_SUCCESS(s_wait_on_streams_completed_count(num_to_acquire));
-    ASSERT_TRUE((int)s_tester.acquiring_stream_errors == 0);
-    ASSERT_TRUE((int)s_tester.stream_200_count == 0);
-
-    return s_tester_clean_up();
-}
-
 static void s_sm_tester_on_connection_setup(struct aws_http_connection *connection, int error_code, void *user_data) {
     if (s_tester.release_sm_during_connection_acquiring) {
         aws_http2_stream_manager_release(s_tester.stream_manager);
@@ -1288,6 +1266,27 @@ TEST_CASE(h2_sm_closing_before_connection_acquired) {
     return s_tester_clean_up();
 }
 
+TEST_CASE(localhost_integ_h2_sm_close_connection_on_server_error) {
+    (void)ctx;
+    /* server that will return 500 status code all the time. */
+    struct aws_byte_cursor uri_cursor = aws_byte_cursor_from_c_str("https://localhost:3443/expect500");
+    struct sm_tester_options options = {
+        .max_connections = 1,
+        .max_concurrent_streams_per_connection = 10,
+        .alloc = allocator,
+        .uri_cursor = &uri_cursor,
+        .close_connection_on_server_error = true,
+    };
+    ASSERT_SUCCESS(s_tester_init(&options));
+    int num_to_acquire = 50;
+    ASSERT_SUCCESS(s_sm_stream_acquiring(num_to_acquire));
+    ASSERT_SUCCESS(s_wait_on_streams_completed_count(num_to_acquire));
+    ASSERT_TRUE((int)s_tester.acquiring_stream_errors == 0);
+    ASSERT_TRUE((int)s_tester.stream_200_count == 0);
+
+    return s_tester_clean_up();
+}
+
 /* Test our http2 stream manager works with prior knowledge */
 TEST_CASE(localhost_integ_h2_sm_prior_knowledge) {
     (void)ctx;