Bind to a list of Network Interfaces (#471)

Co-authored-by: Michael Graeb <[email protected]>

.github/workflows/ci.yml

@@ -161,7 +161,7 @@ jobs:
         python3 builder.pyz build -p aws-c-http --cmake-extra=-DENABLE_LOCALHOST_INTEGRATION_TESTS=ON --config Debug
 
   localhost-test-mac:
-    runs-on: macos-11 # latest
+    runs-on: macos-13 # latest
     steps:
     - name: Checkout
       uses: actions/checkout@v3

include/aws/http/connection_manager.h

@@ -124,6 +124,19 @@ struct aws_http_connection_manager_options {
      * timeout will be closed automatically.
      */
     uint64_t max_connection_idle_in_milliseconds;
+
+    /**
+     * (Optional)
+     * An array of network interface names. The manager will distribute the
+     * connections across network interface names provided in this array. If any interface name is invalid, goes down,
+     * or has any issues like network access, you will see connection failures. If
+     * `socket_options.network_interface_name` is also set, an `AWS_ERROR_INVALID_ARGUMENT` error will be raised.
+     *
+     * This option is only supported on Linux, MacOS, and platforms that have either SO_BINDTODEVICE or IP_BOUND_IF. It
+     * is not supported on Windows. `AWS_ERROR_PLATFORM_NOT_SUPPORTED` will be raised on unsupported platforms.
+     */
+    const struct aws_byte_cursor *network_interface_names_array;
+    size_t num_network_interface_names;
 };
 
 AWS_EXTERN_C_BEGIN

source/connection_manager.c

@@ -292,6 +292,17 @@ struct aws_http_connection_manager {
      */
     struct aws_task *cull_task;
     struct aws_event_loop *cull_event_loop;
+
+    /*
+     * An aws_array_list<struct aws_string *> of network interface names to distribute the connections using the
+     * round-robin algorithm. We picked round-robin because it is trivial to implement and good enough. We can later
+     * update to a more complex distribution algorithm if required.
+     */
+    struct aws_array_list network_interface_names;
+    /*
+     * Current index in the network_interface_names array_list.
+     */
+    size_t network_interface_names_index;
 };
 
 struct aws_http_connection_manager_snapshot {
@@ -703,6 +714,13 @@ static void s_aws_http_connection_manager_finish_destroy(struct aws_http_connect
         aws_http_proxy_config_destroy(manager->proxy_config);
     }
 
+    for (size_t i = 0; i < aws_array_list_length(&manager->network_interface_names); i++) {
+        struct aws_string *interface_name = NULL;
+        aws_array_list_get_at(&manager->network_interface_names, &interface_name, i);
+        aws_string_destroy(interface_name);
+    }
+    aws_array_list_clean_up(&manager->network_interface_names);
+
     /*
      * If this task exists then we are actually in the corresponding event loop running the final destruction task.
      * In that case, we've already cancelled this task and when you cancel, it runs synchronously.  So in that
@@ -819,6 +837,15 @@ struct aws_http_connection_manager *aws_http_connection_manager_new(
         return NULL;
     }
 
+    if (options->socket_options->network_interface_name[0] != '\0' && options->num_network_interface_names > 0) {
+        AWS_LOGF_ERROR(
+            AWS_LS_HTTP_CONNECTION_MANAGER,
+            "Invalid options - socket_options.network_interface_name and network_interface_names_array cannot be both "
+            "set.");
+        aws_raise_error(AWS_ERROR_INVALID_ARGUMENT);
+        return NULL;
+    }
+
     struct aws_http_connection_manager *manager =
         aws_mem_calloc(allocator, 1, sizeof(struct aws_http_connection_manager));
     if (manager == NULL) {
@@ -896,6 +923,20 @@ struct aws_http_connection_manager *aws_http_connection_manager_new(
     manager->max_closed_streams = options->max_closed_streams;
     manager->http2_conn_manual_window_management = options->http2_conn_manual_window_management;
 
+    manager->network_interface_names_index = 0;
+    if (options->num_network_interface_names > 0) {
+        aws_array_list_init_dynamic(
+            &manager->network_interface_names,
+            allocator,
+            options->num_network_interface_names,
+            sizeof(struct aws_string *));
+        for (size_t i = 0; i < options->num_network_interface_names; i++) {
+            struct aws_byte_cursor interface_name = options->network_interface_names_array[i];
+            struct aws_string *interface_name_str = aws_string_new_from_cursor(allocator, &interface_name);
+            aws_array_list_push_back(&manager->network_interface_names, &interface_name_str);
+        }
+    }
+
     /* NOTHING can fail after here */
     s_schedule_connection_culling(manager);
 
@@ -990,7 +1031,26 @@ static int s_aws_http_connection_manager_new_connection(struct aws_http_connecti
     options.host_name = aws_byte_cursor_from_string(manager->host);
     options.port = manager->port;
     options.initial_window_size = manager->initial_window_size;
-    options.socket_options = &manager->socket_options;
+    struct aws_socket_options socket_options = manager->socket_options;
+    if (aws_array_list_length(&manager->network_interface_names)) {
+        struct aws_string *interface_name = NULL;
+        aws_array_list_get_at(
+            &manager->network_interface_names, &interface_name, manager->network_interface_names_index);
+        manager->network_interface_names_index =
+            (manager->network_interface_names_index + 1) % aws_array_list_length(&manager->network_interface_names);
+#if defined(_MSC_VER)
+#    pragma warning(push)
+#    pragma warning(disable : 4996) /* allow strncpy() */
+#endif
+        /* If the interface_name is too long or not null terminated, it will be caught in the `aws_socket_init` function
+         * so we don't need to worry about that here.*/
+        strncpy(
+            socket_options.network_interface_name, aws_string_c_str(interface_name), AWS_NETWORK_INTERFACE_NAME_MAX);
+#if defined(_MSC_VER)
+#    pragma warning(pop)
+#endif
+    }
+    options.socket_options = &socket_options;
     options.on_setup = s_aws_http_connection_manager_on_connection_setup;
     options.on_shutdown = s_aws_http_connection_manager_on_connection_shutdown;
     options.manual_window_management = manager->enable_read_back_pressure;

tests/CMakeLists.txt

@@ -525,6 +525,7 @@ add_net_test_case(test_connection_manager_idle_culling_single)
 add_net_test_case(test_connection_manager_idle_culling_many)
 add_net_test_case(test_connection_manager_idle_culling_mixture)
 add_net_test_case(test_connection_manager_idle_culling_refcount)
+add_net_test_case(test_connection_manager_with_network_interface_list)
 
 # tests where we establish real connections
 add_net_test_case(test_connection_manager_single_connection)

tests/test_connection_manager.c

@@ -58,6 +58,8 @@ struct cm_tester_options {
     struct aws_http2_setting *initial_settings_array;
     size_t num_initial_settings;
     bool self_lib_init;
+    const struct aws_byte_cursor *verify_network_interface_names_array;
+    size_t num_network_interface_names;
 };
 
 struct cm_tester {
@@ -73,6 +75,8 @@ struct cm_tester {
     struct aws_tls_ctx_options tls_ctx_options;
     struct aws_tls_connection_options tls_connection_options;
     struct aws_http_proxy_options *verify_proxy_options;
+    const struct aws_byte_cursor *verify_network_interface_names_array;
+    size_t num_network_interface_names;
 
     struct aws_mutex lock;
     struct aws_condition_variable signal;
@@ -219,6 +223,8 @@ static int s_cm_tester_init(struct cm_tester_options *options) {
         .http2_prior_knowledge = !options->use_tls && options->http2,
         .initial_settings_array = options->initial_settings_array,
         .num_initial_settings = options->num_initial_settings,
+        .network_interface_names_array = options->verify_network_interface_names_array,
+        .num_network_interface_names = options->num_network_interface_names,
     };
 
     if (options->mock_table) {
@@ -234,6 +240,8 @@ static int s_cm_tester_init(struct cm_tester_options *options) {
     }
 
     tester->mock_table = options->mock_table;
+    tester->verify_network_interface_names_array = options->verify_network_interface_names_array;
+    tester->num_network_interface_names = options->num_network_interface_names;
 
     aws_atomic_store_int(&tester->next_connection_id, 0);
 
@@ -729,6 +737,13 @@ static int s_aws_http_connection_manager_create_connection_sync_mock(
     const struct aws_http_client_connection_options *options) {
     struct cm_tester *tester = &s_tester;
 
+    if (tester->num_network_interface_names) {
+        struct aws_byte_cursor interface_name =
+            tester->verify_network_interface_names_array
+                [aws_atomic_load_int(&tester->next_connection_id) % tester->num_network_interface_names];
+        ASSERT_TRUE(aws_byte_cursor_eq_c_str(&interface_name, options->socket_options->network_interface_name));
+    }
+
     size_t next_connection_id = aws_atomic_fetch_add(&tester->next_connection_id, 1);
 
     ASSERT_SUCCESS(aws_mutex_lock(&tester->lock));
@@ -819,6 +834,39 @@ static struct aws_http_connection_manager_system_vtable s_synchronous_mocks = {
     .aws_http_connection_get_version = s_aws_http_connection_manager_connection_get_version_sync_mock,
 };
 
+static int s_test_connection_manager_with_network_interface_list(struct aws_allocator *allocator, void *ctx) {
+    (void)ctx;
+    struct aws_byte_cursor *interface_names_array = aws_mem_calloc(allocator, 3, sizeof(struct aws_byte_cursor));
+    interface_names_array[0] = aws_byte_cursor_from_c_str("ens32");
+    interface_names_array[1] = aws_byte_cursor_from_c_str("ens64");
+    interface_names_array[2] = aws_byte_cursor_from_c_str("ens96");
+
+    struct cm_tester_options options = {
+        .allocator = allocator,
+        .max_connections = 20,
+        .mock_table = &s_synchronous_mocks,
+        .verify_network_interface_names_array = interface_names_array,
+    };
+
+    ASSERT_SUCCESS(s_cm_tester_init(&options));
+    size_t num_connections = 6;
+    for (size_t i = 0; i < num_connections; ++i) {
+        s_add_mock_connections(1, AWS_NCRT_SUCCESS, i % 1 == 0);
+    }
+    s_acquire_connections(num_connections);
+
+    ASSERT_SUCCESS(s_wait_on_connection_reply_count(num_connections));
+    ASSERT_SUCCESS(s_release_connections(num_connections, false));
+    ASSERT_UINT_EQUALS(0, s_tester.connection_errors);
+
+    ASSERT_SUCCESS(s_cm_tester_clean_up());
+    aws_mem_release(allocator, interface_names_array);
+    return AWS_OP_SUCCESS;
+}
+AWS_TEST_CASE(
+    test_connection_manager_with_network_interface_list,
+    s_test_connection_manager_with_network_interface_list);
+
 static int s_test_connection_manager_acquire_release_mix_synchronous(struct aws_allocator *allocator, void *ctx) {
     (void)ctx;