Apple Network Framework Socket Changes

.github/workflows/ci.yml

@@ -243,7 +243,7 @@ jobs:
       run: |
         python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')"
         chmod a+x builder
-        ./builder build -p ${{ env.PACKAGE_NAME }} --cmake-extra=-DAWS_USE_APPLE_DISPATCH_QUEUE=${{ matrix.eventloop == 'dispatch_queue' && 'ON' || 'OFF' }} --cmake-extra=-DENABLE_SANITIZERS=ON --cmake-extra=-DSANITIZERS="${{ matrix.sanitizers }}"
+        ./builder build -p ${{ env.PACKAGE_NAME }} --cmake-extra=-DAWS_USE_APPLE_NETWORK_FRAMEWORK=${{ matrix.eventloop == 'dispatch_queue' && 'ON' || 'OFF' }} --cmake-extra=-DENABLE_SANITIZERS=ON --cmake-extra=-DSANITIZERS="${{ matrix.sanitizers }}"
 
   macos-x64:
     runs-on: macos-14-large # latest
@@ -274,7 +274,7 @@ jobs:
       run: |
         python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')"
         chmod a+x builder
-        ./builder build -p ${{ env.PACKAGE_NAME }} --cmake-extra=-DAWS_USE_APPLE_DISPATCH_QUEUE=${{ matrix.eventloop == 'dispatch_queue' && 'ON' || 'OFF' }} --cmake-extra=-DENABLE_SANITIZERS=ON --cmake-extra=-DSANITIZERS="${{ matrix.sanitizers }}" --config Debug
+        ./builder build -p ${{ env.PACKAGE_NAME }} --cmake-extra=-DAWS_USE_APPLE_NETWORK_FRAMEWORK=${{ matrix.eventloop == 'dispatch_queue' && 'ON' || 'OFF' }} --cmake-extra=-DENABLE_SANITIZERS=ON --cmake-extra=-DSANITIZERS="${{ matrix.sanitizers }}" --config Debug
 
   freebsd:
     runs-on: ubuntu-24.04  # latest

CMakeLists.txt

@@ -29,6 +29,7 @@ file(GLOB AWS_IO_TESTING_HEADERS
         "include/aws/testing/*.h"
         )
 
+
 file(GLOB AWS_IO_PRIV_HEADERS
         "include/aws/io/private/*.h"
         )

README.md

@@ -151,7 +151,8 @@ Core to Async-IO is the event-loop. We provide an implementation for most platfo
 Platform | Implementation
 --- | ---
 Linux | Edge-Triggered Epoll
-BSD Variants and Apple Devices | KQueue
+BSD Variants | KQueue
+Apple Devices | KQueue or Apple Dispatch Queue
 Windows | IOCP (IO Completion Ports)
 
 Also, you can always implement your own as well.
@@ -645,12 +646,49 @@ All exported functions, simply shim into the v-table and return.
 
 We include a cross-platform API for sockets. We support TCP and UDP using IPv4 and IPv6, and Unix Domain sockets. On Windows,
 we use Named Pipes to support the functionality of Unix Domain sockets. On Windows, this is implemented with winsock2, and on
-all unix platforms we use the posix API.
+all unix platforms we use the posix API. Then we will use Apple Network Framework on Apple platforms. 
 
 Upon a connection being established, the new socket (either as the result of a `connect()` or `start_accept()` call)
 will not be attached to any event loops. It is your responsibility to register it with an event loop to begin receiving
 notifications.
 
+
+#### V-Table
+
+    struct aws_socket_vtable {
+        void (*socket_cleanup_fn)(struct aws_socket *socket);
+        int (*socket_connect_fn)(
+            struct aws_socket *socket,
+            const struct aws_socket_endpoint *remote_endpoint,
+            struct aws_event_loop *event_loop,
+            aws_socket_on_connection_result_fn *on_connection_result,
+            void *user_data);
+        int (*socket_bind_fn)(struct aws_socket *socket, const struct aws_socket_endpoint *local_endpoint);
+        int (*socket_listen_fn)(struct aws_socket *socket, int backlog_size);
+        int (*socket_start_accept_fn)(
+            struct aws_socket *socket,
+            struct aws_event_loop *accept_loop,
+            aws_socket_on_accept_result_fn *on_accept_result,
+            void *user_data);
+        int (*socket_stop_accept_fn)(struct aws_socket *socket);
+        int (*socket_close_fn)(struct aws_socket *socket);
+        int (*socket_shutdown_dir_fn)(struct aws_socket *socket, enum aws_channel_direction dir);
+        int (*socket_set_options_fn)(struct aws_socket *socket, const struct aws_socket_options *options);
+        int (*socket_assign_to_event_loop_fn)(struct aws_socket *socket, struct aws_event_loop *event_loop);
+        int (*socket_subscribe_to_readable_events_fn)(
+            struct aws_socket *socket,
+            aws_socket_on_readable_fn *on_readable,
+            void *user_data);
+        int (*socket_read_fn)(struct aws_socket *socket, struct aws_byte_buf *buffer, size_t *amount_read);
+        int (*socket_write_fn)(
+            struct aws_socket *socket,
+            const struct aws_byte_cursor *cursor,
+            aws_socket_on_write_completed_fn *written_fn,
+            void *user_data);
+        int (*socket_get_error_fn)(struct aws_socket *socket);
+        bool (*socket_is_open_fn)(struct aws_socket *socket);
+    };
+
 #### API
     typedef enum aws_socket_domain {
         AWS_SOCKET_IPV4,
@@ -717,32 +755,33 @@ Shuts down any pending operations on the socket, and cleans up state. The socket
 
     int aws_socket_connect(struct aws_socket *socket, struct aws_socket_endpoint *remote_endpoint);
 
-Connects to a remote endpoint. In UDP, this simply binds the socket to a remote address for use with `aws_socket_write()`,
-and if the operation is successful, the socket can immediately be used for write operations.
+Connects to a remote endpoint. In TCP and all Apple Network Framework connections (regardless it is UDP, TCP or LOCAL), when the connection succeed, you still must wait on  the `on_connection_established()` callback to 
+be invoked before using the socket.
 
-In TCP, this will function will not block. If the return value is successful, then you must wait on the `on_connection_established()`
-callback to be invoked before using the socket.
+In UDP, this simply binds the socket to a remote address for use with `aws_socket_write()`, and if the operation is successful, 
+the socket can immediately be used for write operations.
 
 For LOCAL (Unix Domain Sockets or Named Pipes), the socket will be immediately ready for use upon a successful return.
 
     int aws_socket_bind(struct aws_socket *socket, struct aws_socket_endpoint *local_endpoint);
 
-Binds the socket to a local address. In UDP mode, the socket is ready for `aws_socket_read()` operations. In connection oriented
-modes, you still must call `aws_socket_listen()` and `aws_socket_start_accept()` before using the socket.
+Binds the socket to a local address. In UDP mode, the socket is ready for `aws_socket_read()` operations. In connection oriented 
+modes or if you are using Apple Network Framework (regardless it is UDP or TCP), you still must call `aws_socket_listen()` and 
+`aws_socket_start_accept()` before using the socket.
 
     int aws_socket_listen(struct aws_socket *socket, int backlog_size);
 
-TCP and LOCAL only. Sets up the socket to listen on the address bound to in `aws_socket_bind()`.
+TCP, LOCAL, and Apple Network Framework only. Sets up the socket to listen on the address bound to in `aws_socket_bind()`.
 
     int aws_socket_start_accept(struct aws_socket *socket);
 
-TCP and LOCAL only. The socket will begin accepting new connections. This is an asynchronous operation. New connections will
-arrive via the `on_incoming_connection()` callback.
+TCP, LOCAL, and Apple Network Framework only. The socket will begin accepting new connections. This is an asynchronous operation. New 
+connections will arrive via the `on_incoming_connection()` callback.
 
     int aws_socket_stop_accept(struct aws_socket *socket);
 
-TCP and LOCAL only. The socket will shutdown the listener. It is safe to call `aws_socket_start_accept()` again after this
-operation.
+TCP, LOCAL, and Apple Network Framework only. The socket will shutdown the listener. It is safe to call `aws_socket_start_accept()` 
+again after this operation.
 
     int aws_socket_close(struct aws_socket *socket);
 

include/aws/io/channel_bootstrap.h

@@ -132,6 +132,14 @@ typedef void(aws_server_bootstrap_on_accept_channel_shutdown_fn)(
     struct aws_channel *channel,
     void *user_data);
 
+/**
+ * This function is only used for async listener (Apple Network Framework in this case).
+ * Once the server listener socket is finished setup and starting listening, this fuction
+ * will be invoked.
+ */
+typedef void(
+    aws_server_bootstrap_on_listener_setup_fn)(struct aws_server_bootstrap *bootstrap, int error_code, void *user_data);
+
 /**
  * Once the server listener socket is finished destroying, and all the existing connections are closed, this fuction
  * will be invoked.
@@ -212,6 +220,7 @@ struct aws_server_socket_channel_bootstrap_options {
     const struct aws_tls_connection_options *tls_options;
     aws_server_bootstrap_on_accept_channel_setup_fn *incoming_callback;
     aws_server_bootstrap_on_accept_channel_shutdown_fn *shutdown_callback;
+    aws_server_bootstrap_on_listener_setup_fn *setup_callback;
     aws_server_bootstrap_on_server_listener_destroy_fn *destroy_callback;
     bool enable_read_back_pressure;
     void *user_data;
@@ -296,6 +305,9 @@ AWS_IO_API int aws_server_bootstrap_set_alpn_callback(
 AWS_IO_API struct aws_socket *aws_server_bootstrap_new_socket_listener(
     const struct aws_server_socket_channel_bootstrap_options *bootstrap_options);
 
+AWS_IO_API struct aws_socket *aws_server_bootstrap_new_socket_listener_async(
+    const struct aws_server_socket_channel_bootstrap_options *bootstrap_options);
+
 /**
  * Shuts down 'listener' and cleans up any resources associated with it. Any incoming channels on `listener` will still
  * be active. `destroy_callback` will be invoked after the server socket listener is destroyed, and all associated

include/aws/io/private/socket_impl.h

@@ -18,6 +18,17 @@ typedef void (*aws_ms_fn_ptr)(void);
 void aws_check_and_init_winsock(void);
 aws_ms_fn_ptr aws_winsock_get_connectex_fn(void);
 aws_ms_fn_ptr aws_winsock_get_acceptex_fn(void);
+#else // NOT ON WINDOWS
+struct socket_address {
+    union sock_addr_types {
+        struct sockaddr_in addr_in;
+        struct sockaddr_in6 addr_in6;
+        struct sockaddr_un un_addr;
+#ifdef USE_VSOCK
+        struct sockaddr_vm vm_addr;
+#endif
+    } sock_addr_types;
+};
 #endif
 
 int aws_socket_init_posix(
@@ -48,8 +59,7 @@ struct aws_socket_vtable {
     int (*socket_start_accept_fn)(
         struct aws_socket *socket,
         struct aws_event_loop *accept_loop,
-        aws_socket_on_accept_result_fn *on_accept_result,
-        void *user_data);
+        struct aws_socket_listener_options options);
     int (*socket_stop_accept_fn)(struct aws_socket *socket);
     int (*socket_close_fn)(struct aws_socket *socket);
     int (*socket_shutdown_dir_fn)(struct aws_socket *socket, enum aws_channel_direction dir);
@@ -67,6 +77,11 @@ struct aws_socket_vtable {
         void *user_data);
     int (*socket_get_error_fn)(struct aws_socket *socket);
     bool (*socket_is_open_fn)(struct aws_socket *socket);
+    int (*socket_set_close_callback)(struct aws_socket *socket, aws_socket_on_shutdown_complete_fn fn, void *user_data);
+    int (*socket_set_cleanup_callback)(
+        struct aws_socket *socket,
+        aws_socket_on_shutdown_complete_fn fn,
+        void *user_data);
 };
 
 #endif // AWS_IO_SOCKET_IMPL_H

include/aws/io/socket.h

@@ -50,6 +50,8 @@ enum aws_socket_impl_type {
 
 #define AWS_NETWORK_INTERFACE_NAME_MAX 16
 
+typedef void(aws_socket_on_shutdown_complete_fn)(void *user_data);
+
 struct aws_socket_options {
     enum aws_socket_type type;
     enum aws_socket_domain domain;
@@ -65,6 +67,9 @@ struct aws_socket_options {
     uint16_t keep_alive_max_failed_probes;
     bool keepalive;
 
+    aws_socket_on_shutdown_complete_fn *on_shutdown_complete;
+    void *shutdown_user_data;
+
     /**
      * THIS IS AN EXPERIMENTAL AND UNSTABLE API
      * (Optional)
@@ -90,6 +95,15 @@ struct aws_event_loop;
  */
 typedef void(aws_socket_on_connection_result_fn)(struct aws_socket *socket, int error_code, void *user_data);
 
+/**
+ * Called by a listening socket when a listener accept has successfully initialized or an error has occurred.
+ * If the listener was successful error_code will be AWS_ERROR_SUCCESS and the socket has already been assigned
+ * to the event loop specified in aws_socket_start_accept().
+ *
+ * If an error occurred error_code will be non-zero.
+ */
+typedef void(aws_socket_on_accept_started_fn)(struct aws_socket *socket, int error_code, void *user_data);
+
 /**
  * Called by a listening socket when either an incoming connection has been received or an error occurred.
  *
@@ -116,14 +130,15 @@ typedef void(aws_socket_on_accept_result_fn)(
  * Callback for when the data passed to a call to aws_socket_write() has either completed or failed.
  * On success, error_code will be AWS_ERROR_SUCCESS.
  *
- * `socket` may be NULL in the callback if the socket is released and cleaned up before a callback is triggered.
- * by the system I/O handler,
+ * `socket` may be NULL in the callback if the socket is released and cleaned up before the callback is triggered.
  */
 typedef void(
     aws_socket_on_write_completed_fn)(struct aws_socket *socket, int error_code, size_t bytes_written, void *user_data);
 /**
  * Callback for when socket is either readable (edge-triggered) or when an error has occurred. If the socket is
  * readable, error_code will be AWS_ERROR_SUCCESS.
+ *
+ * `socket` may be NULL in the callback if the socket is released and cleaned up before the callback is triggered.
  */
 typedef void(aws_socket_on_readable_fn)(struct aws_socket *socket, int error_code, void *user_data);
 
@@ -156,6 +171,16 @@ struct aws_socket {
     void *impl;
 };
 
+struct aws_socket_listener_options {
+    aws_socket_on_accept_result_fn *on_accept_result;
+    void *on_accept_result_user_data;
+
+    // This callback is invoked when the listener starts accepting incoming connections.
+    // It is only triggered in asynchronous listener APIs while using nw_socket.
+    aws_socket_on_accept_started_fn *on_accept_start_result;
+    void *on_accept_start_user_data;
+};
+
 struct aws_byte_buf;
 struct aws_byte_cursor;
 
@@ -227,8 +252,7 @@ AWS_IO_API int aws_socket_listen(struct aws_socket *socket, int backlog_size);
 AWS_IO_API int aws_socket_start_accept(
     struct aws_socket *socket,
     struct aws_event_loop *accept_loop,
-    aws_socket_on_accept_result_fn *on_accept_result,
-    void *user_data);
+    struct aws_socket_listener_options options);
 
 /**
  * TCP, LOCAL and VSOCK only. The listening socket will stop accepting new connections.
@@ -239,12 +263,33 @@ AWS_IO_API int aws_socket_start_accept(
  */
 AWS_IO_API int aws_socket_stop_accept(struct aws_socket *socket);
 
+/**
+ * Apple Network Framework only. The callback that will triggered when aws_socket_close() finished. The callback
+ * will be called from the socket event loop.
+ */
+AWS_IO_API int aws_socket_set_close_complete_callback(
+    struct aws_socket *socket,
+    aws_socket_on_shutdown_complete_fn fn,
+    void *user_data);
+
+/**
+ * Apple Network Framework only. The callback that will triggered when aws_socket_cleanup() finished. And
+ * it is only safe to release the socket afterwards. The callback will be called from the socket event loop.
+ */
+AWS_IO_API int aws_socket_set_cleanup_complete_callback(
+    struct aws_socket *socket,
+    aws_socket_on_shutdown_complete_fn fn,
+    void *user_data);
+
 /**
  * Calls `close()` on the socket and unregisters all io operations from the event loop. This function must be called
  * from the event-loop's thread unless this is a listening socket. If it's a listening socket it can be called from any
  * non-event-loop thread or the event-loop the socket is currently assigned to. If called from outside the event-loop,
  * this function will block waiting on the socket to close. If this is called from an event-loop thread other than
  * the one it's assigned to, it presents the possibility of a deadlock, so don't do it.
+ *
+ * If you are using Apple Network Framework, you should always call this function from an event-loop thread regardless
+ * it is a server or client socket.
  */
 AWS_IO_API int aws_socket_close(struct aws_socket *socket);
 
@@ -358,6 +403,12 @@ AWS_IO_API void aws_socket_endpoint_init_local_address_for_test(struct aws_socke
  * network_interface_name on Windows */
 AWS_IO_API bool aws_is_network_interface_name_valid(const char *interface_name);
 
+/**
+ * Get default impl type based on the platform.
+ * For user in internal tests only
+ */
+AWS_IO_API enum aws_socket_impl_type aws_socket_get_default_impl_type(void);
+
 AWS_EXTERN_C_END
 AWS_POP_SANE_WARNING_LEVEL