full_posix_sockets (#7672)

* Add posix sockets proxying option.

* Add test for emscripten_websocket_get_url_length().

* Update docs on which sockets functions are proxied

* Add sockets tests for chrome

* Install websockets before sockets test run

* Install node before ws

* Try to fix ws installation

* Fix websocket send test

* Fix bad _strlen dep in _formatString function.

* Add formatString dep

.circleci/config.yml

@@ -213,7 +213,7 @@ commands:
             openbox --exit
             wait || true # wait for startx to shutdown cleanly, or not
   test-chrome:
-    description: "Runs emscripten tests under firefox"
+    description: "Runs emscripten browser tests under chrome"
     steps:
       - checkout
       - attach_workspace:
@@ -238,6 +238,39 @@ commands:
           command: |
             export EMTEST_BROWSER="/usr/bin/google-chrome $CHROME_FLAGS_BASE $CHROME_FLAGS_HEADLESS $CHROME_FLAGS_WASM $CHROME_FLAGS_NOCACHE"
             python3 tests/runner.py browser
+  test-sockets-chrome:
+    description: "Runs emscripten sockets tests under chrome"
+    steps:
+      - checkout
+      - attach_workspace:
+          # Must be absolute path or relative path from working_directory
+          at: ~/
+      - run:
+          name: download chrome
+          command: |
+            wget -O ~/chrome.deb https://dl.google.com/linux/direct/google-chrome-beta_current_amd64.deb
+            dpkg -i ~/chrome.deb
+      - run:
+          name: install sockets test suite prerequisites
+          command: |
+            wget https://nodejs.org/dist/v12.13.0/node-v12.13.0-linux-x64.tar.xz
+            tar -xf node-v12.13.0-linux-x64.tar.xz
+            export PATH="`pwd`/node-v12.13.0-linux-x64/bin:${PATH}"
+            npm install ws
+      - run:
+          name: run sockets tests
+          environment:
+            EMTEST_LACKS_SOUND_HARDWARE: "1"
+            EMTEST_DETECT_TEMPFILE_LEAKS: "0"
+            # --no-sandbox becasue we are running as root and chrome requires
+            # this flag for now: https://crbug.com/638180
+            CHROME_FLAGS_BASE: "--no-first-run -start-maximized --no-sandbox --use-gl=swiftshader --user-data-dir=/tmp/chrome-emscripten-profile"
+            CHROME_FLAGS_HEADLESS: "--headless --remote-debugging-port=1234"
+            CHROME_FLAGS_WASM: "--enable-features=WebAssembly --enable-features=SharedArrayBuffer --disable-features=WebAssemblyTrapHandler --js-flags=\"--experimental-wasm-threads --harmony-sharedarraybuffer --no-wasm-disable-structured-cloning\""
+            CHROME_FLAGS_NOCACHE: "--disk-cache-dir=/dev/null --disk-cache-size=1 --media-cache-size=1 --disable-application-cache --incognito"
+          command: |
+            export EMTEST_BROWSER="/usr/bin/google-chrome $CHROME_FLAGS_BASE $CHROME_FLAGS_HEADLESS $CHROME_FLAGS_WASM $CHROME_FLAGS_NOCACHE"
+            python tests/runner.py sockets
 
 jobs:
   build:
@@ -309,6 +342,10 @@ jobs:
     executor: bionic
     steps:
       - test-chrome
+  test-sockets-chrome:
+    executor: bionic
+    steps:
+      - test-sockets-chrome
   test-ab:
     executor: bionic
     steps:
@@ -476,6 +513,9 @@ workflows:
       - test-browser-chrome:
           requires:
             - build
+      - test-sockets-chrome:
+          requires:
+            - build
       - test-ab:
           requires:
             - build

embuilder.py

@@ -242,6 +242,20 @@ def main():
       build_port('regal', libname('libregal'), ['-s', 'USE_REGAL=1'])
     elif what == 'boost_headers':
       build_port('boost_headers', libname('libboost_headers'), ['-s', 'USE_BOOST_HEADERS=1'])
+    elif what == 'libsockets':
+      build('''
+        #include <sys/socket.h>
+        int main() {
+          return socket(0,0,0);
+        }
+      ''', [libname('libsockets')])
+    elif what == 'libsockets_proxy':
+      build('''
+        #include <sys/socket.h>
+        int main() {
+          return socket(0,0,0);
+        }
+      ''', [libname('libsockets_proxy')], ['-s', 'PROXY_POSIX_SOCKETS=1', '-s', 'USE_PTHREADS=1', '-s', 'PROXY_TO_PTHREAD=1'])
     else:
       logger.error('unfamiliar build target: ' + what)
       return 1

site/source/docs/porting/index.rst

@@ -17,6 +17,7 @@ The topics in this section cover the main integration points that you need to co
   Audio
   Debugging
   pthreads
+  networking
   simd
   asyncify
   emterpreter

site/source/docs/porting/networking.rst

@@ -0,0 +1,50 @@
+.. Networking:
+
+==============================
+Networking
+==============================
+
+Emscripten compiled applications have a number of ways to connect with online servers. Check the subtopics here to learn about the different strategies that are available.
+
+If you are familiar with networking concepts provided by different web APIs, such as XmlHttpRequest, Fetch, WebSockets and WebRTC, you can quickly get started by leveraging what you already know: by calling out from C/C++ code to JavaScript (see the "Connecting C++ and JavaScript" section), you can establish networked connections by writing regular JavaScript. For C/C++ developers, Emscripten provides a few approaches, described here.
+
+Emscripten WebSockets API
+=========================
+
+WebSockets API provides connection-oriented message-framed bidirectional asynchronous networking communication to the browser. It is the closest to TCP on the web that web sites can access, direct access to TCP sockets is not possible from web browsers.
+
+Emscripten provides a passthrough API for accessing the WebSockets API from C/C++ code. This is useful for developers who would prefer not to write any JavaScript code, or deal with the C/C++ and JavaScript language interop. See the system include file <emscripten/websocket.h> for details. One benefit that the Emscripten WebSockets API provides over manual WebSockets access in JavaScript is the ability to share access to a WebSocket handle across multiple threads, something that can be time consuming to develop from scratch.
+
+To target Emscripten WebSockets API, you must link it in with a "-lwebsocket.js" linker directive.
+
+Emulated POSIX TCP Sockets over WebSockets
+==========================================
+
+If you have existing TCP networking code written in C/C++ that utilizes the Posix Sockets API, by default Emscripten attempts to emulate such connections to take place over the WebSocket protocol instead. For this to work, you will need to use something like WebSockify on the server side to enable the TCP server stack to receive incoming WebSocket connections. This emulation is not very complete at the moment, it is likely that you will run into problems out of the box and need to adapt the code to work within the limitations that this emulation provides.
+
+This is the default build mode for POSIX sockets, no linker flags or option settings are needed to enable it.
+
+Full POSIX Sockets over WebSocket Proxy Server
+==============================================
+
+Emscripten provides a native POSIX Sockets proxy server program, located in directory tools/websocket_to_posix_proxy/, that allows full POSIX Sockets API access from a web browser. This support works by proxying all POSIX Sockets API calls from the browser to the Emscripten POSIX Sockets proxy server (via transparent use of WebSockets API), and the proxy server then performs the native TCP/UDP calls on behalf of the page. This allows a web browser page to run full TCP & UDP connections, act as a server to accept incoming connections, and perform host name lookups and reverse lookups. Because all API calls are individually proxied, this support can be slow. This support is mostly useful for developing testing infrastructure and debugging.
+
+The following POSIX sockets functions are proxied in this manner:
+ - `socket()`, `socketpair()`, `shutdown()`, `bind()`, `connect()`, `listen()`, `accept()`, `getsockname()`, `getpeername()`, `send()`, `recv()`, `sendto()`, `recvfrom()`, `sendmsg()`, `recvmsg()`, `getsockopt()`, `setsockopt()`, `getaddrinfo(), `getnameinfo()`.
+
+The following POSIX sockets functions are currently not proxied (and will not work):
+ - `poll()`, `close()` (use `shutdown()` instead), `select()`
+
+To use POSIX sockets proxying, link the application with flags "-lwebsocket.js -s PROXY_POSIX_SOCKETS=1 -s USE_PTHREADS=1 -s PROXY_TO_PTHREAD=1". That is, POSIX sockets proxying builds on top of the Emscripten WebSockets library, and requires multithreading and proxying the application main() to a pthread.
+
+For an example of how the POSIX Sockets proxy server works in an Emscripten client program, see the file tests/websocket/tcp_echo_client.cpp.
+
+XmlHttpRequests and Fetch API
+=============================
+
+For HTTP transfers, one can use the browser built-in XmlHttpRequest (XHR) API and the newer Fetch API. These can be accessed directly from JavaScript. Emscripten also provides passthrough APIs to perform HTTP requests. For more information, see the emscripten_async_wget*() C API and the Emscripten Fetch API.
+
+WebRTC and UDP
+==============
+
+Direct UDP communication is not available in browsers, but as a close alternative, the WebRTC specification provides a mechanism to perform UDP-like communication with WebRTC Data Channels. Currently Emscripten does not provide a C/C++ API for interacting with WebRTC.

src/deps_info.json

@@ -68,7 +68,7 @@
   "__cxa_end_catch": ["free"],
   "__cxa_allocate_exception": ["malloc"],
   "__cxa_free_exception": ["free"],
-  "emscripten_log": ["strlen"],
+  "_formatString": ["strlen"],
   "setjmp": ["realloc"],
   "saveSetjmp": ["realloc"],
   "glfwSleep": ["sleep"],

src/library.js

@@ -3213,6 +3213,7 @@ LibraryManager.library = {
   // arpa/inet.h
   // ==========================================================================
 
+#if PROXY_POSIX_SOCKETS == 0
   // old ipv4 only functions
   inet_addr__deps: ['_inet_pton4_raw'],
   inet_addr: function(ptr) {
@@ -3950,6 +3951,8 @@ LibraryManager.library = {
                0x0b00000a, 0x0c00000a, 0x0d00000a, 0x0e00000a] /* 0x0100000a is reserved */
   },
 
+#endif // PROXY_POSIX_SOCKETS == 0
+
   // pwd.h
 
   getpwnam: function() { throw 'getpwnam: TODO' },

src/library_syscall.js

@@ -512,6 +512,7 @@ var SyscallsLibrary = {
   __syscall97: function(which, varargs) { // setpriority
     return -{{{ cDefine('EPERM') }}};
   },
+#if PROXY_POSIX_SOCKETS == 0
   __syscall102__deps: ['$SOCKFS', '$DNS', '_read_sockaddr', '_write_sockaddr'],
   __syscall102: function(which, varargs) { // socketcall
     var call = SYSCALLS.get(), socketvararg = SYSCALLS.get();
@@ -728,6 +729,7 @@ var SyscallsLibrary = {
       }
     }
   },
+#endif // ~PROXY_POSIX_SOCKETS==0
   __syscall104: function(which, varargs) { // setitimer
     return -{{{ cDefine('ENOSYS') }}}; // unsupported feature
   },
@@ -1393,9 +1395,17 @@ var SyscallsLibrary = {
     var stream = SYSCALLS.getStreamFromFD(fd);
     FS.close(stream);
 #else
+#if PROXY_POSIX_SOCKETS
+    // close() is a tricky function because it can be used to close both regular file descriptors
+    // and POSIX network socket handles, hence an implementation would need to track for each
+    // file descriptor which kind of item it is. To simplify, when using PROXY_POSIX_SOCKETS
+    // option, use shutdown() to close a socket, and this function should behave like a no-op.
+    warnOnce('To close sockets with PROXY_POSIX_SOCKETS bridge, prefer to use the function shutdown() that is proxied, instead of close()')
+#else
 #if ASSERTIONS
     abort('it should not be possible to operate on streams when !SYSCALLS_REQUIRE_FILESYSTEM');
 #endif
+#endif
 #endif
     return 0;
   },

src/settings.js

@@ -360,6 +360,10 @@ var SOCKET_WEBRTC = 0;
 // where addr and port are derived from the socket connect/bind/accept calls.
 var WEBSOCKET_URL = 'ws://';
 
+// If 1, the POSIX sockets API uses a native bridge process server to proxy sockets calls
+// from browser to native world.
+var PROXY_POSIX_SOCKETS = 0;
+
 // A string containing a comma separated list of WebSocket subprotocols
 // as would be present in the Sec-WebSocket-Protocol header.
 // You can set 'null', if you don't want to specify it.

system/include/emscripten/posix_socket.h

@@ -0,0 +1,13 @@
+#pragma once
+
+#include "websocket.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern EMSCRIPTEN_RESULT emscripten_init_websocket_to_posix_socket_bridge(const char *bridgeUrl);
+
+#ifdef __cplusplus
+}
+#endif

system/lib/libc-sockets.symbols

@@ -0,0 +1,23 @@
+         U __errno_location
+         U __syscall102
+         U __syscall221
+         U __syscall_ret
+         U free
+-------- T accept
+-------- T bind
+-------- T connect
+-------- T freeaddrinfo
+-------- T getpeername
+-------- T getsockname
+-------- T getsockopt
+-------- T listen
+-------- T recv
+-------- T recvfrom
+-------- T recvmsg
+-------- T send
+-------- T sendmsg
+-------- T sendto
+-------- T setsockopt
+-------- T shutdown
+-------- T socket
+-------- T socketpair