Fix bug in our TLS socket creation around OOM. (#1832)

We paniced instead of reportint OOM for a malloc
failure at a bad moment.

src/resources/tls.cc

@@ -326,9 +326,7 @@ MbedTlsSocket::~MbedTlsSocket() {
 
 MODULE_IMPLEMENTATION(tls, MODULE_TLS)
 
-Object* tls_error(BaseMbedTlsSocket* socket, Process* process, int err) {
-  static const size_t BUFFER_LEN = 400;
-  char buffer[BUFFER_LEN];
+bool is_tls_malloc_failure(int err) {
   // For some reason Mbedtls doesn't seem to export this mask.
   static const int MBED_LOW_LEVEL_ERROR_MASK = 0x7f;
   // Error codes are negative so we use or-not instead of and.
@@ -343,8 +341,17 @@ Object* tls_error(BaseMbedTlsSocket* socket, Process* process, int err) {
       hi_error == MBEDTLS_ERR_PK_ALLOC_FAILED ||
       hi_error == MBEDTLS_ERR_SSL_ALLOC_FAILED ||
       hi_error == MBEDTLS_ERR_X509_ALLOC_FAILED) {
+    return true;
+  }
+  return false;
+}
+
+Object* tls_error(BaseMbedTlsSocket* socket, Process* process, int err) {
+  if (is_tls_malloc_failure(err)) {
     FAIL(MALLOC_FAILED);
   }
+  static const size_t BUFFER_LEN = 400;
+  char buffer[BUFFER_LEN];
   const char* issuer = socket ? socket->error_issuer() : null;
   if (err == MBEDTLS_ERR_X509_CERT_VERIFY_FAILED &&
       socket &&
@@ -714,8 +721,8 @@ PRIMITIVE(error) {
 
 bool MbedTlsSocket::init() {
   if (int ret = mbedtls_ssl_setup(&ssl, &conf_)) {
-    if (ret == MBEDTLS_ERR_SSL_ALLOC_FAILED) return false;
-    FATAL("mbedtls_ssl_setup returned %d (not %d)", ret, MBEDTLS_ERR_SSL_ALLOC_FAILED);
+    if (is_tls_malloc_failure(ret)) return false;
+    FATAL("mbedtls_ssl_setup returned %x", ret);
   }
 
   mbedtls_ssl_set_bio(&ssl, this, toit_tls_send, toit_tls_recv, null);

src/resources/tls.h

@@ -55,6 +55,8 @@ enum TLS_STATE {
   TLS_SENT_HELLO_VERIFY = 1 << 3,
 };
 
+bool is_tls_malloc_failure(int err);
+
 // Common base for TLS (stream based) and in the future perhaps DTLS (datagram based) sockets.
 class BaseMbedTlsSocket : public TlsSocket {
  public: