Merge pull request #1008 from sozu-proxy/devel/fdubois/fix/rustls

fix(rustls): read buffer if we received a buffer full error instead of processing new packets
sozu-proxy · Oct 21, 2023 · 983cda1 · 983cda1
2 parents 7d3662c + 437eb12
commit 983cda1
Show file tree

Hide file tree

Showing 3 changed files with 22 additions and 7 deletions.
diff --git a/lib/src/protocol/kawa_h1/mod.rs b/lib/src/protocol/kawa_h1/mod.rs
@@ -1435,8 +1435,9 @@ impl<Front: SocketHandler, L: ListenerHandler + L7ListenerHandler> Http<Front, L
             if self.backend_readiness.event.is_hup() && !self.test_backend_socket() {
                 //retry connecting the backend
                 error!(
-                    "{} error connecting to backend, trying again",
-                    self.log_context()
+                    "{} error connecting to backend, trying again, attempt {}",
+                    self.log_context(),
+                    self.connection_attempts
                 );
 
                 self.connection_attempts += 1;

diff --git a/lib/src/protocol/pipe.rs b/lib/src/protocol/pipe.rs
@@ -221,6 +221,8 @@ impl<Front: SocketHandler, L: ListenerHandler> Pipe<Front, L> {
 
     pub fn log_request_error(&self, metrics: &SessionMetrics, message: &str) {
         incr!("pipe.errors");
+        warn!("could not process request properly got: {}", message);
+        self.print_state(&self.log_context().to_string());
         self.log_request(metrics, Some(message));
     }
 

diff --git a/lib/src/socket.rs b/lib/src/socket.rs
@@ -181,6 +181,7 @@ impl SocketHandler for FrontRustls {
                 break;
             }
 
+            let mut is_rustls_backpressuring = false;
             match self.session.read_tls(&mut self.stream) {
                 Ok(0) => {
                     can_read = false;
@@ -196,6 +197,12 @@ impl SocketHandler for FrontRustls {
                     | ErrorKind::BrokenPipe => {
                         is_closed = true;
                     }
+                    // According to rustls comment here https://github.com/rustls/rustls/blob/main/rustls/src/conn.rs#L482-L500,
+                    // [`ErrorKind::Other`] error signal that the buffer is full, we need to read it before processing new packets.
+                    ErrorKind::Other => {
+                        warn!("rustls buffer is full, we will consume it, before processing new incoming packets, to mitigate this issue, you could try to increase the buffer size, {:?}", e);
+                        is_rustls_backpressuring = true;
+                    }
                     _ => {
                         error!("could not read TLS stream from socket: {:?}", e);
                         is_error = true;
@@ -204,16 +211,21 @@ impl SocketHandler for FrontRustls {
                 },
             }
 
-            if let Err(e) = self.session.process_new_packets() {
-                error!("could not process read TLS packets: {:?}", e);
-                is_error = true;
-                break;
+            if !is_rustls_backpressuring {
+                if let Err(e) = self.session.process_new_packets() {
+                    error!("could not process read TLS packets: {:?}", e);
+                    is_error = true;
+                    break;
+                }
             }
 
             while !self.session.wants_read() {
                 match self.session.reader().read(&mut buf[size..]) {
                     Ok(0) => break,
-                    Ok(sz) => size += sz,
+                    Ok(sz) => {
+                        size += sz;
+                        can_read = true;
+                    },
                     Err(e) => match e.kind() {
                         ErrorKind::WouldBlock => {
                             break;