[PATCH v5 http-server 2/2] apiserver: flush queued data before closing a proxied connection

[Kefu Chai <k.chai@proxmox.com>]

When one side of a proxied WebSocket or SPICE connection closes, the
proxy disconnects right away and drops the data still queued for the
other side. client_do_disconnect() shuts the socket down before its
wbuf is written, and the client handle has linger set to 0. With the
backpressure limit this is up to 640 KB at the end of a transfer.

On EOF from either side, stop reading from the other side and
disconnect from its on_drain callback once the queued data is written.
Also clear the on_drain callback of the closed side, so a paused reader
of the other side is not resumed. When waiting for the client, the 60
second write timeout applies, so a client that stops reading does not
keep the connection open. When waiting for the backend, there is no
timeout, for the same reason as in apply_read_backpressure().

Signed-off-by: Kefu Chai <k.chai@proxmox.com>
---
 src/PVE/APIServer/AnyEvent.pm | 41 +++++++++++++++++++++++++++++++++--
 1 file changed, 39 insertions(+), 2 deletions(-)

diff --git a/src/PVE/APIServer/AnyEvent.pm b/src/PVE/APIServer/AnyEvent.pm
index 826e1ff..c8266df 100644
--- a/src/PVE/APIServer/AnyEvent.pm
+++ b/src/PVE/APIServer/AnyEvent.pm
@@ -259,6 +259,31 @@ sub abort_request {
     if (my $err = $@) { syslog('err', $err); }
 }
 
+# on_eof body for proxied connections: stop forwarding, and disconnect once
+# $peer_hdl has written out what is still queued for it.
+sub disconnect_after_drain {
+    my ($self, $reqstate, $hdl, $peer_hdl, $timeout) = @_;
+
+    if (!$peer_hdl) {
+        $self->abort_request($reqstate);
+        return;
+    }
+
+    # drop a paused $peer_hdl reader, so it is not resumed
+    $hdl->on_drain(undef);
+    $hdl->wtimeout(0);
+    $peer_hdl->on_read();
+    $peer_hdl->stop_read();
+
+    # only pass $timeout if $peer_hdl is the client, see apply_read_backpressure
+    if ($timeout) {
+        $peer_hdl->wtimeout_reset();
+        $peer_hdl->wtimeout($timeout);
+    }
+    # invoked right away if nothing is queued
+    $peer_hdl->on_drain(sub { $self->abort_request($reqstate) });
+}
+
 sub response_stream {
     my ($self, $reqstate, $stream_fh) = @_;
 
@@ -620,7 +645,9 @@ sub websocket_proxy {
                 timeout => 5,
                 on_eof => sub {
                     my ($hdl) = @_;
-                    $self->abort_request($reqstate);
+                    $self->disconnect_after_drain(
+                        $reqstate, $hdl, $reqstate->{hdl}, $backpressure_stall_timeout,
+                    );
                 },
                 on_error => sub {
                     my ($hdl, $fatal, $message) = @_;
@@ -728,6 +755,10 @@ sub websocket_proxy {
             $reqstate->{proxyhdl}->timeout(0);
             $reqstate->{proxyhdl}->on_read($proxyhdlreader);
             $reqstate->{hdl}->on_read($hdlreader);
+            $reqstate->{hdl}->on_eof(sub {
+                my ($hdl) = @_;
+                $self->disconnect_after_drain($reqstate, $hdl, $reqstate->{proxyhdl});
+            });
 
             # FIXME: remove protocol in PVE/PMG 8.x
             #
@@ -1141,7 +1172,9 @@ sub handle_spice_proxy_request {
                 timeout => 5,
                 on_eof => sub {
                     my ($hdl) = @_;
-                    $self->abort_request($reqstate);
+                    $self->disconnect_after_drain(
+                        $reqstate, $hdl, $reqstate->{hdl}, $backpressure_stall_timeout,
+                    );
                 },
                 on_error => sub {
                     my ($hdl, $fatal, $message) = @_;
@@ -1179,6 +1212,10 @@ sub handle_spice_proxy_request {
                 $reqstate->{proxyhdl}->timeout(0);
                 $reqstate->{proxyhdl}->on_read($proxyhdlreader);
                 $reqstate->{hdl}->on_read($hdlreader);
+                $reqstate->{hdl}->on_eof(sub {
+                    my ($hdl) = @_;
+                    $self->disconnect_after_drain($reqstate, $hdl, $reqstate->{proxyhdl});
+                });
 
                 # a response must be followed by an empty line
                 my $res = "$proto 200 OK\015\012\015\012";
-- 
2.47.3