From 96e8ea12455d6a6c31c9be6bbbe0dfcd76c79b57 Mon Sep 17 00:00:00 2001
From: Artem Boldariev <artem@boldariev.com>
Date: Thu, 13 Feb 2025 14:53:18 +0200
Subject: [PATCH] DoH: Track the amount of in flight outgoing data

Previously we would limit the amount of incoming data to process based
solely on the presence of not completed send requests. That worked,
however, it was found to severely degrade performance in certain
cases, as was revealed during extended testing.

Now we switch to keeping track of how much data is in flight (or ready
to be in flight) and limit the amount of processed incoming data when
the amount of in flight data surpasses the given threshold, similarly
to like we do in other transports.

(cherry picked from commit 05e8a508188116e8e367aaf5e68575c8cebb4207)
---
 lib/isc/netmgr/http.c | 27 +++++++++++++++++++++++----
 1 file changed, 23 insertions(+), 4 deletions(-)

diff --git a/lib/isc/netmgr/http.c b/lib/isc/netmgr/http.c
index 9a1249feba..ced80dd9ae 100644
--- a/lib/isc/netmgr/http.c
+++ b/lib/isc/netmgr/http.c
@@ -188,6 +188,8 @@ struct isc_nm_http_session {
 	isc__nm_http_pending_callbacks_t pending_write_callbacks;
 	isc_buffer_t *pending_write_data;
 
+	size_t data_in_flight;
+
 	/*
 	 * The statistical values below are for usage on server-side
 	 * only. They are meant to detect clients that are taking too many
@@ -1054,6 +1056,19 @@ client_submit_request(isc_nm_http_session_t *session, http_cstream_t *stream) {
 	return ISC_R_SUCCESS;
 }
 
+static inline size_t
+http_in_flight_data_size(isc_nm_http_session_t *session) {
+	size_t in_flight = 0;
+
+	if (session->pending_write_data != NULL) {
+		in_flight += isc_buffer_usedlength(session->pending_write_data);
+	}
+
+	in_flight += session->data_in_flight;
+
+	return in_flight;
+}
+
 static ssize_t
 http_process_input_data(isc_nm_http_session_t *session,
 			isc_buffer_t *input_data) {
@@ -1105,13 +1120,14 @@ http_process_input_data(isc_nm_http_session_t *session,
 			(session->received - session->processed);
 
 		/*
-		 * If there are non completed send requests in flight -let's
-		 * not process any incoming data, as it could lead to piling
-		 * up too much send data in send buffers. With many clients
+		 * If there is too much outgoing data in flight - let's not
+		 * process any incoming data, as it could lead to piling up
+		 * too much send data in send buffers. With many clients
 		 * connected it can lead to excessive memory consumption on
 		 * the server instance.
 		 */
-		if (session->sending > 0) {
+		const size_t in_flight = http_in_flight_data_size(session);
+		if (in_flight >= ISC_NETMGR_TCP_SENDBUF_SIZE) {
 			break;
 		}
 
@@ -1320,6 +1336,8 @@ http_writecb(isc_nmhandle_t *handle, isc_result_t result, void *arg) {
 		isc_nmhandle_detach(&req->httphandle);
 	}
 
+	session->data_in_flight -=
+		isc_buffer_usedlength(req->pending_write_data);
 	isc_buffer_free(&req->pending_write_data);
 	session->processed += req->submitted;
 	isc_mem_put(session->mctx, req, sizeof(*req));
@@ -1509,6 +1527,7 @@ http_send_outgoing(isc_nm_http_session_t *session, isc_nmhandle_t *httphandle,
 
 	session->sending++;
 	isc_buffer_usedregion(send->pending_write_data, &send_data);
+	session->data_in_flight += send_data.length;
 	isc_nm_send(transphandle, &send_data, http_writecb, send);
 	return true;