From f36e118b9a5750fef886a4ca179740b99e97821e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ond=C5=99ej=20Sur=C3=BD?= <ondrej@isc.org>
Date: Wed, 16 Aug 2023 16:30:53 +0200
Subject: [PATCH] Limit the number of inactive handles kept for reuse

Instead of growing and never shrinking the list of the inactive
handles (to be reused mostly on the UDP connections), limit the number
of maximum number of inactive handles kept to 64.  Instead of caching
the inactive handles for all listening sockets, enable the caching on on
UDP listening sockets.  For TCP, the handles were cached for each
accepted socket thus reusing the handles only for long-standing TCP
connections, but not reusing the handles across different TCP streams.
---
 lib/isc/netmgr/netmgr-int.h | 11 +++++++++++
 lib/isc/netmgr/netmgr.c     |  7 ++++++-
 lib/isc/netmgr/udp.c        |  1 +
 3 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/lib/isc/netmgr/netmgr-int.h b/lib/isc/netmgr/netmgr-int.h
index e3931d71fb..e525814f3d 100644
--- a/lib/isc/netmgr/netmgr-int.h
+++ b/lib/isc/netmgr/netmgr-int.h
@@ -104,6 +104,13 @@ STATIC_ASSERT(ISC_NETMGR_TCP_RECVBUF_SIZE <= ISC_NETMGR_RECVBUF_SIZE,
  */
 #define NM_MAXSEG (1280 - 20 - 40)
 
+/*%
+ * How many isc_nmhandles and isc_nm_uvreqs will we be
+ * caching for reuse in a socket.
+ */
+#define ISC_NM_NMHANDLES_MAX 64
+#define ISC_NM_UVREQS_MAX    64
+
 /*
  * Define ISC_NETMGR_TRACE to activate tracing of handles and sockets.
  * This will impair performance but enables us to quickly determine,
@@ -245,6 +252,7 @@ struct isc_nmhandle {
 #endif
 	LINK(isc_nmhandle_t) active_link;
 	LINK(isc_nmhandle_t) inactive_link;
+
 	void *opaque;
 
 	isc_job_t job;
@@ -624,6 +632,9 @@ struct isc_nmsocket {
 	 */
 	ISC_LIST(isc_nmhandle_t) inactive_handles;
 
+	size_t inactive_handles_cur;
+	size_t inactive_handles_max;
+
 	/*%
 	 * 'active' handles and uvreqs, mostly for debugging purposes.
 	 */
diff --git a/lib/isc/netmgr/netmgr.c b/lib/isc/netmgr/netmgr.c
index 0f1ab466a7..d4f2c2c722 100644
--- a/lib/isc/netmgr/netmgr.c
+++ b/lib/isc/netmgr/netmgr.c
@@ -806,6 +806,8 @@ dequeue_handle(isc_nmsocket_t *sock) {
 	if (handle != NULL) {
 		ISC_LIST_DEQUEUE(sock->inactive_handles, handle, inactive_link);
 
+		sock->inactive_handles_cur--;
+
 		isc_refcount_init(&handle->references, 1);
 		INSIST(VALID_NMHANDLE(handle));
 		return (handle);
@@ -913,7 +915,10 @@ nmhandle__destroy(isc_nmhandle_t *handle) {
 #if defined(__SANITIZE_ADDRESS__) || defined(__SANITIZE_THREAD__)
 	nmhandle_free(sock, handle);
 #else
-	if (sock->active) {
+	if (sock->active &&
+	    sock->inactive_handles_cur < sock->inactive_handles_max)
+	{
+		sock->inactive_handles_cur++;
 		ISC_LIST_APPEND(sock->inactive_handles, handle, inactive_link);
 	} else {
 		nmhandle_free(sock, handle);
diff --git a/lib/isc/netmgr/udp.c b/lib/isc/netmgr/udp.c
index 14558900f4..56ea1e5fa8 100644
--- a/lib/isc/netmgr/udp.c
+++ b/lib/isc/netmgr/udp.c
@@ -188,6 +188,7 @@ start_udp_child(isc_nm_t *mgr, isc_sockaddr_t *iface, isc_nmsocket_t *sock,
 	isc__nmsocket_init(csock, worker, isc_nm_udpsocket, iface, sock);
 	csock->recv_cb = sock->recv_cb;
 	csock->recv_cbarg = sock->recv_cbarg;
+	csock->inactive_handles_max = ISC_NM_NMHANDLES_MAX;
 
 	if (mgr->load_balance_sockets) {
 		csock->fd = isc__nm_udp_lb_socket(mgr,