Use different allocators for UDP and TCP

Each worker has a receive buffer with space for 20 DNS messages of up
to 2^16 bytes each, and the allocator function passed to uv_read_start()
or uv_udp_recv_start() will reserve a portion of it for use by sockets.
UDP can use recvmmsg() and so it needs that entire space, but TCP reads
one message at a time.

This commit introduces separate allocator functions for TCP and UDP
setting different buffer size limits, so that libuv will provide the
correct buffer sizes to each of them.

lib/isc/netmgr/netmgr-int.h

@@ -559,16 +559,6 @@ isc__nm_enqueue_ievent(isc__networker_t *worker, isc__netievent_t *event);
  * way to use an isc__networker_t from another thread.)
  */
 
-void
-isc__nm_alloc_cb(uv_handle_t *handle, size_t size, uv_buf_t *buf);
-/*%<
- * Allocator for recv operations.
- *
- * Note that as currently implemented, this doesn't actually
- * allocate anything, it just assigns the the isc__networker's UDP
- * receive buffer to a socket, and marks it as "in use".
- */
-
 void
 isc__nm_free_uvbuf(isc_nmsocket_t *sock, const uv_buf_t *buf);
 /*%<

lib/isc/netmgr/netmgr.c

@@ -1003,23 +1003,6 @@ isc__nmsocket_clearcb(isc_nmsocket_t *sock) {
 	sock->accept_cbarg = NULL;
 }
 
-void
-isc__nm_alloc_cb(uv_handle_t *handle, size_t size, uv_buf_t *buf) {
-	isc_nmsocket_t *sock = uv_handle_get_data(handle);
-	isc__networker_t *worker = NULL;
-
-	REQUIRE(VALID_NMSOCK(sock));
-	REQUIRE(isc__nm_in_netthread());
-	REQUIRE(size <= ISC_NETMGR_RECVBUF_SIZE);
-
-	worker = &sock->mgr->workers[sock->tid];
-	INSIST(!worker->recvbuf_inuse);
-
-	buf->base = worker->recvbuf;
-	worker->recvbuf_inuse = true;
-	buf->len = ISC_NETMGR_RECVBUF_SIZE;
-}
-
 void
 isc__nm_free_uvbuf(isc_nmsocket_t *sock, const uv_buf_t *buf) {
 	isc__networker_t *worker = NULL;
@@ -1032,7 +1015,7 @@ isc__nm_free_uvbuf(isc_nmsocket_t *sock, const uv_buf_t *buf) {
 	worker = &sock->mgr->workers[sock->tid];
 
 	REQUIRE(worker->recvbuf_inuse);
-	if (buf->base > worker->recvbuf &&
+	if (sock->type == isc_nm_udpsocket && buf->base > worker->recvbuf &&
 	    buf->base <= worker->recvbuf + ISC_NETMGR_RECVBUF_SIZE)
 	{
 		/* Can happen in case of out-of-order recvmmsg in libuv1.36 */

lib/isc/netmgr/tcp.c

@@ -637,6 +637,30 @@ isc__nm_tcp_read(isc_nmhandle_t *handle, isc_nm_recv_cb_t cb, void *cbarg) {
 	return (ISC_R_SUCCESS);
 }
 
+/*%<
+ * Allocator for TCP read operations. Limited to size 2^16.
+ *
+ * Note this doesn't actually allocate anything, it just assigns the
+ * worker's receive buffer to a socket, and marks it as "in use".
+ */
+static void
+tcp_alloc_cb(uv_handle_t *handle, size_t size, uv_buf_t *buf) {
+	isc_nmsocket_t *sock = uv_handle_get_data(handle);
+	isc__networker_t *worker = NULL;
+
+	REQUIRE(VALID_NMSOCK(sock));
+	REQUIRE(sock->type == isc_nm_tcpsocket);
+	REQUIRE(isc__nm_in_netthread());
+	REQUIRE(size <= 65536);
+
+	worker = &sock->mgr->workers[sock->tid];
+	INSIST(!worker->recvbuf_inuse);
+
+	buf->base = worker->recvbuf;
+	buf->len = size;
+	worker->recvbuf_inuse = true;
+}
+
 void
 isc__nm_async_tcp_startread(isc__networker_t *worker, isc__netievent_t *ev0) {
 	isc__netievent_startread_t *ievent = (isc__netievent_startread_t *)ev0;
@@ -654,7 +678,7 @@ isc__nm_async_tcp_startread(isc__networker_t *worker, isc__netievent_t *ev0) {
 			       0);
 	}
 
-	r = uv_read_start(&sock->uv_handle.stream, isc__nm_alloc_cb, read_cb);
+	r = uv_read_start(&sock->uv_handle.stream, tcp_alloc_cb, read_cb);
 	if (r != 0) {
 		isc__nm_incstats(sock->mgr, sock->statsindex[STATID_RECVFAIL]);
 	}

lib/isc/netmgr/udp.c

@@ -132,6 +132,32 @@ isc_nm_listenudp(isc_nm_t *mgr, isc_nmiface_t *iface, isc_nm_recv_cb_t cb,
 	return (ISC_R_SUCCESS);
 }
 
+/*%<
+ * Allocator for UDP recv operations. Limited to size 20 * (2^16 + 2),
+ * which allows enough space for recvmmsg() to get multiple messages at
+ * a time.
+ *
+ * Note this doesn't actually allocate anything, it just assigns the
+ * worker's receive buffer to a socket, and marks it as "in use".
+ */
+static void
+udp_alloc_cb(uv_handle_t *handle, size_t size, uv_buf_t *buf) {
+	isc_nmsocket_t *sock = uv_handle_get_data(handle);
+	isc__networker_t *worker = NULL;
+
+	REQUIRE(VALID_NMSOCK(sock));
+	REQUIRE(sock->type == isc_nm_udpsocket);
+	REQUIRE(isc__nm_in_netthread());
+	REQUIRE(size <= ISC_NETMGR_RECVBUF_SIZE);
+
+	worker = &sock->mgr->workers[sock->tid];
+	INSIST(!worker->recvbuf_inuse);
+
+	buf->base = worker->recvbuf;
+	buf->len = ISC_NETMGR_RECVBUF_SIZE;
+	worker->recvbuf_inuse = true;
+}
+
 /*
  * handle 'udplisten' async call - start listening on a socket.
  */
@@ -193,7 +219,7 @@ isc__nm_async_udplisten(isc__networker_t *worker, isc__netievent_t *ev0) {
 	uv_send_buffer_size(&sock->uv_handle.handle,
 			    &(int){ ISC_SEND_BUFFER_SIZE });
 #endif
-	uv_udp_recv_start(&sock->uv_handle.udp, isc__nm_alloc_cb, udp_recv_cb);
+	uv_udp_recv_start(&sock->uv_handle.udp, udp_alloc_cb, udp_recv_cb);
 }
 
 static void