TLS code refactoring, fixes and unit-tests

This commit fixes numerous stability issues with TLS transport code as
well as adds unit tests for it.

lib/isc/netmgr/netmgr-int.h

@@ -816,23 +816,22 @@ struct isc_nmsocket {
 	/*% TLS stuff */
 	struct tlsstream {
 		bool server;
-		BIO *app_bio;
+		BIO *bio_in;
+		BIO *bio_out;
 		isc_tls_t *tls;
 		isc_tlsctx_t *ctx;
-		BIO *ssl_bio;
 		isc_nmsocket_t *tlslistener;
 		isc_nmiface_t server_iface;
 		isc_nmiface_t local_iface;
-		bool connect_from_networker;
 		atomic_bool result_updated;
 		enum {
 			TLS_INIT,
 			TLS_HANDSHAKE,
 			TLS_IO,
-			TLS_CLOSING,
 			TLS_CLOSED
 		} state; /*%< The order of these is significant */
 		size_t nsending;
+		bool reading;
 	} tlsstream;
 
 	isc_nmsocket_h2_t h2;

lib/isc/netmgr/tlsstream.c

@@ -34,10 +34,11 @@
 #include <isc/thread.h>
 #include <isc/util.h>
 
+#include "../openssl_shim.h"
 #include "netmgr-int.h"
 #include "uv-compat.h"
 
-#define TLS_BUF_SIZE 65536
+#define TLS_BUF_SIZE (UINT16_MAX)
 
 static isc_result_t
 tls_error_to_result(int tls_err) {
@@ -52,7 +53,15 @@ tls_error_to_result(int tls_err) {
 }
 
 static void
-tls_do_bio(isc_nmsocket_t *sock, isc__nm_uvreq_t *send_data, bool finish);
+tls_failed_read_cb(isc_nmsocket_t *sock, const isc_result_t result);
+
+static void
+tls_do_bio(isc_nmsocket_t *sock, isc_region_t *received_data,
+	   isc__nm_uvreq_t *send_data, bool finish);
+
+static void
+tls_readcb(isc_nmhandle_t *handle, isc_result_t result, isc_region_t *region,
+	   void *cbarg);
 
 static void
 tls_close_direct(isc_nmsocket_t *sock);
@@ -103,7 +112,6 @@ tls_call_connect_cb(isc_nmsocket_t *sock, isc_nmhandle_t *handle,
 		return;
 	}
 	sock->connect_cb(handle, result, sock->connect_cbarg);
-	update_result(sock, result);
 	if (result != ISC_R_SUCCESS) {
 		isc__nmsocket_clearcb(handle->sock);
 	}
@@ -134,17 +142,31 @@ tls_senddone(isc_nmhandle_t *handle, isc_result_t eresult, void *cbarg) {
 	tlssock->tlsstream.nsending--;
 
 	if (finish && eresult == ISC_R_SUCCESS) {
+		tlssock->tlsstream.reading = false;
 		isc_nm_cancelread(handle);
 	} else if (eresult == ISC_R_SUCCESS) {
-		tls_do_bio(tlssock, NULL, false);
+		tls_do_bio(tlssock, NULL, NULL, false);
+	} else if (eresult != ISC_R_SUCCESS &&
+		   tlssock->tlsstream.state <= TLS_HANDSHAKE &&
+		   !tlssock->tlsstream.server)
+	{
+		/*
+		 * We are still waiting for the handshake to complete, but
+		 * it isn't going to happen. Call the connect callback,
+		 * passing the error code there.
+		 *
+		 * (Note: tls_failed_read_cb() calls the connect
+		 * rather than the read callback in this case.
+		 * XXX: clarify?)
+		 */
+		tls_failed_read_cb(tlssock, eresult);
 	}
 
 	isc__nmsocket_detach(&tlssock);
 }
 
 static void
-tls_failed_read_cb(isc_nmsocket_t *sock, isc_nmhandle_t *handle,
+tls_failed_read_cb(isc_nmsocket_t *sock, const isc_result_t result) {
-		   const isc_result_t result) {
 	REQUIRE(VALID_NMSOCK(sock));
 
 	if (!sock->tlsstream.server &&
@@ -152,22 +174,21 @@ tls_failed_read_cb(isc_nmsocket_t *sock, isc_nmhandle_t *handle,
 	     sock->tlsstream.state == TLS_HANDSHAKE) &&
 	    sock->connect_cb != NULL)
 	{
-		INSIST(handle == NULL);
+		isc_nmhandle_t *handle = NULL;
-		handle = isc__nmhandle_get(sock, NULL, NULL);
+		INSIST(sock->statichandle == NULL);
+		handle = isc__nmhandle_get(sock, &sock->peer,
+					   &sock->iface->addr);
 		tls_call_connect_cb(sock, handle, result);
+		isc__nmsocket_clearcb(sock);
 		isc_nmhandle_detach(&handle);
-	} else if (sock->recv_cb != NULL) {
+	} else if (sock->recv_cb != NULL && sock->statichandle != NULL) {
 		isc__nm_uvreq_t *req = NULL;
+		INSIST(VALID_NMHANDLE(sock->statichandle));
 		req = isc__nm_uvreq_get(sock->mgr, sock);
 		req->cb.recv = sock->recv_cb;
 		req->cbarg = sock->recv_cbarg;
 		req->handle = NULL;
-		if (handle != NULL) {
+		isc_nmhandle_attach(sock->statichandle, &req->handle);
-			REQUIRE(VALID_NMHANDLE(handle));
-			isc_nmhandle_attach(handle, &req->handle);
-		} else {
-			req->handle = isc__nmhandle_get(sock, NULL, NULL);
-		}
 		isc__nmsocket_clearcb(sock);
 		isc__nm_readcb(sock, req, result);
 	}
@@ -190,6 +211,7 @@ tls_send_outgoing(isc_nmsocket_t *sock, bool finish, isc_nmhandle_t *tlshandle,
 	isc_nmsocket_tls_send_req_t *send_req = NULL;
 	int pending;
 	int rv;
+	size_t len = 0;
 
 	if (inactive(sock)) {
 		if (cb != NULL) {
@@ -205,7 +227,7 @@ tls_send_outgoing(isc_nmsocket_t *sock, bool finish, isc_nmhandle_t *tlshandle,
 		(void)SSL_shutdown(sock->tlsstream.tls);
 	}
 
-	pending = BIO_pending(sock->tlsstream.app_bio);
+	pending = BIO_pending(sock->tlsstream.bio_out);
 	if (pending <= 0) {
 		return (pending);
 	}
@@ -229,9 +251,10 @@ tls_send_outgoing(isc_nmsocket_t *sock, bool finish, isc_nmhandle_t *tlshandle,
 		isc_nmhandle_attach(tlshandle, &send_req->handle);
 	}
 
-	rv = BIO_read(sock->tlsstream.app_bio, send_req->data.base, pending);
+	rv = BIO_read_ex(sock->tlsstream.bio_out, send_req->data.base, pending,
+			 &len);
 	/* There's something pending, read must succeed */
-	RUNTIME_CHECK(rv == pending);
+	RUNTIME_CHECK(rv == 1);
 
 	INSIST(VALID_NMHANDLE(sock->outerhandle));
 
@@ -241,81 +264,10 @@ tls_send_outgoing(isc_nmsocket_t *sock, bool finish, isc_nmhandle_t *tlshandle,
 	return (pending);
 }
 
-static void
+static int
-tls_do_bio(isc_nmsocket_t *sock, isc__nm_uvreq_t *send_data, bool finish) {
+tls_process_outgoing(isc_nmsocket_t *sock, bool finish,
-	isc_result_t result = ISC_R_SUCCESS;
+		     isc__nm_uvreq_t *send_data) {
-	int pending, tls_err = 0;
+	int pending;
-	int rv;
-	char buf[1];
-	bool sent_shutdown, received_shutdown;
-
-	REQUIRE(VALID_NMSOCK(sock));
-	REQUIRE(sock->tid == isc_nm_tid());
-
-	/* We will resume read if TLS layer wants us to */
-	if (sock->outerhandle != NULL) {
-		REQUIRE(VALID_NMHANDLE(sock->outerhandle));
-		isc_nm_pauseread(sock->outerhandle);
-	}
-
-	if (sock->tlsstream.state == TLS_INIT) {
-		(void)SSL_do_handshake(sock->tlsstream.tls);
-		sock->tlsstream.state = TLS_HANDSHAKE;
-	} else if (sock->tlsstream.state == TLS_CLOSED) {
-		return;
-	}
-
-	received_shutdown = (SSL_get_shutdown(sock->tlsstream.tls) &
-			     SSL_RECEIVED_SHUTDOWN) == SSL_RECEIVED_SHUTDOWN;
-
-	/* Data from TLS to client */
-	if (sock->tlsstream.state >= TLS_IO && sock->recv_cb != NULL &&
-	    !atomic_load(&sock->readpaused))
-	{
-		(void)SSL_peek(sock->tlsstream.tls, buf, 1);
-		while ((pending = SSL_pending(sock->tlsstream.tls)) > 0) {
-			uint8_t recv_buf[TLS_BUF_SIZE];
-			isc_region_t region, dregion;
-
-			if (pending > TLS_BUF_SIZE) {
-				pending = TLS_BUF_SIZE;
-			}
-			region = (isc_region_t){ .base = &recv_buf[0],
-						 .length = pending };
-
-			rv = SSL_read(sock->tlsstream.tls, region.base,
-				      region.length);
-			/* Pending succeded, so should read */
-			RUNTIME_CHECK(rv == pending);
-
-			dregion = (isc_region_t){ region.base, rv };
-			sock->recv_cb(sock->statichandle, ISC_R_SUCCESS,
-				      &dregion, sock->recv_cbarg);
-		}
-	}
-
-	if (send_data != NULL) {
-		INSIST(sock->tlsstream.state > TLS_HANDSHAKE);
-		rv = SSL_write(sock->tlsstream.tls, send_data->uvbuf.base,
-			       send_data->uvbuf.len);
-		if (rv != (int)send_data->uvbuf.len) {
-			result = received_shutdown ? ISC_R_CANCELED
-						   : ISC_R_TLSERROR;
-			send_data->cb.send(send_data->handle, result,
-					   send_data->cbarg);
-			send_data = NULL;
-			if (!received_shutdown) {
-				isc__nmsocket_detach(&sock);
-				return;
-			}
-		}
-	}
-
-	sent_shutdown = (SSL_get_shutdown(sock->tlsstream.tls) &
-			 SSL_SENT_SHUTDOWN) == SSL_SENT_SHUTDOWN;
-
-	/* Peek to move the session forward */
-	(void)SSL_peek(sock->tlsstream.tls, buf, 1);
 
 	/* Data from TLS to network */
 	if (send_data != NULL) {
@@ -323,6 +275,12 @@ tls_do_bio(isc_nmsocket_t *sock, isc__nm_uvreq_t *send_data, bool finish) {
 					    send_data->cb.send,
 					    send_data->cbarg);
 	} else {
+		bool received_shutdown =
+			((SSL_get_shutdown(sock->tlsstream.tls) &
+			  SSL_RECEIVED_SHUTDOWN) != 0);
+		bool sent_shutdown = ((SSL_get_shutdown(sock->tlsstream.tls) &
+				       SSL_SENT_SHUTDOWN) != 0);
+
 		if (received_shutdown && !sent_shutdown) {
 			finish = true;
 			(void)SSL_shutdown(sock->tlsstream.tls);
@@ -330,44 +288,144 @@ tls_do_bio(isc_nmsocket_t *sock, isc__nm_uvreq_t *send_data, bool finish) {
 		pending = tls_send_outgoing(sock, finish, NULL, NULL, NULL);
 	}
 
-	if (pending > 0) {
+	return (pending);
-		/* We'll continue in tls_senddone */
+}
-		return;
+
+static int
+tls_try_handshake(isc_nmsocket_t *sock) {
+	int rv = 0;
+	isc_nmhandle_t *tlshandle = NULL;
+
+	REQUIRE(sock->tlsstream.state == TLS_HANDSHAKE);
+
+	if (SSL_is_init_finished(sock->tlsstream.tls) == 1) {
+		return (0);
 	}
 
-	/* Get the potential error code */
+	rv = SSL_do_handshake(sock->tlsstream.tls);
-	rv = SSL_peek(sock->tlsstream.tls, buf, 1);
+	if (rv == 1) {
-	if (rv < 0) {
+		INSIST(SSL_is_init_finished(sock->tlsstream.tls) == 1);
-		tls_err = SSL_get_error(sock->tlsstream.tls, rv);
+		INSIST(sock->statichandle == NULL);
-	}
+		tlshandle = isc__nmhandle_get(sock, &sock->peer,
-
+					      &sock->iface->addr);
-	/* Only after doing the IO we can check if SSL handshake is done */
-	if (sock->tlsstream.state == TLS_HANDSHAKE &&
-	    SSL_is_init_finished(sock->tlsstream.tls) == 1)
-	{
-		isc_nmhandle_t *tlshandle = isc__nmhandle_get(sock, NULL, NULL);
 		if (sock->tlsstream.server) {
-			sock->listener->accept_cb(sock->statichandle,
+			sock->listener->accept_cb(tlshandle, ISC_R_SUCCESS,
-						  ISC_R_SUCCESS,
 						  sock->listener->accept_cbarg);
 		} else {
 			tls_call_connect_cb(sock, tlshandle, ISC_R_SUCCESS);
 		}
 		isc_nmhandle_detach(&tlshandle);
 		sock->tlsstream.state = TLS_IO;
-		async_tls_do_bio(sock);
+	}
+
+	return (rv);
+}
+
+static void
+tls_do_bio(isc_nmsocket_t *sock, isc_region_t *received_data,
+	   isc__nm_uvreq_t *send_data, bool finish) {
+	isc_result_t result = ISC_R_SUCCESS;
+	int pending, tls_status = SSL_ERROR_NONE;
+	int rv = 0;
+	bool sent_shutdown = false, received_shutdown = false;
+	size_t len = 0;
+
+	REQUIRE(VALID_NMSOCK(sock));
+	REQUIRE(sock->tid == isc_nm_tid());
+
+	/* We will resume read if TLS layer wants us to */
+	if (sock->tlsstream.reading && sock->outerhandle) {
+		REQUIRE(VALID_NMHANDLE(sock->outerhandle));
+		isc_nm_pauseread(sock->outerhandle);
+	}
+
+	if (sock->tlsstream.state == TLS_INIT) {
+		INSIST(received_data == NULL && send_data == NULL);
+		if (sock->tlsstream.server) {
+			SSL_set_accept_state(sock->tlsstream.tls);
+		} else {
+			SSL_set_connect_state(sock->tlsstream.tls);
+		}
+		sock->tlsstream.state = TLS_HANDSHAKE;
+		rv = tls_try_handshake(sock);
+		INSIST(SSL_is_init_finished(sock->tlsstream.tls) == 0);
+	} else if (sock->tlsstream.state == TLS_CLOSED) {
+		return;
+	} else { /* initialised and doing I/O */
+		if (received_data != NULL) {
+			INSIST(send_data == NULL);
+			rv = BIO_write_ex(sock->tlsstream.bio_in,
+					  received_data->base,
+					  received_data->length, &len);
+			if (rv <= 0 || len != received_data->length) {
+				result = ISC_R_TLSERROR;
+				goto error;
+			}
+
+			/*
+			 * Only after doing the IO we can check whether SSL
+			 * handshake is done.
+			 */
+			if (sock->tlsstream.state == TLS_HANDSHAKE) {
+				rv = tls_try_handshake(sock);
+			}
+		} else if (send_data != NULL) {
+			INSIST(received_data == NULL);
+			INSIST(sock->tlsstream.state > TLS_HANDSHAKE);
+			received_shutdown =
+				((SSL_get_shutdown(sock->tlsstream.tls) &
+				  SSL_RECEIVED_SHUTDOWN) != 0);
+			rv = SSL_write_ex(sock->tlsstream.tls,
+					  send_data->uvbuf.base,
+					  send_data->uvbuf.len, &len);
+			if (rv != 1 || len != send_data->uvbuf.len) {
+				result = received_shutdown ? ISC_R_CANCELED
+							   : ISC_R_TLSERROR;
+				send_data->cb.send(send_data->handle, result,
+						   send_data->cbarg);
+				send_data = NULL;
+				if (!received_shutdown) {
+					isc__nmsocket_detach(&sock);
+					return;
+				}
+			}
+		}
+
+		/* Decrypt and pass data from network to client */
+		if (sock->tlsstream.state >= TLS_IO && sock->recv_cb != NULL &&
+		    !atomic_load(&sock->readpaused))
+		{
+			uint8_t recv_buf[TLS_BUF_SIZE];
+			INSIST(sock->tlsstream.state > TLS_HANDSHAKE);
+			while ((rv = SSL_read_ex(sock->tlsstream.tls, recv_buf,
+						 TLS_BUF_SIZE, &len)) == 1)
+			{
+				isc_region_t region;
+				region = (isc_region_t){ .base = &recv_buf[0],
+							 .length = len };
+
+				sock->recv_cb(sock->statichandle, ISC_R_SUCCESS,
+					      &region, sock->recv_cbarg);
+			}
+		}
+	}
+	tls_status = SSL_get_error(sock->tlsstream.tls, rv);
+
+	pending = tls_process_outgoing(sock, finish, send_data);
+	if (pending > 0) {
+		/* We'll continue in tls_senddone */
 		return;
 	}
 
-	switch (tls_err) {
+	switch (tls_status) {
 	case SSL_ERROR_NONE:
+	case SSL_ERROR_ZERO_RETURN:
 		if (sent_shutdown && received_shutdown) {
 			/* clean shutdown */
 			isc_nm_cancelread(sock->outerhandle);
 			isc__nm_tls_close(sock);
 		};
 		return;
-		break;
 	case SSL_ERROR_WANT_WRITE:
 		if (sock->tlsstream.nsending == 0) {
 			/*
@@ -375,79 +433,77 @@ tls_do_bio(isc_nmsocket_t *sock, isc__nm_uvreq_t *send_data, bool finish) {
 			 * already the send callback will call it.
 			 */
 			async_tls_do_bio(sock);
-			return;
-		} else {
-			return;
-		}
-		break;
-	case SSL_ERROR_WANT_READ:
-		if (sock->outerhandle != NULL) {
-			INSIST(VALID_NMHANDLE(sock->outerhandle));
-			isc_nm_resumeread(sock->outerhandle);
 		}
 		return;
-		break;
+	case SSL_ERROR_WANT_READ:
+		if (sock->tlsstream.reading) {
+			INSIST(VALID_NMHANDLE(sock->outerhandle));
+			isc_nm_resumeread(sock->outerhandle);
+		} else if (sock->tlsstream.state == TLS_HANDSHAKE) {
+			sock->tlsstream.reading = true;
+			isc_nm_read(sock->outerhandle, tls_readcb, sock);
+		}
+		return;
 	default:
-		result = tls_error_to_result(tls_err);
+		result = tls_error_to_result(tls_status);
-		goto error;
+		break;
 	}
 
-	return;
-
 error:
 	isc_log_write(isc_lctx, ISC_LOGCATEGORY_GENERAL, ISC_LOGMODULE_NETMGR,
-		      ISC_LOG_ERROR, "SSL error in BIO: %d %s", tls_err,
+		      ISC_LOG_ERROR, "SSL error in BIO: %d %s", tls_status,
 		      isc_result_totext(result));
-	tls_failed_read_cb(sock, sock->statichandle, result);
+	tls_failed_read_cb(sock, result);
 }
 
 static void
 tls_readcb(isc_nmhandle_t *handle, isc_result_t result, isc_region_t *region,
 	   void *cbarg) {
 	isc_nmsocket_t *tlssock = (isc_nmsocket_t *)cbarg;
-	int rv;
 
 	REQUIRE(VALID_NMSOCK(tlssock));
 	REQUIRE(VALID_NMHANDLE(handle));
 	REQUIRE(tlssock->tid == isc_nm_tid());
 
 	if (result != ISC_R_SUCCESS) {
-		tls_failed_read_cb(tlssock, tlssock->statichandle, result);
+		tls_failed_read_cb(tlssock, result);
 		return;
 	}
-	rv = BIO_write(tlssock->tlsstream.app_bio, region->base,
+	tls_do_bio(tlssock, region, NULL, false);
-		       region->length);
-	if (rv != (int)region->length) {
-		/* XXXWPK log it? */
-		tls_failed_read_cb(tlssock, tlssock->statichandle,
-				   ISC_R_TLSERROR);
-		return;
-	}
-	tls_do_bio(tlssock, NULL, false);
 }
 
 static isc_result_t
 initialize_tls(isc_nmsocket_t *sock, bool server) {
 	REQUIRE(sock->tid == isc_nm_tid());
 
-	if (BIO_new_bio_pair(&(sock->tlsstream.ssl_bio), TLS_BUF_SIZE,
+	sock->tlsstream.bio_in = BIO_new(BIO_s_mem());
-			     &(sock->tlsstream.app_bio), TLS_BUF_SIZE) != 1)
+	if (sock->tlsstream.bio_in == NULL) {
-	{
+		isc_tls_free(&sock->tlsstream.tls);
+		return (ISC_R_TLSERROR);
+	}
+	sock->tlsstream.bio_out = BIO_new(BIO_s_mem());
+	if (sock->tlsstream.bio_out == NULL) {
+		BIO_free_all(sock->tlsstream.bio_in);
+		sock->tlsstream.bio_in = NULL;
 		isc_tls_free(&sock->tlsstream.tls);
 		return (ISC_R_TLSERROR);
 	}
 
-	SSL_set_bio(sock->tlsstream.tls, sock->tlsstream.ssl_bio,
+	if (BIO_set_mem_eof_return(sock->tlsstream.bio_in, EOF) != 1 ||
-		    sock->tlsstream.ssl_bio);
+	    BIO_set_mem_eof_return(sock->tlsstream.bio_out, EOF) != 1)
-	if (server) {
+	{
-		SSL_set_accept_state(sock->tlsstream.tls);
+		goto error;
-	} else {
-		SSL_set_connect_state(sock->tlsstream.tls);
 	}
+
+	SSL_set_bio(sock->tlsstream.tls, sock->tlsstream.bio_in,
+		    sock->tlsstream.bio_out);
+	sock->tlsstream.server = server;
 	sock->tlsstream.nsending = 0;
-	isc_nm_read(sock->outerhandle, tls_readcb, sock);
-	tls_do_bio(sock, NULL, false);
 	return (ISC_R_SUCCESS);
+error:
+	isc_tls_free(&sock->tlsstream.tls);
+	sock->tlsstream.bio_out = sock->tlsstream.bio_in = NULL;
+	return (ISC_R_TLSERROR);
 }
 
 static isc_result_t
@@ -487,7 +543,6 @@ tlslisten_acceptcb(isc_nmhandle_t *handle, isc_result_t result, void *cbarg) {
 	tlssock->peer = handle->sock->peer;
 	tlssock->read_timeout = atomic_load(&handle->sock->mgr->init);
 	tlssock->tid = isc_nm_tid();
-	tlssock->tlsstream.server = true;
 	tlssock->tlsstream.state = TLS_INIT;
 
 	tlssock->tlsstream.ctx = tlslistensock->tlsstream.ctx;
@@ -496,6 +551,7 @@ tlslisten_acceptcb(isc_nmhandle_t *handle, isc_result_t result, void *cbarg) {
 	RUNTIME_CHECK(result == ISC_R_SUCCESS);
 	/* TODO: catch failure code, detach tlssock, and log the error */
 
+	tls_do_bio(tlssock, NULL, NULL, false);
 	return (result);
 }
 
@@ -573,11 +629,12 @@ isc__nm_async_tlssend(isc__networker_t *worker, isc__netievent_t *ev0) {
 
 	if (inactive(sock)) {
 		req->cb.send(req->handle, ISC_R_CANCELED, req->cbarg);
-		isc__nm_uvreq_put(&req, sock);
+		goto done;
-		return;
 	}
 
-	tls_do_bio(sock, req, false);
+	tls_do_bio(sock, NULL, req, false);
+done:
+	isc_mem_free(sock->mgr->mctx, req->uvbuf.base);
 	isc__nm_uvreq_put(&req, sock);
 	return;
 }
@@ -606,7 +663,8 @@ isc__nm_tls_send(isc_nmhandle_t *handle, const isc_region_t *region,
 	uvreq->cb.send = cb;
 	uvreq->cbarg = cbarg;
 
-	uvreq->uvbuf.base = (char *)region->base;
+	uvreq->uvbuf.base = isc_mem_allocate(sock->mgr->mctx, region->length);
+	memmove(uvreq->uvbuf.base, region->base, region->length);
 	uvreq->uvbuf.len = region->length;
 
 	/*
@@ -627,7 +685,7 @@ isc__nm_async_tlsstartread(isc__networker_t *worker, isc__netievent_t *ev0) {
 
 	UNUSED(worker);
 
-	tls_do_bio(sock, NULL, false);
+	tls_do_bio(sock, NULL, NULL, false);
 }
 
 void
@@ -638,10 +696,10 @@ isc__nm_tls_read(isc_nmhandle_t *handle, isc_nm_recv_cb_t cb, void *cbarg) {
 	REQUIRE(VALID_NMHANDLE(handle));
 
 	sock = handle->sock;
-
 	REQUIRE(VALID_NMSOCK(sock));
 	REQUIRE(sock->statichandle == handle);
 	REQUIRE(sock->tid == isc_nm_tid());
+	REQUIRE(sock->recv_cb == NULL);
 
 	if (inactive(sock)) {
 		cb(handle, ISC_R_NOTCONNECTED, NULL, cbarg);
@@ -661,9 +719,12 @@ isc__nm_tls_pauseread(isc_nmhandle_t *handle) {
 	REQUIRE(VALID_NMHANDLE(handle));
 	REQUIRE(VALID_NMSOCK(handle->sock));
 
-	atomic_store(&handle->sock->readpaused, true);
+	if (atomic_compare_exchange_strong(&handle->sock->readpaused,
-	if (handle->sock->outerhandle != NULL) {
+					   &(bool){ false }, true))
-		isc_nm_pauseread(handle->sock->outerhandle);
+	{
+		if (handle->sock->outerhandle != NULL) {
+			isc_nm_pauseread(handle->sock->outerhandle);
+		}
 	}
 }
 
@@ -672,8 +733,11 @@ isc__nm_tls_resumeread(isc_nmhandle_t *handle) {
 	REQUIRE(VALID_NMHANDLE(handle));
 	REQUIRE(VALID_NMSOCK(handle->sock));
 
-	atomic_store(&handle->sock->readpaused, false);
+	if (!atomic_compare_exchange_strong(&handle->sock->readpaused,
-	async_tls_do_bio(handle->sock);
+					    &(bool){ false }, false))
+	{
+		async_tls_do_bio(handle->sock);
+	}
 }
 
 static void
@@ -686,6 +750,7 @@ tls_close_direct(isc_nmsocket_t *sock) {
 	 */
 	if (sock->outerhandle != NULL) {
 		isc_nm_pauseread(sock->outerhandle);
+		isc__nmsocket_clearcb(sock->outerhandle->sock);
 		isc_nmhandle_detach(&sock->outerhandle);
 	}
 
@@ -694,7 +759,6 @@ tls_close_direct(isc_nmsocket_t *sock) {
 	}
 
 	/* further cleanup performed in isc__nm_tls_cleanup_data() */
-	atomic_store(&sock->active, false);
 	atomic_store(&sock->closed, true);
 	sock->tlsstream.state = TLS_CLOSED;
 }
@@ -773,7 +837,6 @@ isc_nm_tlsconnect(isc_nm_t *mgr, isc_nmiface_t *local, isc_nmiface_t *peer,
 	nsock->connect_cbarg = cbarg;
 	nsock->connect_timeout = timeout;
 	nsock->tlsstream.ctx = ctx;
-	nsock->tlsstream.connect_from_networker = isc__nm_in_netthread();
 
 	ievent = isc__nm_get_netievent_tlsconnect(mgr, nsock);
 	ievent->local = local->addr;
@@ -782,6 +845,7 @@ isc_nm_tlsconnect(isc_nm_t *mgr, isc_nmiface_t *local, isc_nmiface_t *peer,
 
 	isc__nmsocket_attach(nsock, &tsock);
 	if (isc__nm_in_netthread()) {
+		atomic_store(&nsock->active, true);
 		nsock->tid = isc_nm_tid();
 		isc__nm_async_tlsconnect(&mgr->workers[nsock->tid],
 					 (isc__netievent_t *)ievent);
@@ -821,13 +885,15 @@ tcp_connected(isc_nmhandle_t *handle, isc_result_t result, void *cbarg) {
 		goto error;
 	}
 
-	tlssock->peer = isc_nmhandle_peeraddr(handle);
-	isc_nmhandle_attach(handle, &tlssock->outerhandle);
 	result = initialize_tls(tlssock, false);
 	if (result != ISC_R_SUCCESS) {
 		goto error;
 	}
 
+	tlssock->peer = isc_nmhandle_peeraddr(handle);
+	isc_nmhandle_attach(handle, &tlssock->outerhandle);
+
+	tls_do_bio(tlssock, NULL, NULL, false);
 	return;
 error:
 	tlshandle = isc__nmhandle_get(tlssock, NULL, NULL);
@@ -863,15 +929,23 @@ isc__nm_async_tlsconnect(isc__networker_t *worker, isc__netievent_t *ev0) {
 				   (isc_nmiface_t *)&ievent->peer,
 				   tcp_connected, tlssock,
 				   tlssock->connect_timeout, 0);
-	if (tlssock->tlsstream.connect_from_networker) {
+
-		update_result(tlssock, result);
+	/*
+	 * Sometimes on Linux, socket creation might fail if there are
+	 * already too many socket descriptors. In such a case the
+	 * connect callback is not going to be called.
+	 */
+	if (result != ISC_R_SUCCESS) {
+		goto error;
 	}
+	update_result(tlssock, result);
 	return;
 
 error:
 	tlshandle = isc__nmhandle_get(tlssock, NULL, NULL);
 	atomic_store(&tlssock->closed, true);
 	tls_call_connect_cb(tlssock, tlshandle, result);
+	update_result(tlssock, result);
 	isc_nmhandle_detach(&tlshandle);
 	isc__nmsocket_detach(&tlssock);
 }
@@ -879,8 +953,9 @@ error:
 static void
 tls_cancelread(isc_nmsocket_t *sock) {
 	if (!inactive(sock) && sock->tlsstream.state == TLS_IO) {
-		tls_do_bio(sock, NULL, true);
+		tls_do_bio(sock, NULL, NULL, true);
 	} else if (sock->outerhandle != NULL) {
+		sock->tlsstream.reading = false;
 		isc_nm_cancelread(sock->outerhandle);
 	}
 }
@@ -925,7 +1000,7 @@ isc__nm_async_tlsdobio(isc__networker_t *worker, isc__netievent_t *ev0) {
 
 	UNUSED(worker);
 
-	tls_do_bio(ievent->sock, NULL, false);
+	tls_do_bio(ievent->sock, NULL, NULL, false);
 }
 
 void
@@ -939,11 +1014,8 @@ isc__nm_tls_cleanup_data(isc_nmsocket_t *sock) {
 			isc_tls_free(&sock->tlsstream.tls);
 			/* These are destroyed when we free SSL */
 			sock->tlsstream.ctx = NULL;
-			sock->tlsstream.ssl_bio = NULL;
+			sock->tlsstream.bio_out = NULL;
-		}
+			sock->tlsstream.bio_in = NULL;
-		if (sock->tlsstream.app_bio != NULL) {
-			BIO_free(sock->tlsstream.app_bio);
-			sock->tlsstream.app_bio = NULL;
 		}
 	}
 }

lib/isc/tests/Makefile.am

@@ -47,7 +47,8 @@ TESTS =			\
 	task_test	\
 	taskpool_test	\
 	time_test	\
-	timer_test
+	timer_test	\
+	tls_test
 
 check_PROGRAMS =	\
 	$(TESTS)
@@ -91,6 +92,16 @@ netmgr_test_LDADD =	\
 	$(LDADD)	\
 	$(LIBUV_LIBS)
 
+tls_test_CPPFLAGS =	\
+	$(AM_CPPFLAGS)	\
+	$(LIBUV_CFLAGS)	\
+	$(OPENSSL_CFLAGS)
+
+tls_test_LDADD =	\
+	$(LDADD)	\
+	$(LIBUV_LIBS)	\
+	$(OPENSSL_LIBS)
+
 unit-local: check
 
 EXTRA_DIST = testdata

lib/isc/tests/doh_test.c

@@ -65,7 +65,7 @@ static atomic_uint_fast64_t creads;
 
 static atomic_bool was_error;
 
-static unsigned int workers = 1;
+static unsigned int workers = 0;
 
 static bool reuse_supported = true;
 
@@ -227,9 +227,18 @@ setup_ephemeral_port(isc_sockaddr_t *addr, sa_family_t family) {
 
 static int
 _setup(void **state) {
+	char *p = NULL;
+
 	UNUSED(state);
 
-	workers = isc_os_ncpus();
+	if (workers == 0) {
+		workers = isc_os_ncpus();
+	}
+	p = getenv("ISC_TASK_WORKERS");
+	if (p != NULL) {
+		workers = atoi(p);
+	}
+	INSIST(workers != 0);
 
 	if (isc_test_begin(NULL, false, workers) != ISC_R_SUCCESS) {
 		return (-1);
@@ -587,17 +596,25 @@ static isc_threadresult_t
 doh_connect_thread(isc_threadarg_t arg) {
 	isc_nm_t *connect_nm = (isc_nm_t *)arg;
 	char req_url[256];
+	isc_result_t result;
 
 	sockaddr_to_url(&tcp_listen_addr, atomic_load(&use_TLS), req_url,
 			sizeof(req_url), DOH_PATH);
 
 	while (atomic_load(&nsends) > 0) {
-		(void)connect_send_request(
+		result = connect_send_request(
 			connect_nm, req_url, atomic_load(&POST),
 			&(isc_region_t){ .base = (uint8_t *)send_msg.base,
 					 .length = send_msg.len },
 			doh_receive_send_reply_cb, NULL, atomic_load(&use_TLS),
 			30000);
+		/* protection against "too many open files" */
+#ifndef _WIN32
+		if (result != ISC_R_SUCCESS) {
+			INSIST(result == ISC_R_TOOMANYOPENFILES);
+			usleep(1000 * workers);
+		}
+#endif
 	}
 
 	return ((isc_threadresult_t)0);

lib/isc/tests/isctest.c

@@ -80,14 +80,13 @@ create_managers(unsigned int workers) {
 	if (workers == 0) {
 		workers = isc_os_ncpus();
 	}
-
 	p = getenv("ISC_TASK_WORKERS");
 	if (p != NULL) {
 		workers = atoi(p);
 	}
 	INSIST(workers != 0);
 
-	isc_hp_init(4 * workers);
+	isc_hp_init(6 * workers);
 
 	netmgr = isc_nm_start(test_mctx, workers);
 	CHECK(isc_taskmgr_create(test_mctx, workers, 0, netmgr, &taskmgr));

lib/isc/tests/netmgr_test.c

@@ -60,7 +60,7 @@ static uv_buf_t stop_msg = { .base = (char *)&stop_magic,
 			     .len = sizeof(stop_magic) };
 
 static atomic_bool do_send = ATOMIC_VAR_INIT(false);
-static unsigned int workers = 1;
+static unsigned int workers = 0;
 
 static atomic_int_fast64_t nsends;
 static int_fast64_t esends; /* expected sends */
@@ -172,9 +172,11 @@ static bool skip_long_tests = false;
 
 static int
 _setup(void **state __attribute__((unused))) {
-	char *p;
+	char *p = NULL;
 
-	workers = isc_os_ncpus();
+	if (workers == 0) {
+		workers = isc_os_ncpus();
+	}
 	p = getenv("ISC_TASK_WORKERS");
 	if (p != NULL) {
 		workers = atoi(p);

lib/isc/tests/tls_test.c

@@ -0,0 +1,932 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, you can obtain one at https://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+#if HAVE_CMOCKA
+#include <sched.h> /* IWYU pragma: keep */
+#include <setjmp.h>
+#include <stdarg.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdlib.h>
+#include <time.h>
+#include <unistd.h>
+#include <uv.h>
+
+#define UNIT_TESTING
+#include <cmocka.h>
+
+#include <isc/atomic.h>
+#include <isc/buffer.h>
+#include <isc/condition.h>
+#include <isc/mutex.h>
+#include <isc/netmgr.h>
+#include <isc/nonce.h>
+#include <isc/os.h>
+#include <isc/refcount.h>
+#include <isc/sockaddr.h>
+#include <isc/thread.h>
+#include <isc/tls.h>
+
+#include "uv_wrap.h"
+#define KEEP_BEFORE
+
+#include "../netmgr/netmgr-int.h"
+#include "../netmgr/tlsstream.c"
+#include "../netmgr/uv-compat.c"
+#include "../netmgr/uv-compat.h"
+#include "isctest.h"
+
+#define MAX_NM 2
+
+static isc_sockaddr_t tls_listen_addr;
+
+static uint64_t send_magic = 0;
+static uint64_t stop_magic = 0;
+
+static uv_buf_t send_msg = { .base = (char *)&send_magic,
+			     .len = sizeof(send_magic) };
+static uv_buf_t stop_msg = { .base = (char *)&stop_magic,
+			     .len = sizeof(stop_magic) };
+
+static atomic_uint_fast64_t nsends;
+
+static atomic_uint_fast64_t ssends;
+static atomic_uint_fast64_t sreads;
+
+static atomic_uint_fast64_t cconnects;
+static atomic_uint_fast64_t csends;
+static atomic_uint_fast64_t creads;
+static atomic_uint_fast64_t ctimeouts;
+
+static unsigned int workers = 0;
+
+static bool reuse_supported = true;
+
+static isc_tlsctx_t *server_tlsctx = NULL;
+static isc_tlsctx_t *client_tlsctx = NULL;
+
+#define NSENDS	100
+#define NWRITES 10
+
+#define CHECK_RANGE_FULL(v)                                       \
+	{                                                         \
+		int __v = atomic_load(&v);                        \
+		assert_true(__v > NSENDS * NWRITES * 10 / 100);   \
+		assert_true(__v <= NSENDS * NWRITES * 110 / 100); \
+	}
+
+#define CHECK_RANGE_HALF(v)                                       \
+	{                                                         \
+		int __v = atomic_load(&v);                        \
+		assert_true(__v > NSENDS * NWRITES * 5 / 100);    \
+		assert_true(__v <= NSENDS * NWRITES * 110 / 100); \
+	}
+
+/* Enable this to print values while running tests */
+#undef PRINT_DEBUG
+#ifdef PRINT_DEBUG
+#define X(v) fprintf(stderr, #v " = %" PRIu64 "\n", atomic_load(&v))
+#else
+#define X(v)
+#endif
+
+static int
+setup_ephemeral_port(isc_sockaddr_t *addr, sa_family_t family) {
+	isc_result_t result;
+	socklen_t addrlen = sizeof(*addr);
+	int fd;
+	int r;
+
+	isc_sockaddr_fromin6(addr, &in6addr_loopback, 0);
+
+	fd = socket(AF_INET6, family, 0);
+	if (fd < 0) {
+		perror("setup_ephemeral_port: socket()");
+		return (-1);
+	}
+
+	r = bind(fd, (const struct sockaddr *)&addr->type.sa,
+		 sizeof(addr->type.sin6));
+	if (r != 0) {
+		perror("setup_ephemeral_port: bind()");
+		close(fd);
+		return (r);
+	}
+
+	r = getsockname(fd, (struct sockaddr *)&addr->type.sa, &addrlen);
+	if (r != 0) {
+		perror("setup_ephemeral_port: getsockname()");
+		close(fd);
+		return (r);
+	}
+
+	result = isc__nm_socket_reuse(fd);
+	if (result != ISC_R_SUCCESS && result != ISC_R_NOTIMPLEMENTED) {
+		fprintf(stderr,
+			"setup_ephemeral_port: isc__nm_socket_reuse(): %s",
+			isc_result_totext(result));
+		close(fd);
+		return (-1);
+	}
+
+	result = isc__nm_socket_reuse_lb(fd);
+	if (result != ISC_R_SUCCESS && result != ISC_R_NOTIMPLEMENTED) {
+		fprintf(stderr,
+			"setup_ephemeral_port: isc__nm_socket_reuse_lb(): %s",
+			isc_result_totext(result));
+		close(fd);
+		return (-1);
+	}
+	if (result == ISC_R_NOTIMPLEMENTED) {
+		reuse_supported = false;
+	}
+
+#if IPV6_RECVERR
+#define setsockopt_on(socket, level, name) \
+	setsockopt(socket, level, name, &(int){ 1 }, sizeof(int))
+
+	r = setsockopt_on(fd, IPPROTO_IPV6, IPV6_RECVERR);
+	if (r != 0) {
+		perror("setup_ephemeral_port");
+		close(fd);
+		return (r);
+	}
+#endif
+
+	return (fd);
+}
+
+static int
+_setup(void **state) {
+	char *p = NULL;
+
+	UNUSED(state);
+
+	if (workers == 0) {
+		workers = isc_os_ncpus();
+	}
+	p = getenv("ISC_TASK_WORKERS");
+	if (p != NULL) {
+		workers = atoi(p);
+	}
+	INSIST(workers != 0);
+
+	if (isc_test_begin(NULL, true, workers) != ISC_R_SUCCESS) {
+		return (-1);
+	}
+
+	signal(SIGPIPE, SIG_IGN);
+
+	return (0);
+}
+
+static int
+_teardown(void **state) {
+	UNUSED(state);
+
+	isc_test_end();
+
+	return (0);
+}
+
+/* Generic */
+
+static unsigned int
+noop_accept_cb(isc_nmhandle_t *handle, unsigned int result, void *cbarg) {
+	UNUSED(handle);
+	UNUSED(result);
+	UNUSED(cbarg);
+
+	return (0);
+}
+
+static void
+noop_connect_cb(isc_nmhandle_t *handle, isc_result_t result, void *cbarg) {
+	UNUSED(handle);
+	UNUSED(result);
+	UNUSED(cbarg);
+}
+
+thread_local uint8_t tls_buffer_storage[4096];
+thread_local size_t tls_buffer_length = 0;
+
+static int
+nm_setup(void **state) {
+	size_t nworkers = ISC_MAX(ISC_MIN(workers, 32), 1);
+	int tls_listen_sock = -1;
+	isc_nm_t **nm = NULL;
+
+	tls_listen_addr = (isc_sockaddr_t){ .length = 0 };
+	tls_listen_sock = setup_ephemeral_port(&tls_listen_addr, SOCK_STREAM);
+	if (tls_listen_sock < 0) {
+		return (-1);
+	}
+	close(tls_listen_sock);
+	tls_listen_sock = -1;
+
+	atomic_store(&nsends, NSENDS * NWRITES);
+
+	atomic_store(&csends, 0);
+	atomic_store(&creads, 0);
+	atomic_store(&sreads, 0);
+	atomic_store(&ssends, 0);
+	atomic_store(&ctimeouts, 0);
+	atomic_store(&cconnects, 0);
+
+	isc_nonce_buf(&send_magic, sizeof(send_magic));
+	isc_nonce_buf(&stop_magic, sizeof(stop_magic));
+	if (send_magic == stop_magic) {
+		return (-1);
+	}
+
+	nm = isc_mem_get(test_mctx, MAX_NM * sizeof(nm[0]));
+	for (size_t i = 0; i < MAX_NM; i++) {
+		nm[i] = isc_nm_start(test_mctx, nworkers);
+		assert_non_null(nm[i]);
+	}
+
+	INSIST(server_tlsctx == NULL);
+	isc_tlsctx_createserver(NULL, NULL, &server_tlsctx);
+	INSIST(client_tlsctx == NULL);
+	isc_tlsctx_createclient(&client_tlsctx);
+
+	*state = nm;
+
+	return (0);
+}
+
+static int
+nm_teardown(void **state) {
+	isc_nm_t **nm = (isc_nm_t **)*state;
+
+	for (size_t i = 0; i < MAX_NM; i++) {
+		isc_nm_destroy(&nm[i]);
+		assert_null(nm[i]);
+	}
+	isc_mem_put(test_mctx, nm, MAX_NM * sizeof(nm[0]));
+
+	INSIST(server_tlsctx != NULL);
+	isc_tlsctx_free(&server_tlsctx);
+	INSIST(client_tlsctx != NULL);
+	isc_tlsctx_free(&client_tlsctx);
+
+	return (0);
+}
+
+thread_local size_t nwrites = NWRITES;
+
+/* TLS Connect */
+
+static void
+tls_connect_send_cb(isc_nmhandle_t *handle, isc_result_t eresult, void *cbarg);
+
+static void
+tls_connect_send(isc_nmhandle_t *handle);
+
+static void
+tls_connect_read_cb(isc_nmhandle_t *handle, isc_result_t eresult,
+		    isc_region_t *region, void *cbarg) {
+	uint64_t magic = 0;
+
+	UNUSED(cbarg);
+
+	assert_non_null(handle);
+	if (eresult != ISC_R_SUCCESS) {
+		goto unref;
+	}
+
+	memmove(tls_buffer_storage + tls_buffer_length, region->base,
+		region->length);
+	tls_buffer_length += region->length;
+
+	while (tls_buffer_length >= sizeof(magic)) {
+		atomic_fetch_add(&creads, 1);
+
+		memmove(&magic, tls_buffer_storage, sizeof(magic));
+		assert_true(magic == stop_magic || magic == send_magic);
+
+		tls_buffer_length -= sizeof(magic);
+		memmove(tls_buffer_storage, tls_buffer_storage + sizeof(magic),
+			tls_buffer_length);
+
+		if (magic == send_magic) {
+			tls_connect_send(handle);
+			return;
+		} else if (magic == stop_magic) {
+			/* We are done, so we don't send anything back */
+			/* There should be no more packets in the buffer */
+			assert_int_equal(tls_buffer_length, 0);
+		}
+	}
+unref:
+	isc_nmhandle_detach(&handle);
+}
+
+static void
+tls_connect_send_cb(isc_nmhandle_t *handle, isc_result_t eresult, void *cbarg) {
+	assert_non_null(handle);
+	UNUSED(cbarg);
+
+	if (eresult == ISC_R_SUCCESS) {
+		atomic_fetch_add(&csends, 1);
+		isc_nm_resumeread(handle);
+	} else {
+		/* Send failed, we need to stop reading too */
+		isc_nm_cancelread(handle);
+	}
+}
+
+static void
+tls_connect_shutdown(isc_nmhandle_t *handle, isc_result_t eresult,
+		     void *cbarg) {
+	UNUSED(cbarg);
+
+	assert_non_null(handle);
+
+	if (eresult == ISC_R_SUCCESS) {
+		atomic_fetch_add(&csends, 1);
+	} else {
+		/* Send failed, we need to stop reading too */
+		isc_nm_cancelread(handle);
+	}
+}
+
+static void
+tls_connect_send(isc_nmhandle_t *handle) {
+	uint_fast64_t sends = atomic_load(&nsends);
+
+	while (sends > 0) {
+		/* Continue until we subtract or we are done */
+		if (atomic_compare_exchange_weak(&nsends, &sends, sends - 1)) {
+			sends--;
+			break;
+		}
+	}
+
+	if (sends == 0) {
+		isc_nm_send(handle, (isc_region_t *)&stop_msg,
+			    tls_connect_shutdown, NULL);
+	} else {
+		isc_nm_send(handle, (isc_region_t *)&send_msg,
+			    tls_connect_send_cb, NULL);
+	}
+}
+
+static void
+tls_connect_connect_cb(isc_nmhandle_t *handle, isc_result_t eresult,
+		       void *cbarg) {
+	isc_nmhandle_t *readhandle = NULL;
+
+	UNUSED(cbarg);
+
+	if (eresult != ISC_R_SUCCESS) {
+		uint_fast64_t sends = atomic_load(&nsends);
+
+		/* We failed to connect; try again */
+		while (sends > 0) {
+			/* Continue until we subtract or we are done */
+			if (atomic_compare_exchange_weak(&nsends, &sends,
+							 sends - 1)) {
+				sends--;
+				break;
+			}
+		}
+		return;
+	}
+
+	atomic_fetch_add(&cconnects, 1);
+
+	isc_nmhandle_attach(handle, &readhandle);
+	isc_nm_read(handle, tls_connect_read_cb, NULL);
+
+	tls_connect_send(handle);
+}
+
+static void
+tls_noop(void **state) {
+	isc_nm_t **nm = (isc_nm_t **)*state;
+	isc_nm_t *listen_nm = nm[0];
+	isc_nm_t *connect_nm = nm[1];
+	isc_result_t result = ISC_R_SUCCESS;
+	isc_nmsocket_t *listen_sock = NULL;
+	isc_sockaddr_t tls_connect_addr;
+
+	tls_connect_addr = (isc_sockaddr_t){ .length = 0 };
+	isc_sockaddr_fromin6(&tls_connect_addr, &in6addr_loopback, 0);
+
+	result = isc_nm_listentls(listen_nm, (isc_nmiface_t *)&tls_listen_addr,
+				  noop_accept_cb, NULL, 0, 0, NULL,
+				  server_tlsctx, &listen_sock);
+	assert_int_equal(result, ISC_R_SUCCESS);
+
+	isc_nm_stoplistening(listen_sock);
+	isc_nmsocket_close(&listen_sock);
+	assert_null(listen_sock);
+
+	(void)isc_nm_tlsconnect(connect_nm, (isc_nmiface_t *)&tls_connect_addr,
+				(isc_nmiface_t *)&tls_listen_addr,
+				noop_connect_cb, NULL, client_tlsctx, 1, 0);
+
+	isc_nm_closedown(connect_nm);
+
+	assert_int_equal(0, atomic_load(&cconnects));
+	assert_int_equal(0, atomic_load(&csends));
+	assert_int_equal(0, atomic_load(&creads));
+	assert_int_equal(0, atomic_load(&ctimeouts));
+	assert_int_equal(0, atomic_load(&sreads));
+	assert_int_equal(0, atomic_load(&ssends));
+}
+
+static void
+tls_noresponse(void **state) {
+	isc_nm_t **nm = (isc_nm_t **)*state;
+	isc_nm_t *listen_nm = nm[0];
+	isc_nm_t *connect_nm = nm[1];
+	isc_result_t result = ISC_R_SUCCESS;
+	isc_nmsocket_t *listen_sock = NULL;
+	isc_sockaddr_t tls_connect_addr;
+
+	tls_connect_addr = (isc_sockaddr_t){ .length = 0 };
+	isc_sockaddr_fromin6(&tls_connect_addr, &in6addr_loopback, 0);
+
+	result = isc_nm_listentls(listen_nm, (isc_nmiface_t *)&tls_listen_addr,
+				  noop_accept_cb, NULL, 0, 0, NULL,
+				  server_tlsctx, &listen_sock);
+	assert_int_equal(result, ISC_R_SUCCESS);
+
+	(void)isc_nm_tlsconnect(connect_nm, (isc_nmiface_t *)&tls_connect_addr,
+				(isc_nmiface_t *)&tls_listen_addr,
+				noop_connect_cb, NULL, client_tlsctx, 1, 0);
+
+	isc_nm_stoplistening(listen_sock);
+	isc_nmsocket_close(&listen_sock);
+	assert_null(listen_sock);
+	isc_nm_closedown(connect_nm);
+}
+
+static isc_result_t
+tls_listen_accept_cb(isc_nmhandle_t *handle, isc_result_t result, void *cbarg);
+
+static isc_threadresult_t
+tls_connect_thread(isc_threadarg_t arg) {
+	isc_nm_t *connect_nm = (isc_nm_t *)arg;
+	isc_sockaddr_t tls_connect_addr;
+	isc_result_t result;
+
+	tls_connect_addr = (isc_sockaddr_t){ .length = 0 };
+	isc_sockaddr_fromin6(&tls_connect_addr, &in6addr_loopback, 0);
+
+	while (atomic_load(&nsends) > 0) {
+		result = isc_nm_tlsconnect(
+			connect_nm, (isc_nmiface_t *)&tls_connect_addr,
+			(isc_nmiface_t *)&tls_listen_addr,
+			tls_connect_connect_cb, NULL, client_tlsctx, 1, 0);
+		/* protection against "too many open files" */
+		if (result != ISC_R_SUCCESS) {
+			atomic_fetch_sub(&nsends, 1);
+			usleep(1000 * workers);
+		}
+	}
+
+	return ((isc_threadresult_t)0);
+}
+
+static void
+tls_recv_one(void **state) {
+	isc_nm_t **nm = (isc_nm_t **)*state;
+	isc_nm_t *listen_nm = nm[0];
+	isc_nm_t *connect_nm = nm[1];
+	isc_result_t result = ISC_R_SUCCESS;
+	isc_nmsocket_t *listen_sock = NULL;
+	isc_sockaddr_t tls_connect_addr;
+
+	tls_connect_addr = (isc_sockaddr_t){ .length = 0 };
+	isc_sockaddr_fromin6(&tls_connect_addr, &in6addr_loopback, 0);
+
+	atomic_store(&nsends, 1);
+
+	result = isc_nm_listentls(listen_nm, (isc_nmiface_t *)&tls_listen_addr,
+				  tls_listen_accept_cb, NULL, 0, 0, NULL,
+				  server_tlsctx, &listen_sock);
+	assert_int_equal(result, ISC_R_SUCCESS);
+
+	(void)isc_nm_tlsconnect(connect_nm, (isc_nmiface_t *)&tls_connect_addr,
+				(isc_nmiface_t *)&tls_listen_addr,
+				tls_connect_connect_cb, NULL, client_tlsctx,
+				1000, 0);
+
+	while (atomic_load(&nsends) > 0) {
+		isc_thread_yield();
+	}
+
+	while (atomic_load(&cconnects) != 1 || atomic_load(&ssends) != 0 ||
+	       atomic_load(&sreads) != 1 || atomic_load(&creads) != 0 ||
+	       atomic_load(&csends) != 1)
+	{
+		isc_thread_yield();
+	}
+
+	isc_nm_stoplistening(listen_sock);
+	isc_nmsocket_close(&listen_sock);
+	assert_null(listen_sock);
+	isc_nm_closedown(connect_nm);
+
+	X(cconnects);
+	X(csends);
+	X(creads);
+	X(ctimeouts);
+	X(sreads);
+	X(ssends);
+
+	assert_int_equal(atomic_load(&cconnects), 1);
+	assert_int_equal(atomic_load(&csends), 1);
+	assert_int_equal(atomic_load(&creads), 0);
+	assert_int_equal(atomic_load(&ctimeouts), 0);
+	assert_int_equal(atomic_load(&sreads), 1);
+	assert_int_equal(atomic_load(&ssends), 0);
+}
+
+static void
+tls_recv_two(void **state) {
+	isc_nm_t **nm = (isc_nm_t **)*state;
+	isc_nm_t *listen_nm = nm[0];
+	isc_nm_t *connect_nm = nm[1];
+	isc_result_t result = ISC_R_SUCCESS;
+	isc_nmsocket_t *listen_sock = NULL;
+	isc_sockaddr_t tls_connect_addr;
+
+	tls_connect_addr = (isc_sockaddr_t){ .length = 0 };
+	isc_sockaddr_fromin6(&tls_connect_addr, &in6addr_loopback, 0);
+
+	atomic_store(&nsends, 2);
+
+	result = isc_nm_listentls(listen_nm, (isc_nmiface_t *)&tls_listen_addr,
+				  tls_listen_accept_cb, NULL, 0, 0, NULL,
+				  server_tlsctx, &listen_sock);
+	assert_int_equal(result, ISC_R_SUCCESS);
+
+	result = isc_nm_tlsconnect(
+		connect_nm, (isc_nmiface_t *)&tls_connect_addr,
+		(isc_nmiface_t *)&tls_listen_addr, tls_connect_connect_cb, NULL,
+		client_tlsctx, 100000, 0);
+	assert_int_equal(result, ISC_R_SUCCESS);
+
+	while (atomic_load(&nsends) > 0) {
+		isc_thread_yield();
+	}
+
+	while (atomic_load(&sreads) < 2 || atomic_load(&ssends) < 1 ||
+	       atomic_load(&csends) < 2 || atomic_load(&creads) < 1)
+	{
+		isc_thread_yield();
+	}
+
+	isc_nm_stoplistening(listen_sock);
+	isc_nmsocket_close(&listen_sock);
+	assert_null(listen_sock);
+	isc_nm_closedown(connect_nm);
+
+	X(cconnects);
+	X(csends);
+	X(creads);
+	X(ctimeouts);
+	X(sreads);
+	X(ssends);
+
+	assert_int_equal(atomic_load(&cconnects), 1);
+	assert_true(atomic_load(&csends) >= 2);
+	assert_int_equal(atomic_load(&creads), 1);
+	assert_int_equal(atomic_load(&ctimeouts), 0);
+	assert_true(atomic_load(&sreads) >= 2);
+	assert_int_equal(atomic_load(&ssends), 1);
+}
+
+static void
+tls_recv_send(void **state) {
+	isc_nm_t **nm = (isc_nm_t **)*state;
+	isc_nm_t *listen_nm = nm[0];
+	isc_nm_t *connect_nm = nm[1];
+	isc_result_t result = ISC_R_SUCCESS;
+	isc_nmsocket_t *listen_sock = NULL;
+	size_t nthreads = ISC_MAX(ISC_MIN(workers, 32), 1);
+	isc_thread_t threads[32] = { 0 };
+
+	if (!reuse_supported) {
+		skip();
+		return;
+	}
+
+	result = isc_nm_listentls(listen_nm, (isc_nmiface_t *)&tls_listen_addr,
+				  tls_listen_accept_cb, NULL, 0, 0, NULL,
+				  server_tlsctx, &listen_sock);
+	assert_int_equal(result, ISC_R_SUCCESS);
+
+	for (size_t i = 0; i < nthreads; i++) {
+		isc_thread_create(tls_connect_thread, connect_nm, &threads[i]);
+	}
+
+	for (size_t i = 0; i < nthreads; i++) {
+		isc_thread_join(threads[i], NULL);
+	}
+
+	isc_nm_closedown(connect_nm);
+	isc_nm_stoplistening(listen_sock);
+	isc_nmsocket_close(&listen_sock);
+	assert_null(listen_sock);
+
+	X(cconnects);
+	X(csends);
+	X(creads);
+	X(ctimeouts);
+	X(sreads);
+	X(ssends);
+
+	CHECK_RANGE_FULL(csends);
+	CHECK_RANGE_FULL(creads);
+	CHECK_RANGE_FULL(sreads);
+	CHECK_RANGE_FULL(ssends);
+}
+
+static void
+tls_recv_half_send(void **state) {
+	isc_nm_t **nm = (isc_nm_t **)*state;
+	isc_nm_t *listen_nm = nm[0];
+	isc_nm_t *connect_nm = nm[1];
+	isc_result_t result = ISC_R_SUCCESS;
+	isc_nmsocket_t *listen_sock = NULL;
+	size_t nthreads = ISC_MAX(ISC_MIN(workers, 32), 1);
+	isc_thread_t threads[32] = { 0 };
+
+	if (!reuse_supported) {
+		skip();
+		return;
+	}
+
+	result = isc_nm_listentls(listen_nm, (isc_nmiface_t *)&tls_listen_addr,
+				  tls_listen_accept_cb, NULL, 0, 0, NULL,
+				  server_tlsctx, &listen_sock);
+	assert_int_equal(result, ISC_R_SUCCESS);
+
+	for (size_t i = 0; i < nthreads; i++) {
+		isc_thread_create(tls_connect_thread, connect_nm, &threads[i]);
+	}
+
+	while (atomic_load(&nsends) >= (NSENDS * NWRITES) / 2) {
+		isc_thread_yield();
+	}
+
+	isc_nm_closedown(connect_nm);
+
+	for (size_t i = 0; i < nthreads; i++) {
+		isc_thread_join(threads[i], NULL);
+	}
+
+	isc_nm_stoplistening(listen_sock);
+	isc_nmsocket_close(&listen_sock);
+	assert_null(listen_sock);
+
+	X(cconnects);
+	X(csends);
+	X(creads);
+	X(ctimeouts);
+	X(sreads);
+	X(ssends);
+
+	CHECK_RANGE_HALF(csends);
+	CHECK_RANGE_HALF(creads);
+	CHECK_RANGE_HALF(sreads);
+	CHECK_RANGE_HALF(ssends);
+}
+
+static void
+tls_half_recv_send(void **state) {
+	isc_nm_t **nm = (isc_nm_t **)*state;
+	isc_nm_t *listen_nm = nm[0];
+	isc_nm_t *connect_nm = nm[1];
+	isc_result_t result = ISC_R_SUCCESS;
+	isc_nmsocket_t *listen_sock = NULL;
+	size_t nthreads = ISC_MAX(ISC_MIN(workers, 32), 1);
+	isc_thread_t threads[32] = { 0 };
+
+	if (!reuse_supported) {
+		skip();
+		return;
+	}
+
+	result = isc_nm_listentls(listen_nm, (isc_nmiface_t *)&tls_listen_addr,
+				  tls_listen_accept_cb, NULL, 0, 0, NULL,
+				  server_tlsctx, &listen_sock);
+	assert_int_equal(result, ISC_R_SUCCESS);
+
+	for (size_t i = 0; i < nthreads; i++) {
+		isc_thread_create(tls_connect_thread, connect_nm, &threads[i]);
+	}
+
+	while (atomic_load(&nsends) >= (NSENDS * NWRITES) / 2) {
+		isc_thread_yield();
+	}
+
+	isc_nm_stoplistening(listen_sock);
+	isc_nmsocket_close(&listen_sock);
+	assert_null(listen_sock);
+
+	for (size_t i = 0; i < nthreads; i++) {
+		isc_thread_join(threads[i], NULL);
+	}
+
+	isc_nm_closedown(connect_nm);
+
+	X(cconnects);
+	X(csends);
+	X(creads);
+	X(ctimeouts);
+	X(sreads);
+	X(ssends);
+
+	CHECK_RANGE_HALF(csends);
+	CHECK_RANGE_HALF(creads);
+	CHECK_RANGE_HALF(sreads);
+	CHECK_RANGE_HALF(ssends);
+}
+
+static void
+tls_half_recv_half_send(void **state) {
+	isc_nm_t **nm = (isc_nm_t **)*state;
+	isc_nm_t *listen_nm = nm[0];
+	isc_nm_t *connect_nm = nm[1];
+	isc_result_t result = ISC_R_SUCCESS;
+	isc_nmsocket_t *listen_sock = NULL;
+	size_t nthreads = ISC_MAX(ISC_MIN(workers, 32), 1);
+	isc_thread_t threads[32] = { 0 };
+
+	if (!reuse_supported) {
+		skip();
+		return;
+	}
+
+	result = isc_nm_listentls(listen_nm, (isc_nmiface_t *)&tls_listen_addr,
+				  tls_listen_accept_cb, NULL, 0, 0, NULL,
+				  server_tlsctx, &listen_sock);
+	assert_int_equal(result, ISC_R_SUCCESS);
+
+	for (size_t i = 0; i < nthreads; i++) {
+		isc_thread_create(tls_connect_thread, connect_nm, &threads[i]);
+	}
+
+	while (atomic_load(&nsends) >= (NSENDS * NWRITES) / 2) {
+		isc_thread_yield();
+	}
+
+	isc_nm_closedown(connect_nm);
+	isc_nm_stoplistening(listen_sock);
+	isc_nmsocket_close(&listen_sock);
+	assert_null(listen_sock);
+
+	for (size_t i = 0; i < nthreads; i++) {
+		isc_thread_join(threads[i], NULL);
+	}
+
+	X(cconnects);
+	X(csends);
+	X(creads);
+	X(ctimeouts);
+	X(sreads);
+	X(ssends);
+
+	CHECK_RANGE_HALF(csends);
+	CHECK_RANGE_HALF(creads);
+	CHECK_RANGE_HALF(sreads);
+	CHECK_RANGE_HALF(ssends);
+}
+
+/* TCP Listener */
+
+/*
+ * TODO:
+ * 1. write a timeout test
+ * 2. write a test with quota
+ */
+
+static void
+tls_listen_read_cb(isc_nmhandle_t *handle, isc_result_t eresult,
+		   isc_region_t *region, void *cbarg);
+
+static void
+tls_listen_send_cb(isc_nmhandle_t *handle, isc_result_t eresult, void *cbarg) {
+	UNUSED(eresult);
+	UNUSED(cbarg);
+
+	assert_non_null(handle);
+
+	if (eresult == ISC_R_SUCCESS) {
+		atomic_fetch_add(&ssends, 1);
+		isc_nm_resumeread(handle);
+	} else {
+		isc_nm_cancelread(handle);
+	}
+}
+
+static void
+tls_listen_read_cb(isc_nmhandle_t *handle, isc_result_t eresult,
+		   isc_region_t *region, void *cbarg) {
+	uint64_t magic = 0;
+
+	UNUSED(cbarg);
+
+	assert_non_null(handle);
+
+	if (eresult != ISC_R_SUCCESS) {
+		goto unref;
+	}
+
+	atomic_fetch_add(&sreads, 1);
+
+	memmove(tls_buffer_storage + tls_buffer_length, region->base,
+		region->length);
+	tls_buffer_length += region->length;
+
+	while (tls_buffer_length >= sizeof(magic)) {
+		memmove(&magic, tls_buffer_storage, sizeof(magic));
+		assert_true(magic == stop_magic || magic == send_magic);
+
+		tls_buffer_length -= sizeof(magic);
+		memmove(tls_buffer_storage, tls_buffer_storage + sizeof(magic),
+			tls_buffer_length);
+
+		if (magic == send_magic) {
+			isc_nm_send(handle, region, tls_listen_send_cb, NULL);
+			return;
+		} else if (magic == stop_magic) {
+			/* We are done, so we don't send anything back */
+			/* There should be no more packets in the buffer */
+			assert_int_equal(tls_buffer_length, 0);
+		}
+	}
+
+unref:
+	isc_nmhandle_detach(&handle);
+}
+
+static isc_result_t
+tls_listen_accept_cb(isc_nmhandle_t *handle, isc_result_t result, void *cbarg) {
+	isc_nmhandle_t *readhandle = NULL;
+
+	UNUSED(cbarg);
+
+	if (result != ISC_R_SUCCESS) {
+		return (result);
+	}
+
+	tls_buffer_length = 0;
+
+	/* atomic_fetch_add(&saccept, 1); */
+
+	isc_nmhandle_attach(handle, &readhandle);
+	isc_nm_read(handle, tls_listen_read_cb, NULL);
+
+	return (ISC_R_SUCCESS);
+}
+
+int
+main(void) {
+	const struct CMUnitTest tests[] = {
+		cmocka_unit_test_setup_teardown(tls_noop, nm_setup,
+						nm_teardown),
+		cmocka_unit_test_setup_teardown(tls_noresponse, nm_setup,
+						nm_teardown),
+		cmocka_unit_test_setup_teardown(tls_recv_one, nm_setup,
+						nm_teardown),
+		cmocka_unit_test_setup_teardown(tls_recv_two, nm_setup,
+						nm_teardown),
+		cmocka_unit_test_setup_teardown(tls_recv_send, nm_setup,
+						nm_teardown),
+		cmocka_unit_test_setup_teardown(tls_recv_half_send, nm_setup,
+						nm_teardown),
+		cmocka_unit_test_setup_teardown(tls_half_recv_send, nm_setup,
+						nm_teardown),
+		cmocka_unit_test_setup_teardown(tls_half_recv_half_send,
+						nm_setup, nm_teardown),
+	};
+
+	return (cmocka_run_group_tests(tests, _setup, _teardown));
+}
+
+#else /* HAVE_CMOCKA */
+
+#include <stdio.h>
+
+int
+main(void) {
+	printf("1..0 # Skipped: cmocka not available\n");
+	return (SKIPPED_TEST_EXIT_CODE);
+}
+
+#endif /* if HAVE_CMOCKA */

util/copyrights

@@ -2014,6 +2014,7 @@
 ./lib/isc/tests/testdata/file/keep		X	2014,2018,2019,2020,2021
 ./lib/isc/tests/time_test.c			C	2014,2015,2016,2018,2019,2020,2021
 ./lib/isc/tests/timer_test.c			C	2018,2019,2020,2021
+./lib/isc/tests/tls_test.c			C	2021
 ./lib/isc/tests/uv_wrap.h			C	2020,2021
 ./lib/isc/timer.c				C	1998,1999,2000,2001,2002,2004,2005,2007,2008,2009,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020,2021
 ./lib/isc/tls.c					C	2021