TLS stream/DoH: implement TLS client session resumption

This commit extends TLS stream code and DoH code with TLS client
session resumption support implemented on top of the TLS client
session cache.

bin/dig/dighost.c

@@ -3020,7 +3020,7 @@ start_tcp(dig_query_t *query) {
 			isc_nm_httpconnect(netmgr, &localaddr, &query->sockaddr,
 					   uri, !query->lookup->https_get,
 					   tcp_connected, connectquery, tlsctx,
-					   local_timeout);
+					   sess_cache, local_timeout);
 #endif
 		} else {
 			isc_nm_tcpdnsconnect(netmgr, &localaddr,

bin/tests/test_client.c

@@ -428,7 +428,7 @@ run(void) {
 		}
 		isc_nm_httpconnect(netmgr, &sockaddr_local, &sockaddr_remote,
 				   req_url, is_post, connect_cb, NULL, tls_ctx,
-				   timeout);
+				   NULL, timeout);
 	} break;
 #endif
 	default:

lib/isc/include/isc/netmgr.h

@@ -523,12 +523,15 @@ isc_nm_listentls(isc_nm_t *mgr, uint32_t workers, isc_sockaddr_t *iface,
 void
 isc_nm_tlsconnect(isc_nm_t *mgr, isc_sockaddr_t *local, isc_sockaddr_t *peer,
 		  isc_nm_cb_t cb, void *cbarg, isc_tlsctx_t *ctx,
+		  isc_tlsctx_client_session_cache_t *client_sess_cache,
 		  unsigned int			     timeout);
 
 void
 isc_nm_httpconnect(isc_nm_t *mgr, isc_sockaddr_t *local, isc_sockaddr_t *peer,
 		   const char *uri, bool POST, isc_nm_cb_t cb, void *cbarg,
-		   isc_tlsctx_t *ctx, unsigned int timeout);
+		   isc_tlsctx_t			*ctx,
+		   isc_tlsctx_client_session_cache_t *client_sess_cache,
+		   unsigned int			      timeout);
 
 isc_result_t
 isc_nm_listenhttp(isc_nm_t *mgr, uint32_t workers, isc_sockaddr_t *iface,

lib/isc/netmgr/http.c

@@ -1426,7 +1426,9 @@ error:
 void
 isc_nm_httpconnect(isc_nm_t *mgr, isc_sockaddr_t *local, isc_sockaddr_t *peer,
 		   const char *uri, bool post, isc_nm_cb_t cb, void *cbarg,
-		   isc_tlsctx_t *tlsctx, unsigned int timeout) {
+		   isc_tlsctx_t *tlsctx,
+		   isc_tlsctx_client_session_cache_t *client_sess_cache,
+		   unsigned int timeout) {
 	isc_sockaddr_t local_interface;
 	isc_nmsocket_t *sock = NULL;
 
@@ -1487,7 +1489,7 @@ isc_nm_httpconnect(isc_nm_t *mgr, isc_sockaddr_t *local, isc_sockaddr_t *peer,
 
 	if (tlsctx != NULL) {
 		isc_nm_tlsconnect(mgr, local, peer, transport_connect_cb, sock,
-				  tlsctx, timeout);
+				  tlsctx, client_sess_cache, timeout);
 	} else {
 		isc_nm_tcpconnect(mgr, local, peer, transport_connect_cb, sock,
 				  timeout);

lib/isc/netmgr/netmgr-int.h

@@ -963,6 +963,8 @@ struct isc_nmsocket {
 		isc_tlsctx_t **listener_tls_ctx; /*%< A context reference per
 						    worker */
 		size_t n_listener_tls_ctx;
+		isc_tlsctx_client_session_cache_t *client_sess_cache;
+		bool client_session_saved;
 		isc_nmsocket_t *tlslistener;
 		isc_nmsocket_t *tlssocket;
 		atomic_bool result_updated;
@@ -2145,3 +2147,6 @@ isc__nm_udp_freebind(uv_udp_t *handle, const struct sockaddr *addr,
 int
 isc__nm_tcp_freebind(uv_tcp_t *handle, const struct sockaddr *addr,
 		     unsigned int flags);
+
+void
+isc__nmsocket_log_tls_session_reuse(isc_nmsocket_t *sock, isc_tls_t *tls);

lib/isc/netmgr/netmgr.c

@@ -3361,6 +3361,27 @@ isc_nmsocket_set_tlsctx(isc_nmsocket_t *listener, isc_tlsctx_t *tlsctx) {
 	};
 }
 
+void
+isc__nmsocket_log_tls_session_reuse(isc_nmsocket_t *sock, isc_tls_t *tls) {
+	const int log_level = ISC_LOG_DEBUG(1);
+	char client_sabuf[ISC_SOCKADDR_FORMATSIZE];
+	char local_sabuf[ISC_SOCKADDR_FORMATSIZE];
+
+	REQUIRE(tls != NULL);
+
+	if (!isc_log_wouldlog(isc_lctx, log_level)) {
+		return;
+	};
+
+	isc_sockaddr_format(&sock->peer, client_sabuf, sizeof(client_sabuf));
+	isc_sockaddr_format(&sock->iface, local_sabuf, sizeof(local_sabuf));
+	isc_log_write(isc_lctx, ISC_LOGCATEGORY_GENERAL, ISC_LOGMODULE_NETMGR,
+		      log_level, "TLS %s session %s for %s on %s",
+		      SSL_is_server(tls) ? "server" : "client",
+		      SSL_session_reused(tls) ? "resumed" : "created",
+		      client_sabuf, local_sabuf);
+}
+
 #ifdef NETMGR_TRACE
 /*
  * Dump all active sockets in netmgr. We output to stderr

lib/isc/netmgr/tlsstream.c

@@ -85,6 +85,12 @@ tls_cleanup_listener_tlsctx(isc_nmsocket_t *listener);
 static isc_tlsctx_t *
 tls_get_listener_tlsctx(isc_nmsocket_t *listener, const int tid);
 
+static void
+tls_keep_client_tls_session(isc_nmsocket_t *sock);
+
+static void
+tls_try_shutdown(isc_tls_t *tls, const bool quite);
+
 /*
  * The socket is closing, outerhandle has been detached, listener is
  * inactive, or the netmgr is closing: any operation on it should abort
@@ -127,6 +133,10 @@ tls_senddone(isc_nmhandle_t *handle, isc_result_t eresult, void *cbarg) {
 	tlssock = send_req->tlssock;
 	send_req->tlssock = NULL;
 
+	if (finish) {
+		tls_try_shutdown(tlssock->tlsstream.tls, true);
+	}
+
 	if (send_req->cb != NULL) {
 		INSIST(VALID_NMHANDLE(tlssock->statichandle));
 		send_req->cb(send_req->handle, eresult, send_req->cbarg);
@@ -240,10 +250,9 @@ tls_send_outgoing(isc_nmsocket_t *sock, bool finish, isc_nmhandle_t *tlshandle,
 		return (0);
 	}
 
-	if (finish && (SSL_get_shutdown(sock->tlsstream.tls) &
-		       SSL_SENT_SHUTDOWN) != SSL_SENT_SHUTDOWN)
-	{
-		(void)SSL_shutdown(sock->tlsstream.tls);
+	if (finish) {
+		tls_try_shutdown(sock->tlsstream.tls, false);
+		tls_keep_client_tls_session(sock);
 	}
 
 	pending = BIO_pending(sock->tlsstream.bio_out);
@@ -292,22 +301,21 @@ tls_process_outgoing(isc_nmsocket_t *sock, bool finish,
 		     isc__nm_uvreq_t *send_data) {
 	int pending;
 
-	/* Data from TLS to network */
-	if (send_data != NULL) {
-		pending = tls_send_outgoing(sock, finish, send_data->handle,
-					    send_data->cb.send,
-					    send_data->cbarg);
-	} else {
-		bool received_shutdown =
-			((SSL_get_shutdown(sock->tlsstream.tls) &
+	bool received_shutdown = ((SSL_get_shutdown(sock->tlsstream.tls) &
 				   SSL_RECEIVED_SHUTDOWN) != 0);
 	bool sent_shutdown = ((SSL_get_shutdown(sock->tlsstream.tls) &
 			       SSL_SENT_SHUTDOWN) != 0);
 
 	if (received_shutdown && !sent_shutdown) {
 		finish = true;
-			(void)SSL_shutdown(sock->tlsstream.tls);
 	}
+
+	/* Data from TLS to network */
+	if (send_data != NULL) {
+		pending = tls_send_outgoing(sock, finish, send_data->handle,
+					    send_data->cb.send,
+					    send_data->cbarg);
+	} else {
 		pending = tls_send_outgoing(sock, finish, NULL, NULL, NULL);
 	}
 
@@ -330,6 +338,7 @@ tls_try_handshake(isc_nmsocket_t *sock) {
 		isc_result_t result = ISC_R_SUCCESS;
 		INSIST(SSL_is_init_finished(sock->tlsstream.tls) == 1);
 		INSIST(sock->statichandle == NULL);
+		isc__nmsocket_log_tls_session_reuse(sock, sock->tlsstream.tls);
 		tlshandle = isc__nmhandle_get(sock, &sock->peer, &sock->iface);
 		if (sock->tlsstream.server) {
 			sock->listener->accept_cb(tlshandle, result,
@@ -884,7 +893,8 @@ tcp_connected(isc_nmhandle_t *handle, isc_result_t result, void *cbarg);
 
 void
 isc_nm_tlsconnect(isc_nm_t *mgr, isc_sockaddr_t *local, isc_sockaddr_t *peer,
-		  isc_nm_cb_t cb, void *cbarg, SSL_CTX *ctx,
+		  isc_nm_cb_t cb, void *cbarg, isc_tlsctx_t *ctx,
+		  isc_tlsctx_client_session_cache_t *client_sess_cache,
 		  unsigned int timeout) {
 	isc_nmsocket_t *nsock = NULL;
 #if defined(NETMGR_TRACE) && defined(NETMGR_TRACE_VERBOSE)
@@ -901,6 +911,13 @@ isc_nm_tlsconnect(isc_nm_t *mgr, isc_sockaddr_t *local, isc_sockaddr_t *peer,
 	nsock->connect_cbarg = cbarg;
 	nsock->connect_timeout = timeout;
 	isc_tlsctx_attach(ctx, &nsock->tlsstream.ctx);
+	atomic_init(&nsock->client, true);
+	if (client_sess_cache != NULL) {
+		INSIST(isc_tlsctx_client_session_cache_getctx(
+			       client_sess_cache) == ctx);
+		isc_tlsctx_client_session_cache_attach(
+			client_sess_cache, &nsock->tlsstream.client_sess_cache);
+	}
 
 	isc_nm_tcpconnect(mgr, local, peer, tcp_connected, nsock,
 			  nsock->connect_timeout);
@@ -943,6 +960,12 @@ tcp_connected(isc_nmhandle_t *handle, isc_result_t result, void *cbarg) {
 	isc_nmhandle_attach(handle, &tlssock->outerhandle);
 	atomic_store(&tlssock->active, true);
 
+	if (tlssock->tlsstream.client_sess_cache != NULL) {
+		isc_tlsctx_client_session_cache_reuse_sockaddr(
+			tlssock->tlsstream.client_sess_cache, &tlssock->peer,
+			tlssock->tlsstream.tls);
+	}
+
 	/*
 	 * Hold a reference to tlssock in the TCP socket: it will
 	 * detached in isc__nm_tls_cleanup_data().
@@ -1020,15 +1043,26 @@ isc__nm_tls_cleanup_data(isc_nmsocket_t *sock) {
 	} else if (sock->type == isc_nm_tlslistener) {
 		tls_cleanup_listener_tlsctx(sock);
 	} else if (sock->type == isc_nm_tlssocket) {
-		if (sock->tlsstream.ctx != NULL) {
-			isc_tlsctx_free(&sock->tlsstream.ctx);
-		}
 		if (sock->tlsstream.tls != NULL) {
+			/*
+			 * Let's shutdown the TLS session properly so that the
+			 * session will remain resumable, if required.
+			 */
+			tls_try_shutdown(sock->tlsstream.tls, true);
+			tls_keep_client_tls_session(sock);
 			isc_tls_free(&sock->tlsstream.tls);
 			/* These are destroyed when we free SSL */
 			sock->tlsstream.bio_out = NULL;
 			sock->tlsstream.bio_in = NULL;
 		}
+		if (sock->tlsstream.ctx != NULL) {
+			isc_tlsctx_free(&sock->tlsstream.ctx);
+		}
+		if (sock->tlsstream.client_sess_cache != NULL) {
+			INSIST(atomic_load(&sock->client));
+			isc_tlsctx_client_session_cache_detach(
+				&sock->tlsstream.client_sess_cache);
+		}
 	} else if (sock->type == isc_nm_tcpsocket &&
 		   sock->tlsstream.tlssocket != NULL) {
 		/*
@@ -1158,3 +1192,30 @@ isc__nm_async_tls_set_tlsctx(isc_nmsocket_t *listener, isc_tlsctx_t *tlsctx,
 	isc_tlsctx_free(&listener->tlsstream.listener_tls_ctx[tid]);
 	isc_tlsctx_attach(tlsctx, &listener->tlsstream.listener_tls_ctx[tid]);
 }
+
+static void
+tls_keep_client_tls_session(isc_nmsocket_t *sock) {
+	/*
+	 * Ensure that the isc_tls_t is being accessed from
+	 * within the worker thread the socket is bound to.
+	 */
+	REQUIRE(sock->tid == isc_nm_tid());
+	if (sock->tlsstream.client_sess_cache != NULL &&
+	    sock->tlsstream.client_session_saved == false)
+	{
+		INSIST(atomic_load(&sock->client));
+		isc_tlsctx_client_session_cache_keep_sockaddr(
+			sock->tlsstream.client_sess_cache, &sock->peer,
+			sock->tlsstream.tls);
+		sock->tlsstream.client_session_saved = true;
+	}
+}
+
+static void
+tls_try_shutdown(isc_tls_t *tls, const bool force) {
+	if (force) {
+		(void)SSL_set_shutdown(tls, SSL_SENT_SHUTDOWN);
+	} else if ((SSL_get_shutdown(tls) & SSL_SENT_SHUTDOWN) == 0) {
+		(void)SSL_shutdown(tls);
+	}
+}

lib/isc/tests/doh_test.c

@@ -80,6 +80,7 @@ static atomic_bool slowdown = false;
 static atomic_bool use_TLS = false;
 static isc_tlsctx_t *server_tlsctx = NULL;
 static isc_tlsctx_t *client_tlsctx = NULL;
+static isc_tlsctx_client_session_cache_t *client_sess_cache = NULL;
 
 static isc_quota_t listener_quota;
 static atomic_bool check_listener_quota = false;
@@ -169,7 +170,8 @@ connect_send_request(isc_nm_t *mgr, const char *uri, bool post,
 	}
 
 	isc_nm_httpconnect(mgr, NULL, &tcp_listen_addr, uri, post,
-			   connect_send_cb, data, ctx, timeout);
+			   connect_send_cb, data, ctx, client_sess_cache,
+			   timeout);
 }
 
 static int
@@ -334,6 +336,9 @@ nm_setup(void **state) {
 	client_tlsctx = NULL;
 	isc_tlsctx_createclient(&client_tlsctx);
 	isc_tlsctx_enable_http2client_alpn(client_tlsctx);
+	client_sess_cache = isc_tlsctx_client_session_cache_new(
+		test_mctx, client_tlsctx,
+		ISC_TLSCTX_CLIENT_SESSION_CACHE_DEFAULT_SIZE);
 
 	isc_quota_init(&listener_quota, 0);
 	atomic_store(&check_listener_quota, false);
@@ -363,6 +368,8 @@ nm_teardown(void **state) {
 		isc_tlsctx_free(&client_tlsctx);
 	}
 
+	isc_tlsctx_client_session_cache_detach(&client_sess_cache);
+
 	isc_quota_destroy(&listener_quota);
 
 	isc_nm_http_endpoints_detach(&endpoints);
@@ -666,7 +673,7 @@ doh_timeout_recovery(void **state) {
 			ISC_NM_HTTP_DEFAULT_PATH);
 	isc_nm_httpconnect(connect_nm, NULL, &tcp_listen_addr, req_url,
 			   atomic_load(&POST), timeout_request_cb, NULL, ctx,
-			   T_SOFT);
+			   client_sess_cache, T_SOFT);
 
 	/*
 	 * Sleep until sends reaches 5.
@@ -946,7 +953,7 @@ doh_recv_two(void **state) {
 
 	isc_nm_httpconnect(connect_nm, NULL, &tcp_listen_addr, req_url,
 			   atomic_load(&POST), doh_connect_send_two_requests_cb,
-			   NULL, ctx, 5000);
+			   NULL, ctx, client_sess_cache, 5000);
 
 	while (atomic_load(&nsends) > 0) {
 		if (atomic_load(&was_error)) {

lib/isc/tests/netmgr_test.c

@@ -58,6 +58,7 @@ static isc_sockaddr_t tcp_listen_addr;
 static isc_sockaddr_t tcp_connect_addr;
 static isc_tlsctx_t *tcp_listen_tlsctx = NULL;
 static isc_tlsctx_t *tcp_connect_tlsctx = NULL;
+static isc_tlsctx_client_session_cache_t *tcp_tlsctx_client_sess_cache = NULL;
 
 static uint64_t send_magic = 0;
 static uint64_t stop_magic = 0;
@@ -226,6 +227,10 @@ _setup(void **state __attribute__((unused))) {
 
 	isc_tlsctx_enable_dot_client_alpn(tcp_connect_tlsctx);
 
+	tcp_tlsctx_client_sess_cache = isc_tlsctx_client_session_cache_new(
+		test_mctx, tcp_connect_tlsctx,
+		ISC_TLSCTX_CLIENT_SESSION_CACHE_DEFAULT_SIZE);
+
 	return (0);
 }
 
@@ -234,6 +239,8 @@ _teardown(void **state __attribute__((unused))) {
 	isc_tlsctx_free(&tcp_connect_tlsctx);
 	isc_tlsctx_free(&tcp_listen_tlsctx);
 
+	isc_tlsctx_client_session_cache_detach(&tcp_tlsctx_client_sess_cache);
+
 	isc_test_end();
 
 	return (0);
@@ -1201,7 +1208,8 @@ stream_connect(isc_nm_cb_t cb, void *cbarg, unsigned int timeout) {
 	if (stream_use_TLS) {
 		isc_nm_tlsconnect(connect_nm, &tcp_connect_addr,
 				  &tcp_listen_addr, cb, cbarg,
-				  tcp_connect_tlsctx, timeout);
+				  tcp_connect_tlsctx,
+				  tcp_tlsctx_client_sess_cache, timeout);
 		return;
 	}
 #endif
@@ -2139,7 +2147,7 @@ static void
 tls_connect(isc_nm_t *nm) {
 	isc_nm_tlsconnect(nm, &tcp_connect_addr, &tcp_listen_addr,
 			  connect_connect_cb, NULL, tcp_connect_tlsctx,
-			  T_CONNECT);
+			  tcp_tlsctx_client_sess_cache, T_CONNECT);
 }
 
 static void