diff --git a/lib/isc/netmgr/tlsstream.c b/lib/isc/netmgr/tlsstream.c
index f8a0bc6dc2..4a2497d526 100644
--- a/lib/isc/netmgr/tlsstream.c
+++ b/lib/isc/netmgr/tlsstream.c
@@ -687,10 +687,9 @@ tlslisten_acceptcb(isc_nmhandle_t *handle, isc_result_t result, void *cbarg) {
 	REQUIRE(VALID_NMSOCK(tlslistensock));
 	REQUIRE(tlslistensock->type == isc_nm_tlslistener);
 
-	if (isc__nmsocket_closing(handle->sock) ||
-	    isc__nmsocket_closing(tlslistensock) ||
-	    !atomic_load(&tlslistensock->listening))
-	{
+	if (isc__nm_closing(handle->sock->worker)) {
+		return (ISC_R_SHUTTINGDOWN);
+	} else if (isc__nmsocket_closing(handle->sock)) {
 		return (ISC_R_CANCELED);
 	}