netmgr: add shutdown function

- new function isc_nm_shutdown() shuts down all active TCP connections,
  but does not destroy the netmgr.

bin/named/main.c

@@ -939,11 +939,23 @@ create_managers(void) {
 static void
 destroy_managers(void) {
 	/*
-	 * isc_taskmgr_destroy() will block until all tasks have exited,
+	 * isc_nm_closedown() closes all active connections, freeing
+	 * attached clients and other resources and preventing new
+	 * connections from being established, but it not does not
+	 * stop all processing or destroy the netmgr yet.
+	 */
+	isc_nm_closedown(named_g_nm);
+
+	/*
+	 * isc_taskmgr_destroy() will block until all tasks have exited.
 	 */
 	isc_taskmgr_destroy(&named_g_taskmgr);
 	isc_timermgr_destroy(&named_g_timermgr);
 	isc_socketmgr_destroy(&named_g_socketmgr);
+
+	/*
+	 * At this point is safe to destroy the netmgr.
+	 */
 	isc_nm_destroy(&named_g_nm);
 }
 

lib/isc/include/isc/netmgr.h

@@ -46,7 +46,16 @@ isc_nm_destroy(isc_nm_t **mgr0);
  * for all other references to be gone.
  */
 
-/* Return thread id of current thread, or ISC_NETMGR_TID_UNKNOWN */
+void
+isc_nm_closedown(isc_nm_t *mgr);
+/*%<
+ * Close down all active connections, freeing associated resources;
+ * prevent new connections from being established. This can optionally
+ * be called prior to shutting down the netmgr, to stop all processing
+ * before shutting down the task manager.
+ */
+
+/* Return thread ID of current thread, or ISC_NETMGR_TID_UNKNOWN */
 int
 isc_nm_tid(void);
 

lib/isc/netmgr/netmgr-int.h

@@ -182,7 +182,6 @@ typedef isc__netievent__socket_t isc__netievent_tcpstoplisten_t;
 typedef isc__netievent__socket_t isc__netievent_tcpclose_t;
 typedef isc__netievent__socket_t isc__netievent_startread_t;
 typedef isc__netievent__socket_t isc__netievent_pauseread_t;
-typedef isc__netievent__socket_t isc__netievent_resumeread_t;
 typedef isc__netievent__socket_t isc__netievent_closecb_t;
 
 typedef struct isc__netievent__socket_req {
@@ -242,6 +241,12 @@ struct isc_nm {
 	atomic_uint_fast32_t	maxudp;
 	atomic_bool		paused;
 
+	/*
+	 * Acive connections are being closed and new connections are
+	 * no longer allowed.
+	 */
+	atomic_bool		closing;
+
 	/*
 	 * A worker is actively waiting for other workers, for example to
 	 * stop listening; that means no other thread can do the same thing
@@ -582,15 +587,12 @@ isc__nm_async_startread(isc__networker_t *worker, isc__netievent_t *ievent0);
 void
 isc__nm_async_pauseread(isc__networker_t *worker, isc__netievent_t *ievent0);
 void
-isc__nm_async_resumeread(isc__networker_t *worker, isc__netievent_t *ievent0);
-void
 isc__nm_async_tcpclose(isc__networker_t *worker, isc__netievent_t *ievent0);
 /*%<
  * Callback handlers for asynchronous TCP events (connect, listen,
- * stoplisten, send, read, pauseread, resumeread, close).
+ * stoplisten, send, read, pause, close).
  */
 
-
 isc_result_t
 isc__nm_tcpdns_send(isc_nmhandle_t *handle, isc_region_t *region,
 		    isc_nm_cb_t cb, void *cbarg);

lib/isc/netmgr/netmgr.c

@@ -297,26 +297,34 @@ isc_nm_detach(isc_nm_t **mgr0) {
 	}
 }
 
-
 void
-isc_nm_destroy(isc_nm_t **mgr0) {
+isc_nm_closedown(isc_nm_t *mgr) {
-	isc_nm_t *mgr = NULL;
+	REQUIRE(VALID_NM(mgr));
-	int references;
-
-	REQUIRE(mgr0 != NULL);
-	REQUIRE(VALID_NM(*mgr0));
-
-	mgr = *mgr0;
-	*mgr0 = NULL;
 
+	atomic_store(&mgr->closing, true);
 	for (size_t i = 0; i < mgr->nworkers; i++) {
 		isc__netievent_t *event = NULL;
 		event = isc__nm_get_ievent(mgr, netievent_shutdown);
 		isc__nm_enqueue_ievent(&mgr->workers[i], event);
 	}
+}
+
+void
+isc_nm_destroy(isc_nm_t **mgr0) {
+	isc_nm_t *mgr = NULL;
+
+	REQUIRE(mgr0 != NULL);
+	REQUIRE(VALID_NM(*mgr0));
+
+	mgr = *mgr0;
 
 	/*
-	 * Wait for the manager to be dereferenced elsehwere.
+	 * Close active connections.
+	 */
+	isc_nm_closedown(mgr);
+
+	/*
+	 * Wait for the manager to be dereferenced elsewhere.
 	 */
 	while (isc_refcount_current(&mgr->references) > 1) {
 #ifdef WIN32
@@ -325,11 +333,11 @@ isc_nm_destroy(isc_nm_t **mgr0) {
 			usleep(1000000);
 #endif
 	}
-	references = isc_refcount_decrement(&mgr->references);
+
-	INSIST(references > 0);
+	/*
-	if (references == 1) {
+	 * Detach final reference.
-		nm_destroy(&mgr);
+	 */
-	}
+	isc_nm_detach(mgr0);
 }
 
 void
@@ -1185,8 +1193,6 @@ isc__nm_async_closecb(isc__networker_t *worker, isc__netievent_t *ievent0) {
 
 static void
 shutdown_walk_cb(uv_handle_t *handle, void *arg) {
-	isc_nmsocket_t *sock = NULL;
-
 	UNUSED(arg);
 
 	switch(handle->type) {

lib/isc/netmgr/tcp.c

@@ -438,7 +438,9 @@ accept_connection(isc_nmsocket_t *ssock) {
 	REQUIRE(VALID_NMSOCK(ssock));
 	REQUIRE(ssock->tid == isc_nm_tid());
 
-	if (!atomic_load_relaxed(&ssock->active)) {
+	if (!atomic_load_relaxed(&ssock->active) ||
+	    atomic_load_relaxed(&ssock->mgr->closing))
+	{
 		/* We're closing, bail */
 		return (ISC_R_CANCELED);
 	}
@@ -693,5 +695,7 @@ void
 isc__nm_tcp_shutdown(isc_nmsocket_t *sock) {
 	REQUIRE(VALID_NMSOCK(sock));
 
-	sock->rcb.recv(sock->tcphandle, NULL, sock->rcbarg);
+	if (sock->type == isc_nm_tcpsocket && sock->tcphandle != NULL) {
+		sock->rcb.recv(sock->tcphandle, NULL, sock->rcbarg);
+	}
 }

lib/isc/win32/libisc.def.in

@@ -444,6 +444,7 @@ isc_nmhandle_peeraddr
 isc_nmhandle_ref
 isc_nmhandle_setdata
 isc_nmhandle_unref
+isc_nm_closedown
 isc_nm_destroy
 isc_nm_detach
 isc_nm_listentcpdns