netmgr fixes needed for dispatch

- The read timer must always be stopped when reading stops.

- Read callbacks can now call isc_nm_read() again in TCP, TCPDNS and
  TLSDNS; previously this caused an assertion.

- The wrong failure code could be sent after a UDP recv failure because
  the if statements were in the wrong order. the check for a NULL
  address needs to be after the check for an error code, otherwise the
  result will always be set to ISC_R_EOF.

- When aborting a read or connect because the netmgr is shutting down,
  use ISC_R_SHUTTINGDOWN. (ISC_R_CANCELED is now reserved for when the
  read has been canceled by the caller.)

- A new function isc_nmhandle_timer_running() has been added enabling a
  callback to check whether the timer has been reset after processing a
  timeout.

- Incidental netmgr fix: always use isc__nm_closing() instead of
  referencing sock->mgr->closing directly

- Corrected a few comments that used outdated function names.

lib/isc/netmgr/tcpdns.c

@@ -107,7 +107,7 @@ tcpdns_connect_direct(isc_nmsocket_t *sock, isc__nm_uvreq_t *req) {
 	RUNTIME_CHECK(r == 0);
 
 	if (isc__nm_closing(sock)) {
-		result = ISC_R_CANCELED;
+		result = ISC_R_SHUTTINGDOWN;
 		goto error;
 	}
 
@@ -217,7 +217,11 @@ tcpdns_connect_cb(uv_connect_t *uvreq, int status) {
 	REQUIRE(VALID_NMHANDLE(req->handle));
 
 	if (isc__nmsocket_closing(sock)) {
-		/* Socket was closed midflight by isc__nm_tcpdns_shutdown() */
+		/* Network manager shutting down */
+		result = ISC_R_SHUTTINGDOWN;
+		goto error;
+	} else if (isc__nmsocket_closing(sock)) {
+		/* Connection canceled */
 		result = ISC_R_CANCELED;
 		goto error;
 	} else if (status == UV_ETIMEDOUT) {
@@ -690,8 +694,6 @@ isc__nm_tcpdns_read(isc_nmhandle_t *handle, isc_nm_recv_cb_t cb, void *cbarg) {
 
 	REQUIRE(sock->type == isc_nm_tcpdnssocket);
 	REQUIRE(sock->statichandle == handle);
-	REQUIRE(sock->tid == isc_nm_tid());
-	REQUIRE(!sock->recv_read);
 
 	sock->recv_cb = cb;
 	sock->recv_cbarg = cbarg;
@@ -729,7 +731,7 @@ isc__nm_async_tcpdnsread(isc__networker_t *worker, isc__netievent_t *ev0) {
 	REQUIRE(sock->tid == isc_nm_tid());
 
 	if (isc__nmsocket_closing(sock)) {
-		sock->reading = true;
+		atomic_store(&sock->reading, true);
 		isc__nm_failed_read_cb(sock, ISC_R_CANCELED, false);
 		return;
 	}
@@ -780,8 +782,8 @@ isc__nm_tcpdns_processbuffer(isc_nmsocket_t *sock) {
 	REQUIRE(VALID_UVREQ(req));
 
 	/*
-	 * We need to launch the resume_processing after the buffer has
-	 * been consumed, thus we need to delay the detaching the handle.
+	 * We need to launch isc__nm_resume_processing() after the buffer
+	 * has been consumed, thus we must delay detaching the handle.
 	 */
 	isc_nmhandle_attach(req->handle, &handle);
 
@@ -800,9 +802,10 @@ isc__nm_tcpdns_processbuffer(isc_nmsocket_t *sock) {
 	sock->recv_read = false;
 
 	/*
-	 * The assertion failure here means that there's a errnoneous extra
-	 * nmhandle detach happening in the callback and resume_processing gets
-	 * called while we are still processing the buffer.
+	 * An assertion failure here means that there's an erroneous
+	 * extra nmhandle detach happening in the callback and
+	 * isc__nm_resume_processing() is called while we're
+	 * processing the buffer.
 	 */
 	REQUIRE(sock->processing == false);
 	sock->processing = true;
@@ -829,7 +832,7 @@ isc__nm_tcpdns_read_cb(uv_stream_t *stream, ssize_t nread,
 
 	REQUIRE(VALID_NMSOCK(sock));
 	REQUIRE(sock->tid == isc_nm_tid());
-	REQUIRE(sock->reading);
+	REQUIRE(atomic_load(&sock->reading));
 	REQUIRE(buf != NULL);
 
 	if (isc__nmsocket_closing(sock)) {
@@ -949,7 +952,7 @@ accept_connection(isc_nmsocket_t *ssock, isc_quota_t *quota) {
 	csock->recv_cb = ssock->recv_cb;
 	csock->recv_cbarg = ssock->recv_cbarg;
 	csock->quota = quota;
-	csock->accepting = true;
+	atomic_init(&csock->accepting, true);
 
 	worker = &csock->mgr->workers[csock->tid];
 
@@ -1007,7 +1010,7 @@ accept_connection(isc_nmsocket_t *ssock, isc_quota_t *quota) {
 		goto failure;
 	}
 
-	csock->accepting = false;
+	atomic_store(&csock->accepting, false);
 
 	isc__nm_incstats(csock->mgr, csock->statsindex[STATID_ACCEPT]);
 
@@ -1056,13 +1059,15 @@ failure:
 void
 isc__nm_tcpdns_send(isc_nmhandle_t *handle, isc_region_t *region,
 		    isc_nm_cb_t cb, void *cbarg) {
-	REQUIRE(VALID_NMHANDLE(handle));
-	REQUIRE(VALID_NMSOCK(handle->sock));
-
-	isc_nmsocket_t *sock = handle->sock;
 	isc__netievent_tcpdnssend_t *ievent = NULL;
 	isc__nm_uvreq_t *uvreq = NULL;
+	isc_nmsocket_t *sock = NULL;
 
+	REQUIRE(VALID_NMHANDLE(handle));
+
+	sock = handle->sock;
+
+	REQUIRE(VALID_NMSOCK(sock));
 	REQUIRE(sock->type == isc_nm_tcpdnssocket);
 
 	uvreq = isc__nm_uvreq_get(sock->mgr, sock);
@@ -1107,24 +1112,26 @@ tcpdns_send_cb(uv_write_t *req, int status) {
  */
 void
 isc__nm_async_tcpdnssend(isc__networker_t *worker, isc__netievent_t *ev0) {
+	isc_result_t result;
 	isc__netievent_tcpdnssend_t *ievent =
 		(isc__netievent_tcpdnssend_t *)ev0;
+	isc_nmsocket_t *sock = NULL;
+	isc__nm_uvreq_t *uvreq = NULL;
+	int r, nbufs = 2;
+
+	UNUSED(worker);
 
 	REQUIRE(VALID_UVREQ(ievent->req));
 	REQUIRE(VALID_NMSOCK(ievent->sock));
 	REQUIRE(ievent->sock->type == isc_nm_tcpdnssocket);
 	REQUIRE(ievent->sock->tid == isc_nm_tid());
 
-	isc_result_t result;
-	isc_nmsocket_t *sock = ievent->sock;
-	isc__nm_uvreq_t *uvreq = ievent->req;
+	sock = ievent->sock;
+	uvreq = ievent->req;
+
 	uv_buf_t bufs[2] = { { .base = uvreq->tcplen, .len = 2 },
 			     { .base = uvreq->uvbuf.base,
 			       .len = uvreq->uvbuf.len } };
-	int nbufs = 2;
-	int r;
-
-	UNUSED(worker);
 
 	if (isc__nmsocket_closing(sock)) {
 		result = ISC_R_CANCELED;
@@ -1380,7 +1387,7 @@ isc__nm_tcpdns_shutdown(isc_nmsocket_t *sock) {
 		return;
 	}
 
-	if (sock->accepting) {
+	if (atomic_load(&sock->accepting)) {
 		return;
 	}
 
@@ -1392,7 +1399,11 @@ isc__nm_tcpdns_shutdown(isc_nmsocket_t *sock) {
 	}
 
 	if (sock->statichandle != NULL) {
-		isc__nm_failed_read_cb(sock, ISC_R_CANCELED, false);
+		if (isc__nm_closing(sock)) {
+			isc__nm_failed_read_cb(sock, ISC_R_SHUTTINGDOWN, false);
+		} else {
+			isc__nm_failed_read_cb(sock, ISC_R_CANCELED, false);
+		}
 		return;
 	}