Add SO_REUSEPORT and SO_INCOMING_CPU helper functions

The setting of SO_REUSE**** and SO_INCOMING_CPU have been moved into a
separate helper functions.

lib/isc/netmgr/netmgr-int.h

@@ -838,3 +838,15 @@ isc__nm_socket_freebind(const uv_handle_t *handle);
 /*%<
  * Set the IP_FREEBIND (or equivalent) socket option on the uv_handle
  */
+
+isc_result_t
+isc__nm_socket_reuseport(uv_os_fd_t fd);
+/*%<
+ * Set the SO_REUSEPORT (or equivalent) socket option on the fd
+ */
+
+isc_result_t
+isc__nm_socket_incoming_cpu(uv_os_fd_t fd);
+/*%<
+ * Set the SO_INCOMING_CPU socket option on the fd if available
+ */

lib/isc/netmgr/netmgr.c

@@ -1625,12 +1625,68 @@ isc__nm_socket_freebind(const uv_handle_t *handle) {
 	}
 #else
 	UNUSED(handle);
-	UNUSED(fd);
 	result = ISC_R_NOTIMPLEMENTED;
 #endif
 	return (result);
 }
 
+isc_result_t
+isc__nm_socket_reuseport(uv_os_fd_t fd) {
+	/*
+	 * This is SO_REUSE**** hell:
+	 *
+	 * Generally, the SO_REUSEADDR socket option allows reuse of
+	 * local addresses.  On Windows, it also allows a socket to
+	 * forcibly bind to a port in use by another socket.
+	 *
+	 * On Linux, SO_REUSEPORT socket option allows sockets to be
+	 * bound to an identical socket address. For UDP sockets, the
+	 * use of this option can provide better distribution of
+	 * incoming datagrams to multiple processes (or threads) as
+	 * compared to the traditional technique of having multiple
+	 * processes compete to receive datagrams on the same socket.
+	 *
+	 * On FreeBSD 12+, the same thing is achieved with SO_REUSEPORT_LB.
+	 *
+	 */
+	isc_result_t result = ISC_R_NOTIMPLEMENTED;
+#if defined(SO_REUSEADDR)
+	if (setsockopt_on(fd, SOL_SOCKET, SO_REUSEADDR) == -1) {
+		return (ISC_R_FAILURE);
+	} else {
+		result = ISC_R_SUCCESS;
+	}
+#endif
+#if defined(SO_REUSEPORT_LB)
+	if (setsockopt_on(fd, SOL_SOCKET, SO_REUSEPORT_LB) == -1) {
+		return (ISC_R_FAILURE);
+	} else {
+		result = ISC_R_SUCCESS;
+	}
+#elif defined(SO_REUSEPORT)
+	if (setsockopt_on(fd, SOL_SOCKET, SO_REUSEPORT) == -1) {
+		return (ISC_R_FAILURE);
+	} else {
+		result = ISC_R_SUCCESS;
+	}
+#endif
+	return (result);
+}
+
+isc_result_t
+isc__nm_socket_incoming_cpu(uv_os_fd_t fd) {
+#ifdef SO_INCOMING_CPU
+	if (setsockopt_on(fd, SOL_SOCKET, SO_INCOMING_CPU) == -1) {
+		return (ISC_R_FAILURE);
+	} else {
+		return (ISC_R_SUCCESS);
+	}
+#else
+	UNUSED(fd);
+#endif
+	return (ISC_R_NOTIMPLEMENTED);
+}
+
 #ifdef NETMGR_TRACE
 /*
  * Dump all active sockets in netmgr. We output to stderr

lib/isc/netmgr/udp.c

@@ -65,8 +65,8 @@ isc_nm_listenudp(isc_nm_t *mgr, isc_nmiface_t *iface, isc_nm_recv_cb_t cb,
 	nsock->extrahandlesize = extrahandlesize;
 
 	for (size_t i = 0; i < mgr->nworkers; i++) {
+		isc_result_t result;
 		uint16_t family = iface->addr.type.sa.sa_family;
-		int res = 0;
 
 		isc__netievent_udplisten_t *ievent = NULL;
 		isc_nmsocket_t *csock = &nsock->children[i];
@@ -82,46 +82,16 @@ isc_nm_listenudp(isc_nm_t *mgr, isc_nmiface_t *iface, isc_nm_recv_cb_t cb,
 		csock->fd = socket(family, SOCK_DGRAM, 0);
 		RUNTIME_CHECK(csock->fd >= 0);
 
-		/*
+		result = isc__nm_socket_reuseport(csock->fd);
-		 * This is SO_REUSE**** hell:
+		RUNTIME_CHECK(result == ISC_R_SUCCESS ||
-		 *
+			      result == ISC_R_NOTIMPLEMENTED);
-		 * Generally, the SO_REUSEADDR socket option allows reuse of
-		 * local addresses.  On Windows, it also allows a socket to
-		 * forcibly bind to a port in use by another socket.
-		 *
-		 * On Linux, SO_REUSEPORT socket option allows sockets to be
-		 * bound to an identical socket address. For UDP sockets, the
-		 * use of this option can provide better distribution of
-		 * incoming datagrams to multiple processes (or threads) as
-		 * compared to the traditional technique of having multiple
-		 * processes compete to receive datagrams on the same socket.
-		 *
-		 * On FreeBSD, the same thing is achieved with SO_REUSEPORT_LB.
-		 *
-		 */
-#if defined(SO_REUSEADDR)
-		res = setsockopt(csock->fd, SOL_SOCKET, SO_REUSEADDR,
-				 &(int){ 1 }, sizeof(int));
-		RUNTIME_CHECK(res == 0);
-#endif
-#if defined(SO_REUSEPORT_LB)
-		res = setsockopt(csock->fd, SOL_SOCKET, SO_REUSEPORT_LB,
-				 &(int){ 1 }, sizeof(int));
-		RUNTIME_CHECK(res == 0);
-#elif defined(SO_REUSEPORT)
-		res = setsockopt(csock->fd, SOL_SOCKET, SO_REUSEPORT,
-				 &(int){ 1 }, sizeof(int));
-		RUNTIME_CHECK(res == 0);
-#endif
 
-#ifdef SO_INCOMING_CPU
 		/* We don't check for the result, because SO_INCOMING_CPU can be
 		 * available without the setter on Linux kernel version 4.4, and
 		 * setting SO_INCOMING_CPU is just an optimization.
 		 */
-		(void)setsockopt(csock->fd, SOL_SOCKET, SO_INCOMING_CPU,
+		(void)isc__nm_socket_incoming_cpu(csock->fd);
-				 &(int){ 1 }, sizeof(int));
+
-#endif
 		ievent = isc__nm_get_ievent(mgr, netievent_udplisten);
 		ievent->sock = csock;
 		isc__nm_enqueue_ievent(&mgr->workers[i],