EXP: play with affinity

configure.ac

@@ -552,6 +552,8 @@ AC_COMPILE_IFELSE(
 
 AC_CHECK_FUNCS([pthread_attr_getstacksize pthread_attr_setstacksize pthread_barrier_init pthread_spin_init])
 
+AC_CHECK_FUNCS([pthread_setaffinity_np])
+
 AC_CHECK_HEADERS([sched.h])
 
 AC_SEARCH_LIBS([sched_yield],[rt])

lib/isc/include/isc/thread.h

@@ -56,4 +56,7 @@ isc_thread_setname(isc_thread_t thread, const char *name);
 
 #define isc_thread_self (uintptr_t)pthread_self
 
+void
+isc_thread_setaffinity(isc_thread_t thread, int affinity);
+
 ISC_LANG_ENDDECLS

lib/isc/netmgr/netmgr-int.h

@@ -1332,7 +1332,7 @@ isc__nm_socket_reuse_lb(uv_os_sock_t fd);
  */
 
 isc_result_t
-isc__nm_socket_incoming_cpu(uv_os_sock_t fd);
+isc__nm_socket_incoming_cpu(uv_os_sock_t fd, int affinity);
 /*%<
  * Set the SO_INCOMING_CPU socket option on the fd if available
  */

lib/isc/netmgr/socket.c

@@ -219,9 +219,11 @@ isc__nm_socket_reuse_lb(uv_os_sock_t fd) {
 }
 
 isc_result_t
-isc__nm_socket_incoming_cpu(uv_os_sock_t fd) {
+isc__nm_socket_incoming_cpu(uv_os_sock_t fd, int affinity) {
 #ifdef SO_INCOMING_CPU
-	if (setsockopt_on(fd, SOL_SOCKET, SO_INCOMING_CPU) == -1) {
+	if (setsockopt(fd, SOL_SOCKET, SO_INCOMING_CPU, &affinity,
+		       sizeof(affinity)))
+	{
 		return (ISC_R_FAILURE);
 	} else {
 		return (ISC_R_SUCCESS);

lib/isc/netmgr/tcp.c

@@ -278,14 +278,14 @@ isc_nm_tcpconnect(isc_nm_t *mgr, isc_sockaddr_t *local, isc_sockaddr_t *peer,
 }
 
 static uv_os_sock_t
-isc__nm_tcp_lb_socket(isc_nm_t *mgr, sa_family_t sa_family) {
+isc__nm_tcp_lb_socket(isc_nm_t *mgr, sa_family_t sa_family, int affinity) {
 	isc_result_t result;
 	uv_os_sock_t sock;
 
 	result = isc__nm_socket(sa_family, SOCK_STREAM, 0, &sock);
 	RUNTIME_CHECK(result == ISC_R_SUCCESS);
 
-	(void)isc__nm_socket_incoming_cpu(sock);
+	(void)isc__nm_socket_incoming_cpu(sock, affinity);
 	(void)isc__nm_socket_v6only(sock, sa_family);
 
 	/* FIXME: set mss */
@@ -427,8 +427,8 @@ start_tcp_child(isc_nm_t *mgr, isc_sockaddr_t *iface, isc_nmsocket_t *sock,
 
 	if (mgr->load_balance_sockets) {
 		UNUSED(fd);
-		csock->fd = isc__nm_tcp_lb_socket(mgr,
-						  iface->type.sa.sa_family);
+		csock->fd = isc__nm_tcp_lb_socket(mgr, iface->type.sa.sa_family,
+						  tid);
 	} else {
 		csock->fd = dup(fd);
 	}
@@ -475,7 +475,7 @@ isc_nm_listentcp(isc_nm_t *mgr, uint32_t workers, isc_sockaddr_t *iface,
 	sock->pquota = quota;
 
 	if (!mgr->load_balance_sockets) {
-		fd = isc__nm_tcp_lb_socket(mgr, iface->type.sa.sa_family);
+		fd = isc__nm_tcp_lb_socket(mgr, iface->type.sa.sa_family, -1);
 	}
 
 	start_tcp_child(mgr, iface, sock, fd, 0);

lib/isc/netmgr/udp.c

@@ -67,14 +67,14 @@ static void
 udp_close_cb(uv_handle_t *handle);
 
 static uv_os_sock_t
-isc__nm_udp_lb_socket(isc_nm_t *mgr, sa_family_t sa_family) {
+isc__nm_udp_lb_socket(isc_nm_t *mgr, sa_family_t sa_family, int affinity) {
 	isc_result_t result;
 	uv_os_sock_t sock = -1;
 
 	result = isc__nm_socket(sa_family, SOCK_DGRAM, 0, &sock);
 	RUNTIME_CHECK(result == ISC_R_SUCCESS);
 
-	(void)isc__nm_socket_incoming_cpu(sock);
+	(void)isc__nm_socket_incoming_cpu(sock, affinity);
 	(void)isc__nm_socket_disable_pmtud(sock, sa_family);
 	(void)isc__nm_socket_v6only(sock, sa_family);
 
@@ -191,8 +191,8 @@ start_udp_child(isc_nm_t *mgr, isc_sockaddr_t *iface, isc_nmsocket_t *sock,
 	csock->inactive_handles_max = ISC_NM_NMHANDLES_MAX;
 
 	if (mgr->load_balance_sockets) {
-		csock->fd = isc__nm_udp_lb_socket(mgr,
-						  iface->type.sa.sa_family);
+		csock->fd = isc__nm_udp_lb_socket(mgr, iface->type.sa.sa_family,
+						  tid);
 	} else {
 		csock->fd = dup(fd);
 	}
@@ -241,7 +241,7 @@ isc_nm_listenudp(isc_nm_t *mgr, uint32_t workers, isc_sockaddr_t *iface,
 	sock->recv_cbarg = cbarg;
 
 	if (!mgr->load_balance_sockets) {
-		fd = isc__nm_udp_lb_socket(mgr, iface->type.sa.sa_family);
+		fd = isc__nm_udp_lb_socket(mgr, iface->type.sa.sa_family, -1);
 	}
 
 	start_udp_child(mgr, iface, sock, fd, 0);
@@ -810,7 +810,7 @@ isc_nm_udpconnect(isc_nm_t *mgr, isc_sockaddr_t *local, isc_sockaddr_t *peer,
 	RUNTIME_CHECK(result == ISC_R_SUCCESS ||
 		      result == ISC_R_NOTIMPLEMENTED);
 
-	(void)isc__nm_socket_incoming_cpu(sock->fd);
+	(void)isc__nm_socket_incoming_cpu(sock->fd, sock->tid);
 
 	(void)isc__nm_socket_disable_pmtud(sock->fd, sa_family);
 

lib/isc/os.c

@@ -87,8 +87,9 @@ sched_affinity_ncpus(void) {
 		int i, n = 0;
 
 		for (i = 0; i < CPU_SETSIZE; ++i) {
-			if (CPU_ISSET(i, &cpus))
+			if (CPU_ISSET(i, &cpus)) {
 				++n;
+			}
 		}
 		return (n);
 #endif

lib/isc/thread.c

@@ -13,6 +13,9 @@
 
 /*! \file */
 
+#include <errno.h>
+#include <stdio.h>
+
 #if defined(HAVE_SCHED_H)
 #include <sched.h>
 #endif /* if defined(HAVE_SCHED_H) */
@@ -179,3 +182,18 @@ isc_thread_yield(void) {
 	pthread_yield_np();
 #endif /* if defined(HAVE_SCHED_YIELD) */
 }
+
+void
+isc_thread_setaffinity(isc_thread_t thread, int affinity) {
+#if HAVE_PTHREAD_SETAFFINITY_NP
+	cpu_set_t cpuset;
+
+	CPU_ZERO(&cpuset);
+	CPU_SET(affinity, &cpuset);
+
+	int r = pthread_setaffinity_np(thread, sizeof(cpuset), &cpuset);
+	if (r == -1) {
+		perror("setting affinity failed");
+	}
+#endif /* HAVE_PTHREAD_SETAFFINITY_NP */
+}