Increase rbtdb nodelock counts

This reverts commit d77ea585a0.

lib/dns/rbtdb.c

@@ -309,7 +309,7 @@ typedef ISC_LIST(dns_rbtnode_t)         rbtnodelist_t;
 	(((header)->rdh_ttl > (now)) || \
 	 ((header)->rdh_ttl == (now) && ZEROTTL(header)))
 
-#define DEFAULT_NODE_LOCK_COUNT         7       /*%< Should be prime. */
+#define DEFAULT_NODE_LOCK_COUNT         31       /*%< Should be prime. */
 #define RBTDB_GLUE_TABLE_INIT_SIZE     2U
 
 /*%
@@ -329,10 +329,10 @@ typedef ISC_LIST(dns_rbtnode_t)         rbtnodelist_t;
 #define DEFAULT_CACHE_NODE_LOCK_COUNT DNS_RBTDB_CACHE_NODE_LOCK_COUNT
 #endif
 #else
-#define DEFAULT_CACHE_NODE_LOCK_COUNT   16
+#define DEFAULT_CACHE_NODE_LOCK_COUNT   64
 #endif	/* DNS_RBTDB_CACHE_NODE_LOCK_COUNT */
 
-typedef struct {
+typedef struct rbtdb_nodelock {
 	nodelock_t                      lock;
 	/* Protected in the refcount routines. */
 	isc_refcount_t                  references;

lib/dns/resolver.c

@@ -4832,7 +4832,7 @@ fctx_create(dns_resolver_t *res, const dns_name_t *name, dns_rdatatype_t type,
 	fctx->vresult = ISC_R_SUCCESS;
 	fctx->exitline = -1;	/* sentinel */
 	fctx->logged = false;
-	atomic_store(&fctx->attributes, 0);
+	atomic_init(&fctx->attributes, 0);
 	fctx->spilled = false;
 	fctx->nqueries = 0;
 	fctx->reason = NULL;

lib/isc/hp.c

@@ -133,7 +133,7 @@ void
 isc_hp_destroy(isc_hp_t *hp) {
 	for (int i = 0; i < isc__hp_max_threads; i++) {
 		isc_mem_put(hp->mctx, hp->hp[i],
-			    CLPAD * 2 * sizeof(uintptr_t));
+			    CLPAD * 2 * sizeof(hp->hp[i][0]));
 
 		for (int j = 0; j < hp->rl[i]->size; j++) {
 			void *data = (void *)hp->rl[i]->list[j];

lib/isc/include/isc/netmgr.h

@@ -59,6 +59,10 @@ isc_nm_closedown(isc_nm_t *mgr);
 int
 isc_nm_tid(void);
 
+/* Return number of netmgr worker threads */
+int
+isc_nm_threads(void);
+
 /*
  * isc_nm_freehandle frees a handle, releasing resources
  */

lib/isc/netmgr/netmgr.c

@@ -121,6 +121,9 @@ static const isc_statscounter_t unixstatsindex[] = {
 
 ISC_THREAD_LOCAL int isc__nm_tid_v = ISC_NETMGR_TID_UNKNOWN;
 
+/* An assumption that we'll have one netmgr per named */
+static atomic_uint_fast32_t isc__nm_tid_count_v;
+
 static void
 nmsocket_maybe_destroy(isc_nmsocket_t *sock);
 static void
@@ -136,7 +139,10 @@ int
 isc_nm_tid() {
 	return (isc__nm_tid_v);
 }
-
+int
+isc_nm_threads() {
+	return atomic_load(&isc__nm_tid_count_v);
+}
 bool
 isc__nm_in_netthread() {
 	return (isc__nm_tid_v >= 0);
@@ -151,6 +157,7 @@ isc_nm_start(isc_mem_t *mctx, uint32_t workers) {
 	*mgr = (isc_nm_t) {
 		.nworkers = workers
 	};
+	atomic_init(&isc__nm_tid_count_v, workers);
 
 	isc_mem_attach(mctx, &mgr->mctx);
 	isc_mutex_init(&mgr->lock);
@@ -920,9 +927,9 @@ isc__nmsocket_init(isc_nmsocket_t *sock, isc_nm_t *mgr,
 		.type = type,
 		.iface = iface,
 		.fd = -1,
-		.ah_size = 32,
-		.inactivehandles = isc_astack_new(mgr->mctx, 60),
-		.inactivereqs = isc_astack_new(mgr->mctx, 60)
+		.ah_size = 1000,
+		.inactivehandles = isc_astack_new(mgr->mctx, 1000),
+		.inactivereqs = isc_astack_new(mgr->mctx, 1000)
 	};
 
 	isc_nm_attach(mgr, &sock->mgr);
@@ -933,7 +940,7 @@ isc__nmsocket_init(isc_nmsocket_t *sock, isc_nm_t *mgr,
 	sock->ah_handles = isc_mem_allocate(mgr->mctx,
 					    sock->ah_size *
 					     sizeof(isc_nmhandle_t *));
-	for (size_t i = 0; i < 32; i++) {
+	for (size_t i = 0; i < sock->ah_size; i++) {
 		sock->ah_frees[i] = i;
 		sock->ah_handles[i] = NULL;
 	}

lib/isc/siphash.c

@@ -17,7 +17,7 @@
 #include <isc/util.h>
 #include <isc/siphash.h>
 
-#if HAVE_OPENSSL_SIPHASH
+#if 0 /* HAVE_OPENSSL_SIPHASH */
 #include <openssl/evp.h>
 
 void

lib/isc/task.c

@@ -106,6 +106,7 @@ struct isc__task {
 	isc_mutex_t			lock;
 	/* Locked by task lock. */
 	task_state_t			state;
+	int				pause_cnt;
 	isc_refcount_t			references;
 	isc_eventlist_t			events;
 	isc_eventlist_t			on_shutdown;
@@ -308,6 +309,7 @@ isc_task_create_bound(isc_taskmgr_t *manager0, unsigned int quantum,
 
 	isc_mutex_init(&task->lock);
 	task->state = task_state_idle;
+	task->pause_cnt = 0;
 
 	isc_refcount_init(&task->references, 1);
 	INIT_LIST(task->events);
@@ -407,7 +409,7 @@ task_shutdown(isc__task_t *task) {
 /*
  * Moves a task onto the appropriate run queue.
  *
- * Caller must NOT hold manager lock.
+ * Caller must NOT hold queue lock.
  */
 static inline void
 task_ready(isc__task_t *task) {
@@ -415,7 +417,7 @@ task_ready(isc__task_t *task) {
 	bool has_privilege = isc_task_privilege((isc_task_t *) task);
 
 	REQUIRE(VALID_MANAGER(manager));
-	REQUIRE(task->state == task_state_ready);
+//	REQUIRE(task->state == task_state_ready);
 
 	XTRACE("task_ready");
 	LOCK(&manager->queues[task->threadid].lock);
@@ -967,10 +969,13 @@ pop_readyq(isc__taskmgr_t *manager, int c) {
  * Push 'task' onto the ready_tasks queue.  If 'task' has the privilege
  * flag set, then also push it onto the ready_priority_tasks queue.
  *
- * Caller must hold the task manager lock.
+ * Caller must hold the task queue lock.
  */
 static inline void
 push_readyq(isc__taskmgr_t *manager, isc__task_t *task, int c) {
+	if (ISC_LINK_LINKED(task, ready_link)) {
+		return;
+	}
 	ENQUEUE(manager->queues[c].ready_tasks, task, ready_link);
 	if (TASK_PRIVILEGED(task)) {
 		ENQUEUE(manager->queues[c].ready_priority_tasks, task,
@@ -1128,6 +1133,16 @@ dispatch(isc__taskmgr_t *manager, unsigned int threadid) {
 						  memory_order_acquire);
 
 			LOCK(&task->lock);
+			/*
+			 * It is possible if we're pausing this thread right
+			 * now and we haven't popped it from the queue yet,
+			 * just bail then.
+			 */
+			if (task->state != task_state_ready) {
+				UNLOCK(&task->lock);
+				LOCK(&manager->queues[threadid].lock);
+				continue;
+			}
 			INSIST(task->state == task_state_ready);
 			task->state = task_state_running;
 			XTRACE("running");
@@ -1215,7 +1230,8 @@ dispatch(isc__taskmgr_t *manager, unsigned int threadid) {
 						}
 					}
 					done = true;
-				} else if (dispatch_count >= task->quantum) {
+				} else if (dispatch_count >= task->quantum ||
+					   task->state == task_state_pausing) {
 					/*
 					 * Our quantum has expired, but
 					 * there is more work to be done.
@@ -1690,6 +1706,16 @@ isc_task_pause(isc_task_t *task0) {
 	bool running = false;
 
 	LOCK(&task->lock);
+	task->pause_cnt++;
+	if (task->pause_cnt > 1) {
+		/*
+		 * Someone already paused this thread, just increase
+		 * the number of pausing clients.
+		 */
+		UNLOCK(&task->lock);
+		return;
+	}
+
 	INSIST(task->state == task_state_idle ||
 	       task->state == task_state_ready ||
 	       task->state == task_state_running);
@@ -1699,8 +1725,9 @@ isc_task_pause(isc_task_t *task0) {
 	} else {
 		task->state = task_state_paused;
 	}
-	UNLOCK(&task->lock);
 
+	UNLOCK(&task->lock);
+	
 	if (running) {
 		return;
 	}
@@ -1711,6 +1738,7 @@ isc_task_pause(isc_task_t *task0) {
 			task, ready_link);
 	}
 	UNLOCK(&manager->queues[task->threadid].lock);
+
 }
 
 void
@@ -1721,6 +1749,12 @@ isc_task_unpause(isc_task_t *task0) {
 	REQUIRE(ISCAPI_TASK_VALID(task0));
 
 	LOCK(&task->lock);
+	task->pause_cnt--;
+	if (task->pause_cnt > 0) {
+		UNLOCK(&task->lock);
+		return;
+	}
+	
 	INSIST(task->state == task_state_paused ||
 	       task->state == task_state_pausing);
 	/* If the task was pausing we can't reschedule it */

lib/ns/client.c

@@ -132,6 +132,8 @@ static void compute_cookie(ns_client_t *client, uint32_t when,
 			   isc_buffer_t *buf);
 static void
 get_clientmctx(ns_clientmgr_t *manager, isc_mem_t **mctxp);
+static void
+get_clienttask(ns_clientmgr_t *manager, isc_task_t **taskp);
 
 void
 ns_client_recursing(ns_client_t *client) {
@@ -181,11 +183,13 @@ ns_client_endrequest(ns_client_t *client) {
 
 	CTRACE("endrequest");
 
-	LOCK(&client->manager->reclock);
-	if (ISC_LINK_LINKED(client, rlink)) {
-		ISC_LIST_UNLINK(client->manager->recursing, client, rlink);
+	if (client->state == NS_CLIENTSTATE_RECURSING) {
+		LOCK(&client->manager->reclock);
+		if (ISC_LINK_LINKED(client, rlink)) {
+			ISC_LIST_UNLINK(client->manager->recursing, client, rlink);
+		}
+		UNLOCK(&client->manager->reclock);
 	}
-	UNLOCK(&client->manager->reclock);
 
 	if (client->cleanup != NULL) {
 		(client->cleanup)(client);
@@ -871,7 +875,7 @@ ns_client_error(ns_client_t *client, isc_result_t result) {
 		/*
 		 * SERVFAIL caching: store qname/qtype of failed queries
 		 */
-		isc_time_t expire;
+/*		isc_time_t expire;
 		isc_interval_t i;
 		uint32_t flags = 0;
 
@@ -885,7 +889,7 @@ ns_client_error(ns_client_t *client, isc_result_t result) {
 					 client->query.qname,
 					 client->query.qtype,
 					 true, flags, &expire);
-		}
+		} */
 	}
 
 	ns_client_send(client);
@@ -2212,38 +2216,36 @@ ns__client_tcpconn(isc_nmhandle_t *handle, isc_result_t result, void *arg) {
 static void
 get_clientmctx(ns_clientmgr_t *manager, isc_mem_t **mctxp) {
 	isc_mem_t *clientmctx;
-#if CLIENT_NMCTXS > 0
 	unsigned int nextmctx;
-#endif
-
+	int tid = isc_nm_tid();
+	if (tid < 0) {
+		tid = 0;
+	}
 	MTRACE("clientmctx");
 
-#if CLIENT_NMCTXS > 0
-	LOCK(&manager->lock);
-	if (isc_nm_tid() >= 0) {
-		nextmctx = isc_nm_tid();
-	} else {
-		nextmctx = manager->nextmctx++;
-		if (manager->nextmctx == CLIENT_NMCTXS)
-			manager->nextmctx = 0;
-
-		INSIST(nextmctx < CLIENT_NMCTXS);
-	}
-
+	nextmctx = isc_random_uniform(CLIENT_NMCTXS_PERCPU) * isc_nm_threads() + tid;
 	clientmctx = manager->mctxpool[nextmctx];
-	if (clientmctx == NULL) {
-		isc_mem_create(&clientmctx);
-		isc_mem_setname(clientmctx, "client", NULL);
-		manager->mctxpool[nextmctx] = clientmctx;
-	}
-	UNLOCK(&manager->lock);
-#else
-	clientmctx = manager->mctx;
-#endif
 
 	isc_mem_attach(clientmctx, mctxp);
 }
 
+static void
+get_clienttask(ns_clientmgr_t *manager, isc_task_t **taskp) {
+	isc_task_t *clienttask;
+	unsigned int nexttask;
+	int tid = isc_nm_tid();
+	if (tid < 0) {
+		tid = 0;
+	}
+	MTRACE("clienttask");
+
+	nexttask = isc_random_uniform(CLIENT_NTASKS_PERCPU) * isc_nm_threads() + tid;
+	clienttask = manager->taskpool[nexttask];
+
+//	isc_task_create_bound(manager->taskmgr, 20, taskp, tid);
+	isc_task_attach(clienttask, taskp);
+}
+
 isc_result_t
 ns__client_setup(ns_client_t *client, ns_clientmgr_t *mgr, bool new) {
 	isc_result_t result;
@@ -2260,6 +2262,11 @@ ns__client_setup(ns_client_t *client, ns_clientmgr_t *mgr, bool new) {
 	REQUIRE(VALID_MANAGER(mgr) || !new);
 
 	if (new) {
+		int tid = isc_nm_tid();
+		if (tid < 0) {
+			tid = 0;
+		}
+
 		*client = (ns_client_t) {
 			.magic = 0
 		};
@@ -2267,10 +2274,7 @@ ns__client_setup(ns_client_t *client, ns_clientmgr_t *mgr, bool new) {
 		get_clientmctx(mgr, &client->mctx);
 		clientmgr_attach(mgr, &client->manager);
 		ns_server_attach(mgr->sctx, &client->sctx);
-		result = isc_task_create(mgr->taskmgr, 20,  &client->task);
-		if (result != ISC_R_SUCCESS) {
-			goto cleanup;
-		}
+		get_clienttask(mgr, &client->task);
 		result = dns_message_create(client->mctx,
 					    DNS_MESSAGE_INTENTPARSE,
 					    &client->message);
@@ -2397,21 +2401,18 @@ clientmgr_detach(ns_clientmgr_t **mp) {
 
 static void
 clientmgr_destroy(ns_clientmgr_t *manager) {
-#if CLIENT_NMCTXS > 0
 	int i;
-#endif
 
 	MTRACE("clientmgr_destroy");
 
 	isc_refcount_destroy(&manager->references);
 	manager->magic = 0;
 
-#if CLIENT_NMCTXS > 0
-	for (i = 0; i < CLIENT_NMCTXS; i++) {
+	for (i = 0; i < manager->nummctxs; i++) {
 		if (manager->mctxpool[i] != NULL)
 			isc_mem_detach(&manager->mctxpool[i]);
 	}
-#endif
+	isc_mem_put(manager->mctx, manager->mctxpool, manager->nummctxs * sizeof(isc_mem_t*));
 
 	if (manager->interface != NULL) {
 		ns_interface_detach(&manager->interface);
@@ -2423,13 +2424,13 @@ clientmgr_destroy(ns_clientmgr_t *manager) {
 	if (manager->excl != NULL)
 		isc_task_detach(&manager->excl);
 
-	for (i = 0; i < CLIENT_NTASKS; i++) {
+	for (i = 0; i < manager->numtasks; i++) {
 		if (manager->taskpool[i] != NULL) {
 			isc_task_detach(&manager->taskpool[i]);
 		}
 	}
 	isc_mem_put(manager->mctx, manager->taskpool,
-		    CLIENT_NTASKS * sizeof(isc_task_t *));
+		    manager->numtasks * sizeof(isc_task_t *));
 	ns_server_detach(&manager->sctx);
 
 	isc_mem_put(manager->mctx, manager, sizeof(*manager));
@@ -2442,9 +2443,7 @@ ns_clientmgr_create(isc_mem_t *mctx, ns_server_t *sctx, isc_taskmgr_t *taskmgr,
 {
 	ns_clientmgr_t *manager;
 	isc_result_t result;
-#if CLIENT_NMCTXS > 0
 	int i;
-#endif
 
 	manager = isc_mem_get(mctx, sizeof(*manager));
 	*manager = (ns_clientmgr_t) { .magic = 0 };
@@ -2465,22 +2464,30 @@ ns_clientmgr_create(isc_mem_t *mctx, ns_server_t *sctx, isc_taskmgr_t *taskmgr,
 	ns_interface_attach(interface, &manager->interface);
 
 	manager->exiting = false;
+	manager->numtasks = CLIENT_NTASKS_PERCPU * isc_nm_threads();
+	INSIST(manager->numtasks > 0);
 	manager->taskpool =
-		isc_mem_get(mctx, CLIENT_NTASKS*sizeof(isc_task_t *));
-	for (i = 0; i < CLIENT_NTASKS; i++) {
+		isc_mem_get(mctx, manager->numtasks * sizeof(isc_task_t *));
+	for (i = 0; i < manager->numtasks; i++) {
 		manager->taskpool[i] = NULL;
-		isc_task_create(manager->taskmgr, 20, &manager->taskpool[i]);
+		isc_task_create_bound(manager->taskmgr, 20, &manager->taskpool[i], i);
 	}
 	isc_refcount_init(&manager->references, 1);
 	manager->sctx = NULL;
 	ns_server_attach(sctx, &manager->sctx);
 
 	ISC_LIST_INIT(manager->recursing);
-#if CLIENT_NMCTXS > 0
-	manager->nextmctx = 0;
-	for (i = 0; i < CLIENT_NMCTXS; i++)
-		manager->mctxpool[i] = NULL; /* will be created on-demand */
-#endif
+	atomic_init(&manager->nextmctx, 0);
+	manager->nummctxs = CLIENT_NMCTXS_PERCPU * isc_nm_threads();
+	INSIST(manager->nummctxs > 0);
+	manager->mctxpool = isc_mem_get(manager->mctx,
+					manager->nummctxs * sizeof(isc_mem_t*));
+	
+	for (i = 0; i < manager->nummctxs; i++) {
+		manager->mctxpool[i] = NULL;
+		isc_mem_create(&manager->mctxpool[i]);
+		isc_mem_setname(manager->mctxpool[i], "client", NULL);
+	}
 	manager->magic = MANAGER_MAGIC;
 
 	MTRACE("create");

lib/ns/include/ns/client.h

@@ -85,9 +85,9 @@
 #define NS_CLIENT_SEND_BUFFER_SIZE		4096
 #define NS_CLIENT_RECV_BUFFER_SIZE		4096
 
-#define CLIENT_NMCTXS				100
+#define CLIENT_NMCTXS_PERCPU			16
 /*%<
- * Number of 'mctx pools' for clients. (Should this be configurable?)
+ * Number of 'mctx pools' for clients per CPU. (Should this be configurable?)
  * When enabling threads, we use a pool of memory contexts shared by
  * client objects, since concurrent access to a shared context would cause
  * heavy contentions.  The above constant is expected to be enough for
@@ -95,7 +95,7 @@
  * server.
  */
 
-#define CLIENT_NTASKS				100
+#define CLIENT_NTASKS_PERCPU			16
 /*%<
  * Number of tasks to be used by clients - those are used only when recursing
  */
@@ -169,6 +169,8 @@ struct ns_clientmgr {
 
 	/* Attached by clients, needed for e.g. recursion */
 	isc_task_t **		taskpool;
+	int			numtasks;
+	atomic_uint_fast32_t	nexttask;
 
 	ns_interface_t		*interface;
 
@@ -180,11 +182,10 @@ struct ns_clientmgr {
 	isc_mutex_t		reclock;
 	client_list_t		recursing;    /*%< Recursing clients */
 
-#if CLIENT_NMCTXS > 0
 	/*%< mctx pool for clients. */
-	unsigned int		nextmctx;
-	isc_mem_t *		mctxpool[CLIENT_NMCTXS];
-#endif
+	atomic_uint_fast32_t	nextmctx;
+	int			nummctxs;
+	isc_mem_t **		mctxpool;
 };
 
 /*% nameserver client structure */

lib/ns/query.c

@@ -6116,10 +6116,11 @@ ns__query_sfcache(query_ctx_t *qctx) {
 					  &qctx->client->tnow);
 	}
 #else
-	failcache = dns_badcache_find(qctx->view->failcache,
+/*	failcache = dns_badcache_find(qctx->view->failcache,
 				      qctx->client->query.qname,
 				      qctx->qtype, &flags,
-				      &qctx->client->tnow);
+				      &qctx->client->tnow); */
+	failcache = false;
 #endif
 	if (failcache &&
 	    (((flags & NS_FAILCACHE_CD) != 0) ||