WiP: test affine workers

lib/isc/mem.c

@@ -456,8 +456,70 @@ more_frags(isc__mem_t *ctx, size_t new_size) {
 
 static inline void *
 mem_getunlocked(isc__mem_t *ctx, size_t size) {
-	UNUSED(ctx);
-	return (malloc(size));
+	size_t new_size = quantize(size);
+	void *ret;
+
+	if (new_size >= ctx->max_size) {
+		/*
+		 * memget() was called on something beyond our upper limit.
+		 */
+		if (ctx->quota != 0U && ctx->total + size > ctx->quota) {
+			ret = NULL;
+			goto done;
+		}
+		ret = (ctx->memalloc)(ctx->arg, size);
+		if (ret == NULL) {
+			ctx->memalloc_failures++;
+			goto done;
+		}
+		ctx->total += size;
+		ctx->inuse += size;
+		ctx->stats[ctx->max_size].gets++;
+		ctx->stats[ctx->max_size].totalgets++;
+		ctx->malloced += size;
+		if (ctx->malloced > ctx->maxmalloced)
+			ctx->maxmalloced = ctx->malloced;
+		/*
+		 * If we don't set new_size to size, then the
+		 * ISC_MEMFLAG_FILL code might write over bytes we don't
+		 * own.
+		 */
+		new_size = size;
+		goto done;
+	}
+	/*
+	 * If there are no blocks in the free list for this size, get a chunk
+	 * of memory and then break it up into "new_size"-sized blocks, adding
+	 * them to the free list.
+	 */
+	if (ctx->freelists[new_size] == NULL && !more_frags(ctx, new_size))
+		return (NULL);
+
+	/*
+	 * The free list uses the "rounded-up" size "new_size".
+	 */
+
+	ret = ctx->freelists[new_size];
+	ctx->freelists[new_size] = ctx->freelists[new_size]->next;
+
+
+	/*
+	 * The stats[] uses the _actual_ "size" requested by the
+	 * caller, with the caveat (in the code above) that "size" >= the
+	 * max. size (max_size) ends up getting recorded as a call to
+	 * max_size.
+	 */
+	ctx->stats[size].gets++;
+	ctx->stats[size].totalgets++;
+	ctx->stats[new_size].freefrags--;
+	ctx->inuse += new_size;
+
+ done:
+	if (ISC_UNLIKELY((ctx->flags & ISC_MEMFLAG_FILL) != 0) &&
+	    ISC_LIKELY(ret != NULL))
+		memset(ret, 0xbe, new_size); /* Mnemonic for "beef". */
+
+	return (ret);
 }
 
 #if ISC_MEM_CHECKOVERRUN
@@ -478,9 +540,47 @@ check_overrun(void *mem, size_t size, size_t new_size) {
 /* coverity[+free : arg-1] */
 static inline void
 mem_putunlocked(isc__mem_t *ctx, void *mem, size_t size) {
-	UNUSED(ctx);
-	UNUSED(size);
-	free(mem);
+	size_t new_size = quantize(size);
+
+	if (new_size >= ctx->max_size) {
+		/*
+		 * memput() called on something beyond our upper limit.
+		 */
+		if (ISC_UNLIKELY((ctx->flags & ISC_MEMFLAG_FILL) != 0))
+			memset(mem, 0xde, size); /* Mnemonic for "dead". */
+
+		(ctx->memfree)(ctx->arg, mem);
+		INSIST(ctx->stats[ctx->max_size].gets != 0U);
+		ctx->stats[ctx->max_size].gets--;
+		INSIST(size <= ctx->inuse);
+		ctx->inuse -= size;
+		ctx->malloced -= size;
+		return;
+	}
+
+	if (ISC_UNLIKELY((ctx->flags & ISC_MEMFLAG_FILL) != 0)) {
+#if ISC_MEM_CHECKOVERRUN
+		check_overrun(mem, size, new_size);
+#endif
+		memset(mem, 0xde, new_size); /* Mnemonic for "dead". */
+	}
+
+	/*
+	 * The free list uses the "rounded-up" size "new_size".
+	 */
+	((element *)mem)->next = ctx->freelists[new_size];
+	ctx->freelists[new_size] = (element *)mem;
+
+	/*
+	 * The stats[] uses the _actual_ "size" requested by the
+	 * caller, with the caveat (in the code above) that "size" >= the
+	 * max. size (max_size) ends up getting recorded as a call to
+	 * max_size.
+	 */
+	INSIST(ctx->stats[size].gets != 0U);
+	ctx->stats[size].gets--;
+	ctx->stats[new_size].freefrags++;
+	ctx->inuse -= new_size;
 }
 
 /*!
@@ -968,15 +1068,98 @@ isc_mem_destroy(isc_mem_t **ctxp) {
 
 void *
 isc__mem_get(isc_mem_t *ctx0, size_t size FLARG) {
-	UNUSED(ctx0);
-	return (malloc(size));
+	isc__mem_t *ctx = (isc__mem_t *)ctx0;
+	void *ptr;
+	isc_boolean_t call_water = ISC_FALSE;
+
+	REQUIRE(VALID_CONTEXT(ctx));
+
+	if (ISC_UNLIKELY((isc_mem_debugging &
+			  (ISC_MEM_DEBUGSIZE|ISC_MEM_DEBUGCTX)) != 0))
+		return (isc__mem_allocate(ctx0, size FLARG_PASS));
+
+	if ((ctx->flags & ISC_MEMFLAG_INTERNAL) != 0) {
+		MCTXLOCK(ctx, &ctx->lock);
+		ptr = mem_getunlocked(ctx, size);
+	} else {
+		ptr = mem_get(ctx, size);
+		MCTXLOCK(ctx, &ctx->lock);
+		if (ptr != NULL)
+			mem_getstats(ctx, size);
+	}
+
+	ADD_TRACE(ctx, ptr, size, file, line);
+
+	if (ctx->hi_water != 0U && ctx->inuse > ctx->hi_water) {
+		ctx->is_overmem = ISC_TRUE;
+		if (!ctx->hi_called)
+			call_water = ISC_TRUE;
+	}
+	if (ctx->inuse > ctx->maxinuse) {
+		ctx->maxinuse = ctx->inuse;
+		if (ctx->hi_water != 0U && ctx->inuse > ctx->hi_water &&
+		    (isc_mem_debugging & ISC_MEM_DEBUGUSAGE) != 0)
+			fprintf(stderr, "maxinuse = %lu\n",
+				(unsigned long)ctx->inuse);
+	}
+	MCTXUNLOCK(ctx, &ctx->lock);
+
+	if (call_water && (ctx->water != NULL))
+		(ctx->water)(ctx->water_arg, ISC_MEM_HIWATER);
+
+	return (ptr);
 }
 
 void
 isc__mem_put(isc_mem_t *ctx0, void *ptr, size_t size FLARG) {
-	UNUSED(ctx0);
-	UNUSED(size);
-	free(ptr);
+	isc__mem_t *ctx = (isc__mem_t *)ctx0;
+	isc_boolean_t call_water = ISC_FALSE;
+	size_info *si;
+	size_t oldsize;
+
+	REQUIRE(VALID_CONTEXT(ctx));
+	REQUIRE(ptr != NULL);
+
+	if (ISC_UNLIKELY((isc_mem_debugging &
+			  (ISC_MEM_DEBUGSIZE|ISC_MEM_DEBUGCTX)) != 0))
+	{
+		if ((isc_mem_debugging & ISC_MEM_DEBUGSIZE) != 0) {
+			si = &(((size_info *)ptr)[-1]);
+			oldsize = si->u.size - ALIGNMENT_SIZE;
+			if ((isc_mem_debugging & ISC_MEM_DEBUGCTX) != 0)
+				oldsize -= ALIGNMENT_SIZE;
+			INSIST(oldsize == size);
+		}
+		isc__mem_free((isc_mem_t *)ctx, ptr FLARG_PASS);
+		return;
+	}
+
+	MCTXLOCK(ctx, &ctx->lock);
+
+	DELETE_TRACE(ctx, ptr, size, file, line);
+
+	if ((ctx->flags & ISC_MEMFLAG_INTERNAL) != 0) {
+		mem_putunlocked(ctx, ptr, size);
+	} else {
+		mem_putstats(ctx, ptr, size);
+		mem_put(ctx, ptr, size);
+	}
+
+	/*
+	 * The check against ctx->lo_water == 0 is for the condition
+	 * when the context was pushed over hi_water but then had
+	 * isc_mem_setwater() called with 0 for hi_water and lo_water.
+	 */
+	if ((ctx->inuse < ctx->lo_water) || (ctx->lo_water == 0U)) {
+		ctx->is_overmem = ISC_FALSE;
+		if (ctx->hi_called)
+			call_water = ISC_TRUE;
+	}
+
+	MCTXUNLOCK(ctx, &ctx->lock);
+
+	if (call_water && (ctx->water != NULL))
+		(ctx->water)(ctx->water_arg, ISC_MEM_LOWATER);
 }
 
 void
@@ -1143,20 +1326,136 @@ mem_allocateunlocked(isc_mem_t *ctx0, size_t size) {
 
 void *
 isc__mem_allocate(isc_mem_t *ctx0, size_t size FLARG) {
-	UNUSED(ctx0);
-	return (malloc(size));
+	isc__mem_t *ctx = (isc__mem_t *)ctx0;
+	size_info *si;
+	isc_boolean_t call_water = ISC_FALSE;
+
+	REQUIRE(VALID_CONTEXT(ctx));
+
+	MCTXLOCK(ctx, &ctx->lock);
+	si = mem_allocateunlocked((isc_mem_t *)ctx, size);
+	if (((ctx->flags & ISC_MEMFLAG_INTERNAL) == 0) && (si != NULL))
+		mem_getstats(ctx, si[-1].u.size);
+
+	ADD_TRACE(ctx, si, si[-1].u.size, file, line);
+	if (ctx->hi_water != 0U && ctx->inuse > ctx->hi_water &&
+	    !ctx->is_overmem) {
+		ctx->is_overmem = ISC_TRUE;
+	}
+
+	if (ctx->hi_water != 0U && !ctx->hi_called &&
+	    ctx->inuse > ctx->hi_water) {
+		ctx->hi_called = ISC_TRUE;
+		call_water = ISC_TRUE;
+	}
+	if (ctx->inuse > ctx->maxinuse) {
+		ctx->maxinuse = ctx->inuse;
+		if (ISC_UNLIKELY(ctx->hi_water != 0U &&
+				 ctx->inuse > ctx->hi_water &&
+				 (isc_mem_debugging & ISC_MEM_DEBUGUSAGE) != 0))
+			fprintf(stderr, "maxinuse = %lu\n",
+				(unsigned long)ctx->inuse);
+	}
+	MCTXUNLOCK(ctx, &ctx->lock);
+
+	if (call_water)
+		(ctx->water)(ctx->water_arg, ISC_MEM_HIWATER);
+
+	return (si);
 }
 
 void *
 isc__mem_reallocate(isc_mem_t *ctx0, void *ptr, size_t size FLARG) {
-	UNUSED(ctx0);
-	return (realloc(ptr, size));
+	isc__mem_t *ctx = (isc__mem_t *)ctx0;
+	void *new_ptr = NULL;
+	size_t oldsize, copysize;
+
+	REQUIRE(VALID_CONTEXT(ctx));
+
+	/*
+	 * This function emulates the realloc(3) standard library function:
+	 * - if size > 0, allocate new memory; and if ptr is non NULL, copy
+	 *   as much of the old contents to the new buffer and free the old one.
+	 *   Note that when allocation fails the original pointer is intact;
+	 *   the caller must free it.
+	 * - if size is 0 and ptr is non NULL, simply free the given ptr.
+	 * - this function returns:
+	 *     pointer to the newly allocated memory, or
+	 *     NULL if allocation fails or doesn't happen.
+	 */
+	if (size > 0U) {
+		new_ptr = isc__mem_allocate(ctx0, size FLARG_PASS);
+		if (new_ptr != NULL && ptr != NULL) {
+			oldsize = (((size_info *)ptr)[-1]).u.size;
+			INSIST(oldsize >= ALIGNMENT_SIZE);
+			oldsize -= ALIGNMENT_SIZE;
+			if (ISC_UNLIKELY((isc_mem_debugging &
+					  ISC_MEM_DEBUGCTX) != 0))
+			{
+				INSIST(oldsize >= ALIGNMENT_SIZE);
+				oldsize -= ALIGNMENT_SIZE;
+			}
+			copysize = (oldsize > size) ? size : oldsize;
+			memmove(new_ptr, ptr, copysize);
+			isc__mem_free(ctx0, ptr FLARG_PASS);
+		}
+	} else if (ptr != NULL)
+		isc__mem_free(ctx0, ptr FLARG_PASS);
+
+	return (new_ptr);
 }
 
 void
 isc__mem_free(isc_mem_t *ctx0, void *ptr FLARG) {
-	UNUSED(ctx0);
-	free(ptr);
+	isc__mem_t *ctx = (isc__mem_t *)ctx0;
+	size_info *si;
+	size_t size;
+	isc_boolean_t call_water= ISC_FALSE;
+
+	REQUIRE(VALID_CONTEXT(ctx));
+	REQUIRE(ptr != NULL);
+
+	if (ISC_UNLIKELY((isc_mem_debugging & ISC_MEM_DEBUGCTX) != 0)) {
+		si = &(((size_info *)ptr)[-2]);
+		REQUIRE(si->u.ctx == ctx);
+		size = si[1].u.size;
+	} else {
+		si = &(((size_info *)ptr)[-1]);
+		size = si->u.size;
+	}
+
+	MCTXLOCK(ctx, &ctx->lock);
+
+	DELETE_TRACE(ctx, ptr, size, file, line);
+
+	if ((ctx->flags & ISC_MEMFLAG_INTERNAL) != 0) {
+		mem_putunlocked(ctx, si, size);
+	} else {
+		mem_putstats(ctx, si, size);
+		mem_put(ctx, si, size);
+	}
+
+	/*
+	 * The check against ctx->lo_water == 0 is for the condition
+	 * when the context was pushed over hi_water but then had
+	 * isc_mem_setwater() called with 0 for hi_water and lo_water.
+	 */
+	if (ctx->is_overmem &&
+	    (ctx->inuse < ctx->lo_water || ctx->lo_water == 0U)) {
+		ctx->is_overmem = ISC_FALSE;
+	}
+
+	if (ctx->hi_called &&
+	    (ctx->inuse < ctx->lo_water || ctx->lo_water == 0U)) {
+		ctx->hi_called = ISC_FALSE;
+
+		if (ctx->water != NULL)
+			call_water = ISC_TRUE;
+	}
+	MCTXUNLOCK(ctx, &ctx->lock);
+
+	if (call_water)
+		(ctx->water)(ctx->water_arg, ISC_MEM_LOWATER);
 }
 
 
@@ -1166,8 +1465,21 @@ isc__mem_free(isc_mem_t *ctx0, void *ptr FLARG) {
 
 char *
 isc__mem_strdup(isc_mem_t *mctx0, const char *s FLARG) {
-	UNUSED(mctx0);
-	return (strdup(s));
+	isc__mem_t *mctx = (isc__mem_t *)mctx0;
+	size_t len;
+	char *ns;
+
+	REQUIRE(VALID_CONTEXT(mctx));
+	REQUIRE(s != NULL);
+
+	len = strlen(s) + 1;
+
+	ns = isc__mem_allocate((isc_mem_t *)mctx, len FLARG_PASS);
+
+	if (ns != NULL)
+		strlcpy(ns, s, len);
+
+	return (ns);
 }
 
 void

lib/isc/task.c

@@ -19,6 +19,7 @@
 #include <config.h>
 
 #include <isc/app.h>
+#include <isc/atomic.h>
 #include <isc/condition.h>
 #include <isc/event.h>
 #include <isc/json.h>
@@ -102,6 +103,7 @@ struct isc__task {
 	isc_time_t			tnow;
 	char				name[16];
 	void *				tag;
+	int				c;
 	/* Locked by task manager lock. */
 	LINK(isc__task_t)		link;
 	LINK(isc__task_t)		ready_link;
@@ -124,19 +126,20 @@ struct isc__taskmgr {
 	isc_taskmgr_t			common;
 	isc_mem_t *			mctx;
 	isc_mutex_t			lock;
+	isc_mutex_t			*locks;
 	unsigned int			workers;
 	isc_thread_t *			threads;
 	/* Locked by task manager lock. */
 	unsigned int			default_quantum;
 	LIST(isc__task_t)		tasks;
-	isc__tasklist_t			ready_tasks;
-	isc__tasklist_t			ready_priority_tasks;
+	isc__tasklist_t			*ready_tasks;
+	isc__tasklist_t			*ready_priority_tasks;
 	isc_taskmgrmode_t		mode;
-	isc_condition_t			work_available;
+	isc_condition_t			*work_available;
 	isc_condition_t			exclusive_granted;
 	isc_condition_t			paused;
-	unsigned int			tasks_running;
-	unsigned int			tasks_ready;
+	isc_int32_t			tasks_running;
+	isc_int32_t			tasks_ready;
 	isc_boolean_t			pause_requested;
 	isc_boolean_t			exclusive_requested;
 	isc_boolean_t			exiting;
@@ -173,13 +176,13 @@ isc_taskmgr_setexcltask(isc_taskmgr_t *mgr0, isc_task_t *task0);
 isc_result_t
 isc_taskmgr_excltask(isc_taskmgr_t *mgr0, isc_task_t **taskp);
 static inline isc_boolean_t
-empty_readyq(isc__taskmgr_t *manager);
+empty_readyq(isc__taskmgr_t *manager, int c);
 
 static inline isc__task_t *
-pop_readyq(isc__taskmgr_t *manager);
+pop_readyq(isc__taskmgr_t *manager, int c);
 
 static inline void
-push_readyq(isc__taskmgr_t *manager, isc__task_t *task);
+push_readyq(isc__taskmgr_t *manager, isc__task_t *task, int c);
 
 /***
  *** Tasks.
@@ -188,7 +191,6 @@ push_readyq(isc__taskmgr_t *manager, isc__task_t *task);
 static void
 task_finished(isc__task_t *task) {
 	isc__taskmgr_t *manager = task->manager;
-
 	REQUIRE(EMPTY(task->events));
 	REQUIRE(task->nevents == 0);
 	REQUIRE(EMPTY(task->on_shutdown));
@@ -199,6 +201,7 @@ task_finished(isc__task_t *task) {
 
 	LOCK(&manager->lock);
 	UNLINK(manager->tasks, task, link);
+	UNLOCK(&manager->lock);
 	if (FINISHED(manager)) {
 		/*
 		 * All tasks have completed and the
@@ -206,10 +209,10 @@ task_finished(isc__task_t *task) {
 		 * any idle worker threads so they
 		 * can exit.
 		 */
-		BROADCAST(&manager->work_available);
+		for (int i=0; i<manager->workers; i++) {
+			BROADCAST(&manager->work_available[i]);
+		}
 	}
-	UNLOCK(&manager->lock);
-
 	DESTROYLOCK(&task->lock);
 	task->common.impmagic = 0;
 	task->common.magic = 0;
@@ -233,6 +236,7 @@ isc_task_create(isc_taskmgr_t *manager0, unsigned int quantum,
 		return (ISC_R_NOMEMORY);
 	XTRACE("isc_task_create");
 	task->manager = manager;
+	task->c = isc_random_uniform(manager->workers);
 	result = isc_mutex_init(&task->lock);
 	if (result != ISC_R_SUCCESS) {
 		isc_mem_put(manager->mctx, task, sizeof(*task));
@@ -349,12 +353,11 @@ task_ready(isc__task_t *task) {
 	REQUIRE(task->state == task_state_ready);
 
 	XTRACE("task_ready");
-
-	LOCK(&manager->lock);
-	push_readyq(manager, task);
+	LOCK(&manager->locks[task->c]);
+	push_readyq(manager, task, task->c);
 	if (manager->mode == isc_taskmgrmode_normal || has_privilege)
-		SIGNAL(&manager->work_available);
-	UNLOCK(&manager->lock);
+		BROADCAST(&manager->work_available[task->c]);
+	UNLOCK(&manager->locks[task->c]);
 }
 
 static inline isc_boolean_t
@@ -432,6 +435,7 @@ task_send(isc__task_t *task, isc_event_t **eventp) {
 	if (task->state == task_state_idle) {
 		was_idle = ISC_TRUE;
 		INSIST(EMPTY(task->events));
+		task->c = isc_random_uniform(task->manager->workers);
 		task->state = task_state_ready;
 	}
 	INSIST(task->state == task_state_ready ||
@@ -827,13 +831,13 @@ isc_task_getcurrenttimex(isc_task_t *task0, isc_time_t *t) {
  * Caller must hold the task manager lock.
  */
 static inline isc_boolean_t
-empty_readyq(isc__taskmgr_t *manager) {
+empty_readyq(isc__taskmgr_t *manager, int c) {
 	isc__tasklist_t queue;
 
 	if (manager->mode == isc_taskmgrmode_normal)
-		queue = manager->ready_tasks;
+		queue = manager->ready_tasks[c];
 	else
-		queue = manager->ready_priority_tasks;
+		queue = manager->ready_priority_tasks[c];
 
 	return (ISC_TF(EMPTY(queue)));
 }
@@ -847,18 +851,18 @@ empty_readyq(isc__taskmgr_t *manager) {
  * Caller must hold the task manager lock.
  */
 static inline isc__task_t *
-pop_readyq(isc__taskmgr_t *manager) {
+pop_readyq(isc__taskmgr_t *manager, int c) {
 	isc__task_t *task;
 
 	if (manager->mode == isc_taskmgrmode_normal)
-		task = HEAD(manager->ready_tasks);
+		task = HEAD(manager->ready_tasks[c]);
 	else
-		task = HEAD(manager->ready_priority_tasks);
+		task = HEAD(manager->ready_priority_tasks[c]);
 
 	if (task != NULL) {
-		DEQUEUE(manager->ready_tasks, task, ready_link);
+		DEQUEUE(manager->ready_tasks[c], task, ready_link);
 		if (ISC_LINK_LINKED(task, ready_priority_link))
-			DEQUEUE(manager->ready_priority_tasks, task,
+			DEQUEUE(manager->ready_priority_tasks[c], task,
 				ready_priority_link);
 	}
 
@@ -872,16 +876,16 @@ pop_readyq(isc__taskmgr_t *manager) {
  * Caller must hold the task manager lock.
  */
 static inline void
-push_readyq(isc__taskmgr_t *manager, isc__task_t *task) {
-	ENQUEUE(manager->ready_tasks, task, ready_link);
+push_readyq(isc__taskmgr_t *manager, isc__task_t *task, int c) {
+	ENQUEUE(manager->ready_tasks[c], task, ready_link);
 	if ((task->flags & TASK_F_PRIVILEGED) != 0)
-		ENQUEUE(manager->ready_priority_tasks, task,
+		ENQUEUE(manager->ready_priority_tasks[c], task,
 			ready_priority_link);
-	manager->tasks_ready++;
+	isc_atomic_xadd(&manager->tasks_ready, 1);
 }
 
 static void
-dispatch(isc__taskmgr_t *manager) {
+dispatch(isc__taskmgr_t *manager, int c) {
 	isc__task_t *task;
 
 	REQUIRE(VALID_MANAGER(manager));
@@ -935,9 +939,8 @@ dispatch(isc__taskmgr_t *manager) {
 	 * For N iterations of the loop, this code does N+1 locks and N+1
 	 * unlocks.  The while expression is always protected by the lock.
 	 */
-
-	LOCK(&manager->lock);
-
+	LOCK(&manager->locks[c]);
+	
 	while (!FINISHED(manager)) {
 		/*
 		 * For reasons similar to those given in the comment in
@@ -949,13 +952,13 @@ dispatch(isc__taskmgr_t *manager) {
 		 * If a pause has been requested, don't do any work
 		 * until it's been released.
 		 */
-		while ((empty_readyq(manager) || manager->pause_requested ||
+		while ((empty_readyq(manager, c) || manager->pause_requested ||
 			manager->exclusive_requested) && !FINISHED(manager))
 		{
 			XTHREADTRACE(isc_msgcat_get(isc_msgcat,
 						    ISC_MSGSET_GENERAL,
 						    ISC_MSG_WAIT, "wait"));
-			WAIT(&manager->work_available, &manager->lock);
+			WAIT(&manager->work_available[c], &manager->locks[c]);
 			XTHREADTRACE(isc_msgcat_get(isc_msgcat,
 						    ISC_MSGSET_TASK,
 						    ISC_MSG_AWAKE, "awake"));
@@ -963,7 +966,7 @@ dispatch(isc__taskmgr_t *manager) {
 		XTHREADTRACE(isc_msgcat_get(isc_msgcat, ISC_MSGSET_TASK,
 					    ISC_MSG_WORKING, "working"));
 
-		task = pop_readyq(manager);
+		task = pop_readyq(manager, c);
 		if (task != NULL) {
 			unsigned int dispatch_count = 0;
 			isc_boolean_t done = ISC_FALSE;
@@ -978,13 +981,14 @@ dispatch(isc__taskmgr_t *manager) {
 			 * have a task to do.  We must reacquire the manager
 			 * lock before exiting the 'if (task != NULL)' block.
 			 */
-			manager->tasks_ready--;
-			manager->tasks_running++;
-			UNLOCK(&manager->lock);
+			UNLOCK(&manager->locks[c]);
+			isc_atomic_xadd(&manager->tasks_ready, -1);
+			isc_atomic_xadd(&manager->tasks_running, 1);
 
 			LOCK(&task->lock);
 			INSIST(task->state == task_state_ready);
 			task->state = task_state_running;
+			task->c = c;
 			XTRACE(isc_msgcat_get(isc_msgcat, ISC_MSGSET_GENERAL,
 					      ISC_MSG_RUNNING, "running"));
 			TIME_NOW(&task->tnow);
@@ -1092,8 +1096,7 @@ dispatch(isc__taskmgr_t *manager) {
 			if (finished)
 				task_finished(task);
 
-			LOCK(&manager->lock);
-			manager->tasks_running--;
+			isc_atomic_xadd(&manager->tasks_running, -1);;
 			if (manager->exclusive_requested &&
 			    manager->tasks_running == 1) {
 				SIGNAL(&manager->exclusive_granted);
@@ -1121,7 +1124,9 @@ dispatch(isc__taskmgr_t *manager) {
 				 * were usually nonempty, the 'optimization'
 				 * might even hurt rather than help.
 				 */
-				push_readyq(manager, task);
+				LOCK(&manager->locks[c]);
+				push_readyq(manager, task, c);
+				UNLOCK(&manager->locks[c]);
 			}
 		}
 
@@ -1131,27 +1136,34 @@ dispatch(isc__taskmgr_t *manager) {
 		 * we're stuck.  Automatically drop privileges at that
 		 * point and continue with the regular ready queue.
 		 */
-		if (manager->tasks_running == 0 && empty_readyq(manager)) {
+		if (manager->tasks_running == 0 && empty_readyq(manager, c)) {
 			manager->mode = isc_taskmgrmode_normal;
-			if (!empty_readyq(manager))
-				BROADCAST(&manager->work_available);
+			if (!empty_readyq(manager, c))
+				BROADCAST(&manager->work_available[c]);
 		}
 	}
-
-	UNLOCK(&manager->lock);
+	UNLOCK(&manager->locks[c]);
 }
 
+typedef struct st {
+	isc__taskmgr_t *manager;
+	int c;
+} stt;
+
 static isc_threadresult_t
 #ifdef _WIN32
 WINAPI
 #endif
 run(void *uap) {
-	isc__taskmgr_t *manager = uap;
+	stt *st = uap;
+	isc__taskmgr_t *manager = st->manager;
+	int c = st->c;
+	free(st);
 
 	XTHREADTRACE(isc_msgcat_get(isc_msgcat, ISC_MSGSET_GENERAL,
 				    ISC_MSG_STARTING, "starting"));
 
-	dispatch(manager);
+	dispatch(manager, c);
 
 	XTHREADTRACE(isc_msgcat_get(isc_msgcat, ISC_MSGSET_GENERAL,
 				    ISC_MSG_EXITING, "exiting"));
@@ -1168,7 +1180,7 @@ manager_free(isc__taskmgr_t *manager) {
 	isc_mem_t *mctx;
 
 	(void)isc_condition_destroy(&manager->exclusive_granted);
-	(void)isc_condition_destroy(&manager->work_available);
+	(void)isc_condition_destroy(&manager->work_available[0]);
 	(void)isc_condition_destroy(&manager->paused);
 	isc_mem_free(manager->mctx, manager->threads);
 	DESTROYLOCK(&manager->lock);
@@ -1218,14 +1230,6 @@ isc_taskmgr_create(isc_mem_t *mctx, unsigned int workers,
 		result = ISC_R_NOMEMORY;
 		goto cleanup_lock;
 	}
-	if (isc_condition_init(&manager->work_available) != ISC_R_SUCCESS) {
-		UNEXPECTED_ERROR(__FILE__, __LINE__,
-				 "isc_condition_init() %s",
-				 isc_msgcat_get(isc_msgcat, ISC_MSGSET_GENERAL,
-						ISC_MSG_FAILED, "failed"));
-		result = ISC_R_UNEXPECTED;
-		goto cleanup_threads;
-	}
 	if (isc_condition_init(&manager->exclusive_granted) != ISC_R_SUCCESS) {
 		UNEXPECTED_ERROR(__FILE__, __LINE__,
 				 "isc_condition_init() %s",
@@ -1246,8 +1250,10 @@ isc_taskmgr_create(isc_mem_t *mctx, unsigned int workers,
 		default_quantum = DEFAULT_DEFAULT_QUANTUM;
 	manager->default_quantum = default_quantum;
 	INIT_LIST(manager->tasks);
-	INIT_LIST(manager->ready_tasks);
-	INIT_LIST(manager->ready_priority_tasks);
+	manager->ready_tasks = malloc(workers * sizeof(isc__tasklist_t));
+	manager->locks = malloc(workers * sizeof(isc_mutex_t));
+	manager->work_available = malloc(workers * sizeof(isc_condition_t));
+	manager->ready_priority_tasks = malloc(workers * sizeof(isc__tasklist_t));
 	manager->tasks_running = 0;
 	manager->tasks_ready = 0;
 	manager->exclusive_requested = ISC_FALSE;
@@ -1262,16 +1268,34 @@ isc_taskmgr_create(isc_mem_t *mctx, unsigned int workers,
 	 * Start workers.
 	 */
 	for (i = 0; i < workers; i++) {
-		if (isc_thread_create(run, manager,
-				      &manager->threads[manager->workers]) ==
-		    ISC_R_SUCCESS) {
-			char name[16];	/* thread name limit on Linux */
-			snprintf(name, sizeof(name), "isc-worker%04u", i);
-			isc_thread_setname(manager->threads[manager->workers],
-					   name);
-			manager->workers++;
-			started++;
+		INIT_LIST(manager->ready_tasks[i]);
+		INIT_LIST(manager->ready_priority_tasks[i]);
+		isc_mutex_init(&manager->locks[i]);
+		if (isc_condition_init(&manager->work_available[i]) != ISC_R_SUCCESS) {
+			UNEXPECTED_ERROR(__FILE__, __LINE__,
+					 "isc_condition_init() %s",
+					 isc_msgcat_get(isc_msgcat, ISC_MSGSET_GENERAL,
+							ISC_MSG_FAILED, "failed"));
+			result = ISC_R_UNEXPECTED;
+			goto cleanup_threads;
 		}
+		stt *st = malloc(sizeof(stt));
+		st->manager = manager;
+		st->c = i;
+		if (isc_thread_create(run, st,
+				      &manager->threads[i]) != ISC_R_SUCCESS) {
+			goto cleanup_threads;
+		}
+		char name[16];	/* thread name limit on Linux */
+		snprintf(name, sizeof(name), "isc-worker%04u", i);
+		isc_thread_setname(manager->threads[manager->workers],
+				   name);
+		cpu_set_t cpuset;
+		CPU_ZERO(&cpuset);
+	        CPU_SET(i, &cpuset);
+	        pthread_setaffinity_np(manager->threads[i], sizeof(cpu_set_t), &cpuset);
+		manager->workers++;
+		started++;
 	}
 	UNLOCK(&manager->lock);
 
@@ -1288,7 +1312,7 @@ isc_taskmgr_create(isc_mem_t *mctx, unsigned int workers,
  cleanup_exclusivegranted:
 	(void)isc_condition_destroy(&manager->exclusive_granted);
  cleanup_workavailable:
-	(void)isc_condition_destroy(&manager->work_available);
+	(void)isc_condition_destroy(&manager->work_available[0]);
  cleanup_threads:
 	isc_mem_free(mctx, manager->threads);
  cleanup_lock:
@@ -1360,7 +1384,7 @@ isc_taskmgr_destroy(isc_taskmgr_t **managerp) {
 	     task = NEXT(task, link)) {
 		LOCK(&task->lock);
 		if (task_shutdown(task))
-			push_readyq(manager, task);
+			push_readyq(manager, task, 0);
 		UNLOCK(&task->lock);
 	}
 	/*
@@ -1368,7 +1392,7 @@ isc_taskmgr_destroy(isc_taskmgr_t **managerp) {
 	 * there's work left to do, and if there are already no tasks left
 	 * it will cause the workers to see manager->exiting.
 	 */
-	BROADCAST(&manager->work_available);
+	BROADCAST(&manager->work_available[0]);
 	UNLOCK(&manager->lock);
 
 	/*
@@ -1419,7 +1443,7 @@ isc__taskmgr_resume(isc_taskmgr_t *manager0) {
 	LOCK(&manager->lock);
 	if (manager->pause_requested) {
 		manager->pause_requested = ISC_FALSE;
-		BROADCAST(&manager->work_available);
+		BROADCAST(&manager->work_available[0]);
 	}
 	UNLOCK(&manager->lock);
 }
@@ -1491,7 +1515,9 @@ isc_task_endexclusive(isc_task_t *task0) {
 	LOCK(&manager->lock);
 	REQUIRE(manager->exclusive_requested);
 	manager->exclusive_requested = ISC_FALSE;
-	BROADCAST(&manager->work_available);
+	for (int i=0; i < manager->workers; i++) {
+		BROADCAST(&manager->work_available[i]);
+	}
 	UNLOCK(&manager->lock);
 }
 
@@ -1513,14 +1539,14 @@ isc_task_setprivilege(isc_task_t *task0, isc_boolean_t priv) {
 	if (priv == oldpriv)
 		return;
 
-	LOCK(&manager->lock);
+	LOCK(&manager->locks[task->c]);
 	if (priv && ISC_LINK_LINKED(task, ready_link))
-		ENQUEUE(manager->ready_priority_tasks, task,
+		ENQUEUE(manager->ready_priority_tasks[task->c], task,
 			ready_priority_link);
 	else if (!priv && ISC_LINK_LINKED(task, ready_priority_link))
-		DEQUEUE(manager->ready_priority_tasks, task,
+		DEQUEUE(manager->ready_priority_tasks[task->c], task,
 			ready_priority_link);
-	UNLOCK(&manager->lock);
+	UNLOCK(&manager->locks[task->c]);
 }
 
 isc_boolean_t