Badcache with multiple locks.

Previously badcache used one single mutex for everything, which
was causing performance issues. Use one global rwlock for the whole
hashtable and per-bucket mutexes.

bin/named/server.c

@@ -145,13 +145,13 @@
 #endif /* ifndef SIZE_AS_PERCENT */
 
 #ifdef TUNE_LARGE
-#define RESOLVER_NTASKS 523
-#define UDPBUFFERS	32768
-#define EXCLBUFFERS	32768
-#else /* ifdef TUNE_LARGE */
-#define RESOLVER_NTASKS 31
-#define UDPBUFFERS	1000
-#define EXCLBUFFERS	4096
+#define RESOLVER_NTASKS_PERCPU 32
+#define UDPBUFFERS	       32768
+#define EXCLBUFFERS	       32768
+#else
+#define RESOLVER_NTASKS_PERCPU 8
+#define UDPBUFFERS	       1000
+#define EXCLBUFFERS	       4096
 #endif /* TUNE_LARGE */
 
 #define MAX_TCP_TIMEOUT 65535
@@ -1286,7 +1286,7 @@ get_view_querysource_dispatch(const cfg_obj_t **maps, int af,
 		break;
 	}
 	if (isc_sockaddr_getport(&sa) == 0) {
-		attrs |= DNS_DISPATCHATTR_EXCLUSIVE;
+		//		attrs |= DNS_DISPATCHATTR_EXCLUSIVE;
 		maxdispatchbuffers = EXCLBUFFERS;
 	} else {
 		INSIST(obj != NULL);
@@ -4539,8 +4539,8 @@ configure_view(dns_view_t *view, dns_viewlist_t *viewlist, cfg_obj_t *config,
 
 	ndisp = 4 * ISC_MIN(named_g_udpdisp, MAX_UDP_DISPATCH);
 	CHECK(dns_view_createresolver(
-		view, named_g_taskmgr, RESOLVER_NTASKS, ndisp,
-		named_g_socketmgr, named_g_timermgr, resopts,
+		view, named_g_taskmgr, RESOLVER_NTASKS_PERCPU * named_g_cpus,
+		ndisp, named_g_socketmgr, named_g_timermgr, resopts,
 		named_g_dispatchmgr, dispatch4, dispatch6));
 
 	if (dscp4 == -1) {
@@ -9637,7 +9637,7 @@ run_server(isc_task_t *task, isc_event_t *event) {
 			   named_g_mctx, server->sctx, named_g_taskmgr,
 			   named_g_timermgr, named_g_socketmgr, named_g_nm,
 			   named_g_dispatchmgr, server->task, named_g_udpdisp,
-			   geoip, &server->interfacemgr),
+			   geoip, named_g_cpus, &server->interfacemgr),
 		   "creating interface manager");
 
 	CHECKFATAL(isc_timer_create(named_g_timermgr, isc_timertype_inactive,

config.h.in

@@ -471,6 +471,9 @@
 /* Define to 1 if you have the `uv_handle_set_data' function. */
 #undef HAVE_UV_HANDLE_SET_DATA
 
+/* Define to 1 if you have the `uv_import' function. */
+#undef HAVE_UV_IMPORT
+
 /* Use zlib library */
 #undef HAVE_ZLIB
 

config.h.win32

@@ -355,6 +355,15 @@ typedef __int64 off_t;
 /* Define to 1 if you have the `HMAC_CTX_reset' function. */
 @HAVE_HMAC_CTX_RESET@
 
+/* Define to 1 if you have the `uv_handle_get_data' function. */
+@HAVE_UV_HANDLE_GET_DATA@
+
+/* Define to 1 if you have the `uv_handle_set_data' function. */
+@HAVE_UV_HANDLE_SET_DATA@
+
+/* Define to 1 if you have the `uv_import' function. */
+@HAVE_UV_IMPORT@
+
 /*
  * Define to nothing if C supports flexible array members, and to 1 if it does
  * not. That way, with a declaration like `struct s { int n; double

configure

@@ -15960,7 +15960,7 @@ LIBS="$LIBS $LIBUV_LIBS"
 
 # Those functions are only provided in newer versions of libuv, we'll be emulating them
 # for now
-for ac_func in uv_handle_get_data uv_handle_set_data
+for ac_func in uv_handle_get_data uv_handle_set_data uv_import
 do :
   as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh`
 ac_fn_c_check_func "$LINENO" "$ac_func" "$as_ac_var"

configure.ac

@@ -667,7 +667,7 @@ LIBS="$LIBS $LIBUV_LIBS"
 
 # Those functions are only provided in newer versions of libuv, we'll be emulating them
 # for now
-AC_CHECK_FUNCS([uv_handle_get_data uv_handle_set_data])
+AC_CHECK_FUNCS([uv_handle_get_data uv_handle_set_data uv_import])
 
 #
 # flockfile is usually provided by pthreads

lib/dns/badcache.c

@@ -21,6 +21,7 @@
 #include <isc/mutex.h>
 #include <isc/platform.h>
 #include <isc/print.h>
+#include <isc/rwlock.h>
 #include <isc/string.h>
 #include <isc/time.h>
 #include <isc/util.h>
@@ -34,14 +35,17 @@ typedef struct dns_bcentry dns_bcentry_t;
 
 struct dns_badcache {
 	unsigned int magic;
-	isc_mutex_t lock;
+	isc_rwlock_t lock;
 	isc_mem_t *mctx;
 
+	isc_mutex_t *tlocks;
 	dns_bcentry_t **table;
-	unsigned int count;
+
+	atomic_uint_fast32_t count;
+	atomic_uint_fast32_t sweep;
+
 	unsigned int minsize;
 	unsigned int size;
-	unsigned int sweep;
 };
 
 #define BADCACHE_MAGIC	  ISC_MAGIC('B', 'd', 'C', 'a')
@@ -56,12 +60,13 @@ struct dns_bcentry {
 	dns_name_t name;
 };
 
-static isc_result_t
-badcache_resize(dns_badcache_t *bc, isc_time_t *now, bool grow);
+static void
+badcache_resize(dns_badcache_t *bc, isc_time_t *now);
 
 isc_result_t
 dns_badcache_init(isc_mem_t *mctx, unsigned int size, dns_badcache_t **bcp) {
 	dns_badcache_t *bc = NULL;
+	unsigned int i;
 
 	REQUIRE(bcp != NULL && *bcp == NULL);
 	REQUIRE(mctx != NULL);
@@ -70,15 +75,18 @@ dns_badcache_init(isc_mem_t *mctx, unsigned int size, dns_badcache_t **bcp) {
 	memset(bc, 0, sizeof(dns_badcache_t));
 
 	isc_mem_attach(mctx, &bc->mctx);
-	isc_mutex_init(&bc->lock);
+	isc_rwlock_init(&bc->lock, 0, 0);
 
 	bc->table = isc_mem_get(bc->mctx, sizeof(*bc->table) * size);
-
+	bc->tlocks = isc_mem_get(bc->mctx, sizeof(isc_mutex_t) * size);
+	for (i = 0; i < size; i++) {
+		isc_mutex_init(&bc->tlocks[i]);
+	}
 	bc->size = bc->minsize = size;
 	memset(bc->table, 0, bc->size * sizeof(dns_bcentry_t *));
 
-	bc->count = 0;
-	bc->sweep = 0;
+	atomic_init(&bc->count, 0);
+	atomic_init(&bc->sweep, 0);
 	bc->magic = BADCACHE_MAGIC;
 
 	*bcp = bc;
@@ -88,6 +96,7 @@ dns_badcache_init(isc_mem_t *mctx, unsigned int size, dns_badcache_t **bcp) {
 void
 dns_badcache_destroy(dns_badcache_t **bcp) {
 	dns_badcache_t *bc;
+	unsigned int i;
 
 	REQUIRE(bcp != NULL && *bcp != NULL);
 	bc = *bcp;
@@ -96,15 +105,42 @@ dns_badcache_destroy(dns_badcache_t **bcp) {
 	dns_badcache_flush(bc);
 
 	bc->magic = 0;
-	isc_mutex_destroy(&bc->lock);
+	isc_rwlock_destroy(&bc->lock);
+	for (i = 0; i < bc->size; i++) {
+		isc_mutex_destroy(&bc->tlocks[i]);
+	}
 	isc_mem_put(bc->mctx, bc->table, sizeof(dns_bcentry_t *) * bc->size);
+	isc_mem_put(bc->mctx, bc->tlocks, sizeof(isc_mutex_t) * bc->size);
 	isc_mem_putanddetach(&bc->mctx, bc, sizeof(dns_badcache_t));
 }
 
-static isc_result_t
-badcache_resize(dns_badcache_t *bc, isc_time_t *now, bool grow) {
+static void
+badcache_resize(dns_badcache_t *bc, isc_time_t *now) {
 	dns_bcentry_t **newtable, *bad, *next;
+	isc_mutex_t *newlocks;
 	unsigned int newsize, i;
+	bool grow;
+
+	RWLOCK(&bc->lock, isc_rwlocktype_write);
+
+	/*
+	 * XXXWPK we will have a thundering herd problem here,
+	 * as all threads will wait on the RWLOCK when there's
+	 * a need to resize badcache.
+	 * However, it happens so rarely it should not be a
+	 * performance issue.
+	 */
+	if (atomic_load_relaxed(&bc->count) > bc->size * 8) {
+		grow = true;
+	} else if (atomic_load_relaxed(&bc->count) < bc->size * 2 &&
+		   bc->size > bc->minsize)
+	{
+		grow = false;
+	} else {
+		/* Someone resized it already, bail. */
+		RWUNLOCK(&bc->lock, isc_rwlocktype_write);
+		return;
+	}
 
 	if (grow) {
 		newsize = bc->size * 2 + 1;
@@ -115,13 +151,24 @@ badcache_resize(dns_badcache_t *bc, isc_time_t *now, bool grow) {
 	newtable = isc_mem_get(bc->mctx, sizeof(dns_bcentry_t *) * newsize);
 	memset(newtable, 0, sizeof(dns_bcentry_t *) * newsize);
 
-	for (i = 0; bc->count > 0 && i < bc->size; i++) {
+	newlocks = isc_mem_get(bc->mctx, sizeof(isc_mutex_t) * newsize);
+	for (i = 0; i < newsize && i < bc->size; i++) {
+		newlocks[i] = bc->tlocks[i];
+	}
+	for (i = bc->size; i < newsize; i++) {
+		isc_mutex_init(&newlocks[i]);
+	}
+	for (i = newsize; i < bc->size; i++) {
+		isc_mutex_destroy(&bc->tlocks[i]);
+	}
+
+	for (i = 0; atomic_load_relaxed(&bc->count) > 0 && i < bc->size; i++) {
 		for (bad = bc->table[i]; bad != NULL; bad = next) {
 			next = bad->next;
 			if (isc_time_compare(&bad->expire, now) < 0) {
 				isc_mem_put(bc->mctx, bad,
 					    sizeof(*bad) + bad->name.length);
-				bc->count--;
+				atomic_fetch_sub_relaxed(&bc->count, 1);
 			} else {
 				bad->next = newtable[bad->hashval % newsize];
 				newtable[bad->hashval % newsize] = bad;
@@ -130,11 +177,14 @@ badcache_resize(dns_badcache_t *bc, isc_time_t *now, bool grow) {
 		bc->table[i] = NULL;
 	}
 
+	isc_mem_put(bc->mctx, bc->tlocks, sizeof(isc_mutex_t) * bc->size);
+	bc->tlocks = newlocks;
+
 	isc_mem_put(bc->mctx, bc->table, sizeof(*bc->table) * bc->size);
 	bc->size = newsize;
 	bc->table = newtable;
 
-	return (ISC_R_SUCCESS);
+	RWUNLOCK(&bc->lock, isc_rwlocktype_write);
 }
 
 void
@@ -142,15 +192,16 @@ dns_badcache_add(dns_badcache_t *bc, const dns_name_t *name,
 		 dns_rdatatype_t type, bool update, uint32_t flags,
 		 isc_time_t *expire) {
 	isc_result_t result;
-	unsigned int i, hashval;
+	unsigned int hashval, hash;
 	dns_bcentry_t *bad, *prev, *next;
 	isc_time_t now;
+	bool resize = false;
 
 	REQUIRE(VALID_BADCACHE(bc));
 	REQUIRE(name != NULL);
 	REQUIRE(expire != NULL);
 
-	LOCK(&bc->lock);
+	RWLOCK(&bc->lock, isc_rwlocktype_read);
 
 	result = isc_time_now(&now);
 	if (result != ISC_R_SUCCESS) {
@@ -158,9 +209,10 @@ dns_badcache_add(dns_badcache_t *bc, const dns_name_t *name,
 	}
 
 	hashval = dns_name_hash(name, false);
-	i = hashval % bc->size;
+	hash = hashval % bc->size;
+	LOCK(&bc->tlocks[hash]);
 	prev = NULL;
-	for (bad = bc->table[i]; bad != NULL; bad = next) {
+	for (bad = bc->table[hash]; bad != NULL; bad = next) {
 		next = bad->next;
 		if (bad->type == type && dns_name_equal(name, &bad->name)) {
 			if (update) {
@@ -171,13 +223,13 @@ dns_badcache_add(dns_badcache_t *bc, const dns_name_t *name,
 		}
 		if (isc_time_compare(&bad->expire, &now) < 0) {
 			if (prev == NULL) {
-				bc->table[i] = bad->next;
+				bc->table[hash] = bad->next;
 			} else {
 				prev->next = bad->next;
 			}
 			isc_mem_put(bc->mctx, bad,
 				    sizeof(*bad) + bad->name.length);
-			bc->count--;
+			atomic_fetch_sub_relaxed(&bc->count, 1);
 		} else {
 			prev = bad;
 		}
@@ -193,20 +245,22 @@ dns_badcache_add(dns_badcache_t *bc, const dns_name_t *name,
 		isc_buffer_init(&buffer, bad + 1, name->length);
 		dns_name_init(&bad->name, NULL);
 		dns_name_copy(name, &bad->name, &buffer);
-		bad->next = bc->table[i];
-		bc->table[i] = bad;
-		bc->count++;
-		if (bc->count > bc->size * 8) {
-			badcache_resize(bc, &now, true);
-		}
-		if (bc->count < bc->size * 2 && bc->size > bc->minsize) {
-			badcache_resize(bc, &now, false);
+		bad->next = bc->table[hash];
+		bc->table[hash] = bad;
+		unsigned count = atomic_fetch_add_relaxed(&bc->count, 1);
+		if ((count > bc->size * 8) ||
+		    (count < bc->size * 2 && bc->size > bc->minsize)) {
+			resize = true;
 		}
 	} else {
 		bad->expire = *expire;
 	}
 
-	UNLOCK(&bc->lock);
+	UNLOCK(&bc->tlocks[hash]);
+	RWUNLOCK(&bc->lock, isc_rwlocktype_read);
+	if (resize) {
+		badcache_resize(bc, &now);
+	}
 }
 
 bool
@@ -214,13 +268,13 @@ dns_badcache_find(dns_badcache_t *bc, const dns_name_t *name,
 		  dns_rdatatype_t type, uint32_t *flagp, isc_time_t *now) {
 	dns_bcentry_t *bad, *prev, *next;
 	bool answer = false;
-	unsigned int i;
+	unsigned int i, hash;
 
 	REQUIRE(VALID_BADCACHE(bc));
 	REQUIRE(name != NULL);
 	REQUIRE(now != NULL);
 
-	LOCK(&bc->lock);
+	RWLOCK(&bc->lock, isc_rwlocktype_read);
 
 	/*
 	 * XXXMUKS: dns_name_equal() is expensive as it does a
@@ -234,13 +288,14 @@ dns_badcache_find(dns_badcache_t *bc, const dns_name_t *name,
 	 * name->link to store the type specific part.
 	 */
 
-	if (bc->count == 0) {
+	if (atomic_load_relaxed(&bc->count) == 0) {
 		goto skip;
 	}
 
-	i = dns_name_hash(name, false) % bc->size;
+	hash = dns_name_hash(name, false) % bc->size;
 	prev = NULL;
-	for (bad = bc->table[i]; bad != NULL; bad = next) {
+	LOCK(&bc->tlocks[hash]);
+	for (bad = bc->table[hash]; bad != NULL; bad = next) {
 		next = bad->next;
 		/*
 		 * Search the hash list. Clean out expired records as we go.
@@ -249,12 +304,12 @@ dns_badcache_find(dns_badcache_t *bc, const dns_name_t *name,
 			if (prev != NULL) {
 				prev->next = bad->next;
 			} else {
-				bc->table[i] = bad->next;
+				bc->table[hash] = bad->next;
 			}
 
 			isc_mem_put(bc->mctx, bad,
 				    sizeof(*bad) + bad->name.length);
-			bc->count--;
+			atomic_fetch_sub(&bc->count, 1);
 			continue;
 		}
 		if (bad->type == type && dns_name_equal(name, &bad->name)) {
@@ -266,20 +321,25 @@ dns_badcache_find(dns_badcache_t *bc, const dns_name_t *name,
 		}
 		prev = bad;
 	}
+	UNLOCK(&bc->tlocks[hash]);
 skip:
 
 	/*
 	 * Slow sweep to clean out stale records.
 	 */
-	i = bc->sweep++ % bc->size;
+	i = atomic_fetch_add(&bc->sweep, 1) % bc->size;
 	bad = bc->table[i];
-	if (bad != NULL && isc_time_compare(&bad->expire, now) < 0) {
-		bc->table[i] = bad->next;
-		isc_mem_put(bc->mctx, bad, sizeof(*bad) + bad->name.length);
-		bc->count--;
+	if (isc_mutex_trylock(&bc->tlocks[i]) == ISC_R_SUCCESS) {
+		if (bad != NULL && isc_time_compare(&bad->expire, now) < 0) {
+			bc->table[i] = bad->next;
+			isc_mem_put(bc->mctx, bad,
+				    sizeof(*bad) + bad->name.length);
+			atomic_fetch_sub_relaxed(&bc->count, 1);
+		}
+		UNLOCK(&bc->tlocks[i]);
 	}
 
-	UNLOCK(&bc->lock);
+	RWUNLOCK(&bc->lock, isc_rwlocktype_read);
 	return (answer);
 }
 
@@ -288,17 +348,19 @@ dns_badcache_flush(dns_badcache_t *bc) {
 	dns_bcentry_t *entry, *next;
 	unsigned int i;
 
+	RWLOCK(&bc->lock, isc_rwlocktype_write);
 	REQUIRE(VALID_BADCACHE(bc));
 
-	for (i = 0; bc->count > 0 && i < bc->size; i++) {
+	for (i = 0; atomic_load_relaxed(&bc->count) > 0 && i < bc->size; i++) {
 		for (entry = bc->table[i]; entry != NULL; entry = next) {
 			next = entry->next;
 			isc_mem_put(bc->mctx, entry,
 				    sizeof(*entry) + entry->name.length);
-			bc->count--;
+			atomic_fetch_sub_relaxed(&bc->count, 1);
 		}
 		bc->table[i] = NULL;
 	}
+	RWUNLOCK(&bc->lock, isc_rwlocktype_write);
 }
 
 void
@@ -311,13 +373,14 @@ dns_badcache_flushname(dns_badcache_t *bc, const dns_name_t *name) {
 	REQUIRE(VALID_BADCACHE(bc));
 	REQUIRE(name != NULL);
 
-	LOCK(&bc->lock);
+	RWLOCK(&bc->lock, isc_rwlocktype_read);
 
 	result = isc_time_now(&now);
 	if (result != ISC_R_SUCCESS) {
 		isc_time_settoepoch(&now);
 	}
 	i = dns_name_hash(name, false) % bc->size;
+	LOCK(&bc->tlocks[i]);
 	prev = NULL;
 	for (bad = bc->table[i]; bad != NULL; bad = next) {
 		int n;
@@ -332,13 +395,14 @@ dns_badcache_flushname(dns_badcache_t *bc, const dns_name_t *name) {
 
 			isc_mem_put(bc->mctx, bad,
 				    sizeof(*bad) + bad->name.length);
-			bc->count--;
+			atomic_fetch_sub_relaxed(&bc->count, 1);
 		} else {
 			prev = bad;
 		}
 	}
+	UNLOCK(&bc->tlocks[i]);
 
-	UNLOCK(&bc->lock);
+	RWUNLOCK(&bc->lock, isc_rwlocktype_read);
 }
 
 void
@@ -352,14 +416,14 @@ dns_badcache_flushtree(dns_badcache_t *bc, const dns_name_t *name) {
 	REQUIRE(VALID_BADCACHE(bc));
 	REQUIRE(name != NULL);
 
-	LOCK(&bc->lock);
+	RWLOCK(&bc->lock, isc_rwlocktype_write);
 
 	result = isc_time_now(&now);
 	if (result != ISC_R_SUCCESS) {
 		isc_time_settoepoch(&now);
 	}
 
-	for (i = 0; bc->count > 0 && i < bc->size; i++) {
+	for (i = 0; atomic_load_relaxed(&bc->count) > 0 && i < bc->size; i++) {
 		prev = NULL;
 		for (bad = bc->table[i]; bad != NULL; bad = next) {
 			next = bad->next;
@@ -373,14 +437,14 @@ dns_badcache_flushtree(dns_badcache_t *bc, const dns_name_t *name) {
 
 				isc_mem_put(bc->mctx, bad,
 					    sizeof(*bad) + bad->name.length);
-				bc->count--;
+				atomic_fetch_sub_relaxed(&bc->count, 1);
 			} else {
 				prev = bad;
 			}
 		}
 	}
 
-	UNLOCK(&bc->lock);
+	RWUNLOCK(&bc->lock, isc_rwlocktype_write);
 }
 
 void
@@ -396,11 +460,15 @@ dns_badcache_print(dns_badcache_t *bc, const char *cachename, FILE *fp) {
 	REQUIRE(cachename != NULL);
 	REQUIRE(fp != NULL);
 
-	LOCK(&bc->lock);
+	/*
+	 * We write lock the tree to avoid relocking every node
+	 * individually.
+	 */
+	RWLOCK(&bc->lock, isc_rwlocktype_write);
 	fprintf(fp, ";\n; %s\n;\n", cachename);
 
 	TIME_NOW(&now);
-	for (i = 0; bc->count > 0 && i < bc->size; i++) {
+	for (i = 0; atomic_load_relaxed(&bc->count) > 0 && i < bc->size; i++) {
 		prev = NULL;
 		for (bad = bc->table[i]; bad != NULL; bad = next) {
 			next = bad->next;
@@ -413,7 +481,7 @@ dns_badcache_print(dns_badcache_t *bc, const char *cachename, FILE *fp) {
 
 				isc_mem_put(bc->mctx, bad,
 					    sizeof(*bad) + bad->name.length);
-				bc->count--;
+				atomic_fetch_sub_relaxed(&bc->count, 1);
 				continue;
 			}
 			prev = bad;
@@ -428,5 +496,5 @@ dns_badcache_print(dns_badcache_t *bc, const char *cachename, FILE *fp) {
 				namebuf, typebuf, t);
 		}
 	}
-	UNLOCK(&bc->lock);
+	RWUNLOCK(&bc->lock, isc_rwlocktype_write);
 }

lib/dns/compress.c

@@ -129,6 +129,7 @@ dns_compress_init(dns_compress_t *cctx, int edns, isc_mem_t *mctx) {
 	cctx->mctx = mctx;
 	cctx->count = 0;
 	cctx->allowed = DNS_COMPRESS_ENABLED;
+	cctx->arena_off = 0;
 
 	memset(&cctx->table[0], 0, sizeof(cctx->table));
 
@@ -382,6 +383,7 @@ dns_compress_add(dns_compress_t *cctx, const dns_name_t *name,
 	uint16_t toffset;
 	unsigned char *tmp;
 	isc_region_t r;
+	bool allocated = false;
 
 	REQUIRE(VALID_CCTX(cctx));
 	REQUIRE(dns_name_isabsolute(name));
@@ -407,7 +409,13 @@ dns_compress_add(dns_compress_t *cctx, const dns_name_t *name,
 	start = 0;
 	dns_name_toregion(name, &r);
 	length = r.length;
-	tmp = isc_mem_get(cctx->mctx, length);
+	if (cctx->arena_off + length < DNS_COMPRESS_ARENA_SIZE) {
+		tmp = &cctx->arena[cctx->arena_off];
+		cctx->arena_off += length;
+	} else {
+		allocated = true;
+		tmp = isc_mem_get(cctx->mctx, length);
+	}
 	/*
 	 * Copy name data to 'tmp' and make 'r' use 'tmp'.
 	 */
@@ -448,7 +456,7 @@ dns_compress_add(dns_compress_t *cctx, const dns_name_t *name,
 		 * 'node->r.base' becomes 'tmp' when start == 0.
 		 * Record this by setting 0x8000 so it can be freed later.
 		 */
-		if (start == 0) {
+		if (start == 0 && allocated) {
 			toffset |= 0x8000;
 		}
 		node->offset = toffset;
@@ -466,7 +474,11 @@ dns_compress_add(dns_compress_t *cctx, const dns_name_t *name,
 	}
 
 	if (start == 0) {
-		isc_mem_put(cctx->mctx, tmp, length);
+		if (!allocated) {
+			cctx->arena_off -= length;
+		} else {
+			isc_mem_put(cctx->mctx, tmp, length);
+		}
 	}
 }
 

lib/dns/include/dns/compress.h

@@ -51,7 +51,8 @@ ISC_LANG_BEGINDECLS
 #define DNS_COMPRESS_TABLEBITS	  6
 #define DNS_COMPRESS_TABLESIZE	  (1U << DNS_COMPRESS_TABLEBITS)
 #define DNS_COMPRESS_TABLEMASK	  (DNS_COMPRESS_TABLESIZE - 1)
-#define DNS_COMPRESS_INITIALNODES 16
+#define DNS_COMPRESS_INITIALNODES 24
+#define DNS_COMPRESS_ARENA_SIZE	  640
 
 typedef struct dns_compressnode dns_compressnode_t;
 
@@ -69,6 +70,9 @@ struct dns_compress {
 	int	     edns;    /*%< Edns version or -1. */
 	/*% Global compression table. */
 	dns_compressnode_t *table[DNS_COMPRESS_TABLESIZE];
+	/*% Preallocated arena for names. */
+	unsigned char arena[DNS_COMPRESS_ARENA_SIZE];
+	off_t	      arena_off;
 	/*% Preallocated nodes for the table. */
 	dns_compressnode_t initialnodes[DNS_COMPRESS_INITIALNODES];
 	uint16_t	   count; /*%< Number of nodes. */

lib/dns/rbtdb.c

@@ -174,8 +174,11 @@ typedef isc_rwlock_t nodelock_t;
  * to be 0 by default either with or without threads.
  */
 #ifndef DNS_RBTDB_LIMITLRUUPDATE
-#define DNS_RBTDB_LIMITLRUUPDATE 0
-#endif /* ifndef DNS_RBTDB_LIMITLRUUPDATE */
+#define DNS_RBTDB_LIMITLRUUPDATE 1
+#endif
+
+#define DNS_RBTDB_LRUUPDATE_GLUE    300
+#define DNS_RBTDB_LRUUPDATE_REGULAR 600
 
 /*
  * Allow clients with a virtual time of up to 5 minutes in the past to see
@@ -299,7 +302,7 @@ typedef ISC_LIST(dns_rbtnode_t) rbtnodelist_t;
 	(((header)->rdh_ttl > (now)) || \
 	 ((header)->rdh_ttl == (now) && ZEROTTL(header)))
 
-#define DEFAULT_NODE_LOCK_COUNT	   7 /*%< Should be prime. */
+#define DEFAULT_NODE_LOCK_COUNT	   53 /*%< Should be prime. */
 #define RBTDB_GLUE_TABLE_INIT_SIZE 2U
 
 /*%
@@ -319,7 +322,7 @@ typedef ISC_LIST(dns_rbtnode_t) rbtnodelist_t;
 #define DEFAULT_CACHE_NODE_LOCK_COUNT DNS_RBTDB_CACHE_NODE_LOCK_COUNT
 #endif /* if DNS_RBTDB_CACHE_NODE_LOCK_COUNT <= 1 */
 #else  /* ifdef DNS_RBTDB_CACHE_NODE_LOCK_COUNT */
-#define DEFAULT_CACHE_NODE_LOCK_COUNT 16
+#define DEFAULT_CACHE_NODE_LOCK_COUNT 97
 #endif /* DNS_RBTDB_CACHE_NODE_LOCK_COUNT */
 
 typedef struct {
@@ -2057,25 +2060,7 @@ decrement_reference(dns_rbtdb_t *rbtdb, dns_rbtnode_t *node,
 	 * we will add this node to a linked list of nodes in this locking
 	 * bucket which we will free later.
 	 */
-	if (tlock != isc_rwlocktype_write) {
-		/*
-		 * Locking hierarchy notwithstanding, we don't need to free
-		 * the node lock before acquiring the tree write lock because
-		 * we only do a trylock.
-		 */
-		if (tlock == isc_rwlocktype_read) {
-			result = isc_rwlock_tryupgrade(&rbtdb->tree_lock);
-		} else {
-			result = isc_rwlock_trylock(&rbtdb->tree_lock,
-						    isc_rwlocktype_write);
-		}
-		RUNTIME_CHECK(result == ISC_R_SUCCESS ||
-			      result == ISC_R_LOCKBUSY);
-
-		write_locked = (result == ISC_R_SUCCESS);
-	} else {
-		write_locked = true;
-	}
+	write_locked = (tlock == isc_rwlocktype_write);
 
 	refs = isc_refcount_decrement(&nodelock->references);
 	INSIST(refs > 0);
@@ -10339,12 +10324,12 @@ need_headerupdate(rdatasetheader_t *header, isc_stdtime_t now) {
 		 * Glue records are updated if at least 60 seconds have passed
 		 * since the previous update time.
 		 */
-		return (header->last_used + 60 <= now);
+		return (header->last_used + DNS_RBTDB_LRUUPDATE_GLUE <= now);
 	}
 
 	/* Other records are updated if 5 minutes have passed. */
-	return (header->last_used + 300 <= now);
-#else  /* if DNS_RBTDB_LIMITLRUUPDATE */
+	return (header->last_used + DNS_RBTDB_LRUUPDATE_REGULAR <= now);
+#else
 	UNUSED(now);
 
 	return (true);

lib/isc/netmgr/netmgr-int.h

@@ -33,8 +33,8 @@
 
 #include "uv-compat.h"
 
-#define ISC_NETMGR_TID_UNKNOWN -1
-
+#define ISC_NETMGR_TID_UNKNOWN	-1
+#define ISC_NETMGR_RECVBUF_SIZE (20 * 65536)
 /*
  * Single network event loop worker.
  */
@@ -56,7 +56,7 @@ typedef struct isc__networker {
 				    * worker is paused */
 	isc_refcount_t references;
 	atomic_int_fast64_t pktcount;
-	char recvbuf[65536];
+	char *recvbuf;
 	bool recvbuf_inuse;
 } isc__networker_t;
 

lib/isc/netmgr/netmgr.c

@@ -196,6 +196,7 @@ isc_nm_start(isc_mem_t *mctx, uint32_t workers) {
 
 		worker->ievents = isc_queue_new(mgr->mctx, 128);
 		worker->ievents_prio = isc_queue_new(mgr->mctx, 128);
+		worker->recvbuf = isc_mem_get(mctx, ISC_NETMGR_RECVBUF_SIZE);
 
 		/*
 		 * We need to do this here and not in nm_thread to avoid a
@@ -268,6 +269,8 @@ nm_destroy(isc_nm_t **mgr0) {
 
 		isc_queue_destroy(worker->ievents);
 		isc_queue_destroy(worker->ievents_prio);
+		isc_mem_put(mgr->mctx, worker->recvbuf,
+			    ISC_NETMGR_RECVBUF_SIZE);
 		isc_thread_join(worker->thread, NULL);
 	}
 
@@ -900,8 +903,8 @@ isc__nmsocket_init(isc_nmsocket_t *sock, isc_nm_t *mgr, isc_nmsocket_type type,
 		.iface = iface,
 		.fd = -1,
 		.ah_size = 32,
-		.inactivehandles = isc_astack_new(mgr->mctx, 60),
-		.inactivereqs = isc_astack_new(mgr->mctx, 60)
+		.inactivehandles = isc_astack_new(mgr->mctx, 600),
+		.inactivereqs = isc_astack_new(mgr->mctx, 600)
 	};
 
 	isc_nm_attach(mgr, &sock->mgr);
@@ -960,14 +963,14 @@ isc__nm_alloc_cb(uv_handle_t *handle, size_t size, uv_buf_t *buf) {
 
 	REQUIRE(VALID_NMSOCK(sock));
 	REQUIRE(isc__nm_in_netthread());
-	REQUIRE(size <= 65536);
+	REQUIRE(size <= ISC_NETMGR_RECVBUF_SIZE);
 
 	worker = &sock->mgr->workers[sock->tid];
 	INSIST(!worker->recvbuf_inuse);
 
 	buf->base = worker->recvbuf;
 	worker->recvbuf_inuse = true;
-	buf->len = size;
+	buf->len = ISC_NETMGR_RECVBUF_SIZE;
 }
 
 void
@@ -982,8 +985,13 @@ isc__nm_free_uvbuf(isc_nmsocket_t *sock, const uv_buf_t *buf) {
 	worker = &sock->mgr->workers[sock->tid];
 
 	REQUIRE(worker->recvbuf_inuse);
+	if (buf->base > worker->recvbuf &&
+	    buf->base <= worker->recvbuf + ISC_NETMGR_RECVBUF_SIZE)
+	{
+		/* Can happen in case of recvmmsg */
+		return;
+	}
 	REQUIRE(buf->base == worker->recvbuf);
-
 	worker->recvbuf_inuse = false;
 }
 

lib/isc/netmgr/udp.c

@@ -97,6 +97,10 @@ isc_nm_listenudp(isc_nm_t *mgr, isc_nmiface_t *iface, isc_nm_recv_cb_t cb,
 #endif /* ifdef WIN32 */
 		RUNTIME_CHECK(res == 0);
 
+#ifdef SO_INCOMING_CPU
+		setsockopt(csock->fd, SOL_SOCKET, SO_INCOMING_CPU, &(int){ 1 },
+			   sizeof(int));
+#endif
 		ievent = isc__nm_get_ievent(mgr, netievent_udplisten);
 		ievent->sock = csock;
 		isc__nm_enqueue_ievent(&mgr->workers[i],
@@ -284,8 +288,6 @@ udp_recv_cb(uv_udp_t *handle, ssize_t nrecv, const uv_buf_t *buf,
 	isc_result_t result;
 	isc_nmhandle_t *nmhandle = NULL;
 	isc_sockaddr_t sockaddr;
-	isc_sockaddr_t localaddr;
-	struct sockaddr_storage laddr;
 	isc_nmsocket_t *sock = uv_handle_get_data((uv_handle_t *)handle);
 	isc_region_t region;
 	uint32_t maxudp;
@@ -319,13 +321,7 @@ udp_recv_cb(uv_udp_t *handle, ssize_t nrecv, const uv_buf_t *buf,
 
 	result = isc_sockaddr_fromsockaddr(&sockaddr, addr);
 	RUNTIME_CHECK(result == ISC_R_SUCCESS);
-	uv_udp_getsockname(handle, (struct sockaddr *)&laddr,
-			   &(int){ sizeof(struct sockaddr_storage) });
-	result = isc_sockaddr_fromsockaddr(&localaddr,
-					   (struct sockaddr *)&laddr);
-	RUNTIME_CHECK(result == ISC_R_SUCCESS);
-
-	nmhandle = isc__nmhandle_get(sock, &sockaddr, &localaddr);
+	nmhandle = isc__nmhandle_get(sock, &sockaddr, NULL);
 	region.base = (unsigned char *)buf->base;
 	region.length = nrecv;
 
@@ -383,8 +379,16 @@ isc__nm_udp_send(isc_nmhandle_t *handle, isc_region_t *region, isc_nm_cb_t cb,
 		return (ISC_R_CANCELED);
 	}
 
+	/*
+	 * If we're in netthread - send it directly
+	 * If the original packet was received over a regular socket
+	 * - send it over the same thread (assuming cpu affinity)
+	 * Otherwise - use a random thread.
+	 */
 	if (isc__nm_in_netthread()) {
 		ntid = isc_nm_tid();
+	} else if (sock->type == isc_nm_udpsocket) {
+		ntid = sock->tid;
 	} else {
 		ntid = (int)isc_random_uniform(sock->nchildren);
 	}

lib/isc/netmgr/uv-compat.c

@@ -15,6 +15,7 @@
 
 #include <isc/util.h>
 
+#ifndef HAVE_UV_IMPORT
 /*
  * XXXWPK: This code goes into libuv internals and it's platform dependent.
  * It's ugly, we shouldn't do it, but the alternative with passing sockets
@@ -185,3 +186,5 @@ isc_uv_import(uv_stream_t *stream, isc_uv_stream_info_t *info) {
 	return (uv_tcp_open(tcp, info->fd));
 }
 #endif /* ifdef WIN32 */
+
+#endif /* ifndef HAVE_UV_IMPORT */

lib/isc/netmgr/uv-compat.h

@@ -33,6 +33,14 @@ uv_handle_set_data(uv_handle_t *handle, void *data) {
 }
 #endif /* ifndef HAVE_UV_HANDLE_SET_DATA */
 
+#ifdef HAVE_UV_IMPORT
+
+#define isc_uv_stream_info_t uv_stream_info_t
+#define isc_uv_export	     uv_export
+#define isc_uv_import	     uv_import
+
+#else
+
 /*
  * These functions are not available in libuv, but they're very internal
  * to libuv. We should try to get them merged upstream.
@@ -69,3 +77,5 @@ isc_uv_import(uv_stream_t *stream, isc_uv_stream_info_t *info);
  * Imports uv_stream_info_t value into uv_stream_t to initialize a
  * shared stream.
  */
+
+#endif

lib/isc/result.c

@@ -15,9 +15,9 @@
 #include <stdlib.h>
 
 #include <isc/lib.h>
-#include <isc/mutex.h>
 #include <isc/once.h>
 #include <isc/resultclass.h>
+#include <isc/rwlock.h>
 #include <isc/util.h>
 
 typedef struct resulttable {
@@ -182,7 +182,7 @@ static const char *identifier[ISC_R_NRESULTS] = {
 static isc_once_t once = ISC_ONCE_INIT;
 static resulttable_list_t description_tables;
 static resulttable_list_t identifier_tables;
-static isc_mutex_t lock;
+static isc_rwlock_t lock;
 
 static isc_result_t
 register_table(resulttable_list_t *tables, unsigned int base,
@@ -207,11 +207,11 @@ register_table(resulttable_list_t *tables, unsigned int base,
 	table->set = set;
 	ISC_LINK_INIT(table, link);
 
-	LOCK(&lock);
+	RWLOCK(&lock, isc_rwlocktype_write);
 
 	ISC_LIST_APPEND(*tables, table, link);
 
-	UNLOCK(&lock);
+	RWUNLOCK(&lock, isc_rwlocktype_write);
 
 	return (ISC_R_SUCCESS);
 }
@@ -220,7 +220,7 @@ static void
 initialize_action(void) {
 	isc_result_t result;
 
-	isc_mutex_init(&lock);
+	isc_rwlock_init(&lock, 0, 0);
 	ISC_LIST_INIT(description_tables);
 	ISC_LIST_INIT(identifier_tables);
 
@@ -254,7 +254,7 @@ isc_result_tomany_helper(resulttable_list_t *tables, isc_result_t result) {
 
 	initialize();
 
-	LOCK(&lock);
+	RWLOCK(&lock, isc_rwlocktype_read);
 
 	text = NULL;
 	for (table = ISC_LIST_HEAD(*tables); table != NULL;
@@ -270,7 +270,7 @@ isc_result_tomany_helper(resulttable_list_t *tables, isc_result_t result) {
 		text = "(result code text not available)";
 	}
 
-	UNLOCK(&lock);
+	RWUNLOCK(&lock, isc_rwlocktype_read);
 
 	return (text);
 }

lib/isc/stats.c

@@ -22,11 +22,14 @@
 #include <isc/print.h>
 #include <isc/refcount.h>
 #include <isc/stats.h>
+#include <isc/thread.h>
 #include <isc/util.h>
 
 #define ISC_STATS_MAGIC	   ISC_MAGIC('S', 't', 'a', 't')
 #define ISC_STATS_VALID(x) ISC_MAGIC_VALID(x, ISC_STATS_MAGIC)
 
+#define STATS_BUCKETS 64
+
 #if defined(_WIN32) && !defined(_WIN64)
 typedef atomic_int_fast32_t isc__atomic_statcounter_t;
 #else  /* if defined(_WIN32) && !defined(_WIN64) */
@@ -41,6 +44,19 @@ struct isc_stats {
 	isc__atomic_statcounter_t *counters;
 };
 
+ISC_THREAD_LOCAL int isc__stats_thread_v = -1;
+static atomic_uint_fast32_t isc__stats_thread_n = 0;
+
+static int
+threadhash() {
+	if (isc__stats_thread_v < 0) {
+		isc__stats_thread_v =
+			atomic_fetch_add_relaxed(&isc__stats_thread_n, 1) %
+			STATS_BUCKETS;
+	}
+	return (isc__stats_thread_v);
+}
+
 static isc_result_t
 create_stats(isc_mem_t *mctx, int ncounters, isc_stats_t **statsp) {
 	isc_stats_t *stats;
@@ -49,7 +65,8 @@ create_stats(isc_mem_t *mctx, int ncounters, isc_stats_t **statsp) {
 	REQUIRE(statsp != NULL && *statsp == NULL);
 
 	stats = isc_mem_get(mctx, sizeof(*stats));
-	counters_alloc_size = sizeof(isc__atomic_statcounter_t) * ncounters;
+	counters_alloc_size = sizeof(isc__atomic_statcounter_t) * ncounters *
+			      STATS_BUCKETS;
 	stats->counters = isc_mem_get(mctx, counters_alloc_size);
 	isc_refcount_init(&stats->references, 1);
 	memset(stats->counters, 0, counters_alloc_size);
@@ -84,7 +101,7 @@ isc_stats_detach(isc_stats_t **statsp) {
 		isc_refcount_destroy(&stats->references);
 		isc_mem_put(stats->mctx, stats->counters,
 			    sizeof(isc__atomic_statcounter_t) *
-				    stats->ncounters);
+				    stats->ncounters * STATS_BUCKETS);
 		isc_mem_putanddetach(&stats->mctx, stats, sizeof(*stats));
 	}
 }
@@ -107,8 +124,8 @@ void
 isc_stats_increment(isc_stats_t *stats, isc_statscounter_t counter) {
 	REQUIRE(ISC_STATS_VALID(stats));
 	REQUIRE(counter < stats->ncounters);
-
-	atomic_fetch_add_explicit(&stats->counters[counter], 1,
+	int idx = threadhash() * stats->ncounters + counter;
+	atomic_fetch_add_explicit(&stats->counters[idx], 1,
 				  memory_order_relaxed);
 }
 
@@ -117,7 +134,8 @@ isc_stats_decrement(isc_stats_t *stats, isc_statscounter_t counter) {
 	REQUIRE(ISC_STATS_VALID(stats));
 	REQUIRE(counter < stats->ncounters);
 
-	atomic_fetch_sub_explicit(&stats->counters[counter], 1,
+	int idx = threadhash() * stats->ncounters + counter;
+	atomic_fetch_sub_explicit(&stats->counters[idx], 1,
 				  memory_order_relaxed);
 }
 
@@ -129,8 +147,13 @@ isc_stats_dump(isc_stats_t *stats, isc_stats_dumper_t dump_fn, void *arg,
 	REQUIRE(ISC_STATS_VALID(stats));
 
 	for (i = 0; i < stats->ncounters; i++) {
-		uint32_t counter = atomic_load_explicit(&stats->counters[i],
+		uint32_t counter = 0;
+		int b;
+		for (b = 0; b < STATS_BUCKETS; b++) {
+			int idx = stats->ncounters * b + i;
+			counter += atomic_load_explicit(&stats->counters[idx],
 							memory_order_relaxed);
+		}
 		if ((options & ISC_STATSDUMP_VERBOSE) == 0 && counter == 0) {
 			continue;
 		}
@@ -140,11 +163,17 @@ isc_stats_dump(isc_stats_t *stats, isc_stats_dumper_t dump_fn, void *arg,
 
 void
 isc_stats_set(isc_stats_t *stats, uint64_t val, isc_statscounter_t counter) {
+	int i;
+
 	REQUIRE(ISC_STATS_VALID(stats));
 	REQUIRE(counter < stats->ncounters);
-
 	atomic_store_explicit(&stats->counters[counter], val,
 			      memory_order_relaxed);
+	for (i = 1; i < STATS_BUCKETS; i++) {
+		int idx = stats->ncounters * i + counter;
+		atomic_store_explicit(&stats->counters[idx], val,
+				      memory_order_relaxed);
+	}
 }
 
 void
@@ -153,9 +182,9 @@ isc_stats_update_if_greater(isc_stats_t *stats, isc_statscounter_t counter,
 	REQUIRE(ISC_STATS_VALID(stats));
 	REQUIRE(counter < stats->ncounters);
 
-	isc_statscounter_t curr_value =
-		atomic_load_relaxed(&stats->counters[counter]);
+	isc_statscounter_t curr_value;
 	do {
+		curr_value = atomic_load_relaxed(&stats->counters[counter]);
 		if (curr_value >= value) {
 			break;
 		}
@@ -165,9 +194,15 @@ isc_stats_update_if_greater(isc_stats_t *stats, isc_statscounter_t counter,
 
 isc_statscounter_t
 isc_stats_get_counter(isc_stats_t *stats, isc_statscounter_t counter) {
+	uint32_t value = 0;
+	int i;
 	REQUIRE(ISC_STATS_VALID(stats));
 	REQUIRE(counter < stats->ncounters);
 
-	return (atomic_load_explicit(&stats->counters[counter],
-				     memory_order_relaxed));
+	for (i = 0; i < STATS_BUCKETS; i++) {
+		int idx = i * stats->ncounters + counter;
+		value += atomic_load_explicit(&stats->counters[idx],
+					      memory_order_relaxed);
+	}
+	return (value);
 }

lib/isc/task.c

@@ -107,6 +107,7 @@ struct isc__task {
 	isc_mutex_t lock;
 	/* Locked by task lock. */
 	task_state_t state;
+	int pause_cnt;
 	isc_refcount_t references;
 	isc_eventlist_t events;
 	isc_eventlist_t on_shutdown;
@@ -305,6 +306,7 @@ isc_task_create_bound(isc_taskmgr_t *manager0, unsigned int quantum,
 
 	isc_mutex_init(&task->lock);
 	task->state = task_state_idle;
+	task->pause_cnt = 0;
 
 	isc_refcount_init(&task->references, 1);
 	INIT_LIST(task->events);
@@ -403,7 +405,7 @@ task_shutdown(isc__task_t *task) {
 /*
  * Moves a task onto the appropriate run queue.
  *
- * Caller must NOT hold manager lock.
+ * Caller must NOT hold queue lock.
  */
 static inline void
 task_ready(isc__task_t *task) {
@@ -411,7 +413,6 @@ task_ready(isc__task_t *task) {
 	bool has_privilege = isc_task_privilege((isc_task_t *)task);
 
 	REQUIRE(VALID_MANAGER(manager));
-	REQUIRE(task->state == task_state_ready);
 
 	XTRACE("task_ready");
 	LOCK(&manager->queues[task->threadid].lock);
@@ -952,10 +953,13 @@ pop_readyq(isc__taskmgr_t *manager, int c) {
  * Push 'task' onto the ready_tasks queue.  If 'task' has the privilege
  * flag set, then also push it onto the ready_priority_tasks queue.
  *
- * Caller must hold the task manager lock.
+ * Caller must hold the task queue lock.
  */
 static inline void
 push_readyq(isc__taskmgr_t *manager, isc__task_t *task, int c) {
+	if (ISC_LINK_LINKED(task, ready_link)) {
+		return;
+	}
 	ENQUEUE(manager->queues[c].ready_tasks, task, ready_link);
 	if (TASK_PRIVILEGED(task)) {
 		ENQUEUE(manager->queues[c].ready_priority_tasks, task,
@@ -1115,6 +1119,17 @@ dispatch(isc__taskmgr_t *manager, unsigned int threadid) {
 						  memory_order_acquire);
 
 			LOCK(&task->lock);
+			/*
+			 * It is possible because that we have a paused task
+			 * in the queue - it might have been paused in the
+			 * meantime and we never hold both queue and task lock
+			 * to avoid deadlocks, just bail then.
+			 */
+			if (task->state != task_state_ready) {
+				UNLOCK(&task->lock);
+				LOCK(&manager->queues[threadid].lock);
+				continue;
+			}
 			INSIST(task->state == task_state_ready);
 			task->state = task_state_running;
 			XTRACE("running");
@@ -1203,6 +1218,15 @@ dispatch(isc__taskmgr_t *manager, unsigned int threadid) {
 						}
 					}
 					done = true;
+				} else if (task->state == task_state_pausing) {
+					/*
+					 * We got a pause request on this task,
+					 * stop working on it and switch the
+					 * state to paused.
+					 */
+					XTRACE("pausing");
+					task->state = task_state_paused;
+					done = true;
 				} else if (dispatch_count >= task->quantum) {
 					/*
 					 * Our quantum has expired, but
@@ -1215,17 +1239,8 @@ dispatch(isc__taskmgr_t *manager, unsigned int threadid) {
 					 * so the minimum quantum is one.
 					 */
 					XTRACE("quantum");
-					if (task->state == task_state_running) {
-						/*
-						 * We requeue only if it's
-						 * not paused.
-						 */
-						task->state = task_state_ready;
-						requeue = true;
-					} else if (task->state ==
-						   task_state_pausing) {
-						task->state = task_state_paused;
-					}
+					task->state = task_state_ready;
+					requeue = true;
 					done = true;
 				}
 			} while (!done);
@@ -1675,31 +1690,27 @@ void
 isc_task_pause(isc_task_t *task0) {
 	REQUIRE(ISCAPI_TASK_VALID(task0));
 	isc__task_t *task = (isc__task_t *)task0;
-	isc__taskmgr_t *manager = task->manager;
-	bool running = false;
 
 	LOCK(&task->lock);
+	task->pause_cnt++;
+	if (task->pause_cnt > 1) {
+		/*
+		 * Someone already paused this thread, just increase
+		 * the number of pausing clients.
+		 */
+		UNLOCK(&task->lock);
+		return;
+	}
+
 	INSIST(task->state == task_state_idle ||
 	       task->state == task_state_ready ||
 	       task->state == task_state_running);
 	if (task->state == task_state_running) {
-		running = true;
 		task->state = task_state_pausing;
 	} else {
 		task->state = task_state_paused;
 	}
 	UNLOCK(&task->lock);
-
-	if (running) {
-		return;
-	}
-
-	LOCK(&manager->queues[task->threadid].lock);
-	if (ISC_LINK_LINKED(task, ready_link)) {
-		DEQUEUE(manager->queues[task->threadid].ready_tasks, task,
-			ready_link);
-	}
-	UNLOCK(&manager->queues[task->threadid].lock);
 }
 
 void
@@ -1710,6 +1721,13 @@ isc_task_unpause(isc_task_t *task0) {
 	REQUIRE(ISCAPI_TASK_VALID(task0));
 
 	LOCK(&task->lock);
+	task->pause_cnt--;
+	INSIST(task->pause_cnt >= 0);
+	if (task->pause_cnt > 0) {
+		UNLOCK(&task->lock);
+		return;
+	}
+
 	INSIST(task->state == task_state_paused ||
 	       task->state == task_state_pausing);
 	/* If the task was pausing we can't reschedule it */

lib/isc/unix/socket.c

@@ -1828,8 +1828,6 @@ destroy(isc__socket_t **sockp) {
 
 	socket_log(sock, NULL, CREATION, "destroying");
 
-	isc_refcount_destroy(&sock->references);
-
 	LOCK(&sock->lock);
 	INSIST(ISC_LIST_EMPTY(sock->connect_list));
 	INSIST(ISC_LIST_EMPTY(sock->accept_list));
@@ -1856,6 +1854,7 @@ destroy(isc__socket_t **sockp) {
 	if (ISC_LIST_EMPTY(manager->socklist)) {
 		SIGNAL(&manager->shutdown_ok);
 	}
+	isc_refcount_destroy(&sock->references);
 
 	/* can't unlock manager as its memory context is still used */
 	free_socket(sockp);
@@ -2636,7 +2635,6 @@ isc_socket_detach(isc_socket_t **socketp) {
 	REQUIRE(socketp != NULL);
 	sock = (isc__socket_t *)*socketp;
 	REQUIRE(VALID_SOCKET(sock));
-
 	if (isc_refcount_decrement(&sock->references) == 1) {
 		destroy(&sock);
 	}
@@ -3153,16 +3151,20 @@ process_fd(isc__socketthread_t *thread, int fd, bool readable, bool writeable) {
 		return;
 	}
 
-	if (isc_refcount_increment0(&sock->references) == 0) {
+	LOCK(&sock->lock);
+	if (isc_refcount_current(&sock->references) == 0) {
 		/*
-		 * Sock is being closed, it will be destroyed, bail.
+		 * Sock is being closed - the final external reference
+		 * is gone but it was not yet removed from event loop
+		 * and fdstate[]/fds[] as destroy() is waiting on
+		 * thread->fdlock[lockid] or sock->lock that we're holding.
+		 * Just release the locks and bail.
 		 */
-		(void)isc_refcount_decrement(&sock->references);
+		UNLOCK(&sock->lock);
 		UNLOCK(&thread->fdlock[lockid]);
 		return;
 	}
 
-	LOCK(&sock->lock);
 	if (readable) {
 		if (sock->listener) {
 			internal_accept(sock);
@@ -3181,9 +3183,10 @@ process_fd(isc__socketthread_t *thread, int fd, bool readable, bool writeable) {
 	UNLOCK(&sock->lock);
 
 	UNLOCK(&thread->fdlock[lockid]);
-	if (isc_refcount_decrement(&sock->references) == 1) {
-		destroy(&sock);
-	}
+	/*
+	 * Socket destruction might be pending, it will resume
+	 * after releasing fdlock and sock->lock.
+	 */
 }
 
 /*

lib/isc/win32/libisc.def.in

@@ -687,8 +687,6 @@ isc_timermgr_destroy
 isc_timermgr_poke
 isc_tm_timegm
 isc_tm_strptime
-isc_uv_export
-isc_uv_import
 isc_win32os_versioncheck
 openlog
 @IF PKCS11

lib/ns/client.c

@@ -79,7 +79,6 @@
  * task to change the client, then the client will have to be locked.
  */
 
-#define NS_CLIENT_TRACE
 #ifdef NS_CLIENT_TRACE
 #define CTRACE(m)                                                         \
 	ns_client_log(client, NS_LOGCATEGORY_CLIENT, NS_LOGMODULE_CLIENT, \
@@ -133,6 +132,8 @@ compute_cookie(ns_client_t *client, uint32_t when, uint32_t nonce,
 	       const unsigned char *secret, isc_buffer_t *buf);
 static void
 get_clientmctx(ns_clientmgr_t *manager, isc_mem_t **mctxp);
+static void
+get_clienttask(ns_clientmgr_t *manager, isc_task_t **taskp);
 
 void
 ns_client_recursing(ns_client_t *client) {
@@ -172,21 +173,20 @@ ns_client_settimeout(ns_client_t *client, unsigned int seconds) {
 
 static void
 ns_client_endrequest(ns_client_t *client) {
-	INSIST(client->naccepts == 0);
-	INSIST(client->nreads == 0);
-	INSIST(client->nsends == 0);
-	INSIST(client->nrecvs == 0);
 	INSIST(client->nupdates == 0);
 	INSIST(client->state == NS_CLIENTSTATE_WORKING ||
 	       client->state == NS_CLIENTSTATE_RECURSING);
 
 	CTRACE("endrequest");
 
-	LOCK(&client->manager->reclock);
-	if (ISC_LINK_LINKED(client, rlink)) {
-		ISC_LIST_UNLINK(client->manager->recursing, client, rlink);
+	if (client->state == NS_CLIENTSTATE_RECURSING) {
+		LOCK(&client->manager->reclock);
+		if (ISC_LINK_LINKED(client, rlink)) {
+			ISC_LIST_UNLINK(client->manager->recursing, client,
+					rlink);
+		}
+		UNLOCK(&client->manager->reclock);
 	}
-	UNLOCK(&client->manager->reclock);
 
 	if (client->cleanup != NULL) {
 		(client->cleanup)(client);
@@ -1594,7 +1594,7 @@ ns__client_put_cb(void *client0) {
 		clientmgr_detach(&client->manager);
 	}
 
-	isc_mem_put(client->mctx, client->recvbuf, NS_CLIENT_RECV_BUFFER_SIZE);
+	isc_mem_put(client->mctx, client->sendbuf, NS_CLIENT_SEND_BUFFER_SIZE);
 	if (client->opt != NULL) {
 		INSIST(dns_rdataset_isassociated(client->opt));
 		dns_rdataset_disassociate(client->opt);
@@ -1684,7 +1684,6 @@ ns__client_request(isc_nmhandle_t *handle, isc_region_t *region, void *arg) {
 	}
 
 	client->state = NS_CLIENTSTATE_READY;
-	client->dscp = ifp->dscp;
 
 	isc_task_pause(client->task);
 	if (client->handle == NULL) {
@@ -2229,41 +2228,37 @@ ns__client_tcpconn(isc_nmhandle_t *handle, isc_result_t result, void *arg) {
 }
 
 static void
-get_clientmctx(ns_clientmgr_t *manager, isc_mem_t **mctxp) {
+get_clientmctx(ns_clientmgr_t *manager, isc_mem_t **mctxp)
+{
 	isc_mem_t *clientmctx;
-#if CLIENT_NMCTXS > 0
-	unsigned int nextmctx;
-#endif /* if CLIENT_NMCTXS > 0 */
-
 	MTRACE("clientmctx");
 
-#if CLIENT_NMCTXS > 0
-	LOCK(&manager->lock);
-	if (isc_nm_tid() >= 0) {
-		nextmctx = isc_nm_tid();
-	} else {
-		nextmctx = manager->nextmctx++;
-		if (manager->nextmctx == CLIENT_NMCTXS) {
-			manager->nextmctx = 0;
-		}
-
-		INSIST(nextmctx < CLIENT_NMCTXS);
+	int tid = isc_nm_tid();
+	if (tid < 0) {
+		tid = isc_random_uniform(manager->ncpus);
 	}
-
+	int rand = isc_random_uniform(CLIENT_NMCTXS_PERCPU);
+	int nextmctx = (rand * manager->ncpus) + tid;
 	clientmctx = manager->mctxpool[nextmctx];
-	if (clientmctx == NULL) {
-		isc_mem_create(&clientmctx);
-		isc_mem_setname(clientmctx, "client", NULL);
-		manager->mctxpool[nextmctx] = clientmctx;
-	}
-	UNLOCK(&manager->lock);
-#else  /* if CLIENT_NMCTXS > 0 */
-	clientmctx = manager->mctx;
-#endif /* if CLIENT_NMCTXS > 0 */
 
 	isc_mem_attach(clientmctx, mctxp);
 }
 
+static void
+get_clienttask(ns_clientmgr_t *manager, isc_task_t **taskp)
+{
+	MTRACE("clienttask");
+
+	int tid = isc_nm_tid();
+	if (tid < 0) {
+		tid = isc_random_uniform(manager->ncpus);
+	}
+
+	int rand = isc_random_uniform(CLIENT_NTASKS_PERCPU);
+	int nexttask = (rand * manager->ncpus) + tid;
+	isc_task_attach(manager->taskpool[nexttask], taskp);
+}
+
 isc_result_t
 ns__client_setup(ns_client_t *client, ns_clientmgr_t *mgr, bool new) {
 	isc_result_t result;
@@ -2285,10 +2280,8 @@ ns__client_setup(ns_client_t *client, ns_clientmgr_t *mgr, bool new) {
 		get_clientmctx(mgr, &client->mctx);
 		clientmgr_attach(mgr, &client->manager);
 		ns_server_attach(mgr->sctx, &client->sctx);
-		result = isc_task_create(mgr->taskmgr, 20, &client->task);
-		if (result != ISC_R_SUCCESS) {
-			goto cleanup;
-		}
+		get_clienttask(mgr, &client->task);
+
 		result = dns_message_create(client->mctx,
 					    DNS_MESSAGE_INTENTPARSE,
 					    &client->message);
@@ -2296,8 +2289,8 @@ ns__client_setup(ns_client_t *client, ns_clientmgr_t *mgr, bool new) {
 			goto cleanup;
 		}
 
-		client->recvbuf = isc_mem_get(client->mctx,
-					      NS_CLIENT_RECV_BUFFER_SIZE);
+		client->sendbuf = isc_mem_get(client->mctx,
+					      NS_CLIENT_SEND_BUFFER_SIZE);
 		/*
 		 * Set magic earlier than usual because ns_query_init()
 		 * and the functions it calls will require it.
@@ -2311,7 +2304,7 @@ ns__client_setup(ns_client_t *client, ns_clientmgr_t *mgr, bool new) {
 		ns_clientmgr_t *oldmgr = client->manager;
 		ns_server_t *sctx = client->sctx;
 		isc_task_t *task = client->task;
-		unsigned char *recvbuf = client->recvbuf;
+		unsigned char *sendbuf = client->sendbuf;
 		dns_message_t *message = client->message;
 		isc_mem_t *oldmctx = client->mctx;
 		ns_query_t query = client->query;
@@ -2321,14 +2314,13 @@ ns__client_setup(ns_client_t *client, ns_clientmgr_t *mgr, bool new) {
 					 .manager = oldmgr,
 					 .sctx = sctx,
 					 .task = task,
-					 .recvbuf = recvbuf,
+					 .sendbuf = sendbuf,
 					 .message = message,
 					 .query = query };
 	}
 
 	client->state = NS_CLIENTSTATE_INACTIVE;
 	client->udpsize = 512;
-	client->dscp = -1;
 	client->ednsversion = -1;
 	dns_name_init(&client->signername, NULL);
 	dns_ecs_init(&client->ecs);
@@ -2345,9 +2337,9 @@ ns__client_setup(ns_client_t *client, ns_clientmgr_t *mgr, bool new) {
 	return (ISC_R_SUCCESS);
 
 cleanup:
-	if (client->recvbuf != NULL) {
-		isc_mem_put(client->mctx, client->recvbuf,
-			    NS_CLIENT_RECV_BUFFER_SIZE);
+	if (client->sendbuf != NULL) {
+		isc_mem_put(client->mctx, client->sendbuf,
+			    NS_CLIENT_SEND_BUFFER_SIZE);
 	}
 
 	if (client->message != NULL) {
@@ -2412,22 +2404,19 @@ clientmgr_detach(ns_clientmgr_t **mp) {
 
 static void
 clientmgr_destroy(ns_clientmgr_t *manager) {
-#if CLIENT_NMCTXS > 0
 	int i;
-#endif /* if CLIENT_NMCTXS > 0 */
 
 	MTRACE("clientmgr_destroy");
 
 	isc_refcount_destroy(&manager->references);
 	manager->magic = 0;
 
-#if CLIENT_NMCTXS > 0
-	for (i = 0; i < CLIENT_NMCTXS; i++) {
-		if (manager->mctxpool[i] != NULL) {
-			isc_mem_detach(&manager->mctxpool[i]);
-		}
+	for (i = 0; i < manager->ncpus * CLIENT_NMCTXS_PERCPU; i++) {
+		isc_mem_detach(&manager->mctxpool[i]);
 	}
-#endif /* if CLIENT_NMCTXS > 0 */
+	isc_mem_put(manager->mctx, manager->mctxpool,
+		    manager->ncpus * CLIENT_NMCTXS_PERCPU *
+			    sizeof(isc_mem_t *));
 
 	if (manager->interface != NULL) {
 		ns_interface_detach(&manager->interface);
@@ -2440,13 +2429,14 @@ clientmgr_destroy(ns_clientmgr_t *manager) {
 		isc_task_detach(&manager->excl);
 	}
 
-	for (i = 0; i < CLIENT_NTASKS; i++) {
+	for (i = 0; i < manager->ncpus * CLIENT_NTASKS_PERCPU; i++) {
 		if (manager->taskpool[i] != NULL) {
 			isc_task_detach(&manager->taskpool[i]);
 		}
 	}
 	isc_mem_put(manager->mctx, manager->taskpool,
-		    CLIENT_NTASKS * sizeof(isc_task_t *));
+		    manager->ncpus * CLIENT_NTASKS_PERCPU *
+			    sizeof(isc_task_t *));
 	ns_server_detach(&manager->sctx);
 
 	isc_mem_put(manager->mctx, manager, sizeof(*manager));
@@ -2455,12 +2445,11 @@ clientmgr_destroy(ns_clientmgr_t *manager) {
 isc_result_t
 ns_clientmgr_create(isc_mem_t *mctx, ns_server_t *sctx, isc_taskmgr_t *taskmgr,
 		    isc_timermgr_t *timermgr, ns_interface_t *interface,
-		    ns_clientmgr_t **managerp) {
+		    int ncpus, ns_clientmgr_t **managerp) {
 	ns_clientmgr_t *manager;
 	isc_result_t result;
-#if CLIENT_NMCTXS > 0
 	int i;
-#endif /* if CLIENT_NMCTXS > 0 */
+	int npools;
 
 	manager = isc_mem_get(mctx, sizeof(*manager));
 	*manager = (ns_clientmgr_t){ .magic = 0 };
@@ -2477,27 +2466,35 @@ ns_clientmgr_create(isc_mem_t *mctx, ns_server_t *sctx, isc_taskmgr_t *taskmgr,
 	manager->mctx = mctx;
 	manager->taskmgr = taskmgr;
 	manager->timermgr = timermgr;
+	manager->ncpus = ncpus;
 
 	ns_interface_attach(interface, &manager->interface);
 
 	manager->exiting = false;
-	manager->taskpool = isc_mem_get(mctx,
-					CLIENT_NTASKS * sizeof(isc_task_t *));
-	for (i = 0; i < CLIENT_NTASKS; i++) {
+	int ntasks = CLIENT_NTASKS_PERCPU * manager->ncpus;
+	manager->taskpool = isc_mem_get(mctx, ntasks * sizeof(isc_task_t *));
+	for (i = 0; i < ntasks; i++) {
 		manager->taskpool[i] = NULL;
-		isc_task_create(manager->taskmgr, 20, &manager->taskpool[i]);
+		result = isc_task_create_bound(manager->taskmgr, 20,
+					       &manager->taskpool[i],
+					       i % CLIENT_NTASKS_PERCPU);
+		RUNTIME_CHECK(result == ISC_R_SUCCESS);
 	}
 	isc_refcount_init(&manager->references, 1);
 	manager->sctx = NULL;
 	ns_server_attach(sctx, &manager->sctx);
 
 	ISC_LIST_INIT(manager->recursing);
-#if CLIENT_NMCTXS > 0
-	manager->nextmctx = 0;
-	for (i = 0; i < CLIENT_NMCTXS; i++) {
-		manager->mctxpool[i] = NULL; /* will be created on-demand */
+
+	npools = CLIENT_NMCTXS_PERCPU * manager->ncpus;
+	manager->mctxpool = isc_mem_get(manager->mctx,
+					npools * sizeof(isc_mem_t *));
+	for (i = 0; i < npools; i++) {
+		manager->mctxpool[i] = NULL;
+		isc_mem_create(&manager->mctxpool[i]);
+		isc_mem_setname(manager->mctxpool[i], "client", NULL);
 	}
-#endif /* if CLIENT_NMCTXS > 0 */
+
 	manager->magic = MANAGER_MAGIC;
 
 	MTRACE("create");

lib/ns/include/ns/client.h

@@ -85,7 +85,7 @@
 #define NS_CLIENT_SEND_BUFFER_SIZE 4096
 #define NS_CLIENT_RECV_BUFFER_SIZE 4096
 
-#define CLIENT_NMCTXS 100
+#define CLIENT_NMCTXS_PERCPU 8
 /*%<
  * Number of 'mctx pools' for clients. (Should this be configurable?)
  * When enabling threads, we use a pool of memory contexts shared by
@@ -95,7 +95,7 @@
  * server.
  */
 
-#define CLIENT_NTASKS 100
+#define CLIENT_NTASKS_PERCPU 32
 /*%<
  * Number of tasks to be used by clients - those are used only when recursing
  */
@@ -166,6 +166,7 @@ struct ns_clientmgr {
 	isc_timermgr_t *timermgr;
 	isc_task_t *	excl;
 	isc_refcount_t	references;
+	int		ncpus;
 
 	/* Attached by clients, needed for e.g. recursion */
 	isc_task_t **taskpool;
@@ -180,11 +181,8 @@ struct ns_clientmgr {
 	isc_mutex_t   reclock;
 	client_list_t recursing; /*%< Recursing clients */
 
-#if CLIENT_NMCTXS > 0
 	/*%< mctx pool for clients. */
-	unsigned int nextmctx;
-	isc_mem_t *  mctxpool[CLIENT_NMCTXS];
-#endif /* if CLIENT_NMCTXS > 0 */
+	isc_mem_t **mctxpool;
 };
 
 /*% nameserver client structure */
@@ -195,12 +193,7 @@ struct ns_client {
 	ns_server_t *	 sctx;
 	ns_clientmgr_t * manager;
 	ns_clientstate_t state;
-	int		 naccepts;
-	int		 nreads;
-	int		 nsends;
-	int		 nrecvs;
 	int		 nupdates;
-	int		 nctls;
 	bool		 shuttingdown;
 	unsigned int	 attributes;
 	isc_task_t *	 task;
@@ -209,8 +202,7 @@ struct ns_client {
 	isc_nmhandle_t * handle;
 	unsigned char *	 tcpbuf;
 	dns_message_t *	 message;
-	unsigned char *	 recvbuf;
-	unsigned char	 sendbuf[NS_CLIENT_SEND_BUFFER_SIZE];
+	unsigned char *	 sendbuf;
 	dns_rdataset_t * opt;
 	uint16_t	 udpsize;
 	uint16_t	 extflags;
@@ -364,7 +356,7 @@ ns_client_settimeout(ns_client_t *client, unsigned int seconds);
 
 isc_result_t
 ns_clientmgr_create(isc_mem_t *mctx, ns_server_t *sctx, isc_taskmgr_t *taskmgr,
-		    isc_timermgr_t *timermgr, ns_interface_t *ifp,
+		    isc_timermgr_t *timermgr, ns_interface_t *ifp, int ncpus,
 		    ns_clientmgr_t **managerp);
 /*%<
  * Create a client manager.

lib/ns/include/ns/interfacemgr.h

@@ -104,7 +104,7 @@ ns_interfacemgr_create(isc_mem_t *mctx, ns_server_t *sctx,
 		       isc_socketmgr_t *socketmgr, isc_nm_t *nm,
 		       dns_dispatchmgr_t *dispatchmgr, isc_task_t *task,
 		       unsigned int udpdisp, dns_geoip_databases_t *geoip,
-		       ns_interfacemgr_t **mgrp);
+		       int ncpus, ns_interfacemgr_t **mgrp);
 /*%<
  * Create a new interface manager.
  *

lib/ns/interfacemgr.c

@@ -77,6 +77,7 @@ struct ns_interfacemgr {
 	isc_timermgr_t *timermgr;   /*%< Timer manager. */
 	isc_socketmgr_t *socketmgr; /*%< Socket manager. */
 	isc_nm_t *nm;		    /*%< Net manager. */
+	int ncpus;		    /*%< Number of workers . */
 	dns_dispatchmgr_t *dispatchmgr;
 	unsigned int generation; /*%< Current generation no. */
 	ns_listenlist_t *listenon4;
@@ -183,7 +184,7 @@ ns_interfacemgr_create(isc_mem_t *mctx, ns_server_t *sctx,
 		       isc_socketmgr_t *socketmgr, isc_nm_t *nm,
 		       dns_dispatchmgr_t *dispatchmgr, isc_task_t *task,
 		       unsigned int udpdisp, dns_geoip_databases_t *geoip,
-		       ns_interfacemgr_t **mgrp) {
+		       int ncpus, ns_interfacemgr_t **mgrp) {
 	isc_result_t result;
 	ns_interfacemgr_t *mgr;
 
@@ -220,6 +221,7 @@ ns_interfacemgr_create(isc_mem_t *mctx, ns_server_t *sctx,
 	mgr->listenon4 = NULL;
 	mgr->listenon6 = NULL;
 	mgr->udpdisp = udpdisp;
+	mgr->ncpus = ncpus;
 	atomic_init(&mgr->shuttingdown, false);
 
 	ISC_LIST_INIT(mgr->interfaces);
@@ -426,7 +428,8 @@ ns_interface_create(ns_interfacemgr_t *mgr, isc_sockaddr_t *addr,
 	ifp->magic = IFACE_MAGIC;
 
 	result = ns_clientmgr_create(mgr->mctx, mgr->sctx, mgr->taskmgr,
-				     mgr->timermgr, ifp, &ifp->clientmgr);
+				     mgr->timermgr, ifp, mgr->ncpus,
+				     &ifp->clientmgr);
 	if (result != ISC_R_SUCCESS) {
 		isc_log_write(IFMGR_COMMON_LOGARGS, ISC_LOG_ERROR,
 			      "ns_clientmgr_create() failed: %s",

lib/ns/tests/nstest.c

@@ -233,7 +233,7 @@ create_managers(void) {
 	CHECK(dns_dispatchmgr_create(mctx, &dispatchmgr));
 
 	CHECK(ns_interfacemgr_create(mctx, sctx, taskmgr, timermgr, socketmgr,
-				     nm, dispatchmgr, maintask, ncpus, NULL,
+				     nm, dispatchmgr, maintask, ncpus, NULL, 1,
 				     &interfacemgr));
 
 	CHECK(ns_listenlist_default(mctx, port, -1, true, &listenon));

win32utils/Configure

@@ -223,6 +223,9 @@ my @substdefh = ("CONFIGARGS",
                  "HAVE_HMAC_CTX_GET_MD",
                  "HAVE_HMAC_CTX_NEW",
                  "HAVE_HMAC_CTX_RESET",
+                 "HAVE_UV_HANDLE_GET_DATA",
+                 "HAVE_UV_HANDLE_SET_DATA",
+                 "HAVE_UV_IMPORT",
     );
 
 # for platform.h
@@ -1321,6 +1324,10 @@ if ($use_libuv eq "auto") {
             last;
         }
     }
+    $configdefh{"HAVE_UV_HANDLE_SET_DATA"} = 1;
+    $configdefh{"HAVE_UV_HANDLE_GET_DATA"} = 1;
+    $configdefh{"HAVE_UV_IMPORT"} = 1;
+
 
     # If we have one use it otherwise report the error
     if ($use_libuv eq "auto") {