- Use TIME_NOW to verify the result

- Verify the result of isc_task_create

.gitignore

@@ -10,6 +10,7 @@
 *.rej
 *.so
 *_test
+*.ipch # vscode/intellisense precompiled header
 *~
 .ccache/
 .cproject

.gitlab-ci.yml

@@ -203,7 +203,6 @@ stages:
 .system_test: &system_test_job
   <<: *default_triggering_rules
   stage: system
-  retry: 2
   before_script:
     - *setup_interfaces
     - *setup_softhsm
@@ -675,6 +674,7 @@ unit:gcc:bionic:amd64:
 asan:sid:amd64:
   variables:
     CC: gcc
+    ASAN_OPTIONS: "detect_leaks=0"
     CFLAGS: "-Wall -Wextra -O2 -g -fsanitize=address,undefined -DISC_MEM_USE_INTERNAL_MALLOC=0"
     LDFLAGS: "-fsanitize=address,undefined"
     EXTRA_CONFIGURE: "--with-libidn2"
@@ -901,6 +901,7 @@ msvc:windows:amd64:
          "with-vcredist=C:/Program Files (x86)/Microsoft Visual Studio/2017/BuildTools/VC/Redist/MSVC/14.16.27012/vcredist_x64.exe"
          "with-openssl=C:/OpenSSL"
          "with-libxml2=C:/libxml2"
+         "with-libuv=C:/libuv"
          "without-python"
          "with-system-tests"
          x64'

.uncrustify.cfg

@@ -642,7 +642,7 @@ align_with_tabs                          = true     # false/true
 align_on_tabstop                         = false    # false/true
 
 # Whether to left-align numbers
-align_number_left                        = false    # false/true
+#align_number_left                        = false    # false/true
 
 # Align variable definitions in prototypes and functions
 align_func_params                        = false    # false/true

bin/dig/dighost.c

@@ -1379,7 +1379,7 @@ setup_libs(void) {
 
 	isc_log_setdebuglevel(lctx, 0);
 
-	result = isc_taskmgr_create(mctx, 1, 0, &taskmgr);
+	result = isc_taskmgr_create(mctx, 1, 0, NULL, &taskmgr);
 	check_result(result, "isc_taskmgr_create");
 
 	result = isc_task_create(taskmgr, 0, &global_task);

bin/dnssec/dnssec-signzone.c

@@ -3797,7 +3797,7 @@ main(int argc, char *argv[]) {
 	print_time(outfp);
 	print_version(outfp);
 
-	result = isc_taskmgr_create(mctx, ntasks, 0, &taskmgr);
+	result = isc_taskmgr_create(mctx, ntasks, 0, NULL, &taskmgr);
 	if (result != ISC_R_SUCCESS)
 		fatal("failed to create task manager: %s",
 		      isc_result_totext(result));

bin/named/include/named/globals.h

@@ -19,6 +19,7 @@
 #include <isc/rwlock.h>
 #include <isc/log.h>
 #include <isc/net.h>
+#include <isc/netmgr.h>
 
 #include <isccfg/aclconf.h>
 #include <isccfg/cfg.h>
@@ -62,6 +63,7 @@ EXTERN bool		named_g_run_done	INIT(false);
  */
 EXTERN isc_timermgr_t *		named_g_timermgr	INIT(NULL);
 EXTERN isc_socketmgr_t *	named_g_socketmgr	INIT(NULL);
+EXTERN isc_nm_t *		named_g_nm		INIT(NULL);
 EXTERN cfg_parser_t *		named_g_parser		INIT(NULL);
 EXTERN cfg_parser_t *		named_g_addparser	INIT(NULL);
 EXTERN const char *		named_g_version		INIT(VERSION);

bin/named/main.c

@@ -24,6 +24,7 @@
 #include <isc/file.h>
 #include <isc/hash.h>
 #include <isc/httpd.h>
+#include <isc/netmgr.h>
 #include <isc/os.h>
 #include <isc/platform.h>
 #include <isc/print.h>
@@ -123,7 +124,6 @@ static int		maxudp = 0;
 /*
  * -T options:
  */
-static bool clienttest = false;
 static bool dropedns = false;
 static bool ednsformerr = false;
 static bool ednsnotimp = false;
@@ -602,17 +602,12 @@ parse_T_opt(char *option) {
 	/*
 	 * force the server to behave (or misbehave) in
 	 * specified ways for testing purposes.
-	 *
-	 * clienttest: make clients single shot with their
-	 * 	       own memory context.
 	 * delay=xxxx: delay client responses by xxxx ms to
 	 *	       simulate remote servers.
 	 * dscp=x:     check that dscp values are as
 	 * 	       expected and assert otherwise.
 	 */
-	if (!strcmp(option, "clienttest")) {
-		clienttest = true;
-	} else if (!strncmp(option, "delay=", 6)) {
+	if (!strncmp(option, "delay=", 6)) {
 		delay = atoi(option + 6);
 	} else if (!strcmp(option, "dropedns")) {
 		dropedns = true;
@@ -877,8 +872,15 @@ create_managers(void) {
 		      "using %u UDP listener%s per interface",
 		      named_g_udpdisp, named_g_udpdisp == 1 ? "" : "s");
 
+	named_g_nm = isc_nm_start(named_g_mctx, named_g_cpus);
+	if (named_g_nm == NULL) {
+		UNEXPECTED_ERROR(__FILE__, __LINE__,
+				"isc_nm_start() failed");
+		return (ISC_R_UNEXPECTED);
+	}
+
 	result = isc_taskmgr_create(named_g_mctx, named_g_cpus, 0,
-				    &named_g_taskmgr);
+				    named_g_nm, &named_g_taskmgr);
 	if (result != ISC_R_SUCCESS) {
 		UNEXPECTED_ERROR(__FILE__, __LINE__,
 				 "isc_taskmgr_create() failed: %s",
@@ -903,6 +905,7 @@ create_managers(void) {
 		return (ISC_R_UNEXPECTED);
 	}
 	isc_socketmgr_maxudp(named_g_socketmgr, maxudp);
+	isc_nm_maxudp(named_g_nm, maxudp);
 	result = isc_socketmgr_getmaxsockets(named_g_socketmgr, &socks);
 	if (result == ISC_R_SUCCESS) {
 		isc_log_write(named_g_lctx, NAMED_LOGCATEGORY_GENERAL,
@@ -921,6 +924,7 @@ destroy_managers(void) {
 	isc_taskmgr_destroy(&named_g_taskmgr);
 	isc_timermgr_destroy(&named_g_timermgr);
 	isc_socketmgr_destroy(&named_g_socketmgr);
+	isc_nm_detach(&named_g_nm);
 }
 
 static void
@@ -1234,8 +1238,6 @@ setup(void) {
 	/*
 	 * Modify server context according to command line options
 	 */
-	if (clienttest)
-		ns_server_setoption(sctx, NS_SERVER_CLIENTTEST, true);
 	if (disable4)
 		ns_server_setoption(sctx, NS_SERVER_DISABLE4, true);
 	if (disable6)

bin/named/server.c

@@ -9400,6 +9400,7 @@ run_server(isc_task_t *task, isc_event_t *event) {
 	CHECKFATAL(ns_interfacemgr_create(named_g_mctx, server->sctx,
 					  named_g_taskmgr, named_g_timermgr,
 					  named_g_socketmgr,
+					  named_g_nm,
 					  named_g_dispatchmgr,
 					  server->task, named_g_udpdisp, geoip,
 					  &server->interfacemgr),
@@ -9462,6 +9463,12 @@ shutdown_server(isc_task_t *task, isc_event_t *event) {
 	UNUSED(task);
 	INSIST(task == server->task);
 
+	/*
+	 * We need to shut down the interface before going exclusive
+	 * (and pausing netmgr).
+	 */
+	ns_interfacemgr_shutdown(server->interfacemgr);
+
 	result = isc_task_beginexclusive(server->task);
 	RUNTIME_CHECK(result == ISC_R_SUCCESS);
 
@@ -9511,7 +9518,6 @@ shutdown_server(isc_task_t *task, isc_event_t *event) {
 	isc_timer_detach(&server->pps_timer);
 	isc_timer_detach(&server->tat_timer);
 
-	ns_interfacemgr_shutdown(server->interfacemgr);
 	ns_interfacemgr_detach(&server->interfacemgr);
 
 	dns_dispatchmgr_destroy(&named_g_dispatchmgr);

bin/nsupdate/nsupdate.c

@@ -906,7 +906,7 @@ setup_system(void) {
 	result = isc_timermgr_create(gmctx, &timermgr);
 	check_result(result, "dns_timermgr_create");
 
-	result = isc_taskmgr_create(gmctx, 1, 0, &taskmgr);
+	result = isc_taskmgr_create(gmctx, 1, 0, NULL, &taskmgr);
 	check_result(result, "isc_taskmgr_create");
 
 	result = isc_task_create(taskmgr, 0, &global_task);

bin/rndc/rndc.c

@@ -937,7 +937,8 @@ main(int argc, char **argv) {
 
 	isc_mem_create(&rndc_mctx);
 	DO("create socket manager", isc_socketmgr_create(rndc_mctx, &socketmgr));
-	DO("create task manager", isc_taskmgr_create(rndc_mctx, 1, 0, &taskmgr));
+	DO("create task manager",
+	   isc_taskmgr_create(rndc_mctx, 1, 0, NULL, &taskmgr));
 	DO("create task", isc_task_create(taskmgr, 0, &task));
 
 	DO("create logging context", isc_log_create(rndc_mctx, &log, &logconfig));

bin/tests/optional/adb_test.c

@@ -144,7 +144,7 @@ create_managers(void) {
 	isc_result_t result;
 
 	taskmgr = NULL;
-	result = isc_taskmgr_create(mctx, 5, 0, &taskmgr);
+	result = isc_taskmgr_create(mctx, 5, 0, NULL, &taskmgr);
 	check_result(result, "isc_taskmgr_create");
 
 	timermgr = NULL;

bin/tests/optional/byaddr_test.c

@@ -112,7 +112,7 @@ main(int argc, char *argv[]) {
 	}
 
 	taskmgr = NULL;
-	RUNTIME_CHECK(isc_taskmgr_create(mctx, workers, 0, &taskmgr)
+	RUNTIME_CHECK(isc_taskmgr_create(mctx, workers, 0, NULL, &taskmgr)
 		      == ISC_R_SUCCESS);
 	task = NULL;
 	RUNTIME_CHECK(isc_task_create(taskmgr, 0, &task)

bin/tests/optional/byname_test.c

@@ -226,7 +226,7 @@ main(int argc, char *argv[]) {
 	}
 
 	taskmgr = NULL;
-	RUNTIME_CHECK(isc_taskmgr_create(mctx, workers, 0, &taskmgr) ==
+	RUNTIME_CHECK(isc_taskmgr_create(mctx, workers, 0, NULL, &taskmgr) ==
 		      ISC_R_SUCCESS);
 	task = NULL;
 	RUNTIME_CHECK(isc_task_create(taskmgr, 0, &task) ==

bin/tests/optional/gsstest.c

@@ -470,7 +470,7 @@ main(int argc, char *argv[]) {
 	RUNCHECK(dst_lib_init(mctx, NULL));
 
 	taskmgr = NULL;
-	RUNCHECK(isc_taskmgr_create(mctx, 1, 0, &taskmgr));
+	RUNCHECK(isc_taskmgr_create(mctx, 1, 0, NULL, &taskmgr));
 	task = NULL;
 	RUNCHECK(isc_task_create(taskmgr, 0, &task));
 	timermgr = NULL;

bin/tests/optional/ratelimiter_test.c

@@ -102,7 +102,7 @@ main(int argc, char *argv[]) {
 	isc_interval_set(&linterval, 1, 0);
 
 	isc_mem_create(&mctx);
-	RUNTIME_CHECK(isc_taskmgr_create(mctx, 3, 0, &taskmgr) ==
+	RUNTIME_CHECK(isc_taskmgr_create(mctx, 3, 0, NULL, &taskmgr) ==
 		      ISC_R_SUCCESS);
 	RUNTIME_CHECK(isc_timermgr_create(mctx, &timermgr) ==
 		      ISC_R_SUCCESS);

bin/tests/optional/shutdown_test.c

@@ -181,7 +181,7 @@ main(int argc, char *argv[]) {
 	isc_mem_create(&mctx);
 	mctx2 = NULL;
 	isc_mem_create(&mctx2);
-	RUNTIME_CHECK(isc_taskmgr_create(mctx, workers, 0, &task_manager) ==
+	RUNTIME_CHECK(isc_taskmgr_create(mctx, workers, 0, NULL, &task_manager) ==
 		      ISC_R_SUCCESS);
 	RUNTIME_CHECK(isc_timermgr_create(mctx, &timer_manager) ==
 		      ISC_R_SUCCESS);

bin/tests/optional/sig0_test.c

@@ -226,7 +226,7 @@ main(int argc, char *argv[]) {
 	dst_result_register();
 
 	taskmgr = NULL;
-	RUNTIME_CHECK(isc_taskmgr_create(mctx, 2, 0, &taskmgr) ==
+	RUNTIME_CHECK(isc_taskmgr_create(mctx, 2, 0, NULL, &taskmgr) ==
 		      ISC_R_SUCCESS);
 	task1 = NULL;
 	RUNTIME_CHECK(isc_task_create(taskmgr, 0, &task1) == ISC_R_SUCCESS);

bin/tests/optional/sock_test.c

@@ -292,7 +292,7 @@ main(int argc, char *argv[]) {
 	 * The task manager is independent (other than memory context)
 	 */
 	manager = NULL;
-	RUNTIME_CHECK(isc_taskmgr_create(mctx, workers, 0, &manager) ==
+	RUNTIME_CHECK(isc_taskmgr_create(mctx, workers, 0, NULL, &manager) ==
 		      ISC_R_SUCCESS);
 
 	/*

bin/tests/optional/task_test.c

@@ -79,7 +79,7 @@ main(int argc, char *argv[]) {
 
 	isc_mem_create(&mctx);
 
-	RUNTIME_CHECK(isc_taskmgr_create(mctx, workers, 0, &manager) ==
+	RUNTIME_CHECK(isc_taskmgr_create(mctx, workers, 0, NULL, &manager) ==
 		      ISC_R_SUCCESS);
 
 	RUNTIME_CHECK(isc_task_create(manager, 0, &t1) == ISC_R_SUCCESS);

bin/tests/optional/timer_test.c

@@ -108,7 +108,7 @@ main(int argc, char *argv[]) {
 	printf("%u workers\n", workers);
 
 	isc_mem_create(&mctx1);
-	RUNTIME_CHECK(isc_taskmgr_create(mctx1, workers, 0, &manager) ==
+	RUNTIME_CHECK(isc_taskmgr_create(mctx1, workers, 0, NULL, &manager) ==
 		      ISC_R_SUCCESS);
 	RUNTIME_CHECK(isc_timermgr_create(mctx1, &timgr) == ISC_R_SUCCESS);
 

bin/tests/optional/zone_test.c

@@ -280,7 +280,7 @@ main(int argc, char **argv) {
 
 	RUNTIME_CHECK(isc_app_start() == ISC_R_SUCCESS);
 	isc_mem_create(&mctx);
-	RUNTIME_CHECK(isc_taskmgr_create(mctx, 2, 0, &taskmgr) ==
+	RUNTIME_CHECK(isc_taskmgr_create(mctx, 2, 0, NULL, &taskmgr) ==
 		      ISC_R_SUCCESS);
 	RUNTIME_CHECK(isc_timermgr_create(mctx, &timermgr) == ISC_R_SUCCESS);
 	RUNTIME_CHECK(isc_socketmgr_create(mctx, &socketmgr) == ISC_R_SUCCESS);

bin/tests/system/README

@@ -584,10 +584,6 @@ By default, start.pl starts a "named" server with the following options:
                     preventing multiple instances of this named running in this
                     directory (which could possibly interfere with the test).
 
-In addition, start.pl also sets the following undocumented flag:
-
-    -T clienttest   Makes clients single-shot with their own memory context.
-
 All output is sent to a file called "named.run" in the nameserver directory.
 
 The options used to start named can be altered.  There are three ways of doing
@@ -608,9 +604,9 @@ the named command-line arguments.  The rest of the file is ignored.
 
 3. Tweaking the default command line arguments with "-T" options.  This flag is
 used to alter the behavior of BIND for testing and is not documented in the
-ARM.  The "clienttest" option has already been mentioned, but the presence of
-certain files in the "nsN" directory adds flags to the default command line
-(the content of the files is irrelevant - it is only the presence that counts):
+ARM.  The presence of certain files in the "nsN" directory adds flags to
+the default command line (the content of the files is irrelevant - it
+is only the presence that counts):
 
     named.noaa       Appends "-T noaa" to the command line, which causes
                      "named" to never set the AA bit in an answer.
@@ -635,7 +631,6 @@ certain files in the "nsN" directory adds flags to the default command line
                      the additional section if the response is triggered by RPZ
                      rewriting).
 
-
 Starting Other Nameservers
 ---
 In contrast to "named", nameservers written in Perl or Python (whose script

bin/tests/system/additional/ns1/named.args

@@ -1,2 +1,2 @@
 # this server runs named with only one worker thread
--m record,size,mctx -c named.conf -d 99 -D additional-ns1 -X named.lock -g -T clienttest -n 1
+-m record,size,mctx -c named.conf -d 99 -D additional-ns1 -X named.lock -g -n 1

bin/tests/system/addzone/tests.sh

@@ -696,11 +696,17 @@ $RNDCCMD 10.53.0.3 addzone "test4.baz" '{ type master; file "e.db"; };' > /dev/n
 $RNDCCMD 10.53.0.3 addzone "test5.baz" '{ type master; file "e.db"; };' > /dev/null 2>&1 || ret=1
 $PERL $SYSTEMTESTTOP/stop.pl addzone ns3
 $PERL $SYSTEMTESTTOP/start.pl --noclean --restart --port ${PORT} addzone ns3 || ret=1
-$DIG $DIGOPTS @10.53.0.3 version.bind txt ch > dig.out.test$n || ret=1
-grep "status: NOERROR" dig.out.test$n > /dev/null || ret=1
-n=`expr $n + 1`
+for try in 0 1 2 3 4 5 6 7 8 9; do
+    iret=0
+    $DIG $DIGOPTS @10.53.0.3 version.bind txt ch > dig.out.test$n || iret=1
+    grep "status: NOERROR" dig.out.test$n > /dev/null || iret=1
+    [ "$iret" -eq 0 ] && break
+    sleep 1
+done
+[ "$iret" -ne 0 ] && ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
+n=`expr $n + 1`
 
 echo_i "exit status: $status"
 [ $status -eq 0 ] || exit 1

bin/tests/system/allow-query/ns3/named.args

@@ -1,2 +1,2 @@
 # this server only has 127.0.0.1 in its localhost/localnets ACLs
--m record,size,mctx -c named.conf -d 99 -D allow-query-ns3 -X named.lock -g -T clienttest -T fixedlocal
+-m record,size,mctx -c named.conf -d 99 -D allow-query-ns3 -X named.lock -g -T fixedlocal

bin/tests/system/delzone/ns2/named.args

@@ -1 +1 @@
--D delzone-ns2 -X named.lock -m record,size,mctx -T clienttest -c named.conf -g -U 4
+-D delzone-ns2 -X named.lock -m record,size,mctx -c named.conf -g -U 4

bin/tests/system/dlzexternal/driver.c

@@ -124,13 +124,13 @@ add_name(struct dlz_example_data *state, struct record *list,
 	    strlen(data) >= sizeof(list[i].data))
 		return (ISC_R_NOSPACE);
 
-	strncpy(list[i].name, name, sizeof(list[i].name));
+	strncpy(list[i].name, name, sizeof(list[i].name) - 1);
 	list[i].name[sizeof(list[i].name) - 1] = '\0';
 
-	strncpy(list[i].type, type, sizeof(list[i].type));
+	strncpy(list[i].type, type, sizeof(list[i].type) - 1);
 	list[i].type[sizeof(list[i].type) - 1] = '\0';
 
-	strncpy(list[i].data, data, sizeof(list[i].data));
+	strncpy(list[i].data, data, sizeof(list[i].data) - 1);
 	list[i].data[sizeof(list[i].data) - 1] = '\0';
 
 	list[i].ttl = ttl;

bin/tests/system/dnssec/ns6/named.args

@@ -1 +1 @@
--m record,size,mctx -c named.conf -d 99 -D dnssec-ns6 -X named.lock -g -T nonearest -T clienttest -T tat=1
+-m record,size,mctx -c named.conf -d 99 -D dnssec-ns6 -X named.lock -g -T nonearest -T tat=1

bin/tests/system/dscp/ns1/named.args

@@ -1 +1 @@
--m record,size,mctx -T clienttest -c named.conf -d 99 -D dscp-ns1 -X named.lock -g -U 4 -T dscp=46
+-m record,size,mctx -c named.conf -d 99 -D dscp-ns1 -X named.lock -g -U 4 -T dscp=46

bin/tests/system/dscp/ns2/named.args

@@ -1 +1 @@
--m record,size,mctx -T clienttest -c named.conf -d 99 -D dscp-ns2 -X named.lock -g -U 4 -T dscp=46
+-m record,size,mctx -c named.conf -d 99 -D dscp-ns2 -X named.lock -g -U 4 -T dscp=46

bin/tests/system/dscp/ns3/named.args

@@ -1 +1 @@
--m record,size,mctx -T clienttest -c named.conf -d 99 -D dscp-ns3 -X named.lock -g -U 4 -T dscp=46
+-m record,size,mctx -c named.conf -d 99 -D dscp-ns3 -X named.lock -g -U 4 -T dscp=46

bin/tests/system/dscp/ns4/named.args

@@ -1 +1 @@
--m record,size,mctx -T clienttest -c named.conf -d 99 -D dscp-ns4 -X named.lock -g -U 4 -T dscp=46
+-m record,size,mctx -c named.conf -d 99 -D dscp-ns4 -X named.lock -g -U 4 -T dscp=46

bin/tests/system/dscp/ns5/named.args

@@ -1 +1 @@
--m record,size,mctx -T clienttest -c named.conf -d 99 -D dscp-ns5 -X named.lock -g -U 4 -T dscp=46
+-m record,size,mctx -c named.conf -d 99 -D dscp-ns5 -X named.lock -g -U 4 -T dscp=46

bin/tests/system/dscp/ns6/named.args

@@ -1 +1 @@
--m record,size,mctx -T clienttest -c named.conf -d 99 -D dscp-ns6 -X named.lock -g -U 4 -T dscp=46
+-m record,size,mctx -c named.conf -d 99 -D dscp-ns6 -X named.lock -g -U 4 -T dscp=46

bin/tests/system/dscp/ns7/named.args

@@ -1 +1 @@
--m record,size,mctx -T clienttest -c named.conf -d 99 -D dscp-ns7 -X named.lock -g -U 4 -T dscp=46
+-m record,size,mctx -c named.conf -d 99 -D dscp-ns7 -X named.lock -g -U 4 -T dscp=46

bin/tests/system/dupsigs/ns1/named.args

@@ -1 +1 @@
--D dupsigs-ns1 -X named.lock -m record,size,mctx -T clienttest -c named.conf -d 99 -g -U 4 -T sigvalinsecs
+-D dupsigs-ns1 -X named.lock -m record,size,mctx -c named.conf -d 99 -g -U 4 -T sigvalinsecs

bin/tests/system/ecdsa/tests.sh

@@ -20,7 +20,6 @@ rm -f dig.out.*
 DIGOPTS="+tcp +noau +noadd +nosea +nostat +nocmd +dnssec -p 5300"
 
 # Check the example. domain
-
 echo "I:checking that positive validation works ($n)"
 ret=0
 $DIG $DIGOPTS . @10.53.0.1 soa > dig.out.ns1.test$n || ret=1

bin/tests/system/fetchlimit/ns3/named.args

@@ -1,2 +0,0 @@
-# Don't specify '-T clienttest' as it consumes lots of memory with this test
--D fetchlimit-ns3 -X named.lock -m record,size,mctx -c named.conf -d 99 -g -U 4

bin/tests/system/forward/tests.sh

@@ -98,10 +98,15 @@ status=`expr $status + $ret`
 
 echo_i "checking that forward only zone overrides empty zone"
 ret=0
-$DIG $DIGOPTS 1.0.10.in-addr.arpa TXT @10.53.0.4 > dig.out.f2
-grep "status: NOERROR" dig.out.f2 > /dev/null || ret=1
-$DIG $DIGOPTS 2.0.10.in-addr.arpa TXT @10.53.0.4 > dig.out.f2
-grep "status: NXDOMAIN" dig.out.f2 > /dev/null || ret=1
+# retry loop in case the server restart above causes transient failure
+for try in 0 1 2 3 4 5 6 7 8 9; do
+    $DIG $DIGOPTS 1.0.10.in-addr.arpa TXT @10.53.0.4 > dig.out.f2
+    grep "status: NOERROR" dig.out.f2 > /dev/null || ret=1
+    $DIG $DIGOPTS 2.0.10.in-addr.arpa TXT @10.53.0.4 > dig.out.f2
+    grep "status: NXDOMAIN" dig.out.f2 > /dev/null || ret=1
+    [ "$ret" -eq 0 ] && break
+    sleep 1
+done
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
 

bin/tests/system/legacy/clean.sh

@@ -14,6 +14,7 @@ rm -f ns*/named.run
 rm -f ns*/named.lock
 
 # build.sh
+rm -f ns1/named_dump.db*
 rm -f ns6/K*
 rm -f ns6/dsset-*
 rm -f ns6/edns512.db

bin/tests/system/legacy/ns4/named.args

@@ -1 +1 @@
--m record,size,mctx -T clienttest -c named.conf -d 99 -D legacy-ns4 -X named.lock -g -U 4 -T noedns
+-m record,size,mctx -c named.conf -d 99 -D legacy-ns4 -X named.lock -g -U 4 -T noedns

bin/tests/system/legacy/ns5/named.args

@@ -1 +1 @@
--m record,size,mctx -T clienttest -c named.conf -d 99 -D legacy-ns5 -X named.lock -g -U 4 -T noedns
+-m record,size,mctx -c named.conf -d 99 -D legacy-ns5 -X named.lock -g -U 4 -T noedns

bin/tests/system/legacy/ns6/named.args

@@ -1 +1 @@
--m record,size,mctx -T clienttest -c named.conf -d 99 -D legacy-ns6 -X named.lock -g -U 4 -T maxudp512
+-m record,size,mctx -c named.conf -d 99 -D legacy-ns6 -X named.lock -g -U 4 -T maxudp512

bin/tests/system/legacy/ns7/named.args

@@ -1 +1 @@
--m record,size,mctx -T clienttest -c named.conf -d 99 -D legacy-ns7 -X named.lock -g -U 4 -T maxudp512
+-m record,size,mctx -c named.conf -d 99 -D legacy-ns7 -X named.lock -g -U 4 -T maxudp512

bin/tests/system/legacy/tests.sh

@@ -259,8 +259,13 @@ $PERL $SYSTEMTESTTOP/start.pl --noclean --restart --port ${PORT} legacy ns1
 
 n=`expr $n + 1`
 echo_i "checking recursive lookup to edns 512 + no tcp + trust anchor fails ($n)"
-ret=0
-resolution_fails edns512-notcp. || ret=1
+# retry loop in case the server restart above causes transient failure
+for try in 0 1 2 3 4 5 6 7 8 9; do
+    ret=0
+    resolution_fails edns512-notcp. || ret=1
+    [ "$ret" -eq 0 ] && break
+    sleep 1
+done
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
 

bin/tests/system/logfileconfig/tests.sh

@@ -36,7 +36,7 @@ DLFILE="named_deflog"
 
 PIDFILE="${THISDIR}/${CONFDIR}/named.pid"
 myRNDC="$RNDC -c ${THISDIR}/${CONFDIR}/rndc.conf"
-myNAMED="$NAMED -c ${THISDIR}/${CONFDIR}/named.conf -m record,size,mctx -T clienttest -T nosyslog -d 99 -D logfileconfig-ns1 -X named.lock -U 4"
+myNAMED="$NAMED -c ${THISDIR}/${CONFDIR}/named.conf -m record,size,mctx -T nosyslog -d 99 -D logfileconfig-ns1 -X named.lock -U 4"
 
 # Test given condition.  If true, test again after a second.  Used for testing
 # filesystem-dependent conditions in order to prevent false negatives caused by

bin/tests/system/mirror/ns3/named.args

@@ -1 +1 @@
--D mirror-ns3 -X named.lock -m record,size,mctx -T clienttest -c named.conf -d 99 -g -U 4 -T tat=3
+-D mirror-ns3 -X named.lock -m record,size,mctx -c named.conf -d 99 -g -U 4 -T tat=3

bin/tests/system/mkeys/ns2/named.args

@@ -1 +1 @@
--m record,size,mctx -T clienttest -c named.conf -d 99 -D mkeys-ns2 -X named.lock -g -T mkeytimers=5/10/20 -T tat=1
+-m record,size,mctx -c named.conf -d 99 -D mkeys-ns2 -X named.lock -g -T mkeytimers=5/10/20 -T tat=1

bin/tests/system/mkeys/ns3/named.args

@@ -1 +1 @@
--m record,size,mctx -T clienttest -c named.conf -d 99 -D mkeys-ns3 -X named.lock -g -T mkeytimers=5/10/20
+-m record,size,mctx -c named.conf -d 99 -D mkeys-ns3 -X named.lock -g -T mkeytimers=5/10/20

bin/tests/system/mkeys/ns5/named1.args

@@ -1 +1 @@
--m record,size,mctx -T clienttest -c named.conf -d 99 -X named.lock -g
+-m record,size,mctx -c named.conf -d 99 -X named.lock -g

bin/tests/system/mkeys/ns5/named2.args

@@ -1 +1 @@
--m record,size,mctx -T clienttest -c named.conf -d 99 -X named.lock -g -T mkeytimers=2/20/40
+-m record,size,mctx -c named.conf -d 99 -X named.lock -g -T mkeytimers=2/20/40

bin/tests/system/mkeys/ns6/named.args

@@ -1 +1 @@
--m record,size,mctx -T clienttest -c named.conf -d 99 -X named.lock -g -T mkeytimers=5/10/20
+-m record,size,mctx -c named.conf -d 99 -X named.lock -g -T mkeytimers=5/10/20

bin/tests/system/nsupdate/ns5/named.args

@@ -1 +1 @@
--D nsupdate-ns5 -m record,size,mctx -T clienttest -c named.conf -d 99 -X named.lock -g -U 4 -T fixedlocal
+-D nsupdate-ns5 -m record,size,mctx -c named.conf -d 99 -X named.lock -g -U 4 -T fixedlocal

bin/tests/system/nsupdate/ns6/named.args

@@ -1 +1 @@
--D nsupdate-ns6 -m record,size,mctx -T clienttest -c named.conf -d 99 -X named.lock -g -U 4 -T fixedlocal
+-D nsupdate-ns6 -m record,size,mctx -c named.conf -d 99 -X named.lock -g -U 4 -T fixedlocal

bin/tests/system/nsupdate/tests.sh

@@ -506,7 +506,6 @@ grep "add nsec3param.test.	0	IN	TYPE65534 .# 6 000140000400" jp.out.ns3.$n > /de
 if [ $ret != 0 ] ; then echo_i "failed"; status=`expr $ret + $status`; fi
 
 
-
 ret=0
 echo_i "testing that rndc stop updates the master file"
 $NSUPDATE -k ns1/ddns.key <<END > /dev/null || ret=1
@@ -520,10 +519,16 @@ $PERL $SYSTEMTESTTOP/stop.pl --use-rndc --port ${CONTROLPORT} nsupdate ns1
 # those dumped to the master file by "rndc stop".
 rm -f ns1/*jnl
 $PERL $SYSTEMTESTTOP/start.pl --noclean --restart --port ${PORT} nsupdate ns1
-$DIG $DIGOPTS +tcp +noadd +nosea +nostat +noquest +nocomm +nocmd updated4.example.nil.\
-	@10.53.0.1 a > dig.out.ns1 || status=1
-digcomp knowngood.ns1.afterstop dig.out.ns1 || ret=1
-[ $ret = 0 ] || { echo_i "failed"; status=1; }
+for try in 0 1 2 3 4 5 6 7 8 9; do
+    iret=0
+    $DIG $DIGOPTS +tcp +noadd +nosea +nostat +noquest +nocomm +nocmd \
+	updated4.example.nil. @10.53.0.1 a > dig.out.ns1 || iret=1
+    digcomp knowngood.ns1.afterstop dig.out.ns1 || iret=1
+    [ "$iret" -eq 0 ] && break
+    sleep 1
+done
+[ "$iret" -ne 0 ] && ret=1
+[ "$ret" -eq 0 ] || { echo_i "failed"; status=1; }
 
 ret=0
 echo_i "check that 'nsupdate -l' with a missing keyfile reports the missing file"

bin/tests/system/nzd2nzf/tests.sh

@@ -61,9 +61,14 @@ $PERL $SYSTEMTESTTOP/start.pl --noclean --restart --port ${PORT} nzd2nzf ns1
 
 n=`expr $n + 1`
 echo_i "querying for zone data from migrated zone config ($n)"
-ret=0
-$DIG $DIGOPTS @10.53.0.1 a.added.example a > dig.out.ns1.$n || ret=1
-grep 'status: NOERROR' dig.out.ns1.$n > /dev/null || ret=1
+# retry loop in case the server restart above causes transient failures
+for try in 0 1 2 3 4 5 6 7 8 9; do
+    ret=0
+    $DIG $DIGOPTS @10.53.0.1 a.added.example a > dig.out.ns1.$n || ret=1
+    grep 'status: NOERROR' dig.out.ns1.$n > /dev/null || ret=1
+    [ "$ret" -eq 0 ] && break
+    sleep 1
+done
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`

bin/tests/system/pipelined/pipequeries.c

@@ -277,7 +277,7 @@ main(int argc, char *argv[]) {
 	RUNCHECK(dst_lib_init(mctx, NULL));
 
 	taskmgr = NULL;
-	RUNCHECK(isc_taskmgr_create(mctx, 1, 0, &taskmgr));
+	RUNCHECK(isc_taskmgr_create(mctx, 1, 0, NULL, &taskmgr));
 	task = NULL;
 	RUNCHECK(isc_task_create(taskmgr, 0, &task));
 	timermgr = NULL;

bin/tests/system/resolver/ns7/named.args

@@ -1,2 +0,0 @@
-# this server runs named with the "-T clienttest" option omitted
--m record,size,mctx -c named.conf -d 99 -D resolver-ns7 -X named.lock -g

bin/tests/system/resolver/ns7/named2.conf.in

@@ -12,7 +12,7 @@
 // NS7
 
 options {
-	query-source address 10.53.0.7 port @PORT@ dscp 13;
+	query-source address 10.53.0.7 dscp 13;
 	notify-source 10.53.0.7 dscp 14;
 	transfer-source 10.53.0.7 dscp 15;
 	port @PORT@;

bin/tests/system/rndc/ns6/named.args

@@ -1,3 +1,3 @@
 # teardown of a huge zone with tracing enabled takes way too long 
 # -m none is set so that stop.pl does not timeout
--D rndc-ns6 -X named.lock -m none -T clienttest -c named.conf -d 99 -g -U 4
+-D rndc-ns6 -X named.lock -m none -c named.conf -d 99 -g -U 4

bin/tests/system/rpz/tests.sh

@@ -219,6 +219,7 @@ restart () {
     $PERL $SYSTEMTESTTOP/start.pl --noclean --restart --port ${PORT} rpz ns$1
     load_db
     dnsrps_loaded
+    sleep 1
 }
 
 # $1=server and irrelevant args
@@ -465,6 +466,7 @@ for mode in native dnsrps; do
     else
       echo_i "running DNSRPS sub-test"
       $PERL $SYSTEMTESTTOP/start.pl --noclean --restart --port ${PORT} rpz
+      sleep 3
     fi
     ;;
   esac

bin/tests/system/rpzrecurse/tests.sh

@@ -135,6 +135,7 @@ for mode in native dnsrps; do
     else
       echo_i "running DNSRPS sub-test"
       $PERL $SYSTEMTESTTOP/start.pl --noclean --restart --port ${PORT} rpzrecurse
+      sleep 3
     fi
     ;;
   esac

bin/tests/system/start.pl

@@ -257,7 +257,6 @@ sub construct_ns_command {
 		$command .= "-D $test-$server ";
 		$command .= "-X named.lock ";
 		$command .= "-m record,size,mctx ";
-		$command .= "-T clienttest ";
 
 		foreach my $t_option(
 			"dropedns", "ednsformerr", "ednsnotimp", "ednsrefused",

bin/tests/system/statistics/tests.sh

@@ -71,7 +71,7 @@ $RNDCCMD -s 10.53.0.3 stats > /dev/null 2>&1
 [ -f ns3/named.stats ] || ret=1
 if [ ! "$CYGWIN" ]; then
     nsock0nstat=`grep "UDP/IPv4 sockets active" ns3/named.stats | awk '{print $1}'`
-    [ 0 -ne ${nsock0nstat:-0} ] || ret=1
+    [ 0 -eq ${nsock0nstat:-0} ] || ret=1
 fi
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
@@ -107,7 +107,7 @@ if [ ! "$CYGWIN" ]; then
     ret=0
     echo_i "verifying active sockets output in named.stats ($n)"
     nsock1nstat=`grep "UDP/IPv4 sockets active" ns3/named.stats | awk '{print $1}'`
-    [ `expr $nsock1nstat - $nsock0nstat` -eq 1 ] || ret=1
+    [ `expr ${nsock1nstat:-0} - ${nsock0nstat:-0}` -eq 1 ] || ret=1
     if [ $ret != 0 ]; then echo_i "failed"; fi
     status=`expr $status + $ret`
     n=`expr $n + 1`

bin/tests/system/tkey/keycreate.c

@@ -236,7 +236,7 @@ main(int argc, char *argv[]) {
 	RUNCHECK(dst_lib_init(mctx, NULL));
 
 	taskmgr = NULL;
-	RUNCHECK(isc_taskmgr_create(mctx, 1, 0, &taskmgr));
+	RUNCHECK(isc_taskmgr_create(mctx, 1, 0, NULL, &taskmgr));
 	task = NULL;
 	RUNCHECK(isc_task_create(taskmgr, 0, &task));
 	timermgr = NULL;

bin/tests/system/tkey/keydelete.c

@@ -175,7 +175,7 @@ main(int argc, char **argv) {
 	RUNCHECK(dst_lib_init(mctx, NULL));
 
 	taskmgr = NULL;
-	RUNCHECK(isc_taskmgr_create(mctx, 1, 0, &taskmgr));
+	RUNCHECK(isc_taskmgr_create(mctx, 1, 0, NULL, &taskmgr));
 	task = NULL;
 	RUNCHECK(isc_task_create(taskmgr, 0, &task));
 	timermgr = NULL;

bin/tests/system/unknown/tests.sh

@@ -122,16 +122,24 @@ do
 done
 
 echo_i "checking large unknown record loading on master"
-ret=0
-$DIG $DIGOPTS @10.53.0.1 +tcp +short large.example TYPE45234 > dig.out || { ret=1 ; echo_i "dig failed" ; }
-$DIFF -s large.out dig.out > /dev/null || { ret=1 ; echo_i "$DIFF failed"; }
+for try in 0 1 2 3 4 5 6 7 8 9; do
+    ret=0
+    $DIG $DIGOPTS @10.53.0.1 +tcp +short large.example TYPE45234 > dig.out || { ret=1 ; echo_i "dig failed" ; }
+    $DIFF -s large.out dig.out > /dev/null || { ret=1 ; echo_i "$DIFF failed"; }
+    [ "$ret" -eq 0 ] && break
+    sleep 1
+done
 [ $ret = 0 ] || echo_i "failed"
 status=`expr $status + $ret`
 
 echo_i "checking large unknown record loading on slave"
-ret=0
-$DIG $DIGOPTS @10.53.0.2 +tcp +short large.example TYPE45234 > dig.out || { ret=1 ; echo_i "dig failed" ; }
-$DIFF -s large.out dig.out > /dev/null || { ret=1 ; echo_i "$DIFF failed"; }
+for try in 0 1 2 3 4 5 6 7 8 9; do
+    ret=0
+    $DIG $DIGOPTS @10.53.0.2 +tcp +short large.example TYPE45234 > dig.out || { ret=1 ; echo_i "dig failed" ; }
+    $DIFF -s large.out dig.out > /dev/null || { ret=1 ; echo_i "$DIFF failed"; }
+    [ "$ret" -eq 0 ] && break
+    sleep 1
+done
 [ $ret = 0 ] || echo_i "failed"
 status=`expr $status + $ret`
 
@@ -139,10 +147,16 @@ echo_i "stop and restart slave"
 $PERL $SYSTEMTESTTOP/stop.pl unknown ns2
 $PERL $SYSTEMTESTTOP/start.pl --noclean --restart --port ${PORT} unknown ns2
 
+# server may be answering queries before zones are loaded,
+# so retry a few times if this query fails
 echo_i "checking large unknown record loading on slave"
-ret=0
-$DIG $DIGOPTS @10.53.0.2 +tcp +short large.example TYPE45234 > dig.out || { ret=1 ; echo_i "dig failed" ; }
-$DIFF -s large.out dig.out > /dev/null || { ret=1 ; echo_i "$DIFF failed"; }
+for try in 0 1 2 3 4 5 6 7 8 9; do
+    ret=0
+    $DIG $DIGOPTS @10.53.0.2 +tcp +short large.example TYPE45234 > dig.out || { ret=1 ; echo_i "dig failed" ; }
+    $DIFF -s large.out dig.out > /dev/null || { ret=1 ; echo_i "$DIFF failed"; }
+    [ "$ret" -eq 0 ] && break
+    sleep 1
+done
 [ $ret = 0 ] || echo_i "failed"
 status=`expr $status + $ret`
 
@@ -157,10 +171,16 @@ echo_i "stop and restart inline slave"
 $PERL $SYSTEMTESTTOP/stop.pl unknown ns3
 $PERL $SYSTEMTESTTOP/start.pl --noclean --restart --port ${PORT} unknown ns3
 
+# server may be answering queries before zones are loaded,
+# so retry a few times if this query fails
 echo_i "checking large unknown record loading on inline slave"
-ret=0
-$DIG $DIGOPTS @10.53.0.3 +tcp +short large.example TYPE45234 > dig.out || { ret=1 ; echo_i "dig failed" ; }
-$DIFF large.out dig.out > /dev/null || { ret=1 ; echo_i "$DIFF failed"; }
+for try in 0 1 2 3 4 5 6 7 8 9; do
+    ret=0
+    $DIG $DIGOPTS @10.53.0.3 +tcp +short large.example TYPE45234 > dig.out || { ret=1 ; echo_i "dig failed" ; }
+    $DIFF large.out dig.out > /dev/null || { ret=1 ; echo_i "$DIFF failed"; }
+    [ "$ret" -eq 0 ] && break
+    sleep 1
+done
 [ $ret = 0 ] || echo_i "failed"
 status=`expr $status + $ret`
 

bin/tests/system/upforwd/ns3/named.conf.in

@@ -17,7 +17,7 @@ options {
 	pid-file "named.pid";
 	listen-on { 10.53.0.3; };
 	listen-on-v6 { none; };
-	recursion yes;
+	recursion no;
 	notify yes;
 };
 

bin/tests/system/upforwd/tests.sh

@@ -21,8 +21,6 @@ DIGOPTS="+tcp +noadd +nosea +nostat +noquest +nocomm +nocmd -p ${PORT}"
 status=0
 n=1
 
-sleep 5
-
 echo_i "waiting for servers to be ready for testing ($n)"
 for i in 1 2 3 4 5 6 7 8 9 10
 do

bin/tests/system/xfer/tests.sh

@@ -431,11 +431,17 @@ $DIG -p ${PORT} txt mapped @10.53.0.3 > dig.out.1.$n
 grep "status: NOERROR," dig.out.1.$n > /dev/null || tmp=1
 $PERL $SYSTEMTESTTOP/stop.pl xfer ns3
 $PERL $SYSTEMTESTTOP/start.pl --noclean --restart --port ${PORT} xfer ns3
-$DIG -p ${PORT} txt mapped @10.53.0.3 > dig.out.2.$n
-grep "status: NOERROR," dig.out.2.$n > /dev/null || tmp=1
-$DIG -p ${PORT} axfr mapped @10.53.0.3 > dig.out.3.$n
-digcomp knowngood.mapped dig.out.3.$n || tmp=1
-if test $tmp != 0 ; then echo_i "failed"; fi
+for try in 0 1 2 3 4 5 6 7 8 9; do
+    iret=0
+    $DIG -p ${PORT} txt mapped @10.53.0.3 > dig.out.2.$n
+    grep "status: NOERROR," dig.out.2.$n > /dev/null || iret=1
+    $DIG -p ${PORT} axfr mapped @10.53.0.3 > dig.out.3.$n
+    digcomp knowngood.mapped dig.out.3.$n || iret=1
+    [ "$iret" -eq 0 ] && break
+    sleep 1
+done
+[ "$iret" -eq 0 ] || tmp=1
+[ "$tmp" -ne 0 ] && echo_i "failed"
 status=`expr $status + $tmp`
 
 n=`expr $n + 1`

bin/tools/mdig.c

@@ -2047,7 +2047,7 @@ main(int argc, char *argv[]) {
 		fatal("can't choose between IPv4 and IPv6");
 
 	taskmgr = NULL;
-	RUNCHECK(isc_taskmgr_create(mctx, 1, 0, &taskmgr));
+	RUNCHECK(isc_taskmgr_create(mctx, 1, 0, NULL, &taskmgr));
 	task = NULL;
 	RUNCHECK(isc_task_create(taskmgr, 0, &task));
 	timermgr = NULL;

config.h.in

@@ -366,6 +366,9 @@
 /* define if struct stat has st_mtim.tv_nsec field */
 #undef HAVE_STAT_NSEC
 
+/* Define to 1 if you have the <stdalign.h> header file. */
+#undef HAVE_STDALIGN_H
+
 /* Define to 1 if you have the <stdatomic.h> header file. */
 #undef HAVE_STDATOMIC_H
 

configure

@@ -747,6 +747,8 @@ OPENSSL_LIBS
 OPENSSL_CFLAGS
 INSTALL_LIBRARY
 ALWAYS_DEFINES
+LIBUV_LIBS
+LIBUV_CFLAGS
 PTHREAD_CFLAGS
 PTHREAD_LIBS
 PTHREAD_CC
@@ -848,6 +850,7 @@ infodir
 docdir
 oldincludedir
 includedir
+runstatedir
 localstatedir
 sharedstatedir
 sysconfdir
@@ -967,6 +970,8 @@ PKG_CONFIG_LIBDIR
 MAXMINDDB_CFLAGS
 MAXMINDDB_LIBS
 MAXMINDDB_PREFIX
+LIBUV_CFLAGS
+LIBUV_LIBS
 OPENSSL_CFLAGS
 OPENSSL_LIBS
 LIBXML2_CFLAGS
@@ -1018,6 +1023,7 @@ datadir='${datarootdir}'
 sysconfdir='${prefix}/etc'
 sharedstatedir='${prefix}/com'
 localstatedir='${prefix}/var'
+runstatedir='${localstatedir}/run'
 includedir='${prefix}/include'
 oldincludedir='/usr/include'
 docdir='${datarootdir}/doc/${PACKAGE_TARNAME}'
@@ -1270,6 +1276,15 @@ do
   | -silent | --silent | --silen | --sile | --sil)
     silent=yes ;;
 
+  -runstatedir | --runstatedir | --runstatedi | --runstated \
+  | --runstate | --runstat | --runsta | --runst | --runs \
+  | --run | --ru | --r)
+    ac_prev=runstatedir ;;
+  -runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \
+  | --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \
+  | --run=* | --ru=* | --r=*)
+    runstatedir=$ac_optarg ;;
+
   -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb)
     ac_prev=sbindir ;;
   -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \
@@ -1407,7 +1422,7 @@ fi
 for ac_var in	exec_prefix prefix bindir sbindir libexecdir datarootdir \
 		datadir sysconfdir sharedstatedir localstatedir includedir \
 		oldincludedir docdir infodir htmldir dvidir pdfdir psdir \
-		libdir localedir mandir
+		libdir localedir mandir runstatedir
 do
   eval ac_val=\$$ac_var
   # Remove trailing slashes.
@@ -1560,6 +1575,7 @@ Fine tuning of the installation directories:
   --sysconfdir=DIR        read-only single-machine data [PREFIX/etc]
   --sharedstatedir=DIR    modifiable architecture-independent data [PREFIX/com]
   --localstatedir=DIR     modifiable single-machine data [PREFIX/var]
+  --runstatedir=DIR       modifiable per-process data [LOCALSTATEDIR/run]
   --libdir=DIR            object code libraries [EPREFIX/lib]
   --includedir=DIR        C header files [PREFIX/include]
   --oldincludedir=DIR     C header files for non-gcc [/usr/include]
@@ -1726,6 +1742,9 @@ Some influential environment variables:
               linker flags for MAXMINDDB, overriding pkg-config
   MAXMINDDB_PREFIX
               value of prefix for libmaxminddb, overriding pkg-config
+  LIBUV_CFLAGS
+              C compiler flags for LIBUV, overriding pkg-config
+  LIBUV_LIBS  linker flags for LIBUV, overriding pkg-config
   OPENSSL_CFLAGS
               C compiler flags for OPENSSL, overriding pkg-config
   OPENSSL_LIBS
@@ -4000,7 +4019,7 @@ else
     We can't simply define LARGE_OFF_T to be 9223372036854775807,
     since some C++ compilers masquerading as C compilers
     incorrectly reject 9223372036854775807.  */
-#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
+#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
   int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
 		       && LARGE_OFF_T % 2147483647 == 1)
 		      ? 1 : -1];
@@ -4046,7 +4065,7 @@ else
     We can't simply define LARGE_OFF_T to be 9223372036854775807,
     since some C++ compilers masquerading as C compilers
     incorrectly reject 9223372036854775807.  */
-#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
+#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
   int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
 		       && LARGE_OFF_T % 2147483647 == 1)
 		      ? 1 : -1];
@@ -4070,7 +4089,7 @@ rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
     We can't simply define LARGE_OFF_T to be 9223372036854775807,
     since some C++ compilers masquerading as C compilers
     incorrectly reject 9223372036854775807.  */
-#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
+#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
   int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
 		       && LARGE_OFF_T % 2147483647 == 1)
 		      ? 1 : -1];
@@ -4115,7 +4134,7 @@ else
     We can't simply define LARGE_OFF_T to be 9223372036854775807,
     since some C++ compilers masquerading as C compilers
     incorrectly reject 9223372036854775807.  */
-#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
+#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
   int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
 		       && LARGE_OFF_T % 2147483647 == 1)
 		      ? 1 : -1];
@@ -4139,7 +4158,7 @@ rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
     We can't simply define LARGE_OFF_T to be 9223372036854775807,
     since some C++ compilers masquerading as C compilers
     incorrectly reject 9223372036854775807.  */
-#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
+#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
   int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
 		       && LARGE_OFF_T % 2147483647 == 1)
 		      ? 1 : -1];
@@ -15768,6 +15787,152 @@ fi
 done
 
 
+# libuv
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for libuv >= 1.0.0" >&5
+$as_echo_n "checking for libuv >= 1.0.0... " >&6; }
+
+if test -n "$LIBUV_CFLAGS"; then
+    pkg_cv_LIBUV_CFLAGS="$LIBUV_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libuv >= 1.0.0\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "libuv >= 1.0.0") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LIBUV_CFLAGS=`$PKG_CONFIG --cflags "libuv >= 1.0.0" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+if test -n "$LIBUV_LIBS"; then
+    pkg_cv_LIBUV_LIBS="$LIBUV_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libuv >= 1.0.0\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "libuv >= 1.0.0") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LIBUV_LIBS=`$PKG_CONFIG --libs "libuv >= 1.0.0" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+        _pkg_short_errors_supported=yes
+else
+        _pkg_short_errors_supported=no
+fi
+        if test $_pkg_short_errors_supported = yes; then
+	        LIBUV_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "libuv >= 1.0.0" 2>&1`
+        else
+	        LIBUV_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "libuv >= 1.0.0" 2>&1`
+        fi
+	# Put the nasty error message in config.log where it belongs
+	echo "$LIBUV_PKG_ERRORS" >&5
+
+	as_fn_error $? "The uv library was not found. Please install libuv!" "$LINENO" 5
+elif test $pkg_failed = untried; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+	as_fn_error $? "The uv library was not found. Please install libuv!" "$LINENO" 5
+else
+	LIBUV_CFLAGS=$pkg_cv_LIBUV_CFLAGS
+	LIBUV_LIBS=$pkg_cv_LIBUV_LIBS
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+
+
+  CCASFLAGS_libuv_ax_save_flags=$CCASFLAGS
+
+
+
+  CFLAGS_libuv_ax_save_flags=$CFLAGS
+
+
+
+  CPPFLAGS_libuv_ax_save_flags=$CPPFLAGS
+
+
+
+  CXXFLAGS_libuv_ax_save_flags=$CXXFLAGS
+
+
+
+  ERLCFLAGS_libuv_ax_save_flags=$ERLCFLAGS
+
+
+
+  FCFLAGS_libuv_ax_save_flags=$FCFLAGS
+
+
+
+  FCLIBS_libuv_ax_save_flags=$FCLIBS
+
+
+
+  FFLAGS_libuv_ax_save_flags=$FFLAGS
+
+
+
+  FLIBS_libuv_ax_save_flags=$FLIBS
+
+
+
+  GCJFLAGS_libuv_ax_save_flags=$GCJFLAGS
+
+
+
+  JAVACFLAGS_libuv_ax_save_flags=$JAVACFLAGS
+
+
+
+  LDFLAGS_libuv_ax_save_flags=$LDFLAGS
+
+
+
+  LIBS_libuv_ax_save_flags=$LIBS
+
+
+
+  OBJCFLAGS_libuv_ax_save_flags=$OBJCFLAGS
+
+
+
+  OBJCXXFLAGS_libuv_ax_save_flags=$OBJCXXFLAGS
+
+
+
+  UPCFLAGS_libuv_ax_save_flags=$UPCFLAGS
+
+
+
+  VALAFLAGS_libuv_ax_save_flags=$VALAFLAGS
+
+
+
+
+CFLAGS="$CFLAGS $LIBUV_CFLAGS"
+LIBS="$LIBS $LIBUV_LIBS"
+
 #
 # flockfile is usually provided by pthreads
 #
@@ -18149,7 +18314,7 @@ fi ;; #(
 esac
 
 if test "$GCC" = "yes"; then :
-  STD_CWARNINGS="$STD_CWARNINGS -W -Wall -Wmissing-prototypes -Wcast-qual -Wwrite-strings -Wformat -Wpointer-arith"
+  STD_CWARNINGS="$STD_CWARNINGS -W -Wall -Wmissing-prototypes -Wcast-qual -Wwrite-strings -Wformat -Wpointer-arith -Wno-missing-field-initializers"
 
 fi
 
@@ -19672,6 +19837,19 @@ done
 
 LIBS="$LIBS $ISC_ATOMIC_LIBS"
 
+for ac_header in stdalign.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "stdalign.h" "ac_cv_header_stdalign_h" "$ac_includes_default"
+if test "x$ac_cv_header_stdalign_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_STDALIGN_H 1
+_ACEOF
+
+fi
+
+done
+
+
 for ac_header in uchar.h
 do :
   ac_fn_c_check_header_mongrel "$LINENO" "uchar.h" "ac_cv_header_uchar_h" "$ac_includes_default"
@@ -23306,7 +23484,7 @@ ac_config_commands="$ac_config_commands chmod"
 # elsewhere if there's a good reason for doing so.
 #
 
-ac_config_files="$ac_config_files make/Makefile make/mkdep Makefile bin/Makefile bin/check/Makefile bin/confgen/Makefile bin/confgen/unix/Makefile bin/delv/Makefile bin/dig/Makefile bin/dnssec/Makefile bin/named/Makefile bin/named/unix/Makefile bin/nsupdate/Makefile bin/pkcs11/Makefile bin/plugins/Makefile bin/python/Makefile bin/python/isc/Makefile bin/python/isc/utils.py bin/python/isc/tests/Makefile bin/python/dnssec-checkds.py bin/python/dnssec-coverage.py bin/python/dnssec-keymgr.py bin/python/isc/__init__.py bin/python/isc/checkds.py bin/python/isc/coverage.py bin/python/isc/dnskey.py bin/python/isc/eventlist.py bin/python/isc/keydict.py bin/python/isc/keyevent.py bin/python/isc/keymgr.py bin/python/isc/keyseries.py bin/python/isc/keyzone.py bin/python/isc/policy.py bin/python/isc/rndc.py bin/python/isc/tests/dnskey_test.py bin/python/isc/tests/policy_test.py bin/rndc/Makefile bin/tests/Makefile bin/tests/headerdep_test.sh bin/tests/optional/Makefile bin/tests/pkcs11/Makefile bin/tests/pkcs11/benchmarks/Makefile bin/tests/system/Makefile bin/tests/system/conf.sh bin/tests/system/dlzexternal/Makefile bin/tests/system/dlzexternal/ns1/dlzs.conf bin/tests/system/dyndb/Makefile bin/tests/system/dyndb/driver/Makefile bin/tests/system/pipelined/Makefile bin/tests/system/rndc/Makefile bin/tests/system/rpz/Makefile bin/tests/system/rsabigexponent/Makefile bin/tests/system/tkey/Makefile bin/tools/Makefile contrib/scripts/check-secure-delegation.pl contrib/scripts/zone-edit.sh doc/Makefile doc/arm/Makefile doc/arm/noteversion.xml doc/arm/pkgversion.xml doc/arm/releaseinfo.xml doc/doxygen/Doxyfile doc/doxygen/Makefile doc/doxygen/doxygen-input-filter doc/misc/Makefile doc/tex/Makefile doc/tex/armstyle.sty doc/xsl/Makefile doc/xsl/isc-docbook-chunk.xsl doc/xsl/isc-docbook-html.xsl doc/xsl/isc-manpage.xsl doc/xsl/isc-notes-html.xsl lib/Makefile lib/bind9/Makefile lib/bind9/include/Makefile lib/bind9/include/bind9/Makefile lib/dns/Makefile lib/dns/include/Makefile lib/dns/include/dns/Makefile lib/dns/include/dst/Makefile lib/dns/tests/Makefile lib/irs/Makefile lib/irs/include/Makefile lib/irs/include/irs/Makefile lib/irs/include/irs/netdb.h lib/irs/include/irs/platform.h lib/irs/tests/Makefile lib/isc/pthreads/Makefile lib/isc/pthreads/include/Makefile lib/isc/pthreads/include/isc/Makefile lib/isc/Makefile lib/isc/include/Makefile lib/isc/include/isc/Makefile lib/isc/include/isc/platform.h lib/isc/include/pk11/Makefile lib/isc/include/pkcs11/Makefile lib/isc/tests/Makefile lib/isc/unix/Makefile lib/isc/unix/include/Makefile lib/isc/unix/include/isc/Makefile lib/isccc/Makefile lib/isccc/include/Makefile lib/isccc/include/isccc/Makefile lib/isccc/tests/Makefile lib/isccfg/Makefile lib/isccfg/include/Makefile lib/isccfg/include/isccfg/Makefile lib/isccfg/tests/Makefile lib/ns/Makefile lib/ns/include/Makefile lib/ns/include/ns/Makefile lib/ns/tests/Makefile lib/samples/Makefile lib/samples/Makefile-postinstall unit/unittest.sh fuzz/Makefile"
+ac_config_files="$ac_config_files make/Makefile make/mkdep Makefile bin/Makefile bin/check/Makefile bin/confgen/Makefile bin/confgen/unix/Makefile bin/delv/Makefile bin/dig/Makefile bin/dnssec/Makefile bin/named/Makefile bin/named/unix/Makefile bin/nsupdate/Makefile bin/pkcs11/Makefile bin/plugins/Makefile bin/python/Makefile bin/python/isc/Makefile bin/python/isc/utils.py bin/python/isc/tests/Makefile bin/python/dnssec-checkds.py bin/python/dnssec-coverage.py bin/python/dnssec-keymgr.py bin/python/isc/__init__.py bin/python/isc/checkds.py bin/python/isc/coverage.py bin/python/isc/dnskey.py bin/python/isc/eventlist.py bin/python/isc/keydict.py bin/python/isc/keyevent.py bin/python/isc/keymgr.py bin/python/isc/keyseries.py bin/python/isc/keyzone.py bin/python/isc/policy.py bin/python/isc/rndc.py bin/python/isc/tests/dnskey_test.py bin/python/isc/tests/policy_test.py bin/rndc/Makefile bin/tests/Makefile bin/tests/headerdep_test.sh bin/tests/optional/Makefile bin/tests/pkcs11/Makefile bin/tests/pkcs11/benchmarks/Makefile bin/tests/system/Makefile bin/tests/system/conf.sh bin/tests/system/dlzexternal/Makefile bin/tests/system/dlzexternal/ns1/dlzs.conf bin/tests/system/dyndb/Makefile bin/tests/system/dyndb/driver/Makefile bin/tests/system/pipelined/Makefile bin/tests/system/rndc/Makefile bin/tests/system/rpz/Makefile bin/tests/system/rsabigexponent/Makefile bin/tests/system/tkey/Makefile bin/tools/Makefile contrib/scripts/check-secure-delegation.pl contrib/scripts/zone-edit.sh doc/Makefile doc/arm/Makefile doc/arm/noteversion.xml doc/arm/pkgversion.xml doc/arm/releaseinfo.xml doc/doxygen/Doxyfile doc/doxygen/Makefile doc/doxygen/doxygen-input-filter doc/misc/Makefile doc/tex/Makefile doc/tex/armstyle.sty doc/xsl/Makefile doc/xsl/isc-docbook-chunk.xsl doc/xsl/isc-docbook-html.xsl doc/xsl/isc-manpage.xsl doc/xsl/isc-notes-html.xsl lib/Makefile lib/bind9/Makefile lib/bind9/include/Makefile lib/bind9/include/bind9/Makefile lib/dns/Makefile lib/dns/include/Makefile lib/dns/include/dns/Makefile lib/dns/include/dst/Makefile lib/dns/tests/Makefile lib/irs/Makefile lib/irs/include/Makefile lib/irs/include/irs/Makefile lib/irs/include/irs/netdb.h lib/irs/include/irs/platform.h lib/irs/tests/Makefile lib/isc/pthreads/Makefile lib/isc/pthreads/include/Makefile lib/isc/pthreads/include/isc/Makefile lib/isc/Makefile lib/isc/include/Makefile lib/isc/include/isc/Makefile lib/isc/include/isc/platform.h lib/isc/include/pk11/Makefile lib/isc/include/pkcs11/Makefile lib/isc/netmgr/Makefile lib/isc/tests/Makefile lib/isc/unix/Makefile lib/isc/unix/include/Makefile lib/isc/unix/include/isc/Makefile lib/isccc/Makefile lib/isccc/include/Makefile lib/isccc/include/isccc/Makefile lib/isccc/tests/Makefile lib/isccfg/Makefile lib/isccfg/include/Makefile lib/isccfg/include/isccfg/Makefile lib/isccfg/tests/Makefile lib/ns/Makefile lib/ns/include/Makefile lib/ns/include/ns/Makefile lib/ns/tests/Makefile lib/samples/Makefile lib/samples/Makefile-postinstall unit/unittest.sh fuzz/Makefile"
 
 
 #
@@ -24406,6 +24584,7 @@ do
     "lib/isc/include/isc/platform.h") CONFIG_FILES="$CONFIG_FILES lib/isc/include/isc/platform.h" ;;
     "lib/isc/include/pk11/Makefile") CONFIG_FILES="$CONFIG_FILES lib/isc/include/pk11/Makefile" ;;
     "lib/isc/include/pkcs11/Makefile") CONFIG_FILES="$CONFIG_FILES lib/isc/include/pkcs11/Makefile" ;;
+    "lib/isc/netmgr/Makefile") CONFIG_FILES="$CONFIG_FILES lib/isc/netmgr/Makefile" ;;
     "lib/isc/tests/Makefile") CONFIG_FILES="$CONFIG_FILES lib/isc/tests/Makefile" ;;
     "lib/isc/unix/Makefile") CONFIG_FILES="$CONFIG_FILES lib/isc/unix/Makefile" ;;
     "lib/isc/unix/include/Makefile") CONFIG_FILES="$CONFIG_FILES lib/isc/unix/include/Makefile" ;;

configure.ac

@@ -641,6 +641,14 @@ AC_CHECK_FUNCS([pthread_setaffinity_np cpuset_setaffinity processor_bind sched_s
 AC_CHECK_FUNCS([pthread_setname_np pthread_set_name_np])
 AC_CHECK_HEADERS([pthread_np.h], [], [], [#include <pthread.h>])
 
+# libuv
+PKG_CHECK_MODULES([LIBUV], [libuv >= 1.0.0], [],
+		  [AC_MSG_ERROR([The uv library was not found. Please install libuv!])])
+AX_SAVE_FLAGS([libuv])
+
+CFLAGS="$CFLAGS $LIBUV_CFLAGS"
+LIBS="$LIBS $LIBUV_LIBS"
+
 #
 # flockfile is usually provided by pthreads
 #
@@ -1321,7 +1329,7 @@ AS_CASE([$host],
 		  [MKDEPCFLAGS="-xM"])])
 
 AS_IF([test "$GCC" = "yes"],
-      [STD_CWARNINGS="$STD_CWARNINGS -W -Wall -Wmissing-prototypes -Wcast-qual -Wwrite-strings -Wformat -Wpointer-arith"]
+      [STD_CWARNINGS="$STD_CWARNINGS -W -Wall -Wmissing-prototypes -Wcast-qual -Wwrite-strings -Wformat -Wpointer-arith -Wno-missing-field-initializers"]
       )
 
 AX_CHECK_COMPILE_FLAG([-fno-strict-aliasing],
@@ -1796,6 +1804,8 @@ AC_CHECK_HEADERS(
   ])
 LIBS="$LIBS $ISC_ATOMIC_LIBS"
 
+AC_CHECK_HEADERS([stdalign.h])
+
 AC_CHECK_HEADERS([uchar.h])
 
 #
@@ -2817,6 +2827,7 @@ AC_CONFIG_FILES([
 	lib/isc/include/isc/platform.h
 	lib/isc/include/pk11/Makefile
 	lib/isc/include/pkcs11/Makefile
+	lib/isc/netmgr/Makefile
 	lib/isc/tests/Makefile
 	lib/isc/unix/Makefile
 	lib/isc/unix/include/Makefile

doc/design/netmgr.md

@@ -0,0 +1,30 @@
+# What is netmgr
+netmgr (aka rainbow duck) is the new networking system for BIND. It's based on libuv, although it does not expose any of the libuv api - to keep the API agnostic of underlying library.
+## A bit of history
+Networking in BIND9 up to 9.12 works with a single event loop (epoll on Linux, kqueue on fbsd, etc).
+
+When a client wants to read from a socket it creates a socketevent with a task that will receive this event. isc_socket_{read,write,etc.} operation tries to read directly from the socket, if it succeeds it sends the socketevent to the task provided by the callee. If it doesn't it adds an event to an event loop, and when this event is received the listener is re-set, and an internal task is launched to read the data from the socket. After the internal task is done it launches the task from socketevent provided by the callee. This means that a simple socket operation causes a lot of context switches.
+
+9.14 fixed some of these issues by having multiple event loops in separate threads (one per CPU), that read the data immediately and then call the socketevent - which is still sub-optimal.
+
+## Internals - event loops
+Internally new 
+
+
+## Basic concepts
+
+### `isc_nm_t`
+
+### `isc_nmsocket_t`
+nmsocket is a wrapper around a libuv socket
+
+### `isc_nmhandle_t`
+nmhandle is an interface that can be read or written - for TCP it's just a socket, for UDP it's a socket with peer address. The idea is that he client should not care what the underlying transport is.
+
+## UDP listening
+
+## TCP listening
+
+## TCP listening for DNS
+
+

doc/dev/dev.md

@@ -654,10 +654,6 @@ Items can be removed from the list using `ISC_LIST_UNLINK`:
 
         ISC_LIST_UNLINK(foolist, foo, link);
 
-A similar but smaller set of `ISC_QUEUE` macros, including `ISC_QUEUE_PUSH`
-and `ISC_QUEUE_POP`, are provided to implement strict FIFO lists, with
-built-in fine-grained locking.
-
 #### <a name="names"></a>Names
 
 The `dns_name` API has facilities for processing DNS names and labels,

lib/dns/acl.c

@@ -182,6 +182,7 @@ dns_acl_match(const isc_netaddr_t *reqaddr,
 	isc_result_t result;
 	int match_num = -1;
 	unsigned int i;
+	INSIST(DNS_ACL_VALID(acl));
 
 	REQUIRE(reqaddr != NULL);
 	REQUIRE(matchelt == NULL || *matchelt == NULL);

lib/dns/dispatch.c

@@ -598,7 +598,8 @@ deref_portentry(dns_dispatch_t *disp, dispportentry_t **portentryp) {
 	dns_qid_t *qid;
 
 	REQUIRE(disp->port_table != NULL);
-	REQUIRE(portentry != NULL && isc_refcount_current(&portentry->refs) > 0);
+	REQUIRE(portentry != NULL &&
+		isc_refcount_current(&portentry->refs) > 0);
 
 	if (isc_refcount_decrement(&portentry->refs) == 1) {
 		qid = DNS_QID(disp);

lib/dns/message.c

@@ -1103,7 +1103,7 @@ getquestions(isc_buffer_t *source, dns_message_t *msg, dns_decompress_t *dctx,
 			result = ISC_R_NOMEMORY;
 			goto cleanup;
 		}
-		rdataset =  isc_mempool_get(msg->rdspool);
+		rdataset = isc_mempool_get(msg->rdspool);
 		if (rdataset == NULL) {
 			result = ISC_R_NOMEMORY;
 			goto cleanup;

lib/dns/tests/dnstest.c

@@ -116,7 +116,7 @@ create_managers(void) {
 	isc_result_t result;
 	ncpus = isc_os_ncpus();
 
-	CHECK(isc_taskmgr_create(dt_mctx, ncpus, 0, &taskmgr));
+	CHECK(isc_taskmgr_create(dt_mctx, ncpus, 0, NULL, &taskmgr));
 	CHECK(isc_timermgr_create(dt_mctx, &timermgr));
 	CHECK(isc_socketmgr_create(dt_mctx, &socketmgr));
 	CHECK(isc_task_create(taskmgr, 0, &maintask));

lib/isc/Makefile.in

@@ -50,12 +50,15 @@ OBJS =		pk11.@O@ pk11_result.@O@ \
 		backtrace.@O@ base32.@O@ base64.@O@ \
 		bind9.@O@ buffer.@O@ bufferlist.@O@ \
 		commandline.@O@ counter.@O@ crc64.@O@ error.@O@ entropy.@O@ \
-		event.@O@ hash.@O@ ht.@O@ heap.@O@ hex.@O@ hmac.@O@ \
-		httpd.@O@ iterated_hash.@O@ \
+		event.@O@ hash.@O@ ht.@O@ heap.@O@ hex.@O@ \
+		hmac.@O@ httpd.@O@ hp.@O@ astack.@O@ iterated_hash.@O@ \
 		lex.@O@ lfsr.@O@ lib.@O@ log.@O@ \
 		md.@O@ mem.@O@ mutexblock.@O@ \
+		netmgr/netmgr.@O@ netmgr/tcp.@O@ netmgr/udp.@O@ \
+		netmgr/tcpdns.@O@ netmgr/uverr2result.@O@ \
 		netaddr.@O@ netscope.@O@ nonce.@O@ openssl_shim.@O@ pool.@O@ \
-		parseint.@O@ portset.@O@ quota.@O@ radix.@O@ random.@O@ \
+		parseint.@O@ portset.@O@ queue.@O@ quota.@O@ \
+		radix.@O@ random.@O@ \
 		ratelimiter.@O@ region.@O@ regex.@O@ result.@O@ \
 		rwlock.@O@ \
 		serial.@O@ siphash.@O@ sockaddr.@O@ stats.@O@ \
@@ -69,12 +72,12 @@ SRCS =		pk11.c pk11_result.c \
 		aes.c app.c assertions.c \
 		backtrace.c base32.c base64.c bind9.c \
 		buffer.c bufferlist.c commandline.c counter.c crc64.c \
-		entropy.c error.c event.c hash.c ht.c heap.c hex.c hmac.c \
-		httpd.c iterated_hash.c \
+		entropy.c error.c event.c hash.c ht.c heap.c \
+		hex.c hmac.c hp.c astack.c httpd.c iterated_hash.c \
 		lex.c lfsr.c lib.c log.c \
 		md.c mem.c mutexblock.c \
 		netaddr.c netscope.c nonce.c openssl_shim.c pool.c \
-		parseint.c portset.c quota.c radix.c random.c \
+		parseint.c portset.c queue.c quota.c radix.c random.c \
 		ratelimiter.c region.c regex.c result.c rwlock.c \
 		serial.c siphash.c sockaddr.c stats.c string.c \
 		symtab.c task.c taskpool.c timer.c \
@@ -86,7 +89,7 @@ LIBS =		${OPENSSL_LIBS} @LIBS@
 # Attempt to disable parallel processing.
 .NOTPARALLEL:
 .NO_PARALLEL:
-SUBDIRS =	include unix pthreads
+SUBDIRS =	include netmgr unix pthreads
 TARGETS =	timestamp
 TESTDIRS =	@UNITTESTS@
 

lib/isc/astack.c

@@ -0,0 +1,80 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+#include <inttypes.h>
+#include <string.h>
+
+#include <isc/astack.h>
+#include <isc/atomic.h>
+#include <isc/mem.h>
+#include <isc/mutex.h>
+#include <isc/types.h>
+#include <isc/util.h>
+
+struct isc_astack {
+	isc_mem_t *mctx;
+	size_t size;
+	size_t pos;
+	isc_mutex_t lock;
+	uintptr_t nodes[];
+};
+
+isc_astack_t *
+isc_astack_new(isc_mem_t *mctx, size_t size) {
+	isc_astack_t *stack =
+		isc_mem_get(mctx,
+			    sizeof(isc_astack_t) + size * sizeof(uintptr_t));
+
+	stack->mctx = NULL;
+	isc_mem_attach(mctx, &stack->mctx);
+	stack->size = size;
+	stack->pos = 0;
+	memset(stack->nodes, 0, size * sizeof(uintptr_t));
+	isc_mutex_init(&stack->lock);
+	return (stack);
+}
+
+bool
+isc_astack_trypush(isc_astack_t *stack, void *obj) {
+	if (isc_mutex_trylock(&stack->lock) == false) {
+		if (stack->pos >= stack->size) {
+			isc_mutex_unlock(&stack->lock);
+			return (false);
+		}
+		stack->nodes[stack->pos++] = (uintptr_t) obj;
+		isc_mutex_unlock(&stack->lock);
+		return (true);
+	} else {
+		return (false);
+	}
+}
+
+void *
+isc_astack_pop(isc_astack_t *stack) {
+	isc_mutex_lock(&stack->lock);
+	uintptr_t rv;
+	if (stack->pos == 0) {
+		rv = 0;
+	} else {
+		rv = stack->nodes[--stack->pos];
+	}
+	isc_mutex_unlock(&stack->lock);
+	return ((void*) rv);
+}
+
+void
+isc_astack_destroy(isc_astack_t *stack) {
+	REQUIRE(stack->pos == 0);
+
+	isc_mem_putanddetach(&stack->mctx, stack,
+			     sizeof(struct isc_astack) +
+			      stack->size * sizeof(uintptr_t));
+}

lib/isc/hp.c

@@ -0,0 +1,245 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+/*
+ * Hazard Pointer implementation.
+ *
+ * This work is based on C++ code available from:
+ * https://github.com/pramalhe/ConcurrencyFreaks/
+ *
+ * Copyright (c) 2014-2016, Pedro Ramalhete, Andreia Correia
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Concurrency Freaks nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER>
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <inttypes.h>
+
+#include <isc/atomic.h>
+#include <isc/hp.h>
+#include <isc/string.h>
+#include <isc/mem.h>
+#include <isc/util.h>
+
+#define HP_MAX_THREADS 128
+#define HP_MAX_HPS 4			/* This is named 'K' in the HP paper */
+#define CLPAD (128 / sizeof(uintptr_t))
+#define HP_THRESHOLD_R 0		/* This is named 'R' in the HP paper */
+
+/* Maximum number of retired objects per thread */
+#define MAX_RETIRED (HP_MAX_THREADS * HP_MAX_HPS)
+
+#define TID_UNKNOWN -1
+
+static atomic_int_fast32_t tid_v_base;
+static bool tid_v_initialized;
+
+#if defined(HAVE_TLS)
+#if defined(HAVE_THREAD_LOCAL)
+#include <threads.h>
+static thread_local int tid_v = TID_UNKNOWN;
+#elif defined(HAVE___THREAD)
+static __thread int tid_v = TID_UNKNOWN;
+#elif defined(HAVE___DECLSPEC_THREAD)
+static __declspec( thread ) int tid_v = TID_UNKNOWN;
+#else  /* if defined(HAVE_THREAD_LOCAL) */
+#error "Unknown method for defining a TLS variable!"
+#endif /* if defined(HAVE_THREAD_LOCAL) */
+#else  /* if defined(HAVE_TLS) */
+#error "Thread-local storage support is required!"
+#endif /* if defined(HAVE_TLS) */
+
+typedef struct retirelist {
+	int			size;
+	uintptr_t		list[MAX_RETIRED];
+} retirelist_t;
+
+struct isc_hp {
+	int			max_hps;
+	isc_mem_t		*mctx;
+	atomic_uintptr_t	*hp[HP_MAX_THREADS];
+	retirelist_t		*rl[HP_MAX_THREADS];
+	isc_hp_deletefunc_t	*deletefunc;
+};
+
+static inline int
+tid() {
+	if (!tid_v_initialized) {
+		atomic_init(&tid_v_base, 0);
+		tid_v_initialized = true;
+	}
+	if (tid_v == TID_UNKNOWN) {
+		tid_v = atomic_fetch_add(&tid_v_base, 1);
+		REQUIRE(tid_v < HP_MAX_THREADS);
+	}
+
+	return (tid_v);
+}
+
+isc_hp_t *
+isc_hp_new(isc_mem_t *mctx, size_t max_hps, isc_hp_deletefunc_t *deletefunc) {
+	isc_hp_t *hp = isc_mem_get(mctx, sizeof(*hp));
+
+	if (max_hps == 0) {
+		max_hps = HP_MAX_HPS;
+	}
+
+	*hp = (isc_hp_t){
+		.max_hps = max_hps,
+		.deletefunc = deletefunc
+	};
+
+	isc_mem_attach(mctx, &hp->mctx);
+
+	for (int i = 0; i < HP_MAX_THREADS; i++) {
+		hp->hp[i] = isc_mem_get(mctx, CLPAD * 2 * sizeof(hp->hp[i][0]));
+		hp->rl[i] = isc_mem_get(mctx, sizeof(*hp->rl[0]));
+		*hp->rl[i] = (retirelist_t) { .size = 0 };
+
+		for (int j = 0; j < hp->max_hps; j++) {
+			atomic_init(&hp->hp[i][j], 0);
+		}
+	}
+
+	return (hp);
+}
+
+void
+isc_hp_destroy(isc_hp_t *hp) {
+	for (int i = 0; i < HP_MAX_THREADS; i++) {
+		isc_mem_put(hp->mctx, hp->hp[i],
+			    CLPAD * 2 * sizeof(uintptr_t));
+
+		for (int j = 0; j < hp->rl[i]->size; j++) {
+			void *data = (void *)hp->rl[i]->list[j];
+			hp->deletefunc(data);
+		}
+
+		isc_mem_put(hp->mctx, hp->rl[i], sizeof(*hp->rl[0]));
+	}
+
+	isc_mem_putanddetach(&hp->mctx, hp, sizeof(*hp));
+}
+
+void
+isc_hp_clear(isc_hp_t *hp) {
+	for (int i = 0; i < hp->max_hps; i++) {
+		atomic_store_release(&hp->hp[tid()][i], 0);
+	}
+}
+
+/*
+ * Progress Condition: wait-free population oblivious
+ */
+void isc_hp_clear_one(isc_hp_t *hp, int ihp) {
+	atomic_store_release(&hp->hp[tid()][ihp], 0);
+}
+
+/*
+ * Progress Condition: lock-free
+ */
+uintptr_t
+isc_hp_protect(isc_hp_t *hp, int ihp, atomic_uintptr_t *atom) {
+	uintptr_t n = 0;
+	uintptr_t ret;
+	while ((ret = atomic_load(atom)) != n) {
+		atomic_store(&hp->hp[tid()][ihp], ret);
+		n = ret;
+	}
+	return (ret);
+}
+
+/*
+ * This returns the same value that is passed as ptr, which is sometimes
+ * useful.
+ *
+ * Progress Condition: wait-free population oblivious
+ */
+uintptr_t
+isc_hp_protect_ptr(isc_hp_t *hp, int ihp, atomic_uintptr_t ptr) {
+	atomic_store(&hp->hp[tid()][ihp], atomic_load(&ptr));
+	return (atomic_load(&ptr));
+}
+
+/*
+ * This returns the same value that is passed as ptr, which is sometimes
+ * useful.
+ *
+ * Progress Condition: wait-free population oblivious
+ */
+uintptr_t
+isc_hp_protect_release(isc_hp_t *hp, int ihp, atomic_uintptr_t ptr) {
+	atomic_store_release(&hp->hp[tid()][ihp], atomic_load(&ptr));
+	return (atomic_load(&ptr));
+}
+
+/*
+ * Progress Condition: wait-free bounded (by the number of threads squared)
+ */
+void
+isc_hp_retire(isc_hp_t *hp, uintptr_t ptr) {
+	hp->rl[tid()]->list[hp->rl[tid()]->size++] = ptr;
+	INSIST(hp->rl[tid()]->size < MAX_RETIRED);
+
+	if (hp->rl[tid()]->size < HP_THRESHOLD_R) {
+		return;
+	}
+
+	for (int iret = 0; iret < hp->rl[tid()]->size;) {
+		uintptr_t obj = hp->rl[tid()]->list[iret];
+		bool can_delete = true;
+		for (int itid = 0;
+		     itid < HP_MAX_THREADS && can_delete;
+		     itid++)
+		{
+			for (int ihp = hp->max_hps-1; ihp >= 0; ihp--) {
+				if (atomic_load(&hp->hp[itid][ihp]) == obj) {
+					can_delete = false;
+					break;
+				}
+			}
+		}
+
+		if (can_delete) {
+			size_t bytes = (hp->rl[tid()]->size - iret) *
+				sizeof(hp->rl[tid()]->list[0]);
+			memmove(&hp->rl[tid()]->list[iret],
+				&hp->rl[tid()]->list[iret + 1],
+				bytes);
+			hp->rl[tid()]->size--;
+			hp->deletefunc((void *)obj);
+		}
+
+		iret++;
+	}
+}

lib/isc/include/isc/Makefile.in

@@ -28,7 +28,7 @@ HEADERS =	aes.h app.h assertions.h atomic.h backtrace.h \
 		lang.h lex.h lfsr.h lib.h likely.h list.h log.h \
 		magic.h md.h mem.h meminfo.h mutexblock.h \
 		netaddr.h netscope.h nonce.h os.h parseint.h \
-		pool.h portset.h print.h queue.h quota.h \
+		pool.h portset.h print.h quota.h \
 		radix.h random.h ratelimiter.h refcount.h regex.h \
 		region.h resource.h result.h resultclass.h rwlock.h \
 		safe.h serial.h siphash.h sockaddr.h socket.h \

lib/isc/include/isc/astack.h

@@ -0,0 +1,45 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+#include <inttypes.h>
+#include <isc/mem.h>
+#include <isc/types.h>
+
+
+isc_astack_t *
+isc_astack_new(isc_mem_t *mctx, size_t size);
+/*%<
+ * Allocate and initialize a new array stack of size 'size'.
+ */
+
+void
+isc_astack_destroy(isc_astack_t *stack);
+/*%<
+ * Free an array stack 'stack'.
+ *
+ * Requires:
+ * \li	'stack' is empty.
+ */
+
+bool
+isc_astack_trypush(isc_astack_t *stack, void *obj);
+/*%<
+ * Try to push 'obj' onto array stack 'astack'. On failure, either
+ * because the stack size limit has been reached or because another
+ * thread has already changed the stack pointer, return 'false'.
+ */
+
+void *
+isc_astack_pop(isc_astack_t *stack);
+/*%<
+ * Pop an object off of array stack 'stack'. If the stack is empty,
+ * return NULL.
+ */

lib/isc/include/isc/hp.h

@@ -0,0 +1,77 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+/*
+ * Hazard Pointer implementation.
+ *
+ * This work is based on C++ code available from:
+ * https://github.com/pramalhe/ConcurrencyFreaks/
+ *
+ * Copyright (c) 2014-2016, Pedro Ramalhete, Andreia Correia
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Concurrency Freaks nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER>
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <isc/atomic.h>
+#include <isc/string.h>
+#include <isc/mem.h>
+#include <isc/types.h>
+#include <isc/util.h>
+
+typedef void
+(isc_hp_deletefunc_t)(void *);
+
+isc_hp_t *
+isc_hp_new(isc_mem_t *mctx, size_t max_hps, isc_hp_deletefunc_t *deletefunc);
+
+void
+isc_hp_destroy(isc_hp_t *hp);
+
+void
+isc_hp_clear(isc_hp_t *hp);
+
+void
+isc_hp_clear_one(isc_hp_t *hp, int ihp);
+
+uintptr_t
+isc_hp_protect(isc_hp_t *hp, int ihp, atomic_uintptr_t *atom);
+
+uintptr_t
+isc_hp_protect_ptr(isc_hp_t *hp, int ihp, atomic_uintptr_t ptr);
+
+uintptr_t
+isc_hp_protect_release(isc_hp_t *hp, int ihp, atomic_uintptr_t ptr);
+
+void
+isc_hp_retire(isc_hp_t *hp, uintptr_t ptr);

lib/isc/include/isc/mutexatomic.h

@@ -93,6 +93,11 @@ typedef struct atomic_uint_fast64 {
 	uint64_t v;
 } atomic_uint_fast64_t;
 
+typedef struct atomic_uintptr {
+	isc_mutex_t m;
+	uintptr_t v;
+} atomic_uintptr_t;
+
 typedef struct atomic_bool_s {
 	isc_mutex_t m;
 	bool v;
@@ -198,3 +203,14 @@ typedef struct atomic_bool_s {
 	atomic_compare_exchange_weak_explicit(obj, expected, desired, \
 					      memory_order_seq_cst,   \
 					      memory_order_seq_cst)
+#define atomic_exchange_explicit(obj, desired, order)			\
+	({								\
+		typeof((obj)->v) ___v;					\
+		REQUIRE(isc_mutex_lock(&(obj)->m) == ISC_R_SUCCESS);	\
+		___v = (obj)->v;					\
+		(obj)->v = desired;					\
+		REQUIRE(isc_mutex_unlock(&(obj)->m) == ISC_R_SUCCESS);	\
+		___v;							\
+	})
+#define atomic_exchange(obj, desired) \
+	atomic_exchange_explicit(obj, desired, memory_order_seq_cst)

lib/isc/include/isc/netmgr.h

@@ -0,0 +1,280 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+#pragma once
+
+#include <config.h>
+
+#include <isc/mem.h>
+#include <isc/result.h>
+#include <isc/types.h>
+
+typedef enum {
+	NMEV_READ,
+	NMEV_WRITE,
+	NMEV_ACCEPT,
+	NMEV_CONNECTED,
+	NMEV_CANCELLED,
+	NMEV_SHUTDOWN
+} isc_nm_eventtype;
+
+isc_nm_t *
+isc_nm_start(isc_mem_t *mctx, uint32_t workers);
+/*%<
+ * Creates a new network manager with 'workers' worker threads,
+ * and starts it running.
+ */
+
+void
+isc_nm_attach(isc_nm_t *mgr, isc_nm_t **dst);
+void
+isc_nm_detach(isc_nm_t **mgr0);
+/*%<
+ * Attach/detach a network manager. When all references have been
+ * released, the network manager is shut down, freeing all resources.
+ */
+
+/* Return thread id of current thread, or ISC_NETMGR_TID_UNKNOWN */
+int
+isc_nm_tid(void);
+
+/*
+ * isc_nm_freehandle frees a handle, releasing resources
+ */
+void
+isc_nm_freehandle(isc_nmhandle_t *handle);
+
+void
+isc_nmsocket_attach(isc_nmsocket_t *sock, isc_nmsocket_t **target);
+/*%<
+ * isc_nmsocket_attach attaches to a socket, increasing refcount
+ */
+
+void
+isc_nmsocket_close(isc_nmsocket_t *sock);
+
+void
+isc_nmsocket_detach(isc_nmsocket_t **socketp);
+/*%<
+ * isc_nmsocket_detach detaches from socket, decreasing refcount
+ * and possibly destroying the socket if it's no longer referenced.
+ */
+
+void
+isc_nmhandle_ref(isc_nmhandle_t *handle);
+void
+isc_nmhandle_unref(isc_nmhandle_t *handle);
+/*%<
+ * Increment/decrement the reference counter in a netmgr handle,
+ * but (unlike the attach/detach functions) do not change the pointer
+ * value. If reference counters drop to zero, the handle can be
+ * marked inactive, possibly triggering deletion of its associated
+ * socket.
+ *
+ * (This will be used to prevent a client from being cleaned up when
+ * it's passed to an isc_task event handler. The libuv code would not
+ * otherwise know that the handle was in use and might free it, along
+ * with the client.)
+ */
+
+void *
+isc_nmhandle_getdata(isc_nmhandle_t *handle);
+
+void *
+isc_nmhandle_getextra(isc_nmhandle_t *handle);
+
+typedef void (*isc_nm_opaquecb)(void *arg);
+
+bool
+isc_nmhandle_is_stream(isc_nmhandle_t *handle);
+
+/*
+ * isc_nmhandle_t has a void * opaque field (usually - ns_client_t).
+ * We reuse handle and `opaque` can also be reused between calls.
+ * This function sets this field and two callbacks:
+ * - doreset resets the `opaque` to initial state
+ * - dofree frees everything associated with `opaque`
+ */
+void
+isc_nmhandle_setdata(isc_nmhandle_t *handle, void *arg,
+		     isc_nm_opaquecb doreset, isc_nm_opaquecb dofree);
+
+isc_sockaddr_t
+isc_nmhandle_peeraddr(isc_nmhandle_t *handle);
+isc_sockaddr_t
+isc_nmhandle_localaddr(isc_nmhandle_t *handle);
+
+typedef void (*isc_nm_recv_cb_t)(isc_nmhandle_t *handle, isc_region_t *region,
+				 void *cbarg);
+/*%<
+ * Callback function to be used when receiving a packet.
+ *
+ * 'handle' the handle that can be used to send back the answer.
+ * 'region' contains the received data. It will be freed after
+ *          return by caller.
+ * 'cbarg'  the callback argument passed to isc_nm_listenudp(),
+ *          isc_nm_listentcpdns(), or isc_nm_read().
+ */
+
+typedef void (*isc_nm_cb_t)(isc_nmhandle_t *handle, isc_result_t result,
+			    void *cbarg);
+/*%<
+ * Callback function for other network completion events (send, connect,
+ * accept).
+ *
+ * 'handle' the handle on which the event took place.
+ * 'result' the result of the event.
+ * 'cbarg'  the callback argument passed to isc_nm_send(),
+ *          isc_nm_tcp_connect(), or isc_nm_listentcp()
+ */
+
+isc_result_t
+isc_nm_listenudp(isc_nm_t *mgr, isc_nmiface_t *iface,
+		 isc_nm_recv_cb_t cb, void *cbarg,
+		 size_t extrasize, isc_nmsocket_t **sockp);
+/*%<
+ * Start listening for UDP packets on interface 'iface' using net manager
+ * 'mgr'.
+ *
+ * On success, 'sockp' will be updated to contain a new listening UDP socket.
+ *
+ * When a packet is received on the socket, 'cb' will be called with 'cbarg'
+ * as its argument.
+ *
+ * When handles are allocated for the socket, 'extrasize' additional bytes
+ * will be allocated along with the handle for an associated object
+ * (typically ns_client).
+ */
+
+void
+isc_nm_udp_stoplistening(isc_nmsocket_t *sock);
+/*%<
+ * Stop listening for UDP packets on socket 'sock'.
+ */
+
+void
+isc_nm_pause(isc_nm_t *mgr);
+/*%<
+ * Pause all processing, equivalent to taskmgr exclusive tasks.
+ * It won't return until all workers have been paused.
+ */
+
+void
+isc_nm_resume(isc_nm_t *mgr);
+/*%<
+ * Resume paused processing. It will return immediately
+ * after signalling workers to resume.
+ */
+
+isc_result_t
+isc_nm_read(isc_nmhandle_t *handle, isc_nm_recv_cb_t cb, void *cbarg);
+
+isc_result_t
+isc_nm_pauseread(isc_nmsocket_t *sock);
+/*%<
+ * Pause reading on this socket, while still remembering the callback.
+ */
+
+isc_result_t
+isc_nm_resumeread(isc_nmsocket_t *sock);
+/*%<
+ * Resume reading from socket.
+ *
+ * Requires:
+ * \li	'sock' is a valid netmgr socket
+ * \li	...for which a read/recv callback has been defined.
+ */
+
+isc_result_t
+isc_nm_send(isc_nmhandle_t *handle, isc_region_t *region,
+	    isc_nm_cb_t cb, void *cbarg);
+/*%<
+ * Send the data in 'region' via 'handle'. Afterward, the callback 'cb' is
+ * called with the argument 'cbarg'.
+ *
+ * 'region' is not copied; it has to be allocated beforehand and freed
+ * in 'cb'.
+ */
+
+isc_result_t
+isc_nm_listentcp(isc_nm_t *mgr, isc_nmiface_t *iface,
+		 isc_nm_cb_t cb, void *cbarg,
+		 size_t extrahandlesize, isc_quota_t *quota,
+		 isc_nmsocket_t **rv);
+/*%<
+ * Start listening for raw messages over the TCP interface 'iface', using
+ * net manager 'mgr'.
+ *
+ * On success, 'sockp' will be updated to contain a new listening TCP
+ * socket.
+ *
+ * When a message is received on the socket, 'cb' will be called with 'cbarg'
+ * as its argument.
+ *
+ * When handles are allocated for the socket, 'extrasize' additional bytes
+ * will be allocated along with the handle for an associated object.
+ *
+ * If 'quota' is not NULL, then the socket is attached to the specified
+ * quota. This allows us to enforce TCP client quota limits.
+ *
+ * NOTE: This is currently only called inside isc_nm_listentcpdns(), which
+ * creates a 'wrapper' socket that sends and receives DNS messages -
+ * prepended with a two-byte length field - and handles buffering.
+ */
+
+void
+isc_nm_tcp_stoplistening(isc_nmsocket_t *sock);
+/*%<
+ * Stop listening on TCP socket 'sock'.
+ */
+
+isc_result_t
+isc_nm_listentcpdns(isc_nm_t *mgr, isc_nmiface_t *iface,
+		     isc_nm_recv_cb_t cb, void *arg,
+		     size_t extrahandlesize, isc_quota_t *quota,
+		     isc_nmsocket_t **sockp);
+/*%<
+ * Start listening for DNS messages over the TCP interface 'iface', using
+ * net manager 'mgr'.
+ *
+ * On success, 'sockp' will be updated to contain a new listening TCPDNS
+ * socket. This is a wrapper around a TCP socket, and handles DNS length
+ * processing.
+ *
+ * When a complete DNS message is received on the socket, 'cb' will be
+ * called with 'cbarg' as its argument.
+ *
+ * When handles are allocated for the socket, 'extrasize' additional bytes
+ * will be allocated along with the handle for an associated object
+ * (typically ns_client).
+ */
+
+void
+isc_nm_tcpdns_stoplistening(isc_nmsocket_t *sock);
+/*%<
+ * Stop listening on TCPDNS socket 'sock'.
+ */
+
+void
+isc_nm_tcpdns_sequential(isc_nmhandle_t *handle);
+/*%<
+ * Disable pipelining on this connection. Each DNS packet
+ * will be only processed after the previous completes.
+ *
+ * This cannot be reversed once set for a given connection
+ */
+
+void
+isc_nm_maxudp(isc_nm_t *mgr, uint32_t maxudp);
+/*%<
+ * Simulate a broken firewall that blocks UDP messages larger
+ * than a given size.
+ */

lib/isc/include/isc/queue.h

@@ -9,153 +9,19 @@
  * information regarding copyright ownership.
  */
 
+#pragma once
+#include <isc/mem.h>
 
-/*
- * This is a generic implementation of a two-lock concurrent queue.
- * There are built-in mutex locks for the head and tail of the queue,
- * allowing elements to be safely added and removed at the same time.
- *
- * NULL is "end of list"
- * -1 is "not linked"
- */
+typedef struct isc_queue isc_queue_t;
 
-#ifndef ISC_QUEUE_H
-#define ISC_QUEUE_H 1
+isc_queue_t *
+isc_queue_new(isc_mem_t *mctx, int max_threads);
 
-#include <stdbool.h>
+uintptr_t
+isc_queue_dequeue(isc_queue_t *queue);
 
-#include <isc/assertions.h>
-#include <isc/mutex.h>
+void
+isc_queue_enqueue(isc_queue_t *queue, uintptr_t item);
 
-#ifdef ISC_QUEUE_CHECKINIT
-#define ISC_QLINK_INSIST(x) ISC_INSIST(x)
-#else
-#define ISC_QLINK_INSIST(x) (void)0
-#endif
-
-#define ISC_QLINK(type) struct { type *prev, *next; }
-
-#define ISC_QLINK_INIT(elt, link) \
-	do { \
-		(elt)->link.next = (elt)->link.prev = (void *)(-1); \
-	} while(0)
-
-#define ISC_QLINK_LINKED(elt, link) ((void*)(elt)->link.next != (void*)(-1))
-
-#define ISC_QUEUE(type) struct { \
-	type *head, *tail; \
-	isc_mutex_t headlock, taillock; \
-}
-
-#define ISC_QUEUE_INIT(queue, link) \
-	do { \
-		isc_mutex_init(&(queue).taillock); \
-		isc_mutex_init(&(queue).headlock); \
-		(queue).tail = (queue).head = NULL; \
-	} while (0)
-
-#define ISC_QUEUE_EMPTY(queue) ((queue).head == NULL)
-
-#define ISC_QUEUE_DESTROY(queue) \
-	do { \
-		ISC_QLINK_INSIST(ISC_QUEUE_EMPTY(queue)); \
-		isc_mutex_destroy(&(queue).taillock); \
-		isc_mutex_destroy(&(queue).headlock); \
-	} while (0)
-
-/*
- * queues are meant to separate the locks at either end.  For best effect, that
- * means keeping the ends separate - i.e. non-empty queues work best.
- *
- * a push to an empty queue has to take the pop lock to update
- * the pop side of the queue.
- * Popping the last entry has to take the push lock to update
- * the push side of the queue.
- *
- * The order is (pop, push), because a pop is presumably in the
- * latency path and a push is when we're done.
- *
- * We do an MT hot test in push to see if we need both locks, so we can
- * acquire them in order.  Hopefully that makes the case where we get
- * the push lock and find we need the pop lock (and have to release it) rare.
- *
- * > 1 entry - no collision, push works on one end, pop on the other
- *   0 entry - headlock race
- *     pop wins - return(NULL), push adds new as both head/tail
- *     push wins - updates head/tail, becomes 1 entry case.
- *   1 entry - taillock race
- *     pop wins - return(pop) sets head/tail NULL, becomes 0 entry case
- *     push wins - updates {head,tail}->link.next, pop updates head
- *                 with new ->link.next and doesn't update tail
- *
- */
-#define ISC_QUEUE_PUSH(queue, elt, link) \
-	do { \
-		bool headlocked = false; \
-		ISC_QLINK_INSIST(!ISC_QLINK_LINKED(elt, link)); \
-		if ((queue).head == NULL) { \
-			LOCK(&(queue).headlock); \
-			headlocked = true; \
-		} \
-		LOCK(&(queue).taillock); \
-		if ((queue).tail == NULL && !headlocked) { \
-			UNLOCK(&(queue).taillock); \
-			LOCK(&(queue).headlock); \
-			LOCK(&(queue).taillock); \
-			headlocked = true; \
-		} \
-		(elt)->link.prev = (queue).tail; \
-		(elt)->link.next = NULL; \
-		if ((queue).tail != NULL) \
-			(queue).tail->link.next = (elt); \
-		(queue).tail = (elt); \
-		UNLOCK(&(queue).taillock); \
-		if (headlocked) { \
-			if ((queue).head == NULL) \
-				(queue).head = (elt); \
-			UNLOCK(&(queue).headlock); \
-		} \
-	} while (0)
-
-#define ISC_QUEUE_POP(queue, link, ret) \
-	do { \
-		LOCK(&(queue).headlock); \
-		ret = (queue).head; \
-		while (ret != NULL) { \
-			if (ret->link.next == NULL) { \
-				LOCK(&(queue).taillock); \
-				if (ret->link.next == NULL) { \
-					(queue).head = (queue).tail = NULL; \
-					UNLOCK(&(queue).taillock); \
-					break; \
-				}\
-				UNLOCK(&(queue).taillock); \
-			} \
-			(queue).head = ret->link.next; \
-			(queue).head->link.prev = NULL; \
-			break; \
-		} \
-		UNLOCK(&(queue).headlock); \
-		if (ret != NULL) \
-			(ret)->link.next = (ret)->link.prev = (void *)(-1); \
-	} while(0)
-
-#define ISC_QUEUE_UNLINK(queue, elt, link) \
-	do { \
-		ISC_QLINK_INSIST(ISC_QLINK_LINKED(elt, link)); \
-		LOCK(&(queue).headlock); \
-		LOCK(&(queue).taillock); \
-		if ((elt)->link.prev == NULL) \
-			(queue).head = (elt)->link.next; \
-		else \
-			(elt)->link.prev->link.next = (elt)->link.next; \
-		if ((elt)->link.next == NULL) \
-			(queue).tail = (elt)->link.prev; \
-		else \
-			(elt)->link.next->link.prev = (elt)->link.prev; \
-		UNLOCK(&(queue).taillock); \
-		UNLOCK(&(queue).headlock); \
-		(elt)->link.next = (elt)->link.prev = (void *)(-1); \
-	} while(0)
-
-#endif /* ISC_QUEUE_H */
+void
+isc_queue_destroy(isc_queue_t *queue);

lib/isc/include/isc/sockaddr.h

@@ -230,6 +230,10 @@ isc_sockaddr_frompath(isc_sockaddr_t *sockaddr, const char *path);
  * \li	ISC_R_SUCCESS
  */
 
+isc_result_t
+isc_sockaddr_fromsockaddr(isc_sockaddr_t *isa, const struct sockaddr *sa);
+
+
 #define ISC_SOCKADDR_FORMATSIZE \
 	sizeof("xxxx:xxxx:xxxx:xxxx:xxxx:xxxx:XXX.XXX.XXX.XXX%SSSSSSSSSS#YYYYY")
 /*%<

lib/isc/include/isc/socket.h

@@ -524,7 +524,7 @@ isc_result_t
 isc_socket_filter(isc_socket_t *sock, const char *filter);
 /*%<
  * Inform the kernel that it should perform accept filtering.
- * If filter is NULL the current filter will be removed.:w
+ * If filter is NULL the current filter will be removed.
  */
 
 isc_result_t

lib/isc/include/isc/task.h

@@ -78,6 +78,7 @@
 #include <isc/eventclass.h>
 #include <isc/lang.h>
 #include <isc/stdtime.h>
+#include <isc/netmgr.h>
 #include <isc/types.h>
 
 #define ISC_TASKEVENT_FIRSTEVENT	(ISC_EVENTCLASS_TASK + 0)
@@ -544,6 +545,8 @@ isc_task_beginexclusive(isc_task_t *task);
  * task.  Waits for any other concurrently executing tasks to finish their
  * current event, and prevents any new events from executing in any of the
  * tasks sharing a task manager with 'task'.
+ * It also pauses processing of network events in netmgr if it was provided
+ * when taskmgr was created.
  *
  * The exclusive access must be relinquished by calling
  * isc_task_endexclusive() before returning from the current event handler.
@@ -568,6 +571,22 @@ isc_task_endexclusive(isc_task_t *task);
  *		exclusive access by calling isc_task_spl().
  */
 
+void
+isc_task_pause(isc_task_t *task0);
+void
+isc_task_unpause(isc_task_t *task0);
+/*%<
+ * Pause/unpause this task. Pausing a task removes it from the ready
+ * queue if it is present there; this ensures that the task will not
+ * run again until unpaused. This is necessary when the libuv network
+ * thread executes a function which schedules task manager events; this
+ * prevents the task manager from executing the next event in a task
+ * before the network thread has finished.
+ *
+ * Requires:
+ *\li	'task' is a valid task, and is not already paused or shutting down.
+ */
+
 void
 isc_task_getcurrenttime(isc_task_t *task, isc_stdtime_t *t);
 void
@@ -633,7 +652,8 @@ isc_taskmgr_createinctx(isc_mem_t *mctx,
 			isc_taskmgr_t **managerp);
 isc_result_t
 isc_taskmgr_create(isc_mem_t *mctx, unsigned int workers,
-		   unsigned int default_quantum, isc_taskmgr_t **managerp);
+		   unsigned int default_quantum,
+		   isc_nm_t *nm, isc_taskmgr_t **managerp);
 /*%<
  * Create a new task manager.  isc_taskmgr_createinctx() also associates
  * the new manager with the specified application context.
@@ -650,6 +670,9 @@ isc_taskmgr_create(isc_mem_t *mctx, unsigned int workers,
  *	quantum value when tasks are created.  If zero, then an implementation
  *	defined default quantum will be used.
  *
+ *\li	If 'nm' is set then netmgr is paused when an exclusive task mode
+ *	is requested.
+ *
  * Requires:
  *
  *\li      'mctx' is a valid memory context.

lib/isc/include/isc/types.h

@@ -30,6 +30,7 @@
 
 /* Core Types.  Alphabetized by defined type. */
 
+typedef struct isc_astack		isc_astack_t;		/*%< Array-based fast stack */
 typedef struct isc_appctx		isc_appctx_t;	 	/*%< Application context */
 typedef struct isc_backtrace_symmap	isc_backtrace_symmap_t; /*%< Symbol Table Entry */
 typedef struct isc_buffer		isc_buffer_t;		/*%< Buffer */
@@ -43,6 +44,7 @@ typedef ISC_LIST(isc_event_t)		isc_eventlist_t;	/*%< Event List */
 typedef unsigned int			isc_eventtype_t;	/*%< Event Type */
 typedef uint32_t			isc_fsaccess_t;		/*%< FS Access */
 typedef struct isc_hash			isc_hash_t;		/*%< Hash */
+typedef struct isc_hp			isc_hp_t;		/*%< Hazard pointer */
 typedef struct isc_httpd		isc_httpd_t;		/*%< HTTP client */
 typedef void (isc_httpdfree_t)(isc_buffer_t *, void *);		/*%< HTTP free function */
 typedef struct isc_httpdmgr		isc_httpdmgr_t;		/*%< HTTP manager */
@@ -59,6 +61,10 @@ typedef struct isc_logmodule		isc_logmodule_t;	/*%< Log Module */
 typedef struct isc_mem			isc_mem_t;		/*%< Memory */
 typedef struct isc_mempool		isc_mempool_t;		/*%< Memory Pool */
 typedef struct isc_netaddr		isc_netaddr_t;		/*%< Net Address */
+typedef struct isc_nm			 isc_nm_t;		/*%< Network manager */
+typedef struct isc_nmsocket		 isc_nmsocket_t;	/*%< Network manager socket */
+typedef struct isc_nmiface		 isc_nmiface_t;		/*%< Network manager interface.  */
+typedef struct isc_nmhandle		 isc_nmhandle_t;	/*%< Network manager handle */
 typedef struct isc_portset		isc_portset_t;		/*%< Port Set */
 typedef struct isc_quota		isc_quota_t;		/*%< Quota */
 typedef struct isc_ratelimiter		isc_ratelimiter_t;	/*%< Rate Limiter */

lib/isc/netmgr/Makefile.in

@@ -0,0 +1,33 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+srcdir =	@srcdir@
+VPATH =		@srcdir@
+top_srcdir =	@top_srcdir@
+
+CINCLUDES =	-I${srcdir}/../include \
+		-I${srcdir}/../unix/include \
+		-I${srcdir}/../pthreads/include \
+		-I${srcdir}/.. \
+		${OPENSSL_CFLAGS} \
+		${JSON_C_CFLAGS} \
+		${LIBXML2_CFLAGS}
+
+CDEFINES =
+CWARNINGS =
+
+# Alphabetically
+OBJS =		netmgr.@O@ tcp.@O@ udp.@O@ tcpdns.@O@ uverr2result.@O@
+
+# Alphabetically
+SRCS =		netmgr.c tcp.c udp.c tcpdns.c uverr2result.c
+
+TARGETS =	${OBJS}
+
+@BIND9_MAKE_RULES@

lib/isc/netmgr/netmgr-int.h

@@ -0,0 +1,540 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+#include <unistd.h>
+#include <uv.h>
+
+#include <isc/astack.h>
+#include <isc/atomic.h>
+#include <isc/buffer.h>
+#include <isc/condition.h>
+#include <isc/magic.h>
+#include <isc/mem.h>
+#include <isc/netmgr.h>
+#include <isc/queue.h>
+#include <isc/random.h>
+#include <isc/refcount.h>
+#include <isc/region.h>
+#include <isc/result.h>
+#include <isc/sockaddr.h>
+#include <isc/thread.h>
+#include <isc/util.h>
+
+#define ISC_NETMGR_TID_UNKNOWN -1
+#define ISC_NETMGR_TID_NOTLS -2
+
+/*
+ * Single network event loop worker.
+ */
+typedef struct isc__networker {
+	isc_nm_t *		   mgr;
+	int			   id;          /* thread id */
+	uv_loop_t		   loop;        /* libuv loop structure */
+	uv_async_t		   async;       /* async channel to send
+						 * data to this networker */
+	isc_mutex_t		   lock;
+	isc_mempool_t		   *mpool_bufs;
+	isc_condition_t		   cond;
+	bool			   paused;
+	bool			   finished;
+	isc_thread_t		   thread;
+	isc_queue_t		   *ievents;     /* incoming async events */
+	isc_refcount_t		   references;
+	atomic_int_fast64_t	   pktcount;
+	char			   udprecvbuf[65536];
+	bool			   udprecvbuf_inuse;
+} isc__networker_t;
+
+/*
+ * A general handle for a connection bound to a networker.  For UDP
+ * connections we have peer address here, so both TCP and UDP can be
+ * handled with a simple send-like function
+ */
+#define NMHANDLE_MAGIC                        ISC_MAGIC('N', 'M', 'H', 'D')
+#define VALID_NMHANDLE(t)                     ISC_MAGIC_VALID(t, \
+							      NMHANDLE_MAGIC)
+
+typedef void (*isc__nm_closecb)(isc_nmhandle_t *);
+
+struct isc_nmhandle {
+	int			magic;
+	isc_refcount_t		references;
+
+	/*
+	 * The socket is not 'attached' in the traditional
+	 * reference-counting sense. Instead, we keep all handles in an
+	 * array in the socket object.  This way, we don't have circular
+	 * dependencies and we can close all handles when we're destroying
+	 * the socket.
+	 */
+	isc_nmsocket_t		*sock;
+	size_t			ah_pos;    /* Position in the socket's
+					    * 'active handles' array */
+
+	/*
+	 * The handle is 'inflight' if netmgr is not currently processing
+	 * it in any way - it might mean that e.g. a recursive resolution
+	 * is happening. For an inflight handle we must wait for the
+	 * calling code to finish before we can free it.
+	 */
+	atomic_bool		inflight;
+
+	isc_sockaddr_t		peer;
+	isc_sockaddr_t		local;
+	isc_nm_opaquecb		doreset; /* reset extra callback, external */
+	isc_nm_opaquecb		dofree;  /* free extra callback, external */
+	void *			opaque;
+	char			extra[];
+};
+
+/*
+ * An interface - an address we can listen on.
+ */
+struct isc_nmiface {
+	isc_sockaddr_t        addr;
+};
+
+typedef enum isc__netievent_type {
+	netievent_stop,
+	netievent_udplisten,
+	netievent_udpstoplisten,
+	netievent_udpsend,
+	netievent_udprecv,
+	netievent_tcpconnect,
+	netievent_tcpsend,
+	netievent_tcprecv,
+	netievent_tcpstartread,
+	netievent_tcppauseread,
+	netievent_tcpresumeread,
+	netievent_tcpstopread,
+	netievent_tcplisten,
+	netievent_tcpstoplisten,
+	netievent_tcpclose,
+} isc__netievent_type;
+
+typedef struct isc__netievent_stop {
+	isc__netievent_type        type;
+} isc__netievent_stop_t;
+
+/*
+ * We have to split it because we can read and write on a socket
+ * simultaneously.
+ */
+typedef union {
+	isc_nm_recv_cb_t	recv;
+	isc_nm_cb_t	  	accept;
+} isc__nm_readcb_t;
+
+typedef union {
+	isc_nm_cb_t	   	send;
+	isc_nm_cb_t	   	connect;
+} isc__nm_writecb_t;
+
+typedef union {
+	isc_nm_recv_cb_t	recv;
+	isc_nm_cb_t		accept;
+	isc_nm_cb_t		send;
+	isc_nm_cb_t		connect;
+} isc__nm_cb_t;
+
+/*
+ * Wrapper around uv_req_t with 'our' fields in it.  req->data should
+ * always point to its parent.  Note that we always allocate more than
+ * sizeof(struct) because we make room for different req types;
+ */
+#define UVREQ_MAGIC                        ISC_MAGIC('N', 'M', 'U', 'R')
+#define VALID_UVREQ(t)                     ISC_MAGIC_VALID(t, UVREQ_MAGIC)
+
+typedef struct isc__nm_uvreq {
+	int			magic;
+	isc_nmsocket_t *	sock;
+	isc_nmhandle_t *	handle;
+	uv_buf_t		uvbuf;	/* translated isc_region_t, to be
+					   sent or received */
+	isc_sockaddr_t		local;	/* local address */
+	isc_sockaddr_t		peer;	/* peer address */
+	isc__nm_cb_t		cb;	/* callback */
+	void *			cbarg;	/* callback argument */
+	union {
+		uv_req_t		req;
+		uv_getaddrinfo_t	getaddrinfo;
+		uv_getnameinfo_t	getnameinfo;
+		uv_shutdown_t		shutdown;
+		uv_write_t		write;
+		uv_connect_t		connect;
+		uv_udp_send_t		udp_send;
+		uv_fs_t			fs;
+		uv_work_t		work;
+	} uv_req;
+} isc__nm_uvreq_t;
+
+typedef struct isc__netievent__socket {
+	isc__netievent_type	type;
+	isc_nmsocket_t		*sock;
+} isc__netievent__socket_t;
+
+typedef isc__netievent__socket_t isc__netievent_udplisten_t;
+typedef isc__netievent__socket_t isc__netievent_udpstoplisten_t;
+typedef isc__netievent__socket_t isc__netievent_tcpstoplisten_t;
+typedef isc__netievent__socket_t isc__netievent_tcpclose_t;
+typedef isc__netievent__socket_t isc__netievent_startread_t;
+typedef isc__netievent__socket_t isc__netievent_pauseread_t;
+typedef isc__netievent__socket_t isc__netievent_resumeread_t;
+
+typedef struct isc__netievent__socket_req {
+	isc__netievent_type	type;
+	isc_nmsocket_t		*sock;
+	isc__nm_uvreq_t		*req;
+} isc__netievent__socket_req_t;
+
+typedef isc__netievent__socket_req_t isc__netievent_tcpconnect_t;
+typedef isc__netievent__socket_req_t isc__netievent_tcplisten_t;
+typedef isc__netievent__socket_req_t isc__netievent_tcpsend_t;
+
+typedef struct isc__netievent_udpsend {
+	isc__netievent_type	type;
+	isc_nmsocket_t		*sock;
+	isc_sockaddr_t		peer;
+	isc__nm_uvreq_t		*req;
+} isc__netievent_udpsend_t;
+
+typedef struct isc__netievent {
+	isc__netievent_type	type;
+} isc__netievent_t;
+
+typedef union {
+		isc__netievent_t		  ni;
+		isc__netievent_stop_t		  nis;
+		isc__netievent_udplisten_t	  niul;
+		isc__netievent_udpsend_t	  nius;
+} isc__netievent_storage_t;
+
+/*
+ * Network manager
+ */
+#define NM_MAGIC                        ISC_MAGIC('N', 'E', 'T', 'M')
+#define VALID_NM(t)                     ISC_MAGIC_VALID(t, NM_MAGIC)
+
+struct isc_nm {
+	int			magic;
+	isc_refcount_t		references;
+	isc_mem_t		*mctx;
+	uint32_t		nworkers;
+	isc_mutex_t		lock;
+	isc_condition_t		wkstatecond;
+	isc__networker_t	*workers;
+	atomic_uint_fast32_t	workers_running;
+	atomic_uint_fast32_t	workers_paused;
+	atomic_uint_fast32_t	maxudp;
+	atomic_bool		paused;
+
+	/*
+	 * A worker is actively waiting for other workers, for example to
+	 * stop listening; that means no other thread can do the same thing
+	 * or pause, or we'll deadlock. We have to either re-enqueue our
+	 * event or wait for the other one to finish if we want to pause.
+	 */
+	atomic_bool		interlocked;
+};
+
+typedef enum isc_nmsocket_type {
+	isc_nm_udpsocket,
+	isc_nm_udplistener, /* Aggregate of nm_udpsocks */
+	isc_nm_tcpsocket,
+	isc_nm_tcplistener,
+	isc_nm_tcpdnslistener,
+	isc_nm_tcpdnssocket
+} isc_nmsocket_type;
+
+/*%
+ * A universal structure for either a single socket or a group of
+ * dup'd/SO_REUSE_PORT-using sockets listening on the same interface.
+ */
+#define NMSOCK_MAGIC                    ISC_MAGIC('N', 'M', 'S', 'K')
+#define VALID_NMSOCK(t)                 ISC_MAGIC_VALID(t, NMSOCK_MAGIC)
+
+struct isc_nmsocket {
+	/*% Unlocked, RO */
+	int			magic;
+	int			tid;
+	isc_nmsocket_type	type;
+	isc_nm_t		*mgr;
+	isc_nmsocket_t		*parent;
+	isc_quota_t		*quota;
+	bool			overquota;
+
+	/*% outer socket is for 'wrapped' sockets - e.g. tcpdns in tcp */
+	isc_nmsocket_t		*outer;
+
+	/*% server socket for connections */
+	isc_nmsocket_t		*server;
+
+	/*% children sockets for multi-socket setups */
+	isc_nmsocket_t		*children;
+	int			nchildren;
+	isc_nmiface_t		*iface;
+	isc_nmhandle_t		*tcphandle;
+
+	/*% extra data allocated at the end of each isc_nmhandle_t */
+	size_t			extrahandlesize;
+
+	/*% libuv data */
+	uv_os_sock_t		fd;
+	union uv_any_handle	uv_handle;
+
+	isc_sockaddr_t		peer;
+
+	/* Atomic */
+	/*% Number of running (e.g. listening) children sockets */
+	atomic_int_fast32_t     rchildren;
+
+	/*%
+	 * Socket if active if it's listening, working, etc., if we're
+	 * closing a socket it doesn't make any sense to e.g. still
+	 * push handles or reqs for reuse
+	 */
+	atomic_bool        	active;
+	atomic_bool	   	destroying;
+
+	/*%
+	 * Socket is closed if it's not active and all the possible
+	 * callbacks were fired, there are no active handles, etc.
+	 * active==false, closed==false means the socket is closing.
+	 */
+	atomic_bool	      	closed;
+	atomic_bool	      	listening;
+	isc_refcount_t	      	references;
+
+	/*%
+	 * TCPDNS socket is not pipelining.
+	 */
+	atomic_bool		sequential;
+	/*%
+	 * TCPDNS socket in sequential mode is currently processing a packet,
+	 * we need to wait until it finishes.
+	 */
+	atomic_bool		processing;
+
+	/*%
+	 * 'spare' handles for that can be reused to avoid allocations,
+	 * for UDP.
+	 */
+	isc_astack_t 		*inactivehandles;
+	isc_astack_t 		*inactivereqs;
+
+	/* Used for active/rchildren during shutdown */
+	isc_mutex_t		lock;
+	isc_condition_t		cond;
+
+	/*%
+	 * List of active handles.
+	 * ah_size - size of ah_frees and ah_handles
+	 * ah_cpos - current position in ah_frees;
+	 * ah_handles - array of *handles.
+	 * Adding a handle
+	 *  - if ah_cpos == ah_size, realloc
+	 *  - x = ah_frees[ah_cpos]
+	 *  - ah_frees[ah_cpos++] = 0;
+	 *  - ah_handles[x] = handle
+	 *  - x must be stored with the handle!
+	 * Removing a handle:
+	 *  - ah_frees[--ah_cpos] = x
+	 *  - ah_handles[x] = NULL;
+	 *
+	 * XXXWPK for now this is locked with socket->lock, but we might want
+	 * to change it to something lockless
+	 */
+	size_t			ah_size;
+	size_t			ah_cpos;
+	size_t			*ah_frees;
+	isc_nmhandle_t		**ah_handles;
+
+	/* Buffer for TCPDNS processing, optional */
+	size_t			buf_size;
+	size_t			buf_len;
+	unsigned char		*buf;
+
+	isc__nm_readcb_t	rcb;
+	void			*rcbarg;
+};
+
+bool
+isc__nm_in_netthread(void);
+/*%
+ * Returns 'true' if we're in the network thread.
+ */
+
+void *
+isc__nm_get_ievent(isc_nm_t *mgr, isc__netievent_type type);
+/*%<
+ * Allocate an ievent and set the type.
+ */
+
+void
+isc__nm_enqueue_ievent(isc__networker_t *worker, isc__netievent_t *event);
+/*%<
+ * Enqueue an ievent onto a specific worker queue. (This the only safe
+ * way to use an isc__networker_t from another thread.)
+ */
+
+void
+isc__nm_alloc_cb(uv_handle_t *handle, size_t size, uv_buf_t *buf);
+/*%<
+ * Allocator for recv operations.
+ *
+ * Note that as currently implemented, this doesn't actually
+ * allocate anything, it just assigns the the isc__networker's UDP
+ * receive buffer to a socket, and marks it as "in use".
+ */
+
+void
+isc__nm_free_uvbuf(isc_nmsocket_t *sock, const uv_buf_t *buf);
+/*%<
+ * Free a buffer allocated for a receive operation.
+ *
+ * Note that as currently implemented, this doesn't actually
+ * free anything, marks the isc__networker's UDP receive buffer
+ * as "not in use".
+ */
+
+
+isc_nmhandle_t *
+isc__nmhandle_get(isc_nmsocket_t *sock, isc_sockaddr_t *peer,
+		  isc_sockaddr_t *local);
+/*%<
+ * Get a handle for the socket 'sock', allocating a new one
+ * if there isn't one availbale in 'sock->inactivehandles'.
+ *
+ * If 'peer' is not NULL, set the handle's peer address to 'peer',
+ * otherwise set it to 'sock->peer'.
+ *
+ * If 'local' is not NULL, set the handle's local address to 'local',
+ * otherwise set it to 'sock->iface->addr'.
+ */
+
+isc__nm_uvreq_t *
+isc__nm_uvreq_get(isc_nm_t *mgr, isc_nmsocket_t *sock);
+/*%<
+ * Get a UV request structure for the socket 'sock', allocating a
+ * new one if there isn't one availbale in 'sock->inactivereqs'.
+ */
+
+void
+isc__nm_uvreq_put(isc__nm_uvreq_t **req, isc_nmsocket_t *sock);
+/*%<
+ * Completes the use of a UV request structure, setting '*req' to NULL.
+ *
+ * The UV request is pushed onto the 'sock->inactivereqs' stack or,
+ * if that doesn't work, freed.
+ */
+
+void
+isc__nmsocket_init(isc_nmsocket_t *sock, isc_nm_t *mgr,
+		   isc_nmsocket_type type);
+/*%<
+ * Initialize socket 'sock', attach it to 'mgr', and set it to type 'type'.
+ */
+
+void
+isc__nmsocket_prep_destroy(isc_nmsocket_t *sock);
+/*%<
+ * Market 'sock' as inactive, close it if necessary, and destroy it
+ * if there are no remaining references or active handles.
+ */
+
+isc_result_t
+isc__nm_udp_send(isc_nmhandle_t *handle, isc_region_t *region,
+		 isc_nm_cb_t cb, void *cbarg);
+/*%<
+ * Back-end implemenation of isc_nm_send() for UDP handles.
+ */
+
+void
+isc__nm_async_udplisten(isc__networker_t *worker, isc__netievent_t *ievent0);
+
+void
+isc__nm_async_udpstoplisten(isc__networker_t *worker,
+			    isc__netievent_t *ievent0);
+void
+isc__nm_async_udpsend(isc__networker_t *worker, isc__netievent_t *ievent0);
+/*%<
+ * Callback handlers for asynchronous UDP events (listen, stoplisten, send).
+ */
+
+isc_result_t
+isc__nm_tcp_send(isc_nmhandle_t *handle, isc_region_t *region,
+		 isc_nm_cb_t cb, void *cbarg);
+/*%<
+ * Back-end implemenation of isc_nm_send() for TCP handles.
+ */
+
+void
+isc__nm_tcp_close(isc_nmsocket_t *sock);
+/*%<
+ * Close a TCP socket.
+ */
+
+void
+isc__nm_async_tcpconnect(isc__networker_t *worker, isc__netievent_t *ievent0);
+void
+isc__nm_async_tcplisten(isc__networker_t *worker, isc__netievent_t *ievent0);
+void
+isc__nm_async_tcpstoplisten(isc__networker_t *worker,
+			    isc__netievent_t *ievent0);
+void
+isc__nm_async_tcpsend(isc__networker_t *worker, isc__netievent_t *ievent0);
+void
+isc__nm_async_startread(isc__networker_t *worker, isc__netievent_t *ievent0);
+void
+isc__nm_async_pauseread(isc__networker_t *worker, isc__netievent_t *ievent0);
+void
+isc__nm_async_resumeread(isc__networker_t *worker, isc__netievent_t *ievent0);
+void
+isc__nm_async_tcpclose(isc__networker_t *worker, isc__netievent_t *ievent0);
+/*%<
+ * Callback handlers for asynchronous TCP events (connect, listen,
+ * stoplisten, send, read, pauseread, resumeread, close).
+ */
+
+
+isc_result_t
+isc__nm_tcpdns_send(isc_nmhandle_t *handle, isc_region_t *region,
+		    isc_nm_cb_t cb, void *cbarg);
+/*%<
+ * Back-end implemenation of isc_nm_send() for TCPDNS handles.
+ */
+
+void
+isc__nm_tcpdns_close(isc_nmsocket_t *sock);
+/*%<
+ * Close a TCPDNS socket.
+ */
+
+#define isc__nm_uverr2result(x) \
+	isc___nm_uverr2result(x, true, __FILE__, __LINE__)
+isc_result_t
+isc___nm_uverr2result(int uverr, bool dolog,
+		      const char *file, unsigned int line);
+/*%<
+ * Convert a libuv error value into an isc_result_t.  The
+ * list of supported error values is not complete; new users
+ * of this function should add any expected errors that are
+ * not already there.
+ */
+
+bool
+isc__nm_acquire_interlocked(isc_nm_t *mgr);
+
+void
+isc__nm_drop_interlocked(isc_nm_t *mgr);
+
+void
+isc__nm_acquire_interlocked_force(isc_nm_t *mgr);

lib/isc/netmgr/tcp.c

@@ -0,0 +1,609 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+#include <unistd.h>
+#include <uv.h>
+
+#include <isc/atomic.h>
+#include <isc/buffer.h>
+#include <isc/condition.h>
+#include <isc/magic.h>
+#include <isc/mem.h>
+#include <isc/netmgr.h>
+#include <isc/quota.h>
+#include <isc/random.h>
+#include <isc/refcount.h>
+#include <isc/region.h>
+#include <isc/result.h>
+#include <isc/sockaddr.h>
+#include <isc/thread.h>
+#include <isc/util.h>
+
+#include "netmgr-int.h"
+
+static int
+tcp_connect_direct(isc_nmsocket_t *sock, isc__nm_uvreq_t *req);
+
+static void
+tcp_close_direct(isc_nmsocket_t *sock);
+
+static isc_result_t
+tcp_send_direct(isc_nmsocket_t *sock, isc__nm_uvreq_t *req);
+static void
+tcp_connect_cb(uv_connect_t *uvreq, int status);
+
+static void
+tcp_connection_cb(uv_stream_t *server, int status);
+
+static void
+read_cb(uv_stream_t *stream, ssize_t nread, const uv_buf_t *buf);
+
+static void
+tcp_close_cb(uv_handle_t *uvhandle);
+
+static int
+tcp_connect_direct(isc_nmsocket_t *sock, isc__nm_uvreq_t *req) {
+	isc__networker_t *worker;
+	int r;
+
+	REQUIRE(isc_nm_tid() >= 0);
+
+	worker = &sock->mgr->workers[isc_nm_tid()];
+
+	r = uv_tcp_init(&worker->loop, &sock->uv_handle.tcp);
+	if (r != 0) {
+		return (r);
+	}
+
+	if (req->local.length != 0) {
+		r = uv_tcp_bind(&sock->uv_handle.tcp, &req->local.type.sa, 0);
+		if (r != 0) {
+			tcp_close_direct(sock);
+			return (r);
+		}
+	}
+
+	r = uv_tcp_connect(&req->uv_req.connect, &sock->uv_handle.tcp,
+			   &req->peer.type.sa, tcp_connect_cb);
+	return (r);
+}
+
+void
+isc__nm_async_tcpconnect(isc__networker_t *worker, isc__netievent_t *ievent0) {
+	isc__netievent_tcpconnect_t *ievent =
+		(isc__netievent_tcpconnect_t *) ievent0;
+	isc_nmsocket_t *sock = ievent->sock;
+	isc__nm_uvreq_t *req = ievent->req;
+	int r;
+
+	REQUIRE(sock->type == isc_nm_tcpsocket);
+	REQUIRE(worker->id == ievent->req->sock->mgr->workers[isc_nm_tid()].id);
+
+	r = tcp_connect_direct(sock, req);
+	if (r != 0) {
+		/* We need to issue callbacks ourselves */
+		tcp_connect_cb(&req->uv_req.connect, r);
+	}
+}
+
+static void
+tcp_connect_cb(uv_connect_t *uvreq, int status) {
+	isc__nm_uvreq_t *req = (isc__nm_uvreq_t *) uvreq->data;
+	isc_nmsocket_t *sock = uvreq->handle->data;
+
+	REQUIRE(VALID_UVREQ(req));
+
+	if (status == 0) {
+		isc_result_t result;
+		isc_nmhandle_t *handle = NULL;
+		struct sockaddr_storage ss;
+
+		uv_tcp_getpeername(&sock->uv_handle.tcp,
+				   (struct sockaddr *) &ss,
+				   &(int){sizeof(ss)});
+		result = isc_sockaddr_fromsockaddr(&sock->peer,
+						   (struct sockaddr *) &ss);
+		RUNTIME_CHECK(result == ISC_R_SUCCESS);
+
+		handle = isc__nmhandle_get(sock, NULL, NULL);
+		req->cb.connect(handle, ISC_R_SUCCESS, req->cbarg);
+	} else {
+		/* TODO handle it properly, free sock, translate code */
+		req->cb.connect(NULL, ISC_R_FAILURE, req->cbarg);
+	}
+
+	isc__nm_uvreq_put(&req, sock);
+}
+
+isc_result_t
+isc_nm_listentcp(isc_nm_t *mgr, isc_nmiface_t *iface,
+		 isc_nm_cb_t cb, void *cbarg,
+		 size_t extrahandlesize, isc_quota_t *quota,
+		 isc_nmsocket_t **rv)
+{
+	isc__netievent_tcplisten_t *ievent = NULL;
+	isc_nmsocket_t *nsock = NULL;
+
+	REQUIRE(VALID_NM(mgr));
+
+	nsock = isc_mem_get(mgr->mctx, sizeof(*nsock));
+	isc__nmsocket_init(nsock, mgr, isc_nm_tcplistener);
+	nsock->iface = iface;
+	nsock->rcb.accept = cb;
+	nsock->rcbarg = cbarg;
+	nsock->extrahandlesize = extrahandlesize;
+	if (quota != NULL) {
+		/*
+		 * We need to force it to make sure we get it attached.
+		 * An example failure mode would be server under attack
+		 * reconfiguring interfaces - that might cause weak attach
+		 * to fail and leave this listening socket without limits.
+		 * We can ignore the result.
+		 */
+		isc_quota_force(quota, &nsock->quota);
+	}
+	nsock->tid = isc_random_uniform(mgr->nworkers);
+
+	/*
+	 * Listening to TCP is rare enough not to care about the
+	 * added overhead from passing this to another thread.
+	 */
+	ievent = isc__nm_get_ievent(mgr, netievent_tcplisten);
+	ievent->sock = nsock;
+	isc__nm_enqueue_ievent(&mgr->workers[nsock->tid],
+			       (isc__netievent_t *) ievent);
+	*rv = nsock;
+
+	return (ISC_R_SUCCESS);
+}
+
+void
+isc__nm_async_tcplisten(isc__networker_t *worker, isc__netievent_t *ievent0) {
+	isc__netievent_tcplisten_t *ievent =
+		(isc__netievent_tcplisten_t *) ievent0;
+	isc_nmsocket_t *sock = ievent->sock;
+	int r;
+
+	REQUIRE(isc_nm_tid() >= 0);
+	REQUIRE(sock->type == isc_nm_tcplistener);
+
+	r = uv_tcp_init(&worker->loop, &sock->uv_handle.tcp);
+	if (r != 0) {
+		return;
+	}
+
+	uv_tcp_bind(&sock->uv_handle.tcp, &sock->iface->addr.type.sa, 0);
+	r = uv_listen((uv_stream_t *) &sock->uv_handle.tcp,
+		      10,
+		      tcp_connection_cb);
+	if (r != 0) {
+		return;
+	}
+	atomic_store(&sock->listening, true);
+	return;
+	/* issue callback? */
+}
+
+void
+isc_nm_tcp_stoplistening(isc_nmsocket_t *sock) {
+	isc__netievent_tcpstoplisten_t *ievent = NULL;
+
+	REQUIRE(VALID_NMSOCK(sock));
+	REQUIRE(!isc__nm_in_netthread());
+
+	ievent = isc__nm_get_ievent(sock->mgr, netievent_tcpstoplisten);
+	isc_nmsocket_attach(sock, &ievent->sock);
+	isc__nm_enqueue_ievent(&sock->mgr->workers[sock->tid],
+			       (isc__netievent_t *) ievent);
+}
+
+static void
+stoplistening_cb(uv_handle_t *handle) {
+	isc_nmsocket_t *sock = handle->data;
+
+	LOCK(&sock->lock);
+	atomic_store(&sock->listening, false);
+	atomic_store(&sock->closed, true);
+	SIGNAL(&sock->cond);
+	UNLOCK(&sock->lock);
+
+	if (sock->quota != NULL) {
+		isc_quota_detach(&sock->quota);
+	}
+
+	isc_nmsocket_detach(&sock);
+}
+
+void
+isc__nm_async_tcpstoplisten(isc__networker_t *worker,
+			    isc__netievent_t *ievent0)
+{
+	isc__netievent_tcpstoplisten_t *ievent =
+		(isc__netievent_tcpstoplisten_t *) ievent0;
+	isc_nmsocket_t *sock = ievent->sock;
+
+	UNUSED(worker);
+
+	REQUIRE(isc_nm_tid() >= 0);
+	REQUIRE(VALID_NMSOCK(sock));
+	REQUIRE(sock->type == isc_nm_tcplistener);
+
+	uv_close(&sock->uv_handle.handle, stoplistening_cb);
+}
+
+isc_result_t
+isc_nm_read(isc_nmhandle_t *handle, isc_nm_recv_cb_t cb, void *cbarg) {
+	isc_nmsocket_t *sock = NULL;
+
+	REQUIRE(VALID_NMHANDLE(handle));
+	REQUIRE(VALID_NMSOCK(handle->sock));
+
+	sock = handle->sock;
+	sock->rcb.recv = cb;
+	sock->rcbarg = cbarg; /* That's obviously broken... */
+	if (sock->tid == isc_nm_tid()) {
+		int r = uv_read_start(&sock->uv_handle.stream,
+				      isc__nm_alloc_cb, read_cb);
+		INSIST(r == 0);
+	} else {
+		isc__netievent_startread_t *ievent =
+			isc__nm_get_ievent(sock->mgr,
+					   netievent_tcpstartread);
+		ievent->sock = sock;
+		isc__nm_enqueue_ievent(&sock->mgr->workers[sock->tid],
+				       (isc__netievent_t *) ievent);
+	}
+
+	return (ISC_R_SUCCESS);
+}
+
+void
+isc__nm_async_startread(isc__networker_t *worker, isc__netievent_t *ievent0) {
+	isc__netievent_startread_t *ievent =
+		(isc__netievent_startread_t *) ievent0;
+	isc_nmsocket_t *sock = ievent->sock;
+
+	REQUIRE(worker->id == isc_nm_tid());
+
+	uv_read_start(&sock->uv_handle.stream, isc__nm_alloc_cb, read_cb);
+}
+
+isc_result_t
+isc_nm_pauseread(isc_nmsocket_t *sock) {
+	REQUIRE(VALID_NMSOCK(sock));
+
+	if (sock->tid == isc_nm_tid()) {
+		int r = uv_read_stop(&sock->uv_handle.stream);
+		INSIST(r == 0);
+	} else {
+		isc__netievent_pauseread_t *ievent =
+			isc__nm_get_ievent(sock->mgr,
+					   netievent_tcppauseread);
+		ievent->sock = sock;
+		isc__nm_enqueue_ievent(&sock->mgr->workers[sock->tid],
+				       (isc__netievent_t *) ievent);
+	}
+
+	return (ISC_R_SUCCESS);
+}
+
+void
+isc__nm_async_pauseread(isc__networker_t *worker, isc__netievent_t *ievent0) {
+	isc__netievent_pauseread_t *ievent =
+		(isc__netievent_pauseread_t *) ievent0;
+	isc_nmsocket_t *sock = ievent->sock;
+	REQUIRE(VALID_NMSOCK(sock));
+
+	REQUIRE(worker->id == isc_nm_tid());
+
+	uv_read_stop(&sock->uv_handle.stream);
+}
+
+isc_result_t
+isc_nm_resumeread(isc_nmsocket_t *sock) {
+	REQUIRE(VALID_NMSOCK(sock));
+	REQUIRE(sock->rcb.recv != NULL);
+
+	if (sock->tid == isc_nm_tid()) {
+		int r = uv_read_start(&sock->uv_handle.stream,
+				      isc__nm_alloc_cb, read_cb);
+		INSIST(r == 0);
+	} else {
+		/* It's the same as startread */
+		isc__netievent_startread_t *ievent =
+			isc__nm_get_ievent(sock->mgr,
+					   netievent_tcpstartread);
+		ievent->sock = sock;
+		isc__nm_enqueue_ievent(&sock->mgr->workers[sock->tid],
+				       (isc__netievent_t *) ievent);
+	}
+
+	return (ISC_R_SUCCESS);
+}
+
+static void
+read_cb(uv_stream_t *stream, ssize_t nread, const uv_buf_t *buf) {
+	isc_nmsocket_t *sock = stream->data;
+
+	REQUIRE(VALID_NMSOCK(sock));
+	REQUIRE(buf != NULL);
+
+	if (nread >= 0) {
+		isc_region_t region = {
+			.base = (unsigned char *) buf->base,
+			.length = nread
+		};
+
+		INSIST(sock->rcb.recv != NULL);
+		sock->rcb.recv(sock->tcphandle, &region, sock->rcbarg);
+		isc__nm_free_uvbuf(sock, buf);
+		return;
+	}
+
+	isc__nm_free_uvbuf(sock, buf);
+	sock->rcb.recv(sock->tcphandle, NULL, sock->rcbarg);
+
+	/*
+	 * XXXWPK TODO clean up handles, close the connection,
+	 * reclaim quota
+	 */
+
+}
+
+static isc_result_t
+accept_connection(isc_nmsocket_t *ssock) {
+	isc_result_t result;
+	isc_quota_t *quota = NULL;
+	isc_nmsocket_t *csock = NULL;
+	isc__networker_t *worker = NULL;
+	isc_nmhandle_t *handle = NULL;
+	struct sockaddr_storage ss;
+	isc_sockaddr_t local;
+	int r;
+
+	REQUIRE(VALID_NMSOCK(ssock));
+	REQUIRE(ssock->tid == isc_nm_tid());
+
+	if (!atomic_load_relaxed(&ssock->active)) {
+		/* We're closing, bail */
+		return (ISC_R_CANCELED);
+	}
+
+	if (ssock->quota != NULL) {
+		result = isc_quota_attach(ssock->quota, &quota);
+		if (result != ISC_R_SUCCESS) {
+			return (result);
+		}
+	}
+
+	csock = isc_mem_get(ssock->mgr->mctx, sizeof(isc_nmsocket_t));
+	isc__nmsocket_init(csock, ssock->mgr, isc_nm_tcpsocket);
+	csock->tid = isc_nm_tid();
+	csock->extrahandlesize = ssock->extrahandlesize;
+	csock->iface = ssock->iface;
+	csock->quota = quota;
+	quota = NULL;
+
+	worker = &ssock->mgr->workers[isc_nm_tid()];
+	uv_tcp_init(&worker->loop, &csock->uv_handle.tcp);
+
+	r = uv_accept(&ssock->uv_handle.stream, &csock->uv_handle.stream);
+	if (r != 0) {
+		if (csock->quota != NULL) {
+			isc_quota_detach(&csock->quota);
+		}
+		isc_mem_put(ssock->mgr->mctx, csock, sizeof(isc_nmsocket_t));
+
+		return (isc__nm_uverr2result(r));
+	}
+
+	isc_nmsocket_attach(ssock, &csock->server);
+
+	uv_tcp_getpeername(&csock->uv_handle.tcp, (struct sockaddr *) &ss,
+			   &(int){sizeof(ss)});
+
+	result = isc_sockaddr_fromsockaddr(&csock->peer,
+					   (struct sockaddr *) &ss);
+	RUNTIME_CHECK(result == ISC_R_SUCCESS);
+	uv_tcp_getsockname(&csock->uv_handle.tcp, (struct sockaddr *) &ss,
+			   &(int){sizeof(ss)});
+	result = isc_sockaddr_fromsockaddr(&local,
+					   (struct sockaddr *) &ss);
+	RUNTIME_CHECK(result == ISC_R_SUCCESS);
+
+	handle = isc__nmhandle_get(csock, NULL, &local);
+
+	INSIST(ssock->rcb.accept != NULL);
+	ssock->rcb.accept(handle, ISC_R_SUCCESS, ssock->rcbarg);
+	isc_nmsocket_detach(&csock);
+
+	return (ISC_R_SUCCESS);
+}
+
+static void
+tcp_connection_cb(uv_stream_t *server, int status) {
+	isc_nmsocket_t *ssock = server->data;
+	isc_result_t result = accept_connection(ssock);
+
+	UNUSED(status);
+
+	if (result != ISC_R_SUCCESS) {
+		if (result == ISC_R_QUOTA || result == ISC_R_SOFTQUOTA) {
+			ssock->overquota = true;
+		}
+		/* XXXWPK TODO LOG */
+	}
+}
+
+/*
+ * isc__nm_tcp_send sends buf to a peer on a socket.
+ */
+isc_result_t
+isc__nm_tcp_send(isc_nmhandle_t *handle, isc_region_t *region,
+		 isc_nm_cb_t cb, void *cbarg)
+{
+	isc_nmsocket_t *sock = handle->sock;
+	isc__netievent_tcpsend_t *ievent = NULL;
+	isc__nm_uvreq_t *uvreq = NULL;
+
+	REQUIRE(sock->type == isc_nm_tcpsocket);
+
+	uvreq = isc__nm_uvreq_get(sock->mgr, sock);
+	uvreq->uvbuf.base = (char *) region->base;
+	uvreq->uvbuf.len = region->length;
+	uvreq->handle = handle;
+	isc_nmhandle_ref(uvreq->handle);
+	uvreq->cb.send = cb;
+	uvreq->cbarg = cbarg;
+
+	if (sock->tid == isc_nm_tid()) {
+		/*
+		 * If we're in the same thread as the socket we can send the
+		 * data directly
+		 */
+		return (tcp_send_direct(sock, uvreq));
+	} else {
+		/*
+		 * We need to create an event and pass it using async channel
+		 */
+		ievent = isc__nm_get_ievent(sock->mgr, netievent_tcpsend);
+		ievent->sock = sock;
+		ievent->req = uvreq;
+		isc__nm_enqueue_ievent(&sock->mgr->workers[sock->tid],
+				       (isc__netievent_t *) ievent);
+		return (ISC_R_SUCCESS);
+	}
+
+	return (ISC_R_UNEXPECTED);
+}
+
+static void
+tcp_send_cb(uv_write_t *req, int status) {
+	isc_result_t result = ISC_R_SUCCESS;
+	isc__nm_uvreq_t *uvreq = (isc__nm_uvreq_t *) req->data;
+
+	REQUIRE(VALID_UVREQ(uvreq));
+	REQUIRE(VALID_NMHANDLE(uvreq->handle));
+
+	if (status < 0) {
+		result = isc__nm_uverr2result(status);
+	}
+
+	uvreq->cb.send(uvreq->handle, result, uvreq->cbarg);
+	isc_nmhandle_unref(uvreq->handle);
+	isc__nm_uvreq_put(&uvreq, uvreq->handle->sock);
+}
+
+/*
+ * Handle 'tcpsend' async event - send a packet on the socket
+ */
+void
+isc__nm_async_tcpsend(isc__networker_t *worker, isc__netievent_t *ievent0) {
+	isc_result_t result;
+	isc__netievent_tcpsend_t *ievent = (isc__netievent_tcpsend_t *) ievent0;
+
+	REQUIRE(worker->id == ievent->sock->tid);
+
+	if (!atomic_load(&ievent->sock->active)) {
+		return;
+	}
+
+	result = tcp_send_direct(ievent->sock, ievent->req);
+	if (result != ISC_R_SUCCESS) {
+		ievent->req->cb.send(ievent->req->handle,
+				     result, ievent->req->cbarg);
+		isc__nm_uvreq_put(&ievent->req, ievent->req->handle->sock);
+	}
+}
+
+static isc_result_t
+tcp_send_direct(isc_nmsocket_t *sock, isc__nm_uvreq_t *req) {
+	int r;
+
+	REQUIRE(sock->tid == isc_nm_tid());
+	REQUIRE(sock->type == isc_nm_tcpsocket);
+
+	isc_nmhandle_ref(req->handle);
+	r = uv_write(&req->uv_req.write, &sock->uv_handle.stream,
+		     &req->uvbuf, 1, tcp_send_cb);
+	if (r < 0) {
+		req->cb.send(NULL, isc__nm_uverr2result(r), req->cbarg);
+		isc__nm_uvreq_put(&req, sock);
+		return (isc__nm_uverr2result(r));
+	}
+
+	return (ISC_R_SUCCESS);
+}
+
+static void
+tcp_close_cb(uv_handle_t *uvhandle) {
+	isc_nmsocket_t *sock = uvhandle->data;
+
+	REQUIRE(VALID_NMSOCK(sock));
+
+	atomic_store(&sock->closed, true);
+	isc__nmsocket_prep_destroy(sock);
+}
+
+static void
+tcp_close_direct(isc_nmsocket_t *sock) {
+	REQUIRE(VALID_NMSOCK(sock));
+	REQUIRE(sock->tid == isc_nm_tid());
+	REQUIRE(sock->type == isc_nm_tcpsocket);
+
+	if (sock->quota != NULL) {
+		isc_nmsocket_t *ssock = sock->server;
+
+		isc_quota_detach(&sock->quota);
+
+		if (ssock->overquota) {
+			/* XXXWPK TODO we should loop here */
+			isc_result_t result = accept_connection(ssock);
+			if (result != ISC_R_QUOTA && result != ISC_R_SOFTQUOTA)
+			{
+				ssock->overquota = false;
+			}
+		}
+	}
+
+	isc_nmsocket_detach(&sock->server);
+	uv_close(&sock->uv_handle.handle, tcp_close_cb);
+}
+
+void
+isc__nm_tcp_close(isc_nmsocket_t *sock) {
+	REQUIRE(VALID_NMSOCK(sock));
+	REQUIRE(sock->type == isc_nm_tcpsocket);
+
+	if (sock->tid == isc_nm_tid()) {
+		tcp_close_direct(sock);
+	} else {
+		/*
+		 * We need to create an event and pass it using async channel
+		 */
+		isc__netievent_tcpclose_t *ievent =
+			isc__nm_get_ievent(sock->mgr, netievent_tcpclose);
+
+		ievent->sock = sock;
+		isc__nm_enqueue_ievent(&sock->mgr->workers[sock->tid],
+				       (isc__netievent_t *) ievent);
+	}
+}
+
+void
+isc__nm_async_tcpclose(isc__networker_t *worker, isc__netievent_t *ievent0) {
+	isc__netievent_tcpclose_t *ievent =
+		(isc__netievent_tcpclose_t *) ievent0;
+
+	REQUIRE(worker->id == ievent->sock->tid);
+
+	tcp_close_direct(ievent->sock);
+}

lib/isc/netmgr/tcpdns.c

@@ -0,0 +1,403 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+#include <unistd.h>
+#include <uv.h>
+
+#include <isc/atomic.h>
+#include <isc/buffer.h>
+#include <isc/condition.h>
+#include <isc/magic.h>
+#include <isc/mem.h>
+#include <isc/netmgr.h>
+#include <isc/random.h>
+#include <isc/refcount.h>
+#include <isc/region.h>
+#include <isc/result.h>
+#include <isc/sockaddr.h>
+#include <isc/thread.h>
+#include <isc/util.h>
+
+#include "netmgr-int.h"
+
+static void
+dnslisten_readcb(isc_nmhandle_t *handle, isc_region_t *region, void *arg);
+
+static inline size_t
+dnslen(unsigned char* base) {
+	return ((base[0] << 8) + (base[1]));
+}
+
+#define NM_REG_BUF 4096
+#define NM_BIG_BUF 65536
+static inline void
+alloc_dnsbuf(isc_nmsocket_t *sock, size_t len) {
+	REQUIRE(len <= NM_BIG_BUF);
+
+	if (sock->buf == NULL) {
+		/* We don't have the buffer at all */
+		size_t alloc_len = len < NM_REG_BUF ? NM_REG_BUF : NM_BIG_BUF;
+		sock->buf = isc_mem_get(sock->mgr->mctx, alloc_len);
+		sock->buf_size = alloc_len;
+	} else {
+		/* We have the buffer but it's too small */
+		sock->buf = isc_mem_reallocate(sock->mgr->mctx, sock->buf,
+					       NM_BIG_BUF);
+		sock->buf_size = NM_BIG_BUF;
+	}
+}
+
+
+/*
+ * Accept callback for TCP-DNS connection
+ */
+static void
+dnslisten_acceptcb(isc_nmhandle_t *handle, isc_result_t result, void *cbarg) {
+	isc_nmsocket_t *dnslistensock = (isc_nmsocket_t *) cbarg;
+	isc_nmsocket_t *dnssock = NULL;
+
+	REQUIRE(VALID_NMSOCK(dnslistensock));
+	REQUIRE(dnslistensock->type == isc_nm_tcpdnslistener);
+
+	/* If accept() was unnsuccessful we can't do anything */
+	if (result != ISC_R_SUCCESS) {
+		return;
+	}
+
+	/* We need to create a 'wrapper' dnssocket for this connection */
+	dnssock = isc_mem_get(handle->sock->mgr->mctx, sizeof(*dnssock));
+	isc__nmsocket_init(dnssock, handle->sock->mgr,
+			   isc_nm_tcpdnssocket);
+
+	/* We need to copy read callbacks from outer socket */
+	dnssock->rcb.recv = dnslistensock->rcb.recv;
+	dnssock->rcbarg = dnslistensock->rcbarg;
+	dnssock->extrahandlesize = dnslistensock->extrahandlesize;
+	isc_nmsocket_attach(handle->sock, &dnssock->outer);
+	dnssock->peer = handle->sock->peer;
+	dnssock->iface = handle->sock->iface;
+
+	isc_nm_read(handle, dnslisten_readcb, dnssock);
+}
+
+/*
+ * We've got a read on our underlying socket, need to check if we have
+ * a complete DNS packet and, if so - call the callback
+ */
+static void
+dnslisten_readcb(isc_nmhandle_t *handle, isc_region_t *region, void *arg) {
+	isc_nmsocket_t *dnssock = (isc_nmsocket_t *) arg;
+	isc_sockaddr_t local;
+	unsigned char *base = NULL;
+	size_t len;
+
+	REQUIRE(VALID_NMSOCK(dnssock));
+	REQUIRE(VALID_NMHANDLE(handle));
+
+	if (region == NULL) {
+		/* Connection closed */
+		atomic_store(&dnssock->closed, true);
+		isc_nmsocket_detach(&dnssock->outer);
+		isc_nmsocket_detach(&dnssock);
+		return;
+	}
+
+	local = isc_nmhandle_localaddr(handle);
+
+	base = region->base;
+	len = region->length;
+
+	/*
+	 * We have something in the buffer, we need to glue it.
+	 */
+	if (dnssock->buf_len > 0) {
+		size_t plen;
+
+		if (dnssock->buf_len == 1) {
+			/* Make sure we have the length */
+			dnssock->buf[1] = base[0];
+			dnssock->buf_len = 2;
+			base++;
+			len--;
+		}
+
+		/* At this point we definitely have 2 bytes there. */
+		plen = ISC_MIN(len, (dnslen(dnssock->buf) + 2 -
+				     dnssock->buf_len));
+		if (plen > dnssock->buf_size) {
+			alloc_dnsbuf(dnssock, plen);
+		}
+
+		memmove(dnssock->buf + dnssock->buf_len, base, plen);
+		dnssock->buf_len += plen;
+		base += plen;
+		len -= plen;
+
+		/* Do we have a complete packet in the buffer? */
+		if (dnslen(dnssock->buf) == dnssock->buf_len - 2) {
+			isc_nmhandle_t *dnshandle = NULL;
+			isc_region_t r2 = {
+				.base = dnssock->buf + 2,
+				.length = dnslen(dnssock->buf)
+			};
+			dnshandle = isc__nmhandle_get(dnssock, NULL, &local);
+			atomic_store(&dnssock->processing, true);
+			dnssock->rcb.recv(dnshandle, &r2, dnssock->rcbarg);
+			dnssock->buf_len = 0;
+
+			/*
+			 * If the recv callback wants to hold on to the
+			 * handle, it needs to attach to it.
+			 */
+			isc_nmhandle_unref(dnshandle);
+		}
+	}
+
+	/*
+	 * At this point we've processed whatever was previously in the
+	 * socket buffer. If there are more messages to be found in what
+	 * we've read, and if we're either pipelining or not processing
+	 * anything else, then we can process those messages now.
+	 */
+	while (len >= 2 && dnslen(base) <= len - 2 &&
+	       !(atomic_load(&dnssock->sequential) &&
+		 atomic_load(&dnssock->processing)))
+	{
+		isc_nmhandle_t *dnshandle = NULL;
+		isc_region_t r2 = {
+			.base = base + 2,
+			.length = dnslen(base)
+		};
+
+		len -= dnslen(base) + 2;
+		base += dnslen(base) + 2;
+
+		dnshandle = isc__nmhandle_get(dnssock, NULL, &local);
+		atomic_store(&dnssock->processing, true);
+		dnssock->rcb.recv(dnshandle, &r2, dnssock->rcbarg);
+
+		/*
+		 * If the recv callback wants to hold on to the
+		 * handle, it needs to attach to it.
+		 */
+		isc_nmhandle_unref(dnshandle);
+	}
+
+	/*
+	 * We have less than a full message remaining; it can be
+	 * stored in the socket buffer for next time.
+	 */
+	if (len > 0) {
+		if (len > dnssock->buf_size) {
+			alloc_dnsbuf(dnssock, len);
+		}
+
+		INSIST(len <= dnssock->buf_size);
+		memmove(dnssock->buf, base, len);
+		dnssock->buf_len = len;
+	}
+}
+
+/* Process all complete packets out of incoming buffer */
+static void
+processbuffer(isc_nmsocket_t *dnssock) {
+	REQUIRE(VALID_NMSOCK(dnssock));
+
+	/* While we have a complete packet in the buffer */
+	while (dnssock->buf_len > 2 &&
+	       dnslen(dnssock->buf) <= dnssock->buf_len - 2)
+	{
+		isc_nmhandle_t *dnshandle = NULL;
+		isc_region_t r2 = {
+			.base = dnssock->buf + 2,
+			.length = dnslen(dnssock->buf)
+		};
+		size_t len;
+
+		dnshandle = isc__nmhandle_get(dnssock, NULL, NULL);
+		atomic_store(&dnssock->processing, true);
+		dnssock->rcb.recv(dnshandle, &r2, dnssock->rcbarg);
+
+		/*
+		 * If the recv callback wants to hold on to the
+		 * handle, it needs to attach to it.
+		 */
+		isc_nmhandle_unref(dnshandle);
+
+		len = dnslen(dnssock->buf) + 2;
+		dnssock->buf_len -= len;
+		if (len > 0) {
+			memmove(dnssock->buf, dnssock->buf + len,
+				dnssock->buf_len);
+		}
+
+		/* Check here to make sure we do the processing at least once */
+		if (atomic_load(&dnssock->processing)) {
+			return;
+		}
+	}
+}
+
+/*
+ * isc_nm_listentcpdns listens for connections and accepts
+ * them immediately, then calls the cb for each incoming DNS packet
+ * (with 2-byte length stripped) - just like for UDP packet.
+ */
+isc_result_t
+isc_nm_listentcpdns(isc_nm_t *mgr, isc_nmiface_t *iface,
+		    isc_nm_recv_cb_t cb, void *cbarg,
+		    size_t extrahandlesize, isc_quota_t *quota,
+		    isc_nmsocket_t **rv)
+{
+	/* A 'wrapper' socket object with outer set to true TCP socket */
+	isc_nmsocket_t *dnslistensock =
+		isc_mem_get(mgr->mctx, sizeof(*dnslistensock));
+	isc_result_t result;
+
+	isc__nmsocket_init(dnslistensock, mgr, isc_nm_tcpdnslistener);
+	dnslistensock->iface = iface;
+	dnslistensock->rcb.recv = cb;
+	dnslistensock->rcbarg = cbarg;
+	dnslistensock->extrahandlesize = extrahandlesize;
+
+	/* We set dnslistensock->outer to a true listening socket */
+	result = isc_nm_listentcp(mgr, iface, dnslisten_acceptcb,
+				  dnslistensock, extrahandlesize,
+				  quota, &dnslistensock->outer);
+
+	atomic_store(&dnslistensock->listening, true);
+	*rv = dnslistensock;
+	return (result);
+}
+
+void
+isc_nm_tcpdns_stoplistening(isc_nmsocket_t *sock) {
+	REQUIRE(VALID_NMSOCK(sock));
+	REQUIRE(sock->type == isc_nm_tcpdnslistener);
+
+	atomic_store(&sock->listening, false);
+	atomic_store(&sock->closed, true);
+
+	if (sock->outer != NULL) {
+		isc_nm_tcp_stoplistening(sock->outer);
+		isc_nmsocket_detach(&sock->outer);
+	}
+}
+
+void
+isc_nm_tcpdns_sequential(isc_nmhandle_t *handle) {
+	REQUIRE(VALID_NMHANDLE(handle));
+
+	if (handle->sock->type != isc_nm_tcpdnssocket ||
+	    handle->sock->outer == NULL)
+	{
+		return;
+	}
+
+	/*
+	 * We don't want pipelining on this connection. That means
+	 * that we can launch query processing only when the previous
+	 * one returned.
+	 *
+	 * The socket MUST be unpaused after the query is processed.
+	 * This is done by isc_nm_resumeread() in tcpdnssend_cb() below.
+	 *
+	 * XXX: The callback is not currently executed in failure cases!
+	 */
+	isc_nm_pauseread(handle->sock->outer);
+	atomic_store(&handle->sock->sequential, true);
+}
+
+typedef struct tcpsend {
+	isc_mem_t		*mctx;
+	isc_nmhandle_t		*handle;
+	isc_region_t		region;
+	isc_nmhandle_t		*orighandle;
+	isc_nm_cb_t		cb;
+	void 			*cbarg;
+} tcpsend_t;
+
+static void
+tcpdnssend_cb(isc_nmhandle_t *handle, isc_result_t result, void *cbarg) {
+	tcpsend_t *ts = (tcpsend_t *) cbarg;
+
+	UNUSED(handle);
+
+	ts->cb(ts->orighandle, result, ts->cbarg);
+	isc_mem_put(ts->mctx, ts->region.base, ts->region.length);
+
+	/*
+	 * The response was sent, if we're in sequential mode resume
+	 * processing.
+	 */
+	if (atomic_load(&ts->orighandle->sock->sequential)) {
+		atomic_store(&ts->orighandle->sock->processing, false);
+		processbuffer(ts->orighandle->sock);
+		isc_nm_resumeread(handle->sock);
+	}
+
+	isc_nmhandle_unref(ts->orighandle);
+	isc_mem_putanddetach(&ts->mctx, ts, sizeof(*ts));
+}
+
+/*
+ * isc__nm_tcp_send sends buf to a peer on a socket.
+ */
+isc_result_t
+isc__nm_tcpdns_send(isc_nmhandle_t *handle, isc_region_t *region,
+		    isc_nm_cb_t cb, void *cbarg)
+{
+	tcpsend_t *t = NULL;
+
+	REQUIRE(VALID_NMHANDLE(handle));
+
+	isc_nmsocket_t *sock = handle->sock;
+
+	REQUIRE(VALID_NMSOCK(sock));
+	REQUIRE(sock->type == isc_nm_tcpdnssocket);
+
+	if (sock->outer == NULL) {
+		/* The socket is closed, just issue the callback */
+		cb(handle, ISC_R_FAILURE, cbarg);
+		return (ISC_R_NOTCONNECTED);
+	}
+
+	t = isc_mem_get(sock->mgr->mctx, sizeof(*t));
+	*t = (tcpsend_t) {
+		.cb = cb,
+		.cbarg = cbarg,
+		.handle = handle->sock->outer->tcphandle,
+	};
+
+	isc_mem_attach(sock->mgr->mctx, &t->mctx);
+	t->orighandle = handle;
+	isc_nmhandle_ref(t->orighandle);
+
+	t->region = (isc_region_t) {
+		.base = isc_mem_get(t->mctx, region->length + 2),
+		.length = region->length + 2
+	};
+
+	*(uint16_t *) t->region.base = htons(region->length);
+	memmove(t->region.base + 2, region->base, region->length);
+
+	return (isc__nm_tcp_send(t->handle, &t->region, tcpdnssend_cb, t));
+}
+
+void
+isc__nm_tcpdns_close(isc_nmsocket_t *sock) {
+	if (sock->outer != NULL) {
+		isc_nmsocket_detach(&sock->outer);
+	}
+
+	atomic_store(&sock->closed, true);
+	isc__nmsocket_prep_destroy(sock);
+}

lib/isc/netmgr/udp.c

@@ -0,0 +1,459 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+#include <unistd.h>
+#include <uv.h>
+
+#include <isc/atomic.h>
+#include <isc/buffer.h>
+#include <isc/condition.h>
+#include <isc/magic.h>
+#include <isc/mem.h>
+#include <isc/netmgr.h>
+#include <isc/random.h>
+#include <isc/refcount.h>
+#include <isc/region.h>
+#include <isc/result.h>
+#include <isc/sockaddr.h>
+#include <isc/thread.h>
+#include <isc/util.h>
+#include "netmgr-int.h"
+
+static isc_result_t
+udp_send_direct(isc_nmsocket_t *sock, isc__nm_uvreq_t *req,
+		isc_sockaddr_t *peer);
+
+static void
+udp_recv_cb(uv_udp_t *handle, ssize_t nrecv, const uv_buf_t *buf,
+	    const struct sockaddr *addr, unsigned flags);
+
+static void
+udp_send_cb(uv_udp_send_t *req, int status);
+
+isc_result_t
+isc_nm_listenudp(isc_nm_t *mgr, isc_nmiface_t *iface,
+		 isc_nm_recv_cb_t cb, void *cbarg,
+		 size_t extrahandlesize, isc_nmsocket_t **sockp)
+{
+	isc_nmsocket_t *nsock = NULL;
+
+	/*
+	 * We are creating mgr->nworkers duplicated sockets, one
+	 * socket for each worker thread.
+	 */
+	nsock = isc_mem_get(mgr->mctx, sizeof(isc_nmsocket_t));
+	isc__nmsocket_init(nsock, mgr, isc_nm_udplistener);
+	nsock->iface = iface;
+	nsock->nchildren = mgr->nworkers;
+	atomic_init(&nsock->rchildren, mgr->nworkers);
+	nsock->children = isc_mem_get(mgr->mctx,
+				      mgr->nworkers * sizeof(*nsock));
+	memset(nsock->children, 0, mgr->nworkers * sizeof(*nsock));
+
+	INSIST(nsock->rcb.recv == NULL && nsock->rcbarg == NULL);
+	nsock->rcb.recv = cb;
+	nsock->rcbarg = cbarg;
+	nsock->extrahandlesize = extrahandlesize;
+
+	for (size_t i = 0; i < mgr->nworkers; i++) {
+		uint16_t family = iface->addr.type.sa.sa_family;
+		int res;
+
+		isc__netievent_udplisten_t *ievent = NULL;
+		isc_nmsocket_t *csock = &nsock->children[i];
+
+		isc__nmsocket_init(csock, mgr, isc_nm_udpsocket);
+		csock->parent = nsock;
+		csock->iface = iface;
+		csock->tid = i;
+		csock->extrahandlesize = extrahandlesize;
+
+		INSIST(csock->rcb.recv == NULL && csock->rcbarg == NULL);
+		csock->rcb.recv = cb;
+		csock->rcbarg = cbarg;
+		csock->fd = socket(family, SOCK_DGRAM, 0);
+		INSIST(csock->fd >= 0);
+
+		/*
+		 * This is SO_REUSE**** hell:
+		 * On Linux SO_REUSEPORT allows multiple sockets to bind to
+		 * the same host:port pair.
+		 * On Windows the same thing is achieved with SO_REUSEADDR
+		 */
+#ifdef WIN32
+		res = setsockopt(csock->fd, SOL_SOCKET, SO_REUSEADDR,
+				 &(int){1}, sizeof(int));
+#else
+		res = setsockopt(csock->fd, SOL_SOCKET, SO_REUSEPORT,
+				 &(int){1}, sizeof(int));
+#endif
+		RUNTIME_CHECK(res == 0);
+
+		ievent = isc__nm_get_ievent(mgr, netievent_udplisten);
+		ievent->sock = csock;
+		isc__nm_enqueue_ievent(&mgr->workers[i],
+				       (isc__netievent_t *) ievent);
+	}
+
+	*sockp = nsock;
+	return (ISC_R_SUCCESS);
+}
+
+/*
+ * handle 'udplisten' async call - start listening on a socket.
+ */
+void
+isc__nm_async_udplisten(isc__networker_t *worker, isc__netievent_t *ievent0) {
+	isc__netievent_udplisten_t *ievent =
+		(isc__netievent_udplisten_t *) ievent0;
+	isc_nmsocket_t *sock = ievent->sock;
+
+	REQUIRE(sock->type == isc_nm_udpsocket);
+	REQUIRE(sock->iface != NULL);
+	REQUIRE(sock->parent != NULL);
+
+	uv_udp_init(&worker->loop, &sock->uv_handle.udp);
+	sock->uv_handle.udp.data = NULL;
+	isc_nmsocket_attach(sock,
+			    (isc_nmsocket_t **)&sock->uv_handle.udp.data);
+
+	uv_udp_open(&sock->uv_handle.udp, sock->fd);
+	uv_udp_bind(&sock->uv_handle.udp,
+		    &sock->parent->iface->addr.type.sa, 0);
+	uv_recv_buffer_size(&sock->uv_handle.handle,
+			    &(int){16 * 1024 * 1024});
+	uv_send_buffer_size(&sock->uv_handle.handle,
+			    &(int){16 * 1024 * 1024});
+	uv_udp_recv_start(&sock->uv_handle.udp, isc__nm_alloc_cb,
+			  udp_recv_cb);
+}
+
+static void
+udp_close_cb(uv_handle_t *handle) {
+	isc_nmsocket_t *sock = handle->data;
+	atomic_store(&sock->closed, true);
+
+	isc_nmsocket_detach((isc_nmsocket_t **)&sock->uv_handle.udp.data);
+}
+
+static void
+stop_udp_child(isc_nmsocket_t *sock) {
+	INSIST(sock->type == isc_nm_udpsocket);
+
+	uv_udp_recv_stop(&sock->uv_handle.udp);
+	uv_close((uv_handle_t *) &sock->uv_handle.udp, udp_close_cb);
+
+	LOCK(&sock->parent->lock);
+	atomic_fetch_sub(&sock->parent->rchildren, 1);
+	UNLOCK(&sock->parent->lock);
+	BROADCAST(&sock->parent->cond);
+}
+
+static void
+stoplistening(isc_nmsocket_t *sock) {
+	/*
+	 * Socket is already closing; there's nothing to do.
+	 */
+	if (uv_is_closing((uv_handle_t *) &sock->uv_handle.udp)) {
+		return;
+	}
+
+	INSIST(sock->type == isc_nm_udplistener);
+
+	for (int i = 0; i < sock->nchildren; i++) {
+		isc__netievent_udplisten_t *event = NULL;
+
+		if (i == sock->tid) {
+			stop_udp_child(&sock->children[i]);
+			continue;
+		}
+
+		event = isc__nm_get_ievent(sock->mgr, netievent_udpstoplisten);
+		event->sock = &sock->children[i];
+		isc__nm_enqueue_ievent(&sock->mgr->workers[i],
+				       (isc__netievent_t *) event);
+	}
+
+	LOCK(&sock->lock);
+	while (atomic_load(&sock->rchildren) > 0) {
+		WAIT(&sock->cond, &sock->lock);
+	}
+	atomic_store(&sock->closed, true);
+	UNLOCK(&sock->lock);
+
+	isc__nmsocket_prep_destroy(sock);
+}
+
+void
+isc_nm_udp_stoplistening(isc_nmsocket_t *sock) {
+	isc__netievent_udpstoplisten_t *ievent = NULL;
+
+	/* We can't be launched from network thread, we'd deadlock */
+	REQUIRE(!isc__nm_in_netthread());
+	REQUIRE(VALID_NMSOCK(sock));
+	REQUIRE(sock->type == isc_nm_udplistener);
+
+	/*
+	 * If the manager is interlocked, re-enqueue this as an asynchronous
+	 * event. Otherwise, go ahead and stop listening right away.
+	 */
+	if (!isc__nm_acquire_interlocked(sock->mgr)) {
+		ievent = isc__nm_get_ievent(sock->mgr, netievent_udpstoplisten);
+		ievent->sock = sock;
+		isc__nm_enqueue_ievent(&sock->mgr->workers[sock->tid],
+				       (isc__netievent_t *) ievent);
+	} else {
+		stoplistening(sock);
+		isc__nm_drop_interlocked(sock->mgr);
+	}
+}
+
+/*
+ * handle 'udpstoplisten' async call - stop listening on a socket.
+ */
+void
+isc__nm_async_udpstoplisten(isc__networker_t *worker,
+			    isc__netievent_t *ievent0)
+{
+	isc__netievent_udplisten_t *ievent =
+		(isc__netievent_udplisten_t *) ievent0;
+	isc_nmsocket_t *sock = ievent->sock;
+
+	REQUIRE(sock->iface != NULL);
+	UNUSED(worker);
+
+	/*
+	 * If this is a child socket, stop listening and return.
+	 */
+	if (sock->parent != NULL) {
+		stop_udp_child(sock);
+		return;
+	}
+
+	/*
+	 * If network manager is paused, re-enqueue the event for later.
+	 */
+	if (!isc__nm_acquire_interlocked(sock->mgr)) {
+		isc__netievent_udplisten_t *event = NULL;
+
+		event = isc__nm_get_ievent(sock->mgr, netievent_udpstoplisten);
+		event->sock = sock;
+		isc__nm_enqueue_ievent(&sock->mgr->workers[sock->tid],
+				       (isc__netievent_t *) event);
+	} else {
+		stoplistening(sock);
+		isc__nm_drop_interlocked(sock->mgr);
+	}
+}
+
+/*
+ * udp_recv_cb handles incoming UDP packet from uv.
+ * The buffer here is reused for a series of packets,
+ * so we need to allocate a new one. This new one can
+ * be reused to send the response then.
+ */
+static void
+udp_recv_cb(uv_udp_t *handle, ssize_t nrecv, const uv_buf_t *buf,
+	    const struct sockaddr *addr, unsigned flags)
+{
+	isc_result_t result;
+	isc_nmhandle_t *nmhandle = NULL;
+	isc_sockaddr_t sockaddr;
+	isc_sockaddr_t localaddr;
+	struct sockaddr_storage laddr;
+	isc_nmsocket_t *sock = (isc_nmsocket_t *) handle->data;
+	isc_region_t region;
+	uint32_t maxudp;
+
+	REQUIRE(VALID_NMSOCK(sock));
+
+	/* XXXWPK TODO handle it! */
+	UNUSED(flags);
+
+	/*
+	 * If addr == NULL that's the end of stream - we can
+	 * free the buffer and bail.
+	 */
+	if (addr == NULL) {
+		isc__nm_free_uvbuf(sock, buf);
+		return;
+	}
+
+	/*
+	 * Simulate a firewall blocking UDP packets bigger than
+	 * 'maxudp' bytes.
+	 */
+	maxudp = atomic_load(&sock->mgr->maxudp);
+	if (maxudp != 0 && (uint32_t)nrecv > maxudp) {
+		return;
+	}
+
+	result = isc_sockaddr_fromsockaddr(&sockaddr, addr);
+	RUNTIME_CHECK(result == ISC_R_SUCCESS);
+	uv_udp_getsockname(handle, (struct sockaddr *) &laddr,
+			   &(int){sizeof(struct sockaddr_storage)});
+	result = isc_sockaddr_fromsockaddr(&localaddr,
+					   (struct sockaddr *) &laddr);
+	RUNTIME_CHECK(result == ISC_R_SUCCESS);
+
+	nmhandle = isc__nmhandle_get(sock, &sockaddr, &localaddr);
+	region.base = (unsigned char *) buf->base;
+	region.length = nrecv;
+
+	INSIST(sock->rcb.recv != NULL);
+	sock->rcb.recv(nmhandle, &region, sock->rcbarg);
+	isc__nm_free_uvbuf(sock, buf);
+
+	/*
+	 * If the recv callback wants to hold on to the handle,
+	 * it needs to attach to it.
+	 */
+	isc_nmhandle_unref(nmhandle);
+}
+
+/*
+ * isc__nm_udp_send sends buf to a peer on a socket.
+ * It tries to find a proper sibling/child socket so that we won't have
+ * to jump to other thread.
+ */
+isc_result_t
+isc__nm_udp_send(isc_nmhandle_t *handle, isc_region_t *region,
+		 isc_nm_cb_t cb, void *cbarg)
+{
+	isc_nmsocket_t *psock = NULL, *rsock = NULL;
+	isc_nmsocket_t *sock = handle->sock;
+	isc_sockaddr_t *peer = &handle->peer;
+	isc__netievent_udpsend_t *ievent;
+	isc__nm_uvreq_t *uvreq = NULL;
+	int ntid;
+	uint32_t maxudp = atomic_load(&sock->mgr->maxudp);
+
+	/*
+	 * Simulate a firewall blocking UDP packets bigger than
+	 * 'maxudp' bytes.
+	 */
+	if (maxudp != 0 && region->length > maxudp) {
+		isc_nmhandle_unref(handle);
+		return (ISC_R_SUCCESS);
+	}
+
+	if (sock->type == isc_nm_udpsocket) {
+		INSIST(sock->parent != NULL);
+		psock = sock->parent;
+	} else if (sock->type == isc_nm_udplistener) {
+		psock = sock;
+	} else {
+		isc_nmhandle_unref(handle);
+		return (ISC_R_UNEXPECTED);
+	}
+
+	if (isc__nm_in_netthread()) {
+		ntid = isc_nm_tid();
+	} else {
+		ntid = (int) isc_random_uniform(sock->nchildren);
+	}
+
+	rsock = &psock->children[ntid];
+
+	uvreq = isc__nm_uvreq_get(sock->mgr, sock);
+	uvreq->uvbuf.base = (char *) region->base;
+	uvreq->uvbuf.len = region->length;
+
+	uvreq->handle = handle;
+	isc_nmhandle_ref(uvreq->handle);
+
+	uvreq->cb.send = cb;
+	uvreq->cbarg = cbarg;
+
+	if (isc_nm_tid() == rsock->tid) {
+		/*
+		 * If we're in the same thread as the socket we can send the
+		 * data directly
+		 */
+		return (udp_send_direct(rsock, uvreq, peer));
+	} else {
+		/*
+		 * We need to create an event and pass it using async channel
+		 */
+		ievent = isc__nm_get_ievent(sock->mgr, netievent_udpsend);
+		ievent->sock = rsock;
+		ievent->peer = *peer;
+		ievent->req = uvreq;
+
+		isc__nm_enqueue_ievent(&sock->mgr->workers[rsock->tid],
+				       (isc__netievent_t *) ievent);
+		return (ISC_R_SUCCESS);
+	}
+}
+
+
+/*
+ * handle 'udpsend' async event - send a packet on the socket
+ */
+void
+isc__nm_async_udpsend(isc__networker_t *worker, isc__netievent_t *ievent0) {
+	isc__netievent_udpsend_t *ievent =
+		(isc__netievent_udpsend_t *) ievent0;
+
+	REQUIRE(worker->id == ievent->sock->tid);
+
+	if (atomic_load(&ievent->sock->active)) {
+		udp_send_direct(ievent->sock, ievent->req, &ievent->peer);
+	} else {
+		ievent->req->cb.send(ievent->req->handle,
+				     ISC_R_CANCELED, ievent->req->cbarg);
+		isc__nm_uvreq_put(&ievent->req, ievent->req->sock);
+	}
+}
+
+/*
+ * udp_send_cb - callback
+ */
+static void
+udp_send_cb(uv_udp_send_t *req, int status) {
+	isc_result_t result = ISC_R_SUCCESS;
+	isc__nm_uvreq_t *uvreq = (isc__nm_uvreq_t *)req->data;
+
+	REQUIRE(VALID_UVREQ(uvreq));
+	REQUIRE(VALID_NMHANDLE(uvreq->handle));
+
+	if (status < 0) {
+		result = isc__nm_uverr2result(status);
+	}
+
+	uvreq->cb.send(uvreq->handle, result, uvreq->cbarg);
+	isc_nmhandle_unref(uvreq->handle);
+	isc__nm_uvreq_put(&uvreq, uvreq->sock);
+}
+
+/*
+ * udp_send_direct sends buf to a peer on a socket. Sock has to be in
+ * the same thread as the callee.
+ */
+static isc_result_t
+udp_send_direct(isc_nmsocket_t *sock, isc__nm_uvreq_t *req,
+		isc_sockaddr_t *peer)
+{
+	int rv;
+
+	REQUIRE(sock->tid == isc_nm_tid());
+	REQUIRE(sock->type == isc_nm_udpsocket);
+
+	isc_nmhandle_ref(req->handle);
+	rv = uv_udp_send(&req->uv_req.udp_send,
+			 &sock->uv_handle.udp, &req->uvbuf, 1,
+			 &peer->type.sa, udp_send_cb);
+	if (rv < 0) {
+		return (isc__nm_uverr2result(rv));
+	}
+
+	return (ISC_R_SUCCESS);
+}

lib/isc/netmgr/uverr2result.c

@@ -0,0 +1,91 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+#include <stdbool.h>
+#include <uv.h>
+
+#include <isc/platform.h>
+#include <isc/result.h>
+#include <isc/strerr.h>
+#include <isc/string.h>
+#include <isc/util.h>
+
+#include "netmgr-int.h"
+
+/*%
+ * Convert a libuv error value into an isc_result_t.  The
+ * list of supported error values is not complete; new users
+ * of this function should add any expected errors that are
+ * not already there.
+ */
+isc_result_t
+isc___nm_uverr2result(int uverr, bool dolog,
+		     const char *file, unsigned int line)
+{
+	switch (uverr) {
+	case UV_ENOTDIR:
+	case UV_ELOOP:
+	case UV_EINVAL:		/* XXX sometimes this is not for files */
+	case UV_ENAMETOOLONG:
+	case UV_EBADF:
+		return (ISC_R_INVALIDFILE);
+	case UV_ENOENT:
+		return (ISC_R_FILENOTFOUND);
+	case UV_EACCES:
+	case UV_EPERM:
+		return (ISC_R_NOPERM);
+	case UV_EEXIST:
+		return (ISC_R_FILEEXISTS);
+	case UV_EIO:
+		return (ISC_R_IOERROR);
+	case UV_ENOMEM:
+		return (ISC_R_NOMEMORY);
+	case UV_ENFILE:
+	case UV_EMFILE:
+		return (ISC_R_TOOMANYOPENFILES);
+	case UV_ENOSPC:
+		return (ISC_R_DISCFULL);
+	case UV_EPIPE:
+	case UV_ECONNRESET:
+	case UV_ECONNABORTED:
+		return (ISC_R_CONNECTIONRESET);
+	case UV_ENOTCONN:
+		return (ISC_R_NOTCONNECTED);
+	case UV_ETIMEDOUT:
+		return (ISC_R_TIMEDOUT);
+	case UV_ENOBUFS:
+		return (ISC_R_NORESOURCES);
+	case UV_EAFNOSUPPORT:
+		return (ISC_R_FAMILYNOSUPPORT);
+	case UV_ENETDOWN:
+		return (ISC_R_NETDOWN);
+	case UV_EHOSTDOWN:
+		return (ISC_R_HOSTDOWN);
+	case UV_ENETUNREACH:
+		return (ISC_R_NETUNREACH);
+	case UV_EHOSTUNREACH:
+		return (ISC_R_HOSTUNREACH);
+	case UV_EADDRINUSE:
+		return (ISC_R_ADDRINUSE);
+	case UV_EADDRNOTAVAIL:
+		return (ISC_R_ADDRNOTAVAIL);
+	case UV_ECONNREFUSED:
+		return (ISC_R_CONNREFUSED);
+	default:
+		if (dolog) {
+			UNEXPECTED_ERROR(file, line,
+					 "unable to convert libuv "
+					 "error code to isc_result: %d: %s",
+					 uverr, uv_strerror(uverr));
+		}
+		return (ISC_R_UNEXPECTED);
+	}
+}