WIP: Add F&A Array Queue implementation

Make a copy of ns__client_request as ns__client_request_old

The 'old' one will be used by not-yet-reworked parts, while we
keep history of ns__client_request.

Start netmgr with named, shutdown at the end

- udplistener in interfacemgr uses netmgr
- use per-client buffer for udp send
- first steps at making ns_client_request use netmgr for UDP
- uncrustify

Make interfacemgr and interface references isc_refcount_t instead of locked ints

Use clock_gettime instead of gettimeofday

Cleanup get_client, use one interface for clientmgr instead of attaching it to each client

'extra' field at the end of handle - for client data

WiP: no TCP, UDP works, no resolver

.gitignore

@@ -10,6 +10,7 @@
 *.rej
 *.so
 *_test
+*.ipch # vscode/intellisense precompiled header
 *~
 .ccache/
 .cproject

.gitlab-ci.yml

@@ -300,30 +300,6 @@ unit:gcc:alpine3.10:amd64:
     - gcc:alpine3.10:amd64
   needs: ["gcc:alpine3.10:amd64"]
 
-# Jobs for regular GCC builds on CentOS 6 (amd64)
-
-gcc:centos6:amd64:
-  variables:
-    CC: gcc
-    CFLAGS: "-Wall -Wextra -O2 -g"
-    EXTRA_CONFIGURE: "--with-libidn2 --disable-warn-error"
-  <<: *centos_centos6_amd64_image
-  <<: *build_job
-
-system:gcc:centos6:amd64:
-  <<: *centos_centos6_amd64_image
-  <<: *system_test_job
-  dependencies:
-    - gcc:centos6:amd64
-  needs: ["gcc:centos6:amd64"]
-
-unit:gcc:centos6:amd64:
-  <<: *centos_centos6_amd64_image
-  <<: *unit_test_job
-  dependencies:
-    - gcc:centos6:amd64
-  needs: ["gcc:centos6:amd64"]
-
 # Jobs for regular GCC builds on CentOS 7 (amd64)
 
 gcc:centos7:amd64:
@@ -348,30 +324,6 @@ unit:gcc:centos7:amd64:
     - gcc:centos7:amd64
   needs: ["gcc:centos7:amd64"]
 
-# Jobs for regular GCC builds on Debian 8 Jessie (amd64)
-
-gcc:jessie:amd64:
-  variables:
-    CC: gcc
-    CFLAGS: "-Wall -Wextra -O2 -g"
-    EXTRA_CONFIGURE: "--without-cmocka --with-python --disable-geoip"
-  <<: *debian_jessie_amd64_image
-  <<: *build_job
-
-system:gcc:jessie:amd64:
-  <<: *debian_jessie_amd64_image
-  <<: *system_test_job
-  dependencies:
-    - gcc:jessie:amd64
-  needs: ["gcc:jessie:amd64"]
-
-unit:gcc:jessie:amd64:
-  <<: *debian_jessie_amd64_image
-  <<: *unit_test_job
-  dependencies:
-    - gcc:jessie:amd64
-  needs: ["gcc:jessie:amd64"]
-
 # Jobs for regular GCC builds on Debian 9 Stretch (amd64)
 
 gcc:stretch:amd64:

.uncrustify.cfg

@@ -642,7 +642,7 @@ align_with_tabs                          = true     # false/true
 align_on_tabstop                         = false    # false/true
 
 # Whether to left-align numbers
-align_number_left                        = false    # false/true
+#align_number_left                        = false    # false/true
 
 # Align variable definitions in prototypes and functions
 align_func_params                        = false    # false/true

bin/dig/dighost.c

@@ -1381,7 +1381,7 @@ setup_libs(void) {
 
 	isc_log_setdebuglevel(lctx, 0);
 
-	result = isc_taskmgr_create(mctx, 1, 0, &taskmgr);
+	result = isc_taskmgr_create(mctx, 1, 0, NULL, &taskmgr);
 	check_result(result, "isc_taskmgr_create");
 
 	result = isc_task_create(taskmgr, 0, &global_task);

bin/dnssec/dnssec-signzone.c

@@ -3798,7 +3798,7 @@ main(int argc, char *argv[]) {
 	print_time(outfp);
 	print_version(outfp);
 
-	result = isc_taskmgr_create(mctx, ntasks, 0, &taskmgr);
+	result = isc_taskmgr_create(mctx, ntasks, 0, NULL, &taskmgr);
 	if (result != ISC_R_SUCCESS)
 		fatal("failed to create task manager: %s",
 		      isc_result_totext(result));

bin/named/include/named/globals.h

@@ -19,6 +19,7 @@
 #include <isc/rwlock.h>
 #include <isc/log.h>
 #include <isc/net.h>
+#include <isc/netmgr.h>
 
 #include <isccfg/aclconf.h>
 #include <isccfg/cfg.h>
@@ -62,6 +63,7 @@ EXTERN bool		named_g_run_done	INIT(false);
  */
 EXTERN isc_timermgr_t *		named_g_timermgr	INIT(NULL);
 EXTERN isc_socketmgr_t *	named_g_socketmgr	INIT(NULL);
+EXTERN isc_nm_t *		named_g_nm		INIT(NULL);
 EXTERN cfg_parser_t *		named_g_parser		INIT(NULL);
 EXTERN cfg_parser_t *		named_g_addparser	INIT(NULL);
 EXTERN const char *		named_g_version		INIT(VERSION);

bin/named/main.c

@@ -24,6 +24,7 @@
 #include <isc/file.h>
 #include <isc/hash.h>
 #include <isc/httpd.h>
+#include <isc/netmgr.h>
 #include <isc/os.h>
 #include <isc/platform.h>
 #include <isc/print.h>
@@ -123,7 +124,6 @@ static int		maxudp = 0;
 /*
  * -T options:
  */
-static bool clienttest = false;
 static bool dropedns = false;
 static bool ednsformerr = false;
 static bool ednsnotimp = false;
@@ -602,17 +602,12 @@ parse_T_opt(char *option) {
 	/*
 	 * force the server to behave (or misbehave) in
 	 * specified ways for testing purposes.
-	 *
-	 * clienttest: make clients single shot with their
-	 * 	       own memory context.
-	 * delay=xxxx: delay client responses by xxxx ms to
+	 *	 * delay=xxxx: delay client responses by xxxx ms to
 	 *	       simulate remote servers.
 	 * dscp=x:     check that dscp values are as
 	 * 	       expected and assert otherwise.
 	 */
-	if (!strcmp(option, "clienttest")) {
-		clienttest = true;
-	} else if (!strncmp(option, "delay=", 6)) {
+	if (!strncmp(option, "delay=", 6)) {
 		delay = atoi(option + 6);
 	} else if (!strcmp(option, "dropedns")) {
 		dropedns = true;
@@ -877,8 +872,15 @@ create_managers(void) {
 		      "using %u UDP listener%s per interface",
 		      named_g_udpdisp, named_g_udpdisp == 1 ? "" : "s");
 
+	named_g_nm = isc_nm_start(named_g_mctx, named_g_cpus);
+	if (named_g_nm == NULL) {
+		UNEXPECTED_ERROR(__FILE__, __LINE__,
+				"isc_nm_start() failed");
+		return (ISC_R_UNEXPECTED);
+	}
+
 	result = isc_taskmgr_create(named_g_mctx, named_g_cpus, 0,
-				    &named_g_taskmgr);
+				    named_g_nm, &named_g_taskmgr);
 	if (result != ISC_R_SUCCESS) {
 		UNEXPECTED_ERROR(__FILE__, __LINE__,
 				 "isc_taskmgr_create() failed: %s",
@@ -921,6 +923,7 @@ destroy_managers(void) {
 	isc_taskmgr_destroy(&named_g_taskmgr);
 	isc_timermgr_destroy(&named_g_timermgr);
 	isc_socketmgr_destroy(&named_g_socketmgr);
+	isc_nm_shutdown(&named_g_nm);
 }
 
 static void
@@ -1234,8 +1237,6 @@ setup(void) {
 	/*
 	 * Modify server context according to command line options
 	 */
-	if (clienttest)
-		ns_server_setoption(sctx, NS_SERVER_CLIENTTEST, true);
 	if (disable4)
 		ns_server_setoption(sctx, NS_SERVER_DISABLE4, true);
 	if (disable6)

bin/named/server.c

@@ -9399,6 +9399,7 @@ run_server(isc_task_t *task, isc_event_t *event) {
 	CHECKFATAL(ns_interfacemgr_create(named_g_mctx, server->sctx,
 					  named_g_taskmgr, named_g_timermgr,
 					  named_g_socketmgr,
+					  named_g_nm,
 					  named_g_dispatchmgr,
 					  server->task, named_g_udpdisp, geoip,
 					  &server->interfacemgr),
@@ -9461,6 +9462,10 @@ shutdown_server(isc_task_t *task, isc_event_t *event) {
 	UNUSED(task);
 	INSIST(task == server->task);
 
+	/* We need to shutdown the interface before going exclusive (and
+	 * pausing netmgr. */
+	ns_interfacemgr_shutdown(server->interfacemgr);
+
 	result = isc_task_beginexclusive(server->task);
 	RUNTIME_CHECK(result == ISC_R_SUCCESS);
 
@@ -9510,7 +9515,6 @@ shutdown_server(isc_task_t *task, isc_event_t *event) {
 	isc_timer_detach(&server->pps_timer);
 	isc_timer_detach(&server->tat_timer);
 
-	ns_interfacemgr_shutdown(server->interfacemgr);
 	ns_interfacemgr_detach(&server->interfacemgr);
 
 	dns_dispatchmgr_destroy(&named_g_dispatchmgr);

bin/nsupdate/nsupdate.c

@@ -906,7 +906,7 @@ setup_system(void) {
 	result = isc_timermgr_create(gmctx, &timermgr);
 	check_result(result, "dns_timermgr_create");
 
-	result = isc_taskmgr_create(gmctx, 1, 0, &taskmgr);
+	result = isc_taskmgr_create(gmctx, 1, 0, NULL, &taskmgr);
 	check_result(result, "isc_taskmgr_create");
 
 	result = isc_task_create(taskmgr, 0, &global_task);

bin/rndc/rndc.c

@@ -932,7 +932,8 @@ main(int argc, char **argv) {
 
 	DO("create memory context", isc_mem_create(0, 0, &rndc_mctx));
 	DO("create socket manager", isc_socketmgr_create(rndc_mctx, &socketmgr));
-	DO("create task manager", isc_taskmgr_create(rndc_mctx, 1, 0, &taskmgr));
+	DO("create task manager",
+	   isc_taskmgr_create(rndc_mctx, 1, 0, NULL, &taskmgr));
 	DO("create task", isc_task_create(taskmgr, 0, &task));
 
 	DO("create logging context", isc_log_create(rndc_mctx, &log, &logconfig));

bin/tests/optional/adb_test.c

@@ -144,7 +144,7 @@ create_managers(void) {
 	isc_result_t result;
 
 	taskmgr = NULL;
-	result = isc_taskmgr_create(mctx, 5, 0, &taskmgr);
+	result = isc_taskmgr_create(mctx, 5, 0, NULL, &taskmgr);
 	check_result(result, "isc_taskmgr_create");
 
 	timermgr = NULL;

bin/tests/optional/byaddr_test.c

@@ -112,7 +112,7 @@ main(int argc, char *argv[]) {
 	}
 
 	taskmgr = NULL;
-	RUNTIME_CHECK(isc_taskmgr_create(mctx, workers, 0, &taskmgr)
+	RUNTIME_CHECK(isc_taskmgr_create(mctx, workers, 0, NULL, &taskmgr)
 		      == ISC_R_SUCCESS);
 	task = NULL;
 	RUNTIME_CHECK(isc_task_create(taskmgr, 0, &task)

bin/tests/optional/byname_test.c

@@ -226,7 +226,7 @@ main(int argc, char *argv[]) {
 	}
 
 	taskmgr = NULL;
-	RUNTIME_CHECK(isc_taskmgr_create(mctx, workers, 0, &taskmgr) ==
+	RUNTIME_CHECK(isc_taskmgr_create(mctx, workers, 0, NULL, &taskmgr) ==
 		      ISC_R_SUCCESS);
 	task = NULL;
 	RUNTIME_CHECK(isc_task_create(taskmgr, 0, &task) ==

bin/tests/optional/gsstest.c

@@ -470,7 +470,7 @@ main(int argc, char *argv[]) {
 	RUNCHECK(dst_lib_init(mctx, NULL));
 
 	taskmgr = NULL;
-	RUNCHECK(isc_taskmgr_create(mctx, 1, 0, &taskmgr));
+	RUNCHECK(isc_taskmgr_create(mctx, 1, 0, NULL, &taskmgr));
 	task = NULL;
 	RUNCHECK(isc_task_create(taskmgr, 0, &task));
 	timermgr = NULL;

bin/tests/optional/ratelimiter_test.c

@@ -102,7 +102,7 @@ main(int argc, char *argv[]) {
 	isc_interval_set(&linterval, 1, 0);
 
 	RUNTIME_CHECK(isc_mem_create(0, 0, &mctx) == ISC_R_SUCCESS);
-	RUNTIME_CHECK(isc_taskmgr_create(mctx, 3, 0, &taskmgr) ==
+	RUNTIME_CHECK(isc_taskmgr_create(mctx, 3, 0, NULL, &taskmgr) ==
 		      ISC_R_SUCCESS);
 	RUNTIME_CHECK(isc_timermgr_create(mctx, &timermgr) ==
 		      ISC_R_SUCCESS);

bin/tests/optional/shutdown_test.c

@@ -181,7 +181,7 @@ main(int argc, char *argv[]) {
 	RUNTIME_CHECK(isc_mem_create(0, 0, &mctx) == ISC_R_SUCCESS);
 	mctx2 = NULL;
 	RUNTIME_CHECK(isc_mem_create(0, 0, &mctx2) == ISC_R_SUCCESS);
-	RUNTIME_CHECK(isc_taskmgr_create(mctx, workers, 0, &task_manager) ==
+	RUNTIME_CHECK(isc_taskmgr_create(mctx, workers, 0, NULL, &task_manager) ==
 		      ISC_R_SUCCESS);
 	RUNTIME_CHECK(isc_timermgr_create(mctx, &timer_manager) ==
 		      ISC_R_SUCCESS);

bin/tests/optional/sig0_test.c

@@ -226,7 +226,7 @@ main(int argc, char *argv[]) {
 	dst_result_register();
 
 	taskmgr = NULL;
-	RUNTIME_CHECK(isc_taskmgr_create(mctx, 2, 0, &taskmgr) ==
+	RUNTIME_CHECK(isc_taskmgr_create(mctx, 2, 0, NULL, &taskmgr) ==
 		      ISC_R_SUCCESS);
 	task1 = NULL;
 	RUNTIME_CHECK(isc_task_create(taskmgr, 0, &task1) == ISC_R_SUCCESS);

bin/tests/optional/sock_test.c

@@ -292,7 +292,7 @@ main(int argc, char *argv[]) {
 	 * The task manager is independent (other than memory context)
 	 */
 	manager = NULL;
-	RUNTIME_CHECK(isc_taskmgr_create(mctx, workers, 0, &manager) ==
+	RUNTIME_CHECK(isc_taskmgr_create(mctx, workers, 0, NULL, &manager) ==
 		      ISC_R_SUCCESS);
 
 	/*

bin/tests/optional/task_test.c

@@ -79,7 +79,7 @@ main(int argc, char *argv[]) {
 
 	RUNTIME_CHECK(isc_mem_create(0, 0, &mctx) == ISC_R_SUCCESS);
 
-	RUNTIME_CHECK(isc_taskmgr_create(mctx, workers, 0, &manager) ==
+	RUNTIME_CHECK(isc_taskmgr_create(mctx, workers, 0, NULL, &manager) ==
 		      ISC_R_SUCCESS);
 
 	RUNTIME_CHECK(isc_task_create(manager, 0, &t1) == ISC_R_SUCCESS);

bin/tests/optional/timer_test.c

@@ -108,7 +108,7 @@ main(int argc, char *argv[]) {
 	printf("%u workers\n", workers);
 
 	RUNTIME_CHECK(isc_mem_create(0, 0, &mctx1) == ISC_R_SUCCESS);
-	RUNTIME_CHECK(isc_taskmgr_create(mctx1, workers, 0, &manager) ==
+	RUNTIME_CHECK(isc_taskmgr_create(mctx1, workers, 0, NULL, &manager) ==
 		      ISC_R_SUCCESS);
 	RUNTIME_CHECK(isc_timermgr_create(mctx1, &timgr) == ISC_R_SUCCESS);
 

bin/tests/optional/zone_test.c

@@ -281,7 +281,7 @@ main(int argc, char **argv) {
 
 	RUNTIME_CHECK(isc_app_start() == ISC_R_SUCCESS);
 	RUNTIME_CHECK(isc_mem_create(0, 0, &mctx) == ISC_R_SUCCESS);
-	RUNTIME_CHECK(isc_taskmgr_create(mctx, 2, 0, &taskmgr) ==
+	RUNTIME_CHECK(isc_taskmgr_create(mctx, 2, 0, NULL, &taskmgr) ==
 		      ISC_R_SUCCESS);
 	RUNTIME_CHECK(isc_timermgr_create(mctx, &timermgr) == ISC_R_SUCCESS);
 	RUNTIME_CHECK(isc_socketmgr_create(mctx, &socketmgr) == ISC_R_SUCCESS);

bin/tests/system/ecdsa/tests.sh

@@ -20,7 +20,7 @@ rm -f dig.out.*
 DIGOPTS="+tcp +noau +noadd +nosea +nostat +nocmd +dnssec -p 5300"
 
 # Check the example. domain
-
+sleep 5
 echo "I:checking that positive validation works ($n)"
 ret=0
 $DIG $DIGOPTS . @10.53.0.1 soa > dig.out.ns1.test$n || ret=1

bin/tests/system/pipelined/pipequeries.c

@@ -277,7 +277,7 @@ main(int argc, char *argv[]) {
 	RUNCHECK(dst_lib_init(mctx, NULL));
 
 	taskmgr = NULL;
-	RUNCHECK(isc_taskmgr_create(mctx, 1, 0, &taskmgr));
+	RUNCHECK(isc_taskmgr_create(mctx, 1, 0, NULL, &taskmgr));
 	task = NULL;
 	RUNCHECK(isc_task_create(taskmgr, 0, &task));
 	timermgr = NULL;

bin/tests/system/tkey/keycreate.c

@@ -236,7 +236,7 @@ main(int argc, char *argv[]) {
 	RUNCHECK(dst_lib_init(mctx, NULL));
 
 	taskmgr = NULL;
-	RUNCHECK(isc_taskmgr_create(mctx, 1, 0, &taskmgr));
+	RUNCHECK(isc_taskmgr_create(mctx, 1, 0, NULL, &taskmgr));
 	task = NULL;
 	RUNCHECK(isc_task_create(taskmgr, 0, &task));
 	timermgr = NULL;

bin/tests/system/tkey/keydelete.c

@@ -175,7 +175,7 @@ main(int argc, char **argv) {
 	RUNCHECK(dst_lib_init(mctx, NULL));
 
 	taskmgr = NULL;
-	RUNCHECK(isc_taskmgr_create(mctx, 1, 0, &taskmgr));
+	RUNCHECK(isc_taskmgr_create(mctx, 1, 0, NULL, &taskmgr));
 	task = NULL;
 	RUNCHECK(isc_task_create(taskmgr, 0, &task));
 	timermgr = NULL;

bin/tools/mdig.c

@@ -2047,7 +2047,7 @@ main(int argc, char *argv[]) {
 		fatal("can't choose between IPv4 and IPv6");
 
 	taskmgr = NULL;
-	RUNCHECK(isc_taskmgr_create(mctx, 1, 0, &taskmgr));
+	RUNCHECK(isc_taskmgr_create(mctx, 1, 0, NULL, &taskmgr));
 	task = NULL;
 	RUNCHECK(isc_task_create(taskmgr, 0, &task));
 	timermgr = NULL;

configure

@@ -850,6 +850,7 @@ infodir
 docdir
 oldincludedir
 includedir
+runstatedir
 localstatedir
 sharedstatedir
 sysconfdir
@@ -1020,6 +1021,7 @@ datadir='${datarootdir}'
 sysconfdir='${prefix}/etc'
 sharedstatedir='${prefix}/com'
 localstatedir='${prefix}/var'
+runstatedir='${localstatedir}/run'
 includedir='${prefix}/include'
 oldincludedir='/usr/include'
 docdir='${datarootdir}/doc/${PACKAGE_TARNAME}'
@@ -1272,6 +1274,15 @@ do
   | -silent | --silent | --silen | --sile | --sil)
     silent=yes ;;
 
+  -runstatedir | --runstatedir | --runstatedi | --runstated \
+  | --runstate | --runstat | --runsta | --runst | --runs \
+  | --run | --ru | --r)
+    ac_prev=runstatedir ;;
+  -runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \
+  | --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \
+  | --run=* | --ru=* | --r=*)
+    runstatedir=$ac_optarg ;;
+
   -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb)
     ac_prev=sbindir ;;
   -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \
@@ -1409,7 +1420,7 @@ fi
 for ac_var in	exec_prefix prefix bindir sbindir libexecdir datarootdir \
 		datadir sysconfdir sharedstatedir localstatedir includedir \
 		oldincludedir docdir infodir htmldir dvidir pdfdir psdir \
-		libdir localedir mandir
+		libdir localedir mandir runstatedir
 do
   eval ac_val=\$$ac_var
   # Remove trailing slashes.
@@ -1562,6 +1573,7 @@ Fine tuning of the installation directories:
   --sysconfdir=DIR        read-only single-machine data [PREFIX/etc]
   --sharedstatedir=DIR    modifiable architecture-independent data [PREFIX/com]
   --localstatedir=DIR     modifiable single-machine data [PREFIX/var]
+  --runstatedir=DIR       modifiable per-process data [LOCALSTATEDIR/run]
   --libdir=DIR            object code libraries [EPREFIX/lib]
   --includedir=DIR        C header files [PREFIX/include]
   --oldincludedir=DIR     C header files for non-gcc [/usr/include]
@@ -4002,7 +4014,7 @@ else
     We can't simply define LARGE_OFF_T to be 9223372036854775807,
     since some C++ compilers masquerading as C compilers
     incorrectly reject 9223372036854775807.  */
-#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
+#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
   int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
 		       && LARGE_OFF_T % 2147483647 == 1)
 		      ? 1 : -1];
@@ -4048,7 +4060,7 @@ else
     We can't simply define LARGE_OFF_T to be 9223372036854775807,
     since some C++ compilers masquerading as C compilers
     incorrectly reject 9223372036854775807.  */
-#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
+#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
   int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
 		       && LARGE_OFF_T % 2147483647 == 1)
 		      ? 1 : -1];
@@ -4072,7 +4084,7 @@ rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
     We can't simply define LARGE_OFF_T to be 9223372036854775807,
     since some C++ compilers masquerading as C compilers
     incorrectly reject 9223372036854775807.  */
-#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
+#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
   int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
 		       && LARGE_OFF_T % 2147483647 == 1)
 		      ? 1 : -1];
@@ -4117,7 +4129,7 @@ else
     We can't simply define LARGE_OFF_T to be 9223372036854775807,
     since some C++ compilers masquerading as C compilers
     incorrectly reject 9223372036854775807.  */
-#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
+#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
   int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
 		       && LARGE_OFF_T % 2147483647 == 1)
 		      ? 1 : -1];
@@ -4141,7 +4153,7 @@ rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
     We can't simply define LARGE_OFF_T to be 9223372036854775807,
     since some C++ compilers masquerading as C compilers
     incorrectly reject 9223372036854775807.  */
-#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
+#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
   int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
 		       && LARGE_OFF_T % 2147483647 == 1)
 		      ? 1 : -1];
@@ -15770,6 +15782,66 @@ fi
 done
 
 
+# libuv
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing uv_run" >&5
+$as_echo_n "checking for library containing uv_run... " >&6; }
+if ${ac_cv_search_uv_run+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+  ac_func_search_save_LIBS=$LIBS
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+/* Override any GCC internal prototype to avoid an error.
+   Use char because int might match the return type of a GCC
+   builtin and then its argument prototype would still apply.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+char uv_run ();
+int
+main ()
+{
+return uv_run ();
+  ;
+  return 0;
+}
+_ACEOF
+for ac_lib in '' uv; do
+  if test -z "$ac_lib"; then
+    ac_res="none required"
+  else
+    ac_res=-l$ac_lib
+    LIBS="-l$ac_lib  $ac_func_search_save_LIBS"
+  fi
+  if ac_fn_c_try_link "$LINENO"; then :
+  ac_cv_search_uv_run=$ac_res
+fi
+rm -f core conftest.err conftest.$ac_objext \
+    conftest$ac_exeext
+  if ${ac_cv_search_uv_run+:} false; then :
+  break
+fi
+done
+if ${ac_cv_search_uv_run+:} false; then :
+
+else
+  ac_cv_search_uv_run=no
+fi
+rm conftest.$ac_ext
+LIBS=$ac_func_search_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_uv_run" >&5
+$as_echo "$ac_cv_search_uv_run" >&6; }
+ac_res=$ac_cv_search_uv_run
+if test "$ac_res" != no; then :
+  test "$ac_res" = "none required" || LIBS="$ac_res $LIBS"
+
+else
+  as_fn_error $? "The uv library was not found. Please install libuv!" "$LINENO" 5
+fi
+
+
 #
 # flockfile is usually provided by pthreads
 #
@@ -23356,7 +23428,7 @@ ac_config_commands="$ac_config_commands chmod"
 # elsewhere if there's a good reason for doing so.
 #
 
-ac_config_files="$ac_config_files make/Makefile make/mkdep Makefile bin/Makefile bin/check/Makefile bin/confgen/Makefile bin/confgen/unix/Makefile bin/delv/Makefile bin/dig/Makefile bin/dnssec/Makefile bin/named/Makefile bin/named/unix/Makefile bin/nsupdate/Makefile bin/pkcs11/Makefile bin/plugins/Makefile bin/python/Makefile bin/python/isc/Makefile bin/python/isc/utils.py bin/python/isc/tests/Makefile bin/python/dnssec-checkds.py bin/python/dnssec-coverage.py bin/python/dnssec-keymgr.py bin/python/isc/__init__.py bin/python/isc/checkds.py bin/python/isc/coverage.py bin/python/isc/dnskey.py bin/python/isc/eventlist.py bin/python/isc/keydict.py bin/python/isc/keyevent.py bin/python/isc/keymgr.py bin/python/isc/keyseries.py bin/python/isc/keyzone.py bin/python/isc/policy.py bin/python/isc/rndc.py bin/python/isc/tests/dnskey_test.py bin/python/isc/tests/policy_test.py bin/rndc/Makefile bin/tests/Makefile bin/tests/headerdep_test.sh bin/tests/optional/Makefile bin/tests/pkcs11/Makefile bin/tests/pkcs11/benchmarks/Makefile bin/tests/system/Makefile bin/tests/system/conf.sh bin/tests/system/dlzexternal/Makefile bin/tests/system/dlzexternal/ns1/dlzs.conf bin/tests/system/dyndb/Makefile bin/tests/system/dyndb/driver/Makefile bin/tests/system/pipelined/Makefile bin/tests/system/rndc/Makefile bin/tests/system/rpz/Makefile bin/tests/system/rsabigexponent/Makefile bin/tests/system/tkey/Makefile bin/tools/Makefile contrib/scripts/check-secure-delegation.pl contrib/scripts/zone-edit.sh doc/Makefile doc/arm/Makefile doc/arm/noteversion.xml doc/arm/pkgversion.xml doc/arm/releaseinfo.xml doc/doxygen/Doxyfile doc/doxygen/Makefile doc/doxygen/doxygen-input-filter doc/misc/Makefile doc/tex/Makefile doc/tex/armstyle.sty doc/xsl/Makefile doc/xsl/isc-docbook-chunk.xsl doc/xsl/isc-docbook-html.xsl doc/xsl/isc-manpage.xsl doc/xsl/isc-notes-html.xsl lib/Makefile lib/bind9/Makefile lib/bind9/include/Makefile lib/bind9/include/bind9/Makefile lib/dns/Makefile lib/dns/include/Makefile lib/dns/include/dns/Makefile lib/dns/include/dst/Makefile lib/dns/tests/Makefile lib/irs/Makefile lib/irs/include/Makefile lib/irs/include/irs/Makefile lib/irs/include/irs/netdb.h lib/irs/include/irs/platform.h lib/irs/tests/Makefile lib/isc/pthreads/Makefile lib/isc/pthreads/include/Makefile lib/isc/pthreads/include/isc/Makefile lib/isc/Makefile lib/isc/include/Makefile lib/isc/include/isc/Makefile lib/isc/include/isc/platform.h lib/isc/include/pk11/Makefile lib/isc/include/pkcs11/Makefile lib/isc/tests/Makefile lib/isc/unix/Makefile lib/isc/unix/include/Makefile lib/isc/unix/include/isc/Makefile lib/isc/unix/include/pkcs11/Makefile lib/isccc/Makefile lib/isccc/include/Makefile lib/isccc/include/isccc/Makefile lib/isccc/tests/Makefile lib/isccfg/Makefile lib/isccfg/include/Makefile lib/isccfg/include/isccfg/Makefile lib/isccfg/tests/Makefile lib/ns/Makefile lib/ns/include/Makefile lib/ns/include/ns/Makefile lib/ns/tests/Makefile lib/samples/Makefile lib/samples/Makefile-postinstall unit/unittest.sh fuzz/Makefile"
+ac_config_files="$ac_config_files make/Makefile make/mkdep Makefile bin/Makefile bin/check/Makefile bin/confgen/Makefile bin/confgen/unix/Makefile bin/delv/Makefile bin/dig/Makefile bin/dnssec/Makefile bin/named/Makefile bin/named/unix/Makefile bin/nsupdate/Makefile bin/pkcs11/Makefile bin/plugins/Makefile bin/python/Makefile bin/python/isc/Makefile bin/python/isc/utils.py bin/python/isc/tests/Makefile bin/python/dnssec-checkds.py bin/python/dnssec-coverage.py bin/python/dnssec-keymgr.py bin/python/isc/__init__.py bin/python/isc/checkds.py bin/python/isc/coverage.py bin/python/isc/dnskey.py bin/python/isc/eventlist.py bin/python/isc/keydict.py bin/python/isc/keyevent.py bin/python/isc/keymgr.py bin/python/isc/keyseries.py bin/python/isc/keyzone.py bin/python/isc/policy.py bin/python/isc/rndc.py bin/python/isc/tests/dnskey_test.py bin/python/isc/tests/policy_test.py bin/rndc/Makefile bin/tests/Makefile bin/tests/headerdep_test.sh bin/tests/optional/Makefile bin/tests/pkcs11/Makefile bin/tests/pkcs11/benchmarks/Makefile bin/tests/system/Makefile bin/tests/system/conf.sh bin/tests/system/dlzexternal/Makefile bin/tests/system/dlzexternal/ns1/dlzs.conf bin/tests/system/dyndb/Makefile bin/tests/system/dyndb/driver/Makefile bin/tests/system/pipelined/Makefile bin/tests/system/rndc/Makefile bin/tests/system/rpz/Makefile bin/tests/system/rsabigexponent/Makefile bin/tests/system/tkey/Makefile bin/tools/Makefile contrib/scripts/check-secure-delegation.pl contrib/scripts/zone-edit.sh doc/Makefile doc/arm/Makefile doc/arm/noteversion.xml doc/arm/pkgversion.xml doc/arm/releaseinfo.xml doc/doxygen/Doxyfile doc/doxygen/Makefile doc/doxygen/doxygen-input-filter doc/misc/Makefile doc/tex/Makefile doc/tex/armstyle.sty doc/xsl/Makefile doc/xsl/isc-docbook-chunk.xsl doc/xsl/isc-docbook-html.xsl doc/xsl/isc-manpage.xsl doc/xsl/isc-notes-html.xsl lib/Makefile lib/bind9/Makefile lib/bind9/include/Makefile lib/bind9/include/bind9/Makefile lib/dns/Makefile lib/dns/include/Makefile lib/dns/include/dns/Makefile lib/dns/include/dst/Makefile lib/dns/tests/Makefile lib/irs/Makefile lib/irs/include/Makefile lib/irs/include/irs/Makefile lib/irs/include/irs/netdb.h lib/irs/include/irs/platform.h lib/irs/tests/Makefile lib/isc/pthreads/Makefile lib/isc/pthreads/include/Makefile lib/isc/pthreads/include/isc/Makefile lib/isc/Makefile lib/isc/include/Makefile lib/isc/include/isc/Makefile lib/isc/include/isc/platform.h lib/isc/include/pk11/Makefile lib/isc/include/pkcs11/Makefile lib/isc/netmgr/Makefile lib/isc/tests/Makefile lib/isc/unix/Makefile lib/isc/unix/include/Makefile lib/isc/unix/include/isc/Makefile lib/isc/unix/include/pkcs11/Makefile lib/isccc/Makefile lib/isccc/include/Makefile lib/isccc/include/isccc/Makefile lib/isccc/tests/Makefile lib/isccfg/Makefile lib/isccfg/include/Makefile lib/isccfg/include/isccfg/Makefile lib/isccfg/tests/Makefile lib/ns/Makefile lib/ns/include/Makefile lib/ns/include/ns/Makefile lib/ns/tests/Makefile lib/samples/Makefile lib/samples/Makefile-postinstall unit/unittest.sh fuzz/Makefile"
 
 
 #
@@ -24456,6 +24528,7 @@ do
     "lib/isc/include/isc/platform.h") CONFIG_FILES="$CONFIG_FILES lib/isc/include/isc/platform.h" ;;
     "lib/isc/include/pk11/Makefile") CONFIG_FILES="$CONFIG_FILES lib/isc/include/pk11/Makefile" ;;
     "lib/isc/include/pkcs11/Makefile") CONFIG_FILES="$CONFIG_FILES lib/isc/include/pkcs11/Makefile" ;;
+    "lib/isc/netmgr/Makefile") CONFIG_FILES="$CONFIG_FILES lib/isc/netmgr/Makefile" ;;
     "lib/isc/tests/Makefile") CONFIG_FILES="$CONFIG_FILES lib/isc/tests/Makefile" ;;
     "lib/isc/unix/Makefile") CONFIG_FILES="$CONFIG_FILES lib/isc/unix/Makefile" ;;
     "lib/isc/unix/include/Makefile") CONFIG_FILES="$CONFIG_FILES lib/isc/unix/include/Makefile" ;;

configure.ac

@@ -641,6 +641,9 @@ AC_CHECK_FUNCS([pthread_setaffinity_np cpuset_setaffinity processor_bind sched_s
 AC_CHECK_FUNCS([pthread_setname_np pthread_set_name_np])
 AC_CHECK_HEADERS([pthread_np.h], [], [], [#include <pthread.h>])
 
+# libuv
+AC_SEARCH_LIBS([uv_run], [uv], [], AC_MSG_ERROR([The uv library was not found. Please install libuv!]))
+
 #
 # flockfile is usually provided by pthreads
 #
@@ -2823,6 +2826,7 @@ AC_CONFIG_FILES([
 	lib/isc/include/isc/platform.h
 	lib/isc/include/pk11/Makefile
 	lib/isc/include/pkcs11/Makefile
+	lib/isc/netmgr/Makefile
 	lib/isc/tests/Makefile
 	lib/isc/unix/Makefile
 	lib/isc/unix/include/Makefile

doc/design/netmgr.md

@@ -0,0 +1,30 @@
+# What is netmgr
+netmgr (aka rainbow duck) is the new networking system for BIND. It's based on libuv, although it does not expose any of the libuv api - to keep the API agnostic of underlying library.
+## A bit of history
+Networking in BIND9 up to 9.12 works with a single event loop (epoll on Linux, kqueue on fbsd, etc).
+
+When a client wants to read from a socket it creates a socketevent with a task that will receive this event. isc_socket_{read,write,etc.} operation tries to read directly from the socket, if it succeeds it sends the socketevent to the task provided by the callee. If it doesn't it adds an event to an event loop, and when this event is received the listener is re-set, and an internal task is launched to read the data from the socket. After the internal task is done it launches the task from socketevent provided by the callee. This means that a simple socket operation causes a lot of context switches.
+
+9.14 fixed some of these issues by having multiple event loops in separate threads (one per CPU), that read the data immediately and then call the socketevent - which is still sub-optimal.
+
+## Internals - event loops
+Internally new 
+
+
+## Basic concepts
+
+### `isc_nm_t`
+
+### `isc_nmsocket_t`
+nmsocket is a wrapper around a libuv socket
+
+### `isc_nmhandle_t`
+nmhandle is an interface that can be read or written - for TCP it's just a socket, for UDP it's a socket with peer address. The idea is that he client should not care what the underlying transport is.
+
+## UDP listening
+
+## TCP listening
+
+## TCP listening for DNS
+
+

lib/dns/acl.c

@@ -182,6 +182,7 @@ dns_acl_match(const isc_netaddr_t *reqaddr,
 	isc_result_t result;
 	int match_num = -1;
 	unsigned int i;
+	INSIST(DNS_ACL_VALID(acl));
 
 	REQUIRE(reqaddr != NULL);
 	REQUIRE(matchelt == NULL || *matchelt == NULL);

lib/dns/tests/dnstest.c

@@ -116,7 +116,7 @@ create_managers(void) {
 	isc_result_t result;
 	ncpus = isc_os_ncpus();
 
-	CHECK(isc_taskmgr_create(dt_mctx, ncpus, 0, &taskmgr));
+	CHECK(isc_taskmgr_create(dt_mctx, ncpus, 0, NULL, &taskmgr));
 	CHECK(isc_timermgr_create(dt_mctx, &timermgr));
 	CHECK(isc_socketmgr_create(dt_mctx, &socketmgr));
 	CHECK(isc_task_create(taskmgr, 0, &maintask));

lib/isc/Makefile.in

@@ -54,6 +54,8 @@ OBJS =		pk11.@O@ pk11_result.@O@ \
 		httpd.@O@ iterated_hash.@O@ \
 		lex.@O@ lfsr.@O@ lib.@O@ log.@O@ \
 		md.@O@ mem.@O@ mutexblock.@O@ \
+		netmgr/netmgr.@O@ netmgr/tcp.@O@ netmgr/udp.@O@ \
+		netmgr/tcpdns.@O@ \
 		netaddr.@O@ netscope.@O@ nonce.@O@ openssl_shim.@O@ pool.@O@ \
 		parseint.@O@ portset.@O@ quota.@O@ radix.@O@ random.@O@ \
 		ratelimiter.@O@ region.@O@ regex.@O@ result.@O@ \
@@ -86,7 +88,7 @@ LIBS =		${OPENSSL_LIBS} @LIBS@
 # Attempt to disable parallel processing.
 .NOTPARALLEL:
 .NO_PARALLEL:
-SUBDIRS =	include unix pthreads
+SUBDIRS =	include netmgr unix pthreads
 TARGETS =	timestamp
 TESTDIRS =	@UNITTESTS@
 

lib/isc/faaa_queue.c

@@ -0,0 +1,108 @@
+#include <stdalign.h>
+
+#include <isc/atomic.h>
+#include <isc/string.h>
+#include <isc/mem.h>
+#include <isc/hp.h>
+
+#define BUFFER_SIZE 1024
+
+#define MAX_THREADS 128
+
+static uintptr_t taken_uint = UINTPTR_MAX;;
+
+typedef struct node {
+	atomic_uint_fast32_t deqidx;
+	atomic_uintptr_t items[BUFFER_SIZE];
+	atomic_uint_fast32_t enqidx;
+	atomic_uintptr_t next;
+	isc_mem_t *mctx;
+} node_t;
+
+/* we just need one Hazard Pointer */
+#define HP_TAIL 0
+#define HP_HEAD 0
+
+struct isc_faaa_queue {
+	alignas(128) atomic_uintptr_t head;
+	alignas(128) atomic_uintptr_t tail;
+	isc_mem_t *mctx;
+	int max_threads;
+	uintptr_t *taken;
+	isc_hp_t *hp;
+};
+
+typedef struct isc_faaa_queue isc_faaa_queue_t;
+
+static node_t *
+node_new(isc_mem_t *mctx, uintptr_t item) {
+	node_t *node = isc_mem_get(mctx, sizeof(*node));
+	*node = (node_t){
+		.deqidx = 0,
+		.enqidx = 1,
+		.next = 0,
+		.items = { item, 0 }
+	};
+	isc_mem_attach(mctx, &node->mctx);
+	return (node);
+}
+
+static void
+node_destroy(void *node0) {
+	node_t *node = (node_t *)node0;
+	isc_mem_putanddetach(&node->mctx, node, sizeof(*node));
+}
+
+static bool
+node_cas_next(atomic_uintptr_t node, uintptr_t cmp, const node_t *val) {
+	return (atomic_compare_exchange_strong(&node, &cmp, (uintptr_t)val));
+}
+
+bool
+queue_cas_tail(isc_faaa_queue_t *queue, uintptr_t cmp, const node_t *val) {
+	return (atomic_compare_exchange_strong(&queue->tail, &cmp, (uintptr_t)val));
+}
+
+bool
+queue_cas_head(isc_faaa_queue_t *queue, uintptr_t cmp, const node_t *val) {
+	return (atomic_compare_exchange_strong(&queue->head, &cmp, (uintptr_t)val));
+}
+
+isc_faaa_queue_t *
+isc_faaa_queue_new(isc_mem_t *mctx, int max_threads) {
+	isc_faaa_queue_t *queue = isc_mem_get(mctx, sizeof(*queue));
+	node_t *sentinel = node_new(mctx, (uintptr_t)NULL);
+
+	if (max_threads == 0) {
+		max_threads = MAX_THREADS;
+	}
+
+	*queue = (isc_faaa_queue_t){
+		.max_threads = max_threads,
+		.taken = &taken_uint,
+	};
+
+	isc_mem_attach(mctx, &queue->mctx);
+
+	queue->hp = isc_hp_new(mctx, 0, max_threads, node_destroy);
+
+	atomic_init(&sentinel->enqidx, 0);
+	atomic_init(&queue->head, (uintptr_t)sentinel);
+	atomic_init(&queue->tail, (uintptr_t)sentinel);
+
+	return (queue);
+}
+
+uintptr_t
+isc_faaa_queue_dequeue(isc_faaa_queue_t *queue, const int tid);
+
+void
+isc_faaa_queue_destroy(isc_faaa_queue_t *queue) {
+	node_t *last;
+
+	while (isc_faaa_queue_dequeue(queue, 0) != (uintptr_t)NULL);
+	last = (node_t *)atomic_load_relaxed(&queue->head);
+	node_destroy(last);
+	isc_hp_destroy(queue->hp);
+	isc_mem_putanddetach(&queue->mctx, queue, sizeof(*queue));
+}

lib/isc/hp.c

@@ -0,0 +1,187 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+/*
+ * This work is based on C++ code available from:
+ * https://github.com/pramalhe/ConcurrencyFreaks/
+ *
+ * Copyright (c) 2014-2016, Pedro Ramalhete, Andreia Correia
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Concurrency Freaks nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ ******************************************************************************
+ */
+
+#include <isc/atomic.h>
+#include <isc/hp.h>
+#include <isc/string.h>
+#include <isc/mem.h>
+#include <isc/util.h>
+
+#define HP_MAX_THREADS 128
+#define HP_MAX_HPS 4 /* This is named 'K' in the HP paper */
+#define CLPAD (128 / sizeof(uintptr_t))
+#define HP_THRESHOLD_R 0 /* This is named 'R' in the HP paper */
+#define MAX_RETIRED (HP_MAX_THREADS * HP_MAX_HPS) /* Maximum number of retired objects per thread */
+
+struct isc__hp_rl {
+	int size;
+	uintptr_t list[MAX_RETIRED];
+};
+
+struct isc_hp {
+	int max_hps;
+	int max_threads;
+	isc_mem_t *mctx;
+	atomic_uintptr_t *hp[HP_MAX_THREADS];
+	struct isc__hp_rl *rl[HP_MAX_THREADS*CLPAD];
+	isc_hp_deletefunc_t *deletefunc;
+};
+
+isc_hp_t *
+isc_hp_new(isc_mem_t *mctx, size_t max_hps, size_t max_threads, isc_hp_deletefunc_t *deletefunc) {
+	isc_hp_t *hp = isc_mem_get(mctx, sizeof(*hp));
+
+	hp->max_hps = HP_MAX_HPS;
+	hp->max_threads = HP_MAX_THREADS;
+	isc_mem_attach(mctx, &hp->mctx);
+
+	*hp = (isc_hp_t){ .max_hps = max_hps,
+			  .max_threads = max_threads,
+			  .deletefunc = deletefunc };
+
+	for (int i = 0; i < hp->max_threads; i++) {
+		hp->hp[i] = isc_mem_get(mctx, CLPAD * 2 * sizeof(hp->hp[i][0]));
+		hp->rl[i*CLPAD] = isc_mem_get(mctx, sizeof(hp->rl[0]));
+		for (int j = 0; j < hp->max_hps; j++) {
+			atomic_init(&hp->hp[i][j], 0);
+		}
+	}
+	return (hp);
+}
+
+void
+isc_hp_destroy(isc_hp_t *hp) {
+	for (int i = 0; i < hp->max_threads; i++) {
+		isc_mem_put(hp->mctx, hp->hp[i], CLPAD * 2 * sizeof(uintptr_t));
+
+		for (int j = 0; j < hp->rl[i*CLPAD]->size; j++) {
+			void *data = (void *)hp->rl[i*CLPAD]->list[j];
+			hp->deletefunc(data);
+		}
+	}
+	isc_mem_putanddetach(&hp->mctx, hp, sizeof(*hp));
+}
+
+void
+isc_hp_clear(isc_hp_t *hp, const int tid) {
+	REQUIRE(tid < hp->max_threads);
+	for (int i = 0; i < hp->max_hps; i++) {
+		atomic_store_release(&hp->hp[tid][i], 0);
+	}
+}
+
+
+/**
+ * Progress Condition: wait-free population oblivious
+ */
+void isc_hp_clear_one(isc_hp_t *hp, int ihp, const int tid) {
+	atomic_store_release(&hp->hp[tid][ihp], 0);
+}
+
+/**
+ * Progress Condition: lock-free
+ */
+uintptr_t
+isc_hp_protect(isc_hp_t *hp, int ihp, const atomic_uintptr_t atom, const int tid) {
+	uintptr_t n = 0;
+	uintptr_t ret;
+	while ((ret = atomic_load(&atom)) != n) {
+		atomic_store(&hp->hp[tid][ihp], ret);
+		n = ret;
+	}
+	return (ret);
+}
+
+/**
+ * This returns the same value that is passed as ptr, which is sometimes useful
+ * Progress Condition: wait-free population oblivious
+ */
+uintptr_t
+isc_hp_protect_ptr(isc_hp_t *hp, int ihp, const atomic_uintptr_t ptr, const int tid) {
+	atomic_store(&hp->hp[tid][ihp], ptr);
+	return (ptr);
+}
+
+/**
+ * This returns the same value that is passed as ptr, which is sometimes useful
+ * Progress Condition: wait-free population oblivious
+ */
+uintptr_t
+isc_hp_protect_release(isc_hp_t *hp, int ihp, const atomic_uintptr_t ptr, const int tid) {
+	atomic_store_release(&hp->hp[tid][ihp], ptr);
+	return (ptr);
+}
+
+/**
+ * Progress Condition: wait-free bounded (by the number of threads squared)
+ */
+void
+isc_hp_retire(isc_hp_t *hp, const atomic_uintptr_t ptr, const int tid) {
+	hp->rl[tid*CLPAD]->list[hp->rl[tid*CLPAD]->size++] = ptr;
+	INSIST(hp->rl[tid*CLPAD]->size < MAX_RETIRED);
+
+	if (hp->rl[tid*CLPAD]->size < HP_THRESHOLD_R) {
+		return;
+	}
+
+	for (int iret = 0; iret < hp->rl[tid*CLPAD]->size;) {
+		uintptr_t obj = hp->rl[tid*CLPAD]->list[iret];
+		bool can_delete = true;
+		for (int itid = 0; itid < hp->max_threads && can_delete; itid++) {
+			for (int ihp = hp->max_hps-1; ihp >= 0; ihp--) {
+				if (atomic_load(&hp->hp[itid][ihp]) == obj) {
+					can_delete = false;
+					break;
+				}
+			}
+		}
+		if (can_delete) {
+			size_t bytes = (hp->rl[tid*CLPAD]->size - iret) *
+				sizeof(hp->rl[tid*CLPAD]->list[0]);
+			memmove(&hp->rl[tid*CLPAD]->list[iret],
+				&hp->rl[tid*CLPAD]->list[iret + 1],
+				bytes);
+			hp->rl[tid*CLPAD]->size--;
+			hp->deletefunc((void *)obj);
+		}
+		iret++;
+	}
+}

lib/isc/include/isc/hp.h

@@ -0,0 +1,73 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+/*
+ * This work is based on C++ code available from:
+ * https://github.com/pramalhe/ConcurrencyFreaks/
+ *
+ * Copyright (c) 2014-2016, Pedro Ramalhete, Andreia Correia
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Concurrency Freaks nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ ******************************************************************************
+ */
+
+#include <isc/atomic.h>
+#include <isc/string.h>
+#include <isc/mem.h>
+#include <isc/util.h>
+
+typedef void
+(isc_hp_deletefunc_t)(void *);
+
+typedef struct isc_hp isc_hp_t;
+
+isc_hp_t *
+isc_hp_new(isc_mem_t *mctx, size_t max_hps, size_t max_threads, isc_hp_deletefunc_t *deletefunc);
+
+void
+isc_hp_destroy(isc_hp_t *hp);
+
+void
+isc_hp_clear(isc_hp_t *hp, const int tid);
+
+void isc_hp_clear_one(isc_hp_t *hp, int ihp, const int tid);
+
+uintptr_t
+isc_hp_protect(isc_hp_t *hp, int ihp, const atomic_uintptr_t atom, const int tid);
+
+uintptr_t
+isc_hp_protect_ptr(isc_hp_t *hp, int ihp, const atomic_uintptr_t ptr, const int tid);
+
+uintptr_t
+isc_hp_protect_release(isc_hp_t *hp, int ihp, const atomic_uintptr_t ptr, const int tid);
+
+void
+isc_hp_retire(isc_hp_t *hp, const atomic_uintptr_t ptr, const int tid);

lib/isc/include/isc/netmgr.h

@@ -0,0 +1,211 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+#pragma once
+
+#include <config.h>
+
+#include <isc/mem.h>
+#include <isc/result.h>
+
+typedef struct isc_nm isc_nm_t;
+typedef struct isc_nmsocket isc_nmsocket_t;
+typedef struct isc_nmiface isc_nmiface_t;
+typedef struct isc_nmhandle isc_nmhandle_t;
+
+typedef enum {
+	NMEV_READ,
+	NMEV_WRITE,
+	NMEV_ACCEPT,
+	NMEV_CONNECTED,
+	NMEV_CANCELLED,
+	NMEV_SHUTDOWN
+} isc_nmev_type;
+
+/*
+ * isc_nm_start creates and starts a netmgr
+ */
+isc_nm_t*
+isc_nm_start(isc_mem_t *mctx, int workers);
+
+/*
+ * isc_nm_shutdown shutdowns netmgr, freeing all the resources
+ */
+void
+isc_nm_shutdown(isc_nm_t**mgr);
+
+void
+isc_nm_attach(isc_nm_t *mgr, isc_nm_t **dst);
+
+void
+isc_nm_detach(isc_nm_t **mgr0);
+
+/* Return thread id of current thread, or ISC_NETMGR_TID_UNKNOWN */
+int
+isc_nm_tid(void);
+
+/*
+ * isc_nm_freehandle frees a handle, releasing resources
+ */
+void
+isc_nm_freehandle(isc_nmhandle_t *handle);
+
+/*
+ * isc_nmsocket_attach attaches to a socket, increasing refcount
+ */
+void
+isc_nmsocket_attach(isc_nmsocket_t *socket, isc_nmsocket_t **target);
+
+void
+isc_nmsocket_close(isc_nmsocket_t *socket);
+/*
+ * isc_nmsocket_detach detaches from socket, decreasing refcount
+ * and possibly destroying the socket if it's no longer referenced.
+ */
+void
+isc_nmsocket_detach(isc_nmsocket_t **socketp);
+
+void
+isc_nmhandle_attach(isc_nmhandle_t *handle, isc_nmhandle_t **handlep);
+
+void
+isc_nmhandle_detach(isc_nmhandle_t **handlep);
+
+void *
+isc_nmhandle_getdata(isc_nmhandle_t *handle);
+
+void*
+isc_nmhandle_getextra(isc_nmhandle_t *handle);
+
+typedef void (*isc_nm_opaquecb)(void *arg);
+
+bool
+isc_nmhandle_is_stream(isc_nmhandle_t *handle);
+
+/*
+ * isc_nmhandle_t has a void* opaque field (usually - ns_client_t).
+ * We reuse handle and `opaque` can also be reused between calls.
+ * This function sets this field and two callbacks:
+ * - doreset resets the `opaque` to initial state
+ * - dofree frees everything associated with `opaque`
+ */
+void
+isc_nmhandle_setdata(isc_nmhandle_t *handle,
+		     void *arg,
+		     isc_nm_opaquecb doreset,
+		     isc_nm_opaquecb dofree);
+
+isc_sockaddr_t
+isc_nmhandle_peeraddr(isc_nmhandle_t *handle);
+
+/*
+ * Callback for receiving a packet.
+ * arg is the argument passed to isc_nm_listen_udp
+ * handle - handle that can be used to send back the answer
+ * region - contains the received data. It will be freed after
+ *          return by caller
+ */
+typedef void (*isc_nm_recv_cb_t)(void *arg, isc_nmhandle_t*handle,
+				 isc_region_t *region);
+
+/*
+ * isc_nm_udp_listen starts listening for UDP packets on iface using mgr.
+ * When a packet is received cb is called with cbarg as its first argument
+ */
+isc_result_t
+isc_nm_udp_listen(isc_nm_t *mgr,
+		  isc_nmiface_t *iface,
+		  isc_nm_recv_cb_t cb,
+		  size_t extrasize,
+		  void *cbarg,
+		  isc_nmsocket_t **rv);
+
+void
+isc_nm_udp_stoplistening(isc_nmsocket_t *socket);
+
+
+/* XXXWPK TODOs */
+typedef void (*isc_nm_send_cb_t)(isc_nmhandle_t *handle, isc_result_t result,
+				 void *cbarg);
+typedef void (*isc_nm_connect_cb_t)(isc_nmhandle_t *handle,
+				    isc_result_t result, void *cbarg);
+typedef void (*isc_nm_accept_cb_t)(isc_nmhandle_t *handle, isc_result_t result,
+				   void *cbarg);
+
+isc_result_t
+isc_nm_listen_tcp(isc_nm_t *mgr, isc_nm_accept_cb_t *cb);
+
+void
+isc_nm_tcp_stoplistening(isc_nmsocket_t *socket);
+
+/*
+ * isc_nm_pause pauses all processing, equivalent to taskmgr exclusive tasks.
+ * It won't return until all workers are paused.
+ */
+void
+isc_nm_pause(isc_nm_t *mgr);
+
+/*
+ * isc_nm_resume resumes paused processing. It will return immediately
+ * after signalling workers to resume.
+ */
+void
+isc_nm_resume(isc_nm_t *mgr);
+
+isc_result_t
+isc_nm_tcp_connect(isc_nm_t*mgr,
+		   isc_nmiface_t *iface,
+		   isc_sockaddr_t *peer,
+		   isc_nm_connect_cb_t cb,
+		   void *cbarg);
+
+isc_nmsocket_t*
+isc_nm_udp_socket();
+
+isc_result_t
+isc_nm_dnsread(isc_nmsocket_t *socket, isc_buffer_t*buf);
+
+
+isc_result_t
+isc_nm_read(isc_nmhandle_t *handle, isc_nm_recv_cb_t cb, void*cbarg);
+
+/*
+ * isc_nm_send sends region to handle, after finishing cb is called.
+ * region is not copied, it has to be allocated beforehand and freed in cb.
+ * Callback can be invoked directly from the calling thread, or called later.
+ */
+isc_result_t
+isc_nm_send(isc_nmhandle_t *handle,
+	    isc_region_t *region,
+	    isc_nm_send_cb_t cb,
+	    void *cbarg);
+
+isc_result_t
+isc_nm_tcp_listen(isc_nm_t *mgr,
+		  isc_nmiface_t *iface,
+		  isc_nm_accept_cb_t cb,
+		  size_t extrahandlesize,
+		  void *cbarg,
+		  isc_quota_t *quota,
+		  isc_nmsocket_t **rv);
+
+
+isc_result_t
+isc_nm_tcp_dnslisten(isc_nm_t *mgr,
+		     isc_nmiface_t *iface,
+		     isc_nm_recv_cb_t cb,
+		     size_t extrahandlesize,
+		     void *arg,
+		     isc_quota_t *quota,
+		     isc_nmsocket_t **rv);
+
+void
+isc_nm_tcpdns_stoplistening(isc_nmsocket_t *socket);

lib/isc/include/isc/sockaddr.h

@@ -230,6 +230,10 @@ isc_sockaddr_frompath(isc_sockaddr_t *sockaddr, const char *path);
  * \li	ISC_R_SUCCESS
  */
 
+isc_result_t
+isc_sockaddr_fromsockaddr(isc_sockaddr_t *isa, const struct sockaddr *sa);
+
+
 #define ISC_SOCKADDR_FORMATSIZE \
 	sizeof("xxxx:xxxx:xxxx:xxxx:xxxx:xxxx:XXX.XXX.XXX.XXX%SSSSSSSSSS#YYYYY")
 /*%<

lib/isc/include/isc/task.h

@@ -78,6 +78,7 @@
 #include <isc/eventclass.h>
 #include <isc/lang.h>
 #include <isc/stdtime.h>
+#include <isc/netmgr.h>
 #include <isc/types.h>
 
 #define ISC_TASKEVENT_FIRSTEVENT	(ISC_EVENTCLASS_TASK + 0)
@@ -544,6 +545,8 @@ isc_task_beginexclusive(isc_task_t *task);
  * task.  Waits for any other concurrently executing tasks to finish their
  * current event, and prevents any new events from executing in any of the
  * tasks sharing a task manager with 'task'.
+ * It also pauses processing of network events in netmgr if it was provided
+ * when taskmgr was created.
  *
  * The exclusive access must be relinquished by calling
  * isc_task_endexclusive() before returning from the current event handler.
@@ -568,6 +571,11 @@ isc_task_endexclusive(isc_task_t *task);
  *		exclusive access by calling isc_task_spl().
  */
 
+void
+isc_task_pause(isc_task_t *task0);
+void
+isc_task_unpause(isc_task_t *task0);
+
 void
 isc_task_getcurrenttime(isc_task_t *task, isc_stdtime_t *t);
 void
@@ -633,7 +641,8 @@ isc_taskmgr_createinctx(isc_mem_t *mctx,
 			isc_taskmgr_t **managerp);
 isc_result_t
 isc_taskmgr_create(isc_mem_t *mctx, unsigned int workers,
-		   unsigned int default_quantum, isc_taskmgr_t **managerp);
+		   unsigned int default_quantum,
+		   isc_nm_t *nm, isc_taskmgr_t **managerp);
 /*%<
  * Create a new task manager.  isc_taskmgr_createinctx() also associates
  * the new manager with the specified application context.
@@ -650,6 +659,9 @@ isc_taskmgr_create(isc_mem_t *mctx, unsigned int workers,
  *	quantum value when tasks are created.  If zero, then an implementation
  *	defined default quantum will be used.
  *
+ *\li	If 'nm' is set then netmgr is paused when an exclusive task mode
+ *	is requested.
+ *
  * Requires:
  *
  *\li      'mctx' is a valid memory context.

lib/isc/netmgr/Makefile.in

@@ -0,0 +1,33 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+srcdir =	@srcdir@
+VPATH =		@srcdir@
+top_srcdir =	@top_srcdir@
+
+CINCLUDES =	-I${srcdir}/../include \
+		-I${srcdir}/../unix/include \
+		-I${srcdir}/../pthreads/include \
+		-I${srcdir}/.. \
+		${OPENSSL_CFLAGS} \
+		${JSON_C_CFLAGS} \
+		${LIBXML2_CFLAGS}
+
+CDEFINES =
+CWARNINGS =
+
+# Alphabetically
+OBJS =		netmgr.@O@ tcp.@O@ udp.@O@ tcpdns.@O@
+
+# Alphabetically
+SRCS =		netmgr.c tcp.c udp.c tcpdns.c
+
+TARGETS =	${OBJS}
+
+@BIND9_MAKE_RULES@

lib/isc/netmgr/netmgr-int.h

@@ -0,0 +1,460 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+#include "config.h"
+
+#include <unistd.h>
+#include <uv.h>
+#include <ck_fifo.h>
+#include <ck_stack.h>
+
+#include <isc/atomic.h>
+#include <isc/buffer.h>
+#include <isc/condition.h>
+#include <isc/magic.h>
+#include <isc/mem.h>
+#include <isc/netmgr.h>
+#include <isc/random.h>
+#include <isc/refcount.h>
+#include <isc/region.h>
+#include <isc/result.h>
+#include <isc/sockaddr.h>
+#include <isc/thread.h>
+#include <isc/util.h>
+
+#define ISC_NETMGR_TID_UNKNOWN -1
+#define ISC_NETMGR_TID_NOTLS -2
+
+/*
+ * Single network event loop worker.
+ */
+typedef struct isc__networker {
+	isc_nm_t *		   mgr;
+	int			   id;          /* thread id */
+	uv_loop_t		   loop;        /* libuv loop structure */
+	uv_async_t		   async;       /* async channel to send
+						 * data to this networker */
+	isc_mutex_t		   lock;
+	isc_mempool_t *		   mpool_bufs;
+	isc_condition_t		   cond;
+	bool			   paused;
+	bool			   finished;
+	isc_thread_t		   thread;
+	struct ck_fifo_mpmc	   ievents;     /* incoming async events */
+	isc_refcount_t		   references;
+	atomic_int_fast64_t	   pktcount;
+	char			   udprecvbuf[65536];
+	bool			   udprecvbuf_inuse;
+} isc__networker_t;
+
+/*
+ * A general handle for a connection bound to a networker.
+ * For UDP connections we have peer address here,
+ * so both TCP and UDP can be handled with a simple send-like
+ * function
+ */
+#define NMHANDLE_MAGIC                        ISC_MAGIC('N', 'M', 'H', 'D')
+#define VALID_NMHANDLE(t)                     ISC_MAGIC_VALID(t, \
+							      NMHANDLE_MAGIC)
+
+typedef void (*isc__nm_closecb)(isc_nmhandle_t*);
+
+struct isc_nmhandle {
+	int		      magic;
+	isc_refcount_t	      references;
+	/* The socket is not 'attached' in the traditional reference-counting
+	 * sense. Instead, we keep all handles in an array in the socket object.
+	 * This way, we don't have circular dependencies and we can close all
+	 * handles when we're destroying the socket. */
+	isc_nmsocket_t *      socket;
+	size_t		      ah_pos;    /* Position in socket active handles
+					  * array */
+	/* The handle is 'inflight' if netmgr is not currently processing it in
+	 * any way - it might mean that e.g. a recursive resolution is
+	 * happening. For an inflight handle we must wait for the calling
+	 * code to finish before we can free it. */
+	atomic_bool		inflight;
+	isc_sockaddr_t		peer;
+	ck_stack_entry_t	ilink;
+	isc_nm_opaquecb		doreset; /* reset extra callback, external */
+	isc_nm_opaquecb		dofree;  /* free extra callback, external */
+	void *			opaque;
+	char			extra[];
+};
+
+CK_STACK_CONTAINER(struct isc_nmhandle, ilink, nm_handle_is_get)
+
+/*
+ * An interface - an address we can listen on.
+ */
+struct isc_nmiface {
+	isc_sockaddr_t        addr;
+};
+
+typedef enum isc__netievent_type {
+	netievent_stop,
+	netievent_udplisten,
+	netievent_udpstoplisten,
+	netievent_udpsend,
+	netievent_udprecv,
+	netievent_tcpconnect,
+	netievent_tcpsend,
+	netievent_tcprecv,
+	netievent_tcpstartread,
+	netievent_tcpstopread,
+	netievent_tcplisten,
+	netievent_tcpstoplisten,
+	netievent_tcpclose,
+} isc__netievent_type;
+
+typedef struct isc__netievent_stop {
+	isc__netievent_type        type;
+} isc__netievent_stop_t;
+
+/* We have to split it because we can read and write on a socket simultaneously
+ * */
+typedef union {
+	isc_nm_recv_cb_t	  recv;
+	isc_nm_accept_cb_t	  accept;
+} isc__nm_readcb_t;
+
+typedef union {
+	isc_nm_send_cb_t	   send;
+	isc_nm_connect_cb_t	   connect;
+} isc__nm_writecb_t;
+
+typedef union {
+	isc_nm_recv_cb_t	   recv;
+	isc_nm_accept_cb_t	   accept;
+	isc_nm_send_cb_t	   send;
+	isc_nm_connect_cb_t	   connect;
+} isc__nm_cb_t;
+
+/*
+ * Wrapper around uv_req_t with 'our' fields in it.
+ * req->data should always point to it's parent.
+ * Note that we always allocate more than sizeof(struct)
+ * because we make room for different req types;
+ */
+#define UVREQ_MAGIC                        ISC_MAGIC('N', 'M', 'U', 'R')
+#define VALID_UVREQ(t)                     ISC_MAGIC_VALID(t, UVREQ_MAGIC)
+
+typedef struct isc__nm_uvreq {
+	int			magic;
+	isc_nm_t *		mgr;
+	uv_buf_t		uvbuf; /* translated isc_region_t, to be sent or
+					* received */
+	isc_sockaddr_t		local; /* local address */
+	isc_sockaddr_t		peer; /* peer address */
+	isc__nm_cb_t		cb;  /* callback */
+	void *			cbarg;
+	isc_nmhandle_t *	handle;
+	ck_stack_entry_t	ilink;
+	union {
+		uv_req_t		req;
+		uv_getaddrinfo_t	getaddrinfo;
+		uv_getnameinfo_t	getnameinfo;
+		uv_shutdown_t		shutdown;
+		uv_write_t		write;
+		uv_connect_t		connect;
+		uv_udp_send_t		udp_send;
+		uv_fs_t			fs;
+		uv_work_t		work;
+	} uv_req;
+} isc__nm_uvreq_t;
+
+CK_STACK_CONTAINER(struct isc__nm_uvreq, ilink, uvreq_is_get);
+
+/*
+ * TODO: unify the events.
+ */
+
+typedef struct isc__netievent_udplisten {
+	isc__netievent_type	   type;
+	isc_nmsocket_t *	   socket;
+} isc__netievent_udplisten_t;
+
+typedef struct isc__netievent_udpstoplisten {
+	isc__netievent_type	   type;
+	isc_nmsocket_t *	   socket;
+} isc__netievent_udpstoplisten_t;
+
+typedef struct isc__netievent_udpsend {
+	isc__netievent_type	   type;
+	isc_nmsocket_t *	   socket;
+	isc_sockaddr_t		   peer;
+	isc__nm_uvreq_t *	   req;
+} isc__netievent_udpsend_t;
+
+typedef struct isc__netievent_tcpconnect {
+	isc__netievent_type	   type;
+	isc_nmsocket_t *	   socket;
+	isc__nm_uvreq_t *	   req;
+} isc__netievent_tcpconnect_t;
+
+typedef struct isc__netievent_tcplisten {
+	isc__netievent_type	   type;
+	isc_nmsocket_t *	   socket;
+	isc__nm_uvreq_t *	   req;
+} isc__netievent_tcplisten_t;
+
+typedef struct isc__netievent_tcpstoplisten {
+	isc__netievent_type	   type;
+	isc_nmsocket_t *	   socket;
+} isc__netievent_tcpstoplisten_t;
+
+typedef struct isc__netievent_tcpclose {
+	isc__netievent_type	   type;
+	isc_nmsocket_t *	   socket;
+	isc__nm_uvreq_t *	   req;
+} isc__netievent_tcpclose_t;
+
+typedef struct isc__netievent_tcpsend {
+	isc__netievent_type	   type;
+	isc_nmsocket_t *	   socket;
+	isc__nm_uvreq_t *	   req;
+} isc__netievent_tcpsend_t;
+
+typedef struct isc__netievent_startread {
+	isc__netievent_type	   type;
+	isc_nmsocket_t *	   socket;
+	isc__nm_uvreq_t *	   req;
+} isc__netievent_startread_t;
+
+typedef struct isc__netievent {
+	isc__netievent_type        type;
+} isc__netievent_t;
+
+typedef struct isc__netievent_storage {
+	union {
+		isc__netievent_t		  ni;
+		isc__netievent_stop_t		  nis;
+		isc__netievent_udplisten_t	  niul;
+		isc__netievent_udpsend_t	  nius;
+	};
+} isc__netievent_storage_t;
+
+/*
+ * Network manager
+ */
+#define NM_MAGIC                        ISC_MAGIC('N', 'E', 'T', 'M')
+#define VALID_NM(t)                     ISC_MAGIC_VALID(t, NM_MAGIC)
+
+struct isc_nm {
+	int			    magic;
+	isc_refcount_t		    references;
+	isc_mem_t *		    mctx;
+	uint32_t		    nworkers;
+	isc_mutex_t		    lock;
+	isc_condition_t		    wkstatecond;
+	isc__networker_t *	    workers;
+	atomic_uint_fast32_t	    workers_running;
+	atomic_uint_fast32_t	    workers_paused;
+};
+
+
+typedef enum isc_nmsocket_type {
+	isc_nm_udpsocket,
+	isc_nm_udplistener, /* Aggregate of nm_udpsocks */
+	isc_nm_tcpsocket,
+	isc_nm_tcplistener,
+	isc_nm_tcpdnslistener,
+	isc_nm_tcpdnssocket
+} isc_nmsocket_type;
+
+
+/*
+ * An universal structure for either a single socket or
+ * a group of dup'd/SO_REUSE_PORT-using sockets listening
+ * on the same interface.
+ */
+#define NMSOCK_MAGIC                    ISC_MAGIC('N', 'M', 'S', 'K')
+#define VALID_NMSOCK(t)                 ISC_MAGIC_VALID(t, NMSOCK_MAGIC)
+struct isc_nmsocket {
+	/* Unlocked, RO */
+	int			   magic;
+	int			   tid;
+	isc_nmsocket_type	   type;
+	isc_nm_t *		   mgr;
+	isc_nmsocket_t *	   parent;
+	isc_quota_t *		   quota;
+	bool			   overquota;
+	/* outer socket is for 'wrapped' sockets - e.g. tcpdns in tcp */
+	isc_nmsocket_t *	   outer;
+	/* server socket for connections */
+	isc_nmsocket_t *	   server;
+	/* children sockets for multi-socket setups */
+	isc_nmsocket_t *	   children;
+	int			   nchildren;
+	isc_nmiface_t *		   iface;
+	isc_nmhandle_t *	   tcphandle;
+
+	/* extra data allocated at the end of each isc_nmhandle_t */
+	size_t			   extrahandlesize;
+
+	/* libuv data */
+	uv_os_sock_t		   fd;
+	union uv_any_handle	   uv_handle;
+
+	/* Atomic */
+	/* Number of running (e.g. listening) children sockets */
+	atomic_int_fast32_t        rchildren;
+	/*
+	 * Socket if active if it's listening, working, etc., if we're
+	 * closing a socket it doesn't make any sense to e.g. still
+	 * push handles or reqs for reuse
+	 */
+	atomic_bool        active;
+	/*
+	 * Socket is closed if it's not active and all the possible callbacks
+	 * were fired, there are no active handles, etc.
+	 * active==false, closed == false means the socket is closing.
+	 */
+	atomic_bool	      closed;
+	atomic_bool	      listening;
+	isc_refcount_t	      references;
+	/*
+	 * 'spare' handles for that can be reused to avoid allocations,
+	 * for UDP.
+	 */
+	ck_stack_t inactivehandles	  CK_CC_CACHELINE;
+	ck_stack_t inactivereqs		  CK_CC_CACHELINE;
+
+	/* Used for active/rchildren during shutdown */
+	isc_mutex_t			  lock;
+	isc_condition_t			  cond;
+
+	/*
+	 * List of active handles.
+	 * ah_size - size of ah_frees and ah_handles
+	 * ah_cpos - current position in ah_frees;
+	 * ah_handles - array of *handles.
+	 * Adding a handle
+	 *  - if ah_cpos == ah_size, realloc
+	 *  - x = ah_frees[ah_cpos]
+	 *  - ah_frees[ah_cpos++] = 0;
+	 *  - ah_handles[x] = handle
+	 *  - x must be stored with the handle!
+	 * Removing a handle:
+	 *  - ah_frees[--ah_cpos] = x
+	 *  - ah_handles[x] = NULL;
+	 *
+	 * XXXWPK for now this is locked with socket->lock, but we might want
+	 * to change it to something lockless
+	 */
+	size_t			ah_size;
+	size_t			ah_cpos;
+	size_t *		ah_frees;
+	isc_nmhandle_t **	ah_handles;
+
+	/* XXXWPK can it be not locked? */
+	isc__nm_readcb_t	rcb;
+	void *			rcbarg;
+};
+
+/* Are we in network thread? */
+bool
+isc__nm_in_netthread(void);
+
+void
+isc__nmhandle_free(isc_nmsocket_t *socket, isc_nmhandle_t *handle);
+
+void *
+isc__nm_get_ievent(isc_nm_t *mgr, isc__netievent_type type);
+
+void
+isc__nm_enqueue_ievent(isc__networker_t *worker, isc__netievent_t *event);
+
+void
+isc__nm_alloc_cb(uv_handle_t *handle, size_t size, uv_buf_t *buf);
+
+void
+isc__nm_free_uvbuf(isc_nmsocket_t *socket, const uv_buf_t *buf);
+
+isc_nmhandle_t *
+isc__nmhandle_get(isc_nmsocket_t *socket, isc_sockaddr_t *peer);
+
+isc__nm_uvreq_t *
+isc__nm_uvreq_get(isc_nm_t *mgr, isc_nmsocket_t *socket);
+
+void
+isc__nm_uvreq_put(isc__nm_uvreq_t **req, isc_nmsocket_t *socket);
+
+void
+isc__nmsocket_init(isc_nmsocket_t *socket,
+		   isc_nm_t *mgr,
+		   isc_nmsocket_type type);
+
+void
+isc__nmsocket_prep_destroy(isc_nmsocket_t *socket);
+
+/*
+ * Send for UDP handle
+ */
+isc_result_t
+isc__nm_udp_send(isc_nmhandle_t *handle,
+		 isc_region_t *region,
+		 isc_nm_send_cb_t cb,
+		 void *cbarg);
+/*
+ * Async callbacks for UDP
+ */
+void
+isc__nm_handle_udplisten(isc__networker_t *worker, isc__netievent_t *ievent0);
+void
+isc__nm_handle_udpstoplisten(isc__networker_t *worker,
+			     isc__netievent_t *ievent0);
+void
+isc__nm_handle_udpsend(isc__networker_t *worker, isc__netievent_t *ievent0);
+
+
+/*
+ * Send for TCP handle
+ */
+isc_result_t
+isc__nm_tcp_send(isc_nmhandle_t *handle,
+		 isc_region_t *region,
+		 isc_nm_send_cb_t cb,
+		 void *cbarg);
+
+void
+isc__nm_tcp_close(isc_nmsocket_t *socket);
+
+
+/*
+ * Async callbacks for TCP
+ */
+
+void
+isc__nm_handle_tcpconnect(isc__networker_t *worker, isc__netievent_t *ievent0);
+void
+isc__nm_handle_tcplisten(isc__networker_t *worker, isc__netievent_t *ievent0);
+void
+isc__nm_handle_tcpstoplistening(isc__networker_t *worker,
+				isc__netievent_t *ievent0);
+void
+isc__nm_handle_tcpsend(isc__networker_t *worker, isc__netievent_t *ievent0);
+void
+isc__nm_handle_startread(isc__networker_t *worker, isc__netievent_t *ievent0);
+void
+isc__nm_handle_tcpclose(isc__networker_t *worker, isc__netievent_t *ievent0);
+/* static void
+ * handle_stopread(isc__networker_t *worker, isc__netievent_t *ievent0);
+ */
+
+
+isc_result_t
+isc__nm_tcpdns_send(isc_nmhandle_t *handle,
+		    isc_region_t *region,
+		    isc_nm_send_cb_t cb,
+		    void *cbarg);
+void
+isc__nm_tcpdns_close(isc_nmsocket_t *socket);

lib/isc/netmgr/netmgr.c

@@ -0,0 +1,808 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+#include "config.h"
+
+#include <unistd.h>
+#include <uv.h>
+#include <ck_fifo.h>
+#include <ck_stack.h>
+
+#include <isc/atomic.h>
+#include <isc/buffer.h>
+#include <isc/condition.h>
+#include <isc/magic.h>
+#include <isc/mem.h>
+#include <isc/netmgr.h>
+#include <isc/quota.h>
+#include <isc/print.h>
+#include <isc/random.h>
+#include <isc/refcount.h>
+#include <isc/region.h>
+#include <isc/result.h>
+#include <isc/sockaddr.h>
+#include <isc/thread.h>
+#include <isc/util.h>
+
+#include "netmgr-int.h"
+/*
+ * libuv is not thread safe but has mechanisms to pass messages
+ * between threads. Each socket is owned by a thread. For UDP
+ * sockets we have a set of sockets for each interface and we can
+ * choose a sibling and send the message directly. For TCP or if
+ * we're calling from a not networking thread we need to pass the
+ * request using async_cb
+ */
+
+#if defined(HAVE_TLS)
+#if defined(HAVE_THREAD_LOCAL)
+#include <threads.h>
+static thread_local int isc__nm_tid_v = ISC_NETMGR_TID_UNKNOWN;
+#elif defined(HAVE___THREAD)
+static __thread int isc__nm_tid_v = ISC_NETMGR_TID_UNKNOWN;
+#elif defined(HAVE___DECLSPEC_THREAD)
+static __declspec( thread ) int isc__nm_tid_v = ISC_NETMGR_TID_UNKNOWN;
+#else  /* if defined(HAVE_THREAD_LOCAL) */
+#error "Unknown method for defining a TLS variable!"
+#endif /* if defined(HAVE_THREAD_LOCAL) */
+#else  /* if defined(HAVE_TLS) */
+static int isc__nm_tid_v = ISC_NETMGR_TID_NOTLS;
+#endif /* if defined(HAVE_TLS) */
+
+static void *
+nm_thread(void *worker0);
+
+static void
+async_cb(uv_async_t *handle);
+
+
+
+int
+isc_nm_tid() {
+	return (isc__nm_tid_v);
+}
+
+bool
+isc__nm_in_netthread() {
+	return (isc__nm_tid_v >= 0);
+}
+
+/*
+ * isc_nm_start creates and starts a network manager, with `workers` workers.
+ */
+isc_nm_t*
+isc_nm_start(isc_mem_t *mctx, int workers) {
+	int i;
+	isc_nm_t*mgr;
+	int r;
+	char name[32];
+
+	mgr = isc_mem_get(mctx, sizeof(*mgr));
+	*mgr = (isc_nm_t) { .nworkers = workers};
+	isc_mem_attach(mctx, &mgr->mctx);
+	isc_mutex_init(&mgr->lock);
+	isc_condition_init(&mgr->wkstatecond);
+	isc_refcount_init(&mgr->references, 1);
+	mgr->workers = isc_mem_get(mctx, workers * sizeof(isc__networker_t));
+	for (i = 0; i < workers; i++) {
+		isc__networker_t *worker = &mgr->workers[i];
+		*worker = (isc__networker_t) {
+			.mgr = mgr,
+			.id = i,
+			.loop.data = &mgr->workers[i]
+		};
+
+		r = uv_loop_init(&worker->loop);
+		RUNTIME_CHECK(r == 0);
+
+		r = uv_async_init(&worker->loop,
+				  &worker->async, async_cb);
+		RUNTIME_CHECK(r == 0);
+
+		isc_mutex_init(&worker->lock);
+		isc_condition_init(&worker->cond);
+
+		isc_mempool_create(mgr->mctx, 65536, &worker->mpool_bufs);
+		struct ck_fifo_mpmc_entry *stub =
+			isc_mem_get(mgr->mctx, sizeof(*stub));
+		ck_fifo_mpmc_init(&worker->ievents, stub);
+
+		isc_thread_create(nm_thread, &mgr->workers[i],
+				  &worker->thread);
+
+		snprintf(name, sizeof(name), "isc-net-%04u", i);
+		isc_thread_setname(worker->thread, name);
+	}
+	mgr->magic = NM_MAGIC;
+	return (mgr);
+}
+
+/*
+ * isc_nm_shutdown shuts down the network manager.
+ * TODO we need to clean up properly - launch all missing callbacks,
+ * destroy all listeners, etc.
+ */
+void
+isc_nm_shutdown(isc_nm_t **mgr0) {
+	INSIST(mgr0 != NULL);
+	isc_nm_t *mgr = *mgr0;
+	INSIST(VALID_NM(mgr));
+	INSIST(!isc__nm_in_netthread());
+
+	for (size_t i = 0; i < mgr->nworkers; i++) {
+		LOCK(&mgr->workers[i].lock);
+		mgr->workers[i].finished = true;
+		UNLOCK(&mgr->workers[i].lock);
+		isc__netievent_t *ievent = isc__nm_get_ievent(mgr,
+							      netievent_stop);
+		isc__nm_enqueue_ievent(&mgr->workers[i], ievent);
+	}
+	LOCK(&mgr->lock);
+	while (mgr->workers_running > 0) {
+		isc_condition_wait(&mgr->wkstatecond, &mgr->lock);
+	}
+	for (size_t i = 0; i < mgr->nworkers; i++) {
+		struct ck_fifo_mpmc_entry *garbage;
+		ck_fifo_mpmc_deinit(&mgr->workers[i].ievents, &garbage);
+		isc_mem_put(mgr->mctx, garbage, sizeof(*garbage));
+		isc_mempool_destroy(&mgr->workers[i].mpool_bufs);
+	}
+	UNLOCK(&mgr->lock);
+	isc_condition_destroy(&mgr->wkstatecond);
+	isc_mutex_destroy(&mgr->lock);
+	isc_mem_put(mgr->mctx, mgr->workers,
+		    mgr->nworkers * sizeof(isc__networker_t));
+	isc_mem_putanddetach(&mgr->mctx, mgr, sizeof(*mgr));
+	*mgr0 = NULL;
+}
+
+void
+isc_nm_pause(isc_nm_t *mgr) {
+	INSIST(VALID_NM(mgr));
+	INSIST(!isc__nm_in_netthread());
+
+	for (size_t i = 0; i < mgr->nworkers; i++) {
+		LOCK(&mgr->workers[i].lock);
+		mgr->workers[i].paused = true;
+		UNLOCK(&mgr->workers[i].lock);
+		/* We have to issue a stop, otherwise the uv_run loop will
+		 * run indefinitely! */
+		isc__netievent_t *ievent = isc__nm_get_ievent(mgr,
+							      netievent_stop);
+		isc__nm_enqueue_ievent(&mgr->workers[i], ievent);
+	}
+	LOCK(&mgr->lock);
+	while (atomic_load_relaxed(&mgr->workers_paused) != mgr->nworkers) {
+		isc_condition_wait(&mgr->wkstatecond, &mgr->lock);
+	}
+	UNLOCK(&mgr->lock);
+}
+
+void
+isc_nm_resume(isc_nm_t *mgr) {
+	INSIST(VALID_NM(mgr));
+	INSIST(!isc__nm_in_netthread());
+
+	for (size_t i = 0; i < mgr->nworkers; i++) {
+		LOCK(&mgr->workers[i].lock);
+		mgr->workers[i].paused = false;
+		isc_condition_signal(&mgr->workers[i].cond);
+		UNLOCK(&mgr->workers[i].lock);
+	}
+	/* We're not waiting for all the workers to come back to life, they
+	 * eventually will, we don't care */
+}
+
+void
+isc_nm_attach(isc_nm_t *mgr, isc_nm_t **dst) {
+	INSIST(mgr != NULL);
+	INSIST(dst != NULL && *dst == NULL);
+	INSIST(isc_refcount_increment(&mgr->references) > 0);
+	*dst = mgr;
+}
+
+void
+isc_nm_detach(isc_nm_t **mgr0) {
+	isc_nm_t *mgr;
+	int references;
+
+	INSIST(mgr0 != NULL);
+	mgr = *mgr0;
+	INSIST(VALID_NM(mgr));
+
+	references = isc_refcount_decrement(&mgr->references);
+	INSIST(references > 0);
+	if (references == 1) {
+		/* XXXWPK TODO */
+	}
+	mgr0 = NULL;
+}
+
+
+/*
+ * nm_thread is a single worker thread, that runs uv_run event loop
+ * until asked to stop.
+ */
+static void *
+nm_thread(void *worker0) {
+	isc__networker_t *worker = (isc__networker_t*) worker0;
+	atomic_fetch_add_explicit(&worker->mgr->workers_running, 1,
+				  memory_order_relaxed);
+	isc__nm_tid_v = worker->id;
+	isc_thread_setaffinity(isc__nm_tid_v);
+	while (true) {
+		int r = uv_run(&worker->loop, UV_RUN_DEFAULT);
+		/*
+		 * or there's nothing to do. In the first case - wait
+		 * for condition. In the latter - timedwait
+		 */
+		LOCK(&worker->lock);
+		while (worker->paused) {
+			LOCK(&worker->mgr->lock);
+			atomic_fetch_add_explicit(&worker->mgr->workers_paused,
+						  1, memory_order_acquire);
+			isc_condition_signal(&worker->mgr->wkstatecond);
+			UNLOCK(&worker->mgr->lock);
+			isc_condition_wait(&worker->cond, &worker->lock);
+		}
+		atomic_fetch_sub_explicit(&worker->mgr->workers_paused, 1,
+					  memory_order_release);
+		UNLOCK(&worker->lock);
+		if (worker->finished) {
+			/* TODO walk the handles and free them! */
+			break;
+		}
+
+		if (r == 0) {
+			/* TODO */
+			usleep(100000);
+		}
+	}
+	atomic_fetch_sub_explicit(&worker->mgr->workers_running, 1,
+				  memory_order_relaxed);
+	isc_condition_signal(&worker->mgr->wkstatecond);
+	return (NULL);
+}
+
+
+/*
+ * async_cb is an universal callback for 'async' events sent to event loop.
+ * It's the only way to safely pass data to libuv event loop. We use a single
+ * async event and a lockless queue of 'isc__netievent_t' structures passed
+ * from other threads.
+ */
+static void
+async_cb(uv_async_t *handle) {
+	isc__networker_t *worker = (isc__networker_t *) handle->loop->data;
+	isc__netievent_t *ievent;
+	struct ck_fifo_mpmc_entry *garbage;
+	/*
+	 * We only try dequeue to not waste time, libuv guarantees
+	 * that if someone calls uv_async_send -after- async_cb was called
+	 * then async_cb will be called again, we won't loose any signals.
+	 */
+	while (ck_fifo_mpmc_trydequeue(&worker->ievents, &ievent, &garbage)) {
+		switch (ievent->type) {
+		case netievent_stop:
+			uv_stop(handle->loop);
+			break;
+		case netievent_udplisten:
+			isc__nm_handle_udplisten(worker, ievent);
+			break;
+		case netievent_udpstoplisten:
+			isc__nm_handle_udpstoplisten(worker, ievent);
+			break;
+		case netievent_udpsend:
+			isc__nm_handle_udpsend(worker, ievent);
+			break;
+		case netievent_tcpconnect:
+			isc__nm_handle_tcpconnect(worker, ievent);
+			break;
+		case netievent_tcplisten:
+			isc__nm_handle_tcplisten(worker, ievent);
+			break;
+		case netievent_tcpstartread:
+			isc__nm_handle_startread(worker, ievent);
+			break;
+/*              case netievent_tcpstopread:
+ *                      handle_stopread(worker, ievent);
+ *                      break; */
+		case netievent_tcpsend:
+			isc__nm_handle_tcpsend(worker, ievent);
+			break;
+		case netievent_tcpstoplisten:
+			isc__nm_handle_tcpstoplistening(worker, ievent);
+			break;
+		case netievent_tcpclose:
+			isc__nm_handle_tcpclose(worker, ievent);
+			break;
+		default:
+			INSIST(0);
+		}
+		isc_mem_put(worker->mgr->mctx, ievent,
+			    sizeof(isc__netievent_storage_t));
+		isc_mem_put(worker->mgr->mctx, garbage, sizeof(*garbage));
+	}
+}
+/*
+ * isc__nm_get_ievent allocates an ievent and sets the type
+ * xxxwpk: use pool?
+ */
+void *
+isc__nm_get_ievent(isc_nm_t *mgr, isc__netievent_type type) {
+	isc__netievent_storage_t *event =
+		isc_mem_get(mgr->mctx, sizeof(isc__netievent_storage_t));
+	*event = (isc__netievent_storage_t) { .ni.type = type };
+	return (event);
+}
+
+/*
+ * enqueue ievent on a specific worker queue. This the only safe
+ * way to use isc__networker_t from another thread.
+ */
+void
+isc__nm_enqueue_ievent(isc__networker_t *worker, isc__netievent_t *event) {
+	struct ck_fifo_mpmc_entry *entry = isc_mem_get(worker->mgr->mctx,
+						       sizeof(*entry));
+	ck_fifo_mpmc_enqueue(&worker->ievents, entry, event);
+	uv_async_send(&worker->async);
+}
+
+static bool
+isc__nmsocket_active(isc_nmsocket_t *socket) {
+	if (socket->parent != NULL) {
+		return (atomic_load(&socket->parent->active));
+	} else {
+		return (atomic_load(&socket->active));
+	}
+}
+
+void
+isc_nmsocket_attach(isc_nmsocket_t *socket, isc_nmsocket_t **target) {
+	REQUIRE(VALID_NMSOCK(socket));
+	REQUIRE(target != NULL && *target == NULL);
+	if (socket->parent != NULL) {
+		INSIST(socket->parent->parent == NULL); /* sanity check */
+		isc_refcount_increment(&socket->parent->references);
+	} else {
+		isc_refcount_increment(&socket->references);
+	}
+	*target = socket;
+}
+
+/*
+ * Destroy socket (and its children), freeing all resources and optionally
+ * socket itself.
+ */
+static void
+isc__nmsocket_destroy(isc_nmsocket_t *socket, bool dofree) {
+	INSIST(VALID_NMSOCK(socket));
+	REQUIRE(socket->ah_cpos == 0);  /* We don't have active handles */
+	REQUIRE(!isc__nmsocket_active(socket));
+	isc_nm_t *mgr = socket->mgr;
+	for (int i = 0; i < socket->nchildren; i++) {
+		isc__nmsocket_destroy(&socket->children[i], false);
+	}
+
+	if (socket->tcphandle != NULL) {
+		isc_nmhandle_detach(&socket->tcphandle);
+	}
+
+	if (socket->quota != NULL) {
+		isc_quota_detach(&socket->quota);
+	}
+
+	ck_stack_entry_t *se;
+	while ((se = ck_stack_pop_mpmc(&socket->inactivehandles)) != NULL) {
+		isc_nmhandle_t *handle = nm_handle_is_get(se);
+		isc__nmhandle_free(socket, handle);
+	}
+
+	while ((se = ck_stack_pop_mpmc(&socket->inactivereqs)) != NULL) {
+		isc__nm_uvreq_t *uvreq = uvreq_is_get(se);
+		isc_mem_put(mgr->mctx, uvreq, sizeof(*uvreq));
+	}
+	isc_mem_free(mgr->mctx, socket->ah_frees);
+	isc_mem_free(mgr->mctx, socket->ah_handles);
+	if (dofree) {
+		isc_mem_put(mgr->mctx, socket, sizeof(*socket));
+	}
+	isc_nm_detach(&mgr);
+}
+
+
+static void
+isc__nmsocket_maybe_destroy(isc_nmsocket_t *socket) {
+	isc_mutex_lock(&socket->lock);
+	INSIST(!isc__nmsocket_active(socket));
+	/* XXXWPK TODO destroy non-inflight handles, launching callbacks */
+	bool destroy = (socket->ah_cpos == 0);
+	isc_mutex_unlock(&socket->lock);
+	if (destroy) {
+		isc__nmsocket_destroy(socket, true);
+	}
+}
+
+/* The final external reference to the socket is gone, we can try destroying the
+ * socket, but we have to wait for all the inflight handles to finish first. */
+void
+isc__nmsocket_prep_destroy(isc_nmsocket_t *socket) {
+	INSIST(socket->parent == NULL);
+	atomic_store(&socket->active, false);
+	/*
+	 * XXXWPK if we're here we already stopped listening, otherwise
+	 * we'd have a hanging reference from the listening process.
+	 *
+	 * If that's a regular socket we need to close it.
+	 */
+	if (atomic_load(&socket->closed)) {
+		isc__nmsocket_maybe_destroy(socket);
+	} else {
+		switch (socket->type) {
+		case isc_nm_tcpsocket:
+			isc__nm_tcp_close(socket);
+			break;
+		case isc_nm_tcpdnssocket:
+			isc__nm_tcpdns_close(socket);
+			break;
+		default: /* This socket does not require closing */
+			isc__nmsocket_maybe_destroy(socket);
+			break;
+		}
+	}
+}
+
+void
+isc_nmsocket_detach(isc_nmsocket_t **socketp) {
+	isc_nmsocket_t *socket, *rsocket;
+	int references;
+	REQUIRE(socketp != NULL && *socketp != NULL);
+	socket = *socketp;
+	REQUIRE(VALID_NMSOCK(socket));
+
+	/*
+	 * If the socket is a part of a set (a child socket) we are
+	 * counting references for the whole set at the parent.
+	 */
+	if (socket->parent != NULL) {
+		rsocket = socket->parent;
+		INSIST(rsocket->parent == NULL); /* Sanity check */
+	} else {
+		rsocket = socket;
+	}
+
+	references = isc_refcount_decrement(&rsocket->references);
+	INSIST(references > 0);
+	if (references == 1) {
+		isc__nmsocket_prep_destroy(rsocket);
+	}
+	*socketp = NULL;
+}
+
+
+
+void
+isc__nmsocket_init(isc_nmsocket_t *socket,
+		   isc_nm_t *mgr,
+		   isc_nmsocket_type type)
+{
+	*socket = (isc_nmsocket_t) {
+		.type = type,
+		.fd = -1
+	};
+
+	isc_nm_attach(mgr, &socket->mgr);
+	uv_handle_set_data(&socket->uv_handle.handle, socket);
+
+	ck_stack_init(&socket->inactivehandles);
+	ck_stack_init(&socket->inactivereqs);
+
+	socket->ah_cpos = 0;
+	socket->ah_size = 32;
+	socket->ah_frees =
+		isc_mem_allocate(mgr->mctx, socket->ah_size * sizeof(size_t));
+	socket->ah_handles = isc_mem_allocate(mgr->mctx,
+					      socket->ah_size *
+					      sizeof(isc_nmhandle_t*));
+	for (size_t i = 0; i < 32; i++) {
+		socket->ah_frees[i] = i;
+		socket->ah_handles[i] = NULL;
+	}
+
+	isc_mutex_init(&socket->lock);
+	isc_condition_init(&socket->cond);
+	socket->magic = NMSOCK_MAGIC;
+	isc_refcount_init(&socket->references, 1);
+	atomic_init(&socket->active, true);
+}
+/*
+ * alloc_cb for recv operations. XXXWPK TODO use a pool
+ */
+void
+isc__nm_alloc_cb(uv_handle_t *handle, size_t size, uv_buf_t *buf) {
+	isc_nmsocket_t *socket = (isc_nmsocket_t *) handle->data;
+	REQUIRE(VALID_NMSOCK(socket));
+	REQUIRE(socket->tid >= 0);
+	INSIST(size <= 65536);
+	/* TODO that's for UDP only! */
+	isc__networker_t *worker = &socket->mgr->workers[socket->tid];
+	REQUIRE(!worker->udprecvbuf_inuse);
+	buf->base = worker->udprecvbuf;
+/*	buf->base = isc_mempool_get(worker->mpool_bufs); */
+	worker->udprecvbuf_inuse = true;
+	buf->len = size;
+}
+
+void
+isc__nm_free_uvbuf(isc_nmsocket_t *socket, const uv_buf_t *buf) {
+	REQUIRE(VALID_NMSOCK(socket));
+	(void) buf;
+	isc__networker_t *worker = &socket->mgr->workers[socket->tid];
+	REQUIRE(worker->udprecvbuf_inuse);
+	REQUIRE(buf->base == worker->udprecvbuf);
+	worker->udprecvbuf_inuse = false;
+#if 0
+	void *b = buf->base;
+	if (b != NULL) {
+		isc_mempool_put(worker->mpool_bufs, b);
+	}
+#endif
+}
+
+/*
+ * alloc_handle allocates a handle. XXXWPK TODO use a pool
+ */
+
+static isc_nmhandle_t *
+alloc_handle(isc_nmsocket_t *socket);
+
+static isc_nmhandle_t *
+alloc_handle(isc_nmsocket_t *socket) {
+	isc_nmhandle_t *handle;
+	handle = isc_mem_get(socket->mgr->mctx,
+			     sizeof(isc_nmhandle_t) +
+			     socket->extrahandlesize);
+	*handle = (isc_nmhandle_t) {.magic = NMHANDLE_MAGIC};
+	isc_refcount_init(&handle->references, 1);
+	return (handle);
+}
+
+isc_nmhandle_t *
+isc__nmhandle_get(isc_nmsocket_t *socket, isc_sockaddr_t *peer) {
+	isc_nmhandle_t *handle = NULL;
+	ck_stack_entry_t *sentry;
+	REQUIRE(VALID_NMSOCK(socket));
+	INSIST(peer != NULL);
+	sentry = ck_stack_pop_mpmc(&socket->inactivehandles);
+	if (sentry != NULL) {
+		handle = nm_handle_is_get(sentry);
+	}
+	if (handle == NULL) {
+		handle = alloc_handle(socket);
+	} else {
+		INSIST(VALID_NMHANDLE(handle));
+		isc_refcount_increment(&handle->references);
+	}
+
+	handle->socket = socket;
+	memcpy(&handle->peer, peer, sizeof(isc_sockaddr_t));
+
+	isc_mutex_lock(&socket->lock);
+	if (socket->ah_cpos == socket->ah_size) {
+		socket->ah_frees = isc_mem_reallocate(socket->mgr->mctx,
+						      socket->ah_frees,
+						      socket->ah_size * 2 *
+						      sizeof(size_t));
+		socket->ah_handles = isc_mem_reallocate(socket->mgr->mctx,
+							socket->ah_handles,
+							socket->ah_size * 2 *
+							sizeof(isc_nmhandle_t*));
+		for (size_t i = socket->ah_size; i < socket->ah_size * 2;
+		     i++)
+		{
+			socket->ah_frees[i] = i;
+			socket->ah_handles[i] = NULL;
+		}
+		socket->ah_size *= 2;
+	}
+	int pos = socket->ah_frees[socket->ah_cpos++];
+	INSIST(socket->ah_handles[pos] == NULL);
+	socket->ah_handles[pos] = handle;
+	handle->ah_pos = pos;
+	isc_mutex_unlock(&socket->lock);
+
+	if (socket->type == isc_nm_tcpsocket) {
+		INSIST(socket->tcphandle == NULL);
+		socket->tcphandle = handle;
+	}
+	return(handle);
+}
+
+void
+isc_nmhandle_attach(isc_nmhandle_t *handle, isc_nmhandle_t **handlep) {
+	REQUIRE(VALID_NMHANDLE(handle));
+	REQUIRE(handlep != NULL && *handlep == NULL);
+	INSIST(isc_refcount_increment(&handle->references) > 0);
+	*handlep = handle;
+}
+
+bool
+isc_nmhandle_is_stream(isc_nmhandle_t *handle) {
+	return(handle->socket->type == isc_nm_tcpsocket ||
+	       handle->socket->type == isc_nm_tcpdnssocket);
+}
+
+void
+isc__nmhandle_free(isc_nmsocket_t *socket, isc_nmhandle_t *handle) {
+	size_t extra = socket->extrahandlesize;
+	if (handle->dofree) {
+		handle->dofree(handle->opaque);
+	}
+	*handle = (isc_nmhandle_t) {};
+	isc_mem_put(socket->mgr->mctx,
+		    handle,
+		    sizeof(isc_nmhandle_t) + extra);
+}
+
+void
+isc_nmhandle_detach(isc_nmhandle_t **handlep) {
+	isc_nmhandle_t *handle = *handlep;
+	REQUIRE(VALID_NMHANDLE(handle));
+	if (isc_refcount_decrement(&handle->references) == 1) {
+		isc_nmsocket_t *socket = handle->socket;
+		handle->socket = NULL;
+		if (handle->doreset != NULL) {
+			handle->doreset(handle->opaque);
+		}
+
+		/*
+		 * We do it all under lock to avoid races with socket
+		 * destruction.
+		 */
+		isc_mutex_lock(&socket->lock);
+		INSIST(socket->ah_handles[handle->ah_pos] == handle);
+		INSIST(socket->ah_size > handle->ah_pos);
+		INSIST(socket->ah_cpos > 0);
+		socket->ah_handles[handle->ah_pos] = NULL;
+		socket->ah_frees[--socket->ah_cpos] = handle->ah_pos;
+		handle->ah_pos = 0;
+
+		bool reuse = false;
+		if (atomic_load(&socket->active)) {
+			reuse = ck_stack_trypush_mpmc(
+				&socket->inactivehandles,
+				&handle->ilink);
+		}
+
+		isc_mutex_unlock(&socket->lock);
+
+		if (!reuse) {
+			isc__nmhandle_free(socket, handle);
+		}
+
+		if (!atomic_load(&socket->active)) {
+			isc__nmsocket_maybe_destroy(socket);
+		}
+	}
+	*handlep = NULL;
+}
+
+
+void *
+isc_nmhandle_getdata(isc_nmhandle_t *handle) {
+	INSIST(VALID_NMHANDLE(handle));
+	return (handle->opaque);
+}
+
+void
+isc_nmhandle_setdata(isc_nmhandle_t *handle,
+		     void *arg,
+		     isc_nm_opaquecb doreset,
+		     isc_nm_opaquecb dofree)
+{
+	INSIST(VALID_NMHANDLE(handle));
+	handle->opaque = arg;
+	handle->doreset = doreset;
+	handle->dofree = dofree;
+}
+
+void*
+isc_nmhandle_getextra(isc_nmhandle_t *handle) {
+	INSIST(VALID_NMHANDLE(handle));
+	return (handle->extra);
+}
+
+isc_sockaddr_t
+isc_nmhandle_peeraddr(isc_nmhandle_t *handle) {
+	INSIST(VALID_NMHANDLE(handle));
+	return (handle->peer);
+}
+
+/*
+ * isc__nm_uvreq_get returns an uv_req_t of specified type
+ * with data field set up.
+ */
+
+isc__nm_uvreq_t *
+isc__nm_uvreq_get(isc_nm_t *mgr, isc_nmsocket_t *socket) {
+	isc__nm_uvreq_t *req = NULL;
+	if (socket != NULL && atomic_load(&socket->active)) {
+		/* Try to reuse one */
+		ck_stack_entry_t *sentry =
+			ck_stack_pop_mpmc(&socket->inactivereqs);
+		if (sentry != NULL) {
+			req = uvreq_is_get(sentry);
+		}
+	}
+	if (req == NULL) {
+		req = isc_mem_get(mgr->mctx, sizeof(isc__nm_uvreq_t));
+	}
+	*req = (isc__nm_uvreq_t) {};
+	uv_req_set_data(&req->uv_req.req, req);
+	isc_nm_attach(mgr, &req->mgr);
+	req->magic = UVREQ_MAGIC;
+	return (req);
+}
+
+/*
+ * isc__nm_uvreq_put frees uv_req_t of specified type
+ */
+void
+isc__nm_uvreq_put(isc__nm_uvreq_t **req0, isc_nmsocket_t *socket) {
+	isc__nm_uvreq_t *req;
+	isc_nm_t *mgr;
+
+	INSIST(req0 != NULL);
+	req = *req0;
+	INSIST(VALID_UVREQ(req));
+	req0 = NULL;
+	req->magic = 0;
+	if (req->handle != NULL) {
+		isc_nmhandle_detach(&req->handle);
+	}
+	/*
+	 * We need to do it this way to make sure that mgr won't disappear
+	 * in the meantime, taking mctx with it.
+	 */
+	mgr = req->mgr;
+	req->mgr = NULL;
+	if (socket == NULL || !atomic_load(&socket->active) ||
+	    !ck_stack_trypush_mpmc(&socket->inactivereqs, &req->ilink))
+	{
+		isc_mem_put(mgr->mctx, req, sizeof(isc__nm_uvreq_t));
+	}
+	isc_nm_detach(&mgr);
+}
+
+/*
+ * isc_nm_send sends data in 'region' to endpoint specified by 'handle'
+ * handle can be of any type.
+ */
+isc_result_t
+isc_nm_send(isc_nmhandle_t *handle,
+	    isc_region_t *region,
+	    isc_nm_send_cb_t cb,
+	    void *cbarg)
+{
+	switch (handle->socket->type) {
+	case isc_nm_udpsocket:
+	case isc_nm_udplistener:
+		return (isc__nm_udp_send(handle, region, cb, cbarg));
+	case isc_nm_tcpsocket:
+		return (isc__nm_tcp_send(handle, region, cb, cbarg));
+	case isc_nm_tcpdnssocket:
+		return (isc__nm_tcpdns_send(handle, region, cb, cbarg));
+	default:
+		INSIST(0);
+		ISC_UNREACHABLE();
+	}
+	return (ISC_R_FAILURE);
+}

lib/isc/netmgr/tcp.c

@@ -0,0 +1,554 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+#include "config.h"
+
+#include <unistd.h>
+#include <uv.h>
+#include <ck_fifo.h>
+#include <ck_stack.h>
+
+#include <isc/atomic.h>
+#include <isc/buffer.h>
+#include <isc/condition.h>
+#include <isc/magic.h>
+#include <isc/mem.h>
+#include <isc/netmgr.h>
+#include <isc/quota.h>
+#include <isc/random.h>
+#include <isc/refcount.h>
+#include <isc/region.h>
+#include <isc/result.h>
+#include <isc/sockaddr.h>
+#include <isc/thread.h>
+#include <isc/util.h>
+
+#include "netmgr-int.h"
+/*
+ * TTTTT   CCC   PPP
+ *   T    C   C  P  P
+ *   T    C      PPP
+ *   T    C   C  P
+ *   T     CCC   P
+ */
+
+static int
+tcp_connect_direct(isc_nmsocket_t *socket, isc__nm_uvreq_t *req);
+
+static isc_result_t
+tcp_send_direct(isc_nmsocket_t *socket, isc__nm_uvreq_t *req);
+static void
+tcp_connect_cb(uv_connect_t *uvreq, int status);
+
+static void
+tcp_connection_cb(uv_stream_t *server, int status);
+
+static void
+read_cb(uv_stream_t*stream, ssize_t nread, const uv_buf_t*buf);
+
+
+/*
+ * isc_nm_tcp_connect connects to 'peer' using (optional) 'iface' as the
+ * source IP.
+ * If the result is ISC_R_SUCCESS then the cb is always called.
+ */
+isc_result_t
+isc_nm_tcp_connect(isc_nm_t*mgr,
+		   isc_nmiface_t *iface,
+		   isc_sockaddr_t *peer,
+		   isc_nm_connect_cb_t cb,
+		   void *cbarg)
+{
+	isc_nmsocket_t *sock;
+	isc__nm_uvreq_t *req;
+	isc_result_t result;
+
+	sock = isc_mem_get(mgr->mctx, sizeof(isc_nmsocket_t));
+	isc__nmsocket_init(sock, mgr, isc_nm_tcpsocket);
+
+	sock->uv_handle.tcp.data = sock;
+
+	req = isc__nm_uvreq_get(sock->mgr, sock);
+	memcpy(&req->peer, peer, peer->length);
+	if (iface != NULL) {
+		memcpy(&req->local, iface, iface->addr.length);
+	} else {
+		req->local.length = 0;
+	}
+	req->cb.connect = cb;
+	req->cbarg = cbarg;
+
+	result = tcp_connect_direct(sock, req);
+	if (result != ISC_R_SUCCESS) {
+		isc__nm_uvreq_put(&req, NULL);
+		isc_mem_put(mgr->mctx, sock, sizeof(isc_nmsocket_t));
+	}
+	return (result);
+}
+
+static int
+tcp_connect_direct(isc_nmsocket_t *socket, isc__nm_uvreq_t *req)
+{
+	isc__networker_t *worker;
+	int r;
+
+	REQUIRE(isc_nm_tid() >= 0);
+
+	worker = &req->mgr->workers[isc_nm_tid()];
+
+	r = uv_tcp_init(&worker->loop, &socket->uv_handle.tcp);
+	if (r != 0) {
+		return (r);
+	}
+	if (req->local.length != 0) {
+		r = uv_tcp_bind(&socket->uv_handle.tcp,
+				&req->local.type.sa,
+				0);
+		if (r != 0) {
+			/* TODO uv_close() */
+			return (r);
+		}
+	}
+	r = uv_tcp_connect(&req->uv_req.connect,
+			   &socket->uv_handle.tcp,
+			   &req->peer.type.sa,
+			   tcp_connect_cb);
+	return (r);
+}
+
+void
+isc__nm_handle_tcpconnect(isc__networker_t *worker,
+			  isc__netievent_t *ievent0) {
+	int r;
+	isc__netievent_tcpconnect_t *ievent =
+		(isc__netievent_tcpconnect_t *) ievent0;
+	isc_nmsocket_t *socket = ievent->socket;
+	isc__nm_uvreq_t *req = ievent->req;
+
+	REQUIRE(socket->type == isc_nm_tcpsocket);
+	REQUIRE(worker->id == ievent->req->mgr->workers[isc_nm_tid()].id);
+
+	r = tcp_connect_direct(socket, req);
+	if (r != 0) {
+		/* We need to issue callbacks ourselves */
+		tcp_connect_cb(&req->uv_req.connect, r);
+	}
+}
+
+static void
+tcp_connect_cb(uv_connect_t *uvreq, int status) {
+	isc__nm_uvreq_t *req = (isc__nm_uvreq_t*) uvreq->data;
+	isc_nmsocket_t *socket = uvreq->handle->data;
+	INSIST(VALID_UVREQ(req));
+
+	if (status == 0) {
+		isc_sockaddr_t peer;
+		struct sockaddr_storage ss;
+		uv_tcp_getpeername(&socket->uv_handle.tcp,
+				   (struct sockaddr*) &ss,
+				   &(int){sizeof(ss)});
+		isc_result_t result =
+			isc_sockaddr_fromsockaddr(&peer,
+						  (struct sockaddr*) &ss);
+		INSIST(result == ISC_R_SUCCESS);
+
+		isc_nmhandle_t *handle = isc__nmhandle_get(socket, &peer);
+		/* handle->peer = NULL; */
+		req->cb.connect(handle, ISC_R_SUCCESS, req->cbarg);
+	} else {
+		/* TODO handle it properly, free socket, translate code */
+		req->cb.connect(NULL, ISC_R_FAILURE, req->cbarg);
+	}
+	isc__nm_uvreq_put(&req, socket);
+}
+
+isc_result_t
+isc_nm_tcp_listen(isc_nm_t *mgr,
+		  isc_nmiface_t *iface,
+		  isc_nm_accept_cb_t cb,
+		  size_t extrahandlesize,
+		  void *cbarg,
+		  isc_quota_t *quota,
+		  isc_nmsocket_t **rv)
+{
+	isc__netievent_tcplisten_t *ievent;
+	INSIST(VALID_NM(mgr));
+	isc_nmsocket_t *nsocket;
+
+	nsocket = isc_mem_get(mgr->mctx, sizeof(*nsocket));
+	isc__nmsocket_init(nsocket, mgr, isc_nm_tcplistener);
+	nsocket->iface = iface;
+	nsocket->rcb.accept = cb;
+	nsocket->rcbarg = cbarg;
+	nsocket->extrahandlesize = extrahandlesize;
+	if (quota != NULL) {
+		/*
+		 * We need to force it to make sure we get it attached.
+		 * An example failure mode would be server under attack
+		 * reconfiguring interfaces - that might cause weak attach
+		 * to fail and leave this listening socket without limits.
+		 * We can ignore the result.
+		 */
+		isc_quota_force(quota, &nsocket->quota);
+	}
+	nsocket->tid = isc_random_uniform(mgr->nworkers);
+	/*
+	 * Listening to TCP is rare enough not to care about the
+	 * added overhead from passing this to another thread.
+	 */
+	ievent = isc__nm_get_ievent(mgr, netievent_tcplisten);
+	ievent->socket = nsocket;
+	isc__nm_enqueue_ievent(&mgr->workers[nsocket->tid],
+			       (isc__netievent_t*) ievent);
+	*rv = nsocket;
+	return (ISC_R_SUCCESS);
+}
+
+void
+isc__nm_handle_tcplisten(isc__networker_t *worker, isc__netievent_t *ievent0) {
+	int r;
+	isc__netievent_tcplisten_t *ievent =
+		(isc__netievent_tcplisten_t *) ievent0;
+	isc_nmsocket_t *socket = ievent->socket;
+
+	REQUIRE(isc_nm_tid() >= 0);
+	REQUIRE(socket->type == isc_nm_tcplistener);
+
+	r = uv_tcp_init(&worker->loop, &socket->uv_handle.tcp);
+	if (r != 0) {
+		return;
+	}
+
+	uv_tcp_bind(&socket->uv_handle.tcp, &socket->iface->addr.type.sa, 0);
+	r = uv_listen((uv_stream_t*) &socket->uv_handle.tcp,
+		      10,
+		      tcp_connection_cb);
+	socket->listening = true;
+	return;
+	/* issue callback? */
+}
+
+
+void
+isc_nm_tcp_stoplistening(isc_nmsocket_t *socket) {
+	isc__netievent_tcpstoplisten_t *ievent;
+	INSIST(VALID_NMSOCK(socket));
+	REQUIRE(!isc__nm_in_netthread());
+
+	ievent = isc__nm_get_ievent(socket->mgr, netievent_tcpstoplisten);
+	isc_nmsocket_attach(socket, &ievent->socket);
+	isc__nm_enqueue_ievent(&socket->mgr->workers[socket->tid],
+			       (isc__netievent_t*) ievent);
+	isc_mutex_lock(&socket->lock);
+	while (atomic_load(&socket->listening) == true) {
+		isc_condition_wait(&socket->cond, &socket->lock);
+	}
+	isc_mutex_unlock(&socket->lock);
+	isc_nmsocket_detach(&socket);
+}
+
+static void
+stoplistening_cb(uv_handle_t *handle) {
+	isc_nmsocket_t *socket = handle->data;
+	isc_mutex_lock(&socket->lock);
+	atomic_store(&socket->listening, false);
+	isc_condition_signal(&socket->cond);
+	isc_mutex_unlock(&socket->lock);
+}
+
+void
+isc__nm_handle_tcpstoplistening(isc__networker_t *worker,
+				isc__netievent_t *ievent0) {
+	(void)worker;
+	isc__netievent_tcpstoplisten_t *ievent =
+		(isc__netievent_tcpstoplisten_t *) ievent0;
+	isc_nmsocket_t *socket = ievent->socket;
+
+	REQUIRE(isc_nm_tid() >= 0);
+	REQUIRE(VALID_NMSOCK(socket));
+	REQUIRE(socket->type == isc_nm_tcplistener);
+
+	uv_close(&socket->uv_handle.handle, stoplistening_cb);
+}
+
+
+isc_result_t
+isc_nm_read(isc_nmhandle_t *handle, isc_nm_recv_cb_t cb, void *cbarg) {
+	INSIST(VALID_NMHANDLE(handle));
+	isc_nmsocket_t *socket = handle->socket;
+	INSIST(VALID_NMSOCK(socket));
+	socket->rcb.recv = cb;
+	socket->rcbarg = cbarg; /* That's obviously broken... */
+	if (socket->tid == isc_nm_tid()) {
+		int r = uv_read_start(&socket->uv_handle.stream,
+				      isc__nm_alloc_cb,
+				      read_cb);
+		INSIST(r == 0);
+	} else {
+		isc__netievent_startread_t *ievent =
+			isc__nm_get_ievent(socket->mgr,
+					   netievent_tcpstartread);
+		ievent->socket = socket;
+		isc__nm_enqueue_ievent(&socket->mgr->workers[socket->tid],
+				       (isc__netievent_t*) ievent);
+	}
+	return (ISC_R_SUCCESS);
+}
+
+void
+isc__nm_handle_startread(isc__networker_t *worker, isc__netievent_t *ievent0) {
+	isc__netievent_startread_t *ievent =
+		(isc__netievent_startread_t *) ievent0;
+	REQUIRE(worker->id == isc_nm_tid());
+
+	isc_nmsocket_t *socket = ievent->socket;
+
+	uv_read_start(&socket->uv_handle.stream, isc__nm_alloc_cb, read_cb);
+}
+
+static void
+read_cb(uv_stream_t *stream, ssize_t nread, const uv_buf_t*buf) {
+	isc_nmsocket_t *socket = uv_handle_get_data((uv_handle_t*) stream);
+	INSIST(buf != NULL);
+	if (nread < 0) {
+		isc__nm_free_uvbuf(socket, buf);
+		socket->rcb.recv(socket->rcbarg, socket->tcphandle, NULL);
+		/* XXXWPK TODO clean up handles, close the connection, reclaim
+		 * quota */
+		return;
+	}
+	isc_region_t region = { .base = (unsigned char *) buf->base,
+				.length = nread };
+
+	INSIST(VALID_NMSOCK(socket));
+	INSIST(socket->rcb.recv != NULL);
+
+	socket->rcb.recv(socket->rcbarg, socket->tcphandle, &region);
+	isc__nm_free_uvbuf(socket, buf);
+}
+
+static isc_result_t
+accept_connection(isc_nmsocket_t *ssocket) {
+	REQUIRE(VALID_NMSOCK(ssocket));
+	REQUIRE(ssocket->tid == isc_nm_tid());
+	if (!atomic_load_relaxed(&ssocket->active)) {
+		/* We're closing, bail */
+		return (ISC_R_CANCELED);
+	}
+	INSIST(ssocket->rcb.accept != NULL);
+
+	isc_quota_t *quota = NULL;
+	if (ssocket->quota != NULL) {
+		isc_result_t result = isc_quota_attach(ssocket->quota, &quota);
+		if (result != ISC_R_SUCCESS) {
+			return (result);
+		}
+	}
+
+	isc_nmsocket_t *csocket =
+		isc_mem_get(ssocket->mgr->mctx, sizeof(isc_nmsocket_t));
+	isc__nmsocket_init(csocket, ssocket->mgr, isc_nm_tcpsocket);
+	csocket->tid = isc_nm_tid();
+	csocket->extrahandlesize = ssocket->extrahandlesize;
+	csocket->quota = quota;
+	quota = NULL;
+
+	isc__networker_t *worker = &ssocket->mgr->workers[isc_nm_tid()];
+	uv_tcp_init(&worker->loop, &csocket->uv_handle.tcp);
+	int r = uv_accept(&ssocket->uv_handle.stream,
+			  &csocket->uv_handle.stream);
+	if (r != 0) {
+		if (csocket->quota != NULL) {
+			isc_quota_detach(&csocket->quota);
+		}
+		isc_mem_put(ssocket->mgr->mctx, csocket,
+			    sizeof(isc_nmsocket_t));
+		return (ISC_R_FAILURE); /* XXXWPK TODO translate ! */
+	}
+
+	isc_nmsocket_attach(ssocket, &csocket->server);
+
+	isc_sockaddr_t peer;
+	struct sockaddr_storage ss;
+	uv_tcp_getpeername(&csocket->uv_handle.tcp, (struct sockaddr*) &ss,
+			   &(int){sizeof(ss)});
+	isc_result_t result =
+		isc_sockaddr_fromsockaddr(&peer, (struct sockaddr*) &ss);
+	INSIST(result == ISC_R_SUCCESS);
+	isc_nmhandle_t *handle = isc__nmhandle_get(csocket, &peer);
+
+	ssocket->rcb.accept(handle, ISC_R_SUCCESS, ssocket->rcbarg);
+	isc_nmsocket_detach(&csocket);
+	return (ISC_R_SUCCESS);
+}
+
+static void
+tcp_connection_cb(uv_stream_t *server, int status) {
+	(void) status;
+	isc_nmsocket_t *ssocket = uv_handle_get_data((uv_handle_t*) server);
+	isc_result_t result = accept_connection(ssocket);
+	if (result != ISC_R_SUCCESS) {
+		if (result == ISC_R_QUOTA || result == ISC_R_SOFTQUOTA) {
+			ssocket->overquota = true;
+		}
+		/* XXXWPK TODO LOG */
+	}
+}
+
+
+
+/*
+ * isc__nm_tcp_send sends buf to a peer on a socket.
+ */
+isc_result_t
+isc__nm_tcp_send(isc_nmhandle_t *handle,
+		 isc_region_t *region,
+		 isc_nm_send_cb_t cb,
+		 void *cbarg)
+{
+	isc_nmsocket_t *socket = handle->socket;
+	isc__netievent_udpsend_t *ievent;
+	isc__nm_uvreq_t *uvreq;
+
+	INSIST(socket->type == isc_nm_tcpsocket);
+
+	uvreq = isc__nm_uvreq_get(socket->mgr, socket);
+	uvreq->uvbuf.base = (char *) region->base;
+	uvreq->uvbuf.len = region->length;
+	isc_nmhandle_attach(handle, &uvreq->handle);
+	uvreq->cb.send = cb;
+	uvreq->cbarg = cbarg;
+
+	if (socket->tid == isc_nm_tid()) {
+		/*
+		 * If we're in the same thread as the socket we can send the
+		 * data directly
+		 */
+		return (tcp_send_direct(socket, uvreq));
+	} else {
+		/*
+		 * We need to create an event and pass it using async channel
+		 */
+		ievent = isc__nm_get_ievent(socket->mgr, netievent_tcpsend);
+		ievent->socket = socket;
+		ievent->req = uvreq;
+		isc__nm_enqueue_ievent(&socket->mgr->workers[socket->tid],
+				       (isc__netievent_t*) ievent);
+		return (ISC_R_SUCCESS);
+	}
+	return (ISC_R_UNEXPECTED);
+}
+
+static void
+tcp_send_cb(uv_write_t *req, int status) {
+	isc_result_t result;
+	isc__nm_uvreq_t *uvreq = (isc__nm_uvreq_t*)req->data;
+	INSIST(VALID_UVREQ(uvreq));
+	if (status == 0) {
+		result = ISC_R_SUCCESS;
+	} else {
+		result = ISC_R_FAILURE;
+	}
+	INSIST(VALID_NMHANDLE(uvreq->handle));
+	uvreq->cb.send(uvreq->handle, result, uvreq->cbarg);
+	isc__nm_uvreq_put(&uvreq, uvreq->handle->socket);
+}
+
+/*
+ * handle 'tcpsend' async event - send a packet on the socket
+ */
+void
+isc__nm_handle_tcpsend(isc__networker_t *worker, isc__netievent_t *ievent0) {
+	isc__netievent_tcpsend_t *ievent =
+		(isc__netievent_tcpsend_t *) ievent0;
+	INSIST(worker->id == ievent->socket->tid);
+	isc_result_t result = tcp_send_direct(ievent->socket,
+					      ievent->req);
+	if (result != ISC_R_SUCCESS) {
+		ievent->req->cb.send(NULL, result, ievent->req->cbarg);
+		isc__nm_uvreq_put(&ievent->req, ievent->req->handle->socket);
+	}
+}
+
+static isc_result_t
+tcp_send_direct(isc_nmsocket_t *socket, isc__nm_uvreq_t *req)
+{
+	int rv;
+	INSIST(socket->tid == isc_nm_tid());
+	INSIST(socket->type == isc_nm_tcpsocket);
+
+	rv = uv_write(&req->uv_req.write,
+		      &socket->uv_handle.stream,
+		      &req->uvbuf,
+		      1,
+		      tcp_send_cb);
+	if (rv == 0) {
+		return (ISC_R_SUCCESS);
+	} else {
+		/* TODO call cb! */
+		return (ISC_R_FAILURE);
+	}
+}
+
+static void
+tcp_close_cb(uv_handle_t *uvhandle) {
+	isc_nmsocket_t *socket = uvhandle->data;
+	INSIST(VALID_NMSOCK(socket));
+	atomic_store(&socket->closed, true);
+	isc__nmsocket_prep_destroy(socket);
+}
+
+static void
+tcp_close_direct(isc_nmsocket_t *socket) {
+	INSIST(VALID_NMSOCK(socket));
+	INSIST(socket->tid == isc_nm_tid());
+	INSIST(socket->type == isc_nm_tcpsocket);
+	if (socket->quota != NULL) {
+		isc_quota_detach(&socket->quota);
+		isc_nmsocket_t *ssocket = socket->server;
+		if (ssocket->overquota) {
+			/* XXXWPK TODO we should loop here */
+			isc_result_t result = accept_connection(ssocket);
+			if (result != ISC_R_QUOTA &&
+			    result != ISC_R_SOFTQUOTA) {
+				ssocket->overquota = false;
+			}
+		}
+	}
+	isc_nmsocket_detach(&socket->server);
+	uv_close(&socket->uv_handle.handle, tcp_close_cb);
+}
+
+void
+isc__nm_tcp_close(isc_nmsocket_t *socket) {
+	INSIST(VALID_NMSOCK(socket));
+	INSIST(socket->type == isc_nm_tcpsocket);
+	if (socket->tid == isc_nm_tid()) {
+		tcp_close_direct(socket);
+	} else {
+		/*
+		 * We need to create an event and pass it using async channel
+		 */
+		isc__netievent_tcpclose_t *ievent;
+		ievent = isc__nm_get_ievent(socket->mgr, netievent_tcpclose);
+		ievent->socket = socket;
+		isc__nm_enqueue_ievent(&socket->mgr->workers[socket->tid],
+				       (isc__netievent_t*) ievent);
+	}
+}
+
+void
+isc__nm_handle_tcpclose(isc__networker_t *worker, isc__netievent_t *ievent0) {
+	isc__netievent_tcpclose_t *ievent =
+		(isc__netievent_tcpclose_t *) ievent0;
+	INSIST(worker->id == ievent->socket->tid);
+	tcp_close_direct(ievent->socket);
+}

lib/isc/netmgr/tcpdns.c

@@ -0,0 +1,185 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+#include "config.h"
+
+#include <unistd.h>
+#include <uv.h>
+#include <ck_fifo.h>
+#include <ck_stack.h>
+
+#include <isc/atomic.h>
+#include <isc/buffer.h>
+#include <isc/condition.h>
+#include <isc/magic.h>
+#include <isc/mem.h>
+#include <isc/netmgr.h>
+#include <isc/random.h>
+#include <isc/refcount.h>
+#include <isc/region.h>
+#include <isc/result.h>
+#include <isc/sockaddr.h>
+#include <isc/thread.h>
+#include <isc/util.h>
+
+#include "netmgr-int.h"
+
+
+static void
+dnslisten_readcb(void *arg, isc_nmhandle_t*handle, isc_region_t *region);
+
+
+/*
+ * Accept callback for TCP-DNS connection
+ */
+static void
+dnslisten_acceptcb(isc_nmhandle_t *handle, isc_result_t result, void *cbarg) {
+	INSIST(result == ISC_R_SUCCESS); /* XXXWPK TODO */
+	isc_nmsocket_t *dnslistensocket = (isc_nmsocket_t*) cbarg;
+	INSIST(VALID_NMSOCK(dnslistensocket));
+	INSIST(dnslistensocket->type == isc_nm_tcpdnslistener);
+
+	/* We need to create a 'wrapper' dnssocket for this connection */
+	isc_nmsocket_t *dnssocket = isc_mem_get(handle->socket->mgr->mctx,
+						sizeof(*dnssocket));
+	isc__nmsocket_init(dnssocket, handle->socket->mgr,
+			   isc_nm_tcpdnssocket);
+	/* We need to copy read callbacks from outer socket */
+	dnssocket->rcb.recv = dnslistensocket->rcb.recv;
+	dnssocket->rcbarg = dnslistensocket->rcbarg;
+	dnssocket->extrahandlesize = dnslistensocket->extrahandlesize;
+	isc_nmsocket_attach(handle->socket, &dnssocket->outer);
+	isc_nm_read(handle, dnslisten_readcb, dnssocket);
+}
+
+/*
+ * We've got a read on our underlying socket, need to check if we have
+ * a complete DNS packet and, if so - call the callback
+ */
+static void
+dnslisten_readcb(void *arg, isc_nmhandle_t *handle, isc_region_t *region) {
+	/* A 'wrapper' handle */
+	(void)handle;
+	isc_nmsocket_t *dnssocket = (isc_nmsocket_t*) arg;
+	if (region == NULL) {
+		/* Connection closed */
+		atomic_store(&dnssocket->closed, true);
+		isc_nmhandle_detach(&handle);
+		isc_nmsocket_detach(&dnssocket->outer);
+		isc_nmsocket_detach(&dnssocket);
+		return;
+	}
+	isc_nmhandle_t *dnshandle =
+		isc__nmhandle_get(dnssocket, &handle->peer);
+	/* XXXWPK for the love of all that is holy fix it, that's so wrong */
+	INSIST(((region->base[0] << 8) + (region->base[1]) ==
+		(int) region->length - 2));
+	isc_region_t r2 =
+	{.base = region->base + 2, .length = region->length - 2};
+	dnssocket->rcb.recv(dnssocket->rcbarg, dnshandle, &r2);
+	isc_nmhandle_detach(&dnshandle);
+}
+
+/*
+ * isc_nm_tcp_dnslistens listens for connections and accepts
+ * them immediately, then calls the cb for each incoming DNS packet
+ * (with 2-byte length stripped) - just like for UDP packet.
+ */
+isc_result_t
+isc_nm_tcp_dnslisten(isc_nm_t *mgr,
+		     isc_nmiface_t *iface,
+		     isc_nm_recv_cb_t cb,
+		     size_t extrahandlesize,
+		     void *cbarg,
+		     isc_quota_t *quota,
+		     isc_nmsocket_t **rv)
+{
+	isc_result_t result;
+
+	/* A 'wrapper' socket object with outer set to true TCP socket */
+	isc_nmsocket_t *dnslistensocket =
+		isc_mem_get(mgr->mctx, sizeof(*dnslistensocket));
+	isc__nmsocket_init(dnslistensocket, mgr, isc_nm_tcpdnslistener);
+	dnslistensocket->iface = iface;
+	dnslistensocket->rcb.recv = cb;
+	dnslistensocket->rcbarg = cbarg;
+	dnslistensocket->extrahandlesize = extrahandlesize;
+
+	/* We set dnslistensocket->outer to a true listening socket */
+	result = isc_nm_tcp_listen(mgr,
+				   iface,
+				   dnslisten_acceptcb,
+				   extrahandlesize,
+				   dnslistensocket,
+				   quota,
+				   &dnslistensocket->outer);
+	dnslistensocket->listening = true;
+	*rv = dnslistensocket;
+	return (result);
+}
+
+void
+isc_nm_tcpdns_stoplistening(isc_nmsocket_t *socket) {
+	INSIST(socket->type == isc_nm_tcpdnslistener);
+	isc_nm_tcp_stoplistening(socket->outer);
+	atomic_store(&socket->listening, false);
+	isc_nmsocket_detach(&socket->outer);
+}
+
+
+typedef struct tcpsend {
+	isc_nmhandle_t *	handle;
+	isc_region_t		region;
+	isc_nmhandle_t *	orighandle;
+	isc_nm_send_cb_t	cb;
+	void*			cbarg;
+} tcpsend_t;
+
+static void
+tcpdnssend_cb(isc_nmhandle_t *handle, isc_result_t result, void *cbarg) {
+	tcpsend_t *ts = (tcpsend_t *) cbarg;
+	(void) handle;
+	ts->cb(ts->orighandle, result, ts->cbarg);
+	isc_nmhandle_detach(&ts->orighandle);
+}
+/*
+ * isc__nm_tcp_send sends buf to a peer on a socket.
+ */
+isc_result_t
+isc__nm_tcpdns_send(isc_nmhandle_t *handle,
+		    isc_region_t *region,
+		    isc_nm_send_cb_t cb,
+		    void *cbarg)
+{
+	isc_nmsocket_t *socket = handle->socket;
+
+	INSIST(socket->type == isc_nm_tcpdnssocket);
+	tcpsend_t *t = malloc(sizeof(*t));
+	*t = (tcpsend_t) {};
+	t->handle = handle->socket->outer->tcphandle;
+	t->cb = cb;
+	t->cbarg = cbarg;
+	t->region =
+		(isc_region_t) { .base = malloc(region->length + 2),
+				 .length = region->length + 2 };
+	memmove(t->region.base + 2, region->base, region->length);
+	t->region.base[0] = (uint8_t) (region->length >> 8);
+	t->region.base[1] = (uint8_t) (region->length & 0xff);
+	isc_nmhandle_attach(handle, &t->orighandle);
+	return (isc__nm_tcp_send(t->handle, &t->region, tcpdnssend_cb, t));
+}
+
+void
+isc__nm_tcpdns_close(isc_nmsocket_t *socket) {
+	isc_nmsocket_detach(&socket->outer);
+	socket->closed = true;
+	isc__nmsocket_prep_destroy(socket);
+}

lib/isc/netmgr/udp.c

@@ -0,0 +1,365 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+#include "config.h"
+
+#include <unistd.h>
+#include <uv.h>
+#include <ck_fifo.h>
+#include <ck_stack.h>
+
+#include <isc/atomic.h>
+#include <isc/buffer.h>
+#include <isc/condition.h>
+#include <isc/magic.h>
+#include <isc/mem.h>
+#include <isc/netmgr.h>
+#include <isc/random.h>
+#include <isc/refcount.h>
+#include <isc/region.h>
+#include <isc/result.h>
+#include <isc/sockaddr.h>
+#include <isc/thread.h>
+#include <isc/util.h>
+#include "netmgr-int.h"
+
+/*
+ *  U   U  DDD   PPP
+ *  U   U  D  D  P  P
+ *  U   U  D  D  PPP
+ *  U   U  D  D  P
+ *   UUU   DDD   P
+ */
+
+static isc_result_t
+udp_send_direct(isc_nmsocket_t *socket,
+		isc__nm_uvreq_t *req,
+		isc_sockaddr_t *peer);
+
+static void
+udp_recv_cb(uv_udp_t *handle,
+	    ssize_t nrecv,
+	    const uv_buf_t *buf,
+	    const struct sockaddr *addr,
+	    unsigned flags);
+
+static void
+udp_send_cb(uv_udp_send_t *req, int status);
+
+
+isc_result_t
+isc_nm_udp_listen(isc_nm_t *mgr,
+		  isc_nmiface_t *iface,
+		  isc_nm_recv_cb_t cb,
+		  size_t extrahandlesize,
+		  void *arg,
+		  isc_nmsocket_t **rv) {
+	isc_nmsocket_t *nsocket;
+	int res;
+
+	/*
+	 * We are creating nworkers duplicated sockets, one for each worker
+	 * thread
+	 */
+
+	nsocket = isc_mem_get(mgr->mctx, sizeof(*nsocket));
+	isc__nmsocket_init(nsocket, mgr, isc_nm_udplistener);
+	nsocket->iface = iface;
+	nsocket->nchildren = mgr->nworkers;
+	atomic_init(&nsocket->rchildren, mgr->nworkers);
+	nsocket->children = malloc(mgr->nworkers * sizeof(*nsocket));
+/* TODO for debugging		isc_mem_get(mgr->mctx, mgr->nworkers *
+ * sizeof(*nsocket)); */
+	INSIST(nsocket->rcb.recv == NULL && nsocket->rcbarg == NULL);
+	nsocket->rcb.recv = cb;
+	nsocket->rcbarg = arg;
+	nsocket->extrahandlesize = extrahandlesize;
+
+	for (size_t i = 0; i < mgr->nworkers; i++) {
+		isc__netievent_udplisten_t *ievent;
+		isc_nmsocket_t *csocket = &nsocket->children[i];
+		isc__nmsocket_init(csocket, mgr, isc_nm_udpsocket);
+		csocket->parent = nsocket;
+		csocket->iface = iface;
+		csocket->tid = i;
+		csocket->extrahandlesize = extrahandlesize;
+		INSIST(csocket->rcb.recv == NULL && csocket->rcbarg == NULL);
+		csocket->rcb.recv = cb;
+		csocket->rcbarg = arg;
+		csocket->fd = socket(AF_INET, SOCK_DGRAM, 0);
+		INSIST(csocket->fd >= 0);
+		res = setsockopt(csocket->fd,
+				 SOL_SOCKET,
+				 SO_REUSEPORT,
+				 &(int){1},
+				 sizeof(int));
+		INSIST(res == 0);
+		ievent = isc__nm_get_ievent(mgr, netievent_udplisten);
+		ievent->socket = csocket;
+		isc__nm_enqueue_ievent(&mgr->workers[i],
+				       (isc__netievent_t*) ievent);
+	}
+	*rv = nsocket;
+	return (ISC_R_SUCCESS);
+}
+
+void
+isc_nm_udp_stoplistening(isc_nmsocket_t *socket) {
+	int i;
+
+	INSIST(VALID_NMSOCK(socket));
+	/* We can't be launched from network thread, we'd deadlock */
+	REQUIRE(!isc__nm_in_netthread());
+	INSIST(socket->type == isc_nm_udplistener);
+
+	for (i = 0; i < socket->nchildren; i++) {
+		isc__netievent_udpstoplisten_t *ievent;
+		ievent = isc__nm_get_ievent(socket->mgr,
+					    netievent_udpstoplisten);
+		ievent->socket = &socket->children[i];
+		isc__nm_enqueue_ievent(&socket->mgr->workers[i],
+				       (isc__netievent_t*) ievent);
+	}
+	isc_mutex_lock(&socket->lock);
+	while (atomic_load(&socket->rchildren) > 0) {
+		isc_condition_wait(&socket->cond, &socket->lock);
+	}
+	isc_mutex_unlock(&socket->lock);
+}
+
+/*
+ * handle 'udplisten' async call - start listening on a socket.
+ */
+void
+isc__nm_handle_udplisten(isc__networker_t *worker, isc__netievent_t *ievent0) {
+	isc__netievent_udplisten_t *ievent =
+		(isc__netievent_udplisten_t *) ievent0;
+	isc_nmsocket_t *socket = ievent->socket;
+
+	REQUIRE(socket->type == isc_nm_udpsocket);
+	REQUIRE(socket->iface != NULL);
+	REQUIRE(socket->parent != NULL);
+
+	uv_udp_init(&worker->loop, &socket->uv_handle.udp);
+	socket->uv_handle.udp.data = NULL;
+	isc_nmsocket_attach(socket,
+			    (isc_nmsocket_t**)&socket->uv_handle.udp.data);
+	uv_udp_open(&socket->uv_handle.udp, socket->fd);
+	uv_udp_bind(&socket->uv_handle.udp,
+		    &socket->parent->iface->addr.type.sa,
+		    0);
+	uv_recv_buffer_size(&socket->uv_handle.handle,
+			    &(int){16 * 1024 * 1024});
+	uv_send_buffer_size(&socket->uv_handle.handle,
+			    &(int){16 * 1024 * 1024});
+	uv_udp_recv_start(&socket->uv_handle.udp, isc__nm_alloc_cb,
+			  udp_recv_cb);
+}
+
+static void
+udp_close_cb(uv_handle_t *handle) {
+	isc_nmsocket_t *socket = handle->data;
+	isc_nmsocket_detach((isc_nmsocket_t**)&socket->uv_handle.udp.data);
+}
+/*
+ * handle 'udpstoplisten' async call - stop listening on a socket.
+ */
+void
+isc__nm_handle_udpstoplisten(isc__networker_t *worker,
+			     isc__netievent_t *ievent0) {
+	(void) worker;
+	isc__netievent_udplisten_t *ievent =
+		(isc__netievent_udplisten_t *) ievent0;
+	isc_nmsocket_t *socket = ievent->socket;
+
+	REQUIRE(socket->type == isc_nm_udpsocket);
+	REQUIRE(socket->iface != NULL);
+	REQUIRE(socket->parent != NULL);
+
+	/* XXXWPK TODO do it properly! */
+	if (uv_is_closing((uv_handle_t*) &socket->uv_handle.udp)) {
+		return;
+	}
+	uv_udp_recv_stop(&socket->uv_handle.udp);
+	uv_close((uv_handle_t*) &socket->uv_handle.udp, udp_close_cb);
+	isc_mutex_lock(&socket->parent->lock);
+	atomic_fetch_sub(&socket->parent->rchildren, 1);
+	isc_mutex_unlock(&socket->parent->lock);
+	isc_condition_broadcast(&socket->parent->cond);
+}
+
+
+/*
+ * udp_recv_cb handles incoming UDP packet from uv.
+ * The buffer here is reused for a series of packets,
+ * so we need to allocate a new one. This new one can
+ * be reused to send the response then.
+ */
+
+static void
+udp_recv_cb(uv_udp_t *handle,
+	    ssize_t nrecv,
+	    const uv_buf_t *buf,
+	    const struct sockaddr *addr,
+	    unsigned flags)
+{
+	isc_result_t result;
+	isc_nmhandle_t *nmhandle;
+	isc_sockaddr_t sockaddr;
+	isc_region_t region;
+	/* XXXWPK TODO handle it! */
+	(void) flags;
+	isc_nmsocket_t *socket = (isc_nmsocket_t *) handle->data;
+	REQUIRE(VALID_NMSOCK(socket));
+	INSIST(socket->rcb.recv != NULL);
+
+	/*
+	 * If addr == NULL that's the end of stream - we can
+	 * free the buffer and bail.
+	 */
+	if (addr == NULL) {
+		isc__nm_free_uvbuf(socket, buf);
+		return;
+	}
+
+	result = isc_sockaddr_fromsockaddr(&sockaddr, addr);
+	REQUIRE(result == ISC_R_SUCCESS);
+
+	nmhandle = isc__nmhandle_get(socket, &sockaddr);
+	region.base = (unsigned char *) buf->base;
+	/* not buf->len, that'd be the whole buffer! */
+	region.length = nrecv;
+
+	socket->rcb.recv(socket->rcbarg, nmhandle, &region);
+	isc__nm_free_uvbuf(socket, buf);
+
+	/* If recv callback wants it it should attach to it */
+	isc_nmhandle_detach(&nmhandle);
+}
+
+/*
+ * isc__nm_udp_send sends buf to a peer on a socket.
+ * It tries to find a proper sibling/child socket so that we won't have
+ * to jump to other thread.
+ */
+isc_result_t
+isc__nm_udp_send(isc_nmhandle_t *handle,
+		 isc_region_t *region,
+		 isc_nm_send_cb_t cb,
+		 void *cbarg) {
+	isc_nmsocket_t *psocket, *rsocket;
+	isc_nmsocket_t *socket = handle->socket;
+	isc_sockaddr_t *peer = &handle->peer;
+	isc__netievent_udpsend_t *ievent;
+	isc__nm_uvreq_t *uvreq;
+	int ntid = socket->tid;
+
+	if (socket->type == isc_nm_udpsocket) {
+		INSIST(socket->parent != NULL);
+		psocket = socket->parent;
+	} else if (socket->type == isc_nm_udplistener) {
+		psocket = socket;
+	} else {
+		return (ISC_R_UNEXPECTED);
+	}
+
+	ntid = isc_nm_tid() >= 0 ?
+	       isc_nm_tid() : (int) isc_random_uniform(socket->nchildren);
+
+	rsocket = &psocket->children[ntid];
+
+	uvreq = isc__nm_uvreq_get(socket->mgr, socket);
+	uvreq->uvbuf.base = (char *) region->base;
+	uvreq->uvbuf.len = region->length;
+	isc_nmhandle_attach(handle, &uvreq->handle);
+	uvreq->cb.send = cb;
+	uvreq->cbarg = cbarg;
+
+	if (isc_nm_tid() == rsocket->tid) {
+		/*
+		 * If we're in the same thread as the socket we can send the
+		 * data directly
+		 */
+		return (udp_send_direct(rsocket, uvreq, peer));
+	} else {
+		/*
+		 * We need to create an event and pass it using async channel
+		 */
+		ievent = isc__nm_get_ievent(socket->mgr, netievent_udpsend);
+		ievent->socket = rsocket;
+		ievent->peer = *peer;
+		ievent->req = uvreq;
+		isc__nm_enqueue_ievent(&socket->mgr->workers[rsocket->tid],
+				       (isc__netievent_t*) ievent);
+		return (ISC_R_SUCCESS);
+	}
+	return (ISC_R_UNEXPECTED);
+}
+
+
+/*
+ * handle 'udpsend' async event - send a packet on the socket
+ */
+void
+isc__nm_handle_udpsend(isc__networker_t *worker, isc__netievent_t *ievent0) {
+	isc__netievent_udpsend_t *ievent =
+		(isc__netievent_udpsend_t *) ievent0;
+	INSIST(worker->id == ievent->socket->tid);
+	udp_send_direct(ievent->socket,
+			ievent->req,
+			&ievent->peer);
+}
+
+/*
+ * udp_send_cb - callback
+ */
+static void
+udp_send_cb(uv_udp_send_t *req, int status) {
+	isc_result_t result;
+	isc__nm_uvreq_t *uvreq = (isc__nm_uvreq_t*)req->data;
+	INSIST(VALID_UVREQ(uvreq));
+	if (status == 0) {
+		result = ISC_R_SUCCESS;
+	} else {
+		result = ISC_R_FAILURE;
+	}
+	INSIST(VALID_NMHANDLE(uvreq->handle));
+	uvreq->cb.send(uvreq->handle, result, uvreq->cbarg);
+	isc__nm_uvreq_put(&uvreq, uvreq->handle->socket);
+}
+
+/*
+ * udp_send_direct sends buf to a peer on a socket. Sock has to be in
+ * the same thread as the callee.
+ */
+static isc_result_t
+udp_send_direct(isc_nmsocket_t *socket,
+		isc__nm_uvreq_t *req,
+		isc_sockaddr_t *peer)
+{
+	int rv;
+	INSIST(socket->tid == isc_nm_tid());
+	INSIST(socket->type == isc_nm_udpsocket);
+
+	rv = uv_udp_send(&req->uv_req.udp_send,
+			 &socket->uv_handle.udp,
+			 &req->uvbuf,
+			 1,
+			 &peer->type.sa,
+			 udp_send_cb);
+	if (rv == 0) {
+		return (ISC_R_SUCCESS);
+	} else {
+		printf("Fail\n");
+		return (ISC_R_FAILURE);
+	}
+}
+

lib/isc/sockaddr.c

@@ -472,3 +472,26 @@ isc_sockaddr_frompath(isc_sockaddr_t *sockaddr, const char *path) {
 	return (ISC_R_NOTIMPLEMENTED);
 #endif
 }
+
+isc_result_t
+isc_sockaddr_fromsockaddr(isc_sockaddr_t *isa, const struct sockaddr *sa) {
+	unsigned int length = 0;
+	switch (sa->sa_family) {
+		case AF_INET:
+			length = sizeof(isa->type.sin);
+			break;
+		case AF_INET6:
+			length = sizeof(isa->type.sin6);
+			break;
+		case AF_UNIX:
+			length = sizeof(isa->type.sunix);
+			break;
+	}
+	if (length == 0) {
+		return (ISC_R_NOTIMPLEMENTED);
+	}
+	memset(isa, 0, sizeof(isc_sockaddr_t));
+	memcpy(isa, sa, length);
+	isa->length = length;
+	return (ISC_R_SUCCESS);
+}

lib/isc/task.c

@@ -78,8 +78,8 @@
  ***/
 
 typedef enum {
-	task_state_idle, task_state_ready, task_state_running,
-	task_state_done
+	task_state_idle, task_state_ready, task_state_paused,
+	task_state_running, task_state_done
 } task_state_t;
 
 #if defined(HAVE_LIBXML2) || defined(HAVE_JSON_C)
@@ -155,6 +155,7 @@ struct isc__taskmgr {
 	atomic_uint_fast32_t		curq;
 	atomic_uint_fast32_t		tasks_count;
 	isc__taskqueue_t		*queues;
+	isc_nm_t			*nm;
 
 	/* Locked by task manager lock. */
 	unsigned int			default_quantum;
@@ -489,7 +490,8 @@ task_send(isc__task_t *task, isc_event_t **eventp, int c) {
 		task->state = task_state_ready;
 	}
 	INSIST(task->state == task_state_ready ||
-	       task->state == task_state_running);
+	       task->state == task_state_running ||
+	       task->state == task_state_paused);
 	ENQUEUE(task->events, event, ev_link);
 	task->nevents++;
 	*eventp = NULL;
@@ -1108,6 +1110,7 @@ dispatch(isc__taskmgr_t *manager, unsigned int threadid) {
 						  memory_order_acquire);
 
 			LOCK(&task->lock);
+			/* XXXWPK TODO */
 			INSIST(task->state == task_state_ready);
 			task->state = task_state_running;
 			XTRACE("running");
@@ -1323,7 +1326,8 @@ manager_free(isc__taskmgr_t *manager) {
 
 isc_result_t
 isc_taskmgr_create(isc_mem_t *mctx, unsigned int workers,
-		    unsigned int default_quantum, isc_taskmgr_t **managerp)
+		    unsigned int default_quantum,
+		    isc_nm_t *nm, isc_taskmgr_t **managerp)
 {
 	unsigned int i;
 	isc__taskmgr_t *manager;
@@ -1353,6 +1357,9 @@ isc_taskmgr_create(isc_mem_t *mctx, unsigned int workers,
 		default_quantum = DEFAULT_DEFAULT_QUANTUM;
 	}
 	manager->default_quantum = default_quantum;
+
+	manager->nm = nm;
+
 	INIT_LIST(manager->tasks);
 	atomic_store(&manager->tasks_count, 0);
 	manager->queues = isc_mem_get(mctx,
@@ -1599,6 +1606,9 @@ isc_task_beginexclusive(isc_task_t *task0) {
 		WAIT(&manager->halt_cond, &manager->halt_lock);
 	}
 	UNLOCK(&manager->halt_lock);
+	if (manager->nm != NULL) {
+		isc_nm_pause(manager->nm);
+	}
 	return (ISC_R_SUCCESS);
 }
 
@@ -1609,6 +1619,9 @@ isc_task_endexclusive(isc_task_t *task0) {
 
 	REQUIRE(VALID_TASK(task));
 	REQUIRE(task->state == task_state_running);
+	if (manager->nm != NULL) {
+		isc_nm_resume(manager->nm);
+	}
 	LOCK(&manager->halt_lock);
 	REQUIRE(atomic_load_relaxed(&manager->exclusive_req) == true);
 	atomic_store_relaxed(&manager->exclusive_req, false);
@@ -1617,6 +1630,44 @@ isc_task_endexclusive(isc_task_t *task0) {
 		WAIT(&manager->halt_cond, &manager->halt_lock);
 	}
 	UNLOCK(&manager->halt_lock);
+
+}
+
+void
+isc_task_pause(isc_task_t *task0) {
+	REQUIRE(ISCAPI_TASK_VALID(task0));
+	isc__task_t *task = (isc__task_t *)task0;
+	isc__taskmgr_t *manager = task->manager;
+	LOCK(&task->lock);
+	INSIST(task->state == task_state_idle || task->state == task_state_ready);
+	task->state = task_state_paused;
+	UNLOCK(&task->lock);
+	/* XXXWPK TODO optimize it - 'lazy' removal in runner? */
+	LOCK(&manager->queues[task->threadid].lock);
+	if (ISC_LINK_LINKED(task, ready_link)) {
+		DEQUEUE(manager->queues[task->threadid].ready_tasks,
+			task, ready_link);
+	}
+	UNLOCK(&manager->queues[task->threadid].lock);
+}
+
+void
+isc_task_unpause(isc_task_t *task0) {
+	REQUIRE(ISCAPI_TASK_VALID(task0));
+	isc__task_t *task = (isc__task_t *)task0;
+	LOCK(&task->lock);
+	INSIST(task->state == task_state_paused);
+	bool was_idle = false;
+	if (!EMPTY(task->events)) {
+		task->state = task_state_ready;
+		was_idle = true;
+	} else {
+		task->state = task_state_idle;
+	}
+	UNLOCK(&task->lock);
+	if (was_idle) {
+		task_ready(task);
+	}
 }
 
 void
@@ -1884,8 +1935,8 @@ isc_taskmgr_createinctx(isc_mem_t *mctx,
 {
 	isc_result_t result;
 
-	result = isc_taskmgr_create(mctx, workers, default_quantum,
-				       managerp);
+	result = isc_taskmgr_create(mctx, workers, default_quantum, NULL,
+				    managerp);
 
 	return (result);
 }

lib/isc/tests/isctest.c

@@ -33,6 +33,7 @@ isc_log_t *lctx = NULL;
 isc_taskmgr_t *taskmgr = NULL;
 isc_timermgr_t *timermgr = NULL;
 isc_socketmgr_t *socketmgr = NULL;
+isc_nm_t *netmgr = NULL;
 isc_task_t *maintask = NULL;
 int ncpus;
 
@@ -55,6 +56,9 @@ static isc_logcategory_t categories[] = {
 
 static void
 cleanup_managers(void) {
+	if (netmgr != NULL) {
+		isc_nm_shutdown(&netmgr);
+	}
 	if (maintask != NULL) {
 		isc_task_shutdown(maintask);
 		isc_task_destroy(&maintask);
@@ -84,12 +88,13 @@ create_managers(unsigned int workers) {
 		workers = atoi(p);
 	}
 
-	CHECK(isc_taskmgr_create(mctx, workers, 0, &taskmgr));
+	CHECK(isc_taskmgr_create(mctx, workers, 0, NULL, &taskmgr));
 	CHECK(isc_task_create(taskmgr, 0, &maintask));
 	isc_taskmgr_setexcltask(taskmgr, maintask);
 
 	CHECK(isc_timermgr_create(mctx, &timermgr));
 	CHECK(isc_socketmgr_create(mctx, &socketmgr));
+	netmgr = isc_nm_start(mctx, 3);
 	return (ISC_R_SUCCESS);
 
  cleanup:

lib/isc/tests/isctest.h

@@ -17,6 +17,7 @@
 #include <isc/buffer.h>
 #include <isc/hash.h>
 #include <isc/log.h>
+#include <isc/netmgr.h>
 #include <isc/mem.h>
 #include <isc/print.h>
 #include <isc/result.h>
@@ -37,6 +38,7 @@ extern isc_log_t *lctx;
 extern isc_taskmgr_t *taskmgr;
 extern isc_timermgr_t *timermgr;
 extern isc_socketmgr_t *socketmgr;
+extern isc_nm_t *netmgr;
 extern int ncpus;
 
 isc_result_t

lib/isc/tests/task_test.c

@@ -711,7 +711,7 @@ manytasks(void **state) {
 	result = isc_mem_create(0, 0, &mctx);
 	assert_int_equal(result, ISC_R_SUCCESS);
 
-	result = isc_taskmgr_create(mctx, 4, 0, &taskmgr);
+	result = isc_taskmgr_create(mctx, 4, 0, NULL, &taskmgr);
 	assert_int_equal(result, ISC_R_SUCCESS);
 
 	done = false;

lib/isc/unix/time.c

@@ -36,15 +36,10 @@
 #define NS_PER_MS	1000000		/*%< Nanoseconds per millisecond. */
 #define US_PER_S	1000000		/*%< Microseconds per second. */
 
-/*
- * All of the INSIST()s checks of nanoseconds < NS_PER_S are for
- * consistency checking of the type. In lieu of magic numbers, it
- * is the best we've got.  The check is only performed on functions which
- * need an initialized type.
- */
-
-#ifndef ISC_FIX_TV_USEC
-#define ISC_FIX_TV_USEC 1
+#ifdef CLOCK_REALTIME_COARSE
+#define CLOCKSOURCE CLOCK_REALTIME_COARSE
+#else
+#define CLOCKSOURCE CLOCK_REALTIME
 #endif
 
 /*%
@@ -54,32 +49,6 @@
 static const isc_interval_t zero_interval = { 0, 0 };
 const isc_interval_t * const isc_interval_zero = &zero_interval;
 
-#if ISC_FIX_TV_USEC
-static inline void
-fix_tv_usec(struct timeval *tv) {
-	bool fixed = false;
-
-	if (tv->tv_usec < 0) {
-		fixed = true;
-		do {
-			tv->tv_sec -= 1;
-			tv->tv_usec += US_PER_S;
-		} while (tv->tv_usec < 0);
-	} else if (tv->tv_usec >= US_PER_S) {
-		fixed = true;
-		do {
-			tv->tv_sec += 1;
-			tv->tv_usec -= US_PER_S;
-		} while (tv->tv_usec >=US_PER_S);
-	}
-	/*
-	 * Call syslog directly as was are called from the logging functions.
-	 */
-	if (fixed)
-		(void)syslog(LOG_ERR, "gettimeofday returned bad tv_usec: corrected");
-}
-#endif
-
 void
 isc_interval_set(isc_interval_t *i,
 		 unsigned int seconds, unsigned int nanoseconds)
@@ -141,76 +110,52 @@ isc_time_isepoch(const isc_time_t *t) {
 
 isc_result_t
 isc_time_now(isc_time_t *t) {
-	struct timeval tv;
+	struct timespec ts;
 	char strbuf[ISC_STRERRORSIZE];
 
 	REQUIRE(t != NULL);
 
-	if (gettimeofday(&tv, NULL) == -1) {
+	if (clock_gettime(CLOCKSOURCE, &ts) == -1) {
 		strerror_r(errno, strbuf, sizeof(strbuf));
 		UNEXPECTED_ERROR(__FILE__, __LINE__, "%s", strbuf);
 		return (ISC_R_UNEXPECTED);
 	}
 
-	/*
-	 * Does POSIX guarantee the signedness of tv_sec and tv_usec?  If not,
-	 * then this test will generate warnings for platforms on which it is
-	 * unsigned.  In any event, the chances of any of these problems
-	 * happening are pretty much zero, but since the libisc library ensures
-	 * certain things to be true ...
-	 */
-#if ISC_FIX_TV_USEC
-	fix_tv_usec(&tv);
-	if (tv.tv_sec < 0)
+	if (ts.tv_sec < 0 || ts.tv_nsec < 0 || ts.tv_nsec >= NS_PER_S) {
 		return (ISC_R_UNEXPECTED);
-#else
-	if (tv.tv_sec < 0 || tv.tv_usec < 0 || tv.tv_usec >= US_PER_S)
-		return (ISC_R_UNEXPECTED);
-#endif
+	}
 
 	/*
 	 * Ensure the tv_sec value fits in t->seconds.
 	 */
-	if (sizeof(tv.tv_sec) > sizeof(t->seconds) &&
-	    ((tv.tv_sec | (unsigned int)-1) ^ (unsigned int)-1) != 0U)
+	if (sizeof(ts.tv_sec) > sizeof(t->seconds) &&
+	    ((ts.tv_sec | (unsigned int)-1) ^ (unsigned int)-1) != 0U)
 		return (ISC_R_RANGE);
 
-	t->seconds = tv.tv_sec;
-	t->nanoseconds = tv.tv_usec * NS_PER_US;
+	t->seconds = ts.tv_sec;
+	t->nanoseconds = ts.tv_nsec;
 
 	return (ISC_R_SUCCESS);
 }
 
 isc_result_t
 isc_time_nowplusinterval(isc_time_t *t, const isc_interval_t *i) {
-	struct timeval tv;
+	struct timespec ts;
 	char strbuf[ISC_STRERRORSIZE];
 
 	REQUIRE(t != NULL);
 	REQUIRE(i != NULL);
 	INSIST(i->nanoseconds < NS_PER_S);
 
-	if (gettimeofday(&tv, NULL) == -1) {
+	if (clock_gettime(CLOCKSOURCE, &ts) == -1) {
 		strerror_r(errno, strbuf, sizeof(strbuf));
 		UNEXPECTED_ERROR(__FILE__, __LINE__, "%s", strbuf);
 		return (ISC_R_UNEXPECTED);
 	}
 
-	/*
-	 * Does POSIX guarantee the signedness of tv_sec and tv_usec?  If not,
-	 * then this test will generate warnings for platforms on which it is
-	 * unsigned.  In any event, the chances of any of these problems
-	 * happening are pretty much zero, but since the libisc library ensures
-	 * certain things to be true ...
-	 */
-#if ISC_FIX_TV_USEC
-	fix_tv_usec(&tv);
-	if (tv.tv_sec < 0)
+	if (ts.tv_sec < 0 || ts.tv_nsec < 0 || ts.tv_nsec >= NS_PER_S) {
 		return (ISC_R_UNEXPECTED);
-#else
-	if (tv.tv_sec < 0 || tv.tv_usec < 0 || tv.tv_usec >= US_PER_S)
-		return (ISC_R_UNEXPECTED);
-#endif
+	}
 
 	/*
 	 * Ensure the resulting seconds value fits in the size of an
@@ -218,12 +163,12 @@ isc_time_nowplusinterval(isc_time_t *t, const isc_interval_t *i) {
 	 * note that even if both values == INT_MAX, then when added
 	 * and getting another 1 added below the result is UINT_MAX.)
 	 */
-	if ((tv.tv_sec > INT_MAX || i->seconds > INT_MAX) &&
-	    ((long long)tv.tv_sec + i->seconds > UINT_MAX))
+	if ((ts.tv_sec > INT_MAX || i->seconds > INT_MAX) &&
+	    ((long long)ts.tv_sec + i->seconds > UINT_MAX))
 		return (ISC_R_RANGE);
 
-	t->seconds = tv.tv_sec + i->seconds;
-	t->nanoseconds = tv.tv_usec * NS_PER_US + i->nanoseconds;
+	t->seconds = ts.tv_sec + i->seconds;
+	t->nanoseconds = ts.tv_nsec + i->nanoseconds;
 	if (t->nanoseconds >= NS_PER_S) {
 		t->seconds++;
 		t->nanoseconds -= NS_PER_S;

lib/isc/win32/libisc.def.in

@@ -504,6 +504,7 @@ isc_sockaddr_eqaddr
 isc_sockaddr_eqaddrprefix
 isc_sockaddr_equal
 isc_sockaddr_format
+isc_sockaddr_fromsockaddr
 isc_sockaddr_fromin
 isc_sockaddr_fromin6
 isc_sockaddr_fromnetaddr
@@ -564,6 +565,7 @@ isc_task_exiting
 isc_task_getcurrenttime
 isc_task_getcurrenttimex
 isc_task_onshutdown
+isc_task_pause
 isc_task_privilege
 isc_task_purge
 isc_task_purgeevent
@@ -575,6 +577,7 @@ isc_task_sendtoanddetach
 isc_task_setname
 isc_task_setprivilege
 isc_task_shutdown
+isc_task_unpause
 isc_task_unsend
 isc_taskmgr_create
 isc_taskmgr_createinctx

lib/ns/include/ns/client.h

@@ -59,6 +59,7 @@
 
 #include <isc/buffer.h>
 #include <isc/magic.h>
+#include <isc/netmgr.h>
 #include <isc/stdtime.h>
 #include <isc/quota.h>
 #include <isc/queue.h>
@@ -87,10 +88,71 @@ typedef struct ns_tcpconn {
 	bool			pipelined;
 } ns_tcpconn_t;
 
+#define NS_CLIENT_TCP_BUFFER_SIZE			(65535 + 2)
+#define NS_CLIENT_SEND_BUFFER_SIZE		4096
+#define NS_CLIENT_RECV_BUFFER_SIZE		4096
+
+#define CLIENT_NMCTXS				100
+/*%<
+ * Number of 'mctx pools' for clients. (Should this be configurable?)
+ * When enabling threads, we use a pool of memory contexts shared by
+ * client objects, since concurrent access to a shared context would cause
+ * heavy contentions.  The above constant is expected to be enough for
+ * completely avoiding contentions among threads for an authoritative-only
+ * server.
+ */
+
+#define CLIENT_NTASKS				100
+/*%<
+ * Number of tasks to be used by clients - those are used only when recursing
+ */
+
+typedef ISC_QUEUE(ns_client_t) client_queue_t;
+typedef ISC_LIST(ns_client_t) client_list_t;
+
+/*% nameserver client manager structure */
+struct ns_clientmgr {
+	/* Unlocked. */
+	unsigned int			magic;
+
+	/* The queue object has its own locks */
+	client_queue_t			inactive;     /*%< To be recycled */
+
+	isc_mem_t *			mctx;
+	ns_server_t *			sctx;
+	isc_taskmgr_t *			taskmgr;
+	isc_timermgr_t *		timermgr;
+	isc_task_t *			excl;
+
+	/* Attached by clients, needed for e.g. recursion */
+	isc_task_t **			taskpool;
+
+	ns_interface_t			*interface;
+
+	/* Lock covers manager state. */
+	isc_mutex_t			lock;
+	bool			exiting;
+
+	/* Lock covers the clients list */
+	isc_mutex_t			listlock;
+	client_list_t			clients;      /*%< All active clients */
+
+	/* Lock covers the recursing list */
+	isc_mutex_t			reclock;
+	client_list_t			recursing;    /*%< Recursing clients */
+
+#if CLIENT_NMCTXS > 0
+	/*%< mctx pool for clients. */
+	unsigned int			nextmctx;
+	isc_mem_t *			mctxpool[CLIENT_NMCTXS];
+#endif
+};
+
 /*% nameserver client structure */
 struct ns_client {
 	unsigned int		magic;
 	isc_mem_t		*mctx;
+	bool			allocated;	/* Do we need to free it? */
 	ns_server_t		*sctx;
 	ns_clientmgr_t		*manager;
 	int			state;
@@ -114,19 +176,17 @@ struct ns_client {
 	isc_task_t		*task;
 	dns_view_t		*view;
 	dns_dispatch_t		*dispatch;
+	isc_nmhandle_t		*handle;
 	isc_socket_t		*udpsocket;
 	isc_socket_t		*tcplistener;
 	isc_socket_t		*tcpsocket;
 	unsigned char		*tcpbuf;
 	dns_tcpmsg_t		tcpmsg;
 	bool			tcpmsg_valid;
-	isc_timer_t		*timer;
 	isc_timer_t		*delaytimer;
-	bool 			timerset;
 	dns_message_t		*message;
-	isc_socketevent_t	*sendevent;
-	isc_socketevent_t	*recvevent;
 	unsigned char		*recvbuf;
+	unsigned char		sendbuf[NS_CLIENT_SEND_BUFFER_SIZE];
 	dns_rdataset_t		*opt;
 	uint16_t		udpsize;
 	uint16_t		extflags;
@@ -143,7 +203,6 @@ struct ns_client {
 	bool			mortal;	      /*%< Die after handling request */
 	ns_tcpconn_t		*tcpconn;
 	isc_quota_t		*recursionquota;
-	ns_interface_t		*interface;
 
 	isc_sockaddr_t		peeraddr;
 	bool			peeraddr_valid;
@@ -154,7 +213,6 @@ struct ns_client {
 
 	struct in6_pktinfo	pktinfo;
 	isc_dscp_t		dscp;
-	isc_event_t		ctlevent;
 	/*%
 	 * Information about recent FORMERR response(s), for
 	 * FORMERR loop avoidance.  This is separate for each
@@ -187,9 +245,6 @@ struct ns_client {
 	int32_t			rcode_override;
 };
 
-typedef ISC_QUEUE(ns_client_t) client_queue_t;
-typedef ISC_LIST(ns_client_t) client_list_t;
-
 #define NS_CLIENT_MAGIC			ISC_MAGIC('N','S','C','c')
 #define NS_CLIENT_VALID(c)		ISC_MAGIC_VALID(c, NS_CLIENT_MAGIC)
 
@@ -296,7 +351,8 @@ ns_client_settimeout(ns_client_t *client, unsigned int seconds);
 
 isc_result_t
 ns_clientmgr_create(isc_mem_t *mctx, ns_server_t *sctx, isc_taskmgr_t *taskmgr,
-		    isc_timermgr_t *timermgr, ns_clientmgr_t **managerp);
+		    isc_timermgr_t *timermgr, ns_interface_t *ifp,
+		    ns_clientmgr_t **managerp);
 /*%<
  * Create a client manager.
  */
@@ -308,15 +364,6 @@ ns_clientmgr_destroy(ns_clientmgr_t **managerp);
  * managed by it.
  */
 
-isc_result_t
-ns_clientmgr_createclients(ns_clientmgr_t *manager, unsigned int n,
-			   ns_interface_t *ifp, bool tcp);
-/*%<
- * Create up to 'n' clients listening on interface 'ifp'.
- * If 'tcp' is true, the clients will listen for TCP connections,
- * otherwise for UDP requests.
- */
-
 isc_sockaddr_t *
 ns_client_getsockaddr(ns_client_t *client);
 /*%<
@@ -427,16 +474,17 @@ isc_result_t
 ns_client_addopt(ns_client_t *client, dns_message_t *message,
 		 dns_rdataset_t **opt);
 
-isc_result_t
-ns__clientmgr_getclient(ns_clientmgr_t *manager, ns_interface_t *ifp,
-			bool tcp, ns_client_t **clientp);
 /*
  * Get a client object from the inactive queue, or create one, as needed.
  * (Not intended for use outside this module and associated tests.)
  */
 
 void
-ns__client_request(isc_task_t *task, isc_event_t *event);
+ns__client_request(void *arg,
+		   isc_nmhandle_t *handle,
+		   struct isc_region *region);
+void
+ns__client_request_old(isc_task_t *task, isc_event_t *event);
 /*
  * Handle client requests.
  * (Not intended for use outside this module and associated tests.)

lib/ns/include/ns/interfacemgr.h

@@ -44,6 +44,7 @@
 
 #include <isc/magic.h>
 #include <isc/mem.h>
+#include <isc/netmgr.h>
 #include <isc/socket.h>
 #include <isc/refcount.h>
 
@@ -76,6 +77,8 @@ struct ns_interface {
 	dns_dispatch_t *	udpdispatch[MAX_UDP_DISPATCH];
 						/*%< UDP dispatchers. */
 	isc_socket_t *		tcpsocket;	/*%< TCP socket. */
+	isc_nmsocket_t *	udplistensocket;
+	isc_nmsocket_t *	tcplistensocket;
 	isc_dscp_t		dscp;		/*%< "listen-on" DSCP value */
 	isc_refcount_t		ntcpaccepting;	/*%< Number of clients
 						     ready to accept new
@@ -100,6 +103,7 @@ ns_interfacemgr_create(isc_mem_t *mctx,
 		       isc_taskmgr_t *taskmgr,
 		       isc_timermgr_t *timermgr,
 		       isc_socketmgr_t *socketmgr,
+		       isc_nm_t *nm,
 		       dns_dispatchmgr_t *dispatchmgr,
 		       isc_task_t *task,
 		       unsigned int udpdisp,

lib/ns/include/ns/server.h

@@ -36,7 +36,6 @@
 #define NS_SERVER_NOAA		0x00000002U	/*%< -T noaa */
 #define NS_SERVER_NOSOA		0x00000004U	/*%< -T nosoa */
 #define NS_SERVER_NONEAREST	0x00000008U	/*%< -T nonearest */
-#define NS_SERVER_CLIENTTEST	0x00000010U	/*%< -T clienttest */
 #define NS_SERVER_NOEDNS	0x00000020U	/*%< -T noedns */
 #define NS_SERVER_DROPEDNS	0x00000040U	/*%< -T dropedns */
 #define NS_SERVER_NOTCP		0x00000080U	/*%< -T notcp */

lib/ns/interfacemgr.c

@@ -14,6 +14,7 @@
 #include <stdbool.h>
 
 #include <isc/interfaceiter.h>
+#include <isc/netmgr.h>
 #include <isc/os.h>
 #include <isc/random.h>
 #include <isc/string.h>
@@ -72,6 +73,7 @@ struct ns_interfacemgr {
 	isc_task_t *		excl;		/*%< Exclusive task. */
 	isc_timermgr_t *	timermgr;	/*%< Timer manager. */
 	isc_socketmgr_t *	socketmgr;	/*%< Socket manager. */
+	isc_nm_t *		nm;		/*%< Net manager. */
 	dns_dispatchmgr_t *	dispatchmgr;
 	unsigned int		generation;	/*%< Current generation no. */
 	ns_listenlist_t *	listenon4;
@@ -172,6 +174,7 @@ ns_interfacemgr_create(isc_mem_t *mctx,
 		       isc_taskmgr_t *taskmgr,
 		       isc_timermgr_t *timermgr,
 		       isc_socketmgr_t *socketmgr,
+		       isc_nm_t	*nm,
 		       dns_dispatchmgr_t *dispatchmgr,
 		       isc_task_t *task,
 		       unsigned int udpdisp,
@@ -208,6 +211,7 @@ ns_interfacemgr_create(isc_mem_t *mctx,
 	mgr->taskmgr = taskmgr;
 	mgr->timermgr = timermgr;
 	mgr->socketmgr = socketmgr;
+	mgr->nm = nm;
 	mgr->dispatchmgr = dispatchmgr;
 	mgr->generation = 1;
 	mgr->listenon4 = NULL;
@@ -249,8 +253,9 @@ ns_interfacemgr_create(isc_mem_t *mctx,
 	}
 
 	mgr->task = NULL;
-	if (mgr->route != NULL)
+	if (mgr->route != NULL) {
 		isc_task_attach(task, &mgr->task);
+	}
 	isc_refcount_init(&mgr->references, (mgr->route != NULL) ? 2 : 1);
 #else
 	isc_refcount_init(&mgr->references, 1);
@@ -390,7 +395,7 @@ ns_interface_create(ns_interfacemgr_t *mgr, isc_sockaddr_t *addr,
 	isc_mutex_init(&ifp->lock);
 
 	result = ns_clientmgr_create(mgr->mctx, mgr->sctx,
-				     mgr->taskmgr, mgr->timermgr,
+				     mgr->taskmgr, mgr->timermgr, ifp,
 				     &ifp->clientmgr);
 	if (result != ISC_R_SUCCESS) {
 		isc_log_write(IFMGR_COMMON_LOGARGS, ISC_LOG_ERROR,
@@ -440,127 +445,42 @@ ns_interface_create(ns_interfacemgr_t *mgr, isc_sockaddr_t *addr,
 static isc_result_t
 ns_interface_listenudp(ns_interface_t *ifp) {
 	isc_result_t result;
-	unsigned int attrs;
-	unsigned int attrmask;
-	int disp, i;
-
-	attrs = 0;
-	attrs |= DNS_DISPATCHATTR_UDP;
-	if (isc_sockaddr_pf(&ifp->addr) == AF_INET)
-		attrs |= DNS_DISPATCHATTR_IPV4;
-	else
-		attrs |= DNS_DISPATCHATTR_IPV6;
-	attrs |= DNS_DISPATCHATTR_NOLISTEN;
-	attrs |= DNS_DISPATCHATTR_CANREUSE;
-	attrmask = 0;
-	attrmask |= DNS_DISPATCHATTR_UDP | DNS_DISPATCHATTR_TCP;
-	attrmask |= DNS_DISPATCHATTR_IPV4 | DNS_DISPATCHATTR_IPV6;
-
-	ifp->nudpdispatch = ISC_MIN(ifp->mgr->udpdisp, MAX_UDP_DISPATCH);
-	for (disp = 0; disp < ifp->nudpdispatch; disp++) {
-		result = dns_dispatch_getudp_dup(ifp->mgr->dispatchmgr,
-						 ifp->mgr->socketmgr,
-						 ifp->mgr->taskmgr, &ifp->addr,
-						 4096, UDPBUFFERS,
-						 32768, 8219, 8237,
-						 attrs, attrmask,
-						 &ifp->udpdispatch[disp],
-						 disp == 0
-						    ? NULL
-						    : ifp->udpdispatch[0]);
-		if (result != ISC_R_SUCCESS) {
-			isc_log_write(IFMGR_COMMON_LOGARGS, ISC_LOG_ERROR,
-				      "could not listen on UDP socket: %s",
-				      isc_result_totext(result));
-			goto udp_dispatch_failure;
-		}
-
-	}
-
-	result = ns_clientmgr_createclients(ifp->clientmgr, ifp->nudpdispatch,
-					    ifp, false);
-	if (result != ISC_R_SUCCESS) {
-		UNEXPECTED_ERROR(__FILE__, __LINE__,
-				 "UDP ns_clientmgr_createclients(): %s",
-				 isc_result_totext(result));
-		goto addtodispatch_failure;
-	}
-
-	return (ISC_R_SUCCESS);
-
- addtodispatch_failure:
-	for (i = disp - 1; i >= 0; i--) {
-		dns_dispatch_changeattributes(ifp->udpdispatch[i], 0,
-					      DNS_DISPATCHATTR_NOLISTEN);
-		dns_dispatch_detach(&(ifp->udpdispatch[i]));
-	}
-	ifp->nudpdispatch = 0;
-
- udp_dispatch_failure:
+	/* XXXWPK TODO we have to increase refcount on ifp, for both UDP and TCP */
+	result = isc_nm_udp_listen(ifp->mgr->nm,
+				   (isc_nmiface_t*) &ifp->addr, /* XXXWPK TODO! */
+				   ns__client_request,
+				   sizeof(ns_client_t),
+				   ifp,
+				   &ifp->udplistensocket);
 	return (result);
 }
 
 static isc_result_t
-ns_interface_accepttcp(ns_interface_t *ifp) {
+ns_interface_listentcp(ns_interface_t *ifp) {
 	isc_result_t result;
-
-	/*
-	 * Open a TCP socket.
-	 */
-	result = isc_socket_create(ifp->mgr->socketmgr,
-				   isc_sockaddr_pf(&ifp->addr),
-				   isc_sockettype_tcp,
-				   &ifp->tcpsocket);
+	result = isc_nm_tcp_dnslisten(ifp->mgr->nm,
+				      (isc_nmiface_t*) &ifp->addr, /* XXXWPK TODO! */
+				      ns__client_request,
+				      sizeof(ns_client_t),
+				      ifp,
+				      &ifp->mgr->sctx->tcpquota,
+				      &ifp->tcplistensocket);
 	if (result != ISC_R_SUCCESS) {
 		isc_log_write(IFMGR_COMMON_LOGARGS, ISC_LOG_ERROR,
 				 "creating TCP socket: %s",
 				 isc_result_totext(result));
-		goto tcp_socket_failure;
+		return (result);
 	}
-	isc_socket_setname(ifp->tcpsocket, "dispatcher", NULL);
+/* 	isc_socket_setname(ifp->tcpsocket, "dispatcher", NULL);
 #ifndef ISC_ALLOW_MAPPED
 	isc_socket_ipv6only(ifp->tcpsocket, true);
 #endif
-	result = isc_socket_bind(ifp->tcpsocket, &ifp->addr,
-				 ISC_SOCKET_REUSEADDRESS);
-	if (result != ISC_R_SUCCESS) {
-		isc_log_write(IFMGR_COMMON_LOGARGS, ISC_LOG_ERROR,
-				 "binding TCP socket: %s",
-				 isc_result_totext(result));
-		goto tcp_bind_failure;
-	}
-
-	if (ifp->dscp != -1)
+*/
+/*	if (ifp->dscp != -1)
 		isc_socket_dscp(ifp->tcpsocket, ifp->dscp);
 
-	result = isc_socket_listen(ifp->tcpsocket, ifp->mgr->backlog);
-	if (result != ISC_R_SUCCESS) {
-		isc_log_write(IFMGR_COMMON_LOGARGS, ISC_LOG_ERROR,
-				 "listening on TCP socket: %s",
-				 isc_result_totext(result));
-		goto tcp_listen_failure;
-	}
-
-	/*
-	 * If/when there a multiple filters listen to the
-	 * result.
-	 */
 	(void)isc_socket_filter(ifp->tcpsocket, "dataready");
-
-	result = ns_clientmgr_createclients(ifp->clientmgr, 1, ifp, true);
-	if (result != ISC_R_SUCCESS) {
-		UNEXPECTED_ERROR(__FILE__, __LINE__,
-				 "TCP ns_clientmgr_createclients(): %s",
-				 isc_result_totext(result));
-		goto accepttcp_failure;
-	}
-	return (ISC_R_SUCCESS);
-
- accepttcp_failure:
- tcp_listen_failure:
- tcp_bind_failure:
-	isc_socket_detach(&ifp->tcpsocket);
- tcp_socket_failure:
+*/
 	return (result);
 }
 
@@ -591,7 +511,7 @@ ns_interface_setup(ns_interfacemgr_t *mgr, isc_sockaddr_t *addr,
 	if (((mgr->sctx->options & NS_SERVER_NOTCP) == 0) &&
 	    accept_tcp == true)
 	{
-		result = ns_interface_accepttcp(ifp);
+		result = ns_interface_listentcp(ifp);
 		if (result != ISC_R_SUCCESS) {
 			if ((result == ISC_R_ADDRINUSE) &&
 			    (addr_in_use != NULL))
@@ -617,8 +537,17 @@ ns_interface_setup(ns_interfacemgr_t *mgr, isc_sockaddr_t *addr,
 
 void
 ns_interface_shutdown(ns_interface_t *ifp) {
-	if (ifp->clientmgr != NULL)
+	if (ifp->udplistensocket != NULL) {
+		isc_nm_udp_stoplistening(ifp->udplistensocket);
+		isc_nmsocket_detach(&ifp->udplistensocket);
+	}
+	if (ifp->tcplistensocket != NULL) {
+		isc_nm_tcpdns_stoplistening(ifp->tcplistensocket);
+		isc_nmsocket_detach(&ifp->tcplistensocket);
+	}
+	if (ifp->clientmgr != NULL) {
 		ns_clientmgr_destroy(&ifp->clientmgr);
+	}
 }
 
 static void

lib/ns/query.c

@@ -2488,15 +2488,6 @@ query_prefetch(ns_client_t *client, dns_name_t *qname,
 	if (client->recursionquota == NULL) {
 		result = isc_quota_attach(&client->sctx->recursionquota,
 					  &client->recursionquota);
-		if (result == ISC_R_SUCCESS || result == ISC_R_SOFTQUOTA) {
-			ns_stats_increment(client->sctx->nsstats,
-					   ns_statscounter_recursclients);
-		}
-		if (result == ISC_R_SUCCESS && !client->mortal &&
-		    !TCP(client))
-		{
-			result = ns_client_replace(client);
-		}
 		if (result != ISC_R_SUCCESS) {
 			return;
 		}
@@ -2701,15 +2692,6 @@ query_rpzfetch(ns_client_t *client, dns_name_t *qname, dns_rdatatype_t type) {
 	if (client->recursionquota == NULL) {
 		result = isc_quota_attach(&client->sctx->recursionquota,
 					  &client->recursionquota);
-		if (result == ISC_R_SUCCESS || result == ISC_R_SOFTQUOTA) {
-			ns_stats_increment(client->sctx->nsstats,
-					   ns_statscounter_recursclients);
-		}
-		if (result == ISC_R_SUCCESS && !client->mortal &&
-		    !TCP(client))
-		{
-			result = ns_client_replace(client);
-		}
 		if (result != ISC_R_SUCCESS) {
 			return;
 		}
@@ -4644,7 +4626,7 @@ dns64_aaaaok(ns_client_t *client, dns_rdataset_t *rdataset,
 	     dns_rdataset_t *sigrdataset)
 {
 	isc_netaddr_t netaddr;
-	dns_aclenv_t *env = ns_interfacemgr_getaclenv(client->interface->mgr);
+	dns_aclenv_t *env = ns_interfacemgr_getaclenv(client->manager->interface->mgr);
 	dns_dns64_t *dns64 = ISC_LIST_HEAD(client->view->dns64);
 	unsigned int flags = 0;
 	unsigned int i, count;
@@ -5801,20 +5783,6 @@ ns_query_recurse(ns_client_t *client, dns_rdatatype_t qtype, dns_name_t *qname,
 			}
 			ns_client_killoldestquery(client);
 		}
-		if (result == ISC_R_SUCCESS && !client->mortal &&
-		    !TCP(client)) {
-			result = ns_client_replace(client);
-			if (result != ISC_R_SUCCESS) {
-				ns_client_log(client, NS_LOGCATEGORY_CLIENT,
-					      NS_LOGMODULE_QUERY,
-					      ISC_LOG_WARNING,
-					      "ns_client_replace() failed: %s",
-					      isc_result_totext(result));
-				isc_quota_detach(&client->recursionquota);
-				ns_stats_decrement(client->sctx->nsstats,
-					   ns_statscounter_recursclients);
-			}
-		}
 		if (result != ISC_R_SUCCESS)
 			return (result);
 		ns_client_recursing(client);
@@ -7349,7 +7317,7 @@ query_respond(query_ctx_t *qctx) {
 static isc_result_t
 query_dns64(query_ctx_t *qctx) {
 	ns_client_t *client = qctx->client;
-	dns_aclenv_t *env = ns_interfacemgr_getaclenv(client->interface->mgr);
+	dns_aclenv_t *env = ns_interfacemgr_getaclenv(client->manager->interface->mgr);
 	dns_name_t *name, *mname;
 	dns_rdata_t *dns64_rdata;
 	dns_rdata_t rdata = DNS_RDATA_INIT;
@@ -10616,7 +10584,7 @@ static void
 query_setup_sortlist(query_ctx_t *qctx) {
 	isc_netaddr_t netaddr;
 	ns_client_t *client = qctx->client;
-	dns_aclenv_t *env = ns_interfacemgr_getaclenv(client->interface->mgr);
+	dns_aclenv_t *env = ns_interfacemgr_getaclenv(client->manager->interface->mgr);
 	const void *order_arg = NULL;
 
 	isc_netaddr_fromsockaddr(&netaddr, &client->peeraddr);
@@ -10958,22 +10926,6 @@ ns_query_start(ns_client_t *client) {
 
 	CTRACE(ISC_LOG_DEBUG(3), "ns_query_start");
 
-	/*
-	 * Test only.
-	 */
-	if (((client->sctx->options & NS_SERVER_CLIENTTEST) != 0) &&
-	    !TCP(client))
-	{
-		result = ns_client_replace(client);
-		if (result == ISC_R_SHUTTINGDOWN) {
-			ns_client_next(client, result);
-			return;
-		} else if (result != ISC_R_SUCCESS) {
-			query_error(client, result, __LINE__);
-			return;
-		}
-	}
-
 	/*
 	 * Ensure that appropriate cleanups occur.
 	 */

lib/ns/tests/nstest.c

@@ -22,6 +22,7 @@
 #include <isc/file.h>
 #include <isc/hash.h>
 #include <isc/mem.h>
+#include <isc/netmgr.h>
 #include <isc/os.h>
 #include <isc/print.h>
 #include <isc/random.h>
@@ -55,6 +56,7 @@ isc_taskmgr_t *taskmgr = NULL;
 isc_task_t *maintask = NULL;
 isc_timermgr_t *timermgr = NULL;
 isc_socketmgr_t *socketmgr = NULL;
+isc_nm_t *nm = NULL;
 dns_zonemgr_t *zonemgr = NULL;
 dns_dispatchmgr_t *dispatchmgr = NULL;
 ns_clientmgr_t *clientmgr = NULL;
@@ -151,6 +153,12 @@ cleanup_managers(void) {
 	if (socketmgr != NULL) {
 		isc_socketmgr_destroy(&socketmgr);
 	}
+	if (socketmgr != NULL) {
+		isc_socketmgr_destroy(&socketmgr);
+	}
+	if (nm != NULL ){
+		isc_nm_shutdown(&nm);
+	}
 	if (taskmgr != NULL) {
 		isc_taskmgr_destroy(&taskmgr);
 	}
@@ -177,7 +185,7 @@ create_managers(void) {
 	isc_event_t *event = NULL;
 	ncpus = isc_os_ncpus();
 
-	CHECK(isc_taskmgr_create(mctx, ncpus, 0, &taskmgr));
+	CHECK(isc_taskmgr_create(mctx, ncpus, 0, NULL, &taskmgr));
 	CHECK(isc_task_create(taskmgr, 0, &maintask));
 	isc_taskmgr_setexcltask(taskmgr, maintask);
 	CHECK(isc_task_onshutdown(maintask, shutdown_managers, NULL));
@@ -186,15 +194,17 @@ create_managers(void) {
 
 	CHECK(isc_socketmgr_create(mctx, &socketmgr));
 
+	nm = isc_nm_start(mctx, ncpus);
+
 	CHECK(ns_server_create(mctx, matchview, &sctx));
 
 	CHECK(dns_dispatchmgr_create(mctx, &dispatchmgr));
 
 	CHECK(ns_interfacemgr_create(mctx, sctx, taskmgr, timermgr,
-				     socketmgr, dispatchmgr, maintask,
+				     socketmgr, nm, dispatchmgr, maintask,
 				     ncpus, NULL, &interfacemgr));
 
-	CHECK(ns_clientmgr_create(mctx, sctx, taskmgr, timermgr,
+	CHECK(ns_clientmgr_create(mctx, sctx, taskmgr, timermgr, NULL,
 				  &clientmgr));
 
 	CHECK(ns_listenlist_default(mctx, 5300, -1, true, &listenon));
@@ -509,6 +519,8 @@ isc_result_t
 ns_test_getclient(ns_interface_t *ifp0, bool tcp,
 		  ns_client_t **clientp)
 {
+	(void) tcp;
+	(void) clientp;
 	isc_result_t result;
 	ns_interface_t *ifp = ifp0;
 
@@ -519,7 +531,8 @@ ns_test_getclient(ns_interface_t *ifp0, bool tcp,
 		return (ISC_R_FAILURE);
 	}
 
-	result = ns__clientmgr_getclient(clientmgr, ifp, tcp, clientp);
+//	result = ns__clientmgr_getclient(clientmgr, tcp, clientp);
+	result = ISC_R_SUCCESS;
 	return (result);
 }
 

lib/ns/update.c

@@ -2518,7 +2518,7 @@ update_action(isc_task_t *task, isc_event_t *event) {
 	dns_ttl_t maxttl = 0;
 	uint32_t maxrecords;
 	uint64_t records;
-	dns_aclenv_t *env = ns_interfacemgr_getaclenv(client->interface->mgr);
+	dns_aclenv_t *env = ns_interfacemgr_getaclenv(client->manager->interface->mgr);
 
 	INSIST(event->ev_type == DNS_EVENT_UPDATE);
 
@@ -3475,12 +3475,6 @@ send_forward_event(ns_client_t *client, dns_zone_t *zone) {
 	isc_task_t *zonetask = NULL;
 	ns_client_t *evclient;
 
-	/*
-	 * This may take some time so replace this client.
-	 */
-	if (!client->mortal && (client->attributes & NS_CLIENTATTR_TCP) == 0)
-		CHECK(ns_client_replace(client));
-
 	event = (update_event_t *)
 		isc_event_allocate(client->mctx, client, DNS_EVENT_UPDATE,
 				   forward_action, NULL, sizeof(*event));

lib/ns/xfrout.c

@@ -669,6 +669,7 @@ typedef struct {
 						   names and rdatas */
 	isc_buffer_t 		txlenbuf;	/* Transmit length buffer */
 	isc_buffer_t		txbuf;		/* Transmit message buffer */
+	size_t			cbytes;		/* Lenght of current message */
 	void 			*txmem;
 	unsigned int 		txmemlen;
 	dns_tsigkey_t		*tsigkey;	/* Key used to create TSIG */
@@ -699,7 +700,7 @@ static void
 sendstream(xfrout_ctx_t *xfr);
 
 static void
-xfrout_senddone(isc_task_t *task, isc_event_t *event);
+xfrout_senddone(isc_nmhandle_t *handle, isc_result_t result, void* arg);
 
 static void
 xfrout_fail(xfrout_ctx_t *xfr, isc_result_t result, const char *msg);
@@ -1202,6 +1203,8 @@ xfrout_ctx_create(isc_mem_t *mctx, ns_client_t *client, unsigned int id,
 		  unsigned int maxtime, unsigned int idletime,
 		  bool many_answers, xfrout_ctx_t **xfrp)
 {
+	(void) maxtime;
+	(void) idletime;
 	xfrout_ctx_t *xfr;
 	isc_result_t result;
 	unsigned int len;
@@ -1271,8 +1274,8 @@ xfrout_ctx_create(isc_mem_t *mctx, ns_client_t *client, unsigned int id,
 	xfr->txmem = mem;
 	xfr->txmemlen = len;
 
-	CHECK(dns_timer_setidle(xfr->client->timer,
-				maxtime, idletime, false));
+//	CHECK(dns_timer_setidle(xfr->client->timer,
+//				maxtime, idletime, false));
 
 	/*
 	 * Register a shutdown callback with the client, so that we
@@ -1291,7 +1294,6 @@ xfrout_ctx_create(isc_mem_t *mctx, ns_client_t *client, unsigned int id,
 	*xfrp = xfr;
 	return (ISC_R_SUCCESS);
 
-failure:
 	xfrout_ctx_destroy(&xfr);
 	return (result);
 }
@@ -1310,8 +1312,6 @@ sendstream(xfrout_ctx_t *xfr) {
 	dns_message_t *tcpmsg = NULL;
 	dns_message_t *msg = NULL; /* Client message if UDP, tcpmsg if TCP */
 	isc_result_t result;
-	isc_region_t used;
-	isc_region_t region;
 	dns_rdataset_t *qrdataset;
 	dns_name_t *msgname = NULL;
 	dns_rdata_t *msgrdata = NULL;
@@ -1545,6 +1545,7 @@ sendstream(xfrout_ctx_t *xfr) {
 	}
 
 	if (is_tcp) {
+		isc_region_t used;
 		CHECK(dns_compress_init(&cctx, -1, xfr->mctx));
 		dns_compress_setsensitive(&cctx, true);
 		cleanup_cctx = true;
@@ -1556,18 +1557,15 @@ sendstream(xfrout_ctx_t *xfr) {
 		cleanup_cctx = false;
 
 		isc_buffer_usedregion(&xfr->txbuf, &used);
-		isc_buffer_putuint16(&xfr->txlenbuf,
-				     (uint16_t)used.length);
-		region.base = xfr->txlenbuf.base;
-		region.length = 2 + used.length;
 		xfrout_log(xfr, ISC_LOG_DEBUG(8),
 			   "sending TCP message of %d bytes",
 			   used.length);
-		CHECK(isc_socket_send(xfr->client->tcpsocket, /* XXX */
-				      &region, xfr->client->task,
-				      xfrout_senddone,
-				      xfr));
+		CHECK(isc_nm_send(xfr->client->handle,
+				  &used,
+				  xfrout_senddone,
+				  xfr));
 		xfr->sends++;
+		xfr->cbytes = used.length+2;
 	} else {
 		xfrout_log(xfr, ISC_LOG_DEBUG(8), "sending IXFR UDP response");
 		ns_client_send(xfr->client);
@@ -1652,14 +1650,10 @@ xfrout_ctx_destroy(xfrout_ctx_t **xfrp) {
 }
 
 static void
-xfrout_senddone(isc_task_t *task, isc_event_t *event) {
-	isc_socketevent_t *sev = (isc_socketevent_t *)event;
-	xfrout_ctx_t *xfr = (xfrout_ctx_t *)event->ev_arg;
-	isc_result_t evresult = sev->result;
+xfrout_senddone(isc_nmhandle_t *handle, isc_result_t result, void* arg) {
+	xfrout_ctx_t *xfr = (xfrout_ctx_t *)arg;
+	(void) handle;
 
-	UNUSED(task);
-
-	INSIST(event->ev_type == ISC_SOCKEVENT_SENDDONE);
 	INSIST((xfr->client->attributes & NS_CLIENTATTR_TCP) != 0);
 
 	xfr->sends--;
@@ -1669,18 +1663,16 @@ xfrout_senddone(isc_task_t *task, isc_event_t *event) {
 	 * Update transfer statistics if sending succeeded, accounting for the
 	 * two-byte TCP length prefix included in the number of bytes sent.
 	 */
-	if (evresult == ISC_R_SUCCESS) {
+	if (result == ISC_R_SUCCESS) {
 		xfr->stats.nmsg++;
-		xfr->stats.nbytes += sev->region.length - 2;
+		xfr->stats.nbytes += xfr->cbytes;
 	}
 
-	isc_event_free(&event);
-
-	(void)isc_timer_touch(xfr->client->timer);
+//	(void)isc_timer_touch(xfr->client->timer);
 	if (xfr->shuttingdown == true) {
 		xfrout_maybe_destroy(xfr);
-	} else if (evresult != ISC_R_SUCCESS) {
-		xfrout_fail(xfr, evresult, "send");
+	} else if (result != ISC_R_SUCCESS) {
+		xfrout_fail(xfr, result, "send");
 	} else if (xfr->end_of_stream == false) {
 		sendstream(xfr);
 	} else {

util/copyrights

@@ -2169,6 +2169,7 @@
 ./lib/isc/include/isc/mutexatomic.h		C	2019
 ./lib/isc/include/isc/mutexblock.h		C	1999,2000,2001,2004,2005,2006,2007,2016,2018,2019
 ./lib/isc/include/isc/netaddr.h			C	1998,1999,2000,2001,2002,2004,2005,2006,2007,2009,2015,2016,2017,2018,2019
+./lib/isc/include/isc/netmgr.h			C	2019
 ./lib/isc/include/isc/netscope.h		C	2002,2004,2005,2006,2007,2009,2016,2018,2019
 ./lib/isc/include/isc/nonce.h			C	2018,2019
 ./lib/isc/include/isc/os.h			C	2000,2001,2004,2005,2006,2007,2016,2018,2019
@@ -2225,6 +2226,11 @@
 ./lib/isc/mem_p.h				C	2018,2019
 ./lib/isc/mutexblock.c				C	1999,2000,2001,2004,2005,2007,2011,2012,2016,2018,2019
 ./lib/isc/netaddr.c				C	1999,2000,2001,2002,2004,2005,2007,2010,2011,2012,2014,2015,2016,2017,2018,2019
+./lib/isc/netmgr/netmgr-int.h			C	2019
+./lib/isc/netmgr/netmgr.c			C	2019
+./lib/isc/netmgr/tcp.c				C	2019
+./lib/isc/netmgr/tcpdns.c			C	2019
+./lib/isc/netmgr/udp.c				C	2019
 ./lib/isc/netscope.c				C	2002,2004,2005,2006,2007,2016,2018,2019
 ./lib/isc/nonce.c				C	2018,2019
 ./lib/isc/openssl_shim.c			C	2018,2019