WiP/checkpoint2 - working XFRIN with netmgr

WiP checkpoint: xfrin using netmgr
Rename isc_nmsocket_t->tcphandle to statichandle
2020-06-15 19:04:28 +02:00 · 2020-06-15 19:02:50 +02:00 · 2020-06-15 19:01:50 +02:00 · 2020-06-10 10:45:38 +02:00 · 2020-06-10 01:19:15 -07:00 · 2020-06-10 01:19:15 -07:00
25 changed files with 1436 additions and 1106 deletions
@@ -1,3 +1,13 @@
+5438.	[func]		The control channel and the rndc utility have been
+			updated to use the new network manager. To support
+			this, the network manager was updated to enable
+			wthe initiation of client TCP connections. Its
+			internal reference counting has been refactored.
+
+			As a side effect of these changes, rndc cannot now be
+			used with UNIX-domain sockets, and its default timeout
+			has changed from 60 seconds to 30. [GL #1759]
+
 5437.	[bug]		Fix a data race in resolver log_formerr. [GL #1808]

 5436.	[placeholder]
@@ -38,6 +38,7 @@
 #define NAMED_EVENTCLASS    ISC_EVENTCLASS(0x4E43)
 #define NAMED_EVENT_RELOAD  (NAMED_EVENTCLASS + 0)
 #define NAMED_EVENT_DELZONE (NAMED_EVENTCLASS + 1)
+#define NAMED_EVENT_COMMAND (NAMED_EVENTCLASS + 2)

 /*%
 * Name server state.  Better here than in lots of separate global variables.
@@ -80,8 +81,6 @@ struct named_server {
 	uint32_t interface_interval;
 	uint32_t heartbeat_interval;

-	isc_mutex_t    reload_event_lock;
-	isc_event_t *  reload_event;
 	named_reload_t reload_status;

 	bool flushonshutdown;
@@ -9908,14 +9908,6 @@ named_server_create(isc_mem_t *mctx, named_server_t **serverp) {
 				     &server->in_roothints),
 		   "setting up root hints");

-	isc_mutex_init(&server->reload_event_lock);
-
-	server->reload_event = isc_event_allocate(
-		named_g_mctx, server, NAMED_EVENT_RELOAD, named_server_reload,
-		server, sizeof(isc_event_t));
-	CHECKFATAL(server->reload_event == NULL ? ISC_R_NOMEMORY
-						: ISC_R_SUCCESS,
-		   "allocating reload event");
 	server->reload_status = NAMED_RELOAD_IN_PROGRESS;

 	/*
@@ -9962,7 +9954,7 @@ named_server_create(isc_mem_t *mctx, named_server_t **serverp) {

 	CHECKFATAL(dns_zonemgr_create(named_g_mctx, named_g_taskmgr,
 				      named_g_timermgr, named_g_socketmgr,
-				      &server->zonemgr),
+				      named_g_nm, &server->zonemgr),
 		   "dns_zonemgr_create");
 	CHECKFATAL(dns_zonemgr_setsize(server->zonemgr, 1000), "dns_zonemgr_"
 							       "setsize");
@@ -10085,9 +10077,6 @@ named_server_destroy(named_server_t **serverp) {

 	dst_lib_destroy();

-	isc_event_free(&server->reload_event);
-	isc_mutex_destroy(&server->reload_event_lock);
-
 	INSIST(ISC_LIST_EMPTY(server->kasplist));
 	INSIST(ISC_LIST_EMPTY(server->viewlist));
 	INSIST(ISC_LIST_EMPTY(server->cachelist));
@@ -10265,7 +10254,7 @@ cleanup:
 */
 static void
 named_server_reload(isc_task_t *task, isc_event_t *event) {
-	named_server_t *server = (named_server_t *)event->ev_arg;
+	named_server_t *server = (named_server_t *)event->ev_sender;

 	INSIST(task == server->task);
 	UNUSED(task);
@@ -10275,19 +10264,15 @@ named_server_reload(isc_task_t *task, isc_event_t *event) {
 		      "received SIGHUP signal to reload zones");
 	(void)reload(server);

-	LOCK(&server->reload_event_lock);
-	INSIST(server->reload_event == NULL);
-	server->reload_event = event;
-	UNLOCK(&server->reload_event_lock);
+	isc_event_free(&event);
 }

 void
 named_server_reloadwanted(named_server_t *server) {
-	LOCK(&server->reload_event_lock);
-	if (server->reload_event != NULL) {
-		isc_task_send(server->task, &server->reload_event);
-	}
-	UNLOCK(&server->reload_event_lock);
+	isc_event_t *event = isc_event_allocate(
+		named_g_mctx, server, NAMED_EVENT_RELOAD, named_server_reload,
+		NULL, sizeof(isc_event_t));
+	isc_task_send(server->task, &event);
 }

 void
@@ -12058,12 +12043,10 @@ cleanup:

 isc_result_t
 named_server_tsiglist(named_server_t *server, isc_buffer_t **text) {
-	isc_result_t result;
+	isc_result_t result = ISC_R_SUCCESS;
 	dns_view_t *view;
 	unsigned int foundkeys = 0;

-	result = isc_task_beginexclusive(server->task);
-	RUNTIME_CHECK(result == ISC_R_SUCCESS);
 	for (view = ISC_LIST_HEAD(server->viewlist); view != NULL;
 	     view = ISC_LIST_NEXT(view, link))
 	{
@@ -12072,7 +12055,6 @@ named_server_tsiglist(named_server_t *server, isc_buffer_t **text) {
 				       &foundkeys);
 		RWUNLOCK(&view->statickeys->lock, isc_rwlocktype_read);
 		if (result != ISC_R_SUCCESS) {
-			isc_task_endexclusive(server->task);
 			return (result);
 		}
 		RWLOCK(&view->dynamickeys->lock, isc_rwlocktype_read);
@@ -12080,11 +12062,9 @@ named_server_tsiglist(named_server_t *server, isc_buffer_t **text) {
 				       &foundkeys);
 		RWUNLOCK(&view->dynamickeys->lock, isc_rwlocktype_read);
 		if (result != ISC_R_SUCCESS) {
-			isc_task_endexclusive(server->task);
 			return (result);
 		}
 	}
-	isc_task_endexclusive(server->task);

 	if (foundkeys == 0) {
 		CHECK(putstr(text, "no tsig keys found."));
@@ -14105,7 +14085,6 @@ named_server_showzone(named_server_t *server, isc_lex_t *lex,
 	dns_view_t *view = NULL;
 	dns_zone_t *zone = NULL;
 	ns_cfgctx_t *cfg = NULL;
-	bool exclusive = false;
 #ifdef HAVE_LMDB
 	cfg_obj_t *nzconfig = NULL;
 #endif /* HAVE_LMDB */
@@ -14130,10 +14109,6 @@ named_server_showzone(named_server_t *server, isc_lex_t *lex,
 		goto cleanup;
 	}

-	result = isc_task_beginexclusive(server->task);
-	RUNTIME_CHECK(result == ISC_R_SUCCESS);
-	exclusive = true;
-
 	if (!added) {
 		/* Find the view statement */
 		vconfig = find_name_in_list_from_map(cfg->config, "view",
@@ -14192,9 +14167,6 @@ cleanup:
 	if (isc_buffer_usedlength(*text) > 0) {
 		(void)putnull(text);
 	}
-	if (exclusive) {
-		isc_task_endexclusive(server->task);
-	}

 	return (result);
 }
@@ -55,11 +55,14 @@

 #define SERVERADDRS 10

-const char *progname;
+const char *progname = NULL;
 bool verbose;

-static const char *admin_conffile;
-static const char *admin_keyfile;
+static isc_taskmgr_t *taskmgr = NULL;
+static isc_task_t *rndc_task = NULL;
+
+static const char *admin_conffile = NULL;
+static const char *admin_keyfile = NULL;
 static const char *version = PACKAGE_VERSION;
 static const char *servername = NULL;
 static isc_sockaddr_t serveraddrs[SERVERADDRS];
@@ -68,27 +71,27 @@ static bool local4set = false, local6set = false;
 static int nserveraddrs;
 static int currentaddr = 0;
 static unsigned int remoteport = 0;
-static isc_socketmgr_t *socketmgr = NULL;
-static isc_buffer_t *databuf;
-static isccc_ccmsg_t ccmsg;
+static isc_nm_t *netmgr = NULL;
+static isc_buffer_t *databuf = NULL;
+static isccc_ccmsg_t rndc_ccmsg;
 static uint32_t algorithm;
 static isccc_region_t secret;
 static bool failed = false;
 static bool c_flag = false;
-static isc_mem_t *rndc_mctx;
+static isc_mem_t *rndc_mctx = NULL;
 static atomic_uint_fast32_t sends = ATOMIC_VAR_INIT(0);
 static atomic_uint_fast32_t recvs = ATOMIC_VAR_INIT(0);
 static atomic_uint_fast32_t connects = ATOMIC_VAR_INIT(0);
-static char *command;
-static char *args;
+static char *command = NULL;
+static char *args = NULL;
 static char program[256];
-static isc_socket_t *sock = NULL;
 static uint32_t serial;
 static bool quiet = false;
 static bool showresult = false;
+static bool shuttingdown = false;

 static void
-rndc_startconnect(isc_sockaddr_t *addr, isc_task_t *task);
+rndc_startconnect(isc_sockaddr_t *addr);

 ISC_NORETURN static void
 usage(int status);
@@ -111,7 +114,7 @@ command is one of the following:\n\
 		Close, rename and re-open the DNSTAP output file(s).\n\
  dumpdb [-all|-cache|-zones|-adb|-bad|-fail] [view ...]\n\
 		Dump cache(s) to the dump file (named_dump.db).\n\
-  flush 	Flushes all of the server's caches.\n\
+  flush         Flushes all of the server's caches.\n\
  flush [view]	Flushes the server's cache for a view.\n\
  flushname name [view]\n\
 		Flush the given name from the server's cache(s)\n\
@@ -275,51 +278,55 @@ get_addresses(const char *host, in_port_t port) {
 }

 static void
-rndc_senddone(isc_task_t *task, isc_event_t *event) {
-	isc_socketevent_t *sevent = (isc_socketevent_t *)event;
+rndc_senddone(isc_nmhandle_t *handle, isc_result_t result, void *arg) {
+	UNUSED(handle);
+	UNUSED(arg);

-	UNUSED(task);
-
-	if (sevent->result != ISC_R_SUCCESS) {
-		fatal("send failed: %s", isc_result_totext(sevent->result));
+	if (result != ISC_R_SUCCESS) {
+		fatal("send failed: %s", isc_result_totext(result));
 	}
-	isc_event_free(&event);
+
 	if (atomic_fetch_sub_release(&sends, 1) == 1 &&
 	    atomic_load_acquire(&recvs) == 0)
 	{
-		isc_socket_detach(&sock);
-		isc_task_shutdown(task);
+		shuttingdown = true;
+		isc_task_shutdown(rndc_task);
 		isc_app_shutdown();
+		isc_nmhandle_unref(handle);
 	}
 }

 static void
-rndc_recvdone(isc_task_t *task, isc_event_t *event) {
+rndc_recvdone(isc_nmhandle_t *handle, isc_result_t result, void *arg) {
+	isccc_ccmsg_t *ccmsg = (isccc_ccmsg_t *)arg;
 	isccc_sexpr_t *response = NULL;
 	isccc_sexpr_t *data;
 	isccc_region_t source;
 	char *errormsg = NULL;
 	char *textmsg = NULL;
-	isc_result_t result;
+
+	UNUSED(handle);

 	atomic_fetch_sub_release(&recvs, 1);

-	if (ccmsg.result == ISC_R_EOF) {
-		fatal("connection to remote host closed\n"
-		      "This may indicate that\n"
-		      "* the remote server is using an older version of"
-		      " the command protocol,\n"
+	if (shuttingdown && (result == ISC_R_EOF || result == ISC_R_CANCELED)) {
+		isc_nmhandle_unref(handle);
+		return;
+	} else if (result == ISC_R_EOF) {
+		fatal("connection to remote host closed.\n"
+		      "* This may indicate that the\n"
+		      "* remote server is using an older\n"
+		      "* version of the command protocol,\n"
 		      "* this host is not authorized to connect,\n"
-		      "* the clocks are not synchronized, or\n"
-		      "* the key is invalid.");
+		      "* the clocks are not synchronized,\n"
+		      "* the key signing algorithm is incorrect,\n"
+		      "* or the key is invalid.");
+	} else if (result != ISC_R_SUCCESS && result != ISC_R_CANCELED) {
+		fatal("recv failed: %s", isc_result_totext(result));
 	}

-	if (ccmsg.result != ISC_R_SUCCESS) {
-		fatal("recv failed: %s", isc_result_totext(ccmsg.result));
-	}
-
-	source.rstart = isc_buffer_base(&ccmsg.buffer);
-	source.rend = isc_buffer_used(&ccmsg.buffer);
+	source.rstart = isc_buffer_base(ccmsg->buffer);
+	source.rend = isc_buffer_used(ccmsg->buffer);

 	DO("parse message",
 	   isccc_cc_fromwire(&source, &response, algorithm, &secret));
@@ -359,22 +366,23 @@ rndc_recvdone(isc_task_t *task, isc_event_t *event) {
 		}
 	}

-	isc_event_free(&event);
 	isccc_sexpr_free(&response);
+
 	if (atomic_load_acquire(&sends) == 0 &&
 	    atomic_load_acquire(&recvs) == 0) {
-		isc_socket_detach(&sock);
-		isc_task_shutdown(task);
+		shuttingdown = true;
+		isc_task_shutdown(rndc_task);
 		isc_app_shutdown();
+		isc_nmhandle_unref(handle);
 	}
 }

 static void
-rndc_recvnonce(isc_task_t *task, isc_event_t *event) {
+rndc_recvnonce(isc_nmhandle_t *handle, isc_result_t result, void *arg) {
+	isccc_ccmsg_t *ccmsg = (isccc_ccmsg_t *)arg;
 	isccc_sexpr_t *response = NULL;
 	isccc_sexpr_t *_ctrl;
 	isccc_region_t source;
-	isc_result_t result;
 	uint32_t nonce;
 	isccc_sexpr_t *request = NULL;
 	isccc_time_t now;
@@ -384,23 +392,24 @@ rndc_recvnonce(isc_task_t *task, isc_event_t *event) {

 	atomic_fetch_sub_release(&recvs, 1);

-	if (ccmsg.result == ISC_R_EOF) {
-		fatal("connection to remote host closed\n"
-		      "This may indicate that\n"
-		      "* the remote server is using an older version of"
-		      " the command protocol,\n"
+	if (shuttingdown && result == ISC_R_EOF) {
+		isc_nmhandle_unref(handle);
+		return;
+	} else if (result == ISC_R_EOF) {
+		fatal("connection to remote host closed.\n"
+		      "* This may indicate that the\n"
+		      "* remote server is using an older\n"
+		      "* version of the command protocol,\n"
 		      "* this host is not authorized to connect,\n"
 		      "* the clocks are not synchronized,\n"
-		      "* the key signing algorithm is incorrect, or\n"
-		      "* the key is invalid.");
+		      "* the key signing algorithm is incorrect\n"
+		      "* or the key is invalid.");
+	} else if (result != ISC_R_SUCCESS) {
+		fatal("recv failed: %s", isc_result_totext(result));
 	}

-	if (ccmsg.result != ISC_R_SUCCESS) {
-		fatal("recv failed: %s", isc_result_totext(ccmsg.result));
-	}
-
-	source.rstart = isc_buffer_base(&ccmsg.buffer);
-	source.rend = isc_buffer_used(&ccmsg.buffer);
+	source.rstart = isc_buffer_base(ccmsg->buffer);
+	source.rend = isc_buffer_used(ccmsg->buffer);

 	DO("parse message",
 	   isccc_cc_fromwire(&source, &response, algorithm, &secret));
@@ -448,48 +457,42 @@ rndc_recvnonce(isc_task_t *task, isc_event_t *event) {
 	r.base = databuf->base;
 	r.length = databuf->used;

-	isccc_ccmsg_cancelread(&ccmsg);
 	DO("schedule recv",
-	   isccc_ccmsg_readmessage(&ccmsg, task, rndc_recvdone, NULL));
+	   isccc_ccmsg_readmessage(ccmsg, rndc_recvdone, ccmsg));
 	atomic_fetch_add_relaxed(&recvs, 1);
-	DO("send message",
-	   isc_socket_send(sock, &r, task, rndc_senddone, NULL));
+
+	DO("send message", isc_nm_send(handle, &r, rndc_senddone, NULL));
 	atomic_fetch_add_relaxed(&sends, 1);

-	isc_event_free(&event);
 	isccc_sexpr_free(&response);
 	isccc_sexpr_free(&request);
 	return;
 }

 static void
-rndc_connected(isc_task_t *task, isc_event_t *event) {
+rndc_connected(isc_nmhandle_t *handle, isc_result_t result, void *arg) {
+	isccc_ccmsg_t *ccmsg = (isccc_ccmsg_t *)arg;
 	char socktext[ISC_SOCKADDR_FORMATSIZE];
-	isc_socketevent_t *sevent = (isc_socketevent_t *)event;
 	isccc_sexpr_t *request = NULL;
 	isccc_sexpr_t *data;
 	isccc_time_t now;
 	isc_region_t r;
 	isc_buffer_t b;
-	isc_result_t result;

 	atomic_fetch_sub_release(&connects, 1);

-	if (sevent->result != ISC_R_SUCCESS) {
+	if (result != ISC_R_SUCCESS) {
 		isc_sockaddr_format(&serveraddrs[currentaddr], socktext,
 				    sizeof(socktext));
-		if (sevent->result != ISC_R_CANCELED &&
-		    ++currentaddr < nserveraddrs) {
+		if (++currentaddr < nserveraddrs) {
 			notify("connection failed: %s: %s", socktext,
-			       isc_result_totext(sevent->result));
-			isc_socket_detach(&sock);
-			isc_event_free(&event);
-			rndc_startconnect(&serveraddrs[currentaddr], task);
+			       isc_result_totext(result));
+			rndc_startconnect(&serveraddrs[currentaddr]);
 			return;
-		} else {
-			fatal("connect failed: %s: %s", socktext,
-			      isc_result_totext(sevent->result));
 		}
+
+		fatal("connect failed: %s: %s", socktext,
+		      isc_result_totext(result));
 	}

 	isc_stdtime_get(&now);
@@ -516,50 +519,52 @@ rndc_connected(isc_task_t *task, isc_event_t *event) {
 	r.base = databuf->base;
 	r.length = databuf->used;

-	isccc_ccmsg_init(rndc_mctx, sock, &ccmsg);
-	isccc_ccmsg_setmaxsize(&ccmsg, 1024 * 1024);
+	isccc_ccmsg_init(rndc_mctx, handle, ccmsg);
+	isccc_ccmsg_setmaxsize(ccmsg, 1024 * 1024);
+
+	isc_nmhandle_ref(handle);

 	DO("schedule recv",
-	   isccc_ccmsg_readmessage(&ccmsg, task, rndc_recvnonce, NULL));
+	   isccc_ccmsg_readmessage(ccmsg, rndc_recvnonce, ccmsg));
 	atomic_fetch_add_relaxed(&recvs, 1);
-	DO("send message",
-	   isc_socket_send(sock, &r, task, rndc_senddone, NULL));
+
+	DO("send message", isc_nm_send(handle, &r, rndc_senddone, NULL));
 	atomic_fetch_add_relaxed(&sends, 1);
-	isc_event_free(&event);
+
 	isccc_sexpr_free(&request);
 }

 static void
-rndc_startconnect(isc_sockaddr_t *addr, isc_task_t *task) {
+rndc_startconnect(isc_sockaddr_t *addr) {
 	isc_result_t result;
-	int pf;
-	isc_sockettype_t type;
-
 	char socktext[ISC_SOCKADDR_FORMATSIZE];
+	isc_sockaddr_t *local = NULL;

 	isc_sockaddr_format(addr, socktext, sizeof(socktext));

 	notify("using server %s (%s)", servername, socktext);

-	pf = isc_sockaddr_pf(addr);
-	if (pf == AF_INET || pf == AF_INET6) {
-		type = isc_sockettype_tcp;
-	} else {
-		type = isc_sockettype_unix;
-	}
-	DO("create socket", isc_socket_create(socketmgr, pf, type, &sock));
 	switch (isc_sockaddr_pf(addr)) {
 	case AF_INET:
-		DO("bind socket", isc_socket_bind(sock, &local4, 0));
+		local = &local4;
 		break;
 	case AF_INET6:
-		DO("bind socket", isc_socket_bind(sock, &local6, 0));
+		local = &local6;
 		break;
+	case AF_UNIX:
+		/*
+		 * TODO: support UNIX domain sockets in netgmr.
+		 */
+		fatal("UNIX domain sockets not currently supported");
 	default:
-		break;
+		INSIST(0);
+		ISC_UNREACHABLE();
 	}
-	DO("connect",
-	   isc_socket_connect(sock, addr, task, rndc_connected, NULL));
+
+	DO("create connection",
+	   isc_nm_tcpconnect(netmgr, (isc_nmiface_t *)local,
+			     (isc_nmiface_t *)addr, rndc_connected, &rndc_ccmsg,
+			     0));
 	atomic_fetch_add_relaxed(&connects, 1);
 }

@@ -567,8 +572,10 @@ static void
 rndc_start(isc_task_t *task, isc_event_t *event) {
 	isc_event_free(&event);

+	UNUSED(task);
+
 	currentaddr = 0;
-	rndc_startconnect(&serveraddrs[currentaddr], task);
+	rndc_startconnect(&serveraddrs[currentaddr]);
 }

 static void
@@ -649,7 +656,7 @@ parse_config(isc_mem_t *mctx, isc_log_t *log, const char *keyname,
 		if (servers != NULL) {
 			for (elt = cfg_list_first(servers); elt != NULL;
 			     elt = cfg_list_next(elt)) {
-				const char *name;
+				const char *name = NULL;
 				server = cfg_listelt_value(elt);
 				name = cfg_obj_asstring(
 					cfg_map_getname(server));
@@ -686,9 +693,11 @@ parse_config(isc_mem_t *mctx, isc_log_t *log, const char *keyname,
 		DO("get config key list", cfg_map_get(config, "key", &keys));
 		for (elt = cfg_list_first(keys); elt != NULL;
 		     elt = cfg_list_next(elt)) {
+			const char *name = NULL;
+
 			key = cfg_listelt_value(elt);
-			if (strcasecmp(cfg_obj_asstring(cfg_map_getname(key)),
-				       keyname) == 0) {
+			name = cfg_obj_asstring(cfg_map_getname(key));
+			if (strcasecmp(name, keyname) == 0) {
 				break;
 			}
 		}
@@ -848,8 +857,6 @@ int
 main(int argc, char **argv) {
 	isc_result_t result = ISC_R_SUCCESS;
 	bool show_final_mem = false;
-	isc_taskmgr_t *taskmgr = NULL;
-	isc_task_t *task = NULL;
 	isc_log_t *log = NULL;
 	isc_logconfig_t *logconfig = NULL;
 	isc_logdestination_t logdest;
@@ -975,19 +982,23 @@ main(int argc, char **argv) {
 	argc -= isc_commandline_index;
 	argv += isc_commandline_index;

-	if (argc < 1) {
+	if (argv[0] == NULL) {
 		usage(1);
+	} else {
+		command = argv[0];
+		if (strcmp(command, "restart") == 0) {
+			fatal("'%s' is not implemented", command);
+		}
+		notify("%s", command);
 	}

 	serial = isc_random32();

 	isc_mem_create(&rndc_mctx);
-	DO("create socket manager",
-	   isc_socketmgr_create(rndc_mctx, &socketmgr));
+	netmgr = isc_nm_start(rndc_mctx, 1);
 	DO("create task manager",
 	   isc_taskmgr_create(rndc_mctx, 1, 0, NULL, &taskmgr));
-	DO("create task", isc_task_create(taskmgr, 0, &task));
-
+	DO("create task", isc_task_create(taskmgr, 0, &rndc_task));
 	isc_log_create(rndc_mctx, &log, &logconfig);
 	isc_log_setcontext(log);
 	isc_log_settag(logconfig, progname);
@@ -1005,8 +1016,6 @@ main(int argc, char **argv) {

 	isccc_result_register();

-	command = *argv;
-
 	isc_buffer_allocate(rndc_mctx, &databuf, 2048);

 	/*
@@ -1033,32 +1042,30 @@ main(int argc, char **argv) {
 	*p++ = '\0';
 	INSIST(p == args + argslen);

-	notify("%s", command);
-
-	if (strcmp(command, "restart") == 0) {
-		fatal("'%s' is not implemented", command);
-	}
-
 	if (nserveraddrs == 0 && servername != NULL) {
 		get_addresses(servername, (in_port_t)remoteport);
 	}

-	DO("post event", isc_app_onrun(rndc_mctx, task, rndc_start, NULL));
+	DO("post event", isc_app_onrun(rndc_mctx, rndc_task, rndc_start, NULL));

 	result = isc_app_run();
 	if (result != ISC_R_SUCCESS) {
 		fatal("isc_app_run() failed: %s", isc_result_totext(result));
 	}

-	if (atomic_load_acquire(&connects) > 0 ||
-	    atomic_load_acquire(&sends) > 0 || atomic_load_acquire(&recvs) > 0)
-	{
-		isc_socket_cancel(sock, task, ISC_SOCKCANCEL_ALL);
-	}
-
-	isc_task_detach(&task);
+	isc_task_detach(&rndc_task);
 	isc_taskmgr_destroy(&taskmgr);
-	isc_socketmgr_destroy(&socketmgr);
+
+	isc_nm_destroy(&netmgr);
+
+	/*
+	 * XXX: when netmgr TCP connections are shut down, there will
+	 * be a final call to the isccc callback routine with &rndc_ccmsg
+	 * as its argument. We therefore need to delay invalidating it
+	 * until after the netmgr is destroyed.
+	 */
+	isccc_ccmsg_invalidate(&rndc_ccmsg);
+
 	isc_log_destroy(&log);
 	isc_log_setcontext(NULL);

@@ -1066,7 +1073,6 @@ main(int argc, char **argv) {
 	cfg_parser_destroy(&pctx);

 	isc_mem_put(rndc_mctx, args, argslen);
-	isccc_ccmsg_invalidate(&ccmsg);

 	isc_buffer_free(&databuf);

@@ -166,13 +166,16 @@ options {
        dnssec-secure-to-insecure <boolean>;
        dnssec-update-mode ( maintain | no-resign );
        dnssec-validation ( yes | no | auto );
-        dnstap { ( all | auth | client | forwarder | resolver | update ) [
-            ( query | response ) ]; ... };
-        dnstap-identity ( <quoted_string> | none | hostname );
-        dnstap-output ( file | unix ) <quoted_string> [ size ( unlimited |
-            <size> ) ] [ versions ( unlimited | <integer> ) ] [ suffix (
-            increment | timestamp ) ];
-        dnstap-version ( <quoted_string> | none );
+        dnstap { ( all | auth | client | forwarder |
+            resolver | update ) [ ( query | response ) ];
+            ... }; // not configured
+        dnstap-identity ( <quoted_string> | none |
+            hostname ); // not configured
+        dnstap-output ( file | unix ) <quoted_string> [
+            size ( unlimited | <size> ) ] [ versions (
+            unlimited | <integer> ) ] [ suffix ( increment
+            | timestamp ) ]; // not configured
+        dnstap-version ( <quoted_string> | none ); // not configured
        dscp <integer>;
        dual-stack-servers [ port <integer> ] { ( <quoted_string> [ port
            <integer> ] [ dscp <integer> ] | <ipv4_address> [ port
@@ -196,13 +199,13 @@ options {
        forward ( first | only );
        forwarders [ port <integer> ] [ dscp <integer> ] { ( <ipv4_address>
            | <ipv6_address> ) [ port <integer> ] [ dscp <integer> ]; ... };
-        fstrm-set-buffer-hint <integer>;
-        fstrm-set-flush-timeout <integer>;
-        fstrm-set-input-queue-size <integer>;
-        fstrm-set-output-notify-threshold <integer>;
-        fstrm-set-output-queue-model ( mpsc | spsc );
-        fstrm-set-output-queue-size <integer>;
-        fstrm-set-reopen-interval <duration>;
+        fstrm-set-buffer-hint <integer>; // not configured
+        fstrm-set-flush-timeout <integer>; // not configured
+        fstrm-set-input-queue-size <integer>; // not configured
+        fstrm-set-output-notify-threshold <integer>; // not configured
+        fstrm-set-output-queue-model ( mpsc | spsc ); // not configured
+        fstrm-set-output-queue-size <integer>; // not configured
+        fstrm-set-reopen-interval <duration>; // not configured
        geoip-directory ( <quoted_string> | none );
        geoip-use-ecs <boolean>; // obsolete
        glue-cache <boolean>;
@@ -547,8 +550,9 @@ view <string> [ <class> ] {
        dnssec-secure-to-insecure <boolean>;
        dnssec-update-mode ( maintain | no-resign );
        dnssec-validation ( yes | no | auto );
-        dnstap { ( all | auth | client | forwarder | resolver | update ) [
-            ( query | response ) ]; ... };
+        dnstap { ( all | auth | client | forwarder |
+            resolver | update ) [ ( query | response ) ];
+            ... }; // not configured
        dual-stack-servers [ port <integer> ] { ( <quoted_string> [ port
            <integer> ] [ dscp <integer> ] | <ipv4_address> [ port
            <integer> ] [ dscp <integer> ] | <ipv6_address> [ port
@@ -153,13 +153,16 @@ options {
        dnssec-secure-to-insecure <boolean>;
        dnssec-update-mode ( maintain | no-resign );
        dnssec-validation ( yes | no | auto );
-        dnstap { ( all | auth | client | forwarder | resolver | update ) [
-            ( query | response ) ]; ... };
-        dnstap-identity ( <quoted_string> | none | hostname );
-        dnstap-output ( file | unix ) <quoted_string> [ size ( unlimited |
-            <size> ) ] [ versions ( unlimited | <integer> ) ] [ suffix (
-            increment | timestamp ) ];
-        dnstap-version ( <quoted_string> | none );
+        dnstap { ( all | auth | client | forwarder |
+            resolver | update ) [ ( query | response ) ];
+            ... }; // not configured
+        dnstap-identity ( <quoted_string> | none |
+            hostname ); // not configured
+        dnstap-output ( file | unix ) <quoted_string> [
+            size ( unlimited | <size> ) ] [ versions (
+            unlimited | <integer> ) ] [ suffix ( increment
+            | timestamp ) ]; // not configured
+        dnstap-version ( <quoted_string> | none ); // not configured
        dscp <integer>;
        dual-stack-servers [ port <integer> ] { ( <quoted_string> [ port
            <integer> ] [ dscp <integer> ] | <ipv4_address> [ port
@@ -178,13 +181,13 @@ options {
        forward ( first | only );
        forwarders [ port <integer> ] [ dscp <integer> ] { ( <ipv4_address>
            | <ipv6_address> ) [ port <integer> ] [ dscp <integer> ]; ... };
-        fstrm-set-buffer-hint <integer>;
-        fstrm-set-flush-timeout <integer>;
-        fstrm-set-input-queue-size <integer>;
-        fstrm-set-output-notify-threshold <integer>;
-        fstrm-set-output-queue-model ( mpsc | spsc );
-        fstrm-set-output-queue-size <integer>;
-        fstrm-set-reopen-interval <duration>;
+        fstrm-set-buffer-hint <integer>; // not configured
+        fstrm-set-flush-timeout <integer>; // not configured
+        fstrm-set-input-queue-size <integer>; // not configured
+        fstrm-set-output-notify-threshold <integer>; // not configured
+        fstrm-set-output-queue-model ( mpsc | spsc ); // not configured
+        fstrm-set-output-queue-size <integer>; // not configured
+        fstrm-set-reopen-interval <duration>; // not configured
        geoip-directory ( <quoted_string> | none );
        glue-cache <boolean>;
        heartbeat-interval <integer>;
@@ -492,8 +495,9 @@ view <string> [ <class> ] {
        dnssec-secure-to-insecure <boolean>;
        dnssec-update-mode ( maintain | no-resign );
        dnssec-validation ( yes | no | auto );
-        dnstap { ( all | auth | client | forwarder | resolver | update ) [
-            ( query | response ) ]; ... };
+        dnstap { ( all | auth | client | forwarder |
+            resolver | update ) [ ( query | response ) ];
+            ... }; // not configured
        dual-stack-servers [ port <integer> ] { ( <quoted_string> [ port
            <integer> ] [ dscp <integer> ] | <ipv4_address> [ port
            <integer> ] [ dscp <integer> ] | <ipv6_address> [ port
@@ -82,12 +82,15 @@
  	dnssec-secure-to-insecure <boolean>;
  	dnssec-update-mode ( maintain | no-resign );
  	dnssec-validation ( yes | no | auto );
-  	dnstap { ( all | auth | client | forwarder | resolver | update ) [
-  	    ( query | response ) ]; ... };
-  	dnstap-identity ( <quoted_string> | none | hostname );
-  	dnstap-output ( file | unix ) <quoted_string> [ size ( unlimited |
-  	    <size> ) ] [ versions ( unlimited | <integer> ) ] [ suffix (
-  	    increment | timestamp ) ];
+  	dnstap { ( all | auth | client | forwarder |
+  	    resolver | update ) [ ( query | response ) ];
+  	    ... };
+  	dnstap-identity ( <quoted_string> | none |
+  	    hostname );
+  	dnstap-output ( file | unix ) <quoted_string> [
+  	    size ( unlimited | <size> ) ] [ versions (
+  	    unlimited | <integer> ) ] [ suffix ( increment
+  	    | timestamp ) ];
  	dnstap-version ( <quoted_string> | none );
  	dscp <integer>;
  	dual-stack-servers [ port <integer> ] { ( <quoted_string> [ port
@@ -76,6 +76,15 @@ New Features
   at the zone apex via UPDATE will be logged but otherwise ignored.
   DS records belong in the parent zone, not at the zone apex. [GL #1798]

+- ``rndc`` has been updated to use the new BIND network manager API.
+  This change had the side effect of altering the TCP timeout for RNDC
+  connections from 60 seconds to the ``tcp-idle-timeout`` value, which
+  defaults to 30 seconds. Also, because the network manager currently
+  has no support for UNIX-domain sockets, those cannot now be used
+  with ``rndc``. This will be addressed in a future release, either by
+  restoring UNIX-domain socket support or by formally declaring them
+  to be obsolete in the control channel. [GL #1759]
+
 Feature Changes
 ~~~~~~~~~~~~~~~

@@ -49,9 +49,8 @@ dns_xfrin_create(dns_zone_t *zone, dns_rdatatype_t xfrtype,
 		 const isc_sockaddr_t *masteraddr,
 		 const isc_sockaddr_t *sourceaddr, isc_dscp_t dscp,
 		 dns_tsigkey_t *tsigkey, isc_mem_t *mctx,
-		 isc_timermgr_t *timermgr, isc_socketmgr_t *socketmgr,
-		 isc_task_t *task, dns_xfrindone_t done,
-		 dns_xfrin_ctx_t **xfrp);
+		 isc_timermgr_t *timermgr, isc_nm_t *netmgr, isc_task_t *task,
+		 dns_xfrindone_t done, dns_xfrin_ctx_t **xfrp);
 /*%<
 * Attempt to start an incoming zone transfer of 'zone'
 * from 'masteraddr', creating a dns_xfrin_ctx_t object to
@@ -1629,7 +1629,7 @@ dns_zone_getkeydirectory(dns_zone_t *zone);
 isc_result_t
 dns_zonemgr_create(isc_mem_t *mctx, isc_taskmgr_t *taskmgr,
 		   isc_timermgr_t *timermgr, isc_socketmgr_t *socketmgr,
-		   dns_zonemgr_t **zmgrp);
+		   isc_nm_t *netmgr, dns_zonemgr_t **zmgrp);
 /*%<
 * Create a zone manager.  Note: the zone manager will not be able to
 * manage any zones until dns_zonemgr_setsize() has been run.
@@ -100,7 +100,7 @@ struct dns_xfrin_ctx {

 	isc_task_t *task;
 	isc_timer_t *timer;
-	isc_socketmgr_t *socketmgr;
+	isc_nm_t *netmgr;

 	int connects; /*%< Connect in progress */
 	int sends;    /*%< Send in progress */
@@ -123,16 +123,12 @@ struct dns_xfrin_ctx {

 	isc_sockaddr_t masteraddr;
 	isc_sockaddr_t sourceaddr;
-	isc_socket_t *socket;
+	isc_nmhandle_t *nmhandle;

 	/*% Buffer for IXFR/AXFR request message */
 	isc_buffer_t qbuffer;
 	unsigned char qbuffer_data[512];

-	/*% Incoming reply TCP message */
-	dns_tcpmsg_t tcpmsg;
-	bool tcpmsg_valid;
-
 	/*%
 	 * Whether the zone originally had a database attached at the time this
 	 * transfer context was created.  Used by maybe_free() when making
@@ -190,11 +186,10 @@ struct dns_xfrin_ctx {

 static isc_result_t
 xfrin_create(isc_mem_t *mctx, dns_zone_t *zone, dns_db_t *db, isc_task_t *task,
-	     isc_timermgr_t *timermgr, isc_socketmgr_t *socketmgr,
-	     dns_name_t *zonename, dns_rdataclass_t rdclass,
-	     dns_rdatatype_t reqtype, const isc_sockaddr_t *masteraddr,
-	     const isc_sockaddr_t *sourceaddr, isc_dscp_t dscp,
-	     dns_tsigkey_t *tsigkey, dns_xfrin_ctx_t **xfrp);
+	     isc_timermgr_t *timermgr, isc_nm_t *netmgr, dns_name_t *zonename,
+	     dns_rdataclass_t rdclass, dns_rdatatype_t reqtype,
+	     const isc_sockaddr_t *masteraddr, const isc_sockaddr_t *sourceaddr,
+	     isc_dscp_t dscp, dns_tsigkey_t *tsigkey, dns_xfrin_ctx_t **xfrp);

 static isc_result_t
 axfr_init(dns_xfrin_ctx_t *xfr);
@@ -228,13 +223,14 @@ static isc_result_t
 xfrin_start(dns_xfrin_ctx_t *xfr);

 static void
-xfrin_connect_done(isc_task_t *task, isc_event_t *event);
+xfrin_connect_done(isc_nmhandle_t *handle, isc_result_t result, void *cbarg);
 static isc_result_t
 xfrin_send_request(dns_xfrin_ctx_t *xfr);
 static void
-xfrin_send_done(isc_task_t *task, isc_event_t *event);
+xfrin_send_done(isc_nmhandle_t *handle, isc_result_t result, void *cbarg);
 static void
-xfrin_recv_done(isc_task_t *task, isc_event_t *event);
+xfrin_recv_done(isc_nmhandle_t *handle, isc_result_t result,
+		isc_region_t *region, void *cbarg);
 static void
 xfrin_timeout(isc_task_t *task, isc_event_t *event);

@@ -658,9 +654,8 @@ dns_xfrin_create(dns_zone_t *zone, dns_rdatatype_t xfrtype,
 		 const isc_sockaddr_t *masteraddr,
 		 const isc_sockaddr_t *sourceaddr, isc_dscp_t dscp,
 		 dns_tsigkey_t *tsigkey, isc_mem_t *mctx,
-		 isc_timermgr_t *timermgr, isc_socketmgr_t *socketmgr,
-		 isc_task_t *task, dns_xfrindone_t done,
-		 dns_xfrin_ctx_t **xfrp) {
+		 isc_timermgr_t *timermgr, isc_nm_t *netmgr, isc_task_t *task,
+		 dns_xfrindone_t done, dns_xfrin_ctx_t **xfrp) {
 	dns_name_t *zonename = dns_zone_getorigin(zone);
 	dns_xfrin_ctx_t *xfr = NULL;
 	isc_result_t result;
@@ -674,7 +669,7 @@ dns_xfrin_create(dns_zone_t *zone, dns_rdatatype_t xfrtype,
 		REQUIRE(db != NULL);
 	}

-	CHECK(xfrin_create(mctx, zone, db, task, timermgr, socketmgr, zonename,
+	CHECK(xfrin_create(mctx, zone, db, task, timermgr, netmgr, zonename,
 			   dns_zone_getclass(zone), xfrtype, masteraddr,
 			   sourceaddr, dscp, tsigkey, &xfr));

@@ -728,14 +723,20 @@ dns_xfrin_detach(dns_xfrin_ctx_t **xfrp) {

 static void
 xfrin_cancelio(dns_xfrin_ctx_t *xfr) {
-	if (xfr->connects > 0) {
-		isc_socket_cancel(xfr->socket, xfr->task,
-				  ISC_SOCKCANCEL_CONNECT);
-	} else if (xfr->recvs > 0) {
-		dns_tcpmsg_cancelread(&xfr->tcpmsg);
-	} else if (xfr->sends > 0) {
-		isc_socket_cancel(xfr->socket, xfr->task, ISC_SOCKCANCEL_SEND);
-	}
+	UNUSED(xfr);
+	/* XXXWPK TODO
+		if (xfr->connects > 0) {
+			isc_socket_cancel(xfr->socket, xfr->task,
+					  ISC_SOCKCANCEL_CONNECT);
+		} else if (xfr->recvs > 0) {
+			dns_tcpmsg_cancelread(&xfr->tcpmsg);
+		} else if (xfr->sends > 0) {
+			isc_socket_cancel(xfr->socket, xfr->task,
+	   ISC_SOCKCANCEL_SEND);
+		}
+	*/
+	isc_nmhandle_unref(xfr->nmhandle);
+	xfr->nmhandle = NULL;
 }

 static void
@@ -746,8 +747,9 @@ xfrin_reset(dns_xfrin_ctx_t *xfr) {

 	xfrin_cancelio(xfr);

-	if (xfr->socket != NULL) {
-		isc_socket_detach(&xfr->socket);
+	if (xfr->nmhandle != NULL) {
+		isc_nmhandle_unref(xfr->nmhandle);
+		xfr->nmhandle = NULL;
 	}

 	if (xfr->lasttsig != NULL) {
@@ -765,11 +767,6 @@ xfrin_reset(dns_xfrin_ctx_t *xfr) {
 		(void)dns_db_endload(xfr->db, &xfr->axfr);
 	}

-	if (xfr->tcpmsg_valid) {
-		dns_tcpmsg_invalidate(&xfr->tcpmsg);
-		xfr->tcpmsg_valid = false;
-	}
-
 	if (xfr->ver != NULL) {
 		dns_db_closeversion(xfr->db, &xfr->ver, false);
 	}
@@ -803,11 +800,10 @@ xfrin_fail(dns_xfrin_ctx_t *xfr, isc_result_t result, const char *msg) {

 static isc_result_t
 xfrin_create(isc_mem_t *mctx, dns_zone_t *zone, dns_db_t *db, isc_task_t *task,
-	     isc_timermgr_t *timermgr, isc_socketmgr_t *socketmgr,
-	     dns_name_t *zonename, dns_rdataclass_t rdclass,
-	     dns_rdatatype_t reqtype, const isc_sockaddr_t *masteraddr,
-	     const isc_sockaddr_t *sourceaddr, isc_dscp_t dscp,
-	     dns_tsigkey_t *tsigkey, dns_xfrin_ctx_t **xfrp) {
+	     isc_timermgr_t *timermgr, isc_nm_t *netmgr, dns_name_t *zonename,
+	     dns_rdataclass_t rdclass, dns_rdatatype_t reqtype,
+	     const isc_sockaddr_t *masteraddr, const isc_sockaddr_t *sourceaddr,
+	     isc_dscp_t dscp, dns_tsigkey_t *tsigkey, dns_xfrin_ctx_t **xfrp) {
 	dns_xfrin_ctx_t *xfr = NULL;
 	isc_result_t result;

@@ -820,7 +816,7 @@ xfrin_create(isc_mem_t *mctx, dns_zone_t *zone, dns_db_t *db, isc_task_t *task,
 	xfr->task = NULL;
 	isc_task_attach(task, &xfr->task);
 	xfr->timer = NULL;
-	xfr->socketmgr = socketmgr;
+	xfr->netmgr = netmgr;
 	xfr->done = NULL;

 	xfr->connects = 0;
@@ -836,11 +832,10 @@ xfrin_create(isc_mem_t *mctx, dns_zone_t *zone, dns_db_t *db, isc_task_t *task,
 	xfr->dscp = dscp;

 	/* sockaddr */
-	xfr->socket = NULL;
+	xfr->nmhandle = NULL;
 	/* qbuffer */
 	/* qbuffer_data */
 	/* tcpmsg */
-	xfr->tcpmsg_valid = false;

 	xfr->zone_had_db = false;
 	xfr->db = NULL;
@@ -926,17 +921,9 @@ failure:
 static isc_result_t
 xfrin_start(dns_xfrin_ctx_t *xfr) {
 	isc_result_t result;
-	CHECK(isc_socket_create(xfr->socketmgr,
-				isc_sockaddr_pf(&xfr->sourceaddr),
-				isc_sockettype_tcp, &xfr->socket));
-	isc_socket_setname(xfr->socket, "xfrin", NULL);
-#ifndef BROKEN_TCP_BIND_BEFORE_CONNECT
-	CHECK(isc_socket_bind(xfr->socket, &xfr->sourceaddr,
-			      ISC_SOCKET_REUSEADDRESS));
-#endif /* ifndef BROKEN_TCP_BIND_BEFORE_CONNECT */
-	isc_socket_dscp(xfr->socket, xfr->dscp);
-	CHECK(isc_socket_connect(xfr->socket, &xfr->masteraddr, xfr->task,
-				 xfrin_connect_done, xfr));
+	CHECK(isc_nm_tcpdnsconnect(xfr->netmgr, (isc_nmiface_t *)&xfr->sourceaddr,
+				(isc_nmiface_t *) &xfr->masteraddr, xfrin_connect_done, xfr, 0));
+	/* TODO	isc_socket_dscp(xfr->socket, xfr->dscp); */
 	xfr->connects++;
 	return (ISC_R_SUCCESS);
 failure:
@@ -972,10 +959,8 @@ failure:
 * A connection has been established.
 */
 static void
-xfrin_connect_done(isc_task_t *task, isc_event_t *event) {
-	isc_socket_connev_t *cev = (isc_socket_connev_t *)event;
-	dns_xfrin_ctx_t *xfr = (dns_xfrin_ctx_t *)event->ev_arg;
-	isc_result_t result = cev->result;
+xfrin_connect_done(isc_nmhandle_t *nmhandle, isc_result_t result, void *cbarg) {
+	dns_xfrin_ctx_t *xfr = (dns_xfrin_ctx_t *)cbarg;
 	char sourcetext[ISC_SOCKADDR_FORMATSIZE];
 	char signerbuf[DNS_NAME_FORMATSIZE];
 	const char *signer = "", *sep = "";
@@ -985,11 +970,6 @@ xfrin_connect_done(isc_task_t *task, isc_event_t *event) {

 	REQUIRE(VALID_XFRIN(xfr));

-	UNUSED(task);
-
-	INSIST(event->ev_type == ISC_SOCKEVENT_CONNECT);
-	isc_event_free(&event);
-
 	xfr->connects--;
 	if (xfr->shuttingdown) {
 		maybe_free(xfr);
@@ -1009,13 +989,11 @@ xfrin_connect_done(isc_task_t *task, isc_event_t *event) {
 		}
 	}

-	result = isc_socket_getsockname(xfr->socket, &sockaddr);
-	if (result == ISC_R_SUCCESS) {
-		isc_sockaddr_format(&sockaddr, sourcetext, sizeof(sourcetext));
-	} else {
-		strlcpy(sourcetext, "<UNKNOWN>", sizeof(sourcetext));
-	}
-
+	sockaddr = isc_nmhandle_peeraddr(nmhandle);
+	isc_sockaddr_format(&sockaddr, sourcetext, sizeof(sourcetext));
+	isc_nmhandle_ref(nmhandle);
+	xfr->nmhandle = nmhandle;
+	
 	if (xfr->tsigkey != NULL && xfr->tsigkey->key != NULL) {
 		dns_name_format(dst_key_name(xfr->tsigkey->key), signerbuf,
 				sizeof(signerbuf));
@@ -1026,9 +1004,6 @@ xfrin_connect_done(isc_task_t *task, isc_event_t *event) {
 	xfrin_log(xfr, ISC_LOG_INFO, "connected using %s%s%s", sourcetext, sep,
 		  signer);

-	dns_tcpmsg_init(xfr->mctx, xfr->socket, &xfr->tcpmsg);
-	xfr->tcpmsg_valid = true;
-
 	CHECK(xfrin_send_request(xfr));
 failure:
 	if (result != ISC_R_SUCCESS) {
@@ -1167,16 +1142,7 @@ xfrin_send_request(dns_xfrin_ctx_t *xfr) {
 	isc_buffer_usedregion(&xfr->qbuffer, &region);
 	INSIST(region.length <= 65535);

-	/*
-	 * Record message length and adjust region to include TCP
-	 * length field.
-	 */
-	xfr->qbuffer_data[0] = (region.length >> 8) & 0xff;
-	xfr->qbuffer_data[1] = region.length & 0xff;
-	region.base -= 2;
-	region.length += 2;
-	CHECK(isc_socket_send(xfr->socket, &region, xfr->task, xfrin_send_done,
-			      xfr));
+	CHECK(isc_nm_send(xfr->nmhandle, &region, xfrin_send_done, xfr));
 	xfr->sends++;

 failure:
@@ -1199,58 +1165,45 @@ failure:
 }

 static void
-xfrin_send_done(isc_task_t *task, isc_event_t *event) {
-	isc_socketevent_t *sev = (isc_socketevent_t *)event;
-	dns_xfrin_ctx_t *xfr = (dns_xfrin_ctx_t *)event->ev_arg;
-	isc_result_t result;
-
+xfrin_send_done(isc_nmhandle_t *handle, isc_result_t result, void *cbarg) {
+	dns_xfrin_ctx_t *xfr = (dns_xfrin_ctx_t *)cbarg;
 	REQUIRE(VALID_XFRIN(xfr));
-
-	UNUSED(task);
-
-	INSIST(event->ev_type == ISC_SOCKEVENT_SENDDONE);
-
+	UNUSED(handle);
 	xfr->sends--;
-	xfrin_log(xfr, ISC_LOG_DEBUG(3), "sent request data");
-	CHECK(sev->result);

-	CHECK(dns_tcpmsg_readmessage(&xfr->tcpmsg, xfr->task, xfrin_recv_done,
-				     xfr));
+	xfrin_log(xfr, ISC_LOG_DEBUG(3), "sent request data");
+	CHECK(result);
+
+	CHECK(isc_nm_read(xfr->nmhandle, xfrin_recv_done, xfr));
+
 	xfr->recvs++;
+
 failure:
-	isc_event_free(&event);
 	if (result != ISC_R_SUCCESS) {
 		xfrin_fail(xfr, result, "failed sending request data");
 	}
 }

 static void
-xfrin_recv_done(isc_task_t *task, isc_event_t *ev) {
-	dns_xfrin_ctx_t *xfr = (dns_xfrin_ctx_t *)ev->ev_arg;
-	isc_result_t result;
+xfrin_recv_done(isc_nmhandle_t *handle, isc_result_t result,
+		isc_region_t *region, void *cbarg) {
+	dns_xfrin_ctx_t *xfr = (dns_xfrin_ctx_t *)cbarg;
+
 	dns_message_t *msg = NULL;
 	dns_name_t *name;
-	dns_tcpmsg_t *tcpmsg;
 	const dns_name_t *tsigowner = NULL;

 	REQUIRE(VALID_XFRIN(xfr));

-	UNUSED(task);
-
-	INSIST(ev->ev_type == DNS_EVENT_TCPMSG);
-	tcpmsg = ev->ev_sender;
-	isc_event_free(&ev);
-
 	xfr->recvs--;
 	if (xfr->shuttingdown) {
 		maybe_free(xfr);
 		return;
 	}

-	CHECK(tcpmsg->result);
+	CHECK(result);

-	xfrin_log(xfr, ISC_LOG_DEBUG(7), "received %u bytes",
-		  tcpmsg->buffer.used);
+	xfrin_log(xfr, ISC_LOG_DEBUG(7), "received %u bytes", region->length);

 	CHECK(isc_timer_touch(xfr->timer));

@@ -1267,13 +1220,17 @@ xfrin_recv_done(isc_task_t *task, isc_event_t *ev) {
 	if (xfr->nmsg > 0) {
 		msg->tcp_continuation = 1;
 	}
+	isc_buffer_t buffer;
+	isc_buffer_init(&buffer, region->base, region->length);
+	isc_buffer_add(&buffer, region->length);

-	result = dns_message_parse(msg, &tcpmsg->buffer,
+	result = dns_message_parse(msg, &buffer,
 				   DNS_MESSAGEPARSE_PRESERVEORDER);

+	isc_sockaddr_t peer = isc_nmhandle_peeraddr(handle);
 	if (result == ISC_R_SUCCESS) {
-		dns_message_logpacket(msg, "received message from",
-				      &tcpmsg->address, DNS_LOGCATEGORY_XFER_IN,
+		dns_message_logpacket(msg, "received message from", &peer,
+				      DNS_LOGCATEGORY_XFER_IN,
 				      DNS_LOGMODULE_XFER_IN, ISC_LOG_DEBUG(10),
 				      xfr->mctx);
 	} else {
@@ -1455,7 +1412,7 @@ xfrin_recv_done(isc_task_t *task, isc_event_t *ev) {
 	/*
 	 * Update the number of bytes received.
 	 */
-	xfr->nbytes += tcpmsg->buffer.used;
+	xfr->nbytes += buffer.used;

 	/*
 	 * Take the context back.
@@ -1502,8 +1459,8 @@ xfrin_recv_done(isc_task_t *task, isc_event_t *ev) {
 		/*
 		 * Read the next message.
 		 */
-		CHECK(dns_tcpmsg_readmessage(&xfr->tcpmsg, xfr->task,
-					     xfrin_recv_done, xfr));
+		CHECK(isc_nm_read(xfr->nmhandle, xfrin_recv_done, xfr));
+
 		xfr->recvs++;
 	}
 	return;
@@ -1575,8 +1532,9 @@ maybe_free(dns_xfrin_ctx_t *xfr) {
 		  (unsigned int)(msecs / 1000), (unsigned int)(msecs % 1000),
 		  (unsigned int)persec, xfr->end_serial);

-	if (xfr->socket != NULL) {
-		isc_socket_detach(&xfr->socket);
+	if (xfr->nmhandle != NULL) {
+		isc_nmhandle_unref(xfr->nmhandle);
+		xfr->nmhandle = NULL;
 	}

 	if (xfr->timer != NULL) {
@@ -1605,10 +1563,6 @@ maybe_free(dns_xfrin_ctx_t *xfr) {
 		(void)dns_db_endload(xfr->db, &xfr->axfr);
 	}

-	if (xfr->tcpmsg_valid) {
-		dns_tcpmsg_invalidate(&xfr->tcpmsg);
-	}
-
 	if (xfr->tsigctx != NULL) {
 		dst_context_destroy(&xfr->tsigctx);
 	}
@@ -545,6 +545,7 @@ struct dns_zonemgr {
 	isc_taskmgr_t *taskmgr;
 	isc_timermgr_t *timermgr;
 	isc_socketmgr_t *socketmgr;
+	isc_nm_t *netmgr;
 	isc_taskpool_t *zonetasks;
 	isc_taskpool_t *loadtasks;
 	isc_task_t *task;
@@ -17090,7 +17091,7 @@ got_transfer_quota(isc_task_t *task, isc_event_t *event) {
 	INSIST(isc_sockaddr_pf(&masteraddr) == isc_sockaddr_pf(&sourceaddr));
 	result = dns_xfrin_create(zone, xfrtype, &masteraddr, &sourceaddr, dscp,
 				  zone->tsigkey, zone->mctx,
-				  zone->zmgr->timermgr, zone->zmgr->socketmgr,
+				  zone->zmgr->timermgr, zone->zmgr->netmgr,
 				  zone->task, zone_xfrdone, &zone->xfr);
 	if (result == ISC_R_SUCCESS) {
 		LOCK_ZONE(zone);
@@ -17398,7 +17399,7 @@ dns_zone_first(dns_zonemgr_t *zmgr, dns_zone_t **first) {
 isc_result_t
 dns_zonemgr_create(isc_mem_t *mctx, isc_taskmgr_t *taskmgr,
 		   isc_timermgr_t *timermgr, isc_socketmgr_t *socketmgr,
-		   dns_zonemgr_t **zmgrp) {
+		   isc_nm_t *netmgr, dns_zonemgr_t **zmgrp) {
 	dns_zonemgr_t *zmgr;
 	isc_result_t result;

@@ -17409,6 +17410,7 @@ dns_zonemgr_create(isc_mem_t *mctx, isc_taskmgr_t *taskmgr,
 	zmgr->taskmgr = taskmgr;
 	zmgr->timermgr = timermgr;
 	zmgr->socketmgr = socketmgr;
+	zmgr->netmgr = netmgr;
 	zmgr->zonetasks = NULL;
 	zmgr->loadtasks = NULL;
 	zmgr->mctxpool = NULL;
@@ -12,17 +12,19 @@
 #pragma once

 #include <isc/mem.h>
+#include <isc/region.h>
 #include <isc/result.h>
 #include <isc/types.h>

+/*
+ * Replacement for isc_sockettype_t provided by socket.h.
+ */
 typedef enum {
-	NMEV_READ,
-	NMEV_WRITE,
-	NMEV_ACCEPT,
-	NMEV_CONNECTED,
-	NMEV_CANCELLED,
-	NMEV_SHUTDOWN
-} isc_nm_eventtype;
+	isc_socktype_tcp = 1,
+	isc_socktype_udp = 2,
+	isc_socktype_unix = 3,
+	isc_socktype_raw = 4
+} isc_socktype_t;

 isc_nm_t *
 isc_nm_start(isc_mem_t *mctx, uint32_t workers);
@@ -57,26 +59,13 @@ isc_nm_closedown(isc_nm_t *mgr);
 int
 isc_nm_tid(void);

-/*
- * isc_nm_freehandle frees a handle, releasing resources
- */
 void
-isc_nm_freehandle(isc_nmhandle_t *handle);
-
-void
-isc_nmsocket_attach(isc_nmsocket_t *sock, isc_nmsocket_t **target);
+isc_nmsocket_close(isc_nmsocket_t **sockp);
 /*%<
- * isc_nmsocket_attach attaches to a socket, increasing refcount
- */
-
-void
-isc_nmsocket_close(isc_nmsocket_t *sock);
-
-void
-isc_nmsocket_detach(isc_nmsocket_t **socketp);
-/*%<
- * isc_nmsocket_detach detaches from socket, decreasing refcount
- * and possibly destroying the socket if it's no longer referenced.
+ * isc_nmsocket_close() detaches a listening socket that was
+ * created by isc_nm_listenudp(), isc_nm_listentcp(), or
+ * isc_nm_listentcpdns(). Once there are no remaining child
+ * sockets with active handles, the socket will be closed.
 */

 void
@@ -135,26 +124,27 @@ isc_nmhandle_netmgr(isc_nmhandle_t *handle);
 * Return a pointer to the netmgr object for the given handle.
 */

-typedef void (*isc_nm_recv_cb_t)(isc_nmhandle_t *handle, isc_region_t *region,
-				 void *cbarg);
+typedef void (*isc_nm_recv_cb_t)(isc_nmhandle_t *handle, isc_result_t eresult,
+				 isc_region_t *region, void *cbarg);
 /*%<
 * Callback function to be used when receiving a packet.
 *
 * 'handle' the handle that can be used to send back the answer.
- * 'region' contains the received data. It will be freed after
- *          return by caller.
+ * 'eresult' the result of the event.
+ * 'region' contains the received data, if any. It will be freed
+ *          after return by caller.
 * 'cbarg'  the callback argument passed to isc_nm_listenudp(),
 *          isc_nm_listentcpdns(), or isc_nm_read().
 */

-typedef void (*isc_nm_cb_t)(isc_nmhandle_t *handle, isc_result_t result,
+typedef void (*isc_nm_cb_t)(isc_nmhandle_t *handle, isc_result_t eresult,
 			    void *cbarg);
 /*%<
 * Callback function for other network completion events (send, connect,
 * accept).
 *
 * 'handle' the handle on which the event took place.
- * 'result' the result of the event.
+ * 'eresult' the result of the event.
 * 'cbarg'  the callback argument passed to isc_nm_send(),
 *          isc_nm_tcp_connect(), or isc_nm_listentcp()
 */
@@ -205,6 +195,17 @@ isc_nm_pauseread(isc_nmsocket_t *sock);
 * Pause reading on this socket, while still remembering the callback.
 */

+void
+isc_nm_cancelread(isc_nmhandle_t *handle);
+/*%<
+ * Cancel reading on a connected socket. Calls the read/recv callback on
+ * active handles with a result code of ISC_R_CANCELED.
+ *
+ * Requires:
+ * \li	'sock' is a valid netmgr socket
+ * \li	...for which a read/recv callback has been defined.
+ */
+
 isc_result_t
 isc_nm_resumeread(isc_nmsocket_t *sock);
 /*%<
@@ -251,6 +252,21 @@ isc_nm_listentcp(isc_nm_t *mgr, isc_nmiface_t *iface, isc_nm_cb_t cb,
 * prepended with a two-byte length field, and handles buffering.
 */

+isc_result_t
+isc_nm_tcpconnect(isc_nm_t *mgr, isc_nmiface_t *local, isc_nmiface_t *peer,
+		  isc_nm_cb_t cb, void *cbarg, size_t extrahandlesize);
+/*%<
+ * Create a socket using netmgr 'mgr', bind it to the address 'local',
+ * and connect it to the address 'peer'.
+ *
+ * When the connection is complete, call 'cb' with argument 'cbarg'.
+ * Allocate 'extrahandlesize' additional bytes along with the handle to use
+ * for an associated object.
+ *
+ * The connected socket can only be accessed via the handle passed to
+ * 'cb'.
+ */
+
 isc_result_t
 isc_nm_listentcpdns(isc_nm_t *mgr, isc_nmiface_t *iface, isc_nm_recv_cb_t cb,
 		    void *cbarg, isc_nm_cb_t accept_cb, void *accept_cbarg,
@@ -351,3 +367,7 @@ isc_nm_setstats(isc_nm_t *mgr, isc_stats_t *stats);
 *\li	stats is a valid set of statistics counters supporting the
 *	full range of socket-related stats counter numbers.
 */
+
+isc_result_t
+isc_nm_tcpdnsconnect(isc_nm_t *mgr, isc_nmiface_t *local, isc_nmiface_t *peer,
+		     isc_nm_cb_t cb, void *cbarg, size_t extrahandlesize);
@@ -96,8 +96,7 @@ struct isc_nmhandle {
 	 * the socket.
 	 */
 	isc_nmsocket_t *sock;
-	size_t ah_pos; /* Position in the socket's
-			* 'active handles' array */
+	size_t ah_pos; /* Position in the socket's 'active handles' array */

 	/*
 	 * The handle is 'inflight' if netmgr is not currently processing
@@ -136,11 +135,14 @@ typedef enum isc__netievent_type {
 	netievent_tcpaccept,
 	netievent_tcpstop,
 	netievent_tcpclose,
+
 	netievent_tcpdnsclose,
+	netievent_tcpdnsread,

 	netievent_closecb,
 	netievent_shutdown,
 	netievent_stop,
+
 	netievent_prio = 0xff, /* event type values higher than this
 				* will be treated as high-priority
 				* events, which can be processed
@@ -213,10 +215,11 @@ typedef isc__netievent__socket_t isc__netievent_udplisten_t;
 typedef isc__netievent__socket_t isc__netievent_udpstop_t;
 typedef isc__netievent__socket_t isc__netievent_tcpstop_t;
 typedef isc__netievent__socket_t isc__netievent_tcpclose_t;
-typedef isc__netievent__socket_t isc__netievent_tcpdnsclose_t;
 typedef isc__netievent__socket_t isc__netievent_startread_t;
 typedef isc__netievent__socket_t isc__netievent_pauseread_t;
 typedef isc__netievent__socket_t isc__netievent_closecb_t;
+typedef isc__netievent__socket_t isc__netievent_tcpdnsclose_t;
+typedef isc__netievent__socket_t isc__netievent_tcpdnsread_t;

 typedef struct isc__netievent__socket_req {
 	isc__netievent_type type;
@@ -371,6 +374,8 @@ struct isc_nmsocket {
 	isc_nmsocket_t *parent;
 	/*% Listener socket this connection was accepted on */
 	isc_nmsocket_t *listener;
+	/*% Self, for self-contained unreferenced sockets (tcpdns) */
+	isc_nmsocket_t *self;

 	/*%
 	 * quota is the TCP client, attached when a TCP connection
@@ -404,7 +409,8 @@ struct isc_nmsocket {
 	isc_nmsocket_t *children;
 	int nchildren;
 	isc_nmiface_t *iface;
-	isc_nmhandle_t *tcphandle;
+	isc_nmhandle_t *statichandle;
+	isc_nmhandle_t *outerhandle;

 	/*% Extra data allocated at the end of each isc_nmhandle_t */
 	size_t extrahandlesize;
@@ -440,8 +446,16 @@ struct isc_nmsocket {
 	atomic_bool closed;
 	atomic_bool listening;
 	atomic_bool listen_error;
+	atomic_bool connected;
+	atomic_bool connect_error;
 	isc_refcount_t references;

+
+	/*%
+	 * Socket was connected, not connected to.
+	 */
+	atomic_bool client;
+
 	/*%
 	 * TCPDNS socket has been set not to pipeliine.
 	 */
@@ -589,6 +603,9 @@ isc__nmhandle_get(isc_nmsocket_t *sock, isc_sockaddr_t *peer,
 *
 * If 'local' is not NULL, set the handle's local address to 'local',
 * otherwise set it to 'sock->iface->addr'.
+ *
+ * 'sock' will be attached to 'handle->sock'. The caller may need
+ * to detach the socket afterward.
 */

 isc__nm_uvreq_t *
@@ -615,6 +632,19 @@ isc__nmsocket_init(isc_nmsocket_t *sock, isc_nm_t *mgr, isc_nmsocket_type type,
 * and its interface to 'iface'.
 */

+void
+isc__nmsocket_attach(isc_nmsocket_t *sock, isc_nmsocket_t **target);
+/*%<
+ * Attach to a socket, increasing refcount
+ */
+
+void
+isc__nmsocket_detach(isc_nmsocket_t **socketp);
+/*%<
+ * Detach from socket, decreasing refcount and possibly destroying the
+ * socket if it's no longer referenced.
+ */
+
 void
 isc__nmsocket_prep_destroy(isc_nmsocket_t *sock);
 /*%<
@@ -694,7 +724,14 @@ isc__nm_tcp_resumeread(isc_nmsocket_t *sock);
 void
 isc__nm_tcp_shutdown(isc_nmsocket_t *sock);
 /*%<
- * Called on shutdown to close and clean up a listening TCP socket.
+ * Called during the shutdown process to close and clean up connected
+ * sockets.
+ */
+
+void
+isc__nm_tcp_cancelread(isc_nmsocket_t *sock);
+/*%<
+ * Stop reading on a connected socket.
 */

 void
@@ -746,6 +783,13 @@ isc__nm_tcpdns_stoplistening(isc_nmsocket_t *sock);
 void
 isc__nm_async_tcpdnsclose(isc__networker_t *worker, isc__netievent_t *ev0);

+void
+isc__nm_async_tcpdnsread(isc__networker_t *worker, isc__netievent_t *ev0);
+
+isc_result_t
+isc__nm_tcpdns_read(isc_nmhandle_t *handle, isc_nm_recv_cb_t cb, void *cbarg);
+
+
 #define isc__nm_uverr2result(x) \
 	isc___nm_uverr2result(x, true, __FILE__, __LINE__)
 isc_result_t
@@ -417,9 +417,9 @@ isc_nm_destroy(isc_nm_t **mgr0) {
 		isc_nm_pause(mgr);
 		isc_nm_resume(mgr);
 #ifdef WIN32
-		_sleep(1000);
+		_sleep(10);
 #else  /* ifdef WIN32 */
-		usleep(1000000);
+		usleep(10000);
 #endif /* ifdef WIN32 */
 	}

@@ -630,6 +630,9 @@ process_queue(isc__networker_t *worker, isc_queue_t *queue) {
 		case netievent_tcpdnsclose:
 			isc__nm_async_tcpdnsclose(worker, ievent);
 			break;
+		case netievent_tcpdnsread:
+			isc__nm_async_tcpdnsread(worker, ievent);
+			break;
 		case netievent_closecb:
 			isc__nm_async_closecb(worker, ievent);
 			break;
@@ -686,7 +689,7 @@ isc__nmsocket_active(isc_nmsocket_t *sock) {
 }

 void
-isc_nmsocket_attach(isc_nmsocket_t *sock, isc_nmsocket_t **target) {
+isc__nmsocket_attach(isc_nmsocket_t *sock, isc_nmsocket_t **target) {
 	REQUIRE(VALID_NMSOCK(sock));
 	REQUIRE(target != NULL && *target == NULL);

@@ -736,9 +739,15 @@ nmsocket_cleanup(isc_nmsocket_t *sock, bool dofree) {
 		isc__nm_decstats(sock->mgr, sock->statsindex[STATID_ACTIVE]);
 	}

-	if (sock->tcphandle != NULL) {
-		isc_nmhandle_unref(sock->tcphandle);
-		sock->tcphandle = NULL;
+	sock->statichandle = NULL;
+
+	if (sock->outerhandle != NULL) {
+		isc_nmhandle_unref(sock->outerhandle);
+		sock->outerhandle = NULL;
+	}
+
+	if (sock->outer != NULL) {
+		isc__nmsocket_detach(&sock->outer);
 	}

 	while ((handle = isc_astack_pop(sock->inactivehandles)) != NULL) {
@@ -824,7 +833,7 @@ nmsocket_maybe_destroy(isc_nmsocket_t *sock) {
 		}
 	}

-	if (active_handles == 0 || sock->tcphandle != NULL) {
+	if (active_handles == 0 || sock->statichandle != NULL) {
 		destroy = true;
 	}
 	UNLOCK(&sock->lock);
@@ -878,7 +887,7 @@ isc__nmsocket_prep_destroy(isc_nmsocket_t *sock) {
 }

 void
-isc_nmsocket_detach(isc_nmsocket_t **sockp) {
+isc__nmsocket_detach(isc_nmsocket_t **sockp) {
 	REQUIRE(sockp != NULL && *sockp != NULL);
 	REQUIRE(VALID_NMSOCK(*sockp));

@@ -901,6 +910,17 @@ isc_nmsocket_detach(isc_nmsocket_t **sockp) {
 	}
 }

+void
+isc_nmsocket_close(isc_nmsocket_t **sockp) {
+	REQUIRE(sockp != NULL);
+	REQUIRE(VALID_NMSOCK(*sockp));
+	REQUIRE((*sockp)->type == isc_nm_udplistener ||
+		(*sockp)->type == isc_nm_tcplistener ||
+		(*sockp)->type == isc_nm_tcpdnslistener);
+
+	isc__nmsocket_detach(sockp);
+}
+
 void
 isc__nmsocket_init(isc_nmsocket_t *sock, isc_nm_t *mgr, isc_nmsocket_type type,
 		   isc_nmiface_t *iface) {
@@ -1039,7 +1059,8 @@ isc__nmhandle_get(isc_nmsocket_t *sock, isc_sockaddr_t *peer,
 		isc_refcount_increment0(&handle->references);
 	}

-	handle->sock = sock;
+	isc__nmsocket_attach(sock, &handle->sock);
+
 	if (peer != NULL) {
 		memcpy(&handle->peer, peer, sizeof(isc_sockaddr_t));
 	} else {
@@ -1083,8 +1104,8 @@ isc__nmhandle_get(isc_nmsocket_t *sock, isc_sockaddr_t *peer,
 	UNLOCK(&sock->lock);

 	if (sock->type == isc_nm_tcpsocket) {
-		INSIST(sock->tcphandle == NULL);
-		sock->tcphandle = handle;
+		INSIST(sock->statichandle == NULL);
+		sock->statichandle = handle;
 	}

 	return (handle);
@@ -1122,6 +1143,9 @@ nmhandle_free(isc_nmsocket_t *sock, isc_nmhandle_t *handle) {

 static void
 nmhandle_deactivate(isc_nmsocket_t *sock, isc_nmhandle_t *handle) {
+	size_t handlenum;
+	bool reuse = false;
+
 	/*
 	 * We do all of this under lock to avoid races with socket
 	 * destruction.  We have to do this now, because at this point the
@@ -1134,10 +1158,9 @@ nmhandle_deactivate(isc_nmsocket_t *sock, isc_nmhandle_t *handle) {
 	INSIST(atomic_load(&sock->ah) > 0);

 	sock->ah_handles[handle->ah_pos] = NULL;
-	size_t handlenum = atomic_fetch_sub(&sock->ah, 1) - 1;
+	handlenum = atomic_fetch_sub(&sock->ah, 1) - 1;
 	sock->ah_frees[handlenum] = handle->ah_pos;
 	handle->ah_pos = 0;
-	bool reuse = false;
 	if (atomic_load(&sock->active)) {
 		reuse = isc_astack_trypush(sock->inactivehandles, handle);
 	}
@@ -1149,7 +1172,7 @@ nmhandle_deactivate(isc_nmsocket_t *sock, isc_nmhandle_t *handle) {

 void
 isc_nmhandle_unref(isc_nmhandle_t *handle) {
-	isc_nmsocket_t *sock = NULL, *tmp = NULL;
+	isc_nmsocket_t *sock = NULL;

 	REQUIRE(VALID_NMHANDLE(handle));

@@ -1164,12 +1187,6 @@ isc_nmhandle_unref(isc_nmhandle_t *handle) {
 		handle->doreset(handle->opaque);
 	}

-	/*
-	 * Temporarily reference the socket to ensure that it can't
-	 * be deleted by another thread while we're deactivating the
-	 * handle.
-	 */
-	isc_nmsocket_attach(sock, &tmp);
 	nmhandle_deactivate(sock, handle);

 	/*
@@ -1187,13 +1204,13 @@ isc_nmhandle_unref(isc_nmhandle_t *handle) {
 			 * The socket will be finally detached by the closecb
 			 * event handler.
 			 */
-			isc_nmsocket_attach(sock, &event->sock);
+			isc__nmsocket_attach(sock, &event->sock);
 			isc__nm_enqueue_ievent(&sock->mgr->workers[sock->tid],
 					       (isc__netievent_t *)event);
 		}
 	}

-	isc_nmsocket_detach(&tmp);
+	isc__nmsocket_detach(&sock);
 }

 void *
@@ -1260,7 +1277,7 @@ isc__nm_uvreq_get(isc_nm_t *mgr, isc_nmsocket_t *sock) {

 	*req = (isc__nm_uvreq_t){ .magic = 0 };
 	req->uv_req.req.data = req;
-	isc_nmsocket_attach(sock, &req->sock);
+	isc__nmsocket_attach(sock, &req->sock);
 	req->magic = UVREQ_MAGIC;

 	return (req);
@@ -1297,7 +1314,7 @@ isc__nm_uvreq_put(isc__nm_uvreq_t **req0, isc_nmsocket_t *sock) {
 		isc_nmhandle_unref(handle);
 	}

-	isc_nmsocket_detach(&sock);
+	isc__nmsocket_detach(&sock);
 }

 isc_result_t
@@ -1326,6 +1343,22 @@ isc_nm_read(isc_nmhandle_t *handle, isc_nm_recv_cb_t cb, void *cbarg) {
 	switch (handle->sock->type) {
 	case isc_nm_tcpsocket:
 		return (isc__nm_tcp_read(handle, cb, cbarg));
+	case isc_nm_tcpdnssocket:
+		return (isc__nm_tcpdns_read(handle, cb, cbarg));
+	default:
+		INSIST(0);
+		ISC_UNREACHABLE();
+	}
+}
+
+void
+isc_nm_cancelread(isc_nmhandle_t *handle) {
+	REQUIRE(VALID_NMHANDLE(handle));
+
+	switch (handle->sock->type) {
+	case isc_nm_tcpsocket:
+		isc__nm_tcp_cancelread(handle->sock);
+		break;
 	default:
 		INSIST(0);
 		ISC_UNREACHABLE();
@@ -1335,6 +1368,7 @@ isc_nm_read(isc_nmhandle_t *handle, isc_nm_recv_cb_t cb, void *cbarg) {
 isc_result_t
 isc_nm_pauseread(isc_nmsocket_t *sock) {
 	REQUIRE(VALID_NMSOCK(sock));
+
 	switch (sock->type) {
 	case isc_nm_tcpsocket:
 		return (isc__nm_tcp_pauseread(sock));
@@ -1347,6 +1381,7 @@ isc_nm_pauseread(isc_nmsocket_t *sock) {
 isc_result_t
 isc_nm_resumeread(isc_nmsocket_t *sock) {
 	REQUIRE(VALID_NMSOCK(sock));
+
 	switch (sock->type) {
 	case isc_nm_tcpsocket:
 		return (isc__nm_tcp_resumeread(sock));
@@ -1359,6 +1394,7 @@ isc_nm_resumeread(isc_nmsocket_t *sock) {
 void
 isc_nm_stoplistening(isc_nmsocket_t *sock) {
 	REQUIRE(VALID_NMSOCK(sock));
+
 	switch (sock->type) {
 	case isc_nm_udplistener:
 		isc__nm_udp_stoplistening(sock);
@@ -1386,7 +1422,7 @@ isc__nm_async_closecb(isc__networker_t *worker, isc__netievent_t *ev0) {
 	UNUSED(worker);

 	ievent->sock->closehandle_cb(ievent->sock);
-	isc_nmsocket_detach(&ievent->sock);
+	isc__nmsocket_detach(&ievent->sock);
 }

 static void
@@ -85,6 +85,10 @@ tcp_connect_direct(isc_nmsocket_t *sock, isc__nm_uvreq_t *req) {
 	r = uv_tcp_init(&worker->loop, &sock->uv_handle.tcp);
 	if (r != 0) {
 		isc__nm_incstats(sock->mgr, sock->statsindex[STATID_OPENFAIL]);
+		/* Socket was never opened; no need for tcp_close_direct() */
+		atomic_store(&sock->closed, true);
+		sock->result = isc__nm_uverr2result(r);
+		atomic_store(&sock->connect_error, true);
 		return (r);
 	}

@@ -93,13 +97,23 @@ tcp_connect_direct(isc_nmsocket_t *sock, isc__nm_uvreq_t *req) {
 		if (r != 0) {
 			isc__nm_incstats(sock->mgr,
 					 sock->statsindex[STATID_BINDFAIL]);
+			sock->result = isc__nm_uverr2result(r);
+			atomic_store(&sock->connect_error, true);
 			tcp_close_direct(sock);
 			return (r);
 		}
 	}
+
 	uv_handle_set_data(&sock->uv_handle.handle, sock);
 	r = uv_tcp_connect(&req->uv_req.connect, &sock->uv_handle.tcp,
 			   &req->peer.type.sa, tcp_connect_cb);
+	if (r != 0) {
+		isc__nm_incstats(sock->mgr,
+				 sock->statsindex[STATID_CONNECTFAIL]);
+		sock->result = isc__nm_uverr2result(r);
+		atomic_store(&sock->connect_error, true);
+		tcp_close_direct(sock);
+	}
 	return (r);
 }

@@ -111,29 +125,38 @@ isc__nm_async_tcpconnect(isc__networker_t *worker, isc__netievent_t *ev0) {
 	isc__nm_uvreq_t *req = ievent->req;
 	int r;

-	REQUIRE(sock->type == isc_nm_tcpsocket);
-	REQUIRE(worker->id == ievent->req->sock->mgr->workers[isc_nm_tid()].id);
+	UNUSED(worker);

 	r = tcp_connect_direct(sock, req);
 	if (r != 0) {
 		/* We need to issue callbacks ourselves */
 		tcp_connect_cb(&req->uv_req.connect, r);
+		goto done;
 	}
+
+	atomic_store(&sock->connected, true);
+
+done:
+	LOCK(&sock->lock);
+	SIGNAL(&sock->cond);
+	UNLOCK(&sock->lock);
 }

 static void
 tcp_connect_cb(uv_connect_t *uvreq, int status) {
 	isc__nm_uvreq_t *req = (isc__nm_uvreq_t *)uvreq->data;
 	isc_nmsocket_t *sock = NULL;
+
 	sock = uv_handle_get_data((uv_handle_t *)uvreq->handle);

 	REQUIRE(VALID_UVREQ(req));

 	if (status == 0) {
 		isc_result_t result;
-		isc_nmhandle_t *handle = NULL;
 		struct sockaddr_storage ss;
+		isc_nmhandle_t *handle = NULL;

+		sock = uv_handle_get_data((uv_handle_t *)uvreq->handle);
 		isc__nm_incstats(sock->mgr, sock->statsindex[STATID_CONNECT]);
 		uv_tcp_getpeername(&sock->uv_handle.tcp, (struct sockaddr *)&ss,
 				   &(int){ sizeof(ss) });
@@ -143,17 +166,77 @@ tcp_connect_cb(uv_connect_t *uvreq, int status) {

 		handle = isc__nmhandle_get(sock, NULL, NULL);
 		req->cb.connect(handle, ISC_R_SUCCESS, req->cbarg);
+
+		isc__nm_uvreq_put(&req, sock);
+
+		/*
+		 * The sock is now attached to the handle.
+		 */
+		isc__nmsocket_detach(&sock);
+
+		/*
+		 * If the connect callback wants to hold on to the handle,
+		 * it needs to attach to it.
+		 */
+		isc_nmhandle_unref(handle);
 	} else {
 		/*
 		 * TODO:
 		 * Handle the connect error properly and free the socket.
 		 */
-		isc__nm_incstats(sock->mgr,
-				 sock->statsindex[STATID_CONNECTFAIL]);
 		req->cb.connect(NULL, isc__nm_uverr2result(status), req->cbarg);
+		isc__nm_uvreq_put(&req, sock);
+	}
+}
+
+isc_result_t
+isc_nm_tcpconnect(isc_nm_t *mgr, isc_nmiface_t *local, isc_nmiface_t *peer,
+		  isc_nm_cb_t cb, void *cbarg, size_t extrahandlesize) {
+	isc_nmsocket_t *nsock = NULL;
+	isc__netievent_tcpconnect_t *ievent = NULL;
+	isc__nm_uvreq_t *req = NULL;
+
+	REQUIRE(VALID_NM(mgr));
+
+	nsock = isc_mem_get(mgr->mctx, sizeof(*nsock));
+	isc__nmsocket_init(nsock, mgr, isc_nm_tcpsocket, local);
+	nsock->extrahandlesize = extrahandlesize;
+	nsock->result = ISC_R_SUCCESS;
+
+	req = isc__nm_uvreq_get(mgr, nsock);
+	req->cb.connect = cb;
+	req->cbarg = cbarg;
+	req->peer = peer->addr;
+
+	ievent = isc__nm_get_ievent(mgr, netievent_tcpconnect);
+	ievent->sock = nsock;
+	ievent->req = req;
+
+	if (isc__nm_in_netthread()) {
+		nsock->tid = isc_nm_tid();
+		isc__nm_async_tcpconnect(&mgr->workers[nsock->tid],
+					 (isc__netievent_t *)ievent);
+		isc__nm_put_ievent(mgr, ievent);
+	} else {
+		nsock->tid = isc_random_uniform(mgr->nworkers);
+		isc__nm_enqueue_ievent(&mgr->workers[nsock->tid],
+				       (isc__netievent_t *)ievent);
+
+		LOCK(&nsock->lock);
+		while (!atomic_load(&nsock->connected) &&
+		       !atomic_load(&nsock->connect_error)) {
+			WAIT(&nsock->cond, &nsock->lock);
+		}
+		UNLOCK(&nsock->lock);
 	}

-	isc__nm_uvreq_put(&req, sock);
+	if (nsock->result != ISC_R_SUCCESS) {
+		isc_result_t result = nsock->result;
+		isc__nmsocket_detach(&nsock);
+		return (result);
+	}
+
+	return (ISC_R_SUCCESS);
 }

 isc_result_t
@@ -167,8 +250,8 @@ isc_nm_listentcp(isc_nm_t *mgr, isc_nmiface_t *iface, isc_nm_cb_t cb,

 	nsock = isc_mem_get(mgr->mctx, sizeof(*nsock));
 	isc__nmsocket_init(nsock, mgr, isc_nm_tcplistener, iface);
-	nsock->rcb.accept = cb;
-	nsock->rcbarg = cbarg;
+	nsock->accept_cb.accept = cb;
+	nsock->accept_cbarg = cbarg;
 	nsock->extrahandlesize = extrahandlesize;
 	nsock->backlog = backlog;
 	nsock->result = ISC_R_SUCCESS;
@@ -205,7 +288,7 @@ isc_nm_listentcp(isc_nm_t *mgr, isc_nmiface_t *iface, isc_nm_cb_t cb,
 		return (ISC_R_SUCCESS);
 	} else {
 		isc_result_t result = nsock->result;
-		isc_nmsocket_detach(&nsock);
+		isc__nmsocket_detach(&nsock);
 		return (result);
 	}
 }
@@ -228,8 +311,8 @@ isc__nm_async_tcplisten(isc__networker_t *worker, isc__netievent_t *ev0) {

 	r = uv_tcp_init(&worker->loop, &sock->uv_handle.tcp);
 	if (r != 0) {
-		/* It was never opened */
 		isc__nm_incstats(sock->mgr, sock->statsindex[STATID_OPENFAIL]);
+		/* The socket was never opened, so no need for uv_close() */
 		atomic_store(&sock->closed, true);
 		sock->result = isc__nm_uverr2result(r);
 		atomic_store(&sock->listen_error, true);
@@ -375,14 +458,24 @@ isc__nm_async_tcpchildaccept(isc__networker_t *worker, isc__netievent_t *ev0) {
 		goto error;
 	}

-	isc_nmsocket_attach(ssock, &csock->server);
+	isc__nmsocket_attach(ssock, &csock->server);

 	handle = isc__nmhandle_get(csock, NULL, &local);

-	INSIST(ssock->rcb.accept != NULL);
+	INSIST(ssock->accept_cb.accept != NULL);
 	csock->read_timeout = ssock->mgr->init;
-	ssock->rcb.accept(handle, ISC_R_SUCCESS, ssock->rcbarg);
-	isc_nmsocket_detach(&csock);
+	ssock->accept_cb.accept(handle, ISC_R_SUCCESS, ssock->accept_cbarg);
+
+	/*
+	 * csock is now attached to the handle.
+	 */
+	isc__nmsocket_detach(&csock);
+
+	/*
+	 * If the accept callback wants to hold on to the handle,
+	 * it needs to attach to it.
+	 */
+	isc_nmhandle_unref(handle);
 	return;

 error:
@@ -401,7 +494,7 @@ error:
 	/*
 	 * Detach the socket properly to make sure uv_close() is called.
 	 */
-	isc_nmsocket_detach(&csock);
+	isc__nmsocket_detach(&csock);
 }

 void
@@ -412,7 +505,7 @@ isc__nm_tcp_stoplistening(isc_nmsocket_t *sock) {
 	REQUIRE(!isc__nm_in_netthread());

 	ievent = isc__nm_get_ievent(sock->mgr, netievent_tcpstop);
-	isc_nmsocket_attach(sock, &ievent->sock);
+	isc__nmsocket_attach(sock, &ievent->sock);
 	isc__nm_enqueue_ievent(&sock->mgr->workers[sock->tid],
 			       (isc__netievent_t *)ievent);
 }
@@ -458,7 +551,7 @@ tcp_listenclose_cb(uv_handle_t *handle) {
 	sock->pquota = NULL;
 	UNLOCK(&sock->lock);

-	isc_nmsocket_detach(&sock);
+	isc__nmsocket_detach(&sock);
 }

 static void
@@ -484,7 +577,10 @@ readtimeout_cb(uv_timer_t *handle) {
 	if (sock->quota) {
 		isc_quota_detach(&sock->quota);
 	}
-	sock->rcb.recv(sock->tcphandle, NULL, sock->rcbarg);
+	if (sock->rcb.recv != NULL) {
+		sock->rcb.recv(sock->statichandle, ISC_R_TIMEDOUT, NULL,
+			       sock->rcbarg);
+	}
 }

 isc_result_t
@@ -617,7 +713,8 @@ read_cb(uv_stream_t *stream, ssize_t nread, const uv_buf_t *buf) {
 					.length = nread };

 		if (sock->rcb.recv != NULL) {
-			sock->rcb.recv(sock->tcphandle, &region, sock->rcbarg);
+			sock->rcb.recv(sock->statichandle, ISC_R_SUCCESS,
+				       &region, sock->rcbarg);
 		}

 		sock->read_timeout = (atomic_load(&sock->keepalive)
@@ -642,7 +739,8 @@ read_cb(uv_stream_t *stream, ssize_t nread, const uv_buf_t *buf) {
 	 */
 	if (sock->rcb.recv != NULL) {
 		isc__nm_incstats(sock->mgr, sock->statsindex[STATID_RECVFAIL]);
-		sock->rcb.recv(sock->tcphandle, NULL, sock->rcbarg);
+		sock->rcb.recv(sock->statichandle, ISC_R_EOF, NULL,
+			       sock->rcbarg);
 	}

 	/*
@@ -693,7 +791,7 @@ isc__nm_async_tcpaccept(isc__networker_t *worker, isc__netievent_t *ev0) {
 	/*
 	 * The socket was attached just before we called isc_quota_attach_cb().
 	 */
-	isc_nmsocket_detach(&ievent->sock);
+	isc__nmsocket_detach(&ievent->sock);
 }

 /*
@@ -737,7 +835,7 @@ accept_connection(isc_nmsocket_t *ssock, isc_quota_t *quota) {
 		 * we need to.)
 		 */
 		isc_nmsocket_t *tsock = NULL;
-		isc_nmsocket_attach(ssock, &tsock);
+		isc__nmsocket_attach(ssock, &tsock);
 		isc_quota_cb_init(&ssock->quotacb, quota_accept_cb, tsock);
 		result = isc_quota_attach_cb(ssock->pquota, &quota,
 					     &ssock->quotacb);
@@ -752,7 +850,7 @@ accept_connection(isc_nmsocket_t *ssock, isc_quota_t *quota) {
 		 * clear the quota callback and and detach the socket.
 		 */
 		isc_quota_cb_init(&ssock->quotacb, NULL, NULL);
-		isc_nmsocket_detach(&tsock);
+		isc__nmsocket_detach(&tsock);
 	}

 	isc__nm_incstats(ssock->mgr, ssock->statsindex[STATID_ACCEPT]);
@@ -839,6 +937,7 @@ static void
 tcp_send_cb(uv_write_t *req, int status) {
 	isc_result_t result = ISC_R_SUCCESS;
 	isc__nm_uvreq_t *uvreq = (isc__nm_uvreq_t *)req->data;
+	isc_nmsocket_t *sock = NULL;

 	REQUIRE(VALID_UVREQ(uvreq));
 	REQUIRE(VALID_NMHANDLE(uvreq->handle));
@@ -850,8 +949,10 @@ tcp_send_cb(uv_write_t *req, int status) {
 	}

 	uvreq->cb.send(uvreq->handle, result, uvreq->cbarg);
+
+	sock = uvreq->handle->sock;
 	isc_nmhandle_unref(uvreq->handle);
-	isc__nm_uvreq_put(&uvreq, uvreq->handle->sock);
+	isc__nm_uvreq_put(&uvreq, sock);
 }

 /*
@@ -904,6 +1005,7 @@ tcp_close_cb(uv_handle_t *uvhandle) {

 	isc__nm_incstats(sock->mgr, sock->statsindex[STATID_CLOSE]);
 	atomic_store(&sock->closed, true);
+	atomic_store(&sock->connected, false);
 	isc__nmsocket_prep_destroy(sock);
 }

@@ -913,7 +1015,9 @@ timer_close_cb(uv_handle_t *uvhandle) {

 	REQUIRE(VALID_NMSOCK(sock));

-	isc_nmsocket_detach(&sock->server);
+	if (sock->server != NULL) {
+		isc__nmsocket_detach(&sock->server);
+	}
 	uv_close(&sock->uv_handle.handle, tcp_close_cb);
 }

@@ -931,7 +1035,7 @@ tcp_close_direct(isc_nmsocket_t *sock) {
 		uv_close((uv_handle_t *)&sock->timer, timer_close_cb);
 	} else {
 		if (sock->server != NULL) {
-			isc_nmsocket_detach(&sock->server);
+			isc__nmsocket_detach(&sock->server);
 		}
 		uv_close(&sock->uv_handle.handle, tcp_close_cb);
 	}
@@ -970,9 +1074,22 @@ void
 isc__nm_tcp_shutdown(isc_nmsocket_t *sock) {
 	REQUIRE(VALID_NMSOCK(sock));

-	if (sock->type == isc_nm_tcpsocket && sock->tcphandle != NULL &&
+	if (sock->type == isc_nm_tcpsocket && sock->statichandle != NULL &&
 	    sock->rcb.recv != NULL)
 	{
-		sock->rcb.recv(sock->tcphandle, NULL, sock->rcbarg);
+		sock->rcb.recv(sock->statichandle, ISC_R_CANCELED, NULL,
+			       sock->rcbarg);
+	}
+}
+
+void
+isc__nm_tcp_cancelread(isc_nmsocket_t *sock) {
+	REQUIRE(VALID_NMSOCK(sock));
+
+	if (sock->type == isc_nm_tcpsocket && sock->statichandle != NULL &&
+	    sock->rcb.recv != NULL)
+	{
+		sock->rcb.recv(sock->statichandle, ISC_R_CANCELED, NULL,
+			       sock->rcbarg);
 	}
 }
@@ -38,7 +38,8 @@
 */

 static void
-dnslisten_readcb(isc_nmhandle_t *handle, isc_region_t *region, void *arg);
+dnslisten_readcb(isc_nmhandle_t *handle, isc_result_t eresult,
+		 isc_region_t *region, void *arg);

 static void
 resume_processing(void *arg);
@@ -82,7 +83,8 @@ static void
 timer_close_cb(uv_handle_t *handle) {
 	isc_nmsocket_t *sock = (isc_nmsocket_t *)uv_handle_get_data(handle);
 	INSIST(VALID_NMSOCK(sock));
-	isc_nmsocket_detach(&sock);
+	atomic_store(&sock->closed, true);
+	tcpdns_close_direct(sock);
 }

 static void
@@ -92,7 +94,9 @@ dnstcp_readtimeout(uv_timer_t *timer) {

 	REQUIRE(VALID_NMSOCK(sock));
 	REQUIRE(sock->tid == isc_nm_tid());
-	tcpdns_close_direct(sock);
+	/* Close the TCP connection, it's closing should fire 'our' closing */
+	isc_nmhandle_unref(sock->outerhandle);
+	sock->outerhandle = NULL;
 }

 /*
@@ -122,8 +126,13 @@ dnslisten_acceptcb(isc_nmhandle_t *handle, isc_result_t result, void *cbarg) {
 			   handle->sock->iface);

 	dnssock->extrahandlesize = dnslistensock->extrahandlesize;
-	isc_nmsocket_attach(dnslistensock, &dnssock->listener);
-	isc_nmsocket_attach(handle->sock, &dnssock->outer);
+	isc__nmsocket_attach(dnslistensock, &dnssock->listener);
+
+	isc__nmsocket_attach(dnssock, &dnssock->self);
+
+	dnssock->outerhandle = handle;
+	isc_nmhandle_ref(dnssock->outerhandle);
+
 	dnssock->peer = handle->sock->peer;
 	dnssock->read_timeout = handle->sock->mgr->init;
 	dnssock->tid = isc_nm_tid();
@@ -135,8 +144,12 @@ dnslisten_acceptcb(isc_nmhandle_t *handle, isc_result_t result, void *cbarg) {
 	dnssock->timer_initialized = true;
 	uv_timer_start(&dnssock->timer, dnstcp_readtimeout,
 		       dnssock->read_timeout, 0);
-
-	isc_nm_read(handle, dnslisten_readcb, dnssock);
+	isc_nmhandle_ref(handle);
+	result = isc_nm_read(handle, dnslisten_readcb, dnssock);
+	if (result != ISC_R_SUCCESS) {
+		isc_nmhandle_unref(handle);
+	}
+	isc__nmsocket_detach(&dnssock);
 }

 /*
@@ -162,6 +175,11 @@ processbuffer(isc_nmsocket_t *dnssock, isc_nmhandle_t **handlep) {
 	if (dnssock->buf_len < 2) {
 		return (ISC_R_NOMORE);
 	}
+	
+	if (dnssock->listener == NULL && dnssock->rcb.recv == NULL) {
+		/* Nobody waits for us, pause. */
+		return (ISC_R_DISABLED);
+	}

 	/*
 	 * Process the first packet from the buffer, leaving
@@ -169,16 +187,35 @@ processbuffer(isc_nmsocket_t *dnssock, isc_nmhandle_t **handlep) {
 	 */
 	len = dnslen(dnssock->buf);
 	if (len <= dnssock->buf_len - 2) {
-		isc_nmhandle_t *dnshandle = isc__nmhandle_get(dnssock, NULL,
-							      NULL);
+		isc_nmhandle_t *dnshandle;
+		if (dnssock->statichandle != NULL) {
+			dnshandle = dnssock->statichandle;
+			isc_nmhandle_ref(dnshandle);
+		} else {
+			dnshandle = isc__nmhandle_get(dnssock, NULL, NULL);
+		}
+		
 		isc_nmsocket_t *listener = dnssock->listener;

 		if (listener != NULL && listener->rcb.recv != NULL) {
 			listener->rcb.recv(
-				dnshandle,
+				dnshandle, ISC_R_SUCCESS,
 				&(isc_region_t){ .base = dnssock->buf + 2,
 						 .length = len },
 				listener->rcbarg);
+		} else if (dnssock->rcb.recv != NULL) {
+			/*
+			 * We need to clear the callback before issuing it -
+			 * as the callback itself might replace it.
+			 */
+			isc_nm_recv_cb_t cb = dnssock->rcb.recv;
+			void* cbarg = dnssock->rcbarg;
+			dnssock->rcb.recv = NULL;
+			dnssock->rcbarg = NULL;
+			cb(dnshandle, ISC_R_SUCCESS,
+			   &(isc_region_t){ .base = dnssock->buf + 2,
+					    .length = len },
+			   cbarg);
 		}

 		len += 2;
@@ -200,7 +237,8 @@ processbuffer(isc_nmsocket_t *dnssock, isc_nmhandle_t **handlep) {
 * a complete DNS packet and, if so - call the callback
 */
 static void
-dnslisten_readcb(isc_nmhandle_t *handle, isc_region_t *region, void *arg) {
+dnslisten_readcb(isc_nmhandle_t *handle, isc_result_t eresult,
+		 isc_region_t *region, void *arg) {
 	isc_nmsocket_t *dnssock = (isc_nmsocket_t *)arg;
 	unsigned char *base = NULL;
 	bool done = false;
@@ -210,9 +248,13 @@ dnslisten_readcb(isc_nmhandle_t *handle, isc_region_t *region, void *arg) {
 	REQUIRE(VALID_NMHANDLE(handle));
 	REQUIRE(dnssock->tid == isc_nm_tid());

-	if (region == NULL) {
+	if (region == NULL || eresult != ISC_R_SUCCESS) {
 		/* Connection closed */
-		isc__nm_tcpdns_close(dnssock);
+		isc_nmhandle_unref(handle);
+		dnssock->result = eresult;
+		if (dnssock->self != NULL) {
+			isc__nmsocket_detach(&dnssock->self);
+		}
 		return;
 	}

@@ -234,17 +276,24 @@ dnslisten_readcb(isc_nmhandle_t *handle, isc_region_t *region, void *arg) {
 		isc_nmhandle_t *dnshandle = NULL;

 		result = processbuffer(dnssock, &dnshandle);
-		if (result != ISC_R_SUCCESS) {
+		if (result == ISC_R_DISABLED) {
 			/*
-			 * There wasn't anything in the buffer to process.
+			 * Nobody is waiting on the callback, pause reading.
+			 */
+			isc_nm_pauseread(dnssock->outerhandle->sock);
+			return;
+		} else if (result == ISC_R_NOMORE) {
+			/*
+			 * There wasn't anything in the buffer to process
 			 */
 			return;
 		}
+		INSIST(result == ISC_R_SUCCESS);

 		/*
 		 * We have a packet: stop timeout timers
 		 */
-		atomic_store(&dnssock->outer->processing, true);
+		atomic_store(&dnssock->outerhandle->sock->processing, true);
 		if (dnssock->timer_initialized) {
 			uv_timer_stop(&dnssock->timer);
 		}
@@ -254,8 +303,14 @@ dnslisten_readcb(isc_nmhandle_t *handle, isc_region_t *region, void *arg) {
 			 * We're in sequential mode and we processed
 			 * one packet, so we're done until the next read
 			 * completes.
+			 * If we're a client - clear the callback.
 			 */
-			isc_nm_pauseread(dnssock->outer);
+		} else if (dnssock->client && dnssock->rcb.recv == NULL) {
+			/*
+			 * We're in client mode and we don't have a callback -
+			 * pause the read.
+			 */
+			isc_nm_pauseread(dnssock->outerhandle->sock);
 			done = true;
 		} else {
 			/*
@@ -267,7 +322,7 @@ dnslisten_readcb(isc_nmhandle_t *handle, isc_region_t *region, void *arg) {
 			 */
 			if (atomic_load(&dnssock->ah) >=
 			    TCPDNS_CLIENTS_PER_CONN) {
-				isc_nm_pauseread(dnssock->outer);
+				isc_nm_pauseread(dnssock->outerhandle->sock);
 				done = true;
 			}
 		}
@@ -310,7 +365,7 @@ isc_nm_listentcpdns(isc_nm_t *mgr, isc_nmiface_t *iface, isc_nm_recv_cb_t cb,
 		return (ISC_R_SUCCESS);
 	} else {
 		atomic_store(&dnslistensock->closed, true);
-		isc_nmsocket_detach(&dnslistensock);
+		isc__nmsocket_detach(&dnslistensock);
 		return (result);
 	}
 }
@@ -326,8 +381,8 @@ isc__nm_tcpdns_stoplistening(isc_nmsocket_t *sock) {
 	sock->rcbarg = NULL;

 	if (sock->outer != NULL) {
-		isc_nm_stoplistening(sock->outer);
-		isc_nmsocket_detach(&sock->outer);
+		isc__nm_tcp_stoplistening(sock->outer);
+		isc__nmsocket_detach(&sock->outer);
 	}
 }

@@ -336,7 +391,8 @@ isc_nm_tcpdns_sequential(isc_nmhandle_t *handle) {
 	REQUIRE(VALID_NMHANDLE(handle));

 	if (handle->sock->type != isc_nm_tcpdnssocket ||
-	    handle->sock->outer == NULL) {
+	    handle->sock->outerhandle == NULL)
+	{
 		return;
 	}

@@ -348,7 +404,7 @@ isc_nm_tcpdns_sequential(isc_nmhandle_t *handle) {
 	 * closehandle_cb callback, called whenever a handle
 	 * is released.
 	 */
-	isc_nm_pauseread(handle->sock->outer);
+	isc_nm_pauseread(handle->sock->outerhandle->sock);
 	atomic_store(&handle->sock->sequential, true);
 }

@@ -357,12 +413,13 @@ isc_nm_tcpdns_keepalive(isc_nmhandle_t *handle) {
 	REQUIRE(VALID_NMHANDLE(handle));

 	if (handle->sock->type != isc_nm_tcpdnssocket ||
-	    handle->sock->outer == NULL) {
+	    handle->sock->outerhandle == NULL)
+	{
 		return;
 	}

 	atomic_store(&handle->sock->keepalive, true);
-	atomic_store(&handle->sock->outer->keepalive, true);
+	atomic_store(&handle->sock->outerhandle->sock->keepalive, true);
 }

 typedef struct tcpsend {
@@ -382,13 +439,13 @@ resume_processing(void *arg) {
 	REQUIRE(VALID_NMSOCK(sock));
 	REQUIRE(sock->tid == isc_nm_tid());

-	if (sock->type != isc_nm_tcpdnssocket || sock->outer == NULL) {
+	if (sock->type != isc_nm_tcpdnssocket || sock->outerhandle == NULL) {
 		return;
 	}

 	if (atomic_load(&sock->ah) == 0) {
 		/* Nothing is active; sockets can timeout now */
-		atomic_store(&sock->outer->processing, false);
+		atomic_store(&sock->outerhandle->sock->processing, false);
 		if (sock->timer_initialized) {
 			uv_timer_start(&sock->timer, dnstcp_readtimeout,
 				       sock->read_timeout, 0);
@@ -404,13 +461,14 @@ resume_processing(void *arg) {

 		result = processbuffer(sock, &handle);
 		if (result == ISC_R_SUCCESS) {
-			atomic_store(&sock->outer->processing, true);
+			atomic_store(&sock->outerhandle->sock->processing,
+				     true);
 			if (sock->timer_initialized) {
 				uv_timer_stop(&sock->timer);
 			}
 			isc_nmhandle_unref(handle);
-		} else if (sock->outer != NULL) {
-			isc_nm_resumeread(sock->outer);
+		} else if (sock->outerhandle != NULL) {
+			isc_nm_resumeread(sock->outerhandle->sock);
 		}

 		return;
@@ -428,8 +486,8 @@ resume_processing(void *arg) {
 			/*
 			 * Nothing in the buffer; resume reading.
 			 */
-			if (sock->outer != NULL) {
-				isc_nm_resumeread(sock->outer);
+			if (sock->outerhandle != NULL) {
+				isc_nm_resumeread(sock->outerhandle->sock);
 			}

 			break;
@@ -438,7 +496,7 @@ resume_processing(void *arg) {
 		if (sock->timer_initialized) {
 			uv_timer_stop(&sock->timer);
 		}
-		atomic_store(&sock->outer->processing, true);
+		atomic_store(&sock->outerhandle->sock->processing, true);
 		isc_nmhandle_unref(dnshandle);
 	} while (atomic_load(&sock->ah) < TCPDNS_CLIENTS_PER_CONN);
 }
@@ -447,13 +505,13 @@ static void
 tcpdnssend_cb(isc_nmhandle_t *handle, isc_result_t result, void *cbarg) {
 	tcpsend_t *ts = (tcpsend_t *)cbarg;

-	UNUSED(handle);
-
 	ts->cb(ts->orighandle, result, ts->cbarg);
 	isc_mem_put(ts->mctx, ts->region.base, ts->region.length);

 	isc_nmhandle_unref(ts->orighandle);
 	isc_mem_putanddetach(&ts->mctx, ts, sizeof(*ts));
+
+	isc_nmhandle_unref(handle);
 }

 /*
@@ -471,7 +529,7 @@ isc__nm_tcpdns_send(isc_nmhandle_t *handle, isc_region_t *region,
 	REQUIRE(VALID_NMSOCK(sock));
 	REQUIRE(sock->type == isc_nm_tcpdnssocket);

-	if (sock->outer == NULL) {
+	if (sock->outerhandle == NULL) {
 		/* The socket is closed */
 		return (ISC_R_NOTCONNECTED);
 	}
@@ -480,12 +538,13 @@ isc__nm_tcpdns_send(isc_nmhandle_t *handle, isc_region_t *region,
 	*t = (tcpsend_t){
 		.cb = cb,
 		.cbarg = cbarg,
-		.handle = handle->sock->outer->tcphandle,
+		.handle = handle->sock->outerhandle,
 	};

 	isc_mem_attach(sock->mgr->mctx, &t->mctx);
 	t->orighandle = handle;
 	isc_nmhandle_ref(t->orighandle);
+	isc_nmhandle_ref(t->handle);

 	t->region = (isc_region_t){ .base = isc_mem_get(t->mctx,
 							region->length + 2),
@@ -500,6 +559,7 @@ isc__nm_tcpdns_send(isc_nmhandle_t *handle, isc_region_t *region,
 static void
 tcpdns_close_direct(isc_nmsocket_t *sock) {
 	REQUIRE(sock->tid == isc_nm_tid());
+
 	/* We don't need atomics here, it's all in single network thread */
 	if (sock->timer_initialized) {
 		/*
@@ -510,19 +570,23 @@ tcpdns_close_direct(isc_nmsocket_t *sock) {
 		sock->timer_initialized = false;
 		uv_timer_stop(&sock->timer);
 		uv_close((uv_handle_t *)&sock->timer, timer_close_cb);
+	} else if (sock->self != NULL) {
+		isc__nmsocket_detach(&sock->self);
 	} else {
 		/*
 		 * At this point we're certain that there are no external
 		 * references, we can close everything.
 		 */
-		if (sock->outer != NULL) {
-			sock->outer->rcb.recv = NULL;
-			isc_nmsocket_detach(&sock->outer);
+		if (sock->outerhandle != NULL) {
+			sock->outerhandle->sock->rcb.recv = NULL;
+			isc_nmhandle_unref(sock->outerhandle);
+			sock->outerhandle = NULL;
 		}
 		if (sock->listener != NULL) {
-			isc_nmsocket_detach(&sock->listener);
+			isc__nmsocket_detach(&sock->listener);
 		}
 		atomic_store(&sock->closed, true);
+		isc__nmsocket_prep_destroy(sock);
 	}
 }

@@ -552,3 +616,137 @@ isc__nm_async_tcpdnsclose(isc__networker_t *worker, isc__netievent_t *ev0) {

 	tcpdns_close_direct(ievent->sock);
 }
+
+typedef struct tcpconnect {
+	isc_mem_t *mctx;
+	isc_nm_cb_t cb;
+	void *cbarg;
+	size_t extrahandlesize;
+} tcpconnect_t;
+
+static void
+tcpdnsconnect_cb(isc_nmhandle_t *handle, isc_result_t result, void *ncbarg_) {
+	tcpconnect_t *ncbarg = (tcpconnect_t *)ncbarg_;
+	isc_nm_cb_t cb = ncbarg->cb;
+	void *cbarg = ncbarg->cbarg;
+	size_t extrahandlesize = ncbarg->extrahandlesize;
+	isc_mem_putanddetach(&ncbarg->mctx, ncbarg, sizeof(*ncbarg));
+
+	if (result != ISC_R_SUCCESS) {
+		cb(NULL, result, cbarg);
+		return;
+	}
+	INSIST(VALID_NMHANDLE(handle));
+
+	isc_nmsocket_t *dnssock = isc_mem_get(handle->sock->mgr->mctx,
+					      sizeof(*dnssock));
+	isc__nmsocket_init(dnssock, handle->sock->mgr, isc_nm_tcpdnssocket,
+			   handle->sock->iface);
+
+	dnssock->extrahandlesize = extrahandlesize;
+	dnssock->outerhandle = handle;
+	isc_nmhandle_ref(dnssock->outerhandle);
+
+	dnssock->peer = handle->sock->peer;
+	dnssock->read_timeout = handle->sock->mgr->init;
+	dnssock->tid = isc_nm_tid();
+
+	dnssock->client = true;
+	dnssock->statichandle = isc__nmhandle_get(dnssock, NULL, NULL);
+	
+	uv_timer_init(&dnssock->mgr->workers[isc_nm_tid()].loop,
+		      &dnssock->timer);
+	dnssock->timer.data = dnssock;
+	dnssock->timer_initialized = true;
+	uv_timer_start(&dnssock->timer, dnstcp_readtimeout,
+		       dnssock->read_timeout, 0);
+	/*
+	 * We start reading not asked to - we'll read and buffer
+	 * at most one packet.
+	 */
+	result = isc_nm_read(handle, dnslisten_readcb, dnssock);
+	if (result != ISC_R_SUCCESS) {
+		isc_nmhandle_unref(handle);
+	}
+
+	cb(dnssock->statichandle, ISC_R_SUCCESS, cbarg);
+	isc_nmhandle_unref(dnssock->statichandle);
+	isc__nmsocket_detach(&dnssock);
+}
+
+isc_result_t
+isc_nm_tcpdnsconnect(isc_nm_t *mgr, isc_nmiface_t *local, isc_nmiface_t *peer,
+		     isc_nm_cb_t cb, void *cbarg, size_t extrahandlesize) {
+	tcpconnect_t *ncbarg = isc_mem_get(mgr->mctx, sizeof(tcpconnect_t));
+	*ncbarg = (tcpconnect_t){ .cb = cb,
+				  .cbarg = cbarg,
+				  .extrahandlesize = extrahandlesize };
+	isc_mem_attach(mgr->mctx, &ncbarg->mctx);
+	return (isc_nm_tcpconnect(mgr, local, peer, tcpdnsconnect_cb, ncbarg,
+				  0));
+}
+
+isc_result_t
+isc__nm_tcpdns_read(isc_nmhandle_t *handle, isc_nm_recv_cb_t cb, void *cbarg) {
+	/*
+	 * This HAS to be done asynchronously - read is often called from the
+	 * read callback, we'd clash in processbuffer() AND grow the stack
+	 * indefinitely.
+	 */
+	isc_nmsocket_t *sock = handle->sock;
+	INSIST(handle == sock->statichandle);
+	INSIST(sock->rcb.recv == NULL);
+	isc__netievent_tcpdnsread_t *ievent =
+		isc__nm_get_ievent(sock->mgr, netievent_tcpdnsread);
+	ievent->sock = sock;
+	sock->rcb.recv = cb;
+	sock->rcbarg = cbarg;
+	isc__nm_enqueue_ievent(&sock->mgr->workers[sock->tid],
+			       (isc__netievent_t *)ievent);
+	return (ISC_R_SUCCESS);
+}
+
+void
+isc__nm_async_tcpdnsread(isc__networker_t *worker, isc__netievent_t *ev0) {
+	isc__netievent_tcpdnsread_t *ievent =
+		(isc__netievent_tcpdnsclose_t *)ev0;
+	isc_nmsocket_t *sock = ievent->sock;
+
+	REQUIRE(VALID_NMSOCK(sock));
+	REQUIRE(worker->id == sock->tid);
+	isc_nmhandle_t *handle = sock->statichandle;
+
+	isc_result_t result;
+
+	if (sock->type != isc_nm_tcpdnssocket || sock->outerhandle == NULL) {
+		sock->rcb.recv(handle, ISC_R_NOTCONNECTED, NULL, sock->rcbarg);
+		return;
+	}
+
+	/* Maybe we have a packet already? */
+	isc_nmhandle_t *newhandle = NULL;
+	result = processbuffer(sock, &newhandle);
+	if (result == ISC_R_SUCCESS) {
+		atomic_store(&sock->outerhandle->sock->processing,
+			     true);
+		if (sock->timer_initialized) {
+			uv_timer_stop(&sock->timer);
+		}
+		isc_nmhandle_unref(handle);
+	} else if (sock->outerhandle != NULL) {
+		/* Restart reading, wait for the callback */
+		atomic_store(&sock->outerhandle->sock->processing,
+			     false);
+		if (sock->timer_initialized) {
+			uv_timer_start(&sock->timer, dnstcp_readtimeout,
+				       sock->read_timeout, 0);
+		}
+		isc_nm_resumeread(sock->outerhandle->sock);
+	} else {
+		isc_nm_recv_cb_t cb = sock->rcb.recv;
+		void *cbarg = sock->rcbarg;
+		sock->rcb.recv = NULL;
+		sock->rcbarg = NULL;
+		cb(handle, ISC_R_NOTCONNECTED, NULL, cbarg);
+	}
+}
@@ -152,7 +152,8 @@ isc__nm_async_udplisten(isc__networker_t *worker, isc__netievent_t *ev0) {
 #endif
 	uv_udp_init_ex(&worker->loop, &sock->uv_handle.udp, uv_init_flags);
 	uv_handle_set_data(&sock->uv_handle.handle, NULL);
-	isc_nmsocket_attach(sock, (isc_nmsocket_t **)&sock->uv_handle.udp.data);
+	isc__nmsocket_attach(sock,
+			     (isc_nmsocket_t **)&sock->uv_handle.udp.data);

 	r = uv_udp_open(&sock->uv_handle.udp, sock->fd);
 	if (r == 0) {
@@ -186,7 +187,7 @@ udp_close_cb(uv_handle_t *handle) {
 	isc_nmsocket_t *sock = uv_handle_get_data(handle);
 	atomic_store(&sock->closed, true);

-	isc_nmsocket_detach((isc_nmsocket_t **)&sock->uv_handle.udp.data);
+	isc__nmsocket_detach((isc_nmsocket_t **)&sock->uv_handle.udp.data);
 }

 static void
@@ -316,12 +317,17 @@ udp_recv_cb(uv_udp_t *handle, ssize_t nrecv, const uv_buf_t *buf,
 	isc_result_t result;
 	isc_nmhandle_t *nmhandle = NULL;
 	isc_sockaddr_t sockaddr;
-	isc_nmsocket_t *sock = uv_handle_get_data((uv_handle_t *)handle);
+	isc_nmsocket_t *sock = NULL;
 	isc_region_t region;
 	uint32_t maxudp;
 	bool free_buf = true;

-	REQUIRE(VALID_NMSOCK(sock));
+	/*
+	 * Even though destruction of the socket can only happen from the
+	 * network thread that we're in, we still attach to the socket here
+	 * to ensure it won't be destroyed by the recv callback.
+	 */
+	isc__nmsocket_attach(uv_handle_get_data((uv_handle_t *)handle), &sock);

 #ifdef UV_UDP_MMSG_CHUNK
 	free_buf = ((flags & UV_UDP_MMSG_CHUNK) == 0);
@@ -337,6 +343,7 @@ udp_recv_cb(uv_udp_t *handle, ssize_t nrecv, const uv_buf_t *buf,
 		if (free_buf) {
 			isc__nm_free_uvbuf(sock, buf);
 		}
+		isc__nmsocket_detach(&sock);
 		return;
 	}

@@ -346,6 +353,7 @@ udp_recv_cb(uv_udp_t *handle, ssize_t nrecv, const uv_buf_t *buf,
 	 */
 	maxudp = atomic_load(&sock->mgr->maxudp);
 	if (maxudp != 0 && (uint32_t)nrecv > maxudp) {
+		isc__nmsocket_detach(&sock);
 		return;
 	}

@@ -356,11 +364,16 @@ udp_recv_cb(uv_udp_t *handle, ssize_t nrecv, const uv_buf_t *buf,
 	region.length = nrecv;

 	INSIST(sock->rcb.recv != NULL);
-	sock->rcb.recv(nmhandle, &region, sock->rcbarg);
+	sock->rcb.recv(nmhandle, ISC_R_SUCCESS, &region, sock->rcbarg);
 	if (free_buf) {
 		isc__nm_free_uvbuf(sock, buf);
 	}

+	/*
+	 * The sock is now attached to the handle, we can detach our ref.
+	 */
+	isc__nmsocket_detach(&sock);
+
 	/*
 	 * If the recv callback wants to hold on to the handle,
 	 * it needs to attach to it.
@@ -449,22 +449,26 @@ isc_nmhandle_peeraddr
 isc_nmhandle_ref
 isc_nmhandle_setdata
 isc_nmhandle_unref
+isc_nm_cancelread
 isc_nm_closedown
 isc_nm_destroy
 isc_nm_detach
 isc_nm_listentcpdns
+isc_nm_listentcp
 isc_nm_listenudp
 isc_nm_maxudp
+isc_nm_read
 isc_nm_send
 isc_nm_setstats
 isc_nm_start
 isc_nm_stoplistening
+isc_nm_tcpconnect
 isc_nm_tcp_gettimeouts
 isc_nm_tcp_settimeouts
 isc_nm_tcpdns_keepalive
 isc_nm_tcpdns_sequential
 isc_nm_tid
-isc_nmsocket_detach
+isc_nmsocket_close
 isc__nm_acquire_interlocked
 isc__nm_drop_interlocked
 isc__nm_acquire_interlocked_force
@@ -28,8 +28,9 @@
 #include <inttypes.h>

 #include <isc/mem.h>
+#include <isc/netmgr.h>
 #include <isc/result.h>
-#include <isc/task.h>
+#include <isc/string.h>
 #include <isc/util.h>

 #include <isccc/ccmsg.h>
@@ -39,111 +40,138 @@
 #define VALID_CCMSG(foo) ISC_MAGIC_VALID(foo, CCMSG_MAGIC)

 static void
-recv_length(isc_task_t *, isc_event_t *);
-static void
-recv_message(isc_task_t *, isc_event_t *);
+recv_message(isc_nmhandle_t *handle, isc_result_t eresult, isc_region_t *region,
+	     void *arg);

 static void
-recv_length(isc_task_t *task, isc_event_t *ev_in) {
-	isc_socketevent_t *ev = (isc_socketevent_t *)ev_in;
-	isc_event_t *dev;
-	isccc_ccmsg_t *ccmsg = ev_in->ev_arg;
-	isc_region_t region;
+recv_nonce(isc_nmhandle_t *handle, isc_result_t eresult, isc_region_t *region,
+	   void *arg) {
+	isccc_ccmsg_t *ccmsg = arg;
 	isc_result_t result;

+	if (eresult == ISC_R_CANCELED) {
+		return;
+	}
+
 	INSIST(VALID_CCMSG(ccmsg));

-	dev = &ccmsg->event;
-
-	if (ev->result != ISC_R_SUCCESS) {
-		ccmsg->result = ev->result;
-		goto send_and_free;
+	if (region == NULL && eresult == ISC_R_SUCCESS) {
+		ccmsg->result = ISC_R_EOF;
+		goto done;
+	} else if (eresult != ISC_R_SUCCESS) {
+		ccmsg->result = eresult;
+		goto done;
+	} else {
+		ccmsg->result = eresult;
 	}

-	/*
-	 * Success.
-	 */
-	ccmsg->size = ntohl(ccmsg->size);
+	if (region->length < sizeof(uint32_t)) {
+		ccmsg->result = ISC_R_UNEXPECTEDEND;
+		goto done;
+	}
+
+	ccmsg->size = ntohl(*(uint32_t *)region->base);
 	if (ccmsg->size == 0) {
 		ccmsg->result = ISC_R_UNEXPECTEDEND;
-		goto send_and_free;
+		goto done;
 	}
 	if (ccmsg->size > ccmsg->maxsize) {
 		ccmsg->result = ISC_R_RANGE;
-		goto send_and_free;
+		goto done;
 	}

-	region.base = isc_mem_get(ccmsg->mctx, ccmsg->size);
-	region.length = ccmsg->size;
-	if (region.base == NULL) {
-		ccmsg->result = ISC_R_NOMEMORY;
-		goto send_and_free;
+	isc_region_consume(region, sizeof(uint32_t));
+	isc_buffer_allocate(ccmsg->mctx, &ccmsg->buffer, ccmsg->size);
+
+	/*
+	 * If there's more of the message waiting, pass it to
+	 * recv_message() directly.
+	 */
+	if (region->length != 0) {
+		recv_message(handle, ISC_R_SUCCESS, region, ccmsg);
+		return;
 	}

-	isc_buffer_init(&ccmsg->buffer, region.base, region.length);
-	result = isc_socket_recv(ccmsg->sock, &region, 0, task, recv_message,
-				 ccmsg);
-	if (result != ISC_R_SUCCESS) {
-		ccmsg->result = result;
-		goto send_and_free;
+	/*
+	 * Otherwise, continue reading and handle it in
+	 * recv_message().
+	 */
+	result = isc_nm_read(handle, recv_message, ccmsg);
+	if (result == ISC_R_SUCCESS) {
+		return;
 	}

-	isc_event_free(&ev_in);
-	return;
+	ccmsg->result = result;

-send_and_free:
-	isc_task_send(ccmsg->task, &dev);
-	ccmsg->task = NULL;
-	isc_event_free(&ev_in);
-	return;
+done:
+	ccmsg->cb(handle, ccmsg->result, ccmsg->cbarg);
+	isc_nmhandle_unref(handle);
 }

 static void
-recv_message(isc_task_t *task, isc_event_t *ev_in) {
-	isc_socketevent_t *ev = (isc_socketevent_t *)ev_in;
-	isc_event_t *dev;
-	isccc_ccmsg_t *ccmsg = ev_in->ev_arg;
+recv_message(isc_nmhandle_t *handle, isc_result_t eresult, isc_region_t *region,
+	     void *arg) {
+	isc_result_t result;
+	isccc_ccmsg_t *ccmsg = arg;
+	size_t size;

-	(void)task;
+	if (eresult == ISC_R_CANCELED) {
+		return;
+	}

 	INSIST(VALID_CCMSG(ccmsg));

-	dev = &ccmsg->event;
-
-	if (ev->result != ISC_R_SUCCESS) {
-		ccmsg->result = ev->result;
-		goto send_and_free;
+	if (region == NULL && eresult == ISC_R_SUCCESS) {
+		ccmsg->result = ISC_R_EOF;
+		goto done;
+	} else if (eresult != ISC_R_SUCCESS) {
+		ccmsg->result = eresult;
+		goto done;
+	} else {
+		ccmsg->result = eresult;
 	}

-	ccmsg->result = ISC_R_SUCCESS;
-	isc_buffer_add(&ccmsg->buffer, ev->n);
-	ccmsg->address = ev->address;
+	if (region->length == 0) {
+		ccmsg->result = ISC_R_UNEXPECTEDEND;
+		goto done;
+	}

-send_and_free:
-	isc_task_send(ccmsg->task, &dev);
-	ccmsg->task = NULL;
-	isc_event_free(&ev_in);
+	size = ISC_MIN(isc_buffer_availablelength(ccmsg->buffer),
+		       region->length);
+	isc_buffer_putmem(ccmsg->buffer, region->base, size);
+	isc_region_consume(region, size);
+
+	if (isc_buffer_usedlength(ccmsg->buffer) == ccmsg->size) {
+		ccmsg->result = ISC_R_SUCCESS;
+		goto done;
+	}
+
+	result = isc_nm_read(handle, recv_message, ccmsg);
+	if (result == ISC_R_SUCCESS) {
+		return;
+	}
+
+	ccmsg->result = result;
+
+done:
+	ccmsg->cb(handle, ccmsg->result, ccmsg->cbarg);
+	isc_nmhandle_unref(handle);
 }

 void
-isccc_ccmsg_init(isc_mem_t *mctx, isc_socket_t *sock, isccc_ccmsg_t *ccmsg) {
+isccc_ccmsg_init(isc_mem_t *mctx, isc_nmhandle_t *handle,
+		 isccc_ccmsg_t *ccmsg) {
 	REQUIRE(mctx != NULL);
-	REQUIRE(sock != NULL);
+	REQUIRE(handle != NULL);
 	REQUIRE(ccmsg != NULL);

-	ccmsg->magic = CCMSG_MAGIC;
-	ccmsg->size = 0;
-	ccmsg->buffer.base = NULL;
-	ccmsg->buffer.length = 0;
-	ccmsg->maxsize = 4294967295U; /* Largest message possible. */
-	ccmsg->mctx = mctx;
-	ccmsg->sock = sock;
-	ccmsg->task = NULL;		  /* None yet. */
-	ccmsg->result = ISC_R_UNEXPECTED; /* None yet. */
-					  /*
-					   * Should probably initialize the
-					   *event here, but it can wait.
-					   */
+	*ccmsg = (isccc_ccmsg_t){
+		.magic = CCMSG_MAGIC,
+		.maxsize = 0xffffffffU, /* Largest message possible. */
+		.mctx = mctx,
+		.handle = handle,
+		.result = ISC_R_UNEXPECTED /* None yet. */
+	};
 }

 void
@@ -154,37 +182,23 @@ isccc_ccmsg_setmaxsize(isccc_ccmsg_t *ccmsg, unsigned int maxsize) {
 }

 isc_result_t
-isccc_ccmsg_readmessage(isccc_ccmsg_t *ccmsg, isc_task_t *task,
-			isc_taskaction_t action, void *arg) {
+isccc_ccmsg_readmessage(isccc_ccmsg_t *ccmsg, isc_nm_cb_t cb, void *cbarg) {
 	isc_result_t result;
-	isc_region_t region;

 	REQUIRE(VALID_CCMSG(ccmsg));
-	REQUIRE(task != NULL);
-	REQUIRE(ccmsg->task == NULL); /* not currently in use */

-	if (ccmsg->buffer.base != NULL) {
-		isc_mem_put(ccmsg->mctx, ccmsg->buffer.base,
-			    ccmsg->buffer.length);
-		ccmsg->buffer.base = NULL;
-		ccmsg->buffer.length = 0;
+	if (ccmsg->buffer != NULL) {
+		isc_buffer_free(&ccmsg->buffer);
 	}

-	ccmsg->task = task;
-	ccmsg->action = action;
-	ccmsg->arg = arg;
+	ccmsg->cb = cb;
+	ccmsg->cbarg = cbarg;
 	ccmsg->result = ISC_R_UNEXPECTED; /* unknown right now */

-	ISC_EVENT_INIT(&ccmsg->event, sizeof(isc_event_t), 0, 0,
-		       ISCCC_EVENT_CCMSG, action, arg, ccmsg, NULL, NULL);
-
-	region.base = (unsigned char *)&ccmsg->size;
-	region.length = 4; /* uint32_t */
-	result = isc_socket_recv(ccmsg->sock, &region, 0, ccmsg->task,
-				 recv_length, ccmsg);
-
+	isc_nmhandle_ref(ccmsg->handle);
+	result = isc_nm_read(ccmsg->handle, recv_nonce, ccmsg);
 	if (result != ISC_R_SUCCESS) {
-		ccmsg->task = NULL;
+		isc_nmhandle_unref(ccmsg->handle);
 	}

 	return (result);
@@ -194,34 +208,16 @@ void
 isccc_ccmsg_cancelread(isccc_ccmsg_t *ccmsg) {
 	REQUIRE(VALID_CCMSG(ccmsg));

-	isc_socket_cancel(ccmsg->sock, NULL, ISC_SOCKCANCEL_RECV);
+	isc_nm_cancelread(ccmsg->handle);
 }

-#if 0
-void
-isccc_ccmsg_freebuffer(isccc_ccmsg_t*ccmsg) {
-	REQUIRE(VALID_CCMSG(ccmsg));
-
-	if (ccmsg->buffer.base == NULL) {
-		return;
-	}
-
-	isc_mem_put(ccmsg->mctx,ccmsg->buffer.base,ccmsg->buffer.length);
-	ccmsg->buffer.base = NULL;
-	ccmsg->buffer.length = 0;
-}
-#endif /* if 0 */
-
 void
 isccc_ccmsg_invalidate(isccc_ccmsg_t *ccmsg) {
 	REQUIRE(VALID_CCMSG(ccmsg));

 	ccmsg->magic = 0;

-	if (ccmsg->buffer.base != NULL) {
-		isc_mem_put(ccmsg->mctx, ccmsg->buffer.base,
-			    ccmsg->buffer.length);
-		ccmsg->buffer.base = NULL;
-		ccmsg->buffer.length = 0;
+	if (ccmsg->buffer != NULL) {
+		isc_buffer_free(&ccmsg->buffer);
 	}
 }
@@ -32,39 +32,37 @@

 #include <isc/buffer.h>
 #include <isc/lang.h>
-#include <isc/socket.h>
+#include <isc/netmgr.h>
+#include <isc/sockaddr.h>

 /*% ISCCC Message Structure */
 typedef struct isccc_ccmsg {
 	/* private (don't touch!) */
-	unsigned int	 magic;
-	uint32_t	 size;
-	isc_buffer_t	 buffer;
-	unsigned int	 maxsize;
-	isc_mem_t *	 mctx;
-	isc_socket_t *	 sock;
-	isc_task_t *	 task;
-	isc_taskaction_t action;
-	void *		 arg;
-	isc_event_t	 event;
+	unsigned int	magic;
+	uint32_t	size;
+	isc_buffer_t *	buffer;
+	unsigned int	maxsize;
+	isc_mem_t *	mctx;
+	isc_nmhandle_t *handle;
+	isc_nm_cb_t	cb;
+	void *		cbarg;
 	/* public (read-only) */
-	isc_result_t   result;
-	isc_sockaddr_t address;
+	isc_result_t result;
 } isccc_ccmsg_t;

 ISC_LANG_BEGINDECLS

 void
-isccc_ccmsg_init(isc_mem_t *mctx, isc_socket_t *sock, isccc_ccmsg_t *ccmsg);
+isccc_ccmsg_init(isc_mem_t *mctx, isc_nmhandle_t *handle, isccc_ccmsg_t *ccmsg);
 /*%
 * Associate a cc message state with a given memory context and
- * TCP socket.
+ * netmgr handle.
 *
 * Requires:
 *
- *\li	"mctx" and "sock" be non-NULL and valid types.
+ *\li	"mctx" be a valid memory context.
 *
- *\li	"sock" be a read/write TCP socket.
+ *\li	"handle" be a netmgr handle for a stream socket.
 *
 *\li	"ccmsg" be non-NULL and an uninitialized or invalidated structure.
 *
@@ -86,8 +84,7 @@ isccc_ccmsg_setmaxsize(isccc_ccmsg_t *ccmsg, unsigned int maxsize);
 */

 isc_result_t
-isccc_ccmsg_readmessage(isccc_ccmsg_t *ccmsg, isc_task_t *task,
-			isc_taskaction_t action, void *arg);
+isccc_ccmsg_readmessage(isccc_ccmsg_t *ccmsg, isc_nm_cb_t cb, void *cbarg);
 /*%
 * Schedule an event to be delivered when a command channel message is
 * readable, or when an error occurs on the socket.
@@ -96,12 +93,10 @@ isccc_ccmsg_readmessage(isccc_ccmsg_t *ccmsg, isc_task_t *task,
 *
 *\li	"ccmsg" be valid.
 *
- *\li	"task", "taskaction", and "arg" be valid.
- *
 * Returns:
 *
 *\li	#ISC_R_SUCCESS		-- no error
- *\li	Anything that the isc_socket_recv() call can return.  XXXMLG
+ *\li	Anything that the isc_nm_read() call can return.
 *
 * Notes:
 *
@@ -1660,7 +1660,8 @@ ns__client_put_cb(void *client0) {
 * or tcpmsg (TCP case).
 */
 void
-ns__client_request(isc_nmhandle_t *handle, isc_region_t *region, void *arg) {
+ns__client_request(isc_nmhandle_t *handle, isc_result_t eresult,
+		   isc_region_t *region, void *arg) {
 	ns_client_t *client;
 	bool newclient = false;
 	ns_clientmgr_t *mgr;
@@ -1684,6 +1685,8 @@ ns__client_request(isc_nmhandle_t *handle, isc_region_t *region, void *arg) {
 #endif /* ifdef HAVE_DNSTAP */
 	ifp = (ns_interface_t *)arg;

+	UNUSED(eresult);
+
 	mgr = ifp->clientmgr;
 	if (mgr == NULL) {
 		/* The interface was shut down in the meantime, just bail */
@@ -468,7 +468,8 @@ ns_client_addopt(ns_client_t *client, dns_message_t *message,
 */

 void
-ns__client_request(isc_nmhandle_t *handle, isc_region_t *region, void *arg);
+ns__client_request(isc_nmhandle_t *handle, isc_result_t eresult,
+		   isc_region_t *region, void *arg);

 /*%<
 * Handle client requests.
@@ -553,11 +553,11 @@ void
 ns_interface_shutdown(ns_interface_t *ifp) {
 	if (ifp->udplistensocket != NULL) {
 		isc_nm_stoplistening(ifp->udplistensocket);
-		isc_nmsocket_detach(&ifp->udplistensocket);
+		isc_nmsocket_close(&ifp->udplistensocket);
 	}
 	if (ifp->tcplistensocket != NULL) {
 		isc_nm_stoplistening(ifp->tcplistensocket);
-		isc_nmsocket_detach(&ifp->tcplistensocket);
+		isc_nmsocket_close(&ifp->tcplistensocket);
 	}
 	if (ifp->clientmgr != NULL) {
 		ns_clientmgr_destroy(&ifp->clientmgr);
Author	SHA1	Message	Date
Witold Kręcicki	7c86c4fa18	WiP/checkpoint2 - working XFRIN with netmgr	2020-06-15 19:04:28 +02:00
Witold Kręcicki	47df857b98	WiP checkpoint: xfrin using netmgr	2020-06-15 19:02:50 +02:00
Witold Kręcicki	210a1729f9	Rename isc_nmsocket_t->tcphandle to statichandle	2020-06-15 19:01:50 +02:00
Witold Kręcicki	ef00bf8eca	fixup! allocate controlconnection_t as extra data	2020-06-10 10:45:38 +02:00
Evan Hunt	9f6fb12384	CHANGES, release note	2020-06-10 01:19:15 -07:00
Evan Hunt	40e589ada9	allocate controlconnection_t as extra data when a new control channel connection is estabslished, allocate the controlconnection object as data associated with the network manager handle.	2020-06-10 01:19:15 -07:00
Evan Hunt	f2e681e418	use isc_nm_cancelread() in isccc_ccmsg_cancelread() when shuttting down a control channel connection, if no valid message has arrived, cancel the read.	2020-06-09 18:15:53 -07:00
Evan Hunt	1fdaf3465d	implement isc_nm_cancelread() The isc_nm_cancelread() function cancels reading on a connected socket and calls its read callback function with a 'result' parameter of ISC_R_CANCELED.	2020-06-09 18:14:29 -07:00
Evan Hunt	57e66deb4a	use an isc_task to execute rndc commands - separate control_recvmessage() into two functions, control_recvmessage() and control_respond(). the respond function can be called directly from control_recvmessage() when processing a nonce, or indirectly after returning from named_control_docommand().	2020-06-09 18:14:29 -07:00
Evan Hunt	0174352352	don't use exclusive mode for rndc commands that don't need it "showzone" and "tsig-list" both used exclusive mode unnecessarily.	2020-06-09 18:14:29 -07:00
Evan Hunt	e8e86ddbbb	convert rndc and control channel to use netmgr - updated libisccc to use netmgr events - updated rndc to use isc_nm_tcpconnect() to establish connections - updated control channel to use isc_nm_listentcp() open issues: - the control channel timeout was previously 60 seconds, but it is now overridden by the TCP idle timeout setting, which defaults to 30 seconds. we should add a function that sets the timeout value for a specific listener socket, instead of always using the global value set in the netmgr. (for the moment, since 30 seconds is a reasonable timeout for the control channel, I'm not prioritizing this.) - the netmgr currently has no support for UNIX-domain sockets; until this is addressed, it will not be possible to configure rndc to use them. we will need to either fix this or document the change in behavior.	2020-06-09 18:13:19 -07:00
Evan Hunt	d1dfcadb4d	change the signature of recv callbacks to include a result code this will allow recv event handlers to distinguish between cases in which the region is NULL because of error, shutdown, or cancelation.	2020-06-09 18:13:19 -07:00
Evan Hunt	d366c8b148	shorten the sleep in isc_nm_destroy() when isc_nm_destroy() is called, there's a loop that waits for other references to be detached, pausing and unpausing the netmgr to ensure that all the workers' events are run, followed by a 1-second sleep. this caused a noticeable delay when shutting down rndc; so the delay has now been reduced to a hundredth of a second.	2020-06-09 18:13:19 -07:00
Evan Hunt	3c6dcbb841	implement isc_nm_tcpconnect() the isc_nm_tcpconnect() function establishes a client connection via TCP. once the connection is esablished, a callback function will be called with a newly created network manager handle.	2020-06-09 18:11:29 -07:00
Evan Hunt	15060430f0	style cleanup clean up style in rndc and the control channel in preparation for changing them to use the new network manager.	2020-06-09 18:10:32 -07:00
Witold KręcickiandEvan Hunt	28b8ac016b	allow tcpdns sockets to self-reference while connected A TCPDNS socket creates a handle for each complete DNS message. Previously, when all the handles were disconnected, the socket would be closed, but the wrapped TCP socket might still have more to read. Now, when a connection is established, the TCPDNS socket creates a reference to itself by attaching itself to sock->self. This reference isn't cleared until the connection is closed via EOF, timeout, or server shutdown. This allows the socket to remain open even when there are no active handles for it.	2020-06-09 17:55:44 -07:00
Evan Hunt	402f9d75c7	modify reference counting within netmgr - isc__nmhandle_get() now attaches to the sock in the nmhandle object. the caller is responsible for dereferencing the original socket pointer when necessary. - tcpdns listener sockets attach sock->outer to the outer tcp listener socket. tcpdns connected sockets attach sock->outerhandle to the handle for the tcp connected socket. - only listener sockets need to be attached/detached directly. connected sockets should only be accessed and reference-counted via their associated handles.	2020-06-09 17:55:16 -07:00
Evan Hunt	6f5812831a	make isc_nmsocket_{attach,detach}{} functions private there is no need for a caller to reference-count socket objects. they need tto be able tto close listener sockets (i.e., those returned by isc_nm_listen{udp,tcp,tcpdns}), and an isc_nmsocket_close() function has been added for that. other sockets are only accessed via handles.	2020-06-09 17:06:00 -07:00