fixup! checkpoint: convert dig/host/nslookup to use the netmgr

fixup! add netmgr functions to support outgoing DNS queries
2020-09-06 21:01:05 -07:00 · 2020-09-06 18:34:42 -07:00 · 2020-09-06 18:34:25 -07:00 · 2020-09-06 17:29:21 -07:00 · 2020-09-06 17:14:43 -07:00 · 2020-09-06 14:39:11 -07:00
36 changed files with 1492 additions and 814 deletions
@@ -1,3 +1,7 @@
+5503.	[bug]		Cleaned up reference counting of network manager
+			handles, now using isc_nmhandle_attach() and _detach()
+			instead of _ref() and _unref(). [GL #2122]
+
 5502.	[func]		'dig +bufsize=0' no longer disables EDNS. [GL #2054]

 5501.	[func]		Log CDS/CDNSKEY publication. [GL #1748]
@@ -642,7 +642,6 @@ printmessage(dig_query_t *query, const isc_buffer_t *msgbuf, dns_message_t *msg,
 	if (yaml) {
 		enum { Q = 0x1, R = 0x2 }; /* Q:query; R:ecursive */
 		unsigned int tflag = 0;
-		isc_sockaddr_t saddr;
 		char sockstr[ISC_SOCKADDR_FORMATSIZE];
 		uint16_t sport;
 		char *hash;
@@ -713,10 +712,9 @@ printmessage(dig_query_t *query, const isc_buffer_t *msgbuf, dns_message_t *msg,
 			printf("    response_port: %u\n", sport);
 		}

-		if (query->sock != NULL &&
-		    isc_socket_getsockname(query->sock, &saddr) ==
-			    ISC_R_SUCCESS)
-		{
+		if (query->handle != NULL) {
+			isc_sockaddr_t saddr =
+				isc_nmhandle_localaddr(query->handle);
 			sport = isc_sockaddr_getport(&saddr);
 			isc_sockaddr_format(&saddr, sockstr, sizeof(sockstr));
 			hash = strchr(sockstr, '#');
@@ -1969,10 +1967,10 @@ dash_option(char *option, char *next, dig_lookup_t **lookup,
 			srcport = 0;
 		}
 		if (have_ipv6 && inet_pton(AF_INET6, value, &in6) == 1) {
-			isc_sockaddr_fromin6(&bind_address, &in6, srcport);
+			isc_sockaddr_fromin6(&localaddr, &in6, srcport);
 			isc_net_disableipv4();
 		} else if (have_ipv4 && inet_pton(AF_INET, value, &in4) == 1) {
-			isc_sockaddr_fromin(&bind_address, &in4, srcport);
+			isc_sockaddr_fromin(&localaddr, &in4, srcport);
 			isc_net_disableipv6();
 		} else {
 			if (hash != NULL) {
@@ -172,7 +172,7 @@ struct dig_query {
 	unsigned int magic;
 	dig_lookup_t *lookup;
 	bool waiting_connect, pending_free, waiting_senddone, first_pass,
-		first_soa_rcvd, second_rr_rcvd, first_repeat_rcvd, recv_made,
+		first_soa_rcvd, second_rr_rcvd, first_repeat_rcvd,
 		warn_id, timedout;
 	uint32_t first_rr_serial;
 	uint32_t second_rr_serial;
@@ -181,9 +181,11 @@ struct dig_query {
 	bool ixfr_axfr;
 	char *servname;
 	char *userarg;
-	isc_buffer_t recvbuf, lengthbuf, tmpsendbuf, sendbuf;
+	isc_buffer_t sendbuf;
 	char *recvspace, *tmpsendspace, lengthspace[4];
-	isc_socket_t *sock;
+	isc_nmhandle_t *handle;
+	isc_nmhandle_t *readhandle;
+	isc_nmhandle_t *sendhandle;
 	ISC_LINK(dig_query_t) link;
 	ISC_LINK(dig_query_t) clink;
 	isc_sockaddr_t sockaddr;
@@ -225,7 +227,7 @@ extern int sendcount;
 extern int ndots;
 extern int lookup_counter;
 extern int exitcode;
-extern isc_sockaddr_t bind_address;
+extern isc_sockaddr_t localaddr;
 extern char keynametext[MXNAME];
 extern char keyfile[MXNAME];
 extern char keysecret[MXNAME];
@@ -915,13 +915,10 @@ flush_lookup_list(void) {
 	while (l != NULL) {
 		q = ISC_LIST_HEAD(l->q);
 		while (q != NULL) {
-			if (q->sock != NULL) {
-				isc_socket_cancel(q->sock, NULL,
-						  ISC_SOCKCANCEL_ALL);
-				isc_socket_detach(&q->sock);
+			if (q->handle != NULL) {
+				isc_nm_cancelread(q->handle);
+				isc_nmhandle_detach(&q->handle);
 			}
-			isc_buffer_invalidate(&q->recvbuf);
-			isc_buffer_invalidate(&q->lengthbuf);
 			qp = q;
 			q = ISC_LIST_NEXT(q, link);
 			ISC_LIST_DEQUEUE(l->q, qp, link);
@@ -68,8 +68,11 @@ struct controlkey {

 struct controlconnection {
 	isc_nmhandle_t *handle;
+	isc_nmhandle_t *readhandle;
+	isc_nmhandle_t *sendhandle;
+	isc_nmhandle_t *cmdhandle;
 	isccc_ccmsg_t ccmsg;
-	bool ccmsg_valid;
+	bool reading;
 	bool sending;
 	controllistener_t *listener;
 	isccc_sexpr_t *ctrl;
@@ -221,10 +224,8 @@ control_senddone(isc_nmhandle_t *handle, isc_result_t result, void *arg) {

 	conn->sending = false;

-	isc_nmhandle_unref(handle);
-
-	if (result == ISC_R_CANCELED) {
-		return;
+	if (listener->controls->shuttingdown || result == ISC_R_CANCELED) {
+		goto cleanup;
 	} else if (result != ISC_R_SUCCESS) {
 		char socktext[ISC_SOCKADDR_FORMATSIZE];

@@ -233,16 +234,24 @@ control_senddone(isc_nmhandle_t *handle, isc_result_t result, void *arg) {
 			      NAMED_LOGMODULE_CONTROL, ISC_LOG_WARNING,
 			      "error sending command response to %s: %s",
 			      socktext, isc_result_totext(result));
-		return;
+		goto cleanup;
 	}

+	if (conn->readhandle == NULL) {
+		isc_nmhandle_attach(handle, &conn->readhandle);
+	}
+	conn->reading = true;
 	result = isccc_ccmsg_readmessage(&conn->ccmsg, control_recvmessage,
 					 conn);
 	if (result != ISC_R_SUCCESS) {
+		isc_nmhandle_detach(&conn->readhandle);
 		maybe_free_listener(listener);
 	}

 	listener->listening = true;
+
+cleanup:
+	isc_nmhandle_detach(&conn->sendhandle);
 }

 static inline void
@@ -257,6 +266,25 @@ log_invalid(isccc_ccmsg_t *ccmsg, isc_result_t result) {
 		      isc_result_totext(result));
 }

+static void
+conn_cleanup(controlconnection_t *conn) {
+	controllistener_t *listener = conn->listener;
+
+	if (conn->response != NULL) {
+		isccc_sexpr_free(&conn->response);
+	}
+	if (conn->request != NULL) {
+		isccc_sexpr_free(&conn->request);
+	}
+	if (conn->secret.rstart != NULL) {
+		isc_mem_put(listener->mctx, conn->secret.rstart,
+			    REGION_SIZE(conn->secret));
+	}
+	if (conn->text != NULL) {
+		isc_buffer_free(&conn->text);
+	}
+}
+
 static void
 control_respond(isc_nmhandle_t *handle, isc_result_t result, void *arg) {
 	controlconnection_t *conn = (controlconnection_t *)arg;
@@ -324,29 +352,16 @@ control_respond(isc_nmhandle_t *handle, isc_result_t result, void *arg) {
 	r.base = conn->buffer->base;
 	r.length = conn->buffer->used;

-	isc_nmhandle_ref(handle);
+	isc_nmhandle_attach(handle, &conn->sendhandle);
 	conn->sending = true;
-	result = isc_nm_send(handle, &r, control_senddone, conn);
+	result = isc_nm_send(conn->sendhandle, &r, control_senddone, conn);
 	if (result != ISC_R_SUCCESS) {
-		isc_nmhandle_unref(handle);
+		isc_nmhandle_detach(&conn->sendhandle);
 		conn->sending = false;
 	}

 cleanup:
-	if (conn->response != NULL) {
-		isccc_sexpr_free(&conn->response);
-	}
-	if (conn->request != NULL) {
-		isccc_sexpr_free(&conn->request);
-	}
-
-	if (conn->secret.rstart != NULL) {
-		isc_mem_put(listener->mctx, conn->secret.rstart,
-			    REGION_SIZE(conn->secret));
-	}
-	if (conn->text != NULL) {
-		isc_buffer_free(&conn->text);
-	}
+	conn_cleanup(conn);
 }

 static void
@@ -356,18 +371,20 @@ control_command(isc_task_t *task, isc_event_t *event) {

 	UNUSED(task);

-	/*
-	 * An extra ref and two unrefs are needed here to
-	 * ensure the handle isn't cleaned up if we're running
-	 * an "rndc stop" command.
-	 */
-	isc_nmhandle_ref(conn->handle);
+	if (listener->controls->shuttingdown) {
+		conn_cleanup(conn);
+		goto done;
+	}
+
 	conn->result = named_control_docommand(conn->request,
 					       listener->readonly, &conn->text);
 	control_respond(conn->handle, conn->result, conn);
-	isc_nmhandle_unref(conn->handle);
-	isc_nmhandle_unref(conn->handle);
+
+done:
 	isc_event_free(&event);
+	if (conn->cmdhandle != NULL) {
+		isc_nmhandle_detach(&conn->cmdhandle);
+	}
 }

 static void
@@ -380,26 +397,21 @@ control_recvmessage(isc_nmhandle_t *handle, isc_result_t result, void *arg) {
 	isccc_time_t exp;
 	uint32_t nonce;

-	conn->ccmsg_valid = false;
+	conn->reading = false;

 	/* Is the server shutting down? */
 	if (listener->controls->shuttingdown) {
-		return;
+		goto cleanup_readhandle;
 	}

 	if (result != ISC_R_SUCCESS) {
 		if (result == ISC_R_CANCELED) {
-			/*
-			 * Don't bother with any more scheduled command events.
-			 */
 			listener->controls->shuttingdown = true;
-			isc_task_purge(named_g_server->task, NULL,
-				       NAMED_EVENT_COMMAND, NULL);
 		} else if (result != ISC_R_EOF) {
 			log_invalid(&conn->ccmsg, result);
 		}

-		return;
+		goto cleanup_readhandle;
 	}

 	for (key = ISC_LIST_HEAD(listener->keys); key != NULL;
@@ -424,7 +436,7 @@ control_recvmessage(isc_nmhandle_t *handle, isc_result_t result, void *arg) {
 			    REGION_SIZE(conn->secret));
 		if (result != ISCCC_R_BADAUTH) {
 			log_invalid(&conn->ccmsg, result);
-			return;
+			goto cleanup;
 		}
 	}

@@ -496,8 +508,6 @@ control_recvmessage(isc_nmhandle_t *handle, isc_result_t result, void *arg) {

 	isc_buffer_allocate(listener->mctx, &conn->text, 2 * 2048);

-	conn->ccmsg_valid = true;
-
 	if (conn->nonce == 0) {
 		/*
 		 * Establish nonce.
@@ -513,26 +523,19 @@ control_recvmessage(isc_nmhandle_t *handle, isc_result_t result, void *arg) {
 	/*
 	 * Trigger the command.
 	 */
-	isc_nmhandle_ref(handle);
+	isc_nmhandle_attach(handle, &conn->cmdhandle);
 	event = isc_event_allocate(listener->mctx, conn, NAMED_EVENT_COMMAND,
 				   control_command, conn, sizeof(isc_event_t));
 	isc_task_send(named_g_server->task, &event);
+	isc_nmhandle_detach(&conn->readhandle);
 	return;

 cleanup:
-	if (conn->response != NULL) {
-		isccc_sexpr_free(&conn->response);
-	}
-	if (conn->request != NULL) {
-		isccc_sexpr_free(&conn->request);
-	}
+	conn_cleanup(conn);

-	if (conn->secret.rstart != NULL) {
-		isc_mem_put(listener->mctx, conn->secret.rstart,
-			    REGION_SIZE(conn->secret));
-	}
-	if (conn->text != NULL) {
-		isc_buffer_free(&conn->text);
+cleanup_readhandle:
+	if (conn->readhandle != NULL) {
+		isc_nmhandle_detach(&conn->readhandle);
 	}
 }

@@ -545,7 +548,7 @@ conn_reset(void *arg) {
 		isc_buffer_free(&conn->buffer);
 	}

-	if (conn->ccmsg_valid) {
+	if (conn->reading) {
 		isccc_ccmsg_cancelread(&conn->ccmsg);
 		return;
 	}
@@ -594,7 +597,7 @@ newconnection(controllistener_t *listener, isc_nmhandle_t *handle) {

 	*conn = (controlconnection_t){ .handle = handle,
 				       .listener = listener,
-				       .ccmsg_valid = true,
+				       .reading = false,
 				       .alg = DST_ALG_UNKNOWN };

 	isccc_ccmsg_init(listener->mctx, handle, &conn->ccmsg);
@@ -604,9 +607,14 @@ newconnection(controllistener_t *listener, isc_nmhandle_t *handle) {

 	ISC_LINK_INIT(conn, link);

+	if (conn->readhandle == NULL) {
+		isc_nmhandle_attach(handle, &conn->readhandle);
+	}
+	conn->reading = true;
 	result = isccc_ccmsg_readmessage(&conn->ccmsg, control_recvmessage,
 					 conn);
 	if (result != ISC_R_SUCCESS) {
+		isc_nmhandle_detach(&conn->readhandle);
 		goto cleanup;
 	}

@@ -14865,14 +14865,15 @@ named_server_zonestatus(named_server_t *server, isc_lex_t *lex,

 	/* Serial number */
 	result = dns_zone_getserial(mayberaw, &serial);
-	/* XXXWPK TODO this is to mirror old behavior with dns_zone_getserial */
+
+	/* This is to mirror old behavior with dns_zone_getserial */
 	if (result != ISC_R_SUCCESS) {
 		serial = 0;
 	}
+
 	snprintf(serbuf, sizeof(serbuf), "%u", serial);
 	if (hasraw) {
 		result = dns_zone_getserial(zone, &signed_serial);
-		/* XXXWPK TODO ut supra */
 		if (result != ISC_R_SUCCESS) {
 			serial = 0;
 		}
@@ -24,6 +24,7 @@
 #include <isc/log.h>
 #include <isc/mem.h>
 #include <isc/net.h>
+#include <isc/netmgr.h>
 #include <isc/print.h>
 #include <isc/random.h>
 #include <isc/refcount.h>
@@ -89,6 +90,7 @@ static uint32_t serial;
 static bool quiet = false;
 static bool showresult = false;
 static bool shuttingdown = false;
+static isc_nmhandle_t *readhandle = NULL, *sendhandle = NULL;

 static void
 rndc_startconnect(isc_sockaddr_t *addr);
@@ -288,19 +290,21 @@ get_addresses(const char *host, in_port_t port) {

 static void
 rndc_senddone(isc_nmhandle_t *handle, isc_result_t result, void *arg) {
+	REQUIRE(handle == sendhandle);
+
 	UNUSED(arg);

 	if (result != ISC_R_SUCCESS) {
 		fatal("send failed: %s", isc_result_totext(result));
 	}

+	isc_nmhandle_detach(&sendhandle);
 	if (atomic_fetch_sub_release(&sends, 1) == 1 &&
 	    atomic_load_acquire(&recvs) == 0)
 	{
 		shuttingdown = true;
 		isc_task_shutdown(rndc_task);
 		isc_app_shutdown();
-		isc_nmhandle_unref(handle);
 	}
 }

@@ -318,7 +322,10 @@ rndc_recvdone(isc_nmhandle_t *handle, isc_result_t result, void *arg) {
 	atomic_fetch_sub_release(&recvs, 1);

 	if (shuttingdown && (result == ISC_R_EOF || result == ISC_R_CANCELED)) {
-		isc_nmhandle_unref(handle);
+		if (readhandle != NULL) {
+			INSIST(handle == readhandle);
+			isc_nmhandle_detach(&readhandle);
+		}
 		return;
 	} else if (result == ISC_R_EOF) {
 		fatal("connection to remote host closed.\n"
@@ -378,8 +385,8 @@ rndc_recvdone(isc_nmhandle_t *handle, isc_result_t result, void *arg) {

 	if (atomic_load_acquire(&sends) == 0 &&
 	    atomic_load_acquire(&recvs) == 0) {
+		isc_nmhandle_detach(&readhandle);
 		shuttingdown = true;
-		isc_nmhandle_unref(handle);
 		isc_task_shutdown(rndc_task);
 		isc_app_shutdown();
 	}
@@ -398,12 +405,13 @@ rndc_recvnonce(isc_nmhandle_t *handle, isc_result_t result, void *arg) {
 	isccc_sexpr_t *data;
 	isc_buffer_t b;

+	REQUIRE(handle == readhandle);
 	REQUIRE(ccmsg != NULL);

 	atomic_fetch_sub_release(&recvs, 1);

 	if (shuttingdown && result == ISC_R_EOF) {
-		isc_nmhandle_unref(handle);
+		isc_nmhandle_detach(&readhandle);
 		return;
 	} else if (result == ISC_R_EOF) {
 		fatal("connection to remote host closed.\n"
@@ -471,6 +479,9 @@ rndc_recvnonce(isc_nmhandle_t *handle, isc_result_t result, void *arg) {
 	   isccc_ccmsg_readmessage(ccmsg, rndc_recvdone, ccmsg));
 	atomic_fetch_add_relaxed(&recvs, 1);

+	if (sendhandle == NULL) {
+		isc_nmhandle_attach(handle, &sendhandle);
+	}
 	DO("send message", isc_nm_send(handle, &r, rndc_senddone, NULL));
 	atomic_fetch_add_relaxed(&sends, 1);

@@ -534,12 +545,16 @@ rndc_connected(isc_nmhandle_t *handle, isc_result_t result, void *arg) {
 	isccc_ccmsg_init(rndc_mctx, handle, ccmsg);
 	isccc_ccmsg_setmaxsize(ccmsg, 1024 * 1024);

-	isc_nmhandle_ref(handle);
-
+	if (readhandle == NULL) {
+		isc_nmhandle_attach(handle, &readhandle);
+	}
 	DO("schedule recv",
 	   isccc_ccmsg_readmessage(ccmsg, rndc_recvnonce, ccmsg));
 	atomic_fetch_add_relaxed(&recvs, 1);

+	if (sendhandle == NULL) {
+		isc_nmhandle_attach(handle, &sendhandle);
+	}
 	DO("send message", isc_nm_send(handle, &r, rndc_senddone, NULL));
 	atomic_fetch_add_relaxed(&sends, 1);

@@ -1009,7 +1024,7 @@ main(int argc, char **argv) {
 	isc_mem_create(&rndc_mctx);
 	netmgr = isc_nm_start(rndc_mctx, 1);
 	DO("create task manager",
-	   isc_taskmgr_create(rndc_mctx, 1, 0, NULL, &taskmgr));
+	   isc_taskmgr_create(rndc_mctx, 1, 0, netmgr, &taskmgr));
 	DO("create task", isc_task_create(taskmgr, 0, &rndc_task));
 	isc_log_create(rndc_mctx, &log, &logconfig);
 	isc_log_setcontext(log);
@@ -82,7 +82,7 @@ if [ -x "$NSLOOKUP" -a $checkupdate -eq 1 ] ; then
  n=$((n+1))
  echo_i "check nslookup handles UPDATE response ($n)"
  ret=0
-  "$NSLOOKUP" -q=CNAME "-port=$PORT" foo.bar 10.53.0.7 > nslookup.out.test$n 2>&1 && ret=1
+  "$NSLOOKUP" -q=CNAME -timeout=1 "-port=$PORT" foo.bar 10.53.0.7 > nslookup.out.test$n 2>&1 && ret=1
  grep "Opcode mismatch" nslookup.out.test$n > /dev/null || ret=1
  if [ $ret -ne 0 ]; then echo_i "failed"; fi
  status=$((status+ret))
@@ -94,7 +94,7 @@ if [ -x "$HOST" -a $checkupdate -eq 1 ] ; then
  n=$((n+1))
  echo_i "check host handles UPDATE response ($n)"
  ret=0
-  "$HOST" -t CNAME -p $PORT foo.bar 10.53.0.7 > host.out.test$n 2>&1 && ret=1
+  "$HOST" -W 1 -t CNAME -p $PORT foo.bar 10.53.0.7 > host.out.test$n 2>&1 && ret=1
  grep "Opcode mismatch" host.out.test$n > /dev/null || ret=1
  if [ $ret -ne 0 ]; then echo_i "failed"; fi
  status=$((status+ret))
@@ -108,7 +108,7 @@ if [ -x "$DIG" ] ; then
    n=$((n+1))
    echo_i "check dig handles UPDATE response ($n)"
    ret=0
-    dig_with_opts @10.53.0.7 cname foo.bar > dig.out.test$n 2>&1 && ret=1
+    dig_with_opts @10.53.0.7 +tries=1 +timeout=1 cname foo.bar > dig.out.test$n 2>&1 && ret=1
    grep "Opcode mismatch" dig.out.test$n > /dev/null || ret=1
    if [ $ret -ne 0 ]; then echo_i "failed"; fi
    status=$((status+ret))
@@ -922,7 +922,7 @@ if [ -x "$DIG" ] ; then
  n=$((n+1))
  echo_i "check that dig +unexpected works ($n)"
  ret=0
-  dig_with_opts @10.53.0.6 +unexpected a a.example > dig.out.test$n || ret=1
+  dig_with_opts @10.53.0.6 +tries=1 +time=2 +unexpected a a.example > dig.out.test$n || ret=1
  grep 'reply from unexpected source' dig.out.test$n > /dev/null || ret=1
  grep 'status: NOERROR' dig.out.test$n > /dev/null || ret=1
  if [ $ret -ne 0 ]; then echo_i "failed"; fi
@@ -55,6 +55,44 @@ sub usage {
    exit 1;
 }

+my $sock;
+my $proto;
+
+sub dumppacket {
+    use Net::DNS;
+    use Net::DNS::Packet;
+
+    my $rin;
+    my $rout;
+    $rin = '';
+    vec($rin, fileno($sock), 1) = 1;
+    select($rout = $rin, undef, undef, 1);
+    if (vec($rout, fileno($sock), 1)) {
+        my $buf;
+
+        if ($proto eq "udp") {
+            $sock->recv($buf, 512);
+        } else {
+            my $n = $sock->sysread($buf, 2);
+            return unless $n == 2;
+            my $len = unpack("n", $buf);
+            $n = $sock->sysread($buf, $len);
+            return unless $n == $len;
+        }
+
+        my $response;
+        if ($Net::DNS::VERSION > 0.68) {
+            $response = new Net::DNS::Packet(\$buf, 0);
+            $@ and die $@;
+        } else {
+            my $err;
+            ($response, $err) = new Net::DNS::Packet(\$buf, 0);
+            $err and die $err;
+        }
+        $response->print;
+    }
+}
+
 my %options={};
 getopts("a:dp:t:r:", \%options);

@@ -64,7 +102,7 @@ $addr = $options{a} if defined $options{a};
 my $port = 53;
 $port = $options{p} if defined $options{p};

-my $proto = "udp";
+$proto = "udp";
 $proto = lc $options{t} if defined $options{t};
 usage if ($proto !~ /^(udp|tcp)$/);

@@ -92,7 +130,7 @@ my $len = length $data;
 my $output = unpack("H*", $data);
 print ("sending $repeats time(s): $output\n");

-my $sock = IO::Socket::INET->new(PeerAddr => $addr, PeerPort => $port,
+$sock = IO::Socket::INET->new(PeerAddr => $addr, PeerPort => $port,
 				 Blocking => 0,
 				 Proto => $proto,) or die "$!";

@@ -115,40 +153,8 @@ while ($repeats > 0) {
 }

 $sock->shutdown(SHUT_WR);
-
-my $rin;
-my $rout;
-$rin = '';
-vec($rin, fileno($sock), 1) = 1;
-select($rout = $rin, undef, undef, 1);
-if (vec($rout, fileno($sock), 1)) {
-    my $buf;
-
-    if ($proto eq "udp") {
-	$sock->recv($buf, 512);
-    } else {
-	my $n = $sock->sysread($buf, 2);
-	last unless $n == 2;
-	my $len = unpack("n", $buf);
-	$n = $sock->sysread($buf, $len);
-	last unless $n == $len;
-    }
-
-    if (defined $options{d}) {
-	use Net::DNS;
-	use Net::DNS::Packet;
-
-	my $response;
-	if ($Net::DNS::VERSION > 0.68) {
-	    $response = new Net::DNS::Packet(\$buf, 0);
-	    $@ and die $@;
-	} else {
-	    my $err;
-	    ($response, $err) = new Net::DNS::Packet(\$buf, 0);
-	    $err and die $err;
-	}
-	$response->print;
-    }
+if (defined $options{d}) {
+    dumppacket;
 }

 $sock->close;
@@ -26,12 +26,10 @@ $DIFF ref output > /dev/null && { ret=1 ; echo_i "diff out of order failed"; }
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`

-# flush resolver so queries will be from others again
-$RNDCCMD 10.53.0.4 flush
-sleep 1
-
 echo_i "check pipelined TCP queries using mdig"
 ret=0
+$RNDCCMD 10.53.0.4 flush
+sleep 1
 $MDIG $MDIGOPTS +noall +answer +vc -f input -b 10.53.0.4 @10.53.0.4 > raw.mdig
 awk '{ print $1 " " $5 }' < raw.mdig > output.mdig
 sort < output.mdig > output-sorted.mdig
@@ -42,6 +40,8 @@ status=`expr $status + $ret`

 echo_i "check keep-response-order"
 ret=0
+$RNDCCMD 10.53.0.4 flush
+sleep 1
 $PIPEQUERIES -p ${PORT} ++ < inputb > rawb || ret=1
 awk '{ print $1 " " $5 }' < rawb > outputb
 $DIFF refb outputb || ret=1
@@ -50,6 +50,8 @@ status=`expr $status + $ret`

 echo_i "check keep-response-order using mdig"
 ret=0
+$RNDCCMD 10.53.0.4 flush
+sleep 1
 $MDIG $MDIGOPTS +noall +answer +vc -f inputb -b 10.53.0.7 @10.53.0.4 > rawb.mdig
 awk '{ print $1 " " $5 }' < rawb.mdig > outputb.mdig
 $DIFF refb outputb.mdig || ret=1
@@ -58,6 +60,8 @@ status=`expr $status + $ret`

 echo_i "check mdig -4 -6"
 ret=0
+$RNDCCMD 10.53.0.4 flush
+sleep 1
 $MDIG $MDIGOPTS -4 -6 -f input @10.53.0.4 > output46.mdig 2>&1 && ret=1
 grep "only one of -4 and -6 allowed" output46.mdig > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
@@ -15,6 +15,7 @@ options {
 	listen-on { 10.53.0.4; };
 	listen-on-v6 { none; };
 	recursion yes;
+    dnssec-validation yes;
 };

 view normal {
@@ -8,11 +8,10 @@
 # information regarding copyright ownership.

 rm -f ns*/*.jnl
-rm -f ns*/named.conf
 rm -f ns*/named.lock
 rm -f ns*/named.memstats
-rm -f ns*/named.run
 rm -f ns*/rpz*.txt
-rm -f resolver/named.conf
+rm -f */named.conf
+rm -f */named.run
 rm -rf __pycache__
 rm -f *.status
@@ -1,3 +1,11 @@
+logging {
+	channel basic {
+		file "named.run";
+		severity debug 999;
+		print-time yes;
+	};
+	category default { basic; };
+};
 key rndc_key {
    secret "1234abcd8765";
 	algorithm hmac-sha256;
@@ -1320,16 +1320,16 @@ AM_CONDITIONAL([HAVE_CMOCKA], [test "$with_cmocka" = "yes"])
 LD_WRAP_TESTS=false
 AC_MSG_CHECKING([for linker support for --wrap option])
 AX_SAVE_FLAGS([wrap])
-LDFLAGS="-Wl,-wrap,exit"
+LDFLAGS="-Wl,--wrap,exit"
 AC_RUN_IFELSE(
    [AC_LANG_PROGRAM([[#include <stdlib.h>
 		       void __real_exit (int status);
 		       void __wrap_exit (int status) { __real_exit (status); }
 		      ]],
 		     [[exit (1);]])],
+    [AC_MSG_RESULT([no])],
    [LD_WRAP_TESTS=true
-     AC_MSG_RESULT([yes])],
-    [AC_MSG_RESULT([no])])
+     AC_MSG_RESULT([yes])])
 AX_RESTORE_FLAGS([wrap])

 AM_CONDITIONAL([HAVE_LD_WRAP], [test "$LD_WRAP_TESTS" = "true"])
@@ -75,7 +75,10 @@ struct isc_httpd {
 	isc_httpdmgr_t *mgr; /*%< our parent */
 	ISC_LINK(isc_httpd_t) link;

-	isc_nmhandle_t *handle;
+	isc_nmhandle_t *handle;	    /* Permanent pointer to handle */
+	isc_nmhandle_t *readhandle; /* Waiting for a read callback */
+	isc_nmhandle_t *sendhandle; /* Waiting for a send callback */
+
 	state_t state;
 	int flags;

@@ -636,6 +639,7 @@ new_httpd(isc_httpdmgr_t *httpdmgr, isc_nmhandle_t *handle) {

 	if (httpd->handle == NULL) {
 		isc_nmhandle_setdata(handle, httpd, httpd_reset, httpd_put);
+		httpd->handle = handle;
 	} else {
 		INSIST(httpd->handle == handle);
 	}
@@ -663,10 +667,10 @@ new_httpd(isc_httpdmgr_t *httpdmgr, isc_nmhandle_t *handle) {
 	ISC_LIST_APPEND(httpdmgr->running, httpd, link);
 	UNLOCK(&httpdmgr->lock);

-	isc_nmhandle_ref(handle);
+	isc_nmhandle_attach(handle, &httpd->readhandle);
 	result = isc_nm_read(handle, httpd_request, httpdmgr);
 	if (result != ISC_R_SUCCESS) {
-		isc_nmhandle_unref(handle);
+		isc_nmhandle_detach(&httpd->readhandle);
 	}

 	return (result);
@@ -958,18 +962,18 @@ httpd_request(isc_nmhandle_t *handle, isc_result_t eresult,
 	isc_nm_pauseread(handle);
 	httpd->state = SEND;

-	isc_nmhandle_ref(handle);
+	isc_nmhandle_attach(handle, &httpd->sendhandle);
 	result = isc_nm_send(handle, &r, httpd_senddone, httpd);
 	if (result != ISC_R_SUCCESS) {
 		isc_nm_resumeread(handle);
-		isc_nmhandle_unref(handle);
+		isc_nmhandle_detach(&httpd->sendhandle);

 		httpd->state = RECV;
 	}

 	return;
 done:
-	isc_nmhandle_unref(handle);
+	isc_nmhandle_detach(&httpd->readhandle);
 }

 void
@@ -990,7 +994,7 @@ isc_httpdmgr_shutdown(isc_httpdmgr_t **httpdmgrp) {

 	httpd = ISC_LIST_HEAD(httpdmgr->running);
 	while (httpd != NULL) {
-		isc_nmhandle_unref(httpd->handle);
+		isc_nmhandle_detach(&httpd->readhandle);
 		httpd = ISC_LIST_NEXT(httpd, link);
 	}
 	UNLOCK(&httpdmgr->lock);
@@ -1139,7 +1143,7 @@ httpd_senddone(isc_nmhandle_t *handle, isc_result_t result, void *arg) {

 	httpd->state = RECV;
 	isc_nm_resumeread(handle);
-	isc_nmhandle_unref(handle);
+	isc_nmhandle_detach(&httpd->sendhandle);
 }

 isc_result_t
@@ -110,9 +110,9 @@ isc_nmsocket_close(isc_nmsocket_t **sockp);
 */

 void
-isc_nmhandle_ref(isc_nmhandle_t *handle);
+isc_nmhandle_attach(isc_nmhandle_t *handle, isc_nmhandle_t **dest);
 void
-isc_nmhandle_unref(isc_nmhandle_t *handle);
+isc_nmhandle_detach(isc_nmhandle_t **handlep);
 /*%<
 * Increment/decrement the reference counter in a netmgr handle,
 * but (unlike the attach/detach functions) do not change the pointer
@@ -176,8 +176,21 @@ isc_nm_listenudp(isc_nm_t *mgr, isc_nmiface_t *iface, isc_nm_recv_cb_t cb,
 * as its argument.
 *
 * When handles are allocated for the socket, 'extrasize' additional bytes
- * will be allocated along with the handle for an associated object
- * (typically ns_client).
+ * can be allocated along with the handle for an associated object, which
+ * can then be freed automatically when the handle is destroyed.
+ */
+
+isc_result_t
+isc_nm_udpconnect(isc_nm_t *mgr, isc_nmiface_t *local, isc_nmiface_t *peer,
+		  isc_nm_cb_t cb, void *cbarg, size_t extrahandlesize);
+/*%<
+ * Open a UDP socket, bind to 'local' and connect to 'peer', and
+ * immediately call 'cb' with a handle so that the caller can begin
+ * sending packets over UDP.
+ *
+ * When handles are allocated for the socket, 'extrasize' additional bytes
+ * can be allocated along with the handle for an associated object, which
+ * can then be freed automatically when the handle is destroyed.
 */

 void
@@ -196,12 +209,17 @@ isc_nm_pause(isc_nm_t *mgr);
 void
 isc_nm_resume(isc_nm_t *mgr);
 /*%<
- * Resume paused processing. It will return immediately
- * after signalling workers to resume.
+ * Resume paused processing. It will return immediately after signalling
+ * workers to resume.
 */

 isc_result_t
 isc_nm_read(isc_nmhandle_t *handle, isc_nm_recv_cb_t cb, void *cbarg);
+/*
+ * Begin (or continue) reading on the socket associated with 'handle', and
+ * update its recv callback to 'cb', which will be called as soon as there
+ * is data to process.
+ */

 isc_result_t
 isc_nm_pauseread(isc_nmhandle_t *handle);
@@ -385,3 +403,10 @@ isc_nm_setstats(isc_nm_t *mgr, isc_stats_t *stats);
 *\li	stats is a valid set of statistics counters supporting the
 *	full range of socket-related stats counter numbers.
 */
+
+isc_result_t
+isc_nm_tcpdnsconnect(isc_nm_t *mgr, isc_nmiface_t *local, isc_nmiface_t *peer,
+		     isc_nm_cb_t cb, void *cbarg, size_t extrahandlesize);
+/*%
+ * Establish a DNS client connection over a TCP socket.
+ */
@@ -49,6 +49,16 @@
 #define ISC_NETMGR_RECVBUF_SIZE (65536)
 #endif

+/*
+ * Define NETMGR_TRACE to activate tracing of handles and sockets.
+ * This will impair performance but enables us to quickly determine,
+ * if netmgr resources haven't been cleaned up on shutdown, which ones
+ * are still in use.
+ */
+#ifdef NETMGR_TRACE
+#define TRACE_SIZE 8
+#endif
+
 /*
 * Single network event loop worker.
 */
@@ -104,6 +114,11 @@ struct isc_nmhandle {
 	isc_sockaddr_t local;
 	isc_nm_opaquecb_t doreset; /* reset extra callback, external */
 	isc_nm_opaquecb_t dofree;  /* free extra callback, external */
+#ifdef NETMGR_TRACE
+	void *backtrace[TRACE_SIZE];
+	int backtrace_size;
+	LINK(isc_nmhandle_t) active_link;
+#endif
 	void *opaque;
 	char extra[];
 };
@@ -116,13 +131,14 @@ struct isc_nmiface {
 };

 typedef enum isc__netievent_type {
+	netievent_udpconnect,
 	netievent_udpsend,
-	netievent_udprecv,
+	netievent_udpread,
 	netievent_udpstop,
+	netievent_udpclose,

 	netievent_tcpconnect,
 	netievent_tcpsend,
-	netievent_tcprecv,
 	netievent_tcpstartread,
 	netievent_tcppauseread,
 	netievent_tcpchildaccept,
@@ -130,8 +146,9 @@ typedef enum isc__netievent_type {
 	netievent_tcpstop,
 	netievent_tcpclose,

-	netievent_tcpdnsclose,
 	netievent_tcpdnssend,
+	netievent_tcpdnsread,
+	netievent_tcpdnsclose,

 	netievent_closecb,
 	netievent_shutdown,
@@ -146,20 +163,12 @@ typedef enum isc__netievent_type {
 	netievent_tcplisten,
 } isc__netievent_type;

-/*
- * We have to split it because we can read and write on a socket
- * simultaneously.
- */
 typedef union {
 	isc_nm_recv_cb_t recv;
+	isc_nm_cb_t connect;
 	isc_nm_accept_cb_t accept;
 } isc__nm_readcb_t;

-typedef union {
-	isc_nm_cb_t send;
-	isc_nm_cb_t connect;
-} isc__nm_writecb_t;
-
 typedef union {
 	isc_nm_recv_cb_t recv;
 	isc_nm_accept_cb_t accept;
@@ -206,13 +215,16 @@ typedef struct isc__netievent__socket {
 } isc__netievent__socket_t;

 typedef isc__netievent__socket_t isc__netievent_udplisten_t;
+typedef isc__netievent__socket_t isc__netievent_udpread_t;
 typedef isc__netievent__socket_t isc__netievent_udpstop_t;
+typedef isc__netievent__socket_t isc__netievent_udpclose_t;
 typedef isc__netievent__socket_t isc__netievent_tcpstop_t;
 typedef isc__netievent__socket_t isc__netievent_tcpclose_t;
-typedef isc__netievent__socket_t isc__netievent_tcpdnsclose_t;
 typedef isc__netievent__socket_t isc__netievent_startread_t;
 typedef isc__netievent__socket_t isc__netievent_pauseread_t;
 typedef isc__netievent__socket_t isc__netievent_closecb_t;
+typedef isc__netievent__socket_t isc__netievent_tcpdnsclose_t;
+typedef isc__netievent__socket_t isc__netievent_tcpdnsread_t;

 typedef struct isc__netievent__socket_req {
 	isc__netievent_type type;
@@ -256,6 +268,12 @@ typedef struct isc__netievent_udpsend {
 	isc__nm_uvreq_t *req;
 } isc__netievent_udpsend_t;

+typedef struct isc__netievent_udpconnect {
+	isc__netievent_type type;
+	isc_nmsocket_t *sock;
+	isc_sockaddr_t peer;
+} isc__netievent_udpconnect_t;
+
 typedef struct isc__netievent {
 	isc__netievent_type type;
 } isc__netievent_t;
@@ -325,6 +343,10 @@ struct isc_nm {
 	uint32_t idle;
 	uint32_t keepalive;
 	uint32_t advertised;
+
+#ifdef NETMGR_TRACE
+	ISC_LIST(isc_nmsocket_t) active_sockets;
+#endif
 };

 typedef enum isc_nmsocket_type {
@@ -333,7 +355,7 @@ typedef enum isc_nmsocket_type {
 	isc_nm_tcpsocket,
 	isc_nm_tcplistener,
 	isc_nm_tcpdnslistener,
-	isc_nm_tcpdnssocket
+	isc_nm_tcpdnssocket,
 } isc_nmsocket_type;

 /*%
@@ -403,7 +425,7 @@ struct isc_nmsocket {
 	isc_nmsocket_t *children;
 	int nchildren;
 	isc_nmiface_t *iface;
-	isc_nmhandle_t *tcphandle;
+	isc_nmhandle_t *statichandle;
 	isc_nmhandle_t *outerhandle;

 	/*% Extra data allocated at the end of each isc_nmhandle_t */
@@ -445,7 +467,12 @@ struct isc_nmsocket {
 	isc_refcount_t references;

 	/*%
-	 * TCPDNS socket has been set not to pipeliine.
+	 * Established an outgoing connection, as client not server.
+	 */
+	atomic_bool client;
+
+	/*%
+	 * TCPDNS socket has been set not to pipeline.
 	 */
 	atomic_bool sequential;

@@ -536,6 +563,12 @@ struct isc_nmsocket {

 	isc__nm_cb_t accept_cb;
 	void *accept_cbarg;
+#ifdef NETMGR_TRACE
+	void *backtrace[TRACE_SIZE];
+	int backtrace_size;
+	LINK(isc_nmsocket_t) active_link;
+	ISC_LIST(isc_nmhandle_t) active_handles;
+#endif
 };

 bool
@@ -663,16 +696,34 @@ isc__nm_udp_send(isc_nmhandle_t *handle, isc_region_t *region, isc_nm_cb_t cb,
 * Back-end implementation of isc_nm_send() for UDP handles.
 */

+isc_result_t
+isc__nm_udp_read(isc_nmhandle_t *handle, isc_nm_recv_cb_t cb, void *cbarg);
+/*
+ * Back-end implementation of isc_nm_read() for UDP handles.
+ */
+
+void
+isc__nm_udp_close(isc_nmsocket_t *sock);
+/*%<
+ * Close a UDP socket.
+ */
+
 void
 isc__nm_udp_stoplistening(isc_nmsocket_t *sock);

 void
 isc__nm_async_udplisten(isc__networker_t *worker, isc__netievent_t *ev0);
+void
+isc__nm_async_udpconnect(isc__networker_t *worker, isc__netievent_t *ev0);

 void
 isc__nm_async_udpstop(isc__networker_t *worker, isc__netievent_t *ev0);
 void
 isc__nm_async_udpsend(isc__networker_t *worker, isc__netievent_t *ev0);
+void
+isc__nm_async_udpread(isc__networker_t *worker, isc__netievent_t *ev0);
+void
+isc__nm_async_udpclose(isc__networker_t *worker, isc__netievent_t *ev0);
 /*%<
 * Callback handlers for asynchronous UDP events (listen, stoplisten, send).
 */
@@ -686,6 +737,9 @@ isc__nm_tcp_send(isc_nmhandle_t *handle, isc_region_t *region, isc_nm_cb_t cb,

 isc_result_t
 isc__nm_tcp_read(isc_nmhandle_t *handle, isc_nm_recv_cb_t cb, void *cbarg);
+/*
+ * Back-end implementation of isc_nm_read() for TCP handles.
+ */

 void
 isc__nm_tcp_close(isc_nmsocket_t *sock);
@@ -713,9 +767,9 @@ isc__nm_tcp_shutdown(isc_nmsocket_t *sock);
 */

 void
-isc__nm_tcp_cancelread(isc_nmsocket_t *sock);
+isc__nm_tcp_cancelread(isc_nmhandle_t *handle);
 /*%<
- * Stop reading on a connected socket.
+ * Stop reading on a connected TCP handle.
 */

 void
@@ -770,6 +824,18 @@ isc__nm_async_tcpdnsclose(isc__networker_t *worker, isc__netievent_t *ev0);
 void
 isc__nm_async_tcpdnssend(isc__networker_t *worker, isc__netievent_t *ev0);

+void
+isc__nm_async_tcpdnsread(isc__networker_t *worker, isc__netievent_t *ev0);
+
+isc_result_t
+isc__nm_tcpdns_read(isc_nmhandle_t *handle, isc_nm_recv_cb_t cb, void *cbarg);
+
+void
+isc__nm_tcpdns_cancelread(isc_nmhandle_t *handle);
+/*%<
+ * Stop reading on a connected TCPDNS handle.
+ */
+
 #define isc__nm_uverr2result(x) \
 	isc___nm_uverr2result(x, true, __FILE__, __LINE__)
 isc_result_t
@@ -33,6 +33,10 @@
 #include "netmgr-int.h"
 #include "uv-compat.h"

+#ifdef NETMGR_TRACE
+#include <execinfo.h>
+#endif
+
 /*%
 * How many isc_nmhandles and isc_nm_uvreqs will we be
 * caching for reuse in a socket.
@@ -157,6 +161,10 @@ isc_nm_start(isc_mem_t *mctx, uint32_t workers) {
 	atomic_init(&mgr->paused, false);
 	atomic_init(&mgr->interlocked, false);

+#ifdef NETMGR_TRACE
+	ISC_LIST_INIT(mgr->active_sockets);
+#endif
+
 	/*
 	 * Default TCP timeout values.
 	 * May be updated by isc_nm_tcptimeouts().
@@ -393,6 +401,7 @@ isc_nm_closedown(isc_nm_t *mgr) {
 void
 isc_nm_destroy(isc_nm_t **mgr0) {
 	isc_nm_t *mgr = NULL;
+	int counter = 0;

 	REQUIRE(mgr0 != NULL);
 	REQUIRE(VALID_NM(*mgr0));
@@ -407,7 +416,7 @@ isc_nm_destroy(isc_nm_t **mgr0) {
 	/*
 	 * Wait for the manager to be dereferenced elsewhere.
 	 */
-	while (isc_refcount_current(&mgr->references) > 1) {
+	while (isc_refcount_current(&mgr->references) > 1 && counter++ < 1000) {
 		/*
 		 * Sometimes libuv gets stuck, pausing and unpausing
 		 * netmgr goes over all events in async queue for all
@@ -422,6 +431,10 @@ isc_nm_destroy(isc_nm_t **mgr0) {
 		usleep(10000);
 #endif /* ifdef WIN32 */
 	}
+#ifdef NETMGR_TRACE
+	INSIST(ISC_LIST_EMPTY(mgr->active_sockets));
+#endif
+	INSIST(counter <= 1000);

 	/*
 	 * Detach final reference.
@@ -591,6 +604,10 @@ process_queue(isc__networker_t *worker, isc_queue_t *queue) {
 			uv_stop(&worker->loop);
 			isc_mempool_put(worker->mgr->evpool, ievent);
 			return;
+
+		case netievent_udpconnect:
+			isc__nm_async_udpconnect(worker, ievent);
+			break;
 		case netievent_udplisten:
 			isc__nm_async_udplisten(worker, ievent);
 			break;
@@ -600,6 +617,13 @@ process_queue(isc__networker_t *worker, isc_queue_t *queue) {
 		case netievent_udpsend:
 			isc__nm_async_udpsend(worker, ievent);
 			break;
+		case netievent_udpread:
+			isc__nm_async_udpread(worker, ievent);
+			break;
+		case netievent_udpclose:
+			isc__nm_async_udpclose(worker, ievent);
+			break;
+
 		case netievent_tcpconnect:
 			isc__nm_async_tcpconnect(worker, ievent);
 			break;
@@ -630,9 +654,14 @@ process_queue(isc__networker_t *worker, isc_queue_t *queue) {
 		case netievent_tcpclose:
 			isc__nm_async_tcpclose(worker, ievent);
 			break;
+
 		case netievent_tcpdnsclose:
 			isc__nm_async_tcpdnsclose(worker, ievent);
 			break;
+		case netievent_tcpdnsread:
+			isc__nm_async_tcpdnsread(worker, ievent);
+			break;
+
 		case netievent_closecb:
 			isc__nm_async_closecb(worker, ievent);
 			break;
@@ -739,11 +768,10 @@ nmsocket_cleanup(isc_nmsocket_t *sock, bool dofree) {
 		isc__nm_decstats(sock->mgr, sock->statsindex[STATID_ACTIVE]);
 	}

-	sock->tcphandle = NULL;
+	sock->statichandle = NULL;

 	if (sock->outerhandle != NULL) {
-		isc_nmhandle_unref(sock->outerhandle);
-		sock->outerhandle = NULL;
+		isc_nmhandle_detach(&sock->outerhandle);
 	}

 	if (sock->outer != NULL) {
@@ -786,7 +814,11 @@ nmsocket_cleanup(isc_nmsocket_t *sock, bool dofree) {
 	isc_mem_free(sock->mgr->mctx, sock->ah_handles);
 	isc_mutex_destroy(&sock->lock);
 	isc_condition_destroy(&sock->cond);
-
+#ifdef NETMGR_TRACE
+	LOCK(&sock->mgr->lock);
+	ISC_LIST_UNLINK(sock->mgr->active_sockets, sock, active_link);
+	UNLOCK(&sock->mgr->lock);
+#endif
 	if (dofree) {
 		isc_nm_t *mgr = sock->mgr;
 		isc_mem_put(mgr->mctx, sock, sizeof(*sock));
@@ -833,7 +865,7 @@ nmsocket_maybe_destroy(isc_nmsocket_t *sock) {
 		}
 	}

-	if (active_handles == 0 || sock->tcphandle != NULL) {
+	if (active_handles == 0 || sock->statichandle != NULL) {
 		destroy = true;
 	}

@@ -875,6 +907,9 @@ isc__nmsocket_prep_destroy(isc_nmsocket_t *sock) {
 	 */
 	if (!atomic_load(&sock->closed)) {
 		switch (sock->type) {
+		case isc_nm_udpsocket:
+			isc__nm_udp_close(sock);
+			return;
 		case isc_nm_tcpsocket:
 			isc__nm_tcp_close(sock);
 			return;
@@ -944,6 +979,15 @@ isc__nmsocket_init(isc_nmsocket_t *sock, isc_nm_t *mgr, isc_nmsocket_type type,
 				  .inactivereqs = isc_astack_new(
 					  mgr->mctx, ISC_NM_REQS_STACK_SIZE) };

+#ifdef NETMGR_TRACE
+	sock->backtrace_size = backtrace(sock->backtrace, TRACE_SIZE);
+	ISC_LINK_INIT(sock, active_link);
+	ISC_LIST_INIT(sock->active_handles);
+	LOCK(&mgr->lock);
+	ISC_LIST_APPEND(mgr->active_sockets, sock, active_link);
+	UNLOCK(&mgr->lock);
+#endif
+
 	isc_nm_attach(mgr, &sock->mgr);
 	sock->uv_handle.handle.data = sock;

@@ -1032,6 +1076,9 @@ alloc_handle(isc_nmsocket_t *sock) {
 			    sizeof(isc_nmhandle_t) + sock->extrahandlesize);

 	*handle = (isc_nmhandle_t){ .magic = NMHANDLE_MAGIC };
+#ifdef NETMGR_TRACE
+	ISC_LINK_INIT(handle, active_link);
+#endif
 	isc_refcount_init(&handle->references, 1);

 	return (handle);
@@ -1051,12 +1098,16 @@ isc__nmhandle_get(isc_nmsocket_t *sock, isc_sockaddr_t *peer,
 	if (handle == NULL) {
 		handle = alloc_handle(sock);
 	} else {
-		isc_refcount_increment0(&handle->references);
+		isc_refcount_init(&handle->references, 1);
 		INSIST(VALID_NMHANDLE(handle));
 	}

 	isc__nmsocket_attach(sock, &handle->sock);

+#ifdef NETMGR_TRACE
+	handle->backtrace_size = backtrace(handle->backtrace, TRACE_SIZE);
+#endif
+
 	if (peer != NULL) {
 		memcpy(&handle->peer, peer, sizeof(isc_sockaddr_t));
 	} else {
@@ -1097,21 +1148,35 @@ isc__nmhandle_get(isc_nmsocket_t *sock, isc_sockaddr_t *peer,
 	INSIST(sock->ah_handles[pos] == NULL);
 	sock->ah_handles[pos] = handle;
 	handle->ah_pos = pos;
+#ifdef NETMGR_TRACE
+	ISC_LIST_APPEND(sock->active_handles, handle, active_link);
+#endif
 	UNLOCK(&sock->lock);

-	if (sock->type == isc_nm_tcpsocket) {
-		INSIST(sock->tcphandle == NULL);
-		sock->tcphandle = handle;
+	if (sock->type == isc_nm_tcpsocket ||
+	    (sock->type == isc_nm_udpsocket && atomic_load(&sock->client)))
+	{
+		INSIST(sock->statichandle == NULL);
+
+		/*
+		 * statichandle must be assigned, not attached;
+		 * otherwise, if a handle was detached elsewhere
+		 * it could never reach 0 references, and the
+		 * handle and socket would never be freed.
+		 */
+		sock->statichandle = handle;
 	}

 	return (handle);
 }

 void
-isc_nmhandle_ref(isc_nmhandle_t *handle) {
+isc_nmhandle_attach(isc_nmhandle_t *handle, isc_nmhandle_t **handlep) {
 	REQUIRE(VALID_NMHANDLE(handle));
+	REQUIRE(handlep != NULL && *handlep == NULL);

 	isc_refcount_increment(&handle->references);
+	*handlep = handle;
 }

 bool
@@ -1153,6 +1218,10 @@ nmhandle_deactivate(isc_nmsocket_t *sock, isc_nmhandle_t *handle) {
 	INSIST(sock->ah_size > handle->ah_pos);
 	INSIST(atomic_load(&sock->ah) > 0);

+#ifdef NETMGR_TRACE
+	ISC_LIST_UNLINK(sock->active_handles, handle, active_link);
+#endif
+
 	sock->ah_handles[handle->ah_pos] = NULL;
 	handlenum = atomic_fetch_sub(&sock->ah, 1) - 1;
 	sock->ah_frees[handlenum] = handle->ah_pos;
@@ -1167,14 +1236,20 @@ nmhandle_deactivate(isc_nmsocket_t *sock, isc_nmhandle_t *handle) {
 }

 void
-isc_nmhandle_unref(isc_nmhandle_t *handle) {
+isc_nmhandle_detach(isc_nmhandle_t **handlep) {
 	isc_nmsocket_t *sock = NULL;
+	isc_nmhandle_t *handle = NULL;

-	REQUIRE(VALID_NMHANDLE(handle));
+	REQUIRE(handlep != NULL);
+	REQUIRE(VALID_NMHANDLE(*handlep));
+
+	handle = *handlep;
+	*handlep = NULL;

 	if (isc_refcount_decrement(&handle->references) > 1) {
 		return;
 	}
+
 	/* We need an acquire memory barrier here */
 	(void)isc_refcount_current(&handle->references);

@@ -1208,6 +1283,11 @@ isc_nmhandle_unref(isc_nmhandle_t *handle) {
 		}
 	}

+	if (handle == sock->statichandle) {
+		/* statichandle is assigned, not attached. */
+		sock->statichandle = NULL;
+	}
+
 	isc__nmsocket_detach(&sock);
 }

@@ -1309,7 +1389,7 @@ isc__nm_uvreq_put(isc__nm_uvreq_t **req0, isc_nmsocket_t *sock) {
 	}

 	if (handle != NULL) {
-		isc_nmhandle_unref(handle);
+		isc_nmhandle_detach(&handle);
 	}

 	isc__nmsocket_detach(&sock);
@@ -1339,8 +1419,12 @@ isc_nm_read(isc_nmhandle_t *handle, isc_nm_recv_cb_t cb, void *cbarg) {
 	REQUIRE(VALID_NMHANDLE(handle));

 	switch (handle->sock->type) {
+	case isc_nm_udpsocket:
+		return (isc__nm_udp_read(handle, cb, cbarg));
 	case isc_nm_tcpsocket:
 		return (isc__nm_tcp_read(handle, cb, cbarg));
+	case isc_nm_tcpdnssocket:
+		return (isc__nm_tcpdns_read(handle, cb, cbarg));
 	default:
 		INSIST(0);
 		ISC_UNREACHABLE();
@@ -1353,7 +1437,10 @@ isc_nm_cancelread(isc_nmhandle_t *handle) {

 	switch (handle->sock->type) {
 	case isc_nm_tcpsocket:
-		isc__nm_tcp_cancelread(handle->sock);
+		isc__nm_tcp_cancelread(handle);
+		break;
+	case isc_nm_tcpdnssocket:
+		isc__nm_tcpdns_cancelread(handle);
 		break;
 	default:
 		INSIST(0);
@@ -147,49 +147,47 @@ done:

 static void
 tcp_connect_cb(uv_connect_t *uvreq, int status) {
+	isc_result_t result;
 	isc__nm_uvreq_t *req = (isc__nm_uvreq_t *)uvreq->data;
 	isc_nmsocket_t *sock = NULL;
+	struct sockaddr_storage ss;
+	isc_nmhandle_t *handle = NULL;

 	sock = uv_handle_get_data((uv_handle_t *)uvreq->handle);

 	REQUIRE(VALID_UVREQ(req));

-	if (status == 0) {
-		isc_result_t result;
-		struct sockaddr_storage ss;
-		isc_nmhandle_t *handle = NULL;
-
-		sock = uv_handle_get_data((uv_handle_t *)uvreq->handle);
-		isc__nm_incstats(sock->mgr, sock->statsindex[STATID_CONNECT]);
-		uv_tcp_getpeername(&sock->uv_handle.tcp, (struct sockaddr *)&ss,
-				   &(int){ sizeof(ss) });
-		result = isc_sockaddr_fromsockaddr(&sock->peer,
-						   (struct sockaddr *)&ss);
-		RUNTIME_CHECK(result == ISC_R_SUCCESS);
-
-		handle = isc__nmhandle_get(sock, NULL, NULL);
-		req->cb.connect(handle, ISC_R_SUCCESS, req->cbarg);
-
-		isc__nm_uvreq_put(&req, sock);
-
-		/*
-		 * The sock is now attached to the handle.
-		 */
-		isc__nmsocket_detach(&sock);
-
-		/*
-		 * If the connect callback wants to hold on to the handle,
-		 * it needs to attach to it.
-		 */
-		isc_nmhandle_unref(handle);
-	} else {
-		/*
-		 * TODO:
-		 * Handle the connect error properly and free the socket.
-		 */
+	if (status != 0) {
 		req->cb.connect(NULL, isc__nm_uverr2result(status), req->cbarg);
 		isc__nm_uvreq_put(&req, sock);
+		isc__nmsocket_detach(&sock);
+		return;
 	}
+
+	sock = uv_handle_get_data((uv_handle_t *)uvreq->handle);
+	isc__nm_incstats(sock->mgr, sock->statsindex[STATID_CONNECT]);
+	uv_tcp_getpeername(&sock->uv_handle.tcp, (struct sockaddr *)&ss,
+			   &(int){ sizeof(ss) });
+	result = isc_sockaddr_fromsockaddr(&sock->peer, (struct sockaddr *)&ss);
+	RUNTIME_CHECK(result == ISC_R_SUCCESS);
+
+	handle = isc__nmhandle_get(sock, NULL, NULL);
+	req->cb.connect(handle, ISC_R_SUCCESS, req->cbarg);
+
+	isc__nm_uvreq_put(&req, sock);
+
+	atomic_init(&sock->client, true);
+
+	/*
+	 * The sock is now attached to the handle.
+	 */
+	isc__nmsocket_detach(&sock);
+
+	/*
+	 * The connect callback should have attached to the handle.
+	 * If it didn't, the socket will be closed now.
+	 */
+	isc_nmhandle_detach(&handle);
 }

 isc_result_t
@@ -201,6 +199,8 @@ isc_nm_tcpconnect(isc_nm_t *mgr, isc_nmiface_t *local, isc_nmiface_t *peer,
 	isc_result_t result = ISC_R_SUCCESS;

 	REQUIRE(VALID_NM(mgr));
+	REQUIRE(local != NULL);
+	REQUIRE(peer != NULL);

 	nsock = isc_mem_get(mgr->mctx, sizeof(*nsock));
 	isc__nmsocket_init(nsock, mgr, isc_nm_tcpsocket, local);
@@ -211,6 +211,7 @@ isc_nm_tcpconnect(isc_nm_t *mgr, isc_nmiface_t *local, isc_nmiface_t *peer,
 	req->cb.connect = cb;
 	req->cbarg = cbarg;
 	req->peer = peer->addr;
+	req->local = local->addr;

 	ievent = isc__nm_get_ievent(mgr, netievent_tcpconnect);
 	ievent->sock = nsock;
@@ -498,10 +499,10 @@ isc__nm_async_tcpchildaccept(isc__networker_t *worker, isc__netievent_t *ev0) {
 	isc__nmsocket_detach(&csock);

 	/*
-	 * If the accept callback wants to hold on to the handle,
-	 * it needs to attach to it.
+	 * The accept callback should have attached to the handle.
+	 * If it didn't, the socket will be closed now.
 	 */
-	isc_nmhandle_unref(handle);
+	isc_nmhandle_detach(&handle);
 	return;

 error:
@@ -604,7 +605,7 @@ readtimeout_cb(uv_timer_t *handle) {
 		isc_quota_detach(&sock->quota);
 	}
 	if (sock->rcb.recv != NULL) {
-		sock->rcb.recv(sock->tcphandle, ISC_R_TIMEDOUT, NULL,
+		sock->rcb.recv(sock->statichandle, ISC_R_TIMEDOUT, NULL,
 			       sock->rcbarg);
 		isc__nmsocket_clearcb(sock);
 	}
@@ -766,8 +767,8 @@ read_cb(uv_stream_t *stream, ssize_t nread, const uv_buf_t *buf) {
 					.length = nread };

 		if (sock->rcb.recv != NULL) {
-			sock->rcb.recv(sock->tcphandle, ISC_R_SUCCESS, &region,
-				       sock->rcbarg);
+			sock->rcb.recv(sock->statichandle, ISC_R_SUCCESS,
+				       &region, sock->rcbarg);
 		}

 		sock->read_timeout = (atomic_load(&sock->keepalive)
@@ -792,7 +793,8 @@ read_cb(uv_stream_t *stream, ssize_t nread, const uv_buf_t *buf) {
 	 */
 	if (sock->rcb.recv != NULL) {
 		isc__nm_incstats(sock->mgr, sock->statsindex[STATID_RECVFAIL]);
-		sock->rcb.recv(sock->tcphandle, ISC_R_EOF, NULL, sock->rcbarg);
+		sock->rcb.recv(sock->statichandle, ISC_R_EOF, NULL,
+			       sock->rcbarg);
 		isc__nmsocket_clearcb(sock);
 	}

@@ -958,8 +960,7 @@ isc__nm_tcp_send(isc_nmhandle_t *handle, isc_region_t *region, isc_nm_cb_t cb,
 	uvreq = isc__nm_uvreq_get(sock->mgr, sock);
 	uvreq->uvbuf.base = (char *)region->base;
 	uvreq->uvbuf.len = region->length;
-	uvreq->handle = handle;
-	isc_nmhandle_ref(uvreq->handle);
+	isc_nmhandle_attach(handle, &uvreq->handle);
 	uvreq->cb.send = cb;
 	uvreq->cbarg = cbarg;

@@ -1002,7 +1003,6 @@ tcp_send_cb(uv_write_t *req, int status) {
 	uvreq->cb.send(uvreq->handle, result, uvreq->cbarg);

 	sock = uvreq->handle->sock;
-	isc_nmhandle_unref(uvreq->handle);
 	isc__nm_uvreq_put(&uvreq, sock);
 }

@@ -1034,8 +1034,6 @@ tcp_send_direct(isc_nmsocket_t *sock, isc__nm_uvreq_t *req) {

 	REQUIRE(sock->tid == isc_nm_tid());
 	REQUIRE(sock->type == isc_nm_tcpsocket);
-
-	isc_nmhandle_ref(req->handle);
 	r = uv_write(&req->uv_req.write, &sock->uv_handle.stream, &req->uvbuf,
 		     1, tcp_send_cb);
 	if (r < 0) {
@@ -1125,24 +1123,27 @@ void
 isc__nm_tcp_shutdown(isc_nmsocket_t *sock) {
 	REQUIRE(VALID_NMSOCK(sock));

-	if (sock->type == isc_nm_tcpsocket && sock->tcphandle != NULL &&
+	if (sock->type == isc_nm_tcpsocket && sock->statichandle != NULL &&
 	    sock->rcb.recv != NULL)
 	{
-		sock->rcb.recv(sock->tcphandle, ISC_R_CANCELED, NULL,
+		sock->rcb.recv(sock->statichandle, ISC_R_CANCELED, NULL,
 			       sock->rcbarg);
 		isc__nmsocket_clearcb(sock);
 	}
 }

 void
-isc__nm_tcp_cancelread(isc_nmsocket_t *sock) {
-	REQUIRE(VALID_NMSOCK(sock));
+isc__nm_tcp_cancelread(isc_nmhandle_t *handle) {
+	isc_nmsocket_t *sock = NULL;

-	if (sock->type == isc_nm_tcpsocket && sock->tcphandle != NULL &&
-	    sock->rcb.recv != NULL)
-	{
-		sock->rcb.recv(sock->tcphandle, ISC_R_CANCELED, NULL,
-			       sock->rcbarg);
+	REQUIRE(VALID_NMHANDLE(handle));
+
+	sock = handle->sock;
+
+	REQUIRE(sock->type == isc_nm_tcpsocket);
+
+	if (atomic_load(&sock->client) && sock->rcb.recv != NULL) {
+		sock->rcb.recv(handle, ISC_R_EOF, NULL, sock->rcbarg);
 		isc__nmsocket_clearcb(sock);
 	}
 }
@@ -82,7 +82,9 @@ alloc_dnsbuf(isc_nmsocket_t *sock, size_t len) {
 static void
 timer_close_cb(uv_handle_t *handle) {
 	isc_nmsocket_t *sock = (isc_nmsocket_t *)uv_handle_get_data(handle);
-	INSIST(VALID_NMSOCK(sock));
+
+	REQUIRE(VALID_NMSOCK(sock));
+
 	atomic_store(&sock->closed, true);
 	tcpdns_close_direct(sock);
 }
@@ -94,9 +96,9 @@ dnstcp_readtimeout(uv_timer_t *timer) {

 	REQUIRE(VALID_NMSOCK(sock));
 	REQUIRE(sock->tid == isc_nm_tid());
-	/* Close the TCP connection, it's closing should fire 'our' closing */
-	isc_nmhandle_unref(sock->outerhandle);
-	sock->outerhandle = NULL;
+
+	/* Close the TCP connection; its closure should fire ours. */
+	isc_nmhandle_detach(&sock->outerhandle);
 }

 /*
@@ -106,6 +108,7 @@ static isc_result_t
 dnslisten_acceptcb(isc_nmhandle_t *handle, isc_result_t result, void *cbarg) {
 	isc_nmsocket_t *dnslistensock = (isc_nmsocket_t *)cbarg;
 	isc_nmsocket_t *dnssock = NULL;
+	isc_nmhandle_t *readhandle = NULL;

 	REQUIRE(VALID_NMSOCK(dnslistensock));
 	REQUIRE(dnslistensock->type == isc_nm_tcpdnslistener);
@@ -132,8 +135,7 @@ dnslisten_acceptcb(isc_nmhandle_t *handle, isc_result_t result, void *cbarg) {

 	isc__nmsocket_attach(dnssock, &dnssock->self);

-	dnssock->outerhandle = handle;
-	isc_nmhandle_ref(dnssock->outerhandle);
+	isc_nmhandle_attach(handle, &dnssock->outerhandle);

 	dnssock->peer = handle->sock->peer;
 	dnssock->read_timeout = handle->sock->mgr->init;
@@ -147,10 +149,14 @@ dnslisten_acceptcb(isc_nmhandle_t *handle, isc_result_t result, void *cbarg) {
 	uv_timer_start(&dnssock->timer, dnstcp_readtimeout,
 		       dnssock->read_timeout, 0);

-	isc_nmhandle_ref(handle);
-	result = isc_nm_read(handle, dnslisten_readcb, dnssock);
+	/*
+	 * Add a reference to handle to keep it from being freed by
+	 * the caller; it will be detached in dnslisten_readcb().
+	 */
+	isc_nmhandle_attach(handle, &readhandle);
+	result = isc_nm_read(readhandle, dnslisten_readcb, dnssock);
 	if (result != ISC_R_SUCCESS) {
-		isc_nmhandle_unref(handle);
+		isc_nmhandle_detach(&readhandle);
 	}
 	isc__nmsocket_detach(&dnssock);

@@ -187,8 +193,14 @@ processbuffer(isc_nmsocket_t *dnssock, isc_nmhandle_t **handlep) {
 	 */
 	len = dnslen(dnssock->buf);
 	if (len <= dnssock->buf_len - 2) {
-		isc_nmhandle_t *dnshandle = isc__nmhandle_get(dnssock, NULL,
-							      NULL);
+		isc_nmhandle_t *dnshandle = NULL;
+		if (atomic_load(&dnssock->client) &&
+		    dnssock->statichandle != NULL) {
+			isc_nmhandle_attach(dnssock->statichandle, &dnshandle);
+		} else {
+			dnshandle = isc__nmhandle_get(dnssock, NULL, NULL);
+		}
+
 		isc_nmsocket_t *listener = dnssock->listener;

 		if (listener != NULL && listener->rcb.recv != NULL) {
@@ -197,6 +209,20 @@ processbuffer(isc_nmsocket_t *dnssock, isc_nmhandle_t **handlep) {
 				&(isc_region_t){ .base = dnssock->buf + 2,
 						 .length = len },
 				listener->rcbarg);
+		} else if (dnssock->rcb.recv != NULL) {
+			isc_nm_recv_cb_t cb = dnssock->rcb.recv;
+			void *cbarg = dnssock->rcbarg;
+
+			/*
+			 * We need to clear the read callback *before*
+			 * calling it, because it might make another
+			 * call to isc_nm_read() and set up a new callback.
+			 */
+			isc__nmsocket_clearcb(dnssock);
+			cb(dnshandle, ISC_R_SUCCESS,
+			   &(isc_region_t){ .base = dnssock->buf + 2,
+					    .length = len },
+			   cbarg);
 		}

 		len += 2;
@@ -227,19 +253,30 @@ dnslisten_readcb(isc_nmhandle_t *handle, isc_result_t eresult,

 	REQUIRE(VALID_NMSOCK(dnssock));
 	REQUIRE(VALID_NMHANDLE(handle));
-	REQUIRE(dnssock->tid == isc_nm_tid());

 	if (region == NULL || eresult != ISC_R_SUCCESS) {
 		/* Connection closed */
-		isc_nmhandle_unref(handle);
 		dnssock->result = eresult;
+		if (atomic_load(&dnssock->client) &&
+		    dnssock->rcb.recv != NULL) {
+			dnssock->rcb.recv(dnssock->statichandle, eresult,
+					  NULL, dnssock->rcbarg);
+		}
 		if (dnssock->self != NULL) {
 			isc__nmsocket_detach(&dnssock->self);
 		}
 		isc__nmsocket_clearcb(dnssock);
 		if (dnssock->outerhandle != NULL) {
-			isc_nmhandle_unref(dnssock->outerhandle);
-			dnssock->outerhandle = NULL;
+			isc_nmhandle_detach(&dnssock->outerhandle);
+		}
+
+		/*
+		 * Server connections will hold two handle references when
+		 * shut down, but client (tcpdnsconnect) connections have
+		 * only one.
+		 */
+		if (!atomic_load(&dnssock->client)) {
+			isc_nmhandle_detach(&handle);
 		}
 		return;
 	}
@@ -277,11 +314,14 @@ dnslisten_readcb(isc_nmhandle_t *handle, isc_result_t eresult,
 			uv_timer_stop(&dnssock->timer);
 		}

-		if (atomic_load(&dnssock->sequential)) {
+		if (atomic_load(&dnssock->sequential) ||
+		    dnssock->rcb.recv == NULL) {
 			/*
-			 * We're in sequential mode and we processed
-			 * one packet, so we're done until the next read
-			 * completes.
+			 * There are two reasons we might want to pause here:
+			 * - We're in sequential mode and we've received
+			 *   a whole packet, so we're done until it's been
+			 *   processed; or
+			 * - We no longer have a read callback.
 			 */
 			isc_nm_pauseread(dnssock->outerhandle);
 			done = true;
@@ -300,7 +340,7 @@ dnslisten_readcb(isc_nmhandle_t *handle, isc_result_t eresult,
 			}
 		}

-		isc_nmhandle_unref(dnshandle);
+		isc_nmhandle_detach(&dnshandle);
 	} while (!done);
 }

@@ -314,7 +354,6 @@ isc_nm_listentcpdns(isc_nm_t *mgr, isc_nmiface_t *iface, isc_nm_recv_cb_t cb,
 		    void *cbarg, isc_nm_accept_cb_t accept_cb,
 		    void *accept_cbarg, size_t extrahandlesize, int backlog,
 		    isc_quota_t *quota, isc_nmsocket_t **sockp) {
-	/* A 'wrapper' socket object with outer set to true TCP socket */
 	isc_nmsocket_t *dnslistensock = isc_mem_get(mgr->mctx,
 						    sizeof(*dnslistensock));
 	isc_result_t result;
@@ -328,7 +367,11 @@ isc_nm_listentcpdns(isc_nm_t *mgr, isc_nmiface_t *iface, isc_nm_recv_cb_t cb,
 	dnslistensock->accept_cbarg = accept_cbarg;
 	dnslistensock->extrahandlesize = extrahandlesize;

-	/* We set dnslistensock->outer to a true listening socket */
+	/*
+	 * dnslistensock will be a DNS 'wrapper' around a connected
+	 * stream. We set dnslistensock->outer to a socket listening
+	 * for a TCP connection.
+	 */
 	result = isc_nm_listentcp(mgr, iface, dnslisten_acceptcb, dnslistensock,
 				  extrahandlesize, backlog, quota,
 				  &dnslistensock->outer);
@@ -429,12 +472,11 @@ resume_processing(void *arg) {
 			if (sock->timer_initialized) {
 				uv_timer_stop(&sock->timer);
 			}
-			isc_nmhandle_unref(handle);
+			isc_nmhandle_detach(&handle);
 		} else if (sock->outerhandle != NULL) {
 			result = isc_nm_resumeread(sock->outerhandle);
 			if (result != ISC_R_SUCCESS) {
-				isc_nmhandle_unref(sock->outerhandle);
-				sock->outerhandle = NULL;
+				isc_nmhandle_detach(&sock->outerhandle);
 			}
 		}

@@ -464,7 +506,7 @@ resume_processing(void *arg) {
 			uv_timer_stop(&sock->timer);
 		}
 		atomic_store(&sock->outerhandle->sock->processing, true);
-		isc_nmhandle_unref(dnshandle);
+		isc_nmhandle_detach(&dnshandle);
 	} while (atomic_load(&sock->ah) < TCPDNS_CLIENTS_PER_CONN);
 }

@@ -495,8 +537,7 @@ isc__nm_async_tcpdnssend(isc__networker_t *worker, isc__netievent_t *ev0) {

 		r.base = (unsigned char *)req->uvbuf.base;
 		r.length = req->uvbuf.len;
-		result = isc__nm_tcp_send(sock->outerhandle, &r, tcpdnssend_cb,
-					  req);
+		result = isc_nm_send(sock->outerhandle, &r, tcpdnssend_cb, req);
 	}

 	if (result != ISC_R_SUCCESS) {
@@ -522,8 +563,7 @@ isc__nm_tcpdns_send(isc_nmhandle_t *handle, isc_region_t *region,
 	REQUIRE(sock->type == isc_nm_tcpdnssocket);

 	uvreq = isc__nm_uvreq_get(sock->mgr, sock);
-	uvreq->handle = handle;
-	isc_nmhandle_ref(uvreq->handle);
+	isc_nmhandle_attach(handle, &uvreq->handle);
 	uvreq->cb.send = cb;
 	uvreq->cbarg = cbarg;

@@ -538,8 +578,8 @@ isc__nm_tcpdns_send(isc_nmhandle_t *handle, isc_region_t *region,
 		r.base = (unsigned char *)uvreq->uvbuf.base;
 		r.length = uvreq->uvbuf.len;

-		return (isc__nm_tcp_send(sock->outerhandle, &r, tcpdnssend_cb,
-					 uvreq));
+		return (isc_nm_send(sock->outerhandle, &r, tcpdnssend_cb,
+				    uvreq));
 	} else {
 		isc__netievent_tcpdnssend_t *ievent = NULL;

@@ -579,8 +619,7 @@ tcpdns_close_direct(isc_nmsocket_t *sock) {
 		 */
 		if (sock->outerhandle != NULL) {
 			isc__nmsocket_clearcb(sock->outerhandle->sock);
-			isc_nmhandle_unref(sock->outerhandle);
-			sock->outerhandle = NULL;
+			isc_nmhandle_detach(&sock->outerhandle);
 		}
 		if (sock->listener != NULL) {
 			isc__nmsocket_detach(&sock->listener);
@@ -616,3 +655,169 @@ isc__nm_async_tcpdnsclose(isc__networker_t *worker, isc__netievent_t *ev0) {

 	tcpdns_close_direct(ievent->sock);
 }
+
+typedef struct {
+	isc_mem_t *mctx;
+	isc_nm_cb_t cb;
+	void *cbarg;
+	size_t extrahandlesize;
+} tcpconnect_t;
+
+static void
+tcpdnsconnect_cb(isc_nmhandle_t *handle, isc_result_t result, void *arg) {
+	tcpconnect_t *conn = (tcpconnect_t *)arg;
+	isc_nm_cb_t cb = conn->cb;
+	void *cbarg = conn->cbarg;
+	size_t extrahandlesize = conn->extrahandlesize;
+	isc_nmsocket_t *dnssock = NULL;
+
+	REQUIRE(result != ISC_R_SUCCESS || VALID_NMHANDLE(handle));
+
+	isc_mem_putanddetach(&conn->mctx, conn, sizeof(*conn));
+
+	if (result != ISC_R_SUCCESS) {
+		cb(NULL, result, cbarg);
+		return;
+	}
+
+	dnssock = isc_mem_get(handle->sock->mgr->mctx, sizeof(*dnssock));
+	isc__nmsocket_init(dnssock, handle->sock->mgr, isc_nm_tcpdnssocket,
+			   handle->sock->iface);
+
+	dnssock->extrahandlesize = extrahandlesize;
+	isc_nmhandle_attach(handle, &dnssock->outerhandle);
+
+	dnssock->peer = handle->sock->peer;
+	dnssock->read_timeout = handle->sock->mgr->init;
+	dnssock->tid = isc_nm_tid();
+
+	atomic_init(&dnssock->client, true);
+	dnssock->statichandle = isc__nmhandle_get(dnssock, NULL, NULL);
+
+	uv_timer_init(&dnssock->mgr->workers[isc_nm_tid()].loop,
+		      &dnssock->timer);
+	dnssock->timer.data = dnssock;
+	dnssock->timer_initialized = true;
+	uv_timer_start(&dnssock->timer, dnstcp_readtimeout,
+		       dnssock->read_timeout, 0);
+
+	/*
+	 * The connection is now established; we start reading immediately,
+	 * before we've been asked to. We'll read and buffer at most one
+	 * packet.
+	 */
+	result = isc_nm_read(handle, dnslisten_readcb, dnssock);
+	cb(dnssock->statichandle, result, cbarg);
+	isc__nmsocket_detach(&dnssock);
+}
+
+isc_result_t
+isc_nm_tcpdnsconnect(isc_nm_t *mgr, isc_nmiface_t *local, isc_nmiface_t *peer,
+		     isc_nm_cb_t cb, void *cbarg, size_t extrahandlesize) {
+	tcpconnect_t *conn = isc_mem_get(mgr->mctx, sizeof(tcpconnect_t));
+
+	*conn = (tcpconnect_t){ .cb = cb,
+				.cbarg = cbarg,
+				.extrahandlesize = extrahandlesize };
+	isc_mem_attach(mgr->mctx, &conn->mctx);
+	return (isc_nm_tcpconnect(mgr, local, peer, tcpdnsconnect_cb, conn, 0));
+}
+
+isc_result_t
+isc__nm_tcpdns_read(isc_nmhandle_t *handle, isc_nm_recv_cb_t cb, void *cbarg) {
+	isc_nmsocket_t *sock = handle->sock;
+	isc__netievent_tcpdnsread_t *ievent = NULL;
+	isc_nmhandle_t *eventhandle = NULL;
+
+	REQUIRE(handle == sock->statichandle);
+	REQUIRE(sock->rcb.recv == NULL);
+	REQUIRE(atomic_load(&sock->client));
+
+	/*
+	 * This MUST be done asynchronously, no matter which thread we're
+	 * in. The callback function for isc_nm_read() often calls
+	 * isc_nm_read() again; if we tried to do that synchronously
+	 * we'd clash in processbuffer() and grow the stack indefinitely.
+	 */
+	ievent = isc__nm_get_ievent(sock->mgr, netievent_tcpdnsread);
+	ievent->sock = sock;
+	sock->rcb.recv = cb;
+	sock->rcbarg = cbarg;
+
+	/*
+	 * Add a reference to the handle to keep it from being freed by
+	 * the caller; it will be detached in in isc__nm_async_tcpdnsread().
+	 */
+	isc_nmhandle_attach(handle, &eventhandle);
+	isc__nm_enqueue_ievent(&sock->mgr->workers[sock->tid],
+			       (isc__netievent_t *)ievent);
+	return (ISC_R_SUCCESS);
+}
+
+void
+isc__nm_async_tcpdnsread(isc__networker_t *worker, isc__netievent_t *ev0) {
+	isc_result_t result;
+	isc__netievent_tcpdnsread_t *ievent =
+		(isc__netievent_tcpdnsclose_t *)ev0;
+	isc_nmsocket_t *sock = ievent->sock;
+	isc_nmhandle_t *handle = NULL, *newhandle = NULL;
+
+	REQUIRE(VALID_NMSOCK(sock));
+	REQUIRE(worker->id == sock->tid);
+
+	handle = sock->statichandle;
+
+	if (sock->type != isc_nm_tcpdnssocket || sock->outerhandle == NULL) {
+		if (sock->rcb.recv != NULL) {
+			sock->rcb.recv(handle, ISC_R_NOTCONNECTED, NULL,
+				       sock->rcbarg);
+		}
+		isc_nmhandle_detach(&handle);
+		return;
+	}
+
+	/*
+	 * Maybe we have a packet already?
+	 */
+	result = processbuffer(sock, &newhandle);
+	if (result == ISC_R_SUCCESS) {
+		atomic_store(&sock->outerhandle->sock->processing, true);
+		if (sock->timer_initialized) {
+			uv_timer_stop(&sock->timer);
+		}
+		isc_nmhandle_detach(&newhandle);
+	} else if (sock->outerhandle != NULL) {
+		/* Restart reading, wait for the callback */
+		atomic_store(&sock->outerhandle->sock->processing, false);
+		if (sock->timer_initialized) {
+			uv_timer_start(&sock->timer, dnstcp_readtimeout,
+				       sock->read_timeout, 0);
+		}
+		isc_nm_resumeread(sock->outerhandle);
+	} else {
+		isc_nm_recv_cb_t cb = sock->rcb.recv;
+		void *cbarg = sock->rcbarg;
+
+		isc__nmsocket_clearcb(sock);
+		cb(handle, ISC_R_NOTCONNECTED, NULL, cbarg);
+	}
+
+	isc_nmhandle_detach(&handle);
+}
+
+void
+isc__nm_tcpdns_cancelread(isc_nmhandle_t *handle) {
+	isc_nmsocket_t *sock = NULL;
+
+	REQUIRE(VALID_NMHANDLE(handle));
+
+	sock = handle->sock;
+
+	REQUIRE(sock->type == isc_nm_tcpdnssocket);
+
+	if (atomic_load(&sock->client) && sock->rcb.recv != NULL) {
+		sock->rcb.recv(handle, ISC_R_EOF, NULL, sock->rcbarg);
+		isc__nmsocket_clearcb(sock);
+		isc__nm_tcp_cancelread(sock->outerhandle);
+	}
+}
@@ -15,6 +15,7 @@
 #include <isc/atomic.h>
 #include <isc/buffer.h>
 #include <isc/condition.h>
+#include <isc/errno.h>
 #include <isc/magic.h>
 #include <isc/mem.h>
 #include <isc/netmgr.h>
@@ -40,6 +41,9 @@ udp_recv_cb(uv_udp_t *handle, ssize_t nrecv, const uv_buf_t *buf,
 static void
 udp_send_cb(uv_udp_send_t *req, int status);

+static void
+udp_close_cb(uv_handle_t *uvhandle);
+
 isc_result_t
 isc_nm_listenudp(isc_nm_t *mgr, isc_nmiface_t *iface, isc_nm_recv_cb_t cb,
 		 void *cbarg, size_t extrahandlesize, isc_nmsocket_t **sockp) {
@@ -80,7 +84,7 @@ isc_nm_listenudp(isc_nm_t *mgr, isc_nmiface_t *iface, isc_nm_recv_cb_t cb,
 		csock->rcb.recv = cb;
 		csock->rcbarg = cbarg;
 		csock->fd = socket(family, SOCK_DGRAM, 0);
-		INSIST(csock->fd >= 0);
+		RUNTIME_CHECK(csock->fd >= 0);

 		/*
 		 * This is SO_REUSE**** hell:
@@ -159,7 +163,7 @@ udp_alloc_cb(uv_handle_t *handle, size_t size, uv_buf_t *buf) {
 }

 /*
- * handle 'udplisten' async call - start listening on a socket.
+ * Asynchronous 'udplisten' call handler: start listening on a UDP socket.
 */
 void
 isc__nm_async_udplisten(isc__networker_t *worker, isc__netievent_t *ev0) {
@@ -223,7 +227,7 @@ isc__nm_async_udplisten(isc__networker_t *worker, isc__netievent_t *ev0) {
 }

 static void
-udp_close_cb(uv_handle_t *handle) {
+udp_stop_cb(uv_handle_t *handle) {
 	isc_nmsocket_t *sock = uv_handle_get_data(handle);
 	atomic_store(&sock->closed, true);

@@ -236,7 +240,7 @@ stop_udp_child(isc_nmsocket_t *sock) {
 	REQUIRE(sock->tid == isc_nm_tid());

 	uv_udp_recv_stop(&sock->uv_handle.udp);
-	uv_close((uv_handle_t *)&sock->uv_handle.udp, udp_close_cb);
+	uv_close((uv_handle_t *)&sock->uv_handle.udp, udp_stop_cb);

 	isc__nm_incstats(sock->mgr, sock->statsindex[STATID_CLOSE]);

@@ -311,7 +315,7 @@ isc__nm_udp_stoplistening(isc_nmsocket_t *sock) {
 }

 /*
- * handle 'udpstop' async call - stop listening on a socket.
+ * Asynchronous 'udpstop' call handler: stop listening on a UDP socket.
 */
 void
 isc__nm_async_udpstop(isc__networker_t *worker, isc__netievent_t *ev0) {
@@ -375,9 +379,9 @@ udp_recv_cb(uv_udp_t *handle, ssize_t nrecv, const uv_buf_t *buf,
 #endif

 	/*
-	 * Three reasons to return now without processing:
-	 * - If addr == NULL that's the end of stream - we can
-	 *   free the buffer and bail.
+	 * Three possible reasons to return now without processing:
+	 * - If addr == NULL, in which case it's the end of stream;
+	 *   we can free the buffer and bail.
 	 * - If we're simulating a firewall blocking UDP packets
 	 *   bigger than 'maxudp' bytes for testing purposes.
 	 * - If the socket is no longer active.
@@ -395,7 +399,11 @@ udp_recv_cb(uv_udp_t *handle, ssize_t nrecv, const uv_buf_t *buf,

 	result = isc_sockaddr_fromsockaddr(&sockaddr, addr);
 	RUNTIME_CHECK(result == ISC_R_SUCCESS);
-	nmhandle = isc__nmhandle_get(sock, &sockaddr, NULL);
+	if (!atomic_load(&sock->connected)) {
+		nmhandle = isc__nmhandle_get(sock, &sockaddr, NULL);
+	} else {
+		nmhandle = sock->statichandle;
+	}
 	region.base = (unsigned char *)buf->base;
 	region.length = nrecv;

@@ -414,24 +422,24 @@ udp_recv_cb(uv_udp_t *handle, ssize_t nrecv, const uv_buf_t *buf,
 	 * If the recv callback wants to hold on to the handle,
 	 * it needs to attach to it.
 	 */
-	isc_nmhandle_unref(nmhandle);
+	isc_nmhandle_detach(&nmhandle);
 }

 /*
- * isc__nm_udp_send sends buf to a peer on a socket.
- * It tries to find a proper sibling/child socket so that we won't have
- * to jump to another thread.
+ * Send the data in 'region' to a peer via a UDP socket. We try to find
+ * a proper sibling/child socket so that we won't have to jump to another
+ * thread.
 */
 isc_result_t
 isc__nm_udp_send(isc_nmhandle_t *handle, isc_region_t *region, isc_nm_cb_t cb,
 		 void *cbarg) {
-	isc_nmsocket_t *psock = NULL, *rsock = NULL;
 	isc_nmsocket_t *sock = handle->sock;
+	isc_nmsocket_t *psock = NULL, *rsock = sock;
 	isc_sockaddr_t *peer = &handle->peer;
 	isc__netievent_udpsend_t *ievent = NULL;
 	isc__nm_uvreq_t *uvreq = NULL;
-	int ntid;
 	uint32_t maxudp = atomic_load(&sock->mgr->maxudp);
+	int ntid;

 	/*
 	 * We're simulating a firewall blocking UDP packets bigger than
@@ -442,16 +450,16 @@ isc__nm_udp_send(isc_nmhandle_t *handle, isc_region_t *region, isc_nm_cb_t cb,
 	 * we need to do so here.
 	 */
 	if (maxudp != 0 && region->length > maxudp) {
-		isc_nmhandle_unref(handle);
+		isc_nmhandle_detach(&handle);
 		return (ISC_R_SUCCESS);
 	}

-	if (sock->type == isc_nm_udpsocket) {
+	if (sock->type == isc_nm_udpsocket && !atomic_load(&sock->client)) {
 		INSIST(sock->parent != NULL);
 		psock = sock->parent;
 	} else if (sock->type == isc_nm_udplistener) {
 		psock = sock;
-	} else {
+	} else if (!atomic_load(&sock->client)) {
 		INSIST(0);
 		ISC_UNREACHABLE();
 	}
@@ -467,20 +475,22 @@ isc__nm_udp_send(isc_nmhandle_t *handle, isc_region_t *region, isc_nm_cb_t cb,
 	 */
 	if (isc__nm_in_netthread()) {
 		ntid = isc_nm_tid();
-	} else if (sock->type == isc_nm_udpsocket) {
+	} else if (sock->type == isc_nm_udpsocket &&
+		   !atomic_load(&sock->client)) {
 		ntid = sock->tid;
 	} else {
 		ntid = (int)isc_random_uniform(sock->nchildren);
 	}

-	rsock = &psock->children[ntid];
+	if (psock != NULL) {
+		rsock = &psock->children[ntid];
+	}

 	uvreq = isc__nm_uvreq_get(sock->mgr, sock);
 	uvreq->uvbuf.base = (char *)region->base;
 	uvreq->uvbuf.len = region->length;

-	uvreq->handle = handle;
-	isc_nmhandle_ref(uvreq->handle);
+	isc_nmhandle_attach(handle, &uvreq->handle);

 	uvreq->cb.send = cb;
 	uvreq->cbarg = cbarg;
@@ -507,7 +517,7 @@ isc__nm_udp_send(isc_nmhandle_t *handle, isc_region_t *region, isc_nm_cb_t cb,
 }

 /*
- * handle 'udpsend' async event - send a packet on the socket
+ * Asynchronous 'udpsend' event handler: send a packet on a UDP socket.
 */
 void
 isc__nm_async_udpsend(isc__networker_t *worker, isc__netievent_t *ev0) {
@@ -524,9 +534,6 @@ isc__nm_async_udpsend(isc__networker_t *worker, isc__netievent_t *ev0) {
 	}
 }

-/*
- * udp_send_cb - callback
- */
 static void
 udp_send_cb(uv_udp_send_t *req, int status) {
 	isc_result_t result = ISC_R_SUCCESS;
@@ -542,7 +549,6 @@ udp_send_cb(uv_udp_send_t *req, int status) {
 	}

 	uvreq->cb.send(uvreq->handle, result, uvreq->cbarg);
-	isc_nmhandle_unref(uvreq->handle);
 	isc__nm_uvreq_put(&uvreq, uvreq->sock);
 }

@@ -553,6 +559,7 @@ udp_send_cb(uv_udp_send_t *req, int status) {
 static isc_result_t
 udp_send_direct(isc_nmsocket_t *sock, isc__nm_uvreq_t *req,
 		isc_sockaddr_t *peer) {
+	const struct sockaddr *sa = NULL;
 	int rv;

 	REQUIRE(sock->tid == isc_nm_tid());
@@ -561,9 +568,10 @@ udp_send_direct(isc_nmsocket_t *sock, isc__nm_uvreq_t *req,
 	if (!isc__nmsocket_active(sock)) {
 		return (ISC_R_CANCELED);
 	}
-	isc_nmhandle_ref(req->handle);
+
+	sa = atomic_load(&sock->connected) ? NULL : &peer->type.sa;
 	rv = uv_udp_send(&req->uv_req.udp_send, &sock->uv_handle.udp,
-			 &req->uvbuf, 1, &peer->type.sa, udp_send_cb);
+			 &req->uvbuf, 1, sa, udp_send_cb);
 	if (rv < 0) {
 		isc__nm_incstats(req->sock->mgr,
 				 req->sock->statsindex[STATID_SENDFAIL]);
@@ -572,3 +580,279 @@ udp_send_direct(isc_nmsocket_t *sock, isc__nm_uvreq_t *req,

 	return (ISC_R_SUCCESS);
 }
+
+/*
+ * Asynchronous 'udpconnect' call handler: open a new UDP socket and call
+ * the 'open' callback with a handle.
+ */
+void
+isc__nm_async_udpconnect(isc__networker_t *worker, isc__netievent_t *ev0) {
+	isc__netievent_udpconnect_t *ievent =
+		(isc__netievent_udpconnect_t *)ev0;
+	isc_nmsocket_t *sock = ievent->sock;
+	isc_nmhandle_t *handle = NULL;
+	int uv_bind_flags = UV_UDP_REUSEADDR;
+	int r;
+
+	REQUIRE(sock->type == isc_nm_udpsocket);
+	REQUIRE(sock->iface != NULL);
+	REQUIRE(sock->parent == NULL);
+	REQUIRE(sock->tid == isc_nm_tid());
+
+	uv_udp_init(&worker->loop, &sock->uv_handle.udp);
+	uv_handle_set_data(&sock->uv_handle.handle, NULL);
+	uv_handle_set_data(&sock->uv_handle.handle, sock);
+	handle = isc__nmhandle_get(sock, &ievent->peer, &sock->iface->addr);
+
+	r = uv_udp_open(&sock->uv_handle.udp, sock->fd);
+	if (r != 0) {
+		isc__nm_incstats(sock->mgr, sock->statsindex[STATID_OPENFAIL]);
+		atomic_store(&sock->closed, true);
+		atomic_store(&sock->connect_error, true);
+		sock->result = isc__nm_uverr2result(r);
+		goto done;
+	}
+	isc__nm_incstats(sock->mgr, sock->statsindex[STATID_OPEN]);
+
+	if (sock->iface->addr.type.sa.sa_family == AF_INET6) {
+		uv_bind_flags |= UV_UDP_IPV6ONLY;
+	}
+
+	r = uv_udp_bind(&sock->uv_handle.udp, &sock->iface->addr.type.sa,
+			uv_bind_flags);
+	if (r != 0) {
+		isc__nm_incstats(sock->mgr, sock->statsindex[STATID_BINDFAIL]);
+		atomic_store(&sock->connect_error, true);
+		sock->result = isc__nm_uverr2result(r);
+		goto done;
+	}
+
+#if UV_VERSION_MAJOR == 1 && UV_VERSION_MINOR < 27
+	do {
+		int addrlen = (ievent->peer.type.sa.sa_family == AF_INET)
+			? sizeof(struct sockaddr_in)
+			: sizeof(struct sockaddr_in6);
+
+		errno = 0;
+		r = connect(sock->fd, &ievent->peer.type.sa, addrlen);
+	} while (r == -1 && errno == EINTR);
+
+	if (r == -1) {
+		isc__nm_incstats(sock->mgr,
+				 sock->statsindex[STATID_CONNECTFAIL]);
+		atomic_store(&sock->connect_error, true);
+		sock->result = isc_errno_toresult(errno);
+		goto done;
+	}
+	isc__nm_incstats(sock->mgr, sock->statsindex[STATID_CONNECT]);
+#else
+	r = uv_udp_connect(&sock->uv_handle.udp, &ievent->peer.type.sa);
+	if (r != 0) {
+		isc__nm_incstats(sock->mgr,
+				 sock->statsindex[STATID_CONNECTFAIL]);
+		atomic_store(&sock->connect_error, true);
+		sock->result = isc__nm_uverr2result(r);
+		goto done;
+	}
+	isc__nm_incstats(sock->mgr, sock->statsindex[STATID_CONNECT]);
+#endif
+
+#ifdef ISC_RECV_BUFFER_SIZE
+	uv_recv_buffer_size(&sock->uv_handle.handle,
+			    &(int){ ISC_RECV_BUFFER_SIZE });
+#endif
+#ifdef ISC_SEND_BUFFER_SIZE
+	uv_send_buffer_size(&sock->uv_handle.handle,
+			    &(int){ ISC_SEND_BUFFER_SIZE });
+#endif
+
+	atomic_store(&sock->connected, true);
+	sock->result = ISC_R_SUCCESS;
+
+done:
+	sock->rcb.connect(handle, sock->result, sock->rcbarg);
+
+	LOCK(&sock->lock);
+	SIGNAL(&sock->cond);
+	UNLOCK(&sock->lock);
+
+	isc__nmsocket_detach(&sock);
+}
+
+isc_result_t
+isc_nm_udpconnect(isc_nm_t *mgr, isc_nmiface_t *local, isc_nmiface_t *peer,
+		  isc_nm_cb_t cb, void *cbarg, size_t extrahandlesize) {
+	isc_result_t result = ISC_R_SUCCESS;
+	isc_nmsocket_t *sock = NULL, *tmp = NULL;
+	isc__netievent_udpconnect_t *event = NULL;
+	int r = 0;
+
+	REQUIRE(VALID_NM(mgr));
+	REQUIRE(local != NULL);
+	REQUIRE(peer != NULL);
+
+	sock = isc_mem_get(mgr->mctx, sizeof(isc_nmsocket_t));
+	isc__nmsocket_init(sock, mgr, isc_nm_udpsocket, local);
+
+	INSIST(sock->rcb.recv == NULL && sock->rcbarg == NULL);
+	sock->rcb.connect = cb;
+	sock->rcbarg = cbarg;
+	sock->extrahandlesize = extrahandlesize;
+	sock->peer = peer->addr;
+	atomic_init(&sock->client, true);
+
+	sock->fd = socket(peer->addr.type.sa.sa_family, SOCK_DGRAM, 0);
+	RUNTIME_CHECK(sock->fd >= 0);
+
+	/*
+	 * Set up SO_REUSE* (see comments in isc_nm_listenudp() for
+	 * details).
+	 */
+#if defined(SO_REUSEADDR)
+	r = setsockopt(sock->fd, SOL_SOCKET, SO_REUSEADDR, &(int){ 1 },
+		       sizeof(int));
+	RUNTIME_CHECK(r == 0);
+#endif
+#if defined(SO_REUSEPORT_LB)
+	r = setsockopt(sock->fd, SOL_SOCKET, SO_REUSEPORT_LB, &(int){ 1 },
+		       sizeof(int));
+	RUNTIME_CHECK(r == 0);
+#elif defined(SO_REUSEPORT)
+	r = setsockopt(sock->fd, SOL_SOCKET, SO_REUSEPORT, &(int){ 1 },
+		       sizeof(int));
+	RUNTIME_CHECK(r == 0);
+#endif
+
+#ifdef SO_INCOMING_CPU
+	(void)setsockopt(sock->fd, SOL_SOCKET, SO_INCOMING_CPU, &(int){ 1 },
+			 sizeof(int));
+#endif
+
+	event = isc__nm_get_ievent(mgr, netievent_udpconnect);
+	event->sock = sock;
+	event->peer = *((isc_sockaddr_t *)peer);
+
+	/*
+	 * Hold an additional sock reference so async callbacks
+	 * can't destroy it until we're ready.
+	 */
+	isc__nmsocket_attach(sock, &tmp);
+
+	r = isc_random_uniform(mgr->nworkers);
+	if (r == isc_nm_tid()) {
+		isc__nm_async_udpconnect(&mgr->workers[r],
+					 (isc__netievent_t *)event);
+		isc__nm_put_ievent(mgr, event);
+	} else {
+		isc__nm_enqueue_ievent(&mgr->workers[r],
+				       (isc__netievent_t *)event);
+
+		LOCK(&sock->lock);
+		while (!atomic_load(&sock->connected) &&
+		       !atomic_load(&sock->connect_error)) {
+			WAIT(&sock->cond, &sock->lock);
+		}
+		UNLOCK(&sock->lock);
+	}
+
+	if (sock->result != ISC_R_SUCCESS) {
+		result = sock->result;
+		isc__nmsocket_detach(&sock);
+	}
+
+	isc__nmsocket_detach(&tmp);
+
+	return (result);
+}
+
+static void
+udp_read_cb(uv_udp_t *handle, ssize_t nrecv, const uv_buf_t *buf,
+	    const struct sockaddr *addr, unsigned flags) {
+	isc_nmsocket_t *sock = uv_handle_get_data((uv_handle_t *)handle);
+
+	udp_recv_cb(handle, nrecv, buf, addr, flags);
+	uv_udp_recv_stop(&sock->uv_handle.udp);
+}
+
+/*
+ * Asynchronous 'udpread' call handler: start or resume reading on a socket;
+ * pause reading and call the 'recv' callback after each datagram.
+ */
+void
+isc__nm_async_udpread(isc__networker_t *worker, isc__netievent_t *ev0) {
+	isc__netievent_udpread_t *ievent = (isc__netievent_udpread_t *)ev0;
+	isc_nmsocket_t *sock = ievent->sock;
+
+	UNUSED(worker);
+
+	uv_udp_recv_start(&sock->uv_handle.udp, udp_alloc_cb, udp_read_cb);
+}
+
+isc_result_t
+isc__nm_udp_read(isc_nmhandle_t *handle, isc_nm_recv_cb_t cb, void *cbarg) {
+	isc_nmsocket_t *sock = NULL;
+	isc__netievent_startread_t *ievent = NULL;
+
+	REQUIRE(VALID_NMHANDLE(handle));
+	REQUIRE(VALID_NMSOCK(handle->sock));
+	REQUIRE(handle->sock->type == isc_nm_udpsocket);
+
+	sock = handle->sock;
+	sock->rcb.recv = cb;
+	sock->rcbarg = cbarg;
+
+	ievent = isc__nm_get_ievent(sock->mgr, netievent_udpread);
+	ievent->sock = sock;
+
+	if (sock->tid == isc_nm_tid()) {
+		isc__nm_async_udpread(&sock->mgr->workers[sock->tid],
+				      (isc__netievent_t *)ievent);
+		isc__nm_put_ievent(sock->mgr, ievent);
+	} else {
+		isc__nm_enqueue_ievent(&sock->mgr->workers[sock->tid],
+				       (isc__netievent_t *)ievent);
+	}
+
+	return (ISC_R_SUCCESS);
+}
+
+static void
+udp_close_cb(uv_handle_t *uvhandle) {
+	isc_nmsocket_t *sock = uv_handle_get_data(uvhandle);
+
+	REQUIRE(VALID_NMSOCK(sock));
+
+	isc__nm_incstats(sock->mgr, sock->statsindex[STATID_CLOSE]);
+	atomic_store(&sock->closed, true);
+	isc__nmsocket_prep_destroy(sock);
+}
+
+void
+isc__nm_async_udpclose(isc__networker_t *worker, isc__netievent_t *ev0) {
+	isc__netievent_udpclose_t *ievent = (isc__netievent_udpclose_t *)ev0;
+	isc_nmsocket_t *sock = ievent->sock;
+
+	REQUIRE(worker->id == ievent->sock->tid);
+
+	uv_close(&sock->uv_handle.handle, udp_close_cb);
+}
+
+void
+isc__nm_udp_close(isc_nmsocket_t *sock) {
+	isc__netievent_udpclose_t *ievent = NULL;
+
+	REQUIRE(VALID_NMSOCK(sock));
+	REQUIRE(sock->type == isc_nm_udpsocket);
+
+	ievent = isc__nm_get_ievent(sock->mgr, netievent_udpclose);
+	ievent->sock = sock;
+
+	if (sock->tid == isc_nm_tid()) {
+		isc__nm_async_udpclose(&sock->mgr->workers[sock->tid],
+				       (isc__netievent_t *)ievent);
+		isc__nm_put_ievent(sock->mgr, ievent);
+	} else {
+		isc__nm_enqueue_ievent(&sock->mgr->workers[sock->tid],
+				       (isc__netievent_t *)ievent);
+	}
+}
@@ -436,15 +436,15 @@ isc_netaddr_setzone
 isc_netaddr_totext
 isc_netaddr_unspec
 isc_netscope_pton
+isc_nmhandle_attach
+isc_nmhandle_detach
 isc_nmhandle_getdata
 isc_nmhandle_getextra
 isc_nmhandle_is_stream
 isc_nmhandle_netmgr
 isc_nmhandle_localaddr
 isc_nmhandle_peeraddr
-isc_nmhandle_ref
 isc_nmhandle_setdata
-isc_nmhandle_unref
 isc_nm_cancelread
 isc_nm_closedown
 isc_nm_destroy
@@ -461,6 +461,7 @@ isc_nm_setstats
 isc_nm_start
 isc_nm_stoplistening
 isc_nm_tcpconnect
+isc_nm_tcpdnsconnect
 isc_nm_tcp_gettimeouts
 isc_nm_tcp_settimeouts
 isc_nm_tcpdns_keepalive
@@ -108,7 +108,6 @@ recv_data(isc_nmhandle_t *handle, isc_result_t eresult, isc_region_t *region,
 done:
 	isc_nm_pauseread(handle);
 	ccmsg->cb(handle, ccmsg->result, ccmsg->cbarg);
-	isc_nmhandle_unref(handle);
 }

 void
@@ -149,18 +148,14 @@ isccc_ccmsg_readmessage(isccc_ccmsg_t *ccmsg, isc_nm_cb_t cb, void *cbarg) {
 	ccmsg->result = ISC_R_UNEXPECTED; /* unknown right now */
 	ccmsg->length_received = false;

-	isc_nmhandle_ref(ccmsg->handle);
 	if (ccmsg->reading) {
 		result = isc_nm_resumeread(ccmsg->handle);
 	} else {
 		result = isc_nm_read(ccmsg->handle, recv_data, ccmsg);
 		ccmsg->reading = true;
 	}
-	if (result == ISC_R_CANCELED) {
+	if (result != ISC_R_SUCCESS) {
 		ccmsg->reading = false;
-	} else if (result != ISC_R_SUCCESS) {
-		ccmsg->reading = false;
-		isc_nmhandle_unref(ccmsg->handle);
 	}

 	return (result);
@@ -172,6 +167,7 @@ isccc_ccmsg_cancelread(isccc_ccmsg_t *ccmsg) {

 	if (ccmsg->reading) {
 		isc_nm_cancelread(ccmsg->handle);
+		ccmsg->reading = false;
 	}
 }

@@ -58,7 +58,9 @@ void
 isccc_ccmsg_init(isc_mem_t *mctx, isc_nmhandle_t *handle, isccc_ccmsg_t *ccmsg);
 /*%
 * Associate a cc message state with a given memory context and
- * netmgr handle.
+ * netmgr handle. (Note that the caller must hold a reference to
+ * the handle during asynchronous ccmsg operations; the ccmsg code
+ * does not hold the reference itself.)
 *
 * Requires:
 *
@@ -275,7 +275,7 @@ static void
 client_senddone(isc_nmhandle_t *handle, isc_result_t result, void *cbarg) {
 	ns_client_t *client = cbarg;

-	REQUIRE(client->handle == handle);
+	REQUIRE(client->sendhandle == handle);

 	CTRACE("senddone");
 	if (result != ISC_R_SUCCESS) {
@@ -284,7 +284,7 @@ client_senddone(isc_nmhandle_t *handle, isc_result_t result, void *cbarg) {
 			      "send failed: %s", isc_result_totext(result));
 	}

-	isc_nmhandle_unref(handle);
+	isc_nmhandle_detach(&client->sendhandle);
 }

 static void
@@ -325,13 +325,18 @@ client_allocsendbuf(ns_client_t *client, isc_buffer_t *buffer,

 static isc_result_t
 client_sendpkg(ns_client_t *client, isc_buffer_t *buffer) {
+	isc_result_t result;
 	isc_region_t r;

+	REQUIRE(client->sendhandle == NULL);
+
 	isc_buffer_usedregion(buffer, &r);
-
-	INSIST(client->handle != NULL);
-
-	return (isc_nm_send(client->handle, &r, client_senddone, client));
+	isc_nmhandle_attach(client->handle, &client->sendhandle);
+	result = isc_nm_send(client->handle, &r, client_senddone, client);
+	if (result != ISC_R_SUCCESS) {
+		isc_nmhandle_detach(&client->sendhandle);
+	}
+	return (result);
 }

 void
@@ -383,7 +388,6 @@ done:
 	}

 	ns_client_drop(client, result);
-	isc_nmhandle_unref(client->handle);
 }

 void
@@ -405,17 +409,13 @@ ns_client_send(ns_client_t *client) {
 	isc_region_t zr;
 #endif /* HAVE_DNSTAP */

+	REQUIRE(NS_CLIENT_VALID(client));
+
 	/*
 	 * XXXWPK TODO
 	 * Delay the response according to the -T delay option
 	 */

-	REQUIRE(NS_CLIENT_VALID(client));
-	/*
-	 * We need to do it to make sure the client and handle
-	 * won't disappear from under us with client_senddone.
-	 */
-
 	env = ns_interfacemgr_getaclenv(client->manager->interface->mgr);

 	CTRACE("send");
@@ -591,12 +591,7 @@ renderend:

 		respsize = isc_buffer_usedlength(&buffer);

-		isc_nmhandle_ref(client->handle);
 		result = client_sendpkg(client, &buffer);
-		if (result != ISC_R_SUCCESS) {
-			/* We won't get a callback to clean it up */
-			isc_nmhandle_unref(client->handle);
-		}

 		switch (isc_sockaddr_pf(&client->peeraddr)) {
 		case AF_INET:
@@ -626,12 +621,7 @@ renderend:

 		respsize = isc_buffer_usedlength(&buffer);

-		isc_nmhandle_ref(client->handle);
 		result = client_sendpkg(client, &buffer);
-		if (result != ISC_R_SUCCESS) {
-			/* We won't get a callback to clean it up */
-			isc_nmhandle_unref(client->handle);
-		}

 		switch (isc_sockaddr_pf(&client->peeraddr)) {
 		case AF_INET:
@@ -1683,6 +1673,7 @@ ns__client_request(isc_nmhandle_t *handle, isc_result_t eresult,
 				     ns__client_put_cb);
 		client->handle = handle;
 	}
+
 	if (isc_nmhandle_is_stream(handle)) {
 		client->attributes |= NS_CLIENTATTR_TCP;
 	}
@@ -1697,8 +1688,7 @@ ns__client_request(isc_nmhandle_t *handle, isc_result_t eresult,
 	isc_buffer_add(&tbuffer, region->length);
 	buffer = &tbuffer;

-	client->peeraddr = isc_nmhandle_peeraddr(client->handle);
-
+	client->peeraddr = isc_nmhandle_peeraddr(handle);
 	client->peeraddr_valid = true;

 	reqsize = isc_buffer_usedlength(buffer);
@@ -1968,8 +1958,7 @@ ns__client_request(isc_nmhandle_t *handle, isc_result_t eresult,
 		isc_netaddr_fromsockaddr(&client->destaddr,
 					 &client->manager->interface->addr);
 	} else {
-		isc_sockaddr_t sockaddr =
-			isc_nmhandle_localaddr(client->handle);
+		isc_sockaddr_t sockaddr = isc_nmhandle_localaddr(handle);
 		isc_netaddr_fromsockaddr(&client->destaddr, &sockaddr);
 	}

@@ -2174,7 +2163,7 @@ ns__client_request(isc_nmhandle_t *handle, isc_result_t eresult,
 			    &client->requesttime, NULL, buffer);
 #endif /* HAVE_DNSTAP */

-		isc_nmhandle_ref(client->handle);
+		isc_nmhandle_attach(handle, &client->reqhandle);
 		ns_query_start(client);
 		break;
 	case dns_opcode_update:
@@ -2185,13 +2174,13 @@ ns__client_request(isc_nmhandle_t *handle, isc_result_t eresult,
 			    &client->requesttime, NULL, buffer);
 #endif /* HAVE_DNSTAP */
 		ns_client_settimeout(client, 60);
-		isc_nmhandle_ref(client->handle);
+		isc_nmhandle_attach(handle, &client->reqhandle);
 		ns_update_start(client, sigresult);
 		break;
 	case dns_opcode_notify:
 		CTRACE("notify");
 		ns_client_settimeout(client, 60);
-		isc_nmhandle_ref(client->handle);
+		isc_nmhandle_attach(handle, &client->reqhandle);
 		ns_notify_start(client);
 		break;
 	case dns_opcode_iquery:
@@ -183,14 +183,19 @@ struct ns_client {
 	isc_task_t *	 task;
 	dns_view_t *	 view;
 	dns_dispatch_t * dispatch;
-	isc_nmhandle_t * handle;
-	unsigned char *	 tcpbuf;
-	dns_message_t *	 message;
-	unsigned char *	 sendbuf;
-	dns_rdataset_t * opt;
-	uint16_t	 udpsize;
-	uint16_t	 extflags;
-	int16_t		 ednsversion; /* -1 noedns */
+	isc_nmhandle_t * handle;      /* Permanent pointer to handle */
+	isc_nmhandle_t * sendhandle;  /* Waiting for send callback */
+	isc_nmhandle_t * reqhandle;   /* Waiting for request callback
+					 (query, update, notify) */
+	isc_nmhandle_t *fetchhandle;  /* Waiting for recursive fetch */
+	isc_nmhandle_t *updatehandle; /* Waiting for update callback */
+	unsigned char * tcpbuf;
+	dns_message_t * message;
+	unsigned char * sendbuf;
+	dns_rdataset_t *opt;
+	uint16_t	udpsize;
+	uint16_t	extflags;
+	int16_t		ednsversion; /* -1 noedns */
 	void (*cleanup)(ns_client_t *);
 	void (*shutdown)(void *arg, isc_result_t result);
 	void *	      shutdown_arg;
@@ -54,7 +54,7 @@ respond(ns_client_t *client, isc_result_t result) {
 	}
 	if (msg_result != ISC_R_SUCCESS) {
 		ns_client_drop(client, msg_result);
-		isc_nmhandle_unref(client->handle);
+		isc_nmhandle_detach(&client->reqhandle);
 		return;
 	}
 	message->rcode = rcode;
@@ -65,7 +65,7 @@ respond(ns_client_t *client, isc_result_t result) {
 	}

 	ns_client_send(client);
-	isc_nmhandle_unref(client->handle);
+	isc_nmhandle_detach(&client->reqhandle);
 }

 void
@@ -550,7 +550,7 @@ query_send(ns_client_t *client) {

 	inc_stats(client, counter);
 	ns_client_send(client);
-	isc_nmhandle_unref(client->handle);
+	isc_nmhandle_detach(&client->reqhandle);
 }

 static void
@@ -577,7 +577,7 @@ query_error(ns_client_t *client, isc_result_t result, int line) {
 	log_queryerror(client, result, line, loglevel);

 	ns_client_error(client, result);
-	isc_nmhandle_unref(client->handle);
+	isc_nmhandle_detach(&client->reqhandle);
 }

 static void
@@ -590,7 +590,7 @@ query_next(ns_client_t *client, isc_result_t result) {
 		inc_stats(client, ns_statscounter_failure);
 	}
 	ns_client_drop(client, result);
-	isc_nmhandle_unref(client->handle);
+	isc_nmhandle_detach(&client->reqhandle);
 }

 static inline void
@@ -2475,7 +2475,7 @@ prefetch_done(isc_task_t *task, isc_event_t *event) {
 	}

 	free_devent(client, &event, &devent);
-	isc_nmhandle_unref(client->handle);
+	isc_nmhandle_detach(&client->fetchhandle);
 }

 static void
@@ -2518,7 +2518,7 @@ query_prefetch(ns_client_t *client, dns_name_t *qname,
 		peeraddr = NULL;
 	}

-	isc_nmhandle_ref(client->handle);
+	isc_nmhandle_attach(client->handle, &client->fetchhandle);
 	options = client->query.fetchoptions | DNS_FETCHOPT_PREFETCH;
 	result = dns_resolver_createfetch(
 		client->view->resolver, qname, rdataset->type, NULL, NULL, NULL,
@@ -2527,7 +2527,7 @@ query_prefetch(ns_client_t *client, dns_name_t *qname,
 		&client->query.prefetch);
 	if (result != ISC_R_SUCCESS) {
 		ns_client_putrdataset(client, &tmprdataset);
-		isc_nmhandle_unref(client->handle);
+		isc_nmhandle_detach(&client->fetchhandle);
 	}

 	dns_rdataset_clearprefetch(rdataset);
@@ -2732,7 +2732,7 @@ query_rpzfetch(ns_client_t *client, dns_name_t *qname, dns_rdatatype_t type) {
 	}

 	options = client->query.fetchoptions;
-	isc_nmhandle_ref(client->handle);
+	isc_nmhandle_attach(client->handle, &client->fetchhandle);
 	result = dns_resolver_createfetch(
 		client->view->resolver, qname, type, NULL, NULL, NULL, peeraddr,
 		client->message->id, options, 0, NULL, client->task,
@@ -2740,7 +2740,7 @@ query_rpzfetch(ns_client_t *client, dns_name_t *qname, dns_rdatatype_t type) {
 		&client->query.prefetch);
 	if (result != ISC_R_SUCCESS) {
 		ns_client_putrdataset(client, &tmprdataset);
-		isc_nmhandle_unref(client->handle);
+		isc_nmhandle_detach(&client->fetchhandle);
 	}
 }

@@ -5703,6 +5703,12 @@ fetch_callback(isc_task_t *task, isc_event_t *event) {
 	}
 	UNLOCK(&client->manager->reclock);

+	if (client->reqhandle == NULL) {
+		isc_nmhandle_attach(client->handle, &client->reqhandle);
+	}
+
+	isc_nmhandle_detach(&client->fetchhandle);
+
 	client->query.attributes &= ~NS_QUERYATTR_RECURSING;
 	client->state = NS_CLIENTSTATE_WORKING;

@@ -5748,7 +5754,6 @@ fetch_callback(isc_task_t *task, isc_event_t *event) {
 	}

 	dns_resolver_destroyfetch(&fetch);
-	isc_nmhandle_unref(client->handle);
 }

 /*%
@@ -5940,14 +5945,14 @@ ns_query_recurse(ns_client_t *client, dns_rdatatype_t qtype, dns_name_t *qname,
 		peeraddr = &client->peeraddr;
 	}

-	isc_nmhandle_ref(client->handle);
+	isc_nmhandle_attach(client->handle, &client->fetchhandle);
 	result = dns_resolver_createfetch(
 		client->view->resolver, qname, qtype, qdomain, nameservers,
 		NULL, peeraddr, client->message->id, client->query.fetchoptions,
 		0, NULL, client->task, fetch_callback, client, rdataset,
 		sigrdataset, &client->query.fetch);
 	if (result != ISC_R_SUCCESS) {
-		isc_nmhandle_unref(client->handle);
+		isc_nmhandle_detach(&client->fetchhandle);
 		ns_client_putrdataset(client, &rdataset);
 		if (sigrdataset != NULL) {
 			ns_client_putrdataset(client, &sigrdataset);
@@ -32,7 +32,8 @@ notify_test_SOURCES =	\

 notify_test_LDFLAGS =			\
 	$(LDFLAGS)			\
-	-Wl,--wrap=isc_nmhandle_unref
+	-Wl,--wrap=isc_nmhandle_attach	\
+	-Wl,--wrap=isc_nmhandle_detach

 query_test_SOURCES =	\
 	query_test.c	\
@@ -40,7 +41,7 @@ query_test_SOURCES =	\

 query_test_LDFLAGS =			\
 	$(LDFLAGS)			\
-	-Wl,--wrap=isc_nmhandle_unref
+	-Wl,--wrap=isc_nmhandle_detach

 endif

@@ -86,6 +86,7 @@ static void
 notify_start(void **state) {
 	isc_result_t result;
 	ns_client_t *client = NULL;
+	isc_nmhandle_t *handle = NULL;
 	dns_message_t *nmsg = NULL;
 	unsigned char ndata[4096];
 	isc_buffer_t nbuf;
@@ -130,13 +131,17 @@ notify_start(void **state) {
 	client->message = nmsg;
 	nmsg = NULL;
 	client->sendcb = check_response;
+	isc_nmhandle_attach(client->handle, &client->reqhandle);
 	ns_notify_start(client);

 	/*
 	 * Clean up
 	 */
 	ns_test_cleanup_zone();
-	isc_nmhandle_unref(client->handle);
+
+	handle = client->handle;
+	isc_nmhandle_detach(&client->handle);
+	isc_nmhandle_detach(&handle);
 }

 int
@@ -78,10 +78,32 @@ atomic_uint_fast32_t client_refs[32];
 atomic_uintptr_t client_addrs[32];

 void
-__wrap_isc_nmhandle_unref(isc_nmhandle_t *handle);
+__wrap_isc_nmhandle_attach(isc_nmhandle_t *source, isc_nmhandle_t **targetp);
+void
+__wrap_isc_nmhandle_detach(isc_nmhandle_t **handlep);

 void
-__wrap_isc_nmhandle_unref(isc_nmhandle_t *handle) {
+__wrap_isc_nmhandle_attach(isc_nmhandle_t *source, isc_nmhandle_t **targetp) {
+	ns_client_t *client = (ns_client_t *)source;
+	int i;
+
+	for (i = 0; i < 32; i++) {
+		if (atomic_load(&client_addrs[i]) == (uintptr_t)client) {
+			break;
+		}
+	}
+	INSIST(i < 32);
+	INSIST(atomic_load(&client_refs[i]) > 0);
+
+	atomic_fetch_add(&client_refs[i], 1);
+
+	*targetp = source;
+	return;
+}
+
+void
+__wrap_isc_nmhandle_detach(isc_nmhandle_t **handlep) {
+	isc_nmhandle_t *handle = *handlep;
 	ns_client_t *client = (ns_client_t *)handle;
 	int i;

@@ -90,7 +112,7 @@ __wrap_isc_nmhandle_unref(isc_nmhandle_t *handle) {
 			break;
 		}
 	}
-	REQUIRE(i < 32);
+	INSIST(i < 32);

 	if (atomic_fetch_sub(&client_refs[i], 1) == 1) {
 		dns_view_detach(&client->view);
@@ -99,6 +121,8 @@ __wrap_isc_nmhandle_unref(isc_nmhandle_t *handle) {
 		ns__client_put_cb(client);
 		isc_mem_put(mctx, client, sizeof(ns_client_t));
 	}
+
+	*handlep = NULL;
 	return;
 }

@@ -760,6 +784,7 @@ ns_test_qctx_create(const ns_test_qctx_create_params_t *params,
 		    query_ctx_t **qctxp) {
 	ns_client_t *client = NULL;
 	isc_result_t result;
+	isc_nmhandle_t *handle = NULL;

 	REQUIRE(params != NULL);
 	REQUIRE(params->qname != NULL);
@@ -810,17 +835,19 @@ ns_test_qctx_create(const ns_test_qctx_create_params_t *params,
 	}

 	/*
-	 * Reference count for "client" is now at 2, so decrement it in order
-	 * for it to drop to zero when "qctx" gets destroyed.
+	 * The reference count for "client" is now at 2, so we need to
+	 * decrement it in order for it to drop to zero when "qctx" gets
+	 * destroyed.
 	 */
-	isc_nmhandle_unref(client->handle);
+	handle = client->handle;
+	isc_nmhandle_detach(&handle);

 	return (ISC_R_SUCCESS);

 destroy_query:
 	dns_message_destroy(&client->message);
 detach_client:
-	isc_nmhandle_unref(client->handle);
+	isc_nmhandle_detach(&client->handle);

 	return (result);
 }
@@ -842,7 +869,7 @@ ns_test_qctx_destroy(query_ctx_t **qctxp) {
 		dns_db_detach(&qctx->db);
 	}
 	if (qctx->client != NULL) {
-		isc_nmhandle_unref(qctx->client->handle);
+		isc_nmhandle_detach(&qctx->client->handle);
 	}

 	isc_mem_put(mctx, qctx, sizeof(*qctx));
@@ -26,15 +26,22 @@
 #include <ns/client.h>

 /*
- * This overrides calls to isc_nmhandle_unref(), sending them to
- * __wrap_isc_nmhandle_unref(), when libtool is in use and LD_WRAP
- * can't be used.
+ * This overrides calls to isc_nmhandle_attach/detach(), sending them to
+ * __wrap_isc_nmhandle_attach/detach() instead, when libtool is in use
+ * and LD_WRAP can't be used.
 */

+void
+__wrap_isc_nmhandle_attach(isc_nmhandle_t *source, isc_nmhandle_t **targetp);
 extern void
-__wrap_isc_nmhandle_unref(isc_nmhandle_t *handle);
+__wrap_isc_nmhandle_detach(isc_nmhandle_t **handlep);

 void
-isc_nmhandle_unref(isc_nmhandle_t *handle) {
-	__wrap_isc_nmhandle_unref(handle);
+isc_nmhandle_attach(isc_nmhandle_t *source, isc_nmhandle_t **targetp) {
+	__wrap_isc_nmhandle_attach(source, targetp);
+}
+
+void
+isc_nmhandle_detach(isc_nmhandle_t **handlep) {
+	__wrap_isc_nmhandle_detach(handlep);
 }
@@ -1567,7 +1567,9 @@ send_update_event(ns_client_t *client, dns_zone_t *zone) {
 	client->nupdates++;
 	event->ev_arg = client;

-	isc_nmhandle_ref(client->handle);
+	if (client->updatehandle == NULL) {
+		isc_nmhandle_attach(client->handle, &client->updatehandle);
+	}
 	dns_zone_gettask(zone, &zonetask);
 	isc_task_send(zonetask, ISC_EVENT_PTR(&event));

@@ -1585,7 +1587,6 @@ respond(ns_client_t *client, isc_result_t result) {
 	client->message->rcode = dns_result_torcode(result);

 	ns_client_send(client);
-	isc_nmhandle_unref(client->handle);
 	return;

 msg_failure:
@@ -1594,7 +1595,7 @@ msg_failure:
 		      "could not create update response message: %s",
 		      isc_result_totext(msg_result));
 	ns_client_drop(client, msg_result);
-	isc_nmhandle_unref(client->handle);
+	isc_nmhandle_detach(&client->reqhandle);
 }

 void
@@ -1673,6 +1674,8 @@ ns_update_start(ns_client_t *client, isc_result_t sigresult) {
 	default:
 		FAILC(DNS_R_NOTAUTH, "not authoritative for update zone");
 	}
+
+	isc_nmhandle_detach(&client->reqhandle);
 	return;

 failure:
@@ -1690,6 +1693,7 @@ failure:
 	if (zone != NULL) {
 		dns_zone_detach(&zone);
 	}
+	isc_nmhandle_detach(&client->reqhandle);
 }

 /*%
@@ -3497,6 +3501,10 @@ common:
 	}
 	uev->ev_type = DNS_EVENT_UPDATEDONE;
 	uev->ev_action = updatedone_action;
+
+	if (client->updatehandle == NULL) {
+		isc_nmhandle_attach(client->handle, &client->updatehandle);
+	}
 	isc_task_send(client->task, &event);

 	INSIST(ver == NULL);
@@ -3510,8 +3518,9 @@ updatedone_action(isc_task_t *task, isc_event_t *event) {

 	UNUSED(task);

-	INSIST(event->ev_type == DNS_EVENT_UPDATEDONE);
-	INSIST(task == client->task);
+	REQUIRE(event->ev_type == DNS_EVENT_UPDATEDONE);
+	REQUIRE(task == client->task);
+	REQUIRE(client->updatehandle == client->handle);

 	INSIST(client->nupdates > 0);
 	switch (uev->result) {
@@ -3528,16 +3537,18 @@ updatedone_action(isc_task_t *task, isc_event_t *event) {
 	if (uev->zone != NULL) {
 		dns_zone_detach(&uev->zone);
 	}
+
 	client->nupdates--;
+
 	respond(client, uev->result);
+
 	isc_event_free(&event);
-	isc_nmhandle_unref(client->handle);
+	isc_nmhandle_detach(&client->updatehandle);
 }

 /*%
 * Update forwarding support.
 */
-
 static void
 forward_fail(isc_task_t *task, isc_event_t *event) {
 	ns_client_t *client = (ns_client_t *)event->ev_arg;
@@ -3548,7 +3559,7 @@ forward_fail(isc_task_t *task, isc_event_t *event) {
 	client->nupdates--;
 	respond(client, DNS_R_SERVFAIL);
 	isc_event_free(&event);
-	isc_nmhandle_unref(client->handle);
+	isc_nmhandle_detach(&client->updatehandle);
 }

 static void
@@ -3568,6 +3579,10 @@ forward_callback(void *arg, isc_result_t result, dns_message_t *answer) {
 		uev->answer = answer;
 		inc_stats(client, zone, ns_statscounter_updaterespfwd);
 	}
+
+	if (client->updatehandle == NULL) {
+		isc_nmhandle_attach(client->handle, &client->updatehandle);
+	}
 	isc_task_send(client->task, ISC_EVENT_PTR(&uev));
 	dns_zone_detach(&zone);
 }
@@ -3584,7 +3599,7 @@ forward_done(isc_task_t *task, isc_event_t *event) {
 	ns_client_sendraw(client, uev->answer);
 	dns_message_destroy(&uev->answer);
 	isc_event_free(&event);
-	isc_nmhandle_unref(client->handle);
+	isc_nmhandle_detach(&client->updatehandle);
 }

 static void
@@ -3594,6 +3609,10 @@ forward_action(isc_task_t *task, isc_event_t *event) {
 	ns_client_t *client = (ns_client_t *)event->ev_arg;
 	isc_result_t result;

+	if (client->updatehandle == NULL) {
+		isc_nmhandle_attach(client->handle, &client->updatehandle);
+	}
+
 	result = dns_zone_forwardupdate(zone, client->message, forward_callback,
 					event);
 	if (result != ISC_R_SUCCESS) {
@@ -3636,7 +3655,9 @@ send_forward_event(ns_client_t *client, dns_zone_t *zone) {
 		      namebuf, classbuf);

 	dns_zone_gettask(zone, &zonetask);
-	isc_nmhandle_ref(client->handle);
+	if (client->updatehandle == NULL) {
+		isc_nmhandle_attach(client->handle, &client->updatehandle);
+	}
 	isc_task_send(zonetask, ISC_EVENT_PTR(&event));

 	if (event != NULL) {
@@ -1192,7 +1192,7 @@ failure:
 			      NS_LOGMODULE_XFER_OUT, ISC_LOG_DEBUG(3),
 			      "zone transfer setup failed");
 		ns_client_error(client, result);
-		isc_nmhandle_unref(client->handle);
+		isc_nmhandle_detach(&client->handle);
 	}
 }

@@ -1580,7 +1580,7 @@ sendstream(xfrout_ctx_t *xfr) {
 		xfrout_log(xfr, ISC_LOG_DEBUG(8), "sending IXFR UDP response");
 		ns_client_send(xfr->client);
 		xfr->stream->methods->pause(xfr->stream);
-		isc_nmhandle_unref(xfr->client->handle);
+		isc_nmhandle_detach(&xfr->client->handle);
 		xfrout_ctx_destroy(&xfr);
 		return;
 	}
@@ -1718,7 +1718,7 @@ xfrout_senddone(isc_nmhandle_t *handle, isc_result_t result, void *arg) {

 		xfrout_ctx_destroy(&xfr);
 		/* We're done, unreference the handle */
-		isc_nmhandle_unref(handle);
+		isc_nmhandle_detach(&handle);
 	}
 }

@@ -1744,7 +1744,7 @@ xfrout_maybe_destroy(xfrout_ctx_t *xfr) {
 	} else {
 #endif /* if 0 */
 	ns_client_drop(xfr->client, ISC_R_CANCELED);
-	isc_nmhandle_unref(xfr->client->handle);
+	isc_nmhandle_detach(&xfr->client->handle);
 	xfrout_ctx_destroy(&xfr);
 #if 0
 }
@@ -68,7 +68,7 @@ AC_DEFUN([AX_CHECK_OPENSSL], [
 	    AS_IF([test -f "$ssldir/include/openssl/ssl.h"],
 	        [
 		    OPENSSL_CFLAGS="-I$ssldir/include"
-                    OPENSSL_LIBS="-L$ssldir/lib -lcrypto"
+                    OPENSSL_LIBS="-L$ssldir/lib -lcrypto -lssl"
                    found=true
                    AC_MSG_RESULT([yes])
                    break
Author	SHA1	Message	Date
Evan Hunt	a768277b24	fixup! checkpoint: convert dig/host/nslookup to use the netmgr	2020-09-06 21:01:05 -07:00
Evan Hunt	4bd14084ea	fixup! add netmgr functions to support outgoing DNS queries	2020-09-06 18:34:42 -07:00
Evan Hunt	23106798be	fixup! checkpoint: convert dig/host/nslookup to use the netmgr	2020-09-06 18:34:25 -07:00
Evan Hunt	ad902a0f09	fixup! checkpoint: convert dig/host/nslookup to use the netmgr	2020-09-06 17:29:21 -07:00
Evan Hunt	ef33c28c0a	fixup! checkpoint: convert dig/host/nslookup to use the netmgr	2020-09-06 17:14:43 -07:00
Evan Hunt	87ba699f4f	fixup! checkpoint: convert dig/host/nslookup to use the netmgr	2020-09-06 14:39:11 -07:00
Evan Hunt	9c7683872b	fixup! add netmgr functions to support outgoing DNS queries	2020-09-06 14:39:00 -07:00
Evan Hunt	3c12b263e3	checkpoint: convert dig/host/nslookup to use the netmgr (only partially working at this point; basic queries over UDP and TCP do work now but edge and error cases may fail.)	2020-09-06 00:45:05 -07:00
Evan Hunt	31192119d7	shim code for UDP connect in older libuv versions uv_udp_connect() was added in libuv 1.27; we need a method of connecting UDP sockets for older versions.	2020-09-06 00:45:05 -07:00
Evan Hunt	a832204ec9	add netmgr functions to support outgoing DNS queries - isc_nm_tcpdnsconnect() sets up up an outgoing TCP DNS connection - isc_nm_udpconnect() sets up a connected UDP socket - isc_nm_read() now supports UDP; it reads a single datagram and then stops until the next time it's called. these functions will later be used to support outgoing queries in dig, dispatch, etc.	2020-09-06 00:45:05 -07:00
Evan Hunt	343c10f964	fix LD_WRAP tests I noticed that query_test and notify_test still had references to isc_nmhandle_unref(), which hadn't been causing problems because they weren't being run. this appears to be due to a logic reversal in configure; the test program returns 0 if -Wl,--wrap is not supported. after fixing this the unit tests needed to be cleaned up in order to pass.	2020-09-05 12:24:42 -07:00
Evan Hunt	04ef1f730e	work around perl error some versions of perl failed to run packet.pl because the 'last' keyword can't be used outside of a loop block. this commit changes the packet dumping code to a function so we can use 'return' instead.	2020-09-05 00:24:10 -07:00
Evan Hunt	5866c4a3be	fixup! change from isc_nmhandle_ref/unref to isc_nmhandle attach/detach	2020-09-04 23:43:38 -07:00
Evan Hunt	feddbcce10	fixup! assorted small netmgr-related changes	2020-09-04 23:27:51 -07:00
Evan Hunt	d427a20056	fixup! change from isc_nmhandle_ref/unref to isc_nmhandle attach/detach	2020-09-04 22:55:30 -07:00
Evan Hunt	6a6dd6ff0a	fixup! change from isc_nmhandle_ref/unref to isc_nmhandle attach/detach	2020-09-04 14:53:20 -07:00
Evan Hunt	f7f068dc64	CHANGES	2020-09-04 14:25:15 -07:00
Witold KręcickiandEvan Hunt	26c28bd7b5	tracing of active sockets and handles If NETMGR_TRACE is defined, we now maintain a list of active sockets in the netmgr object and a list of active handles in each socket object; by walking the list and printing `backtrace` in a debugger we can see where they were created. This will assist in debugging reference counting errors.	2020-09-04 14:25:15 -07:00
Evan Hunt	fb852e1d31	limit the time we wait for netmgr to be destroyed if more than 10 seconds pass while we wait for netmgr events to finish running on shutdown, something is almost certainly wrong and we should assert and crash.	2020-09-04 14:25:14 -07:00
Evan Hunt	b3bc26af7d	add more logging to the shutdown system test	2020-09-04 14:25:14 -07:00
Evan Hunt	32a453674a	change from isc_nmhandle_ref/unref to isc_nmhandle attach/detach Attaching and detaching handle pointers will make it easier to determine where and why reference counting errors have occurred. A handle needs to be referenced more than once when multiple asynchronous operations are in flight, so callers must now maintain multiple handle pointers for each pending operation. For example, ns_client objects now contain: - reqhandle: held while waiting for a request callback (query, notify, update) - sendhandle: held while waiting for a send callback - fetchhandle: held while waiting for a recursive fetch to complete - updatehandle: held while waiting for an update-forwarding task to complete control channel connection objects now contain: - readhandle: held while waiting for a read callback - sendhandle: held while waiting for a send callback - cmdhandle: held while an rndc command is running httpd connections contain: - readhandle: held while waiting for a read callback - sendhandle: held while waiting for a send callback	2020-09-04 14:25:06 -07:00
Witold KręcickiandEvan Hunt	c07df818c5	assorted small netmgr-related changes - cancelread functions now take handles instead of sockets - add a 'client' flag in socket objects, currently unused, to indicate whether it is to used as a client or server socket.	2020-09-04 14:25:06 -07:00
Witold KręcickiandEvan Hunt	96e119cd19	rename isc_nmsocket_t->tcphandle to statichandle	2020-09-04 14:25:06 -07:00