DO NOT MERGE: WIP: run multiple different resolvers

Previously, taskmgr was using ISC_LIST for keeping the enqueued tasks
and the ISC_LIST have an inherent property that the task would not get
enqueued twice.  When taskmgr switched to using netmgr, we are using
isc_queue_t which can enqueue same item multiple times, and this was
causing problem when the previous isc_task_run() would find the task to
be empty and shutting down.  It would directly destroy the task and
leave the following netievents with already freed task.

This commit adds references counting around task_ready(++) and
task_run(--) and adds isc_task_ready() public API wrapper around
task_ready, so we can call it from netmgr when the event quantum would
be reached.

bin/delv/delv.c

@@ -34,6 +34,7 @@
 #include <isc/hex.h>
 #include <isc/lib.h>
 #include <isc/log.h>
+#include <isc/managers.h>
 #include <isc/md.h>
 #include <isc/mem.h>
 #include <isc/netmgr.h>
@@ -1761,10 +1762,8 @@ main(int argc, char *argv[]) {
 	isc_mem_create(&mctx);
 
 	CHECK(isc_appctx_create(mctx, &actx));
-	netmgr = isc_nm_start(mctx, 1);
-	CHECK(isc_taskmgr_create(mctx, 0, netmgr, &taskmgr));
-	CHECK(isc_socketmgr_create(mctx, &socketmgr));
-	CHECK(isc_timermgr_create(mctx, &timermgr));
+	isc_managers_create(mctx, 1, 0, 0, &netmgr, &taskmgr, &timermgr,
+			    &socketmgr);
 
 	parse_args(argc, argv);
 
@@ -1867,18 +1866,7 @@ cleanup:
 	if (client != NULL) {
 		dns_client_destroy(&client);
 	}
-	if (taskmgr != NULL) {
-		isc_taskmgr_destroy(&taskmgr);
-	}
-	if (netmgr != NULL) {
-		isc_nm_destroy(&netmgr);
-	}
-	if (timermgr != NULL) {
-		isc_timermgr_destroy(&timermgr);
-	}
-	if (socketmgr != NULL) {
-		isc_socketmgr_destroy(&socketmgr);
-	}
+	isc_managers_destroy(&netmgr, &taskmgr, &timermgr, &socketmgr);
 	if (actx != NULL) {
 		isc_appctx_destroy(&actx);
 	}

bin/dig/dighost.c

@@ -41,6 +41,7 @@
 #include <isc/hex.h>
 #include <isc/lang.h>
 #include <isc/log.h>
+#include <isc/managers.h>
 #include <isc/netaddr.h>
 #include <isc/netdb.h>
 #include <isc/nonce.h>
@@ -106,9 +107,9 @@ unsigned int timeout = 0;
 unsigned int extrabytes;
 isc_mem_t *mctx = NULL;
 isc_log_t *lctx = NULL;
+isc_nm_t *netmgr = NULL;
 isc_taskmgr_t *taskmgr = NULL;
 isc_task_t *global_task = NULL;
-isc_nm_t *netmgr = NULL;
 isc_sockaddr_t localaddr;
 isc_refcount_t sendcount = ATOMIC_VAR_INIT(0);
 isc_refcount_t recvcount = ATOMIC_VAR_INIT(0);
@@ -226,8 +227,9 @@ void (*dighost_shutdown)(void);
 
 /* forward declarations */
 
+#define cancel_lookup(l) _cancel_lookup(l, __FILE__, __LINE__)
 static void
-cancel_lookup(dig_lookup_t *lookup);
+_cancel_lookup(dig_lookup_t *lookup, const char *file, unsigned int line);
 
 static void
 recv_done(isc_nmhandle_t *handle, isc_result_t eresult, isc_region_t *region,
@@ -1360,10 +1362,7 @@ setup_libs(void) {
 
 	isc_log_setdebuglevel(lctx, 0);
 
-	netmgr = isc_nm_start(mctx, 1);
-
-	result = isc_taskmgr_create(mctx, 0, netmgr, &taskmgr);
-	check_result(result, "isc_taskmgr_create");
+	isc_managers_create(mctx, 1, 0, 0, &netmgr, &taskmgr, NULL, NULL);
 
 	result = isc_task_create(taskmgr, 0, &global_task);
 	check_result(result, "isc_task_create");
@@ -1696,6 +1695,9 @@ _query_detach(dig_query_t **queryp, const char *file, unsigned int line) {
 	      isc_refcount_current(&query->references) - 1);
 
 	if (isc_refcount_decrement(&query->references) == 1) {
+		INSIST(query->readhandle == NULL);
+		INSIST(query->sendhandle == NULL);
+
 		if (ISC_LINK_LINKED(query, link)) {
 			ISC_LIST_UNLINK(lookup->q, query, link);
 		}
@@ -1752,13 +1754,18 @@ start_lookup(void) {
  * decremented, current_lookup will not be set to NULL.)
  */
 static void
-clear_current_lookup() {
+clear_current_lookup(void) {
 	dig_lookup_t *lookup = current_lookup;
 
 	INSIST(!free_now);
 
 	debug("clear_current_lookup()");
 
+	if (lookup == NULL) {
+		debug("current_lookup is already detached");
+		return;
+	}
+
 	if (ISC_LIST_HEAD(lookup->q) != NULL) {
 		debug("still have a worker");
 		return;
@@ -2671,11 +2678,12 @@ send_done(isc_nmhandle_t *handle, isc_result_t eresult, void *arg) {
 /*%
  * Cancel a lookup, sending canceling reads on all existing sockets.
  */
+
 static void
-cancel_lookup(dig_lookup_t *lookup) {
+_cancel_lookup(dig_lookup_t *lookup, const char *file, unsigned int line) {
 	dig_query_t *query, *next;
 
-	debug("cancel_lookup()");
+	debug("%s:%u:%s()", file, line, __func__);
 	query = ISC_LIST_HEAD(lookup->q);
 	while (query != NULL) {
 		REQUIRE(DIG_VALID_QUERY(query));
@@ -2941,6 +2949,7 @@ static void
 start_udp(dig_query_t *query) {
 	isc_result_t result;
 	dig_query_t *next = NULL;
+	dig_query_t *connectquery = NULL;
 
 	REQUIRE(DIG_VALID_QUERY(query));
 
@@ -2992,8 +3001,10 @@ start_udp(dig_query_t *query) {
 		}
 	}
 
+	query_attach(query, &connectquery);
 	isc_nm_udpconnect(netmgr, (isc_nmiface_t *)&localaddr,
-			  (isc_nmiface_t *)&query->sockaddr, udp_ready, query,
+			  (isc_nmiface_t *)&query->sockaddr, udp_ready,
+			  connectquery,
 			  (timeout ? timeout : UDP_TIMEOUT) * 1000, 0);
 }
 
@@ -3568,16 +3579,19 @@ recv_done(isc_nmhandle_t *handle, isc_result_t eresult, isc_region_t *region,
 	      region, arg);
 
 	LOCK_LOOKUP;
-	lookup_attach(query->lookup, &l);
 
 	isc_refcount_decrement0(&recvcount);
 	debug("recvcount=%" PRIuFAST32, isc_refcount_current(&recvcount));
 
 	if (eresult == ISC_R_CANCELED) {
 		debug("recv_done: cancel");
-		goto detach_query;
+		isc_nmhandle_detach(&query->readhandle);
+		query_detach(&query);
+		return;
 	}
 
+	lookup_attach(query->lookup, &l);
+
 	if (query->lookup->use_usec) {
 		TIME_NOW_HIRES(&query->time_recv);
 	} else {
@@ -4188,19 +4202,23 @@ cancel_all(void) {
 		return;
 	}
 	atomic_store(&cancel_now, true);
-	if (current_lookup != NULL) {
+	while (current_lookup != NULL) {
 		for (q = ISC_LIST_HEAD(current_lookup->q); q != NULL; q = nq) {
 			nq = ISC_LIST_NEXT(q, link);
 			debug("canceling pending query %p, belonging to %p", q,
 			      current_lookup);
 			if (q->readhandle != NULL) {
-				isc_refcount_decrement0(&recvcount);
-				debug("recvcount=%" PRIuFAST32,
-				      isc_refcount_current(&recvcount));
+				isc_nm_cancelread(q->readhandle);
 			}
 			query_detach(&q);
 		}
-		lookup_detach(&current_lookup);
+
+		/*
+		 * current_lookup could have been detached via query_detach().
+		 */
+		if (current_lookup != NULL) {
+			lookup_detach(&current_lookup);
+		}
 	}
 	l = ISC_LIST_HEAD(lookup_list);
 	while (l != NULL) {
@@ -4226,20 +4244,8 @@ destroy_libs(void) {
 		debug("freeing task");
 		isc_task_detach(&global_task);
 	}
-	/*
-	 * The taskmgr_destroy() and isc_nm_destroy() calls block until
-	 * all events are cleared.
-	 */
-	if (taskmgr != NULL) {
-		debug("freeing taskmgr");
-		isc_taskmgr_destroy(&taskmgr);
-	}
 
-	debug("closing down netmgr");
-	isc_nm_closedown(netmgr);
-
-	debug("destroy netmgr");
-	isc_nm_destroy(&netmgr);
+	isc_managers_destroy(&netmgr, &taskmgr, NULL, NULL);
 
 	LOCK_LOOKUP;
 	isc_refcount_destroy(&recvcount);

bin/dnssec/dnssec-signzone.c

@@ -40,6 +40,7 @@
 #include <isc/file.h>
 #include <isc/hash.h>
 #include <isc/hex.h>
+#include <isc/managers.h>
 #include <isc/md.h>
 #include <isc/mem.h>
 #include <isc/mutex.h>
@@ -3963,13 +3964,7 @@ main(int argc, char *argv[]) {
 	print_time(outfp);
 	print_version(outfp);
 
-	netmgr = isc_nm_start(mctx, ntasks);
-
-	result = isc_taskmgr_create(mctx, 0, netmgr, &taskmgr);
-	if (result != ISC_R_SUCCESS) {
-		fatal("failed to create task manager: %s",
-		      isc_result_totext(result));
-	}
+	isc_managers_create(mctx, ntasks, 0, 0, &netmgr, &taskmgr, NULL, NULL);
 
 	master = NULL;
 	result = isc_task_create(taskmgr, 0, &master);
@@ -4020,8 +4015,7 @@ main(int argc, char *argv[]) {
 	for (i = 0; i < (int)ntasks; i++) {
 		isc_task_detach(&tasks[i]);
 	}
-	isc_taskmgr_destroy(&taskmgr);
-	isc_nm_destroy(&netmgr);
+	isc_managers_destroy(&netmgr, &taskmgr, NULL, NULL);
 	isc_mem_put(mctx, tasks, ntasks * sizeof(isc_task_t *));
 	postsign();
 	TIME_NOW(&sign_finish);

bin/named/controlconf.c

@@ -1164,10 +1164,10 @@ add_listener(named_controls_t *cp, controllistener_t **listenerp,
 	}
 #endif
 
-	CHECK(isc_nm_listentcp(named_g_nm, (isc_nmiface_t *)&listener->address,
-			       control_newconn, listener,
-			       sizeof(controlconnection_t), 5, NULL,
-			       &listener->sock));
+	CHECK(isc_nm_listentcp(
+		named_g_netmgr, (isc_nmiface_t *)&listener->address,
+		control_newconn, listener, sizeof(controlconnection_t), 5, NULL,
+		&listener->sock));
 #if 0
 	/* XXX: no unix socket support yet */
 	if (type == isc_socktype_unix) {

bin/named/include/named/globals.h

@@ -62,7 +62,7 @@ EXTERN bool named_g_run_done INIT(false);
  */
 EXTERN isc_timermgr_t *named_g_timermgr INIT(NULL);
 EXTERN isc_socketmgr_t *named_g_socketmgr INIT(NULL);
-EXTERN isc_nm_t *named_g_nm INIT(NULL);
+EXTERN isc_nm_t *named_g_netmgr INIT(NULL);
 EXTERN cfg_parser_t *named_g_parser INIT(NULL);
 EXTERN cfg_parser_t *named_g_addparser INIT(NULL);
 EXTERN const char *named_g_version     INIT(PACKAGE_VERSION);

bin/named/main.c

@@ -31,6 +31,7 @@
 #include <isc/hash.h>
 #include <isc/hp.h>
 #include <isc/httpd.h>
+#include <isc/managers.h>
 #include <isc/netmgr.h>
 #include <isc/os.h>
 #include <isc/platform.h>
@@ -937,45 +938,17 @@ create_managers(void) {
 		      "using %u UDP listener%s per interface", named_g_udpdisp,
 		      named_g_udpdisp == 1 ? "" : "s");
 
-	/*
-	 * We have ncpus network threads, ncpus worker threads, ncpus
-	 * old network threads - make it 4x just to be safe. The memory
-	 * impact is negligible.
-	 */
-	isc_hp_init(4 * named_g_cpus);
-	named_g_nm = isc_nm_start(named_g_mctx, named_g_cpus);
-	if (named_g_nm == NULL) {
-		UNEXPECTED_ERROR(__FILE__, __LINE__, "isc_nm_start() failed");
-		return (ISC_R_UNEXPECTED);
+	result = isc_managers_create(named_g_mctx, named_g_cpus,
+				     0 /* quantum */, maxsocks, &named_g_netmgr,
+				     &named_g_taskmgr, &named_g_timermgr,
+				     &named_g_socketmgr);
+	if (result != ISC_R_SUCCESS) {
+		return (result);
 	}
 
-	result = isc_taskmgr_create(named_g_mctx, 0, named_g_nm,
-				    &named_g_taskmgr);
-	if (result != ISC_R_SUCCESS) {
-		UNEXPECTED_ERROR(__FILE__, __LINE__,
-				 "isc_taskmgr_create() failed: %s",
-				 isc_result_totext(result));
-		return (ISC_R_UNEXPECTED);
-	}
-
-	result = isc_timermgr_create(named_g_mctx, &named_g_timermgr);
-	if (result != ISC_R_SUCCESS) {
-		UNEXPECTED_ERROR(__FILE__, __LINE__,
-				 "isc_timermgr_create() failed: %s",
-				 isc_result_totext(result));
-		return (ISC_R_UNEXPECTED);
-	}
-
-	result = isc_socketmgr_create2(named_g_mctx, &named_g_socketmgr,
-				       maxsocks, named_g_cpus);
-	if (result != ISC_R_SUCCESS) {
-		UNEXPECTED_ERROR(__FILE__, __LINE__,
-				 "isc_socketmgr_create() failed: %s",
-				 isc_result_totext(result));
-		return (ISC_R_UNEXPECTED);
-	}
 	isc_socketmgr_maxudp(named_g_socketmgr, maxudp);
-	isc_nm_maxudp(named_g_nm, maxudp);
+	isc_nm_maxudp(named_g_netmgr, maxudp);
+
 	result = isc_socketmgr_getmaxsockets(named_g_socketmgr, &socks);
 	if (result == ISC_R_SUCCESS) {
 		isc_log_write(named_g_lctx, NAMED_LOGCATEGORY_GENERAL,
@@ -988,21 +961,8 @@ create_managers(void) {
 
 static void
 destroy_managers(void) {
-	/*
-	 * isc_nm_closedown() closes all active connections, freeing
-	 * attached clients and other resources and preventing new
-	 * connections from being established, but it not does not
-	 * stop all processing or destroy the netmgr yet.
-	 */
-	isc_nm_closedown(named_g_nm);
-
-	/*
-	 * isc_taskmgr_destroy() will block until all tasks have exited.
-	 */
-	isc_taskmgr_destroy(&named_g_taskmgr);
-	isc_nm_destroy(&named_g_nm);
-	isc_timermgr_destroy(&named_g_timermgr);
-	isc_socketmgr_destroy(&named_g_socketmgr);
+	isc_managers_destroy(&named_g_netmgr, &named_g_taskmgr,
+			     &named_g_timermgr, &named_g_socketmgr);
 }
 
 static void

bin/named/server.c

@@ -8840,7 +8840,8 @@ load_configuration(const char *filename, named_server_t *server,
 		advertised = MAX_ADVERTISED_TIMEOUT;
 	}
 
-	isc_nm_settimeouts(named_g_nm, initial, idle, keepalive, advertised);
+	isc_nm_settimeouts(named_g_netmgr, initial, idle, keepalive,
+			   advertised);
 
 	/*
 	 * Configure sets of UDP query source ports.
@@ -9950,7 +9951,7 @@ run_server(isc_task_t *task, isc_event_t *event) {
 
 	CHECKFATAL(ns_interfacemgr_create(
 			   named_g_mctx, server->sctx, named_g_taskmgr,
-			   named_g_timermgr, named_g_socketmgr, named_g_nm,
+			   named_g_timermgr, named_g_socketmgr, named_g_netmgr,
 			   named_g_dispatchmgr, server->task, named_g_udpdisp,
 			   geoip, named_g_cpus, &server->interfacemgr),
 		   "creating interface manager");
@@ -10220,7 +10221,7 @@ named_server_create(isc_mem_t *mctx, named_server_t **serverp) {
 
 	CHECKFATAL(dns_zonemgr_create(named_g_mctx, named_g_taskmgr,
 				      named_g_timermgr, named_g_socketmgr,
-				      named_g_nm, &server->zonemgr),
+				      named_g_netmgr, &server->zonemgr),
 		   "dns_zonemgr_create");
 	CHECKFATAL(dns_zonemgr_setsize(server->zonemgr, 1000), "dns_zonemgr_"
 							       "setsize");
@@ -10260,7 +10261,7 @@ named_server_create(isc_mem_t *mctx, named_server_t **serverp) {
 				    isc_sockstatscounter_max),
 		   "isc_stats_create");
 	isc_socketmgr_setstats(named_g_socketmgr, server->sockstats);
-	isc_nm_setstats(named_g_nm, server->sockstats);
+	isc_nm_setstats(named_g_netmgr, server->sockstats);
 
 	CHECKFATAL(isc_stats_create(named_g_mctx, &server->zonestats,
 				    dns_zonestatscounter_max),
@@ -16342,7 +16343,7 @@ named_server_tcptimeouts(isc_lex_t *lex, isc_buffer_t **text) {
 		return (ISC_R_UNEXPECTEDEND);
 	}
 
-	isc_nm_gettimeouts(named_g_nm, &initial, &idle, &keepalive,
+	isc_nm_gettimeouts(named_g_netmgr, &initial, &idle, &keepalive,
 			   &advertised);
 
 	/* Look for optional arguments. */
@@ -16396,7 +16397,7 @@ named_server_tcptimeouts(isc_lex_t *lex, isc_buffer_t **text) {
 		result = isc_task_beginexclusive(named_g_server->task);
 		RUNTIME_CHECK(result == ISC_R_SUCCESS);
 
-		isc_nm_settimeouts(named_g_nm, initial, idle, keepalive,
+		isc_nm_settimeouts(named_g_netmgr, initial, idle, keepalive,
 				   advertised);
 
 		isc_task_endexclusive(named_g_server->task);

bin/named/statschannel.c

@@ -3612,7 +3612,7 @@ add_listener(named_server_t *server, named_statschannel_t **listenerp,
 		CHECK(ISC_R_FAMILYNOSUPPORT);
 	}
 
-	CHECK(isc_httpdmgr_create(named_g_nm, server->mctx, addr, client_ok,
+	CHECK(isc_httpdmgr_create(named_g_netmgr, server->mctx, addr, client_ok,
 				  destroy_listener, listener,
 				  &listener->httpdmgr));
 

bin/nsupdate/nsupdate.c

@@ -29,6 +29,7 @@
 #include <isc/hash.h>
 #include <isc/lex.h>
 #include <isc/log.h>
+#include <isc/managers.h>
 #include <isc/mem.h>
 #include <isc/nonce.h>
 #include <isc/parseint.h>
@@ -923,16 +924,8 @@ setup_system(void) {
 	result = dns_dispatchmgr_create(gmctx, &dispatchmgr);
 	check_result(result, "dns_dispatchmgr_create");
 
-	result = isc_socketmgr_create(gmctx, &socketmgr);
-	check_result(result, "dns_socketmgr_create");
-
-	result = isc_timermgr_create(gmctx, &timermgr);
-	check_result(result, "dns_timermgr_create");
-
-	netmgr = isc_nm_start(gmctx, 1);
-
-	result = isc_taskmgr_create(gmctx, 0, netmgr, &taskmgr);
-	check_result(result, "isc_taskmgr_create");
+	isc_managers_create(gmctx, 1, 0, 0, &netmgr, &taskmgr, &timermgr,
+			    &socketmgr);
 
 	result = isc_task_create(taskmgr, 0, &global_task);
 	check_result(result, "isc_task_create");
@@ -3325,21 +3318,12 @@ cleanup(void) {
 		dst_key_free(&sig0key);
 	}
 
-	ddebug("Shutting down task manager");
-	isc_taskmgr_destroy(&taskmgr);
-
-	ddebug("Shutting down network manager");
-	isc_nm_destroy(&netmgr);
+	ddebug("Shutting down managers");
+	isc_managers_destroy(&netmgr, &taskmgr, &timermgr, &socketmgr);
 
 	ddebug("Destroying event");
 	isc_event_free(&global_event);
 
-	ddebug("Shutting down socket manager");
-	isc_socketmgr_destroy(&socketmgr);
-
-	ddebug("Shutting down timer manager");
-	isc_timermgr_destroy(&timermgr);
-
 #ifdef HAVE_GSSAPI
 	/*
 	 * Cleanup GSSAPI resources after taskmgr has been destroyed.

bin/rndc/rndc.c

@@ -22,6 +22,7 @@
 #include <isc/commandline.h>
 #include <isc/file.h>
 #include <isc/log.h>
+#include <isc/managers.h>
 #include <isc/mem.h>
 #include <isc/net.h>
 #include <isc/netmgr.h>
@@ -59,6 +60,7 @@
 const char *progname = NULL;
 bool verbose;
 
+static isc_nm_t *netmgr = NULL;
 static isc_taskmgr_t *taskmgr = NULL;
 static isc_task_t *rndc_task = NULL;
 
@@ -72,7 +74,6 @@ static bool local4set = false, local6set = false;
 static int nserveraddrs;
 static int currentaddr = 0;
 static unsigned int remoteport = 0;
-static isc_nm_t *netmgr = NULL;
 static isc_buffer_t *databuf = NULL;
 static isccc_ccmsg_t rndc_ccmsg;
 static uint32_t algorithm;
@@ -1030,9 +1031,7 @@ main(int argc, char **argv) {
 	serial = isc_random32();
 
 	isc_mem_create(&rndc_mctx);
-	netmgr = isc_nm_start(rndc_mctx, 1);
-	DO("create task manager",
-	   isc_taskmgr_create(rndc_mctx, 0, netmgr, &taskmgr));
+	isc_managers_create(rndc_mctx, 1, 0, 0, &netmgr, &taskmgr, NULL, NULL);
 	DO("create task", isc_task_create(taskmgr, 0, &rndc_task));
 	isc_log_create(rndc_mctx, &log, &logconfig);
 	isc_log_setcontext(log);
@@ -1089,9 +1088,7 @@ main(int argc, char **argv) {
 	}
 
 	isc_task_detach(&rndc_task);
-	isc_taskmgr_destroy(&taskmgr);
-
-	isc_nm_closedown(netmgr);
+	isc_managers_destroy(&netmgr, &taskmgr, NULL, NULL);
 
 	/*
 	 * Note: when TCP connections are shut down, there will be a final
@@ -1101,8 +1098,6 @@ main(int argc, char **argv) {
 	 */
 	isccc_ccmsg_invalidate(&rndc_ccmsg);
 
-	isc_nm_destroy(&netmgr);
-
 	isc_log_destroy(&log);
 	isc_log_setcontext(NULL);
 

bin/tests/system/pipelined/pipequeries.c

@@ -20,6 +20,7 @@
 #include <isc/commandline.h>
 #include <isc/hash.h>
 #include <isc/log.h>
+#include <isc/managers.h>
 #include <isc/mem.h>
 #include <isc/net.h>
 #include <isc/parseint.h>
@@ -208,10 +209,10 @@ main(int argc, char *argv[]) {
 	isc_logconfig_t *lcfg;
 	isc_nm_t *netmgr = NULL;
 	isc_taskmgr_t *taskmgr = NULL;
-	isc_task_t *task;
-	isc_timermgr_t *timermgr;
-	isc_socketmgr_t *socketmgr;
-	dns_dispatchmgr_t *dispatchmgr;
+	isc_task_t *task = NULL;
+	isc_timermgr_t *timermgr = NULL;
+	isc_socketmgr_t *socketmgr = NULL;
+	dns_dispatchmgr_t *dispatchmgr = NULL;
 	unsigned int attrs, attrmask;
 	dns_dispatch_t *dispatchv4;
 	dns_view_t *view;
@@ -277,17 +278,10 @@ main(int argc, char *argv[]) {
 
 	RUNCHECK(dst_lib_init(mctx, NULL));
 
-	netmgr = isc_nm_start(mctx, 1);
+	isc_managers_create(mctx, 1, 0, 0, &netmgr, &taskmgr, &timermgr,
+			    &socketmgr);
 
-	RUNCHECK(isc_taskmgr_create(mctx, 0, netmgr, &taskmgr));
-	task = NULL;
 	RUNCHECK(isc_task_create(taskmgr, 0, &task));
-	timermgr = NULL;
-
-	RUNCHECK(isc_timermgr_create(mctx, &timermgr));
-	socketmgr = NULL;
-	RUNCHECK(isc_socketmgr_create(mctx, &socketmgr));
-	dispatchmgr = NULL;
 	RUNCHECK(dns_dispatchmgr_create(mctx, &dispatchmgr));
 
 	attrs = DNS_DISPATCHATTR_UDP | DNS_DISPATCHATTR_MAKEQUERY |
@@ -318,13 +312,10 @@ main(int argc, char *argv[]) {
 	dns_dispatch_detach(&dispatchv4);
 	dns_dispatchmgr_destroy(&dispatchmgr);
 
-	isc_socketmgr_destroy(&socketmgr);
-	isc_timermgr_destroy(&timermgr);
-
 	isc_task_shutdown(task);
 	isc_task_detach(&task);
-	isc_taskmgr_destroy(&taskmgr);
-	isc_nm_destroy(&netmgr);
+
+	isc_managers_destroy(&netmgr, &taskmgr, &timermgr, &socketmgr);
 
 	dst_lib_destroy();
 

bin/tests/system/resolve.c

@@ -29,6 +29,7 @@
 #include <isc/buffer.h>
 #include <isc/commandline.h>
 #include <isc/lib.h>
+#include <isc/managers.h>
 #include <isc/mem.h>
 #include <isc/print.h>
 #include <isc/sockaddr.h>
@@ -66,25 +67,8 @@ isc_timermgr_t *ctxs_timermgr = NULL;
 
 static void
 ctxs_destroy(void) {
-	if (ctxs_netmgr != NULL) {
-		isc_nm_closedown(ctxs_netmgr);
-	}
-
-	if (ctxs_taskmgr != NULL) {
-		isc_taskmgr_destroy(&ctxs_taskmgr);
-	}
-
-	if (ctxs_netmgr != NULL) {
-		isc_nm_destroy(&ctxs_netmgr);
-	}
-
-	if (ctxs_timermgr != NULL) {
-		isc_timermgr_destroy(&ctxs_timermgr);
-	}
-
-	if (ctxs_socketmgr != NULL) {
-		isc_socketmgr_destroy(&ctxs_socketmgr);
-	}
+	isc_managers_destroy(&ctxs_netmgr, &ctxs_taskmgr, &ctxs_timermgr,
+			     &ctxs_socketmgr);
 
 	if (ctxs_actx != NULL) {
 		isc_appctx_destroy(&ctxs_actx);
@@ -106,22 +90,8 @@ ctxs_init(void) {
 		goto fail;
 	}
 
-	ctxs_netmgr = isc_nm_start(ctxs_mctx, 1);
-
-	result = isc_taskmgr_create(ctxs_mctx, 0, ctxs_netmgr, &ctxs_taskmgr);
-	if (result != ISC_R_SUCCESS) {
-		goto fail;
-	}
-
-	result = isc_socketmgr_create(ctxs_mctx, &ctxs_socketmgr);
-	if (result != ISC_R_SUCCESS) {
-		goto fail;
-	}
-
-	result = isc_timermgr_create(ctxs_mctx, &ctxs_timermgr);
-	if (result != ISC_R_SUCCESS) {
-		goto fail;
-	}
+	isc_managers_create(ctxs_mctx, 1, 0, 0, &ctxs_netmgr, &ctxs_taskmgr,
+			    &ctxs_timermgr, &ctxs_socketmgr);
 
 	return (ISC_R_SUCCESS);
 

bin/tests/system/tkey/keycreate.c

@@ -16,6 +16,7 @@
 #include <isc/base64.h>
 #include <isc/hash.h>
 #include <isc/log.h>
+#include <isc/managers.h>
 #include <isc/mem.h>
 #include <isc/nonce.h>
 #include <isc/print.h>
@@ -191,21 +192,21 @@ sendquery(isc_task_t *task, isc_event_t *event) {
 
 int
 main(int argc, char *argv[]) {
-	char *ourkeyname;
+	char *ourkeyname = NULL;
 	isc_nm_t *netmgr = NULL;
 	isc_taskmgr_t *taskmgr = NULL;
-	isc_timermgr_t *timermgr;
-	isc_socketmgr_t *socketmgr;
-	isc_socket_t *sock;
+	isc_timermgr_t *timermgr = NULL;
+	isc_socketmgr_t *socketmgr = NULL;
+	isc_socket_t *sock = NULL;
 	unsigned int attrs, attrmask;
 	isc_sockaddr_t bind_any;
-	dns_dispatchmgr_t *dispatchmgr;
-	dns_dispatch_t *dispatchv4;
-	dns_view_t *view;
-	dns_tkeyctx_t *tctx;
-	isc_log_t *log;
-	isc_logconfig_t *logconfig;
-	isc_task_t *task;
+	dns_dispatchmgr_t *dispatchmgr = NULL;
+	dns_dispatch_t *dispatchv4 = NULL;
+	dns_view_t *view = NULL;
+	dns_tkeyctx_t *tctx = NULL;
+	isc_log_t *log = NULL;
+	isc_logconfig_t *logconfig = NULL;
+	isc_task_t *task = NULL;
 	isc_result_t result;
 	int type;
 
@@ -235,17 +236,12 @@ main(int argc, char *argv[]) {
 
 	RUNCHECK(dst_lib_init(mctx, NULL));
 
-	netmgr = isc_nm_start(mctx, 1);
+	isc_managers_create(mctx, 1, 0, 0, &netmgr, &taskmgr, &timermgr,
+			    &socketmgr);
 
-	RUNCHECK(isc_taskmgr_create(mctx, 0, netmgr, &taskmgr));
-	task = NULL;
 	RUNCHECK(isc_task_create(taskmgr, 0, &task));
-	timermgr = NULL;
-	RUNCHECK(isc_timermgr_create(mctx, &timermgr));
-	socketmgr = NULL;
-	RUNCHECK(isc_socketmgr_create(mctx, &socketmgr));
-	dispatchmgr = NULL;
 	RUNCHECK(dns_dispatchmgr_create(mctx, &dispatchmgr));
+
 	isc_sockaddr_any(&bind_any);
 	attrs = DNS_DISPATCHATTR_UDP | DNS_DISPATCHATTR_MAKEQUERY |
 		DNS_DISPATCHATTR_IPV4;
@@ -293,11 +289,8 @@ main(int argc, char *argv[]) {
 	dns_dispatchmgr_destroy(&dispatchmgr);
 	isc_task_shutdown(task);
 	isc_task_detach(&task);
-	isc_taskmgr_destroy(&taskmgr);
-	isc_nm_destroy(&netmgr);
 	isc_socket_detach(&sock);
-	isc_socketmgr_destroy(&socketmgr);
-	isc_timermgr_destroy(&timermgr);
+	isc_managers_destroy(&netmgr, &taskmgr, &timermgr, &socketmgr);
 
 	dst_key_free(&ourkey);
 	dns_tsigkey_detach(&initialkey);

bin/tests/system/tkey/keydelete.c

@@ -16,6 +16,7 @@
 #include <isc/base64.h>
 #include <isc/hash.h>
 #include <isc/log.h>
+#include <isc/managers.h>
 #include <isc/mem.h>
 #include <isc/netmgr.h>
 #include <isc/print.h>
@@ -135,22 +136,22 @@ sendquery(isc_task_t *task, isc_event_t *event) {
 
 int
 main(int argc, char **argv) {
-	char *keyname;
-	isc_nm_t *netmgr;
+	char *keyname = NULL;
+	isc_nm_t *netmgr = NULL;
 	isc_taskmgr_t *taskmgr = NULL;
-	isc_timermgr_t *timermgr;
-	isc_socketmgr_t *socketmgr;
-	isc_socket_t *sock;
+	isc_timermgr_t *timermgr = NULL;
+	isc_socketmgr_t *socketmgr = NULL;
+	isc_socket_t *sock = NULL;
 	unsigned int attrs, attrmask;
 	isc_sockaddr_t bind_any;
-	dns_dispatchmgr_t *dispatchmgr;
-	dns_dispatch_t *dispatchv4;
-	dns_view_t *view;
-	dns_tkeyctx_t *tctx;
-	dst_key_t *dstkey;
-	isc_log_t *log;
-	isc_logconfig_t *logconfig;
-	isc_task_t *task;
+	dns_dispatchmgr_t *dispatchmgr = NULL;
+	dns_dispatch_t *dispatchv4 = NULL;
+	dns_view_t *view = NULL;
+	dns_tkeyctx_t *tctx = NULL;
+	dst_key_t *dstkey = NULL;
+	isc_log_t *log = NULL;
+	isc_logconfig_t *logconfig = NULL;
+	isc_task_t *task = NULL;
 	isc_result_t result;
 	int type;
 
@@ -179,16 +180,10 @@ main(int argc, char **argv) {
 
 	RUNCHECK(dst_lib_init(mctx, NULL));
 
-	netmgr = isc_nm_start(mctx, 1);
+	isc_managers_create(mctx, 1, 0, 0, &netmgr, &taskmgr, &timermgr,
+			    &socketmgr);
 
-	RUNCHECK(isc_taskmgr_create(mctx, 0, netmgr, &taskmgr));
-	task = NULL;
 	RUNCHECK(isc_task_create(taskmgr, 0, &task));
-	timermgr = NULL;
-	RUNCHECK(isc_timermgr_create(mctx, &timermgr));
-	socketmgr = NULL;
-	RUNCHECK(isc_socketmgr_create(mctx, &socketmgr));
-	dispatchmgr = NULL;
 	RUNCHECK(dns_dispatchmgr_create(mctx, &dispatchmgr));
 	isc_sockaddr_any(&bind_any);
 	attrs = DNS_DISPATCHATTR_UDP | DNS_DISPATCHATTR_MAKEQUERY |
@@ -237,11 +232,8 @@ main(int argc, char **argv) {
 	dns_dispatchmgr_destroy(&dispatchmgr);
 	isc_task_shutdown(task);
 	isc_task_detach(&task);
-	isc_taskmgr_destroy(&taskmgr);
-	isc_nm_destroy(&netmgr);
 	isc_socket_detach(&sock);
-	isc_socketmgr_destroy(&socketmgr);
-	isc_timermgr_destroy(&timermgr);
+	isc_managers_destroy(&netmgr, &taskmgr, &timermgr, &socketmgr);
 
 	dns_tsigkeyring_detach(&ring);
 

bin/tests/test_client.c

@@ -23,6 +23,7 @@
 #include <sys/types.h>
 #include <unistd.h>
 
+#include <isc/managers.h>
 #include <isc/mem.h>
 #include <isc/netaddr.h>
 #include <isc/netmgr.h>
@@ -307,7 +308,7 @@ setup(void) {
 
 	isc_mem_create(&mctx);
 
-	netmgr = isc_nm_start(mctx, workers);
+	isc_managers_create(mctx, workers, 0, 0, &netmgr, NULL, NULL, NULL);
 }
 
 static void
@@ -316,7 +317,7 @@ teardown(void) {
 		close(out);
 	}
 
-	isc_nm_destroy(&netmgr);
+	isc_managers_destroy(&netmgr, NULL, NULL, NULL);
 	isc_mem_destroy(&mctx);
 	if (tls_ctx) {
 		isc_tlsctx_free(&tls_ctx);
@@ -464,8 +465,6 @@ run(void) {
 	}
 
 	waitforsignal();
-
-	isc_nm_closedown(netmgr);
 }
 
 int

bin/tests/test_server.c

@@ -17,6 +17,7 @@
 #include <stdlib.h>
 #include <strings.h>
 
+#include <isc/managers.h>
 #include <isc/mem.h>
 #include <isc/netaddr.h>
 #include <isc/netmgr.h>
@@ -188,12 +189,12 @@ setup(void) {
 
 	isc_mem_create(&mctx);
 
-	netmgr = isc_nm_start(mctx, workers);
+	isc_managers_create(mctx, workers, 0, 0, &netmgr, NULL, NULL, NULL);
 }
 
 static void
 teardown(void) {
-	isc_nm_destroy(&netmgr);
+	isc_managers_destroy(&netmgr, NULL, NULL, NULL);
 	isc_mem_destroy(&mctx);
 	if (tls_ctx) {
 		isc_tlsctx_free(&tls_ctx);

bin/tools/mdig.c

@@ -21,6 +21,7 @@
 #include <isc/hash.h>
 #include <isc/hex.h>
 #include <isc/log.h>
+#include <isc/managers.h>
 #include <isc/mem.h>
 #include <isc/net.h>
 #include <isc/nonce.h>
@@ -2070,10 +2071,10 @@ main(int argc, char *argv[]) {
 	isc_logconfig_t *lcfg;
 	isc_nm_t *netmgr = NULL;
 	isc_taskmgr_t *taskmgr = NULL;
-	isc_task_t *task;
-	isc_timermgr_t *timermgr;
-	isc_socketmgr_t *socketmgr;
-	dns_dispatchmgr_t *dispatchmgr;
+	isc_task_t *task = NULL;
+	isc_timermgr_t *timermgr = NULL;
+	isc_socketmgr_t *socketmgr = NULL;
+	dns_dispatchmgr_t *dispatchmgr = NULL;
 	unsigned int attrs, attrmask;
 	dns_dispatch_t *dispatchvx;
 	dns_view_t *view;
@@ -2130,17 +2131,11 @@ main(int argc, char *argv[]) {
 		fatal("can't choose between IPv4 and IPv6");
 	}
 
-	netmgr = isc_nm_start(mctx, 1);
+	isc_managers_create(mctx, 1, 0, 0, &netmgr, &taskmgr, &timermgr,
+			    &socketmgr);
 
-	RUNCHECK(isc_taskmgr_create(mctx, 0, netmgr, &taskmgr));
-	task = NULL;
 	RUNCHECK(isc_task_create(taskmgr, 0, &task));
-	timermgr = NULL;
 
-	RUNCHECK(isc_timermgr_create(mctx, &timermgr));
-	socketmgr = NULL;
-	RUNCHECK(isc_socketmgr_create(mctx, &socketmgr));
-	dispatchmgr = NULL;
 	RUNCHECK(dns_dispatchmgr_create(mctx, &dispatchmgr));
 
 	attrs = DNS_DISPATCHATTR_UDP | DNS_DISPATCHATTR_MAKEQUERY;
@@ -2206,13 +2201,10 @@ main(int argc, char *argv[]) {
 	dns_dispatch_detach(&dispatchvx);
 	dns_dispatchmgr_destroy(&dispatchmgr);
 
-	isc_socketmgr_destroy(&socketmgr);
-	isc_timermgr_destroy(&timermgr);
-
 	isc_task_shutdown(task);
 	isc_task_detach(&task);
-	isc_taskmgr_destroy(&taskmgr);
-	isc_nm_destroy(&netmgr);
+
+	isc_managers_destroy(&netmgr, &taskmgr, &timermgr, &socketmgr);
 
 	dst_lib_destroy();
 

lib/dns/tests/dnstest.c

@@ -32,6 +32,7 @@
 #include <isc/hash.h>
 #include <isc/hex.h>
 #include <isc/lex.h>
+#include <isc/managers.h>
 #include <isc/mem.h>
 #include <isc/os.h>
 #include <isc/print.h>
@@ -95,18 +96,10 @@ cleanup_managers(void) {
 		isc_task_shutdown(maintask);
 		isc_task_destroy(&maintask);
 	}
-	if (socketmgr != NULL) {
-		isc_socketmgr_destroy(&socketmgr);
-	}
-	if (taskmgr != NULL) {
-		isc_taskmgr_destroy(&taskmgr);
-	}
-	if (netmgr != NULL) {
-		isc_nm_destroy(&netmgr);
-	}
-	if (timermgr != NULL) {
-		isc_timermgr_destroy(&timermgr);
-	}
+	isc_managers_destroy(netmgr == NULL ? NULL : &netmgr,
+			     taskmgr == NULL ? NULL : &taskmgr,
+			     timermgr == NULL ? NULL : &timermgr,
+			     socketmgr == NULL ? NULL : &socketmgr);
 	if (app_running) {
 		isc_app_finish();
 	}
@@ -117,10 +110,8 @@ create_managers(void) {
 	isc_result_t result;
 	ncpus = isc_os_ncpus();
 
-	netmgr = isc_nm_start(dt_mctx, ncpus);
-	CHECK(isc_taskmgr_create(dt_mctx, 0, netmgr, &taskmgr));
-	CHECK(isc_timermgr_create(dt_mctx, &timermgr));
-	CHECK(isc_socketmgr_create(dt_mctx, &socketmgr));
+	isc_managers_create(dt_mctx, ncpus, 0, 0, &netmgr, &taskmgr, &timermgr,
+			    &socketmgr);
 	CHECK(isc_task_create(taskmgr, 0, &maintask));
 	return (ISC_R_SUCCESS);
 

lib/isc/Makefile.am

@@ -46,6 +46,7 @@ libisc_la_HEADERS =			\
 	include/isc/list.h		\
 	include/isc/log.h		\
 	include/isc/magic.h		\
+	include/isc/managers.h		\
 	include/isc/md.h		\
 	include/isc/mem.h		\
 	include/isc/meminfo.h		\
@@ -182,6 +183,7 @@ libisc_la_SOURCES =		\
 	lex.c			\
 	lib.c			\
 	log.c			\
+	managers.c		\
 	md.c			\
 	mem.c			\
 	mutexblock.c		\
@@ -222,6 +224,10 @@ libisc_la_SOURCES =		\
 	pthreads/thread.c	\
 	entropy_private.h	\
 	fsaccess_common_p.h	\
+	task_p.h		\
+	timer_p.h		\
+	socket_p.h		\
+	netmgr_p.h		\
 	lib_p.h			\
 	mem_p.h			\
 	tls_p.h

lib/isc/hp.c

@@ -82,6 +82,9 @@ tid(void) {
 
 void
 isc_hp_init(int max_threads) {
+	if (isc__hp_max_threads > max_threads) {
+		return;
+	}
 	isc__hp_max_threads = max_threads;
 	isc__hp_max_retired = max_threads * HP_MAX_HPS;
 }

lib/isc/include/isc/hp.h

@@ -66,8 +66,10 @@ typedef void(isc_hp_deletefunc_t)(void *);
 void
 isc_hp_init(int max_threads);
 /*%<
- * Initialize hazard pointer constants - isc__hp_max_threads. If more threads
- * will try to access hp it will assert.
+ * Initialize hazard pointer constants, isc__hp_max_threads and
+ * isc__hp_max_retired. If more threads try to access hp, it
+ * will assert. Calling this function repeatedly can be used
+ * to increase the limits, but cannot reduce them.
  */
 
 isc_hp_t *

lib/isc/include/isc/managers.h

@@ -0,0 +1,30 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, you can obtain one at https://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+#pragma once
+
+#include <isc/netmgr.h>
+#include <isc/result.h>
+#include <isc/socket.h>
+#include <isc/task.h>
+#include <isc/timer.h>
+
+typedef struct isc_managers isc_managers_t;
+
+isc_result_t
+isc_managers_create(isc_mem_t *mctx, size_t workers, size_t quantum,
+		    size_t sockets, isc_nm_t **netmgrp,
+		    isc_taskmgr_t **taskmgrp, isc_timermgr_t **timermgrp,
+		    isc_socketmgr_t **socketmgrp);
+
+void
+isc_managers_destroy(isc_nm_t **netmgrp, isc_taskmgr_t **taskmgrp,
+		     isc_timermgr_t **timermgrp, isc_socketmgr_t **socketmgrp);

lib/isc/include/isc/netmgr.h

@@ -68,19 +68,10 @@ typedef void (*isc_nm_opaquecb_t)(void *arg);
  * callbacks.
  */
 
-isc_nm_t *
-isc_nm_start(isc_mem_t *mctx, uint32_t workers);
-/*%<
- * Creates a new network manager with 'workers' worker threads,
- * and starts it running.
- */
-
 void
 isc_nm_attach(isc_nm_t *mgr, isc_nm_t **dst);
 void
 isc_nm_detach(isc_nm_t **mgr0);
-void
-isc_nm_destroy(isc_nm_t **mgr0);
 /*%<
  * Attach/detach a network manager. When all references have been
  * released, the network manager is shut down, freeing all resources.
@@ -88,15 +79,6 @@ isc_nm_destroy(isc_nm_t **mgr0);
  * for all other references to be gone.
  */
 
-void
-isc_nm_closedown(isc_nm_t *mgr);
-/*%<
- * Close down all active connections, freeing associated resources;
- * prevent new connections from being established. This can optionally
- * be called prior to shutting down the netmgr, to stop all processing
- * before shutting down the task manager.
- */
-
 /* Return thread ID of current thread, or ISC_NETMGR_TID_UNKNOWN */
 int
 isc_nm_tid(void);

lib/isc/include/isc/socket.h

@@ -9,8 +9,7 @@
  * information regarding copyright ownership.
  */
 
-#ifndef ISC_SOCKET_H
-#define ISC_SOCKET_H 1
+#pragma once
 
 /*****
 ***** Module Info
@@ -676,42 +675,6 @@ isc_socket_sendto2(isc_socket_t *sock, isc_region_t *region, isc_task_t *task,
  */
 /*@}*/
 
-isc_result_t
-isc_socketmgr_create(isc_mem_t *mctx, isc_socketmgr_t **managerp);
-
-isc_result_t
-isc_socketmgr_create2(isc_mem_t *mctx, isc_socketmgr_t **managerp,
-		      unsigned int maxsocks, int nthreads);
-/*%<
- * Create a socket manager.  If "maxsocks" is non-zero, it specifies the
- * maximum number of sockets that the created manager should handle.
- * isc_socketmgr_create() is equivalent of isc_socketmgr_create2() with
- * "maxsocks" being zero.
- *
- * Notes:
- *
- *\li	All memory will be allocated in memory context 'mctx'.
- *
- * Requires:
- *
- *\li	'mctx' is a valid memory context.
- *
- *\li	'managerp' points to a NULL isc_socketmgr_t.
- *
- *\li	'actx' is a valid application context (for createinctx()).
- *
- * Ensures:
- *
- *\li	'*managerp' is a valid isc_socketmgr_t.
- *
- * Returns:
- *
- *\li	#ISC_R_SUCCESS
- *\li	#ISC_R_NOMEMORY
- *\li	#ISC_R_UNEXPECTED
- *\li	#ISC_R_NOTIMPLEMENTED
- */
-
 isc_result_t
 isc_socketmgr_getmaxsockets(isc_socketmgr_t *manager, unsigned int *nsockp);
 /*%<
@@ -741,31 +704,6 @@ isc_socketmgr_setstats(isc_socketmgr_t *manager, isc_stats_t *stats);
  *	(see above).
  */
 
-void
-isc_socketmgr_destroy(isc_socketmgr_t **managerp);
-/*%<
- * Destroy a socket manager.
- *
- * Notes:
- *
- *\li	This routine blocks until there are no sockets left in the manager,
- *	so if the caller holds any socket references using the manager, it
- *	must detach them before calling isc_socketmgr_destroy() or it will
- *	block forever.
- *
- * Requires:
- *
- *\li	'*managerp' is a valid isc_socketmgr_t.
- *
- *\li	All sockets managed by this manager are fully detached.
- *
- * Ensures:
- *
- *\li	*managerp == NULL
- *
- *\li	All resources used by the manager have been freed.
- */
-
 isc_sockettype_t
 isc_socket_gettype(isc_socket_t *sock);
 /*%<
@@ -908,5 +846,3 @@ typedef isc_result_t (*isc_socketmgrcreatefunc_t)(isc_mem_t *	    mctx,
 						  isc_socketmgr_t **managerp);
 
 ISC_LANG_ENDDECLS
-
-#endif /* ISC_SOCKET_H */

lib/isc/include/isc/task.h

@@ -9,8 +9,7 @@
  * information regarding copyright ownership.
  */
 
-#ifndef ISC_TASK_H
-#define ISC_TASK_H 1
+#pragma once
 
 /*****
 ***** Module Info
@@ -137,6 +136,12 @@ isc_task_create_bound(isc_taskmgr_t *manager, unsigned int quantum,
  *\li	#ISC_R_SHUTTINGDOWN
  */
 
+void
+isc_task_ready(isc_task_t *task);
+/*%<
+ * Enqueue the task onto netmgr queue.
+ */
+
 isc_result_t
 isc_task_run(isc_task_t *task);
 /*%<
@@ -626,81 +631,11 @@ isc_task_privilege(isc_task_t *task);
 ***** Task Manager.
 *****/
 
-isc_result_t
-isc_taskmgr_create(isc_mem_t *mctx, unsigned int default_quantum, isc_nm_t *nm,
-		   isc_taskmgr_t **managerp);
-/*%<
- * Create a new task manager.
- *
- * Notes:
- *
- *\li	If 'default_quantum' is non-zero, then it will be used as the default
- *	quantum value when tasks are created.  If zero, then an implementation
- *	defined default quantum will be used.
- *
- *\li	If 'nm' is set then netmgr is paused when an exclusive task mode
- *	is requested.
- *
- * Requires:
- *
- *\li      'mctx' is a valid memory context.
- *
- *\li	managerp != NULL && *managerp == NULL
- *
- * Ensures:
- *
- *\li	On success, '*managerp' will be attached to the newly created task
- *	manager.
- *
- * Returns:
- *
- *\li	#ISC_R_SUCCESS
- *\li	#ISC_R_NOMEMORY
- *\li	#ISC_R_NOTHREADS		No threads could be created.
- *\li	#ISC_R_UNEXPECTED		An unexpected error occurred.
- *\li	#ISC_R_SHUTTINGDOWN		The non-threaded, shared, task
- *					manager shutting down.
- */
-
 void
 isc_taskmgr_attach(isc_taskmgr_t *, isc_taskmgr_t **);
 void
 isc_taskmgr_detach(isc_taskmgr_t *);
 
-void
-isc_taskmgr_destroy(isc_taskmgr_t **managerp);
-/*%<
- * Destroy '*managerp'.
- *
- * Notes:
- *
- *\li	Calling isc_taskmgr_destroy() will shutdown all tasks managed by
- *	*managerp that haven't already been shutdown.  The call will block
- *	until all tasks have entered the done state.
- *
- *\li	isc_taskmgr_destroy() must not be called by a task event action,
- *	because it would block forever waiting for the event action to
- *	complete.  An event action that wants to cause task manager shutdown
- *	should request some non-event action thread of execution to do the
- *	shutdown, e.g. by signaling a condition variable or using
- *	isc_app_shutdown().
- *
- *\li	Task manager references are not reference counted, so the caller
- *	must ensure that no attempt will be made to use the manager after
- *	isc_taskmgr_destroy() returns.
- *
- * Requires:
- *
- *\li	'*managerp' is a valid task manager.
- *
- *\li	isc_taskmgr_destroy() has not be called previously on '*managerp'.
- *
- * Ensures:
- *
- *\li	All resources used by the task manager, and any tasks it managed,
- *	have been freed.
- */
-
 void
 isc_taskmgr_setexcltask(isc_taskmgr_t *mgr, isc_task_t *task);
 /*%<
@@ -736,5 +671,3 @@ isc_taskmgr_renderjson(isc_taskmgr_t *mgr, void *tasksobj0);
 #endif /* HAVE_JSON_C */
 
 ISC_LANG_ENDDECLS
-
-#endif /* ISC_TASK_H */

lib/isc/include/isc/timer.h

@@ -9,8 +9,7 @@
  * information regarding copyright ownership.
  */
 
-#ifndef ISC_TIMER_H
-#define ISC_TIMER_H 1
+#pragma once
 
 /*****
 ***** Module Info
@@ -276,58 +275,7 @@ isc_timer_gettype(isc_timer_t *timer);
  *\li	'timer' to be a valid timer.
  */
 
-isc_result_t
-isc_timermgr_create(isc_mem_t *mctx, isc_timermgr_t **managerp);
-/*%<
- * Create a timer manager.
- *
- * Notes:
- *
- *\li	All memory will be allocated in memory context 'mctx'.
- *
- * Requires:
- *
- *\li	'mctx' is a valid memory context.
- *
- *\li	'managerp' points to a NULL isc_timermgr_t.
- *
- * Ensures:
- *
- *\li	'*managerp' is a valid isc_timermgr_t.
- *
- * Returns:
- *
- *\li	Success
- *\li	No memory
- *\li	Unexpected error
- */
-
-void
-isc_timermgr_destroy(isc_timermgr_t **managerp);
-/*%<
- * Destroy a timer manager.
- *
- * Notes:
- *
- *\li	This routine blocks until there are no timers left in the manager,
- *	so if the caller holds any timer references using the manager, it
- *	must detach them before calling isc_timermgr_destroy() or it will
- *	block forever.
- *
- * Requires:
- *
- *\li	'*managerp' is a valid isc_timermgr_t.
- *
- * Ensures:
- *
- *\li	*managerp == NULL
- *
- *\li	All resources used by the manager have been freed.
- */
-
 void
 isc_timermgr_poke(isc_timermgr_t *m);
 
 ISC_LANG_ENDDECLS
-
-#endif /* ISC_TIMER_H */

lib/isc/managers.c

@@ -0,0 +1,144 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, you can obtain one at https://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+#include <isc/hp.h>
+#include <isc/managers.h>
+#include <isc/util.h>
+
+#include "netmgr_p.h"
+#include "socket_p.h"
+#include "task_p.h"
+#include "timer_p.h"
+
+isc_result_t
+isc_managers_create(isc_mem_t *mctx, size_t workers, size_t quantum,
+		    size_t sockets, isc_nm_t **netmgrp,
+		    isc_taskmgr_t **taskmgrp, isc_timermgr_t **timermgrp,
+		    isc_socketmgr_t **socketmgrp) {
+	isc_result_t result;
+	isc_nm_t *netmgr = NULL;
+	isc_socketmgr_t *socketmgr = NULL;
+	isc_taskmgr_t *taskmgr = NULL;
+	isc_timermgr_t *timermgr = NULL;
+
+	/*
+	 * We have ncpus network threads, ncpus old network threads - make
+	 * it 4x just to be on the safe side.
+	 */
+	isc_hp_init(4 * workers);
+
+	REQUIRE(netmgrp != NULL && *netmgrp == NULL);
+	isc__netmgr_create(mctx, workers, &netmgr);
+	*netmgrp = netmgr;
+	INSIST(netmgr != NULL);
+
+	REQUIRE(taskmgrp == NULL || *taskmgrp == NULL);
+	if (taskmgrp != NULL) {
+		INSIST(netmgr != NULL);
+		result = isc__taskmgr_create(mctx, quantum, netmgr, &taskmgr);
+		if (result != ISC_R_SUCCESS) {
+			UNEXPECTED_ERROR(__FILE__, __LINE__,
+					 "isc_taskmgr_create() failed: %s",
+					 isc_result_totext(result));
+			goto fail;
+		}
+		*taskmgrp = taskmgr;
+	}
+
+	REQUIRE(timermgrp == NULL || *timermgrp == NULL);
+	if (timermgrp != NULL) {
+		result = isc__timermgr_create(mctx, &timermgr);
+		if (result != ISC_R_SUCCESS) {
+			UNEXPECTED_ERROR(__FILE__, __LINE__,
+					 "isc_timermgr_create() failed: %s",
+					 isc_result_totext(result));
+			goto fail;
+		}
+		*timermgrp = timermgr;
+	}
+
+	REQUIRE(socketmgrp == NULL || *socketmgrp == NULL);
+	if (socketmgrp != NULL) {
+		result = isc__socketmgr_create(mctx, &socketmgr, sockets,
+					       workers);
+		if (result != ISC_R_SUCCESS) {
+			UNEXPECTED_ERROR(__FILE__, __LINE__,
+					 "isc_socketmgr_create() failed: %s",
+					 isc_result_totext(result));
+			goto fail;
+		}
+		*socketmgrp = socketmgr;
+	}
+
+	return (ISC_R_SUCCESS);
+fail:
+	isc_managers_destroy(netmgrp, taskmgrp, timermgrp, socketmgrp);
+
+	return (result);
+}
+
+void
+isc_managers_destroy(isc_nm_t **netmgrp, isc_taskmgr_t **taskmgrp,
+		     isc_timermgr_t **timermgrp, isc_socketmgr_t **socketmgrp) {
+	/*
+	 * If we have a taskmgr to clean up, then we must also have a netmgr.
+	 */
+	REQUIRE(taskmgrp != NULL || netmgrp == NULL);
+
+	/*
+	 * The sequence of operations here is important:
+	 *
+	 * 1. Initiate shutdown of the taskmgr, sending shutdown events to
+	 * all tasks that are not already shutting down.
+	 */
+	if (taskmgrp != NULL) {
+		INSIST(*taskmgrp != NULL);
+		isc__taskmgr_shutdown(*taskmgrp);
+	}
+
+	/*
+	 * 2. Initiate shutdown of the network manager, freeing clients
+	 * and other resources and preventing new connections, but do
+	 * not stop processing of existing events.
+	 */
+	if (netmgrp != NULL) {
+		INSIST(*netmgrp != NULL);
+		isc__netmgr_shutdown(*netmgrp);
+	}
+
+	/*
+	 * 3. Finish destruction of the task manager when all tasks
+	 * have completed.
+	 */
+	if (taskmgrp != NULL) {
+		isc__taskmgr_destroy(taskmgrp);
+	}
+
+	/*
+	 * 4. Finish destruction of the netmgr, and wait until all
+	 * references have been released.
+	 */
+	if (netmgrp != NULL) {
+		isc__netmgr_destroy(netmgrp);
+	}
+
+	/*
+	 * 5. Clean up the remaining managers.
+	 */
+	if (timermgrp != NULL) {
+		INSIST(*timermgrp != NULL);
+		isc__timermgr_destroy(timermgrp);
+	}
+	if (socketmgrp != NULL) {
+		INSIST(*socketmgrp != NULL);
+		isc__socketmgr_destroy(socketmgrp);
+	}
+}

lib/isc/netmgr/http.c

@@ -1193,7 +1193,7 @@ isc_nm_httpconnect(isc_nm_t *mgr, isc_nmiface_t *local, isc_nmiface_t *peer,
 
 	sock->extrahandlesize = extrahandlesize;
 	sock->connect_timeout = timeout;
-	sock->result = ISC_R_DEFAULT;
+	sock->result = ISC_R_UNSET;
 	sock->connect_cb = cb;
 	sock->connect_cbarg = cbarg;
 	atomic_init(&sock->client, true);
@@ -2164,7 +2164,7 @@ isc_nm_listenhttp(isc_nm_t *mgr, isc_nmiface_t *iface, int backlog,
 	isc__nmsocket_attach(sock, &sock->outer->h2.httpserver);
 
 	sock->nchildren = sock->outer->nchildren;
-	sock->result = ISC_R_DEFAULT;
+	sock->result = ISC_R_UNSET;
 	sock->tid = isc_random_uniform(sock->nchildren);
 	sock->fd = (uv_os_sock_t)-1;
 

lib/isc/netmgr/netmgr-int.h

@@ -39,6 +39,8 @@
 
 #include "uv-compat.h"
 
+#define ISC_NETMGR_QUANTUM_DEFAULT 16
+
 #define ISC_NETMGR_TID_UNKNOWN -1
 
 /* Must be different from ISC_NETMGR_TID_UNKNOWN */
@@ -172,7 +174,6 @@ typedef struct isc__networker {
 	uv_async_t async; /* async channel to send
 			   * data to this networker */
 	isc_mutex_t lock;
-	isc_condition_t cond;
 	bool paused;
 	bool finished;
 	isc_thread_t thread;
@@ -183,11 +184,13 @@ typedef struct isc__networker {
 				    * used for listening etc.
 				    * can be processed while
 				    * worker is paused */
+	isc_condition_t cond_prio;
 	isc_refcount_t references;
 	atomic_int_fast64_t pktcount;
 	char *recvbuf;
 	char *sendbuf;
 	bool recvbuf_inuse;
+	unsigned int quantum;
 } isc__networker_t;
 
 /*
@@ -668,7 +671,7 @@ struct isc_nm {
 	isc_mutex_t evlock;
 
 	uint_fast32_t workers_running;
-	uint_fast32_t workers_paused;
+	atomic_uint_fast32_t workers_paused;
 	atomic_uint_fast32_t maxudp;
 
 	atomic_bool paused;
@@ -699,6 +702,11 @@ struct isc_nm {
 	atomic_uint_fast32_t keepalive;
 	atomic_uint_fast32_t advertised;
 
+	uv_barrier_t startlistening;
+	uv_barrier_t stoplistening;
+	uv_barrier_t pausing;
+	uv_barrier_t resuming;
+
 #ifdef NETMGR_TRACE
 	ISC_LIST(isc_nmsocket_t) active_sockets;
 #endif
@@ -927,7 +935,7 @@ struct isc_nmsocket {
 
 	/* Atomic */
 	/*% Number of running (e.g. listening) child sockets */
-	uint_fast32_t rchildren;
+	atomic_uint_fast32_t rchildren;
 
 	/*%
 	 * Socket is active if it's listening, working, etc. If it's

lib/isc/netmgr/netmgr.c

@@ -37,6 +37,7 @@
 #include <isc/util.h>
 
 #include "netmgr-int.h"
+#include "netmgr_p.h"
 #include "openssl_shim.h"
 #include "uv-compat.h"
 
@@ -134,15 +135,29 @@ nm_thread(isc_threadarg_t worker0);
 static void
 async_cb(uv_async_t *handle);
 static bool
-process_queue(isc__networker_t *worker, isc_queue_t *queue);
+process_netievent(isc__networker_t *worker, isc__netievent_t *ievent);
 static bool
-process_priority_queue(isc__networker_t *worker);
+process_queue(isc__networker_t *worker, isc_queue_t *queue,
+	      unsigned int *quantump);
 static void
-process_privilege_queue(isc__networker_t *worker);
-static void
-process_tasks_queue(isc__networker_t *worker);
-static void
-process_normal_queue(isc__networker_t *worker);
+wait_for_priority_queue(isc__networker_t *worker);
+static bool
+process_priority_queue(isc__networker_t *worker, unsigned int *quantump);
+static bool
+process_privilege_queue(isc__networker_t *worker, unsigned int *quantump);
+static bool
+process_task_queue(isc__networker_t *worker, unsigned int *quantump);
+static bool
+process_normal_queue(isc__networker_t *worker, unsigned int *quantump);
+
+#define drain_priority_queue(worker) \
+	(void)process_priority_queue(worker, &(unsigned int){ UINT_MAX })
+#define drain_privilege_queue(worker) \
+	(void)process_privilege_queue(worker, &(unsigned int){ UINT_MAX })
+#define drain_task_queue(worker) \
+	(void)process_task_queue(worker, &(unsigned int){ UINT_MAX })
+#define drain_normal_queue(worker) \
+	(void)process_normal_queue(worker, &(unsigned int){ UINT_MAX })
 
 static void
 isc__nm_async_stop(isc__networker_t *worker, isc__netievent_t *ev0);
@@ -206,8 +221,8 @@ isc__nm_winsock_destroy(void) {
 }
 #endif /* WIN32 */
 
-isc_nm_t *
-isc_nm_start(isc_mem_t *mctx, uint32_t workers) {
+void
+isc__netmgr_create(isc_mem_t *mctx, uint32_t workers, isc_nm_t **netmgrp) {
 	isc_nm_t *mgr = NULL;
 	char name[32];
 
@@ -227,6 +242,7 @@ isc_nm_start(isc_mem_t *mctx, uint32_t workers) {
 	isc_refcount_init(&mgr->references, 1);
 	atomic_init(&mgr->maxudp, 0);
 	atomic_init(&mgr->interlocked, ISC_NETMGR_NON_INTERLOCKED);
+	atomic_init(&mgr->workers_paused, 0);
 
 #ifdef NETMGR_TRACE
 	ISC_LIST_INIT(mgr->active_sockets);
@@ -256,6 +272,11 @@ isc_nm_start(isc_mem_t *mctx, uint32_t workers) {
 	isc_mempool_associatelock(mgr->evpool, &mgr->evlock);
 	isc_mempool_setfillcount(mgr->evpool, 32);
 
+	uv_barrier_init(&mgr->startlistening, workers);
+	uv_barrier_init(&mgr->stoplistening, workers);
+	uv_barrier_init(&mgr->pausing, workers);
+	uv_barrier_init(&mgr->resuming, workers);
+
 	mgr->workers = isc_mem_get(mctx, workers * sizeof(isc__networker_t));
 	for (size_t i = 0; i < workers; i++) {
 		int r;
@@ -263,6 +284,7 @@ isc_nm_start(isc_mem_t *mctx, uint32_t workers) {
 		*worker = (isc__networker_t){
 			.mgr = mgr,
 			.id = i,
+			.quantum = ISC_NETMGR_QUANTUM_DEFAULT,
 		};
 
 		r = uv_loop_init(&worker->loop);
@@ -274,12 +296,13 @@ isc_nm_start(isc_mem_t *mctx, uint32_t workers) {
 		RUNTIME_CHECK(r == 0);
 
 		isc_mutex_init(&worker->lock);
-		isc_condition_init(&worker->cond);
 
 		worker->ievents = isc_queue_new(mgr->mctx, 128);
-		worker->ievents_priv = isc_queue_new(mgr->mctx, 128);
 		worker->ievents_task = isc_queue_new(mgr->mctx, 128);
+		worker->ievents_priv = isc_queue_new(mgr->mctx, 128);
 		worker->ievents_prio = isc_queue_new(mgr->mctx, 128);
+		isc_condition_init(&worker->cond_prio);
+
 		worker->recvbuf = isc_mem_get(mctx, ISC_NETMGR_RECVBUF_SIZE);
 		worker->sendbuf = isc_mem_get(mctx, ISC_NETMGR_SENDBUF_SIZE);
 
@@ -296,7 +319,7 @@ isc_nm_start(isc_mem_t *mctx, uint32_t workers) {
 	}
 
 	mgr->magic = NM_MAGIC;
-	return (mgr);
+	*netmgrp = mgr;
 }
 
 /*
@@ -348,6 +371,7 @@ nm_destroy(isc_nm_t **mgr0) {
 		{
 			isc_mempool_put(mgr->evpool, ievent);
 		}
+		isc_condition_destroy(&worker->cond_prio);
 
 		r = uv_loop_close(&worker->loop);
 		INSIST(r == 0);
@@ -357,7 +381,6 @@ nm_destroy(isc_nm_t **mgr0) {
 		isc_queue_destroy(worker->ievents_task);
 		isc_queue_destroy(worker->ievents_prio);
 		isc_mutex_destroy(&worker->lock);
-		isc_condition_destroy(&worker->cond);
 
 		isc_mem_put(mgr->mctx, worker->sendbuf,
 			    ISC_NETMGR_SENDBUF_SIZE);
@@ -370,6 +393,11 @@ nm_destroy(isc_nm_t **mgr0) {
 		isc_stats_detach(&mgr->stats);
 	}
 
+	uv_barrier_destroy(&mgr->resuming);
+	uv_barrier_destroy(&mgr->pausing);
+	uv_barrier_destroy(&mgr->stoplistening);
+	uv_barrier_destroy(&mgr->startlistening);
+
 	isc_condition_destroy(&mgr->wkstatecond);
 	isc_condition_destroy(&mgr->wkpausecond);
 	isc_mutex_destroy(&mgr->lock);
@@ -410,13 +438,18 @@ isc_nm_pause(isc_nm_t *mgr) {
 		}
 	}
 
+	if (isc__nm_in_netthread()) {
+		uv_barrier_wait(&mgr->pausing);
+	}
+
 	LOCK(&mgr->lock);
-	while (mgr->workers_paused != pausing) {
+	while (atomic_load(&mgr->workers_paused) != pausing) {
 		WAIT(&mgr->wkstatecond, &mgr->lock);
 	}
+	UNLOCK(&mgr->lock);
+
 	REQUIRE(atomic_compare_exchange_strong(&mgr->paused, &(bool){ false },
 					       true));
-	UNLOCK(&mgr->lock);
 }
 
 void
@@ -436,14 +469,19 @@ isc_nm_resume(isc_nm_t *mgr) {
 		}
 	}
 
+	if (isc__nm_in_netthread()) {
+		uv_barrier_wait(&mgr->resuming);
+	}
+
 	LOCK(&mgr->lock);
-	while (mgr->workers_paused != 0) {
+	while (atomic_load(&mgr->workers_paused) != 0) {
 		WAIT(&mgr->wkstatecond, &mgr->lock);
 	}
+	UNLOCK(&mgr->lock);
+
 	REQUIRE(atomic_compare_exchange_strong(&mgr->paused, &(bool){ true },
 					       false));
-	BROADCAST(&mgr->wkpausecond);
-	UNLOCK(&mgr->lock);
+
 	isc__nm_drop_interlocked(mgr);
 }
 
@@ -473,7 +511,7 @@ isc_nm_detach(isc_nm_t **mgr0) {
 }
 
 void
-isc_nm_closedown(isc_nm_t *mgr) {
+isc__netmgr_shutdown(isc_nm_t *mgr) {
 	REQUIRE(VALID_NM(mgr));
 
 	atomic_store(&mgr->closing, true);
@@ -485,19 +523,18 @@ isc_nm_closedown(isc_nm_t *mgr) {
 }
 
 void
-isc_nm_destroy(isc_nm_t **mgr0) {
+isc__netmgr_destroy(isc_nm_t **netmgrp) {
 	isc_nm_t *mgr = NULL;
 	int counter = 0;
 
-	REQUIRE(mgr0 != NULL);
-	REQUIRE(VALID_NM(*mgr0));
+	REQUIRE(VALID_NM(*netmgrp));
 
-	mgr = *mgr0;
+	mgr = *netmgrp;
 
 	/*
 	 * Close active connections.
 	 */
-	isc_nm_closedown(mgr);
+	isc__netmgr_shutdown(mgr);
 
 	/*
 	 * Wait for the manager to be dereferenced elsewhere.
@@ -524,7 +561,7 @@ isc_nm_destroy(isc_nm_t **mgr0) {
 	/*
 	 * Detach final reference.
 	 */
-	isc_nm_detach(mgr0);
+	isc_nm_detach(netmgrp);
 }
 
 void
@@ -615,43 +652,29 @@ nm_thread(isc_threadarg_t worker0) {
 		if (worker->paused) {
 			INSIST(atomic_load(&mgr->interlocked) != isc_nm_tid());
 
-			/*
-			 * We need to lock the worker first; otherwise
-			 * isc_nm_resume() might slip in before WAIT() in
-			 * the while loop starts, then the signal never
-			 * gets delivered and we are stuck forever in the
-			 * paused loop.
-			 */
-			LOCK(&worker->lock);
-			LOCK(&mgr->lock);
-			mgr->workers_paused++;
-			SIGNAL(&mgr->wkstatecond);
-			UNLOCK(&mgr->lock);
+			atomic_fetch_add(&mgr->workers_paused, 1);
+			if (uv_barrier_wait(&mgr->pausing) > 0) {
+				LOCK(&mgr->lock);
+				SIGNAL(&mgr->wkstatecond);
+				UNLOCK(&mgr->lock);
+			}
 
 			while (worker->paused) {
-				WAIT(&worker->cond, &worker->lock);
-				UNLOCK(&worker->lock);
-				(void)process_priority_queue(worker);
-				LOCK(&worker->lock);
+				wait_for_priority_queue(worker);
 			}
 
-			LOCK(&mgr->lock);
-			mgr->workers_paused--;
-			SIGNAL(&mgr->wkstatecond);
-			UNLOCK(&mgr->lock);
-			UNLOCK(&worker->lock);
-
 			/*
-			 * All workers must run the privileged event
+			 * All workers must drain the privileged event
 			 * queue before we resume from pause.
 			 */
-			process_privilege_queue(worker);
+			drain_privilege_queue(worker);
 
-			LOCK(&mgr->lock);
-			while (atomic_load(&mgr->paused)) {
-				WAIT(&mgr->wkpausecond, &mgr->lock);
+			atomic_fetch_sub(&mgr->workers_paused, 1);
+			if (uv_barrier_wait(&mgr->resuming) > 0) {
+				LOCK(&mgr->lock);
+				SIGNAL(&mgr->wkstatecond);
+				UNLOCK(&mgr->lock);
 			}
-			UNLOCK(&mgr->lock);
 		}
 
 		if (r == 0) {
@@ -660,16 +683,6 @@ nm_thread(isc_threadarg_t worker0) {
 		}
 
 		INSIST(!worker->finished);
-
-		/*
-		 * We've fully resumed from pause. Drain the normal
-		 * asynchronous event queues before resuming the uv_run()
-		 * loop. (This is not strictly necessary, it just ensures
-		 * that all pending events are processed before another
-		 * pause can slip in.)
-		 */
-		process_tasks_queue(worker);
-		process_normal_queue(worker);
 	}
 
 	/*
@@ -677,8 +690,8 @@ nm_thread(isc_threadarg_t worker0) {
 	 * (they may include shutdown events) but do not process
 	 * the netmgr event queue.
 	 */
-	process_privilege_queue(worker);
-	process_tasks_queue(worker);
+	drain_privilege_queue(worker);
+	drain_task_queue(worker);
 
 	LOCK(&mgr->lock);
 	mgr->workers_running--;
@@ -688,6 +701,23 @@ nm_thread(isc_threadarg_t worker0) {
 	return ((isc_threadresult_t)0);
 }
 
+static bool
+process_all_queues(isc__networker_t *worker, unsigned int quantum) {
+	/*
+	 * The queue processing functions will return false when the
+	 * system is pausing or stopping, or if we have completed
+	 * 'quantum' events.
+	 *
+	 * We don't want to proceed to a new queue until the previous one
+	 * has been fully drained, so whenever one queue is interrupted,
+	 * we skip all the later ones.
+	 */
+	return (process_priority_queue(worker, &quantum) &&
+		process_privilege_queue(worker, &quantum) &&
+		process_task_queue(worker, &quantum) &&
+		process_normal_queue(worker, &quantum));
+}
+
 /*
  * async_cb() is a universal callback for 'async' events sent to event loop.
  * It's the only way to safely pass data to the libuv event loop. We use a
@@ -697,18 +727,14 @@ nm_thread(isc_threadarg_t worker0) {
 static void
 async_cb(uv_async_t *handle) {
 	isc__networker_t *worker = (isc__networker_t *)handle->loop->data;
+	unsigned int quantum = worker->quantum;
 
-	/*
-	 * process_priority_queue() returns false when pausing or stopping,
-	 * so we don't want to process the other queues in that case.
-	 */
-	if (!process_priority_queue(worker)) {
-		return;
+	if (!process_all_queues(worker, quantum)) {
+		/* If we didn't process all the events, we need to enqueue
+		 * async_cb to be run in the next iteration of the uv_loop
+		 */
+		uv_async_send(handle);
 	}
-
-	process_privilege_queue(worker);
-	process_tasks_queue(worker);
-	process_normal_queue(worker);
 }
 
 static void
@@ -775,8 +801,7 @@ isc__nm_async_task(isc__networker_t *worker, isc__netievent_t *ev0) {
 
 	switch (result) {
 	case ISC_R_QUOTA:
-		isc_nm_task_enqueue(worker->mgr, (isc_task_t *)ievent->task,
-				    isc_nm_tid());
+		isc_task_ready(ievent->task);
 		return;
 	case ISC_R_SUCCESS:
 		return;
@@ -786,24 +811,52 @@ isc__nm_async_task(isc__networker_t *worker, isc__netievent_t *ev0) {
 	}
 }
 
+static void
+wait_for_priority_queue(isc__networker_t *worker) {
+	isc_queue_t *queue = worker->ievents_prio;
+	isc_condition_t *cond = &worker->cond_prio;
+	bool wait_for_work = true;
+
+	while (true) {
+		isc__netievent_t *ievent;
+		LOCK(&worker->lock);
+		ievent = (isc__netievent_t *)isc_queue_dequeue(queue);
+		if (wait_for_work) {
+			while (ievent == NULL) {
+				WAIT(cond, &worker->lock);
+				ievent = (isc__netievent_t *)isc_queue_dequeue(
+					queue);
+			}
+		}
+		UNLOCK(&worker->lock);
+		wait_for_work = false;
+
+		if (ievent == NULL) {
+			return;
+		}
+
+		(void)process_netievent(worker, ievent);
+	}
+}
+
 static bool
-process_priority_queue(isc__networker_t *worker) {
-	return (process_queue(worker, worker->ievents_prio));
+process_priority_queue(isc__networker_t *worker, unsigned int *quantump) {
+	return (process_queue(worker, worker->ievents_prio, quantump));
 }
 
-static void
-process_privilege_queue(isc__networker_t *worker) {
-	(void)process_queue(worker, worker->ievents_priv);
+static bool
+process_privilege_queue(isc__networker_t *worker, unsigned int *quantump) {
+	return (process_queue(worker, worker->ievents_priv, quantump));
 }
 
-static void
-process_tasks_queue(isc__networker_t *worker) {
-	(void)process_queue(worker, worker->ievents_task);
+static bool
+process_task_queue(isc__networker_t *worker, unsigned int *quantump) {
+	return (process_queue(worker, worker->ievents_task, quantump));
 }
 
-static void
-process_normal_queue(isc__networker_t *worker) {
-	(void)process_queue(worker, worker->ievents);
+static bool
+process_normal_queue(isc__networker_t *worker, unsigned int *quantump) {
+	return (process_queue(worker, worker->ievents, quantump));
 }
 
 /*
@@ -906,16 +959,27 @@ process_netievent(isc__networker_t *worker, isc__netievent_t *ievent) {
 }
 
 static bool
-process_queue(isc__networker_t *worker, isc_queue_t *queue) {
-	isc__netievent_t *ievent = NULL;
+process_queue(isc__networker_t *worker, isc_queue_t *queue,
+	      unsigned int *quantump) {
+	while (*quantump > 0) {
+		isc__netievent_t *ievent =
+			(isc__netievent_t *)isc_queue_dequeue(queue);
+
+		if (ievent == NULL) {
+			/* We fully drained this queue */
+			return (true);
+		}
+
+		(*quantump)--;
 
-	while ((ievent = (isc__netievent_t *)isc_queue_dequeue(queue)) != NULL)
-	{
 		if (!process_netievent(worker, ievent)) {
+			/* Netievent told us to stop */
 			return (false);
 		}
 	}
-	return (true);
+
+	/* No more quantum */
+	return (false);
 }
 
 void *
@@ -1018,7 +1082,7 @@ isc__nm_enqueue_ievent(isc__networker_t *worker, isc__netievent_t *event) {
 		 */
 		LOCK(&worker->lock);
 		isc_queue_enqueue(worker->ievents_prio, (uintptr_t)event);
-		SIGNAL(&worker->cond);
+		SIGNAL(&worker->cond_prio);
 		UNLOCK(&worker->lock);
 	} else if (event->type == netievent_privilegedtask) {
 		isc_queue_enqueue(worker->ievents_priv, (uintptr_t)event);
@@ -1153,7 +1217,6 @@ nmsocket_cleanup(isc_nmsocket_t *sock, bool dofree FLARG) {
 	isc_mem_free(sock->mgr->mctx, sock->ah_frees);
 	isc_mem_free(sock->mgr->mctx, sock->ah_handles);
 	isc_mutex_destroy(&sock->lock);
-	isc_condition_destroy(&sock->cond);
 	isc_condition_destroy(&sock->scond);
 	isc__nm_tls_cleanup_data(sock);
 	isc__nm_http_cleanup_data(sock);
@@ -1400,7 +1463,6 @@ isc___nmsocket_init(isc_nmsocket_t *sock, isc_nm_t *mgr, isc_nmsocket_type type,
 	}
 
 	isc_mutex_init(&sock->lock);
-	isc_condition_init(&sock->cond);
 	isc_condition_init(&sock->scond);
 	isc_refcount_init(&sock->references, 1);
 

lib/isc/netmgr/tcp.c

@@ -116,7 +116,7 @@ failed_accept_cb(isc_nmsocket_t *sock, isc_result_t eresult) {
 static isc_result_t
 tcp_connect_direct(isc_nmsocket_t *sock, isc__nm_uvreq_t *req) {
 	isc__networker_t *worker = NULL;
-	isc_result_t result = ISC_R_DEFAULT;
+	isc_result_t result = ISC_R_UNSET;
 	int r;
 
 	REQUIRE(VALID_NMSOCK(sock));
@@ -302,7 +302,7 @@ isc_nm_tcpconnect(isc_nm_t *mgr, isc_nmiface_t *local, isc_nmiface_t *peer,
 
 	sock->extrahandlesize = extrahandlesize;
 	sock->connect_timeout = timeout;
-	sock->result = ISC_R_DEFAULT;
+	sock->result = ISC_R_UNSET;
 	sock->fd = (uv_os_sock_t)-1;
 	atomic_init(&sock->client, true);
 
@@ -340,7 +340,7 @@ isc_nm_tcpconnect(isc_nm_t *mgr, isc_nmiface_t *local, isc_nmiface_t *peer,
 				       (isc__netievent_t *)ievent);
 	}
 	LOCK(&sock->lock);
-	while (sock->result == ISC_R_DEFAULT) {
+	while (sock->result == ISC_R_UNSET) {
 		WAIT(&sock->cond, &sock->lock);
 	}
 	atomic_store(&sock->active, true);
@@ -371,6 +371,38 @@ isc__nm_tcp_lb_socket(sa_family_t sa_family) {
 	return (sock);
 }
 
+static void
+start_tcp_child(isc_nm_t *mgr, isc_nmiface_t *iface, isc_nmsocket_t *sock,
+		int tid) {
+	isc__netievent_tcplisten_t *ievent = NULL;
+	isc_nmsocket_t *csock = &sock->children[tid];
+
+	isc__nmsocket_init(csock, mgr, isc_nm_tcpsocket, iface);
+	csock->parent = sock;
+	csock->accept_cb = sock->accept_cb;
+	csock->accept_cbarg = sock->accept_cbarg;
+	csock->extrahandlesize = sock->extrahandlesize;
+	csock->backlog = sock->backlog;
+	csock->tid = tid;
+	/*
+	 * We don't attach to quota, just assign - to avoid
+	 * increasing quota unnecessarily.
+	 */
+	csock->pquota = sock->pquota;
+	isc_quota_cb_init(&csock->quotacb, quota_accept_cb, csock);
+
+#if HAVE_SO_REUSEPORT_LB || defined(WIN32)
+	csock->fd = isc__nm_tcp_lb_socket(iface->addr.type.sa.sa_family);
+#else
+	csock->fd = dup(fd);
+#endif
+	REQUIRE(csock->fd >= 0);
+
+	ievent = isc__nm_get_netievent_tcplisten(mgr, csock);
+	isc__nm_maybe_enqueue_ievent(&mgr->workers[tid],
+				     (isc__netievent_t *)ievent);
+}
+
 isc_result_t
 isc_nm_listentcp(isc_nm_t *mgr, isc_nmiface_t *iface,
 		 isc_nm_accept_cb_t accept_cb, void *accept_cbarg,
@@ -378,18 +410,14 @@ isc_nm_listentcp(isc_nm_t *mgr, isc_nmiface_t *iface,
 		 isc_nmsocket_t **sockp) {
 	isc_result_t result = ISC_R_SUCCESS;
 	isc_nmsocket_t *sock = NULL;
-	sa_family_t sa_family = iface->addr.type.sa.sa_family;
 	size_t children_size = 0;
-#if !HAVE_SO_REUSEPORT_LB && !defined(WIN32)
-	uv_os_sock_t fd = -1;
-#endif
 
 	REQUIRE(VALID_NM(mgr));
 
 	sock = isc_mem_get(mgr->mctx, sizeof(*sock));
 	isc__nmsocket_init(sock, mgr, isc_nm_tcplistener, iface);
 
-	sock->rchildren = 0;
+	atomic_init(&sock->rchildren, 0);
 #if defined(WIN32)
 	sock->nchildren = 1;
 #else
@@ -399,64 +427,58 @@ isc_nm_listentcp(isc_nm_t *mgr, isc_nmiface_t *iface,
 	sock->children = isc_mem_get(mgr->mctx, children_size);
 	memset(sock->children, 0, children_size);
 
-	sock->result = ISC_R_DEFAULT;
-	sock->tid = isc_random_uniform(sock->nchildren);
-	sock->fd = -1;
+	sock->result = ISC_R_UNSET;
 
-#if !HAVE_SO_REUSEPORT_LB && !defined(WIN32)
-	fd = isc__nm_tcp_lb_socket(sa_family);
-#endif
+	sock->accept_cb = accept_cb;
+	sock->accept_cbarg = accept_cbarg;
+	sock->extrahandlesize = extrahandlesize;
+	sock->backlog = backlog;
+	sock->pquota = quota;
 
-	for (size_t i = 0; i < sock->nchildren; i++) {
-		isc__netievent_tcplisten_t *ievent = NULL;
-		isc_nmsocket_t *csock = &sock->children[i];
-
-		isc__nmsocket_init(csock, mgr, isc_nm_tcpsocket, iface);
-		csock->parent = sock;
-		csock->accept_cb = accept_cb;
-		csock->accept_cbarg = accept_cbarg;
-		csock->extrahandlesize = extrahandlesize;
-		csock->backlog = backlog;
-		csock->tid = i;
-		/*
-		 * We don't attach to quota, just assign - to avoid
-		 * increasing quota unnecessarily.
-		 */
-		csock->pquota = quota;
-		isc_quota_cb_init(&csock->quotacb, quota_accept_cb, csock);
-
-#if HAVE_SO_REUSEPORT_LB || defined(WIN32)
-		csock->fd = isc__nm_tcp_lb_socket(sa_family);
-#else
-		csock->fd = dup(fd);
-#endif
-		REQUIRE(csock->fd >= 0);
-
-		ievent = isc__nm_get_netievent_tcplisten(mgr, csock);
-		isc__nm_maybe_enqueue_ievent(&mgr->workers[i],
-					     (isc__netievent_t *)ievent);
+	if (isc__nm_in_netthread()) {
+		sock->tid = isc_nm_tid();
+	} else {
+		sock->tid = isc_random_uniform(sock->nchildren);
 	}
 
 #if !HAVE_SO_REUSEPORT_LB && !defined(WIN32)
-	isc__nm_closesocket(fd);
+	sock->fd = isc__nm_tcp_lb_socket(iface->addr.type.sa.sa_family);
+#else
+	sock->fd = -1;
+#endif
+
+	for (size_t i = 0; i < sock->nchildren; i++) {
+		if ((int)i == isc_nm_tid()) {
+			continue;
+		}
+		start_tcp_child(mgr, iface, sock, i);
+	}
+
+	if (isc__nm_in_netthread()) {
+		start_tcp_child(mgr, iface, sock, isc_nm_tid());
+	}
+
+#if !HAVE_SO_REUSEPORT_LB && !defined(WIN32)
+	isc__nm_closesocket(sock->fd);
+	sock->fd = -1;
 #endif
 
 	LOCK(&sock->lock);
-	while (sock->rchildren != sock->nchildren) {
+	while (atomic_load(&sock->rchildren) != sock->nchildren) {
 		WAIT(&sock->cond, &sock->lock);
 	}
 	result = sock->result;
 	atomic_store(&sock->active, true);
-	BROADCAST(&sock->scond);
 	UNLOCK(&sock->lock);
-	INSIST(result != ISC_R_DEFAULT);
+
+	INSIST(result != ISC_R_UNSET);
 
 	if (result == ISC_R_SUCCESS) {
-		REQUIRE(sock->rchildren == sock->nchildren);
+		REQUIRE(atomic_load(&sock->rchildren) == sock->nchildren);
 		*sockp = sock;
 	} else {
 		atomic_store(&sock->active, false);
-		isc__nm_tcp_stoplistening(sock);
+		isc_nm_stoplistening(sock);
 		isc_nmsocket_close(&sock);
 	}
 
@@ -561,15 +583,11 @@ done:
 		sock->pquota = NULL;
 	}
 
-	sock->parent->rchildren += 1;
-	if (sock->parent->result == ISC_R_DEFAULT) {
+	atomic_fetch_add(&sock->parent->rchildren, 1);
+	if (sock->parent->result == ISC_R_UNSET) {
 		sock->parent->result = result;
 	}
 	SIGNAL(&sock->parent->cond);
-	if (!atomic_load(&sock->parent->active)) {
-		WAIT(&sock->parent->scond, &sock->parent->lock);
-	}
-	INSIST(atomic_load(&sock->parent->active));
 	UNLOCK(&sock->parent->lock);
 }
 
@@ -633,7 +651,15 @@ isc__nm_tcp_stoplistening(isc_nmsocket_t *sock) {
 		INSIST(0);
 		ISC_UNREACHABLE();
 	}
-	enqueue_stoplistening(sock);
+
+	if (!isc__nm_in_netthread()) {
+		enqueue_stoplistening(sock);
+	} else if (!isc__nm_acquire_interlocked(sock->mgr)) {
+		enqueue_stoplistening(sock);
+	} else {
+		stop_tcp_parent(sock);
+		isc__nm_drop_interlocked(sock->mgr);
+	}
 }
 
 void
@@ -651,7 +677,12 @@ isc__nm_async_tcpstop(isc__networker_t *worker, isc__netievent_t *ev0) {
 		return;
 	}
 
-	stop_tcp_parent(sock);
+	if (!isc__nm_acquire_interlocked(sock->mgr)) {
+		enqueue_stoplistening(sock);
+	} else {
+		stop_tcp_parent(sock);
+		isc__nm_drop_interlocked(sock->mgr);
+	}
 }
 
 void
@@ -1190,8 +1221,6 @@ timer_close_cb(uv_handle_t *handle) {
 
 static void
 stop_tcp_child(isc_nmsocket_t *sock) {
-	bool last_child = false;
-
 	REQUIRE(sock->type == isc_nm_tcpsocket);
 	REQUIRE(sock->tid == isc_nm_tid());
 
@@ -1202,33 +1231,42 @@ stop_tcp_child(isc_nmsocket_t *sock) {
 
 	tcp_close_direct(sock);
 
-	LOCK(&sock->parent->lock);
-	sock->parent->rchildren -= 1;
-	last_child = (sock->parent->rchildren == 0);
-	UNLOCK(&sock->parent->lock);
+	atomic_fetch_sub(&sock->parent->rchildren, 1);
 
-	if (last_child) {
-		atomic_store(&sock->parent->closed, true);
-		isc__nmsocket_prep_destroy(sock->parent);
-	}
+	(void)uv_barrier_wait(&sock->mgr->stoplistening);
 }
 
 static void
 stop_tcp_parent(isc_nmsocket_t *sock) {
+	isc_nmsocket_t *csock = NULL;
 	REQUIRE(VALID_NMSOCK(sock));
 	REQUIRE(sock->type == isc_nm_tcplistener);
 
 	for (size_t i = 0; i < sock->nchildren; i++) {
 		isc__netievent_tcpstop_t *ievent = NULL;
-		isc_nmsocket_t *csock = &sock->children[i];
+		csock = &sock->children[i];
 		REQUIRE(VALID_NMSOCK(csock));
 
+		if ((int)i == isc_nm_tid()) {
+			/*
+			 * We need to schedule closing the other sockets first
+			 */
+			continue;
+		}
+
 		atomic_store(&csock->active, false);
 
 		ievent = isc__nm_get_netievent_tcpstop(sock->mgr, csock);
 		isc__nm_enqueue_ievent(&sock->mgr->workers[csock->tid],
 				       (isc__netievent_t *)ievent);
 	}
+
+	csock = &sock->children[isc_nm_tid()];
+	atomic_store(&csock->active, false);
+	stop_tcp_child(csock);
+
+	atomic_store(&sock->closed, true);
+	isc__nmsocket_prep_destroy(sock);
 }
 
 static void

lib/isc/netmgr/tcpdns.c

@@ -85,7 +85,7 @@ stop_tcpdns_child(isc_nmsocket_t *sock);
 static isc_result_t
 tcpdns_connect_direct(isc_nmsocket_t *sock, isc__nm_uvreq_t *req) {
 	isc__networker_t *worker = NULL;
-	isc_result_t result = ISC_R_DEFAULT;
+	isc_result_t result = ISC_R_UNSET;
 	int r;
 
 	REQUIRE(VALID_NMSOCK(sock));
@@ -269,7 +269,7 @@ isc_nm_tcpdnsconnect(isc_nm_t *mgr, isc_nmiface_t *local, isc_nmiface_t *peer,
 
 	sock->extrahandlesize = extrahandlesize;
 	sock->connect_timeout = timeout;
-	sock->result = ISC_R_DEFAULT;
+	sock->result = ISC_R_UNSET;
 	atomic_init(&sock->client, true);
 
 	req = isc__nm_uvreq_get(mgr, sock);
@@ -311,7 +311,7 @@ isc_nm_tcpdnsconnect(isc_nm_t *mgr, isc_nmiface_t *local, isc_nmiface_t *peer,
 	}
 
 	LOCK(&sock->lock);
-	while (sock->result == ISC_R_DEFAULT) {
+	while (sock->result == ISC_R_UNSET) {
 		WAIT(&sock->cond, &sock->lock);
 	}
 	atomic_store(&sock->active, true);
@@ -342,6 +342,39 @@ isc__nm_tcpdns_lb_socket(sa_family_t sa_family) {
 	return (sock);
 }
 
+static void
+start_tcpdns_child(isc_nm_t *mgr, isc_nmiface_t *iface, isc_nmsocket_t *sock,
+		   int tid) {
+	isc__netievent_tcpdnslisten_t *ievent = NULL;
+	isc_nmsocket_t *csock = &sock->children[tid];
+
+	isc__nmsocket_init(csock, mgr, isc_nm_tcpdnssocket, iface);
+	csock->parent = sock;
+	csock->accept_cb = sock->accept_cb;
+	csock->accept_cbarg = sock->accept_cbarg;
+	csock->recv_cb = sock->recv_cb;
+	csock->recv_cbarg = sock->recv_cbarg;
+	csock->extrahandlesize = sock->extrahandlesize;
+	csock->backlog = sock->backlog;
+	csock->tid = tid;
+	/*
+	 * We don't attach to quota, just assign - to avoid
+	 * increasing quota unnecessarily.
+	 */
+	csock->pquota = sock->pquota;
+	isc_quota_cb_init(&csock->quotacb, quota_accept_cb, csock);
+
+#if HAVE_SO_REUSEPORT_LB || defined(WIN32)
+	csock->fd = isc__nm_tcpdns_lb_socket(iface->addr.type.sa.sa_family);
+#else
+	csock->fd = dup(sock->fd);
+#endif
+	REQUIRE(csock->fd >= 0);
+
+	ievent = isc__nm_get_netievent_tcpdnslisten(mgr, csock);
+	isc__nm_maybe_enqueue_ievent(&mgr->workers[tid],
+				     (isc__netievent_t *)ievent);
+}
 isc_result_t
 isc_nm_listentcpdns(isc_nm_t *mgr, isc_nmiface_t *iface,
 		    isc_nm_recv_cb_t recv_cb, void *recv_cbarg,
@@ -350,18 +383,14 @@ isc_nm_listentcpdns(isc_nm_t *mgr, isc_nmiface_t *iface,
 		    isc_nmsocket_t **sockp) {
 	isc_result_t result = ISC_R_SUCCESS;
 	isc_nmsocket_t *sock = NULL;
-	sa_family_t sa_family = iface->addr.type.sa.sa_family;
 	size_t children_size = 0;
-#if !HAVE_SO_REUSEPORT_LB && !defined(WIN32)
-	uv_os_sock_t fd = -1;
-#endif
 
 	REQUIRE(VALID_NM(mgr));
 
 	sock = isc_mem_get(mgr->mctx, sizeof(*sock));
 	isc__nmsocket_init(sock, mgr, isc_nm_tcpdnslistener, iface);
 
-	sock->rchildren = 0;
+	atomic_init(&sock->rchildren, 0);
 #if defined(WIN32)
 	sock->nchildren = 1;
 #else
@@ -371,66 +400,59 @@ isc_nm_listentcpdns(isc_nm_t *mgr, isc_nmiface_t *iface,
 	sock->children = isc_mem_get(mgr->mctx, children_size);
 	memset(sock->children, 0, children_size);
 
-	sock->result = ISC_R_DEFAULT;
-	sock->tid = isc_random_uniform(sock->nchildren);
-	sock->fd = -1;
+	sock->result = ISC_R_UNSET;
+	sock->accept_cb = accept_cb;
+	sock->accept_cbarg = accept_cbarg;
+	sock->recv_cb = recv_cb;
+	sock->recv_cbarg = recv_cbarg;
+	sock->extrahandlesize = extrahandlesize;
+	sock->backlog = backlog;
+	sock->pquota = quota;
 
-#if !HAVE_SO_REUSEPORT_LB && !defined(WIN32)
-	fd = isc__nm_tcpdns_lb_socket(sa_family);
-#endif
-
-	for (size_t i = 0; i < sock->nchildren; i++) {
-		isc__netievent_tcpdnslisten_t *ievent = NULL;
-		isc_nmsocket_t *csock = &sock->children[i];
-
-		isc__nmsocket_init(csock, mgr, isc_nm_tcpdnssocket, iface);
-		csock->parent = sock;
-		csock->accept_cb = accept_cb;
-		csock->accept_cbarg = accept_cbarg;
-		csock->recv_cb = recv_cb;
-		csock->recv_cbarg = recv_cbarg;
-		csock->extrahandlesize = extrahandlesize;
-		csock->backlog = backlog;
-		csock->tid = i;
-		/*
-		 * We don't attach to quota, just assign - to avoid
-		 * increasing quota unnecessarily.
-		 */
-		csock->pquota = quota;
-		isc_quota_cb_init(&csock->quotacb, quota_accept_cb, csock);
-
-#if HAVE_SO_REUSEPORT_LB || defined(WIN32)
-		csock->fd = isc__nm_tcpdns_lb_socket(sa_family);
-#else
-		csock->fd = dup(fd);
-#endif
-		REQUIRE(csock->fd >= 0);
-
-		ievent = isc__nm_get_netievent_tcpdnslisten(mgr, csock);
-		isc__nm_maybe_enqueue_ievent(&mgr->workers[i],
-					     (isc__netievent_t *)ievent);
+	if (isc__nm_in_netthread()) {
+		sock->tid = isc_nm_tid();
+	} else {
+		sock->tid = isc_random_uniform(sock->nchildren);
 	}
 
 #if !HAVE_SO_REUSEPORT_LB && !defined(WIN32)
-	isc__nm_closesocket(fd);
+	sock->fd = isc__nm_tcpdns_lb_socket(iface->addr.type.sa.sa_family);
+#else
+	sock->fd = -1;
+#endif
+
+	for (size_t i = 0; i < sock->nchildren; i++) {
+		if ((int)i == isc_nm_tid()) {
+			continue;
+		}
+		start_tcpdns_child(mgr, iface, sock, i);
+	}
+
+	if (isc__nm_in_netthread()) {
+		start_tcpdns_child(mgr, iface, sock, isc_nm_tid());
+	}
+
+#if !HAVE_SO_REUSEPORT_LB && !defined(WIN32)
+	isc__nm_closesocket(sock->fd);
+	sock->fd = -1;
 #endif
 
 	LOCK(&sock->lock);
-	while (sock->rchildren != sock->nchildren) {
+	while (atomic_load(&sock->rchildren) != sock->nchildren) {
 		WAIT(&sock->cond, &sock->lock);
 	}
 	result = sock->result;
 	atomic_store(&sock->active, true);
-	BROADCAST(&sock->scond);
 	UNLOCK(&sock->lock);
-	INSIST(result != ISC_R_DEFAULT);
+
+	INSIST(result != ISC_R_UNSET);
 
 	if (result == ISC_R_SUCCESS) {
-		REQUIRE(sock->rchildren == sock->nchildren);
+		REQUIRE(atomic_load(&sock->rchildren) == sock->nchildren);
 		*sockp = sock;
 	} else {
 		atomic_store(&sock->active, false);
-		isc__nm_tcpdns_stoplistening(sock);
+		isc_nm_stoplistening(sock);
 		isc_nmsocket_close(&sock);
 	}
 
@@ -446,7 +468,7 @@ isc__nm_async_tcpdnslisten(isc__networker_t *worker, isc__netievent_t *ev0) {
 	int r;
 	int flags = 0;
 	isc_nmsocket_t *sock = NULL;
-	isc_result_t result = ISC_R_DEFAULT;
+	isc_result_t result = ISC_R_UNSET;
 
 	REQUIRE(VALID_NMSOCK(ievent->sock));
 	REQUIRE(ievent->sock->tid == isc_nm_tid());
@@ -534,16 +556,14 @@ done:
 		sock->pquota = NULL;
 	}
 
-	sock->parent->rchildren += 1;
-	if (sock->parent->result == ISC_R_DEFAULT) {
+	atomic_fetch_add(&sock->parent->rchildren, 1);
+	if (sock->parent->result == ISC_R_UNSET) {
 		sock->parent->result = result;
 	}
 	SIGNAL(&sock->parent->cond);
-	if (!atomic_load(&sock->parent->active)) {
-		WAIT(&sock->parent->scond, &sock->parent->lock);
-	}
-	INSIST(atomic_load(&sock->parent->active));
 	UNLOCK(&sock->parent->lock);
+
+	uv_barrier_wait(&sock->mgr->startlistening);
 }
 
 static void
@@ -607,7 +627,15 @@ isc__nm_tcpdns_stoplistening(isc_nmsocket_t *sock) {
 		INSIST(0);
 		ISC_UNREACHABLE();
 	}
-	enqueue_stoplistening(sock);
+
+	if (!isc__nm_in_netthread()) {
+		enqueue_stoplistening(sock);
+	} else if (!isc__nm_acquire_interlocked(sock->mgr)) {
+		enqueue_stoplistening(sock);
+	} else {
+		stop_tcpdns_parent(sock);
+		isc__nm_drop_interlocked(sock->mgr);
+	}
 }
 
 void
@@ -626,7 +654,15 @@ isc__nm_async_tcpdnsstop(isc__networker_t *worker, isc__netievent_t *ev0) {
 		return;
 	}
 
-	stop_tcpdns_parent(sock);
+	/*
+	 * If network manager is paused, re-enqueue the event for later.
+	 */
+	if (!isc__nm_acquire_interlocked(sock->mgr)) {
+		enqueue_stoplistening(sock);
+	} else {
+		stop_tcpdns_parent(sock);
+		isc__nm_drop_interlocked(sock->mgr);
+	}
 }
 
 void
@@ -1224,8 +1260,6 @@ timer_close_cb(uv_handle_t *timer) {
 
 static void
 stop_tcpdns_child(isc_nmsocket_t *sock) {
-	bool last_child = false;
-
 	REQUIRE(sock->type == isc_nm_tcpdnssocket);
 	REQUIRE(sock->tid == isc_nm_tid());
 
@@ -1236,33 +1270,43 @@ stop_tcpdns_child(isc_nmsocket_t *sock) {
 
 	tcpdns_close_direct(sock);
 
-	LOCK(&sock->parent->lock);
-	sock->parent->rchildren -= 1;
-	last_child = (sock->parent->rchildren == 0);
-	UNLOCK(&sock->parent->lock);
+	atomic_fetch_sub(&sock->parent->rchildren, 1);
 
-	if (last_child) {
-		atomic_store(&sock->parent->closed, true);
-		isc__nmsocket_prep_destroy(sock->parent);
-	}
+	(void)uv_barrier_wait(&sock->mgr->stoplistening);
 }
 
 static void
 stop_tcpdns_parent(isc_nmsocket_t *sock) {
+	isc_nmsocket_t *csock = NULL;
+
 	REQUIRE(VALID_NMSOCK(sock));
 	REQUIRE(sock->type == isc_nm_tcpdnslistener);
 
 	for (size_t i = 0; i < sock->nchildren; i++) {
 		isc__netievent_tcpdnsstop_t *ievent = NULL;
-		isc_nmsocket_t *csock = &sock->children[i];
+		csock = &sock->children[i];
 		REQUIRE(VALID_NMSOCK(csock));
 
+		if ((int)i == isc_nm_tid()) {
+			/*
+			 * We need to schedule closing the other sockets first
+			 */
+			continue;
+		}
+
 		atomic_store(&csock->active, false);
 
 		ievent = isc__nm_get_netievent_tcpdnsstop(sock->mgr, csock);
 		isc__nm_enqueue_ievent(&sock->mgr->workers[csock->tid],
 				       (isc__netievent_t *)ievent);
 	}
+
+	csock = &sock->children[isc_nm_tid()];
+	atomic_store(&csock->active, false);
+	stop_tcpdns_child(csock);
+
+	atomic_store(&sock->closed, true);
+	isc__nmsocket_prep_destroy(sock);
 }
 
 static void

lib/isc/netmgr/tlsdns.c

@@ -97,7 +97,7 @@ can_log_tlsdns_quota(void) {
 static isc_result_t
 tlsdns_connect_direct(isc_nmsocket_t *sock, isc__nm_uvreq_t *req) {
 	isc__networker_t *worker = NULL;
-	isc_result_t result = ISC_R_DEFAULT;
+	isc_result_t result = ISC_R_UNSET;
 	int r;
 
 	REQUIRE(VALID_NMSOCK(sock));
@@ -324,7 +324,7 @@ isc_nm_tlsdnsconnect(isc_nm_t *mgr, isc_nmiface_t *local, isc_nmiface_t *peer,
 
 	sock->extrahandlesize = extrahandlesize;
 	sock->connect_timeout = timeout;
-	sock->result = ISC_R_DEFAULT;
+	sock->result = ISC_R_UNSET;
 	sock->tls.ctx = sslctx;
 	atomic_init(&sock->client, true);
 	atomic_init(&sock->connecting, true);
@@ -364,7 +364,7 @@ isc_nm_tlsdnsconnect(isc_nm_t *mgr, isc_nmiface_t *local, isc_nmiface_t *peer,
 				       (isc__netievent_t *)ievent);
 	}
 	LOCK(&sock->lock);
-	while (sock->result == ISC_R_DEFAULT) {
+	while (sock->result == ISC_R_UNSET) {
 		WAIT(&sock->cond, &sock->lock);
 	}
 	atomic_store(&sock->active, true);
@@ -407,6 +407,42 @@ isc__nm_tlsdns_lb_socket(sa_family_t sa_family) {
 	return (sock);
 }
 
+static void
+start_tlsdns_child(isc_nm_t *mgr, isc_nmiface_t *iface, isc_nmsocket_t *sock,
+		   int tid) {
+	isc__netievent_tlsdnslisten_t *ievent = NULL;
+	isc_nmsocket_t *csock = &sock->children[tid];
+
+	isc__nmsocket_init(csock, mgr, isc_nm_tlsdnssocket, iface);
+	csock->parent = sock;
+	csock->accept_cb = sock->accept_cb;
+	csock->accept_cbarg = sock->accept_cbarg;
+	csock->recv_cb = sock->recv_cb;
+	csock->recv_cbarg = sock->recv_cbarg;
+	csock->extrahandlesize = sock->extrahandlesize;
+	csock->backlog = sock->backlog;
+	csock->tid = tid;
+	csock->tls.ctx = sock->tls.ctx;
+
+	/*
+	 * We don't attach to quota, just assign - to avoid
+	 * increasing quota unnecessarily.
+	 */
+	csock->pquota = sock->pquota;
+	isc_quota_cb_init(&csock->quotacb, quota_accept_cb, csock);
+
+#if HAVE_SO_REUSEPORT_LB || defined(WIN32)
+	csock->fd = isc__nm_tlsdns_lb_socket(iface->addr.type.sa.sa_family);
+#else
+	csock->fd = dup(sock->fd);
+#endif
+	REQUIRE(csock->fd >= 0);
+
+	ievent = isc__nm_get_netievent_tlsdnslisten(mgr, csock);
+	isc__nm_maybe_enqueue_ievent(&mgr->workers[tid],
+				     (isc__netievent_t *)ievent);
+}
+
 isc_result_t
 isc_nm_listentlsdns(isc_nm_t *mgr, isc_nmiface_t *iface,
 		    isc_nm_recv_cb_t recv_cb, void *recv_cbarg,
@@ -415,18 +451,14 @@ isc_nm_listentlsdns(isc_nm_t *mgr, isc_nmiface_t *iface,
 		    isc_tlsctx_t *sslctx, isc_nmsocket_t **sockp) {
 	isc_result_t result = ISC_R_SUCCESS;
 	isc_nmsocket_t *sock = NULL;
-	sa_family_t sa_family = iface->addr.type.sa.sa_family;
 	size_t children_size = 0;
-#if !HAVE_SO_REUSEPORT_LB && !defined(WIN32)
-	uv_os_sock_t fd = -1;
-#endif
 
 	REQUIRE(VALID_NM(mgr));
 
 	sock = isc_mem_get(mgr->mctx, sizeof(*sock));
 	isc__nmsocket_init(sock, mgr, isc_nm_tlsdnslistener, iface);
 
-	sock->rchildren = 0;
+	atomic_init(&sock->rchildren, 0);
 #if defined(WIN32)
 	sock->nchildren = 1;
 #else
@@ -436,69 +468,61 @@ isc_nm_listentlsdns(isc_nm_t *mgr, isc_nmiface_t *iface,
 	sock->children = isc_mem_get(mgr->mctx, children_size);
 	memset(sock->children, 0, children_size);
 
-	sock->result = ISC_R_DEFAULT;
-	sock->tid = isc_random_uniform(sock->nchildren);
-	sock->fd = -1;
+	sock->result = ISC_R_UNSET;
+	sock->result = ISC_R_UNSET;
+	sock->accept_cb = accept_cb;
+	sock->accept_cbarg = accept_cbarg;
+	sock->recv_cb = recv_cb;
+	sock->recv_cbarg = recv_cbarg;
+	sock->extrahandlesize = extrahandlesize;
+	sock->backlog = backlog;
+	sock->pquota = quota;
+
+	if (isc__nm_in_netthread()) {
+		sock->tid = isc_nm_tid();
+	} else {
+		sock->tid = isc_random_uniform(sock->nchildren);
+	}
+
 	sock->tls.ctx = sslctx;
 
 #if !HAVE_SO_REUSEPORT_LB && !defined(WIN32)
-	fd = isc__nm_tlsdns_lb_socket(sa_family);
+	sock->fd = isc__nm_tlsdns_lb_socket(iface->addr.type.sa.sa_family);
+#else
+	sock->fd = -1;
 #endif
 
 	for (size_t i = 0; i < sock->nchildren; i++) {
-		isc__netievent_tlsdnslisten_t *ievent = NULL;
-		isc_nmsocket_t *csock = &sock->children[i];
+		if ((int)i == isc_nm_tid()) {
+			continue;
+		}
+		start_tlsdns_child(mgr, iface, sock, i);
+	}
 
-		isc__nmsocket_init(csock, mgr, isc_nm_tlsdnssocket, iface);
-		csock->parent = sock;
-		csock->accept_cb = accept_cb;
-		csock->accept_cbarg = accept_cbarg;
-		csock->recv_cb = recv_cb;
-		csock->recv_cbarg = recv_cbarg;
-		csock->extrahandlesize = extrahandlesize;
-		csock->backlog = backlog;
-		csock->tid = i;
-		csock->tls.ctx = sslctx;
-
-		/*
-		 * We don't attach to quota, just assign - to avoid
-		 * increasing quota unnecessarily.
-		 */
-		csock->pquota = quota;
-		isc_quota_cb_init(&csock->quotacb, quota_accept_cb, csock);
-
-#if HAVE_SO_REUSEPORT_LB || defined(WIN32)
-		csock->fd = isc__nm_tlsdns_lb_socket(sa_family);
-#else
-		csock->fd = dup(fd);
-#endif
-		REQUIRE(csock->fd >= 0);
-
-		ievent = isc__nm_get_netievent_tlsdnslisten(mgr, csock);
-		isc__nm_maybe_enqueue_ievent(&mgr->workers[i],
-					     (isc__netievent_t *)ievent);
+	if (isc__nm_in_netthread()) {
+		start_tlsdns_child(mgr, iface, sock, isc_nm_tid());
 	}
 
 #if !HAVE_SO_REUSEPORT_LB && !defined(WIN32)
-	isc__nm_closesocket(fd);
+	isc__nm_closesocket(sock->fd);
 #endif
 
 	LOCK(&sock->lock);
-	while (sock->rchildren != sock->nchildren) {
+	while (atomic_load(&sock->rchildren) != sock->nchildren) {
 		WAIT(&sock->cond, &sock->lock);
 	}
 	result = sock->result;
 	atomic_store(&sock->active, true);
-	BROADCAST(&sock->scond);
 	UNLOCK(&sock->lock);
-	INSIST(result != ISC_R_DEFAULT);
+
+	INSIST(result != ISC_R_UNSET);
 
 	if (result == ISC_R_SUCCESS) {
-		REQUIRE(sock->rchildren == sock->nchildren);
+		REQUIRE(atomic_load(&sock->rchildren) == sock->nchildren);
 		*sockp = sock;
 	} else {
 		atomic_store(&sock->active, false);
-		isc__nm_tlsdns_stoplistening(sock);
+		isc_nm_stoplistening(sock);
 		isc_nmsocket_close(&sock);
 	}
 
@@ -514,7 +538,7 @@ isc__nm_async_tlsdnslisten(isc__networker_t *worker, isc__netievent_t *ev0) {
 	int r;
 	int flags = 0;
 	isc_nmsocket_t *sock = NULL;
-	isc_result_t result = ISC_R_DEFAULT;
+	isc_result_t result = ISC_R_UNSET;
 
 	REQUIRE(VALID_NMSOCK(ievent->sock));
 	REQUIRE(ievent->sock->tid == isc_nm_tid());
@@ -603,16 +627,14 @@ done:
 		sock->pquota = NULL;
 	}
 
-	sock->parent->rchildren += 1;
-	if (sock->parent->result == ISC_R_DEFAULT) {
+	atomic_fetch_add(&sock->parent->rchildren, 1);
+	if (sock->parent->result == ISC_R_UNSET) {
 		sock->parent->result = result;
 	}
 	SIGNAL(&sock->parent->cond);
-	if (!atomic_load(&sock->parent->active)) {
-		WAIT(&sock->parent->scond, &sock->parent->lock);
-	}
-	INSIST(atomic_load(&sock->parent->active));
 	UNLOCK(&sock->parent->lock);
+
+	uv_barrier_wait(&sock->mgr->startlistening);
 }
 
 static void
@@ -676,7 +698,15 @@ isc__nm_tlsdns_stoplistening(isc_nmsocket_t *sock) {
 		INSIST(0);
 		ISC_UNREACHABLE();
 	}
-	enqueue_stoplistening(sock);
+
+	if (!isc__nm_in_netthread()) {
+		enqueue_stoplistening(sock);
+	} else if (!isc__nm_acquire_interlocked(sock->mgr)) {
+		enqueue_stoplistening(sock);
+	} else {
+		stop_tlsdns_parent(sock);
+		isc__nm_drop_interlocked(sock->mgr);
+	}
 }
 
 static void
@@ -770,7 +800,15 @@ isc__nm_async_tlsdnsstop(isc__networker_t *worker, isc__netievent_t *ev0) {
 		return;
 	}
 
-	stop_tlsdns_parent(sock);
+	/*
+	 * If network manager is paused, re-enqueue the event for later.
+	 */
+	if (!isc__nm_acquire_interlocked(sock->mgr)) {
+		enqueue_stoplistening(sock);
+	} else {
+		stop_tlsdns_parent(sock);
+		isc__nm_drop_interlocked(sock->mgr);
+	}
 }
 
 void
@@ -1770,8 +1808,6 @@ timer_close_cb(uv_handle_t *handle) {
 
 static void
 stop_tlsdns_child(isc_nmsocket_t *sock) {
-	bool last_child = false;
-
 	REQUIRE(sock->type == isc_nm_tlsdnssocket);
 	REQUIRE(sock->tid == isc_nm_tid());
 
@@ -1782,34 +1818,44 @@ stop_tlsdns_child(isc_nmsocket_t *sock) {
 
 	tlsdns_close_direct(sock);
 
-	LOCK(&sock->parent->lock);
-	sock->parent->rchildren -= 1;
-	last_child = (sock->parent->rchildren == 0);
-	UNLOCK(&sock->parent->lock);
+	atomic_fetch_sub(&sock->parent->rchildren, 1);
 
-	if (last_child) {
-		atomic_store(&sock->parent->closed, true);
-		isc__nmsocket_prep_destroy(sock->parent);
-	}
+	(void)uv_barrier_wait(&sock->mgr->stoplistening);
 }
 
 static void
 stop_tlsdns_parent(isc_nmsocket_t *sock) {
+	isc_nmsocket_t *csock = NULL;
+
 	REQUIRE(VALID_NMSOCK(sock));
 	REQUIRE(sock->type == isc_nm_tlsdnslistener);
 
 	for (size_t i = 0; i < sock->nchildren; i++) {
 		isc__netievent_tlsdnsstop_t *ievent = NULL;
-		isc_nmsocket_t *csock = &sock->children[i];
+		csock = &sock->children[i];
 
 		REQUIRE(VALID_NMSOCK(csock));
 
+		if ((int)i == isc_nm_tid()) {
+			/*
+			 * We need to schedule closing the other sockets first
+			 */
+			continue;
+		}
+
 		atomic_store(&csock->active, false);
 
 		ievent = isc__nm_get_netievent_tlsdnsstop(sock->mgr, csock);
 		isc__nm_enqueue_ievent(&sock->mgr->workers[csock->tid],
 				       (isc__netievent_t *)ievent);
 	}
+
+	csock = &sock->children[isc_nm_tid()];
+	atomic_store(&csock->active, false);
+	stop_tlsdns_child(csock);
+
+	atomic_store(&sock->closed, true);
+	isc__nmsocket_prep_destroy(sock);
 }
 
 static void

lib/isc/netmgr/tlsstream.c

@@ -133,8 +133,14 @@ tls_senddone(isc_nmhandle_t *handle, isc_result_t eresult, void *cbarg) {
 	send_req->tlssock = NULL;
 
 	if (send_req->cb != NULL) {
+		INSIST(VALID_NMHANDLE(tlssock->statichandle));
 		send_req->cb(send_req->handle, eresult, send_req->cbarg);
 		isc_nmhandle_detach(&send_req->handle);
+		/* The last handle has been just detached: close the underlying
+		 * socket. */
+		if (tlssock->statichandle == NULL) {
+			finish = true;
+		}
 	}
 
 	isc_mem_put(handle->sock->mgr->mctx, send_req->data.base,
@@ -633,7 +639,7 @@ isc_nm_listentls(isc_nm_t *mgr, isc_nmiface_t *iface,
 	tlssock->tlsstream.server_iface = *iface;
 	ISC_LINK_INIT(&tlssock->tlsstream.server_iface.addr, link);
 	tlssock->iface = &tlssock->tlsstream.server_iface;
-	tlssock->result = ISC_R_DEFAULT;
+	tlssock->result = ISC_R_UNSET;
 	tlssock->accept_cb = accept_cb;
 	tlssock->accept_cbarg = accept_cbarg;
 	tlssock->extrahandlesize = extrahandlesize;
@@ -655,19 +661,12 @@ isc_nm_listentls(isc_nm_t *mgr, isc_nmiface_t *iface,
 
 	/* wait for listen result */
 	isc__nmsocket_attach(tlssock->outer, &tsock);
-	LOCK(&tlssock->outer->lock);
-	while (tlssock->outer->rchildren != tlssock->outer->nchildren) {
-		WAIT(&tlssock->outer->cond, &tlssock->outer->lock);
-	}
-	result = tlssock->outer->result;
 	tlssock->result = result;
 	atomic_store(&tlssock->active, true);
 	INSIST(tlssock->outer->tlsstream.tlslistener == NULL);
 	isc__nmsocket_attach(tlssock, &tlssock->outer->tlsstream.tlslistener);
-	BROADCAST(&tlssock->outer->scond);
-	UNLOCK(&tlssock->outer->lock);
 	isc__nmsocket_detach(&tsock);
-	INSIST(result != ISC_R_DEFAULT);
+	INSIST(result != ISC_R_UNSET);
 
 	if (result == ISC_R_SUCCESS) {
 		atomic_store(&tlssock->listening, true);
@@ -894,7 +893,7 @@ isc_nm_tlsconnect(isc_nm_t *mgr, isc_nmiface_t *local, isc_nmiface_t *peer,
 	ISC_LINK_INIT(&nsock->tlsstream.local_iface.addr, link);
 	nsock->iface = &nsock->tlsstream.local_iface;
 	nsock->extrahandlesize = extrahandlesize;
-	nsock->result = ISC_R_DEFAULT;
+	nsock->result = ISC_R_UNSET;
 	nsock->connect_cb = cb;
 	nsock->connect_cbarg = cbarg;
 	nsock->connect_timeout = timeout;
@@ -919,7 +918,7 @@ isc_nm_tlsconnect(isc_nm_t *mgr, isc_nmiface_t *local, isc_nmiface_t *peer,
 	}
 
 	LOCK(&nsock->lock);
-	while (nsock->result == ISC_R_DEFAULT) {
+	while (nsock->result == ISC_R_UNSET) {
 		WAIT(&nsock->cond, &nsock->lock);
 	}
 	atomic_store(&nsock->active, true);

lib/isc/netmgr/udp.c

@@ -77,17 +77,41 @@ isc__nm_udp_lb_socket(sa_family_t sa_family) {
 	return (sock);
 }
 
+static void
+start_udp_child(isc_nm_t *mgr, isc_nmiface_t *iface, isc_nmsocket_t *sock,
+		int tid) {
+	isc_nmsocket_t *csock;
+	isc__netievent_udplisten_t *ievent = NULL;
+
+	csock = &sock->children[tid];
+
+	isc__nmsocket_init(csock, mgr, isc_nm_udpsocket, iface);
+	csock->parent = sock;
+	csock->iface = sock->iface;
+	csock->reading = true;
+	csock->recv_cb = sock->recv_cb;
+	csock->recv_cbarg = sock->recv_cbarg;
+	csock->extrahandlesize = sock->extrahandlesize;
+	csock->tid = tid;
+
+#if HAVE_SO_REUSEPORT_LB || defined(WIN32)
+	csock->fd = isc__nm_udp_lb_socket(iface->addr.type.sa.sa_family);
+#else
+	csock->fd = dup(sock->fd);
+#endif
+	REQUIRE(csock->fd >= 0);
+
+	ievent = isc__nm_get_netievent_udplisten(mgr, csock);
+	isc__nm_maybe_enqueue_ievent(&mgr->workers[tid],
+				     (isc__netievent_t *)ievent);
+}
+
 isc_result_t
 isc_nm_listenudp(isc_nm_t *mgr, isc_nmiface_t *iface, isc_nm_recv_cb_t cb,
 		 void *cbarg, size_t extrahandlesize, isc_nmsocket_t **sockp) {
 	isc_result_t result = ISC_R_SUCCESS;
 	isc_nmsocket_t *sock = NULL;
-	sa_family_t sa_family = iface->addr.type.sa.sa_family;
 	size_t children_size = 0;
-#if !HAVE_SO_REUSEPORT_LB && !defined(WIN32)
-	uv_os_sock_t fd = -1;
-#endif
-
 	REQUIRE(VALID_NM(mgr));
 
 	/*
@@ -97,7 +121,7 @@ isc_nm_listenudp(isc_nm_t *mgr, isc_nmiface_t *iface, isc_nm_recv_cb_t cb,
 	sock = isc_mem_get(mgr->mctx, sizeof(isc_nmsocket_t));
 	isc__nmsocket_init(sock, mgr, isc_nm_udplistener, iface);
 
-	sock->rchildren = 0;
+	atomic_init(&sock->rchildren, 0);
 #if defined(WIN32)
 	sock->nchildren = 1;
 #else
@@ -111,59 +135,51 @@ isc_nm_listenudp(isc_nm_t *mgr, isc_nmiface_t *iface, isc_nm_recv_cb_t cb,
 	sock->recv_cb = cb;
 	sock->recv_cbarg = cbarg;
 	sock->extrahandlesize = extrahandlesize;
-	sock->result = ISC_R_DEFAULT;
-	sock->tid = isc_random_uniform(sock->nchildren);
-	sock->fd = -1;
-
-#if !HAVE_SO_REUSEPORT_LB && !defined(WIN32)
-	fd = isc__nm_udp_lb_socket(sa_family);
-#endif
-
-	for (size_t i = 0; i < sock->nchildren; i++) {
-		isc__netievent_udplisten_t *ievent = NULL;
-		isc_nmsocket_t *csock = &sock->children[i];
-
-		isc__nmsocket_init(csock, mgr, isc_nm_udpsocket, iface);
-		csock->parent = sock;
-		csock->iface = sock->iface;
-		csock->reading = true;
-		csock->recv_cb = cb;
-		csock->recv_cbarg = cbarg;
-		csock->extrahandlesize = sock->extrahandlesize;
-		csock->tid = i;
-
-#if HAVE_SO_REUSEPORT_LB || defined(WIN32)
-		csock->fd = isc__nm_udp_lb_socket(sa_family);
-#else
-		csock->fd = dup(fd);
-#endif
-		REQUIRE(csock->fd >= 0);
-
-		ievent = isc__nm_get_netievent_udplisten(mgr, csock);
-		isc__nm_maybe_enqueue_ievent(&mgr->workers[i],
-					     (isc__netievent_t *)ievent);
+	sock->result = ISC_R_UNSET;
+	if (isc__nm_in_netthread()) {
+		sock->tid = isc_nm_tid();
+	} else {
+		sock->tid = isc_random_uniform(sock->nchildren);
 	}
 
 #if !HAVE_SO_REUSEPORT_LB && !defined(WIN32)
-	isc__nm_closesocket(fd);
+	sock->fd = isc__nm_udp_lb_socket(iface->addr.type.sa.sa_family);
+#else
+	sock->fd = -1;
+#endif
+
+	for (size_t i = 0; i < sock->nchildren; i++) {
+		if ((int)i == isc_nm_tid()) {
+			continue;
+		}
+		start_udp_child(mgr, iface, sock, i);
+	}
+
+	if (isc__nm_in_netthread()) {
+		start_udp_child(mgr, iface, sock, isc_nm_tid());
+	}
+
+#if !HAVE_SO_REUSEPORT_LB && !defined(WIN32)
+	isc__nm_closesocket(sock->fd);
+	sock->fd = -1;
 #endif
 
 	LOCK(&sock->lock);
-	while (sock->rchildren != sock->nchildren) {
+	while (atomic_load(&sock->rchildren) != sock->nchildren) {
 		WAIT(&sock->cond, &sock->lock);
 	}
 	result = sock->result;
 	atomic_store(&sock->active, true);
-	BROADCAST(&sock->scond);
 	UNLOCK(&sock->lock);
-	INSIST(result != ISC_R_DEFAULT);
+
+	INSIST(result != ISC_R_UNSET);
 
 	if (result == ISC_R_SUCCESS) {
-		REQUIRE(sock->rchildren == sock->nchildren);
+		REQUIRE(atomic_load(&sock->rchildren) == sock->nchildren);
 		*sockp = sock;
 	} else {
 		atomic_store(&sock->active, false);
-		isc__nm_udp_stoplistening(sock);
+		isc_nm_stoplistening(sock);
 		isc_nmsocket_close(&sock);
 	}
 
@@ -181,7 +197,7 @@ isc__nm_async_udplisten(isc__networker_t *worker, isc__netievent_t *ev0) {
 	int r, uv_bind_flags = 0;
 	int uv_init_flags = 0;
 	sa_family_t sa_family;
-	isc_result_t result = ISC_R_DEFAULT;
+	isc_result_t result = ISC_R_UNSET;
 
 	REQUIRE(VALID_NMSOCK(ievent->sock));
 	REQUIRE(ievent->sock->tid == isc_nm_tid());
@@ -269,16 +285,14 @@ isc__nm_async_udplisten(isc__networker_t *worker, isc__netievent_t *ev0) {
 
 done:
 	result = isc__nm_uverr2result(r);
-	sock->parent->rchildren += 1;
-	if (sock->parent->result == ISC_R_DEFAULT) {
+	atomic_fetch_add(&sock->parent->rchildren, 1);
+	if (sock->parent->result == ISC_R_UNSET) {
 		sock->parent->result = result;
 	}
 	SIGNAL(&sock->parent->cond);
-	if (!atomic_load(&sock->parent->active)) {
-		WAIT(&sock->parent->scond, &sock->parent->lock);
-	}
-	INSIST(atomic_load(&sock->parent->active));
 	UNLOCK(&sock->parent->lock);
+
+	uv_barrier_wait(&sock->mgr->startlistening);
 }
 
 static void
@@ -300,7 +314,14 @@ isc__nm_udp_stoplistening(isc_nmsocket_t *sock) {
 		ISC_UNREACHABLE();
 	}
 
-	enqueue_stoplistening(sock);
+	if (!isc__nm_in_netthread()) {
+		enqueue_stoplistening(sock);
+	} else if (!isc__nm_acquire_interlocked(sock->mgr)) {
+		enqueue_stoplistening(sock);
+	} else {
+		stop_udp_parent(sock);
+		isc__nm_drop_interlocked(sock->mgr);
+	}
 }
 
 /*
@@ -324,7 +345,12 @@ isc__nm_async_udpstop(isc__networker_t *worker, isc__netievent_t *ev0) {
 	/*
 	 * If network manager is paused, re-enqueue the event for later.
 	 */
-	stop_udp_parent(sock);
+	if (!isc__nm_acquire_interlocked(sock->mgr)) {
+		enqueue_stoplistening(sock);
+	} else {
+		stop_udp_parent(sock);
+		isc__nm_drop_interlocked(sock->mgr);
+	}
 }
 
 /*
@@ -590,7 +616,7 @@ static isc_result_t
 udp_connect_direct(isc_nmsocket_t *sock, isc__nm_uvreq_t *req) {
 	isc__networker_t *worker = NULL;
 	int uv_bind_flags = UV_UDP_REUSEADDR;
-	isc_result_t result = ISC_R_DEFAULT;
+	isc_result_t result = ISC_R_UNSET;
 	int tries = 3;
 	int r;
 
@@ -734,7 +760,7 @@ isc_nm_udpconnect(isc_nm_t *mgr, isc_nmiface_t *local, isc_nmiface_t *peer,
 	sock->read_timeout = timeout;
 	sock->extrahandlesize = extrahandlesize;
 	sock->peer = peer->addr;
-	sock->result = ISC_R_DEFAULT;
+	sock->result = ISC_R_UNSET;
 	atomic_init(&sock->client, true);
 
 	req = isc__nm_uvreq_get(mgr, sock);
@@ -782,7 +808,7 @@ isc_nm_udpconnect(isc_nm_t *mgr, isc_nmiface_t *local, isc_nmiface_t *peer,
 				       (isc__netievent_t *)event);
 	}
 	LOCK(&sock->lock);
-	while (sock->result == ISC_R_DEFAULT) {
+	while (sock->result == ISC_R_UNSET) {
 		WAIT(&sock->cond, &sock->lock);
 	}
 	atomic_store(&sock->active, true);
@@ -970,8 +996,6 @@ stop_udp_child(isc_nmsocket_t *sock) {
 	REQUIRE(sock->type == isc_nm_udpsocket);
 	REQUIRE(sock->tid == isc_nm_tid());
 
-	bool last_child = false;
-
 	if (!atomic_compare_exchange_strong(&sock->closing, &(bool){ false },
 					    true)) {
 		return;
@@ -979,33 +1003,43 @@ stop_udp_child(isc_nmsocket_t *sock) {
 
 	udp_close_direct(sock);
 
-	LOCK(&sock->parent->lock);
-	sock->parent->rchildren -= 1;
-	last_child = (sock->parent->rchildren == 0);
-	UNLOCK(&sock->parent->lock);
+	atomic_fetch_sub(&sock->parent->rchildren, 1);
 
-	if (last_child) {
-		atomic_store(&sock->parent->closed, true);
-		isc__nmsocket_prep_destroy(sock->parent);
-	}
+	(void)uv_barrier_wait(&sock->mgr->stoplistening);
 }
 
 static void
 stop_udp_parent(isc_nmsocket_t *sock) {
+	isc_nmsocket_t *csock = NULL;
+
 	REQUIRE(VALID_NMSOCK(sock));
 	REQUIRE(sock->type == isc_nm_udplistener);
 
 	for (size_t i = 0; i < sock->nchildren; i++) {
 		isc__netievent_udpstop_t *ievent = NULL;
-		isc_nmsocket_t *csock = &sock->children[i];
+		csock = &sock->children[i];
 		REQUIRE(VALID_NMSOCK(csock));
 
+		if ((int)i == isc_nm_tid()) {
+			/*
+			 * We need to schedule closing the other sockets first
+			 */
+			continue;
+		}
+
 		atomic_store(&csock->active, false);
 
 		ievent = isc__nm_get_netievent_udpstop(sock->mgr, csock);
 		isc__nm_enqueue_ievent(&sock->mgr->workers[i],
 				       (isc__netievent_t *)ievent);
 	}
+
+	csock = &sock->children[isc_nm_tid()];
+	atomic_store(&csock->active, false);
+	stop_udp_child(csock);
+
+	atomic_store(&sock->closed, true);
+	isc__nmsocket_prep_destroy(sock);
 }
 
 static void

lib/isc/netmgr_p.h

@@ -0,0 +1,36 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, you can obtain one at https://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+#pragma once
+
+#include <isc/mem.h>
+#include <isc/result.h>
+
+void
+isc__netmgr_create(isc_mem_t *mctx, uint32_t workers, isc_nm_t **netgmrp);
+/*%<
+ * Creates a new network manager with 'workers' worker threads,
+ * and starts it running.
+ */
+
+void
+isc__netmgr_destroy(isc_nm_t **netmgrp);
+/*%<
+ * Similar to isc_nm_detach(), but actively waits for all other references
+ * to be gone before returning.
+ */
+
+void
+isc__netmgr_shutdown(isc_nm_t *mgr);
+/*%<
+ * Shut down all active connections, freeing associated resources;
+ * prevent new connections from being established.
+ */

lib/isc/socket_p.h

@@ -0,0 +1,81 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, you can obtain one at https://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+#include <isc/mem.h>
+#include <isc/result.h>
+#include <isc/socket.h>
+
+isc_result_t
+isc__socketmgr_create(isc_mem_t *mctx, isc_socketmgr_t **managerp,
+		      unsigned int maxsocks, int nthreads);
+/*%<
+ * Create a socket manager.  If "maxsocks" is non-zero, it specifies the
+ * maximum number of sockets that the created manager should handle.
+ *
+ * Notes:
+ *
+ *\li	All memory will be allocated in memory context 'mctx'.
+ *
+ * Requires:
+ *
+ *\li	'mctx' is a valid memory context.
+ *
+ *\li	'managerp' points to a NULL isc_socketmgr_t.
+ *
+ * Ensures:
+ *
+ *\li	'*managerp' is a valid isc_socketmgr_t.
+ *
+ * Returns:
+ *
+ *\li	#ISC_R_SUCCESS
+ *\li	#ISC_R_NOMEMORY
+ *\li	#ISC_R_UNEXPECTED
+ *\li	#ISC_R_NOTIMPLEMENTED
+ */
+
+void
+isc__socketmgr_destroy(isc_socketmgr_t **managerp);
+/*%<
+ * Destroy a socket manager.
+ *
+ * Notes:
+ *
+ *\li	This routine blocks until there are no sockets left in the manager,
+ *	so if the caller holds any socket references using the manager, it
+ *	must detach them before calling isc_socketmgr_destroy() or it will
+ *	block forever.
+ *
+ * Requires:
+ *
+ *\li	'*managerp' is a valid isc_socketmgr_t.
+ *
+ *\li	All sockets managed by this manager are fully detached.
+ *
+ * Ensures:
+ *
+ *\li	*managerp == NULL
+ *
+ *\li	All resources used by the manager have been freed.
+ */
+
+#if !WIN32
+
+#include <sys/time.h>
+
+typedef struct isc_socketwait isc_socketwait_t;
+int
+isc__socketmgr_waitevents(isc_socketmgr_t *, struct timeval *,
+			  isc_socketwait_t **);
+isc_result_t
+isc__socketmgr_dispatch(isc_socketmgr_t *, isc_socketwait_t *);
+
+#endif

lib/isc/task.c

@@ -17,6 +17,7 @@
  */
 
 #include <stdbool.h>
+#include <unistd.h>
 
 #include <isc/app.h>
 #include <isc/atomic.h>
@@ -44,9 +45,7 @@
 #include <json_object.h>
 #endif /* HAVE_JSON_C */
 
-#ifdef OPENSSL_LEAKS
-#include <openssl/err.h>
-#endif /* ifdef OPENSSL_LEAKS */
+#include "task_p.h"
 
 /*
  * Task manager is built around 'as little locking as possible' concept.
@@ -104,6 +103,7 @@ struct isc_task {
 	task_state_t state;
 	int pause_cnt;
 	isc_refcount_t references;
+	isc_refcount_t running;
 	isc_eventlist_t events;
 	isc_eventlist_t on_shutdown;
 	unsigned int nevents;
@@ -114,21 +114,14 @@ struct isc_task {
 	void *tag;
 	bool bound;
 	/* Protected by atomics */
-	atomic_uint_fast32_t flags;
+	atomic_bool shuttingdown;
+	atomic_bool privileged;
 	/* Locked by task manager lock. */
 	LINK(isc_task_t) link;
 };
 
-#define TASK_F_SHUTTINGDOWN 0x01
-#define TASK_F_PRIVILEGED   0x02
-
-#define TASK_SHUTTINGDOWN(t) \
-	((atomic_load_acquire(&(t)->flags) & TASK_F_SHUTTINGDOWN) != 0)
-#define TASK_PRIVILEGED(t) \
-	((atomic_load_acquire(&(t)->flags) & TASK_F_PRIVILEGED) != 0)
-
-#define TASK_FLAG_SET(t, f) atomic_fetch_or_release(&(t)->flags, (f))
-#define TASK_FLAG_CLR(t, f) atomic_fetch_and_release(&(t)->flags, ~(f))
+#define TASK_SHUTTINGDOWN(t) (atomic_load_acquire(&(t)->shuttingdown))
+#define TASK_PRIVILEGED(t)   (atomic_load_acquire(&(t)->privileged))
 
 #define TASK_MANAGER_MAGIC ISC_MAGIC('T', 'S', 'K', 'M')
 #define VALID_MANAGER(m)   ISC_MAGIC_VALID(m, TASK_MANAGER_MAGIC)
@@ -139,10 +132,8 @@ struct isc_taskmgr {
 	isc_refcount_t references;
 	isc_mem_t *mctx;
 	isc_mutex_t lock;
-	atomic_uint_fast32_t tasks_running;
-	atomic_uint_fast32_t tasks_ready;
 	atomic_uint_fast32_t tasks_count;
-	isc_nm_t *nm;
+	isc_nm_t *netmgr;
 
 	/* Locked by task manager lock. */
 	unsigned int default_quantum;
@@ -150,9 +141,6 @@ struct isc_taskmgr {
 	atomic_bool exclusive_req;
 	atomic_bool exiting;
 
-	/* Locked by halt_lock */
-	unsigned int halted;
-
 	/*
 	 * Multiple threads can read/write 'excl' at the same time, so we need
 	 * to protect the access.  We can't use 'lock' since isc_task_detach()
@@ -163,9 +151,6 @@ struct isc_taskmgr {
 };
 
 #define DEFAULT_DEFAULT_QUANTUM 25
-#define FINISHED(m)                              \
-	(atomic_load_relaxed(&((m)->exiting)) && \
-	 atomic_load(&(m)->tasks_count) == 0)
 
 /*%
  * The following are intended for internal use (indicated by "isc__"
@@ -195,6 +180,7 @@ task_finished(isc_task_t *task) {
 
 	XTRACE("task_finished");
 
+	isc_refcount_destroy(&task->running);
 	isc_refcount_destroy(&task->references);
 
 	LOCK(&manager->lock);
@@ -253,11 +239,13 @@ isc_task_create_bound(isc_taskmgr_t *manager, unsigned int quantum,
 	task->pause_cnt = 0;
 
 	isc_refcount_init(&task->references, 1);
+	isc_refcount_init(&task->running, 0);
 	INIT_LIST(task->events);
 	INIT_LIST(task->on_shutdown);
 	task->nevents = 0;
 	task->quantum = (quantum > 0) ? quantum : manager->default_quantum;
-	atomic_init(&task->flags, 0);
+	atomic_init(&task->shuttingdown, false);
+	atomic_init(&task->privileged, false);
 	task->now = 0;
 	isc_time_settoepoch(&task->tnow);
 	memset(task->name, 0, sizeof(task->name));
@@ -313,9 +301,9 @@ task_shutdown(isc_task_t *task) {
 
 	XTRACE("task_shutdown");
 
-	if (!TASK_SHUTTINGDOWN(task)) {
+	if (atomic_compare_exchange_strong(&task->shuttingdown,
+					   &(bool){ false }, true)) {
 		XTRACE("shutting down");
-		TASK_FLAG_SET(task, TASK_F_SHUTTINGDOWN);
 		if (task->state == task_state_idle) {
 			INSIST(EMPTY(task->events));
 			task->state = task_state_ready;
@@ -352,7 +340,14 @@ task_ready(isc_task_t *task) {
 	REQUIRE(VALID_MANAGER(manager));
 
 	XTRACE("task_ready");
-	isc_nm_task_enqueue(manager->nm, task, task->threadid);
+
+	isc_refcount_increment0(&task->running);
+	isc_nm_task_enqueue(manager->netmgr, task, task->threadid);
+}
+
+void
+isc_task_ready(isc_task_t *task) {
+	task_ready(task);
 }
 
 static inline bool
@@ -822,8 +817,7 @@ task_run(isc_task_t *task) {
 	 * and task lock to avoid deadlocks, just bail then.
 	 */
 	if (task->state != task_state_ready) {
-		UNLOCK(&task->lock);
-		return (ISC_R_SUCCESS);
+		goto done;
 	}
 
 	INSIST(task->state == task_state_ready);
@@ -888,7 +882,6 @@ task_run(isc_task_t *task) {
 				 * The task is done.
 				 */
 				XTRACE("done");
-				finished = true;
 				task->state = task_state_done;
 			} else {
 				if (task->state == task_state_running) {
@@ -922,6 +915,13 @@ task_run(isc_task_t *task) {
 			break;
 		}
 	}
+
+done:
+	if (isc_refcount_decrement(&task->running) == 1 &&
+	    task->state == task_state_done)
+	{
+		finished = true;
+	}
 	UNLOCK(&task->lock);
 
 	if (finished) {
@@ -939,7 +939,7 @@ isc_task_run(isc_task_t *task) {
 static void
 manager_free(isc_taskmgr_t *manager) {
 	isc_refcount_destroy(&manager->references);
-	isc_nm_detach(&manager->nm);
+	isc_nm_detach(&manager->netmgr);
 
 	isc_mutex_destroy(&manager->lock);
 	isc_mutex_destroy(&manager->excl_lock);
@@ -967,8 +967,8 @@ isc_taskmgr_detach(isc_taskmgr_t *manager) {
 }
 
 isc_result_t
-isc_taskmgr_create(isc_mem_t *mctx, unsigned int default_quantum, isc_nm_t *nm,
-		   isc_taskmgr_t **managerp) {
+isc__taskmgr_create(isc_mem_t *mctx, unsigned int default_quantum, isc_nm_t *nm,
+		    isc_taskmgr_t **managerp) {
 	isc_taskmgr_t *manager;
 
 	/*
@@ -990,13 +990,10 @@ isc_taskmgr_create(isc_mem_t *mctx, unsigned int default_quantum, isc_nm_t *nm,
 	manager->default_quantum = default_quantum;
 
 	if (nm != NULL) {
-		isc_nm_attach(nm, &manager->nm);
+		isc_nm_attach(nm, &manager->netmgr);
 	}
 
 	INIT_LIST(manager->tasks);
-	atomic_init(&manager->tasks_count, 0);
-	atomic_init(&manager->tasks_running, 0);
-	atomic_init(&manager->tasks_ready, 0);
 	atomic_init(&manager->exiting, false);
 	atomic_store_relaxed(&manager->exclusive_req, false);
 
@@ -1010,19 +1007,12 @@ isc_taskmgr_create(isc_mem_t *mctx, unsigned int default_quantum, isc_nm_t *nm,
 }
 
 void
-isc_taskmgr_destroy(isc_taskmgr_t **managerp) {
-	isc_taskmgr_t *manager;
+isc__taskmgr_shutdown(isc_taskmgr_t *manager) {
 	isc_task_t *task;
 
-	/*
-	 * Destroy '*managerp'.
-	 */
-
-	REQUIRE(managerp != NULL);
-	manager = *managerp;
 	REQUIRE(VALID_MANAGER(manager));
 
-	XTHREADTRACE("isc_taskmgr_destroy");
+	XTHREADTRACE("isc_taskmgr_shutdown");
 	/*
 	 * Only one non-worker thread may ever call this routine.
 	 * If a worker thread wants to initiate shutdown of the
@@ -1072,10 +1062,23 @@ isc_taskmgr_destroy(isc_taskmgr_t **managerp) {
 	}
 
 	UNLOCK(&manager->lock);
+}
 
-	isc_taskmgr_detach(manager);
+void
+isc__taskmgr_destroy(isc_taskmgr_t **managerp) {
+	REQUIRE(managerp != NULL && VALID_MANAGER(*managerp));
 
+	isc_taskmgr_t *manager = *managerp;
 	*managerp = NULL;
+
+	XTHREADTRACE("isc_taskmgr_destroy");
+
+	while (isc_refcount_current(&manager->references) > 1) {
+		usleep(1000);
+	}
+
+	REQUIRE(isc_refcount_decrement(&manager->references) == 1);
+	manager_free(manager);
 }
 
 void
@@ -1130,7 +1133,7 @@ isc_task_beginexclusive(isc_task_t *task) {
 		return (ISC_R_LOCKBUSY);
 	}
 
-	isc_nm_pause(manager->nm);
+	isc_nm_pause(manager->netmgr);
 
 	return (ISC_R_SUCCESS);
 }
@@ -1143,7 +1146,7 @@ isc_task_endexclusive(isc_task_t *task) {
 	REQUIRE(task->state == task_state_running);
 	manager = task->manager;
 
-	isc_nm_resume(manager->nm);
+	isc_nm_resume(manager->netmgr);
 	REQUIRE(atomic_compare_exchange_strong(&manager->exclusive_req,
 					       &(bool){ true }, false));
 }
@@ -1210,20 +1213,8 @@ isc_task_unpause(isc_task_t *task) {
 void
 isc_task_setprivilege(isc_task_t *task, bool priv) {
 	REQUIRE(VALID_TASK(task));
-	uint_fast32_t oldflags, newflags;
 
-	oldflags = atomic_load_acquire(&task->flags);
-	do {
-		if (priv) {
-			newflags = oldflags | TASK_F_PRIVILEGED;
-		} else {
-			newflags = oldflags & ~TASK_F_PRIVILEGED;
-		}
-		if (newflags == oldflags) {
-			return;
-		}
-	} while (!atomic_compare_exchange_weak_acq_rel(&task->flags, &oldflags,
-						       newflags));
+	atomic_store_release(&task->privileged, priv);
 }
 
 bool
@@ -1269,21 +1260,6 @@ isc_taskmgr_renderxml(isc_taskmgr_t *mgr, void *writer0) {
 					    mgr->default_quantum));
 	TRY0(xmlTextWriterEndElement(writer)); /* default-quantum */
 
-	TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "tasks-count"));
-	TRY0(xmlTextWriterWriteFormatString(
-		writer, "%d", (int)atomic_load_relaxed(&mgr->tasks_count)));
-	TRY0(xmlTextWriterEndElement(writer)); /* tasks-count */
-
-	TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "tasks-running"));
-	TRY0(xmlTextWriterWriteFormatString(
-		writer, "%d", (int)atomic_load_relaxed(&mgr->tasks_running)));
-	TRY0(xmlTextWriterEndElement(writer)); /* tasks-running */
-
-	TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "tasks-ready"));
-	TRY0(xmlTextWriterWriteFormatString(
-		writer, "%d", (int)atomic_load_relaxed(&mgr->tasks_ready)));
-	TRY0(xmlTextWriterEndElement(writer)); /* tasks-ready */
-
 	TRY0(xmlTextWriterEndElement(writer)); /* thread-model */
 
 	TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "tasks"));
@@ -1373,18 +1349,6 @@ isc_taskmgr_renderjson(isc_taskmgr_t *mgr, void *tasks0) {
 	CHECKMEM(obj);
 	json_object_object_add(tasks, "default-quantum", obj);
 
-	obj = json_object_new_int(atomic_load_relaxed(&mgr->tasks_count));
-	CHECKMEM(obj);
-	json_object_object_add(tasks, "tasks-count", obj);
-
-	obj = json_object_new_int(atomic_load_relaxed(&mgr->tasks_running));
-	CHECKMEM(obj);
-	json_object_object_add(tasks, "tasks-running", obj);
-
-	obj = json_object_new_int(atomic_load_relaxed(&mgr->tasks_ready));
-	CHECKMEM(obj);
-	json_object_object_add(tasks, "tasks-ready", obj);
-
 	array = json_object_new_array();
 	CHECKMEM(array);
 

lib/isc/task_p.h

@@ -0,0 +1,104 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, you can obtain one at https://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+#pragma once
+
+#include <isc/mem.h>
+#include <isc/result.h>
+#include <isc/task.h>
+
+isc_result_t
+isc__taskmgr_create(isc_mem_t *mctx, unsigned int default_quantum, isc_nm_t *nm,
+		    isc_taskmgr_t **managerp);
+/*%<
+ * Create a new task manager.
+ *
+ * Notes:
+ *
+ *\li	If 'default_quantum' is non-zero, then it will be used as the default
+ *	quantum value when tasks are created.  If zero, then an implementation
+ *	defined default quantum will be used.
+ *
+ *\li	If 'nm' is set then netmgr is paused when an exclusive task mode
+ *	is requested.
+ *
+ * Requires:
+ *
+ *\li      'mctx' is a valid memory context.
+ *
+ *\li	managerp != NULL && *managerp == NULL
+ *
+ * Ensures:
+ *
+ *\li	On success, '*managerp' will be attached to the newly created task
+ *	manager.
+ *
+ * Returns:
+ *
+ *\li	#ISC_R_SUCCESS
+ *\li	#ISC_R_NOMEMORY
+ *\li	#ISC_R_NOTHREADS		No threads could be created.
+ *\li	#ISC_R_UNEXPECTED		An unexpected error occurred.
+ *\li	#ISC_R_SHUTTINGDOWN		The non-threaded, shared, task
+ *					manager shutting down.
+ */
+
+void
+isc__taskmgr_destroy(isc_taskmgr_t **managerp);
+/*%<
+ * Destroy '*managerp'.
+ *
+ * Notes:
+ *
+ *\li	Calling isc_taskmgr_destroy() will shutdown all tasks managed by
+ *	*managerp that haven't already been shutdown.  The call will block
+ *	until all tasks have entered the done state.
+ *
+ *\li	isc_taskmgr_destroy() must not be called by a task event action,
+ *	because it would block forever waiting for the event action to
+ *	complete.  An event action that wants to cause task manager shutdown
+ *	should request some non-event action thread of execution to do the
+ *	shutdown, e.g. by signaling a condition variable or using
+ *	isc_app_shutdown().
+ *
+ *\li	Task manager references are not reference counted, so the caller
+ *	must ensure that no attempt will be made to use the manager after
+ *	isc_taskmgr_destroy() returns.
+ *
+ * Requires:
+ *
+ *\li	'*managerp' is a valid task manager.
+ *
+ *\li   'isc__taskmgr_shutdown()' and isc__netmgr_shutdown() have been
+ *	called.
+ */
+
+void
+isc__taskmgr_shutdown(isc_taskmgr_t *manager);
+/*%>
+ * Shutdown 'manager'.
+ *
+ * Notes:
+ *
+ *\li	Calling isc__taskmgr_shutdown() will shut down all tasks managed by
+ *	*managerp that haven't already been shut down.
+ *
+ * Requires:
+ *
+ *\li   'manager' is a valid task manager.
+ *
+ *\li	isc_taskmgr_destroy() has not be called previously on '*managerp'.
+ *
+ * Ensures:
+ *
+ *\li	All resources used by the task manager, and any tasks it managed,
+ *	have been freed.
+ */

lib/isc/tests/Makefile.am

@@ -10,7 +10,7 @@ LDADD +=			\
 
 check_LTLIBRARIES = libisctest.la
 libisctest_la_SOURCES =		\
-	../unix/socket_p.h	\
+	../socket_p.h		\
 	isctest.c		\
 	isctest.h		\
 	uv_wrap.h

lib/isc/tests/doh_test.c

@@ -43,6 +43,7 @@
 #include "../netmgr/netmgr-int.h"
 #include "../netmgr/uv-compat.c"
 #include "../netmgr/uv-compat.h"
+#include "../netmgr_p.h"
 #include "isctest.h"
 
 #define MAX_NM 2
@@ -312,7 +313,7 @@ nm_setup(void **state) {
 
 	nm = isc_mem_get(test_mctx, MAX_NM * sizeof(nm[0]));
 	for (size_t i = 0; i < MAX_NM; i++) {
-		nm[i] = isc_nm_start(test_mctx, nworkers);
+		isc__netmgr_create(test_mctx, nworkers, &nm[i]);
 		assert_non_null(nm[i]);
 	}
 
@@ -332,7 +333,7 @@ nm_teardown(void **state) {
 	isc_nm_t **nm = (isc_nm_t **)*state;
 
 	for (size_t i = 0; i < MAX_NM; i++) {
-		isc_nm_destroy(&nm[i]);
+		isc__netmgr_destroy(&nm[i]);
 		assert_null(nm[i]);
 	}
 	isc_mem_put(test_mctx, nm, MAX_NM * sizeof(nm[0]));
@@ -492,7 +493,7 @@ doh_noop(void **state) {
 					      .length = send_msg.len },
 			     noop_read_cb, NULL, atomic_load(&use_TLS), 30000);
 
-	isc_nm_closedown(connect_nm);
+	isc__netmgr_shutdown(connect_nm);
 
 	assert_int_equal(0, atomic_load(&csends));
 	assert_int_equal(0, atomic_load(&creads));
@@ -539,7 +540,7 @@ doh_noresponse(void **state) {
 	isc_nm_stoplistening(listen_sock);
 	isc_nmsocket_close(&listen_sock);
 	assert_null(listen_sock);
-	isc_nm_closedown(connect_nm);
+	isc__netmgr_shutdown(connect_nm);
 }
 
 static void
@@ -660,7 +661,7 @@ doh_timeout_recovery(void **state) {
 	isc_nm_stoplistening(listen_sock);
 	isc_nmsocket_close(&listen_sock);
 	assert_null(listen_sock);
-	isc_nm_closedown(connect_nm);
+	isc__netmgr_shutdown(connect_nm);
 }
 
 static void
@@ -779,7 +780,7 @@ doh_recv_one(void **state) {
 	isc_nm_stoplistening(listen_sock);
 	isc_nmsocket_close(&listen_sock);
 	assert_null(listen_sock);
-	isc_nm_closedown(connect_nm);
+	isc__netmgr_shutdown(connect_nm);
 
 	X(csends);
 	X(creads);
@@ -900,7 +901,7 @@ doh_recv_two(void **state) {
 	isc_nm_stoplistening(listen_sock);
 	isc_nmsocket_close(&listen_sock);
 	assert_null(listen_sock);
-	isc_nm_closedown(connect_nm);
+	isc__netmgr_shutdown(connect_nm);
 
 	X(csends);
 	X(creads);
@@ -966,7 +967,7 @@ doh_recv_send(void **state) {
 		isc_thread_join(threads[i], NULL);
 	}
 
-	isc_nm_closedown(connect_nm);
+	isc__netmgr_shutdown(connect_nm);
 	isc_nm_stoplistening(listen_sock);
 	isc_nmsocket_close(&listen_sock);
 	assert_null(listen_sock);
@@ -1035,7 +1036,7 @@ doh_recv_half_send(void **state) {
 		isc_thread_yield();
 	}
 
-	isc_nm_closedown(connect_nm);
+	isc__netmgr_shutdown(connect_nm);
 
 	for (size_t i = 0; i < nthreads; i++) {
 		isc_thread_join(threads[i], NULL);
@@ -1117,7 +1118,7 @@ doh_half_recv_send(void **state) {
 		isc_thread_join(threads[i], NULL);
 	}
 
-	isc_nm_closedown(connect_nm);
+	isc__netmgr_shutdown(connect_nm);
 
 	X(csends);
 	X(creads);
@@ -1183,7 +1184,7 @@ doh_half_recv_half_send(void **state) {
 		isc_thread_yield();
 	}
 
-	isc_nm_closedown(connect_nm);
+	isc__netmgr_shutdown(connect_nm);
 	isc_nm_stoplistening(listen_sock);
 	isc_nmsocket_close(&listen_sock);
 	assert_null(listen_sock);

lib/isc/tests/isctest.c

@@ -20,6 +20,7 @@
 #include <isc/buffer.h>
 #include <isc/hash.h>
 #include <isc/hp.h>
+#include <isc/managers.h>
 #include <isc/mem.h>
 #include <isc/os.h>
 #include <isc/socket.h>
@@ -58,18 +59,10 @@ cleanup_managers(void) {
 		isc_task_shutdown(maintask);
 		isc_task_destroy(&maintask);
 	}
-	if (socketmgr != NULL) {
-		isc_socketmgr_destroy(&socketmgr);
-	}
-	if (taskmgr != NULL) {
-		isc_taskmgr_destroy(&taskmgr);
-	}
-	if (netmgr != NULL) {
-		isc_nm_destroy(&netmgr);
-	}
-	if (timermgr != NULL) {
-		isc_timermgr_destroy(&timermgr);
-	}
+	isc_managers_destroy(netmgr == NULL ? NULL : &netmgr,
+			     taskmgr == NULL ? NULL : &taskmgr,
+			     timermgr == NULL ? NULL : &timermgr,
+			     socketmgr == NULL ? NULL : &socketmgr);
 }
 
 static isc_result_t
@@ -87,14 +80,12 @@ create_managers(unsigned int workers) {
 	INSIST(workers != 0);
 
 	isc_hp_init(6 * workers);
+	isc_managers_create(test_mctx, workers, 0, 0, &netmgr, &taskmgr,
+			    &timermgr, &socketmgr);
 
-	netmgr = isc_nm_start(test_mctx, workers);
-	CHECK(isc_taskmgr_create(test_mctx, 0, netmgr, &taskmgr));
 	CHECK(isc_task_create(taskmgr, 0, &maintask));
 	isc_taskmgr_setexcltask(taskmgr, maintask);
 
-	CHECK(isc_timermgr_create(test_mctx, &timermgr));
-	CHECK(isc_socketmgr_create(test_mctx, &socketmgr));
 	return (ISC_R_SUCCESS);
 
 cleanup:
@@ -150,9 +141,6 @@ isc_test_end(void) {
 	if (maintask != NULL) {
 		isc_task_detach(&maintask);
 	}
-	if (taskmgr != NULL) {
-		isc_taskmgr_destroy(&taskmgr);
-	}
 
 	cleanup_managers();
 

lib/isc/tests/netmgr_test.c

@@ -37,6 +37,7 @@
 #include "../netmgr/udp.c"
 #include "../netmgr/uv-compat.c"
 #include "../netmgr/uv-compat.h"
+#include "../netmgr_p.h"
 #include "isctest.h"
 
 typedef void (*stream_connect_function)(isc_nm_t *nm);
@@ -335,12 +336,12 @@ nm_setup(void **state __attribute__((unused))) {
 		return (-1);
 	}
 
-	listen_nm = isc_nm_start(test_mctx, workers);
+	isc__netmgr_create(test_mctx, workers, &listen_nm);
 	assert_non_null(listen_nm);
 	isc_nm_settimeouts(listen_nm, T_INIT, T_IDLE, T_KEEPALIVE,
 			   T_ADVERTISED);
 
-	connect_nm = isc_nm_start(test_mctx, workers);
+	isc__netmgr_create(test_mctx, workers, &connect_nm);
 	assert_non_null(connect_nm);
 	isc_nm_settimeouts(connect_nm, T_INIT, T_IDLE, T_KEEPALIVE,
 			   T_ADVERTISED);
@@ -358,10 +359,10 @@ static int
 nm_teardown(void **state __attribute__((unused))) {
 	UNUSED(state);
 
-	isc_nm_destroy(&connect_nm);
+	isc__netmgr_destroy(&connect_nm);
 	assert_null(connect_nm);
 
-	isc_nm_destroy(&listen_nm);
+	isc__netmgr_destroy(&listen_nm);
 	assert_null(listen_nm);
 
 	WAIT_FOR_EQ(active_cconnects, 0);
@@ -677,7 +678,7 @@ mock_udpconnect_uv_udp_open(void **state __attribute__((unused))) {
 	isc_nm_udpconnect(connect_nm, (isc_nmiface_t *)&udp_connect_addr,
 			  (isc_nmiface_t *)&udp_listen_addr, connect_connect_cb,
 			  NULL, T_CONNECT, 0);
-	isc_nm_closedown(connect_nm);
+	isc__netmgr_shutdown(connect_nm);
 
 	RESET_RETURN;
 }
@@ -691,7 +692,7 @@ mock_udpconnect_uv_udp_bind(void **state __attribute__((unused))) {
 	isc_nm_udpconnect(connect_nm, (isc_nmiface_t *)&udp_connect_addr,
 			  (isc_nmiface_t *)&udp_listen_addr, connect_connect_cb,
 			  NULL, T_CONNECT, 0);
-	isc_nm_closedown(connect_nm);
+	isc__netmgr_shutdown(connect_nm);
 
 	RESET_RETURN;
 }
@@ -706,7 +707,7 @@ mock_udpconnect_uv_udp_connect(void **state __attribute__((unused))) {
 	isc_nm_udpconnect(connect_nm, (isc_nmiface_t *)&udp_connect_addr,
 			  (isc_nmiface_t *)&udp_listen_addr, connect_connect_cb,
 			  NULL, T_CONNECT, 0);
-	isc_nm_closedown(connect_nm);
+	isc__netmgr_shutdown(connect_nm);
 
 	RESET_RETURN;
 }
@@ -721,7 +722,7 @@ mock_udpconnect_uv_recv_buffer_size(void **state __attribute__((unused))) {
 	isc_nm_udpconnect(connect_nm, (isc_nmiface_t *)&udp_connect_addr,
 			  (isc_nmiface_t *)&udp_listen_addr, connect_connect_cb,
 			  NULL, T_CONNECT, 0);
-	isc_nm_closedown(connect_nm);
+	isc__netmgr_shutdown(connect_nm);
 
 	RESET_RETURN;
 }
@@ -735,7 +736,7 @@ mock_udpconnect_uv_send_buffer_size(void **state __attribute__((unused))) {
 	isc_nm_udpconnect(connect_nm, (isc_nmiface_t *)&udp_connect_addr,
 			  (isc_nmiface_t *)&udp_listen_addr, connect_connect_cb,
 			  NULL, T_CONNECT, 0);
-	isc_nm_closedown(connect_nm);
+	isc__netmgr_shutdown(connect_nm);
 
 	RESET_RETURN;
 }
@@ -758,7 +759,7 @@ udp_noop(void **state __attribute__((unused))) {
 	isc_nm_udpconnect(connect_nm, (isc_nmiface_t *)&udp_connect_addr,
 			  (isc_nmiface_t *)&udp_listen_addr, connect_connect_cb,
 			  NULL, T_CONNECT, 0);
-	isc_nm_closedown(connect_nm);
+	isc__netmgr_shutdown(connect_nm);
 
 	atomic_assert_int_eq(cconnects, 0);
 	atomic_assert_int_eq(csends, 0);
@@ -787,7 +788,7 @@ udp_noresponse(void **state __attribute__((unused))) {
 	isc_nm_stoplistening(listen_sock);
 	isc_nmsocket_close(&listen_sock);
 	assert_null(listen_sock);
-	isc_nm_closedown(connect_nm);
+	isc__netmgr_shutdown(connect_nm);
 
 	X(cconnects);
 	X(csends);
@@ -860,7 +861,7 @@ udp_timeout_recovery(void **state __attribute__((unused))) {
 	isc_nm_stoplistening(listen_sock);
 	isc_nmsocket_close(&listen_sock);
 	assert_null(listen_sock);
-	isc_nm_closedown(connect_nm);
+	isc__netmgr_shutdown(connect_nm);
 }
 
 static void
@@ -889,7 +890,7 @@ udp_recv_one(void **state __attribute__((unused))) {
 	isc_nm_stoplistening(listen_sock);
 	isc_nmsocket_close(&listen_sock);
 	assert_null(listen_sock);
-	isc_nm_closedown(connect_nm);
+	isc__netmgr_shutdown(connect_nm);
 
 	X(cconnects);
 	X(csends);
@@ -937,7 +938,7 @@ udp_recv_two(void **state __attribute__((unused))) {
 	isc_nm_stoplistening(listen_sock);
 	isc_nmsocket_close(&listen_sock);
 	assert_null(listen_sock);
-	isc_nm_closedown(connect_nm);
+	isc__netmgr_shutdown(connect_nm);
 
 	X(cconnects);
 	X(csends);
@@ -980,7 +981,7 @@ udp_recv_send(void **state __attribute__((unused))) {
 		isc_thread_join(threads[i], NULL);
 	}
 
-	isc_nm_closedown(connect_nm);
+	isc__netmgr_shutdown(connect_nm);
 	isc_nm_stoplistening(listen_sock);
 	isc_nmsocket_close(&listen_sock);
 	assert_null(listen_sock);
@@ -1020,7 +1021,7 @@ udp_recv_half_send(void **state __attribute__((unused))) {
 	WAIT_FOR_GE(ssends, esends / 2);
 	WAIT_FOR_GE(creads, esends / 2);
 
-	isc_nm_closedown(connect_nm);
+	isc__netmgr_shutdown(connect_nm);
 
 	DONE();
 	for (size_t i = 0; i < workers; i++) {
@@ -1073,7 +1074,7 @@ udp_half_recv_send(void **state __attribute__((unused))) {
 	/* Try to send a little while longer */
 	isc_test_nap((esends / 2) * 10000);
 
-	isc_nm_closedown(connect_nm);
+	isc__netmgr_shutdown(connect_nm);
 
 	DONE();
 	for (size_t i = 0; i < workers; i++) {
@@ -1115,7 +1116,7 @@ udp_half_recv_half_send(void **state __attribute__((unused))) {
 	WAIT_FOR_GE(ssends, esends / 2);
 	WAIT_FOR_GE(creads, esends / 2);
 
-	isc_nm_closedown(connect_nm);
+	isc__netmgr_shutdown(connect_nm);
 	isc_nm_stoplistening(listen_sock);
 	isc_nmsocket_close(&listen_sock);
 	assert_null(listen_sock);
@@ -1218,7 +1219,7 @@ stream_noop(void **state __attribute__((unused))) {
 	connect_readcb = NULL;
 	isc_refcount_increment0(&active_cconnects);
 	stream_connect(connect_connect_cb, NULL, T_CONNECT, 0);
-	isc_nm_closedown(connect_nm);
+	isc__netmgr_shutdown(connect_nm);
 
 	atomic_assert_int_eq(cconnects, 0);
 	atomic_assert_int_eq(csends, 0);
@@ -1244,7 +1245,7 @@ stream_noresponse(void **state __attribute__((unused))) {
 	isc_nm_stoplistening(listen_sock);
 	isc_nmsocket_close(&listen_sock);
 	assert_null(listen_sock);
-	isc_nm_closedown(connect_nm);
+	isc__netmgr_shutdown(connect_nm);
 
 	X(cconnects);
 	X(csends);
@@ -1294,7 +1295,7 @@ stream_timeout_recovery(void **state __attribute__((unused))) {
 	isc_nm_stoplistening(listen_sock);
 	isc_nmsocket_close(&listen_sock);
 	assert_null(listen_sock);
-	isc_nm_closedown(connect_nm);
+	isc__netmgr_shutdown(connect_nm);
 }
 
 static void
@@ -1322,7 +1323,7 @@ stream_recv_one(void **state __attribute__((unused))) {
 	isc_nm_stoplistening(listen_sock);
 	isc_nmsocket_close(&listen_sock);
 	assert_null(listen_sock);
-	isc_nm_closedown(connect_nm);
+	isc__netmgr_shutdown(connect_nm);
 
 	X(cconnects);
 	X(csends);
@@ -1367,7 +1368,7 @@ stream_recv_two(void **state __attribute__((unused))) {
 	isc_nm_stoplistening(listen_sock);
 	isc_nmsocket_close(&listen_sock);
 	assert_null(listen_sock);
-	isc_nm_closedown(connect_nm);
+	isc__netmgr_shutdown(connect_nm);
 
 	X(cconnects);
 	X(csends);
@@ -1416,7 +1417,7 @@ stream_recv_send(void **state __attribute__((unused))) {
 		isc_thread_join(threads[i], NULL);
 	}
 
-	isc_nm_closedown(connect_nm);
+	isc__netmgr_shutdown(connect_nm);
 	isc_nm_stoplistening(listen_sock);
 	isc_nmsocket_close(&listen_sock);
 	assert_null(listen_sock);
@@ -1462,7 +1463,7 @@ stream_recv_half_send(void **state __attribute__((unused))) {
 	WAIT_FOR_GE(ssends, esends / 2);
 	WAIT_FOR_GE(creads, esends / 2);
 
-	isc_nm_closedown(connect_nm);
+	isc__netmgr_shutdown(connect_nm);
 
 	DONE();
 	for (size_t i = 0; i < workers; i++) {
@@ -1521,7 +1522,7 @@ stream_half_recv_send(void **state __attribute__((unused))) {
 	/* Try to send a little while longer */
 	isc_test_nap((esends / 2) * 10000);
 
-	isc_nm_closedown(connect_nm);
+	isc__netmgr_shutdown(connect_nm);
 
 	DONE();
 	for (size_t i = 0; i < workers; i++) {
@@ -1569,7 +1570,7 @@ stream_half_recv_half_send(void **state __attribute__((unused))) {
 	WAIT_FOR_GE(ssends, esends / 2);
 	WAIT_FOR_GE(creads, esends / 2);
 
-	isc_nm_closedown(connect_nm);
+	isc__netmgr_shutdown(connect_nm);
 	isc_nm_stoplistening(listen_sock);
 	isc_nmsocket_close(&listen_sock);
 	assert_null(listen_sock);
@@ -1767,7 +1768,7 @@ tcpdns_noop(void **state __attribute__((unused))) {
 	isc_nm_tcpdnsconnect(connect_nm, (isc_nmiface_t *)&tcp_connect_addr,
 			     (isc_nmiface_t *)&tcp_listen_addr,
 			     connect_connect_cb, NULL, T_CONNECT, 0);
-	isc_nm_closedown(connect_nm);
+	isc__netmgr_shutdown(connect_nm);
 
 	atomic_assert_int_eq(cconnects, 0);
 	atomic_assert_int_eq(csends, 0);
@@ -1801,7 +1802,7 @@ tcpdns_noresponse(void **state __attribute__((unused))) {
 	isc_nm_stoplistening(listen_sock);
 	isc_nmsocket_close(&listen_sock);
 	assert_null(listen_sock);
-	isc_nm_closedown(connect_nm);
+	isc__netmgr_shutdown(connect_nm);
 
 	X(cconnects);
 	X(csends);
@@ -1856,7 +1857,7 @@ tcpdns_timeout_recovery(void **state __attribute__((unused))) {
 	isc_nm_stoplistening(listen_sock);
 	isc_nmsocket_close(&listen_sock);
 	assert_null(listen_sock);
-	isc_nm_closedown(connect_nm);
+	isc__netmgr_shutdown(connect_nm);
 }
 
 static void
@@ -1886,7 +1887,7 @@ tcpdns_recv_one(void **state __attribute__((unused))) {
 	isc_nm_stoplistening(listen_sock);
 	isc_nmsocket_close(&listen_sock);
 	assert_null(listen_sock);
-	isc_nm_closedown(connect_nm);
+	isc__netmgr_shutdown(connect_nm);
 
 	X(cconnects);
 	X(csends);
@@ -1936,7 +1937,7 @@ tcpdns_recv_two(void **state __attribute__((unused))) {
 	isc_nm_stoplistening(listen_sock);
 	isc_nmsocket_close(&listen_sock);
 	assert_null(listen_sock);
-	isc_nm_closedown(connect_nm);
+	isc__netmgr_shutdown(connect_nm);
 
 	X(cconnects);
 	X(csends);
@@ -1980,7 +1981,7 @@ tcpdns_recv_send(void **state __attribute__((unused))) {
 		isc_thread_join(threads[i], NULL);
 	}
 
-	isc_nm_closedown(connect_nm);
+	isc__netmgr_shutdown(connect_nm);
 	isc_nm_stoplistening(listen_sock);
 	isc_nmsocket_close(&listen_sock);
 	assert_null(listen_sock);
@@ -2021,7 +2022,7 @@ tcpdns_recv_half_send(void **state __attribute__((unused))) {
 	WAIT_FOR_GE(ssends, esends / 2);
 	WAIT_FOR_GE(creads, esends / 2);
 
-	isc_nm_closedown(connect_nm);
+	isc__netmgr_shutdown(connect_nm);
 
 	DONE();
 	for (size_t i = 0; i < workers; i++) {
@@ -2075,7 +2076,7 @@ tcpdns_half_recv_send(void **state __attribute__((unused))) {
 	/* Try to send a little while longer */
 	isc_test_nap((esends / 2) * 10000);
 
-	isc_nm_closedown(connect_nm);
+	isc__netmgr_shutdown(connect_nm);
 
 	DONE();
 	for (size_t i = 0; i < workers; i++) {
@@ -2118,7 +2119,7 @@ tcpdns_half_recv_half_send(void **state __attribute__((unused))) {
 	WAIT_FOR_GE(ssends, esends / 2);
 	WAIT_FOR_GE(creads, esends / 2);
 
-	isc_nm_closedown(connect_nm);
+	isc__netmgr_shutdown(connect_nm);
 	isc_nm_stoplistening(listen_sock);
 	isc_nmsocket_close(&listen_sock);
 	assert_null(listen_sock);
@@ -2275,6 +2276,7 @@ static void
 tls_half_recv_send_quota(void **state) {
 	SKIP_IN_CI;
 	stream_use_TLS = true;
+	atomic_store(&check_listener_quota, true);
 	stream_half_recv_send(state);
 }
 
@@ -2282,6 +2284,7 @@ static void
 tls_half_recv_half_send_quota(void **state) {
 	SKIP_IN_CI;
 	stream_use_TLS = true;
+	atomic_store(&check_listener_quota, true);
 	stream_half_recv_half_send(state);
 }
 
@@ -2353,7 +2356,7 @@ tlsdns_noop(void **state __attribute__((unused))) {
 			     connect_connect_cb, NULL, T_CONNECT, 0,
 			     tcp_connect_tlsctx);
 
-	isc_nm_closedown(connect_nm);
+	isc__netmgr_shutdown(connect_nm);
 
 	atomic_assert_int_eq(cconnects, 0);
 	atomic_assert_int_eq(csends, 0);
@@ -2389,7 +2392,7 @@ tlsdns_noresponse(void **state __attribute__((unused))) {
 	isc_nm_stoplistening(listen_sock);
 	isc_nmsocket_close(&listen_sock);
 	assert_null(listen_sock);
-	isc_nm_closedown(connect_nm);
+	isc__netmgr_shutdown(connect_nm);
 
 	X(cconnects);
 	X(csends);
@@ -2450,7 +2453,7 @@ tlsdns_timeout_recovery(void **state __attribute__((unused))) {
 	isc_nm_stoplistening(listen_sock);
 	isc_nmsocket_close(&listen_sock);
 	assert_null(listen_sock);
-	isc_nm_closedown(connect_nm);
+	isc__netmgr_shutdown(connect_nm);
 }
 
 static void
@@ -2482,7 +2485,7 @@ tlsdns_recv_one(void **state __attribute__((unused))) {
 	isc_nm_stoplistening(listen_sock);
 	isc_nmsocket_close(&listen_sock);
 	assert_null(listen_sock);
-	isc_nm_closedown(connect_nm);
+	isc__netmgr_shutdown(connect_nm);
 
 	X(cconnects);
 	X(csends);
@@ -2535,7 +2538,7 @@ tlsdns_recv_two(void **state __attribute__((unused))) {
 	isc_nm_stoplistening(listen_sock);
 	isc_nmsocket_close(&listen_sock);
 	assert_null(listen_sock);
-	isc_nm_closedown(connect_nm);
+	isc__netmgr_shutdown(connect_nm);
 
 	X(cconnects);
 	X(csends);
@@ -2580,7 +2583,7 @@ tlsdns_recv_send(void **state __attribute__((unused))) {
 		isc_thread_join(threads[i], NULL);
 	}
 
-	isc_nm_closedown(connect_nm);
+	isc__netmgr_shutdown(connect_nm);
 	isc_nm_stoplistening(listen_sock);
 	isc_nmsocket_close(&listen_sock);
 	assert_null(listen_sock);
@@ -2622,7 +2625,7 @@ tlsdns_recv_half_send(void **state __attribute__((unused))) {
 	WAIT_FOR_GE(ssends, esends / 2);
 	WAIT_FOR_GE(creads, esends / 2);
 
-	isc_nm_closedown(connect_nm);
+	isc__netmgr_shutdown(connect_nm);
 
 	DONE();
 	for (size_t i = 0; i < workers; i++) {
@@ -2677,7 +2680,7 @@ tlsdns_half_recv_send(void **state __attribute__((unused))) {
 	/* Try to send a little while longer */
 	isc_test_nap((esends / 2) * 10000);
 
-	isc_nm_closedown(connect_nm);
+	isc__netmgr_shutdown(connect_nm);
 
 	DONE();
 	for (size_t i = 0; i < workers; i++) {
@@ -2721,7 +2724,7 @@ tlsdns_half_recv_half_send(void **state __attribute__((unused))) {
 	WAIT_FOR_GE(ssends, esends / 2);
 	WAIT_FOR_GE(creads, esends / 2);
 
-	isc_nm_closedown(connect_nm);
+	isc__netmgr_shutdown(connect_nm);
 	isc_nm_stoplistening(listen_sock);
 	isc_nmsocket_close(&listen_sock);
 	assert_null(listen_sock);

lib/isc/tests/socket_test.c

@@ -30,7 +30,7 @@
 #include <isc/socket.h>
 #include <isc/task.h>
 
-#include "../unix/socket_p.h"
+#include "../socket_p.h"
 #include "isctest.h"
 
 static bool recv_dscp;

lib/isc/tests/task_test.c

@@ -29,6 +29,7 @@
 #include <isc/cmocka.h>
 #include <isc/commandline.h>
 #include <isc/condition.h>
+#include <isc/managers.h>
 #include <isc/mem.h>
 #include <isc/platform.h>
 #include <isc/print.h>
@@ -784,7 +785,6 @@ maxtask_cb(isc_task_t *task, isc_event_t *event) {
 static void
 manytasks(void **state) {
 	isc_mem_t *mctx = NULL;
-	isc_result_t result;
 	isc_event_t *event = NULL;
 	uintptr_t ntasks = 10000;
 
@@ -801,9 +801,7 @@ manytasks(void **state) {
 	isc_mem_debugging = ISC_MEM_DEBUGRECORD;
 	isc_mem_create(&mctx);
 
-	netmgr = isc_nm_start(mctx, 4);
-	result = isc_taskmgr_create(mctx, 0, netmgr, &taskmgr);
-	assert_int_equal(result, ISC_R_SUCCESS);
+	isc_managers_create(mctx, 4, 0, 0, &netmgr, &taskmgr, NULL, NULL);
 
 	atomic_init(&done, false);
 
@@ -818,8 +816,8 @@ manytasks(void **state) {
 	}
 	UNLOCK(&lock);
 
-	isc_taskmgr_destroy(&taskmgr);
-	isc_nm_destroy(&netmgr);
+	isc_managers_destroy(&netmgr, &taskmgr, NULL, NULL);
+
 	isc_mem_destroy(&mctx);
 	isc_condition_destroy(&cv);
 	isc_mutex_destroy(&lock);
@@ -899,7 +897,7 @@ sd_event2(isc_task_t *task, isc_event_t *event) {
 }
 
 static void
-shutdown(void **state) {
+task_shutdown(void **state) {
 	isc_result_t result;
 	isc_eventtype_t event_type;
 	isc_event_t *event = NULL;
@@ -1545,7 +1543,8 @@ main(int argc, char **argv) {
 		cmocka_unit_test_setup_teardown(purgeevent_notpurge, _setup,
 						_teardown),
 		cmocka_unit_test_setup_teardown(purgerange, _setup, _teardown),
-		cmocka_unit_test_setup_teardown(shutdown, _setup4, _teardown),
+		cmocka_unit_test_setup_teardown(task_shutdown, _setup4,
+						_teardown),
 		cmocka_unit_test_setup_teardown(task_exclusive, _setup4,
 						_teardown),
 	};

lib/isc/timer.c

@@ -29,9 +29,7 @@
 #include <isc/timer.h>
 #include <isc/util.h>
 
-#ifdef OPENSSL_LEAKS
-#include <openssl/err.h>
-#endif /* ifdef OPENSSL_LEAKS */
+#include "timer_p.h"
 
 #ifdef ISC_TIMER_TRACE
 #define XTRACE(s)      fprintf(stderr, "%s\n", (s))
@@ -630,10 +628,6 @@ static isc_threadresult_t
 	}
 	UNLOCK(&manager->lock);
 
-#ifdef OPENSSL_LEAKS
-	ERR_remove_state(0);
-#endif /* ifdef OPENSSL_LEAKS */
-
 	return ((isc_threadresult_t)0);
 }
 
@@ -663,7 +657,7 @@ set_index(void *what, unsigned int index) {
 }
 
 isc_result_t
-isc_timermgr_create(isc_mem_t *mctx, isc_timermgr_t **managerp) {
+isc__timermgr_create(isc_mem_t *mctx, isc_timermgr_t **managerp) {
 	isc_timermgr_t *manager;
 	isc_result_t result;
 
@@ -707,7 +701,7 @@ isc_timermgr_poke(isc_timermgr_t *manager) {
 }
 
 void
-isc_timermgr_destroy(isc_timermgr_t **managerp) {
+isc__timermgr_destroy(isc_timermgr_t **managerp) {
 	isc_timermgr_t *manager;
 
 	/*

lib/isc/timer_p.h

@@ -0,0 +1,65 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, you can obtain one at https://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+#pragma once
+
+#include <isc/mem.h>
+#include <isc/result.h>
+#include <isc/timer.h>
+
+isc_result_t
+isc__timermgr_create(isc_mem_t *mctx, isc_timermgr_t **managerp);
+/*%<
+ * Create a timer manager.
+ *
+ * Notes:
+ *
+ *\li	All memory will be allocated in memory context 'mctx'.
+ *
+ * Requires:
+ *
+ *\li	'mctx' is a valid memory context.
+ *
+ *\li	'managerp' points to a NULL isc_timermgr_t.
+ *
+ * Ensures:
+ *
+ *\li	'*managerp' is a valid isc_timermgr_t.
+ *
+ * Returns:
+ *
+ *\li	Success
+ *\li	No memory
+ *\li	Unexpected error
+ */
+
+void
+isc__timermgr_destroy(isc_timermgr_t **managerp);
+/*%<
+ * Destroy a timer manager.
+ *
+ * Notes:
+ *
+ *\li	This routine blocks until there are no timers left in the manager,
+ *	so if the caller holds any timer references using the manager, it
+ *	must detach them before calling isc_timermgr_destroy() or it will
+ *	block forever.
+ *
+ * Requires:
+ *
+ *\li	'*managerp' is a valid isc_timermgr_t.
+ *
+ * Ensures:
+ *
+ *\li	*managerp == NULL
+ *
+ *\li	All resources used by the manager have been freed.
+ */

lib/isc/unix/socket.c

@@ -74,6 +74,7 @@
 #include <netinet/tcp.h>
 
 #include "errno2result.h"
+#include "socket_p.h"
 
 #ifdef ENABLE_TCP_FASTOPEN
 #include <netinet/tcp.h>
@@ -3752,12 +3753,7 @@ cleanup_thread(isc_mem_t *mctx, isc__socketthread_t *thread) {
 }
 
 isc_result_t
-isc_socketmgr_create(isc_mem_t *mctx, isc_socketmgr_t **managerp) {
-	return (isc_socketmgr_create2(mctx, managerp, 0, 1));
-}
-
-isc_result_t
-isc_socketmgr_create2(isc_mem_t *mctx, isc_socketmgr_t **managerp,
+isc__socketmgr_create(isc_mem_t *mctx, isc_socketmgr_t **managerp,
 		      unsigned int maxsocks, int nthreads) {
 	int i;
 	isc_socketmgr_t *manager;
@@ -3828,7 +3824,7 @@ isc_socketmgr_setstats(isc_socketmgr_t *manager, isc_stats_t *stats) {
 }
 
 void
-isc_socketmgr_destroy(isc_socketmgr_t **managerp) {
+isc__socketmgr_destroy(isc_socketmgr_t **managerp) {
 	isc_socketmgr_t *manager;
 
 	/*

lib/isc/unix/socket_p.h

@@ -1,25 +0,0 @@
-/*
- * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
- *
- * This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, you can obtain one at https://mozilla.org/MPL/2.0/.
- *
- * See the COPYRIGHT file distributed with this work for additional
- * information regarding copyright ownership.
- */
-
-#ifndef ISC_SOCKET_P_H
-#define ISC_SOCKET_P_H
-
-/*! \file */
-
-#include <sys/time.h>
-
-typedef struct isc_socketwait isc_socketwait_t;
-int
-isc__socketmgr_waitevents(isc_socketmgr_t *, struct timeval *,
-			  isc_socketwait_t **);
-isc_result_t
-isc__socketmgr_dispatch(isc_socketmgr_t *, isc_socketwait_t *);
-#endif /* ISC_SOCKET_P_H */

lib/isc/win32/libisc.def.in

@@ -96,14 +96,12 @@ isc_socket_send
 isc_socket_sendto
 isc_socket_sendto2
 isc_socket_setname
-isc_socketmgr_create
-isc_socketmgr_create2
-isc_socketmgr_destroy
 isc_socketmgr_getmaxsockets
 isc_socketmgr_setreserved
 isc_socketmgr_setstats
 isc_task_getname
 isc_task_gettag
+isc_task_ready
 isc_task_run
 isc_task_unsendrange
 isc_taskmgr_attach
@@ -352,6 +350,8 @@ isc_logconfig_create
 isc_logconfig_destroy
 isc_logconfig_use
 isc_logfile_roll
+isc_managers_create
+isc_managers_destroy
 isc_md_new
 isc_md_init
 isc_md_reset
@@ -452,8 +452,6 @@ isc_nmhandle_peeraddr
 isc_nmhandle_setdata
 isc_nmhandle_settimeout
 isc_nm_cancelread
-isc_nm_closedown
-isc_nm_destroy
 isc_nm_detach
 isc_nm_http_endpoint
 isc_nm_httpconnect
@@ -469,7 +467,6 @@ isc_nm_read
 isc_nm_resumeread
 isc_nm_send
 isc_nm_setstats
-isc_nm_start
 isc_nm_stoplistening
 isc_nm_tcpconnect
 isc_nm_tcpdnsconnect
@@ -653,8 +650,6 @@ isc_task_setprivilege
 isc_task_shutdown
 isc_task_unpause
 isc_task_unsend
-isc_taskmgr_create
-isc_taskmgr_destroy
 isc_taskmgr_excltask
 @IF NOTYET
 isc_taskmgr_renderjson
@@ -706,8 +701,6 @@ isc_timer_detach
 isc_timer_gettype
 isc_timer_reset
 isc_timer_touch
-isc_timermgr_create
-isc_timermgr_destroy
 isc_timermgr_poke
 isc_tls_get_http2_alpn
 isc_tls_create

lib/isc/win32/libisc.vcxproj.filters.in

@@ -140,6 +140,9 @@
     <ClInclude Include="..\include\isc\magic.h">
       <Filter>Library Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="..\include\isc\managers.h">
+      <Filter>Library Header Files</Filter>
+    </ClInclude>
     <ClInclude Include="..\include\isc\mem.h">
       <Filter>Library Header Files</Filter>
     </ClInclude>
@@ -530,6 +533,9 @@
     <ClCompile Include="..\log.c">
       <Filter>Library Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\managers.c">
+      <Filter>Library Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="..\mem.c">
       <Filter>Library Source Files</Filter>
     </ClCompile>

lib/isc/win32/libisc.vcxproj.in

@@ -302,6 +302,7 @@ copy InstallFiles ..\Build\Release\
     <ClInclude Include="..\include\isc\list.h" />
     <ClInclude Include="..\include\isc\log.h" />
     <ClInclude Include="..\include\isc\magic.h" />
+    <ClInclude Include="..\include\isc\managers.h" />
     <ClInclude Include="..\include\isc\md.h" />
     <ClInclude Include="..\include\isc\mem.h" />
     <ClInclude Include="..\include\isc\meminfo.h" />
@@ -412,6 +413,7 @@ copy InstallFiles ..\Build\Release\
     <ClCompile Include="..\lex.c" />
     <ClCompile Include="..\lib.c" />
     <ClCompile Include="..\log.c" />
+    <ClCompile Include="..\managers.c" />
     <ClCompile Include="..\md.c" />
     <ClCompile Include="..\mem.c" />
     <ClCompile Include="..\mutexblock.c" />

lib/isc/win32/socket.c

@@ -82,6 +82,7 @@
 #endif /* HAVE_LIBXML2 */
 
 #include "errno2result.h"
+#include "socket_p.h"
 
 /*
  * Set by the -T dscp option on the command line. If set to a value
@@ -2548,12 +2549,7 @@ SocketIoThread(LPVOID ThreadContext) {
  * Create a new socket manager.
  */
 isc_result_t
-isc_socketmgr_create(isc_mem_t *mctx, isc_socketmgr_t **managerp) {
-	return (isc_socketmgr_create2(mctx, managerp, 0, 1));
-}
-
-isc_result_t
-isc_socketmgr_create2(isc_mem_t *mctx, isc_socketmgr_t **managerp,
+isc__socketmgr_create(isc_mem_t *mctx, isc_socketmgr_t **managerp,
 		      unsigned int maxsocks, int nthreads) {
 	isc_socketmgr_t *manager;
 
@@ -2611,7 +2607,7 @@ isc_socketmgr_setstats(isc_socketmgr_t *manager, isc_stats_t *stats) {
 }
 
 void
-isc_socketmgr_destroy(isc_socketmgr_t **managerp) {
+isc__socketmgr_destroy(isc_socketmgr_t **managerp) {
 	isc_socketmgr_t *manager;
 
 	/*

lib/ns/tests/nstest.c

@@ -22,6 +22,7 @@
 #include <isc/buffer.h>
 #include <isc/file.h>
 #include <isc/hash.h>
+#include <isc/managers.h>
 #include <isc/mem.h>
 #include <isc/netmgr.h>
 #include <isc/os.h>
@@ -56,7 +57,6 @@ isc_taskmgr_t *taskmgr = NULL;
 isc_task_t *maintask = NULL;
 isc_timermgr_t *timermgr = NULL;
 isc_socketmgr_t *socketmgr = NULL;
-isc_nm_t *nm = NULL;
 dns_zonemgr_t *zonemgr = NULL;
 dns_dispatchmgr_t *dispatchmgr = NULL;
 ns_clientmgr_t *clientmgr = NULL;
@@ -198,28 +198,12 @@ cleanup_managers(void) {
 	if (interfacemgr != NULL) {
 		ns_interfacemgr_detach(&interfacemgr);
 	}
-	if (socketmgr != NULL) {
-		isc_socketmgr_destroy(&socketmgr);
-	}
-	ns_test_nap(500000);
-	if (nm != NULL) {
-		/*
-		 * Force something in the workqueue as a workaround
-		 * for libuv bug - not sending uv_close callback.
-		 */
-		isc_nm_pause(nm);
-		isc_nm_resume(nm);
-		isc_nm_detach(&nm);
-	}
-	if (taskmgr != NULL) {
-		isc_taskmgr_destroy(&taskmgr);
-	}
-	if (netmgr != NULL) {
-		isc_nm_destroy(&netmgr);
-	}
-	if (timermgr != NULL) {
-		isc_timermgr_destroy(&timermgr);
-	}
+
+	isc_managers_destroy(netmgr == NULL ? NULL : &netmgr,
+			     taskmgr == NULL ? NULL : &taskmgr,
+			     timermgr == NULL ? NULL : &timermgr,
+			     socketmgr == NULL ? NULL : &socketmgr);
+
 	if (app_running) {
 		isc_app_finish();
 	}
@@ -241,24 +225,18 @@ create_managers(void) {
 	isc_event_t *event = NULL;
 	ncpus = isc_os_ncpus();
 
-	netmgr = isc_nm_start(mctx, ncpus);
-	CHECK(isc_taskmgr_create(mctx, 0, netmgr, &taskmgr));
+	isc_managers_create(mctx, ncpus, 0, 0, &netmgr, &taskmgr, &timermgr,
+			    &socketmgr);
 	CHECK(isc_task_create(taskmgr, 0, &maintask));
 	isc_taskmgr_setexcltask(taskmgr, maintask);
 	CHECK(isc_task_onshutdown(maintask, shutdown_managers, NULL));
 
-	CHECK(isc_timermgr_create(mctx, &timermgr));
-
-	CHECK(isc_socketmgr_create(mctx, &socketmgr));
-
-	nm = isc_nm_start(mctx, ncpus);
-
 	CHECK(ns_server_create(mctx, matchview, &sctx));
 
 	CHECK(dns_dispatchmgr_create(mctx, &dispatchmgr));
 
 	CHECK(ns_interfacemgr_create(mctx, sctx, taskmgr, timermgr, socketmgr,
-				     nm, dispatchmgr, maintask, ncpus, NULL,
+				     netmgr, dispatchmgr, maintask, ncpus, NULL,
 				     ncpus, &interfacemgr));
 
 	CHECK(ns_listenlist_default(mctx, port, -1, true, &listenon));

util/copyrights

@@ -1887,6 +1887,7 @@
 ./lib/isc/include/isc/list.h			C	1997,1998,1999,2000,2001,2002,2004,2006,2007,2011,2012,2013,2016,2018,2019,2020,2021
 ./lib/isc/include/isc/log.h			C	1999,2000,2001,2002,2004,2005,2006,2007,2009,2014,2016,2017,2018,2019,2020,2021
 ./lib/isc/include/isc/magic.h			C	1999,2000,2001,2004,2005,2006,2007,2016,2017,2018,2019,2020,2021
+./lib/isc/include/isc/managers.h		C	2021
 ./lib/isc/include/isc/md.h			C	2018,2019,2020,2021
 ./lib/isc/include/isc/mem.h			C	1997,1998,1999,2000,2001,2004,2005,2006,2007,2008,2009,2010,2011,2012,2013,2015,2016,2017,2018,2019,2020,2021
 ./lib/isc/include/isc/meminfo.h			C	2015,2016,2018,2019,2020,2021
@@ -1944,6 +1945,7 @@
 ./lib/isc/lib.c					C	1999,2000,2001,2004,2005,2007,2009,2013,2014,2015,2016,2018,2019,2020,2021
 ./lib/isc/lib_p.h				C	2021
 ./lib/isc/log.c					C	1999,2000,2001,2002,2003,2004,2005,2006,2007,2009,2011,2012,2013,2014,2016,2017,2018,2019,2020,2021
+./lib/isc/managers.c				C	2021
 ./lib/isc/md.c					C	2018,2019,2020,2021
 ./lib/isc/mem.c					C	1997,1998,1999,2000,2001,2002,2003,2004,2005,2006,2007,2008,2009,2010,2012,2013,2014,2015,2016,2017,2018,2019,2020,2021
 ./lib/isc/mem_p.h				C	2018,2019,2020,2021
@@ -1960,6 +1962,7 @@
 ./lib/isc/netmgr/uv-compat.c			C	2020,2021
 ./lib/isc/netmgr/uv-compat.h			C	2019,2020,2021
 ./lib/isc/netmgr/uverr2result.c			C	2019,2020,2021
+./lib/isc/netmgr_p.h				C	2021
 ./lib/isc/netscope.c				C	2002,2004,2005,2006,2007,2016,2018,2019,2020,2021
 ./lib/isc/nonce.c				C	2018,2019,2020,2021
 ./lib/isc/openssl_shim.c			C	2018,2019,2020,2021
@@ -1989,10 +1992,12 @@
 ./lib/isc/serial.c				C	1999,2000,2001,2004,2005,2007,2016,2018,2019,2020,2021
 ./lib/isc/siphash.c				C	2019,2020,2021
 ./lib/isc/sockaddr.c				C	1999,2000,2001,2002,2003,2004,2005,2006,2007,2010,2011,2012,2014,2015,2016,2017,2018,2019,2020,2021
+./lib/isc/socket_p.h				C	2021
 ./lib/isc/stats.c				C	2009,2012,2013,2014,2015,2016,2017,2018,2019,2020,2021
 ./lib/isc/string.c				C	1999,2000,2001,2003,2004,2005,2006,2007,2011,2012,2014,2015,2016,2018,2019,2020,2021
 ./lib/isc/symtab.c				C	1996,1997,1998,1999,2000,2001,2004,2005,2007,2011,2012,2013,2016,2018,2019,2020,2021
 ./lib/isc/task.c				C	1998,1999,2000,2001,2002,2003,2004,2005,2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020,2021
+./lib/isc/task_p.h				C	2021
 ./lib/isc/taskpool.c				C	1999,2000,2001,2004,2005,2007,2011,2012,2013,2016,2018,2019,2020,2021
 ./lib/isc/tests/aes_test.c			C	2014,2016,2018,2019,2020,2021
 ./lib/isc/tests/buffer_test.c			C	2014,2015,2016,2017,2018,2019,2020,2021
@@ -2031,6 +2036,7 @@
 ./lib/isc/tests/timer_test.c			C	2018,2019,2020,2021
 ./lib/isc/tests/uv_wrap.h			C	2020,2021
 ./lib/isc/timer.c				C	1998,1999,2000,2001,2002,2004,2005,2007,2008,2009,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020,2021
+./lib/isc/timer_p.h				C	2021
 ./lib/isc/tls.c					C	2021
 ./lib/isc/tls_p.h				C	2021
 ./lib/isc/tm.c					C	2014,2016,2018,2019,2020,2021
@@ -2059,7 +2065,6 @@
 ./lib/isc/unix/pk11_api.c			C	2014,2016,2018,2019,2020,2021
 ./lib/isc/unix/resource.c			C	2000,2001,2004,2007,2008,2009,2016,2018,2019,2020,2021
 ./lib/isc/unix/socket.c				C	1998,1999,2000,2001,2002,2003,2004,2005,2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020,2021
-./lib/isc/unix/socket_p.h			C	2000,2001,2004,2005,2007,2008,2009,2016,2018,2019,2020,2021
 ./lib/isc/unix/stdio.c				C	2000,2001,2004,2007,2011,2012,2013,2014,2016,2018,2019,2020,2021
 ./lib/isc/unix/stdtime.c			C	1999,2000,2001,2004,2005,2007,2016,2018,2019,2020,2021
 ./lib/isc/unix/syslog.c				C	2001,2004,2005,2007,2016,2018,2019,2020,2021