system tests: if USE_RR environmental variable is set then the tests are run under rr in chaos mode

netmgr: make tcp listening multithreaded.

See merge request isc-projects/bind9!2659

CHANGES

@@ -1,3 +1,5 @@
+5335.	[func]		Make TCP listening code multithreaded. [GL !2659]
+
 5334.	[doc]		Update documentation with dnssec-policy clarifications.
 			Also change some defaults.
 

bin/tests/system/addzone/clean.sh

@@ -9,32 +9,34 @@
 # See the COPYRIGHT file distributed with this work for additional
 # information regarding copyright ownership.
 
-rm -f dig.out.*
-rm -f rndc.out*
-rm -f showzone.out*
-rm -f zonestatus.out*
-rm -f */named.conf
-rm -f */named.memstats
-rm -f ns1/*.nzf ns1/*.nzf~
-rm -f ns1/*.nzd ns1/*.nzd-lock
-rm -f ns2/*.nzf ns2/*.nzf~
-rm -f ns2/*.nzd ns2/*.nzd-lock
-rm -f ns3/*.nzf ns3/*.nzf~
-rm -f ns3/*.nzd ns3/*.nzd-lock
-rm -f ns2/core*
-rm -f ns2/inline.db.jbk
-rm -f ns2/inline.db.signed
-rm -f ns2/inlineslave.bk*
-rm -rf ns2/new-zones
-rm -f ns*/named.lock
-rm -f ns*/named.run
-rm -f ns2/nzf-*
-rm -f ns3/named.conf
-rm -f ns3/*.nzf ns3/*.nzf~
-rm -f ns3/*.nzd ns3/*.nzd-lock
-rm -f ns3/inlineslave.db
-rm -f ns1/redirect.db
-rm -f ns2/redirect.db
-rm -f ns2/redirect.bk
-rm -f ns3/redirect.db
-rm -f ns*/managed-keys.bind* ns*/*.mkeys*
+rm -f ./dig.out.*
+rm -f ./rndc.out*
+rm -f ./showzone.out*
+rm -f ./zonestatus.out*
+rm -f ./*/named.conf
+rm -f ./*/named.memstats
+rm -f ./ns1/*.nzf ./ns1/*.nzf~
+rm -f ./ns1/*.nzd ./ns1/*.nzd-lock
+rm -f ./ns2/*.nzf ./ns2/*.nzf~
+rm -f ./ns2/*.nzd ./ns2/*.nzd-lock
+rm -f ./ns3/*.nzf ./ns3/*.nzf~
+rm -f ./ns3/*.nzd ./ns3/*.nzd-lock
+rm -f ./ns2/core*
+rm -f ./ns2/inline.db.jbk
+rm -f ./ns2/inline.db.signed
+rm -f ./ns2/inlineslave.bk*
+rm -rf ./ns2/new-zones
+rm -f ./ns*/named.lock
+rm -f ./ns*/named.run ./ns*/named.run.prev
+rm -f ./ns2/nzf-*
+rm -f ./ns3/named.conf
+rm -f ./ns3/*.nzf ./ns3/*.nzf~
+rm -f ./ns3/*.nzd ns3/*.nzd-lock
+rm -f ./ns3/inlineslave.db
+rm -f ./ns1/redirect.db
+rm -f ./ns2/redirect.db
+rm -f ./ns2/redirect.bk
+rm -f ./ns3/redirect.db
+rm -f ./ns*/managed-keys.bind* ns*/*.mkeys*
+rm -f ./nzd2nzf.out.*
+rm -f ./wait_for_message.*

bin/tests/system/addzone/tests.sh

@@ -15,18 +15,15 @@ SYSTEMTESTTOP=..
 DIGOPTS="+tcp +nosea +nostat +nocmd +norec +noques +noauth +noadd +nostats +dnssec -p ${PORT}"
 RNDCCMD="$RNDC -c $SYSTEMTESTTOP/common/rndc.conf -p ${CONTROLPORT} -s"
 
+check_zonestatus() (
+    $RNDCCMD "10.53.0.$1" zonestatus -redirect > "zonestatus.out.ns$1.$n" &&
+    grep "type: redirect" "zonestatus.out.ns$1.$n" > /dev/null &&
+    grep "serial: 1" "zonestatus.out.ns$1.$n" > /dev/null
+)
+
 status=0
 n=0
 
-_wait_for_message() (
-	nextpartpeek "$1" > wait_for_message.$n
-	grep -F "$2" wait_for_message.$n >/dev/null
-)
-
-wait_for_message() (
-	retry_quiet 20 _wait_for_message "$@"
-)
-
 echo_i "checking normally loaded zone ($n)"
 ret=0
 $DIG $DIGOPTS @10.53.0.2 a.normal.example a > dig.out.ns2.$n || ret=1
@@ -58,11 +55,11 @@ fi
 echo_i "adding new zone ($n)"
 ret=0
 $RNDCCMD 10.53.0.2 addzone 'added.example { type master; file "added.db"; };' 2>&1 | sed 's/^/I:ns2 /'
-_check_adding_new_zone () {
-	$DIG $DIGOPTS @10.53.0.2 a.added.example a > dig.out.ns2.$n || return 1
-	grep 'status: NOERROR' dig.out.ns2.$n > /dev/null || return 1
-	grep '^a.added.example' dig.out.ns2.$n > /dev/null || return 1
-}
+_check_adding_new_zone () (
+	$DIG $DIGOPTS @10.53.0.2 a.added.example a > dig.out.ns2.$n &&
+	grep 'status: NOERROR' dig.out.ns2.$n > /dev/null &&
+	grep '^a.added.example' dig.out.ns2.$n > /dev/null
+)
 retry_quiet 10 _check_adding_new_zone || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
@@ -72,7 +69,7 @@ nextpart ns2/named.run >/dev/null
 echo_i "checking addzone errors are logged correctly"
 ret=0
 $RNDCCMD 10.53.0.2 addzone bad.example '{ type mister; };' 2>&1 | grep 'unexpected token' > /dev/null 2>&1 || ret=1
-wait_for_message ns2/named.run "addzone: 'mister' unexpected" || ret=1
+wait_for_log_peek 20 "addzone: 'mister' unexpected" ns2/named.run || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
@@ -81,7 +78,7 @@ nextpart ns2/named.run >/dev/null
 echo_i "checking modzone errors are logged correctly"
 ret=0
 $RNDCCMD 10.53.0.2 modzone added.example '{ type mister; };' 2>&1 | grep 'unexpected token' > /dev/null 2>&1 || ret=1
-wait_for_message ns2/named.run "modzone: 'mister' unexpected" || ret=1
+wait_for_log_peek 20 "modzone: 'mister' unexpected" ns2/named.run || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
@@ -89,11 +86,11 @@ status=`expr $status + $ret`
 echo_i "adding a zone that requires quotes ($n)"
 ret=0
 $RNDCCMD 10.53.0.2 addzone '"32/1.0.0.127-in-addr.added.example" { check-names ignore; type master; file "added.db"; };' 2>&1 | sed 's/^/I:ns2 /'
-_check_zone_that_requires_quotes() {
-	$DIG $DIGOPTS @10.53.0.2 "a.32/1.0.0.127-in-addr.added.example" a > dig.out.ns2.$n || return 1
-	grep 'status: NOERROR' dig.out.ns2.$n > /dev/null || return 1
-	grep '^a.32/1.0.0.127-in-addr.added.example' dig.out.ns2.$n > /dev/null || return 1
-}
+_check_zone_that_requires_quotes() (
+	$DIG $DIGOPTS @10.53.0.2 "a.32/1.0.0.127-in-addr.added.example" a > dig.out.ns2.$n &&
+	grep 'status: NOERROR' dig.out.ns2.$n > /dev/null &&
+	grep '^a.32/1.0.0.127-in-addr.added.example' dig.out.ns2.$n > /dev/null
+)
 retry_quiet 10 _check_zone_that_requires_quotes || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
@@ -102,11 +99,11 @@ status=`expr $status + $ret`
 echo_i "adding a zone with a quote in the name ($n)"
 ret=0
 $RNDCCMD 10.53.0.2 addzone '"foo\"bar.example" { check-names ignore; type master; file "added.db"; };' 2>&1 | sed 's/^/I:ns2 /'
-_check_zone_with_a_quote() {
-	$DIG $DIGOPTS @10.53.0.2 "a.foo\"bar.example" a > dig.out.ns2.$n || return 1
-	grep 'status: NOERROR' dig.out.ns2.$n > /dev/null || return 1
-	grep '^a.foo\\"bar.example' dig.out.ns2.$n > /dev/null || return 1
-}
+_check_zone_with_a_quote() (
+	$DIG $DIGOPTS @10.53.0.2 "a.foo\"bar.example" a > dig.out.ns2.$n &&
+	grep 'status: NOERROR' dig.out.ns2.$n > /dev/null &&
+	grep '^a.foo\\"bar.example' dig.out.ns2.$n > /dev/null
+)
 retry_quiet 10 _check_zone_with_a_quote || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
@@ -155,25 +152,24 @@ fi
 echo_i "deleting previously added zone ($n)"
 ret=0
 $RNDCCMD 10.53.0.2 delzone previous.example 2>&1 | sed 's/^/I:ns2 /'
-_check_deleting_previously_added_zone() {
-	$DIG $DIGOPTS @10.53.0.2 a.previous.example a > dig.out.ns2.$n
-	grep 'status: REFUSED' dig.out.ns2.$n > /dev/null || return 1
-	grep '^a.previous.example' dig.out.ns2.$n > /dev/null && return 1
-	return 0
-}
+_check_deleting_previously_added_zone() (
+	$DIG $DIGOPTS @10.53.0.2 a.previous.example a > dig.out.ns2.$n &&
+	grep 'status: REFUSED' dig.out.ns2.$n > /dev/null &&
+	! grep '^a.previous.example' dig.out.ns2.$n > /dev/null
+)
 retry_quiet 10 _check_deleting_previously_added_zone || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
 
+check_nzd2nzf() (
+	$NZD2NZF ns2/_default.nzd > nzd2nzf.out.$n &&
+	! grep previous.example nzd2nzf.out.$n > /dev/null
+)
+
 if [ -n "$NZD" ]; then
     echo_i "checking zone was deleted from NZD ($n)"
-    for i in 0 1 2 3 4 5 6 7 8 9; do
-        ret=0
-        $NZD2NZF ns2/_default.nzd | grep previous.example > /dev/null && ret=1
-        [ $ret = 0 ] && break
-        sleep 1
-    done
+    retry_quiet 10 check_nzd2nzf || ret=1
     if [ $ret != 0 ]; then echo_i "failed"; fi
     status=`expr $status + $ret`
 fi
@@ -191,12 +187,11 @@ fi
 echo_i "deleting newly added zone added.example ($n)"
 ret=0
 $RNDCCMD 10.53.0.2 delzone added.example 2>&1 | sed 's/^/I:ns2 /'
-_check_deleting_newly_added_zone() {
-	$DIG $DIGOPTS @10.53.0.2 a.added.example a > dig.out.ns2.$n
-	grep 'status: REFUSED' dig.out.ns2.$n > /dev/null || return 1
-	grep '^a.added.example' dig.out.ns2.$n > /dev/null && return 1
-	return 0
-}
+_check_deleting_newly_added_zone() (
+	$DIG $DIGOPTS @10.53.0.2 a.added.example a > dig.out.ns2.$n &&
+	grep 'status: REFUSED' dig.out.ns2.$n > /dev/null &&
+	! grep '^a.added.example' dig.out.ns2.$n > /dev/null
+)
 retry_quiet 10 _check_deleting_newly_added_zone || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
@@ -205,12 +200,11 @@ status=`expr $status + $ret`
 echo_i "deleting newly added zone with escaped quote ($n)"
 ret=0
 $RNDCCMD 10.53.0.2 delzone "foo\\\"bar.example" 2>&1 | sed 's/^/I:ns2 /'
-_check_deleting_newly_added_zone_quote() {
-	$DIG $DIGOPTS @10.53.0.2 "a.foo\"bar.example" a > dig.out.ns2.$n
-	grep 'status: REFUSED' dig.out.ns2.$n > /dev/null || return 1
-	grep "^a.foo\"bar.example" dig.out.ns2.$n > /dev/null && return 1
-	return 0
-}
+_check_deleting_newly_added_zone_quote() (
+	$DIG $DIGOPTS @10.53.0.2 "a.foo\"bar.example" a > dig.out.ns2.$n &&
+	grep 'status: REFUSED' dig.out.ns2.$n > /dev/null &&
+	! grep "^a.foo\"bar.example" dig.out.ns2.$n > /dev/null
+)
 retry_quiet 10 _check_deleting_newly_added_zone_quote || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
@@ -257,12 +251,7 @@ ret=0
 sleep 1
 cp -f ns1/redirect.db.2 ns1/redirect.db
 $RNDCCMD 10.53.0.1 reload -redirect > rndc.out.ns1.$n
-_check_rndc_reload_with_a_normal_zone() {
-    $RNDCCMD 10.53.0.1 zonestatus -redirect > zonestatus.out.ns1.$n || return 1
-    grep "type: redirect" zonestatus.out.ns1.$n > /dev/null || return 1
-    grep "serial: 1" zonestatus.out.ns1.$n > /dev/null || return 1
-}
-retry 5 _check_rndc_reload_with_a_normal_zone || ret=1
+retry_quiet 5 check_zonestatus 1 || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
@@ -273,11 +262,11 @@ $RNDCCMD 10.53.0.2 delzone normal.example > rndc.out.ns2.$n 2>&1
 grep "is no longer active and will be deleted" rndc.out.ns2.$n > /dev/null || ret=11
 grep "To keep it from returning when the server is restarted" rndc.out.ns2.$n > /dev/null || ret=1
 grep "must also be removed from named.conf." rndc.out.ns2.$n > /dev/null || ret=1
-_check_delete_normally_loaded_zone() {
-	$DIG $DIGOPTS @10.53.0.2 a.normal.example a > dig.out.ns2.$n
-	grep 'status: REFUSED' dig.out.ns2.$n > /dev/null || return 1
-}
-retry 5 _check_delete_normally_loaded_zone || ret=1
+_check_delete_normally_loaded_zone() (
+	$DIG $DIGOPTS @10.53.0.2 a.normal.example a > dig.out.ns2.$n &&
+	grep 'status: REFUSED' dig.out.ns2.$n > /dev/null
+)
+retry_quiet 5 _check_delete_normally_loaded_zone || ret=1
 
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
@@ -285,11 +274,11 @@ status=`expr $status + $ret`
 
 echo_i "attempting to add master zone with inline signing ($n)"
 $RNDCCMD 10.53.0.2 addzone 'inline.example { type master; file "inline.db"; inline-signing yes; };' 2>&1 | sed 's/^/I:ns2 /'
-_check_add_master_zone_with_inline() {
-	$DIG $DIGOPTS @10.53.0.2 a.inline.example a > dig.out.ns2.$n || return 1
-	grep 'status: NOERROR' dig.out.ns2.$n > /dev/null || return 1
-	grep '^a.inline.example' dig.out.ns2.$n > /dev/null || return 1
-}
+_check_add_master_zone_with_inline() (
+	$DIG $DIGOPTS @10.53.0.2 a.inline.example a > dig.out.ns2.$n &&
+	grep 'status: NOERROR' dig.out.ns2.$n > /dev/null &&
+	grep '^a.inline.example' dig.out.ns2.$n > /dev/null
+)
 retry_quiet 5 _check_add_master_zone_with_inline || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
@@ -305,11 +294,11 @@ status=`expr $status + $ret`
 
 echo_i "attempting to add slave zone with inline signing ($n)"
 $RNDCCMD 10.53.0.2 addzone 'inlineslave.example { type slave; masters { 10.53.0.1; }; file "inlineslave.bk"; inline-signing yes; };' 2>&1 | sed 's/^/I:ns2 /'
-_check_add_slave_with_inline() {
-	$DIG $DIGOPTS @10.53.0.2 a.inlineslave.example a > dig.out.ns2.$n || return 1
-	grep 'status: NOERROR' dig.out.ns2.$n > /dev/null || return 1
-	grep '^a.inlineslave.example' dig.out.ns2.$n > /dev/null || return 1
-}
+_check_add_slave_with_inline() (
+	$DIG $DIGOPTS @10.53.0.2 a.inlineslave.example a > dig.out.ns2.$n &&
+	grep 'status: NOERROR' dig.out.ns2.$n > /dev/null &&
+	grep '^a.inlineslave.example' dig.out.ns2.$n > /dev/null
+)
 retry_quiet 5 _check_add_slave_with_inline || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
@@ -317,12 +306,8 @@ status=`expr $status + $ret`
 
 echo_i "attempting to delete slave zone with inline signing ($n)"
 ret=0
-for i in 0 1 2 3 4 5 6 7 8 9
-do
-	test -f ns2/inlineslave.bk.signed -a -f ns2/inlineslave.bk && break
-	sleep 1
-done
-$RNDCCMD 10.53.0.2 delzone inlineslave.example 2>&1 > rndc.out2.test$n
+retry_quiet 10 test -f ns2/inlineslave.bk.signed -a -f ns2/inlineslave.bk || ret=1
+$RNDCCMD 10.53.0.2 delzone inlineslave.example > rndc.out2.test$n 2>&1 || ret=1
 test -f inlineslave.bk ||
 grep '^inlineslave.bk$' rndc.out2.test$n > /dev/null || {
 	echo_i "failed to report inlineslave.bk"; ret=1;
@@ -336,11 +321,11 @@ status=`expr $status + $ret`
 
 echo_i "restoring slave zone with inline signing ($n)"
 $RNDCCMD 10.53.0.2 addzone 'inlineslave.example { type slave; masters { 10.53.0.1; }; file "inlineslave.bk"; inline-signing yes; };' 2>&1 | sed 's/^/I:ns2 /'
-_check_restoring_slave_with_inline() {
-	$DIG $DIGOPTS @10.53.0.2 a.inlineslave.example a > dig.out.ns2.$n || return 1
-	grep 'status: NOERROR' dig.out.ns2.$n > /dev/null || return 1
-	grep '^a.inlineslave.example' dig.out.ns2.$n > /dev/null || return 1
-}
+_check_restoring_slave_with_inline() (
+	$DIG $DIGOPTS @10.53.0.2 a.inlineslave.example a > dig.out.ns2.$n &&
+	grep 'status: NOERROR' dig.out.ns2.$n > /dev/null &&
+	grep '^a.inlineslave.example' dig.out.ns2.$n > /dev/null
+)
 retry_quiet 5 _check_restoring_slave_with_inline || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
@@ -348,28 +333,18 @@ status=`expr $status + $ret`
 
 echo_i "deleting slave zone with automatic zone file removal ($n)"
 ret=0
-for i in 0 1 2 3 4 5 6 7 8 9
-do
-	test -f ns2/inlineslave.bk.signed -a -f ns2/inlineslave.bk && break
-	sleep 1
-done
-$RNDCCMD 10.53.0.2 delzone -clean inlineslave.example 2>&1 > /dev/null
-for i in 0 1 2 3 4 5 6 7 8 9
-do
-        ret=0
-	test -f ns2/inlineslave.bk.signed -a -f ns2/inlineslave.bk && ret=1
-        [ $ret = 0 ] && break
-	sleep 1
-done
+retry_quiet 10 test -f ns2/inlineslave.bk.signed -a -f ns2/inlineslave.bk || ret=1
+$RNDCCMD 10.53.0.2 delzone -clean inlineslave.example > /dev/null 2>&1
+retry_quiet 10 test ! -f ns2/inlineslave.bk.signed -a ! -f ns2/inlineslave.bk
 n=`expr $n + 1`
 status=`expr $status + $ret`
 
 echo_i "modifying zone configuration ($n)"
 ret=0
-$RNDCCMD 10.53.0.2 addzone 'mod.example { type master; file "added.db"; };' 2>&1 | sed 's/^/I:ns2 /'
+$RNDCCMD 10.53.0.2 addzone 'mod.example { type master; file "added.db"; };' 2>&1 | sed 's/^/ns2 /' | cat_i
 $DIG +norec $DIGOPTS @10.53.0.2 mod.example ns > dig.out.ns2.1.$n || ret=1
 grep 'status: NOERROR' dig.out.ns2.1.$n > /dev/null || ret=1
-$RNDCCMD 10.53.0.2 modzone 'mod.example { type master; file "added.db"; allow-query { none; }; };' 2>&1 | sed 's/^/I:ns2 /'
+$RNDCCMD 10.53.0.2 modzone 'mod.example { type master; file "added.db"; allow-query { none; }; };' 2>&1 | sed 's/^/ns2 /' | cat_i
 $DIG +norec $DIGOPTS @10.53.0.2 mod.example ns > dig.out.ns2.2.$n || ret=1
 $RNDCCMD 10.53.0.2 showzone mod.example | grep 'allow-query { "none"; };' > /dev/null 2>&1 || ret=1
 n=`expr $n + 1`
@@ -393,13 +368,13 @@ status=`expr $status + $ret`
 echo_i "check that adding a 'master redirect' zone works ($n)"
 ret=0
 $RNDCCMD 10.53.0.2 addzone '"." { type redirect; file "redirect.db"; };' > rndc.out.ns2.$n 2>&1 || ret=1
-_check_add_master_redirect() {
-	$RNDCCMD 10.53.0.2 showzone -redirect > showzone.out.ns2.$n 2>&1 || return 1
-	grep "type redirect;" showzone.out.ns2.$n > /dev/null || return 1
-	$RNDCCMD 10.53.0.2 zonestatus -redirect > zonestatus.out.ns2.$n 2>&1 || return 1
-	grep "type: redirect" zonestatus.out.ns2.$n > /dev/null || return 1
-	grep "serial: 0" zonestatus.out.ns2.$n > /dev/null || return 1
-}
+_check_add_master_redirect() (
+	$RNDCCMD 10.53.0.2 showzone -redirect > showzone.out.ns2.$n 2>&1 &&
+	grep "type redirect;" showzone.out.ns2.$n > /dev/null &&
+	$RNDCCMD 10.53.0.2 zonestatus -redirect > zonestatus.out.ns2.$n 2>&1 &&
+	grep "type: redirect" zonestatus.out.ns2.$n > /dev/null &&
+	grep "serial: 0" zonestatus.out.ns2.$n > /dev/null
+)
 retry_quiet 10 _check_add_master_redirect || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
@@ -410,12 +385,7 @@ ret=0
 sleep 1
 cp -f ns2/redirect.db.2 ns2/redirect.db
 $RNDCCMD 10.53.0.2 reload -redirect > rndc.out.ns2.$n
-_check_reload_master_redirect() {
-	$RNDCCMD 10.53.0.2 zonestatus -redirect > zonestatus.out.ns2.$n 2>&1 || return 1
-	grep "type: redirect" zonestatus.out.ns2.$n > /dev/null || return 1
-	grep "serial: 1" zonestatus.out.ns2.$n > /dev/null || return 1
-}
-retry_quiet 10 _check_reload_master_redirect || ret=1
+retry_quiet 10 check_zonestatus 2 || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
@@ -430,10 +400,10 @@ status=`expr $status + $ret`
 echo_i "check that deleting a 'master redirect' zone works ($n)"
 ret=0
 $RNDCCMD 10.53.0.2 delzone -redirect > rndc.out.ns2.$n 2>&1 || ret=1
-_check_deleting_master_redirect() {
-	$RNDCCMD 10.53.0.2 showzone -redirect > showzone.out.ns2.$n 2>&1
-	grep 'not found' showzone.out.ns2.$n > /dev/null || return 1
-}
+_check_deleting_master_redirect() (
+	$RNDCCMD 10.53.0.2 showzone -redirect > showzone.out.ns2.$n 2>&1 || true
+	grep 'not found' showzone.out.ns2.$n > /dev/null
+)
 retry_quiet 10 _check_deleting_master_redirect || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
@@ -442,13 +412,13 @@ status=`expr $status + $ret`
 echo_i "check that adding a 'slave redirect' zone works ($n)"
 ret=0
 $RNDCCMD 10.53.0.2 addzone '"." { type redirect; masters { 10.53.0.3;}; file "redirect.bk"; };' > rndc.out.ns2.$n 2>&1 || ret=1
-_check_adding_slave_redirect() {
-	$RNDCCMD 10.53.0.2 showzone -redirect > showzone.out.ns2.$n 2>&1 || return 1
-	grep "type redirect;" showzone.out.ns2.$n > /dev/null || return 1
-	$RNDCCMD 10.53.0.2 zonestatus -redirect > zonestatus.out.ns2.$n 2>&1 || return 1
-	grep "type: redirect" zonestatus.out.ns2.$n > /dev/null || return 1
-	grep "serial: 0" zonestatus.out.ns2.$n > /dev/null || return 1
-}
+_check_adding_slave_redirect() (
+	$RNDCCMD 10.53.0.2 showzone -redirect > showzone.out.ns2.$n 2>&1 &&
+	grep "type redirect;" showzone.out.ns2.$n > /dev/null &&
+	$RNDCCMD 10.53.0.2 zonestatus -redirect > zonestatus.out.ns2.$n 2>&1 &&
+	grep "type: redirect" zonestatus.out.ns2.$n > /dev/null &&
+	grep "serial: 0" zonestatus.out.ns2.$n > /dev/null
+)
 retry_quiet 10 _check_adding_slave_redirect || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
@@ -458,12 +428,12 @@ echo_i "check that retransfering a added 'slave redirect' zone works ($n)"
 ret=0
 cp -f ns3/redirect.db.2 ns3/redirect.db
 $RNDCCMD 10.53.0.3 reload . > showzone.out.ns3.$n 2>&1 || ret=1
-_check_retransfering_slave_redirect() {
-	$RNDCCMD 10.53.0.2 retransfer -redirect > rndc.out.ns2.$n 2>&1 || return 1
-	$RNDCCMD 10.53.0.2 zonestatus -redirect > zonestatus.out.ns2.$n 2>&1 || return 1
-	grep "type: redirect" zonestatus.out.ns2.$n > /dev/null || return 1
-	grep "serial: 1" zonestatus.out.ns2.$n > /dev/null || return 1
-}
+_check_retransfering_slave_redirect() (
+	$RNDCCMD 10.53.0.2 retransfer -redirect > rndc.out.ns2.$n 2>&1 &&
+	$RNDCCMD 10.53.0.2 zonestatus -redirect > zonestatus.out.ns2.$n 2>&1 &&
+	grep "type: redirect" zonestatus.out.ns2.$n > /dev/null &&
+	grep "serial: 1" zonestatus.out.ns2.$n > /dev/null
+)
 retry_quiet 10 _check_retransfering_slave_redirect || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
@@ -472,10 +442,10 @@ status=`expr $status + $ret`
 echo_i "check that deleting a 'slave redirect' zone works ($n)"
 ret=0
 $RNDCCMD 10.53.0.2 delzone -redirect > rndc.out.ns2.$n 2>&1 || ret=1
-_check_deleting_slave_redirect() {
-	$RNDCCMD 10.53.0.2 showzone -redirect > showzone.out.ns2.$n 2>&1
-	grep 'not found' showzone.out.ns2.$n > /dev/null || return 1
-}
+_check_deleting_slave_redirect() (
+	$RNDCCMD 10.53.0.2 showzone -redirect > showzone.out.ns2.$n 2>&1 || true
+	grep 'not found' showzone.out.ns2.$n > /dev/null
+)
 retry_quiet 10 _check_deleting_slave_redirect || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
@@ -561,29 +531,29 @@ fi
 
 echo_i "checking rndc reload causes named to reload the external view's new zone config ($n)"
 ret=0
-$RNDCCMD 10.53.0.2 reload 2>&1 | sed 's/^/I:ns2 /'
-_check_rndc_reload_external_view_config() {
-	$DIG +norec $DIGOPTS @10.53.0.2 -b 10.53.0.2 a.added.example a > dig.out.ns2.int.$n || return 1
-	grep 'status: NOERROR' dig.out.ns2.int.$n > /dev/null || return 1
-	$DIG +norec $DIGOPTS @10.53.0.4 -b 10.53.0.4 a.added.example a > dig.out.ns2.ext.$n || return 1
-	grep 'status: NOERROR' dig.out.ns2.ext.$n > /dev/null || return 1
-	grep '^a.added.example' dig.out.ns2.ext.$n > /dev/null || return 1
-}
+$RNDCCMD 10.53.0.2 reload 2>&1 | sed 's/^/ns2 /' | cat_i
+_check_rndc_reload_external_view_config() (
+	$DIG +norec $DIGOPTS @10.53.0.2 -b 10.53.0.2 a.added.example a > dig.out.ns2.int.$n &&
+	grep 'status: NOERROR' dig.out.ns2.int.$n > /dev/null &&
+	$DIG +norec $DIGOPTS @10.53.0.4 -b 10.53.0.4 a.added.example a > dig.out.ns2.ext.$n &&
+	grep 'status: NOERROR' dig.out.ns2.ext.$n > /dev/null &&
+	grep '^a.added.example' dig.out.ns2.ext.$n > /dev/null
+)
 retry_quiet 10 _check_rndc_reload_external_view_config || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
 
 echo_i "checking rndc showzone with newly added zone ($n)"
-_check_rndc_showzone_newly_added() {
-	$RNDCCMD 10.53.0.2 showzone added.example in external > rndc.out.ns2.$n 2>/dev/null
+_check_rndc_showzone_newly_added() (
 	if [ -z "$NZD" ]; then
 		expected='zone "added.example" in external { type master; file "added.db"; };'
 	else
 		expected='zone "added.example" { type master; file "added.db"; };'
 	fi
-	[ "`cat rndc.out.ns2.$n`" = "$expected" ] || return 1
-}
+	$RNDCCMD 10.53.0.2 showzone added.example in external > rndc.out.ns2.$n 2>/dev/null &&
+	[ "`cat rndc.out.ns2.$n`" = "$expected" ]
+)
 retry_quiet 10  _check_rndc_showzone_newly_added || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
@@ -592,12 +562,11 @@ status=`expr $status + $ret`
 echo_i "deleting newly added zone ($n)"
 ret=0
 $RNDCCMD 10.53.0.2 delzone 'added.example in external' 2>&1 | sed 's/^/I:ns2 /'
-_check_deleting_newly_added_zone() {
-	$DIG $DIGOPTS @10.53.0.4 -b 10.53.0.4 a.added.example a > dig.out.ns2.$n || return1
-	grep 'status: REFUSED' dig.out.ns2.$n > /dev/null || return 1
-	grep '^a.added.example' dig.out.ns2.$n > /dev/null && return 1
-	return 0
-}
+_check_deleting_newly_added_zone() (
+	$DIG $DIGOPTS @10.53.0.4 -b 10.53.0.4 a.added.example a > dig.out.ns2.$n &&
+	grep 'status: REFUSED' dig.out.ns2.$n > /dev/null &&
+	! grep '^a.added.example' dig.out.ns2.$n > /dev/null
+)
 retry_quiet 10 _check_deleting_newly_added_zone || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
@@ -628,13 +597,13 @@ status=`expr $status + $ret`
 echo_i "adding new zone again to external view ($n)"
 ret=0
 $RNDCCMD 10.53.0.2 addzone 'added.example in external { type master; file "added.db"; };' 2>&1 | sed 's/^/I:ns2 /'
-_check_adding_new_zone_again_external() {
-	$DIG +norec $DIGOPTS @10.53.0.2 -b 10.53.0.2 a.added.example a > dig.out.ns2.int.$n || return 1
-	grep 'status: NOERROR' dig.out.ns2.int.$n > /dev/null || return 1
-	$DIG +norec $DIGOPTS @10.53.0.4 -b 10.53.0.4 a.added.example a > dig.out.ns2.ext.$n || return 1
-	grep 'status: NOERROR' dig.out.ns2.ext.$n > /dev/null || return 1
-	grep '^a.added.example' dig.out.ns2.ext.$n > /dev/null || return 1
-}
+_check_adding_new_zone_again_external() (
+	$DIG +norec $DIGOPTS @10.53.0.2 -b 10.53.0.2 a.added.example a > dig.out.ns2.int.$n &&
+	grep 'status: NOERROR' dig.out.ns2.int.$n > /dev/null &&
+	$DIG +norec $DIGOPTS @10.53.0.4 -b 10.53.0.4 a.added.example a > dig.out.ns2.ext.$n &&
+	grep 'status: NOERROR' dig.out.ns2.ext.$n > /dev/null &&
+	grep '^a.added.example' dig.out.ns2.ext.$n > /dev/null
+)
 retry_quiet 10 _check_adding_new_zone_again_external || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
@@ -740,20 +709,18 @@ then
     status=`expr $status + $ret`
 fi
 
+_check_version_bind() (
+	$DIG $DIGOPTS @10.53.0.3 version.bind txt ch > dig.out.test$n &&
+	grep "status: NOERROR" dig.out.test$n > /dev/null
+)
+
 echo_i "check that named restarts with multiple added zones ($n)"
 ret=0
 $RNDCCMD 10.53.0.3 addzone "test4.baz" '{ type master; file "e.db"; };' > /dev/null 2>&1 || ret=1
 $RNDCCMD 10.53.0.3 addzone "test5.baz" '{ type master; file "e.db"; };' > /dev/null 2>&1 || ret=1
 $PERL $SYSTEMTESTTOP/stop.pl addzone ns3
 $PERL $SYSTEMTESTTOP/start.pl --noclean --restart --port ${PORT} addzone ns3 || ret=1
-for try in 0 1 2 3 4 5 6 7 8 9; do
-    iret=0
-    $DIG $DIGOPTS @10.53.0.3 version.bind txt ch > dig.out.test$n || iret=1
-    grep "status: NOERROR" dig.out.test$n > /dev/null || iret=1
-    [ "$iret" -eq 0 ] && break
-    sleep 1
-done
-[ "$iret" -ne 0 ] && ret=1
+retry_quiet 10 _check_version_bind || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
 n=`expr $n + 1`

bin/tests/system/autosign/clean.sh

@@ -14,7 +14,7 @@ rm -f */core
 rm -f */example.bk
 rm -f */named.conf
 rm -f */named.memstats
-rm -f */named.run
+rm -f */named.run*
 rm -f */trusted.conf */private.conf
 rm -f activate-now-publish-1day.key
 rm -f active.key inact.key del.key delzsk.key unpub.key standby.key rev.key
@@ -63,6 +63,7 @@ rm -f ns3/secure.nsec3.example.db
 rm -f ns3/secure.optout.example.db
 rm -f ns3/sync.example.db
 rm -f ns3/ttl*.db
+rm -f ns3/settime.out.*
 rm -f nsupdate.out
 rm -f settime.out.*
 rm -f signing.out.*

bin/tests/system/autosign/tests.sh

@@ -50,6 +50,13 @@ checkprivate () {
     return 1
 }
 
+# wait until notifies for zone $1 are sent by server $2. This is an indication
+# that the zone is signed with the active keys, and the changes have been
+# committed.
+wait_for_notifies () {
+	wait_for_log 10 "zone ${1}/IN: sending notifies" "${2}/named.run" || return 1
+}
+
 freq() {
 	_file=$1
 	# remove first and last line that has incomplete set and skews the distribution
@@ -177,6 +184,9 @@ do
 	$DIG $DIGOPTS $z @10.53.0.3 axfr | awk '$4 == "RRSIG" {print $9}' | sort | uniq -c | cat_i
 done
 
+# Set logfile offset for wait_for_log usage.
+nextpartreset ns3/named.run
+
 #
 # Check that DNSKEY is initially signed with a KSK and not a ZSK.
 #
@@ -207,8 +217,8 @@ test $count -eq 3 || ret=1
 awk='$4 == "RRSIG" && $5 == "DNSKEY" { printf "%05u\n", $11 }'
 id=`awk "${awk}" dig.out.ns3.test$n`
 
-$SETTIME -D now+5 ns3/Kinacksk3.example.+007+${id} > /dev/null 2>&1
-$RNDCCMD 10.53.0.3 loadkeys inacksk3.example 2>&1 | sed 's/^/ns3 /' | cat_i
+$SETTIME -D now+5 ns3/Kinacksk3.example.+007+${id} > settime.out.test$n || ret=1
+($RNDCCMD 10.53.0.3 loadkeys inacksk3.example 2>&1 | sed 's/^/ns3 /' | cat_i) || ret=1
 
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
@@ -235,8 +245,8 @@ count=`awk 'BEGIN { count = 0 }
        END {print count}' dig.out.ns3.test$n`
 test $count -eq 3 || ret=1
 id=`awk '$4 == "RRSIG" && $5 == "CNAME" { printf "%05u\n", $11 }' dig.out.ns3.test$n`
-$SETTIME -D now+5 ns3/Kinaczsk3.example.+007+${id} > /dev/null 2>&1
-$RNDCCMD 10.53.0.3 loadkeys inaczsk3.example 2>&1 | sed 's/^/ns3 /' | cat_i
+$SETTIME -D now+5 ns3/Kinaczsk3.example.+007+${id} > settime.out.test$n || ret=1
+($RNDCCMD 10.53.0.3 loadkeys inaczsk3.example 2>&1 | sed 's/^/ns3 /' | cat_i) || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
@@ -332,9 +342,9 @@ status=`expr $status + $ret`
 echo_i "signing preset nsec3 zone"
 zsk=`cat autozsk.key`
 ksk=`cat autoksk.key`
-$SETTIME -K ns3 -P now -A now $zsk > /dev/null 2>&1
-$SETTIME -K ns3 -P now -A now $ksk > /dev/null 2>&1
-$RNDCCMD 10.53.0.3 loadkeys autonsec3.example. 2>&1 | sed 's/^/ns3 /' | cat_i
+$SETTIME -K ns3 -P now -A now $zsk > settime.out.test$n.zsk || ret=1
+$SETTIME -K ns3 -P now -A now $ksk > settime.out.test$n.ksk || ret=1
+($RNDCCMD 10.53.0.3 loadkeys autonsec3.example. 2>&1 | sed 's/^/ns3 /' | cat_i) || ret=1
 
 echo_i "waiting for changes to take effect"
 sleep 3
@@ -387,9 +397,9 @@ status=`expr $status + $ret`
 # Send rndc sync command to ns1, ns2 and ns3, to force the dynamically
 # signed zones to be dumped to their zone files
 echo_i "dumping zone files"
-$RNDCCMD 10.53.0.1 sync 2>&1 | sed 's/^/ns1 /' | cat_i
-$RNDCCMD 10.53.0.2 sync 2>&1 | sed 's/^/ns2 /' | cat_i
-$RNDCCMD 10.53.0.3 sync 2>&1 | sed 's/^/ns3 /' | cat_i
+($RNDCCMD 10.53.0.1 sync 2>&1 | sed 's/^/ns1 /' | cat_i) || ret=1
+($RNDCCMD 10.53.0.2 sync 2>&1 | sed 's/^/ns2 /' | cat_i) || ret=1
+($RNDCCMD 10.53.0.3 sync 2>&1 | sed 's/^/ns3 /' | cat_i) || ret=1
 
 now="$(TZ=UTC date +%Y%m%d%H%M%S)"
 check_expiry() (
@@ -488,7 +498,7 @@ echo_i "checking TTLs of imported DNSKEYs (no default) ($n)"
 ret=0
 $DIG $DIGOPTS +tcp +noall +answer dnskey ttl1.example. @10.53.0.3 > dig.out.ns3.test$n || ret=1
 [ -s dig.out.ns3.test$n ] || ret=1
-awk 'BEGIN {r=0} $2 != 300 {r=1; print "found TTL " $2} END {exit r}' dig.out.ns3.test$n || ret=1 | cat_i
+(awk 'BEGIN {r=0} $2 != 300 {r=1; print "found TTL " $2} END {exit r}' dig.out.ns3.test$n | cat_i) || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
@@ -497,7 +507,7 @@ echo_i "checking TTLs of imported DNSKEYs (with default) ($n)"
 ret=0
 $DIG $DIGOPTS +tcp +noall +answer dnskey ttl2.example. @10.53.0.3 > dig.out.ns3.test$n || ret=1
 [ -s dig.out.ns3.test$n ] || ret=1
-awk 'BEGIN {r=0} $2 != 60 {r=1; print "found TTL " $2} END {exit r}' dig.out.ns3.test$n || ret=1 | cat_i
+(awk 'BEGIN {r=0} $2 != 60 {r=1; print "found TTL " $2} END {exit r}' dig.out.ns3.test$n | cat_i) || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
@@ -506,7 +516,7 @@ echo_i "checking TTLs of imported DNSKEYs (mismatched) ($n)"
 ret=0
 $DIG $DIGOPTS +tcp +noall +answer dnskey ttl3.example. @10.53.0.3 > dig.out.ns3.test$n || ret=1
 [ -s dig.out.ns3.test$n ] || ret=1
-awk 'BEGIN {r=0} $2 != 30 {r=1; print "found TTL " $2} END {exit r}' dig.out.ns3.test$n || ret=1 | cat_i
+(awk 'BEGIN {r=0} $2 != 30 {r=1; print "found TTL " $2} END {exit r}' dig.out.ns3.test$n | cat_i) || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
@@ -515,7 +525,7 @@ echo_i "checking TTLs of imported DNSKEYs (existing RRset) ($n)"
 ret=0
 $DIG $DIGOPTS +tcp +noall +answer dnskey ttl4.example. @10.53.0.3 > dig.out.ns3.test$n || ret=1
 [ -s dig.out.ns3.test$n ] || ret=1
-awk 'BEGIN {r=0} $2 != 30 {r=1; print "found TTL " $2} END {exit r}' dig.out.ns3.test$n || ret=1 | cat_i
+(awk 'BEGIN {r=0} $2 != 30 {r=1; print "found TTL " $2} END {exit r}' dig.out.ns3.test$n | cat_i) || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
@@ -1006,10 +1016,10 @@ status=`expr $status + $ret`
 echo_i "checking secure-to-insecure transition, scheduled ($n)"
 ret=0
 file="ns3/`cat del1.key`.key"
-$SETTIME -I now -D now $file > /dev/null
+$SETTIME -I now -D now $file > settime.out.test$n.1 || ret=1
 file="ns3/`cat del2.key`.key"
-$SETTIME -I now -D now $file > /dev/null
-$RNDCCMD 10.53.0.3 sign secure-to-insecure2.example. 2>&1 | sed 's/^/ns3 /' | cat_i
+$SETTIME -I now -D now $file > settime.out.test$n.2 || ret=1
+($RNDCCMD 10.53.0.3 sign secure-to-insecure2.example. 2>&1 | sed 's/^/ns3 /' | cat_i) || ret=1
 for i in 0 1 2 3 4 5 6 7 8 9; do
 	ret=0
 	$DIG $DIGOPTS axfr secure-to-insecure2.example @10.53.0.3 > dig.out.ns3.test$n || ret=1
@@ -1035,13 +1045,13 @@ END
 # Create DNSSEC keys in the zone directory.
 $KEYGEN -a rsasha1 -3 -q -K ns3 jitter.nsec3.example > /dev/null
 # Trigger zone signing.
-$RNDCCMD 10.53.0.3 sign jitter.nsec3.example. 2>&1 | sed 's/^/ns3 /' | cat_i
+($RNDCCMD 10.53.0.3 sign jitter.nsec3.example. 2>&1 | sed 's/^/ns3 /' | cat_i) || ret=1
 # Wait until zone has been signed.
 check_if_nsec3param_exists() {
 	$DIG $DIGOPTS NSEC3PARAM jitter.nsec3.example @10.53.0.3 > dig.out.ns3.1.test$n || return 1
 	grep -q "^jitter\.nsec3\.example\..*NSEC3PARAM" dig.out.ns3.1.test$n || return 1
 }
-retry_quiet 50 check_if_nsec3param_exists || ret=1
+retry_quiet 20 check_if_nsec3param_exists || ret=1
 $DIG $DIGOPTS AXFR jitter.nsec3.example @10.53.0.3 > dig.out.ns3.2.test$n || ret=1
 # Check jitter distribution.
 checkjitter dig.out.ns3.2.test$n || ret=1
@@ -1056,7 +1066,7 @@ oldinception=`$DIG $DIGOPTS +short soa prepub.example @10.53.0.3 | awk '/SOA/ {p
 
 $KEYGEN -a rsasha1 -3 -q -K ns3 -P 0 -A +6d -I +38d -D +45d prepub.example > /dev/null
 
-$RNDCCMD 10.53.0.3 sign prepub.example 2>&1 | sed 's/^/ns1 /' | cat_i
+($RNDCCMD 10.53.0.3 sign prepub.example 2>&1 | sed 's/^/ns1 /' | cat_i) || ret=1
 newserial=$oldserial
 try=0
 while [ $oldserial -eq $newserial -a $try -lt 42 ]
@@ -1087,19 +1097,19 @@ oldfile=`cat active.key`
 oldid=$(keyfile_to_key_id "$(cat active.key)")
 newfile=`cat standby.key`
 newid=$(keyfile_to_key_id "$(cat standby.key)")
-$SETTIME -K ns1 -I now+2s -D now+25 $oldfile > /dev/null
-$SETTIME -K ns1 -i 0 -S $oldfile $newfile > /dev/null
+$SETTIME -K ns1 -I now+2s -D now+25 $oldfile > settime.out.test$n.1 || ret=1
+$SETTIME -K ns1 -i 0 -S $oldfile $newfile > settime.out.test$n.2 || ret=1
 
 # note previous zone serial number
 oldserial=`$DIG $DIGOPTS +short soa . @10.53.0.1 | awk '{print $3}'`
 
-$RNDCCMD 10.53.0.1 loadkeys . 2>&1 | sed 's/^/ns1 /' | cat_i
+($RNDCCMD 10.53.0.1 loadkeys . 2>&1 | sed 's/^/ns1 /' | cat_i) || ret=1
 sleep 4
 
 echo_i "revoking key to duplicated key ID"
-$SETTIME -R now -K ns2 Kbar.+005+30676.key > /dev/null 2>&1
+$SETTIME -R now -K ns2 Kbar.+005+30676.key > settime.out.test$n.3 || ret=1
 
-$RNDCCMD 10.53.0.2 loadkeys bar. 2>&1 | sed 's/^/ns2 /' | cat_i
+($RNDCCMD 10.53.0.2 loadkeys bar. 2>&1 | sed 's/^/ns2 /' | cat_i) || ret=1
 
 echo_i "waiting for changes to take effect"
 sleep 5
@@ -1152,7 +1162,7 @@ n=`expr $n + 1`
 status=`expr $status + $ret`
 
 echo_i "forcing full sign"
-$RNDCCMD 10.53.0.1 sign . 2>&1 | sed 's/^/ns1 /' | cat_i
+($RNDCCMD 10.53.0.1 sign . 2>&1 | sed 's/^/ns1 /' | cat_i) || ret=1
 
 echo_i "waiting for change to take effect"
 sleep 5
@@ -1178,8 +1188,10 @@ ret=0
 zsk=`cat delayzsk.key`
 ksk=`cat delayksk.key`
 # publication and activation times should be unset
-$SETTIME -K ns3 -pA -pP $zsk | grep -v UNSET > /dev/null 2>&1 && ret=1
-$SETTIME -K ns3 -pA -pP $ksk | grep -v UNSET > /dev/null 2>&1 && ret=1
+$SETTIME -K ns3 -pA -pP $zsk > settime.out.test$n.zsk || ret=1
+grep -v UNSET settime.out.test$n.zsk >/dev/null && ret=1
+$SETTIME -K ns3 -pA -pP $ksk > settime.out.test$n.ksk || ret=1
+grep -v UNSET settime.out.test$n.ksk >/dev/null && ret=1
 $DIG $DIGOPTS +noall +answer dnskey delay.example. @10.53.0.3 > dig.out.ns3.test$n || ret=1
 # DNSKEY not expected:
 awk 'BEGIN {r=1} $4=="DNSKEY" {r=0} END {exit r}' dig.out.ns3.test$n && ret=1
@@ -1189,12 +1201,14 @@ status=`expr $status + $ret`
 
 echo_i "checking scheduled key publication, not activation ($n)"
 ret=0
-$SETTIME -K ns3 -P now+3s -A none $zsk > /dev/null 2>&1
-$SETTIME -K ns3 -P now+3s -A none $ksk > /dev/null 2>&1
-$RNDCCMD 10.53.0.3 loadkeys delay.example. 2>&1 | sed 's/^/ns2 /' | cat_i
-
+# Ensure initial zone is loaded.
+wait_for_notifies "delay.example" "ns3" || ret=1
+$SETTIME -K ns3 -P now+3s -A none $zsk > settime.out.test$n.zsk || ret=1
+$SETTIME -K ns3 -P now+3s -A none $ksk > settime.out.test$n.ksk || ret=1
+($RNDCCMD 10.53.0.3 loadkeys delay.example. 2>&1 | sed 's/^/ns2 /' | cat_i) || ret=1
 echo_i "waiting for changes to take effect"
-sleep 5
+sleep 3
+wait_for_notifies "delay.example" "ns3" || ret=1
 
 $DIG $DIGOPTS +noall +answer dnskey delay.example. @10.53.0.3 > dig.out.ns3.test$n || ret=1
 # DNSKEY expected:
@@ -1207,13 +1221,12 @@ status=`expr $status + $ret`
 
 echo_i "checking scheduled key activation ($n)"
 ret=0
-$SETTIME -K ns3 -A now+3s $zsk > /dev/null 2>&1
-$SETTIME -K ns3 -A now+3s $ksk > /dev/null 2>&1
-$RNDCCMD 10.53.0.3 loadkeys delay.example. 2>&1 | sed 's/^/ns2 /' | cat_i
-
+$SETTIME -K ns3 -A now+3s $zsk > settime.out.test$n.zsk || ret=1
+$SETTIME -K ns3 -A now+3s $ksk > settime.out.test$n.ksk || ret=1
+($RNDCCMD 10.53.0.3 loadkeys delay.example. 2>&1 | sed 's/^/ns2 /' | cat_i) || ret=1
 echo_i "waiting for changes to take effect"
-sleep 5
-
+sleep 3
+wait_for_notifies "delay.example" "ns3" || ret=1
 $DIG $DIGOPTS +noall +answer dnskey delay.example. @10.53.0.3 > dig.out.ns3.1.test$n || ret=1
 # DNSKEY expected:
 awk 'BEGIN {r=1} $4=="DNSKEY" {r=0} END {exit r}' dig.out.ns3.1.test$n || ret=1
@@ -1323,7 +1336,7 @@ status=`expr $status + $ret`
 echo_i "forcing full sign with unreadable keys ($n)"
 ret=0
 chmod 0 ns1/K.+*+*.key ns1/K.+*+*.private || ret=1
-$RNDCCMD 10.53.0.1 sign . 2>&1 | sed 's/^/ns1 /' | cat_i
+($RNDCCMD 10.53.0.1 sign . 2>&1 | sed 's/^/ns1 /' | cat_i) || ret=1
 $DIG $DIGOPTS . @10.53.0.1 dnskey > dig.out.ns1.test$n || ret=1
 grep "status: NOERROR" dig.out.ns1.test$n > /dev/null || ret=1
 n=`expr $n + 1`
@@ -1333,11 +1346,11 @@ status=`expr $status + $ret`
 echo_i "test turning on auto-dnssec during reconfig ($n)"
 ret=0
 # first create a zone that doesn't have auto-dnssec
-$RNDCCMD 10.53.0.3 addzone reconf.example '{ type master; file "reconf.example.db"; };' 2>&1 | sed 's/^/ns3 /' | cat_i
+($RNDCCMD 10.53.0.3 addzone reconf.example '{ type master; file "reconf.example.db"; };' 2>&1 | sed 's/^/ns3 /' | cat_i) || ret=1
 rekey_calls=`grep "zone reconf.example.*next key event" ns3/named.run | wc -l`
 [ "$rekey_calls" -eq 0 ] || ret=1
 # ...then we add auto-dnssec and reconfigure
-$RNDCCMD 10.53.0.3 modzone reconf.example '{ type master; file "reconf.example.db"; allow-update { any; }; auto-dnssec maintain; };' 2>&1 | sed 's/^/ns3 /' | cat_i
+($RNDCCMD 10.53.0.3 modzone reconf.example '{ type master; file "reconf.example.db"; allow-update { any; }; auto-dnssec maintain; };' 2>&1 | sed 's/^/ns3 /' | cat_i) || ret=1
 rndc_reconfig ns3 10.53.0.3
 for i in 0 1 2 3 4 5 6 7 8 9; do
     lret=0
@@ -1393,8 +1406,8 @@ if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
 
 echo_i "setting CDS and CDNSKEY deletion times and calling 'rndc loadkeys'"
-$SETTIME -D sync now `cat sync.key` > /dev/null
-$RNDCCMD 10.53.0.3 loadkeys sync.example | sed 's/^/ns3 /' | cat_i
+$SETTIME -D sync now `cat sync.key` > settime.out.test$n || ret=1
+($RNDCCMD 10.53.0.3 loadkeys sync.example | sed 's/^/ns3 /' | cat_i) || ret=1
 
 echo_i "checking that the CDS and CDNSKEY are deleted ($n)"
 ret=0
@@ -1411,16 +1424,16 @@ status=`expr $status + $ret`
 
 echo_i "check that dnssec-settime -p Dsync works ($n)"
 ret=0
-$SETTIME -p Dsync `cat sync.key` > settime.out.$n|| ret=0
-grep "SYNC Delete:" settime.out.$n >/dev/null || ret=0
+$SETTIME -p Dsync `cat sync.key` > settime.out.test$n || ret=1
+grep "SYNC Delete:" settime.out.test$n >/dev/null || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
 
 echo_i "check that dnssec-settime -p Psync works ($n)"
 ret=0
-$SETTIME -p Psync `cat sync.key` > settime.out.$n|| ret=0
-grep "SYNC Publish:" settime.out.$n >/dev/null || ret=0
+$SETTIME -p Psync `cat sync.key` > settime.out.test$n || ret=1
+grep "SYNC Publish:" settime.out.test$n >/dev/null || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
@@ -1527,9 +1540,9 @@ if [ $_ret -ne 0 ]; then
 fi
 # Mark the inactive ZSK as pending removal.
 file="ns3/`cat delzsk.key`.key"
-$SETTIME -D now-1h $file > settime.out.test$n 2>&1 || ret=1
+$SETTIME -D now-1h $file > settime.out.test$n || ret=1
 # Trigger removal of the inactive ZSK and wait until its completion.
-$RNDCCMD 10.53.0.3 loadkeys delzsk.example 2>&1 | sed 's/^/ns3 /' | cat_i
+($RNDCCMD 10.53.0.3 loadkeys delzsk.example 2>&1 | sed 's/^/ns3 /' | cat_i) || ret=1
 for i in 0 1 2 3 4 5 6 7 8 9; do
 	_ret=1
 	$RNDCCMD 10.53.0.3 signing -list delzsk.example > signing.out.3.test$n 2>&1

bin/tests/system/conf.sh.common

@@ -374,6 +374,45 @@ nextpartpeek() {
 	nextpartread $1 2> /dev/null
 }
 
+# _search_log: look for message $1 in file $2 with nextpart().
+_search_log() (
+	msg="$1"
+	file="$2"
+	nextpart "$file" | grep -F "$msg" > /dev/null
+)
+
+# _search_log_peek: look for message $1 in file $2 with nextpartpeek().
+_search_log_peek() (
+	msg="$1"
+	file="$2"
+	nextpartpeek "$file" | grep -F "$msg" > /dev/null
+)
+
+# wait_for_log: wait until message $2 in file $3 appears.  Bail out after
+# $1 seconds.  This needs to be used in conjunction with a prior call to
+# nextpart() or nextpartreset() on the same file to guarantee the offset is
+# set correctly.  Tests using wait_for_log() are responsible for cleaning up
+# the created <file>.prev files.
+wait_for_log() (
+	timeout="$1"
+	msg="$2"
+	file="$3"
+	retry_quiet "$timeout" _search_log "$msg" "$file" && return 0
+	echo_i "exceeded time limit waiting for '$msg' in $file"
+        return 1
+)
+
+# wait_for_log_peek: similar to wait_for_log() but peeking, so the file offset
+# does not change.
+wait_for_log_peek() (
+	timeout="$1"
+	msg="$2"
+	file="$3"
+	retry_quiet "$timeout" _search_log_peek "$msg" "$file" && return 0
+	echo_i "exceeded time limit waiting for '$msg' in $file"
+        return 1
+)
+
 # _retry: keep running a command until it succeeds, up to $1 times, with
 # one-second intervals, optionally printing a message upon every attempt
 _retry() {

bin/tests/system/dnssec/tests.sh

@@ -40,25 +40,13 @@ rndccmd() {
     "$RNDC" -c "$SYSTEMTESTTOP/common/rndc.conf" -p "$CONTROLPORT" -s "$@"
 }
 
-# TODO: Move wait_for_log and loadkeys_on to conf.sh.common
-wait_for_log() {
-        msg=$1
-        file=$2
-
-        for i in 1 2 3 4 5 6 7 8 9 10; do
-                nextpart "$file" | grep "$msg" > /dev/null && return
-                sleep 1
-        done
-        echo_i "exceeded time limit waiting for '$msg' in $file"
-        ret=1
-}
-
+# TODO: Move loadkeys_on to conf.sh.common
 dnssec_loadkeys_on() {
 	nsidx=$1
 	zone=$2
 	nextpart ns${nsidx}/named.run > /dev/null
 	rndccmd 10.53.0.${nsidx} loadkeys ${zone} | sed "s/^/ns${nsidx} /" | cat_i
-	wait_for_log "next key event" ns${nsidx}/named.run
+	wait_for_log 20 "next key event" ns${nsidx}/named.run || return 1
 }
 
 # convert private-type records to readable form

bin/tests/system/forward/clean.sh

@@ -10,9 +10,9 @@
 #
 # Clean up after forward tests.
 #
-rm -f dig.out.*
-rm -f */named.conf
-rm -f */named.memstats
-rm -f */named.run
-rm -f ns*/named.lock
-rm -f ns*/managed-keys.bind*
+rm -f ./dig.out.*
+rm -f ./*/named.conf
+rm -f ./*/named.memstats
+rm -f ./*/named.run ./*/named.run.prev
+rm -f ./ns*/named.lock
+rm -f ./ns*/managed-keys.bind*

bin/tests/system/forward/tests.sh

@@ -7,11 +7,17 @@
 # See the COPYRIGHT file distributed with this work for additional
 # information regarding copyright ownership.
 
+#shellcheck source=conf.sh
 SYSTEMTESTTOP=..
-. $SYSTEMTESTTOP/conf.sh
+. "$SYSTEMTESTTOP/conf.sh"
 
-DIGOPTS="-p ${PORT}"
-SENDCMD="$PERL ../send.pl 10.53.0.6 $EXTRAPORT1"
+dig_with_opts() (
+	"$DIG" -p "$PORT" "$@"
+)
+
+sendcmd() (
+	"$PERL" ../send.pl 10.53.0.6 "$EXTRAPORT1"
+)
 
 root=10.53.0.1
 hidden=10.53.0.2
@@ -19,160 +25,197 @@ f1=10.53.0.3
 f2=10.53.0.4
 
 status=0
+n=0
 
-echo_i "checking that a forward zone overrides global forwarders"
+n=$((n+1))
+echo_i "checking that a forward zone overrides global forwarders ($n)"
 ret=0
-$DIG $DIGOPTS +noadd +noauth txt.example1. txt @$hidden > dig.out.hidden || ret=1
-$DIG $DIGOPTS +noadd +noauth txt.example1. txt @$f1 > dig.out.f1 || ret=1
-digcomp dig.out.hidden dig.out.f1 || ret=1
+dig_with_opts +noadd +noauth txt.example1. txt @$hidden > dig.out.$n.hidden || ret=1
+dig_with_opts +noadd +noauth txt.example1. txt @$f1 > dig.out.$n.f1 || ret=1
+digcomp dig.out.$n.hidden dig.out.$n.f1 || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-echo_i "checking that a forward first zone no forwarders recurses"
+n=$((n+1))
+echo_i "checking that a forward first zone no forwarders recurses ($n)"
 ret=0
-$DIG $DIGOPTS +noadd +noauth txt.example2. txt @$root > dig.out.root || ret=1
-$DIG $DIGOPTS +noadd +noauth txt.example2. txt @$f1 > dig.out.f1 || ret=1
-digcomp dig.out.root dig.out.f1 || ret=1
+dig_with_opts +noadd +noauth txt.example2. txt @$root > dig.out.$n.root || ret=1
+dig_with_opts +noadd +noauth txt.example2. txt @$f1 > dig.out.$n.f1 || ret=1
+digcomp dig.out.$n.root dig.out.$n.f1 || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-echo_i "checking that a forward only zone no forwarders fails"
+n=$((n+1))
+echo_i "checking that a forward only zone no forwarders fails ($n)"
 ret=0
-$DIG $DIGOPTS +noadd +noauth txt.example2. txt @$root > dig.out.root || ret=1
-$DIG $DIGOPTS +noadd +noauth txt.example2. txt @$f1 > dig.out.f1 || ret=1
-digcomp dig.out.root dig.out.f1 || ret=1
+dig_with_opts +noadd +noauth txt.example2. txt @$root > dig.out.$n.root || ret=1
+dig_with_opts +noadd +noauth txt.example2. txt @$f1 > dig.out.$n.f1 || ret=1
+digcomp dig.out.$n.root dig.out.$n.f1 || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-echo_i "checking that global forwarders work"
+n=$((n+1))
+echo_i "checking that global forwarders work ($n)"
 ret=0
-$DIG $DIGOPTS +noadd +noauth txt.example4. txt @$hidden > dig.out.hidden || ret=1
-$DIG $DIGOPTS +noadd +noauth txt.example4. txt @$f1 > dig.out.f1 || ret=1
-digcomp dig.out.hidden dig.out.f1 || ret=1
+dig_with_opts +noadd +noauth txt.example4. txt @$hidden > dig.out.$n.hidden || ret=1
+dig_with_opts +noadd +noauth txt.example4. txt @$f1 > dig.out.$n.f1 || ret=1
+digcomp dig.out.$n.hidden dig.out.$n.f1 || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-echo_i "checking that a forward zone works"
+n=$((n+1))
+echo_i "checking that a forward zone works ($n)"
 ret=0
-$DIG $DIGOPTS +noadd +noauth txt.example1. txt @$hidden > dig.out.hidden || ret=1
-$DIG $DIGOPTS +noadd +noauth txt.example1. txt @$f2 > dig.out.f2 || ret=1
-digcomp dig.out.hidden dig.out.f2 || ret=1
+dig_with_opts +noadd +noauth txt.example1. txt @$hidden > dig.out.$n.hidden || ret=1
+dig_with_opts +noadd +noauth txt.example1. txt @$f2 > dig.out.$n.f2 || ret=1
+digcomp dig.out.$n.hidden dig.out.$n.f2 || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-echo_i "checking that forwarding doesn't spontaneously happen"
+n=$((n+1))
+echo_i "checking that forwarding doesn't spontaneously happen ($n)"
 ret=0
-$DIG $DIGOPTS +noadd +noauth txt.example2. txt @$root > dig.out.root || ret=1
-$DIG $DIGOPTS +noadd +noauth txt.example2. txt @$f2 > dig.out.f2 || ret=1
-digcomp dig.out.root dig.out.f2 || ret=1
+dig_with_opts +noadd +noauth txt.example2. txt @$root > dig.out.$n.root || ret=1
+dig_with_opts +noadd +noauth txt.example2. txt @$f2 > dig.out.$n.f2 || ret=1
+digcomp dig.out.$n.root dig.out.$n.f2 || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-echo_i "checking that a forward zone with no specified policy works"
+n=$((n+1))
+echo_i "checking that a forward zone with no specified policy works ($n)"
 ret=0
-$DIG $DIGOPTS +noadd +noauth txt.example3. txt @$hidden > dig.out.hidden || ret=1
-$DIG $DIGOPTS +noadd +noauth txt.example3. txt @$f2 > dig.out.f2 || ret=1
-digcomp dig.out.hidden dig.out.f2 || ret=1
+dig_with_opts +noadd +noauth txt.example3. txt @$hidden > dig.out.$n.hidden || ret=1
+dig_with_opts +noadd +noauth txt.example3. txt @$f2 > dig.out.$n.f2 || ret=1
+digcomp dig.out.$n.hidden dig.out.$n.f2 || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-echo_i "checking that a forward only doesn't recurse"
+n=$((n+1))
+echo_i "checking that a forward only doesn't recurse ($n)"
 ret=0
-$DIG $DIGOPTS txt.example5. txt @$f2 > dig.out.f2 || ret=1
-grep "SERVFAIL" dig.out.f2 > /dev/null || ret=1
+dig_with_opts txt.example5. txt @$f2 > dig.out.$n.f2 || ret=1
+grep "SERVFAIL" dig.out.$n.f2 > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-echo_i "checking for negative caching of forwarder response"
+n=$((n+1))
+echo_i "checking for negative caching of forwarder response ($n)"
 # prime the cache, shutdown the forwarder then check that we can
 # get the answer from the cache.  restart forwarder.
 ret=0
-$DIG $DIGOPTS nonexist. txt @10.53.0.5 > dig.out.f2 || ret=1
-grep "status: NXDOMAIN" dig.out.f2 > /dev/null || ret=1
+dig_with_opts nonexist. txt @10.53.0.5 > dig.out.$n.f2 || ret=1
+grep "status: NXDOMAIN" dig.out.$n.f2 > /dev/null || ret=1
 $PERL ../stop.pl forward ns4 || ret=1
-$DIG $DIGOPTS nonexist. txt @10.53.0.5 > dig.out.f2 || ret=1
-grep "status: NXDOMAIN" dig.out.f2 > /dev/null || ret=1
-$PERL ../start.pl --restart --noclean --port ${PORT} forward ns4 || ret=1
+dig_with_opts nonexist. txt @10.53.0.5 > dig.out.$n.f2 || ret=1
+grep "status: NXDOMAIN" dig.out.$n.f2 > /dev/null || ret=1
+$PERL ../start.pl --restart --noclean --port "${PORT}" forward ns4 || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-echo_i "checking that forward only zone overrides empty zone"
+check_override() (
+    dig_with_opts 1.0.10.in-addr.arpa TXT @10.53.0.4 > dig.out.$n.f2 &&
+    grep "status: NOERROR" dig.out.$n.f2 > /dev/null &&
+    dig_with_opts 2.0.10.in-addr.arpa TXT @10.53.0.4 > dig.out.$n.f2 &&
+    grep "status: NXDOMAIN" dig.out.$n.f2 > /dev/null
+)
+
+n=$((n+1))
+echo_i "checking that forward only zone overrides empty zone ($n)"
 ret=0
 # retry loop in case the server restart above causes transient failure
-for try in 0 1 2 3 4 5 6 7 8 9; do
-    $DIG $DIGOPTS 1.0.10.in-addr.arpa TXT @10.53.0.4 > dig.out.f2
-    grep "status: NOERROR" dig.out.f2 > /dev/null || ret=1
-    $DIG $DIGOPTS 2.0.10.in-addr.arpa TXT @10.53.0.4 > dig.out.f2
-    grep "status: NXDOMAIN" dig.out.f2 > /dev/null || ret=1
-    [ "$ret" -eq 0 ] && break
-    sleep 1
-done
+retry_quiet 10 check_override || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-echo_i "checking that DS lookups for grafting forward zones are isolated"
+n=$((n+1))
+echo_i "checking that DS lookups for grafting forward zones are isolated ($n)"
 ret=0
-$DIG $DIGOPTS grafted A @10.53.0.4 > dig.out.q1
-$DIG $DIGOPTS grafted DS @10.53.0.4 > dig.out.q2
-$DIG $DIGOPTS grafted A @10.53.0.4 > dig.out.q3
-$DIG $DIGOPTS grafted AAAA @10.53.0.4 > dig.out.q4
-grep "status: NOERROR" dig.out.q1 > /dev/null || ret=1
-grep "status: NXDOMAIN" dig.out.q2 > /dev/null || ret=1
-grep "status: NOERROR" dig.out.q3 > /dev/null || ret=1
-grep "status: NOERROR" dig.out.q4 > /dev/null || ret=1
+dig_with_opts grafted A @10.53.0.4 > dig.out.$n.q1 || ret=1
+dig_with_opts grafted DS @10.53.0.4 > dig.out.$n.q2 || ret=1
+dig_with_opts grafted A @10.53.0.4 > dig.out.$n.q3 || ret=1
+dig_with_opts grafted AAAA @10.53.0.4 > dig.out.$n.q4 || ret=1
+grep "status: NOERROR" dig.out.$n.q1 > /dev/null || ret=1
+grep "status: NXDOMAIN" dig.out.$n.q2 > /dev/null || ret=1
+grep "status: NOERROR" dig.out.$n.q3 > /dev/null || ret=1
+grep "status: NOERROR" dig.out.$n.q4 > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-echo_i "checking that rfc1918 inherited 'forward first;' zones are warned about"
+n=$((n+1))
+echo_i "checking that rfc1918 inherited 'forward first;' zones are warned about ($n)"
 ret=0
 $CHECKCONF rfc1918-inherited.conf | grep "forward first;" >/dev/null || ret=1
 $CHECKCONF rfc1918-notinherited.conf | grep "forward first;" >/dev/null && ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-echo_i "checking that ULA inherited 'forward first;' zones are warned about"
+n=$((n+1))
+echo_i "checking that ULA inherited 'forward first;' zones are warned about ($n)"
 ret=0
 $CHECKCONF ula-inherited.conf | grep "forward first;" >/dev/null || ret=1
 $CHECKCONF ula-notinherited.conf | grep "forward first;" >/dev/null && ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-echo_i "checking that a forwarder timeout prevents it from being reused in the same fetch context"
+count_sent() (
+	logfile="$1"
+	start_pattern="$2"
+	pattern="$3"
+	nextpartpeek "$logfile" | sed -n "/$start_pattern/,/^\$/p" | grep -c "$pattern"
+)
+
+check_sent() (
+	expected="$1"
+	shift
+	count=$(count_sent "$@")
+	[ "$expected" = "$count" ]
+)
+
+wait_for_log() (
+	nextpartpeek "$1" | grep "$2" >/dev/null
+
+)
+
+n=$((n+1))
+echo_i "checking that a forwarder timeout prevents it from being reused in the same fetch context ($n)"
 ret=0
 # Make ans6 receive queries without responding to them.
-echo "//" | $SENDCMD
+echo "//" | sendcmd
 # Query for a record in a zone which is forwarded to a non-responding forwarder
 # and is delegated from the root to check whether the forwarder will be retried
 # when a delegation is encountered after falling back to full recursive
 # resolution.
-$DIG $DIGOPTS txt.example7. txt @$f1 > dig.out.f1 || ret=1
+nextpart ns3/named.run >/dev/null
+dig_with_opts txt.example7. txt @$f1 > dig.out.$n.f1 || ret=1
 # The forwarder for the "example7" zone should only be queried once.
-sent=`tr -d '\r' < ns3/named.run | sed -n '/sending packet to 10.53.0.6/,/^$/p' | grep ";txt.example7.*IN.*TXT" | wc -l`
-if [ $sent -ne 1 ]; then ret=1; fi
+start_pattern="sending packet to 10\.53\.0\.6"
+retry_quiet 5 wait_for_log ns3/named.run "$start_pattern"
+check_sent 1 ns3/named.run "$start_pattern" ";txt\.example7\.[[:space:]]*IN[[:space:]]*TXT$" || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-echo_i "checking that priming queries are not forwarded"
+n=$((n+1))
+echo_i "checking that priming queries are not forwarded ($n)"
 ret=0
-$DIG $DIGOPTS +noadd +noauth txt.example1. txt @10.53.0.7 > dig.out.f7 || ret=1
-check_priming_queries () {
-	sent=`tr -d '\r' < ns7/named.run | sed -n '/sending packet to 10.53.0.1/,/^$/p' | grep ";.*IN.*NS" | wc -l`
-	[ $sent -eq 1 ] || return 1
-	sent=`grep "10.53.0.7#.* (.): query '\./NS/IN' approved" ns4/named.run | wc -l`
-	[ $sent -eq 0 ] || return 1
-	sent=`grep "10.53.0.7#.* (.): query '\./NS/IN' approved" ns1/named.run | wc -l`
-	[ $sent -eq 1 ] || return 1
-}
-retry_quiet 20 check_priming_queries || ret=1
+nextpart ns7/named.run >/dev/null
+dig_with_opts +noadd +noauth txt.example1. txt @10.53.0.7 > dig.out.$n.f7 || ret=1
+start_pattern="sending packet to 10\.53\.0\.1"
+retry_quiet 5 wait_for_log ns7/named.run "$start_pattern" || ret=1
+check_sent 1 ns7/named.run "$start_pattern" ";\.[[:space:]]*IN[[:space:]]*NS$" || ret=1
+sent=$(grep -c "10.53.0.7#.* (.): query '\./NS/IN' approved" ns4/named.run)
+[ "$sent" -eq 0 ] || ret=1
+sent=$(grep -c "10.53.0.7#.* (.): query '\./NS/IN' approved" ns1/named.run)
+[ "$sent" -eq 1 ] || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-echo_i "checking recovery from forwarding to a non-recursive server"
+n=$((n+1))
+echo_i "checking recovery from forwarding to a non-recursive server ($n)"
 ret=0
-$DIG $DIGOPTS xxx.sld.tld txt @10.53.0.8  > dig.out.f8
-grep "status: NOERROR" dig.out.f8 > /dev/null || ret=1
+dig_with_opts xxx.sld.tld txt @10.53.0.8  > dig.out.$n.f8 || ret=1
+grep "status: NOERROR" dig.out.$n.f8 > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
 echo_i "exit status: $status"
 [ $status -eq 0 ] || exit 1

bin/tests/system/kasp/tests.sh

@@ -240,13 +240,13 @@ key_states() {
 # KEY_FILE="${BASE_FILE}.key"
 # PRIVATE_FILE="${BASE_FILE}.private"
 # STATE_FILE="${BASE_FILE}.state"
-# KEY_ID=$(echo $1 | sed 's/^0*//')
+# KEY_ID=$(echo $1 | sed 's/^0\{0,4\}//')
 check_key() {
 	_dir="$DIR"
 	_zone="$ZONE"
 	_role=$(key_get "$1" ROLE)
 	_key_idpad="$2"
-	_key_id=$(echo "$_key_idpad" | sed 's/^0*//')
+	_key_id=$(echo "$_key_idpad" | sed 's/^0\{0,4\}//')
 	_alg_num=$(key_get "$1" ALG_NUM)
 	_alg_numpad=$(printf "%03d" "$_alg_num")
 	_alg_string=$(key_get "$1" ALG_STR)
@@ -288,7 +288,7 @@ check_key() {
 	PRIVATE_FILE="${BASE_FILE}.private"
 	STATE_FILE="${BASE_FILE}.state"
 	KEY_ID="${_key_id}"
-	
+
 	test $_log -eq 1 && echo_i "check key $BASE_FILE"
 
 	# Check the public key file.
@@ -409,12 +409,12 @@ check_key() {
 # KEY_FILE="${BASE_FILE}.key"
 # PRIVATE_FILE="${BASE_FILE}.private"
 # STATE_FILE="${BASE_FILE}.state"
-# KEY_ID=$(echo $1 | sed 's/^0*//')
+# KEY_ID=$(echo $1 | sed 's/^0\{0,4\}//')
 key_unused() {
 	_dir=$DIR
 	_zone=$ZONE
 	_key_idpad=$1
-	_key_id=$(echo "$_key_idpad" | sed 's/^0*//')
+	_key_id=$(echo "$_key_idpad" | sed 's/^0\{0,4\}//')
 	_alg_num=$(key_get KEY1 ALG_NUM)
         _alg_numpad=$(printf "%03d" "$_alg_num")
 

bin/tests/system/mkeys/tests.sh

@@ -27,20 +27,6 @@ rndccmd() (
 	"$RNDC" -c "$SYSTEMTESTTOP/common/rndc.conf" -p "${CONTROLPORT}" -s "$@"
 )
 
-search_log() (
-	msg=$1
-	file=$2
-	nextpart "$file" | grep -F "$msg" > /dev/null
-)
-
-wait_for_log() (
-	msg="$1"
-	file="$2"
-	retry_quiet 20 search_log "$msg" "$file" && return 0
-	echo_i "exceeded time limit waiting for '$msg' in $file"
-	return 1
-)
-
 mkeys_reconfig_on() (
 	nsidx=$1
 	rndccmd "10.53.0.${nsidx}" reconfig . | sed "s/^/ns${nsidx} /" | cat_i
@@ -50,21 +36,21 @@ mkeys_reload_on() (
 	nsidx=$1
 	nextpart "ns${nsidx}"/named.run > /dev/null
 	rndc_reload "ns${nsidx}" "10.53.0.${nsidx}"
-	wait_for_log "loaded serial" "ns${nsidx}"/named.run
+	wait_for_log 20 "loaded serial" "ns${nsidx}"/named.run || return 1
 )
 
 mkeys_loadkeys_on() (
 	nsidx=$1
 	nextpart "ns${nsidx}"/named.run > /dev/null
 	rndccmd "10.53.0.${nsidx}" loadkeys . | sed "s/^/ns${nsidx} /" | cat_i
-	wait_for_log "next key event" "ns${nsidx}"/named.run
+	wait_for_log 20 "next key event" "ns${nsidx}"/named.run || return 1
 )
 
 mkeys_refresh_on() (
 	nsidx=$1
 	nextpart "ns${nsidx}"/named.run > /dev/null
 	rndccmd "10.53.0.${nsidx}" managed-keys refresh | sed "s/^/ns${nsidx} /" | cat_i
-	wait_for_log "Returned from key fetch in keyfetch_done()" "ns${nsidx}"/named.run
+	wait_for_log 20 "Returned from key fetch in keyfetch_done()" "ns${nsidx}"/named.run || return 1
 )
 
 mkeys_sync_on() (
@@ -75,7 +61,7 @@ mkeys_sync_on() (
 	nsidx=$1
 	nextpart "ns${nsidx}"/named.run > /dev/null
 	rndccmd "10.53.0.${nsidx}" managed-keys sync | sed "s/^/ns${nsidx} /" | cat_i
-	wait_for_log "dump_done" "ns${nsidx}"/named.run
+	wait_for_log 20 "dump_done" "ns${nsidx}"/named.run || return 1
 )
 
 mkeys_status_on() (
@@ -322,7 +308,7 @@ $PERL $SYSTEMTESTTOP/start.pl --noclean --restart --port "${PORT}" mkeys ns2
 n=$((n+1))
 echo_i "check that no key from bind.keys is marked as an initializing key ($n)"
 ret=0
-wait_for_log "Returned from key fetch in keyfetch_done()" ns2/named.run || ret=1
+wait_for_log 20 "Returned from key fetch in keyfetch_done()" ns2/named.run || ret=1
 mkeys_secroots_on 2 || ret=1
 grep '; initializing' ns2/named.secroots > /dev/null 2>&1 && ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
@@ -338,7 +324,7 @@ $PERL $SYSTEMTESTTOP/start.pl --noclean --restart --port "${PORT}" mkeys ns2
 n=$((n+1))
 echo_i "check that standby key is now trusted ($n)"
 ret=0
-wait_for_log "Returned from key fetch in keyfetch_done()" ns2/named.run || ret=1
+wait_for_log 20 "Returned from key fetch in keyfetch_done()" ns2/named.run || ret=1
 mkeys_status_on 2 > rndc.out.$n 2>&1 || ret=1
 # two keys listed
 count=$(grep -c "keyid: " rndc.out.$n) || true
@@ -495,7 +481,7 @@ $PERL $SYSTEMTESTTOP/start.pl --noclean --restart --port "${PORT}" mkeys ns2
 n=$((n+1))
 echo_i "check positive validation ($n)"
 ret=0
-wait_for_log "Returned from key fetch in keyfetch_done()" ns2/named.run || ret=1
+wait_for_log 20 "Returned from key fetch in keyfetch_done()" ns2/named.run || ret=1
 dig_with_opts +noauth example. @10.53.0.2 txt > dig.out.ns2.test$n || ret=1
 grep "flags:.*ad.*QUERY" dig.out.ns2.test$n > /dev/null || ret=1
 grep "example..*.RRSIG..*TXT" dig.out.ns2.test$n > /dev/null || ret=1
@@ -591,7 +577,7 @@ rm -f ns1/root.db.signed.jnl
 cp ns1/root.db ns1/root.db.signed
 nextpart ns1/named.run > /dev/null
 $PERL $SYSTEMTESTTOP/start.pl --noclean --restart --port "${PORT}" mkeys ns1
-wait_for_log "all zones loaded" ns1/named.run || ret=1
+wait_for_log 20 "all zones loaded" ns1/named.run || ret=1
 mkeys_refresh_on 2 || ret=1
 mkeys_status_on 2 > rndc.out.2.$n 2>&1 || ret=1
 # one key listed
@@ -625,7 +611,7 @@ rm -f ns1/root.db.signed.jnl
 cat ns1/K*.key >> ns1/root.db.signed
 nextpart ns1/named.run > /dev/null
 $PERL $SYSTEMTESTTOP/start.pl --noclean --restart --port "${PORT}" mkeys ns1
-wait_for_log "all zones loaded" ns1/named.run || ret=1
+wait_for_log 20 "all zones loaded" ns1/named.run || ret=1
 # Less than a second may have passed since the last time ns2 received a
 # ./DNSKEY response from ns1.  Ensure keys are refreshed at a different
 # timestamp to prevent minimal update from resetting it to the same timestamp.
@@ -725,7 +711,7 @@ ret=0
 $PERL $SYSTEMTESTTOP/stop.pl --use-rndc --port "${CONTROLPORT}" mkeys ns5
 nextpart ns5/named.run > /dev/null
 $PERL $SYSTEMTESTTOP/start.pl --noclean --restart --port "${PORT}" mkeys ns5
-wait_for_log "Returned from key fetch in keyfetch_done()" ns5/named.run || ret=1
+wait_for_log 20 "Returned from key fetch in keyfetch_done()" ns5/named.run || ret=1
 # ns5/named.run will contain logs from both the old instance and the new
 # instance.  In order for the test to pass, both must attempt a fetch.
 count=$(grep -c "Creating key fetch" ns5/named.run) || true
@@ -744,7 +730,7 @@ rm -f ns5/managed-keys.bind*
 cp ns5/named2.args ns5/named.args
 nextpart ns5/named.run > /dev/null
 $PERL $SYSTEMTESTTOP/start.pl --noclean --restart --port "${PORT}" mkeys ns5
-wait_for_log "Returned from key fetch in keyfetch_done() for '.': failure" ns5/named.run || ret=1
+wait_for_log 20 "Returned from key fetch in keyfetch_done() for '.': failure" ns5/named.run || ret=1
 mkeys_secroots_on 5 || ret=1
 grep '; initializing managed' ns5/named.secroots > /dev/null 2>&1 || ret=1
 # ns1 should still REFUSE queries from ns5, so resolving should be impossible
@@ -757,7 +743,7 @@ copy_setports ns1/named3.conf.in ns1/named.conf
 rm -f ns1/root.db.signed.jnl
 nextpart ns5/named.run > /dev/null
 mkeys_reconfig_on 1 || ret=1
-wait_for_log "Returned from key fetch in keyfetch_done() for '.': success" ns5/named.run || ret=1
+wait_for_log 20 "Returned from key fetch in keyfetch_done() for '.': success" ns5/named.run || ret=1
 mkeys_secroots_on 5 || ret=1
 grep '; managed' ns5/named.secroots > /dev/null || ret=1
 # ns1 should not longer REFUSE queries from ns5, so managed keys should be
@@ -777,7 +763,7 @@ rm -f ns6/managed-keys.bind*
 nextpart ns6/named.run > /dev/null
 $PERL $SYSTEMTESTTOP/start.pl --noclean --restart --port "${PORT}" mkeys ns6
 # log when an unsupported algorithm is encountered during startup
-wait_for_log "ignoring initial-key for 'unsupported.': algorithm is unsupported" ns6/named.run || ret=1
+wait_for_log 20 "ignoring initial-key for 'unsupported.': algorithm is unsupported" ns6/named.run || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=$((status+ret))
 
@@ -819,7 +805,7 @@ count=$(grep -c "keyid: " rndc.out.$n) || true
 count=$(grep -c "trust" rndc.out.$n) || true
 [ "$count" -eq 2 ] || ret=1
 # log when an unsupported algorithm is encountered during rollover
-wait_for_log "Cannot compute tag for key in zone .: algorithm is unsupported" ns6/named.run || ret=1
+wait_for_log 20 "Cannot compute tag for key in zone .: algorithm is unsupported" ns6/named.run || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=$((status+ret))
 

bin/tests/system/start.pl

@@ -230,6 +230,9 @@ sub construct_ns_command {
 		}
 
 		$command .= "$NAMED -m none -M external ";
+	} elsif ($ENV{'USE_RR'}) {
+		$ENV{'_RR_TRACE_DIR'} = $testdir . "/" . $server . "/";
+		$command = "rr record -h $NAMED -n 4 ";
 	} else {
 		$command = "$NAMED ";
 	}

bin/tests/system/statschannel/tests.sh

@@ -76,25 +76,13 @@ getzones() {
     return $result
 }
 
-# TODO: Move wait_for_log and loadkeys_on to conf.sh.common
-wait_for_log() {
-    msg=$1
-    file=$2
-
-    for i in 1 2 3 4 5 6 7 8 9 10; do
-        nextpart "$file" | grep "$msg" > /dev/null && return
-        sleep 1
-    done
-    echo_i "exceeded time limit waiting for '$msg' in $file"
-    ret=1
-}
-
+# TODO: Move loadkeys_on to conf.sh.common
 loadkeys_on() {
     nsidx=$1
     zone=$2
     nextpart ns${nsidx}/named.run > /dev/null
     $RNDCCMD 10.53.0.${nsidx} loadkeys ${zone} | sed "s/^/ns${nsidx} /" | cat_i
-    wait_for_log "next key event" ns${nsidx}/named.run
+    wait_for_log 20 "next key event" ns${nsidx}/named.run
 }
 
 status=0

bin/tests/system/tcp/tests.sh

@@ -78,22 +78,11 @@ refresh_tcp_stats() {
 	TCP_HIGH="$(sed -n "s/^TCP high-water: \([0-9][0-9]*\)/\1/p" rndc.out.$n)"
 }
 
-wait_for_log() {
-	msg=$1
-	file=$2
-	for _ in 1 2 3 4 5 6 7 8 9 10; do
-		nextpartpeek "$file" | grep "$msg" > /dev/null && return
-		sleep 1
-	done
-	echo_i "exceeded time limit waiting for '$msg' in $file"
-	ret=1
-}
-
 # Send a command to the tool script listening on 10.53.0.6.
 send_command() {
 	nextpart ans6/ans.run > /dev/null
 	echo "$*" | "${PERL}" "${SYSTEMTESTTOP}/send.pl" 10.53.0.6 "${CONTROLPORT}"
-	wait_for_log "result=" ans6/ans.run
+	wait_for_log_peek 10 "result=" ans6/ans.run || ret=1
 	if ! nextpartpeek ans6/ans.run | grep -qF "result=OK"; then
 		return 1
 	fi

bin/tests/system/zero/tests.sh

@@ -1,3 +1,5 @@
+#!/bin/sh
+#
 # Copyright (C) Internet Systems Consortium, Inc. ("ISC")
 #
 # This Source Code Form is subject to the terms of the Mozilla Public
@@ -7,84 +9,112 @@
 # See the COPYRIGHT file distributed with this work for additional
 # information regarding copyright ownership.
 
+#shellcheck source=conf.sh
 SYSTEMTESTTOP=..
-. $SYSTEMTESTTOP/conf.sh
+. "$SYSTEMTESTTOP/conf.sh"
 
-DIGOPTS="-p ${PORT}"
+dig_with_opts() {
+	"$DIG" -p "${PORT}" "$@"
+}
+
+wait_for_pid() (
+	for pid in "$@"; do
+		kill -0 "$pid" 2>/dev/null && return 1
+	done
+	return 0
+)
 
 status=0
 n=0
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "check lookups against TTL=0 records ($n)"
 i=0
+ret=0
 passes=10
-$DIG $DIGOPTS @10.53.0.2 axfr example | grep -v "^ds0" |
+dig_with_opts @10.53.0.2 axfr example | grep -v "^ds0" | \
 awk '$2 == "0" { print "-q", $1, $4; print "-q", "zzz"$1, $4;}' > query.list
+
+# add 1/5 second per query
+timeout=$(($(wc -l < query.list) / 5))
 while [ $i -lt $passes ]
 do
-	ret=0
-	$DIG $DIGOPTS @10.53.0.3 -f query.list > dig.out$i.1.test$n &
-	$DIG $DIGOPTS @10.53.0.3 -f query.list > dig.out$i.2.test$n &
-	$DIG $DIGOPTS @10.53.0.3 -f query.list > dig.out$i.3.test$n &
-	$DIG $DIGOPTS @10.53.0.3 -f query.list > dig.out$i.4.test$n &
-	$DIG $DIGOPTS @10.53.0.3 -f query.list > dig.out$i.5.test$n &
-	$DIG $DIGOPTS @10.53.0.3 -f query.list > dig.out$i.6.test$n &
-	wait
-	grep "status: SERVFAIL" dig.out$i.1.test$n > /dev/null && ret=1
-	grep "status: SERVFAIL" dig.out$i.2.test$n > /dev/null && ret=1
-	grep "status: SERVFAIL" dig.out$i.3.test$n > /dev/null && ret=1
-	grep "status: SERVFAIL" dig.out$i.4.test$n > /dev/null && ret=1
-	grep "status: SERVFAIL" dig.out$i.5.test$n > /dev/null && ret=1
-	grep "status: SERVFAIL" dig.out$i.6.test$n > /dev/null && ret=1
+	(dig_with_opts @10.53.0.3 -f query.list > "dig.out$i.1.test$n") & pid1="$!"
+	(dig_with_opts @10.53.0.3 -f query.list > "dig.out$i.2.test$n") & pid2="$!"
+	(dig_with_opts @10.53.0.3 -f query.list > "dig.out$i.3.test$n") & pid3="$!"
+	(dig_with_opts @10.53.0.3 -f query.list > "dig.out$i.4.test$n") & pid4="$!"
+	(dig_with_opts @10.53.0.3 -f query.list > "dig.out$i.5.test$n") & pid5="$!"
+	(dig_with_opts @10.53.0.3 -f query.list > "dig.out$i.6.test$n") & pid6="$!"
+
+	retry_quiet "$timeout" wait_for_pid "$pid1" "$pid2" "$pid3" "$pid4" "$pid5" "$pid6" || ret=1
+	kill -TERM "$pid1" "$pid2" "$pid3" "$pid4" "$pid5" "$pid6" 2>/dev/null
+
+	wait "$pid1" || ret=1
+	wait "$pid2" || ret=1
+	wait "$pid3" || ret=1
+	wait "$pid4" || ret=1
+	wait "$pid5" || ret=1
+	wait "$pid6" || ret=1
+
+	grep "status: SERVFAIL" "dig.out$i.1.test$n" > /dev/null && ret=1
+	grep "status: SERVFAIL" "dig.out$i.2.test$n" > /dev/null && ret=1
+	grep "status: SERVFAIL" "dig.out$i.3.test$n" > /dev/null && ret=1
+	grep "status: SERVFAIL" "dig.out$i.4.test$n" > /dev/null && ret=1
+	grep "status: SERVFAIL" "dig.out$i.5.test$n" > /dev/null && ret=1
+	grep "status: SERVFAIL" "dig.out$i.6.test$n" > /dev/null && ret=1
 	[ $ret = 1 ] && break
-	i=`expr $i + 1`
+	i=$((i+1))
 	echo_i "successfully completed pass $i of $passes"
 done
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+repeat_query() (
+	i=0
+	while [ "$i" -lt "$1" ]; do
+		dig_with_opts +short "@$2" "$3" | tee "dig.out$i.test$n" || return 1
+		i=$((i+1))
+	done
+)
+
+count_unique() (
+	repeat_query "$@" | sort -u | wc -l
+)
+
+n=$((n+1))
 echo_i "check repeated recursive lookups of non recurring TTL=0 responses get new values ($n)"
 ret=0
-count=`(
-$DIG $DIGOPTS +short @10.53.0.3 foo.increment
-$DIG $DIGOPTS +short @10.53.0.3 foo.increment
-$DIG $DIGOPTS +short @10.53.0.3 foo.increment
-$DIG $DIGOPTS +short @10.53.0.3 foo.increment
-$DIG $DIGOPTS +short @10.53.0.3 foo.increment
-$DIG $DIGOPTS +short @10.53.0.3 foo.increment
-$DIG $DIGOPTS +short @10.53.0.3 foo.increment
-) | sort -u | wc -l `
-if [ $count -ne 7 ] ; then echo_i "failed (count=$count)"; ret=1; fi
-status=`expr $status + $ret`
+repeats=9
+count=$(count_unique "$repeats" 10.53.0.3 foo.increment)
+if [ "$count" -ne "$repeats" ] ; then echo_i "failed (count=$count, repeats=$repeats)"; ret=1; fi
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "check lookups against TTL=1 records ($n)"
 i=0
 passes=10
+ret=0
 while [ $i -lt $passes ]
 do
-	ret=0
-	$DIG $DIGOPTS @10.53.0.3 www.one.tld > dig.out$i.1.test$n
-	$DIG $DIGOPTS @10.53.0.3 www.one.tld > dig.out$i.2.test$n
-	$DIG $DIGOPTS @10.53.0.3 www.one.tld > dig.out$i.3.test$n
-	$DIG $DIGOPTS @10.53.0.3 www.one.tld > dig.out$i.4.test$n
-	$DIG $DIGOPTS @10.53.0.3 www.one.tld > dig.out$i.5.test$n
-	$DIG $DIGOPTS @10.53.0.3 www.one.tld > dig.out$i.6.test$n
-	grep "status: SERVFAIL" dig.out$i.1.test$n > /dev/null && ret=1
-	grep "status: SERVFAIL" dig.out$i.2.test$n > /dev/null && ret=1
-	grep "status: SERVFAIL" dig.out$i.3.test$n > /dev/null && ret=1
-	grep "status: SERVFAIL" dig.out$i.4.test$n > /dev/null && ret=1
-	grep "status: SERVFAIL" dig.out$i.5.test$n > /dev/null && ret=1
-	grep "status: SERVFAIL" dig.out$i.6.test$n > /dev/null && ret=1
+	dig_with_opts @10.53.0.3 www.one.tld > "dig.out$i.1.test$n" || ret=1
+	dig_with_opts @10.53.0.3 www.one.tld > "dig.out$i.2.test$n" || ret=1
+	dig_with_opts @10.53.0.3 www.one.tld > "dig.out$i.3.test$n" || ret=1
+	dig_with_opts @10.53.0.3 www.one.tld > "dig.out$i.4.test$n" || ret=1
+	dig_with_opts @10.53.0.3 www.one.tld > "dig.out$i.5.test$n" || ret=1
+	dig_with_opts @10.53.0.3 www.one.tld > "dig.out$i.6.test$n" || ret=1
+	grep "status: SERVFAIL" "dig.out$i.1.test$n" > /dev/null && ret=1
+	grep "status: SERVFAIL" "dig.out$i.2.test$n" > /dev/null && ret=1
+	grep "status: SERVFAIL" "dig.out$i.3.test$n" > /dev/null && ret=1
+	grep "status: SERVFAIL" "dig.out$i.4.test$n" > /dev/null && ret=1
+	grep "status: SERVFAIL" "dig.out$i.5.test$n" > /dev/null && ret=1
+	grep "status: SERVFAIL" "dig.out$i.6.test$n" > /dev/null && ret=1
 	[ $ret = 1 ] && break
-	i=`expr $i + 1`
+	i=$((i+1))
 	echo_i "successfully completed pass $i of $passes"
-	$PERL -e 'select(undef, undef, undef, 0.3);'
+	sleep 1
 done
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
 echo_i "exit status: $status"
-[ $status -eq 0 ] || exit 1
+[ "$status" -eq 0 ] || exit 1

config.h.in

@@ -465,6 +465,12 @@
 /* Define to 1 if you have the `usleep' function. */
 #undef HAVE_USLEEP
 
+/* Define to 1 if you have the `uv_handle_get_data' function. */
+#undef HAVE_UV_HANDLE_GET_DATA
+
+/* Define to 1 if you have the `uv_handle_set_data' function. */
+#undef HAVE_UV_HANDLE_SET_DATA
+
 /* Use zlib library */
 #undef HAVE_ZLIB
 

configure

@@ -15943,6 +15943,21 @@ fi
 CFLAGS="$CFLAGS $LIBUV_CFLAGS"
 LIBS="$LIBS $LIBUV_LIBS"
 
+# Those functions are only provided in newer versions of libuv, we'll be emulating them
+# for now
+for ac_func in uv_handle_get_data uv_handle_set_data
+do :
+  as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh`
+ac_fn_c_check_func "$LINENO" "$ac_func" "$as_ac_var"
+if eval test \"x\$"$as_ac_var"\" = x"yes"; then :
+  cat >>confdefs.h <<_ACEOF
+#define `$as_echo "HAVE_$ac_func" | $as_tr_cpp` 1
+_ACEOF
+
+fi
+done
+
+
 #
 # flockfile is usually provided by pthreads
 #

configure.ac

@@ -663,6 +663,10 @@ AX_SAVE_FLAGS([libuv])
 CFLAGS="$CFLAGS $LIBUV_CFLAGS"
 LIBS="$LIBS $LIBUV_LIBS"
 
+# Those functions are only provided in newer versions of libuv, we'll be emulating them
+# for now
+AC_CHECK_FUNCS([uv_handle_get_data uv_handle_set_data])
+
 #
 # flockfile is usually provided by pthreads
 #

lib/dns/rbtdb.c

@@ -1824,6 +1824,31 @@ new_reference(dns_rbtdb_t *rbtdb, dns_rbtnode_t *node) {
 	}
 }
 
+/*%
+ * The tree lock must be held for the result to be valid.
+ */
+static inline bool
+is_leaf(dns_rbtnode_t *node) {
+	return (node->parent != NULL && node->parent->down == node &&
+		node->left == NULL && node->right == NULL);
+}
+
+static inline void
+send_to_prune_tree(dns_rbtdb_t *rbtdb, dns_rbtnode_t *node) {
+	isc_event_t *ev;
+	dns_db_t *db;
+
+	ev = isc_event_allocate(rbtdb->common.mctx, NULL,
+				DNS_EVENT_RBTPRUNE,
+				prune_tree, node,
+				sizeof(isc_event_t));
+	new_reference(rbtdb, node);
+	db = NULL;
+	attach((dns_db_t *)rbtdb, &db);
+	ev->ev_sender = db;
+	isc_task_send(rbtdb->task, &ev);
+}
+
 /*%
  * Clean up dead nodes.  These are nodes which have no references, and
  * have no data.  They are dead but we could not or chose not to delete
@@ -1844,28 +1869,29 @@ cleanup_dead_nodes(dns_rbtdb_t *rbtdb, int bucketnum) {
 		/*
 		 * Since we're holding a tree write lock, it should be
 		 * impossible for this node to be referenced by others.
+		 *
+		 * decrement_reference may not have tested node->down, as
+		 * the tree_lock was not held, before adding the node to
+		 * deadnodes so we test it here.
 		 */
 		INSIST(isc_refcount_current(&node->references) == 0 &&
 		       node->data == NULL);
 
-		if (node->parent != NULL &&
-		    node->parent->down == node && node->left == NULL &&
-		    node->right == NULL && rbtdb->task != NULL)
-		{
-			isc_event_t *ev;
-			dns_db_t *db;
-
-			ev = isc_event_allocate(rbtdb->common.mctx, NULL,
-						DNS_EVENT_RBTPRUNE,
-						prune_tree, node,
-						sizeof(isc_event_t));
-			new_reference(rbtdb, node);
-			db = NULL;
-			attach((dns_db_t *)rbtdb, &db);
-			ev->ev_sender = db;
-			isc_task_send(rbtdb->task, &ev);
-		} else {
+		if (is_leaf(node) && rbtdb->task != NULL) {
+			send_to_prune_tree(rbtdb, node);
+		} else if (node->down == NULL && node->data == NULL) {
+			/*
+			 * Not a interior node and not needing to be
+			 * reactivated.
+			 */
 			delete_node(rbtdb, node);
+		} else if (node->data == NULL) {
+			/*
+			 * A interior node without data. Leave linked to
+			 * to be cleaned up when node->down becomes NULL.
+			 */
+			ISC_LIST_APPEND(rbtdb->deadnodes[bucketnum],
+					node, deadlink);
 		}
 		node = ISC_LIST_HEAD(rbtdb->deadnodes[bucketnum]);
 		count--;
@@ -1945,6 +1971,7 @@ decrement_reference(dns_rbtdb_t *rbtdb, dns_rbtnode_t *node,
 {
 	isc_result_t result;
 	bool write_locked;
+	bool locked = tlock != isc_rwlocktype_none;
 	rbtdb_nodelock_t *nodelock;
 	int bucket = node->locknum;
 	bool no_reference = true;
@@ -1952,12 +1979,12 @@ decrement_reference(dns_rbtdb_t *rbtdb, dns_rbtnode_t *node,
 
 	nodelock = &rbtdb->node_locks[bucket];
 
-#define KEEP_NODE(n, r) \
-	((n)->data != NULL || (n)->down != NULL || \
+#define KEEP_NODE(n, r, l) \
+	((n)->data != NULL || ((l) && (n)->down != NULL) || \
 	 (n) == (r)->origin_node || (n) == (r)->nsec3_origin_node)
 
 	/* Handle easy and typical case first. */
-	if (!node->dirty && KEEP_NODE(node, rbtdb)) {
+	if (!node->dirty && KEEP_NODE(node, rbtdb, locked)) {
 		if (isc_refcount_decrement(&node->references) == 1) {
 			refs = isc_refcount_decrement(&nodelock->references);
 			INSIST(refs > 0);
@@ -2024,8 +2051,9 @@ decrement_reference(dns_rbtdb_t *rbtdb, dns_rbtnode_t *node,
 	refs = isc_refcount_decrement(&nodelock->references);
 	INSIST(refs > 0);
 
-	if (KEEP_NODE(node, rbtdb))
+	if (KEEP_NODE(node, rbtdb, locked || write_locked)) {
 		goto restore_locks;
+	}
 
 #undef KEEP_NODE
 
@@ -2050,21 +2078,8 @@ decrement_reference(dns_rbtdb_t *rbtdb, dns_rbtnode_t *node,
 		 * it's their responsibility to purge stale leaves (e.g. by
 		 * periodic walk-through).
 		 */
-		if (!pruning && node->parent != NULL &&
-		    node->parent->down == node && node->left == NULL &&
-		    node->right == NULL && rbtdb->task != NULL) {
-			isc_event_t *ev;
-			dns_db_t *db;
-
-			ev = isc_event_allocate(rbtdb->common.mctx, NULL,
-						DNS_EVENT_RBTPRUNE,
-						prune_tree, node,
-						sizeof(isc_event_t));
-			new_reference(rbtdb, node);
-			db = NULL;
-			attach((dns_db_t *)rbtdb, &db);
-			ev->ev_sender = db;
-			isc_task_send(rbtdb->task, &ev);
+		if (!pruning && is_leaf(node) && rbtdb->task != NULL) {
+			send_to_prune_tree(rbtdb, node);
 			no_reference = false;
 		} else {
 			delete_node(rbtdb, node);

lib/isc/netmgr/netmgr-int.h

@@ -46,7 +46,11 @@ typedef struct isc__networker {
 	bool			   paused;
 	bool			   finished;
 	isc_thread_t		   thread;
-	isc_queue_t		   *ievents;     /* incoming async events */
+	isc_queue_t		   *ievents;      /* incoming async events */
+	isc_queue_t		   *ievents_prio; /* priority async events
+						   * used for listening etc.
+						   * can be processed while
+						   * worker is paused */
 	isc_refcount_t		   references;
 	atomic_int_fast64_t	   pktcount;
 	char			   recvbuf[65536];
@@ -103,9 +107,6 @@ struct isc_nmiface {
 };
 
 typedef enum isc__netievent_type {
-	netievent_stop,
-	netievent_udplisten,
-	netievent_udpstoplisten,
 	netievent_udpsend,
 	netievent_udprecv,
 	netievent_tcpconnect,
@@ -113,11 +114,22 @@ typedef enum isc__netievent_type {
 	netievent_tcprecv,
 	netievent_tcpstartread,
 	netievent_tcppauseread,
-	netievent_tcplisten,
-	netievent_tcpstoplisten,
-	netievent_tcpclose,
+	netievent_tcpchildlisten,
+	netievent_tcpchildstop,
 	netievent_closecb,
 	netievent_shutdown,
+	netievent_stop,
+	netievent_udpstop,
+	netievent_tcpstop,
+	netievent_tcpclose,
+	netievent_tcpdnsclose,
+	netievent_prio = 0xff,	/* event type values higher than this
+				 * will be treated as high-priority
+				 * events, which can be processed
+				 * while the netmgr is paused.
+				 */
+	netievent_udplisten,
+	netievent_tcplisten,
 } isc__netievent_type;
 
 /*
@@ -154,11 +166,13 @@ typedef struct isc__nm_uvreq {
 	isc_nmsocket_t *	sock;
 	isc_nmhandle_t *	handle;
 	uv_buf_t		uvbuf;	/* translated isc_region_t, to be
-					   sent or received */
+					 * sent or received */
 	isc_sockaddr_t		local;	/* local address */
 	isc_sockaddr_t		peer;	/* peer address */
 	isc__nm_cb_t		cb;	/* callback */
 	void *			cbarg;	/* callback argument */
+	uv_pipe_t		ipc;	/* used for sending socket
+					 * uv_handles to other threads */
 	union {
 		uv_req_t		req;
 		uv_getaddrinfo_t	getaddrinfo;
@@ -178,12 +192,13 @@ typedef struct isc__netievent__socket {
 } isc__netievent__socket_t;
 
 typedef isc__netievent__socket_t isc__netievent_udplisten_t;
-typedef isc__netievent__socket_t isc__netievent_udpstoplisten_t;
-typedef isc__netievent__socket_t isc__netievent_tcpstoplisten_t;
+typedef isc__netievent__socket_t isc__netievent_udpstop_t;
+typedef isc__netievent__socket_t isc__netievent_tcpstop_t;
+typedef isc__netievent__socket_t isc__netievent_tcpchildstop_t;
 typedef isc__netievent__socket_t isc__netievent_tcpclose_t;
+typedef isc__netievent__socket_t isc__netievent_tcpdnsclose_t;
 typedef isc__netievent__socket_t isc__netievent_startread_t;
 typedef isc__netievent__socket_t isc__netievent_pauseread_t;
-typedef isc__netievent__socket_t isc__netievent_closecb_t;
 
 typedef struct isc__netievent__socket_req {
 	isc__netievent_type	type;
@@ -193,8 +208,18 @@ typedef struct isc__netievent__socket_req {
 
 typedef isc__netievent__socket_req_t isc__netievent_tcpconnect_t;
 typedef isc__netievent__socket_req_t isc__netievent_tcplisten_t;
+typedef isc__netievent__socket_req_t isc__netievent_tcpchildlisten_t;
 typedef isc__netievent__socket_req_t isc__netievent_tcpsend_t;
 
+typedef struct isc__netievent__socket_handle {
+	isc__netievent_type	type;
+	isc_nmsocket_t		*sock;
+	isc_nmhandle_t		*handle;
+} isc__netievent__socket_handle_t;
+
+typedef isc__netievent__socket_handle_t isc__netievent_closecb_t;
+
+
 typedef struct isc__netievent_udpsend {
 	isc__netievent_type	type;
 	isc_nmsocket_t		*sock;
@@ -274,6 +299,7 @@ typedef enum isc_nmsocket_type {
 	isc_nm_udplistener, /* Aggregate of nm_udpsocks */
 	isc_nm_tcpsocket,
 	isc_nm_tcplistener,
+	isc_nm_tcpchildlistener,
 	isc_nm_tcpdnslistener,
 	isc_nm_tcpdnssocket
 } isc_nmsocket_type;
@@ -293,7 +319,7 @@ struct isc_nmsocket {
 	isc_nm_t		*mgr;
 	isc_nmsocket_t		*parent;
 
-	/*
+	/*%
 	 * quota is the TCP client, attached when a TCP connection
 	 * is established. pquota is a non-attached pointer to the
 	 * TCP client quota, stored in listening sockets but only
@@ -303,7 +329,7 @@ struct isc_nmsocket {
 	isc_quota_t		*pquota;
 	bool			overquota;
 
-	/*
+	/*%
 	 * TCP read timeout timer.
 	 */
 	uv_timer_t		timer;
@@ -316,13 +342,18 @@ struct isc_nmsocket {
 	/*% server socket for connections */
 	isc_nmsocket_t		*server;
 
-	/*% children sockets for multi-socket setups */
+	/*% Child sockets for multi-socket setups */
 	isc_nmsocket_t		*children;
 	int			nchildren;
 	isc_nmiface_t		*iface;
 	isc_nmhandle_t		*tcphandle;
 
-	/*% extra data allocated at the end of each isc_nmhandle_t */
+	/*% Used to transfer listening TCP sockets to children */
+	uv_pipe_t		ipc;
+	char			ipc_pipe_name[64];
+	atomic_int_fast32_t	schildren;
+
+	/*% Extra data allocated at the end of each isc_nmhandle_t */
 	size_t			extrahandlesize;
 
 	/*% TCP backlog */
@@ -332,16 +363,17 @@ struct isc_nmsocket {
 	uv_os_sock_t		fd;
 	union uv_any_handle	uv_handle;
 
+	/*% Peer address */
 	isc_sockaddr_t		peer;
 
 	/* Atomic */
-	/*% Number of running (e.g. listening) children sockets */
+	/*% Number of running (e.g. listening) child sockets */
 	atomic_int_fast32_t     rchildren;
 
 	/*%
-	 * Socket if active if it's listening, working, etc., if we're
-	 * closing a socket it doesn't make any sense to e.g. still
-	 * push handles or reqs for reuse
+	 * Socket is active if it's listening, working, etc. If it's
+	 * closing, then it doesn't make a sense, for example, to
+	 * push handles or reqs for reuse.
 	 */
 	atomic_bool        	active;
 	atomic_bool	   	destroying;
@@ -349,7 +381,8 @@ struct isc_nmsocket {
 	/*%
 	 * Socket is closed if it's not active and all the possible
 	 * callbacks were fired, there are no active handles, etc.
-	 * active==false, closed==false means the socket is closing.
+	 * If active==false but closed==false, that means the socket
+	 * is closing.
 	 */
 	atomic_bool	      	closed;
 	atomic_bool	      	listening;
@@ -391,10 +424,19 @@ struct isc_nmsocket {
 	isc_astack_t 		*inactivehandles;
 	isc_astack_t 		*inactivereqs;
 
-	/* Used for active/rchildren during shutdown */
+	/*%
+	 * Used to wait for TCP listening events to complete, and
+	 * for the number of running children to reach zero during
+	 * shutdown.
+	 */
 	isc_mutex_t		lock;
 	isc_condition_t		cond;
 
+	/*%
+	 * Used to pass a result back from TCP listening events.
+	 */
+	isc_result_t		result;
+
 	/*%
 	 * List of active handles.
 	 * ah - current position in 'ah_frees'; this represents the
@@ -421,12 +463,12 @@ struct isc_nmsocket {
 	size_t			*ah_frees;
 	isc_nmhandle_t		**ah_handles;
 
-	/* Buffer for TCPDNS processing, optional */
+	/*% Buffer for TCPDNS processing */
 	size_t			buf_size;
 	size_t			buf_len;
 	unsigned char		*buf;
 
-	/*
+	/*%
 	 * This function will be called with handle->sock
 	 * as the argument whenever a handle's references drop
 	 * to zero, after its reset callback has been called.
@@ -524,13 +566,13 @@ isc__nmsocket_prep_destroy(isc_nmsocket_t *sock);
  */
 
 void
-isc__nm_async_closecb(isc__networker_t *worker, isc__netievent_t *ievent0);
+isc__nm_async_closecb(isc__networker_t *worker, isc__netievent_t *ev0);
 /*%<
  * Issue a 'handle closed' callback on the socket.
  */
 
 void
-isc__nm_async_shutdown(isc__networker_t *worker, isc__netievent_t *ievent0);
+isc__nm_async_shutdown(isc__networker_t *worker, isc__netievent_t *ev0);
 /*%<
  * Walk through all uv handles, get the underlying sockets and issue
  * close on them.
@@ -544,13 +586,12 @@ isc__nm_udp_send(isc_nmhandle_t *handle, isc_region_t *region,
  */
 
 void
-isc__nm_async_udplisten(isc__networker_t *worker, isc__netievent_t *ievent0);
+isc__nm_async_udplisten(isc__networker_t *worker, isc__netievent_t *ev0);
 
 void
-isc__nm_async_udpstoplisten(isc__networker_t *worker,
-			    isc__netievent_t *ievent0);
+isc__nm_async_udpstop(isc__networker_t *worker, isc__netievent_t *ev0);
 void
-isc__nm_async_udpsend(isc__networker_t *worker, isc__netievent_t *ievent0);
+isc__nm_async_udpsend(isc__networker_t *worker, isc__netievent_t *ev0);
 /*%<
  * Callback handlers for asynchronous UDP events (listen, stoplisten, send).
  */
@@ -575,20 +616,23 @@ isc__nm_tcp_shutdown(isc_nmsocket_t *sock);
  */
 
 void
-isc__nm_async_tcpconnect(isc__networker_t *worker, isc__netievent_t *ievent0);
+isc__nm_async_tcpconnect(isc__networker_t *worker, isc__netievent_t *ev0);
 void
-isc__nm_async_tcplisten(isc__networker_t *worker, isc__netievent_t *ievent0);
+isc__nm_async_tcplisten(isc__networker_t *worker, isc__netievent_t *ev0);
 void
-isc__nm_async_tcpstoplisten(isc__networker_t *worker,
-			    isc__netievent_t *ievent0);
+isc__nm_async_tcpchildlisten(isc__networker_t *worker, isc__netievent_t *ev0);
 void
-isc__nm_async_tcpsend(isc__networker_t *worker, isc__netievent_t *ievent0);
+isc__nm_async_tcpstop(isc__networker_t *worker, isc__netievent_t *ev0);
 void
-isc__nm_async_startread(isc__networker_t *worker, isc__netievent_t *ievent0);
+isc__nm_async_tcpchildstop(isc__networker_t *worker, isc__netievent_t *ev0);
 void
-isc__nm_async_pauseread(isc__networker_t *worker, isc__netievent_t *ievent0);
+isc__nm_async_tcpsend(isc__networker_t *worker, isc__netievent_t *ev0);
 void
-isc__nm_async_tcpclose(isc__networker_t *worker, isc__netievent_t *ievent0);
+isc__nm_async_startread(isc__networker_t *worker, isc__netievent_t *ev0);
+void
+isc__nm_async_pauseread(isc__networker_t *worker, isc__netievent_t *ev0);
+void
+isc__nm_async_tcpclose(isc__networker_t *worker, isc__netievent_t *ev0);
 /*%<
  * Callback handlers for asynchronous TCP events (connect, listen,
  * stoplisten, send, read, pause, close).
@@ -607,6 +651,9 @@ isc__nm_tcpdns_close(isc_nmsocket_t *sock);
  * Close a TCPDNS socket.
  */
 
+void
+isc__nm_async_tcpdnsclose(isc__networker_t *worker, isc__netievent_t *ev0);
+
 #define isc__nm_uverr2result(x) \
 	isc___nm_uverr2result(x, true, __FILE__, __LINE__)
 isc_result_t

lib/isc/netmgr/netmgr.c

@@ -29,6 +29,7 @@
 #include <isc/thread.h>
 #include <isc/util.h>
 
+#include "uv-compat.h"
 #include "netmgr-int.h"
 
 /*
@@ -42,6 +43,12 @@
 
 ISC_THREAD_LOCAL int isc__nm_tid_v = ISC_NETMGR_TID_UNKNOWN;
 
+#ifdef WIN32
+#define NAMED_PIPE_PATTERN "\\\\.\\pipe\\named-%d-%u.pipe"
+#else
+#define NAMED_PIPE_PATTERN "/tmp/named-%d-%u.pipe"
+#endif
+
 static void
 nmsocket_maybe_destroy(isc_nmsocket_t *sock);
 static void
@@ -50,6 +57,8 @@ static void *
 nm_thread(void *worker0);
 static void
 async_cb(uv_async_t *handle);
+static void
+process_queue(isc__networker_t *worker, isc_queue_t *queue);
 
 int
 isc_nm_tid() {
@@ -128,6 +137,7 @@ isc_nm_start(isc_mem_t *mctx, uint32_t workers) {
 		isc_condition_init(&worker->cond);
 
 		worker->ievents = isc_queue_new(mgr->mctx, 128);
+		worker->ievents_prio = isc_queue_new(mgr->mctx, 128);
 
 		/*
 		 * We need to do this here and not in nm_thread to avoid a
@@ -175,17 +185,29 @@ nm_destroy(isc_nm_t **mgr0) {
 	UNLOCK(&mgr->lock);
 
 	for (size_t i = 0; i < mgr->nworkers; i++) {
-		/* Empty the async event queue */
-		isc__netievent_t *ievent;
+		isc__networker_t *worker = &mgr->workers[i];
+		isc__netievent_t *ievent = NULL;
+		int r;
+
+		/* Empty the async event queues */
 		while ((ievent = (isc__netievent_t *)
-			isc_queue_dequeue(mgr->workers[i].ievents)) != NULL)
+			isc_queue_dequeue(worker->ievents)) != NULL)
 		{
 			isc_mempool_put(mgr->evpool, ievent);
 		}
-		int r = uv_loop_close(&mgr->workers[i].loop);
+
+		while ((ievent = (isc__netievent_t *)
+			isc_queue_dequeue(worker->ievents_prio)) != NULL)
+		{
+			isc_mempool_put(mgr->evpool, ievent);
+		}
+
+		r = uv_loop_close(&worker->loop);
 		INSIST(r == 0);
-		isc_queue_destroy(mgr->workers[i].ievents);
-		isc_thread_join(mgr->workers[i].thread, NULL);
+
+		isc_queue_destroy(worker->ievents);
+		isc_queue_destroy(worker->ievents_prio);
+		isc_thread_join(worker->thread, NULL);
 	}
 
 	isc_condition_destroy(&mgr->wkstatecond);
@@ -315,10 +337,18 @@ isc_nm_destroy(isc_nm_t **mgr0) {
 	 * Wait for the manager to be dereferenced elsewhere.
 	 */
 	while (isc_refcount_current(&mgr->references) > 1) {
+		/*
+		 * Sometimes libuv gets stuck, pausing and unpausing
+		 * netmgr goes over all events in async queue for all
+		 * the workers, and since it's done only on shutdown it
+		 * doesn't cost us anything.
+		 */
+		isc_nm_pause(mgr);
+		isc_nm_resume(mgr);
 #ifdef WIN32
-			_sleep(1000);
+		_sleep(1000);
 #else
-			usleep(1000000);
+		usleep(1000000);
 #endif
 	}
 
@@ -403,6 +433,9 @@ nm_thread(void *worker0) {
 			UNLOCK(&worker->mgr->lock);
 
 			WAIT(&worker->cond, &worker->lock);
+
+			/* Process priority events */
+			process_queue(worker, worker->ievents_prio);
 		}
 		if (pausing) {
 			uint32_t wp = atomic_fetch_sub_explicit(
@@ -452,7 +485,8 @@ nm_thread(void *worker0) {
 		/*
 		 * Empty the async queue.
 		 */
-		async_cb(&worker->async);
+		process_queue(worker, worker->ievents_prio);
+		process_queue(worker, worker->ievents);
 	}
 
 	LOCK(&worker->mgr->lock);
@@ -472,21 +506,27 @@ nm_thread(void *worker0) {
 static void
 async_cb(uv_async_t *handle) {
 	isc__networker_t *worker = (isc__networker_t *) handle->loop->data;
-	isc__netievent_t *ievent;
+	process_queue(worker, worker->ievents_prio);
+	process_queue(worker, worker->ievents);
+}
+
+static void
+process_queue(isc__networker_t *worker, isc_queue_t *queue) {
+	isc__netievent_t *ievent = NULL;
 
 	while ((ievent = (isc__netievent_t *)
-		isc_queue_dequeue(worker->ievents)) != NULL)
+		isc_queue_dequeue(queue)) != NULL)
 	{
 		switch (ievent->type) {
 		case netievent_stop:
-			uv_stop(handle->loop);
+			uv_stop(&worker->loop);
 			isc_mempool_put(worker->mgr->evpool, ievent);
 			return;
 		case netievent_udplisten:
 			isc__nm_async_udplisten(worker, ievent);
 			break;
-		case netievent_udpstoplisten:
-			isc__nm_async_udpstoplisten(worker, ievent);
+		case netievent_udpstop:
+			isc__nm_async_udpstop(worker, ievent);
 			break;
 		case netievent_udpsend:
 			isc__nm_async_udpsend(worker, ievent);
@@ -497,6 +537,9 @@ async_cb(uv_async_t *handle) {
 		case netievent_tcplisten:
 			isc__nm_async_tcplisten(worker, ievent);
 			break;
+		case netievent_tcpchildlisten:
+			isc__nm_async_tcpchildlisten(worker, ievent);
+			break;
 		case netievent_tcpstartread:
 			isc__nm_async_startread(worker, ievent);
 			break;
@@ -506,12 +549,18 @@ async_cb(uv_async_t *handle) {
 		case netievent_tcpsend:
 			isc__nm_async_tcpsend(worker, ievent);
 			break;
-		case netievent_tcpstoplisten:
-			isc__nm_async_tcpstoplisten(worker, ievent);
+		case netievent_tcpstop:
+			isc__nm_async_tcpstop(worker, ievent);
+			break;
+		case netievent_tcpchildstop:
+			isc__nm_async_tcpchildstop(worker, ievent);
 			break;
 		case netievent_tcpclose:
 			isc__nm_async_tcpclose(worker, ievent);
 			break;
+		case netievent_tcpdnsclose:
+			isc__nm_async_tcpdnsclose(worker, ievent);
+			break;
 		case netievent_closecb:
 			isc__nm_async_closecb(worker, ievent);
 			break;
@@ -544,7 +593,18 @@ isc__nm_put_ievent(isc_nm_t *mgr, void *ievent) {
 
 void
 isc__nm_enqueue_ievent(isc__networker_t *worker, isc__netievent_t *event) {
-	isc_queue_enqueue(worker->ievents, (uintptr_t)event);
+	if (event->type > netievent_prio) {
+		/*
+		 * We need to make sure this signal will be delivered and
+		 * the queue will be processed.
+		 */
+		LOCK(&worker->lock);
+		isc_queue_enqueue(worker->ievents_prio, (uintptr_t)event);
+		SIGNAL(&worker->cond);
+		UNLOCK(&worker->lock);
+	} else {
+		isc_queue_enqueue(worker->ievents, (uintptr_t)event);
+	}
 	uv_async_send(&worker->async);
 }
 
@@ -626,8 +686,9 @@ nmsocket_cleanup(isc_nmsocket_t *sock, bool dofree) {
 	sock->pquota = NULL;
 
 	if (sock->timer_initialized) {
-		uv_close((uv_handle_t *)&sock->timer, NULL);
 		sock->timer_initialized = false;
+		uv_timer_stop(&sock->timer);
+		uv_close((uv_handle_t *)&sock->timer, NULL);
 	}
 
 	isc_astack_destroy(sock->inactivehandles);
@@ -652,11 +713,9 @@ nmsocket_cleanup(isc_nmsocket_t *sock, bool dofree) {
 
 static void
 nmsocket_maybe_destroy(isc_nmsocket_t *sock) {
-	int active_handles = 0;
+	int active_handles;
 	bool destroy = false;
 
-	REQUIRE(!isc__nmsocket_active(sock));
-
 	if (sock->parent != NULL) {
 		/*
 		 * This is a child socket and cannot be destroyed except
@@ -673,7 +732,14 @@ nmsocket_maybe_destroy(isc_nmsocket_t *sock) {
 	 * accept destruction.
 	 */
 	LOCK(&sock->lock);
-	active_handles += atomic_load(&sock->ah);
+	if (atomic_load(&sock->active) || atomic_load(&sock->destroying) ||
+	    !atomic_load(&sock->closed) || atomic_load(&sock->references) != 0)
+	{
+		UNLOCK(&sock->lock);
+		return;
+	}
+
+	active_handles = atomic_load(&sock->ah);
 	if (sock->children != NULL) {
 		for (int i = 0; i < sock->nchildren; i++) {
 			LOCK(&sock->children[i].lock);
@@ -682,10 +748,7 @@ nmsocket_maybe_destroy(isc_nmsocket_t *sock) {
 		}
 	}
 
-	if (atomic_load(&sock->closed) &&
-	    atomic_load(&sock->references) == 0 &&
-	    (active_handles == 0 || sock->tcphandle != NULL))
-	{
+	if (active_handles == 0 || sock->tcphandle != NULL) {
 		destroy = true;
 	}
 	UNLOCK(&sock->lock);
@@ -790,6 +853,16 @@ isc__nmsocket_init(isc_nmsocket_t *sock, isc_nm_t *mgr,
 		sock->ah_handles[i] = NULL;
 	}
 
+	/*
+	 * Use a random number in the named pipe name. Also add getpid()
+	 * to the name to make sure we don't get a conflict between
+	 * different unit tests running at the same time, where the PRNG
+	 * is initialized to a constant seed.
+	 */
+	snprintf(sock->ipc_pipe_name, sizeof(sock->ipc_pipe_name),
+		 NAMED_PIPE_PATTERN, getpid(), isc_random32());
+	sock->ipc_pipe_name[sizeof(sock->ipc_pipe_name) - 1] = '\0';
+
 	isc_mutex_init(&sock->lock);
 	isc_condition_init(&sock->cond);
 	isc_refcount_init(&sock->references, 1);
@@ -805,7 +878,7 @@ isc__nmsocket_init(isc_nmsocket_t *sock, isc_nm_t *mgr,
 
 void
 isc__nm_alloc_cb(uv_handle_t *handle, size_t size, uv_buf_t *buf) {
-	isc_nmsocket_t *sock = (isc_nmsocket_t *) handle->data;
+	isc_nmsocket_t *sock = uv_handle_get_data(handle);
 	isc__networker_t *worker = NULL;
 
 	REQUIRE(VALID_NMSOCK(sock));
@@ -958,11 +1031,37 @@ nmhandle_free(isc_nmsocket_t *sock, isc_nmhandle_t *handle) {
 	isc_mem_put(sock->mgr->mctx, handle, sizeof(isc_nmhandle_t) + extra);
 }
 
+static void
+nmhandle_deactivate(isc_nmsocket_t *sock, isc_nmhandle_t *handle) {
+	/*
+	 * We do all of this under lock to avoid races with socket
+	 * destruction.  We have to do this now, because at this point the
+	 * socket is either unused or still attached to event->sock.
+	 */
+	LOCK(&sock->lock);
+
+	INSIST(sock->ah_handles[handle->ah_pos] == handle);
+	INSIST(sock->ah_size > handle->ah_pos);
+	INSIST(atomic_load(&sock->ah) > 0);
+
+	sock->ah_handles[handle->ah_pos] = NULL;
+	size_t handlenum = atomic_fetch_sub(&sock->ah, 1) - 1;
+	sock->ah_frees[handlenum] = handle->ah_pos;
+	handle->ah_pos = 0;
+	bool reuse = false;
+	if (atomic_load(&sock->active)) {
+		reuse = isc_astack_trypush(sock->inactivehandles,
+					   handle);
+	}
+	if (!reuse) {
+		nmhandle_free(sock, handle);
+	}
+	UNLOCK(&sock->lock);
+}
+
 void
 isc_nmhandle_unref(isc_nmhandle_t *handle) {
 	isc_nmsocket_t *sock = NULL;
-	size_t handlenum;
-	bool reuse = false;
 	int refs;
 
 	REQUIRE(VALID_NMHANDLE(handle));
@@ -980,61 +1079,34 @@ isc_nmhandle_unref(isc_nmhandle_t *handle) {
 		handle->doreset(handle->opaque);
 	}
 
-	/*
-	 * We do all of this under lock to avoid races with socket
-	 * destruction.
-	 */
-	LOCK(&sock->lock);
-
-	INSIST(sock->ah_handles[handle->ah_pos] == handle);
-	INSIST(sock->ah_size > handle->ah_pos);
-	INSIST(atomic_load(&sock->ah) > 0);
-
-	sock->ah_handles[handle->ah_pos] = NULL;
-	handlenum = atomic_fetch_sub(&sock->ah, 1) - 1;
-	sock->ah_frees[handlenum] = handle->ah_pos;
-	handle->ah_pos = 0;
-
-	if (atomic_load(&sock->active)) {
-		reuse = isc_astack_trypush(sock->inactivehandles,
-					   handle);
-	}
-
-	UNLOCK(&sock->lock);
-
-	if (!reuse) {
-		nmhandle_free(sock, handle);
-	}
-
 	/*
 	 * The handle is closed. If the socket has a callback configured
 	 * for that (e.g., to perform cleanup after request processing),
-	 * call it now.
+	 * call it now, or schedule it to run asynchronously.
 	 */
 	if (sock->closehandle_cb != NULL) {
 		if (sock->tid == isc_nm_tid()) {
 			sock->closehandle_cb(sock);
 		} else {
-			isc__netievent_closecb_t * event =
+			isc__netievent_closecb_t *event =
 				isc__nm_get_ievent(sock->mgr,
 						   netievent_closecb);
 			isc_nmsocket_attach(sock, &event->sock);
+			event->handle = handle;
 			isc__nm_enqueue_ievent(&sock->mgr->workers[sock->tid],
 					       (isc__netievent_t *) event);
+
 			/*
-			 * If we do this asynchronously then the async event
-			 * will clean the socket, so just exit.
+			 * If we're doing this asynchronously, then the
+			 * async event will take care of cleaning up the
+			 * handle and closing the socket.
 			 */
 			return;
 		}
 	}
 
-	if (atomic_load(&sock->ah) == 0 &&
-	    !atomic_load(&sock->active) &&
-	    !atomic_load(&sock->destroying))
-	{
-		nmsocket_maybe_destroy(sock);
-	}
+	nmhandle_deactivate(sock, handle);
+	nmsocket_maybe_destroy(sock);
 }
 
 void *
@@ -1166,9 +1238,9 @@ isc_nm_send(isc_nmhandle_t *handle, isc_region_t *region,
 }
 
 void
-isc__nm_async_closecb(isc__networker_t *worker, isc__netievent_t *ievent0) {
+isc__nm_async_closecb(isc__networker_t *worker, isc__netievent_t *ev0) {
 	isc__netievent_closecb_t *ievent =
-		(isc__netievent_closecb_t *) ievent0;
+		(isc__netievent_closecb_t *) ev0;
 
 	REQUIRE(VALID_NMSOCK(ievent->sock));
 	REQUIRE(ievent->sock->tid == isc_nm_tid());
@@ -1176,6 +1248,8 @@ isc__nm_async_closecb(isc__networker_t *worker, isc__netievent_t *ievent0) {
 
 	UNUSED(worker);
 
+	nmhandle_deactivate(ievent->sock, ievent->handle);
+
 	ievent->sock->closehandle_cb(ievent->sock);
 	isc_nmsocket_detach(&ievent->sock);
 }
@@ -1186,7 +1260,7 @@ shutdown_walk_cb(uv_handle_t *handle, void *arg) {
 
 	switch(handle->type) {
 	case UV_TCP:
-		isc__nm_tcp_shutdown((isc_nmsocket_t *) handle->data);
+		isc__nm_tcp_shutdown(uv_handle_get_data(handle));
 		break;
 	default:
 		break;
@@ -1194,8 +1268,8 @@ shutdown_walk_cb(uv_handle_t *handle, void *arg) {
 }
 
 void
-isc__nm_async_shutdown(isc__networker_t *worker, isc__netievent_t *ievent0) {
-	UNUSED(ievent0);
+isc__nm_async_shutdown(isc__networker_t *worker, isc__netievent_t *ev0) {
+	UNUSED(ev0);
 	uv_walk(&worker->loop, shutdown_walk_cb, NULL);
 }
 

lib/isc/netmgr/tcp.c

@@ -10,6 +10,7 @@
  */
 
 #include <unistd.h>
+#include <libgen.h>
 #include <uv.h>
 
 #include <isc/atomic.h>
@@ -28,6 +29,7 @@
 #include <isc/thread.h>
 #include <isc/util.h>
 
+#include "uv-compat.h"
 #include "netmgr-int.h"
 
 static int
@@ -50,9 +52,24 @@ read_cb(uv_stream_t *stream, ssize_t nread, const uv_buf_t *buf);
 static void
 tcp_close_cb(uv_handle_t *uvhandle);
 
+static void
+ipc_connection_cb(uv_stream_t *stream, int status);
+static void
+ipc_write_cb(uv_write_t *uvreq, int status);
+static void
+ipc_close_cb(uv_handle_t *handle);
+static void
+childlisten_ipc_connect_cb(uv_connect_t *uvreq, int status);
+static void
+childlisten_read_cb(uv_stream_t *stream, ssize_t nread, const uv_buf_t *buf);
+static void
+stoplistening(isc_nmsocket_t *sock);
+static void
+tcp_listenclose_cb(uv_handle_t *handle);
+
 static int
 tcp_connect_direct(isc_nmsocket_t *sock, isc__nm_uvreq_t *req) {
-	isc__networker_t *worker;
+	isc__networker_t *worker = NULL;
 	int r;
 
 	REQUIRE(isc__nm_in_netthread());
@@ -71,16 +88,16 @@ tcp_connect_direct(isc_nmsocket_t *sock, isc__nm_uvreq_t *req) {
 			return (r);
 		}
 	}
-
+	uv_handle_set_data(&sock->uv_handle.handle, sock);
 	r = uv_tcp_connect(&req->uv_req.connect, &sock->uv_handle.tcp,
 			   &req->peer.type.sa, tcp_connect_cb);
 	return (r);
 }
 
 void
-isc__nm_async_tcpconnect(isc__networker_t *worker, isc__netievent_t *ievent0) {
+isc__nm_async_tcpconnect(isc__networker_t *worker, isc__netievent_t *ev0) {
 	isc__netievent_tcpconnect_t *ievent =
-		(isc__netievent_tcpconnect_t *) ievent0;
+		(isc__netievent_tcpconnect_t *) ev0;
 	isc_nmsocket_t *sock = ievent->sock;
 	isc__nm_uvreq_t *req = ievent->req;
 	int r;
@@ -98,7 +115,8 @@ isc__nm_async_tcpconnect(isc__networker_t *worker, isc__netievent_t *ievent0) {
 static void
 tcp_connect_cb(uv_connect_t *uvreq, int status) {
 	isc__nm_uvreq_t *req = (isc__nm_uvreq_t *) uvreq->data;
-	isc_nmsocket_t *sock = uvreq->handle->data;
+	isc_nmsocket_t *sock = NULL;
+	sock = uv_handle_get_data((uv_handle_t *) uvreq->handle);
 
 	REQUIRE(VALID_UVREQ(req));
 
@@ -131,21 +149,26 @@ isc_result_t
 isc_nm_listentcp(isc_nm_t *mgr, isc_nmiface_t *iface,
 		 isc_nm_cb_t cb, void *cbarg,
 		 size_t extrahandlesize, int backlog,
-		 isc_quota_t *quota,
-		 isc_nmsocket_t **sockp)
+		 isc_quota_t *quota, isc_nmsocket_t **sockp)
 {
-	isc__netievent_tcplisten_t *ievent = NULL;
 	isc_nmsocket_t *nsock = NULL;
+	isc__netievent_tcplisten_t *ievent = NULL;
 
 	REQUIRE(VALID_NM(mgr));
 
 	nsock = isc_mem_get(mgr->mctx, sizeof(*nsock));
 	isc__nmsocket_init(nsock, mgr, isc_nm_tcplistener);
 	nsock->iface = iface;
+	nsock->nchildren = mgr->nworkers;
+	atomic_init(&nsock->rchildren, mgr->nworkers);
+	nsock->children = isc_mem_get(mgr->mctx,
+				      mgr->nworkers * sizeof(*nsock));
+	memset(nsock->children, 0, mgr->nworkers * sizeof(*nsock));
 	nsock->rcb.accept = cb;
 	nsock->rcbarg = cbarg;
 	nsock->extrahandlesize = extrahandlesize;
 	nsock->backlog = backlog;
+	nsock->result = ISC_R_SUCCESS;
 	if (quota != NULL) {
 		/*
 		 * We don't attach to quota, just assign - to avoid
@@ -153,82 +176,330 @@ isc_nm_listentcp(isc_nm_t *mgr, isc_nmiface_t *iface,
 		 */
 		nsock->pquota = quota;
 	}
-	nsock->tid = isc_random_uniform(mgr->nworkers);
 
-	/*
-	 * Listening to TCP is rare enough not to care about the
-	 * added overhead from passing this to another thread.
-	 */
 	ievent = isc__nm_get_ievent(mgr, netievent_tcplisten);
 	ievent->sock = nsock;
-	isc__nm_enqueue_ievent(&mgr->workers[nsock->tid],
-			       (isc__netievent_t *) ievent);
-	*sockp = nsock;
+	if (isc__nm_in_netthread()) {
+		nsock->tid = isc_nm_tid();
+		isc__nm_async_tcplisten(&mgr->workers[nsock->tid],
+				       (isc__netievent_t *) ievent);
+		isc__nm_put_ievent(mgr, ievent);
+	} else {
+		nsock->tid = isc_random_uniform(mgr->nworkers);
+		LOCK(&nsock->lock);
+		isc__nm_enqueue_ievent(&mgr->workers[nsock->tid],
+				       (isc__netievent_t *) ievent);
+		WAIT(&nsock->cond, &nsock->lock);
+		UNLOCK(&nsock->lock);
+	}
 
-	return (ISC_R_SUCCESS);
+	if (nsock->result == ISC_R_SUCCESS) {
+		*sockp = nsock;
+		return (ISC_R_SUCCESS);
+	} else {
+		isc_result_t result = nsock->result;
+		isc_nmsocket_detach(&nsock);
+		return (result);
+	}
 }
 
+#ifndef WIN32
+/*
+ * Run fsync() on the directory containing a socket's IPC pipe, to
+ * ensure that all threads can see the pipe.
+ */
+static void
+syncdir(const isc_nmsocket_t *sock) {
+	 char *pipe = isc_mem_strdup(sock->mgr->mctx, sock->ipc_pipe_name);
+	 int fd = open(dirname(pipe), O_RDONLY);
+
+	 RUNTIME_CHECK(fd >= 0);
+
+	 isc_mem_free(sock->mgr->mctx, pipe);
+
+	 fsync(fd);
+	 close(fd);
+}
+#endif
+
+/*
+ * For multi-threaded TCP listening, we create a single "parent" socket,
+ * bind to it, and then pass its uv_handle to a set of child sockets, one
+ * per worker. For thread safety, the passing of the socket's uv_handle has
+ * to be done via IPC socket.
+ *
+ * This design pattern is ugly but it's what's recommended by the libuv
+ * documentation.  (A prior version of libuv had uv_export() and
+ * uv_import() functions which would have simplified this greatly, but
+ * they have been deprecated and removed.)
+ */
 void
-isc__nm_async_tcplisten(isc__networker_t *worker, isc__netievent_t *ievent0) {
+isc__nm_async_tcplisten(isc__networker_t *worker, isc__netievent_t *ev0) {
 	isc__netievent_tcplisten_t *ievent =
-		(isc__netievent_tcplisten_t *) ievent0;
+		(isc__netievent_tcplisten_t *) ev0;
 	isc_nmsocket_t *sock = ievent->sock;
 	int r;
 
 	REQUIRE(isc__nm_in_netthread());
 	REQUIRE(sock->type == isc_nm_tcplistener);
 
-	r = uv_tcp_init(&worker->loop, &sock->uv_handle.tcp);
-	if (r != 0) {
-		return;
+	/* Initialize children now to make cleaning up easier */
+	for (int i = 0; i < sock->nchildren; i++) {
+		isc_nmsocket_t *csock = &sock->children[i];
+
+		isc__nmsocket_init(csock, sock->mgr, isc_nm_tcpchildlistener);
+		csock->parent = sock;
+		csock->iface = sock->iface;
+		csock->tid = i;
+		csock->pquota = sock->pquota;
+		csock->backlog = sock->backlog;
+		csock->extrahandlesize = sock->extrahandlesize;
+
+		INSIST(csock->rcb.recv == NULL && csock->rcbarg == NULL);
+		csock->rcb.accept = sock->rcb.accept;
+		csock->rcbarg = sock->rcbarg;
+		csock->fd = -1;
 	}
 
-	uv_tcp_bind(&sock->uv_handle.tcp, &sock->iface->addr.type.sa, 0);
-	r = uv_listen((uv_stream_t *) &sock->uv_handle.tcp, sock->backlog,
-		      tcp_connection_cb);
+	r = uv_tcp_init(&worker->loop, &sock->uv_handle.tcp);
 	if (r != 0) {
-		return;
+		/* It was never opened */
+		atomic_store(&sock->closed, true);
+		sock->result = isc__nm_uverr2result(r);
+		goto done;
+	}
+
+	r = uv_tcp_bind(&sock->uv_handle.tcp, &sock->iface->addr.type.sa, 0);
+	if (r != 0) {
+		uv_close(&sock->uv_handle.handle, tcp_close_cb);
+		sock->result = isc__nm_uverr2result(r);
+		goto done;
+	}
+	uv_handle_set_data(&sock->uv_handle.handle, sock);
+
+	/*
+	 * uv_pipe_init() is incorrectly documented in libuv, and the
+	 * example in benchmark-multi-accept.c is also wrong.
+	 *
+	 * The third parameter ('ipc') indicates that the pipe will be
+	 * used to *send* a handle to other threads. Therefore, it must
+	 * must be set to 0 for a 'listening' IPC socket, and 1
+	 * only for sockets that are really passing FDs between
+	 * threads.
+	 */
+	r = uv_pipe_init(&worker->loop, &sock->ipc, 0);
+	RUNTIME_CHECK(r == 0);
+
+	uv_handle_set_data((uv_handle_t *)&sock->ipc, sock);
+	r = uv_pipe_bind(&sock->ipc, sock->ipc_pipe_name);
+	RUNTIME_CHECK(r == 0);
+
+	r = uv_listen((uv_stream_t *) &sock->ipc, sock->nchildren,
+		      ipc_connection_cb);
+	RUNTIME_CHECK(r == 0);
+
+#ifndef WIN32
+	/*
+	 * On Unices a child thread might not see the pipe yet;
+	 * that happened quite often in unit tests on FreeBSD.
+	 * Syncing the directory ensures that the pipe is visible
+	 * to everyone.
+	 * This isn't done on Windows because named pipes exist
+	 * within a different namespace, not on VFS.
+	 */
+	syncdir(sock);
+#endif
+
+	/*
+	 * For each worker we send a 'tcpchildlisten' event. The child
+	 * listener will then receive its version of the socket
+	 * uv_handle via IPC in isc__nm_async_tcpchildlisten().
+	 */
+	for (int i = 0; i < sock->nchildren; i++) {
+		isc_nmsocket_t *csock = &sock->children[i];
+		isc__netievent_tcpchildlisten_t *event = NULL;
+
+		event = isc__nm_get_ievent(csock->mgr,
+					   netievent_tcpchildlisten);
+		event->sock = csock;
+		if (csock->tid == isc_nm_tid()) {
+			isc__nm_async_tcpchildlisten(&sock->mgr->workers[i],
+						  (isc__netievent_t *) event);
+			isc__nm_put_ievent(sock->mgr, event);
+		} else {
+			isc__nm_enqueue_ievent(&sock->mgr->workers[i],
+					       (isc__netievent_t *) event);
+		}
 	}
 
 	atomic_store(&sock->listening, true);
 
+ done:
+	LOCK(&sock->lock);
+	SIGNAL(&sock->cond);
+	UNLOCK(&sock->lock);
 	return;
 }
 
+/*
+ * The parent received an IPC connection from a child and can now send
+ * the uv_handle to the child for listening.
+ */
+static void
+ipc_connection_cb(uv_stream_t *stream, int status) {
+	isc_nmsocket_t *sock = uv_handle_get_data((uv_handle_t *) stream);
+	isc__networker_t *worker = &sock->mgr->workers[isc_nm_tid()];
+	isc__nm_uvreq_t *nreq = isc__nm_uvreq_get(sock->mgr, sock);
+	int r;
+
+	REQUIRE(status == 0);
+
+	/*
+	 * The buffer can be anything, it will be ignored, but it has to
+	 * be something that won't disappear.
+	 */
+	nreq->uvbuf = uv_buf_init((char *)nreq, 1);
+	uv_pipe_init(&worker->loop, &nreq->ipc, 1);
+	uv_handle_set_data((uv_handle_t *)&nreq->ipc, nreq);
+
+	r = uv_accept((uv_stream_t *) &sock->ipc, (uv_stream_t *) &nreq->ipc);
+	RUNTIME_CHECK(r == 0);
+
+	r = uv_write2(&nreq->uv_req.write,
+		      (uv_stream_t *) &nreq->ipc, &nreq->uvbuf, 1,
+		      (uv_stream_t *) &sock->uv_handle.stream, ipc_write_cb);
+	RUNTIME_CHECK(r == 0);
+}
+
+/*
+ * The call to send a socket uv_handle is complete; we may be able
+ * to close the IPC connection.
+ */
+static void
+ipc_write_cb(uv_write_t *uvreq, int status) {
+	isc__nm_uvreq_t *req = uvreq->data;
+
+	UNUSED(status);
+
+	/*
+	 * We want all children to get the socket. Once that's done,
+	 * we can stop listening on the IPC socket.
+	 */
+	if (atomic_fetch_add(&req->sock->schildren, 1) ==
+	    req->sock->nchildren - 1)
+	{
+		uv_close((uv_handle_t *) &req->sock->ipc, NULL);
+	}
+	uv_close((uv_handle_t *) &req->ipc, ipc_close_cb);
+}
+
+/*
+ * The IPC socket is closed: free its resources.
+ */
+static void
+ipc_close_cb(uv_handle_t *handle) {
+	isc__nm_uvreq_t *req = uv_handle_get_data(handle);
+	isc__nm_uvreq_put(&req, req->sock);
+}
+
+/*
+ * Connect to the parent socket and be ready to receive the uv_handle
+ * for the socket we'll be listening on.
+ */
+void
+isc__nm_async_tcpchildlisten(isc__networker_t *worker, isc__netievent_t *ev0) {
+	isc__netievent_tcpchildlisten_t *ievent =
+		(isc__netievent_tcpchildlisten_t *) ev0;
+	isc_nmsocket_t *sock = ievent->sock;
+	isc__nm_uvreq_t *req = NULL;
+	int r;
+
+	REQUIRE(isc__nm_in_netthread());
+	REQUIRE(sock->type == isc_nm_tcpchildlistener);
+
+	r = uv_pipe_init(&worker->loop, &sock->ipc, 1);
+	RUNTIME_CHECK(r == 0);
+
+	uv_handle_set_data((uv_handle_t *) &sock->ipc, sock);
+
+	req = isc__nm_uvreq_get(sock->mgr, sock);
+	uv_pipe_connect(&req->uv_req.connect, &sock->ipc,
+			sock->parent->ipc_pipe_name,
+			childlisten_ipc_connect_cb);
+}
+
+/* Child is now connected to parent via IPC and can begin reading. */
+static void
+childlisten_ipc_connect_cb(uv_connect_t *uvreq, int status) {
+	isc__nm_uvreq_t *req = uvreq->data;
+	isc_nmsocket_t *sock = req->sock;
+	int r;
+
+	UNUSED(status);
+
+	isc__nm_uvreq_put(&req, sock);
+
+	r = uv_read_start((uv_stream_t *) &sock->ipc, isc__nm_alloc_cb,
+			  childlisten_read_cb);
+	RUNTIME_CHECK(r == 0);
+}
+
+/*
+ * Child has received the socket uv_handle via IPC, and can now begin
+ * listening for connections and can close the IPC socket.
+ */
+static void
+childlisten_read_cb(uv_stream_t *stream, ssize_t nread, const uv_buf_t *buf) {
+	isc_nmsocket_t *sock = uv_handle_get_data((uv_handle_t *) stream);
+	isc__networker_t *worker = NULL;
+	uv_pipe_t *ipc = NULL;
+	uv_handle_type type;
+	int r;
+
+	UNUSED(nread);
+
+	REQUIRE(VALID_NMSOCK(sock));
+	REQUIRE(buf != NULL);
+
+	ipc = (uv_pipe_t *) stream;
+	type = uv_pipe_pending_type(ipc);
+	INSIST(type == UV_TCP);
+
+	isc__nm_free_uvbuf(sock, buf);
+	worker = &sock->mgr->workers[isc_nm_tid()];
+	uv_tcp_init(&worker->loop, (uv_tcp_t *) &sock->uv_handle.tcp);
+	uv_handle_set_data(&sock->uv_handle.handle, sock);
+
+	uv_accept(stream, &sock->uv_handle.stream);
+	r = uv_listen((uv_stream_t *) &sock->uv_handle.tcp, sock->backlog,
+		      tcp_connection_cb);
+	uv_close((uv_handle_t *) ipc, NULL);
+	if (r != 0) {
+		isc_log_write(isc_lctx, ISC_LOGCATEGORY_GENERAL,
+			      ISC_LOGMODULE_NETMGR, ISC_LOG_ERROR,
+			      "IPC socket close failed: %s",
+			      isc_result_totext(isc__nm_uverr2result(r)));
+		return;
+	}
+}
+
+
 void
 isc_nm_tcp_stoplistening(isc_nmsocket_t *sock) {
-	isc__netievent_tcpstoplisten_t *ievent = NULL;
+	isc__netievent_tcpstop_t *ievent = NULL;
 
 	REQUIRE(VALID_NMSOCK(sock));
 	REQUIRE(!isc__nm_in_netthread());
 
-	ievent = isc__nm_get_ievent(sock->mgr, netievent_tcpstoplisten);
+	ievent = isc__nm_get_ievent(sock->mgr, netievent_tcpstop);
 	isc_nmsocket_attach(sock, &ievent->sock);
 	isc__nm_enqueue_ievent(&sock->mgr->workers[sock->tid],
 			       (isc__netievent_t *) ievent);
 }
 
-static void
-stoplistening_cb(uv_handle_t *handle) {
-	isc_nmsocket_t *sock = handle->data;
-
-	LOCK(&sock->lock);
-	atomic_store(&sock->listening, false);
-	atomic_store(&sock->closed, true);
-	SIGNAL(&sock->cond);
-	UNLOCK(&sock->lock);
-
-	sock->pquota = NULL;
-
-	isc_nmsocket_detach(&sock);
-}
-
 void
-isc__nm_async_tcpstoplisten(isc__networker_t *worker,
-			    isc__netievent_t *ievent0)
-{
-	isc__netievent_tcpstoplisten_t *ievent =
-		(isc__netievent_tcpstoplisten_t *) ievent0;
+isc__nm_async_tcpstop(isc__networker_t *worker, isc__netievent_t *ev0) {
+	isc__netievent_tcpstop_t *ievent = (isc__netievent_tcpstop_t *) ev0;
 	isc_nmsocket_t *sock = ievent->sock;
 
 	UNUSED(worker);
@@ -237,12 +508,103 @@ isc__nm_async_tcpstoplisten(isc__networker_t *worker,
 	REQUIRE(VALID_NMSOCK(sock));
 	REQUIRE(sock->type == isc_nm_tcplistener);
 
-	uv_close(&sock->uv_handle.handle, stoplistening_cb);
+	/*
+	 * If network manager is interlocked, re-enqueue the event for later.
+	 */
+	if (!isc__nm_acquire_interlocked(sock->mgr)) {
+		isc__netievent_tcpstop_t *event = NULL;
+
+		event = isc__nm_get_ievent(sock->mgr,
+					   netievent_tcpstop);
+		event->sock = sock;
+		isc__nm_enqueue_ievent(&sock->mgr->workers[sock->tid],
+				       (isc__netievent_t *) event);
+	} else {
+		stoplistening(sock);
+		isc__nm_drop_interlocked(sock->mgr);
+	}
+}
+
+static void
+stoplistening(isc_nmsocket_t *sock) {
+	for (int i = 0; i < sock->nchildren; i++) {
+		isc__netievent_tcpchildstop_t *event = NULL;
+
+		/*
+		 * We can ignore the overhead of event allocation because
+		 * stoplistening is a rare event, and doing it this way
+		 * simplifies sock reference counting.
+		 */
+		event = isc__nm_get_ievent(sock->mgr, netievent_tcpchildstop);
+		isc_nmsocket_attach(&sock->children[i], &event->sock);
+
+		if (i == sock->tid) {
+			isc__nm_async_tcpchildstop(&sock->mgr->workers[i],
+						   (isc__netievent_t *) event);
+			isc__nm_put_ievent(sock->mgr, event);
+		} else {
+			isc__nm_enqueue_ievent(&sock->mgr->workers[i],
+					       (isc__netievent_t *) event);
+		}
+	}
+
+	LOCK(&sock->lock);
+	while (atomic_load_relaxed(&sock->rchildren) > 0) {
+		WAIT(&sock->cond, &sock->lock);
+	}
+	UNLOCK(&sock->lock);
+	uv_close((uv_handle_t *) &sock->uv_handle.tcp, tcp_listenclose_cb);
+}
+
+void
+isc__nm_async_tcpchildstop(isc__networker_t *worker, isc__netievent_t *ev0) {
+	isc__netievent_tcpchildstop_t *ievent =
+		(isc__netievent_tcpchildstop_t *) ev0;
+	isc_nmsocket_t *sock = ievent->sock;
+
+	UNUSED(worker);
+
+	REQUIRE(isc_nm_tid() == sock->tid);
+	REQUIRE(VALID_NMSOCK(sock));
+	REQUIRE(sock->type == isc_nm_tcpchildlistener);
+	REQUIRE(sock->parent != NULL);
+
+	/*
+	 * rchildren is atomic, but we still need to change it
+	 * under a lock because the parent is waiting on conditional
+	 * and without it we might deadlock.
+	 */
+	LOCK(&sock->parent->lock);
+	atomic_fetch_sub(&sock->parent->rchildren, 1);
+	UNLOCK(&sock->parent->lock);
+
+	uv_close((uv_handle_t *) &sock->uv_handle.tcp, tcp_listenclose_cb);
+	BROADCAST(&sock->parent->cond);
+}
+
+/*
+ * This callback is used for closing both child and parent listening
+ * sockets; that's why we need to choose the proper lock.
+ */
+static void
+tcp_listenclose_cb(uv_handle_t *handle) {
+	isc_nmsocket_t *sock = uv_handle_get_data(handle);
+	isc_mutex_t *lock = ((sock->parent != NULL)
+			     ? &sock->parent->lock
+			     : &sock->lock);
+
+	LOCK(lock);
+	atomic_store(&sock->closed, true);
+	atomic_store(&sock->listening, false);
+	sock->pquota = NULL;
+	UNLOCK(lock);
+
+	isc_nmsocket_detach(&sock);
 }
 
 static void
 readtimeout_cb(uv_timer_t *handle) {
-	isc_nmsocket_t *sock = (isc_nmsocket_t *) handle->data;
+	isc_nmsocket_t *sock = uv_handle_get_data((uv_handle_t *)handle);
 
 	REQUIRE(VALID_NMSOCK(sock));
 	REQUIRE(sock->tid == isc_nm_tid());
@@ -294,16 +656,16 @@ isc_nm_read(isc_nmhandle_t *handle, isc_nm_recv_cb_t cb, void *cbarg) {
 }
 
 void
-isc__nm_async_startread(isc__networker_t *worker, isc__netievent_t *ievent0) {
+isc__nm_async_startread(isc__networker_t *worker, isc__netievent_t *ev0) {
 	isc__netievent_startread_t *ievent =
-		(isc__netievent_startread_t *) ievent0;
+		(isc__netievent_startread_t *) ev0;
 	isc_nmsocket_t *sock = ievent->sock;
 
 	REQUIRE(worker->id == isc_nm_tid());
 	if (sock->read_timeout != 0) {
 		if (!sock->timer_initialized) {
 			uv_timer_init(&worker->loop, &sock->timer);
-			sock->timer.data = sock;
+			uv_handle_set_data((uv_handle_t *)&sock->timer, sock);
 			sock->timer_initialized = true;
 		}
 		uv_timer_start(&sock->timer, readtimeout_cb,
@@ -340,9 +702,9 @@ isc_nm_pauseread(isc_nmsocket_t *sock) {
 }
 
 void
-isc__nm_async_pauseread(isc__networker_t *worker, isc__netievent_t *ievent0) {
+isc__nm_async_pauseread(isc__networker_t *worker, isc__netievent_t *ev0) {
 	isc__netievent_pauseread_t *ievent =
-		(isc__netievent_pauseread_t *) ievent0;
+		(isc__netievent_pauseread_t *) ev0;
 	isc_nmsocket_t *sock = ievent->sock;
 
 	REQUIRE(VALID_NMSOCK(sock));
@@ -384,7 +746,7 @@ isc_nm_resumeread(isc_nmsocket_t *sock) {
 
 static void
 read_cb(uv_stream_t *stream, ssize_t nread, const uv_buf_t *buf) {
-	isc_nmsocket_t *sock = stream->data;
+	isc_nmsocket_t *sock = uv_handle_get_data((uv_handle_t *) stream);
 
 	REQUIRE(VALID_NMSOCK(sock));
 	REQUIRE(buf != NULL);
@@ -394,10 +756,14 @@ read_cb(uv_stream_t *stream, ssize_t nread, const uv_buf_t *buf) {
 			.base = (unsigned char *) buf->base,
 			.length = nread
 		};
-
-		INSIST(sock->rcb.recv != NULL);
-		sock->rcb.recv(sock->tcphandle, &region, sock->rcbarg);
-
+		/*
+		 * This might happen if the inner socket is closing.
+		 * It means that it's detached, so the socket will
+		 * be closed.
+		 */
+		if (sock->rcb.recv != NULL) {
+			sock->rcb.recv(sock->tcphandle, &region, sock->rcbarg);
+		}
 		sock->read_timeout = (atomic_load(&sock->keepalive)
 				      ? sock->mgr->keepalive
 				      : sock->mgr->idle);
@@ -415,8 +781,14 @@ read_cb(uv_stream_t *stream, ssize_t nread, const uv_buf_t *buf) {
 	if (sock->quota) {
 		isc_quota_detach(&sock->quota);
 	}
-	sock->rcb.recv(sock->tcphandle, NULL, sock->rcbarg);
-
+	/*
+	 * This might happen if the inner socket is closing.
+	 * It means that it's detached, so the socket will
+	 * be closed.
+	 */
+	if (sock->rcb.recv != NULL) {
+		sock->rcb.recv(sock->tcphandle, NULL, sock->rcbarg);
+	}
 	/*
 	 * We don't need to clean up now; the socket will be closed and
 	 * resources and quota reclaimed when handle is freed in
@@ -499,7 +871,7 @@ accept_connection(isc_nmsocket_t *ssock) {
 
 static void
 tcp_connection_cb(uv_stream_t *server, int status) {
-	isc_nmsocket_t *ssock = server->data;
+	isc_nmsocket_t *ssock = uv_handle_get_data((uv_handle_t *) server);
 	isc_result_t result;
 
 	UNUSED(status);
@@ -576,9 +948,9 @@ tcp_send_cb(uv_write_t *req, int status) {
  * Handle 'tcpsend' async event - send a packet on the socket
  */
 void
-isc__nm_async_tcpsend(isc__networker_t *worker, isc__netievent_t *ievent0) {
+isc__nm_async_tcpsend(isc__networker_t *worker, isc__netievent_t *ev0) {
 	isc_result_t result;
-	isc__netievent_tcpsend_t *ievent = (isc__netievent_tcpsend_t *) ievent0;
+	isc__netievent_tcpsend_t *ievent = (isc__netievent_tcpsend_t *) ev0;
 
 	REQUIRE(worker->id == ievent->sock->tid);
 
@@ -615,7 +987,7 @@ tcp_send_direct(isc_nmsocket_t *sock, isc__nm_uvreq_t *req) {
 
 static void
 tcp_close_cb(uv_handle_t *uvhandle) {
-	isc_nmsocket_t *sock = uvhandle->data;
+	isc_nmsocket_t *sock = uv_handle_get_data(uvhandle);
 
 	REQUIRE(VALID_NMSOCK(sock));
 
@@ -625,7 +997,7 @@ tcp_close_cb(uv_handle_t *uvhandle) {
 
 static void
 timer_close_cb(uv_handle_t *uvhandle) {
-	isc_nmsocket_t *sock = uvhandle->data;
+	isc_nmsocket_t *sock = uv_handle_get_data(uvhandle);
 
 	REQUIRE(VALID_NMSOCK(sock));
 
@@ -653,8 +1025,9 @@ tcp_close_direct(isc_nmsocket_t *sock) {
 		}
 	}
 	if (sock->timer_initialized) {
-		uv_close((uv_handle_t *)&sock->timer, timer_close_cb);
 		sock->timer_initialized = false;
+		uv_timer_stop(&sock->timer);
+		uv_close((uv_handle_t *)&sock->timer, timer_close_cb);
 	} else {
 		isc_nmsocket_detach(&sock->server);
 		uv_close(&sock->uv_handle.handle, tcp_close_cb);
@@ -682,9 +1055,8 @@ isc__nm_tcp_close(isc_nmsocket_t *sock) {
 }
 
 void
-isc__nm_async_tcpclose(isc__networker_t *worker, isc__netievent_t *ievent0) {
-	isc__netievent_tcpclose_t *ievent =
-		(isc__netievent_tcpclose_t *) ievent0;
+isc__nm_async_tcpclose(isc__networker_t *worker, isc__netievent_t *ev0) {
+	isc__netievent_tcpclose_t *ievent = (isc__netievent_tcpclose_t *) ev0;
 
 	REQUIRE(worker->id == ievent->sock->tid);
 
@@ -695,7 +1067,10 @@ void
 isc__nm_tcp_shutdown(isc_nmsocket_t *sock) {
 	REQUIRE(VALID_NMSOCK(sock));
 
-	if (sock->type == isc_nm_tcpsocket && sock->tcphandle != NULL) {
+	if (sock->type == isc_nm_tcpsocket &&
+	    sock->tcphandle != NULL &&
+	    sock->rcb.recv != NULL)
+	{
 		sock->rcb.recv(sock->tcphandle, NULL, sock->rcbarg);
 	}
 }

lib/isc/netmgr/tcpdns.c

@@ -26,6 +26,7 @@
 #include <isc/thread.h>
 #include <isc/util.h>
 
+#include "uv-compat.h"
 #include "netmgr-int.h"
 
 #define TCPDNS_CLIENTS_PER_CONN 23
@@ -76,21 +77,22 @@ alloc_dnsbuf(isc_nmsocket_t *sock, size_t len) {
 
 static void
 timer_close_cb(uv_handle_t *handle) {
-	isc_nmsocket_t *sock = (isc_nmsocket_t *) handle->data;
+	isc_nmsocket_t *sock = (isc_nmsocket_t *) uv_handle_get_data(handle);
 	INSIST(VALID_NMSOCK(sock));
-	sock->timer_initialized = false;
 	atomic_store(&sock->closed, true);
 	isc_nmsocket_detach(&sock);
 }
 
 static void
 dnstcp_readtimeout(uv_timer_t *timer) {
-	isc_nmsocket_t *sock = (isc_nmsocket_t *) timer->data;
+	isc_nmsocket_t *sock =
+		(isc_nmsocket_t *) uv_handle_get_data((uv_handle_t *) timer);
 
 	REQUIRE(VALID_NMSOCK(sock));
+	REQUIRE(sock->tid == isc_nm_tid());
 
 	isc_nmsocket_detach(&sock->outer);
-	uv_close((uv_handle_t*) &sock->timer, timer_close_cb);
+	uv_close((uv_handle_t *) &sock->timer, timer_close_cb);
 }
 
 /*
@@ -201,6 +203,7 @@ dnslisten_readcb(isc_nmhandle_t *handle, isc_region_t *region, void *arg) {
 
 	REQUIRE(VALID_NMSOCK(dnssock));
 	REQUIRE(VALID_NMHANDLE(handle));
+	REQUIRE(dnssock->tid == isc_nm_tid());
 
 	if (region == NULL) {
 		/* Connection closed */
@@ -296,11 +299,15 @@ isc_nm_listentcpdns(isc_nm_t *mgr, isc_nmiface_t *iface,
 	result = isc_nm_listentcp(mgr, iface, dnslisten_acceptcb,
 				  dnslistensock, extrahandlesize, backlog,
 				  quota, &dnslistensock->outer);
-
-	atomic_store(&dnslistensock->listening, true);
-	*sockp = dnslistensock;
-
-	return (result);
+	if (result == ISC_R_SUCCESS) {
+		atomic_store(&dnslistensock->listening, true);
+		*sockp = dnslistensock;
+		return (ISC_R_SUCCESS);
+	} else {
+		atomic_store(&dnslistensock->closed, true);
+		isc_nmsocket_detach(&dnslistensock);
+		return (result);
+	}
 }
 
 void
@@ -483,10 +490,44 @@ isc__nm_tcpdns_send(isc_nmhandle_t *handle, isc_region_t *region,
 }
 
 
-void
-isc__nm_tcpdns_close(isc_nmsocket_t *sock) {
+static void
+tcpdns_close_direct(isc_nmsocket_t *sock) {
+	REQUIRE(sock->tid == isc_nm_tid());
 	if (sock->outer != NULL) {
+		sock->outer->rcb.recv = NULL;
 		isc_nmsocket_detach(&sock->outer);
 	}
-	uv_close((uv_handle_t*) &sock->timer, timer_close_cb);
+	/* We don't need atomics here, it's all in single network thread */
+	if (sock->timer_initialized) {
+		sock->timer_initialized = false;
+		uv_timer_stop(&sock->timer);
+		uv_close((uv_handle_t *) &sock->timer, timer_close_cb);
+	}
+}
+
+void
+isc__nm_tcpdns_close(isc_nmsocket_t *sock) {
+	REQUIRE(VALID_NMSOCK(sock));
+	REQUIRE(sock->type == isc_nm_tcpdnssocket);
+
+	if (sock->tid == isc_nm_tid()) {
+		tcpdns_close_direct(sock);
+	} else {
+		isc__netievent_tcpdnsclose_t *ievent =
+			isc__nm_get_ievent(sock->mgr, netievent_tcpdnsclose);
+
+		ievent->sock = sock;
+		isc__nm_enqueue_ievent(&sock->mgr->workers[sock->tid],
+				       (isc__netievent_t *) ievent);
+	}
+}
+
+void
+isc__nm_async_tcpdnsclose(isc__networker_t *worker, isc__netievent_t *ev0) {
+	isc__netievent_tcpdnsclose_t *ievent =
+		(isc__netievent_tcpdnsclose_t *) ev0;
+
+	REQUIRE(worker->id == ievent->sock->tid);
+
+	tcpdns_close_direct(ievent->sock);
 }

lib/isc/netmgr/udp.c

@@ -25,6 +25,8 @@
 #include <isc/sockaddr.h>
 #include <isc/thread.h>
 #include <isc/util.h>
+
+#include "uv-compat.h"
 #include "netmgr-int.h"
 
 static isc_result_t
@@ -113,9 +115,9 @@ isc_nm_listenudp(isc_nm_t *mgr, isc_nmiface_t *iface,
  * handle 'udplisten' async call - start listening on a socket.
  */
 void
-isc__nm_async_udplisten(isc__networker_t *worker, isc__netievent_t *ievent0) {
+isc__nm_async_udplisten(isc__networker_t *worker, isc__netievent_t *ev0) {
 	isc__netievent_udplisten_t *ievent =
-		(isc__netievent_udplisten_t *) ievent0;
+		(isc__netievent_udplisten_t *) ev0;
 	isc_nmsocket_t *sock = ievent->sock;
 
 	REQUIRE(sock->type == isc_nm_udpsocket);
@@ -123,7 +125,7 @@ isc__nm_async_udplisten(isc__networker_t *worker, isc__netievent_t *ievent0) {
 	REQUIRE(sock->parent != NULL);
 
 	uv_udp_init(&worker->loop, &sock->uv_handle.udp);
-	sock->uv_handle.udp.data = NULL;
+	uv_handle_set_data(&sock->uv_handle.handle, NULL);
 	isc_nmsocket_attach(sock,
 			    (isc_nmsocket_t **)&sock->uv_handle.udp.data);
 
@@ -140,7 +142,7 @@ isc__nm_async_udplisten(isc__networker_t *worker, isc__netievent_t *ievent0) {
 
 static void
 udp_close_cb(uv_handle_t *handle) {
-	isc_nmsocket_t *sock = handle->data;
+	isc_nmsocket_t *sock = uv_handle_get_data(handle);
 	atomic_store(&sock->closed, true);
 
 	isc_nmsocket_detach((isc_nmsocket_t **)&sock->uv_handle.udp.data);
@@ -171,14 +173,14 @@ stoplistening(isc_nmsocket_t *sock) {
 	INSIST(sock->type == isc_nm_udplistener);
 
 	for (int i = 0; i < sock->nchildren; i++) {
-		isc__netievent_udplisten_t *event = NULL;
+		isc__netievent_udpstop_t *event = NULL;
 
 		if (i == sock->tid) {
 			stop_udp_child(&sock->children[i]);
 			continue;
 		}
 
-		event = isc__nm_get_ievent(sock->mgr, netievent_udpstoplisten);
+		event = isc__nm_get_ievent(sock->mgr, netievent_udpstop);
 		event->sock = &sock->children[i];
 		isc__nm_enqueue_ievent(&sock->mgr->workers[i],
 				       (isc__netievent_t *) event);
@@ -196,7 +198,7 @@ stoplistening(isc_nmsocket_t *sock) {
 
 void
 isc_nm_udp_stoplistening(isc_nmsocket_t *sock) {
-	isc__netievent_udpstoplisten_t *ievent = NULL;
+	isc__netievent_udpstop_t *ievent = NULL;
 
 	/* We can't be launched from network thread, we'd deadlock */
 	REQUIRE(!isc__nm_in_netthread());
@@ -208,7 +210,7 @@ isc_nm_udp_stoplistening(isc_nmsocket_t *sock) {
 	 * event. Otherwise, go ahead and stop listening right away.
 	 */
 	if (!isc__nm_acquire_interlocked(sock->mgr)) {
-		ievent = isc__nm_get_ievent(sock->mgr, netievent_udpstoplisten);
+		ievent = isc__nm_get_ievent(sock->mgr, netievent_udpstop);
 		ievent->sock = sock;
 		isc__nm_enqueue_ievent(&sock->mgr->workers[sock->tid],
 				       (isc__netievent_t *) ievent);
@@ -219,14 +221,11 @@ isc_nm_udp_stoplistening(isc_nmsocket_t *sock) {
 }
 
 /*
- * handle 'udpstoplisten' async call - stop listening on a socket.
+ * handle 'udpstop' async call - stop listening on a socket.
  */
 void
-isc__nm_async_udpstoplisten(isc__networker_t *worker,
-			    isc__netievent_t *ievent0)
-{
-	isc__netievent_udplisten_t *ievent =
-		(isc__netievent_udplisten_t *) ievent0;
+isc__nm_async_udpstop(isc__networker_t *worker, isc__netievent_t *ev0) {
+	isc__netievent_udpstop_t *ievent = (isc__netievent_udpstop_t *) ev0;
 	isc_nmsocket_t *sock = ievent->sock;
 
 	REQUIRE(sock->iface != NULL);
@@ -246,7 +245,7 @@ isc__nm_async_udpstoplisten(isc__networker_t *worker,
 	if (!isc__nm_acquire_interlocked(sock->mgr)) {
 		isc__netievent_udplisten_t *event = NULL;
 
-		event = isc__nm_get_ievent(sock->mgr, netievent_udpstoplisten);
+		event = isc__nm_get_ievent(sock->mgr, netievent_udpstop);
 		event->sock = sock;
 		isc__nm_enqueue_ievent(&sock->mgr->workers[sock->tid],
 				       (isc__netievent_t *) event);
@@ -271,7 +270,7 @@ udp_recv_cb(uv_udp_t *handle, ssize_t nrecv, const uv_buf_t *buf,
 	isc_sockaddr_t sockaddr;
 	isc_sockaddr_t localaddr;
 	struct sockaddr_storage laddr;
-	isc_nmsocket_t *sock = (isc_nmsocket_t *) handle->data;
+	isc_nmsocket_t *sock = uv_handle_get_data((uv_handle_t *)handle);
 	isc_region_t region;
 	uint32_t maxudp;
 
@@ -337,7 +336,7 @@ isc__nm_udp_send(isc_nmhandle_t *handle, isc_region_t *region,
 	isc_nmsocket_t *psock = NULL, *rsock = NULL;
 	isc_nmsocket_t *sock = handle->sock;
 	isc_sockaddr_t *peer = &handle->peer;
-	isc__netievent_udpsend_t *ievent;
+	isc__netievent_udpsend_t *ievent = NULL;
 	isc__nm_uvreq_t *uvreq = NULL;
 	int ntid;
 	uint32_t maxudp = atomic_load(&sock->mgr->maxudp);
@@ -405,9 +404,9 @@ isc__nm_udp_send(isc_nmhandle_t *handle, isc_region_t *region,
  * handle 'udpsend' async event - send a packet on the socket
  */
 void
-isc__nm_async_udpsend(isc__networker_t *worker, isc__netievent_t *ievent0) {
+isc__nm_async_udpsend(isc__networker_t *worker, isc__netievent_t *ev0) {
 	isc__netievent_udpsend_t *ievent =
-		(isc__netievent_udpsend_t *) ievent0;
+		(isc__netievent_udpsend_t *) ev0;
 
 	REQUIRE(worker->id == ievent->sock->tid);
 

lib/isc/netmgr/uv-compat.h

@@ -0,0 +1,34 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+#pragma once
+#include <uv.h>
+
+/*
+ * Those functions were introduced in newer libuv, we still
+ * want BIND9 compile on older ones so we emulate them.
+ * They're inline to avoid conflicts when running with a newer
+ * library version.
+ */
+
+#ifndef HAVE_UV_HANDLE_GET_DATA
+static inline void*
+uv_handle_get_data(const uv_handle_t* handle) {
+    return (handle->data);
+}
+#endif
+
+#ifndef HAVE_UV_HANDLE_SET_DATA
+static inline void
+uv_handle_set_data(uv_handle_t* handle, void* data) {
+    handle->data = data;
+};
+#endif

lib/ns/interfacemgr.c

@@ -83,6 +83,7 @@ struct ns_interfacemgr {
 	ISC_LIST(isc_sockaddr_t) listenon;
 	int			backlog;	/*%< Listen queue size */
 	unsigned int		udpdisp;	/*%< UDP dispatch count */
+	atomic_bool		shuttingdown;	/*%< Interfacemgr is shutting down */
 #ifdef USE_ROUTE_SOCKET
 	isc_task_t *		task;
 	isc_socket_t *		route;
@@ -217,6 +218,7 @@ ns_interfacemgr_create(isc_mem_t *mctx,
 	mgr->listenon4 = NULL;
 	mgr->listenon6 = NULL;
 	mgr->udpdisp = udpdisp;
+	atomic_init(&mgr->shuttingdown, false);
 
 	ISC_LIST_INIT(mgr->interfaces);
 	ISC_LIST_INIT(mgr->listenon);
@@ -360,6 +362,7 @@ ns_interfacemgr_shutdown(ns_interfacemgr_t *mgr) {
 	 * consider all interfaces "old".
 	 */
 	mgr->generation++;
+	atomic_store(&mgr->shuttingdown, true);
 #ifdef USE_ROUTE_SOCKET
 	LOCK(&mgr->lock);
 	if (mgr->route != NULL) {
@@ -1230,7 +1233,13 @@ ns_interfacemgr_listeningon(ns_interfacemgr_t *mgr,
 	bool result = false;
 
 	REQUIRE(NS_INTERFACEMGR_VALID(mgr));
-
+	/*
+	 * If the manager is shutting down it's safer to
+	 * return true.
+	 */
+	if (atomic_load(&mgr->shuttingdown)) {
+		return (true);
+	}
 	LOCK(&mgr->lock);
 	for (old = ISC_LIST_HEAD(mgr->listenon);
 	     old != NULL;

lib/ns/tests/nstest.c

@@ -75,8 +75,8 @@ static dns_zone_t *served_zone = NULL;
 /*
  * We don't want to use netmgr-based client accounting, we need to emulate it.
  */
-atomic_uint_fast32_t client_refs[16];
-atomic_uintptr_t client_addrs[16];
+atomic_uint_fast32_t client_refs[32];
+atomic_uintptr_t client_addrs[32];
 
 void
 __wrap_isc_nmhandle_unref(isc_nmhandle_t *handle);
@@ -86,12 +86,12 @@ __wrap_isc_nmhandle_unref(isc_nmhandle_t *handle) {
 	ns_client_t *client = (ns_client_t *)handle;
 	int i;
 
-	for (i = 0; i < 16; i++) {
+	for (i = 0; i < 32; i++) {
 		if (atomic_load(&client_addrs[i]) == (uintptr_t) client) {
 			break;
 		}
 	}
-	REQUIRE(i < 16);
+	REQUIRE(i < 32);
 
 	if (atomic_fetch_sub(&client_refs[i], 1) == 1) {
 		dns_view_detach(&client->view);
@@ -561,14 +561,14 @@ ns_test_getclient(ns_interface_t *ifp0, bool tcp,
 
 	result = ns__client_setup(client, clientmgr, true);
 
-	for (i = 0; i < 16; i++) {
+	for (i = 0; i < 32; i++) {
 		if (atomic_load(&client_addrs[i]) == (uintptr_t) NULL ||
 		    atomic_load(&client_addrs[i]) == (uintptr_t) client)
 		{
 			break;
 		}
 	}
-	REQUIRE(i < 16);
+	REQUIRE(i < 32);
 
 	atomic_store(&client_refs[i], 2);
 	atomic_store(&client_addrs[i], (uintptr_t) client);

util/copyrights

@@ -2258,6 +2258,7 @@
 ./lib/isc/netmgr/tcp.c				C	2019
 ./lib/isc/netmgr/tcpdns.c			C	2019
 ./lib/isc/netmgr/udp.c				C	2019
+./lib/isc/netmgr/uv-compat.h			C	2019
 ./lib/isc/netmgr/uverr2result.c			C	2019
 ./lib/isc/netscope.c				C	2002,2004,2005,2006,2007,2016,2018,2019
 ./lib/isc/nonce.c				C	2018,2019