Revert "Make hazard pointers max_threads configurable at runtime."

This reverts commit e83951144b.
Make hazard pointers max_threads configurable at runtime.
2019-12-13 14:36:36 +01:00 · 2019-12-12 23:41:57 +01:00 · 2019-12-12 22:55:57 +01:00 · 2019-12-12 22:55:18 +01:00 · 2019-12-12 14:14:45 +00:00 · 2019-12-12 13:08:34 +01:00
45 changed files with 1924 additions and 864 deletions
@@ -1,2 +1,10 @@
 *.sln.in eol=crlf
 *.vcxproj.* eol=crlf
+
+.gitignore			 export-ignore
+/conftools			 export-ignore
+/doc/design			 export-ignore
+/doc/dev			 export-ignore
+/util/**			 export-ignore
+/util/bindkeys.pl		-export-ignore
+/util/mksymtbl.pl		-export-ignore
@@ -60,3 +60,4 @@ timestamp
 /compile_commands.json
 /cppcheck_html/
 /cppcheck.results
+/tsan
@@ -18,9 +18,14 @@ variables:
  MAKE: make
  CONFIGURE: ./configure
  SCAN_BUILD: scan-build-9
+  SYMBOLIZER: /usr/lib/llvm-9/bin/llvm-symbolizer
+  ASAN_SYMBOLIZER_PATH: "$SYMBOLIZER"

  CFLAGS_COMMON: -fno-omit-frame-pointer -fno-optimize-sibling-calls -O1 -g -Wall -Wextra 

+  TARBALL_COMPRESSOR: xz
+  TARBALL_EXTENSION: xz
+
 stages:
  - precheck
  - build
@@ -29,6 +34,7 @@ stages:
  - docs
  - push
  - postcheck
+  - release

 ### Runner Tag Templates

@@ -104,8 +110,8 @@ stages:

 # Fedora

-.fedora-30-amd64: &fedora_30_amd64_image
-  image: "$CI_REGISTRY_IMAGE:fedora-30-amd64"
+.fedora-31-amd64: &fedora_31_amd64_image
+  image: "$CI_REGISTRY_IMAGE:fedora-31-amd64"
  <<: *linux_amd64

 # Ubuntu
@@ -133,12 +139,14 @@ stages:
    - merge_requests
    - tags
    - web
+    - schedules

 .release-branch-triggering-rules: &release_branch_triggering_rules
  only:
    - merge_requests
    - tags
    - web
+    - schedules
    - master@isc-projects/bind9
    - /^v9_[1-9][0-9]$/@isc-projects/bind9

@@ -174,7 +182,7 @@ stages:
    --without-make-clean \
    $EXTRA_CONFIGURE \
    || cat config.log
-    
+
 .build: &build_job
  <<: *default_triggering_rules
  stage: build
@@ -192,6 +200,36 @@ stages:
    untracked: true
    expire_in: "1 week"

+.windows_build: &windows_build_job
+  stage: build
+  tags:
+    - windows
+    - amd64
+  script:
+    - 'Push-Location "C:/Program Files (x86)/Microsoft Visual Studio/2017/BuildTools/VC/Auxiliary/Build"'
+    - '& cmd.exe /C "vcvarsall.bat x64 & set" | Foreach-Object { if ($_ -match "(.*?)=(.*)") { Set-Item -force -path "Env:\$($matches[1])" -value "$($matches[2])" } }'
+    - 'Pop-Location'
+    - 'Set-Location win32utils'
+    - '& "C:/Strawberry/perl/bin/perl.exe" Configure
+         "with-tools-version=15.0"
+         "with-platform-toolset=v141"
+         "with-platform-version=10.0.17763.0"
+         "with-vcredist=C:/Program Files (x86)/Microsoft Visual Studio/2017/BuildTools/VC/Redist/MSVC/14.16.27012/vcredist_x64.exe"
+         "with-openssl=C:/OpenSSL"
+         "with-libxml2=C:/libxml2"
+         "with-libuv=C:/libuv"
+         "without-python"
+         "with-system-tests"
+         x64'
+    - 'Set-Item -path "Env:CL" -value "/MP$([Math]::Truncate($BUILD_PARALLEL_JOBS/2))"'
+    - '& msbuild.exe /maxCpuCount:2 /t:Build /p:Configuration=$VSCONF bind9.sln'
+  dependencies: []
+  needs:
+    - autoreconf:sid:amd64
+  artifacts:
+    untracked: true
+    expire_in: "1 week"
+
 .setup_interfaces: &setup_interfaces |
    if [ "$(id -u)" -eq "0" ]; then
      sh -x bin/tests/system/ifconfig.sh up;
@@ -200,7 +238,7 @@ stages:
    fi

 .setup_softhsm: &setup_softhsm |
-    sh -x util/prepare-softhsm2.sh
+    sh -x bin/tests/prepare-softhsm2.sh

 .system_test: &system_test_job
  <<: *default_triggering_rules
@@ -223,6 +261,30 @@ stages:
       --results-filter "" \
       --output kyua_html

+.windows_system_test: &windows_system_test_job
+  stage: system
+  tags:
+    - windows
+    - amd64
+  script:
+    - 'Push-Location bin/tests/system'
+    - '$ifIndex = Get-NetIPInterface -AddressFamily IPv4 -InterfaceMetric 75 | Select-Object -ExpandProperty ifIndex'
+    - '& C:/tools/cygwin/bin/sed.exe -i "s/^exit.*/netsh interface ipv4 set dnsservers $ifIndex dhcp/; s/\(name\|interface\)=Loopback/$ifIndex/;" ifconfig.bat'
+    - '& C:/tools/cygwin/bin/sed.exe -i "s/kill -f/kill -W/;" conf.sh stop.pl'
+    - '& cmd.exe /C ifconfig.bat up; ""'
+    - 'Start-Sleep 2'
+    - '$Env:Path = "C:/tools/cygwin/bin;$Env:Path"'
+    - '& sh.exe runall.sh $TEST_PARALLEL_JOBS'
+    - 'If (Test-Path C:/CrashDumps/*) { dir C:/CrashDumps; Throw }'
+  artifacts:
+    untracked: true
+    expire_in: "1 week"
+    when: on_failure
+  only:
+    - schedules
+    - tags
+    - web
+
 .unit_test: &unit_test_job
  <<: *default_triggering_rules
  stage: unit
@@ -316,6 +378,23 @@ misc:sid:amd64:
    - util/check-cocci
    - if test "$(git status --porcelain | grep -Ev '\?\?' | wc -l)" -gt "0"; then git status --short; exit 1; fi

+tarball-create:sid:amd64:
+  <<: *debian_sid_amd64_image
+  stage: precheck
+  script:
+    - source version
+    - export BIND_DIRECTORY="bind-${MAJORVER}.${MINORVER}.${PATCHVER}${RELEASETYPE}${RELEASEVER}"
+    - git archive --prefix="${BIND_DIRECTORY}/" --output="${BIND_DIRECTORY}.tar" HEAD
+    - mkdir "${BIND_DIRECTORY}"
+    - echo "SRCID=$(git rev-list --max-count=1 HEAD | cut -b1-7)" > "${BIND_DIRECTORY}/srcid"
+    - tar --append --file="${BIND_DIRECTORY}.tar" "${BIND_DIRECTORY}/srcid"
+    - ${TARBALL_COMPRESSOR} "${BIND_DIRECTORY}.tar"
+  artifacts:
+    paths:
+      - bind-*.tar.${TARBALL_EXTENSION}
+  only:
+    - tags
+
 # Jobs for doc builds on Debian Sid (amd64)

 docs:sid:amd64:
@@ -576,6 +655,48 @@ cppcheck:gcc:sid:amd64:
    - gcc:sid:amd64
  needs: ["gcc:sid:amd64"]

+# Jobs for tarball GCC builds on Debian Sid (amd64)
+
+tarball:sid:amd64:
+  variables:
+    CC: gcc
+    EXTRA_CONFIGURE: "--enable-dnstap --with-libidn2"
+    RUN_MAKE_INSTALL: 1
+  <<: *debian_sid_amd64_image
+  <<: *build_job
+  before_script:
+    - tar --extract --file bind-*.tar.${TARBALL_EXTENSION}
+    - rm -f bind-*.tar.${TARBALL_EXTENSION}
+    - cd bind-*
+  dependencies:
+    - tarball-create:sid:amd64
+  needs: ["tarball-create:sid:amd64"]
+  only:
+    - tags
+
+system:tarball:sid:amd64:
+  <<: *debian_sid_amd64_image
+  <<: *system_test_job
+  before_script:
+    - cd bind-*
+    - *setup_interfaces
+  dependencies:
+    - tarball:sid:amd64
+  needs: ["tarball:sid:amd64"]
+  only:
+    - tags
+
+unit:tarball:sid:amd64:
+  <<: *debian_sid_amd64_image
+  <<: *unit_test_job
+  before_script:
+    - cd bind-*
+  dependencies:
+    - tarball:sid:amd64
+  needs: ["tarball:sid:amd64"]
+  only:
+    - tags
+
 # Jobs for regular GCC builds on Debian Sid (i386)

 gcc:sid:i386:
@@ -600,29 +721,29 @@ unit:gcc:sid:i386:
    - gcc:sid:i386
  needs: ["gcc:sid:i386"]

-# Jobs for regular GCC builds on Fedora 30 (amd64)
+# Jobs for regular GCC builds on Fedora 31 (amd64)

-gcc:fedora30:amd64:
+gcc:fedora31:amd64:
  variables:
    CC: gcc
    CFLAGS: "${CFLAGS_COMMON} -O1"
    EXTRA_CONFIGURE: "--with-libidn2"
-  <<: *fedora_30_amd64_image
+  <<: *fedora_31_amd64_image
  <<: *build_job

-system:gcc:fedora30:amd64:
-  <<: *fedora_30_amd64_image
+system:gcc:fedora31:amd64:
+  <<: *fedora_31_amd64_image
  <<: *system_test_job
  dependencies:
-    - gcc:fedora30:amd64
-  needs: ["gcc:fedora30:amd64"]
+    - gcc:fedora31:amd64
+  needs: ["gcc:fedora31:amd64"]

-unit:gcc:fedora30:amd64:
-  <<: *fedora_30_amd64_image
+unit:gcc:fedora31:amd64:
+  <<: *fedora_31_amd64_image
  <<: *unit_test_job
  dependencies:
-    - gcc:fedora30:amd64
-  needs: ["gcc:fedora30:amd64"]
+    - gcc:fedora31:amd64
+  needs: ["gcc:fedora31:amd64"]

 # Jobs for regular GCC builds on Ubuntu 16.04 Xenial Xerus (amd64)

@@ -697,6 +818,63 @@ unit:asan:sid:amd64:
    - asan:sid:amd64
  needs: ["asan:sid:amd64"]

+# Jobs for GCC builds with TSAN enabled on Debian Sid (amd64)
+
+tsan:buster:amd64:
+  <<: *debian_buster_amd64_image
+  <<: *build_job
+  variables:
+    CC: clang-9
+    CFLAGS: "${CFLAGS_COMMON} -fsanitize=thread -DISC_MEM_USE_INTERNAL_MALLOC=0"
+    LDFLAGS: "-fsanitize=thread"
+    EXTRA_CONFIGURE: "--with-libidn2 --enable-pthread-rwlock"
+
+system:tsan:buster:amd64:
+  variables:
+    TSAN_OPTIONS: "second_deadlock_stack=1 history_size=7 log_exe_name=true log_path=tsan external_symbolizer_path=$SYMBOLIZER exitcode=0"
+  before_script:
+    - *setup_interfaces
+    - echo $TSAN_OPTIONS
+  <<: *debian_buster_amd64_image
+  <<: *system_test_job
+  dependencies:
+    - tsan:buster:amd64
+  needs: ["tsan:buster:amd64"]
+  allow_failure: true
+  after_script:
+    - find bin -name 'tsan.*' -exec python3 util/parse_tsan.py {} \;
+  artifacts:
+    expire_in: "1 week"
+    paths:
+      - bin/tests/system/*/tsan.*
+      - bin/tests/system/*/*/tsan.*
+      - tsan/
+    when: on_failure
+
+unit:tsan:buster:amd64:
+  variables:
+    TSAN_OPTIONS: "second_deadlock_stack=1 history_size=7 log_exe_name=true log_path=tsan external_symbolizer_path=$SYMBOLIZER"
+  before_script:
+    - echo $TSAN_OPTIONS
+    - lib/isc/tests/result_test
+  <<: *debian_buster_amd64_image
+  <<: *unit_test_job
+  dependencies:
+    - tsan:buster:amd64
+  needs: ["tsan:buster:amd64"]
+  allow_failure: true
+  after_script:
+    - find lib -name 'tsan.*' -exec python3 util/parse_tsan.py {} \;
+  artifacts:
+    expire_in: "1 week"
+    paths:
+      - lib/*/tests/tsan.*
+      - tsan/
+      - kyua.log
+      - kyua.results
+      - kyua_html/
+    when: on_failure
+
 rwlock:sid:amd64:
  variables:
    CC: gcc
@@ -854,6 +1032,7 @@ system:clang:openbsd6.5:amd64:
    - clang:openbsd6.5:amd64
  needs: ["clang:openbsd6.5:amd64"]
  only:
+    - schedules
    - tags
    - web

@@ -884,62 +1063,74 @@ unit:nolibtool:sid:amd64:
 # Jobs for Visual Studio 2017 builds on Windows (amd64)

 msvc:windows:amd64:
+  <<: *windows_build_job
  <<: *default_triggering_rules
-  stage: build
-  tags:
-    - windows
-    - amd64
  variables:
    VSCONF: Release
-  script:
-    - 'Push-Location "C:/Program Files (x86)/Microsoft Visual Studio/2017/BuildTools/VC/Auxiliary/Build"'
-    - '& cmd.exe /C "vcvarsall.bat x64 & set" | Foreach-Object { if ($_ -match "(.*?)=(.*)") { Set-Item -force -path "Env:\$($matches[1])" -value "$($matches[2])" } }'
-    - 'Pop-Location'
-    - 'Set-Location win32utils'
-    - '& "C:/Strawberry/perl/bin/perl.exe" Configure
-         "with-tools-version=15.0"
-         "with-platform-toolset=v141"
-         "with-platform-version=10.0.17763.0"
-         "with-vcredist=C:/Program Files (x86)/Microsoft Visual Studio/2017/BuildTools/VC/Redist/MSVC/14.16.27012/vcredist_x64.exe"
-         "with-openssl=C:/OpenSSL"
-         "with-libxml2=C:/libxml2"
-         "with-libuv=C:/libuv"
-         "without-python"
-         "with-system-tests"
-         x64'
-    - 'Set-Item -path "Env:CL" -value "/MP$([Math]::Truncate($BUILD_PARALLEL_JOBS/2))"'
-    - '& msbuild.exe /maxCpuCount:2 /t:Build /p:Configuration=$VSCONF bind9.sln'
-  dependencies: []
-  needs:
-    - autoreconf:sid:amd64
-  artifacts:
-    untracked: true
-    expire_in: "1 week"

 system:msvc:windows:amd64:
-  stage: system
-  tags:
-    - windows
-    - amd64
+  <<: *windows_system_test_job
  variables:
    VSCONF: Release
-  script:
-    - 'Push-Location bin/tests/system'
-    - '$ifIndex = Get-NetIPInterface -AddressFamily IPv4 -InterfaceMetric 75 | Select-Object -ExpandProperty ifIndex'
-    - '& C:/tools/cygwin/bin/sed.exe -i "s/^exit.*/netsh interface ipv4 set dnsservers $ifIndex dhcp/; s/\(name\|interface\)=Loopback/$ifIndex/;" ifconfig.bat'
-    - '& C:/tools/cygwin/bin/sed.exe -i "s/kill -f/kill -W/;" conf.sh stop.pl'
-    - '& cmd.exe /C ifconfig.bat up; ""'
-    - 'Start-Sleep 2'
-    - '$Env:Path = "C:/tools/cygwin/bin;$Env:Path"'
-    - '& sh.exe runall.sh $TEST_PARALLEL_JOBS'
-    - 'If (Test-Path C:/CrashDumps/*) { dir C:/CrashDumps; Throw }'
  dependencies:
    - msvc:windows:amd64
  needs: ["msvc:windows:amd64"]
-  artifacts:
-    untracked: true
-    expire_in: "1 week"
-    when: on_failure
+
+msvc-debug:windows:amd64:
+  <<: *windows_build_job
+  variables:
+    VSCONF: Debug
  only:
+    - schedules
    - tags
    - web
+
+system:msvc-debug:windows:amd64:
+  <<: *windows_system_test_job
+  variables:
+    VSCONF: Debug
+  dependencies:
+    - msvc-debug:windows:amd64
+  needs: ["msvc-debug:windows:amd64"]
+
+# Job producing a release tarball
+
+release:sid:amd64:
+  <<: *debian_sid_amd64_image
+  stage: release
+  script:
+    # Determine BIND version
+    - source version
+    - export BIND_DIRECTORY="bind-${MAJORVER}.${MINORVER}.${PATCHVER}${RELEASETYPE}${RELEASEVER}"
+    # Remove redundant files and system test utilities from Windows build artifacts
+    - find Build/Release/ -name "*.pdb" -print -delete
+    - find Build/Debug/ \( -name "*.bsc" -o -name "*.idb" \) -print -delete
+    - find Build/ -regextype posix-extended -regex "Build/.*/($(find bin/tests/ -type f | sed -nE "s|^bin/tests(/system)?/win32/(.*)\.vcxproj$|\2|p" | paste -d"|" -s))\..*" -print -delete
+    # Create Windows zips
+    - openssl dgst -sha256 "${BIND_DIRECTORY}.tar.${TARBALL_EXTENSION}" | tee Build/Release/SHA256 Build/Debug/SHA256
+    - ( cd Build/Release; zip "../../BIND${BIND_DIRECTORY#bind-}.x64.zip" * )
+    - ( cd Build/Debug; zip "../../BIND${BIND_DIRECTORY#bind-}.debug.x64.zip" * )
+    # Prepare release tarball contents (tarballs + zips + documentation)
+    - mkdir -p release/doc/arm
+    - pushd release
+    - mv "../${BIND_DIRECTORY}.tar.${TARBALL_EXTENSION}" ../BIND*.zip .
+    - tar --extract --file="${BIND_DIRECTORY}.tar.${TARBALL_EXTENSION}"
+    - mv "${BIND_DIRECTORY}"/{CHANGES*,COPYRIGHT,LICENSE,README,srcid} .
+    - mv "${BIND_DIRECTORY}"/doc/arm/{Bv9ARM{*.html,.pdf},man.*,notes.{html,pdf,txt}} doc/arm/
+    - rm -rf "${BIND_DIRECTORY}"
+    - cp doc/arm/notes.html "RELEASE-NOTES-${BIND_DIRECTORY}.html"
+    - cp doc/arm/notes.pdf "RELEASE-NOTES-${BIND_DIRECTORY}.pdf"
+    - cp doc/arm/notes.txt "RELEASE-NOTES-${BIND_DIRECTORY}.txt"
+    - popd
+    # Create release tarball
+    - tar --create --file="${CI_COMMIT_TAG}.tar.gz" --gzip release/
+  dependencies:
+    - tarball-create:sid:amd64
+    - msvc:windows:amd64
+    - msvc-debug:windows:amd64
+  only:
+    - tags
+  artifacts:
+    paths:
+      - "*.tar.gz"
+    expire_in: "1 week"
@@ -2,8 +2,6 @@

 **Tagging Deadline:**

-**ASN Deadline:**
-
 **Public Release:**

 ## Release Checklist
@@ -12,6 +10,7 @@

 - [ ] ***(QA)*** Check whether all issues assigned to the release milestone are resolved[^1].
 - [ ] ***(QA)*** Ensure that there are no outstanding merge requests in the private repository[^1] (Subscription Edition only).
+ - [ ] ***(QA)*** Ensure all merge requests marked for backporting have been indeed backported.

 ## Before the Tagging Deadline

@@ -33,21 +32,22 @@

 ## Before the ASN Deadline (for ASN Releases) or the Public Release Date (for Regular Releases)

- - [ ] ***(QA)*** Run the `make release` Jenkins jobs to produce the tarballs and zips.
- - [ ] ***(QA)*** Verify the results of `make release` Jenkins jobs and prepare a QA report for the releases to be published.
- - [ ] ***(QA)*** Request signatures for the tarballs.
- - [ ] ***(Signers)*** Sign the tarballs.
- - [ ] ***(QA)*** Check tarball signatures.
- - [ ] ***(QA)*** Notify Support that the releases are ready for publication.
+ - [ ] ***(QA)*** Verify GitLab CI results for the tags created and prepare a QA report for the releases to be published.
+ - [ ] ***(QA)*** Request signatures for the tarballs, providing their location and checksums.
+ - [ ] ***(Signers)*** Validate tarball checksums, sign tarballs, and upload signatures.
+ - [ ] ***(QA)*** Verify tarball signatures and check tarball checksums again.
 - [ ] ***(Support)*** Pre-publish ASN and/or Subscription Edition tarballs so that packages can be built.
 - [ ] ***(QA)*** Build and test ASN and/or Subscription Edition packages.
+ - [ ] ***(QA)*** Notify Support that the releases have been prepared.
 - [ ] ***(Support)*** Send out ASNs (if applicable).

 ## On the Day of Public Release

- - [ ] ***(Support)*** Publish the releases according to the release schedule.
- - [ ] ***(Support)*** Write release email to *bind9-announce*.
- - [ ] ***(Support)*** Write email to *bind9-users* (if a major release).
+ - [ ] ***(Support)*** Wait for clearance from Security Officer to proceed with the public release (if applicable).
+ - [ ] ***(Support)*** Place tarballs in public location on FTP site.
+ - [ ] ***(Support)*** Publish links to downloads on ISC website.
+ - [ ] ***(Support)*** Write release email to *bind-announce*.
+ - [ ] ***(Support)*** Write email to *bind-users* (if a major release).
 - [ ] ***(Support)*** Update tickets in case of waiting support customers.
 - [ ] ***(QA)*** Build and test any outstanding private packages.
 - [ ] ***(QA)*** Build public packages (`*.deb`, RPMs).
@@ -1,3 +1,5 @@
+5335.	[func]		Make TCP listening code multithreaded. [GL !2659]
+
 5334.	[doc]		Update documentation with dnssec-policy clarifications.
 			Also change some defaults.

@@ -39,7 +39,7 @@ following systems:

  * Debian 9, 10
  * Ubuntu LTS 16.04, 18.04
-  * Fedora 30
+  * Fedora 31
  * Red Hat Enterprise Linux / CentOS 7, 8
  * FreeBSD 11.3, 12.0
  * OpenBSD 6.5
@@ -47,7 +47,7 @@ following systems:

 * Debian 9, 10
 * Ubuntu LTS 16.04, 18.04
-* Fedora 30
+* Fedora 31
 * Red Hat Enterprise Linux / CentOS 7, 8
 * FreeBSD 11.3, 12.0
 * OpenBSD 6.5
@@ -9,32 +9,34 @@
 # See the COPYRIGHT file distributed with this work for additional
 # information regarding copyright ownership.

-rm -f dig.out.*
-rm -f rndc.out*
-rm -f showzone.out*
-rm -f zonestatus.out*
-rm -f */named.conf
-rm -f */named.memstats
-rm -f ns1/*.nzf ns1/*.nzf~
-rm -f ns1/*.nzd ns1/*.nzd-lock
-rm -f ns2/*.nzf ns2/*.nzf~
-rm -f ns2/*.nzd ns2/*.nzd-lock
-rm -f ns3/*.nzf ns3/*.nzf~
-rm -f ns3/*.nzd ns3/*.nzd-lock
-rm -f ns2/core*
-rm -f ns2/inline.db.jbk
-rm -f ns2/inline.db.signed
-rm -f ns2/inlineslave.bk*
-rm -rf ns2/new-zones
-rm -f ns*/named.lock
-rm -f ns*/named.run
-rm -f ns2/nzf-*
-rm -f ns3/named.conf
-rm -f ns3/*.nzf ns3/*.nzf~
-rm -f ns3/*.nzd ns3/*.nzd-lock
-rm -f ns3/inlineslave.db
-rm -f ns1/redirect.db
-rm -f ns2/redirect.db
-rm -f ns2/redirect.bk
-rm -f ns3/redirect.db
-rm -f ns*/managed-keys.bind* ns*/*.mkeys*
+rm -f ./dig.out.*
+rm -f ./rndc.out*
+rm -f ./showzone.out*
+rm -f ./zonestatus.out*
+rm -f ./*/named.conf
+rm -f ./*/named.memstats
+rm -f ./ns1/*.nzf ./ns1/*.nzf~
+rm -f ./ns1/*.nzd ./ns1/*.nzd-lock
+rm -f ./ns2/*.nzf ./ns2/*.nzf~
+rm -f ./ns2/*.nzd ./ns2/*.nzd-lock
+rm -f ./ns3/*.nzf ./ns3/*.nzf~
+rm -f ./ns3/*.nzd ./ns3/*.nzd-lock
+rm -f ./ns2/core*
+rm -f ./ns2/inline.db.jbk
+rm -f ./ns2/inline.db.signed
+rm -f ./ns2/inlineslave.bk*
+rm -rf ./ns2/new-zones
+rm -f ./ns*/named.lock
+rm -f ./ns*/named.run ./ns*/named.run.prev
+rm -f ./ns2/nzf-*
+rm -f ./ns3/named.conf
+rm -f ./ns3/*.nzf ./ns3/*.nzf~
+rm -f ./ns3/*.nzd ns3/*.nzd-lock
+rm -f ./ns3/inlineslave.db
+rm -f ./ns1/redirect.db
+rm -f ./ns2/redirect.db
+rm -f ./ns2/redirect.bk
+rm -f ./ns3/redirect.db
+rm -f ./ns*/managed-keys.bind* ns*/*.mkeys*
+rm -f ./nzd2nzf.out.*
+rm -f ./wait_for_message.*
@@ -15,6 +15,12 @@ SYSTEMTESTTOP=..
 DIGOPTS="+tcp +nosea +nostat +nocmd +norec +noques +noauth +noadd +nostats +dnssec -p ${PORT}"
 RNDCCMD="$RNDC -c $SYSTEMTESTTOP/common/rndc.conf -p ${CONTROLPORT} -s"

+check_zonestatus() (
+    $RNDCCMD "10.53.0.$1" zonestatus -redirect > "zonestatus.out.ns$1.$n" &&
+    grep "type: redirect" "zonestatus.out.ns$1.$n" > /dev/null &&
+    grep "serial: 1" "zonestatus.out.ns$1.$n" > /dev/null
+)
+
 status=0
 n=0

@@ -49,25 +55,30 @@ fi
 echo_i "adding new zone ($n)"
 ret=0
 $RNDCCMD 10.53.0.2 addzone 'added.example { type master; file "added.db"; };' 2>&1 | sed 's/^/I:ns2 /'
-$DIG $DIGOPTS @10.53.0.2 a.added.example a > dig.out.ns2.$n || ret=1
-grep 'status: NOERROR' dig.out.ns2.$n > /dev/null || ret=1
-grep '^a.added.example' dig.out.ns2.$n > /dev/null || ret=1
+_check_adding_new_zone () (
+	$DIG $DIGOPTS @10.53.0.2 a.added.example a > dig.out.ns2.$n &&
+	grep 'status: NOERROR' dig.out.ns2.$n > /dev/null &&
+	grep '^a.added.example' dig.out.ns2.$n > /dev/null
+)
+retry_quiet 10 _check_adding_new_zone || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`

+nextpart ns2/named.run >/dev/null
 echo_i "checking addzone errors are logged correctly"
 ret=0
 $RNDCCMD 10.53.0.2 addzone bad.example '{ type mister; };' 2>&1 | grep 'unexpected token' > /dev/null 2>&1 || ret=1
-grep "addzone: 'mister' unexpected" ns2/named.run >/dev/null 2>&1 || ret=1
+wait_for_log_peek 20 "addzone: 'mister' unexpected" ns2/named.run || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`

+nextpart ns2/named.run >/dev/null
 echo_i "checking modzone errors are logged correctly"
 ret=0
 $RNDCCMD 10.53.0.2 modzone added.example '{ type mister; };' 2>&1 | grep 'unexpected token' > /dev/null 2>&1 || ret=1
-grep "modzone: 'mister' unexpected" ns2/named.run >/dev/null 2>&1 || ret=1
+wait_for_log_peek 20 "modzone: 'mister' unexpected" ns2/named.run || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
@@ -75,9 +86,12 @@ status=`expr $status + $ret`
 echo_i "adding a zone that requires quotes ($n)"
 ret=0
 $RNDCCMD 10.53.0.2 addzone '"32/1.0.0.127-in-addr.added.example" { check-names ignore; type master; file "added.db"; };' 2>&1 | sed 's/^/I:ns2 /'
-$DIG $DIGOPTS @10.53.0.2 "a.32/1.0.0.127-in-addr.added.example" a > dig.out.ns2.$n || ret=1
-grep 'status: NOERROR' dig.out.ns2.$n > /dev/null || ret=1
-grep '^a.32/1.0.0.127-in-addr.added.example' dig.out.ns2.$n > /dev/null || ret=1
+_check_zone_that_requires_quotes() (
+	$DIG $DIGOPTS @10.53.0.2 "a.32/1.0.0.127-in-addr.added.example" a > dig.out.ns2.$n &&
+	grep 'status: NOERROR' dig.out.ns2.$n > /dev/null &&
+	grep '^a.32/1.0.0.127-in-addr.added.example' dig.out.ns2.$n > /dev/null
+)
+retry_quiet 10 _check_zone_that_requires_quotes || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
@@ -85,9 +99,12 @@ status=`expr $status + $ret`
 echo_i "adding a zone with a quote in the name ($n)"
 ret=0
 $RNDCCMD 10.53.0.2 addzone '"foo\"bar.example" { check-names ignore; type master; file "added.db"; };' 2>&1 | sed 's/^/I:ns2 /'
-$DIG $DIGOPTS @10.53.0.2 "a.foo\"bar.example" a > dig.out.ns2.$n || ret=1
-grep 'status: NOERROR' dig.out.ns2.$n > /dev/null || ret=1
-grep '^a.foo\\"bar.example' dig.out.ns2.$n > /dev/null || ret=1
+_check_zone_with_a_quote() (
+	$DIG $DIGOPTS @10.53.0.2 "a.foo\"bar.example" a > dig.out.ns2.$n &&
+	grep 'status: NOERROR' dig.out.ns2.$n > /dev/null &&
+	grep '^a.foo\\"bar.example' dig.out.ns2.$n > /dev/null
+)
+retry_quiet 10 _check_zone_with_a_quote || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
@@ -135,21 +152,24 @@ fi
 echo_i "deleting previously added zone ($n)"
 ret=0
 $RNDCCMD 10.53.0.2 delzone previous.example 2>&1 | sed 's/^/I:ns2 /'
-$DIG $DIGOPTS @10.53.0.2 a.previous.example a > dig.out.ns2.$n
-grep 'status: REFUSED' dig.out.ns2.$n > /dev/null || ret=1
-grep '^a.previous.example' dig.out.ns2.$n > /dev/null && ret=1
+_check_deleting_previously_added_zone() (
+	$DIG $DIGOPTS @10.53.0.2 a.previous.example a > dig.out.ns2.$n &&
+	grep 'status: REFUSED' dig.out.ns2.$n > /dev/null &&
+	! grep '^a.previous.example' dig.out.ns2.$n > /dev/null
+)
+retry_quiet 10 _check_deleting_previously_added_zone || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`

+check_nzd2nzf() (
+	$NZD2NZF ns2/_default.nzd > nzd2nzf.out.$n &&
+	! grep previous.example nzd2nzf.out.$n > /dev/null
+)
+
 if [ -n "$NZD" ]; then
    echo_i "checking zone was deleted from NZD ($n)"
-    for i in 0 1 2 3 4 5 6 7 8 9; do
-        ret=0
-        $NZD2NZF ns2/_default.nzd | grep previous.example > /dev/null && ret=1
-        [ $ret = 0 ] && break
-        sleep 1
-    done
+    retry_quiet 10 check_nzd2nzf || ret=1
    if [ $ret != 0 ]; then echo_i "failed"; fi
    status=`expr $status + $ret`
 fi
@@ -167,9 +187,12 @@ fi
 echo_i "deleting newly added zone added.example ($n)"
 ret=0
 $RNDCCMD 10.53.0.2 delzone added.example 2>&1 | sed 's/^/I:ns2 /'
-$DIG $DIGOPTS @10.53.0.2 a.added.example a > dig.out.ns2.$n
-grep 'status: REFUSED' dig.out.ns2.$n > /dev/null || ret=1
-grep '^a.added.example' dig.out.ns2.$n > /dev/null && ret=1
+_check_deleting_newly_added_zone() (
+	$DIG $DIGOPTS @10.53.0.2 a.added.example a > dig.out.ns2.$n &&
+	grep 'status: REFUSED' dig.out.ns2.$n > /dev/null &&
+	! grep '^a.added.example' dig.out.ns2.$n > /dev/null
+)
+retry_quiet 10 _check_deleting_newly_added_zone || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
@@ -177,9 +200,12 @@ status=`expr $status + $ret`
 echo_i "deleting newly added zone with escaped quote ($n)"
 ret=0
 $RNDCCMD 10.53.0.2 delzone "foo\\\"bar.example" 2>&1 | sed 's/^/I:ns2 /'
-$DIG $DIGOPTS @10.53.0.2 "a.foo\"bar.example" a > dig.out.ns2.$n
-grep 'status: REFUSED' dig.out.ns2.$n > /dev/null || ret=1
-grep "^a.foo\"bar.example" dig.out.ns2.$n > /dev/null && ret=1
+_check_deleting_newly_added_zone_quote() (
+	$DIG $DIGOPTS @10.53.0.2 "a.foo\"bar.example" a > dig.out.ns2.$n &&
+	grep 'status: REFUSED' dig.out.ns2.$n > /dev/null &&
+	! grep "^a.foo\"bar.example" dig.out.ns2.$n > /dev/null
+)
+retry_quiet 10 _check_deleting_newly_added_zone_quote || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
@@ -225,12 +251,7 @@ ret=0
 sleep 1
 cp -f ns1/redirect.db.2 ns1/redirect.db
 $RNDCCMD 10.53.0.1 reload -redirect > rndc.out.ns1.$n
-_check_zonestatus() {
-    $RNDCCMD 10.53.0.1 zonestatus -redirect > zonestatus.out.ns1.$n || return 1
-    grep "type: redirect" zonestatus.out.ns1.$n > /dev/null || return 1
-    grep "serial: 1" zonestatus.out.ns1.$n > /dev/null || return 1
-}
-retry 5 _check_zonestatus || ret=1
+retry_quiet 5 check_zonestatus 1 || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
@@ -238,27 +259,27 @@ status=`expr $status + $ret`
 echo_i "delete a normally-loaded zone ($n)"
 ret=0
 $RNDCCMD 10.53.0.2 delzone normal.example > rndc.out.ns2.$n 2>&1
-$DIG $DIGOPTS @10.53.0.2 a.normal.example a > dig.out.ns2.$n
-grep "is no longer active and will be deleted" rndc.out.ns2.$n > /dev/null || ret=1
+grep "is no longer active and will be deleted" rndc.out.ns2.$n > /dev/null || ret=11
 grep "To keep it from returning when the server is restarted" rndc.out.ns2.$n > /dev/null || ret=1
 grep "must also be removed from named.conf." rndc.out.ns2.$n > /dev/null || ret=1
+_check_delete_normally_loaded_zone() (
+	$DIG $DIGOPTS @10.53.0.2 a.normal.example a > dig.out.ns2.$n &&
+	grep 'status: REFUSED' dig.out.ns2.$n > /dev/null
+)
+retry_quiet 5 _check_delete_normally_loaded_zone || ret=1

-grep 'status: REFUSED' dig.out.ns2.$n > /dev/null || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`

 echo_i "attempting to add master zone with inline signing ($n)"
 $RNDCCMD 10.53.0.2 addzone 'inline.example { type master; file "inline.db"; inline-signing yes; };' 2>&1 | sed 's/^/I:ns2 /'
-for i in 1 2 3 4 5
-do
-ret=0
-$DIG $DIGOPTS @10.53.0.2 a.inline.example a > dig.out.ns2.$n || ret=1
-grep 'status: NOERROR' dig.out.ns2.$n > /dev/null || ret=1
-grep '^a.inline.example' dig.out.ns2.$n > /dev/null || ret=1
-[ $ret = 0 ] && break
-sleep 1
-done
+_check_add_master_zone_with_inline() (
+	$DIG $DIGOPTS @10.53.0.2 a.inline.example a > dig.out.ns2.$n &&
+	grep 'status: NOERROR' dig.out.ns2.$n > /dev/null &&
+	grep '^a.inline.example' dig.out.ns2.$n > /dev/null
+)
+retry_quiet 5 _check_add_master_zone_with_inline || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
@@ -273,27 +294,20 @@ status=`expr $status + $ret`

 echo_i "attempting to add slave zone with inline signing ($n)"
 $RNDCCMD 10.53.0.2 addzone 'inlineslave.example { type slave; masters { 10.53.0.1; }; file "inlineslave.bk"; inline-signing yes; };' 2>&1 | sed 's/^/I:ns2 /'
-for i in 1 2 3 4 5
-do
-ret=0
-$DIG $DIGOPTS @10.53.0.2 a.inlineslave.example a > dig.out.ns2.$n || ret=1
-grep 'status: NOERROR' dig.out.ns2.$n > /dev/null || ret=1
-grep '^a.inlineslave.example' dig.out.ns2.$n > /dev/null || ret=1
-[ $ret = 0 ] && break
-sleep 1
-done
+_check_add_slave_with_inline() (
+	$DIG $DIGOPTS @10.53.0.2 a.inlineslave.example a > dig.out.ns2.$n &&
+	grep 'status: NOERROR' dig.out.ns2.$n > /dev/null &&
+	grep '^a.inlineslave.example' dig.out.ns2.$n > /dev/null
+)
+retry_quiet 5 _check_add_slave_with_inline || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`

 echo_i "attempting to delete slave zone with inline signing ($n)"
 ret=0
-for i in 0 1 2 3 4 5 6 7 8 9
-do
-	test -f ns2/inlineslave.bk.signed -a -f ns2/inlineslave.bk && break
-	sleep 1
-done
-$RNDCCMD 10.53.0.2 delzone inlineslave.example 2>&1 > rndc.out2.test$n
+retry_quiet 10 test -f ns2/inlineslave.bk.signed -a -f ns2/inlineslave.bk || ret=1
+$RNDCCMD 10.53.0.2 delzone inlineslave.example > rndc.out2.test$n 2>&1 || ret=1
 test -f inlineslave.bk ||
 grep '^inlineslave.bk$' rndc.out2.test$n > /dev/null || {
 	echo_i "failed to report inlineslave.bk"; ret=1;
@@ -307,43 +321,30 @@ status=`expr $status + $ret`

 echo_i "restoring slave zone with inline signing ($n)"
 $RNDCCMD 10.53.0.2 addzone 'inlineslave.example { type slave; masters { 10.53.0.1; }; file "inlineslave.bk"; inline-signing yes; };' 2>&1 | sed 's/^/I:ns2 /'
-for i in 1 2 3 4 5
-do
-ret=0
-$DIG $DIGOPTS @10.53.0.2 a.inlineslave.example a > dig.out.ns2.$n || ret=1
-grep 'status: NOERROR' dig.out.ns2.$n > /dev/null || ret=1
-grep '^a.inlineslave.example' dig.out.ns2.$n > /dev/null || ret=1
-[ $ret = 0 ] && break
-sleep 1
-done
+_check_restoring_slave_with_inline() (
+	$DIG $DIGOPTS @10.53.0.2 a.inlineslave.example a > dig.out.ns2.$n &&
+	grep 'status: NOERROR' dig.out.ns2.$n > /dev/null &&
+	grep '^a.inlineslave.example' dig.out.ns2.$n > /dev/null
+)
+retry_quiet 5 _check_restoring_slave_with_inline || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`

 echo_i "deleting slave zone with automatic zone file removal ($n)"
 ret=0
-for i in 0 1 2 3 4 5 6 7 8 9
-do
-	test -f ns2/inlineslave.bk.signed -a -f ns2/inlineslave.bk && break
-	sleep 1
-done
-$RNDCCMD 10.53.0.2 delzone -clean inlineslave.example 2>&1 > /dev/null
-for i in 0 1 2 3 4 5 6 7 8 9
-do
-        ret=0
-	test -f ns2/inlineslave.bk.signed -a -f ns2/inlineslave.bk && ret=1
-        [ $ret = 0 ] && break
-	sleep 1
-done
+retry_quiet 10 test -f ns2/inlineslave.bk.signed -a -f ns2/inlineslave.bk || ret=1
+$RNDCCMD 10.53.0.2 delzone -clean inlineslave.example > /dev/null 2>&1
+retry_quiet 10 test ! -f ns2/inlineslave.bk.signed -a ! -f ns2/inlineslave.bk
 n=`expr $n + 1`
 status=`expr $status + $ret`

 echo_i "modifying zone configuration ($n)"
 ret=0
-$RNDCCMD 10.53.0.2 addzone 'mod.example { type master; file "added.db"; };' 2>&1 | sed 's/^/I:ns2 /'
+$RNDCCMD 10.53.0.2 addzone 'mod.example { type master; file "added.db"; };' 2>&1 | sed 's/^/ns2 /' | cat_i
 $DIG +norec $DIGOPTS @10.53.0.2 mod.example ns > dig.out.ns2.1.$n || ret=1
 grep 'status: NOERROR' dig.out.ns2.1.$n > /dev/null || ret=1
-$RNDCCMD 10.53.0.2 modzone 'mod.example { type master; file "added.db"; allow-query { none; }; };' 2>&1 | sed 's/^/I:ns2 /'
+$RNDCCMD 10.53.0.2 modzone 'mod.example { type master; file "added.db"; allow-query { none; }; };' 2>&1 | sed 's/^/ns2 /' | cat_i
 $DIG +norec $DIGOPTS @10.53.0.2 mod.example ns > dig.out.ns2.2.$n || ret=1
 $RNDCCMD 10.53.0.2 showzone mod.example | grep 'allow-query { "none"; };' > /dev/null 2>&1 || ret=1
 n=`expr $n + 1`
@@ -367,11 +368,14 @@ status=`expr $status + $ret`
 echo_i "check that adding a 'master redirect' zone works ($n)"
 ret=0
 $RNDCCMD 10.53.0.2 addzone '"." { type redirect; file "redirect.db"; };' > rndc.out.ns2.$n 2>&1 || ret=1
-$RNDCCMD 10.53.0.2 showzone -redirect > showzone.out.ns2.$n 2>&1 || ret=1
-grep "type redirect;" showzone.out.ns2.$n > /dev/null || ret=1
-$RNDCCMD 10.53.0.2 zonestatus -redirect > zonestatus.out.ns2.$n 2>&1 || ret=1
-grep "type: redirect" zonestatus.out.ns2.$n > /dev/null || ret=1
-grep "serial: 0" zonestatus.out.ns2.$n > /dev/null || ret=1
+_check_add_master_redirect() (
+	$RNDCCMD 10.53.0.2 showzone -redirect > showzone.out.ns2.$n 2>&1 &&
+	grep "type redirect;" showzone.out.ns2.$n > /dev/null &&
+	$RNDCCMD 10.53.0.2 zonestatus -redirect > zonestatus.out.ns2.$n 2>&1 &&
+	grep "type: redirect" zonestatus.out.ns2.$n > /dev/null &&
+	grep "serial: 0" zonestatus.out.ns2.$n > /dev/null
+)
+retry_quiet 10 _check_add_master_redirect || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
@@ -381,9 +385,7 @@ ret=0
 sleep 1
 cp -f ns2/redirect.db.2 ns2/redirect.db
 $RNDCCMD 10.53.0.2 reload -redirect > rndc.out.ns2.$n
-$RNDCCMD 10.53.0.2 zonestatus -redirect > zonestatus.out.ns2.$n 2>&1 || ret=1
-grep "type: redirect" zonestatus.out.ns2.$n > /dev/null || ret=1
-grep "serial: 1" zonestatus.out.ns2.$n > /dev/null || ret=1
+retry_quiet 10 check_zonestatus 2 || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
@@ -398,8 +400,11 @@ status=`expr $status + $ret`
 echo_i "check that deleting a 'master redirect' zone works ($n)"
 ret=0
 $RNDCCMD 10.53.0.2 delzone -redirect > rndc.out.ns2.$n 2>&1 || ret=1
-$RNDCCMD 10.53.0.2 showzone -redirect > showzone.out.ns2.$n 2>&1
-grep 'not found' showzone.out.ns2.$n > /dev/null || ret=1
+_check_deleting_master_redirect() (
+	$RNDCCMD 10.53.0.2 showzone -redirect > showzone.out.ns2.$n 2>&1 || true
+	grep 'not found' showzone.out.ns2.$n > /dev/null
+)
+retry_quiet 10 _check_deleting_master_redirect || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
@@ -407,12 +412,14 @@ status=`expr $status + $ret`
 echo_i "check that adding a 'slave redirect' zone works ($n)"
 ret=0
 $RNDCCMD 10.53.0.2 addzone '"." { type redirect; masters { 10.53.0.3;}; file "redirect.bk"; };' > rndc.out.ns2.$n 2>&1 || ret=1
-$RNDCCMD 10.53.0.2 showzone -redirect > showzone.out.ns2.$n 2>&1 || ret=1
-grep "type redirect;" showzone.out.ns2.$n > /dev/null || ret=1
-sleep 1
-$RNDCCMD 10.53.0.2 zonestatus -redirect > zonestatus.out.ns2.$n 2>&1 || ret=1
-grep "type: redirect" zonestatus.out.ns2.$n > /dev/null || ret=1
-grep "serial: 0" zonestatus.out.ns2.$n > /dev/null || ret=1
+_check_adding_slave_redirect() (
+	$RNDCCMD 10.53.0.2 showzone -redirect > showzone.out.ns2.$n 2>&1 &&
+	grep "type redirect;" showzone.out.ns2.$n > /dev/null &&
+	$RNDCCMD 10.53.0.2 zonestatus -redirect > zonestatus.out.ns2.$n 2>&1 &&
+	grep "type: redirect" zonestatus.out.ns2.$n > /dev/null &&
+	grep "serial: 0" zonestatus.out.ns2.$n > /dev/null
+)
+retry_quiet 10 _check_adding_slave_redirect || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
@@ -421,12 +428,13 @@ echo_i "check that retransfering a added 'slave redirect' zone works ($n)"
 ret=0
 cp -f ns3/redirect.db.2 ns3/redirect.db
 $RNDCCMD 10.53.0.3 reload . > showzone.out.ns3.$n 2>&1 || ret=1
-sleep 1
-$RNDCCMD 10.53.0.2 retransfer -redirect > rndc.out.ns2.$n 2>&1 || ret=1
-sleep 1
-$RNDCCMD 10.53.0.2 zonestatus -redirect > zonestatus.out.ns2.$n 2>&1 || ret=1
-grep "type: redirect" zonestatus.out.ns2.$n > /dev/null || ret=1
-grep "serial: 1" zonestatus.out.ns2.$n > /dev/null || ret=1
+_check_retransfering_slave_redirect() (
+	$RNDCCMD 10.53.0.2 retransfer -redirect > rndc.out.ns2.$n 2>&1 &&
+	$RNDCCMD 10.53.0.2 zonestatus -redirect > zonestatus.out.ns2.$n 2>&1 &&
+	grep "type: redirect" zonestatus.out.ns2.$n > /dev/null &&
+	grep "serial: 1" zonestatus.out.ns2.$n > /dev/null
+)
+retry_quiet 10 _check_retransfering_slave_redirect || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
@@ -434,8 +442,11 @@ status=`expr $status + $ret`
 echo_i "check that deleting a 'slave redirect' zone works ($n)"
 ret=0
 $RNDCCMD 10.53.0.2 delzone -redirect > rndc.out.ns2.$n 2>&1 || ret=1
-$RNDCCMD 10.53.0.2 showzone -redirect > showzone.out.ns2.$n 2>&1
-grep 'not found' showzone.out.ns2.$n > /dev/null || ret=1
+_check_deleting_slave_redirect() (
+	$RNDCCMD 10.53.0.2 showzone -redirect > showzone.out.ns2.$n 2>&1 || true
+	grep 'not found' showzone.out.ns2.$n > /dev/null
+)
+retry_quiet 10 _check_deleting_slave_redirect || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
@@ -520,31 +531,30 @@ fi

 echo_i "checking rndc reload causes named to reload the external view's new zone config ($n)"
 ret=0
-$RNDCCMD 10.53.0.2 reload 2>&1 | sed 's/^/I:ns2 /'
-$DIG +norec $DIGOPTS @10.53.0.2 -b 10.53.0.2 a.added.example a > dig.out.ns2.int.$n || ret=1
-grep 'status: NOERROR' dig.out.ns2.int.$n > /dev/null || ret=1
-$DIG +norec $DIGOPTS @10.53.0.4 -b 10.53.0.4 a.added.example a > dig.out.ns2.ext.$n || ret=1
-grep 'status: NOERROR' dig.out.ns2.ext.$n > /dev/null || ret=1
-grep '^a.added.example' dig.out.ns2.ext.$n > /dev/null || ret=1
+$RNDCCMD 10.53.0.2 reload 2>&1 | sed 's/^/ns2 /' | cat_i
+_check_rndc_reload_external_view_config() (
+	$DIG +norec $DIGOPTS @10.53.0.2 -b 10.53.0.2 a.added.example a > dig.out.ns2.int.$n &&
+	grep 'status: NOERROR' dig.out.ns2.int.$n > /dev/null &&
+	$DIG +norec $DIGOPTS @10.53.0.4 -b 10.53.0.4 a.added.example a > dig.out.ns2.ext.$n &&
+	grep 'status: NOERROR' dig.out.ns2.ext.$n > /dev/null &&
+	grep '^a.added.example' dig.out.ns2.ext.$n > /dev/null
+)
+retry_quiet 10 _check_rndc_reload_external_view_config || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`

 echo_i "checking rndc showzone with newly added zone ($n)"
-# loop because showzone may complain if zones are still being
-# loaded from the NZDB at this point.
-for try in 0 1 2 3 4 5; do
-    ret=0
-$RNDCCMD 10.53.0.2 showzone added.example in external > rndc.out.ns2.$n 2>/dev/null
-    if [ -z "$NZD" ]; then
-      expected='zone "added.example" in external { type master; file "added.db"; };'
-    else
-      expected='zone "added.example" { type master; file "added.db"; };'
-    fi
-    [ "`cat rndc.out.ns2.$n`" = "$expected" ] || ret=1
-    [ $ret -eq 0 ] && break
-    sleep 1
-done
+_check_rndc_showzone_newly_added() (
+	if [ -z "$NZD" ]; then
+		expected='zone "added.example" in external { type master; file "added.db"; };'
+	else
+		expected='zone "added.example" { type master; file "added.db"; };'
+	fi
+	$RNDCCMD 10.53.0.2 showzone added.example in external > rndc.out.ns2.$n 2>/dev/null &&
+	[ "`cat rndc.out.ns2.$n`" = "$expected" ]
+)
+retry_quiet 10  _check_rndc_showzone_newly_added || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
@@ -552,9 +562,12 @@ status=`expr $status + $ret`
 echo_i "deleting newly added zone ($n)"
 ret=0
 $RNDCCMD 10.53.0.2 delzone 'added.example in external' 2>&1 | sed 's/^/I:ns2 /'
-$DIG $DIGOPTS @10.53.0.4 -b 10.53.0.4 a.added.example a > dig.out.ns2.$n || ret=1
-grep 'status: REFUSED' dig.out.ns2.$n > /dev/null || ret=1
-grep '^a.added.example' dig.out.ns2.$n > /dev/null && ret=1
+_check_deleting_newly_added_zone() (
+	$DIG $DIGOPTS @10.53.0.4 -b 10.53.0.4 a.added.example a > dig.out.ns2.$n &&
+	grep 'status: REFUSED' dig.out.ns2.$n > /dev/null &&
+	! grep '^a.added.example' dig.out.ns2.$n > /dev/null
+)
+retry_quiet 10 _check_deleting_newly_added_zone || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
@@ -584,11 +597,14 @@ status=`expr $status + $ret`
 echo_i "adding new zone again to external view ($n)"
 ret=0
 $RNDCCMD 10.53.0.2 addzone 'added.example in external { type master; file "added.db"; };' 2>&1 | sed 's/^/I:ns2 /'
-$DIG +norec $DIGOPTS @10.53.0.2 -b 10.53.0.2 a.added.example a > dig.out.ns2.int.$n || ret=1
-grep 'status: NOERROR' dig.out.ns2.int.$n > /dev/null || ret=1
-$DIG +norec $DIGOPTS @10.53.0.4 -b 10.53.0.4 a.added.example a > dig.out.ns2.ext.$n || ret=1
-grep 'status: NOERROR' dig.out.ns2.ext.$n > /dev/null || ret=1
-grep '^a.added.example' dig.out.ns2.ext.$n > /dev/null || ret=1
+_check_adding_new_zone_again_external() (
+	$DIG +norec $DIGOPTS @10.53.0.2 -b 10.53.0.2 a.added.example a > dig.out.ns2.int.$n &&
+	grep 'status: NOERROR' dig.out.ns2.int.$n > /dev/null &&
+	$DIG +norec $DIGOPTS @10.53.0.4 -b 10.53.0.4 a.added.example a > dig.out.ns2.ext.$n &&
+	grep 'status: NOERROR' dig.out.ns2.ext.$n > /dev/null &&
+	grep '^a.added.example' dig.out.ns2.ext.$n > /dev/null
+)
+retry_quiet 10 _check_adding_new_zone_again_external || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
@@ -693,20 +709,18 @@ then
    status=`expr $status + $ret`
 fi

+_check_version_bind() (
+	$DIG $DIGOPTS @10.53.0.3 version.bind txt ch > dig.out.test$n &&
+	grep "status: NOERROR" dig.out.test$n > /dev/null
+)
+
 echo_i "check that named restarts with multiple added zones ($n)"
 ret=0
 $RNDCCMD 10.53.0.3 addzone "test4.baz" '{ type master; file "e.db"; };' > /dev/null 2>&1 || ret=1
 $RNDCCMD 10.53.0.3 addzone "test5.baz" '{ type master; file "e.db"; };' > /dev/null 2>&1 || ret=1
 $PERL $SYSTEMTESTTOP/stop.pl addzone ns3
 $PERL $SYSTEMTESTTOP/start.pl --noclean --restart --port ${PORT} addzone ns3 || ret=1
-for try in 0 1 2 3 4 5 6 7 8 9; do
-    iret=0
-    $DIG $DIGOPTS @10.53.0.3 version.bind txt ch > dig.out.test$n || iret=1
-    grep "status: NOERROR" dig.out.test$n > /dev/null || iret=1
-    [ "$iret" -eq 0 ] && break
-    sleep 1
-done
-[ "$iret" -ne 0 ] && ret=1
+retry_quiet 10 _check_version_bind || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
 n=`expr $n + 1`
@@ -14,7 +14,7 @@ rm -f */core
 rm -f */example.bk
 rm -f */named.conf
 rm -f */named.memstats
-rm -f */named.run
+rm -f */named.run*
 rm -f */trusted.conf */private.conf
 rm -f activate-now-publish-1day.key
 rm -f active.key inact.key del.key delzsk.key unpub.key standby.key rev.key
@@ -63,6 +63,7 @@ rm -f ns3/secure.nsec3.example.db
 rm -f ns3/secure.optout.example.db
 rm -f ns3/sync.example.db
 rm -f ns3/ttl*.db
+rm -f ns3/settime.out.*
 rm -f nsupdate.out
 rm -f settime.out.*
 rm -f signing.out.*
@@ -50,6 +50,13 @@ checkprivate () {
    return 1
 }

+# wait until notifies for zone $1 are sent by server $2. This is an indication
+# that the zone is signed with the active keys, and the changes have been
+# committed.
+wait_for_notifies () {
+	wait_for_log 10 "zone ${1}/IN: sending notifies" "${2}/named.run" || return 1
+}
+
 freq() {
 	_file=$1
 	# remove first and last line that has incomplete set and skews the distribution
@@ -177,6 +184,9 @@ do
 	$DIG $DIGOPTS $z @10.53.0.3 axfr | awk '$4 == "RRSIG" {print $9}' | sort | uniq -c | cat_i
 done

+# Set logfile offset for wait_for_log usage.
+nextpartreset ns3/named.run
+
 #
 # Check that DNSKEY is initially signed with a KSK and not a ZSK.
 #
@@ -207,8 +217,8 @@ test $count -eq 3 || ret=1
 awk='$4 == "RRSIG" && $5 == "DNSKEY" { printf "%05u\n", $11 }'
 id=`awk "${awk}" dig.out.ns3.test$n`

-$SETTIME -D now+5 ns3/Kinacksk3.example.+007+${id} > /dev/null 2>&1
-$RNDCCMD 10.53.0.3 loadkeys inacksk3.example 2>&1 | sed 's/^/ns3 /' | cat_i
+$SETTIME -D now+5 ns3/Kinacksk3.example.+007+${id} > settime.out.test$n || ret=1
+($RNDCCMD 10.53.0.3 loadkeys inacksk3.example 2>&1 | sed 's/^/ns3 /' | cat_i) || ret=1

 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
@@ -235,8 +245,8 @@ count=`awk 'BEGIN { count = 0 }
       END {print count}' dig.out.ns3.test$n`
 test $count -eq 3 || ret=1
 id=`awk '$4 == "RRSIG" && $5 == "CNAME" { printf "%05u\n", $11 }' dig.out.ns3.test$n`
-$SETTIME -D now+5 ns3/Kinaczsk3.example.+007+${id} > /dev/null 2>&1
-$RNDCCMD 10.53.0.3 loadkeys inaczsk3.example 2>&1 | sed 's/^/ns3 /' | cat_i
+$SETTIME -D now+5 ns3/Kinaczsk3.example.+007+${id} > settime.out.test$n || ret=1
+($RNDCCMD 10.53.0.3 loadkeys inaczsk3.example 2>&1 | sed 's/^/ns3 /' | cat_i) || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
@@ -332,9 +342,9 @@ status=`expr $status + $ret`
 echo_i "signing preset nsec3 zone"
 zsk=`cat autozsk.key`
 ksk=`cat autoksk.key`
-$SETTIME -K ns3 -P now -A now $zsk > /dev/null 2>&1
-$SETTIME -K ns3 -P now -A now $ksk > /dev/null 2>&1
-$RNDCCMD 10.53.0.3 loadkeys autonsec3.example. 2>&1 | sed 's/^/ns3 /' | cat_i
+$SETTIME -K ns3 -P now -A now $zsk > settime.out.test$n.zsk || ret=1
+$SETTIME -K ns3 -P now -A now $ksk > settime.out.test$n.ksk || ret=1
+($RNDCCMD 10.53.0.3 loadkeys autonsec3.example. 2>&1 | sed 's/^/ns3 /' | cat_i) || ret=1

 echo_i "waiting for changes to take effect"
 sleep 3
@@ -387,9 +397,9 @@ status=`expr $status + $ret`
 # Send rndc sync command to ns1, ns2 and ns3, to force the dynamically
 # signed zones to be dumped to their zone files
 echo_i "dumping zone files"
-$RNDCCMD 10.53.0.1 sync 2>&1 | sed 's/^/ns1 /' | cat_i
-$RNDCCMD 10.53.0.2 sync 2>&1 | sed 's/^/ns2 /' | cat_i
-$RNDCCMD 10.53.0.3 sync 2>&1 | sed 's/^/ns3 /' | cat_i
+($RNDCCMD 10.53.0.1 sync 2>&1 | sed 's/^/ns1 /' | cat_i) || ret=1
+($RNDCCMD 10.53.0.2 sync 2>&1 | sed 's/^/ns2 /' | cat_i) || ret=1
+($RNDCCMD 10.53.0.3 sync 2>&1 | sed 's/^/ns3 /' | cat_i) || ret=1

 now="$(TZ=UTC date +%Y%m%d%H%M%S)"
 check_expiry() (
@@ -488,7 +498,7 @@ echo_i "checking TTLs of imported DNSKEYs (no default) ($n)"
 ret=0
 $DIG $DIGOPTS +tcp +noall +answer dnskey ttl1.example. @10.53.0.3 > dig.out.ns3.test$n || ret=1
 [ -s dig.out.ns3.test$n ] || ret=1
-awk 'BEGIN {r=0} $2 != 300 {r=1; print "found TTL " $2} END {exit r}' dig.out.ns3.test$n || ret=1 | cat_i
+(awk 'BEGIN {r=0} $2 != 300 {r=1; print "found TTL " $2} END {exit r}' dig.out.ns3.test$n | cat_i) || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
@@ -497,7 +507,7 @@ echo_i "checking TTLs of imported DNSKEYs (with default) ($n)"
 ret=0
 $DIG $DIGOPTS +tcp +noall +answer dnskey ttl2.example. @10.53.0.3 > dig.out.ns3.test$n || ret=1
 [ -s dig.out.ns3.test$n ] || ret=1
-awk 'BEGIN {r=0} $2 != 60 {r=1; print "found TTL " $2} END {exit r}' dig.out.ns3.test$n || ret=1 | cat_i
+(awk 'BEGIN {r=0} $2 != 60 {r=1; print "found TTL " $2} END {exit r}' dig.out.ns3.test$n | cat_i) || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
@@ -506,7 +516,7 @@ echo_i "checking TTLs of imported DNSKEYs (mismatched) ($n)"
 ret=0
 $DIG $DIGOPTS +tcp +noall +answer dnskey ttl3.example. @10.53.0.3 > dig.out.ns3.test$n || ret=1
 [ -s dig.out.ns3.test$n ] || ret=1
-awk 'BEGIN {r=0} $2 != 30 {r=1; print "found TTL " $2} END {exit r}' dig.out.ns3.test$n || ret=1 | cat_i
+(awk 'BEGIN {r=0} $2 != 30 {r=1; print "found TTL " $2} END {exit r}' dig.out.ns3.test$n | cat_i) || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
@@ -515,7 +525,7 @@ echo_i "checking TTLs of imported DNSKEYs (existing RRset) ($n)"
 ret=0
 $DIG $DIGOPTS +tcp +noall +answer dnskey ttl4.example. @10.53.0.3 > dig.out.ns3.test$n || ret=1
 [ -s dig.out.ns3.test$n ] || ret=1
-awk 'BEGIN {r=0} $2 != 30 {r=1; print "found TTL " $2} END {exit r}' dig.out.ns3.test$n || ret=1 | cat_i
+(awk 'BEGIN {r=0} $2 != 30 {r=1; print "found TTL " $2} END {exit r}' dig.out.ns3.test$n | cat_i) || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
@@ -1006,10 +1016,10 @@ status=`expr $status + $ret`
 echo_i "checking secure-to-insecure transition, scheduled ($n)"
 ret=0
 file="ns3/`cat del1.key`.key"
-$SETTIME -I now -D now $file > /dev/null
+$SETTIME -I now -D now $file > settime.out.test$n.1 || ret=1
 file="ns3/`cat del2.key`.key"
-$SETTIME -I now -D now $file > /dev/null
-$RNDCCMD 10.53.0.3 sign secure-to-insecure2.example. 2>&1 | sed 's/^/ns3 /' | cat_i
+$SETTIME -I now -D now $file > settime.out.test$n.2 || ret=1
+($RNDCCMD 10.53.0.3 sign secure-to-insecure2.example. 2>&1 | sed 's/^/ns3 /' | cat_i) || ret=1
 for i in 0 1 2 3 4 5 6 7 8 9; do
 	ret=0
 	$DIG $DIGOPTS axfr secure-to-insecure2.example @10.53.0.3 > dig.out.ns3.test$n || ret=1
@@ -1035,7 +1045,7 @@ END
 # Create DNSSEC keys in the zone directory.
 $KEYGEN -a rsasha1 -3 -q -K ns3 jitter.nsec3.example > /dev/null
 # Trigger zone signing.
-$RNDCCMD 10.53.0.3 sign jitter.nsec3.example. 2>&1 | sed 's/^/ns3 /' | cat_i
+($RNDCCMD 10.53.0.3 sign jitter.nsec3.example. 2>&1 | sed 's/^/ns3 /' | cat_i) || ret=1
 # Wait until zone has been signed.
 check_if_nsec3param_exists() {
 	$DIG $DIGOPTS NSEC3PARAM jitter.nsec3.example @10.53.0.3 > dig.out.ns3.1.test$n || return 1
@@ -1056,7 +1066,7 @@ oldinception=`$DIG $DIGOPTS +short soa prepub.example @10.53.0.3 | awk '/SOA/ {p

 $KEYGEN -a rsasha1 -3 -q -K ns3 -P 0 -A +6d -I +38d -D +45d prepub.example > /dev/null

-$RNDCCMD 10.53.0.3 sign prepub.example 2>&1 | sed 's/^/ns1 /' | cat_i
+($RNDCCMD 10.53.0.3 sign prepub.example 2>&1 | sed 's/^/ns1 /' | cat_i) || ret=1
 newserial=$oldserial
 try=0
 while [ $oldserial -eq $newserial -a $try -lt 42 ]
@@ -1087,19 +1097,19 @@ oldfile=`cat active.key`
 oldid=$(keyfile_to_key_id "$(cat active.key)")
 newfile=`cat standby.key`
 newid=$(keyfile_to_key_id "$(cat standby.key)")
-$SETTIME -K ns1 -I now+2s -D now+25 $oldfile > /dev/null
-$SETTIME -K ns1 -i 0 -S $oldfile $newfile > /dev/null
+$SETTIME -K ns1 -I now+2s -D now+25 $oldfile > settime.out.test$n.1 || ret=1
+$SETTIME -K ns1 -i 0 -S $oldfile $newfile > settime.out.test$n.2 || ret=1

 # note previous zone serial number
 oldserial=`$DIG $DIGOPTS +short soa . @10.53.0.1 | awk '{print $3}'`

-$RNDCCMD 10.53.0.1 loadkeys . 2>&1 | sed 's/^/ns1 /' | cat_i
+($RNDCCMD 10.53.0.1 loadkeys . 2>&1 | sed 's/^/ns1 /' | cat_i) || ret=1
 sleep 4

 echo_i "revoking key to duplicated key ID"
-$SETTIME -R now -K ns2 Kbar.+005+30676.key > /dev/null 2>&1
+$SETTIME -R now -K ns2 Kbar.+005+30676.key > settime.out.test$n.3 || ret=1

-$RNDCCMD 10.53.0.2 loadkeys bar. 2>&1 | sed 's/^/ns2 /' | cat_i
+($RNDCCMD 10.53.0.2 loadkeys bar. 2>&1 | sed 's/^/ns2 /' | cat_i) || ret=1

 echo_i "waiting for changes to take effect"
 sleep 5
@@ -1152,7 +1162,7 @@ n=`expr $n + 1`
 status=`expr $status + $ret`

 echo_i "forcing full sign"
-$RNDCCMD 10.53.0.1 sign . 2>&1 | sed 's/^/ns1 /' | cat_i
+($RNDCCMD 10.53.0.1 sign . 2>&1 | sed 's/^/ns1 /' | cat_i) || ret=1

 echo_i "waiting for change to take effect"
 sleep 5
@@ -1178,8 +1188,10 @@ ret=0
 zsk=`cat delayzsk.key`
 ksk=`cat delayksk.key`
 # publication and activation times should be unset
-$SETTIME -K ns3 -pA -pP $zsk | grep -v UNSET > /dev/null 2>&1 && ret=1
-$SETTIME -K ns3 -pA -pP $ksk | grep -v UNSET > /dev/null 2>&1 && ret=1
+$SETTIME -K ns3 -pA -pP $zsk > settime.out.test$n.zsk || ret=1
+grep -v UNSET settime.out.test$n.zsk >/dev/null && ret=1
+$SETTIME -K ns3 -pA -pP $ksk > settime.out.test$n.ksk || ret=1
+grep -v UNSET settime.out.test$n.ksk >/dev/null && ret=1
 $DIG $DIGOPTS +noall +answer dnskey delay.example. @10.53.0.3 > dig.out.ns3.test$n || ret=1
 # DNSKEY not expected:
 awk 'BEGIN {r=1} $4=="DNSKEY" {r=0} END {exit r}' dig.out.ns3.test$n && ret=1
@@ -1189,12 +1201,14 @@ status=`expr $status + $ret`

 echo_i "checking scheduled key publication, not activation ($n)"
 ret=0
-$SETTIME -K ns3 -P now+3s -A none $zsk > /dev/null 2>&1
-$SETTIME -K ns3 -P now+3s -A none $ksk > /dev/null 2>&1
-$RNDCCMD 10.53.0.3 loadkeys delay.example. 2>&1 | sed 's/^/ns2 /' | cat_i
-
+# Ensure initial zone is loaded.
+wait_for_notifies "delay.example" "ns3" || ret=1
+$SETTIME -K ns3 -P now+3s -A none $zsk > settime.out.test$n.zsk || ret=1
+$SETTIME -K ns3 -P now+3s -A none $ksk > settime.out.test$n.ksk || ret=1
+($RNDCCMD 10.53.0.3 loadkeys delay.example. 2>&1 | sed 's/^/ns2 /' | cat_i) || ret=1
 echo_i "waiting for changes to take effect"
-sleep 5
+sleep 3
+wait_for_notifies "delay.example" "ns3" || ret=1

 $DIG $DIGOPTS +noall +answer dnskey delay.example. @10.53.0.3 > dig.out.ns3.test$n || ret=1
 # DNSKEY expected:
@@ -1207,13 +1221,12 @@ status=`expr $status + $ret`

 echo_i "checking scheduled key activation ($n)"
 ret=0
-$SETTIME -K ns3 -A now+3s $zsk > /dev/null 2>&1
-$SETTIME -K ns3 -A now+3s $ksk > /dev/null 2>&1
-$RNDCCMD 10.53.0.3 loadkeys delay.example. 2>&1 | sed 's/^/ns2 /' | cat_i
-
+$SETTIME -K ns3 -A now+3s $zsk > settime.out.test$n.zsk || ret=1
+$SETTIME -K ns3 -A now+3s $ksk > settime.out.test$n.ksk || ret=1
+($RNDCCMD 10.53.0.3 loadkeys delay.example. 2>&1 | sed 's/^/ns2 /' | cat_i) || ret=1
 echo_i "waiting for changes to take effect"
-sleep 5
-
+sleep 3
+wait_for_notifies "delay.example" "ns3" || ret=1
 $DIG $DIGOPTS +noall +answer dnskey delay.example. @10.53.0.3 > dig.out.ns3.1.test$n || ret=1
 # DNSKEY expected:
 awk 'BEGIN {r=1} $4=="DNSKEY" {r=0} END {exit r}' dig.out.ns3.1.test$n || ret=1
@@ -1323,7 +1336,7 @@ status=`expr $status + $ret`
 echo_i "forcing full sign with unreadable keys ($n)"
 ret=0
 chmod 0 ns1/K.+*+*.key ns1/K.+*+*.private || ret=1
-$RNDCCMD 10.53.0.1 sign . 2>&1 | sed 's/^/ns1 /' | cat_i
+($RNDCCMD 10.53.0.1 sign . 2>&1 | sed 's/^/ns1 /' | cat_i) || ret=1
 $DIG $DIGOPTS . @10.53.0.1 dnskey > dig.out.ns1.test$n || ret=1
 grep "status: NOERROR" dig.out.ns1.test$n > /dev/null || ret=1
 n=`expr $n + 1`
@@ -1333,11 +1346,11 @@ status=`expr $status + $ret`
 echo_i "test turning on auto-dnssec during reconfig ($n)"
 ret=0
 # first create a zone that doesn't have auto-dnssec
-$RNDCCMD 10.53.0.3 addzone reconf.example '{ type master; file "reconf.example.db"; };' 2>&1 | sed 's/^/ns3 /' | cat_i
+($RNDCCMD 10.53.0.3 addzone reconf.example '{ type master; file "reconf.example.db"; };' 2>&1 | sed 's/^/ns3 /' | cat_i) || ret=1
 rekey_calls=`grep "zone reconf.example.*next key event" ns3/named.run | wc -l`
 [ "$rekey_calls" -eq 0 ] || ret=1
 # ...then we add auto-dnssec and reconfigure
-$RNDCCMD 10.53.0.3 modzone reconf.example '{ type master; file "reconf.example.db"; allow-update { any; }; auto-dnssec maintain; };' 2>&1 | sed 's/^/ns3 /' | cat_i
+($RNDCCMD 10.53.0.3 modzone reconf.example '{ type master; file "reconf.example.db"; allow-update { any; }; auto-dnssec maintain; };' 2>&1 | sed 's/^/ns3 /' | cat_i) || ret=1
 rndc_reconfig ns3 10.53.0.3
 for i in 0 1 2 3 4 5 6 7 8 9; do
    lret=0
@@ -1393,8 +1406,8 @@ if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`

 echo_i "setting CDS and CDNSKEY deletion times and calling 'rndc loadkeys'"
-$SETTIME -D sync now `cat sync.key` > /dev/null
-$RNDCCMD 10.53.0.3 loadkeys sync.example | sed 's/^/ns3 /' | cat_i
+$SETTIME -D sync now `cat sync.key` > settime.out.test$n || ret=1
+($RNDCCMD 10.53.0.3 loadkeys sync.example | sed 's/^/ns3 /' | cat_i) || ret=1

 echo_i "checking that the CDS and CDNSKEY are deleted ($n)"
 ret=0
@@ -1411,16 +1424,16 @@ status=`expr $status + $ret`

 echo_i "check that dnssec-settime -p Dsync works ($n)"
 ret=0
-$SETTIME -p Dsync `cat sync.key` > settime.out.$n|| ret=0
-grep "SYNC Delete:" settime.out.$n >/dev/null || ret=0
+$SETTIME -p Dsync `cat sync.key` > settime.out.test$n || ret=1
+grep "SYNC Delete:" settime.out.test$n >/dev/null || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`

 echo_i "check that dnssec-settime -p Psync works ($n)"
 ret=0
-$SETTIME -p Psync `cat sync.key` > settime.out.$n|| ret=0
-grep "SYNC Publish:" settime.out.$n >/dev/null || ret=0
+$SETTIME -p Psync `cat sync.key` > settime.out.test$n || ret=1
+grep "SYNC Publish:" settime.out.test$n >/dev/null || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
@@ -1527,9 +1540,9 @@ if [ $_ret -ne 0 ]; then
 fi
 # Mark the inactive ZSK as pending removal.
 file="ns3/`cat delzsk.key`.key"
-$SETTIME -D now-1h $file > settime.out.test$n 2>&1 || ret=1
+$SETTIME -D now-1h $file > settime.out.test$n || ret=1
 # Trigger removal of the inactive ZSK and wait until its completion.
-$RNDCCMD 10.53.0.3 loadkeys delzsk.example 2>&1 | sed 's/^/ns3 /' | cat_i
+($RNDCCMD 10.53.0.3 loadkeys delzsk.example 2>&1 | sed 's/^/ns3 /' | cat_i) || ret=1
 for i in 0 1 2 3 4 5 6 7 8 9; do
 	_ret=1
 	$RNDCCMD 10.53.0.3 signing -list delzsk.example > signing.out.3.test$n 2>&1
@@ -38,7 +38,7 @@ _wait_for_rcode() (
 	ns="$3"
 	qname="$4"
 	file="$5"
-	shift 4
+	shift 5
 	dig_with_opts "$ns" "$qtype" "$qname" "$@" >"$file" || return 1
 	grep "status: $rcode" "$file" >/dev/null
 )
@@ -374,6 +374,45 @@ nextpartpeek() {
 	nextpartread $1 2> /dev/null
 }

+# _search_log: look for message $1 in file $2 with nextpart().
+_search_log() (
+	msg="$1"
+	file="$2"
+	nextpart "$file" | grep -F "$msg" > /dev/null
+)
+
+# _search_log_peek: look for message $1 in file $2 with nextpartpeek().
+_search_log_peek() (
+	msg="$1"
+	file="$2"
+	nextpartpeek "$file" | grep -F "$msg" > /dev/null
+)
+
+# wait_for_log: wait until message $2 in file $3 appears.  Bail out after
+# $1 seconds.  This needs to be used in conjunction with a prior call to
+# nextpart() or nextpartreset() on the same file to guarantee the offset is
+# set correctly.  Tests using wait_for_log() are responsible for cleaning up
+# the created <file>.prev files.
+wait_for_log() (
+	timeout="$1"
+	msg="$2"
+	file="$3"
+	retry_quiet "$timeout" _search_log "$msg" "$file" && return 0
+	echo_i "exceeded time limit waiting for '$msg' in $file"
+        return 1
+)
+
+# wait_for_log_peek: similar to wait_for_log() but peeking, so the file offset
+# does not change.
+wait_for_log_peek() (
+	timeout="$1"
+	msg="$2"
+	file="$3"
+	retry_quiet "$timeout" _search_log_peek "$msg" "$file" && return 0
+	echo_i "exceeded time limit waiting for '$msg' in $file"
+        return 1
+)
+
 # _retry: keep running a command until it succeeds, up to $1 times, with
 # one-second intervals, optionally printing a message upon every attempt
 _retry() {
@@ -40,25 +40,13 @@ rndccmd() {
    "$RNDC" -c "$SYSTEMTESTTOP/common/rndc.conf" -p "$CONTROLPORT" -s "$@"
 }

-# TODO: Move wait_for_log and loadkeys_on to conf.sh.common
-wait_for_log() {
-        msg=$1
-        file=$2
-
-        for i in 1 2 3 4 5 6 7 8 9 10; do
-                nextpart "$file" | grep "$msg" > /dev/null && return
-                sleep 1
-        done
-        echo_i "exceeded time limit waiting for '$msg' in $file"
-        ret=1
-}
-
+# TODO: Move loadkeys_on to conf.sh.common
 dnssec_loadkeys_on() {
 	nsidx=$1
 	zone=$2
 	nextpart ns${nsidx}/named.run > /dev/null
 	rndccmd 10.53.0.${nsidx} loadkeys ${zone} | sed "s/^/ns${nsidx} /" | cat_i
-	wait_for_log "next key event" ns${nsidx}/named.run
+	wait_for_log 20 "next key event" ns${nsidx}/named.run || return 1
 }

 # convert private-type records to readable form
@@ -10,9 +10,9 @@
 #
 # Clean up after forward tests.
 #
-rm -f dig.out.*
-rm -f */named.conf
-rm -f */named.memstats
-rm -f */named.run
-rm -f ns*/named.lock
-rm -f ns*/managed-keys.bind*
+rm -f ./dig.out.*
+rm -f ./*/named.conf
+rm -f ./*/named.memstats
+rm -f ./*/named.run ./*/named.run.prev
+rm -f ./ns*/named.lock
+rm -f ./ns*/managed-keys.bind*
@@ -7,11 +7,17 @@
 # See the COPYRIGHT file distributed with this work for additional
 # information regarding copyright ownership.

+#shellcheck source=conf.sh
 SYSTEMTESTTOP=..
-. $SYSTEMTESTTOP/conf.sh
+. "$SYSTEMTESTTOP/conf.sh"

-DIGOPTS="-p ${PORT}"
-SENDCMD="$PERL ../send.pl 10.53.0.6 $EXTRAPORT1"
+dig_with_opts() (
+	"$DIG" -p "$PORT" "$@"
+)
+
+sendcmd() (
+	"$PERL" ../send.pl 10.53.0.6 "$EXTRAPORT1"
+)

 root=10.53.0.1
 hidden=10.53.0.2
@@ -19,157 +25,197 @@ f1=10.53.0.3
 f2=10.53.0.4

 status=0
+n=0

-echo_i "checking that a forward zone overrides global forwarders"
+n=$((n+1))
+echo_i "checking that a forward zone overrides global forwarders ($n)"
 ret=0
-$DIG $DIGOPTS +noadd +noauth txt.example1. txt @$hidden > dig.out.hidden || ret=1
-$DIG $DIGOPTS +noadd +noauth txt.example1. txt @$f1 > dig.out.f1 || ret=1
-digcomp dig.out.hidden dig.out.f1 || ret=1
+dig_with_opts +noadd +noauth txt.example1. txt @$hidden > dig.out.$n.hidden || ret=1
+dig_with_opts +noadd +noauth txt.example1. txt @$f1 > dig.out.$n.f1 || ret=1
+digcomp dig.out.$n.hidden dig.out.$n.f1 || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))

-echo_i "checking that a forward first zone no forwarders recurses"
+n=$((n+1))
+echo_i "checking that a forward first zone no forwarders recurses ($n)"
 ret=0
-$DIG $DIGOPTS +noadd +noauth txt.example2. txt @$root > dig.out.root || ret=1
-$DIG $DIGOPTS +noadd +noauth txt.example2. txt @$f1 > dig.out.f1 || ret=1
-digcomp dig.out.root dig.out.f1 || ret=1
+dig_with_opts +noadd +noauth txt.example2. txt @$root > dig.out.$n.root || ret=1
+dig_with_opts +noadd +noauth txt.example2. txt @$f1 > dig.out.$n.f1 || ret=1
+digcomp dig.out.$n.root dig.out.$n.f1 || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))

-echo_i "checking that a forward only zone no forwarders fails"
+n=$((n+1))
+echo_i "checking that a forward only zone no forwarders fails ($n)"
 ret=0
-$DIG $DIGOPTS +noadd +noauth txt.example2. txt @$root > dig.out.root || ret=1
-$DIG $DIGOPTS +noadd +noauth txt.example2. txt @$f1 > dig.out.f1 || ret=1
-digcomp dig.out.root dig.out.f1 || ret=1
+dig_with_opts +noadd +noauth txt.example2. txt @$root > dig.out.$n.root || ret=1
+dig_with_opts +noadd +noauth txt.example2. txt @$f1 > dig.out.$n.f1 || ret=1
+digcomp dig.out.$n.root dig.out.$n.f1 || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))

-echo_i "checking that global forwarders work"
+n=$((n+1))
+echo_i "checking that global forwarders work ($n)"
 ret=0
-$DIG $DIGOPTS +noadd +noauth txt.example4. txt @$hidden > dig.out.hidden || ret=1
-$DIG $DIGOPTS +noadd +noauth txt.example4. txt @$f1 > dig.out.f1 || ret=1
-digcomp dig.out.hidden dig.out.f1 || ret=1
+dig_with_opts +noadd +noauth txt.example4. txt @$hidden > dig.out.$n.hidden || ret=1
+dig_with_opts +noadd +noauth txt.example4. txt @$f1 > dig.out.$n.f1 || ret=1
+digcomp dig.out.$n.hidden dig.out.$n.f1 || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))

-echo_i "checking that a forward zone works"
+n=$((n+1))
+echo_i "checking that a forward zone works ($n)"
 ret=0
-$DIG $DIGOPTS +noadd +noauth txt.example1. txt @$hidden > dig.out.hidden || ret=1
-$DIG $DIGOPTS +noadd +noauth txt.example1. txt @$f2 > dig.out.f2 || ret=1
-digcomp dig.out.hidden dig.out.f2 || ret=1
+dig_with_opts +noadd +noauth txt.example1. txt @$hidden > dig.out.$n.hidden || ret=1
+dig_with_opts +noadd +noauth txt.example1. txt @$f2 > dig.out.$n.f2 || ret=1
+digcomp dig.out.$n.hidden dig.out.$n.f2 || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))

-echo_i "checking that forwarding doesn't spontaneously happen"
+n=$((n+1))
+echo_i "checking that forwarding doesn't spontaneously happen ($n)"
 ret=0
-$DIG $DIGOPTS +noadd +noauth txt.example2. txt @$root > dig.out.root || ret=1
-$DIG $DIGOPTS +noadd +noauth txt.example2. txt @$f2 > dig.out.f2 || ret=1
-digcomp dig.out.root dig.out.f2 || ret=1
+dig_with_opts +noadd +noauth txt.example2. txt @$root > dig.out.$n.root || ret=1
+dig_with_opts +noadd +noauth txt.example2. txt @$f2 > dig.out.$n.f2 || ret=1
+digcomp dig.out.$n.root dig.out.$n.f2 || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))

-echo_i "checking that a forward zone with no specified policy works"
+n=$((n+1))
+echo_i "checking that a forward zone with no specified policy works ($n)"
 ret=0
-$DIG $DIGOPTS +noadd +noauth txt.example3. txt @$hidden > dig.out.hidden || ret=1
-$DIG $DIGOPTS +noadd +noauth txt.example3. txt @$f2 > dig.out.f2 || ret=1
-digcomp dig.out.hidden dig.out.f2 || ret=1
+dig_with_opts +noadd +noauth txt.example3. txt @$hidden > dig.out.$n.hidden || ret=1
+dig_with_opts +noadd +noauth txt.example3. txt @$f2 > dig.out.$n.f2 || ret=1
+digcomp dig.out.$n.hidden dig.out.$n.f2 || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))

-echo_i "checking that a forward only doesn't recurse"
+n=$((n+1))
+echo_i "checking that a forward only doesn't recurse ($n)"
 ret=0
-$DIG $DIGOPTS txt.example5. txt @$f2 > dig.out.f2 || ret=1
-grep "SERVFAIL" dig.out.f2 > /dev/null || ret=1
+dig_with_opts txt.example5. txt @$f2 > dig.out.$n.f2 || ret=1
+grep "SERVFAIL" dig.out.$n.f2 > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))

-echo_i "checking for negative caching of forwarder response"
+n=$((n+1))
+echo_i "checking for negative caching of forwarder response ($n)"
 # prime the cache, shutdown the forwarder then check that we can
 # get the answer from the cache.  restart forwarder.
 ret=0
-$DIG $DIGOPTS nonexist. txt @10.53.0.5 > dig.out.f2 || ret=1
-grep "status: NXDOMAIN" dig.out.f2 > /dev/null || ret=1
+dig_with_opts nonexist. txt @10.53.0.5 > dig.out.$n.f2 || ret=1
+grep "status: NXDOMAIN" dig.out.$n.f2 > /dev/null || ret=1
 $PERL ../stop.pl forward ns4 || ret=1
-$DIG $DIGOPTS nonexist. txt @10.53.0.5 > dig.out.f2 || ret=1
-grep "status: NXDOMAIN" dig.out.f2 > /dev/null || ret=1
-$PERL ../start.pl --restart --noclean --port ${PORT} forward ns4 || ret=1
+dig_with_opts nonexist. txt @10.53.0.5 > dig.out.$n.f2 || ret=1
+grep "status: NXDOMAIN" dig.out.$n.f2 > /dev/null || ret=1
+$PERL ../start.pl --restart --noclean --port "${PORT}" forward ns4 || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))

-echo_i "checking that forward only zone overrides empty zone"
+check_override() (
+    dig_with_opts 1.0.10.in-addr.arpa TXT @10.53.0.4 > dig.out.$n.f2 &&
+    grep "status: NOERROR" dig.out.$n.f2 > /dev/null &&
+    dig_with_opts 2.0.10.in-addr.arpa TXT @10.53.0.4 > dig.out.$n.f2 &&
+    grep "status: NXDOMAIN" dig.out.$n.f2 > /dev/null
+)
+
+n=$((n+1))
+echo_i "checking that forward only zone overrides empty zone ($n)"
 ret=0
 # retry loop in case the server restart above causes transient failure
-for try in 0 1 2 3 4 5 6 7 8 9; do
-    $DIG $DIGOPTS 1.0.10.in-addr.arpa TXT @10.53.0.4 > dig.out.f2
-    grep "status: NOERROR" dig.out.f2 > /dev/null || ret=1
-    $DIG $DIGOPTS 2.0.10.in-addr.arpa TXT @10.53.0.4 > dig.out.f2
-    grep "status: NXDOMAIN" dig.out.f2 > /dev/null || ret=1
-    [ "$ret" -eq 0 ] && break
-    sleep 1
-done
+retry_quiet 10 check_override || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))

-echo_i "checking that DS lookups for grafting forward zones are isolated"
+n=$((n+1))
+echo_i "checking that DS lookups for grafting forward zones are isolated ($n)"
 ret=0
-$DIG $DIGOPTS grafted A @10.53.0.4 > dig.out.q1
-$DIG $DIGOPTS grafted DS @10.53.0.4 > dig.out.q2
-$DIG $DIGOPTS grafted A @10.53.0.4 > dig.out.q3
-$DIG $DIGOPTS grafted AAAA @10.53.0.4 > dig.out.q4
-grep "status: NOERROR" dig.out.q1 > /dev/null || ret=1
-grep "status: NXDOMAIN" dig.out.q2 > /dev/null || ret=1
-grep "status: NOERROR" dig.out.q3 > /dev/null || ret=1
-grep "status: NOERROR" dig.out.q4 > /dev/null || ret=1
+dig_with_opts grafted A @10.53.0.4 > dig.out.$n.q1 || ret=1
+dig_with_opts grafted DS @10.53.0.4 > dig.out.$n.q2 || ret=1
+dig_with_opts grafted A @10.53.0.4 > dig.out.$n.q3 || ret=1
+dig_with_opts grafted AAAA @10.53.0.4 > dig.out.$n.q4 || ret=1
+grep "status: NOERROR" dig.out.$n.q1 > /dev/null || ret=1
+grep "status: NXDOMAIN" dig.out.$n.q2 > /dev/null || ret=1
+grep "status: NOERROR" dig.out.$n.q3 > /dev/null || ret=1
+grep "status: NOERROR" dig.out.$n.q4 > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))

-echo_i "checking that rfc1918 inherited 'forward first;' zones are warned about"
+n=$((n+1))
+echo_i "checking that rfc1918 inherited 'forward first;' zones are warned about ($n)"
 ret=0
 $CHECKCONF rfc1918-inherited.conf | grep "forward first;" >/dev/null || ret=1
 $CHECKCONF rfc1918-notinherited.conf | grep "forward first;" >/dev/null && ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))

-echo_i "checking that ULA inherited 'forward first;' zones are warned about"
+n=$((n+1))
+echo_i "checking that ULA inherited 'forward first;' zones are warned about ($n)"
 ret=0
 $CHECKCONF ula-inherited.conf | grep "forward first;" >/dev/null || ret=1
 $CHECKCONF ula-notinherited.conf | grep "forward first;" >/dev/null && ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))

-echo_i "checking that a forwarder timeout prevents it from being reused in the same fetch context"
+count_sent() (
+	logfile="$1"
+	start_pattern="$2"
+	pattern="$3"
+	nextpartpeek "$logfile" | tr -d '\r' | sed -n "/$start_pattern/,/^\$/p" | grep -c "$pattern"
+)
+
+check_sent() (
+	expected="$1"
+	shift
+	count=$(count_sent "$@")
+	[ "$expected" = "$count" ]
+)
+
+wait_for_log() (
+	nextpartpeek "$1" | grep "$2" >/dev/null
+
+)
+
+n=$((n+1))
+echo_i "checking that a forwarder timeout prevents it from being reused in the same fetch context ($n)"
 ret=0
 # Make ans6 receive queries without responding to them.
-echo "//" | $SENDCMD
+echo "//" | sendcmd
 # Query for a record in a zone which is forwarded to a non-responding forwarder
 # and is delegated from the root to check whether the forwarder will be retried
 # when a delegation is encountered after falling back to full recursive
 # resolution.
-$DIG $DIGOPTS txt.example7. txt @$f1 > dig.out.f1 || ret=1
+nextpart ns3/named.run >/dev/null
+dig_with_opts txt.example7. txt @$f1 > dig.out.$n.f1 || ret=1
 # The forwarder for the "example7" zone should only be queried once.
-sent=`tr -d '\r' < ns3/named.run | sed -n '/sending packet to 10.53.0.6/,/^$/p' | grep ";txt.example7.*IN.*TXT" | wc -l`
-if [ $sent -ne 1 ]; then ret=1; fi
+start_pattern="sending packet to 10\.53\.0\.6"
+retry_quiet 5 wait_for_log ns3/named.run "$start_pattern"
+check_sent 1 ns3/named.run "$start_pattern" ";txt\.example7\.[[:space:]]*IN[[:space:]]*TXT$" || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))

-echo_i "checking that priming queries are not forwarded"
+n=$((n+1))
+echo_i "checking that priming queries are not forwarded ($n)"
 ret=0
-$DIG $DIGOPTS +noadd +noauth txt.example1. txt @10.53.0.7 > dig.out.f7 || ret=1
-sent=`tr -d '\r' < ns7/named.run | sed -n '/sending packet to 10.53.0.1/,/^$/p' | grep ";.*IN.*NS" | wc -l`
-[ $sent -eq 1 ] || ret=1
-sent=`grep "10.53.0.7#.* (.): query '\./NS/IN' approved" ns4/named.run | wc -l`
-[ $sent -eq 0 ] || ret=1
-sent=`grep "10.53.0.7#.* (.): query '\./NS/IN' approved" ns1/named.run | wc -l`
-[ $sent -eq 1 ] || ret=1
+nextpart ns7/named.run >/dev/null
+dig_with_opts +noadd +noauth txt.example1. txt @10.53.0.7 > dig.out.$n.f7 || ret=1
+start_pattern="sending packet to 10\.53\.0\.1"
+retry_quiet 5 wait_for_log ns7/named.run "$start_pattern" || ret=1
+check_sent 1 ns7/named.run "$start_pattern" ";\.[[:space:]]*IN[[:space:]]*NS$" || ret=1
+sent=$(grep -c "10.53.0.7#.* (.): query '\./NS/IN' approved" ns4/named.run)
+[ "$sent" -eq 0 ] || ret=1
+sent=$(grep -c "10.53.0.7#.* (.): query '\./NS/IN' approved" ns1/named.run)
+[ "$sent" -eq 1 ] || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))

-echo_i "checking recovery from forwarding to a non-recursive server"
+n=$((n+1))
+echo_i "checking recovery from forwarding to a non-recursive server ($n)"
 ret=0
-$DIG $DIGOPTS xxx.sld.tld txt @10.53.0.8  > dig.out.f8
-grep "status: NOERROR" dig.out.f8 > /dev/null || ret=1
+dig_with_opts xxx.sld.tld txt @10.53.0.8  > dig.out.$n.f8 || ret=1
+grep "status: NOERROR" dig.out.$n.f8 > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))

 echo_i "exit status: $status"
 [ $status -eq 0 ] || exit 1
@@ -252,16 +252,19 @@ idna_enabled_test() {
    #            is displayed as the corresponding A-label.
    #
    # The "+[no]idnout" flag has no effect in these cases.
+    saved_LC_ALL="${LC_ALL}"
+    LC_ALL="C"
    text="Checking valid A-label in C locale"
    label="xn--nxasmq6b.com"
-    if command -v idn2 >/dev/null && ! LC_ALL=C idn2 -d "$label" >/dev/null 2>/dev/null; then
-	LC_ALL=C idna_test "$text" ""                   "$label" "$label."
-	LC_ALL=C idna_test "$text" "+noidnin +noidnout" "$label" "$label."
-	LC_ALL=C idna_test "$text" "+noidnin +idnout"   "$label" "$label."
-	LC_ALL=C idna_test "$text" "+idnin +noidnout"   "$label" "$label."
-	LC_ALL=C idna_test "$text" "+idnin +idnout"     "$label" "$label."
-	LC_ALL=C idna_test "$text" "+noidnin +idnout"   "$label" "$label."
+    if command -v idn2 >/dev/null && ! idn2 -d "$label" >/dev/null 2>/dev/null; then
+	idna_test "$text" ""                   "$label" "$label."
+	idna_test "$text" "+noidnin +noidnout" "$label" "$label."
+	idna_test "$text" "+noidnin +idnout"   "$label" "$label."
+	idna_test "$text" "+idnin +noidnout"   "$label" "$label."
+	idna_test "$text" "+idnin +idnout"     "$label" "$label."
+	idna_test "$text" "+noidnin +idnout"   "$label" "$label."
    fi
+    LC_ALL="${saved_LC_ALL}"



@@ -240,13 +240,13 @@ key_states() {
 # KEY_FILE="${BASE_FILE}.key"
 # PRIVATE_FILE="${BASE_FILE}.private"
 # STATE_FILE="${BASE_FILE}.state"
-# KEY_ID=$(echo $1 | sed 's/^0*//')
+# KEY_ID=$(echo $1 | sed 's/^0\{0,4\}//')
 check_key() {
 	_dir="$DIR"
 	_zone="$ZONE"
 	_role=$(key_get "$1" ROLE)
 	_key_idpad="$2"
-	_key_id=$(echo "$_key_idpad" | sed 's/^0*//')
+	_key_id=$(echo "$_key_idpad" | sed 's/^0\{0,4\}//')
 	_alg_num=$(key_get "$1" ALG_NUM)
 	_alg_numpad=$(printf "%03d" "$_alg_num")
 	_alg_string=$(key_get "$1" ALG_STR)
@@ -288,7 +288,7 @@ check_key() {
 	PRIVATE_FILE="${BASE_FILE}.private"
 	STATE_FILE="${BASE_FILE}.state"
 	KEY_ID="${_key_id}"
-	
+
 	test $_log -eq 1 && echo_i "check key $BASE_FILE"

 	# Check the public key file.
@@ -409,12 +409,12 @@ check_key() {
 # KEY_FILE="${BASE_FILE}.key"
 # PRIVATE_FILE="${BASE_FILE}.private"
 # STATE_FILE="${BASE_FILE}.state"
-# KEY_ID=$(echo $1 | sed 's/^0*//')
+# KEY_ID=$(echo $1 | sed 's/^0\{0,4\}//')
 key_unused() {
 	_dir=$DIR
 	_zone=$ZONE
 	_key_idpad=$1
-	_key_id=$(echo "$_key_idpad" | sed 's/^0*//')
+	_key_id=$(echo "$_key_idpad" | sed 's/^0\{0,4\}//')
 	_alg_num=$(key_get KEY1 ALG_NUM)
        _alg_numpad=$(printf "%03d" "$_alg_num")

@@ -27,20 +27,6 @@ rndccmd() (
 	"$RNDC" -c "$SYSTEMTESTTOP/common/rndc.conf" -p "${CONTROLPORT}" -s "$@"
 )

-search_log() (
-	msg=$1
-	file=$2
-	nextpart "$file" | grep -F "$msg" > /dev/null
-)
-
-wait_for_log() (
-	msg="$1"
-	file="$2"
-	retry_quiet 20 search_log "$msg" "$file" && return 0
-	echo_i "exceeded time limit waiting for '$msg' in $file"
-	return 1
-)
-
 mkeys_reconfig_on() (
 	nsidx=$1
 	rndccmd "10.53.0.${nsidx}" reconfig . | sed "s/^/ns${nsidx} /" | cat_i
@@ -50,21 +36,21 @@ mkeys_reload_on() (
 	nsidx=$1
 	nextpart "ns${nsidx}"/named.run > /dev/null
 	rndc_reload "ns${nsidx}" "10.53.0.${nsidx}"
-	wait_for_log "loaded serial" "ns${nsidx}"/named.run
+	wait_for_log 20 "loaded serial" "ns${nsidx}"/named.run || return 1
 )

 mkeys_loadkeys_on() (
 	nsidx=$1
 	nextpart "ns${nsidx}"/named.run > /dev/null
 	rndccmd "10.53.0.${nsidx}" loadkeys . | sed "s/^/ns${nsidx} /" | cat_i
-	wait_for_log "next key event" "ns${nsidx}"/named.run
+	wait_for_log 20 "next key event" "ns${nsidx}"/named.run || return 1
 )

 mkeys_refresh_on() (
 	nsidx=$1
 	nextpart "ns${nsidx}"/named.run > /dev/null
 	rndccmd "10.53.0.${nsidx}" managed-keys refresh | sed "s/^/ns${nsidx} /" | cat_i
-	wait_for_log "Returned from key fetch in keyfetch_done()" "ns${nsidx}"/named.run
+	wait_for_log 20 "Returned from key fetch in keyfetch_done()" "ns${nsidx}"/named.run || return 1
 )

 mkeys_sync_on() (
@@ -75,7 +61,7 @@ mkeys_sync_on() (
 	nsidx=$1
 	nextpart "ns${nsidx}"/named.run > /dev/null
 	rndccmd "10.53.0.${nsidx}" managed-keys sync | sed "s/^/ns${nsidx} /" | cat_i
-	wait_for_log "dump_done" "ns${nsidx}"/named.run
+	wait_for_log 20 "dump_done" "ns${nsidx}"/named.run || return 1
 )

 mkeys_status_on() (
@@ -322,7 +308,7 @@ $PERL $SYSTEMTESTTOP/start.pl --noclean --restart --port "${PORT}" mkeys ns2
 n=$((n+1))
 echo_i "check that no key from bind.keys is marked as an initializing key ($n)"
 ret=0
-wait_for_log "Returned from key fetch in keyfetch_done()" ns2/named.run || ret=1
+wait_for_log 20 "Returned from key fetch in keyfetch_done()" ns2/named.run || ret=1
 mkeys_secroots_on 2 || ret=1
 grep '; initializing' ns2/named.secroots > /dev/null 2>&1 && ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
@@ -338,7 +324,7 @@ $PERL $SYSTEMTESTTOP/start.pl --noclean --restart --port "${PORT}" mkeys ns2
 n=$((n+1))
 echo_i "check that standby key is now trusted ($n)"
 ret=0
-wait_for_log "Returned from key fetch in keyfetch_done()" ns2/named.run || ret=1
+wait_for_log 20 "Returned from key fetch in keyfetch_done()" ns2/named.run || ret=1
 mkeys_status_on 2 > rndc.out.$n 2>&1 || ret=1
 # two keys listed
 count=$(grep -c "keyid: " rndc.out.$n) || true
@@ -495,7 +481,7 @@ $PERL $SYSTEMTESTTOP/start.pl --noclean --restart --port "${PORT}" mkeys ns2
 n=$((n+1))
 echo_i "check positive validation ($n)"
 ret=0
-wait_for_log "Returned from key fetch in keyfetch_done()" ns2/named.run || ret=1
+wait_for_log 20 "Returned from key fetch in keyfetch_done()" ns2/named.run || ret=1
 dig_with_opts +noauth example. @10.53.0.2 txt > dig.out.ns2.test$n || ret=1
 grep "flags:.*ad.*QUERY" dig.out.ns2.test$n > /dev/null || ret=1
 grep "example..*.RRSIG..*TXT" dig.out.ns2.test$n > /dev/null || ret=1
@@ -591,7 +577,7 @@ rm -f ns1/root.db.signed.jnl
 cp ns1/root.db ns1/root.db.signed
 nextpart ns1/named.run > /dev/null
 $PERL $SYSTEMTESTTOP/start.pl --noclean --restart --port "${PORT}" mkeys ns1
-wait_for_log "all zones loaded" ns1/named.run || ret=1
+wait_for_log 20 "all zones loaded" ns1/named.run || ret=1
 mkeys_refresh_on 2 || ret=1
 mkeys_status_on 2 > rndc.out.2.$n 2>&1 || ret=1
 # one key listed
@@ -625,7 +611,7 @@ rm -f ns1/root.db.signed.jnl
 cat ns1/K*.key >> ns1/root.db.signed
 nextpart ns1/named.run > /dev/null
 $PERL $SYSTEMTESTTOP/start.pl --noclean --restart --port "${PORT}" mkeys ns1
-wait_for_log "all zones loaded" ns1/named.run || ret=1
+wait_for_log 20 "all zones loaded" ns1/named.run || ret=1
 # Less than a second may have passed since the last time ns2 received a
 # ./DNSKEY response from ns1.  Ensure keys are refreshed at a different
 # timestamp to prevent minimal update from resetting it to the same timestamp.
@@ -725,7 +711,7 @@ ret=0
 $PERL $SYSTEMTESTTOP/stop.pl --use-rndc --port "${CONTROLPORT}" mkeys ns5
 nextpart ns5/named.run > /dev/null
 $PERL $SYSTEMTESTTOP/start.pl --noclean --restart --port "${PORT}" mkeys ns5
-wait_for_log "Returned from key fetch in keyfetch_done()" ns5/named.run || ret=1
+wait_for_log 20 "Returned from key fetch in keyfetch_done()" ns5/named.run || ret=1
 # ns5/named.run will contain logs from both the old instance and the new
 # instance.  In order for the test to pass, both must attempt a fetch.
 count=$(grep -c "Creating key fetch" ns5/named.run) || true
@@ -744,7 +730,7 @@ rm -f ns5/managed-keys.bind*
 cp ns5/named2.args ns5/named.args
 nextpart ns5/named.run > /dev/null
 $PERL $SYSTEMTESTTOP/start.pl --noclean --restart --port "${PORT}" mkeys ns5
-wait_for_log "Returned from key fetch in keyfetch_done() for '.': failure" ns5/named.run || ret=1
+wait_for_log 20 "Returned from key fetch in keyfetch_done() for '.': failure" ns5/named.run || ret=1
 mkeys_secroots_on 5 || ret=1
 grep '; initializing managed' ns5/named.secroots > /dev/null 2>&1 || ret=1
 # ns1 should still REFUSE queries from ns5, so resolving should be impossible
@@ -757,7 +743,7 @@ copy_setports ns1/named3.conf.in ns1/named.conf
 rm -f ns1/root.db.signed.jnl
 nextpart ns5/named.run > /dev/null
 mkeys_reconfig_on 1 || ret=1
-wait_for_log "Returned from key fetch in keyfetch_done() for '.': success" ns5/named.run || ret=1
+wait_for_log 20 "Returned from key fetch in keyfetch_done() for '.': success" ns5/named.run || ret=1
 mkeys_secroots_on 5 || ret=1
 grep '; managed' ns5/named.secroots > /dev/null || ret=1
 # ns1 should not longer REFUSE queries from ns5, so managed keys should be
@@ -777,7 +763,7 @@ rm -f ns6/managed-keys.bind*
 nextpart ns6/named.run > /dev/null
 $PERL $SYSTEMTESTTOP/start.pl --noclean --restart --port "${PORT}" mkeys ns6
 # log when an unsupported algorithm is encountered during startup
-wait_for_log "ignoring initial-key for 'unsupported.': algorithm is unsupported" ns6/named.run || ret=1
+wait_for_log 20 "ignoring initial-key for 'unsupported.': algorithm is unsupported" ns6/named.run || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=$((status+ret))

@@ -819,7 +805,7 @@ count=$(grep -c "keyid: " rndc.out.$n) || true
 count=$(grep -c "trust" rndc.out.$n) || true
 [ "$count" -eq 2 ] || ret=1
 # log when an unsupported algorithm is encountered during rollover
-wait_for_log "Cannot compute tag for key in zone .: algorithm is unsupported" ns6/named.run || ret=1
+wait_for_log 20 "Cannot compute tag for key in zone .: algorithm is unsupported" ns6/named.run || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=$((status+ret))

@@ -196,7 +196,7 @@ if [ $status != 0 ]; then
 else
    core_dumps="$(find $systest/ -name 'core*' -or -name '*.core' | sort | tr '\n' ' ')"
    assertion_failures=$(find $systest/ -name named.run | xargs grep "assertion failure" | wc -l)
-    sanitizer_summaries=$(find $systest/ -type f | grep '^[-a-zA-Z0-9./_]*$' | xargs grep "SUMMARY: .*Sanitizer" | wc -l)
+    sanitizer_summaries=$(find $systest/ -name 'tsan.*' | wc -l)
    if [ -n "$core_dumps" ]; then
        echoinfo "I:$systest:Test claims success despite crashes: $core_dumps"
        echofail "R:$systest:FAIL"
@@ -214,7 +214,9 @@ else
 		echoinfo "D:$systest:backtrace from $coredump end"
 	done
    elif [ $assertion_failures -ne 0 ]; then
+	SYSTESTDIR="$systest"
        echoinfo "I:$systest:Test claims success despite $assertion_failures assertion failure(s)"
+	grep "SUMMARY: " $(find $systest/ -name 'tsan.*') | sort -u | cat_d
        echofail "R:$systest:FAIL"
        # Do not clean up - we need the evidence.
    elif [ $sanitizer_summaries -ne 0 ]; then
@@ -204,7 +204,7 @@ if [ "$(id -u)" -eq 0 ] && [ -z "$CYGWIN" ]; then
    if [ "$?" -eq 0 ]; then
        copy_setports ns2/named-alt9.conf.in "${TEMP_NAMED_DIR}/named-alt9.conf"
        export SOFTHSM2_CONF="${TEMP_NAMED_DIR}/softhsm2.conf"
-        sh "$TOP/util/prepare-softhsm2.sh"
+        sh "$TOP/bin/tests/prepare-softhsm2.sh"
        chown -R nobody: "${TEMP_NAMED_DIR}"
        chmod 0700 "${TEMP_NAMED_DIR}"
        ( cd "${TEMP_NAMED_DIR}" && $NAMED -u nobody -c named-alt9.conf -d 99 -g -U 4 >> named$n.run 2>&1 & ) || ret=1
@@ -76,25 +76,13 @@ getzones() {
    return $result
 }

-# TODO: Move wait_for_log and loadkeys_on to conf.sh.common
-wait_for_log() {
-    msg=$1
-    file=$2
-
-    for i in 1 2 3 4 5 6 7 8 9 10; do
-        nextpart "$file" | grep "$msg" > /dev/null && return
-        sleep 1
-    done
-    echo_i "exceeded time limit waiting for '$msg' in $file"
-    ret=1
-}
-
+# TODO: Move loadkeys_on to conf.sh.common
 loadkeys_on() {
    nsidx=$1
    zone=$2
    nextpart ns${nsidx}/named.run > /dev/null
    $RNDCCMD 10.53.0.${nsidx} loadkeys ${zone} | sed "s/^/ns${nsidx} /" | cat_i
-    wait_for_log "next key event" ns${nsidx}/named.run
+    wait_for_log 20 "next key event" ns${nsidx}/named.run
 }

 status=0
@@ -78,22 +78,11 @@ refresh_tcp_stats() {
 	TCP_HIGH="$(sed -n "s/^TCP high-water: \([0-9][0-9]*\)/\1/p" rndc.out.$n)"
 }

-wait_for_log() {
-	msg=$1
-	file=$2
-	for _ in 1 2 3 4 5 6 7 8 9 10; do
-		nextpartpeek "$file" | grep "$msg" > /dev/null && return
-		sleep 1
-	done
-	echo_i "exceeded time limit waiting for '$msg' in $file"
-	ret=1
-}
-
 # Send a command to the tool script listening on 10.53.0.6.
 send_command() {
 	nextpart ans6/ans.run > /dev/null
 	echo "$*" | "${PERL}" "${SYSTEMTESTTOP}/send.pl" 10.53.0.6 "${CONTROLPORT}"
-	wait_for_log "result=" ans6/ans.run
+	wait_for_log_peek 10 "result=" ans6/ans.run || ret=1
 	if ! nextpartpeek ans6/ans.run | grep -qF "result=OK"; then
 		return 1
 	fi
@@ -1,3 +1,5 @@
+#!/bin/sh
+#
 # Copyright (C) Internet Systems Consortium, Inc. ("ISC")
 #
 # This Source Code Form is subject to the terms of the Mozilla Public
@@ -7,84 +9,112 @@
 # See the COPYRIGHT file distributed with this work for additional
 # information regarding copyright ownership.

+#shellcheck source=conf.sh
 SYSTEMTESTTOP=..
-. $SYSTEMTESTTOP/conf.sh
+. "$SYSTEMTESTTOP/conf.sh"

-DIGOPTS="-p ${PORT}"
+dig_with_opts() {
+	"$DIG" -p "${PORT}" "$@"
+}
+
+wait_for_pid() (
+	for pid in "$@"; do
+		kill -0 "$pid" 2>/dev/null && return 1
+	done
+	return 0
+)

 status=0
 n=0

-n=`expr $n + 1`
+n=$((n+1))
 echo_i "check lookups against TTL=0 records ($n)"
 i=0
+ret=0
 passes=10
-$DIG $DIGOPTS @10.53.0.2 axfr example | grep -v "^ds0" |
+dig_with_opts @10.53.0.2 axfr example | grep -v "^ds0" | \
 awk '$2 == "0" { print "-q", $1, $4; print "-q", "zzz"$1, $4;}' > query.list
+
+# add 1/5 second per query
+timeout=$(($(wc -l < query.list) / 5))
 while [ $i -lt $passes ]
 do
-	ret=0
-	$DIG $DIGOPTS @10.53.0.3 -f query.list > dig.out$i.1.test$n &
-	$DIG $DIGOPTS @10.53.0.3 -f query.list > dig.out$i.2.test$n &
-	$DIG $DIGOPTS @10.53.0.3 -f query.list > dig.out$i.3.test$n &
-	$DIG $DIGOPTS @10.53.0.3 -f query.list > dig.out$i.4.test$n &
-	$DIG $DIGOPTS @10.53.0.3 -f query.list > dig.out$i.5.test$n &
-	$DIG $DIGOPTS @10.53.0.3 -f query.list > dig.out$i.6.test$n &
-	wait
-	grep "status: SERVFAIL" dig.out$i.1.test$n > /dev/null && ret=1
-	grep "status: SERVFAIL" dig.out$i.2.test$n > /dev/null && ret=1
-	grep "status: SERVFAIL" dig.out$i.3.test$n > /dev/null && ret=1
-	grep "status: SERVFAIL" dig.out$i.4.test$n > /dev/null && ret=1
-	grep "status: SERVFAIL" dig.out$i.5.test$n > /dev/null && ret=1
-	grep "status: SERVFAIL" dig.out$i.6.test$n > /dev/null && ret=1
+	(dig_with_opts @10.53.0.3 -f query.list > "dig.out$i.1.test$n") & pid1="$!"
+	(dig_with_opts @10.53.0.3 -f query.list > "dig.out$i.2.test$n") & pid2="$!"
+	(dig_with_opts @10.53.0.3 -f query.list > "dig.out$i.3.test$n") & pid3="$!"
+	(dig_with_opts @10.53.0.3 -f query.list > "dig.out$i.4.test$n") & pid4="$!"
+	(dig_with_opts @10.53.0.3 -f query.list > "dig.out$i.5.test$n") & pid5="$!"
+	(dig_with_opts @10.53.0.3 -f query.list > "dig.out$i.6.test$n") & pid6="$!"
+
+	retry_quiet "$timeout" wait_for_pid "$pid1" "$pid2" "$pid3" "$pid4" "$pid5" "$pid6" || ret=1
+	kill -TERM "$pid1" "$pid2" "$pid3" "$pid4" "$pid5" "$pid6" 2>/dev/null
+
+	wait "$pid1" || ret=1
+	wait "$pid2" || ret=1
+	wait "$pid3" || ret=1
+	wait "$pid4" || ret=1
+	wait "$pid5" || ret=1
+	wait "$pid6" || ret=1
+
+	grep "status: SERVFAIL" "dig.out$i.1.test$n" > /dev/null && ret=1
+	grep "status: SERVFAIL" "dig.out$i.2.test$n" > /dev/null && ret=1
+	grep "status: SERVFAIL" "dig.out$i.3.test$n" > /dev/null && ret=1
+	grep "status: SERVFAIL" "dig.out$i.4.test$n" > /dev/null && ret=1
+	grep "status: SERVFAIL" "dig.out$i.5.test$n" > /dev/null && ret=1
+	grep "status: SERVFAIL" "dig.out$i.6.test$n" > /dev/null && ret=1
 	[ $ret = 1 ] && break
-	i=`expr $i + 1`
+	i=$((i+1))
 	echo_i "successfully completed pass $i of $passes"
 done
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))

-n=`expr $n + 1`
+repeat_query() (
+	i=0
+	while [ "$i" -lt "$1" ]; do
+		dig_with_opts +short "@$2" "$3" | tee "dig.out$i.test$n" || return 1
+		i=$((i+1))
+	done
+)
+
+count_unique() (
+	repeat_query "$@" | sort -u | wc -l
+)
+
+n=$((n+1))
 echo_i "check repeated recursive lookups of non recurring TTL=0 responses get new values ($n)"
 ret=0
-count=`(
-$DIG $DIGOPTS +short @10.53.0.3 foo.increment
-$DIG $DIGOPTS +short @10.53.0.3 foo.increment
-$DIG $DIGOPTS +short @10.53.0.3 foo.increment
-$DIG $DIGOPTS +short @10.53.0.3 foo.increment
-$DIG $DIGOPTS +short @10.53.0.3 foo.increment
-$DIG $DIGOPTS +short @10.53.0.3 foo.increment
-$DIG $DIGOPTS +short @10.53.0.3 foo.increment
-) | sort -u | wc -l `
-if [ $count -ne 7 ] ; then echo_i "failed (count=$count)"; ret=1; fi
-status=`expr $status + $ret`
+repeats=9
+count=$(count_unique "$repeats" 10.53.0.3 foo.increment)
+if [ "$count" -ne "$repeats" ] ; then echo_i "failed (count=$count, repeats=$repeats)"; ret=1; fi
+status=$((status+ret))

-n=`expr $n + 1`
+n=$((n+1))
 echo_i "check lookups against TTL=1 records ($n)"
 i=0
 passes=10
+ret=0
 while [ $i -lt $passes ]
 do
-	ret=0
-	$DIG $DIGOPTS @10.53.0.3 www.one.tld > dig.out$i.1.test$n
-	$DIG $DIGOPTS @10.53.0.3 www.one.tld > dig.out$i.2.test$n
-	$DIG $DIGOPTS @10.53.0.3 www.one.tld > dig.out$i.3.test$n
-	$DIG $DIGOPTS @10.53.0.3 www.one.tld > dig.out$i.4.test$n
-	$DIG $DIGOPTS @10.53.0.3 www.one.tld > dig.out$i.5.test$n
-	$DIG $DIGOPTS @10.53.0.3 www.one.tld > dig.out$i.6.test$n
-	grep "status: SERVFAIL" dig.out$i.1.test$n > /dev/null && ret=1
-	grep "status: SERVFAIL" dig.out$i.2.test$n > /dev/null && ret=1
-	grep "status: SERVFAIL" dig.out$i.3.test$n > /dev/null && ret=1
-	grep "status: SERVFAIL" dig.out$i.4.test$n > /dev/null && ret=1
-	grep "status: SERVFAIL" dig.out$i.5.test$n > /dev/null && ret=1
-	grep "status: SERVFAIL" dig.out$i.6.test$n > /dev/null && ret=1
+	dig_with_opts @10.53.0.3 www.one.tld > "dig.out$i.1.test$n" || ret=1
+	dig_with_opts @10.53.0.3 www.one.tld > "dig.out$i.2.test$n" || ret=1
+	dig_with_opts @10.53.0.3 www.one.tld > "dig.out$i.3.test$n" || ret=1
+	dig_with_opts @10.53.0.3 www.one.tld > "dig.out$i.4.test$n" || ret=1
+	dig_with_opts @10.53.0.3 www.one.tld > "dig.out$i.5.test$n" || ret=1
+	dig_with_opts @10.53.0.3 www.one.tld > "dig.out$i.6.test$n" || ret=1
+	grep "status: SERVFAIL" "dig.out$i.1.test$n" > /dev/null && ret=1
+	grep "status: SERVFAIL" "dig.out$i.2.test$n" > /dev/null && ret=1
+	grep "status: SERVFAIL" "dig.out$i.3.test$n" > /dev/null && ret=1
+	grep "status: SERVFAIL" "dig.out$i.4.test$n" > /dev/null && ret=1
+	grep "status: SERVFAIL" "dig.out$i.5.test$n" > /dev/null && ret=1
+	grep "status: SERVFAIL" "dig.out$i.6.test$n" > /dev/null && ret=1
 	[ $ret = 1 ] && break
-	i=`expr $i + 1`
+	i=$((i+1))
 	echo_i "successfully completed pass $i of $passes"
-	$PERL -e 'select(undef, undef, undef, 0.3);'
+	sleep 1
 done
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))

 echo_i "exit status: $status"
-[ $status -eq 0 ] || exit 1
+[ "$status" -eq 0 ] || exit 1
@@ -465,6 +465,12 @@
 /* Define to 1 if you have the `usleep' function. */
 #undef HAVE_USLEEP

+/* Define to 1 if you have the `uv_handle_get_data' function. */
+#undef HAVE_UV_HANDLE_GET_DATA
+
+/* Define to 1 if you have the `uv_handle_set_data' function. */
+#undef HAVE_UV_HANDLE_SET_DATA
+
 /* Use zlib library */
 #undef HAVE_ZLIB

@@ -15943,6 +15943,21 @@ fi
 CFLAGS="$CFLAGS $LIBUV_CFLAGS"
 LIBS="$LIBS $LIBUV_LIBS"

+# Those functions are only provided in newer versions of libuv, we'll be emulating them
+# for now
+for ac_func in uv_handle_get_data uv_handle_set_data
+do :
+  as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh`
+ac_fn_c_check_func "$LINENO" "$ac_func" "$as_ac_var"
+if eval test \"x\$"$as_ac_var"\" = x"yes"; then :
+  cat >>confdefs.h <<_ACEOF
+#define `$as_echo "HAVE_$ac_func" | $as_tr_cpp` 1
+_ACEOF
+
+fi
+done
+
+
 #
 # flockfile is usually provided by pthreads
 #
@@ -663,6 +663,10 @@ AX_SAVE_FLAGS([libuv])
 CFLAGS="$CFLAGS $LIBUV_CFLAGS"
 LIBS="$LIBS $LIBUV_LIBS"

+# Those functions are only provided in newer versions of libuv, we'll be emulating them
+# for now
+AC_CHECK_FUNCS([uv_handle_get_data uv_handle_set_data])
+
 #
 # flockfile is usually provided by pthreads
 #
@@ -11,6 +11,17 @@

 <section xml:id="relnotes-9.15.6"><info><title>Notes for BIND 9.15.6</title></info>

+  <section xml:id="relnotes-9.15.6-security"><info><title>Security Fixes</title></info>
+    <itemizedlist>
+      <listitem>
+        <para>
+          Set a limit on the number of concurrently served pipelined TCP
+          queries. This flaw is disclosed in CVE-2019-6477. [GL #1264]
+        </para>
+      </listitem>
+    </itemizedlist>
+  </section>
+
  <section xml:id="relnotes-9.15.6-new"><info><title>New Features</title></info>
    <itemizedlist>
      <listitem>
@@ -34,25 +45,32 @@
        </para>
      </listitem>
      <listitem>
-	<para>
-	  Two new keywords have been added to the
-	  <command>dnssec-keys</command> statement:
-	  <command>initial-ds</command> and <command>static-ds</command>.
-	  These allow the use of trust anchors in DS format instead of
-	  DNSKEY format.  DS format allows trust anchors to be configured
-	  for keys that have not yet been published; this is the format
-	  used by IANA when announcing future root keys.
-	</para>
-	<para>
-	  As with the <command>initial-key</command> and
-	  <command>static-key</command> keywords, <command>initial-ds</command>
-	  configures a dynamic trust anchor to be maintained via RFC 5011, and
-	  <command>static-ds</command> configures a permanent trust anchor.
-	</para>
-	<para>
-	  (Note: Currently, DNSKEY-format and DS-format trust anchors
-	  cannot both be used for the same domain name.) [GL #6] [GL #622]
-	</para>
+        <para>
+          Two new keywords have been added to the
+          <command>dnssec-keys</command> statement:
+          <command>initial-ds</command> and <command>static-ds</command>.
+          These allow the use of trust anchors in DS format instead of
+          DNSKEY format.  DS format allows trust anchors to be configured
+          for keys that have not yet been published; this is the format
+          used by IANA when announcing future root keys.
+        </para>
+        <para>
+          As with the <command>initial-key</command> and
+          <command>static-key</command> keywords, <command>initial-ds</command>
+          configures a dynamic trust anchor to be maintained via RFC 5011, and
+          <command>static-ds</command> configures a permanent trust anchor.
+        </para>
+        <para>
+          (Note: Currently, DNSKEY-format and DS-format trust anchors
+          cannot both be used for the same domain name.) [GL #6] [GL #622]
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          Added a new statistics variable <command>tcp-highwater</command>
+          that reports the maximum number of simultaneous TCP clients BIND
+          has handled while running. [GL #1206]
+        </para>
      </listitem>
    </itemizedlist>
  </section>
@@ -68,23 +86,10 @@
        </para>
      </listitem>
      <listitem>
-	<para>
-	  The DNSSEC validation code has been refactored for clarity and to
-	  reduce code duplication.  [GL #622]
-	</para>
-      </listitem>
-    </itemizedlist>
-  </section>
-
-  <section xml:id="relnotes-9.15.6-security"><info><title>Security Fixes</title></info>
-    <itemizedlist>
-      <listitem>
-	<para>
-	  Too many simultaneous pipelined TCP queries could cause
-	  resource overuse. We now prevent this by enforcing a limit
-	  on the number of simultaneous requests per active connection.
-	  This flaw`is disclosed in CVE-2019-6477. [GL #1264]
-	</para>
+        <para>
+          The DNSSEC validation code has been refactored for clarity and to
+          reduce code duplication.  [GL #622]
+        </para>
      </listitem>
    </itemizedlist>
  </section>
@@ -15,16 +15,17 @@
    <itemizedlist>
      <listitem>
        <para>
-	  The <command>dnssec-keys</command> configuration statement,
-	  which was introduced in 9.15.1 and revised in 9.15.6, has now
-	  been renamed to the more descriptive
-	  <command>trust-anchors</command>.
+          The <command>dnssec-keys</command> configuration statement,
+          which was introduced in 9.15.1 and revised in 9.15.6, has now
+          been renamed to the more descriptive
+          <command>trust-anchors</command>. [GL !2702]
        </para>
        <para>
-	  (See the sections on
-	  <xref linkend="relnotes-9.15.6" xrefstyle="template:9.15.6"/> and
-	  <xref linkend="relnotes-9.15.1" xrefstyle="template:9.15.1"/>
-	  for prior discussion of this feature.)
+          (See release notes for
+          <xref linkend="relnotes-9.15.1-new" xrefstyle="template:BIND 9.15.1"/>
+          and
+          <xref linkend="relnotes-9.15.6-new" xrefstyle="template:BIND 9.15.6"/>
+          for prior discussion of this feature.)
        </para>
      </listitem>
    </itemizedlist>
@@ -24,6 +24,7 @@
  <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="notes-platforms.xml"/>
  <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="notes-download.xml"/>

+  <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="notes-9.15.7.xml"/>
  <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="notes-9.15.6.xml"/>
  <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="notes-9.15.5.xml"/>
  <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="notes-9.15.4.xml"/>
@@ -164,7 +164,7 @@ typedef isc_rwlock_t nodelock_t;
 #define NODE_LOCK(l, t)         RWLOCK((l), (t))
 #define NODE_UNLOCK(l, t)       RWUNLOCK((l), (t))
 #define NODE_TRYUPGRADE(l)      isc_rwlock_tryupgrade(l)
-#define NODE_DOWNGRADE(l)   isc_rwlock_downgrade(l)
+#define NODE_DOWNGRADE(l)       isc_rwlock_downgrade(l)

 /*%
 * Whether to rate-limit updating the LRU to avoid possible thread contention.
@@ -1824,6 +1824,31 @@ new_reference(dns_rbtdb_t *rbtdb, dns_rbtnode_t *node) {
 	}
 }

+/*%
+ * The tree lock must be held for the result to be valid.
+ */
+static inline bool
+is_leaf(dns_rbtnode_t *node) {
+	return (node->parent != NULL && node->parent->down == node &&
+		node->left == NULL && node->right == NULL);
+}
+
+static inline void
+send_to_prune_tree(dns_rbtdb_t *rbtdb, dns_rbtnode_t *node) {
+	isc_event_t *ev;
+	dns_db_t *db;
+
+	ev = isc_event_allocate(rbtdb->common.mctx, NULL,
+				DNS_EVENT_RBTPRUNE,
+				prune_tree, node,
+				sizeof(isc_event_t));
+	new_reference(rbtdb, node);
+	db = NULL;
+	attach((dns_db_t *)rbtdb, &db);
+	ev->ev_sender = db;
+	isc_task_send(rbtdb->task, &ev);
+}
+
 /*%
 * Clean up dead nodes.  These are nodes which have no references, and
 * have no data.  They are dead but we could not or chose not to delete
@@ -1844,28 +1869,29 @@ cleanup_dead_nodes(dns_rbtdb_t *rbtdb, int bucketnum) {
 		/*
 		 * Since we're holding a tree write lock, it should be
 		 * impossible for this node to be referenced by others.
+		 *
+		 * decrement_reference may not have tested node->down, as
+		 * the tree_lock was not held, before adding the node to
+		 * deadnodes so we test it here.
 		 */
 		INSIST(isc_refcount_current(&node->references) == 0 &&
 		       node->data == NULL);

-		if (node->parent != NULL &&
-		    node->parent->down == node && node->left == NULL &&
-		    node->right == NULL && rbtdb->task != NULL)
-		{
-			isc_event_t *ev;
-			dns_db_t *db;
-
-			ev = isc_event_allocate(rbtdb->common.mctx, NULL,
-						DNS_EVENT_RBTPRUNE,
-						prune_tree, node,
-						sizeof(isc_event_t));
-			new_reference(rbtdb, node);
-			db = NULL;
-			attach((dns_db_t *)rbtdb, &db);
-			ev->ev_sender = db;
-			isc_task_send(rbtdb->task, &ev);
-		} else {
+		if (is_leaf(node) && rbtdb->task != NULL) {
+			send_to_prune_tree(rbtdb, node);
+		} else if (node->down == NULL && node->data == NULL) {
+			/*
+			 * Not a interior node and not needing to be
+			 * reactivated.
+			 */
 			delete_node(rbtdb, node);
+		} else if (node->data == NULL) {
+			/*
+			 * A interior node without data. Leave linked to
+			 * to be cleaned up when node->down becomes NULL.
+			 */
+			ISC_LIST_APPEND(rbtdb->deadnodes[bucketnum],
+					node, deadlink);
 		}
 		node = ISC_LIST_HEAD(rbtdb->deadnodes[bucketnum]);
 		count--;
@@ -1945,6 +1971,7 @@ decrement_reference(dns_rbtdb_t *rbtdb, dns_rbtnode_t *node,
 {
 	isc_result_t result;
 	bool write_locked;
+	bool locked = tlock != isc_rwlocktype_none;
 	rbtdb_nodelock_t *nodelock;
 	int bucket = node->locknum;
 	bool no_reference = true;
@@ -1952,12 +1979,12 @@ decrement_reference(dns_rbtdb_t *rbtdb, dns_rbtnode_t *node,

 	nodelock = &rbtdb->node_locks[bucket];

-#define KEEP_NODE(n, r) \
-	((n)->data != NULL || (n)->down != NULL || \
+#define KEEP_NODE(n, r, l) \
+	((n)->data != NULL || ((l) && (n)->down != NULL) || \
 	 (n) == (r)->origin_node || (n) == (r)->nsec3_origin_node)

 	/* Handle easy and typical case first. */
-	if (!node->dirty && KEEP_NODE(node, rbtdb)) {
+	if (!node->dirty && KEEP_NODE(node, rbtdb, locked)) {
 		if (isc_refcount_decrement(&node->references) == 1) {
 			refs = isc_refcount_decrement(&nodelock->references);
 			INSIST(refs > 0);
@@ -2024,8 +2051,9 @@ decrement_reference(dns_rbtdb_t *rbtdb, dns_rbtnode_t *node,
 	refs = isc_refcount_decrement(&nodelock->references);
 	INSIST(refs > 0);

-	if (KEEP_NODE(node, rbtdb))
+	if (KEEP_NODE(node, rbtdb, locked || write_locked)) {
 		goto restore_locks;
+	}

 #undef KEEP_NODE

@@ -2050,21 +2078,8 @@ decrement_reference(dns_rbtdb_t *rbtdb, dns_rbtnode_t *node,
 		 * it's their responsibility to purge stale leaves (e.g. by
 		 * periodic walk-through).
 		 */
-		if (!pruning && node->parent != NULL &&
-		    node->parent->down == node && node->left == NULL &&
-		    node->right == NULL && rbtdb->task != NULL) {
-			isc_event_t *ev;
-			dns_db_t *db;
-
-			ev = isc_event_allocate(rbtdb->common.mctx, NULL,
-						DNS_EVENT_RBTPRUNE,
-						prune_tree, node,
-						sizeof(isc_event_t));
-			new_reference(rbtdb, node);
-			db = NULL;
-			attach((dns_db_t *)rbtdb, &db);
-			ev->ev_sender = db;
-			isc_task_send(rbtdb->task, &ev);
+		if (!pruning && is_leaf(node) && rbtdb->task != NULL) {
+			send_to_prune_tree(rbtdb, node);
 			no_reference = false;
 		} else {
 			delete_node(rbtdb, node);
@@ -6822,9 +6837,10 @@ subtractrdataset(dns_db_t *db, dns_dbnode_t *node, dns_dbversion_t *version,
 	 * this is deferred until closeversion() is called.
 	 */
 	if (result == ISC_R_SUCCESS && version == NULL && !IS_CACHE(rbtdb)) {
-		RBTDB_LOCK(&rbtdb->lock, isc_rwlocktype_write);
-		iszonesecure(db, rbtdb->current_version, rbtdb->origin_node);
-		RBTDB_UNLOCK(&rbtdb->lock, isc_rwlocktype_write);
+		RBTDB_LOCK(&rbtdb->lock, isc_rwlocktype_read);
+		version = rbtdb->current_version;
+		RBTDB_UNLOCK(&rbtdb->lock, isc_rwlocktype_read);
+		iszonesecure(db, version, rbtdb->origin_node);
 	}

 	return (result);
@@ -6880,9 +6896,10 @@ deleterdataset(dns_db_t *db, dns_dbnode_t *node, dns_dbversion_t *version,
 	 * this is deferred until closeversion() is called.
 	 */
 	if (result == ISC_R_SUCCESS && version == NULL && !IS_CACHE(rbtdb)) {
-		RBTDB_LOCK(&rbtdb->lock, isc_rwlocktype_write);
-		iszonesecure(db, rbtdb->current_version, rbtdb->origin_node);
-		RBTDB_UNLOCK(&rbtdb->lock, isc_rwlocktype_write);
+		RBTDB_LOCK(&rbtdb->lock, isc_rwlocktype_read);
+		version = rbtdb->current_version;
+		RBTDB_UNLOCK(&rbtdb->lock, isc_rwlocktype_read);
+		iszonesecure(db, version, rbtdb->origin_node);
 	}

 	return (result);
@@ -7314,9 +7331,12 @@ endload(dns_db_t *db, dns_rdatacallbacks_t *callbacks) {
 	 * zone key, we consider the zone secure.
 	 */
 	if (! IS_CACHE(rbtdb) && rbtdb->origin_node != NULL) {
-		iszonesecure(db, rbtdb->current_version, rbtdb->origin_node);
+		dns_dbversion_t *version = rbtdb->current_version;
+		RBTDB_UNLOCK(&rbtdb->lock, isc_rwlocktype_write);
+		iszonesecure(db, version, rbtdb->origin_node);
+	} else {
+		RBTDB_UNLOCK(&rbtdb->lock, isc_rwlocktype_write);
 	}
-	RBTDB_UNLOCK(&rbtdb->lock, isc_rwlocktype_write);

 	callbacks->add = NULL;
 	callbacks->add_private = NULL;
@@ -320,7 +320,7 @@ struct fetchctx {
 	/*%
 	 * The number of events we're waiting for.
 	 */
-	unsigned int			pending;
+	unsigned int			pending;	/* Bucket lock. */

 	/*%
 	 * The number of times we've "restarted" the current
@@ -349,7 +349,7 @@ struct fetchctx {
 	/*%
 	 * Number of queries that reference this context.
 	 */
-	unsigned int			nqueries;
+	unsigned int			nqueries;	/* Bucket lock. */

 	/*%
 	 * The reason to print when logging a successful
@@ -394,7 +394,7 @@ struct fetchctx {
 #define FCTX_ATTR_HAVEANSWER            0x0001
 #define FCTX_ATTR_GLUING                0x0002
 #define FCTX_ATTR_ADDRWAIT              0x0004
-#define FCTX_ATTR_SHUTTINGDOWN          0x0008
+#define FCTX_ATTR_SHUTTINGDOWN          0x0008		/* Bucket lock */
 #define FCTX_ATTR_WANTCACHE             0x0010
 #define FCTX_ATTR_WANTNCACHE            0x0020
 #define FCTX_ATTR_NEEDEDNS0             0x0040
@@ -512,18 +512,19 @@ struct dns_resolver {
 	unsigned int			spillatmax;
 	unsigned int			spillatmin;
 	isc_timer_t *			spillattimer;
-	bool			zero_no_soa_ttl;
+	bool				zero_no_soa_ttl;
 	unsigned int			query_timeout;
 	unsigned int			maxdepth;
 	unsigned int			maxqueries;
 	isc_result_t			quotaresp[2];

 	/* Additions for serve-stale feature. */
-	unsigned int			retryinterval; /* in milliseconds */
+	unsigned int			retryinterval;	/* in milliseconds */
 	unsigned int			nonbackofftries;

 	/* Atomic */
 	isc_refcount_t			references;
+	atomic_uint_fast32_t		zspill;		/* fetches-per-zone */
 	atomic_bool			exiting;

 	/* Locked by lock. */
@@ -531,9 +532,8 @@ struct dns_resolver {
 	unsigned int			activebuckets;
 	bool				priming;
 	unsigned int			spillat;	/* clients-per-query */
-	unsigned int			zspill;		/* fetches-per-zone */

-	dns_badcache_t  *		badcache;	 /* Bad cache. */
+	dns_badcache_t  *		badcache;	/* Bad cache. */

 	/* Locked by primelock. */
 	dns_fetch_t *			primefetch;
@@ -1137,9 +1137,8 @@ resquery_destroy(resquery_t **queryp) {
 	res = fctx->res;
 	bucket = fctx->bucketnum;

-	fctx->nqueries--;
-
 	LOCK(&res->buckets[bucket].lock);
+	fctx->nqueries--;
 	empty = fctx_decreference(query->fctx);
 	UNLOCK(&res->buckets[bucket].lock);

@@ -1513,7 +1512,7 @@ fcount_incr(fetchctx_t *fctx, bool force) {
 	isc_result_t result = ISC_R_SUCCESS;
 	zonebucket_t *dbucket;
 	fctxcount_t *counter;
-	unsigned int bucketnum, spill;
+	unsigned int bucketnum;

 	REQUIRE(fctx != NULL);
 	REQUIRE(fctx->res != NULL);
@@ -1522,10 +1521,6 @@ fcount_incr(fetchctx_t *fctx, bool force) {
 	bucketnum = dns_name_fullhash(&fctx->domain, false)
 			% RES_DOMAIN_BUCKETS;

-	LOCK(&fctx->res->lock);
-	spill = fctx->res->zspill;
-	UNLOCK(&fctx->res->lock);
-
 	dbucket = &fctx->res->dbuckets[bucketnum];

 	LOCK(&dbucket->lock);
@@ -1550,6 +1545,7 @@ fcount_incr(fetchctx_t *fctx, bool force) {
 			ISC_LIST_APPEND(dbucket->list, counter, link);
 		}
 	} else {
+		uint_fast32_t spill = atomic_load_acquire(&fctx->res->zspill);
 		if (!force && spill != 0 && counter->count >= spill) {
 			counter->dropped++;
 			fcount_logspill(fctx, counter);
@@ -1937,6 +1933,7 @@ fctx_query(fetchctx_t *fctx, dns_adbaddrinfo_t *addrinfo,
 	bool have_addr = false;
 	unsigned int srtt;
 	isc_dscp_t dscp = -1;
+	unsigned int bucketnum;

 	FCTXTRACE("query");

@@ -2158,7 +2155,10 @@ fctx_query(fetchctx_t *fctx, dns_adbaddrinfo_t *addrinfo,
 	fctx->querysent++;

 	ISC_LIST_APPEND(fctx->queries, query, link);
-	query->fctx->nqueries++;
+	bucketnum = fctx->bucketnum;
+	LOCK(&res->buckets[bucketnum].lock);
+	fctx->nqueries++;
+	UNLOCK(&res->buckets[bucketnum].lock);
 	if (isc_sockaddr_pf(&addrinfo->sockaddr) == PF_INET)
 		inc_stats(res, dns_resstatscounter_queryv4);
 	else
@@ -5288,11 +5288,12 @@ maybe_destroy(fetchctx_t *fctx, bool locked) {
 	dns_validator_t *validator, *next_validator;
 	bool dodestroy = false;

-	REQUIRE(SHUTTINGDOWN(fctx));
-
 	bucketnum = fctx->bucketnum;
 	if (!locked)
 		LOCK(&res->buckets[bucketnum].lock);
+
+	REQUIRE(SHUTTINGDOWN(fctx));
+
 	if (fctx->pending != 0 || fctx->nqueries != 0)
 		goto unlock;

@@ -6988,6 +6989,9 @@ fctx_increference(fetchctx_t *fctx) {
 	isc_refcount_increment(&fctx->references);
 }

+/*
+ * Requires bucket lock to be held.
+ */
 static bool
 fctx_decreference(fetchctx_t *fctx) {
 	bool bucket_empty = false;
@@ -9962,7 +9966,7 @@ dns_resolver_create(dns_view_t *view,
 	res->spillatmin = res->spillat = 10;
 	res->spillatmax = 100;
 	res->spillattimer = NULL;
-	res->zspill = 0;
+	atomic_init(&res->zspill, 0);
 	res->zero_no_soa_ttl = false;
 	res->retryinterval = 30000;
 	res->nonbackofftries = 3;
@@ -11242,9 +11246,7 @@ dns_resolver_setfetchesperzone(dns_resolver_t *resolver, uint32_t clients)
 {
 	REQUIRE(VALID_RESOLVER(resolver));

-	LOCK(&resolver->lock);
-	resolver->zspill = clients;
-	UNLOCK(&resolver->lock);
+	atomic_store_release(&resolver->zspill, clients);
 }


@@ -10830,7 +10830,9 @@ zone_maintenance(dns_zone_t *zone) {
 	default:
 		break;
 	}
+	LOCK_ZONE(zone);
 	zone_settimer(zone, &now);
+	UNLOCK_ZONE(zone);
 }

 void
@@ -13659,6 +13661,7 @@ zone_settimer(dns_zone_t *zone, isc_time_t *now) {
 	isc_result_t result;

 	REQUIRE(DNS_ZONE_VALID(zone));
+	REQUIRE(LOCKED_ZONE(zone));
 	ENTER;

 	if (DNS_ZONE_FLAG(zone, DNS_ZONEFLG_EXITING))
@@ -19038,6 +19041,7 @@ zone_rekey(dns_zone_t *zone) {
 		UNLOCK_ZONE(zone);
 	}

+	LOCK_ZONE(zone);
 	isc_time_settoepoch(&zone->refreshkeytime);

 	/*
@@ -19046,13 +19050,11 @@ zone_rekey(dns_zone_t *zone) {
 	if (kasp != NULL && nexttime > 0) {
 		isc_time_t timenext;

-		LOCK_ZONE(zone);
 		DNS_ZONE_TIME_ADD(&timenow, nexttime - now, &timenext);
 		zone->refreshkeytime = timenext;
-		UNLOCK_ZONE(zone);
-
 		zone_settimer(zone, &timenow);
 		isc_time_formattimestamp(&zone->refreshkeytime, timebuf, 80);
+
 		dnssec_log(zone, ISC_LOG_DEBUG(3),
 			   "next key event in %u seconds: %s",
 			   (nexttime - now), timebuf);
@@ -19067,11 +19069,9 @@ zone_rekey(dns_zone_t *zone) {
 		isc_time_t timethen;
 		isc_stdtime_t then;

-		LOCK_ZONE(zone);
 		DNS_ZONE_TIME_ADD(&timenow, zone->refreshkeyinterval,
 				  &timethen);
 		zone->refreshkeytime = timethen;
-		UNLOCK_ZONE(zone);

 		for (key = ISC_LIST_HEAD(dnskeys);
 		     key != NULL;
@@ -19084,13 +19084,11 @@ zone_rekey(dns_zone_t *zone) {
 			}

 			DNS_ZONE_TIME_ADD(&timenow, then - now, &timethen);
-			LOCK_ZONE(zone);
 			if (isc_time_compare(&timethen,
 					     &zone->refreshkeytime) < 0)
 			{
 				zone->refreshkeytime = timethen;
 			}
-			UNLOCK_ZONE(zone);
 		}

 		zone_settimer(zone, &timenow);
@@ -19098,6 +19096,7 @@ zone_rekey(dns_zone_t *zone) {
 		isc_time_formattimestamp(&zone->refreshkeytime, timebuf, 80);
 		dnssec_log(zone, ISC_LOG_INFO, "next key event: %s", timebuf);
 	}
+	UNLOCK_ZONE(zone);

 	result = ISC_R_SUCCESS;

@@ -46,7 +46,11 @@ typedef struct isc__networker {
 	bool			   paused;
 	bool			   finished;
 	isc_thread_t		   thread;
-	isc_queue_t		   *ievents;     /* incoming async events */
+	isc_queue_t		   *ievents;      /* incoming async events */
+	isc_queue_t		   *ievents_prio; /* priority async events
+						   * used for listening etc.
+						   * can be processed while
+						   * worker is paused */
 	isc_refcount_t		   references;
 	atomic_int_fast64_t	   pktcount;
 	char			   recvbuf[65536];
@@ -103,9 +107,6 @@ struct isc_nmiface {
 };

 typedef enum isc__netievent_type {
-	netievent_stop,
-	netievent_udplisten,
-	netievent_udpstoplisten,
 	netievent_udpsend,
 	netievent_udprecv,
 	netievent_tcpconnect,
@@ -113,11 +114,22 @@ typedef enum isc__netievent_type {
 	netievent_tcprecv,
 	netievent_tcpstartread,
 	netievent_tcppauseread,
-	netievent_tcplisten,
-	netievent_tcpstoplisten,
-	netievent_tcpclose,
+	netievent_tcpchildlisten,
+	netievent_tcpchildstop,
 	netievent_closecb,
 	netievent_shutdown,
+	netievent_stop,
+	netievent_udpstop,
+	netievent_tcpstop,
+	netievent_tcpclose,
+	netievent_tcpdnsclose,
+	netievent_prio = 0xff,	/* event type values higher than this
+				 * will be treated as high-priority
+				 * events, which can be processed
+				 * while the netmgr is paused.
+				 */
+	netievent_udplisten,
+	netievent_tcplisten,
 } isc__netievent_type;

 /*
@@ -154,11 +166,13 @@ typedef struct isc__nm_uvreq {
 	isc_nmsocket_t *	sock;
 	isc_nmhandle_t *	handle;
 	uv_buf_t		uvbuf;	/* translated isc_region_t, to be
-					   sent or received */
+					 * sent or received */
 	isc_sockaddr_t		local;	/* local address */
 	isc_sockaddr_t		peer;	/* peer address */
 	isc__nm_cb_t		cb;	/* callback */
 	void *			cbarg;	/* callback argument */
+	uv_pipe_t		ipc;	/* used for sending socket
+					 * uv_handles to other threads */
 	union {
 		uv_req_t		req;
 		uv_getaddrinfo_t	getaddrinfo;
@@ -178,12 +192,13 @@ typedef struct isc__netievent__socket {
 } isc__netievent__socket_t;

 typedef isc__netievent__socket_t isc__netievent_udplisten_t;
-typedef isc__netievent__socket_t isc__netievent_udpstoplisten_t;
-typedef isc__netievent__socket_t isc__netievent_tcpstoplisten_t;
+typedef isc__netievent__socket_t isc__netievent_udpstop_t;
+typedef isc__netievent__socket_t isc__netievent_tcpstop_t;
+typedef isc__netievent__socket_t isc__netievent_tcpchildstop_t;
 typedef isc__netievent__socket_t isc__netievent_tcpclose_t;
+typedef isc__netievent__socket_t isc__netievent_tcpdnsclose_t;
 typedef isc__netievent__socket_t isc__netievent_startread_t;
 typedef isc__netievent__socket_t isc__netievent_pauseread_t;
-typedef isc__netievent__socket_t isc__netievent_closecb_t;

 typedef struct isc__netievent__socket_req {
 	isc__netievent_type	type;
@@ -193,8 +208,18 @@ typedef struct isc__netievent__socket_req {

 typedef isc__netievent__socket_req_t isc__netievent_tcpconnect_t;
 typedef isc__netievent__socket_req_t isc__netievent_tcplisten_t;
+typedef isc__netievent__socket_req_t isc__netievent_tcpchildlisten_t;
 typedef isc__netievent__socket_req_t isc__netievent_tcpsend_t;

+typedef struct isc__netievent__socket_handle {
+	isc__netievent_type	type;
+	isc_nmsocket_t		*sock;
+	isc_nmhandle_t		*handle;
+} isc__netievent__socket_handle_t;
+
+typedef isc__netievent__socket_handle_t isc__netievent_closecb_t;
+
+
 typedef struct isc__netievent_udpsend {
 	isc__netievent_type	type;
 	isc_nmsocket_t		*sock;
@@ -274,6 +299,7 @@ typedef enum isc_nmsocket_type {
 	isc_nm_udplistener, /* Aggregate of nm_udpsocks */
 	isc_nm_tcpsocket,
 	isc_nm_tcplistener,
+	isc_nm_tcpchildlistener,
 	isc_nm_tcpdnslistener,
 	isc_nm_tcpdnssocket
 } isc_nmsocket_type;
@@ -293,7 +319,7 @@ struct isc_nmsocket {
 	isc_nm_t		*mgr;
 	isc_nmsocket_t		*parent;

-	/*
+	/*%
 	 * quota is the TCP client, attached when a TCP connection
 	 * is established. pquota is a non-attached pointer to the
 	 * TCP client quota, stored in listening sockets but only
@@ -303,7 +329,7 @@ struct isc_nmsocket {
 	isc_quota_t		*pquota;
 	bool			overquota;

-	/*
+	/*%
 	 * TCP read timeout timer.
 	 */
 	uv_timer_t		timer;
@@ -316,13 +342,18 @@ struct isc_nmsocket {
 	/*% server socket for connections */
 	isc_nmsocket_t		*server;

-	/*% children sockets for multi-socket setups */
+	/*% Child sockets for multi-socket setups */
 	isc_nmsocket_t		*children;
 	int			nchildren;
 	isc_nmiface_t		*iface;
 	isc_nmhandle_t		*tcphandle;

-	/*% extra data allocated at the end of each isc_nmhandle_t */
+	/*% Used to transfer listening TCP sockets to children */
+	uv_pipe_t		ipc;
+	char			ipc_pipe_name[64];
+	atomic_int_fast32_t	schildren;
+
+	/*% Extra data allocated at the end of each isc_nmhandle_t */
 	size_t			extrahandlesize;

 	/*% TCP backlog */
@@ -332,16 +363,17 @@ struct isc_nmsocket {
 	uv_os_sock_t		fd;
 	union uv_any_handle	uv_handle;

+	/*% Peer address */
 	isc_sockaddr_t		peer;

 	/* Atomic */
-	/*% Number of running (e.g. listening) children sockets */
+	/*% Number of running (e.g. listening) child sockets */
 	atomic_int_fast32_t     rchildren;

 	/*%
-	 * Socket if active if it's listening, working, etc., if we're
-	 * closing a socket it doesn't make any sense to e.g. still
-	 * push handles or reqs for reuse
+	 * Socket is active if it's listening, working, etc. If it's
+	 * closing, then it doesn't make a sense, for example, to
+	 * push handles or reqs for reuse.
 	 */
 	atomic_bool        	active;
 	atomic_bool	   	destroying;
@@ -349,7 +381,8 @@ struct isc_nmsocket {
 	/*%
 	 * Socket is closed if it's not active and all the possible
 	 * callbacks were fired, there are no active handles, etc.
-	 * active==false, closed==false means the socket is closing.
+	 * If active==false but closed==false, that means the socket
+	 * is closing.
 	 */
 	atomic_bool	      	closed;
 	atomic_bool	      	listening;
@@ -391,10 +424,19 @@ struct isc_nmsocket {
 	isc_astack_t 		*inactivehandles;
 	isc_astack_t 		*inactivereqs;

-	/* Used for active/rchildren during shutdown */
+	/*%
+	 * Used to wait for TCP listening events to complete, and
+	 * for the number of running children to reach zero during
+	 * shutdown.
+	 */
 	isc_mutex_t		lock;
 	isc_condition_t		cond;

+	/*%
+	 * Used to pass a result back from TCP listening events.
+	 */
+	isc_result_t		result;
+
 	/*%
 	 * List of active handles.
 	 * ah - current position in 'ah_frees'; this represents the
@@ -421,12 +463,12 @@ struct isc_nmsocket {
 	size_t			*ah_frees;
 	isc_nmhandle_t		**ah_handles;

-	/* Buffer for TCPDNS processing, optional */
+	/*% Buffer for TCPDNS processing */
 	size_t			buf_size;
 	size_t			buf_len;
 	unsigned char		*buf;

-	/*
+	/*%
 	 * This function will be called with handle->sock
 	 * as the argument whenever a handle's references drop
 	 * to zero, after its reset callback has been called.
@@ -524,13 +566,13 @@ isc__nmsocket_prep_destroy(isc_nmsocket_t *sock);
 */

 void
-isc__nm_async_closecb(isc__networker_t *worker, isc__netievent_t *ievent0);
+isc__nm_async_closecb(isc__networker_t *worker, isc__netievent_t *ev0);
 /*%<
 * Issue a 'handle closed' callback on the socket.
 */

 void
-isc__nm_async_shutdown(isc__networker_t *worker, isc__netievent_t *ievent0);
+isc__nm_async_shutdown(isc__networker_t *worker, isc__netievent_t *ev0);
 /*%<
 * Walk through all uv handles, get the underlying sockets and issue
 * close on them.
@@ -544,13 +586,12 @@ isc__nm_udp_send(isc_nmhandle_t *handle, isc_region_t *region,
 */

 void
-isc__nm_async_udplisten(isc__networker_t *worker, isc__netievent_t *ievent0);
+isc__nm_async_udplisten(isc__networker_t *worker, isc__netievent_t *ev0);

 void
-isc__nm_async_udpstoplisten(isc__networker_t *worker,
-			    isc__netievent_t *ievent0);
+isc__nm_async_udpstop(isc__networker_t *worker, isc__netievent_t *ev0);
 void
-isc__nm_async_udpsend(isc__networker_t *worker, isc__netievent_t *ievent0);
+isc__nm_async_udpsend(isc__networker_t *worker, isc__netievent_t *ev0);
 /*%<
 * Callback handlers for asynchronous UDP events (listen, stoplisten, send).
 */
@@ -575,20 +616,23 @@ isc__nm_tcp_shutdown(isc_nmsocket_t *sock);
 */

 void
-isc__nm_async_tcpconnect(isc__networker_t *worker, isc__netievent_t *ievent0);
+isc__nm_async_tcpconnect(isc__networker_t *worker, isc__netievent_t *ev0);
 void
-isc__nm_async_tcplisten(isc__networker_t *worker, isc__netievent_t *ievent0);
+isc__nm_async_tcplisten(isc__networker_t *worker, isc__netievent_t *ev0);
 void
-isc__nm_async_tcpstoplisten(isc__networker_t *worker,
-			    isc__netievent_t *ievent0);
+isc__nm_async_tcpchildlisten(isc__networker_t *worker, isc__netievent_t *ev0);
 void
-isc__nm_async_tcpsend(isc__networker_t *worker, isc__netievent_t *ievent0);
+isc__nm_async_tcpstop(isc__networker_t *worker, isc__netievent_t *ev0);
 void
-isc__nm_async_startread(isc__networker_t *worker, isc__netievent_t *ievent0);
+isc__nm_async_tcpchildstop(isc__networker_t *worker, isc__netievent_t *ev0);
 void
-isc__nm_async_pauseread(isc__networker_t *worker, isc__netievent_t *ievent0);
+isc__nm_async_tcpsend(isc__networker_t *worker, isc__netievent_t *ev0);
 void
-isc__nm_async_tcpclose(isc__networker_t *worker, isc__netievent_t *ievent0);
+isc__nm_async_startread(isc__networker_t *worker, isc__netievent_t *ev0);
+void
+isc__nm_async_pauseread(isc__networker_t *worker, isc__netievent_t *ev0);
+void
+isc__nm_async_tcpclose(isc__networker_t *worker, isc__netievent_t *ev0);
 /*%<
 * Callback handlers for asynchronous TCP events (connect, listen,
 * stoplisten, send, read, pause, close).
@@ -607,6 +651,9 @@ isc__nm_tcpdns_close(isc_nmsocket_t *sock);
 * Close a TCPDNS socket.
 */

+void
+isc__nm_async_tcpdnsclose(isc__networker_t *worker, isc__netievent_t *ev0);
+
 #define isc__nm_uverr2result(x) \
 	isc___nm_uverr2result(x, true, __FILE__, __LINE__)
 isc_result_t
@@ -29,6 +29,7 @@
 #include <isc/thread.h>
 #include <isc/util.h>

+#include "uv-compat.h"
 #include "netmgr-int.h"

 /*
@@ -42,6 +43,12 @@

 ISC_THREAD_LOCAL int isc__nm_tid_v = ISC_NETMGR_TID_UNKNOWN;

+#ifdef WIN32
+#define NAMED_PIPE_PATTERN "\\\\.\\pipe\\named-%d-%u.pipe"
+#else
+#define NAMED_PIPE_PATTERN "/tmp/named-%d-%u.pipe"
+#endif
+
 static void
 nmsocket_maybe_destroy(isc_nmsocket_t *sock);
 static void
@@ -50,6 +57,8 @@ static void *
 nm_thread(void *worker0);
 static void
 async_cb(uv_async_t *handle);
+static void
+process_queue(isc__networker_t *worker, isc_queue_t *queue);

 int
 isc_nm_tid() {
@@ -128,6 +137,7 @@ isc_nm_start(isc_mem_t *mctx, uint32_t workers) {
 		isc_condition_init(&worker->cond);

 		worker->ievents = isc_queue_new(mgr->mctx, 128);
+		worker->ievents_prio = isc_queue_new(mgr->mctx, 128);

 		/*
 		 * We need to do this here and not in nm_thread to avoid a
@@ -156,7 +166,8 @@ nm_destroy(isc_nm_t **mgr0) {

 	isc_nm_t *mgr = *mgr0;

-	LOCK(&mgr->lock);
+	isc_refcount_destroy(&mgr->references);
+
 	mgr->magic = 0;

 	for (size_t i = 0; i < mgr->nworkers; i++) {
@@ -169,23 +180,36 @@ nm_destroy(isc_nm_t **mgr0) {
 		isc__nm_enqueue_ievent(&mgr->workers[i], event);
 	}

+	LOCK(&mgr->lock);
 	while (atomic_load(&mgr->workers_running) > 0) {
 		WAIT(&mgr->wkstatecond, &mgr->lock);
 	}
 	UNLOCK(&mgr->lock);

 	for (size_t i = 0; i < mgr->nworkers; i++) {
-		/* Empty the async event queue */
-		isc__netievent_t *ievent;
+		isc__networker_t *worker = &mgr->workers[i];
+		isc__netievent_t *ievent = NULL;
+		int r;
+
+		/* Empty the async event queues */
 		while ((ievent = (isc__netievent_t *)
-			isc_queue_dequeue(mgr->workers[i].ievents)) != NULL)
+			isc_queue_dequeue(worker->ievents)) != NULL)
 		{
 			isc_mempool_put(mgr->evpool, ievent);
 		}
-		int r = uv_loop_close(&mgr->workers[i].loop);
+
+		while ((ievent = (isc__netievent_t *)
+			isc_queue_dequeue(worker->ievents_prio)) != NULL)
+		{
+			isc_mempool_put(mgr->evpool, ievent);
+		}
+
+		r = uv_loop_close(&worker->loop);
 		INSIST(r == 0);
-		isc_queue_destroy(mgr->workers[i].ievents);
-		isc_thread_join(mgr->workers[i].thread, NULL);
+
+		isc_queue_destroy(worker->ievents);
+		isc_queue_destroy(worker->ievents_prio);
+		isc_thread_join(worker->thread, NULL);
 	}

 	isc_condition_destroy(&mgr->wkstatecond);
@@ -315,10 +339,18 @@ isc_nm_destroy(isc_nm_t **mgr0) {
 	 * Wait for the manager to be dereferenced elsewhere.
 	 */
 	while (isc_refcount_current(&mgr->references) > 1) {
+		/*
+		 * Sometimes libuv gets stuck, pausing and unpausing
+		 * netmgr goes over all events in async queue for all
+		 * the workers, and since it's done only on shutdown it
+		 * doesn't cost us anything.
+		 */
+		isc_nm_pause(mgr);
+		isc_nm_resume(mgr);
 #ifdef WIN32
-			_sleep(1000);
+		_sleep(1000);
 #else
-			usleep(1000000);
+		usleep(1000000);
 #endif
 	}

@@ -403,6 +435,9 @@ nm_thread(void *worker0) {
 			UNLOCK(&worker->mgr->lock);

 			WAIT(&worker->cond, &worker->lock);
+
+			/* Process priority events */
+			process_queue(worker, worker->ievents_prio);
 		}
 		if (pausing) {
 			uint32_t wp = atomic_fetch_sub_explicit(
@@ -452,7 +487,8 @@ nm_thread(void *worker0) {
 		/*
 		 * Empty the async queue.
 		 */
-		async_cb(&worker->async);
+		process_queue(worker, worker->ievents_prio);
+		process_queue(worker, worker->ievents);
 	}

 	LOCK(&worker->mgr->lock);
@@ -472,21 +508,27 @@ nm_thread(void *worker0) {
 static void
 async_cb(uv_async_t *handle) {
 	isc__networker_t *worker = (isc__networker_t *) handle->loop->data;
-	isc__netievent_t *ievent;
+	process_queue(worker, worker->ievents_prio);
+	process_queue(worker, worker->ievents);
+}
+
+static void
+process_queue(isc__networker_t *worker, isc_queue_t *queue) {
+	isc__netievent_t *ievent = NULL;

 	while ((ievent = (isc__netievent_t *)
-		isc_queue_dequeue(worker->ievents)) != NULL)
+		isc_queue_dequeue(queue)) != NULL)
 	{
 		switch (ievent->type) {
 		case netievent_stop:
-			uv_stop(handle->loop);
+			uv_stop(&worker->loop);
 			isc_mempool_put(worker->mgr->evpool, ievent);
 			return;
 		case netievent_udplisten:
 			isc__nm_async_udplisten(worker, ievent);
 			break;
-		case netievent_udpstoplisten:
-			isc__nm_async_udpstoplisten(worker, ievent);
+		case netievent_udpstop:
+			isc__nm_async_udpstop(worker, ievent);
 			break;
 		case netievent_udpsend:
 			isc__nm_async_udpsend(worker, ievent);
@@ -497,6 +539,9 @@ async_cb(uv_async_t *handle) {
 		case netievent_tcplisten:
 			isc__nm_async_tcplisten(worker, ievent);
 			break;
+		case netievent_tcpchildlisten:
+			isc__nm_async_tcpchildlisten(worker, ievent);
+			break;
 		case netievent_tcpstartread:
 			isc__nm_async_startread(worker, ievent);
 			break;
@@ -506,12 +551,18 @@ async_cb(uv_async_t *handle) {
 		case netievent_tcpsend:
 			isc__nm_async_tcpsend(worker, ievent);
 			break;
-		case netievent_tcpstoplisten:
-			isc__nm_async_tcpstoplisten(worker, ievent);
+		case netievent_tcpstop:
+			isc__nm_async_tcpstop(worker, ievent);
+			break;
+		case netievent_tcpchildstop:
+			isc__nm_async_tcpchildstop(worker, ievent);
 			break;
 		case netievent_tcpclose:
 			isc__nm_async_tcpclose(worker, ievent);
 			break;
+		case netievent_tcpdnsclose:
+			isc__nm_async_tcpdnsclose(worker, ievent);
+			break;
 		case netievent_closecb:
 			isc__nm_async_closecb(worker, ievent);
 			break;
@@ -544,7 +595,18 @@ isc__nm_put_ievent(isc_nm_t *mgr, void *ievent) {

 void
 isc__nm_enqueue_ievent(isc__networker_t *worker, isc__netievent_t *event) {
-	isc_queue_enqueue(worker->ievents, (uintptr_t)event);
+	if (event->type > netievent_prio) {
+		/*
+		 * We need to make sure this signal will be delivered and
+		 * the queue will be processed.
+		 */
+		LOCK(&worker->lock);
+		isc_queue_enqueue(worker->ievents_prio, (uintptr_t)event);
+		SIGNAL(&worker->cond);
+		UNLOCK(&worker->lock);
+	} else {
+		isc_queue_enqueue(worker->ievents, (uintptr_t)event);
+	}
 	uv_async_send(&worker->async);
 }

@@ -626,8 +688,9 @@ nmsocket_cleanup(isc_nmsocket_t *sock, bool dofree) {
 	sock->pquota = NULL;

 	if (sock->timer_initialized) {
-		uv_close((uv_handle_t *)&sock->timer, NULL);
 		sock->timer_initialized = false;
+		uv_timer_stop(&sock->timer);
+		uv_close((uv_handle_t *)&sock->timer, NULL);
 	}

 	isc_astack_destroy(sock->inactivehandles);
@@ -652,11 +715,9 @@ nmsocket_cleanup(isc_nmsocket_t *sock, bool dofree) {

 static void
 nmsocket_maybe_destroy(isc_nmsocket_t *sock) {
-	int active_handles = 0;
+	int active_handles;
 	bool destroy = false;

-	REQUIRE(!isc__nmsocket_active(sock));
-
 	if (sock->parent != NULL) {
 		/*
 		 * This is a child socket and cannot be destroyed except
@@ -673,7 +734,14 @@ nmsocket_maybe_destroy(isc_nmsocket_t *sock) {
 	 * accept destruction.
 	 */
 	LOCK(&sock->lock);
-	active_handles += atomic_load(&sock->ah);
+	if (atomic_load(&sock->active) || atomic_load(&sock->destroying) ||
+	    !atomic_load(&sock->closed) || atomic_load(&sock->references) != 0)
+	{
+		UNLOCK(&sock->lock);
+		return;
+	}
+
+	active_handles = atomic_load(&sock->ah);
 	if (sock->children != NULL) {
 		for (int i = 0; i < sock->nchildren; i++) {
 			LOCK(&sock->children[i].lock);
@@ -682,10 +750,7 @@ nmsocket_maybe_destroy(isc_nmsocket_t *sock) {
 		}
 	}

-	if (atomic_load(&sock->closed) &&
-	    atomic_load(&sock->references) == 0 &&
-	    (active_handles == 0 || sock->tcphandle != NULL))
-	{
+	if (active_handles == 0 || sock->tcphandle != NULL) {
 		destroy = true;
 	}
 	UNLOCK(&sock->lock);
@@ -790,6 +855,16 @@ isc__nmsocket_init(isc_nmsocket_t *sock, isc_nm_t *mgr,
 		sock->ah_handles[i] = NULL;
 	}

+	/*
+	 * Use a random number in the named pipe name. Also add getpid()
+	 * to the name to make sure we don't get a conflict between
+	 * different unit tests running at the same time, where the PRNG
+	 * is initialized to a constant seed.
+	 */
+	snprintf(sock->ipc_pipe_name, sizeof(sock->ipc_pipe_name),
+		 NAMED_PIPE_PATTERN, getpid(), isc_random32());
+	sock->ipc_pipe_name[sizeof(sock->ipc_pipe_name) - 1] = '\0';
+
 	isc_mutex_init(&sock->lock);
 	isc_condition_init(&sock->cond);
 	isc_refcount_init(&sock->references, 1);
@@ -805,7 +880,7 @@ isc__nmsocket_init(isc_nmsocket_t *sock, isc_nm_t *mgr,

 void
 isc__nm_alloc_cb(uv_handle_t *handle, size_t size, uv_buf_t *buf) {
-	isc_nmsocket_t *sock = (isc_nmsocket_t *) handle->data;
+	isc_nmsocket_t *sock = uv_handle_get_data(handle);
 	isc__networker_t *worker = NULL;

 	REQUIRE(VALID_NMSOCK(sock));
@@ -947,6 +1022,8 @@ static void
 nmhandle_free(isc_nmsocket_t *sock, isc_nmhandle_t *handle) {
 	size_t extra = sock->extrahandlesize;

+	isc_refcount_destroy(&handle->references);
+
 	if (handle->dofree != NULL) {
 		handle->dofree(handle->opaque);
 	}
@@ -958,11 +1035,37 @@ nmhandle_free(isc_nmsocket_t *sock, isc_nmhandle_t *handle) {
 	isc_mem_put(sock->mgr->mctx, handle, sizeof(isc_nmhandle_t) + extra);
 }

+static void
+nmhandle_deactivate(isc_nmsocket_t *sock, isc_nmhandle_t *handle) {
+	/*
+	 * We do all of this under lock to avoid races with socket
+	 * destruction.  We have to do this now, because at this point the
+	 * socket is either unused or still attached to event->sock.
+	 */
+	LOCK(&sock->lock);
+
+	INSIST(sock->ah_handles[handle->ah_pos] == handle);
+	INSIST(sock->ah_size > handle->ah_pos);
+	INSIST(atomic_load(&sock->ah) > 0);
+
+	sock->ah_handles[handle->ah_pos] = NULL;
+	size_t handlenum = atomic_fetch_sub(&sock->ah, 1) - 1;
+	sock->ah_frees[handlenum] = handle->ah_pos;
+	handle->ah_pos = 0;
+	bool reuse = false;
+	if (atomic_load(&sock->active)) {
+		reuse = isc_astack_trypush(sock->inactivehandles,
+					   handle);
+	}
+	if (!reuse) {
+		nmhandle_free(sock, handle);
+	}
+	UNLOCK(&sock->lock);
+}
+
 void
 isc_nmhandle_unref(isc_nmhandle_t *handle) {
 	isc_nmsocket_t *sock = NULL;
-	size_t handlenum;
-	bool reuse = false;
 	int refs;

 	REQUIRE(VALID_NMHANDLE(handle));
@@ -980,61 +1083,34 @@ isc_nmhandle_unref(isc_nmhandle_t *handle) {
 		handle->doreset(handle->opaque);
 	}

-	/*
-	 * We do all of this under lock to avoid races with socket
-	 * destruction.
-	 */
-	LOCK(&sock->lock);
-
-	INSIST(sock->ah_handles[handle->ah_pos] == handle);
-	INSIST(sock->ah_size > handle->ah_pos);
-	INSIST(atomic_load(&sock->ah) > 0);
-
-	sock->ah_handles[handle->ah_pos] = NULL;
-	handlenum = atomic_fetch_sub(&sock->ah, 1) - 1;
-	sock->ah_frees[handlenum] = handle->ah_pos;
-	handle->ah_pos = 0;
-
-	if (atomic_load(&sock->active)) {
-		reuse = isc_astack_trypush(sock->inactivehandles,
-					   handle);
-	}
-
-	UNLOCK(&sock->lock);
-
-	if (!reuse) {
-		nmhandle_free(sock, handle);
-	}
-
 	/*
 	 * The handle is closed. If the socket has a callback configured
 	 * for that (e.g., to perform cleanup after request processing),
-	 * call it now.
+	 * call it now, or schedule it to run asynchronously.
 	 */
 	if (sock->closehandle_cb != NULL) {
 		if (sock->tid == isc_nm_tid()) {
 			sock->closehandle_cb(sock);
 		} else {
-			isc__netievent_closecb_t * event =
+			isc__netievent_closecb_t *event =
 				isc__nm_get_ievent(sock->mgr,
 						   netievent_closecb);
 			isc_nmsocket_attach(sock, &event->sock);
+			event->handle = handle;
 			isc__nm_enqueue_ievent(&sock->mgr->workers[sock->tid],
 					       (isc__netievent_t *) event);
+
 			/*
-			 * If we do this asynchronously then the async event
-			 * will clean the socket, so just exit.
+			 * If we're doing this asynchronously, then the
+			 * async event will take care of cleaning up the
+			 * handle and closing the socket.
 			 */
 			return;
 		}
 	}

-	if (atomic_load(&sock->ah) == 0 &&
-	    !atomic_load(&sock->active) &&
-	    !atomic_load(&sock->destroying))
-	{
-		nmsocket_maybe_destroy(sock);
-	}
+	nmhandle_deactivate(sock, handle);
+	nmsocket_maybe_destroy(sock);
 }

 void *
@@ -1166,9 +1242,9 @@ isc_nm_send(isc_nmhandle_t *handle, isc_region_t *region,
 }

 void
-isc__nm_async_closecb(isc__networker_t *worker, isc__netievent_t *ievent0) {
+isc__nm_async_closecb(isc__networker_t *worker, isc__netievent_t *ev0) {
 	isc__netievent_closecb_t *ievent =
-		(isc__netievent_closecb_t *) ievent0;
+		(isc__netievent_closecb_t *) ev0;

 	REQUIRE(VALID_NMSOCK(ievent->sock));
 	REQUIRE(ievent->sock->tid == isc_nm_tid());
@@ -1176,6 +1252,8 @@ isc__nm_async_closecb(isc__networker_t *worker, isc__netievent_t *ievent0) {

 	UNUSED(worker);

+	nmhandle_deactivate(ievent->sock, ievent->handle);
+
 	ievent->sock->closehandle_cb(ievent->sock);
 	isc_nmsocket_detach(&ievent->sock);
 }
@@ -1186,7 +1264,7 @@ shutdown_walk_cb(uv_handle_t *handle, void *arg) {

 	switch(handle->type) {
 	case UV_TCP:
-		isc__nm_tcp_shutdown((isc_nmsocket_t *) handle->data);
+		isc__nm_tcp_shutdown(uv_handle_get_data(handle));
 		break;
 	default:
 		break;
@@ -1194,8 +1272,8 @@ shutdown_walk_cb(uv_handle_t *handle, void *arg) {
 }

 void
-isc__nm_async_shutdown(isc__networker_t *worker, isc__netievent_t *ievent0) {
-	UNUSED(ievent0);
+isc__nm_async_shutdown(isc__networker_t *worker, isc__netievent_t *ev0) {
+	UNUSED(ev0);
 	uv_walk(&worker->loop, shutdown_walk_cb, NULL);
 }

@@ -10,6 +10,7 @@
 */

 #include <unistd.h>
+#include <libgen.h>
 #include <uv.h>

 #include <isc/atomic.h>
@@ -28,6 +29,7 @@
 #include <isc/thread.h>
 #include <isc/util.h>

+#include "uv-compat.h"
 #include "netmgr-int.h"

 static int
@@ -50,9 +52,24 @@ read_cb(uv_stream_t *stream, ssize_t nread, const uv_buf_t *buf);
 static void
 tcp_close_cb(uv_handle_t *uvhandle);

+static void
+ipc_connection_cb(uv_stream_t *stream, int status);
+static void
+ipc_write_cb(uv_write_t *uvreq, int status);
+static void
+ipc_close_cb(uv_handle_t *handle);
+static void
+childlisten_ipc_connect_cb(uv_connect_t *uvreq, int status);
+static void
+childlisten_read_cb(uv_stream_t *stream, ssize_t nread, const uv_buf_t *buf);
+static void
+stoplistening(isc_nmsocket_t *sock);
+static void
+tcp_listenclose_cb(uv_handle_t *handle);
+
 static int
 tcp_connect_direct(isc_nmsocket_t *sock, isc__nm_uvreq_t *req) {
-	isc__networker_t *worker;
+	isc__networker_t *worker = NULL;
 	int r;

 	REQUIRE(isc__nm_in_netthread());
@@ -71,16 +88,16 @@ tcp_connect_direct(isc_nmsocket_t *sock, isc__nm_uvreq_t *req) {
 			return (r);
 		}
 	}
-
+	uv_handle_set_data(&sock->uv_handle.handle, sock);
 	r = uv_tcp_connect(&req->uv_req.connect, &sock->uv_handle.tcp,
 			   &req->peer.type.sa, tcp_connect_cb);
 	return (r);
 }

 void
-isc__nm_async_tcpconnect(isc__networker_t *worker, isc__netievent_t *ievent0) {
+isc__nm_async_tcpconnect(isc__networker_t *worker, isc__netievent_t *ev0) {
 	isc__netievent_tcpconnect_t *ievent =
-		(isc__netievent_tcpconnect_t *) ievent0;
+		(isc__netievent_tcpconnect_t *) ev0;
 	isc_nmsocket_t *sock = ievent->sock;
 	isc__nm_uvreq_t *req = ievent->req;
 	int r;
@@ -98,7 +115,8 @@ isc__nm_async_tcpconnect(isc__networker_t *worker, isc__netievent_t *ievent0) {
 static void
 tcp_connect_cb(uv_connect_t *uvreq, int status) {
 	isc__nm_uvreq_t *req = (isc__nm_uvreq_t *) uvreq->data;
-	isc_nmsocket_t *sock = uvreq->handle->data;
+	isc_nmsocket_t *sock = NULL;
+	sock = uv_handle_get_data((uv_handle_t *) uvreq->handle);

 	REQUIRE(VALID_UVREQ(req));

@@ -131,21 +149,26 @@ isc_result_t
 isc_nm_listentcp(isc_nm_t *mgr, isc_nmiface_t *iface,
 		 isc_nm_cb_t cb, void *cbarg,
 		 size_t extrahandlesize, int backlog,
-		 isc_quota_t *quota,
-		 isc_nmsocket_t **sockp)
+		 isc_quota_t *quota, isc_nmsocket_t **sockp)
 {
-	isc__netievent_tcplisten_t *ievent = NULL;
 	isc_nmsocket_t *nsock = NULL;
+	isc__netievent_tcplisten_t *ievent = NULL;

 	REQUIRE(VALID_NM(mgr));

 	nsock = isc_mem_get(mgr->mctx, sizeof(*nsock));
 	isc__nmsocket_init(nsock, mgr, isc_nm_tcplistener);
 	nsock->iface = iface;
+	nsock->nchildren = mgr->nworkers;
+	atomic_init(&nsock->rchildren, mgr->nworkers);
+	nsock->children = isc_mem_get(mgr->mctx,
+				      mgr->nworkers * sizeof(*nsock));
+	memset(nsock->children, 0, mgr->nworkers * sizeof(*nsock));
 	nsock->rcb.accept = cb;
 	nsock->rcbarg = cbarg;
 	nsock->extrahandlesize = extrahandlesize;
 	nsock->backlog = backlog;
+	nsock->result = ISC_R_SUCCESS;
 	if (quota != NULL) {
 		/*
 		 * We don't attach to quota, just assign - to avoid
@@ -153,82 +176,332 @@ isc_nm_listentcp(isc_nm_t *mgr, isc_nmiface_t *iface,
 		 */
 		nsock->pquota = quota;
 	}
-	nsock->tid = isc_random_uniform(mgr->nworkers);

-	/*
-	 * Listening to TCP is rare enough not to care about the
-	 * added overhead from passing this to another thread.
-	 */
 	ievent = isc__nm_get_ievent(mgr, netievent_tcplisten);
 	ievent->sock = nsock;
-	isc__nm_enqueue_ievent(&mgr->workers[nsock->tid],
-			       (isc__netievent_t *) ievent);
-	*sockp = nsock;
+	if (isc__nm_in_netthread()) {
+		nsock->tid = isc_nm_tid();
+		isc__nm_async_tcplisten(&mgr->workers[nsock->tid],
+				       (isc__netievent_t *) ievent);
+		isc__nm_put_ievent(mgr, ievent);
+	} else {
+		nsock->tid = isc_random_uniform(mgr->nworkers);
+		isc__nm_enqueue_ievent(&mgr->workers[nsock->tid],
+				       (isc__netievent_t *) ievent);

-	return (ISC_R_SUCCESS);
+		LOCK(&nsock->lock);
+		while (!atomic_load(&nsock->listening)) {
+			WAIT(&nsock->cond, &nsock->lock);
+		}
+		UNLOCK(&nsock->lock);
+	}
+
+	if (nsock->result == ISC_R_SUCCESS) {
+		*sockp = nsock;
+		return (ISC_R_SUCCESS);
+	} else {
+		isc_result_t result = nsock->result;
+		isc_nmsocket_detach(&nsock);
+		return (result);
+	}
 }

+#ifndef WIN32
+/*
+ * Run fsync() on the directory containing a socket's IPC pipe, to
+ * ensure that all threads can see the pipe.
+ */
+static void
+syncdir(const isc_nmsocket_t *sock) {
+	 char *pipe = isc_mem_strdup(sock->mgr->mctx, sock->ipc_pipe_name);
+	 int fd = open(dirname(pipe), O_RDONLY);
+
+	 RUNTIME_CHECK(fd >= 0);
+
+	 isc_mem_free(sock->mgr->mctx, pipe);
+
+	 fsync(fd);
+	 close(fd);
+}
+#endif
+
+/*
+ * For multi-threaded TCP listening, we create a single "parent" socket,
+ * bind to it, and then pass its uv_handle to a set of child sockets, one
+ * per worker. For thread safety, the passing of the socket's uv_handle has
+ * to be done via IPC socket.
+ *
+ * This design pattern is ugly but it's what's recommended by the libuv
+ * documentation.  (A prior version of libuv had uv_export() and
+ * uv_import() functions which would have simplified this greatly, but
+ * they have been deprecated and removed.)
+ */
 void
-isc__nm_async_tcplisten(isc__networker_t *worker, isc__netievent_t *ievent0) {
+isc__nm_async_tcplisten(isc__networker_t *worker, isc__netievent_t *ev0) {
 	isc__netievent_tcplisten_t *ievent =
-		(isc__netievent_tcplisten_t *) ievent0;
+		(isc__netievent_tcplisten_t *) ev0;
 	isc_nmsocket_t *sock = ievent->sock;
 	int r;

 	REQUIRE(isc__nm_in_netthread());
 	REQUIRE(sock->type == isc_nm_tcplistener);

+	/* Initialize children now to make cleaning up easier */
+	for (int i = 0; i < sock->nchildren; i++) {
+		isc_nmsocket_t *csock = &sock->children[i];
+
+		isc__nmsocket_init(csock, sock->mgr, isc_nm_tcpchildlistener);
+		csock->parent = sock;
+		csock->iface = sock->iface;
+		csock->tid = i;
+		csock->pquota = sock->pquota;
+		csock->backlog = sock->backlog;
+		csock->extrahandlesize = sock->extrahandlesize;
+
+		INSIST(csock->rcb.recv == NULL && csock->rcbarg == NULL);
+		csock->rcb.accept = sock->rcb.accept;
+		csock->rcbarg = sock->rcbarg;
+		csock->fd = -1;
+	}
+
 	r = uv_tcp_init(&worker->loop, &sock->uv_handle.tcp);
 	if (r != 0) {
-		return;
+		/* It was never opened */
+		atomic_store(&sock->closed, true);
+		sock->result = isc__nm_uverr2result(r);
+		goto done;
 	}

-	uv_tcp_bind(&sock->uv_handle.tcp, &sock->iface->addr.type.sa, 0);
-	r = uv_listen((uv_stream_t *) &sock->uv_handle.tcp, sock->backlog,
-		      tcp_connection_cb);
+	r = uv_tcp_bind(&sock->uv_handle.tcp, &sock->iface->addr.type.sa, 0);
 	if (r != 0) {
-		return;
+		uv_close(&sock->uv_handle.handle, tcp_close_cb);
+		sock->result = isc__nm_uverr2result(r);
+		goto done;
+	}
+	uv_handle_set_data(&sock->uv_handle.handle, sock);
+
+	/*
+	 * uv_pipe_init() is incorrectly documented in libuv, and the
+	 * example in benchmark-multi-accept.c is also wrong.
+	 *
+	 * The third parameter ('ipc') indicates that the pipe will be
+	 * used to *send* a handle to other threads. Therefore, it must
+	 * must be set to 0 for a 'listening' IPC socket, and 1
+	 * only for sockets that are really passing FDs between
+	 * threads.
+	 */
+	r = uv_pipe_init(&worker->loop, &sock->ipc, 0);
+	RUNTIME_CHECK(r == 0);
+
+	uv_handle_set_data((uv_handle_t *)&sock->ipc, sock);
+	r = uv_pipe_bind(&sock->ipc, sock->ipc_pipe_name);
+	RUNTIME_CHECK(r == 0);
+
+	r = uv_listen((uv_stream_t *) &sock->ipc, sock->nchildren,
+		      ipc_connection_cb);
+	RUNTIME_CHECK(r == 0);
+
+#ifndef WIN32
+	/*
+	 * On Unices a child thread might not see the pipe yet;
+	 * that happened quite often in unit tests on FreeBSD.
+	 * Syncing the directory ensures that the pipe is visible
+	 * to everyone.
+	 * This isn't done on Windows because named pipes exist
+	 * within a different namespace, not on VFS.
+	 */
+	syncdir(sock);
+#endif
+
+	/*
+	 * For each worker we send a 'tcpchildlisten' event. The child
+	 * listener will then receive its version of the socket
+	 * uv_handle via IPC in isc__nm_async_tcpchildlisten().
+	 */
+	for (int i = 0; i < sock->nchildren; i++) {
+		isc_nmsocket_t *csock = &sock->children[i];
+		isc__netievent_tcpchildlisten_t *event = NULL;
+
+		event = isc__nm_get_ievent(csock->mgr,
+					   netievent_tcpchildlisten);
+		event->sock = csock;
+		if (csock->tid == isc_nm_tid()) {
+			isc__nm_async_tcpchildlisten(&sock->mgr->workers[i],
+						  (isc__netievent_t *) event);
+			isc__nm_put_ievent(sock->mgr, event);
+		} else {
+			isc__nm_enqueue_ievent(&sock->mgr->workers[i],
+					       (isc__netievent_t *) event);
+		}
 	}

+ done:
 	atomic_store(&sock->listening, true);
-
+	LOCK(&sock->lock);
+	SIGNAL(&sock->cond);
+	UNLOCK(&sock->lock);
 	return;
 }

+/*
+ * The parent received an IPC connection from a child and can now send
+ * the uv_handle to the child for listening.
+ */
+static void
+ipc_connection_cb(uv_stream_t *stream, int status) {
+	isc_nmsocket_t *sock = uv_handle_get_data((uv_handle_t *) stream);
+	isc__networker_t *worker = &sock->mgr->workers[isc_nm_tid()];
+	isc__nm_uvreq_t *nreq = isc__nm_uvreq_get(sock->mgr, sock);
+	int r;
+
+	REQUIRE(status == 0);
+
+	/*
+	 * The buffer can be anything, it will be ignored, but it has to
+	 * be something that won't disappear.
+	 */
+	nreq->uvbuf = uv_buf_init((char *)nreq, 1);
+	uv_pipe_init(&worker->loop, &nreq->ipc, 1);
+	uv_handle_set_data((uv_handle_t *)&nreq->ipc, nreq);
+
+	r = uv_accept((uv_stream_t *) &sock->ipc, (uv_stream_t *) &nreq->ipc);
+	RUNTIME_CHECK(r == 0);
+
+	r = uv_write2(&nreq->uv_req.write,
+		      (uv_stream_t *) &nreq->ipc, &nreq->uvbuf, 1,
+		      (uv_stream_t *) &sock->uv_handle.stream, ipc_write_cb);
+	RUNTIME_CHECK(r == 0);
+}
+
+/*
+ * The call to send a socket uv_handle is complete; we may be able
+ * to close the IPC connection.
+ */
+static void
+ipc_write_cb(uv_write_t *uvreq, int status) {
+	isc__nm_uvreq_t *req = uvreq->data;
+
+	UNUSED(status);
+
+	/*
+	 * We want all children to get the socket. Once that's done,
+	 * we can stop listening on the IPC socket.
+	 */
+	if (atomic_fetch_add(&req->sock->schildren, 1) ==
+	    req->sock->nchildren - 1)
+	{
+		uv_close((uv_handle_t *) &req->sock->ipc, NULL);
+	}
+	uv_close((uv_handle_t *) &req->ipc, ipc_close_cb);
+}
+
+/*
+ * The IPC socket is closed: free its resources.
+ */
+static void
+ipc_close_cb(uv_handle_t *handle) {
+	isc__nm_uvreq_t *req = uv_handle_get_data(handle);
+	isc__nm_uvreq_put(&req, req->sock);
+}
+
+/*
+ * Connect to the parent socket and be ready to receive the uv_handle
+ * for the socket we'll be listening on.
+ */
+void
+isc__nm_async_tcpchildlisten(isc__networker_t *worker, isc__netievent_t *ev0) {
+	isc__netievent_tcpchildlisten_t *ievent =
+		(isc__netievent_tcpchildlisten_t *) ev0;
+	isc_nmsocket_t *sock = ievent->sock;
+	isc__nm_uvreq_t *req = NULL;
+	int r;
+
+	REQUIRE(isc__nm_in_netthread());
+	REQUIRE(sock->type == isc_nm_tcpchildlistener);
+
+	r = uv_pipe_init(&worker->loop, &sock->ipc, 1);
+	RUNTIME_CHECK(r == 0);
+
+	uv_handle_set_data((uv_handle_t *) &sock->ipc, sock);
+
+	req = isc__nm_uvreq_get(sock->mgr, sock);
+	uv_pipe_connect(&req->uv_req.connect, &sock->ipc,
+			sock->parent->ipc_pipe_name,
+			childlisten_ipc_connect_cb);
+}
+
+/* Child is now connected to parent via IPC and can begin reading. */
+static void
+childlisten_ipc_connect_cb(uv_connect_t *uvreq, int status) {
+	isc__nm_uvreq_t *req = uvreq->data;
+	isc_nmsocket_t *sock = req->sock;
+	int r;
+
+	UNUSED(status);
+
+	isc__nm_uvreq_put(&req, sock);
+
+	r = uv_read_start((uv_stream_t *) &sock->ipc, isc__nm_alloc_cb,
+			  childlisten_read_cb);
+	RUNTIME_CHECK(r == 0);
+}
+
+/*
+ * Child has received the socket uv_handle via IPC, and can now begin
+ * listening for connections and can close the IPC socket.
+ */
+static void
+childlisten_read_cb(uv_stream_t *stream, ssize_t nread, const uv_buf_t *buf) {
+	isc_nmsocket_t *sock = uv_handle_get_data((uv_handle_t *) stream);
+	isc__networker_t *worker = NULL;
+	uv_pipe_t *ipc = NULL;
+	uv_handle_type type;
+	int r;
+
+	UNUSED(nread);
+
+	REQUIRE(VALID_NMSOCK(sock));
+	REQUIRE(buf != NULL);
+
+	ipc = (uv_pipe_t *) stream;
+	type = uv_pipe_pending_type(ipc);
+	INSIST(type == UV_TCP);
+
+	isc__nm_free_uvbuf(sock, buf);
+	worker = &sock->mgr->workers[isc_nm_tid()];
+	uv_tcp_init(&worker->loop, (uv_tcp_t *) &sock->uv_handle.tcp);
+	uv_handle_set_data(&sock->uv_handle.handle, sock);
+
+	uv_accept(stream, &sock->uv_handle.stream);
+	r = uv_listen((uv_stream_t *) &sock->uv_handle.tcp, sock->backlog,
+		      tcp_connection_cb);
+	uv_close((uv_handle_t *) ipc, NULL);
+	if (r != 0) {
+		isc_log_write(isc_lctx, ISC_LOGCATEGORY_GENERAL,
+			      ISC_LOGMODULE_NETMGR, ISC_LOG_ERROR,
+			      "IPC socket close failed: %s",
+			      isc_result_totext(isc__nm_uverr2result(r)));
+		return;
+	}
+}
+
+
 void
 isc_nm_tcp_stoplistening(isc_nmsocket_t *sock) {
-	isc__netievent_tcpstoplisten_t *ievent = NULL;
+	isc__netievent_tcpstop_t *ievent = NULL;

 	REQUIRE(VALID_NMSOCK(sock));
 	REQUIRE(!isc__nm_in_netthread());

-	ievent = isc__nm_get_ievent(sock->mgr, netievent_tcpstoplisten);
+	ievent = isc__nm_get_ievent(sock->mgr, netievent_tcpstop);
 	isc_nmsocket_attach(sock, &ievent->sock);
 	isc__nm_enqueue_ievent(&sock->mgr->workers[sock->tid],
 			       (isc__netievent_t *) ievent);
 }

-static void
-stoplistening_cb(uv_handle_t *handle) {
-	isc_nmsocket_t *sock = handle->data;
-
-	LOCK(&sock->lock);
-	atomic_store(&sock->listening, false);
-	atomic_store(&sock->closed, true);
-	SIGNAL(&sock->cond);
-	UNLOCK(&sock->lock);
-
-	sock->pquota = NULL;
-
-	isc_nmsocket_detach(&sock);
-}
-
 void
-isc__nm_async_tcpstoplisten(isc__networker_t *worker,
-			    isc__netievent_t *ievent0)
-{
-	isc__netievent_tcpstoplisten_t *ievent =
-		(isc__netievent_tcpstoplisten_t *) ievent0;
+isc__nm_async_tcpstop(isc__networker_t *worker, isc__netievent_t *ev0) {
+	isc__netievent_tcpstop_t *ievent = (isc__netievent_tcpstop_t *) ev0;
 	isc_nmsocket_t *sock = ievent->sock;

 	UNUSED(worker);
@@ -237,12 +510,103 @@ isc__nm_async_tcpstoplisten(isc__networker_t *worker,
 	REQUIRE(VALID_NMSOCK(sock));
 	REQUIRE(sock->type == isc_nm_tcplistener);

-	uv_close(&sock->uv_handle.handle, stoplistening_cb);
+	/*
+	 * If network manager is interlocked, re-enqueue the event for later.
+	 */
+	if (!isc__nm_acquire_interlocked(sock->mgr)) {
+		isc__netievent_tcpstop_t *event = NULL;
+
+		event = isc__nm_get_ievent(sock->mgr,
+					   netievent_tcpstop);
+		event->sock = sock;
+		isc__nm_enqueue_ievent(&sock->mgr->workers[sock->tid],
+				       (isc__netievent_t *) event);
+	} else {
+		stoplistening(sock);
+		isc__nm_drop_interlocked(sock->mgr);
+	}
+}
+
+static void
+stoplistening(isc_nmsocket_t *sock) {
+	for (int i = 0; i < sock->nchildren; i++) {
+		isc__netievent_tcpchildstop_t *event = NULL;
+
+		/*
+		 * We can ignore the overhead of event allocation because
+		 * stoplistening is a rare event, and doing it this way
+		 * simplifies sock reference counting.
+		 */
+		event = isc__nm_get_ievent(sock->mgr, netievent_tcpchildstop);
+		isc_nmsocket_attach(&sock->children[i], &event->sock);
+
+		if (i == sock->tid) {
+			isc__nm_async_tcpchildstop(&sock->mgr->workers[i],
+						   (isc__netievent_t *) event);
+			isc__nm_put_ievent(sock->mgr, event);
+		} else {
+			isc__nm_enqueue_ievent(&sock->mgr->workers[i],
+					       (isc__netievent_t *) event);
+		}
+	}
+
+	LOCK(&sock->lock);
+	while (atomic_load_relaxed(&sock->rchildren) > 0) {
+		WAIT(&sock->cond, &sock->lock);
+	}
+	UNLOCK(&sock->lock);
+	uv_close((uv_handle_t *) &sock->uv_handle.tcp, tcp_listenclose_cb);
+}
+
+void
+isc__nm_async_tcpchildstop(isc__networker_t *worker, isc__netievent_t *ev0) {
+	isc__netievent_tcpchildstop_t *ievent =
+		(isc__netievent_tcpchildstop_t *) ev0;
+	isc_nmsocket_t *sock = ievent->sock;
+
+	UNUSED(worker);
+
+	REQUIRE(VALID_NMSOCK(sock));
+	REQUIRE(isc_nm_tid() == sock->tid);
+	REQUIRE(sock->type == isc_nm_tcpchildlistener);
+	REQUIRE(sock->parent != NULL);
+
+	/*
+	 * rchildren is atomic, but we still need to change it
+	 * under a lock because the parent is waiting on conditional
+	 * and without it we might deadlock.
+	 */
+	LOCK(&sock->parent->lock);
+	atomic_fetch_sub(&sock->parent->rchildren, 1);
+	UNLOCK(&sock->parent->lock);
+
+	uv_close((uv_handle_t *) &sock->uv_handle.tcp, tcp_listenclose_cb);
+	BROADCAST(&sock->parent->cond);
+}
+
+/*
+ * This callback is used for closing both child and parent listening
+ * sockets; that's why we need to choose the proper lock.
+ */
+static void
+tcp_listenclose_cb(uv_handle_t *handle) {
+	isc_nmsocket_t *sock = uv_handle_get_data(handle);
+	isc_mutex_t *lock = ((sock->parent != NULL)
+			     ? &sock->parent->lock
+			     : &sock->lock);
+
+	LOCK(lock);
+	atomic_store(&sock->closed, true);
+	atomic_store(&sock->listening, false);
+	sock->pquota = NULL;
+	UNLOCK(lock);
+
+	isc_nmsocket_detach(&sock);
 }

 static void
 readtimeout_cb(uv_timer_t *handle) {
-	isc_nmsocket_t *sock = (isc_nmsocket_t *) handle->data;
+	isc_nmsocket_t *sock = uv_handle_get_data((uv_handle_t *)handle);

 	REQUIRE(VALID_NMSOCK(sock));
 	REQUIRE(sock->tid == isc_nm_tid());
@@ -294,16 +658,16 @@ isc_nm_read(isc_nmhandle_t *handle, isc_nm_recv_cb_t cb, void *cbarg) {
 }

 void
-isc__nm_async_startread(isc__networker_t *worker, isc__netievent_t *ievent0) {
+isc__nm_async_startread(isc__networker_t *worker, isc__netievent_t *ev0) {
 	isc__netievent_startread_t *ievent =
-		(isc__netievent_startread_t *) ievent0;
+		(isc__netievent_startread_t *) ev0;
 	isc_nmsocket_t *sock = ievent->sock;

 	REQUIRE(worker->id == isc_nm_tid());
 	if (sock->read_timeout != 0) {
 		if (!sock->timer_initialized) {
 			uv_timer_init(&worker->loop, &sock->timer);
-			sock->timer.data = sock;
+			uv_handle_set_data((uv_handle_t *)&sock->timer, sock);
 			sock->timer_initialized = true;
 		}
 		uv_timer_start(&sock->timer, readtimeout_cb,
@@ -340,9 +704,9 @@ isc_nm_pauseread(isc_nmsocket_t *sock) {
 }

 void
-isc__nm_async_pauseread(isc__networker_t *worker, isc__netievent_t *ievent0) {
+isc__nm_async_pauseread(isc__networker_t *worker, isc__netievent_t *ev0) {
 	isc__netievent_pauseread_t *ievent =
-		(isc__netievent_pauseread_t *) ievent0;
+		(isc__netievent_pauseread_t *) ev0;
 	isc_nmsocket_t *sock = ievent->sock;

 	REQUIRE(VALID_NMSOCK(sock));
@@ -384,7 +748,7 @@ isc_nm_resumeread(isc_nmsocket_t *sock) {

 static void
 read_cb(uv_stream_t *stream, ssize_t nread, const uv_buf_t *buf) {
-	isc_nmsocket_t *sock = stream->data;
+	isc_nmsocket_t *sock = uv_handle_get_data((uv_handle_t *) stream);

 	REQUIRE(VALID_NMSOCK(sock));
 	REQUIRE(buf != NULL);
@@ -394,10 +758,14 @@ read_cb(uv_stream_t *stream, ssize_t nread, const uv_buf_t *buf) {
 			.base = (unsigned char *) buf->base,
 			.length = nread
 		};
-
-		INSIST(sock->rcb.recv != NULL);
-		sock->rcb.recv(sock->tcphandle, &region, sock->rcbarg);
-
+		/*
+		 * This might happen if the inner socket is closing.
+		 * It means that it's detached, so the socket will
+		 * be closed.
+		 */
+		if (sock->rcb.recv != NULL) {
+			sock->rcb.recv(sock->tcphandle, &region, sock->rcbarg);
+		}
 		sock->read_timeout = (atomic_load(&sock->keepalive)
 				      ? sock->mgr->keepalive
 				      : sock->mgr->idle);
@@ -415,8 +783,14 @@ read_cb(uv_stream_t *stream, ssize_t nread, const uv_buf_t *buf) {
 	if (sock->quota) {
 		isc_quota_detach(&sock->quota);
 	}
-	sock->rcb.recv(sock->tcphandle, NULL, sock->rcbarg);
-
+	/*
+	 * This might happen if the inner socket is closing.
+	 * It means that it's detached, so the socket will
+	 * be closed.
+	 */
+	if (sock->rcb.recv != NULL) {
+		sock->rcb.recv(sock->tcphandle, NULL, sock->rcbarg);
+	}
 	/*
 	 * We don't need to clean up now; the socket will be closed and
 	 * resources and quota reclaimed when handle is freed in
@@ -499,7 +873,7 @@ accept_connection(isc_nmsocket_t *ssock) {

 static void
 tcp_connection_cb(uv_stream_t *server, int status) {
-	isc_nmsocket_t *ssock = server->data;
+	isc_nmsocket_t *ssock = uv_handle_get_data((uv_handle_t *) server);
 	isc_result_t result;

 	UNUSED(status);
@@ -576,9 +950,9 @@ tcp_send_cb(uv_write_t *req, int status) {
 * Handle 'tcpsend' async event - send a packet on the socket
 */
 void
-isc__nm_async_tcpsend(isc__networker_t *worker, isc__netievent_t *ievent0) {
+isc__nm_async_tcpsend(isc__networker_t *worker, isc__netievent_t *ev0) {
 	isc_result_t result;
-	isc__netievent_tcpsend_t *ievent = (isc__netievent_tcpsend_t *) ievent0;
+	isc__netievent_tcpsend_t *ievent = (isc__netievent_tcpsend_t *) ev0;

 	REQUIRE(worker->id == ievent->sock->tid);

@@ -615,7 +989,7 @@ tcp_send_direct(isc_nmsocket_t *sock, isc__nm_uvreq_t *req) {

 static void
 tcp_close_cb(uv_handle_t *uvhandle) {
-	isc_nmsocket_t *sock = uvhandle->data;
+	isc_nmsocket_t *sock = uv_handle_get_data(uvhandle);

 	REQUIRE(VALID_NMSOCK(sock));

@@ -625,7 +999,7 @@ tcp_close_cb(uv_handle_t *uvhandle) {

 static void
 timer_close_cb(uv_handle_t *uvhandle) {
-	isc_nmsocket_t *sock = uvhandle->data;
+	isc_nmsocket_t *sock = uv_handle_get_data(uvhandle);

 	REQUIRE(VALID_NMSOCK(sock));

@@ -653,8 +1027,9 @@ tcp_close_direct(isc_nmsocket_t *sock) {
 		}
 	}
 	if (sock->timer_initialized) {
-		uv_close((uv_handle_t *)&sock->timer, timer_close_cb);
 		sock->timer_initialized = false;
+		uv_timer_stop(&sock->timer);
+		uv_close((uv_handle_t *)&sock->timer, timer_close_cb);
 	} else {
 		isc_nmsocket_detach(&sock->server);
 		uv_close(&sock->uv_handle.handle, tcp_close_cb);
@@ -682,9 +1057,8 @@ isc__nm_tcp_close(isc_nmsocket_t *sock) {
 }

 void
-isc__nm_async_tcpclose(isc__networker_t *worker, isc__netievent_t *ievent0) {
-	isc__netievent_tcpclose_t *ievent =
-		(isc__netievent_tcpclose_t *) ievent0;
+isc__nm_async_tcpclose(isc__networker_t *worker, isc__netievent_t *ev0) {
+	isc__netievent_tcpclose_t *ievent = (isc__netievent_tcpclose_t *) ev0;

 	REQUIRE(worker->id == ievent->sock->tid);

@@ -695,7 +1069,10 @@ void
 isc__nm_tcp_shutdown(isc_nmsocket_t *sock) {
 	REQUIRE(VALID_NMSOCK(sock));

-	if (sock->type == isc_nm_tcpsocket && sock->tcphandle != NULL) {
+	if (sock->type == isc_nm_tcpsocket &&
+	    sock->tcphandle != NULL &&
+	    sock->rcb.recv != NULL)
+	{
 		sock->rcb.recv(sock->tcphandle, NULL, sock->rcbarg);
 	}
 }
@@ -26,6 +26,7 @@
 #include <isc/thread.h>
 #include <isc/util.h>

+#include "uv-compat.h"
 #include "netmgr-int.h"

 #define TCPDNS_CLIENTS_PER_CONN 23
@@ -76,21 +77,22 @@ alloc_dnsbuf(isc_nmsocket_t *sock, size_t len) {

 static void
 timer_close_cb(uv_handle_t *handle) {
-	isc_nmsocket_t *sock = (isc_nmsocket_t *) handle->data;
+	isc_nmsocket_t *sock = (isc_nmsocket_t *) uv_handle_get_data(handle);
 	INSIST(VALID_NMSOCK(sock));
-	sock->timer_initialized = false;
 	atomic_store(&sock->closed, true);
 	isc_nmsocket_detach(&sock);
 }

 static void
 dnstcp_readtimeout(uv_timer_t *timer) {
-	isc_nmsocket_t *sock = (isc_nmsocket_t *) timer->data;
+	isc_nmsocket_t *sock =
+		(isc_nmsocket_t *) uv_handle_get_data((uv_handle_t *) timer);

 	REQUIRE(VALID_NMSOCK(sock));
+	REQUIRE(sock->tid == isc_nm_tid());

 	isc_nmsocket_detach(&sock->outer);
-	uv_close((uv_handle_t*) &sock->timer, timer_close_cb);
+	uv_close((uv_handle_t *) &sock->timer, timer_close_cb);
 }

 /*
@@ -201,6 +203,7 @@ dnslisten_readcb(isc_nmhandle_t *handle, isc_region_t *region, void *arg) {

 	REQUIRE(VALID_NMSOCK(dnssock));
 	REQUIRE(VALID_NMHANDLE(handle));
+	REQUIRE(dnssock->tid == isc_nm_tid());

 	if (region == NULL) {
 		/* Connection closed */
@@ -296,11 +299,15 @@ isc_nm_listentcpdns(isc_nm_t *mgr, isc_nmiface_t *iface,
 	result = isc_nm_listentcp(mgr, iface, dnslisten_acceptcb,
 				  dnslistensock, extrahandlesize, backlog,
 				  quota, &dnslistensock->outer);
-
-	atomic_store(&dnslistensock->listening, true);
-	*sockp = dnslistensock;
-
-	return (result);
+	if (result == ISC_R_SUCCESS) {
+		atomic_store(&dnslistensock->listening, true);
+		*sockp = dnslistensock;
+		return (ISC_R_SUCCESS);
+	} else {
+		atomic_store(&dnslistensock->closed, true);
+		isc_nmsocket_detach(&dnslistensock);
+		return (result);
+	}
 }

 void
@@ -483,10 +490,44 @@ isc__nm_tcpdns_send(isc_nmhandle_t *handle, isc_region_t *region,
 }


-void
-isc__nm_tcpdns_close(isc_nmsocket_t *sock) {
+static void
+tcpdns_close_direct(isc_nmsocket_t *sock) {
+	REQUIRE(sock->tid == isc_nm_tid());
 	if (sock->outer != NULL) {
+		sock->outer->rcb.recv = NULL;
 		isc_nmsocket_detach(&sock->outer);
 	}
-	uv_close((uv_handle_t*) &sock->timer, timer_close_cb);
+	/* We don't need atomics here, it's all in single network thread */
+	if (sock->timer_initialized) {
+		sock->timer_initialized = false;
+		uv_timer_stop(&sock->timer);
+		uv_close((uv_handle_t *) &sock->timer, timer_close_cb);
+	}
+}
+
+void
+isc__nm_tcpdns_close(isc_nmsocket_t *sock) {
+	REQUIRE(VALID_NMSOCK(sock));
+	REQUIRE(sock->type == isc_nm_tcpdnssocket);
+
+	if (sock->tid == isc_nm_tid()) {
+		tcpdns_close_direct(sock);
+	} else {
+		isc__netievent_tcpdnsclose_t *ievent =
+			isc__nm_get_ievent(sock->mgr, netievent_tcpdnsclose);
+
+		ievent->sock = sock;
+		isc__nm_enqueue_ievent(&sock->mgr->workers[sock->tid],
+				       (isc__netievent_t *) ievent);
+	}
+}
+
+void
+isc__nm_async_tcpdnsclose(isc__networker_t *worker, isc__netievent_t *ev0) {
+	isc__netievent_tcpdnsclose_t *ievent =
+		(isc__netievent_tcpdnsclose_t *) ev0;
+
+	REQUIRE(worker->id == ievent->sock->tid);
+
+	tcpdns_close_direct(ievent->sock);
 }
@@ -25,6 +25,8 @@
 #include <isc/sockaddr.h>
 #include <isc/thread.h>
 #include <isc/util.h>
+
+#include "uv-compat.h"
 #include "netmgr-int.h"

 static isc_result_t
@@ -113,9 +115,9 @@ isc_nm_listenudp(isc_nm_t *mgr, isc_nmiface_t *iface,
 * handle 'udplisten' async call - start listening on a socket.
 */
 void
-isc__nm_async_udplisten(isc__networker_t *worker, isc__netievent_t *ievent0) {
+isc__nm_async_udplisten(isc__networker_t *worker, isc__netievent_t *ev0) {
 	isc__netievent_udplisten_t *ievent =
-		(isc__netievent_udplisten_t *) ievent0;
+		(isc__netievent_udplisten_t *) ev0;
 	isc_nmsocket_t *sock = ievent->sock;

 	REQUIRE(sock->type == isc_nm_udpsocket);
@@ -123,7 +125,7 @@ isc__nm_async_udplisten(isc__networker_t *worker, isc__netievent_t *ievent0) {
 	REQUIRE(sock->parent != NULL);

 	uv_udp_init(&worker->loop, &sock->uv_handle.udp);
-	sock->uv_handle.udp.data = NULL;
+	uv_handle_set_data(&sock->uv_handle.handle, NULL);
 	isc_nmsocket_attach(sock,
 			    (isc_nmsocket_t **)&sock->uv_handle.udp.data);

@@ -140,7 +142,7 @@ isc__nm_async_udplisten(isc__networker_t *worker, isc__netievent_t *ievent0) {

 static void
 udp_close_cb(uv_handle_t *handle) {
-	isc_nmsocket_t *sock = handle->data;
+	isc_nmsocket_t *sock = uv_handle_get_data(handle);
 	atomic_store(&sock->closed, true);

 	isc_nmsocket_detach((isc_nmsocket_t **)&sock->uv_handle.udp.data);
@@ -149,6 +151,7 @@ udp_close_cb(uv_handle_t *handle) {
 static void
 stop_udp_child(isc_nmsocket_t *sock) {
 	INSIST(sock->type == isc_nm_udpsocket);
+	INSIST(sock->tid == isc_nm_tid());

 	uv_udp_recv_stop(&sock->uv_handle.udp);
 	uv_close((uv_handle_t *) &sock->uv_handle.udp, udp_close_cb);
@@ -171,14 +174,14 @@ stoplistening(isc_nmsocket_t *sock) {
 	INSIST(sock->type == isc_nm_udplistener);

 	for (int i = 0; i < sock->nchildren; i++) {
-		isc__netievent_udplisten_t *event = NULL;
+		isc__netievent_udpstop_t *event = NULL;

-		if (i == sock->tid) {
+		if (isc_nm_tid() == sock->tid) {
 			stop_udp_child(&sock->children[i]);
 			continue;
 		}

-		event = isc__nm_get_ievent(sock->mgr, netievent_udpstoplisten);
+		event = isc__nm_get_ievent(sock->mgr, netievent_udpstop);
 		event->sock = &sock->children[i];
 		isc__nm_enqueue_ievent(&sock->mgr->workers[i],
 				       (isc__netievent_t *) event);
@@ -196,7 +199,7 @@ stoplistening(isc_nmsocket_t *sock) {

 void
 isc_nm_udp_stoplistening(isc_nmsocket_t *sock) {
-	isc__netievent_udpstoplisten_t *ievent = NULL;
+	isc__netievent_udpstop_t *ievent = NULL;

 	/* We can't be launched from network thread, we'd deadlock */
 	REQUIRE(!isc__nm_in_netthread());
@@ -208,7 +211,7 @@ isc_nm_udp_stoplistening(isc_nmsocket_t *sock) {
 	 * event. Otherwise, go ahead and stop listening right away.
 	 */
 	if (!isc__nm_acquire_interlocked(sock->mgr)) {
-		ievent = isc__nm_get_ievent(sock->mgr, netievent_udpstoplisten);
+		ievent = isc__nm_get_ievent(sock->mgr, netievent_udpstop);
 		ievent->sock = sock;
 		isc__nm_enqueue_ievent(&sock->mgr->workers[sock->tid],
 				       (isc__netievent_t *) ievent);
@@ -219,14 +222,11 @@ isc_nm_udp_stoplistening(isc_nmsocket_t *sock) {
 }

 /*
- * handle 'udpstoplisten' async call - stop listening on a socket.
+ * handle 'udpstop' async call - stop listening on a socket.
 */
 void
-isc__nm_async_udpstoplisten(isc__networker_t *worker,
-			    isc__netievent_t *ievent0)
-{
-	isc__netievent_udplisten_t *ievent =
-		(isc__netievent_udplisten_t *) ievent0;
+isc__nm_async_udpstop(isc__networker_t *worker, isc__netievent_t *ev0) {
+	isc__netievent_udpstop_t *ievent = (isc__netievent_udpstop_t *) ev0;
 	isc_nmsocket_t *sock = ievent->sock;

 	REQUIRE(sock->iface != NULL);
@@ -246,7 +246,7 @@ isc__nm_async_udpstoplisten(isc__networker_t *worker,
 	if (!isc__nm_acquire_interlocked(sock->mgr)) {
 		isc__netievent_udplisten_t *event = NULL;

-		event = isc__nm_get_ievent(sock->mgr, netievent_udpstoplisten);
+		event = isc__nm_get_ievent(sock->mgr, netievent_udpstop);
 		event->sock = sock;
 		isc__nm_enqueue_ievent(&sock->mgr->workers[sock->tid],
 				       (isc__netievent_t *) event);
@@ -271,7 +271,7 @@ udp_recv_cb(uv_udp_t *handle, ssize_t nrecv, const uv_buf_t *buf,
 	isc_sockaddr_t sockaddr;
 	isc_sockaddr_t localaddr;
 	struct sockaddr_storage laddr;
-	isc_nmsocket_t *sock = (isc_nmsocket_t *) handle->data;
+	isc_nmsocket_t *sock = uv_handle_get_data((uv_handle_t *)handle);
 	isc_region_t region;
 	uint32_t maxudp;

@@ -337,7 +337,7 @@ isc__nm_udp_send(isc_nmhandle_t *handle, isc_region_t *region,
 	isc_nmsocket_t *psock = NULL, *rsock = NULL;
 	isc_nmsocket_t *sock = handle->sock;
 	isc_sockaddr_t *peer = &handle->peer;
-	isc__netievent_udpsend_t *ievent;
+	isc__netievent_udpsend_t *ievent = NULL;
 	isc__nm_uvreq_t *uvreq = NULL;
 	int ntid;
 	uint32_t maxudp = atomic_load(&sock->mgr->maxudp);
@@ -405,9 +405,9 @@ isc__nm_udp_send(isc_nmhandle_t *handle, isc_region_t *region,
 * handle 'udpsend' async event - send a packet on the socket
 */
 void
-isc__nm_async_udpsend(isc__networker_t *worker, isc__netievent_t *ievent0) {
+isc__nm_async_udpsend(isc__networker_t *worker, isc__netievent_t *ev0) {
 	isc__netievent_udpsend_t *ievent =
-		(isc__netievent_udpsend_t *) ievent0;
+		(isc__netievent_udpsend_t *) ev0;

 	REQUIRE(worker->id == ievent->sock->tid);

@@ -0,0 +1,34 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+#pragma once
+#include <uv.h>
+
+/*
+ * Those functions were introduced in newer libuv, we still
+ * want BIND9 compile on older ones so we emulate them.
+ * They're inline to avoid conflicts when running with a newer
+ * library version.
+ */
+
+#ifndef HAVE_UV_HANDLE_GET_DATA
+static inline void*
+uv_handle_get_data(const uv_handle_t* handle) {
+    return (handle->data);
+}
+#endif
+
+#ifndef HAVE_UV_HANDLE_SET_DATA
+static inline void
+uv_handle_set_data(uv_handle_t* handle, void* data) {
+    handle->data = data;
+};
+#endif
@@ -1834,13 +1834,15 @@ destroy(isc__socket_t **sockp) {

 	socket_log(sock, NULL, CREATION, "destroying");

+	isc_refcount_destroy(&sock->references);
+
+	LOCK(&sock->lock);
 	INSIST(ISC_LIST_EMPTY(sock->connect_list));
 	INSIST(ISC_LIST_EMPTY(sock->accept_list));
 	INSIST(ISC_LIST_EMPTY(sock->recv_list));
 	INSIST(ISC_LIST_EMPTY(sock->send_list));
 	INSIST(sock->fd >= -1 && sock->fd < (int)manager->maxsocks);

-	LOCK(&sock->lock);
 	if (sock->fd >= 0) {
 		fd = sock->fd;
 		thread = &manager->threads[sock->threadid];
@@ -1932,12 +1934,14 @@ free_socket(isc__socket_t **socketp) {

 	INSIST(VALID_SOCKET(sock));
 	INSIST(isc_refcount_current(&sock->references) == 0);
+	LOCK(&sock->lock);
 	INSIST(!sock->connecting);
 	INSIST(ISC_LIST_EMPTY(sock->recv_list));
 	INSIST(ISC_LIST_EMPTY(sock->send_list));
 	INSIST(ISC_LIST_EMPTY(sock->accept_list));
 	INSIST(ISC_LIST_EMPTY(sock->connect_list));
 	INSIST(!ISC_LINK_LINKED(sock, link));
+	UNLOCK(&sock->lock);

 	sock->common.magic = 0;
 	sock->common.impmagic = 0;
@@ -83,6 +83,7 @@ struct ns_interfacemgr {
 	ISC_LIST(isc_sockaddr_t) listenon;
 	int			backlog;	/*%< Listen queue size */
 	unsigned int		udpdisp;	/*%< UDP dispatch count */
+	atomic_bool		shuttingdown;	/*%< Interfacemgr is shutting down */
 #ifdef USE_ROUTE_SOCKET
 	isc_task_t *		task;
 	isc_socket_t *		route;
@@ -217,6 +218,7 @@ ns_interfacemgr_create(isc_mem_t *mctx,
 	mgr->listenon4 = NULL;
 	mgr->listenon6 = NULL;
 	mgr->udpdisp = udpdisp;
+	atomic_init(&mgr->shuttingdown, false);

 	ISC_LIST_INIT(mgr->interfaces);
 	ISC_LIST_INIT(mgr->listenon);
@@ -360,6 +362,7 @@ ns_interfacemgr_shutdown(ns_interfacemgr_t *mgr) {
 	 * consider all interfaces "old".
 	 */
 	mgr->generation++;
+	atomic_store(&mgr->shuttingdown, true);
 #ifdef USE_ROUTE_SOCKET
 	LOCK(&mgr->lock);
 	if (mgr->route != NULL) {
@@ -1230,7 +1233,13 @@ ns_interfacemgr_listeningon(ns_interfacemgr_t *mgr,
 	bool result = false;

 	REQUIRE(NS_INTERFACEMGR_VALID(mgr));
-
+	/*
+	 * If the manager is shutting down it's safer to
+	 * return true.
+	 */
+	if (atomic_load(&mgr->shuttingdown)) {
+		return (true);
+	}
 	LOCK(&mgr->lock);
 	for (old = ISC_LIST_HEAD(mgr->listenon);
 	     old != NULL;
@@ -75,8 +75,8 @@ static dns_zone_t *served_zone = NULL;
 /*
 * We don't want to use netmgr-based client accounting, we need to emulate it.
 */
-atomic_uint_fast32_t client_refs[16];
-atomic_uintptr_t client_addrs[16];
+atomic_uint_fast32_t client_refs[32];
+atomic_uintptr_t client_addrs[32];

 void
 __wrap_isc_nmhandle_unref(isc_nmhandle_t *handle);
@@ -86,12 +86,12 @@ __wrap_isc_nmhandle_unref(isc_nmhandle_t *handle) {
 	ns_client_t *client = (ns_client_t *)handle;
 	int i;

-	for (i = 0; i < 16; i++) {
+	for (i = 0; i < 32; i++) {
 		if (atomic_load(&client_addrs[i]) == (uintptr_t) client) {
 			break;
 		}
 	}
-	REQUIRE(i < 16);
+	REQUIRE(i < 32);

 	if (atomic_fetch_sub(&client_refs[i], 1) == 1) {
 		dns_view_detach(&client->view);
@@ -561,14 +561,14 @@ ns_test_getclient(ns_interface_t *ifp0, bool tcp,

 	result = ns__client_setup(client, clientmgr, true);

-	for (i = 0; i < 16; i++) {
+	for (i = 0; i < 32; i++) {
 		if (atomic_load(&client_addrs[i]) == (uintptr_t) NULL ||
 		    atomic_load(&client_addrs[i]) == (uintptr_t) client)
 		{
 			break;
 		}
 	}
-	REQUIRE(i < 16);
+	REQUIRE(i < 32);

 	atomic_store(&client_refs[i], 2);
 	atomic_store(&client_addrs[i], (uintptr_t) client);
@@ -349,6 +349,7 @@
 ./bin/tests/pkcs11/benchmarks/sha1.c		C	2014,2015,2016,2018,2019
 ./bin/tests/pkcs11/benchmarks/sign.c		C	2014,2015,2016,2018,2019
 ./bin/tests/pkcs11/benchmarks/verify.c		C	2014,2015,2016,2018,2019
+./bin/tests/prepare-softhsm2.sh			X	2018,2019
 ./bin/tests/startperf/README			X	2011,2018,2019
 ./bin/tests/startperf/clean.sh			SH	2011,2012,2016,2018,2019
 ./bin/tests/startperf/makenames.pl		PERL	2011,2012,2016,2018,2019
@@ -2258,6 +2259,7 @@
 ./lib/isc/netmgr/tcp.c				C	2019
 ./lib/isc/netmgr/tcpdns.c			C	2019
 ./lib/isc/netmgr/udp.c				C	2019
+./lib/isc/netmgr/uv-compat.h			C	2019
 ./lib/isc/netmgr/uverr2result.c			C	2019
 ./lib/isc/netscope.c				C	2002,2004,2005,2006,2007,2016,2018,2019
 ./lib/isc/nonce.c				C	2018,2019
@@ -2591,7 +2593,7 @@
 ./util/nanny.pl					PERL	2000,2001,2004,2007,2012,2016,2018,2019
 ./util/new-func					PERL	2005,2007,2012,2016,2018,2019
 ./util/nt-kit					SH	1999,2000,2001,2004,2007,2012,2016,2018,2019
-./util/prepare-softhsm2.sh			X	2018,2019
+./util/parse_tsan.py				PYTHON-BIN	2019
 ./util/spacewhack.pl				PERL	2000,2001,2004,2007,2012,2016,2018,2019
 ./util/tabify-changes				SH	2004,2007,2012,2016,2018,2019
 ./util/update-drafts.pl				PERL	2000,2001,2004,2007,2012,2016,2018,2019
@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+############################################################################
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+############################################################################
+
+import sys, os, os.path, re
+from hashlib import sha256
+
+class State:
+    inside = False
+    block = ""
+    last_line = None
+
+    mutexes = {}
+    m_index = 1
+    threads = {}
+    t_index = 1
+    pointers = {}
+    p_index = 1
+
+    def init(self):
+        self.reset()
+
+    def reset(self):
+        self.inside = False
+        self.block = ""
+
+        self.mutexes = {}
+        self.threads = {}
+        self.pointers = {}
+        self.pointers["0x000000000000"] = 0
+
+        self.m_index = 1
+        self.t_index = 1
+        self.p_index = 1
+
+top = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+
+out = os.path.join(top, "tsan")
+
+if not os.path.isdir(out):
+    os.mkdir(out)
+
+# Regular Expressions
+mutex = re.compile(r"M\d+")
+thread = re.compile(r"T\d+")
+stack = re.compile(r"\s\(\S+\+0x\S+\)")
+pointer = re.compile(r"0x[0-9a-f]+")
+pid = re.compile(r"\(pid=\d+,?\)")
+tid = re.compile(r"tid=\d+,?\s*")
+worker = re.compile(r"\s+'(isc-worker|isc-net-)\d+'")
+path = re.compile(top + "/")
+
+s = State()
+
+
+with open(sys.argv[1], "r", encoding='utf-8') as f:
+    lines = f.readlines()
+    for line in lines:
+        if line == "==================\n":
+           if not s.inside:
+               s.inside = True
+           else:
+               dname = os.path.join(out, sha256(s.last_line.encode('utf-8')).hexdigest())
+               if not os.path.isdir(dname):
+                   os.mkdir(dname)
+               fname = os.path.join(dname, sha256(s.block.encode('utf-8')).hexdigest() + ".tsan")
+               if not os.path.isfile(fname):
+                   with open(fname, "w", encoding='utf-8') as w:
+                       w.write(s.block)
+               s.reset()
+        else:
+            for m in mutex.finditer(line):
+                k = m.group()
+                if k not in s.mutexes:
+                    s.mutexes[k] = s.m_index
+                    s.m_index += 1
+            for m in thread.finditer(line):
+                k = m.group()
+                if k not in s.threads:
+                    s.threads[k] = s.t_index
+                    s.t_index += 1
+            for m in pointer.finditer(line):
+                k = m.group()
+                if k not in s.pointers:
+                    s.pointers[k] = s.p_index
+                    s.p_index += 1
+            for k, v in s.mutexes.items():
+                r = re.compile(k)
+                line = r.sub("M%s" % v, line)
+            for k, v in s.threads.items():
+                r = re.compile(k)
+                line = r.sub("T%s" % v, line)
+            for k, v in s.pointers.items():
+                r = re.compile(k)
+                line = r.sub("0x%s" % str(v).zfill(12), line)
+
+            line = stack.sub("", line)
+            line = pid.sub("", line)
+            line = tid.sub("", line)
+            line = worker.sub("", line)
+            line = path.sub("", line)
+
+            s.block += line
+            s.last_line = line