Replace selected setup.sh system test files

These setup.sh scripts only do templating and copying files. Both of
these can be replaced with either jinja templates, or using plain files.
Since each test invocation creates its own temporary directory, copying
files to ensure a "clean" state is no longer necessary.

In cases where named writes some content to the files, a jinja template
can be used instead of a plain file to avoid an artifact check which
would detect a change to a git-tracked file.

.gitlab-ci.yml

@@ -4,7 +4,7 @@ variables:
   # Locale settings do not affect the build, but might affect tests.
   LC_ALL: C
 
-  CI_REGISTRY_IMAGE: registry.gitlab.isc.org/isc-projects/images/bind9
+  CI_REGISTRY_IMAGE: registry.gitlab.isc.org/isc-projects/images/bind9-staging
   CCACHE_DIR: "/ccache"
 
   GIT_DEPTH: 1
@@ -1628,7 +1628,7 @@ shotgun:tcp:
   <<: *shotgun_job
   variables:
     SHOTGUN_SCENARIO: tcp
-    SHOTGUN_TRAFFIC_MULTIPLIER: 6
+    SHOTGUN_TRAFFIC_MULTIPLIER: 13
   when: delayed
   start_in: 5 minutes
 
@@ -1636,7 +1636,7 @@ shotgun:dot:
   <<: *shotgun_job
   variables:
     SHOTGUN_SCENARIO: dot
-    SHOTGUN_TRAFFIC_MULTIPLIER: 3
+    SHOTGUN_TRAFFIC_MULTIPLIER: 6
   when: delayed
   start_in: 5 minutes
 

.mailmap

@@ -1,4 +1,5 @@
 Alan Clegg <aclegg@isc.org>
+Alessio Podda <alessio@isc.org>
 Aram Sargsyan <aram@isc.org>
 Artem Boldariev <artem@isc.org> <artem@boldariev.com>
 Curtis Blackburn <ckb@isc.org> <ckb@freebsd11.local>

bin/named/control.c

@@ -214,8 +214,7 @@ named_control_docommand(isccc_sexpr_t *message, bool readonly,
 	{
 		result = named_server_dnstap(named_g_server, lex, text);
 	} else if (command_compare(command, NAMED_COMMAND_DUMPDB)) {
-		named_server_dumpdb(named_g_server, lex, text);
-		result = ISC_R_SUCCESS;
+		result = named_server_dumpdb(named_g_server, lex, text);
 	} else if (command_compare(command, NAMED_COMMAND_DUMPSTATS)) {
 		result = named_server_dumpstats(named_g_server);
 	} else if (command_compare(command, NAMED_COMMAND_FETCHLIMIT)) {

bin/named/include/named/globals.h

@@ -111,6 +111,7 @@ EXTERN const char *named_g_chrootdir INIT(NULL);
 EXTERN bool named_g_foreground	     INIT(false);
 EXTERN bool named_g_logstderr	     INIT(false);
 EXTERN bool named_g_nosyslog	     INIT(false);
+EXTERN unsigned int named_g_logflags INIT(0);
 EXTERN const char *named_g_logfile   INIT(NULL);
 
 EXTERN const char *named_g_defaultsessionkeyfile INIT(NAMED_LOCALSTATEDIR

bin/named/log.c

@@ -79,6 +79,16 @@ named_log_setdefaultchannels(isc_logconfig_t *lcfg) {
 		isc_log_createchannel(lcfg, "default_debug", ISC_LOG_TOFILE,
 				      ISC_LOG_DYNAMIC, &destination,
 				      ISC_LOG_PRINTTIME | ISC_LOG_DEBUGONLY);
+	} else if (named_g_logstderr && (named_g_logflags != 0)) {
+		/*
+		 * If the option -g is given, but we also requested iso
+		 * timestamps, we'll still need to override the "default_debug"
+		 * logger with a new one.
+		 */
+		isc_log_createchannel(lcfg, "default_debug", ISC_LOG_TOFILEDESC,
+				      ISC_LOG_DYNAMIC,
+				      ISC_LOGDESTINATION_STDERR,
+				      ISC_LOG_PRINTTIME | named_g_logflags);
 	}
 
 	if (named_g_logfile != NULL) {
@@ -118,6 +128,16 @@ named_log_setsafechannels(isc_logconfig_t *lcfg) {
 		 * discarded a bit faster.
 		 */
 		isc_log_setdebuglevel(0);
+	} else if (named_g_logstderr && (named_g_logflags != 0)) {
+		/*
+		 * If the option -g is given, but we also requested iso
+		 * timestamps, we'll still need to override the "default_debug"
+		 * logger with a new one.
+		 */
+		isc_log_createchannel(lcfg, "default_debug", ISC_LOG_TOFILEDESC,
+				      ISC_LOG_DYNAMIC,
+				      ISC_LOGDESTINATION_STDERR,
+				      ISC_LOG_PRINTTIME | named_g_logflags);
 	} else {
 		isc_log_setdebuglevel(named_g_debuglevel);
 	}

bin/named/logconf.c

@@ -239,6 +239,8 @@ channel_fromconf(const cfg_obj_t *channel, isc_logconfig_t *logconfig) {
 				flags |= ISC_LOG_ISO8601;
 			} else if (strcasecmp(s, "iso8601-utc") == 0) {
 				flags |= ISC_LOG_ISO8601 | ISC_LOG_UTC;
+			} else if (strcasecmp(s, "iso8601-tzinfo") == 0) {
+				flags |= ISC_LOG_ISO8601 | ISC_LOG_TZINFO;
 			}
 		}
 	}

bin/named/main.c

@@ -889,6 +889,8 @@ parse_command_line(int argc, char *argv[]) {
 		case 'g':
 			named_g_foreground = true;
 			named_g_logstderr = true;
+			named_g_logflags = ISC_LOG_PRINTTIME | ISC_LOG_ISO8601 |
+					   ISC_LOG_TZINFO;
 			break;
 		case 'L':
 			named_g_logfile = isc_commandline_argument;

bin/tests/system/README.md

@@ -51,6 +51,7 @@ To run system tests, make sure you have the following dependencies installed:
 - perl
 - dnspython
 - pytest-xdist (for parallel execution)
+- python-jinja2 (for tests which use jinja templates)
 
 Individual system tests might also require additional dependencies. If those
 are missing, the affected tests will be skipped and should produce a message
@@ -154,9 +155,17 @@ system test directories may contain the following standard files:
 - `tests_*.py`: These python files are picked up by pytest as modules. If they
   contain any test functions, they're added to the test suite.
 
-- `setup.sh`: This sets up the preconditions for the tests. Although optional,
-  virtually all tests will require such a file to set up the ports they should
-  use for the test.
+- `*.j2`: These jinja2 templates can be used for configuration files or any
+  other files which require certain variables filled in, e.g. ports from the
+  environment variables. During test setup, the pytest runner will automatically
+  fill those in and strip the filename extension .j2, e.g. `ns1/named.conf.j2`
+  becomes `ns1/named.conf`. When using advanced templating to conditionally
+  include/omit entire sections or when filling in custom variables used for the
+  test, ensure the templates always include the defaults. If you don't need the
+  file to be auto-templated during test setup, use `.j2.manual` instead and then
+  no defaults are needed.
+
+- `setup.sh`: This sets up the preconditions for the tests.
 
 - `tests.sh`: Any shell-based tests are located within this file. Runs the
   actual tests.

bin/tests/system/acl/clean.sh

@@ -1,25 +0,0 @@
-#!/bin/sh
-
-# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
-#
-# SPDX-License-Identifier: MPL-2.0
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0.  If a copy of the MPL was not distributed with this
-# file, you can obtain one at https://mozilla.org/MPL/2.0/.
-#
-# See the COPYRIGHT file distributed with this work for additional
-# information regarding copyright ownership.
-
-#
-# Clean up after zone transfer tests.
-#
-
-rm -f dig.out.*
-rm -f ns2/example.db ns2/tsigzone.db ns2/example.db.jnl
-rm -f */named.conf
-rm -f */named.memstats
-rm -f */named.run
-rm -f ns*/_default.nzf
-rm -f ns*/_default.nzd*
-rm -f ns*/managed-keys.bind* ns*/*.mkeys*

bin/tests/system/acl/setup.sh

@@ -13,7 +13,6 @@
 
 . ../conf.sh
 
-$SHELL clean.sh
 $SHELL ${TOP_SRCDIR}/bin/tests/system/genzone.sh 2 3 >ns2/example.db
 $SHELL ${TOP_SRCDIR}/bin/tests/system/genzone.sh 2 3 >ns2/tsigzone.db
 copy_setports ns2/named1.conf.in ns2/named.conf

bin/tests/system/acl/tests_sh_acl.py

@@ -9,6 +9,18 @@
 # See the COPYRIGHT file distributed with this work for additional
 # information regarding copyright ownership.
 
+import pytest
+
+pytestmark = pytest.mark.extra_artifacts(
+    [
+        "ns*/_default.nzf*",
+        "ns*/_default.nzd*",
+        "ns2/example.db",
+        "ns2/tsigzone.db",
+        "dig.out*",
+    ]
+)
+
 
 def test_acl(run_tests_sh):
     run_tests_sh()

bin/tests/system/additional/clean.sh

@@ -1,22 +0,0 @@
-#!/bin/sh
-
-# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
-#
-# SPDX-License-Identifier: MPL-2.0
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0.  If a copy of the MPL was not distributed with this
-# file, you can obtain one at https://mozilla.org/MPL/2.0/.
-#
-# See the COPYRIGHT file distributed with this work for additional
-# information regarding copyright ownership.
-
-#
-# Clean up after tests.
-#
-
-rm -f dig.out.*
-rm -f */named.memstats
-rm -f */named.conf
-rm -f */named.run
-rm -f ns*/managed-keys.bind*

bin/tests/system/additional/setup.sh

@@ -13,7 +13,6 @@
 
 . ../conf.sh
 
-$SHELL clean.sh
 copy_setports ns1/named1.conf.in ns1/named.conf
 copy_setports ns2/named.conf.in ns2/named.conf
 copy_setports ns3/named.conf.in ns3/named.conf

bin/tests/system/additional/tests_sh_additional.py

@@ -9,6 +9,14 @@
 # See the COPYRIGHT file distributed with this work for additional
 # information regarding copyright ownership.
 
+import pytest
+
+pytestmark = pytest.mark.extra_artifacts(
+    [
+        "dig.out.*",
+    ]
+)
+
 
 def test_additional(run_tests_sh):
     run_tests_sh()

bin/tests/system/addzone/clean.sh

@@ -1,43 +0,0 @@
-#!/bin/sh
-
-# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
-#
-# SPDX-License-Identifier: MPL-2.0
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0.  If a copy of the MPL was not distributed with this
-# file, you can obtain one at https://mozilla.org/MPL/2.0/.
-#
-# See the COPYRIGHT file distributed with this work for additional
-# information regarding copyright ownership.
-
-rm -f ./dig.out.*
-rm -f ./rndc.out*
-rm -f ./showzone.out*
-rm -f ./zonestatus.out*
-rm -f ./*/named.conf
-rm -f ./*/named.memstats
-rm -f ./ns1/*.nzf ./ns1/*.nzf~
-rm -f ./ns1/*.nzd ./ns1/*.nzd-lock
-rm -f ./ns2/*.nzf ./ns2/*.nzf~
-rm -f ./ns2/*.nzd ./ns2/*.nzd-lock
-rm -f ./ns3/*.nzf ./ns3/*.nzf~
-rm -f ./ns3/*.nzd ./ns3/*.nzd-lock
-rm -f ./ns2/core*
-rm -f ./ns2/inline.db.jbk
-rm -f ./ns2/inline.db.signed
-rm -f ./ns2/inlinesec.bk*
-rm -rf ./ns2/new-zones
-rm -f ./ns*/named.run ./ns*/named.run.prev
-rm -f ./ns2/nzf-*
-rm -f ./ns3/named.conf
-rm -f ./ns3/*.nzf ./ns3/*.nzf~
-rm -f ./ns3/*.nzd ns3/*.nzd-lock
-rm -f ./ns3/inlinesec.db
-rm -f ./ns1/redirect.db
-rm -f ./ns2/redirect.db
-rm -f ./ns2/redirect.bk
-rm -f ./ns3/redirect.db
-rm -f ./ns*/managed-keys.bind* ns*/*.mkeys*
-rm -f ./nzd2nzf.out.*
-rm -f ./wait_for_message.*

bin/tests/system/addzone/tests_rndc_deadlock.py

@@ -12,8 +12,21 @@
 import concurrent.futures
 import time
 
+import pytest
+
 import isctest
 
+pytestmark = pytest.mark.extra_artifacts(
+    [
+        "ns*/*.nzf*",
+        "ns*/*.nzd*",
+        "ns1/redirect.db",
+        "ns2/new-zones",
+        "ns2/redirect.db",
+        "ns3/redirect.db",
+    ]
+)
+
 
 def rndc_loop(test_state, domain, ns3):
     """

bin/tests/system/addzone/tests_sh_addzone.py

@@ -9,6 +9,35 @@
 # See the COPYRIGHT file distributed with this work for additional
 # information regarding copyright ownership.
 
+import pytest
+
+pytestmark = pytest.mark.extra_artifacts(
+    [
+        "dig.out.*",
+        "nzd2nzf.out.*",
+        "rndc.out*",
+        "showzone.out.*",
+        "zonestatus.out.*",
+        "ns*/*.nzd*",
+        "ns*/*.nzf*",
+        "ns1/redirect.db",
+        "ns2/*.nzf~",
+        "ns2/K*.key",
+        "ns2/K*.private",
+        "ns2/K*.state",
+        "ns2/external.nzd",
+        "ns2/extra.nzd",
+        "ns2/inline.db.jbk",
+        "ns2/inline.db.signed",
+        "ns2/inline.db.signed.jnl",
+        "ns2/inlinesec.bk.jbk",
+        "ns2/new-zones",
+        "ns2/redirect.bk",
+        "ns2/redirect.db",
+        "ns3/redirect.db",
+    ]
+)
+
 
 def test_addzone(run_tests_sh):
     run_tests_sh()

bin/tests/system/allow-query/clean.sh

@@ -1,23 +0,0 @@
-#!/bin/sh
-
-# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
-#
-# SPDX-License-Identifier: MPL-2.0
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0.  If a copy of the MPL was not distributed with this
-# file, you can obtain one at https://mozilla.org/MPL/2.0/.
-#
-# See the COPYRIGHT file distributed with this work for additional
-# information regarding copyright ownership.
-
-#
-# Clean up after allow query tests.
-#
-
-rm -f dig.out.*
-rm -f ns*/named.conf
-rm -f ns2/controls.conf
-rm -f */named.memstats
-rm -f ns*/named.run ns*/named.run.prev
-rm -f ns*/managed-keys.bind* ns*/*.mkeys*

bin/tests/system/allow-query/tests_sh_allow_query.py

@@ -9,6 +9,15 @@
 # See the COPYRIGHT file distributed with this work for additional
 # information regarding copyright ownership.
 
+import pytest
+
+pytestmark = pytest.mark.extra_artifacts(
+    [
+        "dig.out.*",
+        "ns2/controls.conf",
+    ]
+)
+
 
 def test_allow_query(run_tests_sh):
     run_tests_sh()

bin/tests/system/auth/clean.sh

@@ -1,20 +0,0 @@
-#!/bin/sh
-
-# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
-#
-# SPDX-License-Identifier: MPL-2.0
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0.  If a copy of the MPL was not distributed with this
-# file, you can obtain one at https://mozilla.org/MPL/2.0/.
-#
-# See the COPYRIGHT file distributed with this work for additional
-# information regarding copyright ownership.
-
-rm -f */named.memstats
-rm -f */named.run
-rm -f */named.conf
-rm -f dig.out.test*
-rm -f ns2/example.com.bk
-rm -f ns2/example.net.bk
-rm -f ns*/managed-keys.bind* ns*/*mkeys*

bin/tests/system/auth/setup.sh

@@ -1,17 +0,0 @@
-#!/bin/sh
-
-# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
-#
-# SPDX-License-Identifier: MPL-2.0
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0.  If a copy of the MPL was not distributed with this
-# file, you can obtain one at https://mozilla.org/MPL/2.0/.
-#
-# See the COPYRIGHT file distributed with this work for additional
-# information regarding copyright ownership.
-
-. ../conf.sh
-
-copy_setports ns1/named.conf.in ns1/named.conf
-copy_setports ns2/named.conf.in ns2/named.conf

bin/tests/system/auth/tests_sh_auth.py

@@ -9,6 +9,10 @@
 # See the COPYRIGHT file distributed with this work for additional
 # information regarding copyright ownership.
 
+import pytest
+
+pytestmark = pytest.mark.extra_artifacts(["ns*/example.*.bk", "dig.out.test*"])
+
 
 def test_auth(run_tests_sh):
     run_tests_sh()

bin/tests/system/autosign/clean.sh

@@ -1,76 +0,0 @@
-#!/bin/sh
-
-# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
-#
-# SPDX-License-Identifier: MPL-2.0
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0.  If a copy of the MPL was not distributed with this
-# file, you can obtain one at https://mozilla.org/MPL/2.0/.
-#
-# See the COPYRIGHT file distributed with this work for additional
-# information regarding copyright ownership.
-
-rm -f ./dsset-*
-rm -f */K* */dsset-* */*.signed */tmp* */*.jnl */*.bk
-rm -f */core
-rm -f */example.bk
-rm -f */named.conf
-rm -f */named.memstats
-rm -f */named.run*
-rm -f */trusted.conf */private.conf
-rm -f dig.out.*
-rm -f digcomp.out.test*
-rm -f activate-now-publish-1day.key prepub.key
-rm -f active.key inact.key del.key delzsk.key unpub.key standby.key rev.key
-rm -f delayksk.key delayzsk.key autoksk.key autozsk.key
-rm -f noksk-ksk.key nozsk-ksk.key nozsk-zsk.key inaczsk-zsk.key inaczsk-ksk.key
-rm -f nopriv.key vanishing.key del1.key del2.key
-rm -rf ns*/inactive
-rm -f ns*/managed-keys.bind*
-rm -f ns1/root.db ns1/root.db.1 ns1/root.db.2 ns1/root.db.3
-rm -f ns1/signing.out
-rm -f ns2/bar.db
-rm -f ns2/child.nsec3.example.db
-rm -f ns2/child.optout.example.db
-rm -f ns2/example.db
-rm -f ns2/insecure.secure.example.db
-rm -f ns2/nsec3-with-ent.db
-rm -f ns2/private.secure.example.db
-rm -f ns2/signing.*
-rm -f ns3/*.nzd ns3/*.nzd-lock ns3/*.nzf
-rm -f ns3/*.nzf
-rm -f ns3/*.jbk
-rm -f ns3/autonsec3.example.db
-rm -f ns3/delay.example.db ns3/delay.example.1 ns3/delay.example.2
-rm -f ns3/delzsk.example.db
-rm -f ns3/dname-at-apex-nsec3.example.db
-rm -f ns3/inaczsk2.example.db
-rm -f ns3/jitter.nsec3.example.db
-rm -f ns3/kg.out ns3/s.out ns3/st.out
-rm -f ns3/kskonly.example.db
-rm -f ns3/named.ns3.prev
-rm -f ns3/noksk.example.db
-rm -f ns3/nozsk.example.db ns3/inaczsk.example.db
-rm -f ns3/nsec-only.example.db
-rm -f ns3/nsec3-to-nsec.example.db
-rm -f ns3/nsec3.example.db
-rm -f ns3/nsec3.nsec3.example.db
-rm -f ns3/nsec3.optout.example.db
-rm -f ns3/oldsigs.example.db ns3/oldsigs.example.db.bak
-rm -f ns3/optout.example.db
-rm -f ns3/optout.nsec3.example.db
-rm -f ns3/optout.optout.example.db
-rm -f ns3/prepub.example.db
-rm -f ns3/reconf.example.db
-rm -f ns3/rsasha256.example.db ns3/rsasha512.example.db
-rm -f ns3/secure.example.db
-rm -f ns3/secure.nsec3.example.db
-rm -f ns3/secure.optout.example.db
-rm -f ns3/settime.out.*
-rm -f ns3/sync.example.db
-rm -f ns3/ttl*.db
-rm -f nsupdate.out.test*
-rm -f settime.out.*
-rm -f signing.*
-rm -f sync.key

bin/tests/system/autosign/tests_sh_autosign.py

@@ -9,9 +9,147 @@
 # See the COPYRIGHT file distributed with this work for additional
 # information regarding copyright ownership.
 
+import pytest
 
 import isctest.mark
 
+pytestmark = pytest.mark.extra_artifacts(
+    [
+        "activate-now-publish-1day.key",
+        "active.key",
+        "autoksk.key",
+        "autozsk.key",
+        "del.key",
+        "delayksk.key",
+        "delayzsk.key",
+        "delzsk.key",
+        "dig.out.*",
+        "dsset-.",
+        "dsset-bar.",
+        "dsset-delay.example.",
+        "inact.key",
+        "inaczsk-ksk.key",
+        "inaczsk-zsk.key",
+        "noksk-ksk.key",
+        "nopriv.key",
+        "nozsk-ksk.key",
+        "nozsk-zsk.key",
+        "nsupdate.out.*",
+        "prepub.key",
+        "rev.key",
+        "settime.out.*",
+        "signing.*out*",
+        "standby.key",
+        "sync.key",
+        "unpub.key",
+        "vanishing.key",
+        "*/K*",
+        "*/dsset-*",
+        "*/*.signed",
+        "*/*.jnl",
+        "*/*.bk",
+        "ns*/_default.nzf*",
+        "ns*/_default.nzd*",
+        "ns1/root.db",
+        "ns1/root.db.1",
+        "ns1/root.db.2",
+        "ns1/root.db.3",
+        "ns1/signing.out",
+        "ns1/trusted.conf",
+        "ns2/bar.db",
+        "ns2/child.nsec3.example.db",
+        "ns2/child.optout.example.db",
+        "ns2/dsset-dname-at-apex-nsec3.example.",
+        "ns2/dsset-example.",
+        "ns2/dsset-nsec3-to-nsec.example.",
+        "ns2/dsset-nsec3.example.",
+        "ns2/dsset-oldsigs.example.",
+        "ns2/dsset-optout.example.",
+        "ns2/dsset-private.secure.example.",
+        "ns2/dsset-rsasha256.example.",
+        "ns2/dsset-rsasha512.example.",
+        "ns2/dsset-secure.example.",
+        "ns2/example.db",
+        "ns2/insecure.secure.example.db",
+        "ns2/nsec3-with-ent.db",
+        "ns2/private.conf",
+        "ns2/private.secure.example.db",
+        "ns2/signing.bar.out",
+        "ns2/signing.privsec.out",
+        "ns2/trusted.conf",
+        "ns3/autonsec3.example.db",
+        "ns3/delay.example.1",
+        "ns3/delay.example.2",
+        "ns3/delay.example.db",
+        "ns3/delzsk.example.db",
+        "ns3/dname-at-apex-nsec3.example.db",
+        "ns3/dsset-autonsec3.example.",
+        "ns3/dsset-dname-at-apex-nsec3.example.",
+        "ns3/dsset-inaczsk.example.",
+        "ns3/dsset-inaczsk2.example.",
+        "ns3/dsset-kskonly.example.",
+        "ns3/dsset-noksk.example.",
+        "ns3/dsset-nozsk.example.",
+        "ns3/dsset-nsec-only.example.",
+        "ns3/dsset-nsec3-to-nsec.example.",
+        "ns3/dsset-nsec3-to-nsec3.example.",
+        "ns3/dsset-nsec3.example.",
+        "ns3/dsset-nsec3.nsec3.example.",
+        "ns3/dsset-nsec3.optout.example.",
+        "ns3/dsset-oldsigs.example.",
+        "ns3/dsset-optout.example.",
+        "ns3/dsset-optout.nsec3.example.",
+        "ns3/dsset-optout.optout.example.",
+        "ns3/dsset-prepub.example.",
+        "ns3/dsset-rsasha256.example.",
+        "ns3/dsset-rsasha512.example.",
+        "ns3/dsset-secure.example.",
+        "ns3/dsset-secure.nsec3.example.",
+        "ns3/dsset-secure.optout.example.",
+        "ns3/dsset-sync.example.",
+        "ns3/inactive",
+        "ns3/inaczsk.example.db",
+        "ns3/inaczsk2.example.db",
+        "ns3/jitter.nsec3.example.db",
+        "ns3/kg.out",
+        "ns3/kskonly.example.db",
+        "ns3/kskonly.example.db.jbk",
+        "ns3/noksk.example.db",
+        "ns3/nozsk.example.db",
+        "ns3/nsec-only.example.db",
+        "ns3/nsec3-to-nsec.example.db",
+        "ns3/nsec3-to-nsec3.example.db",
+        "ns3/nsec3.example.db",
+        "ns3/nsec3.nsec3.example.db",
+        "ns3/nsec3.optout.example.db",
+        "ns3/oldsigs.example.db",
+        "ns3/oldsigs.example.db.bak",
+        "ns3/optout.example.db",
+        "ns3/optout.example.db.jbk",
+        "ns3/optout.nsec3.example.db",
+        "ns3/optout.optout.example.db",
+        "ns3/prepub.example.db",
+        "ns3/reconf.example.db",
+        "ns3/reconf.example.db.jbk",
+        "ns3/rsasha256.example.db",
+        "ns3/rsasha512.example.db",
+        "ns3/s.out",
+        "ns3/secure.example.db",
+        "ns3/secure.nsec3.example.db",
+        "ns3/secure.optout.example.db",
+        "ns3/st.out",
+        "ns3/sync.example.db",
+        "ns3/trusted.conf",
+        "ns3/ttl1.example.db",
+        "ns3/ttl2.example.db",
+        "ns3/ttl3.example.db",
+        "ns3/ttl4.example.db",
+        "ns4/private.conf",
+        "ns4/trusted.conf",
+        "ns5/trusted.conf",
+    ]
+)
+
 
 @isctest.mark.flaky(max_runs=2)
 def test_autosign(run_tests_sh):

bin/tests/system/builtin/clean.sh

@@ -1,19 +0,0 @@
-#!/bin/sh
-
-# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
-#
-# SPDX-License-Identifier: MPL-2.0
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0.  If a copy of the MPL was not distributed with this
-# file, you can obtain one at https://mozilla.org/MPL/2.0/.
-#
-# See the COPYRIGHT file distributed with this work for additional
-# information regarding copyright ownership.
-
-rm -f ns?/named.run
-rm -f ns?/named.memstats
-rm -f ns?/named.conf
-rm -f rndc.status.ns*
-rm -f dig.out.ns*
-rm -f ns*/managed-keys.bind*

bin/tests/system/builtin/setup.sh

@@ -1,18 +0,0 @@
-#!/bin/sh
-
-# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
-#
-# SPDX-License-Identifier: MPL-2.0
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0.  If a copy of the MPL was not distributed with this
-# file, you can obtain one at https://mozilla.org/MPL/2.0/.
-#
-# See the COPYRIGHT file distributed with this work for additional
-# information regarding copyright ownership.
-
-. ../conf.sh
-
-copy_setports ns1/named.conf.in ns1/named.conf
-copy_setports ns2/named.conf.in ns2/named.conf
-copy_setports ns3/named.conf.in ns3/named.conf

bin/tests/system/builtin/tests_sh_builtin.py

@@ -9,6 +9,15 @@
 # See the COPYRIGHT file distributed with this work for additional
 # information regarding copyright ownership.
 
+import pytest
+
+pytestmark = pytest.mark.extra_artifacts(
+    [
+        "dig.out.*",
+        "rndc.status.ns*",
+    ]
+)
+
 
 def test_builtin(run_tests_sh):
     run_tests_sh()

bin/tests/system/cacheclean/clean.sh

@@ -1,26 +0,0 @@
-#!/bin/sh
-
-# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
-#
-# SPDX-License-Identifier: MPL-2.0
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0.  If a copy of the MPL was not distributed with this
-# file, you can obtain one at https://mozilla.org/MPL/2.0/.
-#
-# See the COPYRIGHT file distributed with this work for additional
-# information regarding copyright ownership.
-
-#
-# Clean up after cache cleaner tests.
-#
-
-rm -f dig.out.ns2
-rm -f dig.out.expire
-rm -f rndc.out.*
-rm -f sed.out.*
-rm -f */named.memstats
-rm -f */named.run
-rm -f */named.conf
-rm -f ns2/named_dump.db.*
-rm -f ns*/managed-keys.bind*

bin/tests/system/cacheclean/setup.sh

@@ -1,17 +0,0 @@
-#!/bin/sh
-
-# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
-#
-# SPDX-License-Identifier: MPL-2.0
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0.  If a copy of the MPL was not distributed with this
-# file, you can obtain one at https://mozilla.org/MPL/2.0/.
-#
-# See the COPYRIGHT file distributed with this work for additional
-# information regarding copyright ownership.
-
-. ../conf.sh
-
-copy_setports ns1/named.conf.in ns1/named.conf
-copy_setports ns2/named.conf.in ns2/named.conf

bin/tests/system/cacheclean/tests.sh

@@ -101,7 +101,7 @@ digcomp --lc dig.out.ns2 knowngood.dig.out || status=1
 
 n=$((n + 1))
 echo_i "only one tcp socket was used ($n)"
-tcpclients=$(awk '$3 == "client" && $5 ~ /10.53.0.7#[0-9]*:/ {print $5}' ns2/named.run | sort | uniq -c | wc -l)
+tcpclients=$(awk '$2 == "client" && $4 ~ /10.53.0.7#[0-9]*:/ {print $4}' ns2/named.run | sort | uniq -c | wc -l)
 
 test $tcpclients -eq 1 || {
   status=1

bin/tests/system/cacheclean/tests_sh_cacheclean.py

@@ -9,6 +9,17 @@
 # See the COPYRIGHT file distributed with this work for additional
 # information regarding copyright ownership.
 
+import pytest
+
+pytestmark = pytest.mark.extra_artifacts(
+    [
+        "dig.out.*",
+        "rndc.out.*",
+        "sed.out.*",
+        "ns2/named_dump.db.*",
+    ]
+)
+
 
 def test_cacheclean(run_tests_sh):
     run_tests_sh()

bin/tests/system/case/clean.sh

@@ -1,23 +0,0 @@
-#!/bin/sh
-
-# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
-#
-# SPDX-License-Identifier: MPL-2.0
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0.  If a copy of the MPL was not distributed with this
-# file, you can obtain one at https://mozilla.org/MPL/2.0/.
-#
-# See the COPYRIGHT file distributed with this work for additional
-# information regarding copyright ownership.
-
-rm -f dig.ns*.test*
-rm -f ns*/named.conf
-rm -f ns*/named.memstats
-rm -f ns*/named.run
-rm -f ns1/dynamic.db
-rm -f ns1/dynamic.db.jnl
-rm -f ns2/dynamic.bk
-rm -f ns2/dynamic.bk.jnl
-rm -f ns2/example.bk
-rm -f ns*/managed-keys.bind*

bin/tests/system/case/setup.sh

@@ -1,18 +0,0 @@
-#!/bin/sh
-
-# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
-#
-# SPDX-License-Identifier: MPL-2.0
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0.  If a copy of the MPL was not distributed with this
-# file, you can obtain one at https://mozilla.org/MPL/2.0/.
-#
-# See the COPYRIGHT file distributed with this work for additional
-# information regarding copyright ownership.
-
-. ../conf.sh
-
-cp -f ns1/dynamic.db.in ns1/dynamic.db
-copy_setports ns1/named.conf.in ns1/named.conf
-copy_setports ns2/named.conf.in ns2/named.conf

bin/tests/system/case/tests_sh_case.py

@@ -9,6 +9,19 @@
 # See the COPYRIGHT file distributed with this work for additional
 # information regarding copyright ownership.
 
+import pytest
+
+pytestmark = pytest.mark.extra_artifacts(
+    [
+        "dig.ns*.test*",
+        "ns1/dynamic.db",
+        "ns1/dynamic.db.jnl",
+        "ns2/dynamic.bk",
+        "ns2/dynamic.bk.jnl",
+        "ns2/example.bk",
+    ]
+)
+
 
 def test_case(run_tests_sh):
     run_tests_sh()

bin/tests/system/catz/clean.sh

@@ -1,36 +0,0 @@
-#!/bin/sh
-
-# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
-#
-# SPDX-License-Identifier: MPL-2.0
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0.  If a copy of the MPL was not distributed with this
-# file, you can obtain one at https://mozilla.org/MPL/2.0/.
-#
-# See the COPYRIGHT file distributed with this work for additional
-# information regarding copyright ownership.
-
-rm -f dig.out.*
-rm -f ns*/*.jnl
-rm -f ns*/*.mkeys
-rm -f ns*/*.nzd ns*/*.nzd-lock
-rm -f ns*/*.nzf
-rm -f ns*/managed-keys.bind*
-rm -f ns*/named.conf
-rm -f ns*/named.memstats
-rm -f ns*/named.run
-rm -f ns*/named.run.prev
-rm -f ns1/*dom*example.db
-rm -f ns1/tls1.example.db
-rm -f ns2/__catz__*db
-rm -f ns2/catalog-bad*.db
-rm -f ns2/named.conf.tmp
-rm -f ns3/dom2.example.db ns3/dom13.example.db ns3/dom14.example.db ns3/dom17.example.db ns3/dom18.example.db
-rm -f ns4/__catz__*db
-rm -f ns4/catalog-self.example.db
-rm -f ns[123]/catalog[1234].example.db
-rm -f ns[14]/catalog-tls.example.db
-rm -f nsupdate.out.*
-rm -f wait_for_message.*
-rm -rf ns2/zonedir

bin/tests/system/catz/setup.sh

@@ -13,8 +13,6 @@
 
 . ../conf.sh
 
-$SHELL clean.sh
-
 copy_setports ns1/named.conf.in ns1/named.conf
 copy_setports ns2/named1.conf.in ns2/named.conf
 copy_setports ns3/named.conf.in ns3/named.conf

bin/tests/system/catz/tests_sh_catz.py

@@ -9,6 +9,26 @@
 # See the COPYRIGHT file distributed with this work for additional
 # information regarding copyright ownership.
 
+import pytest
+
+pytestmark = pytest.mark.extra_artifacts(
+    [
+        "dig.out.*",
+        "nsupdate.out.*",
+        "wait_for_message.*",
+        "ns*/*.jnl",
+        "ns*/*.nzf*",
+        "ns*/*.nzd*",
+        "ns*/catalog*.example.db",
+        "ns*/*dom*.example.db",
+        "ns1/tls1.example.db",
+        "ns2/__catz__*.db",
+        "ns2/named.conf.tmp",
+        "ns2/zonedir",
+        "ns4/__catz__*.db",
+    ]
+)
+
 
 def test_catz(run_tests_sh):
     run_tests_sh()

bin/tests/system/cds/clean.sh

@@ -1,23 +0,0 @@
-#!/bin/sh -e
-
-# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
-#
-# SPDX-License-Identifier: MPL-2.0
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0.  If a copy of the MPL was not distributed with this
-# file, you can obtain one at https://mozilla.org/MPL/2.0/.
-#
-# See the COPYRIGHT file distributed with this work for additional
-# information regarding copyright ownership.
-
-rm -f CDNSKEY* CDS* DS*
-rm -f K*
-rm -f UP*
-rm -f brk.*
-rm -f db.*
-rm -f dsset-*
-rm -f empty
-rm -f sig.*
-rm -f vars.sh
-rm -f err* out* xerr xout

bin/tests/system/cds/tests_sh_cds.py

@@ -9,6 +9,27 @@
 # See the COPYRIGHT file distributed with this work for additional
 # information regarding copyright ownership.
 
+import pytest
+
+pytestmark = pytest.mark.extra_artifacts(
+    [
+        "CDNSKEY.*",
+        "CDS.*",
+        "DS.*",
+        "K*",
+        "UP.*",
+        "brk.*",
+        "db.*",
+        "empty",
+        "err.*",
+        "out.*",
+        "sig.*",
+        "vars.sh",
+        "xerr",
+        "xout",
+    ]
+)
+
 
 def test_cds(run_tests_sh):
     run_tests_sh()

bin/tests/system/chain/clean.sh

@@ -1,18 +0,0 @@
-#!/bin/sh
-
-# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
-#
-# SPDX-License-Identifier: MPL-2.0
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0.  If a copy of the MPL was not distributed with this
-# file, you can obtain one at https://mozilla.org/MPL/2.0/.
-#
-# See the COPYRIGHT file distributed with this work for additional
-# information regarding copyright ownership.
-
-rm -f dig.out.* named*.pid
-rm -f ns*/named.conf
-rm -f */named.memstats */named.recursing */named.run */ans.run
-rm -f ns2/K* ns2/dsset-* ns2/*.db.signed
-rm -f ns*/managed-keys.bind*

bin/tests/system/chain/tests_sh_chain.py

@@ -9,6 +9,18 @@
 # See the COPYRIGHT file distributed with this work for additional
 # information regarding copyright ownership.
 
+import pytest
+
+pytestmark = pytest.mark.extra_artifacts(
+    [
+        "dig.out.*",
+        "*/ans.run",
+        "ns2/K*",
+        "ns2/dsset-*",
+        "ns2/*.db.signed",
+    ]
+)
+
 
 def test_chain(run_tests_sh):
     run_tests_sh()

bin/tests/system/checkconf/clean.sh

@@ -1,26 +0,0 @@
-#!/bin/sh
-
-# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
-#
-# SPDX-License-Identifier: MPL-2.0
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0.  If a copy of the MPL was not distributed with this
-# file, you can obtain one at https://mozilla.org/MPL/2.0/.
-#
-# See the COPYRIGHT file distributed with this work for additional
-# information regarding copyright ownership.
-
-rm -f bad-kasp-keydir1.conf
-rm -f bad-kasp-keydir2.conf
-rm -f bad-kasp-keydir3.conf
-rm -f bad-kasp-keydir4.conf
-rm -f bad-kasp-keydir5.conf
-rm -f bad-tsig.conf
-rm -f checkconf.out*
-rm -f diff.out*
-rm -f good-kasp.conf.in
-rm -f good-server-christmas-tree.conf
-rm -f good.conf good.conf.raw good.conf.out badzero.conf *.out
-rm -rf keys
-rm -rf test.keydir

bin/tests/system/checkconf/good-printtime.conf

@@ -32,4 +32,8 @@ logging {
 		file "five.out";
 		print-time iso8601-utc;
 	};
+	channel six {
+		file "six.out";
+		print-time iso8601-tzinfo;
+	};
 };

bin/tests/system/checkconf/setup.sh

@@ -1,22 +0,0 @@
-#!/bin/sh
-# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
-#
-# SPDX-License-Identifier: MPL-2.0
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0.  If a copy of the MPL was not distributed with this
-# file, you can obtain one at https://mozilla.org/MPL/2.0/.
-#
-# See the COPYRIGHT file distributed with this work for additional
-# information regarding copyright ownership.
-
-. ../conf.sh
-
-copy_setports bad-kasp-keydir1.conf.in bad-kasp-keydir1.conf
-copy_setports bad-kasp-keydir2.conf.in bad-kasp-keydir2.conf
-copy_setports bad-kasp-keydir3.conf.in bad-kasp-keydir3.conf
-copy_setports bad-kasp-keydir4.conf.in bad-kasp-keydir4.conf
-copy_setports bad-kasp-keydir5.conf.in bad-kasp-keydir5.conf
-copy_setports bad-tsig.conf.in bad-tsig.conf
-copy_setports good.conf.in good.conf
-cp -f good-server-christmas-tree.conf.in good-server-christmas-tree.conf

bin/tests/system/checkconf/tests_sh_checkconf.py

@@ -9,6 +9,28 @@
 # See the COPYRIGHT file distributed with this work for additional
 # information regarding copyright ownership.
 
+import pytest
+
+pytestmark = pytest.mark.extra_artifacts(
+    [
+        "*.out",
+        "bad-kasp-keydir1.conf",
+        "bad-kasp-keydir2.conf",
+        "bad-kasp-keydir3.conf",
+        "bad-kasp-keydir4.conf",
+        "bad-kasp-keydir5.conf",
+        "bad-tsig.conf",
+        "badzero.conf",
+        "checkconf.out*",
+        "diff.out*",
+        "good-kasp.conf.in",
+        "good-server-christmas-tree.conf",
+        "good.conf",
+        "good.conf.raw",
+        "keys",
+    ]
+)
+
 
 def test_checkconf(run_tests_sh):
     run_tests_sh()

bin/tests/system/checkds/clean.sh

@@ -1,28 +0,0 @@
-#!/bin/sh
-
-# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
-#
-# SPDX-License-Identifier: MPL-2.0
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0.  If a copy of the MPL was not distributed with this
-# file, you can obtain one at https://mozilla.org/MPL/2.0/.
-#
-# See the COPYRIGHT file distributed with this work for additional
-# information regarding copyright ownership.
-
-set -e
-
-rm -f dig.out*
-rm -f ns*/named.conf ns*/named.memstats ns*/named.run*
-rm -f ns*/*.jnl ns*/*.jbk
-rm -f ns*/K*.private ns*/K*.key ns*/K*.state
-rm -f ns*/*.keyname
-rm -f ns*/dsset-*
-rm -f ns*/*.db ns*/*.jnl ns*/*.jbk ns*/*.db.signed ns*/*.db.infile
-rm -f ns*/keygen.out.* ns*/settime.out.* ns*/signer.out.*
-rm -f ns*/managed-keys.bind*
-rm -f ns*/trusted.conf
-rm -f ns*/*.mkeys
-rm -f ns*/zones
-rm -f ./*.out

bin/tests/system/checkds/setup.sh

@@ -16,8 +16,6 @@
 
 set -e
 
-$SHELL clean.sh
-
 copy_setports ns1/named.conf.in ns1/named.conf
 copy_setports ns2/named.conf.in ns2/named.conf
 copy_setports ns3/named.conf.in ns3/named.conf

bin/tests/system/checkds/tests_checkds.py

@@ -11,6 +11,7 @@
 # See the COPYRIGHT file distributed with this work for additional
 # information regarding copyright ownership.
 
+
 from typing import NamedTuple, Tuple
 
 import os
@@ -24,15 +25,34 @@ pytest.importorskip("dns", minversion="2.0.0")
 import dns.exception
 import dns.message
 import dns.name
-import dns.query
 import dns.rcode
 import dns.rdataclass
 import dns.rdatatype
 
 
-pytestmark = pytest.mark.skipif(
-    sys.version_info < (3, 7), reason="Python >= 3.7 required [GL #3001]"
-)
+pytestmark = [
+    pytest.mark.skipif(
+        sys.version_info < (3, 7), reason="Python >= 3.7 required [GL #3001]"
+    ),
+    pytest.mark.extra_artifacts(
+        [
+            "*.out",
+            "ns*/*.db",
+            "ns*/*.db.infile",
+            "ns*/*.db.signed",
+            "ns*/*.jnl",
+            "ns*/*.jbk",
+            "ns*/*.keyname",
+            "ns*/dsset-*",
+            "ns*/K*",
+            "ns*/keygen.out*",
+            "ns*/settime.out*",
+            "ns*/signer.out*",
+            "ns*/trusted.conf",
+            "ns*/zones",
+        ]
+    ),
+]
 
 
 def has_signed_apex_nsec(zone, response):
@@ -61,16 +81,9 @@ def has_signed_apex_nsec(zone, response):
 
 
 def do_query(server, qname, qtype, tcp=False):
-    query = dns.message.make_query(qname, qtype, use_edns=True, want_dnssec=True)
-    try:
-        if tcp:
-            response = dns.query.tcp(query, server.ip, timeout=3, port=server.ports.dns)
-        else:
-            response = dns.query.udp(query, server.ip, timeout=3, port=server.ports.dns)
-    except dns.exception.Timeout:
-        print(f"error: query timeout for query {qname} {qtype} to {server.ip}")
-        return None
-
+    msg = dns.message.make_query(qname, qtype, use_edns=True, want_dnssec=True)
+    query_func = isctest.query.tcp if tcp else isctest.query.udp
+    response = query_func(msg, server.ip, expected_rcode=dns.rcode.NOERROR)
     return response
 
 
@@ -97,38 +110,26 @@ def verify_zone(zone, transfer):
 
 
 def read_statefile(server, zone):
-    addr = server.ip
     count = 0
     keyid = 0
     state = {}
 
     response = do_query(server, zone, "DS", tcp=True)
-    if not isinstance(response, dns.message.Message):
-        print(f"error: no response for {zone} DS from {addr}")
-        return {}
+    # fetch key id from response.
+    for rr in response.answer:
+        if rr.match(
+            dns.name.from_text(zone),
+            dns.rdataclass.IN,
+            dns.rdatatype.DS,
+            dns.rdatatype.NONE,
+        ):
+            if count == 0:
+                keyid = list(dict(rr.items).items())[0][0].key_tag
+            count += 1
 
-    if response.rcode() == dns.rcode.NOERROR:
-        # fetch key id from response.
-        for rr in response.answer:
-            if rr.match(
-                dns.name.from_text(zone),
-                dns.rdataclass.IN,
-                dns.rdatatype.DS,
-                dns.rdatatype.NONE,
-            ):
-                if count == 0:
-                    keyid = list(dict(rr.items).items())[0][0].key_tag
-                count += 1
-
-        if count != 1:
-            print(
-                f"error: expected a single DS in response for {zone} from {addr}, got {count}"
-            )
-            return {}
-    else:
-        rcode = dns.rcode.to_text(response.rcode())
-        print(f"error: {rcode} response for {zone} DNSKEY from {addr}")
-        return {}
+    assert (
+        count == 1
+    ), f"expected a single DS in response for {zone} from {server.ip}, got {count}"
 
     filename = f"ns9/K{zone}+013+{keyid:05d}.state"
     print(f"read state file {filename}")
@@ -140,7 +141,6 @@ def read_statefile(server, zone):
                     continue
                 key, val = line.strip().split(":", 1)
                 state[key.strip()] = val.strip()
-
     except FileNotFoundError:
         # file may not be written just yet.
         return {}
@@ -149,40 +149,15 @@ def read_statefile(server, zone):
 
 
 def zone_check(server, zone):
-    addr = server.ip
     fqdn = f"{zone}."
 
-    # wait until zone is fully signed.
-    signed = False
-    for _ in range(10):
-        response = do_query(server, fqdn, "NSEC")
-        if not isinstance(response, dns.message.Message):
-            print(f"error: no response for {fqdn} NSEC from {addr}")
-        elif response.rcode() == dns.rcode.NOERROR:
-            signed = has_signed_apex_nsec(fqdn, response)
-        else:
-            rcode = dns.rcode.to_text(response.rcode())
-            print(f"error: {rcode} response for {fqdn} NSEC from {addr}")
-
-        if signed:
-            break
-
-        time.sleep(1)
-
-    assert signed
+    # check zone is fully signed.
+    response = do_query(server, fqdn, "NSEC")
+    assert has_signed_apex_nsec(fqdn, response)
 
     # check if zone if DNSSEC valid.
-    verified = False
     transfer = do_query(server, fqdn, "AXFR", tcp=True)
-    if not isinstance(transfer, dns.message.Message):
-        print(f"error: no response for {fqdn} AXFR from {addr}")
-    elif transfer.rcode() == dns.rcode.NOERROR:
-        verified = verify_zone(fqdn, transfer)
-    else:
-        rcode = dns.rcode.to_text(transfer.rcode())
-        print(f"error: {rcode} response for {fqdn} AXFR from {addr}")
-
-    assert verified
+    assert verify_zone(fqdn, transfer)
 
 
 def keystate_check(server, zone, key):

bin/tests/system/checknames/clean.sh

@@ -1,26 +0,0 @@
-#!/bin/sh
-
-# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
-#
-# SPDX-License-Identifier: MPL-2.0
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0.  If a copy of the MPL was not distributed with this
-# file, you can obtain one at https://mozilla.org/MPL/2.0/.
-#
-# See the COPYRIGHT file distributed with this work for additional
-# information regarding copyright ownership.
-
-rm -f ns*/named.conf
-rm -f dig.out.ns?.test*
-rm -f nsupdate.out.test*
-rm -f ns1/*.example.db
-rm -f ns1/*.update.db
-rm -f ns1/*.update.db.jnl
-rm -f ns4/*.update.db
-rm -f ns4/*.update.db.jnl
-rm -f ns5/*.update.db
-rm -f ns5/*.update.db.jnl
-rm -f */named.memstats
-rm -f */named.run
-rm -f ns*/managed-keys.bind*

bin/tests/system/checknames/setup.sh

@@ -1,34 +0,0 @@
-#!/bin/sh
-
-# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
-#
-# SPDX-License-Identifier: MPL-2.0
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0.  If a copy of the MPL was not distributed with this
-# file, you can obtain one at https://mozilla.org/MPL/2.0/.
-#
-# See the COPYRIGHT file distributed with this work for additional
-# information regarding copyright ownership.
-
-. ../conf.sh
-
-$SHELL clean.sh
-
-copy_setports ns1/named.conf.in ns1/named.conf
-copy_setports ns2/named.conf.in ns2/named.conf
-copy_setports ns3/named.conf.in ns3/named.conf
-copy_setports ns4/named.conf.in ns4/named.conf
-copy_setports ns5/named.conf.in ns5/named.conf
-
-cp ns1/ignore.example.db.in ns1/ignore.example.db
-cp ns1/warn.example.db.in ns1/warn.example.db
-cp ns1/fail.example.db.in ns1/fail.example.db
-
-cp ns1/ignore.update.db.in ns1/ignore.update.db
-cp ns1/warn.update.db.in ns1/warn.update.db
-cp ns1/fail.update.db.in ns1/fail.update.db
-
-cp ns4/primary-ignore.update.db.in ns4/primary-ignore.update.db
-
-cp ns5/master-ignore.update.db.in ns5/master-ignore.update.db

bin/tests/system/checknames/tests_sh_checknames.py

@@ -9,6 +9,22 @@
 # See the COPYRIGHT file distributed with this work for additional
 # information regarding copyright ownership.
 
+import pytest
+
+pytestmark = pytest.mark.extra_artifacts(
+    [
+        "dig.out.ns*.test*",
+        "nsupdate.out.*",
+        "ns1/*.example.db",
+        "ns1/*.update.db",
+        "ns1/*.update.db.jnl",
+        "ns4/*.update.db",
+        "ns4/*.update.db.jnl",
+        "ns5/*.update.db",
+        "ns5/*.update.db.jnl",
+    ]
+)
+
 
 def test_checknames(run_tests_sh):
     run_tests_sh()

bin/tests/system/checkzone/clean.sh

@@ -1,16 +0,0 @@
-#!/bin/sh
-
-# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
-#
-# SPDX-License-Identifier: MPL-2.0
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0.  If a copy of the MPL was not distributed with this
-# file, you can obtain one at https://mozilla.org/MPL/2.0/.
-#
-# See the COPYRIGHT file distributed with this work for additional
-# information regarding copyright ownership.
-
-rm -f test.* good1.db.raw named-compilezone
-rm -f zones/bad-tsig.db
-rm -f zones/zone1_*.txt

bin/tests/system/checkzone/tests_sh_checkzone.py

@@ -9,6 +9,18 @@
 # See the COPYRIGHT file distributed with this work for additional
 # information regarding copyright ownership.
 
+import pytest
+
+pytestmark = pytest.mark.extra_artifacts(
+    [
+        "good1.db.raw",
+        "named-compilezone",
+        "test.*",
+        "zones/bad-tsig.db",
+        "zones/zone1_*.txt",
+    ]
+)
+
 
 def test_checkzone(run_tests_sh):
     run_tests_sh()

bin/tests/system/cipher-suites/clean.sh

@@ -1,26 +0,0 @@
-#!/bin/sh
-
-# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
-#
-# SPDX-License-Identifier: MPL-2.0
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0.  If a copy of the MPL was not distributed with this
-# file, you can obtain one at https://mozilla.org/MPL/2.0/.
-#
-# See the COPYRIGHT file distributed with this work for additional
-# information regarding copyright ownership.
-
-#
-# Clean up after zone transfer tests.
-#
-
-rm -f ./*/named.conf
-rm -f ./*/named.memstats
-rm -f ./*/named.run
-rm -f ./*/named.run.prev
-rm -f ./dig.out.*
-rm -f ./gnutls-cli.*
-rm -f ./sslyze.log.*
-rm -f ./*/example*.db
-rm -rf ./headers.*

bin/tests/system/cipher-suites/tests_sh_cipher_suites.py

@@ -9,6 +9,15 @@
 # See the COPYRIGHT file distributed with this work for additional
 # information regarding copyright ownership.
 
+import pytest
+
+pytestmark = pytest.mark.extra_artifacts(
+    [
+        "dig.out.*",
+        "ns*/example*.db",
+    ]
+)
+
 
 def test_cipher_suites(run_tests_sh):
     run_tests_sh()

bin/tests/system/conf.sh

@@ -232,8 +232,7 @@ private_type_record() {
 # has been appended since the last time we read it.
 #
 # Calling some of these functions causes temporary *.prev files to be
-# created that need to be cleaned up manually (usually by a given system
-# test's clean.sh script).
+# created.
 #
 # Note that unlike other nextpart*() functions, nextpartread() is not
 # meant to be directly used in system tests; its sole purpose is to

bin/tests/system/conftest.py

@@ -10,6 +10,7 @@
 # information regarding copyright ownership.
 
 from functools import partial
+import filecmp
 import os
 from pathlib import Path
 import re
@@ -296,9 +297,30 @@ def logger(request, system_test_name):
     isctest.log.deinit_test_logger()
 
 
+@pytest.fixture(scope="module")
+def expected_artifacts(request):
+    common_artifacts = [
+        "ns*/named.run",
+        "ns*/named.run.prev",
+        "ns*/named.conf",
+        "ns*/named.memstats",
+        "pytest.log.txt",
+    ]
+
+    try:
+        test_specific_artifacts = request.node.get_closest_marker("extra_artifacts")
+    except AttributeError:
+        return None
+
+    if test_specific_artifacts:
+        return common_artifacts + test_specific_artifacts.args[0]
+
+    return common_artifacts
+
+
 @pytest.fixture(scope="module")
 def system_test_dir(
-    request, system_test_name
+    request, system_test_name, expected_artifacts
 ):  # pylint: disable=too-many-statements,too-many-locals
     """
     Temporary directory for executing the test.
@@ -306,8 +328,6 @@ def system_test_dir(
     This fixture is responsible for creating (and potentially removing) a
     copy of the system test directory which is used as a temporary
     directory for the test execution.
-
-    FUTURE: This removes the need to have clean.sh scripts.
     """
 
     def get_test_result():
@@ -347,6 +367,38 @@ def system_test_dir(
         except FileNotFoundError:
             pass
 
+    def check_artifacts(source_dir, run_dir):
+        def check_artifacts_recursive(dcmp):
+            def artifact_expected(path, expected):
+                for glob in expected:
+                    if path.match(glob):
+                        return True
+                return False
+
+            # test must not remove any Git-tracked file, ignore libtool and gcov artifacts
+            for name in dcmp.left_only:
+                assert name.startswith("lt-") or name.endswith(".gcda")
+            assert not dcmp.diff_files, "test must not modify any Git-tracked file"
+
+            dir_path = Path(dcmp.left).relative_to(source_dir)
+            for name in dcmp.right_only:
+                file = dir_path / Path(name)
+                if not artifact_expected(file, expected_artifacts):
+                    unexpected_files.append(str(file))
+            for subdir in dcmp.subdirs.values():
+                check_artifacts_recursive(subdir)
+
+        if expected_artifacts is None:  # skip the check if artifact list is unavailable
+            return
+
+        unexpected_files = []
+        dcmp = filecmp.dircmp(source_dir, run_dir)
+        check_artifacts_recursive(dcmp)
+
+        assert (
+            not unexpected_files
+        ), f"Unexpected files found in test directory: {unexpected_files}"
+
     # Create a temporary directory with a copy of the original system test dir contents
     system_test_root = Path(os.environ["builddir"])
     testdir = Path(
@@ -376,6 +428,9 @@ def system_test_dir(
 
         result = get_test_result()
 
+        if result == "passed":
+            check_artifacts(system_test_root / system_test_name, testdir)
+
         # Clean temporary dir unless it should be kept
         keep = False
         if request.config.getoption("--noclean"):
@@ -408,6 +463,11 @@ def system_test_dir(
             unlink(symlink_dst)
 
 
+@pytest.fixture(scope="module")
+def templates(system_test_dir: Path):
+    return isctest.template.TemplateEngine(system_test_dir)
+
+
 def _run_script(  # pylint: disable=too-many-arguments
     system_test_dir: Path,
     interpreter: str,
@@ -474,6 +534,7 @@ def run_tests_sh(system_test_dir, shell):
 def system_test(  # pylint: disable=too-many-arguments,too-many-statements
     request,
     system_test_dir,
+    templates,
     shell,
     perl,
 ):
@@ -515,6 +576,7 @@ def system_test(  # pylint: disable=too-many-arguments,too-many-statements
             pytest.skip("Prerequisites missing.")
 
     def setup_test():
+        templates.render_auto()
         try:
             shell(f"{system_test_dir}/setup.sh")
         except FileNotFoundError:

bin/tests/system/cookie/clean.sh

@@ -1,23 +0,0 @@
-#!/bin/sh
-
-# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
-#
-# SPDX-License-Identifier: MPL-2.0
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0.  If a copy of the MPL was not distributed with this
-# file, you can obtain one at https://mozilla.org/MPL/2.0/.
-#
-# See the COPYRIGHT file distributed with this work for additional
-# information regarding copyright ownership.
-
-rm -f ns*/named.conf
-rm -f dig.out.*
-rm -f named.run.*
-rm -f rndc.out.*
-rm -f ns1/named_dump.db*
-rm -f ns*/named.memstats
-rm -f ns*/named.run
-rm -f ns*/managed-keys.bind*
-rm -f ns*/named.run.prev
-rm -f ans*/ans.run ans*/ans.log

bin/tests/system/cookie/setup.sh

@@ -1,23 +0,0 @@
-#!/bin/sh
-
-# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
-#
-# SPDX-License-Identifier: MPL-2.0
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0.  If a copy of the MPL was not distributed with this
-# file, you can obtain one at https://mozilla.org/MPL/2.0/.
-#
-# See the COPYRIGHT file distributed with this work for additional
-# information regarding copyright ownership.
-
-. ../conf.sh
-
-copy_setports ns1/named.conf.in ns1/named.conf
-copy_setports ns2/named.conf.in ns2/named.conf
-copy_setports ns3/named.conf.in ns3/named.conf
-copy_setports ns4/named.conf.in ns4/named.conf
-copy_setports ns5/named.conf.in ns5/named.conf
-copy_setports ns6/named.conf.in ns6/named.conf
-copy_setports ns7/named.conf.in ns7/named.conf
-copy_setports ns8/named.conf.in ns8/named.conf

bin/tests/system/cookie/tests_sh_cookie.py

@@ -9,6 +9,19 @@
 # See the COPYRIGHT file distributed with this work for additional
 # information regarding copyright ownership.
 
+import pytest
+
+pytestmark = pytest.mark.extra_artifacts(
+    [
+        "dig.out.*",
+        "named.run.*",
+        "rndc.out.*",
+        "ans*/ans.run",
+        "ans*/query.log",
+        "ns1/named_dump.db*",
+    ]
+)
+
 
 def test_cookie(run_tests_sh):
     run_tests_sh()

bin/tests/system/cpu/clean.sh

@@ -1,16 +0,0 @@
-#!/bin/sh
-
-# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
-#
-# SPDX-License-Identifier: MPL-2.0
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0.  If a copy of the MPL was not distributed with this
-# file, you can obtain one at https://mozilla.org/MPL/2.0/.
-#
-# See the COPYRIGHT file distributed with this work for additional
-# information regarding copyright ownership.
-
-set -e
-
-rm -f ./named.run.*