diff --git a/bin/tests/system/Makefile.in b/bin/tests/system/Makefile.in index dea2f7573d..556bea0498 100644 --- a/bin/tests/system/Makefile.in +++ b/bin/tests/system/Makefile.in @@ -47,32 +47,6 @@ feature-test@EXEEXT@: feature-test.@O@ # Running the scripts below is bypassed when a separate build directory is # used. -# Define the tests that can be run in parallel. This should be identical to -# the definition of PARALLELDIRS in conf.sh. Note: longer-running tests -# such as serve-stale and rpzrecurse are scheduled first to get more -# benefit from parallelism. -PARALLEL = rpzrecurse dnssec \ - acl additional addzone allow-query autosign \ - builtin cacheclean case catz chain \ - checkconf checknames checkzone \ - @CHECKDS@ @COVERAGE@ @KEYMGR@ \ - cookie database digdelv dlv dlz dlzexternal \ - dns64 @DNSTAP@ dscp dsdigest dyndb \ - ednscompliance emptyzones \ - fetchlimit filter-aaaa formerr forward \ - geoip glue idna inline integrity ixfr \ - legacy limits logfileconfig \ - masterfile masterformat metadata mkeys \ - names notify nslookup nsupdate nzd2nzf \ - pending pipelined \ - reclimit redirect resolver rndc rootkeysentinel rpz \ - rrchecker rrl rrsetorder rsabigexponent runtime \ - sfcache smartsign sortlist \ - spf staticstub statistics statschannel stub \ - tcp tsig tsiggss \ - unknown upforwd verify views wildcard \ - xfer xferquota zero zonechecks - # Produce intermediate makefile that assigns unique port numbers to each # parallel test. The start port number of 5,000 is arbitrary - it must just # be greater than the highest privileged port, 1024. @@ -84,26 +58,7 @@ PARALLEL = rpzrecurse dnssec \ .PHONY: parallel.mk parallel.mk: - @PARALLEL_SANITIZED=`echo $(PARALLEL) | sed "s|\([^ ][^ ]*\)|test-\1|g;" | tr _ -` ; \ - echo ".PHONY: $$PARALLEL_SANITIZED" > $@ ; \ - echo "" >> $@ ; \ - echo "check_interfaces:" >> $@ ; \ - echo " @${PERL} testsock.pl > /dev/null 2>&1 || { \\" >> $@ ; \ - echo " echo \"I:NOTE: System tests were skipped because they require that the\"; \\" >> $@ ; \ - echo " echo \"I: IP addresses 10.53.0.1 through 10.53.0.8 be configured\"; \\" >> $@ ; \ - echo " echo \"I: as alias addresses on the loopback interface. Please run\"; \\" >> $@ ; \ - echo " echo \"I: \"bin/tests/system/ifconfig.sh up\" as root to configure them.\"; \\" >> $@ ; \ - echo " exit 1; \\" >> $@ ; \ - echo " }" >> $@ ; \ - echo "" >> $@ ; \ - echo "test check: $$PARALLEL_SANITIZED" >> $@ ; \ - port=$${STARTPORT:-5000} ; \ - for directory in $(PARALLEL) ; do \ - echo "" >> $@ ; \ - echo "test-`echo $$directory | tr _ -`: check_interfaces" >> $@ ; \ - echo " @$(SHELL) ./run.sh -r -p $$port $$directory 2>&1 | tee $$directory/test.output" >> $@ ; \ - port=`expr $$port + 100` ; \ - done + $(SHELL) parallel.sh > parallel.mk # Targets to run the tests. diff --git a/bin/tests/system/conf.sh.in b/bin/tests/system/conf.sh.in index 4909307cda..9ce67442ef 100644 --- a/bin/tests/system/conf.sh.in +++ b/bin/tests/system/conf.sh.in @@ -104,31 +104,32 @@ SEQUENTIALDIRS="ecdsa eddsa gost lwresd @PKCS11_TEST@ tkey" # and 9953). Because separate blocks of ports can be used for teach # test, these tests can be run in parallel. # -# This symbol must be kept in step with the PARALLEL macro in Makefile.in -PARALLELDIRS="acl additional addzone allow-query autosign \ +# Note: some of the longer-running tests are scheduled first, +# in order to get more benefit from parallelism. +PARALLELDIRS="dnssec rpzrecurse \ + acl additional addzone allow-query autosign \ builtin cacheclean case catz chain \ checkconf checknames checkzone \ @CHECKDS@ @COVERAGE@ @KEYMGR@ \ cookie database digdelv dlv dlz dlzexternal \ - dns64 dnssec @DNSTAP@ dscp dsdigest dyndb \ + dns64 @DNSTAP@ dscp dsdigest dyndb \ ednscompliance emptyzones \ fetchlimit filter-aaaa formerr forward \ geoip glue idna inline integrity ixfr \ - legacy limits logfileconfig lwresd \ - masterfile masterformat metadata mkeys \ - names notify nslookup nsupdate nzd2nzf \ - pending pipelined \ - reclimit redirect resolver rndc rootkeysentinel rpz rpzrecurse \ - rrchecker rrl rrsetorder rsabigexponent runtime \ - sfcache smartsign sortlist \ - spf staticstub statistics statschannel stub \ - tcp tools tsig tsiggss \ - unknown upforwd verify views wildcard \ - xfer xferquota zero zonechecks" + legacy limits logfileconfig \ + masterfile masterformat metadata mkeys \ + names notify nslookup nsupdate nzd2nzf \ + pending pipelined \ + reclimit redirect resolver rndc rootkeysentinel rpz \ + rrchecker rrl rrsetorder rsabigexponent runtime \ + sfcache smartsign sortlist \ + spf staticstub statistics statschannel stub \ + tcp tsig tsiggss \ + unknown upforwd verify views wildcard \ + xfer xferquota zero zonechecks" SUBDIRS="$SEQUENTIALDIRS $PARALLELDIRS" - # Things that are different on Windows KILL=kill DIFF=diff @@ -161,9 +162,9 @@ else fi if grep "^#define WANT_IPV6 1" $TOP/config.h > /dev/null 2>&1 ; then - TESTSOCK6="$TESTSOCK6" + TESTSOCK6="$TESTSOCK6" else - TESTSOCK6=false + TESTSOCK6=false fi # Windows process management leave empty @@ -299,13 +300,13 @@ keyfile_to_keys_section() { shift echo "$section_name {" for keyname in $*; do - awk '!/^; /{ - printf "\t\""$1"\" " - printf "'"$key_prefix"'" - printf $4 " " $5 " " $6 " \"" - for (i=7; i<=NF; i++) printf $i - printf "\";\n" - }' $keyname.key + awk '!/^; /{ + printf "\t\""$1"\" " + printf "'"$key_prefix"'" + printf $4 " " $5 " " $6 " \"" + for (i=7; i<=NF; i++) printf $i + printf "\";\n" + }' $keyname.key done echo "};" } @@ -330,7 +331,7 @@ nextpart () { [ -f $1.prev ] || echo "0" > $1.prev prev=`cat $1.prev` awk "NR > $prev "'{ print } - END { print NR > "/dev/stderr" }' $1 2> $1.prev + END { print NR > "/dev/stderr" }' $1 2> $1.prev } # copy_setports - Copy Configuration File and Replace Ports @@ -345,15 +346,15 @@ nextpart () { copy_setports() { sed -e "s/@PORT@/${PORT}/g" \ - -e "s/@EXTRAPORT1@/${EXTRAPORT1}/g" \ - -e "s/@EXTRAPORT2@/${EXTRAPORT2}/g" \ - -e "s/@EXTRAPORT3@/${EXTRAPORT3}/g" \ - -e "s/@EXTRAPORT4@/${EXTRAPORT4}/g" \ - -e "s/@EXTRAPORT5@/${EXTRAPORT5}/g" \ - -e "s/@EXTRAPORT6@/${EXTRAPORT6}/g" \ - -e "s/@EXTRAPORT7@/${EXTRAPORT7}/g" \ - -e "s/@EXTRAPORT8@/${EXTRAPORT8}/g" \ - -e "s/@CONTROLPORT@/${CONTROLPORT}/g" \ + -e "s/@EXTRAPORT1@/${EXTRAPORT1}/g" \ + -e "s/@EXTRAPORT2@/${EXTRAPORT2}/g" \ + -e "s/@EXTRAPORT3@/${EXTRAPORT3}/g" \ + -e "s/@EXTRAPORT4@/${EXTRAPORT4}/g" \ + -e "s/@EXTRAPORT5@/${EXTRAPORT5}/g" \ + -e "s/@EXTRAPORT6@/${EXTRAPORT6}/g" \ + -e "s/@EXTRAPORT7@/${EXTRAPORT7}/g" \ + -e "s/@EXTRAPORT8@/${EXTRAPORT8}/g" \ + -e "s/@CONTROLPORT@/${CONTROLPORT}/g" \ -e "s/@DEFAULT_ALGORITHM@/${DEFAULT_ALGORITHM}/g" \ -e "s/@DEFAULT_ALGORITHM_NUMBER@/${DEFAULT_ALGORITHM_NUMBER}/g" \ -e "s/@DEFAULT_BITS@/${DEFAULT_BITS}/g" \ diff --git a/bin/tests/system/conf.sh.win32 b/bin/tests/system/conf.sh.win32 index 3135e88185..5396401bcd 100644 --- a/bin/tests/system/conf.sh.win32 +++ b/bin/tests/system/conf.sh.win32 @@ -116,31 +116,30 @@ SEQUENTIALDIRS="ecdsa eddsa gost lwresd @PKCS11_TEST@ tkey" # List of tests that use unique ports (other than 5300 and 9953). These # tests can be run in parallel. # -# XXX: Note that tests are not currently run in parallel on Windows, so it -# isn't strictly necessary to maintain these as separate lists in this file, -# but using the same variable names as in conf.sh.in may simplify keeping the -# lists in sync. +# Note: some of the longer-running tests are scheduled first, in +# order to get more benefit from parallelism. # # XXX: removed logfileconfig as it doesn't currently work on windows. -PARALLELDIRS="acl additional addzone allow-query autosign \ +PARALLELDIRS="dnssec rpzrecurse \ + acl additional addzone allow-query autosign \ builtin cacheclean case catz chain \ checkconf checknames checkzone \ @CHECKDS@ @COVERAGE@ @KEYMGR@ \ cookie database digdelv dlv dlz dlzexternal \ - dns64 dnssec @DNSTAP@ dscp dsdigest dyndb \ + dns64 @DNSTAP@ dscp dsdigest dyndb \ ednscompliance emptyzones \ fetchlimit filter-aaaa formerr forward \ geoip glue idna inline integrity ixfr \ legacy limits lwresd masterfile masterformat metadata mkeys \ - names notify nslookup nsupdate nzd2nzf \ - pending pipelined \ - reclimit redirect resolver rndc rootkeysentinel rpz rpzrecurse \ - rrchecker rrl rrsetorder rsabigexponent runtime \ - sfcache smartsign sortlist \ - spf staticstub statistics statschannel stub \ - tcp tools tsig tsiggss \ - unknown upforwd verify views wildcard \ - xfer xferquota zero zonechecks" + names notify nslookup nsupdate nzd2nzf \ + pending pipelined \ + reclimit redirect resolver rndc rootkeysentinel rpz \ + rrchecker rrl rrsetorder rsabigexponent runtime \ + sfcache smartsign sortlist \ + spf staticstub statistics statschannel stub \ + tcp tsig tsiggss \ + unknown upforwd verify views wildcard \ + xfer xferquota zero zonechecks" SUBDIRS="$SEQUENTIALDIRS $PARALLELDIRS" @@ -171,9 +170,9 @@ else fi if grep "^#define WANT_IPV6 1" $TOP/config.h > /dev/null 2>&1 ; then - TESTSOCK6="$TESTSOCK6" + TESTSOCK6="$TESTSOCK6" else - TESTSOCK6=false + TESTSOCK6=false fi # @@ -217,22 +216,22 @@ else fi echofail () { - printf "${COLOR_FAIL}%s${COLOR_NONE}\n" "$*" + printf "${COLOR_FAIL}%s${COLOR_NONE}\n" "$*" } echowarn () { - printf "${COLOR_WARN}%s${COLOR_NONE}\n" "$*" + printf "${COLOR_WARN}%s${COLOR_NONE}\n" "$*" } echopass () { - printf "${COLOR_PASS}%s${COLOR_NONE}\n" "$*" + printf "${COLOR_PASS}%s${COLOR_NONE}\n" "$*" } echoinfo () { - printf "${COLOR_INFO}%s${COLOR_NONE}\n" "$*" + printf "${COLOR_INFO}%s${COLOR_NONE}\n" "$*" } echostart () { - printf "${COLOR_START}%s${COLOR_NONE}\n" "$*" + printf "${COLOR_START}%s${COLOR_NONE}\n" "$*" } echoend () { - printf "${COLOR_END}%s${COLOR_NONE}\n" "$*" + printf "${COLOR_END}%s${COLOR_NONE}\n" "$*" } SYSTESTDIR="`basename $PWD`" @@ -276,13 +275,13 @@ keyfile_to_keys_section() { shift echo "$section_name {" for keyname in $*; do - awk '!/^; /{ - printf "\t\""$1"\" " - printf "'"$key_prefix"'" - printf $4 " " $5 " " $6 " \"" - for (i=7; i<=NF; i++) printf $i - printf "\";\n" - }' $keyname.key + awk '!/^; /{ + printf "\t\""$1"\" " + printf "'"$key_prefix"'" + printf $4 " " $5 " " $6 " \"" + for (i=7; i<=NF; i++) printf $i + printf "\";\n" + }' $keyname.key done echo "};" } @@ -307,7 +306,7 @@ nextpart () { [ -f $1.prev ] || echo "0" > $1.prev prev=`cat $1.prev` awk "NR > $prev "'{ print } - END { print NR > "/dev/stderr" }' $1 2> $1.prev + END { print NR > "/dev/stderr" }' $1 2> $1.prev } # copy_setports - Copy Configuration File and Replace Ports @@ -328,15 +327,15 @@ copy_setports() { # tokens when it processes this file and produces conf.sh. atsign="@" sed -e "s/${atsign}PORT${atsign}/${PORT}/g" \ - -e "s/${atsign}EXTRAPORT1${atsign}/${EXTRAPORT1}/g" \ - -e "s/${atsign}EXTRAPORT2${atsign}/${EXTRAPORT2}/g" \ - -e "s/${atsign}EXTRAPORT3${atsign}/${EXTRAPORT3}/g" \ - -e "s/${atsign}EXTRAPORT4${atsign}/${EXTRAPORT4}/g" \ - -e "s/${atsign}EXTRAPORT5${atsign}/${EXTRAPORT5}/g" \ - -e "s/${atsign}EXTRAPORT6${atsign}/${EXTRAPORT6}/g" \ - -e "s/${atsign}EXTRAPORT7${atsign}/${EXTRAPORT7}/g" \ - -e "s/${atsign}EXTRAPORT8${atsign}/${EXTRAPORT8}/g" \ - -e "s/${atsign}CONTROLPORT${atsign}/${CONTROLPORT}/g" \ + -e "s/${atsign}EXTRAPORT1${atsign}/${EXTRAPORT1}/g" \ + -e "s/${atsign}EXTRAPORT2${atsign}/${EXTRAPORT2}/g" \ + -e "s/${atsign}EXTRAPORT3${atsign}/${EXTRAPORT3}/g" \ + -e "s/${atsign}EXTRAPORT4${atsign}/${EXTRAPORT4}/g" \ + -e "s/${atsign}EXTRAPORT5${atsign}/${EXTRAPORT5}/g" \ + -e "s/${atsign}EXTRAPORT6${atsign}/${EXTRAPORT6}/g" \ + -e "s/${atsign}EXTRAPORT7${atsign}/${EXTRAPORT7}/g" \ + -e "s/${atsign}EXTRAPORT8${atsign}/${EXTRAPORT8}/g" \ + -e "s/${atsign}CONTROLPORT${atsign}/${CONTROLPORT}/g" \ -e "s/${atsign}DEFAULT_ALGORITHM${atsign}/${DEFAULT_ALGORITHM}/g" \ -e "s/${atsign}DEFAULT_ALGORITHM_NUMBER${atsign}/${DEFAULT_ALGORITHM_NUMBER}/g" \ -e "s/${atsign}DEFAULT_BITS${atsign}/${DEFAULT_BITS}/g" \ diff --git a/bin/tests/system/parallel.sh b/bin/tests/system/parallel.sh new file mode 100644 index 0000000000..24b465e105 --- /dev/null +++ b/bin/tests/system/parallel.sh @@ -0,0 +1,34 @@ +#!/bin/sh +# +# Copyright (C) Internet Systems Consortium, Inc. ("ISC") +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# See the COPYRIGHT file distributed with this work for additional +# information regarding copyright ownership. + +. ./conf.sh + +PARALLELS=`echo $PARALLELDIRS | sed "s|\([^ ][^ ]*\)|test-\1|g;" | tr _ -` + +echo ".PHONY: $PARALLELS" +echo +echo "check_interfaces:" +echo " @${PERL} testsock.pl > /dev/null 2>&1 || { \\" +echo " echo \"I:NOTE: System tests were skipped because they require the\"; \\" +echo " echo \"I: test IP addresses 10.53.0.* to be configured as alias\"; \\" +echo " echo \"I: addresses on the loopback interface. Please run\"; \\" +echo " echo \"I: \"bin/tests/system/ifconfig.sh up\" as root to configure them.\"; \\" +echo " exit 1; \\" +echo " }" +echo +echo "test check: $PARALLELS" +port=${STARTPORT:-5000} +for directory in $PARALLELDIRS ; do + echo + echo "test-`echo $directory | tr _ -`: check_interfaces" + echo " @${SHELL} ./run.sh -r -p $port $directory 2>&1 | tee $directory/test.output" + port=`expr $port + 100` +done diff --git a/bin/tests/system/runall.sh b/bin/tests/system/runall.sh index 76f20a4f3a..52721c34c4 100755 --- a/bin/tests/system/runall.sh +++ b/bin/tests/system/runall.sh @@ -68,21 +68,29 @@ export SYSTEMTEST_FORCE_COLOR export SYSTEMTEST_NO_CLEAN status=0 -if [ "$CYGWIN" = "" ]; then - # Running on Unix, use "make" to run tests in parallel. - make -j $numproc check - status=$? + +if [ "$NOPARALLEL" = "" ]; then + if [ "$CYGWIN" = "" ]; then + # Running on Unix, use "make" to run tests in parallel. + make -j $numproc check + status=$? + else + # Running on Windows: Cygwin "make" is available, but isn't being + # used for the build. So we create a special makefile for the purpose + # of parallel execution of system tests, and use that. + $SHELL parallel.sh > parallel.mk + make -f parallel.mk -j $numproc check || status=$? + $SHELL ./runsequential.sh -r || status=$? + $SHELL ./testsummary.sh + fi else - # Running on Windows: no "make" available, so ensure test interfaces are up - # and then run the tests sequentially. (This is simpler than working out - # where "nmake" is likely to be found. Besides, "nmake" does not support - # parallel execution so if "nmake" is used, the tests would be run - # sequentially anyway.) + # the NOPARALLEL environment variable indicates that tests must be + # run sequentially. $PERL testsock.pl || { cat <<-EOF - I:NOTE: System tests were skipped because they require that the - I: IP addresses 10.53.0.1 through 10.53.0.8 be configured - I: as alias addresses on the loopback interface. Please run + I:NOTE: System tests were skipped because they require the + I: test IP addresses 10.53.0.* to be configured as alias + I: addresses on the loopback interface. Please run I: "bin/tests/system/ifconfig.sh up" as root to configure them. EOF exit 1 @@ -93,4 +101,5 @@ else done } 2>&1 | tee "systests.output" fi + exit $status diff --git a/util/copyrights b/util/copyrights index 21db3011fb..3b829d44eb 100644 --- a/util/copyrights +++ b/util/copyrights @@ -1791,6 +1791,7 @@ ./bin/tests/system/org.isc.bind.system SH 2010,2012,2013,2016,2018,2019 ./bin/tests/system/org.isc.bind.system.plist X 2010,2018,2019 ./bin/tests/system/packet.pl PERL 2011,2012,2016,2018,2019 +./bin/tests/system/parallel.sh SH 2019 ./bin/tests/system/pending/clean.sh SH 2009,2012,2014,2016,2018,2019 ./bin/tests/system/pending/ns1/named.conf.in CONF-C 2018,2019 ./bin/tests/system/pending/ns1/root.db.in ZONE 2009,2010,2016,2018,2019