improve test display, preserve artifacts

- when tests fail, store artifacts from initial run in failed-tests.tar
- store the original test summary output
- after retry, display both the original and the new test summary
- tweak echo_i and cat_i functions to get output spacing and colors right

bin/tests/system/.gitignore

@@ -1,4 +1,5 @@
 dig.out*
+failed-tests.tar
 rndc.out*
 nsupdate.out*
 named.lock

bin/tests/system/Makefile.in

@@ -108,9 +108,9 @@ parallel.mk:
 # Targets to run the tests.
 
 test: parallel.mk subdirs
-	@$(MAKE) -f parallel.mk check
 	@$(SHELL) ./runsequential.sh -r
-	@$(SHELL) ./testsummary.sh
+	+@$(MAKE) -f parallel.mk check
+	+@$(SHELL) ./testsummary.sh || $(SHELL) ./retry.sh
 
 check: test
 
@@ -123,6 +123,8 @@ check: test
 testclean clean distclean::
 	if test -f ./cleanall.sh; then $(SHELL) ./cleanall.sh; fi
 	rm -f systests.output
+	rm -f summary.prev
+	rm -f failed-tests.tar
 	rm -f random.data
 	rm -f parallel.mk
 

bin/tests/system/conf.sh.common

@@ -91,32 +91,32 @@ fi
 SYSTESTDIR="`basename $PWD`"
 
 echo_i() {
-    echo "$@" | while read __LINE ; do
+    echo "$@" | while IFS= read -r __LINE ; do
-       echoinfo "I:$SYSTESTDIR:$__LINE"
+       echoinfo "I:$SYSTESTDIR${SYSTESTDIR:+:}$__LINE"
     done
 }
 
 echo_ic() {
     echo "$@" | while read __LINE ; do
-       echoinfo "I:$SYSTESTDIR:  $__LINE"
+       echoinfo "I:$SYSTESTDIR${SYSTESTDIR:+:}  $__LINE"
     done
 }
 
 cat_i() {
-    while read __LINE ; do
+    while IFS= read -r __LINE ; do
-       echoinfo "I:$SYSTESTDIR:$__LINE"
+       echoinfo "I:$SYSTESTDIR${SYSTESTDIR:+:}$__LINE"
     done
 }
 
 echo_d() {
     echo "$@" | while read __LINE ; do
-       echoinfo "D:$SYSTESTDIR:$__LINE"
+       echoinfo "D:$SYSTESTDIR${SYSTESTDIR:+:}$__LINE"
     done
 }
 
 cat_d() {
     while read __LINE ; do
-       echoinfo "D:$SYSTESTDIR:$__LINE"
+       echoinfo "D:$SYSTESTDIR${SYSTESTDIR:+:}$__LINE"
     done
 }
 

bin/tests/system/retry.sh

@@ -0,0 +1,68 @@
+#!/bin/sh
+#
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+# Creates the system tests output file from the various test.output files.  It
+# then searches that file and prints the number of tests passed, failed, not
+# run.  It also checks whether the IP addresses 10.53.0.[1-8] were set up and,
+# if not, prints a warning.
+#
+# Usage:
+#    retry.sh
+#
+# Status return:
+# 0 - no tests failed
+# 1 - one or more tests failed
+
+SYSTEMTESTTOP=.
+. $SYSTEMTESTTOP/conf.sh
+
+SYSTESTDIR=""
+
+display () {
+    while IFS= read -r __LINE ; do
+       echoinfo "$__LINE"
+    done
+}
+
+if [ ! -f systests.output ]; then
+    echofail "I:'systests.output' not found."
+    exit 0
+fi
+
+# first, preserve artifacts from the tests that are failing
+fails=$(grep 'R:[a-z0-9_-][a-z0-9_-]*:[A-Z][A-Z]*' systests.output |
+        awk -F: 'START { print ". ./conf.sh" }
+              $1 == "R" && $3 == "FAIL" { printf "%s ", $2 }
+              END { print "" }')
+
+# if there were no failed tests, we're done
+[ -n "$fails" ] || exit 0
+
+tar cf failed-tests.tar $fails
+sh testsummary.sh > summary.prev
+
+tar uf failed-tests.tar systests.output
+echo_i "Test failures detected"
+echo_i "Artifacts from failed tests stored in 'failed-tests.tar'"
+echo_i "Rerunning failed tests:"
+
+grep 'R:[a-z0-9_-][a-z0-9_-]*:[A-Z][A-Z]*' systests.output | \
+    awk -F: 'START { print ". ./conf.sh" }
+        $1 == "R" && $3 == "FAIL" { retests = retests " test-"$2; }
+        END { if (retests) { print "make -f parallel.mk " retests } }' | \
+    $SHELL | display
+
+echo_i "Original test results (after first pass):"
+cat summary.prev | display
+rm -f summary.prev
+
+echo_i "Updated test results (after second pass):"
+sh testsummary.sh

bin/tests/system/testsummary.sh

@@ -27,33 +27,36 @@
 SYSTEMTESTTOP=.
 . $SYSTEMTESTTOP/conf.sh
 
-keepfile=0
+SYSTESTDIR=""
 
-while getopts "n" flag; do
+keepfile=0 passes=0
+while getopts "np" flag; do
     case $flag in
 	n) keepfile=1 ;;
+        p) passes=1 ;;
     esac
 done
 
-if [ `ls */test.output 2> /dev/null | wc -l` -eq 0 ]; then
+if [ `ls */test.output 2> /dev/null | wc -l` -ne 0 ]; then
-    echowarn "I:No 'test.output' files were found."
-    echowarn "I:Printing summary from pre-existing 'systests.output'."
-else
     cat */test.output > systests.output
     if [ $keepfile -eq 0 ]; then
         rm -f */test.output
     fi
 fi
 
-status=0
+echo_i "System test result summary:"
-echoinfo "I:System test result summary:"
+grep 'R:[a-z0-9_-][a-z0-9_-]*:[A-Z][A-Z]*' systests.output | \
-echoinfo "`grep 'R:[a-z0-9_-][a-z0-9_-]*:[A-Z][A-Z]*' systests.output | cut -d':' -f3 | sort | uniq -c | sed -e 's/^/I:/'`"
+    cut -d':' -f3 | sort | uniq -c | cat_i
 
-FAILED_TESTS=`grep 'R:[a-z0-9_-][a-z0-9_-]*:FAIL' systests.output | cut -d':' -f2 | sort | sed -e 's/^/I:      /'`
+if [ "$passes" -eq 1 ]; then
-if [ -n "${FAILED_TESTS}" ]; then
+    echo_i "The following system tests passed:"
-	echoinfo "I:The following system tests failed:"
+    grep 'R:[a-z0-9_-][a-z0-9_-]*:PASS' systests.output | \
-	echoinfo "${FAILED_TESTS}"
+        cut -d':' -f2 | sort | sed 's/^/	/' | cat_i
-	status=1
 fi
 
-exit $status
+grep 'R:[a-z0-9_-][a-z0-9_-]*:FAIL' systests.output > /dev/null || exit 0
+echo_i "The following system tests failed:"
+grep 'R:[a-z0-9_-][a-z0-9_-]*:FAIL' systests.output | \
+    cut -d':' -f2 | sort | sed 's/^/	/' | cat_i
+
+exit 1

util/copyrights

@@ -918,6 +918,7 @@
 ./bin/tests/system/resolver/prereq.sh		SH	2000,2001,2004,2007,2012,2014,2016,2018,2019
 ./bin/tests/system/resolver/setup.sh		SH	2010,2011,2012,2013,2014,2016,2017,2018,2019
 ./bin/tests/system/resolver/tests.sh		SH	2000,2001,2004,2007,2009,2010,2011,2012,2013,2014,2015,2016,2017,2018,2019
+./bin/tests/system/retry.sh			SH	2019
 ./bin/tests/system/rndc/clean.sh		SH	2011,2012,2013,2014,2015,2016,2017,2018,2019
 ./bin/tests/system/rndc/gencheck.c		C	2014,2015,2016,2018,2019
 ./bin/tests/system/rndc/ns6/named.args		X	2016,2018,2019