From 5813818036ba531e68d021eb6c6ea53ef68f855e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Sur=C3=BD?= Date: Fri, 24 Apr 2020 15:29:54 +0200 Subject: [PATCH] Improve the LOG_DRIVER and LOG_COMPILER for the system tests There are several improvements over the default/previous behaviour of the test log driver and log compiler: * The system-test-driver.sh was dropped (it was used incorrectly) * The run.sh script is now both log compiler and cli script to run individual tests * The custom-test-driver was added as extended version of the automake test-driver with capability to tee the test output to stdout when `--verbose yes` is passed to it (you can use LOG_DRIVER_FLAGS to add the option by default) * Makefile.am has been extended to honor V=1 for the system tests test-driver (e.g. V=1 adds `--verbose yes` to AM_LOG_DRIVER_FLAGS) --- .gitlab-ci.yml | 3 +- bin/tests/system/.gitignore | 2 +- bin/tests/system/Makefile.am | 13 +- bin/tests/system/run.sh.in | 41 ++++-- bin/tests/system/system-test-driver.sh.in | 60 --------- configure.ac | 2 - custom-test-driver | 156 ++++++++++++++++++++++ util/copyrights | 3 +- 8 files changed, 200 insertions(+), 80 deletions(-) delete mode 100644 bin/tests/system/system-test-driver.sh.in create mode 100755 custom-test-driver diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6c74136089..103ca7569e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -255,7 +255,8 @@ stages: - *setup_interfaces - *setup_softhsm script: - - cd bin/tests/system && make -j${TEST_PARALLEL_JOBS:-1} -k check V=1 + - cd bin/tests/system + - make -j${TEST_PARALLEL_JOBS:-1} -k check V=1 after_script: - cat bin/tests/system/test-suite.log diff --git a/bin/tests/system/.gitignore b/bin/tests/system/.gitignore index 998718e758..8205de1dbc 100644 --- a/bin/tests/system/.gitignore +++ b/bin/tests/system/.gitignore @@ -15,7 +15,7 @@ parallel.mk /get_base_port.state /get_base_port.lock /run.sh +/run.log /start.sh /stop.sh -/system-test-driver.sh /ifconfig.sh diff --git a/bin/tests/system/Makefile.am b/bin/tests/system/Makefile.am index cca65fa19e..43e00b9486 100644 --- a/bin/tests/system/Makefile.am +++ b/bin/tests/system/Makefile.am @@ -161,7 +161,18 @@ check: exit 1 endif !HAVE_PERL -LOG_COMPILER = $(builddir)/system-test-driver.sh +LOG_DRIVER_V = $(LOG_DRIVER_V_@AM_V@) +LOG_DRIVER_V_ = $(LOG_DRIVER_V_@AM_DEFAULT_V@) +LOG_DRIVER_V_0 = --verbose no +LOG_DRIVER_V_1 = --verbose yes + +LOG_DRIVER = $(top_srcdir)/custom-test-driver +AM_LOG_DRIVER_FLAGS = $(LOG_DRIVER_V) + +LOG_COMPILER = $(builddir)/run.sh +AM_LOG_FLAGS = -r -p "$$("$(srcdir)/get_base_port.sh")" + +$(TESTS): run.sh clean-local: -rm -f get_base_port.state get_base_port.lock diff --git a/bin/tests/system/run.sh.in b/bin/tests/system/run.sh.in index 1f38dc45a3..f7d9735a73 100644 --- a/bin/tests/system/run.sh.in +++ b/bin/tests/system/run.sh.in @@ -37,24 +37,37 @@ else clean=true fi -while getopts "knp:r-:" flag; do - case "$flag" in - -) case "${OPTARG}" in - keep) stopservers=false ;; - noclean) clean=false ;; - esac - ;; - k) stopservers=false ;; - n) clean=false ;; - p) baseport=$OPTARG ;; - *) echo "invalid option" >&2; exit 1 ;; - esac +do_run=false +log_flags="-r" +while getopts "knp:r-:" OPT; do + log_flags="$log_flags -$OPT$OPTARG" + if [ "$OPT" = "-" ] && [ -n "$OPTARG" ]; then + OPT="${OPTARG%%=*}" + OPTARG="${OPTARG#$OPT}" + OPTARG="${OPTARG#=}" + fi + + case "$OPT" in + k | keep) stopservers=false ;; + n | noclean) clean=false ;; + p | port) baseport=$OPTARG ;; + r | run) do_run=true ;; + s | skip) exit 77 ;; + -) break ;; + *) echo "invalid option" >&2; exit 1 ;; + esac done + shift $((OPTIND-1)) +if ! $do_run; then + env - TESTS="$1" TEST_SUITE_LOG=run.log LOG_DRIVER_FLAGS="--verbose yes --color-tests yes" LOG_FLAGS="$log_flags" make -e check + exit $? +fi + if [ $# -eq 0 ]; then - echofail "Usage: $0 [-k] [-n] [-p ] test-directory [test-options]" >&2; - exit 1 + echofail "Usage: $0 [-k] [-n] [-p ] test-directory [test-options]" >&2; + exit 1 fi systest=$(basename "${1%%/}") diff --git a/bin/tests/system/system-test-driver.sh.in b/bin/tests/system/system-test-driver.sh.in deleted file mode 100644 index edffd7be22..0000000000 --- a/bin/tests/system/system-test-driver.sh.in +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/sh -# shellcheck disable=SC2181 -# shellcheck disable=SC2034 - -builddir=@abs_builddir@ -srcdir=@abs_srcdir@ - -usage() { - echo "$0 --test-name=NAME --log-file=PATH.log --trs-file=PATH.trs --color-tests={yes|no} --expect-failure={yes|no} --enable-hard-errors={yes|no}" -} - -TEST_NAME= -LOG_FILE= -TRS_FILE= -COLOR_TESTS=yes -EXPECT_FAILURE=no -HARD_ERRORS=yes - -while getopts -: OPT; do - if [ "$OPT" = "-" ] && [ -n "$OPTARG" ]; then - OPT="${OPTARG%%=*}" - OPTARG="${OPTARG#$OPT}" - OPTARG="${OPTARG#=}" - fi - case "$OPT" in - test-name) TEST_NAME="$OPTARG" ;; - log-file) LOG_FILE="$OPTARG" ;; - trs-file) TRS_FILE="$OPTARG" ;; - color-tests) COLOR_TESTS="$OPTARG" ;; - expect-failure) EXPECT_FAILURE="$OPTARG" ;; - hard-errors) HARD_ERRORS="$OPTARG" ;; - *) break ;; - esac -done -shift $((OPTIND-1)) - -if [ -z "$1" ]; then - echo "fatal: test name required" - usage - exit 1 -fi - -TEST_PROGRAM="$1" -shift - -if [ -z "$TEST_NAME" ]; then - TEST_NAME="$(basename "$TEST_PROGRAM")" -fi -if [ -z "$LOG_FILE" ]; then - LOG_FILE="$TEST_PROGRAM.log" -fi -if [ -z "$TRS_FILE" ]; then - TRS_FILE="$TEST_PROGRAM.trs" -fi - -echo "Running $TEST_PROGRAM" - -"${builddir}/run.sh" -p "$("${srcdir}/get_base_port.sh")" "$@" "$TEST_PROGRAM" - -exit $? diff --git a/configure.ac b/configure.ac index 36fdcfb3b0..36d038b810 100644 --- a/configure.ac +++ b/configure.ac @@ -1595,8 +1595,6 @@ AC_CONFIG_FILES([bin/tests/system/start.sh], [chmod +x bin/tests/system/start.sh]) AC_CONFIG_FILES([bin/tests/system/stop.sh], [chmod +x bin/tests/system/stop.sh]) -AC_CONFIG_FILES([bin/tests/system/system-test-driver.sh], - [chmod +x bin/tests/system/system-test-driver.sh]) # Misc diff --git a/custom-test-driver b/custom-test-driver new file mode 100755 index 0000000000..e11acd5685 --- /dev/null +++ b/custom-test-driver @@ -0,0 +1,156 @@ +#! /bin/sh +# test-driver - basic testsuite driver script. + +scriptversion=2020-04-24.14; # UTC + +# Copyright (C) 2011-2020 Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# As a special exception to the GNU General Public License, if you +# distribute this file as part of a program that contains a +# configuration script generated by Autoconf, you may include it under +# the same distribution terms that you use for the rest of that program. + +# This file is maintained in Automake, please report +# bugs to or send patches to +# . + +# Make unconditional expansion of undefined variables an error. This +# helps a lot in preventing typo-related bugs. +set -u + +usage_error () +{ + echo "$0: $*" >&2 + print_usage >&2 + exit 2 +} + +print_usage () +{ + cat <&1; echo $? > "$status_file") | tee $log_file +else + "$@" >$log_file 2>&1; echo $? > "$status_file" +fi +read -r estatus < "$status_file" + +if test $enable_hard_errors = no && test $estatus -eq 99; then + tweaked_estatus=1 +else + tweaked_estatus=$estatus +fi + +case $tweaked_estatus:$expect_failure in + 0:yes) col=$red res=XPASS recheck=yes gcopy=yes;; + 0:*) col=$grn res=PASS recheck=no gcopy=no;; + 77:*) col=$blu res=SKIP recheck=no gcopy=yes;; + 99:*) col=$mgn res=ERROR recheck=yes gcopy=yes;; + *:yes) col=$lgn res=XFAIL recheck=no gcopy=yes;; + *:*) col=$red res=FAIL recheck=yes gcopy=yes;; +esac + +# Report the test outcome and exit status in the logs, so that one can +# know whether the test passed or failed simply by looking at the '.log' +# file, without the need of also peaking into the corresponding '.trs' +# file (automake bug#11814). +echo "$res $test_name (exit status: $estatus)" >>$log_file + +# Report outcome to console. +echo "${col}${res}${std}: $test_name" + +# Register the test result, and other relevant metadata. +echo ":test-result: $res" > $trs_file +echo ":global-test-result: $res" >> $trs_file +echo ":recheck: $recheck" >> $trs_file +echo ":copy-in-global-log: $gcopy" >> $trs_file + +# Local Variables: +# mode: shell-script +# sh-indentation: 2 +# eval: (add-hook 'before-save-hook 'time-stamp) +# time-stamp-start: "scriptversion=" +# time-stamp-format: "%:y-%02m-%02d.%02H" +# time-stamp-time-zone: "UTC0" +# time-stamp-end: "; # UTC" +# End: diff --git a/util/copyrights b/util/copyrights index 2aa21cca2b..a16a02e5ec 100644 --- a/util/copyrights +++ b/util/copyrights @@ -888,7 +888,6 @@ ./bin/tests/system/synthfromdnssec/ns1/sign.sh SH 2017,2018,2019,2020 ./bin/tests/system/synthfromdnssec/setup.sh SH 2017,2018,2019,2020 ./bin/tests/system/synthfromdnssec/tests.sh SH 2017,2018,2019,2020 -./bin/tests/system/system-test-driver.sh.in X 2020 ./bin/tests/system/tcp/ans6/ans.py PYTHON 2019,2020 ./bin/tests/system/tcp/clean.sh SH 2014,2016,2018,2019,2020 ./bin/tests/system/tcp/setup.sh SH 2018,2019,2020 @@ -972,6 +971,7 @@ ./bin/tests/system/win32/pipequeries.vcxproj.filters.in X 2016,2018,2019,2020 ./bin/tests/system/win32/pipequeries.vcxproj.in X 2016,2017,2018,2019,2020 ./bin/tests/system/win32/pipequeries.vcxproj.user X 2016,2018,2019,2020 +./bin/tests/system/wire_test.c C 2020 ./bin/tests/system/xfer/ans5/badkeydata X 2011,2018,2019,2020 ./bin/tests/system/xfer/ans5/badmessageid X 2020 ./bin/tests/system/xfer/ans5/goodaxfr X 2011,2018,2019,2020 @@ -1190,6 +1190,7 @@ ./contrib/scripts/named-bootconf.sh SH.PORTION 1999,2000,2001,2004,2006,2007,2012,2014,2016,2018,2019,2020 ./contrib/scripts/nanny.pl PERL 2000,2001,2004,2007,2012,2014,2016,2018,2019,2020 ./contrib/scripts/zone-edit.sh.in SH 2010,2012,2014,2016,2018,2019,2020 +./custom-test-driver X 2020 ./doc/arm/Bv9ARM-book.xml SGML 2000,2001,2002,2003,2004,2005,2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020 ./doc/arm/Bv9ARM.ch01.html X 2000,2001,2003,2004,2005,2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020 ./doc/arm/Bv9ARM.ch02.html X 2000,2001,2003,2004,2005,2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020