From 3c3085be3c02e2ce20372acbae8266ad37531246 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20K=C4=99pie=C5=84?= Date: Fri, 6 Dec 2019 14:11:01 +0100 Subject: [PATCH] Detect missing system test results At the end of each system test suite run, the system test framework collects all existing test.output files from system test subdirectories and produces bin/tests/system/systests.output from those files. However, it does not check whether a test.output file was found for every executed test. Thus, if the test.output file is accidentally deleted by the system test itself (e.g. due to an overly broad file removal wildcard present in clean.sh), its output will not be included in bin/tests/system/systests.output. Since the result of each system test suite run is determined by bin/tests/system/testsummary.sh, which only operates on the contents of bin/tests/system/systests.output, this can lead to test failures being ignored. Fix by ensuring the number of test results found in bin/tests/system/systests.output is equal to the number of tests run and triggering a system test suite failure in case of a discrepancy between these two values. --- bin/tests/system/testsummary.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bin/tests/system/testsummary.sh b/bin/tests/system/testsummary.sh index 06c356671a..1c008cdbad 100644 --- a/bin/tests/system/testsummary.sh +++ b/bin/tests/system/testsummary.sh @@ -56,4 +56,11 @@ if [ -n "${FAILED_TESTS}" ]; then status=1 fi +RESULTS_FOUND=`grep -c 'R:[a-z0-9_-][a-z0-9_-]*:[A-Z][A-Z]*' systests.output` +TESTS_RUN=`echo "${SUBDIRS}" | wc -w` +if [ "${RESULTS_FOUND}" -ne "${TESTS_RUN}" ]; then + echofail "I:Found ${RESULTS_FOUND} test results, but ${TESTS_RUN} tests were run" + status=1 +fi + exit $status