From 61330a7863c0dfe65f3add97bd363ec4998066aa Mon Sep 17 00:00:00 2001 From: Tom Krizek Date: Mon, 19 Dec 2022 17:44:35 +0100 Subject: [PATCH 1/5] testcrypto.sh: run in TMPDIR if possible Avoid creating any temporary files in the current workdir. Additional/changing files in the bin/tests/system directory are problematic for pytest/xdist collection phase, which assumes the list of files doesn't change between the collection phase of the main pytest thread and the subsequent collection phase of the xdist worker threads. Since the testcrypto.sh is also called during pytest initialization through conf.sh.common (to detect feature support), this could occasionally cause a race condition when the list of files would be different for the main pytest thread and the xdist worker. --- bin/tests/system/get_algorithms.py | 5 ++++- bin/tests/system/testcrypto.sh | 12 +++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/bin/tests/system/get_algorithms.py b/bin/tests/system/get_algorithms.py index 5429649758..9cd07d4857 100755 --- a/bin/tests/system/get_algorithms.py +++ b/bin/tests/system/get_algorithms.py @@ -111,7 +111,10 @@ def is_supported(alg: Algorithm) -> bool: f"{TESTCRYPTO} -q {alg.name}", shell=True, check=True, - env={"KEYGEN": KEYGEN}, + env={ + "KEYGEN": KEYGEN, + "TMPDIR": os.getenv("TMPDIR", "/tmp"), + }, stdout=subprocess.DEVNULL, ) except subprocess.CalledProcessError as exc: diff --git a/bin/tests/system/testcrypto.sh b/bin/tests/system/testcrypto.sh index 2be22f3652..ceaaf3726a 100755 --- a/bin/tests/system/testcrypto.sh +++ b/bin/tests/system/testcrypto.sh @@ -14,6 +14,7 @@ prog=$0 args="" quiet=0 +dir="" msg="cryptography" if test -z "$KEYGEN"; then @@ -74,9 +75,18 @@ if test -z "$alg"; then exit 1 fi +if test -n "$TMPDIR"; then + dir=$(mktemp -d "$TMPDIR/XXXXXX") + args="$args -K $dir" +fi + if $KEYGEN $args $alg foo > /dev/null 2>&1 then - rm -f Kfoo* + if test -z "$dir"; then + rm -f Kfoo* + else + rm -rf "$dir" + fi else if test $quiet -eq 0; then echo_i "This test requires support for $msg" >&2 From d9a97200d5e911d06875510ae9207e913ee42480 Mon Sep 17 00:00:00 2001 From: Tom Krizek Date: Tue, 20 Dec 2022 15:09:48 +0100 Subject: [PATCH 2/5] Factor out script to handle system test core dumps Move the core dump detection functionality for system test runs into a separate script. This enables reuse by the pytest runner. The functionality remains the same. --- bin/tests/system/get_core_dumps.sh | 66 ++++++++++++++++++++++++++++++ bin/tests/system/run.sh.in | 48 +--------------------- 2 files changed, 68 insertions(+), 46 deletions(-) create mode 100755 bin/tests/system/get_core_dumps.sh diff --git a/bin/tests/system/get_core_dumps.sh b/bin/tests/system/get_core_dumps.sh new file mode 100755 index 0000000000..5d0255a2fd --- /dev/null +++ b/bin/tests/system/get_core_dumps.sh @@ -0,0 +1,66 @@ +#!/bin/sh + +# Copyright (C) Internet Systems Consortium, Inc. ("ISC") +# +# SPDX-License-Identifier: MPL-2.0 +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, you can obtain one at https://mozilla.org/MPL/2.0/. +# +# See the COPYRIGHT file distributed with this work for additional +# information regarding copyright ownership. + +dir=$(dirname "$0") +. "$dir/conf.sh" + +systest=$1 +status=0 + +export SYSTESTDIR="${TOP_SRCDIR}/bin/tests/system/${systest}" + +get_core_dumps() { + find "$SYSTESTDIR/" \( -name 'core' -or -name 'core.*' -or -name '*.core' \) ! -name '*.gz' ! -name '*.txt' | sort +} + +core_dumps=$(get_core_dumps | tr '\n' ' ') +assertion_failures=$(find "$SYSTESTDIR/" -name named.run -exec grep "assertion failure" {} + | wc -l) +sanitizer_summaries=$(find "$SYSTESTDIR/" -name 'tsan.*' | wc -l) +if [ -n "$core_dumps" ]; then + status=1 + echoinfo "I:$systest:Core dump(s) found: $core_dumps" + get_core_dumps | while read -r coredump; do + echoinfo "D:$systest:backtrace from $coredump:" + echoinfo "D:$systest:--------------------------------------------------------------------------------" + binary=$(gdb --batch --core="$coredump" 2>/dev/null | sed -ne "s|Core was generated by \`\([^' ]*\)[' ].*|\1|p") + if [ ! -f "${binary}" ]; then + binary=$(find "${TOP_BUILDDIR}" -path "*/.libs/${binary}" -type f) + fi + "${TOP_BUILDDIR}/libtool" --mode=execute gdb \ + -batch \ + -ex bt \ + -core="$coredump" \ + -- \ + "$binary" 2>/dev/null | sed -n '/^Core was generated by/,$p' | cat_d + echoinfo "D:$systest:--------------------------------------------------------------------------------" + coredump_backtrace="${coredump}-backtrace.txt" + echoinfo "D:$systest:full backtrace from $coredump saved in $coredump_backtrace" + "${TOP_BUILDDIR}/libtool" --mode=execute gdb \ + -batch \ + -command=run.gdb \ + -core="$coredump" \ + -- \ + "$binary" > "$coredump_backtrace" 2>&1 + echoinfo "D:$systest:core dump $coredump archived as $coredump.gz" + gzip -1 "${coredump}" + done +elif [ "$assertion_failures" -ne 0 ]; then + status=1 + echoinfo "I:$systest:$assertion_failures assertion failure(s) found" + find "$SYSTESTDIR/" -name 'tsan.*' -exec grep "SUMMARY: " {} + | sort -u | cat_d +elif [ "$sanitizer_summaries" -ne 0 ]; then + status=1 + echoinfo "I:$systest:$sanitizer_summaries sanitizer report(s) found" +fi + +exit $status diff --git a/bin/tests/system/run.sh.in b/bin/tests/system/run.sh.in index 212e9ee4f1..d06f587732 100644 --- a/bin/tests/system/run.sh.in +++ b/bin/tests/system/run.sh.in @@ -113,7 +113,7 @@ if [ "${srcdir}" != "${builddir}" ]; then cp -a "${srcdir}/common" "${builddir}" fi # Some tests require additional files to work for out-of-tree test runs. - for file in ckdnsrps.sh conftest.py digcomp.pl ditch.pl fromhex.pl kasp.sh packet.pl pytest_custom_markers.py start.pl stop.pl testcrypto.sh; do + for file in ckdnsrps.sh conftest.py digcomp.pl ditch.pl fromhex.pl get_core_dumps.sh kasp.sh packet.pl pytest_custom_markers.py start.pl stop.pl testcrypto.sh; do if [ ! -r "${file}" ]; then cp -a "${srcdir}/${file}" "${builddir}" fi @@ -265,51 +265,7 @@ else exit $status fi -get_core_dumps() { - find "$systest/" \( -name 'core' -or -name 'core.*' -or -name '*.core' \) ! -name '*.gz' ! -name '*.txt' | sort -} - -core_dumps=$(get_core_dumps | tr '\n' ' ') -assertion_failures=$(find "$systest/" -name named.run -exec grep "assertion failure" {} + | wc -l) -sanitizer_summaries=$(find "$systest/" -name 'tsan.*' | wc -l) -if [ -n "$core_dumps" ]; then - status=1 - echoinfo "I:$systest:Core dump(s) found: $core_dumps" - get_core_dumps | while read -r coredump; do - export SYSTESTDIR="$systest" - echoinfo "D:$systest:backtrace from $coredump:" - echoinfo "D:$systest:--------------------------------------------------------------------------------" - binary=$(gdb --batch --core="$coredump" 2>/dev/null | sed -ne "s|Core was generated by \`\([^' ]*\)[' ].*|\1|p") - if [ ! -f "${binary}" ]; then - binary=$(find "${top_builddir}" -path "*/.libs/${binary}" -type f) - fi - "${top_builddir}/libtool" --mode=execute gdb \ - -batch \ - -ex bt \ - -core="$coredump" \ - -- \ - "$binary" 2>/dev/null | sed -n '/^Core was generated by/,$p' | cat_d - echoinfo "D:$systest:--------------------------------------------------------------------------------" - coredump_backtrace="${coredump}-backtrace.txt" - echoinfo "D:$systest:full backtrace from $coredump saved in $coredump_backtrace" - "${top_builddir}/libtool" --mode=execute gdb \ - -batch \ - -command=run.gdb \ - -core="$coredump" \ - -- \ - "$binary" > "$coredump_backtrace" 2>&1 - echoinfo "D:$systest:core dump $coredump archived as $coredump.gz" - gzip -1 "${coredump}" - done -elif [ "$assertion_failures" -ne 0 ]; then - status=1 - SYSTESTDIR="$systest" - echoinfo "I:$systest:$assertion_failures assertion failure(s) found" - find "$systest/" -name 'tsan.*' -exec grep "SUMMARY: " {} + | sort -u | cat_d -elif [ "$sanitizer_summaries" -ne 0 ]; then - status=1 - echoinfo "I:$systest:$sanitizer_summaries sanitizer report(s) found" -fi +$SHELL get_core_dumps.sh "$systest" || status=1 print_outstanding_files() { if test -d ${srcdir}/../../../.git; then From e24d3b21d0feeb04e92b41422a6b1ac44e938e63 Mon Sep 17 00:00:00 2001 From: Tom Krizek Date: Mon, 2 Jan 2023 17:54:58 +0100 Subject: [PATCH 3/5] Look for ifconfig.sh.in in testsock.pl parent dir Instead of using the current working directory to find the ifconfig.sh script, look for the ifconfig.sh.in template in the directory where the testsock.pl script is located. This enables the testsock.pl script to be called from any working directory. Using the ifconfig.sh.in template is sufficient, since it contains the necessary information to be extracted: the max= value (which is hard-coded in the template). --- bin/tests/system/testsock.pl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bin/tests/system/testsock.pl b/bin/tests/system/testsock.pl index 3677e200b2..e793874d93 100755 --- a/bin/tests/system/testsock.pl +++ b/bin/tests/system/testsock.pl @@ -15,6 +15,8 @@ require 5.001; +use Cwd 'abs_path'; +use File::Basename; use Socket; use Getopt::Long; @@ -27,7 +29,8 @@ my @ids; if ($id != 0) { @ids = ($id); } else { - my $fn = "ifconfig.sh"; + my $dir = dirname(abs_path($0)); + my $fn = "$dir/ifconfig.sh.in"; open FH, "< $fn" or die "open < $fn: $!\n"; while () { @ids = (1..$1) From 9593ff9347b0d1ddc5b679dc5e40711785b1e5e7 Mon Sep 17 00:00:00 2001 From: Tom Krizek Date: Tue, 3 Jan 2023 11:58:43 +0100 Subject: [PATCH 4/5] Tweak dupsigs test output Use a different visual separator, since "====", "----" and "____" is used by pytest to separate the log output. --- bin/tests/system/dupsigs/tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/tests/system/dupsigs/tests.sh b/bin/tests/system/dupsigs/tests.sh index 205a4ed099..85218c68ee 100644 --- a/bin/tests/system/dupsigs/tests.sh +++ b/bin/tests/system/dupsigs/tests.sh @@ -38,7 +38,7 @@ end=$((start + 140)) while [ $now -lt $end ]; do et=$((now - start)) - echo "=============== $et ============" + echo "............... $et ............" $JOURNALPRINT ns1/signing.test.db.signed.jnl | $PERL check_journal.pl $DIG axfr signing.test -p ${PORT} @10.53.0.1 > dig.out.at$et awk '$4 == "RRSIG" { print $11 }' dig.out.at$et | sort | uniq -c From 03d7b45d81268708fd47bebc8d13f8bc5c49f64b Mon Sep 17 00:00:00 2001 From: Tom Krizek Date: Fri, 6 Jan 2023 15:08:27 +0100 Subject: [PATCH 5/5] Update the TEST_PARALLEL_JOBS value in CI The authoritative source for this value is in the project's CI/CD Variables Setting. The reason to keep it in .gitlab-ci.yaml as well is to have functional testing in forks without the need to manually specify this variable in Settings. The tests have been executed with 4 jobs for some time now. This "change" only brings .gitlab-ci.yaml file up to date, it doesn't actually change the number of jobs we currently use to test. --- .gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bd3f0c8b5e..ba74e23ac7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,8 +8,10 @@ variables: CCACHE_DIR: "/ccache" GIT_DEPTH: 1 + + # The following values may be overwritten in GitLab's CI/CD Variables Settings. BUILD_PARALLEL_JOBS: 6 - TEST_PARALLEL_JOBS: 6 + TEST_PARALLEL_JOBS: 4 CONFIGURE: ./configure CLANG_VERSION: 15