ComputerSurge/bind9
3
0
Fork 0
Code Pull Requests 1 Activity

Pytest integration for meson build

Browse Source 
Integrate the system test runner to work with meson build system. On
each compilation, create build-dependent variables in
isctest/vars/.build_env. Then, run a custom script (on every single
rebuild) to put TOP_BUILDDIR into the source directory (as well as
ifconfig.sh for developer convenience).

When pytest is executed, it always reads TOP_BUILDDIR variable from the
source directory where it was placed by the previously. Then it knows
which directory to use to read the remaining build variables and
proceeds as usual.

meson test command was considered for running system tests, but
ultimately it seems that running pytest directly (from source dir as
devs are used to) is a better solution. Some of the issues with meson
test include: different test granularity from `pytest`, having to
maintain extra list of tests, incorrect reporting of skipped tests, no
advance usage (e.g. using pytest's options) and maintaining two slightly
different ways of running system tests.
This commit is contained in:
Nicki Křížek
2024-11-29 17:07:24 +01:00
parent dd47a6a0a5
commit 1c72708c30
21 changed files with 198 additions and 227 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.reuse/dep5
View File

@@ -62,7 +62,7 @@ Files: **/*.after*
bin/tests/system/forward/CA/index.txt
bin/tests/system/forward/CA/index.txt.attr
bin/tests/system/forward/CA/serial
bin/tests/system/isctest/vars/.ac_vars/*
bin/tests/system/isctest/vars/.build_vars/*
bin/tests/system/journal/ns1/managed-keys.bind.in
bin/tests/system/journal/ns1/managed-keys.bind.jnl.in
bin/tests/system/journal/ns2/managed-keys.bind.in

4
bin/tests/system/.gitignore vendored
View File

@@ -21,8 +21,8 @@ named.run
/start.sh
/stop.sh
/ifconfig.sh
/isctest/vars/.ac_vars/*
!/isctest/vars/.ac_vars/*.in
/isctest/vars/.build_vars/*
!/isctest/vars/.build_vars/*.in
# Ignore file names with underscore in their name except python or shell files.
# This is done to ignore the temporary directories and symlinks created by the

5
bin/tests/system/conftest.py
View File

@@ -25,7 +25,7 @@ import pytest
pytest.register_assert_rewrite("isctest")
import isctest
from isctest.vars.dirs import SYSTEM_TEST_DIR_GIT_PATH
from isctest.vars.build import SYSTEM_TEST_DIR_GIT_PATH
# Silence warnings caused by passing a pytest fixture to another fixture.
@@ -424,6 +424,9 @@ def system_test_dir(request, system_test_name, expected_artifacts):
isctest.log.init_module_logger(system_test_name, testdir)
# Log which binaries are used for the test(s)
isctest.log.info("testing binaries from: %s", os.environ.get("TOP_BUILDDIR"))
# System tests are meant to be executed from their directory - switch to it.
old_cwd = os.getcwd()
os.chdir(testdir)

1
bin/tests/system/isctest/vars/.build_vars/CURL.in Normal file
View File

@@ -0,0 +1 @@
@CURL@

1
bin/tests/system/isctest/vars/.build_vars/FSTRM_CAPTURE.in Normal file
View File

@@ -0,0 +1 @@
@FSTRM_CAPTURE@

1
bin/tests/system/isctest/vars/.build_vars/NC.in Normal file
View File

@@ -0,0 +1 @@
@NC@

1
bin/tests/system/isctest/vars/.build_vars/PERL.in Normal file
View File

@@ -0,0 +1 @@
@PERL@

1
bin/tests/system/isctest/vars/.build_vars/PYTEST.in Normal file
View File

@@ -0,0 +1 @@
@PYTEST@

1
bin/tests/system/isctest/vars/.build_vars/PYTHON.in Normal file
View File

@@ -0,0 +1 @@
@PYTHON@

1
bin/tests/system/isctest/vars/.build_vars/SHELL.in Normal file
View File

@@ -0,0 +1 @@
@SHELL@

1
bin/tests/system/isctest/vars/.build_vars/TOP_BUILDDIR.in Normal file
View File

@@ -0,0 +1 @@
@TOP_BUILDDIR@

1
bin/tests/system/isctest/vars/.build_vars/TOP_SRCDIR.in Normal file
View File

@@ -0,0 +1 @@
@TOP_SRCDIR@

1
bin/tests/system/isctest/vars/.build_vars/XSLTPROC.in Normal file
View File

@@ -0,0 +1 @@
@XSLTPROC@

55
bin/tests/system/isctest/vars/.build_vars/meson.build Normal file
View File

@@ -0,0 +1,55 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
configure_file(
configuration: {'TOP_BUILDDIR': meson.project_build_root()},
input: 'TOP_BUILDDIR.in',
output: 'TOP_BUILDDIR',
)
configure_file(
configuration: {'TOP_SRCDIR': meson.project_source_root()},
input: 'TOP_SRCDIR.in',
output: 'TOP_SRCDIR',
)
foreach name, prog : {
'CURL': curl,
'FSTRM_CAPTURE': fstrm_capture,
'NC': nc,
'PERL': perl,
'PYTEST': pytest,
'PYTHON': python,
'SHELL': sh,
'XSLTPROC': xsltproc,
}
if prog.found()
configure_file(
configuration: {name: prog.full_path()},
input: fs.replace_suffix(name, '.in'),
output: name,
)
endif
endforeach
# Run a script to copy required files to the source directory in order to make
# pytest work and find the compiled files. This is executed on every incovation
# of the compile command, so the latest build is always used for system tests.
custom_target('system-test-init',
build_always_stale: true,
build_by_default: true,
command: meson.project_source_root() / 'util' / 'meson-system-test-init.sh',
env: {
'MESON_SOURCE_ROOT': meson.project_source_root(),
'MESON_BUILD_ROOT': meson.project_build_root(),
},
output: 'system-test-init', # not an actual file, but it's mandatory...
)

4
bin/tests/system/isctest/vars/all.py
View File

@@ -12,7 +12,7 @@
from collections import ChainMap
# pylint: disable=import-error
from .meson import MESON_VARS # type: ignore
from .build import BUILD_VARS # type: ignore
# pylint: enable=import-error
from .algorithms import ALG_VARS, CRYPTO_SUPPORTED_VARS
@@ -58,7 +58,7 @@ ALL = VarLookup(
BASIC_VARS,
CRYPTO_SUPPORTED_VARS,
DIR_VARS,
MESON_VARS,
BUILD_VARS,
OPENSSL_VARS,
PORT_VARS,
)

70
bin/tests/system/isctest/vars/basic.py
View File

@@ -12,50 +12,50 @@
import os
# pylint: disable=import-error
from .meson import MESON_VARS # type: ignore
from .build import BUILD_VARS # type: ignore
# pylint: enable=import-error
BASIC_VARS = {
"ARPANAME": f"{MESON_VARS['TOP_BUILDDIR']}/arpaname",
"CDS": f"{MESON_VARS['TOP_BUILDDIR']}/dnssec-cds",
"CHECKCONF": f"{MESON_VARS['TOP_BUILDDIR']}/named-checkconf",
"CHECKZONE": f"{MESON_VARS['TOP_BUILDDIR']}/named-checkzone",
"DIG": f"{MESON_VARS['TOP_BUILDDIR']}/dig",
"DNSTAPREAD": f"{MESON_VARS['TOP_BUILDDIR']}/dnstap-read",
"DSFROMKEY": f"{MESON_VARS['TOP_BUILDDIR']}/dnssec-dsfromkey",
"FEATURETEST": f"{MESON_VARS['TOP_BUILDDIR']}/bin/tests/system/feature-test",
"HOST": f"{MESON_VARS['TOP_BUILDDIR']}/host",
"IMPORTKEY": f"{MESON_VARS['TOP_BUILDDIR']}/dnssec-importkey",
"JOURNALPRINT": f"{MESON_VARS['TOP_BUILDDIR']}/named-journalprint",
"KEYFRLAB": f"{MESON_VARS['TOP_BUILDDIR']}/dnssec-keyfromlabel",
"KEYGEN": f"{MESON_VARS['TOP_BUILDDIR']}/dnssec-keygen",
"KSR": f"{MESON_VARS['TOP_BUILDDIR']}/dnssec-ksr",
"MDIG": f"{MESON_VARS['TOP_BUILDDIR']}/mdig",
"NAMED": f"{MESON_VARS['TOP_BUILDDIR']}/named",
"NSEC3HASH": f"{MESON_VARS['TOP_BUILDDIR']}/nsec3hash",
"NSLOOKUP": f"{MESON_VARS['TOP_BUILDDIR']}/nslookup",
"NSUPDATE": f"{MESON_VARS['TOP_BUILDDIR']}/nsupdate",
"NZD2NZF": f"{MESON_VARS['TOP_BUILDDIR']}/named-nzd2nzf",
"REVOKE": f"{MESON_VARS['TOP_BUILDDIR']}/dnssec-revoke",
"RNDC": f"{MESON_VARS['TOP_BUILDDIR']}/rndc",
"RNDCCONFGEN": f"{MESON_VARS['TOP_BUILDDIR']}/rndc-confgen",
"RRCHECKER": f"{MESON_VARS['TOP_BUILDDIR']}/named-rrchecker",
"SETTIME": f"{MESON_VARS['TOP_BUILDDIR']}/dnssec-settime",
"SIGNER": f"{MESON_VARS['TOP_BUILDDIR']}/dnssec-signzone",
"TSIGKEYGEN": f"{MESON_VARS['TOP_BUILDDIR']}/tsig-keygen",
"VERIFY": f"{MESON_VARS['TOP_BUILDDIR']}/dnssec-verify",
"WIRETEST": f"{MESON_VARS['TOP_BUILDDIR']}/bin/tests/wire_test",
"BIGKEY": f"{MESON_VARS['TOP_BUILDDIR']}/bin/tests/system/rsabigexponent/bigkey",
"GENCHECK": f"{MESON_VARS['TOP_BUILDDIR']}/bin/tests/system/rndc/gencheck",
"MAKEJOURNAL": f"{MESON_VARS['TOP_BUILDDIR']}/bin/tests/system/makejournal",
"PIPEQUERIES": f"{MESON_VARS['TOP_BUILDDIR']}/bin/tests/system/pipelined/pipequeries",
"ARPANAME": f"{BUILD_VARS['TOP_BUILDDIR']}/arpaname",
"CDS": f"{BUILD_VARS['TOP_BUILDDIR']}/dnssec-cds",
"CHECKCONF": f"{BUILD_VARS['TOP_BUILDDIR']}/named-checkconf",
"CHECKZONE": f"{BUILD_VARS['TOP_BUILDDIR']}/named-checkzone",
"DIG": f"{BUILD_VARS['TOP_BUILDDIR']}/dig",
"DNSTAPREAD": f"{BUILD_VARS['TOP_BUILDDIR']}/dnstap-read",
"DSFROMKEY": f"{BUILD_VARS['TOP_BUILDDIR']}/dnssec-dsfromkey",
"FEATURETEST": f"{BUILD_VARS['TOP_BUILDDIR']}/bin/tests/system/feature-test",
"HOST": f"{BUILD_VARS['TOP_BUILDDIR']}/host",
"IMPORTKEY": f"{BUILD_VARS['TOP_BUILDDIR']}/dnssec-importkey",
"JOURNALPRINT": f"{BUILD_VARS['TOP_BUILDDIR']}/named-journalprint",
"KEYFRLAB": f"{BUILD_VARS['TOP_BUILDDIR']}/dnssec-keyfromlabel",
"KEYGEN": f"{BUILD_VARS['TOP_BUILDDIR']}/dnssec-keygen",
"KSR": f"{BUILD_VARS['TOP_BUILDDIR']}/dnssec-ksr",
"MDIG": f"{BUILD_VARS['TOP_BUILDDIR']}/mdig",
"NAMED": f"{BUILD_VARS['TOP_BUILDDIR']}/named",
"NSEC3HASH": f"{BUILD_VARS['TOP_BUILDDIR']}/nsec3hash",
"NSLOOKUP": f"{BUILD_VARS['TOP_BUILDDIR']}/nslookup",
"NSUPDATE": f"{BUILD_VARS['TOP_BUILDDIR']}/nsupdate",
"NZD2NZF": f"{BUILD_VARS['TOP_BUILDDIR']}/named-nzd2nzf",
"REVOKE": f"{BUILD_VARS['TOP_BUILDDIR']}/dnssec-revoke",
"RNDC": f"{BUILD_VARS['TOP_BUILDDIR']}/rndc",
"RNDCCONFGEN": f"{BUILD_VARS['TOP_BUILDDIR']}/rndc-confgen",
"RRCHECKER": f"{BUILD_VARS['TOP_BUILDDIR']}/named-rrchecker",
"SETTIME": f"{BUILD_VARS['TOP_BUILDDIR']}/dnssec-settime",
"SIGNER": f"{BUILD_VARS['TOP_BUILDDIR']}/dnssec-signzone",
"TSIGKEYGEN": f"{BUILD_VARS['TOP_BUILDDIR']}/tsig-keygen",
"VERIFY": f"{BUILD_VARS['TOP_BUILDDIR']}/dnssec-verify",
"WIRETEST": f"{BUILD_VARS['TOP_BUILDDIR']}/bin/tests/wire_test",
"BIGKEY": f"{BUILD_VARS['TOP_BUILDDIR']}/bin/tests/system/rsabigexponent/bigkey",
"GENCHECK": f"{BUILD_VARS['TOP_BUILDDIR']}/bin/tests/system/rndc/gencheck",
"MAKEJOURNAL": f"{BUILD_VARS['TOP_BUILDDIR']}/bin/tests/system/makejournal",
"PIPEQUERIES": f"{BUILD_VARS['TOP_BUILDDIR']}/bin/tests/system/pipelined/pipequeries",
"TMPDIR": os.getenv("TMPDIR", "/tmp"),
"KRB5_CONFIG": "/dev/null", # we don't want a KRB5_CONFIG setting breaking the tests
"KRB5_KTNAME": "dns.keytab", # use local keytab instead of default /etc/krb5.keytab
"DELV": (
f"{MESON_VARS['TOP_BUILDDIR']}/delv"
f"{BUILD_VARS['TOP_BUILDDIR']}/delv"
if not os.getenv("TSAN_OPTIONS", "")
else ":" # workaround for GL#4119
),

63
bin/tests/system/isctest/vars/build.py Normal file
View File

@@ -0,0 +1,63 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
from pathlib import Path
import os
from typing import Dict
SYSTEM_TEST_DIR_GIT_PATH = "bin/tests/system"
def read_var_from_build_file(path) -> str:
if not path.exists():
raise RuntimeError(
f'Uninitialized build variable: "{path}". Did you run `meson compile`?'
)
return path.read_text(encoding="utf-8").strip()
def load_vars_from_build_files() -> Dict[str, str]:
# TOP_BUILDDIR is special, it is always read from the source directory
top_builddir_file = Path(__file__).resolve().parent / ".build_vars" / "TOP_BUILDDIR"
top_builddir = read_var_from_build_file(top_builddir_file)
build_vars = {
"TOP_BUILDDIR": top_builddir,
}
var_targets = [
"SHELL",
"PERL",
"XSLTPROC",
"CURL",
"NC",
"TOP_SRCDIR",
"FSTRM_CAPTURE",
"PYTEST",
"PYTHON",
]
var_dir = (
Path(top_builddir)
/ SYSTEM_TEST_DIR_GIT_PATH
/ "isctest"
/ "vars"
/ ".build_vars"
)
for var in var_targets:
var_file = var_dir / var
var_value = read_var_from_build_file(var_file)
build_vars[var] = var_value
return build_vars
BUILD_VARS = load_vars_from_build_files()

8
bin/tests/system/isctest/vars/dirs.py
View File

@@ -12,16 +12,14 @@
import os
# pylint: disable=import-error
from .meson import MESON_VARS # type: ignore
from .build import BUILD_VARS, SYSTEM_TEST_DIR_GIT_PATH # type: ignore
# pylint: enable=import-error
SYSTEM_TEST_DIR_GIT_PATH = "bin/tests/system"
DIR_VARS = {
"builddir": f"{MESON_VARS['TOP_BUILDDIR']}/{SYSTEM_TEST_DIR_GIT_PATH}",
"srcdir": f"{MESON_VARS['TOP_SRCDIR']}/{SYSTEM_TEST_DIR_GIT_PATH}",
"builddir": f"{BUILD_VARS['TOP_BUILDDIR']}/{SYSTEM_TEST_DIR_GIT_PATH}",
"srcdir": f"{BUILD_VARS['TOP_SRCDIR']}/{SYSTEM_TEST_DIR_GIT_PATH}",
"SYSTESTDIR": None,
}

34
bin/tests/system/isctest/vars/meson.py
View File

@@ -1,34 +0,0 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
import os
from typing import Dict
def load_vars_from_env() -> Dict[str, str]:
var_targets = [
"TOP_BUILDDIR",
"SHELL",
"PERL",
"XSLTPROC",
"CURL",
"NC",
"TOP_SRCDIR",
"FSTRM_CAPTURE",
"PYTEST",
"PYTHON",
]
return {k: v for k in var_targets if (v := os.getenv(k)) is not None}
MESON_VARS = load_vars_from_env()

148
bin/tests/system/meson.build
View File

@@ -63,150 +63,4 @@ if not pytest.found()
subdir_done()
endif
system_test_env = {
'TOP_BUILDDIR': meson.project_build_root(),
'TOP_SRCDIR': meson.project_source_root(),
'SHELL': sh.full_path(),
'PERL': perl.full_path(),
}
foreach env, prog : {
'CURL': curl,
'XSLTPROC': xsltproc,
'NC': nc,
'FSTRM_CAPTURE': fstrm_capture,
'PYTEST': pytest,
'PYTHON': python,
}
if prog.found()
system_test_env += {env: prog.full_path()}
endif
endforeach
foreach system : [
'acl',
'additional',
'addzone',
'allow-query',
'auth',
'autosign',
'builtin',
'cacheclean',
'case',
'catz',
'cds',
'chain',
'checkconf',
'checkds',
'checknames',
'checkzone',
'cipher-suites',
'cookie',
'cpu',
'database',
'digdelv',
'dispatch',
'dlzexternal',
'dns64',
'dnssec',
'dnstap',
'doth',
'dsdigest',
'dyndb',
'ecdsa',
'eddsa',
'ednscompliance',
'emptyzones',
'enginepkcs11',
'fetchlimit',
'filter-aaaa',
'formerr',
'forward',
'geoip2',
'glue',
'hooks',
'host',
'idna',
'include-multiplecfg',
'inline',
'integrity',
'ixfr',
'journal',
'kasp',
'keepalive',
'keyfromlabel',
'ksr',
'legacy',
'limits',
'logfileconfig',
'masterfile',
'masterformat',
'metadata',
'mirror',
'mkeys',
'multisigner',
'names',
'notify',
'nsec3',
'nslookup',
'nsupdate',
'nzd2nzf',
'padding',
'pending',
'pipelined',
'proxy',
'qmin',
'query-source',
'reclimit',
'redirect',
'resolver',
'rndc',
'rootkeysentinel',
'rpz',
'rpzextra',
'rpzrecurse',
'rrchcker',
'rrl',
'rrsetorder',
'rsabigexponent',
'runtime',
'serve-stale',
'sfcache',
'shutdown',
'smartsign',
'sortlist',
'spf',
'staticstub',
'statistics',
'statschannel',
'stress',
'stub',
'synthfromdnssec',
'tcp',
'timeouts',
'tools',
'transport-acl',
'transport-change',
'tsig',
'tsiggss',
'ttl',
'unknown',
'upforwd',
'verify',
'views',
'wildcard',
'xfer',
'xferquota',
'zero',
'zonechecks',
]
test(
system,
pytest,
args: [system],
suite: 'system',
timeout: 360,
env: system_test_env,
workdir: meson.current_source_dir(),
)
endforeach
subdir('isctest' / 'vars' / '.build_vars')

22
util/meson-system-test-init.sh Normal file
View File

@@ -0,0 +1,22 @@
#!/bin/sh
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
set -eou pipefail
if [ -z "$MESON_SOURCE_ROOT" ] || [ -z "$MESON_BUILD_ROOT" ]; then
echo "meson-system-test-init.sh must be run within meson!"
exit 1
fi
cp $MESON_BUILD_ROOT/bin/tests/system/isctest/vars/.build_vars/TOP_BUILDDIR $MESON_SOURCE_ROOT/bin/tests/system/isctest/vars/.build_vars/TOP_BUILDDIR
cp $MESON_BUILD_ROOT/bin/tests/system/ifconfig.sh $MESON_SOURCE_ROOT/bin/tests/system/ifconfig.sh