From 8302db407c836109c0d313ae82060c42aeef0230 Mon Sep 17 00:00:00 2001 From: Tom Krizek Date: Thu, 4 Jan 2024 17:27:32 +0100 Subject: [PATCH] Move test algorithm configuration to isctest Instead of invoking get_algorithms.py script repeatedly (which may yield different results), move the algorithm configuration to an isctest module. This ensures the variables are consistent across the entire test run. --- bin/tests/system/conf.sh | 36 ------ bin/tests/system/conftest.py | 11 ++ bin/tests/system/isctest/vars/__init__.py | 2 + .../vars/algorithms.py} | 110 ++++++++++-------- bin/tests/system/isctest/vars/all.py | 3 +- bin/tests/system/mkeys/setup.sh | 9 +- bin/tests/system/mkeys/tests.sh | 1 - bin/tests/system/mkeys/tests_sh_mkeys.py | 4 + bin/tests/system/pytest.ini | 1 + 9 files changed, 82 insertions(+), 95 deletions(-) rename bin/tests/system/{get_algorithms.py => isctest/vars/algorithms.py} (69%) mode change 100755 => 100644 diff --git a/bin/tests/system/conf.sh b/bin/tests/system/conf.sh index e3e9a08fb9..320cf64795 100644 --- a/bin/tests/system/conf.sh +++ b/bin/tests/system/conf.sh @@ -99,42 +99,6 @@ send() { $PERL "$TOP_SRCDIR/bin/tests/system/send.pl" "$@" } -# -# Useful variables in test scripts -# - -# The following script sets the following algorithm-related variables. These -# are selected randomly at runtime from a list of supported algorithms. The -# randomization is deterministic and remains stable for a period of time for a -# given platform. -# -# Default algorithm for testing. -# DEFAULT_ALGORITHM -# DEFAULT_ALGORITHM_NUMBER -# DEFAULT_BITS -# -# This is an alternative algorithm for test cases that require more than one -# algorithm (for example algorithm rollover). Must be different from -# DEFAULT_ALGORITHM. -# ALTERNATIVE_ALGORITHM -# ALTERNATIVE_ALGORITHM_NUMBER -# ALTERNATIVE_BITS -# -# This is an algorithm that is used for tests against the "disable-algorithms" -# configuration option. Must be different from above algorithms. -# DISABLED_ALGORITHM -# DISABLED_ALGORITHM_NUMBER -# DISABLED_BITS -# -# There are multiple algoritms sets to choose from (see get_algorithms.py). To -# override the default choice, set the ALGORITHM_SET env var (see mkeys system -# test for example). -eval "$($PYTHON "$TOP_SRCDIR/bin/tests/system/get_algorithms.py")" - -# Default HMAC algorithm. -# also update _common/rndc.conf and _common/rndc.key when updating DEFAULT_HMAC -export DEFAULT_HMAC=hmac-sha256 - # # Useful functions in test scripts # diff --git a/bin/tests/system/conftest.py b/bin/tests/system/conftest.py index 5585d1e402..c578e4a8fd 100644 --- a/bin/tests/system/conftest.py +++ b/bin/tests/system/conftest.py @@ -277,6 +277,17 @@ def wait_for_zones_loaded(request, servers): watcher.wait_for_line("all zones loaded") +@pytest.fixture(scope="module", autouse=True) +def configure_algorithm_set(request): + """Configure the algorithm set to use in tests.""" + mark = _get_marker(request.node, "algorithm_set") + if not mark: + name = None + else: + name = mark.args[0] + isctest.vars.set_algorithm_set(name) + + @pytest.fixture(autouse=True) def logger(request, system_test_name): """Sets up logging facility specific to a particular test.""" diff --git a/bin/tests/system/isctest/vars/__init__.py b/bin/tests/system/isctest/vars/__init__.py index 7c06a247a2..6ed7020344 100644 --- a/bin/tests/system/isctest/vars/__init__.py +++ b/bin/tests/system/isctest/vars/__init__.py @@ -12,12 +12,14 @@ import os from .all import ALL +from .algorithms import set_algorithm_set from .openssl import parse_openssl_config from .. import log def init_vars(): """Initializes the environment variables.""" + set_algorithm_set(os.getenv("ALGORITHM_SET")) parse_openssl_config(ALL["OPENSSL_CONF"]) os.environ.update(ALL) diff --git a/bin/tests/system/get_algorithms.py b/bin/tests/system/isctest/vars/algorithms.py old mode 100755 new mode 100644 similarity index 69% rename from bin/tests/system/get_algorithms.py rename to bin/tests/system/isctest/vars/algorithms.py index 663b986333..41888649ec --- a/bin/tests/system/get_algorithms.py +++ b/bin/tests/system/isctest/vars/algorithms.py @@ -1,5 +1,3 @@ -#!/usr/bin/python3 - # Copyright (C) Internet Systems Consortium, Inc. ("ISC") # # SPDX-License-Identifier: MPL-2.0 @@ -11,22 +9,43 @@ # See the COPYRIGHT file distributed with this work for additional # information regarding copyright ownership. -# This script is a 'port' broker. It keeps track of ports given to the -# individual system subtests, so every test is given a unique port range. - -import logging import os from pathlib import Path import platform import random import subprocess import time -from typing import Dict, List, NamedTuple, Union +from typing import Dict, List, NamedTuple, Optional, Union -# Uncomment to enable DEBUG logging -# logging.basicConfig( -# format="get_algorithms.py %(levelname)s %(message)s", level=logging.DEBUG -# ) +from .basic import BASIC_VARS +from .. import log + +# Algorithms are selected randomly at runtime from a list of supported +# algorithms. The randomization is deterministic and remains stable for a +# period of time for a given platform. +ALG_VARS = { + # There are multiple algoritms sets to choose from (see ALGORITHM_SETS). To + # override the default choice, set the ALGORITHM_SET env var prior to + # loading this module or call set_algorithm_set(). + "ALGORITHM_SET": "none", + "DEFAULT_ALGORITHM": "", + "DEFAULT_ALGORITHM_NUMBER": "", + "DEFAULT_BITS": "", + # Alternative algorithm for test cases that require more than one algorithm + # (for example algorithm rollover). Must be different from + # DEFAULT_ALGORITHM. + "ALTERNATIVE_ALGORITHM": "", + "ALTERNATIVE_ALGORITHM_NUMBER": "", + "ALTERNATIVE_BITS": "", + # Algorithm that is used for tests against the "disable-algorithms" + # configuration option. Must be different from above algorithms. + "DISABLED_ALGORITHM": "", + "DISABLED_ALGORITHM_NUMBER": "", + "DISABLED_BITS": "", + # Default HMAC algorithm. Must match the rndc configuration in + # bin/tests/system/_common (rndc.conf, rndc.key) + "DEFAULT_HMAC": "hmac-sha256", +} STABLE_PERIOD = 3600 * 3 """number of secs during which algorithm selection remains stable""" @@ -93,57 +112,46 @@ ALGORITHM_SETS = { # ), } -TESTCRYPTO = Path(__file__).resolve().parent / "testcrypto.sh" - -KEYGEN = os.getenv("KEYGEN", "") -if not KEYGEN: - raise RuntimeError("KEYGEN environment variable has to be set") - -ALGORITHM_SET = os.getenv("ALGORITHM_SET", "stable") -assert ALGORITHM_SET in ALGORITHM_SETS, f'ALGORITHM_SET "{ALGORITHM_SET}" unknown' -logging.debug('choosing from ALGORITHM_SET "%s"', ALGORITHM_SET) +# TODO rewrite testcrypto.sh to python +TESTCRYPTO = Path(__file__).resolve().parent.parent.parent / "testcrypto.sh" -def is_supported(alg: Algorithm) -> bool: +def _is_supported(alg: Algorithm) -> bool: """Test whether a given algorithm is supported on the current platform.""" try: subprocess.run( f"{TESTCRYPTO} -q {alg.name}", shell=True, check=True, - env={ - "KEYGEN": KEYGEN, - "TMPDIR": os.getenv("TMPDIR", "/tmp"), - }, + env=BASIC_VARS, stdout=subprocess.DEVNULL, ) except subprocess.CalledProcessError as exc: - logging.debug(exc) - logging.info("algorithm %s not supported", alg.name) + log.debug(exc) + log.info("algorithm %s not supported", alg.name) return False return True -def filter_supported(algs: AlgorithmSet) -> AlgorithmSet: +def _filter_supported(algs: AlgorithmSet) -> AlgorithmSet: """Select supported algorithms from the set.""" filtered = {} for alg_type in algs._fields: candidates = getattr(algs, alg_type) if isinstance(candidates, Algorithm): candidates = [candidates] - supported = list(filter(is_supported, candidates)) + supported = list(filter(_is_supported, candidates)) if len(supported) == 1: supported = supported.pop() elif not supported: raise RuntimeError( - f'no {alg_type.upper()} algorithm from "{ALGORITHM_SET}" set ' - "supported on this platform" + f"no {alg_type.upper()} algorithm " "supported on this platform" ) filtered[alg_type] = supported return AlgorithmSet(**filtered) -def select_random(algs: AlgorithmSet, stable_period=STABLE_PERIOD) -> AlgorithmSet: +def _select_random(algs: AlgorithmSet, stable_period=STABLE_PERIOD) -> AlgorithmSet: """Select random DEFAULT, ALTERNATIVE and DISABLED algorithms from the set. The algorithm selection is deterministic for a given time period and @@ -200,9 +208,11 @@ def select_random(algs: AlgorithmSet, stable_period=STABLE_PERIOD) -> AlgorithmS return AlgorithmSet(default, alternative, disabled) -def algorithms_env(algs: AlgorithmSet) -> Dict[str, str]: +def _algorithms_env(algs: AlgorithmSet, name: str) -> Dict[str, str]: """Return environment variables with selected algorithms as a dict.""" - algs_env: Dict[str, str] = {} + algs_env = { + "ALGORITHM_SET": name, + } def set_alg_env(alg: Algorithm, prefix): algs_env[f"{prefix}_ALGORITHM"] = alg.name @@ -217,25 +227,23 @@ def algorithms_env(algs: AlgorithmSet) -> Dict[str, str]: set_alg_env(algs.alternative, "ALTERNATIVE") set_alg_env(algs.disabled, "DISABLED") - logging.info("selected algorithms: %s", algs_env) + log.info("selected algorithms: %s", algs_env) return algs_env -def main(): - try: - algs = ALGORITHM_SETS[ALGORITHM_SET] - algs = filter_supported(algs) - algs = select_random(algs) - algs_env = algorithms_env(algs) - except Exception: - # if anything goes wrong, the conf.sh ignores error codes, so make sure - # we set an environment variable to an error value that can be checked - # later by the test runner and/or tests themselves - print("export ALGORITHM_SET=error") - raise - for name, value in algs_env.items(): - print(f"export {name}={value}") +def set_algorithm_set(name: Optional[str]): + if name is None: + name = "stable" + assert name in ALGORITHM_SETS, f'ALGORITHM_SET "{name}" unknown' + if name == ALG_VARS["ALGORITHM_SET"]: + log.debug('algorithm set already configured: "%s"', name) + return + log.debug('choosing from ALGORITHM_SET "%s"', name) + algs = ALGORITHM_SETS[name] + algs = _filter_supported(algs) + algs = _select_random(algs) + algs_env = _algorithms_env(algs, name) -if __name__ == "__main__": - main() + ALG_VARS.update(algs_env) + os.environ.update(algs_env) diff --git a/bin/tests/system/isctest/vars/all.py b/bin/tests/system/isctest/vars/all.py index 3e6d49dc74..3478a84a54 100644 --- a/bin/tests/system/isctest/vars/all.py +++ b/bin/tests/system/isctest/vars/all.py @@ -15,6 +15,7 @@ from collections import ChainMap from .autoconf import AC_VARS # type: ignore # pylint: enable=import-error +from .algorithms import ALG_VARS from .basic import BASIC_VARS from .dirs import DIR_VARS from .openssl import OPENSSL_VARS @@ -52,4 +53,4 @@ class VarLookup(ChainMap): return iter(self.keys()) -ALL = VarLookup(AC_VARS, BASIC_VARS, OPENSSL_VARS, PORT_VARS, DIR_VARS) +ALL = VarLookup(AC_VARS, BASIC_VARS, OPENSSL_VARS, PORT_VARS, DIR_VARS, ALG_VARS) diff --git a/bin/tests/system/mkeys/setup.sh b/bin/tests/system/mkeys/setup.sh index 7251e30f17..4b5e752afd 100644 --- a/bin/tests/system/mkeys/setup.sh +++ b/bin/tests/system/mkeys/setup.sh @@ -11,14 +11,11 @@ # See the COPYRIGHT file distributed with this work for additional # information regarding copyright ownership. +# Explicitly setting ALGORITHM_SET is only needed is the script is executed +# standalone without the pytest runner (e.g. for debugging). export ALGORITHM_SET="ecc_default" -. ../conf.sh -# Ensure the selected algorithm set is okay. -if [ "$ALGORITHM_SET" = "error" ]; then - echofail "Algorithm selection failed." >&2 - exit 1 -fi +. ../conf.sh copy_setports ns1/named1.conf.in ns1/named.conf copy_setports ns2/named.conf.in ns2/named.conf diff --git a/bin/tests/system/mkeys/tests.sh b/bin/tests/system/mkeys/tests.sh index 40248f96d9..b460121b04 100644 --- a/bin/tests/system/mkeys/tests.sh +++ b/bin/tests/system/mkeys/tests.sh @@ -13,7 +13,6 @@ set -e -export ALGORITHM_SET="ecc_default" #shellcheck source=conf.sh . ../conf.sh diff --git a/bin/tests/system/mkeys/tests_sh_mkeys.py b/bin/tests/system/mkeys/tests_sh_mkeys.py index 141ee838aa..48d79acf28 100644 --- a/bin/tests/system/mkeys/tests_sh_mkeys.py +++ b/bin/tests/system/mkeys/tests_sh_mkeys.py @@ -9,6 +9,10 @@ # See the COPYRIGHT file distributed with this work for additional # information regarding copyright ownership. +import pytest + +pytestmark = pytest.mark.algorithm_set("ecc_default") + def test_mkeys(run_tests_sh): run_tests_sh() diff --git a/bin/tests/system/pytest.ini b/bin/tests/system/pytest.ini index ff17998e35..766b3dad2e 100644 --- a/bin/tests/system/pytest.ini +++ b/bin/tests/system/pytest.ini @@ -20,3 +20,4 @@ junit_logging = log junit_log_passing_tests = 0 markers = requires_zones_loaded: ensures the test does not start until the specified named instances load all configured zones + algorithm_set: use to select desired algorithms from isctest/vars/algorithms.py