Move test algorithm configuration to isctest

Instead of invoking get_algorithms.py script repeatedly (which may yield
different results), move the algorithm configuration to an isctest
module. This ensures the variables are consistent across the entire test
run.

bin/tests/system/conf.sh

@@ -99,42 +99,6 @@ send() {
   $PERL "$TOP_SRCDIR/bin/tests/system/send.pl" "$@"
 }
 
-#
-# Useful variables in test scripts
-#
-
-# The following script sets the following algorithm-related variables. These
-# are selected randomly at runtime from a list of supported algorithms. The
-# randomization is deterministic and remains stable for a period of time for a
-# given platform.
-#
-# Default algorithm for testing.
-# DEFAULT_ALGORITHM
-# DEFAULT_ALGORITHM_NUMBER
-# DEFAULT_BITS
-#
-# This is an alternative algorithm for test cases that require more than one
-# algorithm (for example algorithm rollover). Must be different from
-# DEFAULT_ALGORITHM.
-# ALTERNATIVE_ALGORITHM
-# ALTERNATIVE_ALGORITHM_NUMBER
-# ALTERNATIVE_BITS
-#
-# This is an algorithm that is used for tests against the "disable-algorithms"
-# configuration option. Must be different from above algorithms.
-# DISABLED_ALGORITHM
-# DISABLED_ALGORITHM_NUMBER
-# DISABLED_BITS
-#
-# There are multiple algoritms sets to choose from (see get_algorithms.py). To
-# override the default choice, set the ALGORITHM_SET env var (see mkeys system
-# test for example).
-eval "$($PYTHON "$TOP_SRCDIR/bin/tests/system/get_algorithms.py")"
-
-# Default HMAC algorithm.
-# also update _common/rndc.conf and _common/rndc.key when updating DEFAULT_HMAC
-export DEFAULT_HMAC=hmac-sha256
-
 #
 # Useful functions in test scripts
 #

bin/tests/system/conftest.py

@@ -277,6 +277,17 @@ def wait_for_zones_loaded(request, servers):
             watcher.wait_for_line("all zones loaded")
 
 
+@pytest.fixture(scope="module", autouse=True)
+def configure_algorithm_set(request):
+    """Configure the algorithm set to use in tests."""
+    mark = _get_marker(request.node, "algorithm_set")
+    if not mark:
+        name = None
+    else:
+        name = mark.args[0]
+    isctest.vars.set_algorithm_set(name)
+
+
 @pytest.fixture(autouse=True)
 def logger(request, system_test_name):
     """Sets up logging facility specific to a particular test."""

bin/tests/system/isctest/vars/__init__.py

@@ -12,12 +12,14 @@
 import os
 
 from .all import ALL
+from .algorithms import set_algorithm_set
 from .openssl import parse_openssl_config
 from .. import log
 
 
 def init_vars():
     """Initializes the environment variables."""
+    set_algorithm_set(os.getenv("ALGORITHM_SET"))
     parse_openssl_config(ALL["OPENSSL_CONF"])
 
     os.environ.update(ALL)

bin/tests/system/isctest/vars/algorithms.py

@@ -1,5 +1,3 @@
-#!/usr/bin/python3
-
 # Copyright (C) Internet Systems Consortium, Inc. ("ISC")
 #
 # SPDX-License-Identifier: MPL-2.0
@@ -11,22 +9,43 @@
 # See the COPYRIGHT file distributed with this work for additional
 # information regarding copyright ownership.
 
-# This script is a 'port' broker.  It keeps track of ports given to the
-# individual system subtests, so every test is given a unique port range.
-
-import logging
 import os
 from pathlib import Path
 import platform
 import random
 import subprocess
 import time
-from typing import Dict, List, NamedTuple, Union
+from typing import Dict, List, NamedTuple, Optional, Union
 
-# Uncomment to enable DEBUG logging
-# logging.basicConfig(
-#     format="get_algorithms.py %(levelname)s %(message)s", level=logging.DEBUG
-# )
+from .basic import BASIC_VARS
+from .. import log
+
+# Algorithms are selected randomly at runtime from a list of supported
+# algorithms. The randomization is deterministic and remains stable for a
+# period of time for a given platform.
+ALG_VARS = {
+    # There are multiple algoritms sets to choose from (see ALGORITHM_SETS). To
+    # override the default choice, set the ALGORITHM_SET env var prior to
+    # loading this module or call set_algorithm_set().
+    "ALGORITHM_SET": "none",
+    "DEFAULT_ALGORITHM": "",
+    "DEFAULT_ALGORITHM_NUMBER": "",
+    "DEFAULT_BITS": "",
+    # Alternative algorithm for test cases that require more than one algorithm
+    # (for example algorithm rollover). Must be different from
+    # DEFAULT_ALGORITHM.
+    "ALTERNATIVE_ALGORITHM": "",
+    "ALTERNATIVE_ALGORITHM_NUMBER": "",
+    "ALTERNATIVE_BITS": "",
+    # Algorithm that is used for tests against the "disable-algorithms"
+    # configuration option. Must be different from above algorithms.
+    "DISABLED_ALGORITHM": "",
+    "DISABLED_ALGORITHM_NUMBER": "",
+    "DISABLED_BITS": "",
+    # Default HMAC algorithm. Must match the rndc configuration in
+    # bin/tests/system/_common (rndc.conf, rndc.key)
+    "DEFAULT_HMAC": "hmac-sha256",
+}
 
 STABLE_PERIOD = 3600 * 3
 """number of secs during which algorithm selection remains stable"""
@@ -93,57 +112,46 @@ ALGORITHM_SETS = {
     # ),
 }
 
-TESTCRYPTO = Path(__file__).resolve().parent / "testcrypto.sh"
-
-KEYGEN = os.getenv("KEYGEN", "")
-if not KEYGEN:
-    raise RuntimeError("KEYGEN environment variable has to be set")
-
-ALGORITHM_SET = os.getenv("ALGORITHM_SET", "stable")
-assert ALGORITHM_SET in ALGORITHM_SETS, f'ALGORITHM_SET "{ALGORITHM_SET}" unknown'
-logging.debug('choosing from ALGORITHM_SET "%s"', ALGORITHM_SET)
+# TODO rewrite testcrypto.sh to python
+TESTCRYPTO = Path(__file__).resolve().parent.parent.parent / "testcrypto.sh"
 
 
-def is_supported(alg: Algorithm) -> bool:
+def _is_supported(alg: Algorithm) -> bool:
     """Test whether a given algorithm is supported on the current platform."""
     try:
         subprocess.run(
             f"{TESTCRYPTO} -q {alg.name}",
             shell=True,
             check=True,
-            env={
-                "KEYGEN": KEYGEN,
-                "TMPDIR": os.getenv("TMPDIR", "/tmp"),
-            },
+            env=BASIC_VARS,
             stdout=subprocess.DEVNULL,
         )
     except subprocess.CalledProcessError as exc:
-        logging.debug(exc)
-        logging.info("algorithm %s not supported", alg.name)
+        log.debug(exc)
+        log.info("algorithm %s not supported", alg.name)
         return False
     return True
 
 
-def filter_supported(algs: AlgorithmSet) -> AlgorithmSet:
+def _filter_supported(algs: AlgorithmSet) -> AlgorithmSet:
     """Select supported algorithms from the set."""
     filtered = {}
     for alg_type in algs._fields:
         candidates = getattr(algs, alg_type)
         if isinstance(candidates, Algorithm):
             candidates = [candidates]
-        supported = list(filter(is_supported, candidates))
+        supported = list(filter(_is_supported, candidates))
         if len(supported) == 1:
             supported = supported.pop()
         elif not supported:
             raise RuntimeError(
-                f'no {alg_type.upper()} algorithm from "{ALGORITHM_SET}" set '
-                "supported on this platform"
+                f"no {alg_type.upper()} algorithm " "supported on this platform"
             )
         filtered[alg_type] = supported
     return AlgorithmSet(**filtered)
 
 
-def select_random(algs: AlgorithmSet, stable_period=STABLE_PERIOD) -> AlgorithmSet:
+def _select_random(algs: AlgorithmSet, stable_period=STABLE_PERIOD) -> AlgorithmSet:
     """Select random DEFAULT, ALTERNATIVE and DISABLED algorithms from the set.
 
     The algorithm selection is deterministic for a given time period and
@@ -200,9 +208,11 @@ def select_random(algs: AlgorithmSet, stable_period=STABLE_PERIOD) -> AlgorithmS
     return AlgorithmSet(default, alternative, disabled)
 
 
-def algorithms_env(algs: AlgorithmSet) -> Dict[str, str]:
+def _algorithms_env(algs: AlgorithmSet, name: str) -> Dict[str, str]:
     """Return environment variables with selected algorithms as a dict."""
-    algs_env: Dict[str, str] = {}
+    algs_env = {
+        "ALGORITHM_SET": name,
+    }
 
     def set_alg_env(alg: Algorithm, prefix):
         algs_env[f"{prefix}_ALGORITHM"] = alg.name
@@ -217,25 +227,23 @@ def algorithms_env(algs: AlgorithmSet) -> Dict[str, str]:
     set_alg_env(algs.alternative, "ALTERNATIVE")
     set_alg_env(algs.disabled, "DISABLED")
 
-    logging.info("selected algorithms: %s", algs_env)
+    log.info("selected algorithms: %s", algs_env)
     return algs_env
 
 
-def main():
-    try:
-        algs = ALGORITHM_SETS[ALGORITHM_SET]
-        algs = filter_supported(algs)
-        algs = select_random(algs)
-        algs_env = algorithms_env(algs)
-    except Exception:
-        # if anything goes wrong, the conf.sh ignores error codes, so make sure
-        # we set an environment variable to an error value that can be checked
-        # later by the test runner and/or tests themselves
-        print("export ALGORITHM_SET=error")
-        raise
-    for name, value in algs_env.items():
-        print(f"export {name}={value}")
+def set_algorithm_set(name: Optional[str]):
+    if name is None:
+        name = "stable"
+    assert name in ALGORITHM_SETS, f'ALGORITHM_SET "{name}" unknown'
+    if name == ALG_VARS["ALGORITHM_SET"]:
+        log.debug('algorithm set already configured: "%s"', name)
+        return
+    log.debug('choosing from ALGORITHM_SET "%s"', name)
 
+    algs = ALGORITHM_SETS[name]
+    algs = _filter_supported(algs)
+    algs = _select_random(algs)
+    algs_env = _algorithms_env(algs, name)
 
-if __name__ == "__main__":
-    main()
+    ALG_VARS.update(algs_env)
+    os.environ.update(algs_env)

bin/tests/system/isctest/vars/all.py

@@ -15,6 +15,7 @@ from collections import ChainMap
 from .autoconf import AC_VARS  # type: ignore
 
 # pylint: enable=import-error
+from .algorithms import ALG_VARS
 from .basic import BASIC_VARS
 from .dirs import DIR_VARS
 from .openssl import OPENSSL_VARS
@@ -52,4 +53,4 @@ class VarLookup(ChainMap):
         return iter(self.keys())
 
 
-ALL = VarLookup(AC_VARS, BASIC_VARS, OPENSSL_VARS, PORT_VARS, DIR_VARS)
+ALL = VarLookup(AC_VARS, BASIC_VARS, OPENSSL_VARS, PORT_VARS, DIR_VARS, ALG_VARS)

bin/tests/system/mkeys/setup.sh

@@ -11,14 +11,11 @@
 # See the COPYRIGHT file distributed with this work for additional
 # information regarding copyright ownership.
 
+# Explicitly setting ALGORITHM_SET is only needed is the script is executed
+# standalone without the pytest runner (e.g. for debugging).
 export ALGORITHM_SET="ecc_default"
-. ../conf.sh
 
-# Ensure the selected algorithm set is okay.
-if [ "$ALGORITHM_SET" = "error" ]; then
-  echofail "Algorithm selection failed." >&2
-  exit 1
-fi
+. ../conf.sh
 
 copy_setports ns1/named1.conf.in ns1/named.conf
 copy_setports ns2/named.conf.in ns2/named.conf

bin/tests/system/mkeys/tests.sh

@@ -13,7 +13,6 @@
 
 set -e
 
-export ALGORITHM_SET="ecc_default"
 #shellcheck source=conf.sh
 . ../conf.sh
 

bin/tests/system/mkeys/tests_sh_mkeys.py

@@ -9,6 +9,10 @@
 # See the COPYRIGHT file distributed with this work for additional
 # information regarding copyright ownership.
 
+import pytest
+
+pytestmark = pytest.mark.algorithm_set("ecc_default")
+
 
 def test_mkeys(run_tests_sh):
     run_tests_sh()

bin/tests/system/pytest.ini

@@ -20,3 +20,4 @@ junit_logging = log
 junit_log_passing_tests = 0
 markers =
     requires_zones_loaded: ensures the test does not start until the specified named instances load all configured zones
+    algorithm_set: use to select desired algorithms from isctest/vars/algorithms.py