From 71a9e152f1d4fe983a1dd4ff8b702742d3820a72 Mon Sep 17 00:00:00 2001 From: Tom Krizek Date: Thu, 17 Aug 2023 10:30:46 +0200 Subject: [PATCH 1/2] Add custom flaky decorator to handle unstable tests If the flaky plugin for pytest is available, use its decorator to support re-running unstable tests. In case the package is missing, execute the test as usual without attempts to re-run it in case of failure. This is mostly intended to increase the test stability in CI. Using a custom decorator enables us to keep the flaky package as an optional dependency. (cherry picked from commit 5b703de733b1f53f0452e8e654459d7adeb9aa00) --- bin/tests/system/pytest_custom_markers.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/bin/tests/system/pytest_custom_markers.py b/bin/tests/system/pytest_custom_markers.py index 7411afeafd..ba3a9d4894 100644 --- a/bin/tests/system/pytest_custom_markers.py +++ b/bin/tests/system/pytest_custom_markers.py @@ -40,3 +40,21 @@ have_libxml2 = pytest.mark.skipif( have_json_c = pytest.mark.skipif( not feature_test("--have-json-c"), reason="json-c support disabled in the build" ) + + +try: + import flaky as flaky_pkg +except ModuleNotFoundError: + # In case the flaky package is not installed, run the tests as usual + # without any attempts to re-run them. + # pylint: disable=unused-argument + def flaky(*args, **kwargs): + """Mock decorator that doesn't do anything special, just returns the function.""" + + def wrapper(wrapped_obj): + return wrapped_obj + + return wrapper + +else: + flaky = flaky_pkg.flaky From c3cc8aa429e2d4b5dd3e4ebcf6ce2611fc237f96 Mon Sep 17 00:00:00 2001 From: Michal Nowak Date: Tue, 15 Aug 2023 17:23:30 +0200 Subject: [PATCH 2/2] Mark test_send_timeout as flaky In some cases, BIND is not fast enough to fill the send buffer and manages to answer all queries, contrary to what the test expects. Repeat the check up to 3 times to limit this test instability. (cherry picked from commit 681b23c3985e592ff0548eee012183e0d992b99f) --- bin/tests/system/timeouts/tests_tcp_timeouts.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/tests/system/timeouts/tests_tcp_timeouts.py b/bin/tests/system/timeouts/tests_tcp_timeouts.py index 994a9d746c..2e2a4b47e2 100644 --- a/bin/tests/system/timeouts/tests_tcp_timeouts.py +++ b/bin/tests/system/timeouts/tests_tcp_timeouts.py @@ -185,6 +185,7 @@ def test_long_axfr(named_port): assert soa is not None +@pytest_custom_markers.flaky(max_runs=3) def test_send_timeout(named_port): with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: sock.connect(("10.53.0.1", named_port))