diff --git a/bin/tests/system/.gitignore b/bin/tests/system/.gitignore
index 822e911470..0bbcb233c2 100644
--- a/bin/tests/system/.gitignore
+++ b/bin/tests/system/.gitignore
@@ -19,3 +19,4 @@ named.run
 /start.sh
 /stop.sh
 /ifconfig.sh
+/*_tmp_*
diff --git a/bin/tests/system/conftest.py b/bin/tests/system/conftest.py
index 4b9dccdc8e..f2b5be828b 100644
--- a/bin/tests/system/conftest.py
+++ b/bin/tests/system/conftest.py
@@ -156,6 +156,14 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
                 raise exc
             logging.debug(proc.stdout)
 
+    def pytest_ignore_collect(path):
+        # System tests are executed in temporary directories inside
+        # bin/tests/system. These temporary directories contain all files
+        # needed for the system tests - including tests_*.py files. Make sure to
+        # ignore these during test collection phase. Otherwise, test artifacts
+        # from previous runs could mess with the runner.
+        return "_tmp_" in str(path)
+
     @pytest.hookimpl(tryfirst=True, hookwrapper=True)
     def pytest_runtest_makereport(item):
         """Hook that is used to expose test results to session (for use in fixtures)."""