From 9ba5c120cfe693c75614b74ab783d91a92835c45 Mon Sep 17 00:00:00 2001
From: Michal Nowak <mnowak@isc.org>
Date: Wed, 25 Jan 2023 21:38:56 +0100
Subject: [PATCH] Cross-version testing with named configurations

In #3381 (and #3385), we committed a backward-incompatible change to
BIND 9.19.5, 9.18.7, and 9.16.33, explicitly requiring "inline-signing"
for every "dnssec-policy".

We did this backward-incompatible change deliberately, knowing the
consequences for users and their configurations. But if we didn't, say,
we were unaware this is a backward-incompatible change and fixed failing
systems test by "tweaking a knob to make the CI pass", we would not have
a second look before the change hits user configurations.

"cross-version-config-tests" CI job is such a second look. It will run
system tests from the latest release tag specific to the particular
branch (e.g., v9.19.12 for the "main" branch) with BIND 9 binaries from
the current "HEAD" (the future v9.19.13). This Frankenstein build gets
conceived by altering the "TOP_BUILDDIR" variable in
"bin/tests/system/conf.sh".

Caveats:
- Only system test configurations are tested; no actual test code is
  run.
- Problems with namedN.conf configurations are not identified.

When backward-incompatible change is introduced, the CI job is expected
to fail. If the change is deliberate, the job will keep failing until
the version with the backward-incompatible change is tagged, and the
minor version in configure.ac is bumped.

(cherry picked from commit cc54211baa0a5d42ce45d471af8ef11f6b851b07)
---
 .gitlab-ci.yml | 44 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 63cdd32916..c2a2e3a68d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -343,6 +343,50 @@ stages:
         sudo sh -x bin/tests/system/ifconfig.sh up;
       fi
 
+cross-version-config-tests:
+  stage: system
+  <<: *base_image
+  <<: *default_triggering_rules
+  variables:
+    CC: gcc
+    CFLAGS: "${CFLAGS_COMMON}"
+    # Disable option checking to prevent problems with new default options in
+    # the &configure anchor.
+    EXTRA_CONFIGURE: "--disable-option-checking"
+  script:
+    - *configure
+    - *setup_interfaces
+    - make -j${BUILD_PARALLEL_JOBS:-1}
+    # Build system test binaries.
+    - make -C bin/tests/system -j${TEST_PARALLEL_JOBS:-1} check TESTS=""
+    - export BIND_BRANCH="$(sed -n -E "s|^m4_define\(\[bind_VERSION_MINOR\], ([0-9]+)\)dnl$|\1|p" configure.ac)"
+    # When testing a .0 release, compare it against the previous development
+    # release (e.g., 9.19.0 and 9.18.0 should both be compared against 9.17.22).
+    - if [ "$(sed -n -E "s|^m4_define\(\[bind_VERSION_PATCH\], ([0-9]+)\)dnl$|\1|p" configure.ac)" = "0" ]; then export BIND_BRANCH=$((BIND_BRANCH - 1 - (BIND_BRANCH % 2))); fi
+    - BASELINE="$(curl -s "https://gitlab.isc.org/api/v4/projects/1/repository/tags?search=^v9.${BIND_BRANCH}&order_by=version" | jq -r ".[0].name")"
+    - git clone --branch "${BASELINE}" --depth 1 https://gitlab.isc.org/isc-projects/bind9.git "bind-${BASELINE}"
+    - cd "bind-${BASELINE}"
+    - autoreconf -fi
+    - *configure
+    - make -j${BUILD_PARALLEL_JOBS:-1}
+    - *find_pytest
+    - cd bin/tests/system
+    # Run the setup phase of all system tests in the most recently tagged BIND 9
+    # release using the binaries built for the current BIND 9 version.  This
+    # intends to detect obvious backward compatibility issues with the latter.
+    - sed -i -E "s|(export TOP_BUILDDIR)=.*|\1=${CI_PROJECT_DIR}|" conf.sh
+    - >
+      "$PYTEST" --setup-only -n "${TEST_PARALLEL_JOBS:-1}"
+  needs:
+    - job: autoreconf
+      artifacts: true
+  artifacts:
+    paths:
+      - bind-*
+    untracked: true
+    expire_in: "1 day"
+    when: on_failure
+
 .display_pytest_failures: &display_pytest_failures
     - awk '/^=+ FAILURES =+/{flag=1;next}/^=+.*=+$/{flag=0}flag' bin/tests/system/pytest.out.txt || true
     - awk '/^=+ ERRORS =+/{flag=1;next}/^=+.*=+$/{flag=0}flag' bin/tests/system/pytest.out.txt || true