From ef4e2d296458367aa1628dee238bb534087792e4 Mon Sep 17 00:00:00 2001 From: Tom Krizek Date: Wed, 27 Sep 2023 15:26:10 +0200 Subject: [PATCH 1/4] ci: move baseline version detection into separate job Multiple CI jobs may utilize a baseline version, i.e. the version that the current code should be tested against when doing comparative testing. To avoid repeating the non-trivial detection of the baseline version, move it into a separate job which creates an environment file that subsequent jobs may require via `needs` option. It is then possible to use the variable(s) defined in the script section of the new job. --- .gitlab-ci.yml | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0cdf0d98f2..a711d0ab9b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -340,6 +340,24 @@ stages: sudo sh -x bin/tests/system/ifconfig.sh up; fi +ci-variables: + stage: precheck + <<: *precheck_job + script: + - export BIND_BASELINE_BRANCH="$(sed -n -E "s|^m4_define\(\[bind_VERSION_MINOR\], ([0-9]+)\)dnl$|\1|p" configure.ac)" + # When testing a .0 release, compare it against the previous development + # release (e.g., 9.19.0 and 9.18.0 should both be compared against 9.17.22). + - if [ "$(sed -n -E "s|^m4_define\(\[bind_VERSION_PATCH\], ([0-9]+)\)dnl$|\1|p" configure.ac)" = "0" ]; then export BIND_BASELINE_BRANCH=$((BIND_BASELINE_BRANCH - 1 - (BIND_BASELINE_BRANCH % 2))); fi + - export BIND_BASELINE_VERSION="$(curl -s "https://gitlab.isc.org/api/v4/projects/1/repository/tags?search=^v9.${BIND_BASELINE_BRANCH}&order_by=version" | jq -r ".[0].name")" + - echo "BIND_BASELINE_BRANCH=$BIND_BASELINE_BRANCH" >> ci_vars.env + - echo "BIND_BASELINE_VERSION=$BIND_BASELINE_VERSION" >> ci_vars.env + needs: + - job: autoreconf + artifacts: true + artifacts: + reports: + dotenv: ci_vars.env + cross-version-config-tests: stage: system <<: *base_image @@ -354,13 +372,8 @@ cross-version-config-tests: - *configure - *setup_interfaces - make -j${BUILD_PARALLEL_JOBS:-1} - - export BIND_BRANCH="$(sed -n -E "s|^m4_define\(\[bind_VERSION_MINOR\], ([0-9]+)\)dnl$|\1|p" configure.ac)" - # When testing a .0 release, compare it against the previous development - # release (e.g., 9.19.0 and 9.18.0 should both be compared against 9.17.22). - - if [ "$(sed -n -E "s|^m4_define\(\[bind_VERSION_PATCH\], ([0-9]+)\)dnl$|\1|p" configure.ac)" = "0" ]; then export BIND_BRANCH=$((BIND_BRANCH - 1 - (BIND_BRANCH % 2))); fi - - BASELINE="$(curl -s "https://gitlab.isc.org/api/v4/projects/1/repository/tags?search=^v9.${BIND_BRANCH}&order_by=version" | jq -r ".[0].name")" - - git clone --branch "${BASELINE}" --depth 1 https://gitlab.isc.org/isc-projects/bind9.git "bind-${BASELINE}" - - cd "bind-${BASELINE}" + - git clone --branch "${BIND_BASELINE_VERSION}" --depth 1 https://gitlab.isc.org/isc-projects/bind9.git "bind-${BIND_BASELINE_VERSION}" + - cd "bind-${BIND_BASELINE_VERSION}" - autoreconf -fi - *configure - make -j${BUILD_PARALLEL_JOBS:-1} @@ -375,6 +388,8 @@ cross-version-config-tests: needs: - job: autoreconf artifacts: true + - job: ci-variables + artifacts: true artifacts: reports: junit: junit.xml From ff32421a64f1208687892ea8dbbbfd4c84fed028 Mon Sep 17 00:00:00 2001 From: Tom Krizek Date: Wed, 27 Sep 2023 17:41:26 +0200 Subject: [PATCH 2/4] util: script to get DNS Shotgun pipeline results The shotgun performance tests are executed in a different repository, in a couple of different pipelines. To hide away the complexity, this script takes the pipeline ID of the triggered pipeline and then takes care of the rest - waits for the pipeline to finish, locates the child pipeline and the relevant results. The output from this script is a convenient link to the charts with the results once they're available. GitLab also has a mechanism which can wait for another pipeline. However, it can't be utilized here, since there are variables which need to be passed in when the pipeline is triggered (like protocol to be tested, load, runtime etc.). This isn't currently supported by the GitLab feature. --- util/ci-wait-shotgun.py | 104 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100755 util/ci-wait-shotgun.py diff --git a/util/ci-wait-shotgun.py b/util/ci-wait-shotgun.py new file mode 100755 index 0000000000..ae126e615f --- /dev/null +++ b/util/ci-wait-shotgun.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 + +# Copyright (C) Internet Systems Consortium, Inc. ("ISC") +# +# SPDX-License-Identifier: MPL-2.0 +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, you can obtain one at https://mozilla.org/MPL/2.0/. +# +# See the COPYRIGHT file distributed with this work for additional +# information regarding copyright ownership. + +import os +import sys +import time + +import gitlab + + +def init_gitlab_project(): + if os.getenv("CI_SERVER_URL", None) is not None: + url = os.getenv("CI_SERVER_URL") + job_token = os.environ["CI_JOB_TOKEN"] + gl = gitlab.Gitlab(url, job_token=job_token) + else: # running locally on dev machine + gl = gitlab.Gitlab.from_config() + return gl.projects.get("isc-projects/bind9-shotgun-ci") + + +def parse_parent_pipeline_id(): + if len(sys.argv) != 2: + raise RuntimeError("usage: util/ci-wait-shotgun.py PIPELINE_ID") + try: + return int(sys.argv[1]) + except ValueError as exc: + raise RuntimeError("error: PIPELINE_ID must be a number") from exc + + +def wait_until(callback, timeout=1800, retry=10): + start = time.time() + while time.time() - start < timeout: + if callback(): + return + time.sleep(retry) + raise RuntimeError(f"error: timed out after {timeout}s") + + +def get_child_pipeline_id(project, pipeline_id): + pipeline = project.pipelines.get(pipeline_id) + + def pipeline_finished(): + pipeline.refresh() + return pipeline.finished_at is not None + + wait_until(pipeline_finished, timeout=3000) + + bridges = pipeline.bridges.list() + if len(bridges) != 1: + raise RuntimeError("error: expected exactly one child pipeline") + return bridges[0].downstream_pipeline["id"] + + +def get_postproc_job(project, pipeline_id): + pipeline = project.pipelines.get(pipeline_id) + postproc_job = None + + def job_finished(): + nonlocal postproc_job + for job in pipeline.jobs.list(get_all=True): + if job.name == "postproc": + postproc_job = job + if postproc_job is None: + raise RuntimeError("error: failed to find 'postproc' job in child pipeline") + return postproc_job.finished_at is not None + + wait_until(job_finished) + return postproc_job + + +def evaluate_postproc_job(job): + if job.status != "success": + raise RuntimeError("error: 'postproc' job didn't succeed") + index_url = ( + "https://isc-projects.gitlab-pages.isc.org/-/" + f"bind9-shotgun-ci/-/jobs/{job.id}/artifacts/index.html" + ) + print(f"Result ready for manual inspection: {index_url}") + + +def main(): + project = init_gitlab_project() + parent_pipeline_id = parse_parent_pipeline_id() + child_pipeline_id = get_child_pipeline_id(project, parent_pipeline_id) + postproc_job = get_postproc_job(project, child_pipeline_id) + evaluate_postproc_job(postproc_job) + + +if __name__ == "__main__": + try: + main() + except RuntimeError as err: + print(err) + sys.exit(1) From c7fcb1ca6055ac6796e6035f3939814f841a51fc Mon Sep 17 00:00:00 2001 From: Tom Krizek Date: Wed, 27 Sep 2023 15:48:31 +0200 Subject: [PATCH 3/4] ci: trigger a DNS Shotgun performance test Run comparative performance tests against the latest released version of the same branch. This is done for different protocols with an appropriate load the server is expected to be able to handle. Currently, the results need to be inspected manually, since a success of the job doesn't indicate there is no issue. Instead, the job provides an URL to an overview with latency, memory and CPU charts which display the test results with the current code against the reference version. There should be no major unexplained and reproducible differences in the charts. --- .gitlab-ci.yml | 50 +++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a711d0ab9b..2816b7717a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -348,8 +348,7 @@ ci-variables: # When testing a .0 release, compare it against the previous development # release (e.g., 9.19.0 and 9.18.0 should both be compared against 9.17.22). - if [ "$(sed -n -E "s|^m4_define\(\[bind_VERSION_PATCH\], ([0-9]+)\)dnl$|\1|p" configure.ac)" = "0" ]; then export BIND_BASELINE_BRANCH=$((BIND_BASELINE_BRANCH - 1 - (BIND_BASELINE_BRANCH % 2))); fi - - export BIND_BASELINE_VERSION="$(curl -s "https://gitlab.isc.org/api/v4/projects/1/repository/tags?search=^v9.${BIND_BASELINE_BRANCH}&order_by=version" | jq -r ".[0].name")" - - echo "BIND_BASELINE_BRANCH=$BIND_BASELINE_BRANCH" >> ci_vars.env + - BIND_BASELINE_VERSION="$(curl -s "https://gitlab.isc.org/api/v4/projects/1/repository/tags?search=^v9.${BIND_BASELINE_BRANCH}&order_by=version" | jq -r ".[0].name")" - echo "BIND_BASELINE_VERSION=$BIND_BASELINE_VERSION" >> ci_vars.env needs: - job: autoreconf @@ -404,6 +403,26 @@ cross-version-config-tests: - awk '/^=+ FAILURES =+/{flag=1;next}/^=+.*=+$/{flag=0}flag' bin/tests/system/pytest.out.txt || true - awk '/^=+ ERRORS =+/{flag=1;next}/^=+.*=+$/{flag=0}flag' bin/tests/system/pytest.out.txt || true +.shotgun: &shotgun_job + <<: *base_image + <<: *api_schedules_tags_triggers_web_triggering_rules + stage: performance + script: + - if [ -z "$CI_COMMIT_TAG" ]; then export SHOTGUN_ROUNDS=1; else export SHOTGUN_ROUNDS=3; fi + - PIPELINE_ID=$(curl -s -X POST --fail + -F "token=$CI_JOB_TOKEN" + -F ref=main + -F "variables[SHOTGUN_TEST_VERSION]=['$CI_COMMIT_REF_NAME', '$BIND_BASELINE_VERSION']" + -F "variables[SHOTGUN_DURATION]=300" + -F "variables[SHOTGUN_ROUNDS]=$SHOTGUN_ROUNDS" + -F "variables[SHOTGUN_TRAFFIC_MULTIPLIER]=$SHOTGUN_TRAFFIC_MULTIPLIER" + -F "variables[SHOTGUN_SCENARIO]=$SHOTGUN_SCENARIO" + https://gitlab.isc.org/api/v4/projects/188/trigger/pipeline | jq .id) + - util/ci-wait-shotgun.py $PIPELINE_ID + needs: + - job: ci-variables + artifacts: true + .system_test_common: &system_test_common <<: *default_triggering_rules stage: system @@ -1556,7 +1575,32 @@ respdiff-long-third-party: script: - bash respdiff.sh -s third_party -q "${PWD}/100k_mixed.txt" -c 1 -w "${PWD}/rspworkdir" "${CI_PROJECT_DIR}" -# "Stress" tests +# Performance tests + +# Run shotgun:udp right away, but delay other shotgun jobs sligthly in order to +# allow re-use of the built container image. Otherwise, the jobs would do the +# same builds in parallel rather than re-use the already built image. +shotgun:udp: + <<: *shotgun_job + variables: + SHOTGUN_SCENARIO: udp + SHOTGUN_TRAFFIC_MULTIPLIER: 15 + +shotgun:tcp: + <<: *shotgun_job + variables: + SHOTGUN_SCENARIO: tcp + SHOTGUN_TRAFFIC_MULTIPLIER: 6 + when: delayed + start_in: 5 minutes + +shotgun:dot: + <<: *shotgun_job + variables: + SHOTGUN_SCENARIO: dot + SHOTGUN_TRAFFIC_MULTIPLIER: 3 + when: delayed + start_in: 5 minutes # Parallel build in the "make" step is avoided since multiple jobs can be # executed concurrently on the same runner. This may present problems when one From 322dfbf6a9aac06acd2ee7c7eafdd10d4fe9159a Mon Sep 17 00:00:00 2001 From: Tom Krizek Date: Wed, 25 Oct 2023 15:51:23 +0200 Subject: [PATCH 4/4] Update release checklist Add a step to verify results of shotgun:* jobs. --- .gitlab/issue_templates/Release.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab/issue_templates/Release.md b/.gitlab/issue_templates/Release.md index 9aee063af4..b0bb655f69 100644 --- a/.gitlab/issue_templates/Release.md +++ b/.gitlab/issue_templates/Release.md @@ -33,6 +33,7 @@ - [ ] ***(QA)*** Rebase -S editions on top of current open-source versions: `git checkout bind-9.18-sub && git rebase origin/bind-9.18` - [ ] ***(QA)*** [Inform](https://gitlab.isc.org/isc-private/bind-qa/-/blob/master/bind9/releng/inform_supp_marketing.py) Support and Marketing of impending release (and give estimated release dates). - [ ] ***(QA)*** Ensure there are no permanent test failures on any platform. Check [public](https://gitlab.isc.org/isc-projects/bind9/-/pipelines?scope=all&source=schedule) and [private](https://gitlab.isc.org/isc-private/bind9/-/pipelines?scope=all&source=schedule) scheduled pipelines. + - [ ] ***(QA)*** Check charts from `shotgun:*` jobs in the scheduled pipelines to verify there is no unexplained performance drop for any protocol. - [ ] ***(QA)*** Check [Perflab](https://perflab.isc.org/) to ensure there has been no unexplained drop in performance for the versions being released. - [ ] ***(QA)*** Check whether all issues assigned to the release milestone are resolved[^1]. - [ ] ***(QA)*** Ensure that there are no outstanding [merge requests in the private repository](https://gitlab.isc.org/isc-private/bind9/-/merge_requests/)[^1] (Subscription Edition only).