diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0cdf0d98f2..2816b7717a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -340,6 +340,23 @@ stages: sudo sh -x bin/tests/system/ifconfig.sh up; fi +ci-variables: + stage: precheck + <<: *precheck_job + script: + - export BIND_BASELINE_BRANCH="$(sed -n -E "s|^m4_define\(\[bind_VERSION_MINOR\], ([0-9]+)\)dnl$|\1|p" configure.ac)" + # When testing a .0 release, compare it against the previous development + # release (e.g., 9.19.0 and 9.18.0 should both be compared against 9.17.22). + - if [ "$(sed -n -E "s|^m4_define\(\[bind_VERSION_PATCH\], ([0-9]+)\)dnl$|\1|p" configure.ac)" = "0" ]; then export BIND_BASELINE_BRANCH=$((BIND_BASELINE_BRANCH - 1 - (BIND_BASELINE_BRANCH % 2))); fi + - BIND_BASELINE_VERSION="$(curl -s "https://gitlab.isc.org/api/v4/projects/1/repository/tags?search=^v9.${BIND_BASELINE_BRANCH}&order_by=version" | jq -r ".[0].name")" + - echo "BIND_BASELINE_VERSION=$BIND_BASELINE_VERSION" >> ci_vars.env + needs: + - job: autoreconf + artifacts: true + artifacts: + reports: + dotenv: ci_vars.env + cross-version-config-tests: stage: system <<: *base_image @@ -354,13 +371,8 @@ cross-version-config-tests: - *configure - *setup_interfaces - make -j${BUILD_PARALLEL_JOBS:-1} - - export BIND_BRANCH="$(sed -n -E "s|^m4_define\(\[bind_VERSION_MINOR\], ([0-9]+)\)dnl$|\1|p" configure.ac)" - # When testing a .0 release, compare it against the previous development - # release (e.g., 9.19.0 and 9.18.0 should both be compared against 9.17.22). - - if [ "$(sed -n -E "s|^m4_define\(\[bind_VERSION_PATCH\], ([0-9]+)\)dnl$|\1|p" configure.ac)" = "0" ]; then export BIND_BRANCH=$((BIND_BRANCH - 1 - (BIND_BRANCH % 2))); fi - - BASELINE="$(curl -s "https://gitlab.isc.org/api/v4/projects/1/repository/tags?search=^v9.${BIND_BRANCH}&order_by=version" | jq -r ".[0].name")" - - git clone --branch "${BASELINE}" --depth 1 https://gitlab.isc.org/isc-projects/bind9.git "bind-${BASELINE}" - - cd "bind-${BASELINE}" + - git clone --branch "${BIND_BASELINE_VERSION}" --depth 1 https://gitlab.isc.org/isc-projects/bind9.git "bind-${BIND_BASELINE_VERSION}" + - cd "bind-${BIND_BASELINE_VERSION}" - autoreconf -fi - *configure - make -j${BUILD_PARALLEL_JOBS:-1} @@ -375,6 +387,8 @@ cross-version-config-tests: needs: - job: autoreconf artifacts: true + - job: ci-variables + artifacts: true artifacts: reports: junit: junit.xml @@ -389,6 +403,26 @@ cross-version-config-tests: - awk '/^=+ FAILURES =+/{flag=1;next}/^=+.*=+$/{flag=0}flag' bin/tests/system/pytest.out.txt || true - awk '/^=+ ERRORS =+/{flag=1;next}/^=+.*=+$/{flag=0}flag' bin/tests/system/pytest.out.txt || true +.shotgun: &shotgun_job + <<: *base_image + <<: *api_schedules_tags_triggers_web_triggering_rules + stage: performance + script: + - if [ -z "$CI_COMMIT_TAG" ]; then export SHOTGUN_ROUNDS=1; else export SHOTGUN_ROUNDS=3; fi + - PIPELINE_ID=$(curl -s -X POST --fail + -F "token=$CI_JOB_TOKEN" + -F ref=main + -F "variables[SHOTGUN_TEST_VERSION]=['$CI_COMMIT_REF_NAME', '$BIND_BASELINE_VERSION']" + -F "variables[SHOTGUN_DURATION]=300" + -F "variables[SHOTGUN_ROUNDS]=$SHOTGUN_ROUNDS" + -F "variables[SHOTGUN_TRAFFIC_MULTIPLIER]=$SHOTGUN_TRAFFIC_MULTIPLIER" + -F "variables[SHOTGUN_SCENARIO]=$SHOTGUN_SCENARIO" + https://gitlab.isc.org/api/v4/projects/188/trigger/pipeline | jq .id) + - util/ci-wait-shotgun.py $PIPELINE_ID + needs: + - job: ci-variables + artifacts: true + .system_test_common: &system_test_common <<: *default_triggering_rules stage: system @@ -1541,7 +1575,32 @@ respdiff-long-third-party: script: - bash respdiff.sh -s third_party -q "${PWD}/100k_mixed.txt" -c 1 -w "${PWD}/rspworkdir" "${CI_PROJECT_DIR}" -# "Stress" tests +# Performance tests + +# Run shotgun:udp right away, but delay other shotgun jobs sligthly in order to +# allow re-use of the built container image. Otherwise, the jobs would do the +# same builds in parallel rather than re-use the already built image. +shotgun:udp: + <<: *shotgun_job + variables: + SHOTGUN_SCENARIO: udp + SHOTGUN_TRAFFIC_MULTIPLIER: 15 + +shotgun:tcp: + <<: *shotgun_job + variables: + SHOTGUN_SCENARIO: tcp + SHOTGUN_TRAFFIC_MULTIPLIER: 6 + when: delayed + start_in: 5 minutes + +shotgun:dot: + <<: *shotgun_job + variables: + SHOTGUN_SCENARIO: dot + SHOTGUN_TRAFFIC_MULTIPLIER: 3 + when: delayed + start_in: 5 minutes # Parallel build in the "make" step is avoided since multiple jobs can be # executed concurrently on the same runner. This may present problems when one diff --git a/.gitlab/issue_templates/Release.md b/.gitlab/issue_templates/Release.md index 9aee063af4..b0bb655f69 100644 --- a/.gitlab/issue_templates/Release.md +++ b/.gitlab/issue_templates/Release.md @@ -33,6 +33,7 @@ - [ ] ***(QA)*** Rebase -S editions on top of current open-source versions: `git checkout bind-9.18-sub && git rebase origin/bind-9.18` - [ ] ***(QA)*** [Inform](https://gitlab.isc.org/isc-private/bind-qa/-/blob/master/bind9/releng/inform_supp_marketing.py) Support and Marketing of impending release (and give estimated release dates). - [ ] ***(QA)*** Ensure there are no permanent test failures on any platform. Check [public](https://gitlab.isc.org/isc-projects/bind9/-/pipelines?scope=all&source=schedule) and [private](https://gitlab.isc.org/isc-private/bind9/-/pipelines?scope=all&source=schedule) scheduled pipelines. + - [ ] ***(QA)*** Check charts from `shotgun:*` jobs in the scheduled pipelines to verify there is no unexplained performance drop for any protocol. - [ ] ***(QA)*** Check [Perflab](https://perflab.isc.org/) to ensure there has been no unexplained drop in performance for the versions being released. - [ ] ***(QA)*** Check whether all issues assigned to the release milestone are resolved[^1]. - [ ] ***(QA)*** Ensure that there are no outstanding [merge requests in the private repository](https://gitlab.isc.org/isc-private/bind9/-/merge_requests/)[^1] (Subscription Edition only). diff --git a/util/ci-wait-shotgun.py b/util/ci-wait-shotgun.py new file mode 100755 index 0000000000..ae126e615f --- /dev/null +++ b/util/ci-wait-shotgun.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 + +# Copyright (C) Internet Systems Consortium, Inc. ("ISC") +# +# SPDX-License-Identifier: MPL-2.0 +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, you can obtain one at https://mozilla.org/MPL/2.0/. +# +# See the COPYRIGHT file distributed with this work for additional +# information regarding copyright ownership. + +import os +import sys +import time + +import gitlab + + +def init_gitlab_project(): + if os.getenv("CI_SERVER_URL", None) is not None: + url = os.getenv("CI_SERVER_URL") + job_token = os.environ["CI_JOB_TOKEN"] + gl = gitlab.Gitlab(url, job_token=job_token) + else: # running locally on dev machine + gl = gitlab.Gitlab.from_config() + return gl.projects.get("isc-projects/bind9-shotgun-ci") + + +def parse_parent_pipeline_id(): + if len(sys.argv) != 2: + raise RuntimeError("usage: util/ci-wait-shotgun.py PIPELINE_ID") + try: + return int(sys.argv[1]) + except ValueError as exc: + raise RuntimeError("error: PIPELINE_ID must be a number") from exc + + +def wait_until(callback, timeout=1800, retry=10): + start = time.time() + while time.time() - start < timeout: + if callback(): + return + time.sleep(retry) + raise RuntimeError(f"error: timed out after {timeout}s") + + +def get_child_pipeline_id(project, pipeline_id): + pipeline = project.pipelines.get(pipeline_id) + + def pipeline_finished(): + pipeline.refresh() + return pipeline.finished_at is not None + + wait_until(pipeline_finished, timeout=3000) + + bridges = pipeline.bridges.list() + if len(bridges) != 1: + raise RuntimeError("error: expected exactly one child pipeline") + return bridges[0].downstream_pipeline["id"] + + +def get_postproc_job(project, pipeline_id): + pipeline = project.pipelines.get(pipeline_id) + postproc_job = None + + def job_finished(): + nonlocal postproc_job + for job in pipeline.jobs.list(get_all=True): + if job.name == "postproc": + postproc_job = job + if postproc_job is None: + raise RuntimeError("error: failed to find 'postproc' job in child pipeline") + return postproc_job.finished_at is not None + + wait_until(job_finished) + return postproc_job + + +def evaluate_postproc_job(job): + if job.status != "success": + raise RuntimeError("error: 'postproc' job didn't succeed") + index_url = ( + "https://isc-projects.gitlab-pages.isc.org/-/" + f"bind9-shotgun-ci/-/jobs/{job.id}/artifacts/index.html" + ) + print(f"Result ready for manual inspection: {index_url}") + + +def main(): + project = init_gitlab_project() + parent_pipeline_id = parse_parent_pipeline_id() + child_pipeline_id = get_child_pipeline_id(project, parent_pipeline_id) + postproc_job = get_postproc_job(project, child_pipeline_id) + evaluate_postproc_job(postproc_job) + + +if __name__ == "__main__": + try: + main() + except RuntimeError as err: + print(err) + sys.exit(1)