Merge branch 'tkrizek/perf-tests-shotgun' into 'main'

Add DNS Shotgun performance tests to nightly pipelines See merge request isc-projects/bind9!8373
2023-11-01 13:38:35 +00:00
parent 3f205f3218 322dfbf6a9
commit d17d9de8a2
3 changed files with 172 additions and 8 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -340,6 +340,23 @@ stages:
        sudo sh -x bin/tests/system/ifconfig.sh up;
      fi

+ci-variables:
+  stage: precheck
+  <<: *precheck_job
+  script:
+    - export BIND_BASELINE_BRANCH="$(sed -n -E "s|^m4_define\(\[bind_VERSION_MINOR\], ([0-9]+)\)dnl$|\1|p" configure.ac)"
+    # When testing a .0 release, compare it against the previous development
+    # release (e.g., 9.19.0 and 9.18.0 should both be compared against 9.17.22).
+    - if [ "$(sed -n -E "s|^m4_define\(\[bind_VERSION_PATCH\], ([0-9]+)\)dnl$|\1|p" configure.ac)" = "0" ]; then export BIND_BASELINE_BRANCH=$((BIND_BASELINE_BRANCH - 1 - (BIND_BASELINE_BRANCH % 2))); fi
+    - BIND_BASELINE_VERSION="$(curl -s "https://gitlab.isc.org/api/v4/projects/1/repository/tags?search=^v9.${BIND_BASELINE_BRANCH}&order_by=version" | jq -r ".[0].name")"
+    - echo "BIND_BASELINE_VERSION=$BIND_BASELINE_VERSION" >> ci_vars.env
+  needs:
+    - job: autoreconf
+      artifacts: true
+  artifacts:
+    reports:
+      dotenv: ci_vars.env
+
 cross-version-config-tests:
  stage: system
  <<: *base_image
@@ -354,13 +371,8 @@ cross-version-config-tests:
    - *configure
    - *setup_interfaces
    - make -j${BUILD_PARALLEL_JOBS:-1}
-    - export BIND_BRANCH="$(sed -n -E "s|^m4_define\(\[bind_VERSION_MINOR\], ([0-9]+)\)dnl$|\1|p" configure.ac)"
-    # When testing a .0 release, compare it against the previous development
-    # release (e.g., 9.19.0 and 9.18.0 should both be compared against 9.17.22).
-    - if [ "$(sed -n -E "s|^m4_define\(\[bind_VERSION_PATCH\], ([0-9]+)\)dnl$|\1|p" configure.ac)" = "0" ]; then export BIND_BRANCH=$((BIND_BRANCH - 1 - (BIND_BRANCH % 2))); fi
-    - BASELINE="$(curl -s "https://gitlab.isc.org/api/v4/projects/1/repository/tags?search=^v9.${BIND_BRANCH}&order_by=version" | jq -r ".[0].name")"
-    - git clone --branch "${BASELINE}" --depth 1 https://gitlab.isc.org/isc-projects/bind9.git "bind-${BASELINE}"
-    - cd "bind-${BASELINE}"
+    - git clone --branch "${BIND_BASELINE_VERSION}" --depth 1 https://gitlab.isc.org/isc-projects/bind9.git "bind-${BIND_BASELINE_VERSION}"
+    - cd "bind-${BIND_BASELINE_VERSION}"
    - autoreconf -fi
    - *configure
    - make -j${BUILD_PARALLEL_JOBS:-1}
@@ -375,6 +387,8 @@ cross-version-config-tests:
  needs:
    - job: autoreconf
      artifacts: true
+    - job: ci-variables
+      artifacts: true
  artifacts:
    reports:
      junit: junit.xml
@@ -389,6 +403,26 @@ cross-version-config-tests:
    - awk '/^=+ FAILURES =+/{flag=1;next}/^=+.*=+$/{flag=0}flag' bin/tests/system/pytest.out.txt || true
    - awk '/^=+ ERRORS =+/{flag=1;next}/^=+.*=+$/{flag=0}flag' bin/tests/system/pytest.out.txt || true

+.shotgun: &shotgun_job
+  <<: *base_image
+  <<: *api_schedules_tags_triggers_web_triggering_rules
+  stage: performance
+  script:
+    - if [ -z "$CI_COMMIT_TAG" ]; then export SHOTGUN_ROUNDS=1; else export SHOTGUN_ROUNDS=3; fi
+    - PIPELINE_ID=$(curl -s -X POST --fail
+        -F "token=$CI_JOB_TOKEN"
+        -F ref=main
+        -F "variables[SHOTGUN_TEST_VERSION]=['$CI_COMMIT_REF_NAME', '$BIND_BASELINE_VERSION']"
+        -F "variables[SHOTGUN_DURATION]=300"
+        -F "variables[SHOTGUN_ROUNDS]=$SHOTGUN_ROUNDS"
+        -F "variables[SHOTGUN_TRAFFIC_MULTIPLIER]=$SHOTGUN_TRAFFIC_MULTIPLIER"
+        -F "variables[SHOTGUN_SCENARIO]=$SHOTGUN_SCENARIO"
+        https://gitlab.isc.org/api/v4/projects/188/trigger/pipeline | jq .id)
+    - util/ci-wait-shotgun.py $PIPELINE_ID
+  needs:
+    - job: ci-variables
+      artifacts: true
+
 .system_test_common: &system_test_common
  <<: *default_triggering_rules
  stage: system
@@ -1541,7 +1575,32 @@ respdiff-long-third-party:
  script:
    - bash respdiff.sh -s third_party -q "${PWD}/100k_mixed.txt" -c 1 -w "${PWD}/rspworkdir" "${CI_PROJECT_DIR}"

-# "Stress" tests
+# Performance tests
+
+# Run shotgun:udp right away, but delay other shotgun jobs sligthly in order to
+# allow re-use of the built container image. Otherwise, the jobs would do the
+# same builds in parallel rather than re-use the already built image.
+shotgun:udp:
+  <<: *shotgun_job
+  variables:
+    SHOTGUN_SCENARIO: udp
+    SHOTGUN_TRAFFIC_MULTIPLIER: 15
+
+shotgun:tcp:
+  <<: *shotgun_job
+  variables:
+    SHOTGUN_SCENARIO: tcp
+    SHOTGUN_TRAFFIC_MULTIPLIER: 6
+  when: delayed
+  start_in: 5 minutes
+
+shotgun:dot:
+  <<: *shotgun_job
+  variables:
+    SHOTGUN_SCENARIO: dot
+    SHOTGUN_TRAFFIC_MULTIPLIER: 3
+  when: delayed
+  start_in: 5 minutes

 # Parallel build in the "make" step is avoided since multiple jobs can be
 # executed concurrently on the same runner. This may present problems when one
--- a/.gitlab/issue_templates/Release.md
+++ b/.gitlab/issue_templates/Release.md
@@ -33,6 +33,7 @@
 - [ ] ***(QA)*** Rebase -S editions on top of current open-source versions: `git checkout bind-9.18-sub && git rebase origin/bind-9.18`
 - [ ] ***(QA)*** [Inform](https://gitlab.isc.org/isc-private/bind-qa/-/blob/master/bind9/releng/inform_supp_marketing.py) Support and Marketing of impending release (and give estimated release dates).
 - [ ] ***(QA)*** Ensure there are no permanent test failures on any platform. Check [public](https://gitlab.isc.org/isc-projects/bind9/-/pipelines?scope=all&source=schedule) and [private](https://gitlab.isc.org/isc-private/bind9/-/pipelines?scope=all&source=schedule) scheduled pipelines.
+ - [ ] ***(QA)*** Check charts from `shotgun:*` jobs in the scheduled pipelines to verify there is no unexplained performance drop for any protocol.
 - [ ] ***(QA)*** Check [Perflab](https://perflab.isc.org/) to ensure there has been no unexplained drop in performance for the versions being released.
 - [ ] ***(QA)*** Check whether all issues assigned to the release milestone are resolved[^1].
 - [ ] ***(QA)*** Ensure that there are no outstanding [merge requests in the private repository](https://gitlab.isc.org/isc-private/bind9/-/merge_requests/)[^1] (Subscription Edition only).
--- a/util/ci-wait-shotgun.py
+++ b/util/ci-wait-shotgun.py
@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+import os
+import sys
+import time
+
+import gitlab
+
+
+def init_gitlab_project():
+    if os.getenv("CI_SERVER_URL", None) is not None:
+        url = os.getenv("CI_SERVER_URL")
+        job_token = os.environ["CI_JOB_TOKEN"]
+        gl = gitlab.Gitlab(url, job_token=job_token)
+    else:  # running locally on dev machine
+        gl = gitlab.Gitlab.from_config()
+    return gl.projects.get("isc-projects/bind9-shotgun-ci")
+
+
+def parse_parent_pipeline_id():
+    if len(sys.argv) != 2:
+        raise RuntimeError("usage: util/ci-wait-shotgun.py PIPELINE_ID")
+    try:
+        return int(sys.argv[1])
+    except ValueError as exc:
+        raise RuntimeError("error: PIPELINE_ID must be a number") from exc
+
+
+def wait_until(callback, timeout=1800, retry=10):
+    start = time.time()
+    while time.time() - start < timeout:
+        if callback():
+            return
+        time.sleep(retry)
+    raise RuntimeError(f"error: timed out after {timeout}s")
+
+
+def get_child_pipeline_id(project, pipeline_id):
+    pipeline = project.pipelines.get(pipeline_id)
+
+    def pipeline_finished():
+        pipeline.refresh()
+        return pipeline.finished_at is not None
+
+    wait_until(pipeline_finished, timeout=3000)
+
+    bridges = pipeline.bridges.list()
+    if len(bridges) != 1:
+        raise RuntimeError("error: expected exactly one child pipeline")
+    return bridges[0].downstream_pipeline["id"]
+
+
+def get_postproc_job(project, pipeline_id):
+    pipeline = project.pipelines.get(pipeline_id)
+    postproc_job = None
+
+    def job_finished():
+        nonlocal postproc_job
+        for job in pipeline.jobs.list(get_all=True):
+            if job.name == "postproc":
+                postproc_job = job
+        if postproc_job is None:
+            raise RuntimeError("error: failed to find 'postproc' job in child pipeline")
+        return postproc_job.finished_at is not None
+
+    wait_until(job_finished)
+    return postproc_job
+
+
+def evaluate_postproc_job(job):
+    if job.status != "success":
+        raise RuntimeError("error: 'postproc' job didn't succeed")
+    index_url = (
+        "https://isc-projects.gitlab-pages.isc.org/-/"
+        f"bind9-shotgun-ci/-/jobs/{job.id}/artifacts/index.html"
+    )
+    print(f"Result ready for manual inspection: {index_url}")
+
+
+def main():
+    project = init_gitlab_project()
+    parent_pipeline_id = parse_parent_pipeline_id()
+    child_pipeline_id = get_child_pipeline_id(project, parent_pipeline_id)
+    postproc_job = get_postproc_job(project, child_pipeline_id)
+    evaluate_postproc_job(postproc_job)
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except RuntimeError as err:
+        print(err)
+        sys.exit(1)