From 4214c1e8a71d857fc8d602dc577260934c6342f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicki=20K=C5=99=C3=AD=C5=BEek?= Date: Wed, 19 Feb 2025 16:06:22 +0100 Subject: [PATCH 1/2] Run shotgun tests on MRs If the shotgun tests are executed for MRs, compare it against the MR's base rather than the previous release. Only fail the job in case the performance drops (pass on performance improvements). Note that start_in optimization was removed, since it isn't properly supported with rules as of February 2025 (https://gitlab.com/gitlab-org/gitlab/-/issues/424203). Without this optimization, container test images are likely to be re-built unnecessarily when testing different protocols. A workaround for the .gitlab-ci.yml exists, but the extra complexity doesn't seem justified. The container image builds might change or be optimized in the future, so let's just go with the build duplication for now. --- .gitlab-ci.yml | 52 ++++++++++++++++++++++++++++++++++------- util/ci-wait-shotgun.py | 8 ++++--- 2 files changed, 48 insertions(+), 12 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a88e22ff50..ef9c1ee8b4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -56,6 +56,16 @@ variables: # Some jobs may clean up the build artifacts unless this is set to 0. CLEAN_BUILD_ARTIFACTS_ON_SUCCESS: 1 + # DNS Shotgun performance testing defaults + SHOTGUN_ROUNDS: 1 + SHOTGUN_DURATION: 120 + # allow unlimited improvements against baseline + SHOTGUN_EVAL_THRESHOLD_CPU_MIN: '-inf' + SHOTGUN_EVAL_THRESHOLD_MEMORY_MIN: '-inf' + SHOTGUN_EVAL_THRESHOLD_RCODE_MAX: '+inf' + SHOTGUN_EVAL_THRESHOLD_LATENCY_PCTL_MIN: '-inf' + SHOTGUN_EVAL_THRESHOLD_LATENCY_PCTL_DRIFT_MIN: '-inf' + default: # Allow all running CI jobs to be automatically canceled when a new # version of a branch is pushed. @@ -343,18 +353,38 @@ stages: .shotgun: &shotgun_job <<: *base_image - <<: *api_pipelines_schedules_tags_triggers_web_triggering_rules stage: performance + rules: + - &shotgun_rule_mr + if: '$CI_MERGE_REQUEST_DIFF_BASE_SHA != null' + variables: + BASELINE: '$CI_MERGE_REQUEST_DIFF_BASE_SHA' + - &shotgun_rule_tag + if: '$CI_COMMIT_TAG != null' + variables: + SHOTGUN_ROUNDS: 3 + - &shotgun_rule_other + if: '$CI_PIPELINE_SOURCE =~ /^(api|pipeline|schedule|trigger|web)$/' script: - - if [ -z "$CI_COMMIT_TAG" ]; then export SHOTGUN_ROUNDS=1; else export SHOTGUN_ROUNDS=3; fi + - if [ -z "$BASELINE" ]; then export BASELINE=$BIND_BASELINE_VERSION; fi # this dotenv variable can't be set in the rules section, because rules are evaluated before any jobs run - PIPELINE_ID=$(curl -s -X POST --fail -F "token=$CI_JOB_TOKEN" -F ref=main - -F "variables[SHOTGUN_TEST_VERSION]=['$CI_COMMIT_REF_NAME', '$BIND_BASELINE_VERSION']" + -F "variables[SHOTGUN_TEST_VERSION]=['$CI_COMMIT_REF_NAME', '$BASELINE']" -F "variables[SHOTGUN_DURATION]=300" -F "variables[SHOTGUN_ROUNDS]=$SHOTGUN_ROUNDS" -F "variables[SHOTGUN_TRAFFIC_MULTIPLIER]=$SHOTGUN_TRAFFIC_MULTIPLIER" -F "variables[SHOTGUN_SCENARIO]=$SHOTGUN_SCENARIO" + -F "variables[SHOTGUN_EVAL_THRESHOLD_CPU_MIN]=$SHOTGUN_EVAL_THRESHOLD_CPU_MIN" + -F "variables[SHOTGUN_EVAL_THRESHOLD_CPU_MAX]=$SHOTGUN_EVAL_THRESHOLD_CPU_MAX" + -F "variables[SHOTGUN_EVAL_THRESHOLD_MEMORY_MIN]=$SHOTGUN_EVAL_THRESHOLD_MEMORY_MIN" + -F "variables[SHOTGUN_EVAL_THRESHOLD_MEMORY_MAX]=$SHOTGUN_EVAL_THRESHOLD_MEMORY_MAX" + -F "variables[SHOTGUN_EVAL_THRESHOLD_RCODE_MIN]=$SHOTGUN_EVAL_THRESHOLD_RCODE_MIN" + -F "variables[SHOTGUN_EVAL_THRESHOLD_RCODE_MAX]=$SHOTGUN_EVAL_THRESHOLD_RCODE_MAX" + -F "variables[SHOTGUN_EVAL_THRESHOLD_LATENCY_PCTL_MIN]=$SHOTGUN_EVAL_THRESHOLD_LATENCY_PCTL_MIN" + -F "variables[SHOTGUN_EVAL_THRESHOLD_LATENCY_PCTL_MAX]=$SHOTGUN_EVAL_THRESHOLD_LATENCY_PCTL_MAX" + -F "variables[SHOTGUN_EVAL_THRESHOLD_LATENCY_PCTL_DRIFT_MIN]=$SHOTGUN_EVAL_THRESHOLD_LATENCY_PCTL_DRIFT_MIN" + -F "variables[SHOTGUN_EVAL_THRESHOLD_LATENCY_PCTL_DRIFT_MAX]=$SHOTGUN_EVAL_THRESHOLD_LATENCY_PCTL_DRIFT_MAX" https://gitlab.isc.org/api/v4/projects/188/trigger/pipeline | jq .id) - util/ci-wait-shotgun.py $PIPELINE_ID needs: @@ -1670,24 +1700,28 @@ shotgun:tcp: variables: SHOTGUN_SCENARIO: tcp SHOTGUN_TRAFFIC_MULTIPLIER: 13 - when: delayed - start_in: 5 minutes shotgun:dot: <<: *shotgun_job variables: SHOTGUN_SCENARIO: dot SHOTGUN_TRAFFIC_MULTIPLIER: 6 - when: delayed - start_in: 5 minutes + rules: &shotgun_rules_manual_mr + - if: '$CI_MERGE_REQUEST_DIFF_BASE_SHA != null' + variables: + BASELINE: '$CI_MERGE_REQUEST_DIFF_BASE_SHA' + when: manual # don't run on each MR unless requested + allow_failure: true + - *shotgun_rule_tag + - *shotgun_rule_other shotgun:doh-get: <<: *shotgun_job variables: SHOTGUN_SCENARIO: doh-get SHOTGUN_TRAFFIC_MULTIPLIER: 3 - when: delayed - start_in: 5 minutes + SHOTGUN_EVAL_THRESHOLD_LATENCY_PCTL_MAX: 0.3 # bump from the default due to increased tail-end jitter + rules: *shotgun_rules_manual_mr .stress-test: &stress_test stage: performance diff --git a/util/ci-wait-shotgun.py b/util/ci-wait-shotgun.py index c7f5f26eab..894dfa9f57 100755 --- a/util/ci-wait-shotgun.py +++ b/util/ci-wait-shotgun.py @@ -79,13 +79,15 @@ def get_postproc_job(project, pipeline_id): def evaluate_postproc_job(job): - if job.status != "success": - raise RuntimeError("error: 'postproc' job didn't succeed") + print(f"postproc job URL: {job.web_url}") index_url = ( "https://isc-projects.gitlab-pages.isc.org/-/" f"bind9-shotgun-ci/-/jobs/{job.id}/artifacts/index.html" ) - print(f"Result ready for manual inspection: {index_url}") + if job.status in ["success", "failed"]: + print(f"result for manual inspection: {index_url}") + if job.status != "success": + raise RuntimeError("error: 'postproc' job didn't succeed") def main(): From 29fd7564083731373bd132ec65ffc0a9072f8efc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicki=20K=C5=99=C3=AD=C5=BEek?= Date: Tue, 25 Feb 2025 17:48:05 +0100 Subject: [PATCH 2/2] Replace deprecated only/except with rules in .gitlab-ci.yml The keyword rules allows more flexible and complex conditions when deciding whether to create the job and also makes it possible run tweak variables or job properties depending on arbitraty rules. Since it's not possible to combine only/except and rules together, replace all uses of only/except to avoid any potential future issues. --- .gitlab-ci.yml | 57 ++++++++++++++++++-------------------------------- 1 file changed, 20 insertions(+), 37 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ef9c1ee8b4..e0c936e99f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -229,31 +229,18 @@ stages: ### Job Templates .api-pipelines-schedules-tags-triggers-web-triggering-rules: &api_pipelines_schedules_tags_triggers_web_triggering_rules - only: - - api - - pipelines - - schedules - - tags - - triggers - - web + rules: + - if: '$CI_PIPELINE_SOURCE =~ /^(api|pipeline|schedule|trigger|web)$/' + - if: '$CI_COMMIT_TAG != null' .api-pipelines-schedules-triggers-web-triggering-rules: &api_pipelines_schedules_triggers_web_triggering_rules - only: - - api - - pipelines - - schedules - - triggers - - web + rules: + - if: '$CI_PIPELINE_SOURCE =~ /^(api|pipeline|schedule|trigger|web)$/' .default-triggering-rules: &default_triggering_rules - only: - - api - - merge_requests - - pipelines - - schedules - - tags - - triggers - - web + rules: + - if: '$CI_PIPELINE_SOURCE =~ /^(api|merge_request_event|pipeline|schedule|trigger|web)$/' + - if: '$CI_COMMIT_TAG != null' .precheck: &precheck_job <<: *default_triggering_rules @@ -649,9 +636,8 @@ danger: script: - pip install git+https://gitlab.isc.org/isc-projects/hazard.git - hazard - only: - refs: - - merge_requests + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' checkbashisms: <<: *precheck_job @@ -1524,8 +1510,8 @@ release: artifacts: true - job: docs artifacts: true - only: - - tags + rules: + - if: '$CI_COMMIT_TAG != null' artifacts: paths: - "*-release" @@ -1568,8 +1554,8 @@ sign: needs: - job: release artifacts: true - only: - - tags + rules: + - if: '$CI_COMMIT_TAG != null' when: manual allow_failure: false @@ -1621,10 +1607,8 @@ coverity: - cov-int.tar.gz expire_in: "1 week" when: on_failure - only: - variables: - - $COVERITY_SCAN_PROJECT_NAME - - $COVERITY_SCAN_TOKEN + rules: + - if: '$COVERITY_SCAN_PROJECT_NAME != null && $COVERITY_SCAN_TOKEN != null' # Respdiff tests @@ -1760,8 +1744,8 @@ fsck: - git clone https://gitlab.isc.org/isc-projects/bind9.git bind9-full-clone - cd bind9-full-clone/ - git fsck - only: - - schedules + rules: + - if: '$CI_PIPELINE_SOURCE == "schedule"' needs: [] gcov: @@ -1813,9 +1797,8 @@ pairwise: - pairwise-model.txt - pairwise-output.*.txt when: on_failure - only: - variables: - - $PAIRWISE_TESTING + rules: + - if: '$PAIRWISE_TESTING != null' .post_merge_template: &post_merge <<: *base_image