new: ci: Run shotgun tests on MRs

Execute DNS Shotgun performance tests on the regular MRs and compare the changes they introduce against the MR diff base. The results are evaluated automatically - the shotgun jobs will fail if thresholds for CPU/memory/latency difference is exceeded.

Merge branch 'nicki/ci-shotgun-eval' into 'main'

See merge request isc-projects/bind9!10127

.gitlab-ci.yml

@@ -56,6 +56,16 @@ variables:
   # Some jobs may clean up the build artifacts unless this is set to 0.
   CLEAN_BUILD_ARTIFACTS_ON_SUCCESS: 1
 
+  # DNS Shotgun performance testing defaults
+  SHOTGUN_ROUNDS: 1
+  SHOTGUN_DURATION: 120
+  # allow unlimited improvements against baseline
+  SHOTGUN_EVAL_THRESHOLD_CPU_MIN: '-inf'
+  SHOTGUN_EVAL_THRESHOLD_MEMORY_MIN: '-inf'
+  SHOTGUN_EVAL_THRESHOLD_RCODE_MAX: '+inf'
+  SHOTGUN_EVAL_THRESHOLD_LATENCY_PCTL_MIN: '-inf'
+  SHOTGUN_EVAL_THRESHOLD_LATENCY_PCTL_DRIFT_MIN: '-inf'
+
 default:
   # Allow all running CI jobs to be automatically canceled when a new
   # version of a branch is pushed.
@@ -219,31 +229,18 @@ stages:
 ### Job Templates
 
 .api-pipelines-schedules-tags-triggers-web-triggering-rules: &api_pipelines_schedules_tags_triggers_web_triggering_rules
-  only:
+  rules:
-    - api
+    - if: '$CI_PIPELINE_SOURCE =~ /^(api|pipeline|schedule|trigger|web)$/'
-    - pipelines
+    - if: '$CI_COMMIT_TAG != null'
-    - schedules
-    - tags
-    - triggers
-    - web
 
 .api-pipelines-schedules-triggers-web-triggering-rules: &api_pipelines_schedules_triggers_web_triggering_rules
-  only:
+  rules:
-    - api
+    - if: '$CI_PIPELINE_SOURCE =~ /^(api|pipeline|schedule|trigger|web)$/'
-    - pipelines
-    - schedules
-    - triggers
-    - web
 
 .default-triggering-rules: &default_triggering_rules
-  only:
+  rules:
-    - api
+    - if: '$CI_PIPELINE_SOURCE =~ /^(api|merge_request_event|pipeline|schedule|trigger|web)$/'
-    - merge_requests
+    - if: '$CI_COMMIT_TAG != null'
-    - pipelines
-    - schedules
-    - tags
-    - triggers
-    - web
 
 .precheck: &precheck_job
   <<: *default_triggering_rules
@@ -343,18 +340,38 @@ stages:
 
 .shotgun: &shotgun_job
   <<: *base_image
-  <<: *api_pipelines_schedules_tags_triggers_web_triggering_rules
   stage: performance
+  rules:
+    - &shotgun_rule_mr
+      if: '$CI_MERGE_REQUEST_DIFF_BASE_SHA != null'
+      variables:
+        BASELINE: '$CI_MERGE_REQUEST_DIFF_BASE_SHA'
+    - &shotgun_rule_tag
+      if: '$CI_COMMIT_TAG != null'
+      variables:
+        SHOTGUN_ROUNDS: 3
+    - &shotgun_rule_other
+      if: '$CI_PIPELINE_SOURCE =~ /^(api|pipeline|schedule|trigger|web)$/'
   script:
-    - if [ -z "$CI_COMMIT_TAG" ]; then export SHOTGUN_ROUNDS=1; else export SHOTGUN_ROUNDS=3; fi
+    - if [ -z "$BASELINE" ]; then export BASELINE=$BIND_BASELINE_VERSION; fi  # this dotenv variable can't be set in the rules section, because rules are evaluated before any jobs run
     - PIPELINE_ID=$(curl -s -X POST --fail
         -F "token=$CI_JOB_TOKEN"
         -F ref=main
-        -F "variables[SHOTGUN_TEST_VERSION]=['$CI_COMMIT_REF_NAME', '$BIND_BASELINE_VERSION']"
+        -F "variables[SHOTGUN_TEST_VERSION]=['$CI_COMMIT_REF_NAME', '$BASELINE']"
         -F "variables[SHOTGUN_DURATION]=300"
         -F "variables[SHOTGUN_ROUNDS]=$SHOTGUN_ROUNDS"
         -F "variables[SHOTGUN_TRAFFIC_MULTIPLIER]=$SHOTGUN_TRAFFIC_MULTIPLIER"
         -F "variables[SHOTGUN_SCENARIO]=$SHOTGUN_SCENARIO"
+        -F "variables[SHOTGUN_EVAL_THRESHOLD_CPU_MIN]=$SHOTGUN_EVAL_THRESHOLD_CPU_MIN"
+        -F "variables[SHOTGUN_EVAL_THRESHOLD_CPU_MAX]=$SHOTGUN_EVAL_THRESHOLD_CPU_MAX"
+        -F "variables[SHOTGUN_EVAL_THRESHOLD_MEMORY_MIN]=$SHOTGUN_EVAL_THRESHOLD_MEMORY_MIN"
+        -F "variables[SHOTGUN_EVAL_THRESHOLD_MEMORY_MAX]=$SHOTGUN_EVAL_THRESHOLD_MEMORY_MAX"
+        -F "variables[SHOTGUN_EVAL_THRESHOLD_RCODE_MIN]=$SHOTGUN_EVAL_THRESHOLD_RCODE_MIN"
+        -F "variables[SHOTGUN_EVAL_THRESHOLD_RCODE_MAX]=$SHOTGUN_EVAL_THRESHOLD_RCODE_MAX"
+        -F "variables[SHOTGUN_EVAL_THRESHOLD_LATENCY_PCTL_MIN]=$SHOTGUN_EVAL_THRESHOLD_LATENCY_PCTL_MIN"
+        -F "variables[SHOTGUN_EVAL_THRESHOLD_LATENCY_PCTL_MAX]=$SHOTGUN_EVAL_THRESHOLD_LATENCY_PCTL_MAX"
+        -F "variables[SHOTGUN_EVAL_THRESHOLD_LATENCY_PCTL_DRIFT_MIN]=$SHOTGUN_EVAL_THRESHOLD_LATENCY_PCTL_DRIFT_MIN"
+        -F "variables[SHOTGUN_EVAL_THRESHOLD_LATENCY_PCTL_DRIFT_MAX]=$SHOTGUN_EVAL_THRESHOLD_LATENCY_PCTL_DRIFT_MAX"
         https://gitlab.isc.org/api/v4/projects/188/trigger/pipeline | jq .id)
     - util/ci-wait-shotgun.py $PIPELINE_ID
   needs:
@@ -619,9 +636,8 @@ danger:
   script:
     - pip install git+https://gitlab.isc.org/isc-projects/hazard.git
     - hazard
-  only:
+  rules:
-    refs:
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
-      - merge_requests
 
 checkbashisms:
   <<: *precheck_job
@@ -1494,8 +1510,8 @@ release:
       artifacts: true
     - job: docs
       artifacts: true
-  only:
+  rules:
-    - tags
+    - if: '$CI_COMMIT_TAG != null'
   artifacts:
     paths:
       - "*-release"
@@ -1538,8 +1554,8 @@ sign:
   needs:
     - job: release
       artifacts: true
-  only:
+  rules:
-    - tags
+    - if: '$CI_COMMIT_TAG != null'
   when: manual
   allow_failure: false
 
@@ -1591,10 +1607,8 @@ coverity:
       - cov-int.tar.gz
     expire_in: "1 week"
     when: on_failure
-  only:
+  rules:
-    variables:
+    - if: '$COVERITY_SCAN_PROJECT_NAME != null && $COVERITY_SCAN_TOKEN != null'
-      - $COVERITY_SCAN_PROJECT_NAME
-      - $COVERITY_SCAN_TOKEN
 
 # Respdiff tests
 
@@ -1670,24 +1684,28 @@ shotgun:tcp:
   variables:
     SHOTGUN_SCENARIO: tcp
     SHOTGUN_TRAFFIC_MULTIPLIER: 13
-  when: delayed
-  start_in: 5 minutes
 
 shotgun:dot:
   <<: *shotgun_job
   variables:
     SHOTGUN_SCENARIO: dot
     SHOTGUN_TRAFFIC_MULTIPLIER: 6
-  when: delayed
+  rules: &shotgun_rules_manual_mr
-  start_in: 5 minutes
+    - if: '$CI_MERGE_REQUEST_DIFF_BASE_SHA != null'
+      variables:
+        BASELINE: '$CI_MERGE_REQUEST_DIFF_BASE_SHA'
+      when: manual  # don't run on each MR unless requested
+      allow_failure: true
+    - *shotgun_rule_tag
+    - *shotgun_rule_other
 
 shotgun:doh-get:
   <<: *shotgun_job
   variables:
     SHOTGUN_SCENARIO: doh-get
     SHOTGUN_TRAFFIC_MULTIPLIER: 3
-  when: delayed
+    SHOTGUN_EVAL_THRESHOLD_LATENCY_PCTL_MAX: 0.3  # bump from the default due to increased tail-end jitter
-  start_in: 5 minutes
+  rules: *shotgun_rules_manual_mr
 
 .stress-test: &stress_test
   stage: performance
@@ -1726,8 +1744,8 @@ fsck:
     - git clone https://gitlab.isc.org/isc-projects/bind9.git bind9-full-clone
     - cd bind9-full-clone/
     - git fsck
-  only:
+  rules:
-    - schedules
+    - if: '$CI_PIPELINE_SOURCE == "schedule"'
   needs: []
 
 gcov:
@@ -1779,9 +1797,8 @@ pairwise:
       - pairwise-model.txt
       - pairwise-output.*.txt
     when: on_failure
-  only:
+  rules:
-    variables:
+    - if: '$PAIRWISE_TESTING != null'
-      - $PAIRWISE_TESTING
 
 .post_merge_template: &post_merge
   <<: *base_image

util/ci-wait-shotgun.py

@@ -79,13 +79,15 @@ def get_postproc_job(project, pipeline_id):
 
 
 def evaluate_postproc_job(job):
-    if job.status != "success":
+    print(f"postproc job URL: {job.web_url}")
-        raise RuntimeError("error: 'postproc' job didn't succeed")
     index_url = (
         "https://isc-projects.gitlab-pages.isc.org/-/"
         f"bind9-shotgun-ci/-/jobs/{job.id}/artifacts/index.html"
     )
-    print(f"Result ready for manual inspection: {index_url}")
+    if job.status in ["success", "failed"]:
+        print(f"result for manual inspection: {index_url}")
+    if job.status != "success":
+        raise RuntimeError("error: 'postproc' job didn't succeed")
 
 
 def main():