From 7f8226a039b82d587114ee66662c05c673f0d87a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicki=20K=C5=99=C3=AD=C5=BEek?= <nicki@isc.org>
Date: Wed, 12 Mar 2025 17:03:50 +0100
Subject: [PATCH 1/2] Adjust the load factor for shotgun:tcp test

With the slightly decreased load for the TCP test, the results appear to
be a little bit more stable.
---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 5544a0ac7c..d6384fc896 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1701,7 +1701,7 @@ shotgun:tcp:
   <<: *shotgun_job
   variables:
     SHOTGUN_SCENARIO: tcp
-    SHOTGUN_TRAFFIC_MULTIPLIER: 13
+    SHOTGUN_TRAFFIC_MULTIPLIER: 12
 
 shotgun:dot:
   <<: *shotgun_job

From 5eab352478623eef57008a274c3a6505d9c76390 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicki=20K=C5=99=C3=AD=C5=BEek?= <nicki@isc.org>
Date: Wed, 12 Mar 2025 17:24:05 +0100
Subject: [PATCH 2/2] Allow re-run of the shotgun jobs to reduce false positive

The false positive rate is about 10-20 % when evaluating shotgun results
from a single run. Attempt to reduce the false positive rate by allowing
a re-run of failed jobs.

While there is a slight risk that barely noticable decreases in
performance might slip by more easily in MRs, they'd still likely pop up
during nightly or pre-release testing.

Also increase the tolerance threshold for DoH latency comparisons, as
those tests often experience increased jitter in the tail end latencies.
---
 .gitlab-ci.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d6384fc896..c144e42a17 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -383,6 +383,9 @@ stages:
         SHOTGUN_ROUNDS: 3
     - &shotgun_rule_other
       if: '$CI_PIPELINE_SOURCE =~ /^(api|pipeline|schedule|trigger|web)$/'
+  # when using data from a single run, the overall instability of the results
+  # causes quite high false positive rate, rerun the test to attemp to reduce those
+  retry: 1
   script:
     - if [ -z "$BASELINE" ]; then export BASELINE=$BIND_BASELINE_VERSION; fi  # this dotenv variable can't be set in the rules section, because rules are evaluated before any jobs run
     - PIPELINE_ID=$(curl -s -X POST --fail
@@ -1688,9 +1691,6 @@ respdiff-third-party:
 
 # Performance tests
 
-# Run shotgun:udp right away, but delay other shotgun jobs sligthly in order to
-# allow re-use of the built container image. Otherwise, the jobs would do the
-# same builds in parallel rather than re-use the already built image.
 shotgun:udp:
   <<: *shotgun_job
   variables:
@@ -1722,7 +1722,7 @@ shotgun:doh-get:
   variables:
     SHOTGUN_SCENARIO: doh-get
     SHOTGUN_TRAFFIC_MULTIPLIER: 3
-    SHOTGUN_EVAL_THRESHOLD_LATENCY_PCTL_MAX: 0.3  # bump from the default due to increased tail-end jitter
+    SHOTGUN_EVAL_THRESHOLD_LATENCY_PCTL_MAX: 0.4  # bump from the default due to increased tail-end jitter
   rules: *shotgun_rules_manual_mr
 
 .stress-test: &stress_test