From 0584d3f65f4fea05eeb8c3f70c5460467b6637ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicki=20K=C5=99=C3=AD=C5=BEek?= Date: Mon, 13 Jan 2025 14:29:24 +0100 Subject: [PATCH] Set stricter limits for respdiff testing Adjust the limit of maximum disagreements in respdiff results based on recent pipeline results. The respdiff and respdiff:asan seem to have almost identical results, typically around 0.07 % of differences with ocassional spikes up to around 0.11 %. Similar results are for respdiff:tsan, perhaps with more common spikes with values up to around 0.12 %. Set the limit to 0.15 % to allow for some tolerance due to network conditions, time of day etc. The respdiff:third-party has a slightly higher disagreements average, with typical values being around 0.12 %. Set the limit to 0.2 %. Exceeding either of those values should be quite clear indication that some resolution behaviour has changed, since the values appear to be very stable within the newly configured limits. --- .gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 397451fd01..2ccaf19244 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1596,7 +1596,7 @@ respdiff: variables: CC: gcc CFLAGS: "${CFLAGS_COMMON} -Og -DISC_TRACK_PTHREADS_OBJECTS" - MAX_DISAGREEMENTS_PERCENTAGE: "0.5" + MAX_DISAGREEMENTS_PERCENTAGE: "0.15" script: - bash respdiff.sh -m /usr/lib/x86_64-linux-gnu/libjemalloc.so.2 -s named -q "${PWD}/100k_mixed.txt" -c 3 -w "${PWD}/rspworkdir" "${CI_PROJECT_DIR}" "/usr/local/respdiff-reference-bind/sbin/named" @@ -1609,7 +1609,7 @@ respdiff:asan: CFLAGS: "${CFLAGS_COMMON} -Og -fsanitize=address,undefined" LDFLAGS: "-fsanitize=address,undefined" EXTRA_CONFIGURE: "--without-jemalloc" - MAX_DISAGREEMENTS_PERCENTAGE: "0.5" + MAX_DISAGREEMENTS_PERCENTAGE: "0.15" script: - bash respdiff.sh -s named -q "${PWD}/100k_mixed.txt" -c 3 -w "${PWD}/rspworkdir" "${CI_PROJECT_DIR}" "/usr/local/respdiff-reference-bind/sbin/named" @@ -1622,7 +1622,7 @@ respdiff:tsan: CFLAGS: "${CFLAGS_COMMON} -Og -fsanitize=thread" LDFLAGS: "-fsanitize=thread" EXTRA_CONFIGURE: "--enable-pthread-rwlock --without-jemalloc PKG_CONFIG_PATH=/opt/tsan/lib/pkgconfig" - MAX_DISAGREEMENTS_PERCENTAGE: "0.5" + MAX_DISAGREEMENTS_PERCENTAGE: "0.15" TSAN_OPTIONS: "${TSAN_OPTIONS_DEBIAN}" script: - bash respdiff.sh -s named -q "${PWD}/100k_mixed.txt" -c 3 -w "${PWD}/rspworkdir" "${CI_PROJECT_DIR}" "/usr/local/respdiff-reference-bind/sbin/named" @@ -1637,7 +1637,7 @@ respdiff-third-party: variables: CC: gcc CFLAGS: "${CFLAGS_COMMON} -Og" - MAX_DISAGREEMENTS_PERCENTAGE: "0.5" + MAX_DISAGREEMENTS_PERCENTAGE: "0.2" script: - bash respdiff.sh -s third_party -q "${PWD}/100k_mixed.txt" -c 1 -w "${PWD}/rspworkdir" "${CI_PROJECT_DIR}"