From f55cacbbfd5eeaeb8e521895a0ffab8addeaad6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0t=C4=9Bp=C3=A1n=20Bal=C3=A1=C5=BEik?= Date: Tue, 23 Apr 2024 15:48:18 +0200 Subject: [PATCH] Disable deadlines for hypothesis tests when running in CI The times it takes to run tests CI vary significantly enough that it makes hypothesis test reach their deadlines and fail randomly marking the tests as flaky. This commit disables the deadlines when running in CI. --- .gitlab-ci.yml | 2 ++ bin/tests/system/wildcard/tests_wildcard.py | 7 ++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 72b64b8433..0396e175db 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -55,6 +55,8 @@ variables: BIND_STRESS_TEST_OS: linux BIND_STRESS_TEST_ARCH: amd64 + HYPOTHESIS_PROFILE: "ci" + default: # Allow all running CI jobs to be automatically canceled when a new # version of a branch is pushed. diff --git a/bin/tests/system/wildcard/tests_wildcard.py b/bin/tests/system/wildcard/tests_wildcard.py index cd5b278d28..f304f966b3 100755 --- a/bin/tests/system/wildcard/tests_wildcard.py +++ b/bin/tests/system/wildcard/tests_wildcard.py @@ -28,6 +28,7 @@ Limitations - untested properties: - special behavior of rdtypes like CNAME """ +import os import pytest pytest.importorskip("dns", minversion="2.0.0") @@ -47,7 +48,7 @@ try: pytest.importorskip("hypothesis") except ValueError: pytest.importorskip("hypothesis", minversion="4.41.2") -from hypothesis import assume, example, given +from hypothesis import assume, example, given, settings from strategies import dns_names, dns_rdatatypes_without_meta import isctest.check @@ -62,6 +63,10 @@ WILDCARD_RDATA = "192.0.2.1" IP_ADDR = "10.53.0.1" TIMEOUT = 5 # seconds, just a sanity check +# Timing of hypothesis tests is flaky in the CI, so we disable deadlines. +settings.register_profile("ci", deadline=None) +settings.load_profile(os.getenv("HYPOTHESIS_PROFILE", "default")) + @given(name=dns_names(suffix=SUFFIX), rdtype=dns_rdatatypes_without_meta) def test_wildcard_rdtype_mismatch(