chg: test: Improve system test stability

Tweak various system test which have been unstable in the past weeks.

Closes #5406

Merge branch 'nicki/improve-system-test-stability' into 'main'

See merge request isc-projects/bind9!10690

bin/tests/system/autosign/tests_sh_autosign.py

@@ -11,8 +11,6 @@
 
 import pytest
 
-import isctest.mark
-
 pytestmark = pytest.mark.extra_artifacts(
     [
         "active.key",
@@ -151,6 +149,6 @@ pytestmark = pytest.mark.extra_artifacts(
 )
 
 
-@isctest.mark.flaky(max_runs=2)
+@pytest.mark.flaky(max_runs=2)
 def test_autosign(run_tests_sh):
     run_tests_sh()

bin/tests/system/digdelv/tests.sh

@@ -1828,15 +1828,6 @@ if [ -x "$DELV" ]; then
   status=$((status + ret))
 
   if testsock6 fd92:7065:b8e:ffff::2 2>/dev/null; then
-    n=$((n + 1))
-    echo_i "checking delv +ns uses both address families ($n)"
-    ret=0
-    delv_with_opts -a ns1/anchor.dnskey +root +ns +hint=root.hint a a.example >delv.out.test$n || ret=1
-    grep -q 'sending packet from [0-9.]*#[0-9]* to' delv.out.test$n >/dev/null || ret=1
-    grep -q 'sending packet from [0-9a-f:]*#[0-9]* to' delv.out.test$n >/dev/null || ret=1
-    if [ $ret -ne 0 ]; then echo_i "failed"; fi
-    status=$((status + ret))
-
     n=$((n + 1))
     echo_i "checking delv -4 +ns uses only IPv4 ($n)"
     ret=0

bin/tests/system/dnstap/tests_sh_dnstap.py

@@ -31,5 +31,6 @@ pytestmark = [
 ]
 
 
+@pytest.mark.flaky(max_runs=2, rerun_filter=isctest.mark.is_host_freebsd_13)
 def test_dnstap(run_tests_sh):
     run_tests_sh()

bin/tests/system/enginepkcs11/tests_sh_enginepkcs11.py

@@ -11,8 +11,6 @@
 
 import pytest
 
-import isctest.mark
-
 pytestmark = pytest.mark.extra_artifacts(
     [
         "dig.out.*",
@@ -51,6 +49,6 @@ pytestmark = pytest.mark.extra_artifacts(
 )
 
 
-@isctest.mark.flaky(max_runs=3)  # GL#4605
+@pytest.mark.flaky(max_runs=5)  # GL#4605
 def test_enginepkcs11(run_tests_sh):
     run_tests_sh()

bin/tests/system/fetchlimit/tests_sh_fetchlimit.py

@@ -22,9 +22,7 @@ pytestmark = pytest.mark.extra_artifacts(
     ]
 )
 
-import isctest.mark
 
-
+@pytest.mark.flaky(max_runs=3)
-@isctest.mark.flaky(max_runs=2)
 def test_fetchlimit(run_tests_sh):
     run_tests_sh()

bin/tests/system/isctest/mark.py

@@ -12,6 +12,7 @@
 # information regarding copyright ownership.
 
 import os
+import platform
 import shutil
 import subprocess
 
@@ -39,8 +40,8 @@ def feature_test(feature):
     return True
 
 
-def with_tsan(*args):  # pylint: disable=unused-argument
+def is_host_freebsd_13(*_):
-    return feature_test("--tsan")
+    return platform.system() == "FreeBSD" and platform.release().startswith("13")
 
 
 def with_algorithm(name: str):
@@ -79,20 +80,3 @@ softhsm2_environment = pytest.mark.skipif(
     ),
     reason="SOFTHSM2_CONF and SOFTHSM2_MODULE environmental variables must be set and pkcs11-tool and softhsm2-util tools present",
 )
-
-try:
-    import flaky as flaky_pkg  # type: ignore
-except ModuleNotFoundError:
-    # In case the flaky package is not installed, run the tests as usual
-    # without any attempts to re-run them.
-    # pylint: disable=unused-argument
-    def flaky(*args, **kwargs):
-        """Mock decorator that doesn't do anything special, just returns the function."""
-
-        def wrapper(wrapped_obj):
-            return wrapped_obj
-
-        return wrapper
-
-else:
-    flaky = flaky_pkg.flaky

bin/tests/system/kasp/tests_kasp.py

@@ -203,7 +203,7 @@ def cb_ixfr_is_signed(expected_updates, params, ksks=None, zsks=None):
         )
 
     for update in expected_updates:
-        isctest.run.retry_with_timeout(update_is_signed, timeout=5)
+        isctest.run.retry_with_timeout(update_is_signed, timeout=10)
 
 
 def cb_rrsig_refresh(params, ksks=None, zsks=None):
@@ -586,6 +586,9 @@ def cb_remove_keyfiles(params, ksks=None, zsks=None):
                 ],
             },
             id="secondary.kasp",
+            marks=pytest.mark.flaky(
+                max_runs=2, rerun_filter=isctest.mark.is_host_freebsd_13
+            ),
         ),
         pytest.param(
             {

bin/tests/system/keyfromlabel/tests_keyfromlabel.py

@@ -86,7 +86,6 @@ def token_init_and_cleanup():
             raise_on_exception=False,
         ).stdout.decode("utf-8")
         assert re.search("Found token (.*) with matching token label", output)
-        assert re.search("The token (.*) has been deleted", output)
 
 
 # pylint: disable-msg=too-many-locals

bin/tests/system/mkeys/tests_sh_mkeys.py

@@ -11,8 +11,6 @@
 
 import pytest
 
-import isctest.mark
-
 pytestmark = [
     pytest.mark.algorithm_set("ecc_default"),
     pytest.mark.extra_artifacts(
@@ -52,6 +50,6 @@ pytestmark = [
 ]
 
 
-@isctest.mark.flaky(max_runs=2)  # GL#3098
+@pytest.mark.flaky(max_runs=2)  # GL#3098
 def test_mkeys(run_tests_sh):
     run_tests_sh()

bin/tests/system/nsupdate/tests_sh_nsupdate.py

@@ -13,8 +13,6 @@ import platform
 
 import pytest
 
-import isctest.mark
-
 pytestmark = pytest.mark.extra_artifacts(
     [
         "Kxxx*",
@@ -82,6 +80,6 @@ pytestmark = pytest.mark.extra_artifacts(
 MAX_RUNS = 2 if platform.system() == "FreeBSD" else 1  # GL#3846
 
 
-@isctest.mark.flaky(max_runs=MAX_RUNS)
+@pytest.mark.flaky(max_runs=MAX_RUNS)
 def test_nsupdate(run_tests_sh):
     run_tests_sh()

bin/tests/system/qmin/tests_sh_qmin.py

@@ -11,8 +11,6 @@
 
 import pytest
 
-import isctest.mark
-
 pytestmark = pytest.mark.extra_artifacts(
     [
         "dig.out.*",
@@ -25,6 +23,6 @@ pytestmark = pytest.mark.extra_artifacts(
 
 
 # The qmin test is inherently unstable, see GL #904 for details.
-@isctest.mark.flaky(max_runs=3)
+@pytest.mark.flaky(max_runs=3)
 def test_qmin(run_tests_sh):
     run_tests_sh()

bin/tests/system/reclimit/tests_sh_reclimit.py

@@ -11,8 +11,6 @@
 
 import pytest
 
-import isctest.mark
-
 pytestmark = pytest.mark.extra_artifacts(
     [
         "dig.out.*",
@@ -27,6 +25,6 @@ pytestmark = pytest.mark.extra_artifacts(
 
 
 # The reclimit is known to be quite unstable. GL #1587
-@isctest.mark.flaky(max_runs=2)
+@pytest.mark.flaky(max_runs=2)
 def test_reclimit(run_tests_sh):
     run_tests_sh()

bin/tests/system/rrl/tests_sh_rrl.py

@@ -11,8 +11,6 @@
 
 import pytest
 
-import isctest.mark
-
 pytestmark = pytest.mark.extra_artifacts(
     [
         "*mdig.out*",
@@ -24,6 +22,6 @@ pytestmark = pytest.mark.extra_artifacts(
 
 
 # The rrl is known to be quite unstable. GL #172
-@isctest.mark.flaky(max_runs=2)
+@pytest.mark.flaky(max_runs=2)
 def test_rrl(run_tests_sh):
     run_tests_sh()

bin/tests/system/serve-stale/tests_sh_serve_stale.py

@@ -26,5 +26,6 @@ pytestmark = pytest.mark.extra_artifacts(
 )
 
 
+@pytest.mark.flaky(max_runs=2)
 def test_serve_stale(run_tests_sh):
     run_tests_sh()

bin/tests/system/statschannel/tests_json.py

@@ -119,6 +119,6 @@ def test_zone_with_many_keys_json(statsport):
     )
 
 
-@isctest.mark.flaky(max_runs=2, rerun_filter=isctest.mark.with_tsan)
+@pytest.mark.flaky(max_runs=2)
 def test_traffic_json(statsport):
     generic.test_traffic(fetch_traffic_json, statsip="10.53.0.2", statsport=statsport)

bin/tests/system/statschannel/tests_xml.py

@@ -148,6 +148,6 @@ def test_zone_with_many_keys_xml(statsport):
     )
 
 
-@isctest.mark.flaky(max_runs=2, rerun_filter=isctest.mark.with_tsan)
+@pytest.mark.flaky(max_runs=2)
 def test_traffic_xml(statsport):
     generic.test_traffic(fetch_traffic_xml, statsip="10.53.0.2", statsport=statsport)

bin/tests/system/timeouts/tests_tcp_timeouts.py

@@ -13,7 +13,6 @@
 
 # pylint: disable=unused-variable
 
-import platform
 import socket
 import time
 
@@ -68,11 +67,7 @@ def test_initial_timeout(named_port):
                 raise EOFError from e
 
 
-def is_host_freebsd_13(*_):
+@pytest.mark.flaky(max_runs=2, rerun_filter=isctest.mark.is_host_freebsd_13)
-    return platform.system() == "FreeBSD" and platform.release().startswith("13")
-
-
-@isctest.mark.flaky(max_runs=2, rerun_filter=is_host_freebsd_13)
 def test_idle_timeout(named_port):
     #
     # The idle timeout is 5 seconds, so the third message should fail
@@ -194,7 +189,7 @@ def test_long_axfr(named_port):
         assert soa is not None
 
 
-@isctest.mark.flaky(max_runs=3)
+@pytest.mark.flaky(max_runs=3)
 def test_send_timeout(named_port):
     with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
         sock.connect(("10.53.0.1", named_port))