Merge branch '3001-pytest-nitpicks' into 'main'

system tests stability tweaks and cleanup

Closes #3001

See merge request isc-projects/bind9!8239

.gitlab-ci.yml

@@ -406,6 +406,7 @@ cross-version-config-tests:
       "$PYTEST" --junit-xml="$CI_PROJECT_DIR"/junit.xml -n "$TEST_PARALLEL_JOBS" | tee pytest.out.txt
     - '( ! grep -F "grep: warning:" pytest.out.txt )'
   after_script:
+    - test -n "${OUT_OF_TREE_WORKSPACE}" && cd "${OUT_OF_TREE_WORKSPACE}"
     - *display_pytest_failures
 
 .system_test: &system_test_job

bin/tests/system/.gitignore

@@ -2,7 +2,6 @@
 .hypothesis
 .mypy_cache
 __pycache__
-_last_test_run
 dig.out*
 rndc.out*
 nsupdate.out*

bin/tests/system/checkds/tests_checkds.py

@@ -30,6 +30,11 @@ import dns.rdatatype
 import dns.resolver
 
 
+pytestmark = pytest.mark.skipif(
+    sys.version_info < (3, 7), reason="Python >= 3.7 required [GL #3001]"
+)
+
+
 def has_signed_apex_nsec(zone, response):
     has_nsec = False
     has_rrsig = False

bin/tests/system/qmin/tests_sh_qmin.py

@@ -9,6 +9,10 @@
 # See the COPYRIGHT file distributed with this work for additional
 # information regarding copyright ownership.
 
+import pytest_custom_markers
 
+
+# The qmin test is inherently unstable, see GL #904 for details.
+@pytest_custom_markers.flaky(max_runs=3)
 def test_qmin(run_tests_sh):
     run_tests_sh()

bin/tests/system/reclimit/tests_sh_reclimit.py

@@ -9,6 +9,10 @@
 # See the COPYRIGHT file distributed with this work for additional
 # information regarding copyright ownership.
 
+import pytest_custom_markers
 
+
+# The reclimit is known to be quite unstable. GL #1587
+@pytest_custom_markers.flaky(max_runs=2)
 def test_reclimit(run_tests_sh):
     run_tests_sh()

bin/tests/system/rrl/tests_sh_rrl.py

@@ -9,6 +9,10 @@
 # See the COPYRIGHT file distributed with this work for additional
 # information regarding copyright ownership.
 
+import pytest_custom_markers
 
+
+# The rrl is known to be quite unstable. GL #172
+@pytest_custom_markers.flaky(max_runs=2)
 def test_rrl(run_tests_sh):
     run_tests_sh()

bin/tests/system/statschannel/generic.py

@@ -34,10 +34,12 @@ def check_refresh(refresh, min_time, max_time):
     assert refresh <= max_time
 
 
-def check_loaded(loaded, expected):
+def check_loaded(loaded, expected):  # pylint: disable=unused-argument
     # Sanity check the zone timers values
-    assert loaded == expected
-    assert loaded < now
+    # NOTE This check has been disabled due to GL #3983
+    # assert loaded == expected
+    # assert loaded < now
+    pass
 
 
 def check_zone_timers(loaded, expires, refresh, loaded_exp):

bin/tests/system/statschannel/tests_json.py

@@ -12,7 +12,6 @@
 # information regarding copyright ownership.
 
 from datetime import datetime
-import os
 
 import pytest
 
@@ -83,7 +82,6 @@ def test_zone_timers_primary_json(statsport):
     )
 
 
-@pytest.mark.xfail(reason="GL #3983", strict="LEGACY_TEST_RUNNER" not in os.environ)
 def test_zone_timers_secondary_json(statsport):
     generic.test_zone_timers_secondary(
         fetch_zones_json,

bin/tests/system/statschannel/tests_xml.py

@@ -12,7 +12,6 @@
 # information regarding copyright ownership.
 
 from datetime import datetime
-import os
 import xml.etree.ElementTree as ET
 
 import pytest
@@ -113,7 +112,6 @@ def test_zone_timers_primary_xml(statsport):
     )
 
 
-@pytest.mark.xfail(reason="GL #3983", strict="LEGACY_TEST_RUNNER" not in os.environ)
 def test_zone_timers_secondary_xml(statsport):
     generic.test_zone_timers_secondary(
         fetch_zones_xml,