From 8846877dfec6b353e2939cbf4c9c963a82c2731c Mon Sep 17 00:00:00 2001
From: Tom Krizek <tkrizek@isc.org>
Date: Mon, 7 Aug 2023 14:37:40 +0200
Subject: [PATCH 1/4] ci: use pytest system test runner on OpenBSD

A supported pytest version finally made it to OpenBSD repositories,
allowing us to run system tests with the pytest runner.
---
 .gitlab-ci.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d4110b8fff..36df5e4479 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1291,7 +1291,6 @@ clang:openbsd:amd64:
 system:clang:openbsd:amd64:
   <<: *openbsd_amd64_image
   <<: *system_test_job
-  <<: *system_test_legacy
   <<: *api_schedules_triggers_web_triggering_rules
   variables:
     USER: gitlab-runner

From d49d9ec80e0f3a771c1af3d8db68b0bf2739bff3 Mon Sep 17 00:00:00 2001
From: Tom Krizek <tkrizek@isc.org>
Date: Tue, 8 Aug 2023 13:23:20 +0200
Subject: [PATCH 2/4] ci: run out-of-tree system tests with pytest runner

Out-of-tree builds are built in a directory that is different from
source directory. The build directory doesn't contain the non-compiled
test files from bin/tests/system which are the test cases required by
the pytest runner.

In order to run the system tests for out-of-tree build, copy over the
contents (tests) of bin/tests/system/ from the source directory into the
build directory. Then, it is possible to invoke the pytest runner inside
the build directory.
---
 .gitlab-ci.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 36df5e4479..4d443618ff 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -393,7 +393,7 @@ cross-version-config-tests:
   <<: *default_triggering_rules
   stage: system
   before_script:
-    - test -n "${OUT_OF_TREE_WORKSPACE}" && cd "${OUT_OF_TREE_WORKSPACE}"
+    - test -n "${OUT_OF_TREE_WORKSPACE}" && cp -r bin/tests/system/* "${OUT_OF_TREE_WORKSPACE}/bin/tests/system/" && cd "${OUT_OF_TREE_WORKSPACE}"
     - *setup_interfaces
   script:
     - *find_pytest
@@ -953,7 +953,6 @@ system:gcc:out-of-tree:
       artifacts: true
   <<: *base_image
   <<: *system_test_job
-  <<: *system_test_legacy
   <<: *api_schedules_tags_triggers_web_triggering_rules
 
 unit:gcc:out-of-tree:

From bc66bf146f22a479e2d94ed0575460a33dff7100 Mon Sep 17 00:00:00 2001
From: Tom Krizek <tkrizek@isc.org>
Date: Tue, 8 Aug 2023 13:24:58 +0200
Subject: [PATCH 3/4] ci: remove legacy system test runner template

---
 .gitlab-ci.yml | 16 ----------------
 1 file changed, 16 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 4d443618ff..8f7c6bffc2 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -404,22 +404,6 @@ cross-version-config-tests:
   after_script:
     - *display_pytest_failures
 
-.system_test_legacy: &system_test_legacy
-  script:
-    - cd bin/tests/system
-    - make -j${TEST_PARALLEL_JOBS:-1} -k check V=1
-    - if git rev-parse > /dev/null 2>&1; then ( ! grep "^I:.*:file.*not removed$" *.log ); fi
-    - '( ! grep -F "grep: warning:" *.log )'
-  after_script:
-    - test -d bind-* && cd bind-*
-    - REALSOURCEDIR="$PWD"
-    - test -n "${OUT_OF_TREE_WORKSPACE}" && cd "${OUT_OF_TREE_WORKSPACE}"
-    - cat bin/tests/system/test-suite.log
-    - find bin/tests/system -name "*dig.*" | xargs grep "error" || true
-    - *find_python
-    - >
-      "$PYTHON" "$REALSOURCEDIR"/bin/tests/convert-trs-to-junit.py . > "$CI_PROJECT_DIR"/junit.xml
-
 .system_test: &system_test_job
   <<: *system_test_common
   artifacts:

From b1a9d1fe251e09cc359966cdc90b25db2a46669b Mon Sep 17 00:00:00 2001
From: Tom Krizek <tkrizek@isc.org>
Date: Wed, 9 Aug 2023 17:49:27 +0200
Subject: [PATCH 4/4] Improve pytest message when sanitizer report is found

---
 bin/tests/system/conftest.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/tests/system/conftest.py b/bin/tests/system/conftest.py
index f8c7e9c9e5..29799266e4 100644
--- a/bin/tests/system/conftest.py
+++ b/bin/tests/system/conftest.py
@@ -587,7 +587,7 @@ else:
             try:
                 shell("get_core_dumps.sh", [system_test_dir.name])
             except subprocess.CalledProcessError as exc:
-                mlogger.error("Found core dumps")
+                mlogger.error("Found core dumps or sanitizer reports")
                 pytest.fail(f"get_core_dumps.sh exited with {exc.returncode}")
 
         os.environ.update(env)  # Ensure pytests have the same env vars as shell tests.