[9.20] fix: ci: Ensure that junit.xml is present and non-empty after each system/unit test job

Previously, JUnit files were not generated or were generated empty for various reasons for some system/unit test runs. Now, the number of tests collected for a MR is up from about 4k to 5.8k in the "Tests" tab of a pipeline. Additionally, there is a check that ensures that [a somewhat sane](c5a271eb8b) `junit.xml` file is generated after every system/unit test job and fails the job otherwise. Closes #5316 Backport of MR !10556 Merge branch 'backport-5316-ensure-junit-xml-9.20' into 'bind-9.20' See merge request isc-projects/bind9!10649
2025-08-31 14:35:26 +00:00 · 2025-06-26 15:37:05 +00:00
parent bad04f74a5 d8e6609aaa
commit abe7547799
2 changed files with 60 additions and 56 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -334,6 +334,21 @@ stages:
 .fips-feature-test: &fips_feature_test
    - if bin/tests/system/feature-test --have-fips-mode; then fips-mode-setup --check; fips-mode-setup --is-enabled; fi

+.check_for_junit_xml: &check_for_junit_xml
+    # test if junit.xml file exists and is longer 40 bytes
+    # (i.e., contains more than `<testsuites><testsuite /></testsuites>`)
+    - if [ -f "$CI_PROJECT_DIR"/junit.xml ]; then
+        if [ $(wc -c < "$CI_PROJECT_DIR"/junit.xml) -gt 40 ]; then
+          echo "junit.xml file exists and is longer than 40 bytes.";
+        else
+          echo "junit.xml file exists but is too short.";
+          exit 1;
+        fi
+      else
+        echo "junit.xml file does not exist.";
+        exit 1;
+      fi
+
 .build: &build_job
  <<: *default_triggering_rules
  stage: build
@@ -415,28 +430,34 @@ stages:
      artifacts: true
  timeout: 2h

-.system_test_common: &system_test_common
+.system_test_common: &system_test_job
  <<: *default_triggering_rules
  stage: system
  before_script:
    - test -n "${OUT_OF_TREE_WORKSPACE}" && cp -r bin/tests/system/* "${OUT_OF_TREE_WORKSPACE}/bin/tests/system/" && cd "${OUT_OF_TREE_WORKSPACE}"
    - *setup_interfaces
+  # This script needs to: 1) fail if the system tests fail, 2) fail if
+  # the junit.xml file is broken, 3) produce the junit.xml file even if
+  # the system tests fail.  Therefore, $RET is used to "cache" the
+  # result of running pytest as interrupting the script immediately when
+  # system tests fail would make checking the contents of the junit.xml
+  # file impossible (GitLab Runner uses "set -o pipefail").
  script:
    - *fips_feature_test
    - *find_pytest
    - *find_python
    - ( if [ "${CI_DISPOSABLE_ENVIRONMENT}" = "true" ]; then sleep 3000; "$PYTHON" "${CI_PROJECT_DIR}/util/get-running-system-tests.py"; fi ) &
    - cd bin/tests/system
+    - RET=0
    - >
-      "$PYTEST" --junit-xml="$CI_PROJECT_DIR"/junit.xml -n "$TEST_PARALLEL_JOBS" | tee pytest.out.txt
+      ("$PYTEST" --junit-xml="$CI_PROJECT_DIR"/junit.xml -n "$TEST_PARALLEL_JOBS" | tee pytest.out.txt) || RET=1
+    - *check_for_junit_xml
+    - (exit $RET)
    - '( ! grep -F "grep: warning:" pytest.out.txt )'
    - test "$CLEAN_BUILD_ARTIFACTS_ON_SUCCESS" -eq 0 || ( cd ../../.. && make clean >/dev/null 2>&1 )
  after_script:
    - test -n "${OUT_OF_TREE_WORKSPACE}" && cd "${OUT_OF_TREE_WORKSPACE}"
    - *display_pytest_failures
-
-.system_test: &system_test_job
-  <<: *system_test_common
  artifacts:
    untracked: true
    exclude:
@@ -446,81 +467,65 @@ stages:
      junit: junit.xml

 .system_test_make_check: &system_test_make_check_job
-  <<: *system_test_common
+  <<: *system_test_job
+  # This script needs to: 1) fail if the system tests fail, 2) fail if
+  # the junit.xml file is broken, 3) produce the junit.xml file even if
+  # the system tests fail.  Therefore, $RET is used to "cache" the
+  # result of "make check" as interrupting the script immediately when
+  # system tests fail would prevent the junit.xml file from being
+  # produced.
  script:
    - cd bin/tests/system
-    - make -j${TEST_PARALLEL_JOBS:-1} check
+    - RET=0
+    - make -j${TEST_PARALLEL_JOBS:-1} check || RET=$?
+    - cd "$CI_PROJECT_DIR"
+    - *find_python
+    - >
+      "$PYTHON" bin/tests/convert-trs-to-junit.py . > "$CI_PROJECT_DIR"/junit.xml
+    - *check_for_junit_xml
+    - (exit $RET)
  after_script:
    - cat bin/tests/system/test-suite.log || true

-.system_test_gcov: &system_test_gcov_job
-  <<: *system_test_common
-  artifacts:
-    untracked: true
-    exclude:
-      - "**/__pycache__/**/*"
-    when: always
-
 .system_test_tsan: &system_test_tsan_job
-  <<: *system_test_common
+  <<: *system_test_job
  after_script:
    - *display_pytest_failures
    - find bin/tests/system -name "*dig.*" | xargs grep "error" || true
    - *find_python
    - *parse_tsan
-    - >
-      "$PYTHON" bin/tests/convert-trs-to-junit.py . > "$CI_PROJECT_DIR"/junit.xml
-  artifacts:
-    untracked: true
-    exclude:
-      - "**/__pycache__/**/*"
-    when: always
-    reports:
-      junit: junit.xml

-.unit_test_common: &unit_test_common
+.unit_test_common: &unit_test_job
  <<: *default_triggering_rules
  stage: unit
  before_script:
    - test -n "${OUT_OF_TREE_WORKSPACE}" && cd "${OUT_OF_TREE_WORKSPACE}"
+  # This script needs to: 1) fail if the unit tests fail, 2) fail if the
+  # junit.xml file is broken, 3) produce the junit.xml file even if the
+  # unit tests fail.  Therefore, $RET is used to "cache" the result of
+  # "make unit" as interrupting the script immediately when unit tests
+  # fail would prevent the junit.xml file from being produced.
  script:
    - *fips_feature_test
-    - make -j${TEST_PARALLEL_JOBS:-1} -k unit V=1
-    - test "$CLEAN_BUILD_ARTIFACTS_ON_SUCCESS" -eq 0 || make clean >/dev/null 2>&1
-  after_script:
-    - test -d bind-* && cd bind-*
-    - REALSOURCEDIR="$PWD"
-    - test -n "${OUT_OF_TREE_WORKSPACE}" && cd "${OUT_OF_TREE_WORKSPACE}"
+    - RET=0
+    - make -j${TEST_PARALLEL_JOBS:-1} -k unit V=1 || RET=$?
    - *find_python
    - >
-      "$PYTHON" "$REALSOURCEDIR"/bin/tests/convert-trs-to-junit.py . > "$CI_PROJECT_DIR"/junit.xml
-
-.unit_test: &unit_test_job
-  <<: *unit_test_common
+      "$PYTHON" "$CI_PROJECT_DIR"/bin/tests/convert-trs-to-junit.py . > "$CI_PROJECT_DIR"/junit.xml
+    - *check_for_junit_xml
+    - (exit $RET)
+    - test "$CLEAN_BUILD_ARTIFACTS_ON_SUCCESS" -eq 0 || make clean >/dev/null 2>&1
  artifacts:
    untracked: true
    when: always
    reports:
      junit: junit.xml

-.unit_test_gcov: &unit_test_gcov_job
-  <<: *unit_test_common
-  artifacts:
-    untracked: true
-    when: always
-
 .unit_test_tsan: &unit_test_tsan_job
-  <<: *unit_test_common
+  <<: *unit_test_job
  after_script:
    - *find_python
    - *parse_tsan
-    - >
-      "$PYTHON" bin/tests/convert-trs-to-junit.py . > "$CI_PROJECT_DIR"/junit.xml
-  artifacts:
-    untracked: true
-    when: always
-    reports:
-      junit: junit.xml

 .docs: &docs_job
  variables:
@@ -991,7 +996,7 @@ gcc:bookworm:amd64:

 system:gcc:bookworm:amd64:
  <<: *debian_bookworm_amd64_image
-  <<: *system_test_gcov_job
+  <<: *system_test_job
  variables:
    CI_ENABLE_ALL_TESTS: 1
    CLEAN_BUILD_ARTIFACTS_ON_SUCCESS: 0
@@ -1002,7 +1007,7 @@ system:gcc:bookworm:amd64:

 unit:gcc:bookworm:amd64:
  <<: *debian_bookworm_amd64_image
-  <<: *unit_test_gcov_job
+  <<: *unit_test_job
  variables:
    CI_ENABLE_ALL_TESTS: 1
    CLEAN_BUILD_ARTIFACTS_ON_SUCCESS: 0
@@ -1224,7 +1229,6 @@ gcc:tumbleweed:amd64:

 system:gcc:tumbleweed:amd64:
  <<: *tumbleweed_latest_amd64_image
-  <<: *system_test_job
  <<: *system_test_make_check_job
  needs:
    - job: gcc:tumbleweed:amd64
--- a/bin/tests/convert-trs-to-junit.py
+++ b/bin/tests/convert-trs-to-junit.py
@@ -28,14 +28,14 @@ def read_trs_result(filename):
            items = line.split()
            if len(items) < 2:
                raise ValueError("unsupported line in trs file", filename, line)
-            if items[0] != (":test-result:"):
+            if items[0] != (":global-test-result:"):
                continue
            if result is not None:
-                raise NotImplementedError("double :test-result:", filename)
+                raise NotImplementedError("double :global-test-result:", filename)
            result = items[1].upper()

    if result is None:
-        raise ValueError(":test-result: not found", filename)
+        raise ValueError(":global-test-result: not found", filename)

    return result