Merge branch 'pspacek/junit-report' into 'main'

Generate JUnit reports for unit & system tests

See merge request isc-projects/bind9!6088

.gitlab-ci.yml

@@ -323,6 +323,7 @@ stages:
     - make -j${TEST_PARALLEL_JOBS:-1} -k check V=1
     - if git rev-parse > /dev/null 2>&1; then ( ! grep "^I:.*:file.*not removed$" *.log ); fi
   after_script:
+    - (source bin/tests/system/conf.sh; $PYTHON bin/tests/convert-trs-to-junit.py . > junit.xml)
     - test -n "${OUT_OF_TREE_WORKSPACE}" && cd "${OUT_OF_TREE_WORKSPACE}"
     - test -d bind-* && cd bind-*
     - cat bin/tests/system/test-suite.log
@@ -333,7 +334,9 @@ stages:
   artifacts:
     untracked: true
     expire_in: "1 day"
-    when: on_failure
+    when: always
+    reports:
+      junit: junit.xml
 
 .system_test_gcov: &system_test_gcov_job
   <<: *system_test_common
@@ -347,10 +350,13 @@ stages:
   after_script:
     - cat bin/tests/system/test-suite.log
     - find bin -name 'tsan.*' -exec python3 util/parse_tsan.py {} \;
+    - (source bin/tests/system/conf.sh; $PYTHON bin/tests/convert-trs-to-junit.py . > junit.xml)
   artifacts:
     expire_in: "1 day"
     untracked: true
-    when: on_failure
+    when: always
+    reports:
+      junit: junit.xml
 
 .unit_test_common: &unit_test_common
   <<: *default_triggering_rules
@@ -360,6 +366,7 @@ stages:
   script:
     - make -j${TEST_PARALLEL_JOBS:-1} -k unit V=1
   after_script:
+    - (source bin/tests/system/conf.sh; $PYTHON bin/tests/convert-trs-to-junit.py . > junit.xml)
     - *save_out_of_tree_workspace
 
 .unit_test: &unit_test_job
@@ -367,7 +374,9 @@ stages:
   artifacts:
     untracked: true
     expire_in: "1 day"
-    when: on_failure
+    when: always
+    reports:
+      junit: junit.xml
 
 .unit_test_gcov: &unit_test_gcov_job
   <<: *unit_test_common
@@ -380,12 +389,16 @@ stages:
   <<: *unit_test_common
   after_script:
     - find lib -name 'tsan.*' -exec python3 util/parse_tsan.py {} \;
+    - (source bin/tests/system/conf.sh; $PYTHON bin/tests/convert-trs-to-junit.py . > junit.xml)
   artifacts:
     expire_in: "1 day"
     paths:
       - lib/*/tests/tsan.*
       - tsan/
-    when: on_failure
+      - junit.xml
+    when: always
+    reports:
+      junit: junit.xml
 
 .docs: &docs_job
   stage: docs

bin/tests/convert-trs-to-junit.py

@@ -0,0 +1,150 @@
+#!/usr/bin/env python
+#
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# Convert automake .trs files into JUnit format suitable for Gitlab
+
+import argparse
+import os
+import sys
+from xml.etree import ElementTree
+from xml.etree.ElementTree import Element
+from xml.etree.ElementTree import SubElement
+
+
+# getting explicit encoding specification right for Python 2/3 would be messy,
+# so let's hope for the best
+def read_whole_text(filename):
+    with open(filename) as inf:  # pylint: disable-msg=unspecified-encoding
+        return inf.read().strip()
+
+
+def read_trs_result(filename):
+    result = None
+    with open(filename, "r") as trs:  # pylint: disable-msg=unspecified-encoding
+        for line in trs:
+            items = line.split()
+            if len(items) < 2:
+                raise ValueError("unsupported line in trs file", filename, line)
+            if items[0] != (":test-result:"):
+                continue
+            if result is not None:
+                raise NotImplementedError("double :test-result:", filename)
+            result = items[1].upper()
+
+    if result is None:
+        raise ValueError(":test-result: not found", filename)
+
+    return result
+
+
+def find_test_relative_path(source_dir, in_path):
+    """Return {in_path}.c if it exists, with fallback to {in_path}"""
+    candidates_relative = [in_path + ".c", in_path]
+    for relative in candidates_relative:
+        absolute = os.path.join(source_dir, relative)
+        if os.path.exists(absolute):
+            return relative
+    raise KeyError
+
+
+def err_out(exception):
+    raise exception
+
+
+def walk_trss(source_dir):
+    for cur_dir, _dirs, files in os.walk(source_dir, onerror=err_out):
+        for filename in files:
+            if not filename.endswith(".trs"):
+                continue
+
+            filename_prefix = filename[: -len(".trs")]
+            log_name = filename_prefix + ".log"
+            full_trs_path = os.path.join(cur_dir, filename)
+            full_log_path = os.path.join(cur_dir, log_name)
+            sub_dir = os.path.relpath(cur_dir, source_dir)
+            test_name = os.path.join(sub_dir, filename_prefix)
+
+            t = {
+                "name": test_name,
+                "full_log_path": full_log_path,
+                "rel_log_path": os.path.relpath(full_log_path, source_dir),
+            }
+            t["result"] = read_trs_result(full_trs_path)
+
+            # try to find dir/file path for a clickable link
+            try:
+                t["rel_file_path"] = find_test_relative_path(
+                    source_dir, test_name
+                )
+            except KeyError:
+                pass  # no existing path found
+
+            yield t
+
+
+def append_testcase(testsuite, t):
+    # attributes taken from
+    # https://gitlab.com/gitlab-org/gitlab-foss/-/blob/master/lib/gitlab/ci/parsers/test/junit.rb
+    attrs = {"name": t["name"]}
+    if "rel_file_path" in t:
+        attrs["file"] = t["rel_file_path"]
+
+    testcase = SubElement(testsuite, "testcase", attrs)
+
+    # Gitlab accepts only [[ATTACHMENT| links for system-out, not raw text
+    s = SubElement(testcase, "system-out")
+    s.text = "[[ATTACHMENT|" + t["rel_log_path"] + "]]"
+    if t["result"].lower() == "pass":
+        return
+
+    # Gitlab shows output only for failed or skipped tests
+    if t["result"].lower() == "skip":
+        err = SubElement(testcase, "skipped")
+    else:
+        err = SubElement(testcase, "failure")
+    err.text = read_whole_text(t["full_log_path"])
+
+
+def gen_junit(results):
+    testsuites = Element("testsuites")
+    testsuite = SubElement(testsuites, "testsuite")
+    for test in results:
+        append_testcase(testsuite, test)
+    return testsuites
+
+
+def check_directory(path):
+    try:
+        os.listdir(path)
+        return path
+    except OSError as ex:
+        msg = "Path {} cannot be listed as a directory: {}".format(path, ex)
+        raise argparse.ArgumentTypeError(msg)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Recursively search for .trs + .log files and compile "
+        "them into JUnit XML suitable for Gitlab. Paths in the "
+        "XML are relative to the specified top directory."
+    )
+    parser.add_argument(
+        "top_directory",
+        type=check_directory,
+        help="root directory where to start scanning for .trs files",
+    )
+    args = parser.parse_args()
+    junit = gen_junit(walk_trss(args.top_directory))
+
+    # encode results into file format, on Python 3 it produces bytes
+    xml = ElementTree.tostring(junit, "utf-8")
+    # use stdout as a binary file object, Python2/3 compatibility
+    output = getattr(sys.stdout, "buffer", sys.stdout)
+    output.write(xml)
+
+
+if __name__ == "__main__":
+    main()