Generate JUnit reports for unit & system tests

This allows Gitlab to show nice summary for individual tests/test
directories and to expose the results in Gitlab API for consumption
elsewhere.

A catch: As of Gitlab 14.7.7, the detailed results are stored
only in artifacts and thus expire. All consumers (including API) need
to be "fast enough" to get the data before they disappear.
This also forces us to always store the artifacts intead of storing them
only on failure.

.gitlab-ci.yml

@@ -323,6 +323,7 @@ stages:
     - make -j${TEST_PARALLEL_JOBS:-1} -k check V=1
     - if git rev-parse > /dev/null 2>&1; then ( ! grep "^I:.*:file.*not removed$" *.log ); fi
   after_script:
+    - (source bin/tests/system/conf.sh; $PYTHON bin/tests/convert-trs-to-junit.py . > junit.xml)
     - test -n "${OUT_OF_TREE_WORKSPACE}" && cd "${OUT_OF_TREE_WORKSPACE}"
     - test -d bind-* && cd bind-*
     - cat bin/tests/system/test-suite.log
@@ -333,7 +334,9 @@ stages:
   artifacts:
     untracked: true
     expire_in: "1 day"
-    when: on_failure
+    when: always
+    reports:
+      junit: junit.xml
 
 .system_test_gcov: &system_test_gcov_job
   <<: *system_test_common
@@ -347,10 +350,13 @@ stages:
   after_script:
     - cat bin/tests/system/test-suite.log
     - find bin -name 'tsan.*' -exec python3 util/parse_tsan.py {} \;
+    - (source bin/tests/system/conf.sh; $PYTHON bin/tests/convert-trs-to-junit.py . > junit.xml)
   artifacts:
     expire_in: "1 day"
     untracked: true
-    when: on_failure
+    when: always
+    reports:
+      junit: junit.xml
 
 .unit_test_common: &unit_test_common
   <<: *default_triggering_rules
@@ -360,6 +366,7 @@ stages:
   script:
     - make -j${TEST_PARALLEL_JOBS:-1} -k unit V=1
   after_script:
+    - (source bin/tests/system/conf.sh; $PYTHON bin/tests/convert-trs-to-junit.py . > junit.xml)
     - *save_out_of_tree_workspace
 
 .unit_test: &unit_test_job
@@ -367,7 +374,9 @@ stages:
   artifacts:
     untracked: true
     expire_in: "1 day"
-    when: on_failure
+    when: always
+    reports:
+      junit: junit.xml
 
 .unit_test_gcov: &unit_test_gcov_job
   <<: *unit_test_common
@@ -380,12 +389,16 @@ stages:
   <<: *unit_test_common
   after_script:
     - find lib -name 'tsan.*' -exec python3 util/parse_tsan.py {} \;
+    - (source bin/tests/system/conf.sh; $PYTHON bin/tests/convert-trs-to-junit.py . > junit.xml)
   artifacts:
     expire_in: "1 day"
     paths:
       - lib/*/tests/tsan.*
       - tsan/
-    when: on_failure
+      - junit.xml
+    when: always
+    reports:
+      junit: junit.xml
 
 .docs: &docs_job
   stage: docs

bin/tests/convert-trs-to-junit.py

@@ -0,0 +1,150 @@
+#!/usr/bin/env python
+#
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# Convert automake .trs files into JUnit format suitable for Gitlab
+
+import argparse
+import os
+import sys
+from xml.etree import ElementTree
+from xml.etree.ElementTree import Element
+from xml.etree.ElementTree import SubElement
+
+
+# getting explicit encoding specification right for Python 2/3 would be messy,
+# so let's hope for the best
+def read_whole_text(filename):
+    with open(filename) as inf:  # pylint: disable-msg=unspecified-encoding
+        return inf.read().strip()
+
+
+def read_trs_result(filename):
+    result = None
+    with open(filename, "r") as trs:  # pylint: disable-msg=unspecified-encoding
+        for line in trs:
+            items = line.split()
+            if len(items) < 2:
+                raise ValueError("unsupported line in trs file", filename, line)
+            if items[0] != (":test-result:"):
+                continue
+            if result is not None:
+                raise NotImplementedError("double :test-result:", filename)
+            result = items[1].upper()
+
+    if result is None:
+        raise ValueError(":test-result: not found", filename)
+
+    return result
+
+
+def find_test_relative_path(source_dir, in_path):
+    """Return {in_path}.c if it exists, with fallback to {in_path}"""
+    candidates_relative = [in_path + ".c", in_path]
+    for relative in candidates_relative:
+        absolute = os.path.join(source_dir, relative)
+        if os.path.exists(absolute):
+            return relative
+    raise KeyError
+
+
+def err_out(exception):
+    raise exception
+
+
+def walk_trss(source_dir):
+    for cur_dir, _dirs, files in os.walk(source_dir, onerror=err_out):
+        for filename in files:
+            if not filename.endswith(".trs"):
+                continue
+
+            filename_prefix = filename[: -len(".trs")]
+            log_name = filename_prefix + ".log"
+            full_trs_path = os.path.join(cur_dir, filename)
+            full_log_path = os.path.join(cur_dir, log_name)
+            sub_dir = os.path.relpath(cur_dir, source_dir)
+            test_name = os.path.join(sub_dir, filename_prefix)
+
+            t = {
+                "name": test_name,
+                "full_log_path": full_log_path,
+                "rel_log_path": os.path.relpath(full_log_path, source_dir),
+            }
+            t["result"] = read_trs_result(full_trs_path)
+
+            # try to find dir/file path for a clickable link
+            try:
+                t["rel_file_path"] = find_test_relative_path(
+                    source_dir, test_name
+                )
+            except KeyError:
+                pass  # no existing path found
+
+            yield t
+
+
+def append_testcase(testsuite, t):
+    # attributes taken from
+    # https://gitlab.com/gitlab-org/gitlab-foss/-/blob/master/lib/gitlab/ci/parsers/test/junit.rb
+    attrs = {"name": t["name"]}
+    if "rel_file_path" in t:
+        attrs["file"] = t["rel_file_path"]
+
+    testcase = SubElement(testsuite, "testcase", attrs)
+
+    # Gitlab accepts only [[ATTACHMENT| links for system-out, not raw text
+    s = SubElement(testcase, "system-out")
+    s.text = "[[ATTACHMENT|" + t["rel_log_path"] + "]]"
+    if t["result"].lower() == "pass":
+        return
+
+    # Gitlab shows output only for failed or skipped tests
+    if t["result"].lower() == "skip":
+        err = SubElement(testcase, "skipped")
+    else:
+        err = SubElement(testcase, "failure")
+    err.text = read_whole_text(t["full_log_path"])
+
+
+def gen_junit(results):
+    testsuites = Element("testsuites")
+    testsuite = SubElement(testsuites, "testsuite")
+    for test in results:
+        append_testcase(testsuite, test)
+    return testsuites
+
+
+def check_directory(path):
+    try:
+        os.listdir(path)
+        return path
+    except OSError as ex:
+        msg = "Path {} cannot be listed as a directory: {}".format(path, ex)
+        raise argparse.ArgumentTypeError(msg)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Recursively search for .trs + .log files and compile "
+        "them into JUnit XML suitable for Gitlab. Paths in the "
+        "XML are relative to the specified top directory."
+    )
+    parser.add_argument(
+        "top_directory",
+        type=check_directory,
+        help="root directory where to start scanning for .trs files",
+    )
+    args = parser.parse_args()
+    junit = gen_junit(walk_trss(args.top_directory))
+
+    # encode results into file format, on Python 3 it produces bytes
+    xml = ElementTree.tostring(junit, "utf-8")
+    # use stdout as a binary file object, Python2/3 compatibility
+    output = getattr(sys.stdout, "buffer", sys.stdout)
+    output.write(xml)
+
+
+if __name__ == "__main__":
+    main()