From 295890a16b24f740b8bc8b7c75a4ded9802cb3ff Mon Sep 17 00:00:00 2001
From: Tom Krizek <tkrizek@isc.org>
Date: Tue, 5 Sep 2023 10:29:13 +0200
Subject: [PATCH] Modify custom-test-driver to interpret JUnit results

Pytest provides JUnit output and uses different exit codes from
Automake. Use the conversion script to interpret the JUnit test results
from python rather than relying on the status code.
---
 bin/tests/system/.gitignore         |  1 +
 bin/tests/system/custom-test-driver | 22 ++++++++++------------
 2 files changed, 11 insertions(+), 12 deletions(-)

diff --git a/bin/tests/system/.gitignore b/bin/tests/system/.gitignore
index d26b4b0d3d..15eb5b4f44 100644
--- a/bin/tests/system/.gitignore
+++ b/bin/tests/system/.gitignore
@@ -13,6 +13,7 @@ named.run
 /random.data
 /*.log
 /*.trs
+/*.xml
 /resolve
 /legacy.run.sh
 /run.log
diff --git a/bin/tests/system/custom-test-driver b/bin/tests/system/custom-test-driver
index 7499aa0bc0..fed98cccc6 100755
--- a/bin/tests/system/custom-test-driver
+++ b/bin/tests/system/custom-test-driver
@@ -56,7 +56,7 @@ END
 test_name= # Used for reporting.
 log_file=  # Where to save the output of the test script.
 trs_file=  # Where to save the metadata of the test run.
-status_file= # Where to save the status of the test run.
+junit_file=  # Where to save pytest junit output.
 expect_failure=no
 color_tests=no
 enable_hard_errors=yes
@@ -67,8 +67,7 @@ while test $# -gt 0; do
   --version) echo "test-driver $scriptversion"; exit $?;;
   --test-name) test_name=$2; shift;;
   --log-file) log_file=$2; shift;;
-  --trs-file) trs_file=$2; shift;;
-  --status-file) status_file=$2; shift;;
+  --trs-file) trs_file=$2; junit_file=$(echo $trs_file | sed 's/\.trs$/\.xml/'); shift;;
   --color-tests) color_tests=$2; shift;;
   --expect-failure) expect_failure=$2; shift;;
   --enable-hard-errors) enable_hard_errors=$2; shift;;
@@ -104,22 +103,22 @@ else
   red= grn= lgn= blu= mgn= std=
 fi
 
-do_exit='rm -f $log_file $trs_file $status_file; (exit $st); exit $st'
+do_exit='rm -f $log_file $trs_file $junit_file; (exit $st); exit $st'
 trap "st=129; $do_exit" 1
 trap "st=130; $do_exit" 2
 trap "st=141; $do_exit" 13
 trap "st=143; $do_exit" 15
 
-# Set default
-test x"$status_file" = x && status_file=$(mktemp ./custom-test-runner.XXXXXX)
 # Test script is run here.
 if test $verbose = yes; then
-  ("$@" 2>&1; echo $? > "$status_file") | tee $log_file
+  "$@" --junit-xml $PWD/$junit_file 2>&1 | tee $log_file
 else
-  "$@" >$log_file 2>&1; echo $? > "$status_file"
+  "$@" --junit-xml $PWD/$junit_file >$log_file 2>&1
 fi
-read -r estatus < "$status_file"
-rm "$status_file"
+
+# Run junit to trs converter script.
+./convert-junit-to-trs.py $junit_file > $trs_file
+estatus=$?
 
 if test $enable_hard_errors = no && test $estatus -eq 99; then
   tweaked_estatus=1
@@ -145,8 +144,7 @@ echo "$res $test_name (exit status: $estatus)" >>$log_file
 # Report outcome to console.
 echo "${col}${res}${std}: $test_name"
 
-# Register the test result, and other relevant metadata.
-echo ":test-result: $res" > $trs_file
+# Register other relevant test metadata.
 echo ":global-test-result: $res" >> $trs_file
 echo ":recheck: $recheck" >> $trs_file
 echo ":copy-in-global-log: $gcopy" >> $trs_file