Gather debug info on broken unit tests

.gitlab-ci.yml

@@ -23,6 +23,9 @@ variables:
 
   CFLAGS_COMMON: -fno-omit-frame-pointer -fno-optimize-sibling-calls -O1 -g -Wall -Wextra 
 
+  # Pass run-time flags to AddressSanitizer to get core dumps on error.
+  ASAN_OPTIONS_COMMON: abort_on_error=1:disable_coredump=0:unmap_shadow_on_exit=1
+
   TARBALL_COMPRESSOR: xz
   TARBALL_EXTENSION: xz
 
@@ -805,6 +808,8 @@ asan:sid:amd64:
   <<: *build_job
 
 system:asan:sid:amd64:
+  variables:
+    ASAN_OPTIONS: ${ASAN_OPTIONS_COMMON}
   <<: *debian_sid_amd64_image
   <<: *system_test_job
   dependencies:
@@ -812,6 +817,8 @@ system:asan:sid:amd64:
   needs: ["asan:sid:amd64"]
 
 unit:asan:sid:amd64:
+  variables:
+    ASAN_OPTIONS: ${ASAN_OPTIONS_COMMON}
   <<: *debian_sid_amd64_image
   <<: *unit_test_job
   dependencies:

unit/gdb

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+# `kyua debug` command does not work with libtool (see
+# https://github.com/jmmv/kyua/issues/207).  On some distributions `kyua debug`
+# runs the first `gdb` it finds in $PATH, but on Debian and Ubuntu it looks for
+# `/usr/bin/gdb`.  This script expects `gdb` to be moved to `gdb.orig` and
+# executed from there.
+coredump="$6"
+binary=$(gdb.orig --batch --core="${coredump}" 2>/dev/null | sed -ne "s/Core was generated by \`\(.*\)'./\1/p")
+# GDB 6.3 from OpenBSD 6.6 does not tell the full path of the broken binary.
+# We need to fix it. Either the binary or it's libtool script will do.
+if [ ! -e "${binary}" ]; then
+       binary="$(find "${TOP}" -name "${binary}" | head -n 1)"
+fi
+
+# $TOP points to BIND sources and should be set on `kyua debug` invocation.
+"${TOP}/libtool" --mode=execute gdb.orig \
+	--batch \
+	--command="${TOP}/bin/tests/system/run.gdb" \
+	--core="${coredump}" \
+	-- \
+	"${binary}"

unit/unittest.sh.in

@@ -1,8 +1,16 @@
 #!/bin/sh
+
+# Find the top of the BIND9 tree.
+export TOP=@abs_top_builddir@
 KYUA=@KYUA@
 UNITTESTS=@UNITTESTS@
 CMOCKA_MESSAGE_OUTPUT=TAP
 export CMOCKA_MESSAGE_OUTPUT
+GDB="$(command -v gdb)"
+
+kyua_report() {
+	${KYUA} --logfile /dev/null report --results-file "${KYUA_RESULT:-LATEST}"
+}
 
 status=0
 if [ -n "${UNITTESTS}" ] && [ -f Kyuafile ]
@@ -13,14 +21,50 @@ then
 	${KYUA} -v parallelism="${TEST_PARALLEL_JOBS:-1}" --logfile kyua.log --loglevel debug test --results-file "${KYUA_RESULT:-NEW}"
 	status=$?
 
-	${KYUA} --logfile /dev/null report --results-file "${KYUA_RESULT:-LATEST}"
+	kyua_report
 
-	if [ "${status}" -eq "0" ]
+	if command -v sysctl >/dev/null; then
+		if [ "$(uname -s)" = "Linux" ] && [ "$(sysctl -n kernel.core_uses_pid)" -ne 1 ]; then
+			echo "kernel.core_uses_pid is not set on the Linux host"
+			echo "kyua may not find core file of broken tests"
+		fi
+	else
+		echo "sysctl command is not present, can't check kernel.core_uses_pid."
+		echo "kyua may not find core file of broken tests"
+	fi
+
+	# Use kyua-debug(1) facility to gather additional data on failed tests.
+	# Some runs will just show verbose information from the run, some will
+	# show backtrace via gdb(1).
+	broken_tests="$(kyua_report | awk '/Broken tests/ { flag=1; next } /Summary/ { flag=0 } flag' | awk '{ print $1 }')"
+	if [ -n "${CI}" ] && [ "$(id -u)" -eq 0 ] && [ -n "${broken_tests}" ] && [ -n "${GDB}" ]; then
+		if grep '^#define USE_LIBTOOL 1$' "${TOP}/config.h" >/dev/null; then
+			# kyua debug command misidentifies broken binary when libtool is used
+			# to configure BIND (see https://github.com/jmmv/kyua/issues/207).
+			# Here we try "trick" kyua use our custom gdb script instead
+			# of using gdb(1) directly. That's why this part needs to be run as root
+			# and, for safety reasons, only in the CI.
+			mv "${GDB}" "${GDB}.orig"
+			cp "${TOP}/unit/gdb" "${GDB}"
+			for test in ${broken_tests}; do
+				echo
+				${KYUA} debug "${test}"
+			done
+			mv "${GDB}.orig" "${GDB}"
+		else
+			for test in ${broken_tests}; do
+				echo
+				${KYUA} debug "${test}"
+			done
+		fi
+	fi
+
+	if [ "${status}" -eq 0 ]
 	then
 		rm -f kyua.log
 		echo "R:PASS"
 	else
-		echo "R:FAIL"
+		echo "R:FAIL:status:${status}"
 	fi
 	echo "E:unit:$(date)"
 fi

util/copyrights

@@ -2560,6 +2560,7 @@
 ./make/rules.in					MAKE	1998,1999,2000,2001,2002,2003,2004,2005,2006,2007,2008,2009,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020
 ./mkinstalldirs					X	1996,2018,2019,2020
 ./unit/README					X	2011,2013,2018,2019,2020
+./unit/gdb					X	2020
 ./unit/unittest.sh.in				X	2011,2012,2015,2018,2019,2020
 ./util/COPYRIGHT				X	1996,1997,1998,1999,2000,2001,2004,2007,2016,2018,2019,2020
 ./util/COPYRIGHT.BRIEF				X	1996,1997,1998,1999,2000,2001,2004,2016,2018,2019,2020