Merge branch 'mnowak/try-harder-to-analyze-cores' into 'main'

Rationalize backtrace logging, fail on core file presence

See merge request isc-projects/bind9!3867

bin/tests/system/run.sh.in

@@ -131,17 +131,13 @@ start_servers() {
     fi
 }
 
-stop_servers_failed() {
-    echoinfo "I:$systest:stopping servers failed"
-    echofail "R:$systest:FAIL"
-    echoend  "E:$systest:$(date_with_args)"
-    exit 1
-}
-
 stop_servers() {
     if $stopservers; then
         echoinfo "I:$systest:stopping servers"
-        $PERL stop.pl "$systest" || stop_servers_failed
+        if ! $PERL stop.pl "$systest"; then
+            echoinfo "I:$systest:stopping servers failed"
+            return 1
+        fi
     fi
 }
 
@@ -217,7 +213,7 @@ if [ -r "$systest/tests.sh" ]; then
     ( cd "$systest" && $SHELL tests.sh "$@" )
     status=$?
     run=$((run+1))
-    stop_servers
+    stop_servers || status=1
 fi
 
 if [ -n "$PYTEST" ]; then
@@ -232,7 +228,7 @@ if [ -n "$PYTEST" ]; then
 	    test_status=$(cat "$systest/$test.status")
 	fi
 	status=$((status+test_status))
-	stop_servers
+	stop_servers || status=1
     done
 else
     echoinfo "I:$systest:pytest not installed, skipping python tests"
@@ -250,54 +246,64 @@ else
     exit $status
 fi
 
-if [ $status != 0 ]; then
+get_core_dumps() {
+    find "$systest/" \( -name 'core*' -or -name '*.core' \) ! -name '*.gz' ! -name '*.txt' | sort
+}
+
+core_dumps=$(get_core_dumps | tr '\n' ' ')
+assertion_failures=$(find "$systest/" -name named.run -print0 | xargs -0 grep "assertion failure" | wc -l)
+sanitizer_summaries=$(find "$systest/" -name 'tsan.*' | wc -l)
+if [ -n "$core_dumps" ]; then
+    status=1
+    echoinfo "I:$systest:Core dump(s) found: $core_dumps"
+    get_core_dumps | while read -r coredump; do
+        export SYSTESTDIR="$systest"
+        echoinfo "D:$systest:backtrace from $coredump:"
+        echoinfo "D:$systest:--------------------------------------------------------------------------------"
+        binary=$(gdb --batch --core="$coredump" 2>/dev/null | sed -ne "s/Core was generated by \`//;s/ .*'.$//p;")
+        "${top_builddir}/libtool" --mode=execute gdb \
+                                  -batch \
+                                  -ex bt \
+                                  -core="$coredump" \
+                                  -- \
+                                  "$binary" 2>/dev/null | sed -n '/^Core was generated by/,$p' | cat_d
+        echoinfo "D:$systest:--------------------------------------------------------------------------------"
+        coredump_backtrace=$(basename "${coredump}")-backtrace.txt
+        echoinfo "D:$systest:full backtrace from $coredump saved in $coredump_backtrace"
+        "${top_builddir}/libtool" --mode=execute gdb \
+                                  -batch \
+                                  -command=run.gdb \
+                                  -core="$coredump" \
+                                  -- \
+                                  "$binary" > "$coredump_backtrace" 2>&1
+        echoinfo "D:$systest:core dump $coredump archived as $coredump.gz"
+        gzip -1 "${coredump}"
+    done
+elif [ "$assertion_failures" -ne 0 ]; then
+    status=1
+    SYSTESTDIR="$systest"
+    echoinfo "I:$systest:$assertion_failures assertion failure(s) found"
+    find "$systest/" -name 'tsan.*' -print0 | xargs -0 grep "SUMMARY: " | sort -u | cat_d
+elif [ "$sanitizer_summaries" -ne 0 ]; then
+    status=1
+    echoinfo "I:$systest:$sanitizer_summaries sanitizer report(s) found"
+fi
+
+if [ $status -ne 0 ]; then
     echofail "R:$systest:FAIL"
-    # Do not clean up - we need the evidence.
 else
-    core_dumps="$(find "$systest/" -name 'core*' -or -name '*.core' | sort | tr '\n' ' ')"
-    assertion_failures=$(find "$systest/" -name named.run -print0 | xargs -0 grep "assertion failure" | wc -l)
-    sanitizer_summaries=$(find "$systest/" -name 'tsan.*' | wc -l)
-    if [ -n "$core_dumps" ]; then
-        echoinfo "I:$systest:Test claims success despite crashes: $core_dumps"
-        echofail "R:$systest:FAIL"
-        # Do not clean up - we need the evidence.
-	find "$systest/" -name 'core*' -or -name '*.core' | while read -r coredump; do
-		export SYSTESTDIR="$systest"
-		echoinfo "D:$systest:backtrace from $coredump start"
-		binary=$(gdb --batch --core="$coredump" | sed -ne "s/Core was generated by \`//;s/ .*'.$//p;")
-		"${top_builddir}/libtool" --mode=execute gdb \
-					  --batch \
-					  --command=run.gdb \
-					  --core="$coredump" \
-					  -- \
-					  "$binary"
-		echoinfo "D:$systest:backtrace from $coredump end"
-	done
-    elif [ "$assertion_failures" -ne 0 ]; then
-	SYSTESTDIR="$systest"
-        echoinfo "I:$systest:Test claims success despite $assertion_failures assertion failure(s)"
-	find "$systest/" -name 'tsan.*' -print0 | xargs -0 grep "SUMMARY: " | sort -u | cat_d
-        echofail "R:$systest:FAIL"
-        # Do not clean up - we need the evidence.
-    elif [ "$sanitizer_summaries" -ne 0 ]; then
-        echoinfo "I:$systest:Test claims success despite $sanitizer_summaries sanitizer reports(s)"
-        echofail "R:$systest:FAIL"
-    else
-        echopass "R:$systest:PASS"
-        if $clean
-        then
-            ( cd "${systest}" && $SHELL clean.sh "$@" )
-	    if [ "${srcdir}" != "${builddir}" ]; then
-		rm -rf "./${systest}" ## FIXME (this also removes compiled binaries)
-	    fi
-            if test -d ${srcdir}/../../../.git
-            then
-                git status -su --ignored "${systest}" 2>/dev/null | \
-                sed -n -e 's|^?? \(.*\)|I:file \1 not removed|p' \
-                -e 's|^!! \(.*/named.run\)$|I:file \1 not removed|p' \
-                -e 's|^!! \(.*/named.memstats\)$|I:file \1 not removed|p'
-            fi
-        fi
+    echopass "R:$systest:PASS"
+    if $clean; then
+       ( cd "${systest}" && $SHELL clean.sh "$@" )
+       if [ "${srcdir}" != "${builddir}" ]; then
+           rm -rf "./${systest}" ## FIXME (this also removes compiled binaries)
+       fi
+       if test -d ${srcdir}/../../../.git; then
+           git status -su --ignored "${systest}" 2>/dev/null | \
+           sed -n -e 's|^?? \(.*\)|I:file \1 not removed|p' \
+           -e 's|^!! \(.*/named.run\)$|I:file \1 not removed|p' \
+           -e 's|^!! \(.*/named.memstats\)$|I:file \1 not removed|p'
+       fi
     fi
 fi