diff --git a/bin/tests/system/stop.pl b/bin/tests/system/stop.pl
index 05093ceb2d..0008162e7c 100644
--- a/bin/tests/system/stop.pl
+++ b/bin/tests/system/stop.pl
@@ -236,7 +236,12 @@ sub clean_pid_file {
 
 	return unless defined($pid);
 
-	return if (send_signal(0, $pid) == 0);
+	# If we're here, the PID file hasn't been cleaned up yet
+	if (send_signal(0, $pid) == 0) {
+		print "I:$test:$server crashed on shutdown\n";
+		$errors = 1;
+		return;
+	}
 
 	return $server;
 }