Enable ThreadSanitizer enabled build, system and unit tests

.gitignore

@@ -60,3 +60,4 @@ timestamp
 /compile_commands.json
 /cppcheck_html/
 /cppcheck.results
+/tsan

.gitlab-ci.yml

@@ -18,6 +18,8 @@ variables:
   MAKE: make
   CONFIGURE: ./configure
   SCAN_BUILD: scan-build-9
+  SYMBOLIZER: /usr/lib/llvm-9/bin/llvm-symbolizer
+  ASAN_SYMBOLIZER_PATH: "$SYMBOLIZER"
 
   CFLAGS_COMMON: -fno-omit-frame-pointer -fno-optimize-sibling-calls -O1 -g -Wall -Wextra 
 
@@ -180,7 +182,7 @@ stages:
     --without-make-clean \
     $EXTRA_CONFIGURE \
     || cat config.log
-    
+
 .build: &build_job
   <<: *default_triggering_rules
   stage: build
@@ -816,6 +818,63 @@ unit:asan:sid:amd64:
     - asan:sid:amd64
   needs: ["asan:sid:amd64"]
 
+# Jobs for GCC builds with TSAN enabled on Debian Sid (amd64)
+
+tsan:buster:amd64:
+  <<: *debian_buster_amd64_image
+  <<: *build_job
+  variables:
+    CC: clang-9
+    CFLAGS: "${CFLAGS_COMMON} -fsanitize=thread -DISC_MEM_USE_INTERNAL_MALLOC=0"
+    LDFLAGS: "-fsanitize=thread"
+    EXTRA_CONFIGURE: "--with-libidn2 --enable-pthread-rwlock"
+
+system:tsan:buster:amd64:
+  variables:
+    TSAN_OPTIONS: "second_deadlock_stack=1 history_size=7 log_exe_name=true log_path=tsan external_symbolizer_path=$SYMBOLIZER exitcode=0"
+  before_script:
+    - *setup_interfaces
+    - echo $TSAN_OPTIONS
+  <<: *debian_buster_amd64_image
+  <<: *system_test_job
+  dependencies:
+    - tsan:buster:amd64
+  needs: ["tsan:buster:amd64"]
+  allow_failure: true
+  after_script:
+    - find bin -name 'tsan.*' -exec python3 util/parse_tsan.py {} \;
+  artifacts:
+    expire_in: "1 week"
+    paths:
+      - bin/tests/system/*/tsan.*
+      - bin/tests/system/*/*/tsan.*
+      - tsan/
+    when: on_failure
+
+unit:tsan:buster:amd64:
+  variables:
+    TSAN_OPTIONS: "second_deadlock_stack=1 history_size=7 log_exe_name=true log_path=tsan external_symbolizer_path=$SYMBOLIZER"
+  before_script:
+    - echo $TSAN_OPTIONS
+    - lib/isc/tests/result_test
+  <<: *debian_buster_amd64_image
+  <<: *unit_test_job
+  dependencies:
+    - tsan:buster:amd64
+  needs: ["tsan:buster:amd64"]
+  allow_failure: true
+  after_script:
+    - find lib -name 'tsan.*' -exec python3 util/parse_tsan.py {} \;
+  artifacts:
+    expire_in: "1 week"
+    paths:
+      - lib/*/tests/tsan.*
+      - tsan/
+      - kyua.log
+      - kyua.results
+      - kyua_html/
+    when: on_failure
+
 rwlock:sid:amd64:
   variables:
     CC: gcc

bin/tests/system/run.sh

@@ -196,7 +196,7 @@ if [ $status != 0 ]; then
 else
     core_dumps="$(find $systest/ -name 'core*' -or -name '*.core' | sort | tr '\n' ' ')"
     assertion_failures=$(find $systest/ -name named.run | xargs grep "assertion failure" | wc -l)
-    sanitizer_summaries=$(find $systest/ -type f | grep '^[-a-zA-Z0-9./_]*$' | xargs grep "SUMMARY: .*Sanitizer" | wc -l)
+    sanitizer_summaries=$(find $systest/ -name 'tsan.*' | wc -l)
     if [ -n "$core_dumps" ]; then
         echoinfo "I:$systest:Test claims success despite crashes: $core_dumps"
         echofail "R:$systest:FAIL"
@@ -214,7 +214,9 @@ else
 		echoinfo "D:$systest:backtrace from $coredump end"
 	done
     elif [ $assertion_failures -ne 0 ]; then
+	SYSTESTDIR="$systest"
         echoinfo "I:$systest:Test claims success despite $assertion_failures assertion failure(s)"
+	grep "SUMMARY: " $(find $systest/ -name 'tsan.*') | sort -u | cat_d
         echofail "R:$systest:FAIL"
         # Do not clean up - we need the evidence.
     elif [ $sanitizer_summaries -ne 0 ]; then

util/copyrights

@@ -2593,6 +2593,7 @@
 ./util/nanny.pl					PERL	2000,2001,2004,2007,2012,2016,2018,2019
 ./util/new-func					PERL	2005,2007,2012,2016,2018,2019
 ./util/nt-kit					SH	1999,2000,2001,2004,2007,2012,2016,2018,2019
+./util/parse_tsan.py				PYTHON-BIN	2019
 ./util/spacewhack.pl				PERL	2000,2001,2004,2007,2012,2016,2018,2019
 ./util/tabify-changes				SH	2004,2007,2012,2016,2018,2019
 ./util/update-drafts.pl				PERL	2000,2001,2004,2007,2012,2016,2018,2019

util/parse_tsan.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+############################################################################
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+############################################################################
+
+import sys, os, os.path, re
+from hashlib import sha256
+
+class State:
+    inside = False
+    block = ""
+    last_line = None
+
+    mutexes = {}
+    m_index = 1
+    threads = {}
+    t_index = 1
+    pointers = {}
+    p_index = 1
+
+    def init(self):
+        self.reset()
+
+    def reset(self):
+        self.inside = False
+        self.block = ""
+
+        self.mutexes = {}
+        self.threads = {}
+        self.pointers = {}
+        self.pointers["0x000000000000"] = 0
+
+        self.m_index = 1
+        self.t_index = 1
+        self.p_index = 1
+
+top = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+
+out = os.path.join(top, "tsan")
+
+if not os.path.isdir(out):
+    os.mkdir(out)
+
+# Regular Expressions
+mutex = re.compile(r"M\d+")
+thread = re.compile(r"T\d+")
+stack = re.compile(r"\s\(\S+\+0x\S+\)")
+pointer = re.compile(r"0x[0-9a-f]+")
+pid = re.compile(r"\(pid=\d+,?\)")
+tid = re.compile(r"tid=\d+,?\s*")
+worker = re.compile(r"\s+'(isc-worker|isc-net-)\d+'")
+path = re.compile(top + "/")
+
+s = State()
+
+
+with open(sys.argv[1], "r", encoding='utf-8') as f:
+    lines = f.readlines()
+    for line in lines:
+        if line == "==================\n":
+           if not s.inside:
+               s.inside = True
+           else:
+               dname = os.path.join(out, sha256(s.last_line.encode('utf-8')).hexdigest())
+               if not os.path.isdir(dname):
+                   os.mkdir(dname)
+               fname = os.path.join(dname, sha256(s.block.encode('utf-8')).hexdigest() + ".tsan")
+               if not os.path.isfile(fname):
+                   with open(fname, "w", encoding='utf-8') as w:
+                       w.write(s.block)
+               s.reset()
+        else:
+            for m in mutex.finditer(line):
+                k = m.group()
+                if k not in s.mutexes:
+                    s.mutexes[k] = s.m_index
+                    s.m_index += 1
+            for m in thread.finditer(line):
+                k = m.group()
+                if k not in s.threads:
+                    s.threads[k] = s.t_index
+                    s.t_index += 1
+            for m in pointer.finditer(line):
+                k = m.group()
+                if k not in s.pointers:
+                    s.pointers[k] = s.p_index
+                    s.p_index += 1
+            for k, v in s.mutexes.items():
+                r = re.compile(k)
+                line = r.sub("M%s" % v, line)
+            for k, v in s.threads.items():
+                r = re.compile(k)
+                line = r.sub("T%s" % v, line)
+            for k, v in s.pointers.items():
+                r = re.compile(k)
+                line = r.sub("0x%s" % str(v).zfill(12), line)
+
+            line = stack.sub("", line)
+            line = pid.sub("", line)
+            line = tid.sub("", line)
+            line = worker.sub("", line)
+            line = path.sub("", line)
+
+            s.block += line
+            s.last_line = line