Add system test for the deadlock fix

The test spawns 4 parallel workers that keep adding, modifying and
deleting zones, the main thread repeatedly checks wheter rndc
status responds within a reasonable period.

While environment and timing issues may affect the test, in most
test cases the deadlock that was taking place before the fix used to
trigger in less than 7 seconds in a machine with at least 2 cores.

bin/tests/system/addzone/ns3/example.db

@@ -0,0 +1,2 @@
+@ IN SOA localhost. localhost.localhost. 1 10800 3600 605800 86400
+@ IN NS localhost.

bin/tests/system/addzone/tests_rndc_deadlock.py

@@ -0,0 +1,90 @@
+############################################################################
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+############################################################################
+
+import concurrent.futures
+import os
+import subprocess
+import time
+
+
+def run_rndc(server, rndc_command):
+    '''
+    Send the specified 'rndc_command' to 'server' with a timeout of 2 seconds
+    '''
+    rndc = os.getenv('RNDC')
+    port = os.getenv('CONTROLPORT')
+
+    cmdline = [rndc, '-c', '../common/rndc.conf', '-p', port, '-s', server]
+    cmdline.extend(rndc_command)
+
+    subprocess.check_output(cmdline, stderr=subprocess.STDOUT, timeout=2)
+
+
+def rndc_loop(test_state, domain):
+    '''
+    Run "rndc addzone", "rndc modzone", and "rndc delzone" in a tight loop
+    until the test is considered finished, ignoring errors
+    '''
+    rndc_commands = [
+        ['addzone', domain,
+         '{ type master; file "example.db"; };'],
+        ['modzone', domain,
+         '{ type master; file "example.db"; allow-transfer { any; }; };'],
+        ['delzone', domain],
+    ]
+
+    while not test_state['finished']:
+        for command in rndc_commands:
+            try:
+                run_rndc('10.53.0.3', command)
+            except subprocess.SubprocessError:
+                pass
+
+
+def check_if_server_is_responsive():
+    '''
+    Check if server status can be successfully retrieved using "rndc status"
+    '''
+    try:
+        run_rndc('10.53.0.3', ['status'])
+        return True
+    except subprocess.SubprocessError:
+        return False
+
+
+def test_rndc_deadlock():
+    '''
+    Test whether running "rndc addzone", "rndc modzone", and "rndc delzone"
+    commands concurrently does not trigger a deadlock
+    '''
+    test_state = {'finished': False}
+
+    # Create 4 worker threads running "rndc" commands in a loop.
+    executor = concurrent.futures.ThreadPoolExecutor()
+    for i in range(1, 5):
+        domain = 'example%d' % i
+        executor.submit(rndc_loop, test_state, domain)
+
+    # Run "rndc status" in 1-second intervals for a maximum of 10 seconds.  If
+    # any "rndc status" command fails, the loop will be interrupted.
+    server_is_responsive = True
+    attempts = 10
+    while server_is_responsive and attempts > 0:
+        server_is_responsive = check_if_server_is_responsive()
+        attempts -= 1
+        time.sleep(1)
+
+    # Signal worker threads that the test is finished.
+    test_state['finished'] = True
+    executor.shutdown()
+
+    # Check whether all "rndc status" commands succeeded.
+    assert server_is_responsive

util/copyrights

@@ -221,6 +221,7 @@
 ./bin/tests/system/addzone/ns2/default.nzf.in	X	2010,2018,2019,2020
 ./bin/tests/system/addzone/setup.sh		SH	2010,2012,2013,2014,2016,2017,2018,2019,2020,2021
 ./bin/tests/system/addzone/tests.sh		SH	2010,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020,2021
+./bin/tests/system/addzone/tests_rndc_deadlock.py	PYTHON	2021
 ./bin/tests/system/allow-query/clean.sh		SH	2010,2012,2014,2015,2016,2018,2019,2020,2021
 ./bin/tests/system/allow-query/ns3/named.args	X	2018,2019,2020,2021
 ./bin/tests/system/allow-query/setup.sh		SH	2010,2012,2016,2018,2019,2020,2021