Add a system test that tests connections quota for DoH

The system tests stress out the DoH quota by opening many TCP
connections and then running dig instances against the "overloaded"
server to perform some queries. The processes cannot make any
resolutions because the quota is exceeded. Then the opened connections
are getting closed in random order allowing the queries to proceed.

bin/tests/system/doth/stress_http_quota.py

@@ -0,0 +1,242 @@
+#!/usr/bin/env python
+############################################################################
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+############################################################################
+
+import os
+import sys
+import socket
+import subprocess
+import random
+import time
+
+from functools import reduce
+
+# this number should exceed default HTTP quota value
+NCONNECTIONS = 320
+MULTIDIG_INSTANCES = 10
+CONNECT_TRIES = 5
+
+random.seed()
+
+
+# Introduce some random delay
+def jitter():
+    time.sleep((500 + random.randint(0, 250))/1000000.0)
+
+
+# A set of simple procedures to get the test's configuration options
+def get_http_port(http_secure=False):
+    http_port_env = None
+    if http_secure:
+        http_port_env = os.getenv("HTTPSPORT")
+    else:
+        http_port_env = os.getenv("HTTPPORT")
+    if http_port_env:
+        return int(http_port_env)
+    return 443
+
+
+def get_http_host():
+    bind_host = os.getenv("BINDHOST")
+    if bind_host:
+        return bind_host
+    return "localhost"
+
+
+def get_dig_path():
+    dig_path = os.getenv("DIG")
+    if dig_path:
+        return dig_path
+    return "dig"
+
+
+# A simple class which creates the given number of TCP connections to
+# the given host in order to stress the BIND's quota facility
+class TCPConnector:
+    def __init__(self, nconnections, host, port):
+        self.number_of_connections = nconnections
+        self.host = host
+        self.port = port
+        self.connections = []
+
+    def connect_one(self):
+        tries = CONNECT_TRIES
+        while tries > 0:
+            try:
+                sock = socket.create_connection(address=(self.host, self.port),
+                                                timeout=None)
+                self.connections.append(sock)
+                break
+            except ConnectionResetError:
+                # some jitter for BSDs
+                jitter()
+                continue
+            except TimeoutError:
+                jitter()
+                continue
+            finally:
+                tries -= 1
+
+    def connect_all(self):
+        for _ in range(1, self.number_of_connections + 1):
+            self.connect_one()
+
+    # Close an established connection (randomly)
+    def disconnect_random(self):
+        pos = random.randint(0, len(self.connections) - 1)
+        conn = self.connections[pos]
+        try:
+            conn.shutdown(socket.SHUT_RDWR)
+            conn.close()
+        except OSError:
+            conn.close()
+        finally:
+            self.connections.remove(conn)
+
+    def disconnect_all(self):
+        while len(self.connections) != 0:
+            self.disconnect_random()
+
+
+# A simple class which allows running a dig instance under control of
+# the process
+class SubDIG:
+    def __init__(self, http_secure=None, extra_args=None):
+        self.sub_process = None
+        self.dig_path = get_dig_path()
+        self.host = get_http_host()
+        self.port = get_http_port(http_secure=http_secure)
+        if http_secure:
+            self.http_secure = True
+        else:
+            self.http_secure = False
+        self.extra_args = extra_args
+
+    # This method constructs a command string
+    def get_command(self):
+        command = self.dig_path + " -p " + str(self.port) + " "
+        command = command + "+noadd +nosea +nostat +noquest +nocmd +time=30 "
+        if self.http_secure:
+            command = command + "+https "
+        else:
+            command = command + "+http-plain "
+        command = command + "@" + self.host + " "
+        if self.extra_args:
+            command = command + self.extra_args
+        return command
+
+    def run(self):
+        # pylint: disable=consider-using-with
+        with open(os.devnull, 'w') as devnull:
+            self.sub_process = subprocess.Popen(self.get_command(), shell=True,
+                                                stdout=devnull)
+        jitter()
+
+    def wait(self, timeout=None):
+        res = None
+        if timeout is None:
+            return self.sub_process.wait()
+        try:
+            res = self.sub_process.wait(timeout=timeout)
+        except subprocess.TimeoutExpired:
+            return None
+        return res
+
+    def alive(self):
+        return self.sub_process.poll() is None
+
+
+# A simple wrapper class which allows running multiple dig instances
+# and examining their statuses in one logical operation.
+class MultiDIG:
+    def __init__(self, numdigs, http_secure=None,
+                 extra_args=None):
+        assert int(numdigs) > 0
+        digs = []
+        for _ in range(1, int(numdigs) + 1):
+            digs.append(SubDIG(http_secure=http_secure,
+                               extra_args=extra_args))
+        self.digs = digs
+        assert len(self.digs) == int(numdigs)
+
+    def run(self):
+        for p in self.digs:
+            p.run()
+
+    def wait(self):
+        return map(lambda p: (p.wait()), self.digs)
+
+    # Wait for the all instances to terminate with expected given
+    # status. Returns true or false.
+    def wait_for_result(self, result):
+        return reduce(
+            lambda a, b: ((a == result or a is True) and b == result),
+                      self.wait())
+
+    def alive(self):
+        return reduce(lambda a, b: (a and b), map(lambda p: (p.alive()),
+                                                  self.digs))
+
+
+# The test's main logic
+def run_test(http_secure=True):
+    query_args = "SOA ."
+    # Let's try to make a successful query
+    subdig = SubDIG(http_secure=http_secure, extra_args=query_args)
+    subdig.run()
+    assert subdig.wait() == 0, "DIG was expected to succeed"
+    # Let's create a lot of TCP connections to the server stress the
+    # HTTP quota
+    connector = TCPConnector(NCONNECTIONS, get_http_host(),
+                             get_http_port(http_secure=http_secure))
+    # Let's make queries until the quota kicks in
+    subdig = SubDIG(http_secure=http_secure, extra_args=query_args)
+    subdig.run()
+    while True:
+        subdig = SubDIG(http_secure=http_secure, extra_args=query_args)
+        connector.connect_all()
+        time.sleep(2)
+        subdig.run()
+        if subdig.wait(timeout=2) is None:
+            break
+        connector.disconnect_all()
+
+    # At this point quota has kicked in.  Additionally, let's create a
+    # bunch of dig processes all trying to make a query against the
+    # server with exceeded quota
+    multidig = MultiDIG(MULTIDIG_INSTANCES, http_secure=http_secure,
+                        extra_args=query_args)
+    multidig.run()
+    # Wait for the dig instance to complete. Not a single instance has
+    # a chance to complete successfully because of the exceeded quota
+    assert subdig.wait(timeout=5) is None,\
+        "Single DIG instance has stopped prematurely"
+    assert subdig.alive(), "Single DIG instance is expected to be alive"
+    assert multidig.alive(), "Multiple DIG instances are expected to be alive"
+    # Let's close opened connections (in random order) to let all dig
+    # processes to complete
+    connector.disconnect_all()
+    # Wait for all processes to complete successfully
+    assert subdig.wait() == 0, "Single DIG instance failed"
+    assert multidig.wait_for_result(0) is True,\
+        "One or more of DIG instances returned unexpected results"
+
+
+def main():
+    run_test(http_secure=True)
+    run_test(http_secure=False)
+    # If we have reached this point we could safely return 0
+    # (success). If the test fails because of an assert, the whole
+    # program will return non-zero exit code and produce the backtrace
+    return 0
+
+
+sys.exit(main())

bin/tests/system/doth/tests.sh

@@ -254,5 +254,17 @@ test_opcodes NOERROR 0
 test_opcodes NOTIMP 1 2 3 6 7 8 9 10 11 12 13 14 15
 test_opcodes FORMERR 4 5
 
+n=$((n + 1))
+echo_i "checking server quotas for both encrypted and unencrypted HTTP ($n)"
+ret=0
+if [ -x "$PYTHON" ]; then
+	BINDHOST="10.53.0.1" "$PYTHON" "$TOP_SRCDIR/bin/tests/system/doth/stress_http_quota.py"
+	ret=$?
+else
+	echo_i "Python is not available. Skipping the test..."
+fi
+if [ $ret != 0 ]; then echo_i "failed"; fi
+status=$((status + ret))
+
 echo_i "exit status: $status"
 [ $status -eq 0 ] || exit 1

util/copyrights

@@ -298,6 +298,7 @@
 ./bin/tests/system/doth/ns2/cert.pem		X	2021
 ./bin/tests/system/doth/ns2/key.pem		X	2021
 ./bin/tests/system/doth/setup.sh		SH	2021
+./bin/tests/system/doth/stress_http_quota.py	PYTHON-BIN	2021
 ./bin/tests/system/doth/tests.sh		SH	2021
 ./bin/tests/system/dscp/clean.sh		SH	2013,2014,2015,2016,2018,2019,2020,2021
 ./bin/tests/system/dscp/ns1/named.args		X	2013,2014,2018,2019,2020,2021