Merge branch 'wpk/pipelined-test-fix' into 'master'

Fix pipelined test

See merge request isc-projects/bind9!2927

bin/named/main.c

@@ -141,7 +141,6 @@ static bool nonearest = false;
 static bool nosoa = false;
 static bool notcp = false;
 static bool sigvalinsecs = false;
-static unsigned int delay = 0;
 
 /*
  * -4 and -6
@@ -638,14 +637,10 @@ parse_T_opt(char *option) {
 	/*
 	 * force the server to behave (or misbehave) in
 	 * specified ways for testing purposes.
-	 * delay=xxxx: delay client responses by xxxx ms to
-	 *	       simulate remote servers.
 	 * dscp=x:     check that dscp values are as
 	 * 	       expected and assert otherwise.
 	 */
-	if (!strncmp(option, "delay=", 6)) {
-		delay = atoi(option + 6);
-	} else if (!strcmp(option, "dropedns")) {
+	if (!strcmp(option, "dropedns")) {
 		dropedns = true;
 	} else if (!strncmp(option, "dscp=", 5)) {
 		isc_dscp_check_value = atoi(option + 5);
@@ -1318,8 +1313,6 @@ setup(void) {
 		ns_server_setoption(sctx, NS_SERVER_NOTCP, true);
 	if (sigvalinsecs)
 		ns_server_setoption(sctx, NS_SERVER_SIGVALINSECS, true);
-
-	named_g_server->sctx->delay = delay;
 }
 
 static void

bin/tests/system/pipelined/ans5/ans.py

@@ -0,0 +1,194 @@
+############################################################################
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+############################################################################
+
+############################################################################
+#
+# This tool acts as a TCP/UDP proxy and delays all incoming packets by 500
+# miliseconds.
+#
+# We use it to check pipelining - a client sents 8 questions over a
+# pipelined connection - that require asking a normal (examplea) and a
+# slow-responding (exampleb) servers:
+# a.examplea
+# a.exampleb
+# b.examplea
+# b.exampleb
+# c.examplea
+# c.exampleb
+# d.examplea
+# d.exampleb
+#
+# If pipelining works properly the answers will be returned out of order
+# with all answers from examplea returned first, and then all answers
+# from exampleb.
+#
+############################################################################
+
+from __future__ import print_function
+
+import datetime
+import os
+import select
+import signal
+import socket
+import sys
+import time
+import threading
+import struct
+
+DELAY = 0.5
+THREADS = []
+
+def log(msg):
+    print(datetime.datetime.now().strftime('%d-%b-%Y %H:%M:%S.%f ') + msg)
+
+
+def sigterm(*_):
+    log('SIGTERM received, shutting down')
+    for thread in THREADS:
+        thread.close()
+        thread.join()
+    os.remove('ans.pid')
+    sys.exit(0)
+
+class TCPDelayer(threading.Thread):
+    """ For a given TCP connection conn we open a connection to (ip, port),
+        and then we delay each incoming packet by DELAY by putting it in a
+        queue.
+        In the pipelined test TCP should not be used, but it's here for
+        completnes.
+    """
+    def __init__(self, conn, ip, port):
+        threading.Thread.__init__(self)
+        self.conn = conn
+        self.cconn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        self.cconn.connect((ip, port))
+        self.queue = []
+        self.running = True
+
+    def close(self):
+        self.running = False
+
+    def run(self):
+        while self.running:
+            curr_timeout = 0.5
+            try:
+                curr_timeout = self.queue[0][0]-time.time()
+            except StopIteration:
+                pass
+            if curr_timeout > 0:
+                if curr_timeout == 0:
+                    curr_timeout = 0.5
+                rfds, _, _ = select.select([self.conn, self.cconn], [], [], curr_timeout)
+                if self.conn in rfds:
+                    data = self.conn.recv(65535)
+                    if not data:
+                        return
+                    self.queue.append((time.time() + DELAY, data))
+                if self.cconn in rfds:
+                    data = self.cconn.recv(65535)
+                    if not data == 0:
+                        return
+                    self.conn.send(data)
+            try:
+                while self.queue[0][0]-time.time() < 0:
+                    _, data = self.queue.pop(0)
+                    self.cconn.send(data)
+            except StopIteration:
+                pass
+
+class UDPDelayer(threading.Thread):
+    """ Every incoming UDP packet is put in a queue for DELAY time, then
+        it's sent to (ip, port). We remember the query id to send the
+        response we get to a proper source, responsed are not delayed.
+    """
+    def __init__(self, usock, ip, port):
+        threading.Thread.__init__(self)
+        self.sock = usock
+        self.csock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        self.dst = (ip, port)
+        self.queue = []
+        self.qid_mapping = {}
+        self.running = True
+
+    def close(self):
+        self.running = False
+
+    def run(self):
+        while self.running:
+            curr_timeout = 0.5
+            if self.queue:
+                curr_timeout = self.queue[0][0]-time.time()
+            if curr_timeout >= 0:
+                if curr_timeout == 0:
+                    curr_timeout = 0.5
+                rfds, _, _ = select.select([self.sock, self.csock], [], [], curr_timeout)
+                if self.sock in rfds:
+                    data, addr = self.sock.recvfrom(65535)
+                    if not data:
+                        return
+                    self.queue.append((time.time() + DELAY, data))
+                    qid = struct.unpack('>H', data[:2])[0]
+                    log('Received a query from %s, queryid %d' % (str(addr), qid))
+                    self.qid_mapping[qid] = addr
+                if self.csock in rfds:
+                    data, addr = self.csock.recvfrom(65535)
+                    if not data:
+                        return
+                    qid = struct.unpack('>H', data[:2])[0]
+                    dst = self.qid_mapping.get(qid)
+                    if dst is not None:
+                        self.sock.sendto(data, dst)
+                        log('Received a response from %s, queryid %d, sending to %s' % (str(addr), qid, str(dst)))
+            while self.queue and self.queue[0][0]-time.time() < 0:
+                _, data = self.queue.pop(0)
+                qid = struct.unpack('>H', data[:2])[0]
+                log('Sending a query to %s, queryid %d' % (str(self.dst), qid))
+                self.csock.sendto(data, self.dst)
+
+def main():
+    signal.signal(signal.SIGTERM, sigterm)
+    signal.signal(signal.SIGINT, sigterm)
+
+    with open('ans.pid', 'w') as pidfile:
+        print(os.getpid(), file=pidfile)
+
+    listenip = '10.53.0.5'
+    serverip = '10.53.0.2'
+
+    try:
+        port = int(os.environ['PORT'])
+    except KeyError:
+        port = 5300
+
+    log('Listening on %s:%d' % (listenip, port))
+
+    usock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    usock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    usock.bind((listenip, port))
+    thread = UDPDelayer(usock, serverip, port)
+    thread.start()
+    THREADS.append(thread)
+
+    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    sock.bind((listenip, port))
+    sock.listen(1)
+
+    while True:
+        (clientsock, _) = sock.accept()
+        log('Accepted connection from %s' % clientsock)
+        thread = TCPDelayer(clientsock, serverip, port)
+        thread.start()
+        THREADS.append(thread)
+
+if __name__ == '__main__':
+    main()

bin/tests/system/pipelined/ns1/root.db

@@ -19,7 +19,7 @@ $TTL 300
 a.root-servers.nil.	A	10.53.0.1
 
 examplea.		NS	ns2.examplea.
-ns2.examplea.		A	10.53.0.2
+ns2.examplea.		A	10.53.0.5
 
 exampleb.		NS	ns3.exampleb.
 ns3.exampleb.		A	10.53.0.3

bin/tests/system/pipelined/ns2/examplea.db

@@ -17,7 +17,7 @@ examplea		IN SOA	mname1. . (
 				3600       ; minimum (1 hour)
 				)
 examplea.		NS	ns2.examplea.
-ns2.examplea.		A	10.53.0.2
+ns2.examplea.		A	10.53.0.5
 
 $ORIGIN examplea.
 a			A	10.0.1.1

bin/tests/system/pipelined/ns3/named.args

@@ -1 +0,0 @@
--m record,size,mctx -c named.conf -d 99 -D pipelined-ns3 -X named.lock -g -T delay=200

bin/tests/system/pipelined/prereq.sh

@@ -0,0 +1,29 @@
+#!/bin/sh
+#
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+SYSTEMTESTTOP=..
+. $SYSTEMTESTTOP/conf.sh
+
+if test -n "$PYTHON"
+then
+    if $PYTHON -c "import dns" 2> /dev/null
+    then
+        :
+    else
+        echo_i "This test requires the dnspython module." >&2
+        exit 1
+    fi
+else
+    echo_i "This test requires Python and the dnspython module." >&2
+    exit 1
+fi
+
+exit 0

lib/ns/include/ns/server.h

@@ -90,7 +90,6 @@ struct ns_server {
 
 	/*% Test options and other configurables */
 	uint32_t		options;
-	unsigned int		delay;
 
 	dns_acl_t		*blackholeacl;
 	dns_acl_t		*keepresporder;

util/copyrights

@@ -868,11 +868,12 @@
 ./bin/tests/system/pending/ns2/sign.sh		SH	2009,2010,2012,2014,2016,2017,2018,2019,2020
 ./bin/tests/system/pending/setup.sh		SH	2009,2012,2014,2016,2017,2018,2019,2020
 ./bin/tests/system/pending/tests.sh		SH	2009,2010,2012,2015,2016,2018,2019,2020
+./bin/tests/system/pipelined/ans5/ans.py	PYTHON	2020
 ./bin/tests/system/pipelined/clean.sh		SH	2014,2015,2016,2018,2019,2020
 ./bin/tests/system/pipelined/input		X	2014,2015,2018,2019,2020
 ./bin/tests/system/pipelined/inputb		X	2014,2015,2018,2019,2020
-./bin/tests/system/pipelined/ns3/named.args	X	2014,2015,2018,2019,2020
 ./bin/tests/system/pipelined/pipequeries.c	C	2014,2015,2015,2016,2017,2018,2019,2020
+./bin/tests/system/pipelined/prereq.sh		SH	2020
 ./bin/tests/system/pipelined/ref		X	2014,2015,2018,2019,2020
 ./bin/tests/system/pipelined/refb		X	2014,2015,2018,2019,2020
 ./bin/tests/system/pipelined/setup.sh		SH	2014,2015,2016,2017,2018,2019,2020