From 195d25b2229244fb5ffbc485f8ca05bf6e08d282 Mon Sep 17 00:00:00 2001
From: Evan Hunt <each@isc.org>
Date: Wed, 19 Feb 2020 18:26:48 -0800
Subject: [PATCH] send bursts of queries in parallel to reduce test runtime

sending each group of queries simultaneously, and then checking the
output after the last one finishes, reduces the runtime of the
serve-stale test by about six minutes.
---
 .../system/serve-stale/ns1/named1.conf.in     |   1 +
 .../system/serve-stale/ns1/named2.conf.in     |   3 +-
 bin/tests/system/serve-stale/tests.sh         | 563 ++++++++++--------
 3 files changed, 314 insertions(+), 253 deletions(-)

diff --git a/bin/tests/system/serve-stale/ns1/named1.conf.in b/bin/tests/system/serve-stale/ns1/named1.conf.in
index be3551aae9..c29bcd7a44 100644
--- a/bin/tests/system/serve-stale/ns1/named1.conf.in
+++ b/bin/tests/system/serve-stale/ns1/named1.conf.in
@@ -30,6 +30,7 @@ options {
 	max-stale-ttl 3600;
 	stale-answer-ttl 2;
 	stale-answer-enable yes;
+	servfail-ttl 0;
 };
 
 zone "." {
diff --git a/bin/tests/system/serve-stale/ns1/named2.conf.in b/bin/tests/system/serve-stale/ns1/named2.conf.in
index 7ec265e631..a41f07b140 100644
--- a/bin/tests/system/serve-stale/ns1/named2.conf.in
+++ b/bin/tests/system/serve-stale/ns1/named2.conf.in
@@ -27,9 +27,10 @@ options {
 	listen-on { 10.53.0.1; };
 	listen-on-v6 { none; };
 	recursion yes;
-	max-stale-ttl 45;
+	max-stale-ttl 20;
 	stale-answer-ttl 3;
 	stale-answer-enable yes;
+	servfail-ttl 0;
 };
 
 zone "." {
diff --git a/bin/tests/system/serve-stale/tests.sh b/bin/tests/system/serve-stale/tests.sh
index 07f28def52..252eeab994 100755
--- a/bin/tests/system/serve-stale/tests.sh
+++ b/bin/tests/system/serve-stale/tests.sh
@@ -14,75 +14,68 @@ SYSTEMTESTTOP=..
 
 RNDCCMD="$RNDC -c $SYSTEMTESTTOP/common/rndc.conf -p ${CONTROLPORT} -s"
 
+# wait up to ten seconds to ensure that a file has been written
+waitfile () {
+    for try in 0 1 2 3 4 5 6 7 8 9; do
+        [ -s "$1" ] && break
+        sleep 1
+    done
+}
+
 status=0
 n=0
 
-#echo_i "check ans.pl server ($n)"
-#$DIG -p ${PORT} @10.53.0.2 example NS
-#$DIG -p ${PORT} @10.53.0.2 example SOA
-#$DIG -p ${PORT} @10.53.0.2 ns.example A
-#$DIG -p ${PORT} @10.53.0.2 ns.example AAAA
-#$DIG -p ${PORT} @10.53.0.2 txt enable
-#$DIG -p ${PORT} @10.53.0.2 txt disable
-#$DIG -p ${PORT} @10.53.0.2 ns.example AAAA
-#$DIG -p ${PORT} @10.53.0.2 txt enable
-#$DIG -p ${PORT} @10.53.0.2 ns.example AAAA
-#$DIG -p ${PORT} @10.53.0.2 data.example TXT
-#$DIG -p ${PORT} @10.53.0.2 nodata.example TXT
-#$DIG -p ${PORT} @10.53.0.2 nxdomain.example TXT
-#$DIG -p ${PORT} @10.53.0.2 othertype.example CAA
-
 #
 # First test server with serve-stale options set.
 #
 echo_i "test server with serve-stale options set"
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "prime cache longttl.example ($n)"
 ret=0
 $DIG -p ${PORT} @10.53.0.1 longttl.example TXT > dig.out.test$n
 grep "status: NOERROR" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 1," dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "prime cache data.example ($n)"
 ret=0
 $DIG -p ${PORT} @10.53.0.1 data.example TXT > dig.out.test$n
 grep "status: NOERROR" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 1," dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "prime cache othertype.example ($n)"
 ret=0
 $DIG -p ${PORT} @10.53.0.1 othertype.example CAA > dig.out.test$n
 grep "status: NOERROR" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 1," dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "prime cache nodata.example ($n)"
 ret=0
 $DIG -p ${PORT} @10.53.0.1 nodata.example TXT > dig.out.test$n
 grep "status: NOERROR" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 0," dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "prime cache nxdomain.example ($n)"
 ret=0
 $DIG -p ${PORT} @10.53.0.1 nxdomain.example TXT > dig.out.test$n
 grep "status: NXDOMAIN" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 0," dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "verify prime cache statistics ($n)"
 ret=0
 rm -f ns1/named.stats
@@ -97,31 +90,42 @@ grep "2 TXT" ns1/named.stats.$n.cachedb > /dev/null || ret=1
 grep "1 !TXT" ns1/named.stats.$n.cachedb > /dev/null || ret=1
 grep "1 NXDOMAIN" ns1/named.stats.$n.cachedb > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "disable responses from authoritative server ($n)"
 ret=0
 $DIG -p ${PORT} @10.53.0.2 txt disable  > dig.out.test$n
 grep "ANSWER: 1," dig.out.test$n > /dev/null || ret=1
 grep "TXT.\"0\"" dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-sleep 2
-
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "check 'rndc serve-stale status' ($n)"
 ret=0
 $RNDCCMD 10.53.0.1 serve-stale status > rndc.out.test$n 2>&1 || ret=1
 grep '_default: on (stale-answer-ttl=2 max-stale-ttl=3600)' rndc.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+sleep 2
+
+echo_i "sending queries for tests $((n+1))-$((n+4))..."
+$DIG -p ${PORT} @10.53.0.1 data.example TXT > dig.out.test$((n+1)) &
+$DIG -p ${PORT} @10.53.0.1 othertype.example CAA > dig.out.test$((n+2)) &
+$DIG -p ${PORT} @10.53.0.1 nodata.example TXT > dig.out.test$((n+3)) &
+$DIG -p ${PORT} @10.53.0.1 nxdomain.example TXT > dig.out.test$((n+4))
+
+# ensure all files have been written before proceeding
+waitfile dig.out.test$((n+1))
+waitfile dig.out.test$((n+2))
+waitfile dig.out.test$((n+3))
+waitfile dig.out.test$((n+4))
+
+n=$((n+1))
 echo_i "check stale data.example ($n)"
 ret=0
-$DIG -p ${PORT} @10.53.0.1 data.example TXT > dig.out.test$n
 grep "status: NOERROR" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 1," dig.out.test$n > /dev/null || ret=1
 grep "data\.example\..*2.*IN.*TXT.*A text record with a 2 second ttl" dig.out.test$n > /dev/null || ret=1
@@ -135,39 +139,36 @@ awk '/; stale/ { x=$0; getline; print x, $0}' ns1/named_dump.db.test$n |
 awk '/; answer/ { x=$0; getline; print x, $0}' ns1/named_dump.db.test$n |
     grep "; answer longttl\.example.*A text record with a 600 second ttl" > /dev/null 2>&1 || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "check stale othertype.example ($n)"
 ret=0
-$DIG -p ${PORT} @10.53.0.1 othertype.example CAA > dig.out.test$n
 grep "status: NOERROR" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 1," dig.out.test$n > /dev/null || ret=1
 grep "othertype\.example\..*2.*IN.*CAA.*0.*issue" dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "check stale nodata.example ($n)"
 ret=0
-$DIG -p ${PORT} @10.53.0.1 nodata.example TXT > dig.out.test$n
 grep "status: NOERROR" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 0," dig.out.test$n > /dev/null || ret=1
 grep "example\..*2.*IN.*SOA" dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "check stale nxdomain.example ($n)"
 ret=0
-$DIG -p ${PORT} @10.53.0.1 nxdomain.example TXT > dig.out.test$n
 grep "status: NXDOMAIN" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 0," dig.out.test$n > /dev/null || ret=1
 grep "example\..*2.*IN.*SOA" dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "verify stale cache statistics ($n)"
 ret=0
 rm -f ns1/named.stats
@@ -183,189 +184,211 @@ grep "1 #Others" ns1/named.stats.$n.cachedb > /dev/null || ret=1
 grep "1 #TXT" ns1/named.stats.$n.cachedb > /dev/null || ret=1
 grep "1 #!TXT" ns1/named.stats.$n.cachedb > /dev/null || ret=1
 grep "1 #NXDOMAIN" ns1/named.stats.$n.cachedb > /dev/null || ret=1
-status=`expr $status + $ret`
+status=$((status+ret))
 if [ $ret != 0 ]; then echo_i "failed"; fi
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "running 'rndc serve-stale off' ($n)"
 ret=0
 $RNDCCMD 10.53.0.1 serve-stale off || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "check 'rndc serve-stale status' ($n)"
 ret=0
 $RNDCCMD 10.53.0.1 serve-stale status > rndc.out.test$n 2>&1 || ret=1
 grep '_default: off (rndc) (stale-answer-ttl=2 max-stale-ttl=3600)' rndc.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+echo_i "sending queries for tests $((n+1))-$((n+4))..."
+$DIG -p ${PORT} @10.53.0.1 data.example TXT > dig.out.test$((n+1)) &
+$DIG -p ${PORT} @10.53.0.1 othertype.example CAA > dig.out.test$((n+2)) &
+$DIG -p ${PORT} @10.53.0.1 nodata.example TXT > dig.out.test$((n+3)) &
+$DIG -p ${PORT} @10.53.0.1 nxdomain.example TXT > dig.out.test$((n+4))
+
+# ensure all files have been written before proceeding
+waitfile dig.out.test$((n+1))
+waitfile dig.out.test$((n+2))
+waitfile dig.out.test$((n+3))
+waitfile dig.out.test$((n+4))
+
+n=$((n+1))
 echo_i "check stale data.example (serve-stale off) ($n)"
 ret=0
-$DIG -p ${PORT} @10.53.0.1 data.example TXT > dig.out.test$n
 grep "status: SERVFAIL" dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
-if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "check stale othertype.example (serve-stale off) ($n)"
 ret=0
-$DIG -p ${PORT} @10.53.0.1 othertype.example CAA > dig.out.test$n
 grep "status: SERVFAIL" dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "check stale nodata.example (serve-stale off) ($n)"
 ret=0
-$DIG -p ${PORT} @10.53.0.1 nodata.example TXT > dig.out.test$n
 grep "status: SERVFAIL" dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "check stale nxdomain.example (serve-stale off) ($n)"
 ret=0
-$DIG -p ${PORT} @10.53.0.1 nxdomain.example TXT > dig.out.test$n
 grep "status: SERVFAIL" dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "running 'rndc serve-stale on' ($n)"
 ret=0
 $RNDCCMD 10.53.0.1 serve-stale on || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "check 'rndc serve-stale status' ($n)"
 ret=0
 $RNDCCMD 10.53.0.1 serve-stale status > rndc.out.test$n 2>&1 || ret=1
 grep '_default: on (rndc) (stale-answer-ttl=2 max-stale-ttl=3600)' rndc.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+echo_i "sending queries for tests $((n+1))-$((n+4))..."
+$DIG -p ${PORT} @10.53.0.1 data.example TXT > dig.out.test$((n+1)) &
+$DIG -p ${PORT} @10.53.0.1 othertype.example CAA > dig.out.test$((n+2)) &
+$DIG -p ${PORT} @10.53.0.1 nodata.example TXT > dig.out.test$((n+3)) &
+$DIG -p ${PORT} @10.53.0.1 nxdomain.example TXT > dig.out.test$((n+4))
+
+# ensure all files have been written before proceeding
+waitfile dig.out.test$((n+1))
+waitfile dig.out.test$((n+2))
+waitfile dig.out.test$((n+3))
+waitfile dig.out.test$((n+4))
+
+n=$((n+1))
 echo_i "check stale data.example (serve-stale on) ($n)"
 ret=0
-$DIG -p ${PORT} @10.53.0.1 data.example TXT > dig.out.test$n
 grep "status: NOERROR" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 1," dig.out.test$n > /dev/null || ret=1
 grep "data\.example\..*2.*IN.*TXT.*A text record with a 2 second ttl" dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "check stale othertype.example (serve-stale on) ($n)"
 ret=0
-$DIG -p ${PORT} @10.53.0.1 othertype.example CAA > dig.out.test$n
 grep "status: NOERROR" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 1," dig.out.test$n > /dev/null || ret=1
 grep "othertype\.example\..*2.*IN.*CAA.*0.*issue" dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "check stale nodata.example (serve-stale on) ($n)"
 ret=0
-$DIG -p ${PORT} @10.53.0.1 nodata.example TXT > dig.out.test$n
 grep "status: NOERROR" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 0," dig.out.test$n > /dev/null || ret=1
 grep "example\..*2.*IN.*SOA" dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "check stale nxdomain.example (serve-stale on) ($n)"
 ret=0
-$DIG -p ${PORT} @10.53.0.1 nxdomain.example TXT > dig.out.test$n
 grep "status: NXDOMAIN" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 0," dig.out.test$n > /dev/null || ret=1
 grep "example\..*2.*IN.*SOA" dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "running 'rndc serve-stale off' ($n)"
 ret=0
 $RNDCCMD 10.53.0.1 serve-stale off || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "running 'rndc serve-stale reset' ($n)"
 ret=0
 $RNDCCMD 10.53.0.1 serve-stale reset || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "check 'rndc serve-stale status' ($n)"
 ret=0
 $RNDCCMD 10.53.0.1 serve-stale status > rndc.out.test$n 2>&1 || ret=1
 grep '_default: on (stale-answer-ttl=2 max-stale-ttl=3600)' rndc.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+echo_i "sending queries for tests $((n+1))-$((n+4))..."
+$DIG -p ${PORT} @10.53.0.1 data.example TXT > dig.out.test$((n+1)) &
+$DIG -p ${PORT} @10.53.0.1 othertype.example CAA > dig.out.test$((n+2)) &
+$DIG -p ${PORT} @10.53.0.1 nodata.example TXT > dig.out.test$((n+3)) &
+$DIG -p ${PORT} @10.53.0.1 nxdomain.example TXT > dig.out.test$((n+4))
+
+# ensure all files have been written before proceeding
+waitfile dig.out.test$((n+1))
+waitfile dig.out.test$((n+2))
+waitfile dig.out.test$((n+3))
+waitfile dig.out.test$((n+4))
+
+n=$((n+1))
 echo_i "check stale data.example (serve-stale reset) ($n)"
 ret=0
-$DIG -p ${PORT} @10.53.0.1 data.example TXT > dig.out.test$n
 grep "status: NOERROR" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 1," dig.out.test$n > /dev/null || ret=1
 grep "data\.example\..*2.*IN.*TXT.*A text record with a 2 second ttl" dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "check stale othertype.example (serve-stale reset) ($n)"
 ret=0
-$DIG -p ${PORT} @10.53.0.1 othertype.example CAA > dig.out.test$n
 grep "status: NOERROR" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 1," dig.out.test$n > /dev/null || ret=1
 grep "othertype.example\..*2.*IN.*CAA.*0.*issue" dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "check stale nodata.example (serve-stale reset) ($n)"
 ret=0
-$DIG -p ${PORT} @10.53.0.1 nodata.example TXT > dig.out.test$n
 grep "status: NOERROR" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 0," dig.out.test$n > /dev/null || ret=1
 grep "example\..*2.*IN.*SOA" dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "check stale nxdomain.example (serve-stale reset) ($n)"
 ret=0
-$DIG -p ${PORT} @10.53.0.1 nxdomain.example TXT > dig.out.test$n
 grep "status: NXDOMAIN" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 0," dig.out.test$n > /dev/null || ret=1
 grep "example\..*2.*IN.*SOA" dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "running 'rndc serve-stale off' ($n)"
 ret=0
 $RNDCCMD 10.53.0.1 serve-stale off || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "check 'rndc serve-stale status' ($n)"
 ret=0
 $RNDCCMD 10.53.0.1 serve-stale status > rndc.out.test$n 2>&1 || ret=1
 grep '_default: off (rndc) (stale-answer-ttl=2 max-stale-ttl=3600)' rndc.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
 #
 # Update named.conf.
@@ -373,29 +396,29 @@ status=`expr $status + $ret`
 #
 echo_i "test server with serve-stale options set, low max-stale-ttl"
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "updating ns1/named.conf ($n)"
 ret=0
 copy_setports ns1/named2.conf.in ns1/named.conf
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "running 'rndc reload' ($n)"
 ret=0
 rndc_reload ns1 10.53.0.1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "check 'rndc serve-stale status' ($n)"
 ret=0
 $RNDCCMD 10.53.0.1 serve-stale status > rndc.out.test$n 2>&1 || ret=1
-grep '_default: off (rndc) (stale-answer-ttl=3 max-stale-ttl=45)' rndc.out.test$n > /dev/null || ret=1
+grep '_default: off (rndc) (stale-answer-ttl=3 max-stale-ttl=20)' rndc.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "flush cache, re-enable serve-stale and query again ($n)"
 ret=0
 $RNDCCMD 10.53.0.1 flushtree example > rndc.out.test$n.1 2>&1 || ret=1
@@ -404,73 +427,71 @@ $DIG -p ${PORT} @10.53.0.1 data.example TXT > dig.out.test$n
 grep "status: SERVFAIL" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 0," dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "check 'rndc serve-stale status' ($n)"
 ret=0
 $RNDCCMD 10.53.0.1 serve-stale status > rndc.out.test$n 2>&1 || ret=1
-grep '_default: on (rndc) (stale-answer-ttl=3 max-stale-ttl=45)' rndc.out.test$n > /dev/null || ret=1
+grep '_default: on (rndc) (stale-answer-ttl=3 max-stale-ttl=20)' rndc.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "enable responses from authoritative server ($n)"
 ret=0
 $DIG -p ${PORT} @10.53.0.2 txt enable  > dig.out.test$n
 grep "ANSWER: 1," dig.out.test$n > /dev/null || ret=1
 grep "TXT.\"1\"" dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-sleep 2
-
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "prime cache longttl.example (low max-stale-ttl) ($n)"
 ret=0
 $DIG -p ${PORT} @10.53.0.1 longttl.example TXT > dig.out.test$n
 grep "status: NOERROR" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 1," dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "prime cache data.example (low max-stale-ttl) ($n)"
 ret=0
 $DIG -p ${PORT} @10.53.0.1 data.example TXT > dig.out.test$n
 grep "status: NOERROR" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 1," dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "prime cache othertype.example (low max-stale-ttl) ($n)"
 ret=0
 $DIG -p ${PORT} @10.53.0.1 othertype.example CAA > dig.out.test$n
 grep "status: NOERROR" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 1," dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "prime cache nodata.example (low max-stale-ttl) ($n)"
 ret=0
 $DIG -p ${PORT} @10.53.0.1 nodata.example TXT > dig.out.test$n
 grep "status: NOERROR" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 0," dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "prime cache nxdomain.example (low max-stale-ttl) ($n)"
 ret=0
 $DIG -p ${PORT} @10.53.0.1 nxdomain.example TXT > dig.out.test$n
 grep "status: NXDOMAIN" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 0," dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "verify prime cache statistics (low max-stale-ttl) ($n)"
 ret=0
 rm -f ns1/named.stats
@@ -484,61 +505,69 @@ grep "2 TXT" ns1/named.stats.$n.cachedb > /dev/null || ret=1
 grep "1 Others" ns1/named.stats.$n.cachedb > /dev/null || ret=1
 grep "1 !TXT" ns1/named.stats.$n.cachedb > /dev/null || ret=1
 grep "1 NXDOMAIN" ns1/named.stats.$n.cachedb > /dev/null || ret=1
-status=`expr $status + $ret`
+status=$((status+ret))
 if [ $ret != 0 ]; then echo_i "failed"; fi
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "disable responses from authoritative server ($n)"
 ret=0
 $DIG -p ${PORT} @10.53.0.2 txt disable  > dig.out.test$n
 grep "ANSWER: 1," dig.out.test$n > /dev/null || ret=1
 grep "TXT.\"0\"" dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
 sleep 2
 
-n=`expr $n + 1`
+echo_i "sending queries for tests $((n+1))-$((n+4))..."
+$DIG -p ${PORT} @10.53.0.1 data.example TXT > dig.out.test$((n+1)) &
+$DIG -p ${PORT} @10.53.0.1 othertype.example CAA > dig.out.test$((n+2)) &
+$DIG -p ${PORT} @10.53.0.1 nodata.example TXT > dig.out.test$((n+3)) &
+$DIG -p ${PORT} @10.53.0.1 nxdomain.example TXT > dig.out.test$((n+4))
+
+# ensure all files have been written before proceeding
+waitfile dig.out.test$((n+1))
+waitfile dig.out.test$((n+2))
+waitfile dig.out.test$((n+3))
+waitfile dig.out.test$((n+4))
+
+n=$((n+1))
 echo_i "check stale data.example (low max-stale-ttl) ($n)"
 ret=0
-$DIG -p ${PORT} @10.53.0.1 data.example TXT > dig.out.test$n
 grep "status: NOERROR" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 1," dig.out.test$n > /dev/null || ret=1
 grep "data\.example\..*3.*IN.*TXT.*A text record with a 2 second ttl" dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "check stale othertype.example (low max-stale-ttl) ($n)"
 ret=0
-$DIG -p ${PORT} @10.53.0.1 othertype.example CAA > dig.out.test$n
 grep "status: NOERROR" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 1," dig.out.test$n > /dev/null || ret=1
 grep "othertype\.example\..*3.*IN.*CAA.*0.*issue" dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "check stale nodata.example (low max-stale-ttl) ($n)"
 ret=0
-$DIG -p ${PORT} @10.53.0.1 nodata.example TXT > dig.out.test$n
 grep "status: NOERROR" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 0," dig.out.test$n > /dev/null || ret=1
 grep "example\..*3.*IN.*SOA" dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "check stale nxdomain.example (low max-stale-ttl) ($n)"
 ret=0
-$DIG -p ${PORT} @10.53.0.1 nxdomain.example TXT > dig.out.test$n
 grep "status: NXDOMAIN" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 0," dig.out.test$n > /dev/null || ret=1
 grep "example\..*3.*IN.*SOA" dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "verify stale cache statistics (low max-stale-ttl) ($n)"
 ret=0
 rm -f ns1/named.stats
@@ -555,71 +584,77 @@ grep "1 #Others" ns1/named.stats.$n.cachedb > /dev/null || ret=1
 grep "1 #!TXT" ns1/named.stats.$n.cachedb > /dev/null || ret=1
 grep "1 #NXDOMAIN" ns1/named.stats.$n.cachedb > /dev/null || ret=1
 
-status=`expr $status + $ret`
+status=$((status+ret))
 if [ $ret != 0 ]; then echo_i "failed"; fi
 
-sleep 2
+echo_i "sending queries for tests $((n+1))-$((n+4))..."
+$DIG -p ${PORT} @10.53.0.1 data.example TXT > dig.out.test$((n+1)) &
+$DIG -p ${PORT} @10.53.0.1 othertype.example CAA > dig.out.test$((n+2)) &
+$DIG -p ${PORT} @10.53.0.1 nodata.example TXT > dig.out.test$((n+3)) &
+$DIG -p ${PORT} @10.53.0.1 nxdomain.example TXT > dig.out.test$((n+4))
 
-n=`expr $n + 1`
+# ensure all files have been written before proceeding
+waitfile dig.out.test$((n+1))
+waitfile dig.out.test$((n+2))
+waitfile dig.out.test$((n+3))
+waitfile dig.out.test$((n+4))
+
+n=$((n+1))
 echo_i "check ancient data.example (low max-stale-ttl) ($n)"
 ret=0
-$DIG -p ${PORT} @10.53.0.1 data.example TXT > dig.out.test$n
 grep "status: SERVFAIL" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 0," dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "check ancient othertype.example (low max-stale-ttl) ($n)"
 ret=0
-$DIG -p ${PORT} @10.53.0.1 othertype.example CAA > dig.out.test$n
 grep "status: SERVFAIL" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 0," dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "check ancient nodata.example (low max-stale-ttl) ($n)"
 ret=0
-$DIG -p ${PORT} @10.53.0.1 nodata.example TXT > dig.out.test$n
 grep "status: SERVFAIL" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 0," dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "check ancient nxdomain.example (low max-stale-ttl) ($n)"
 ret=0
-$DIG -p ${PORT} @10.53.0.1 nxdomain.example TXT > dig.out.test$n
 grep "status: SERVFAIL" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 0," dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
 #
 # Now test server with no serve-stale options set.
 #
 echo_i "test server with no serve-stale options set"
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "enable responses from authoritative server ($n)"
 ret=0
 $DIG -p ${PORT} @10.53.0.2 txt enable  > dig.out.test$n
 grep "ANSWER: 1," dig.out.test$n > /dev/null || ret=1
 grep "TXT.\"1\"" dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "prime cache longttl.example (max-stale-ttl default) ($n)"
 ret=0
 $DIG -p ${PORT} @10.53.0.3 longttl.example TXT > dig.out.test$n
 grep "status: NOERROR" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 1," dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "prime cache data.example (max-stale-ttl default) ($n)"
 ret=0
 $DIG -p ${PORT} @10.53.0.3 data.example TXT > dig.out.test$n
@@ -627,9 +662,9 @@ grep "status: NOERROR" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 1," dig.out.test$n > /dev/null || ret=1
 grep "data\.example\..*2.*IN.*TXT.*A text record with a 2 second ttl" dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "prime cache othertype.example (max-stale-ttl default) ($n)"
 ret=0
 $DIG -p ${PORT} @10.53.0.3 othertype.example CAA > dig.out.test$n
@@ -637,9 +672,9 @@ grep "status: NOERROR" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 1," dig.out.test$n > /dev/null || ret=1
 grep "othertype\.example\..*2.*IN.*CAA.*0.*issue" dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "prime cache nodata.example (max-stale-ttl default) ($n)"
 ret=0
 $DIG -p ${PORT} @10.53.0.3 nodata.example TXT > dig.out.test$n
@@ -647,9 +682,9 @@ grep "status: NOERROR" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 0," dig.out.test$n > /dev/null || ret=1
 grep "example\..*2.*IN.*SOA" dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "prime cache nxdomain.example (max-stale-ttl default) ($n)"
 ret=0
 $DIG -p ${PORT} @10.53.0.3 nxdomain.example TXT > dig.out.test$n
@@ -657,9 +692,9 @@ grep "status: NXDOMAIN" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 0," dig.out.test$n > /dev/null || ret=1
 grep "example\..*2.*IN.*SOA" dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "verify prime cache statistics (max-stale-ttl default) ($n)"
 ret=0
 rm -f ns3/named.stats
@@ -673,65 +708,73 @@ grep "2 TXT" ns3/named.stats.$n.cachedb > /dev/null || ret=1
 grep "1 Others" ns3/named.stats.$n.cachedb > /dev/null || ret=1
 grep "1 !TXT" ns3/named.stats.$n.cachedb > /dev/null || ret=1
 grep "1 NXDOMAIN" ns3/named.stats.$n.cachedb > /dev/null || ret=1
-status=`expr $status + $ret`
+status=$((status+ret))
 if [ $ret != 0 ]; then echo_i "failed"; fi
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "disable responses from authoritative server ($n)"
 ret=0
 $DIG -p ${PORT} @10.53.0.2 txt disable  > dig.out.test$n
 grep "ANSWER: 1," dig.out.test$n > /dev/null || ret=1
 grep "TXT.\"0\"" dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-sleep 2
-
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "check 'rndc serve-stale status' ($n)"
 ret=0
 $RNDCCMD 10.53.0.3 serve-stale status > rndc.out.test$n 2>&1 || ret=1
 grep '_default: off (stale-answer-ttl=1 max-stale-ttl=604800)' rndc.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+sleep 2
+
+echo_i "sending queries for tests $((n+1))-$((n+4))..."
+$DIG -p ${PORT} @10.53.0.3 data.example TXT > dig.out.test$((n+1)) &
+$DIG -p ${PORT} @10.53.0.3 othertype.example CAA > dig.out.test$((n+2)) &
+$DIG -p ${PORT} @10.53.0.3 nodata.example TXT > dig.out.test$((n+3)) &
+$DIG -p ${PORT} @10.53.0.3 nxdomain.example TXT > dig.out.test$((n+4))
+
+# ensure all files have been written before proceeding
+waitfile dig.out.test$((n+1))
+waitfile dig.out.test$((n+2))
+waitfile dig.out.test$((n+3))
+waitfile dig.out.test$((n+4))
+
+n=$((n+1))
 echo_i "check fail of data.example (max-stale-ttl default) ($n)"
 ret=0
-$DIG -p ${PORT} @10.53.0.3 data.example TXT > dig.out.test$n
 grep "status: SERVFAIL" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 0," dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "check fail of othertype.example (max-stale-ttl default) ($n)"
 ret=0
-$DIG -p ${PORT} @10.53.0.3 othertype.example CAA > dig.out.test$n
 grep "status: SERVFAIL" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 0," dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "check fail of nodata.example (max-stale-ttl default) ($n)"
 ret=0
-$DIG -p ${PORT} @10.53.0.3 nodata.example TXT > dig.out.test$n
 grep "status: SERVFAIL" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 0," dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "check fail of nxdomain.example (max-stale-ttl default) ($n)"
 ret=0
-$DIG -p ${PORT} @10.53.0.3 nxdomain.example TXT > dig.out.test$n
 grep "status: SERVFAIL" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 0," dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "verify stale cache statistics (max-stale-ttl default) ($n)"
 ret=0
 rm -f ns3/named.stats
@@ -748,88 +791,98 @@ grep "1 #Others" ns3/named.stats.$n.cachedb > /dev/null || ret=1
 grep "1 #!TXT" ns3/named.stats.$n.cachedb > /dev/null || ret=1
 grep "1 #NXDOMAIN" ns3/named.stats.$n.cachedb > /dev/null || ret=1
 
-status=`expr $status + $ret`
+status=$((status+ret))
 if [ $ret != 0 ]; then echo_i "failed"; fi
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "check 'rndc serve-stale on' ($n)"
 ret=0
 $RNDCCMD 10.53.0.3 serve-stale on > rndc.out.test$n 2>&1 || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "check 'rndc serve-stale status' ($n)"
 ret=0
 $RNDCCMD 10.53.0.3 serve-stale status > rndc.out.test$n 2>&1 || ret=1
 grep '_default: on (rndc) (stale-answer-ttl=1 max-stale-ttl=604800)' rndc.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+sleep 2
+
+echo_i "sending queries for tests $((n+1))-$((n+4))..."
+$DIG -p ${PORT} @10.53.0.3 data.example TXT > dig.out.test$((n+1)) &
+$DIG -p ${PORT} @10.53.0.3 othertype.example CAA > dig.out.test$((n+2)) &
+$DIG -p ${PORT} @10.53.0.3 nodata.example TXT > dig.out.test$((n+3)) &
+$DIG -p ${PORT} @10.53.0.3 nxdomain.example TXT > dig.out.test$((n+4))
+
+# ensure all files have been written before proceeding
+waitfile dig.out.test$((n+1))
+waitfile dig.out.test$((n+2))
+waitfile dig.out.test$((n+3))
+waitfile dig.out.test$((n+4))
+
+n=$((n+1))
 echo_i "check data.example (max-stale-ttl default) ($n)"
 ret=0
-$DIG -p ${PORT} @10.53.0.3 data.example TXT > dig.out.test$n
 grep "status: NOERROR" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 1," dig.out.test$n > /dev/null || ret=1
 grep "data\.example\..*1.*IN.*TXT.*A text record with a 2 second ttl" dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "check othertype.example (max-stale-ttl default) ($n)"
 ret=0
-$DIG -p ${PORT} @10.53.0.3 othertype.example CAA > dig.out.test$n
 grep "status: NOERROR" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 1," dig.out.test$n > /dev/null || ret=1
 grep "example\..*1.*IN.*CAA.*0.*issue" dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "check nodata.example (max-stale-ttl default) ($n)"
 ret=0
-$DIG -p ${PORT} @10.53.0.3 nodata.example TXT > dig.out.test$n
 grep "status: NOERROR" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 0," dig.out.test$n > /dev/null || ret=1
 grep "example\..*1.*IN.*SOA" dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "check nxdomain.example (max-stale-ttl default) ($n)"
 ret=0
-$DIG -p ${PORT} @10.53.0.3 nxdomain.example TXT > dig.out.test$n
 grep "status: NXDOMAIN" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 0," dig.out.test$n > /dev/null || ret=1
 grep "example\..*1.*IN.*SOA" dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
 #
 # Now test server with serve-stale disabled.
 #
 echo_i "test server with serve-stale disabled"
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "enable responses from authoritative server ($n)"
 ret=0
 $DIG -p ${PORT} @10.53.0.2 txt enable  > dig.out.test$n
 grep "ANSWER: 1," dig.out.test$n > /dev/null || ret=1
 grep "TXT.\"1\"" dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "prime cache longttl.example (serve-stale disabled) ($n)"
 ret=0
 $DIG -p ${PORT} @10.53.0.4 longttl.example TXT > dig.out.test$n
 grep "status: NOERROR" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 1," dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "prime cache data.example (serve-stale disabled) ($n)"
 ret=0
 $DIG -p ${PORT} @10.53.0.4 data.example TXT > dig.out.test$n
@@ -837,9 +890,9 @@ grep "status: NOERROR" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 1," dig.out.test$n > /dev/null || ret=1
 grep "data\.example\..*2.*IN.*TXT.*A text record with a 2 second ttl" dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "prime cache othertype.example (serve-stale disabled) ($n)"
 ret=0
 $DIG -p ${PORT} @10.53.0.4 othertype.example CAA > dig.out.test$n
@@ -847,9 +900,9 @@ grep "status: NOERROR" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 1," dig.out.test$n > /dev/null || ret=1
 grep "othertype\.example\..*2.*IN.*CAA.*0.*issue" dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "prime cache nodata.example (serve-stale disabled) ($n)"
 ret=0
 $DIG -p ${PORT} @10.53.0.4 nodata.example TXT > dig.out.test$n
@@ -857,9 +910,9 @@ grep "status: NOERROR" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 0," dig.out.test$n > /dev/null || ret=1
 grep "example\..*2.*IN.*SOA" dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "prime cache nxdomain.example (serve-stale disabled) ($n)"
 ret=0
 $DIG -p ${PORT} @10.53.0.4 nxdomain.example TXT > dig.out.test$n
@@ -867,9 +920,9 @@ grep "status: NXDOMAIN" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 0," dig.out.test$n > /dev/null || ret=1
 grep "example\..*2.*IN.*SOA" dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "verify prime cache statistics (serve-stale disabled) ($n)"
 ret=0
 rm -f ns4/named.stats
@@ -883,65 +936,73 @@ grep "2 TXT" ns4/named.stats.$n.cachedb > /dev/null || ret=1
 grep "1 Others" ns4/named.stats.$n.cachedb > /dev/null || ret=1
 grep "1 !TXT" ns4/named.stats.$n.cachedb > /dev/null || ret=1
 grep "1 NXDOMAIN" ns4/named.stats.$n.cachedb > /dev/null || ret=1
-status=`expr $status + $ret`
+status=$((status+ret))
 if [ $ret != 0 ]; then echo_i "failed"; fi
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "disable responses from authoritative server ($n)"
 ret=0
 $DIG -p ${PORT} @10.53.0.2 txt disable  > dig.out.test$n
 grep "ANSWER: 1," dig.out.test$n > /dev/null || ret=1
 grep "TXT.\"0\"" dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-sleep 2
-
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "check 'rndc serve-stale status' ($n)"
 ret=0
 $RNDCCMD 10.53.0.4 serve-stale status > rndc.out.test$n 2>&1 || ret=1
 grep '_default: off (stale-answer-ttl=1 max-stale-ttl=604800)' rndc.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+sleep 2
+
+echo_i "sending queries for tests $((n+1))-$((n+4))..."
+$DIG -p ${PORT} @10.53.0.4 data.example TXT > dig.out.test$((n+1)) &
+$DIG -p ${PORT} @10.53.0.4 othertype.example CAA > dig.out.test$((n+2)) &
+$DIG -p ${PORT} @10.53.0.4 nodata.example TXT > dig.out.test$((n+3)) &
+$DIG -p ${PORT} @10.53.0.4 nxdomain.example TXT > dig.out.test$((n+4))
+
+# ensure all files have been written before proceeding
+waitfile dig.out.test$((n+1))
+waitfile dig.out.test$((n+2))
+waitfile dig.out.test$((n+3))
+waitfile dig.out.test$((n+4))
+
+n=$((n+1))
 echo_i "check fail of data.example (serve-stale disabled) ($n)"
 ret=0
-$DIG -p ${PORT} @10.53.0.4 data.example TXT > dig.out.test$n
 grep "status: SERVFAIL" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 0," dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "check fail of othertype.example (serve-stale disabled) ($n)"
 ret=0
-$DIG -p ${PORT} @10.53.0.4 othertype.example CAA > dig.out.test$n
 grep "status: SERVFAIL" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 0," dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "check fail of nodata.example (serve-stale disabled) ($n)"
 ret=0
-$DIG -p ${PORT} @10.53.0.4 nodata.example TXT > dig.out.test$n
 grep "status: SERVFAIL" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 0," dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "check fail of nxdomain.example (serve-stale disabled) ($n)"
 ret=0
-$DIG -p ${PORT} @10.53.0.4 nxdomain.example TXT > dig.out.test$n
 grep "status: SERVFAIL" dig.out.test$n > /dev/null || ret=1
 grep "ANSWER: 0," dig.out.test$n > /dev/null || ret=1
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "verify stale cache statistics (serve-stale disabled) ($n)"
 ret=0
 rm -f ns4/named.stats
@@ -957,11 +1018,11 @@ grep "1 #TXT" ns4/named.stats.$n.cachedb > /dev/null || ret=1
 grep "1 #Others" ns4/named.stats.$n.cachedb > /dev/null || ret=1
 grep "1 #!TXT" ns4/named.stats.$n.cachedb > /dev/null || ret=1
 grep "1 #NXDOMAIN" ns4/named.stats.$n.cachedb > /dev/null || ret=1
-status=`expr $status + $ret`
+status=$((status+ret))
 if [ $ret != 0 ]; then echo_i "failed"; fi
 
 # Dump the cache.
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "dump the cache (serve-stale disabled) ($n)"
 ret=0
 $RNDCCMD 10.53.0.4 dumpdb -cache > rndc.out.test$n 2>&1 || ret=1
@@ -971,7 +1032,7 @@ for i in 0 1 2 3 4 5 6 7 8 9; do
 	if [ $done != 1 ]; then sleep 1; fi
 done
 if [ $done != 1 ]; then ret=1; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 if [ $ret != 0 ]; then echo_i "failed"; fi
 
 echo_i "stop ns4"
@@ -986,20 +1047,18 @@ LASTWEEK=`TZ=UTC perl -e 'my $now = time();
         my ($s, $m, $h, $d, $mo, $y) = (localtime($fiveMinutesAgo))[0, 1, 2, 3, 4, 5];
         printf("%04d%02d%02d%02d%02d%02d", $y+1900, $mo+1, $d, $h, $m, $s);'`
 
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "mock the cache date to $LASTWEEK (serve-stale disabled) ($n)"
 ret=0
 sed -E "s/DATE [0-9]{14}/DATE $LASTWEEK/g" ns4/named_dump4.db > ns4/named_dumpdb4.db.out || ret=1
 cp ns4/named_dumpdb4.db.out ns4/named_dumpdb4.db
 if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
+status=$((status+ret))
 
 echo_i "start ns4"
 $PERL $SYSTEMTESTTOP/start.pl --noclean --restart --port ${PORT} serve-stale ns4
 
-sleep 2
-
-n=`expr $n + 1`
+n=$((n+1))
 echo_i "verify ancient cache statistics (serve-stale disabled) ($n)"
 ret=0
 rm -f ns4/named.stats
@@ -1013,7 +1072,7 @@ grep "#TXT" ns4/named.stats.$n.cachedb > /dev/null && ret=1
 grep "#Others" ns4/named.stats.$n.cachedb > /dev/null && ret=1
 grep "#!TXT" ns4/named.stats.$n.cachedb > /dev/null && ret=1
 grep "#NXDOMAIN" ns4/named.stats.$n.cachedb > /dev/null && ret=1
-status=`expr $status + $ret`
+status=$((status+ret))
 if [ $ret != 0 ]; then echo_i "failed"; fi
 
 echo_i "exit status: $status"