Use optimal message sizes to improve compression in AXFRs (#40996)

CHANGES

@@ -1,3 +1,6 @@
+4280.	[performance]	Use optimal message sizes to improve compression
+			in AXFRs. This reduces network traffic. [RT #40996]
+
 4279.	[test]		Don't use fixed ports when unit testing. [RT #41194]
 
 4278.	[bug]		'delv +short +[no]split[=##]' didn't work as expected.

bin/named/config.c

@@ -113,6 +113,7 @@ options {\n\
 #	tkey-dhkey <none>\n\
 #	tkey-gssapi-credential <none>\n\
 #	tkey-domain <none>\n\
+	transfer-message-size 20480;\n\
 	transfers-per-ns 2;\n\
 	transfers-in 10;\n\
 	transfers-out 10;\n\

bin/named/include/named/server.h

@@ -126,6 +126,8 @@ struct ns_server {
 	dns_dtenv_t		*dtenv;		/*%< Dnstap environment */
 
 	char *			lockfile;
+
+	isc_uint16_t		transfer_tcp_message_size;
 };
 
 #define NS_SERVER_MAGIC			ISC_MAGIC('S','V','E','R')

bin/named/named.conf.docbook

@@ -237,6 +237,7 @@ options {
 	tkey-gssapi-credential <replaceable>quoted_string</replaceable>;
 	tkey-gssapi-keytab <replaceable>quoted_string</replaceable>;
 	tkey-domain <replaceable>quoted_string</replaceable>;
+	transfer-message-size <replaceable>integer</replaceable>;
 	transfers-per-ns <replaceable>integer</replaceable>;
 	transfers-in <replaceable>integer</replaceable>;
 	transfers-out <replaceable>integer</replaceable>;

bin/named/server.c

@@ -5814,6 +5814,7 @@ load_configuration(const char *filename, ns_server_t *server,
 	isc_uint32_t interface_interval;
 	isc_uint32_t reserved;
 	isc_uint32_t udpsize;
+	isc_uint32_t transfer_message_size;
 	ns_cache_t *nsc;
 	ns_cachelist_t cachelist, tmpcachelist;
 	ns_cfgctx_t *nzctx;
@@ -6128,6 +6129,17 @@ load_configuration(const char *filename, ns_server_t *server,
 		udpsize = 4096;
 	ns_g_udpsize = (isc_uint16_t)udpsize;
 
+	/* Set the transfer message size for TCP */
+	obj = NULL;
+	result = ns_config_get(maps, "transfer-message-size", &obj);
+	INSIST(result == ISC_R_SUCCESS);
+	transfer_message_size = cfg_obj_asuint32(obj);
+	if (transfer_message_size < 512)
+		transfer_message_size = 512;
+	else if (transfer_message_size > 65535)
+		transfer_message_size = 65535;
+	server->transfer_tcp_message_size = (isc_uint16_t) transfer_message_size;
+
 	/*
 	 * Configure the zone manager.
 	 */

bin/named/xfrout.c

@@ -1273,6 +1273,7 @@ sendstream(xfrout_ctx_t *xfr) {
 	dns_rdataset_t *msgrds = NULL;
 	dns_compress_t cctx;
 	isc_boolean_t cleanup_cctx = ISC_FALSE;
+	isc_boolean_t is_tcp;
 
 	int n_rrs;
 
@@ -1280,7 +1281,8 @@ sendstream(xfrout_ctx_t *xfr) {
 	isc_buffer_clear(&xfr->txlenbuf);
 	isc_buffer_clear(&xfr->txbuf);
 
-	if ((xfr->client->attributes & NS_CLIENTATTR_TCP) == 0) {
+	is_tcp = ISC_TF((xfr->client->attributes & NS_CLIENTATTR_TCP) != 0);
+	if (!is_tcp) {
 		/*
 		 * In the UDP case, we put the response data directly into
 		 * the client message.
@@ -1482,9 +1484,17 @@ sendstream(xfrout_ctx_t *xfr) {
 
 		if (! xfr->many_answers)
 			break;
+		/*
+		 * At this stage, at least 1 RR has been rendered into
+		 * the message. Check if we want to clamp this message
+		 * here (TCP only).
+		 */
+		if ((isc_buffer_usedlength(&xfr->buf) >=
+		     ns_g_server->transfer_tcp_message_size) && is_tcp)
+			break;
 	}
 
-	if ((xfr->client->attributes & NS_CLIENTATTR_TCP) != 0) {
+	if (is_tcp) {
 		CHECK(dns_compress_init(&cctx, -1, xfr->mctx));
 		dns_compress_setsensitive(&cctx, ISC_TRUE);
 		cleanup_cctx = ISC_TRUE;

bin/tests/system/xfer/clean.sh

@@ -24,6 +24,7 @@
 rm -f dig.out.ns1 dig.out.ns2 dig.out.ns3 dig.out.ns4
 rm -f dig.out.ns5 dig.out.ns6 dig.out.ns7
 rm -f dig.out.soa.ns3
+rm -f dig.out.msgsize
 rm -f axfr.out
 rm -f ns1/slave.db ns2/slave.db
 rm -f ns1/edns-expire.db
@@ -33,6 +34,7 @@ rm -f ns3/master.bk ns3/master.bk.jnl
 rm -f ns4/named.conf ns4/nil.db ns4/root.db
 rm -f ns6/*.db ns6/*.bk ns6/*.jnl
 rm -f ns7/*.db ns7/*.bk ns7/*.jnl
+rm -f ns8/large.db ns8/small.db
 
 rm -f */named.memstats
 rm -f */named.run

bin/tests/system/xfer/ns8/.gitignore

@@ -0,0 +1,2 @@
+/large.db
+/small.db

bin/tests/system/xfer/ns8/example.db

@@ -0,0 +1,28 @@
+; Copyright (C) 2004, 2007, 2011, 2012, 2014  Internet Systems Consortium, Inc. ("ISC")
+; Copyright (C) 2000, 2001  Internet Software Consortium.
+;
+; Permission to use, copy, modify, and/or distribute this software for any
+; purpose with or without fee is hereby granted, provided that the above
+; copyright notice and this permission notice appear in all copies.
+;
+; THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+; REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+; AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+; INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+; LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+; OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+; PERFORMANCE OF THIS SOFTWARE.
+
+$TTL 300	; 5 minutes
+@			SOA	mname1. . (
+				2000062101 ; serial
+				20         ; refresh (20 seconds)
+				20         ; retry (20 seconds)
+				1814400    ; expire (3 weeks)
+				3600       ; minimum (1 hour)
+				)
+			NS	ns
+ns			A	10.53.0.1
+
+$INCLUDE large.db
+$INCLUDE small.db

bin/tests/system/xfer/ns8/named.conf

@@ -0,0 +1,50 @@
+/*
+ * Copyright (C) 2004, 2007, 2011, 2014, 2015  Internet Systems Consortium, Inc. ("ISC")
+ * Copyright (C) 2000, 2001  Internet Software Consortium.
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
+include "../../common/rndc.key";
+
+controls {
+	inet 10.53.0.8 port 9953 allow { any; } keys { rndc_key; };
+};
+
+options {
+	query-source address 10.53.0.8;
+	notify-source 10.53.0.8;
+	transfer-source 10.53.0.8;
+	port 5300;
+	pid-file "named.pid";
+	listen-on { 10.53.0.8; };
+	listen-on-v6 { none; };
+	recursion no;
+	notify no;
+	transfer-message-size 1024;
+};
+
+key key1. {
+	algorithm hmac-md5;
+	secret "1234abcd8765";
+};
+
+acl tzkey {
+	key key1.;
+};
+
+zone "example." {
+	type master;
+	file "example.db";
+	allow-transfer { tzkey; };
+};

bin/tests/system/xfer/setup.sh

@@ -34,3 +34,6 @@ cp -f ns4/named.conf.base ns4/named.conf
 
 cp ns2/slave.db.in ns2/slave.db
 touch -t 200101010000 ns2/slave.db
+
+$PERL -e 'for ($i=0;$i<4096;$i++){ printf("name%u 259200 A 1.2.3.4\nname%u 259200 TXT \"Hello World %u\"\n", $i, $i, $i);}' > ns8/small.db
+$PERL -e 'printf("large IN TYPE45234 \\# 48000 "); for ($i=0;$i<16*3000;$i++) { printf("%02x", $i % 256); } printf("\n");' > ns8/large.db

bin/tests/system/xfer/tests.sh

@@ -380,5 +380,21 @@ test ${expire:-0} -gt 0 -a ${expire:-0} -lt 1814400 || {
     status=1
 }
 
+echo "I:test smaller transfer TCP message size"
+$DIG $DIGOPTS example. @10.53.0.8 axfr -p 5300 \
+	-y key1.:1234abcd8765 > dig.out.msgsize || status=1
+
+bytes=`wc -c < dig.out.msgsize`
+if [ $bytes -ne 459357 ]; then
+	echo "I:failed axfr size check"
+	status=1
+fi
+
+num_messages=`cat ns8/named.run | grep "sending TCP message of" | wc -l`
+if [ $num_messages -le 300 ]; then
+	echo "I:failed transfer message count check"
+	status=1
+fi
+
 echo "I:exit status: $status"
 exit $status

doc/arm/Bv9ARM-book.xml

@@ -4540,6 +4540,7 @@ badresp:1,adberr:0,findfail:0,valfail:0]
     <optional> serial-queries <replaceable>number</replaceable>; </optional>
     <optional> tcp-listen-queue <replaceable>number</replaceable>; </optional>
     <optional> transfer-format <replaceable>( one-answer | many-answers )</replaceable>; </optional>
+    <optional> transfer-message-size  <replaceable>number</replaceable>; </optional>
     <optional> transfers-in  <replaceable>number</replaceable>; </optional>
     <optional> transfers-out <replaceable>number</replaceable>; </optional>
     <optional> transfers-per-ns <replaceable>number</replaceable>; </optional>
@@ -7924,6 +7925,37 @@ avoid-v6-udp-ports {};
 	      </listitem>
 	    </varlistentry>
 
+	    <varlistentry>
+	      <term><command>transfer-message-size</command></term>
+	      <listitem>
+		<para>
+		  This is an upper bound on the uncompressed size of DNS
+		  messages used in zone transfers over TCP.  If a message
+		  grows larger than this size, additional messages will be
+		  used to complete the zone transfer.  (Note, however,
+		  that this is a hint, not a hard limit; if a message
+		  contains a single resource record whose RDATA does not
+		  fit within the size limit, a larger message will be
+		  permitted so the record can be transferred.)
+		</para>
+		<para>
+		  Valid values are between 512 and 65535 octets, and any
+		  values outside that range will be adjusted to the nearest
+		  value within it.  The default is <literal>20480</literal>,
+		  which was selected to improve message compression:
+		  most DNS messages of this size will compress to less
+		  than 16536 bytes.  Larger messages cannot be compressed
+		  as effectively, because 16536 is the largest permissible
+		  compression offset pointer in a DNS message.
+		</para>
+		<para>
+		  This option is mainly intended for server testing;
+		  there is rarely any benefit in setting a value other
+		  than the default.
+		</para>
+	      </listitem>
+	    </varlistentry>
+
 	    <varlistentry>
 	      <term><command>transfers-in</command></term>
 	      <listitem>

doc/misc/options

@@ -311,6 +311,7 @@ options {
             dscp <integer> ];
         transfer-source-v6 ( <ipv6_address> | * ) [ port ( <integer> | * )
             ] [ dscp <integer> ];
+        transfer-message-size <integer>;
         transfers-in <integer>;
         transfers-out <integer>;
         transfers-per-ns <integer>;

lib/isccfg/namedconf.c

@@ -1085,6 +1085,7 @@ options_clauses[] = {
 	{ "tkey-gssapi-credential", &cfg_type_qstring, 0 },
 	{ "tkey-gssapi-keytab", &cfg_type_qstring, 0 },
 	{ "tkey-domain", &cfg_type_qstring, 0 },
+	{ "transfer-message-size", &cfg_type_uint32, 0 },
 	{ "transfers-per-ns", &cfg_type_uint32, 0 },
 	{ "transfers-in", &cfg_type_uint32, 0 },
 	{ "transfers-out", &cfg_type_uint32, 0 },