Add new test cases with DNSSEC signing

kasp-max-types-per-name (named2.conf.in):
An unsigned zone with RR type count on a name right below the
configured limit. Then sign the zone using KASP. Adding a RRSIG would
push it over the RR type limit per name. Signing should fail, but
the server should not crash, nor end up in infinite resign-attempt loop.

kasp-max-records-per-type-dnskey (named1.conf.in):
Test with low max-record-per-rrset limit and a DNSSEC policy requiring
more than the limit. Signing should fail.

kasp-max-types-per-name (named1.conf.in):
Each RRSIG(covered type) is counted as an individual RR type. Test the
corner case where a signed zone, which is just below the limit-1,
adds a new type - doing so would trigger signing for the new type and
thus increase the number of "types" by 2, pushing it over the limit
again.

bin/tests/system/masterformat/ns4/compile.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+# shellcheck source=conf.sh
+. ../../conf.sh
+
+for zone in kasp-max-records-per-type \
+  kasp-max-records-per-type-dnskey \
+  kasp-max-types-per-name; do
+  $CHECKZONE -D -F raw -o $zone.db.raw $zone template.db >/dev/null 2>&1
+done

bin/tests/system/masterformat/ns4/kasp.db

@@ -0,0 +1,28 @@
+; Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+;
+; SPDX-License-Identifier: MPL-2.0
+;
+; This Source Code Form is subject to the terms of the Mozilla Public
+; License, v. 2.0.  If a copy of the MPL was not distributed with this
+; file, you can obtain one at https://mozilla.org/MPL/2.0/.
+;
+; See the COPYRIGHT file distributed with this work for additional
+; information regarding copyright ownership.
+
+$TTL 1D
+
+@			IN SOA	ns hostmaster (
+				1
+				3600
+				1800
+				1814400
+				3
+				)
+			NS	ns
+ns			A	10.53.0.1
+mx			MX	10 mail
+a			A	10.53.0.1
+aaaa			AAAA	2001:db8::53
+cname			CNAME	cname-target
+dname			DNAME	dname-target
+txt			TXT	"this is text"

bin/tests/system/masterformat/ns4/named1.conf.in

@@ -0,0 +1,89 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * SPDX-License-Identifier: MPL-2.0
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, you can obtain one at https://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+// NS4
+
+options {
+	pid-file "named.pid";
+	listen-on port @PORT@ { 10.53.0.4; };
+	port @PORT@;
+	listen-on-v6 { none; };
+	recursion no;
+	notify no;
+	session-keyfile "session.key";
+	servfail-ttl 0;
+	dnssec-validation no;
+
+	/* Ridicously low on purpose */
+	max-records-per-type 1;
+	max-types-per-name 11;
+};
+
+key rndc_key {
+	secret "1234abcd8765";
+	algorithm @DEFAULT_HMAC@;
+};
+
+controls {
+	inet 10.53.0.4 port @CONTROLPORT@ allow { any; } keys { rndc_key; };
+};
+
+dnssec-policy "masterformat" {
+	keys {
+		ksk key-directory lifetime unlimited algorithm @DEFAULT_ALGORITHM@;
+		zsk key-directory lifetime unlimited algorithm @DEFAULT_ALGORITHM@;
+	};
+};
+
+/*
+ * This one should be okay, since the default policy only introduces one DNSKEY
+ * and each signature covering a different type is considered a separate RRset.
+ */
+zone "kasp-max-records-per-type" {
+	type primary;
+	file "kasp-max-records-per-type.db.raw";
+	masterfile-format raw;
+	dnssec-policy "default";
+	inline-signing no;
+	allow-update { any; };
+	allow-transfer { any; };
+};
+
+/*
+ * This one uses a ZSK / KSK, so that is two records in one RRset,
+ * thus it should fail to sign.
+ */
+zone "kasp-max-records-per-type-dnskey" {
+	type primary;
+	file "kasp-max-records-per-type-dnskey.db.raw";
+	masterfile-format raw;
+	dnssec-policy "masterformat";
+	inline-signing no;
+	allow-update { any; };
+	allow-transfer { any; };
+};
+
+/*
+ * The template zone is fine and should be possible to sign, but when
+ * adding an extra type to the apex the max-types-per-name will be exceeded,
+ * meaning the update should fail.
+ */
+zone "kasp-max-types-per-name" {
+	type primary;
+	file "kasp-max-types-per-name.db.raw";
+	masterfile-format raw;
+	dnssec-policy "default";
+	inline-signing no;
+	allow-update { any; };
+	allow-transfer { any; };
+};

bin/tests/system/masterformat/ns4/named2.conf.in

@@ -0,0 +1,53 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * SPDX-License-Identifier: MPL-2.0
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, you can obtain one at https://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+// NS4
+
+options {
+	pid-file "named.pid";
+	listen-on port @PORT@ { 10.53.0.4; };
+	port @PORT@;
+	listen-on-v6 { none; };
+	recursion no;
+	notify no;
+	session-keyfile "session.key";
+	servfail-ttl 0;
+	dnssec-validation no;
+
+	/* Ridicously low on purpose */
+	max-records-per-type 1;
+	max-types-per-name 9;
+};
+
+key rndc_key {
+	secret "1234abcd8765";
+	algorithm @DEFAULT_HMAC@;
+};
+
+controls {
+	inet 10.53.0.4 port @CONTROLPORT@ allow { any; } keys { rndc_key; };
+};
+
+/*
+ * The template zone is fine, but when adding the DNSSEC records to the apex,
+ * the max-types-per-name will be exceeded, meaning signing should fail.
+ */
+zone "kasp-max-types-per-name" {
+	type primary;
+	file "kasp-max-types-per-name.db.raw";
+	masterfile-format raw;
+	dnssec-policy "default";
+	inline-signing no;
+	allow-update { any; };
+	allow-transfer { any; };
+};

bin/tests/system/masterformat/ns4/template.db

@@ -0,0 +1,28 @@
+; Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+;
+; SPDX-License-Identifier: MPL-2.0
+;
+; This Source Code Form is subject to the terms of the Mozilla Public
+; License, v. 2.0.  If a copy of the MPL was not distributed with this
+; file, you can obtain one at https://mozilla.org/MPL/2.0/.
+;
+; See the COPYRIGHT file distributed with this work for additional
+; information regarding copyright ownership.
+
+$TTL 1D
+
+@			IN SOA	ns hostmaster (
+				1
+				3600
+				1800
+				1814400
+				3
+				)
+			NS	ns
+ns			A	10.53.0.1
+mx			MX	10 mail
+a			A	10.53.0.1
+aaaa			AAAA	2001:db8::53
+cname			CNAME	cname-target
+dname			DNAME	dname-target
+txt			TXT	"this is text"

bin/tests/system/masterformat/setup.sh

@@ -19,6 +19,7 @@ $SHELL clean.sh
 copy_setports ns1/named.conf.in ns1/named.conf
 copy_setports ns2/named.conf.in ns2/named.conf
 copy_setports ns3/named.conf.in ns3/named.conf
+copy_setports ns4/named1.conf.in ns4/named.conf
 
 cp ns1/example.db ns2/
 cp ns2/formerly-text.db.in ns2/formerly-text.db
@@ -57,4 +58,5 @@ for ntype in $(seq 65280 65534); do
   echo "m TYPE${ntype} \# 0"
 done >>ns1/255types.db
 echo "m TXT bunny" >>ns1/255types.db
-cd ns1 && $SHELL compile.sh
+(cd ns1 && $SHELL compile.sh)
+(cd ns4 && $SHELL compile.sh)

bin/tests/system/masterformat/tests.sh

@@ -301,7 +301,6 @@ n=$((n + 1))
 [ $ret -eq 0 ] || echo_i "failed"
 status=$((status + ret))
 
-
 echo_i "checking that over-limit rdatasets not loaded ($n)"
 for _attempt in 0 1 2 3 4 5 6 7 8 9; do
   ret=0
@@ -316,6 +315,102 @@ n=$((n + 1))
 [ $ret -eq 0 ] || echo_i "failed"
 status=$((status + ret))
 
+echo_i "checking that kasp-max-records-per-type rdatasets loaded ($n)"
+for _attempt in 0 1 2 3 4 5 6 7 8 9; do
+  ret=0
+  for rrtype in soa dnskey ns; do
+    $DIG +tcp +dnssec $rrtype "kasp-max-records-per-type" @10.53.0.4 -p "${PORT}" >"dig.out.ns4.$rrtype.test$n"
+    grep "status: NOERROR" "dig.out.ns4.$rrtype.test$n" >/dev/null || ret=1
+    grep "RRSIG" "dig.out.ns4.$rrtype.test$n" >/dev/null || ret=1
+  done
+  [ $ret -eq 0 ] && break
+  sleep 1
+done
+n=$((n + 1))
+[ $ret -eq 0 ] || echo_i "failed"
+status=$((status + ret))
+
+echo_i "checking that kasp-max-records-per-type-dnskey rdatasets not signed ($n)"
+for _attempt in 0 1 2 3 4 5 6 7 8 9; do
+  ret=0
+  for rrtype in soa dnskey ns; do
+    $DIG +tcp +dnssec $rrtype "kasp-max-records-per-type-dnskey" @10.53.0.4 -p "${PORT}" >"dig.out.ns4.$rrtype.test$n"
+    grep "status: NOERROR" "dig.out.ns4.$rrtype.test$n" >/dev/null || ret=1
+    grep "RRSIG" "dig.out.ns4.$rrtype.test$n" >/dev/null && ret=1
+  done
+  [ $ret -eq 0 ] && break
+  sleep 1
+done
+n=$((n + 1))
+[ $ret -eq 0 ] || echo_i "failed"
+status=$((status + ret))
+
+echo_i "checking that kasp-max-types-per-name rdatasets loaded ($n)"
+for _attempt in 0 1 2 3 4 5 6 7 8 9; do
+  ret=0
+  for rrtype in soa dnskey ns; do
+    $DIG +tcp +dnssec $rrtype "kasp-max-types-per-name" @10.53.0.4 -p "${PORT}" >"dig.out.ns4.$rrtype.test$n"
+    grep "status: NOERROR" "dig.out.ns4.$rrtype.test$n" >/dev/null || ret=1
+    grep "RRSIG" "dig.out.ns4.$rrtype.test$n" >/dev/null || ret=1
+  done
+  [ $ret -eq 0 ] && break
+  sleep 1
+done
+n=$((n + 1))
+[ $ret -eq 0 ] || echo_i "failed"
+status=$((status + ret))
+
+# Update zone with nsupdate.
+n=$((n + 1))
+echo_i "add new type to zone and check that it fails ($n)"
+ret=0
+(
+  echo zone kasp-max-types-per-name.
+  echo server 10.53.0.4 "$PORT"
+  echo update add kasp-max-types-per-name. 300 TXT KAPUTT
+  echo send
+) | $NSUPDATE && ret=1
+n=$((n + 1))
+[ $ret -eq 0 ] || echo_i "failed"
+status=$((status + ret))
+
+echo_i "checking that kasp-max-types-per-name rdatasets loaded ($n)"
+for _attempt in 0 1 2 3 4 5 6 7 8 9; do
+  ret=0
+  for rrtype in soa dnskey ns txt; do
+    $DIG +tcp +dnssec $rrtype "kasp-max-types-per-name" @10.53.0.4 -p "${PORT}" >"dig.out.ns4.$rrtype.test$n"
+    grep "status: NOERROR" "dig.out.ns4.$rrtype.test$n" >/dev/null || ret=1
+    grep "KAPUTT" "dig.out.ns4.$rrtype.test$n" >/dev/null && ret=1
+  done
+  [ $ret -eq 0 ] && break
+  sleep 1
+done
+n=$((n + 1))
+[ $ret -eq 0 ] || echo_i "failed"
+status=$((status + ret))
+
+# Reconfigure ns4
+echo_i "reconfigure ns4"
+stop_server ns4
+copy_setports ns4/named2.conf.in ns4/named.conf
+# Recompile zone
+$CHECKZONE -D -F raw -o ns4/kasp.db.raw kasp-max-types-per-name ns4/template.db >/dev/null 2>&1
+start_server --noclean --restart --port "${PORT}" ns4
+
+echo_i "checking that kasp-max-types-per-name rdatasets not loaded ($n)"
+for _attempt in 0 1 2 3 4 5 6 7 8 9; do
+  ret=0
+  for rrtype in soa dnskey ns; do
+    $DIG +tcp +dnssec $rrtype "kasp-max-types-per-name" @10.53.0.4 -p "${PORT}" >"dig.out.ns4.$rrtype.test$n"
+    grep "status: SERVFAIL" "dig.out.ns4.$rrtype.test$n" >/dev/null || ret=1
+  done
+  [ $ret -eq 0 ] && break
+  sleep 1
+done
+n=$((n + 1))
+[ $ret -eq 0 ] || echo_i "failed"
+status=$((status + ret))
+
 echo_i "checking that 255 types are loaded ($n)"
 for _attempt in 0 1 2 3 4 5 6 7 8 9; do
   ret=0