Test infrastructure for the qp-trie

This change adds a number of support routines for the unit tests, and
for benchmarks and fuzz tests to be added later. It isn't necessary to
include the support routines in libdns, since they are not needed by
BIND's installed programs. So `libtest` seems like the best place for
them.

The tests themselves verify that dns_qpkey_fromname() behaves as
expected.

Makefile.am

@@ -1,11 +1,15 @@
 include $(top_srcdir)/Makefile.top
 
-SUBDIRS = . lib doc bin fuzz
+SUBDIRS = . lib doc
 
+# build libtest before fuzz/* and bin/tests
 if HAVE_CMOCKA
 SUBDIRS += tests
 endif HAVE_CMOCKA
 
+# run fuzz tests before system tests
+SUBDIRS += fuzz bin
+
 BUILT_SOURCES = bind.keys.h
 CLEANFILES = bind.keys.h
 

tests/dns/Makefile.am

@@ -30,6 +30,7 @@ check_PROGRAMS =		\
 	nsec3_test		\
 	nsec3param_test		\
 	private_test		\
+	qp_test			\
 	rbt_test		\
 	rbtdb_test		\
 	rdata_test		\

tests/dns/qp_test.c

@@ -0,0 +1,137 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * SPDX-License-Identifier: MPL-2.0
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, you can obtain one at https://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+#include <sched.h> /* IWYU pragma: keep */
+#include <setjmp.h>
+#include <stdarg.h>
+#include <stddef.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#define UNIT_TESTING
+#include <cmocka.h>
+
+#include <isc/result.h>
+#include <isc/string.h>
+#include <isc/util.h>
+
+#include <dns/name.h>
+#include <dns/qp.h>
+
+#include <tests/dns.h>
+#include <tests/qp.h>
+
+ISC_RUN_TEST_IMPL(qpkey_name) {
+	struct {
+		const char *namestr;
+		uint8_t key[512];
+		size_t len;
+	} testcases[] = {
+		{
+			.namestr = ".",
+			.key = { 0x01, 0x01 },
+			.len = 1,
+		},
+		{
+			.namestr = "\\000",
+			.key = { 0x02, 0x02, 0x01, 0x01 },
+			.len = 3,
+		},
+		{
+			.namestr = "example.com.",
+			.key = { 0x01, 0x15, 0x21, 0x1f, 0x01, 0x17, 0x2a, 0x13,
+				 0x1f, 0x22, 0x1e, 0x17, 0x01, 0x01 },
+			.len = 13,
+		},
+		{
+			.namestr = "example.com",
+			.key = { 0x15, 0x21, 0x1f, 0x01, 0x17, 0x2a, 0x13, 0x1f,
+				 0x22, 0x1e, 0x17, 0x01, 0x01 },
+			.len = 12,
+		},
+		{
+			.namestr = "EXAMPLE.COM",
+			.key = { 0x15, 0x21, 0x1f, 0x01, 0x17, 0x2a, 0x13, 0x1f,
+				 0x22, 0x1e, 0x17, 0x01, 0x01 },
+			.len = 12,
+		},
+	};
+
+	for (size_t i = 0; i < ARRAY_SIZE(testcases); i++) {
+		size_t len;
+		dns_qpkey_t key;
+		dns_fixedname_t fn1, fn2;
+		dns_name_t *in = NULL, *out = NULL;
+
+		dns_test_namefromstring(testcases[i].namestr, &fn1);
+		in = dns_fixedname_name(&fn1);
+		len = dns_qpkey_fromname(key, in);
+
+		assert_true(testcases[i].len == len);
+		assert_true(memcmp(testcases[i].key, key, len) == 0);
+
+		out = dns_fixedname_initname(&fn2);
+		qp_test_keytoname(key, out);
+		assert_true(dns_name_equal(in, out));
+	}
+}
+
+ISC_RUN_TEST_IMPL(qpkey_sort) {
+	struct {
+		const char *namestr;
+		dns_name_t *name;
+		dns_fixedname_t fixed;
+		size_t len;
+		dns_qpkey_t key;
+	} testcases[] = {
+		{ .namestr = "." },
+		{ .namestr = "\\000." },
+		{ .namestr = "example.com." },
+		{ .namestr = "EXAMPLE.COM." },
+		{ .namestr = "www.example.com." },
+		{ .namestr = "exam.com." },
+		{ .namestr = "exams.com." },
+		{ .namestr = "exam\\000.com." },
+	};
+
+	for (size_t i = 0; i < ARRAY_SIZE(testcases); i++) {
+		dns_test_namefromstring(testcases[i].namestr,
+					&testcases[i].fixed);
+		testcases[i].name = dns_fixedname_name(&testcases[i].fixed);
+		testcases[i].len = dns_qpkey_fromname(testcases[i].key,
+						      testcases[i].name);
+	}
+
+	for (size_t i = 0; i < ARRAY_SIZE(testcases); i++) {
+		for (size_t j = 0; j < ARRAY_SIZE(testcases); j++) {
+			int namecmp = dns_name_compare(testcases[i].name,
+						       testcases[j].name);
+			size_t len = ISC_MIN(testcases[i].len,
+					     testcases[j].len);
+			/* include extra terminating NOBYTE */
+			int keycmp = memcmp(testcases[i].key, testcases[j].key,
+					    len + 1);
+			assert_true((namecmp < 0) == (keycmp < 0));
+			assert_true((namecmp == 0) == (keycmp == 0));
+			assert_true((namecmp > 0) == (keycmp > 0));
+		}
+	}
+}
+
+ISC_TEST_LIST_START
+ISC_TEST_ENTRY(qpkey_name)
+ISC_TEST_ENTRY(qpkey_sort)
+ISC_TEST_LIST_END
+
+ISC_TEST_MAIN

tests/include/tests/qp.h

@@ -0,0 +1,93 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * SPDX-License-Identifier: MPL-2.0
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, you can obtain one at https://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+#pragma once
+
+/*
+ * Cheap but inaccurate conversion from bit numbers to ascii: this is
+ * adequate for displaying the trie structure. It lacks any context
+ * that would be necessary for handling escapes, so unusual characters
+ * are fudged.
+ */
+uint8_t
+qp_test_bittoascii(uint8_t bit);
+
+/*
+ * Simple and incorrect key conversion for display purposes.
+ * Overwrites the key in place, and returns a pointer to the converted key.
+ */
+const char *
+qp_test_keytoascii(dns_qpkey_t key, size_t len);
+
+/*
+ * Convert a trie lookup key back into a DNS name. Unlike the previous
+ * functions, this is a complete inverse of dns_qpkey_fromname().
+ */
+void
+qp_test_keytoname(const dns_qpkey_t key, dns_name_t *name);
+
+/*
+ * The maximum height of the trie
+ */
+size_t
+qp_test_getheight(dns_qp_t *qp);
+
+/*
+ * The maximum length of any key in the trie (for comparison with the trie's
+ * height)
+ */
+size_t
+qp_test_maxkeylen(dns_qp_t *qp);
+
+/*
+ * Print dns_qp_t metadata to stdout
+ */
+void
+qp_test_dumpqp(dns_qp_t *qp);
+
+/*
+ * Print dns_qpread_t metadata to stdout
+ */
+void
+qp_test_dumpread(dns_qpreadable_t qp);
+
+/*
+ * Print dns_qpsnap_t metadata to stdout
+ */
+void
+qp_test_dumpsnap(dns_qpsnap_t *qps);
+
+/*
+ * Print dns_qpmulti_t metadata to stdout
+ */
+void
+qp_test_dumpmulti(dns_qpmulti_t *multi);
+
+/*
+ * Print dns_qp_t chunk arrays to stdout
+ */
+void
+qp_test_dumpchunks(dns_qp_t *qp);
+
+/*
+ * Print out the trie structure to stdout in an ad-hoc text format
+ * that uses indentation to indicate depth
+ */
+void
+qp_test_dumptrie(dns_qpreadable_t qp);
+
+/*
+ * Print out the trie structure to stdout in graphviz dot format
+ */
+void
+qp_test_dumpdot(dns_qp_t *qp);

tests/libtest/Makefile.am

@@ -5,21 +5,24 @@ AM_CPPFLAGS +=			\
 	$(LIBDNS_CFLAGS)	\
 	$(LIBNS_CFLAGS)		\
 	$(LIBUV_CFLAGS)		\
-	-I$(top_srcdir)/lib/isc
+	-I$(top_srcdir)/lib/isc	\
+	-I$(top_srcdir)/lib/dns
 
 LDADD +=			\
 	$(LIBISC_LIBS)		\
 	$(LIBDNS_LIBS)		\
 	$(LIBNS_LIBS)
 
-check_LTLIBRARIES = libtest.la
+noinst_LTLIBRARIES = libtest.la
 
 libtest_la_SOURCES =			\
 	../include/tests/dns.h		\
 	../include/tests/isc.h		\
 	../include/tests/ns.h		\
+	../include/tests/qp.h		\
 	dns.c				\
 	isc.c				\
-	ns.c
+	ns.c				\
+	qp.c
 
 include $(top_srcdir)/Makefile.tests

tests/libtest/qp.c

@@ -0,0 +1,395 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * SPDX-License-Identifier: MPL-2.0
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, you can obtain one at https://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+#include <assert.h>
+#include <stdint.h>
+#include <stdio.h>
+
+#include <isc/buffer.h>
+#include <isc/magic.h>
+#include <isc/refcount.h>
+#include <isc/rwlock.h>
+#include <isc/util.h>
+
+#include <dns/name.h>
+#include <dns/qp.h>
+#include <dns/types.h>
+
+#include "qp_p.h"
+
+#include <tests/qp.h>
+
+/***********************************************************************
+ *
+ *  key reverse conversions
+ */
+
+uint8_t
+qp_test_bittoascii(qp_shift_t bit) {
+	uint8_t byte = dns_qp_byte_for_bit[bit];
+	if (bit == SHIFT_NOBYTE) {
+		return ('.');
+	} else if (qp_common_character(byte)) {
+		return (byte);
+	} else if (byte < '-') {
+		return ('#');
+	} else if (byte < '_') {
+		return ('@');
+	} else {
+		return ('~' - SHIFT_OFFSET + bit);
+	}
+}
+
+const char *
+qp_test_keytoascii(dns_qpkey_t key, size_t len) {
+	for (size_t offset = 0; offset < len; offset++) {
+		key[offset] = qp_test_bittoascii(key[offset]);
+	}
+	key[len] = '\0';
+	return ((const char *)key);
+}
+
+void
+qp_test_keytoname(const dns_qpkey_t key, dns_name_t *name) {
+	size_t locs[128];
+	size_t loc = 0, opos = 0;
+	size_t offset;
+
+	REQUIRE(ISC_MAGIC_VALID(name, DNS_NAME_MAGIC));
+	REQUIRE(name->buffer != NULL);
+	REQUIRE(name->offsets != NULL);
+
+	isc_buffer_clear(name->buffer);
+
+	/* Scan the key looking for label boundaries */
+	for (offset = 0; offset < 512; offset++) {
+		INSIST(key[offset] >= SHIFT_NOBYTE &&
+		       key[offset] < SHIFT_OFFSET);
+		INSIST(loc < 128);
+		if (key[offset] == SHIFT_NOBYTE) {
+			if (key[offset + 1] == SHIFT_NOBYTE) {
+				locs[loc] = offset + 1;
+				break;
+			}
+			locs[loc++] = offset + 1;
+		} else if (offset == 0) {
+			/* This happens for a relative name */
+			locs[loc++] = offset;
+		}
+	}
+
+	/*
+	 * In the key the labels are encoded in reverse order, so
+	 * we step backward through the label boundaries, then forward
+	 * through the labels, to create the DNS wire format data.
+	 */
+	name->labels = loc;
+	while (loc-- > 0) {
+		uint8_t len = 0, *lenp = NULL;
+
+		/* Add a length byte to the name data and set an offset */
+		lenp = isc_buffer_used(name->buffer);
+		isc_buffer_putuint8(name->buffer, 0);
+		name->offsets[opos++] = name->length++;
+
+		/* Convert from escaped byte ranges to ASCII */
+		for (offset = locs[loc]; offset < locs[loc + 1] - 1; offset++) {
+			uint8_t byte = dns_qp_byte_for_bit[key[offset]];
+			if (qp_common_character(byte)) {
+				isc_buffer_putuint8(name->buffer, byte);
+			} else {
+				byte += key[++offset] - SHIFT_BITMAP;
+				isc_buffer_putuint8(name->buffer, byte);
+			}
+			len++;
+		}
+
+		name->length += len;
+		*lenp = len;
+	}
+
+	/* Add a root label for absolute names */
+	if (key[0] == SHIFT_NOBYTE) {
+		name->attributes.absolute = true;
+		isc_buffer_putuint8(name->buffer, 0);
+		name->length++;
+		name->labels++;
+	}
+
+	name->ndata = isc_buffer_base(name->buffer);
+}
+
+/***********************************************************************
+ *
+ *  trie properties
+ */
+
+static size_t
+getheight(dns_qp_t *qp, qp_node_t *n) {
+	if (!is_branch(n)) {
+		return (0);
+	}
+	size_t max_height = 0;
+	qp_weight_t size = branch_twigs_size(n);
+	qp_node_t *twigs = branch_twigs_vector(qp, n);
+	for (qp_weight_t pos = 0; pos < size; pos++) {
+		size_t height = getheight(qp, &twigs[pos]);
+		max_height = ISC_MAX(max_height, height);
+	}
+	return (max_height + 1);
+}
+
+size_t
+qp_test_getheight(dns_qp_t *qp) {
+	return (getheight(qp, &qp->root));
+}
+
+static size_t
+maxkeylen(dns_qp_t *qp, qp_node_t *n) {
+	if (!is_branch(n)) {
+		if (leaf_pval(n) == NULL) {
+			return (0);
+		} else {
+			dns_qpkey_t key;
+			return (leaf_qpkey(qp, n, key));
+		}
+	}
+	size_t max_len = 0;
+	qp_weight_t size = branch_twigs_size(n);
+	qp_node_t *twigs = branch_twigs_vector(qp, n);
+	for (qp_weight_t pos = 0; pos < size; pos++) {
+		size_t len = maxkeylen(qp, &twigs[pos]);
+		max_len = ISC_MAX(max_len, len);
+	}
+	return (max_len);
+}
+
+size_t
+qp_test_maxkeylen(dns_qp_t *qp) {
+	return (maxkeylen(qp, &qp->root));
+}
+
+/***********************************************************************
+ *
+ *  dump to stdout
+ */
+
+static void
+dumpread(dns_qpreadable_t qpr, const char *type, const char *tail) {
+	dns_qpread_t *qp = dns_qpreadable_cast(qpr);
+	printf("%s %p root %p base %p methods %p%s", type, qp, &qp->root,
+	       qp->base, qp->methods, tail);
+}
+
+static void
+dumpqp(dns_qp_t *qp, const char *type) {
+	dumpread(qp, type, " mctx ");
+	printf("%p\n", qp->mctx);
+	printf("%s %p usage %p generation %u "
+	       "chunk_max %u bump %u fender %u\n",
+	       type, qp, qp->usage, qp->generation, qp->chunk_max, qp->bump,
+	       qp->fender);
+	printf("%s %p leaf %u live %u used %u free %u hold %u\n", type, qp,
+	       qp->leaf_count, qp->used_count - qp->free_count, qp->used_count,
+	       qp->free_count, qp->hold_count);
+	printf("%s %p compact_all=%d shared_arrays=%d"
+	       " transaction_mode=%d write_protect=%d\n",
+	       type, qp, qp->compact_all, qp->shared_arrays,
+	       qp->transaction_mode, qp->write_protect);
+}
+
+void
+qp_test_dumpread(dns_qpreadable_t qp) {
+	dumpread(qp, "qpread", "\n");
+	fflush(stdout);
+}
+
+void
+qp_test_dumpsnap(dns_qpsnap_t *qp) {
+	dumpread(qp, "qpsnap", " whence ");
+	printf("%p\n", qp->whence);
+	fflush(stdout);
+}
+
+void
+qp_test_dumpqp(dns_qp_t *qp) {
+	dumpqp(qp, "qp");
+	fflush(stdout);
+}
+
+void
+qp_test_dumpmulti(dns_qpmulti_t *multi) {
+	dumpqp(&multi->phase[0], "qpmulti->phase[0]");
+	dumpqp(&multi->phase[1], "qpmulti->phase[1]");
+	printf("qpmulti %p read %p snapshots %u\n", &multi, multi->read,
+	       multi->snapshots);
+	fflush(stdout);
+}
+
+void
+qp_test_dumpchunks(dns_qp_t *qp) {
+	qp_cell_t used = 0;
+	qp_cell_t free = 0;
+	dumpqp(qp, "qp");
+	for (qp_chunk_t c = 0; c < qp->chunk_max; c++) {
+		printf("qp %p chunk %u base %p used %u free %u generation %u\n",
+		       qp, c, qp->base[c], qp->usage[c].used, qp->usage[c].free,
+		       qp->usage[c].generation);
+		used += qp->usage[c].used;
+		free += qp->usage[c].free;
+	}
+	printf("qp %p total used %u free %u\n", qp, used, free);
+	fflush(stdout);
+}
+
+void
+qp_test_dumptrie(dns_qpreadable_t qpr) {
+	dns_qpread_t *qp = dns_qpreadable_cast(qpr);
+	struct {
+		qp_ref_t ref;
+		qp_shift_t max, pos;
+	} stack[512];
+	size_t sp = 0;
+	qp_cell_t leaf_count = 0;
+
+	/*
+	 * fake up a sentinel stack entry corresponding to the root
+	 * node; the ref is deliberately out of bounds, and pos == max
+	 * so we will immediately stop scanning it
+	 */
+	stack[sp].ref = ~0U;
+	stack[sp].max = 0;
+	stack[sp].pos = 0;
+	qp_node_t *n = &qp->root;
+	printf("%p ROOT\n", n);
+
+	for (;;) {
+		if (is_branch(n)) {
+			qp_ref_t ref = branch_twigs_ref(n);
+			qp_weight_t max = branch_twigs_size(n);
+			qp_node_t *twigs = ref_ptr(qp, ref);
+
+			/* brief list of twigs */
+			dns_qpkey_t bits;
+			size_t len = 0;
+			for (qp_shift_t bit = SHIFT_NOBYTE; bit < SHIFT_OFFSET;
+			     bit++)
+			{
+				if (branch_has_twig(n, bit)) {
+					bits[len++] = bit;
+				}
+			}
+			assert(len == max);
+			qp_test_keytoascii(bits, len);
+			printf("%*s%p BRANCH %p %d %zu %s\n", (int)sp * 2, "",
+			       n, twigs, ref, branch_key_offset(n), bits);
+
+			++sp;
+			stack[sp].ref = ref;
+			stack[sp].max = max;
+			stack[sp].pos = 0;
+		} else {
+			if (leaf_pval(n) != NULL) {
+				dns_qpkey_t key;
+				qp_test_keytoascii(key, leaf_qpkey(qp, n, key));
+				printf("%*s%p LEAF %p %d %s\n", (int)sp * 2, "",
+				       n, leaf_pval(n), leaf_ival(n), key);
+				leaf_count++;
+			} else {
+				assert(n == &qp->root);
+				assert(leaf_count == 0);
+				printf("%p EMPTY", n);
+			}
+		}
+
+		while (stack[sp].pos == stack[sp].max) {
+			if (sp == 0) {
+				printf("LEAVES %d\n", leaf_count);
+				fflush(stdout);
+				return;
+			}
+			--sp;
+		}
+
+		n = ref_ptr(qp, stack[sp].ref) + stack[sp].pos;
+		stack[sp].pos++;
+	}
+}
+
+static void
+dumpdot_name(qp_node_t *n) {
+	if (is_branch(n)) {
+		qp_ref_t ref = branch_twigs_ref(n);
+		printf("c%dn%d", ref_chunk(ref), ref_cell(ref));
+	} else {
+		printf("v%p", leaf_pval(n));
+	}
+}
+
+static void
+dumpdot_twig(dns_qp_t *qp, qp_node_t *n) {
+	if (is_branch(n)) {
+		dumpdot_name(n);
+		printf(" [shape=record, label=\"{ \\N\\noff %zu | ",
+		       branch_key_offset(n));
+		char sep = '{';
+		for (qp_shift_t bit = SHIFT_NOBYTE; bit < SHIFT_OFFSET; bit++) {
+			if (branch_has_twig(n, bit)) {
+				printf("%c <t%d> %c ", sep,
+				       branch_twig_pos(n, bit),
+				       qp_test_bittoascii(bit));
+				sep = '|';
+			}
+		}
+		printf("}}\"];\n");
+
+		qp_weight_t size = branch_twigs_size(n);
+		qp_node_t *twigs = branch_twigs_vector(qp, n);
+
+		for (qp_weight_t pos = 0; pos < size; pos++) {
+			dumpdot_name(n);
+			printf(":t%d:e -> ", pos);
+			dumpdot_name(&twigs[pos]);
+			printf(":w;\n");
+		}
+
+		for (qp_weight_t pos = 0; pos < size; pos++) {
+			dumpdot_twig(qp, &twigs[pos]);
+		}
+
+	} else {
+		dns_qpkey_t key;
+		const char *str;
+		if (leaf_pval(n) == NULL) {
+			str = "EMPTY";
+		} else {
+			str = qp_test_keytoascii(key, leaf_qpkey(qp, n, key));
+		}
+		printf("v%p [shape=oval, label=\"\\N ival %d\\n%s\"];\n",
+		       leaf_pval(n), leaf_ival(n), str);
+	}
+}
+
+void
+qp_test_dumpdot(dns_qp_t *qp) {
+	REQUIRE(VALID_QP(qp));
+	qp_node_t *n = &qp->root;
+	printf("strict digraph {\nrankdir = \"LR\"; ranksep = 1.0;\n");
+	printf("ROOT [shape=point]; ROOT -> ");
+	dumpdot_name(n);
+	printf(":w;\n");
+	dumpdot_twig(qp, n);
+	printf("}\n");
+}
+
+/**********************************************************************/