From 6b77583f54428b83dbb19d3fd9111afdd0fdc84e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20K=C4=99pie=C5=84?= <michal@isc.org>
Date: Thu, 17 Jun 2021 17:09:37 +0200
Subject: [PATCH] Allow resetting hash table size limits for DNS DBs

When "max-cache-size" is changed to "unlimited" (or "0") for a running
named instance (using "rndc reconfig"), the hash table size limit for
each affected cache DB is not reset to the maximum possible value,
preventing those hash tables from being allowed to grow as a result of
new nodes being added.

Extend dns_rbt_adjusthashsize() to interpret "size" set to 0 as a signal
to remove any previously imposed limits on the hash table size.  Adjust
API documentation for dns_db_adjusthashsize() accordingly.  Move the
call to dns_db_adjusthashsize() from dns_cache_setcachesize() so that it
also happens when "size" is set to 0.
---
 lib/dns/cache.c          |  3 ++-
 lib/dns/include/dns/db.h |  9 +++++----
 lib/dns/rbt.c            | 27 ++++++++++++++++++++++-----
 3 files changed, 29 insertions(+), 10 deletions(-)

diff --git a/lib/dns/cache.c b/lib/dns/cache.c
index 633fc5d122..9935275d8f 100644
--- a/lib/dns/cache.c
+++ b/lib/dns/cache.c
@@ -957,8 +957,9 @@ dns_cache_setcachesize(dns_cache_t *cache, size_t size) {
 		 * time, or replacing other limits).
 		 */
 		isc_mem_setwater(cache->mctx, water, cache, hiwater, lowater);
-		dns_db_adjusthashsize(cache->db, size);
 	}
+
+	dns_db_adjusthashsize(cache->db, size);
 }
 
 size_t
diff --git a/lib/dns/include/dns/db.h b/lib/dns/include/dns/db.h
index 8664cf1458..fe36759a2c 100644
--- a/lib/dns/include/dns/db.h
+++ b/lib/dns/include/dns/db.h
@@ -1400,14 +1400,15 @@ dns_db_hashsize(dns_db_t *db);
 isc_result_t
 dns_db_adjusthashsize(dns_db_t *db, size_t size);
 /*%<
- * For database implementations using a hash table, adjust
- * the size of the hash table to store objects with size
- * memory footprint.
+ * For database implementations using a hash table, adjust the size of
+ * the hash table to store objects with a maximum total memory footprint
+ * of 'size' bytes.  If 'size' is set to 0, it means no finite limit is
+ * requested.
  *
  * Requires:
  *
  * \li	'db' is a valid database.
- * \li  'size' is maximum memory footprint of the database
+ * \li  'size' is maximum memory footprint of the database in bytes
  *
  * Returns:
  * \li	#ISC_R_SUCCESS	The registration succeeded
diff --git a/lib/dns/rbt.c b/lib/dns/rbt.c
index a42b7d08ff..a05fcd8609 100644
--- a/lib/dns/rbt.c
+++ b/lib/dns/rbt.c
@@ -1080,11 +1080,28 @@ isc_result_t
 dns_rbt_adjusthashsize(dns_rbt_t *rbt, size_t size) {
 	REQUIRE(VALID_RBT(rbt));
 
-	size_t newsize = size / RBT_HASH_BUCKETSIZE;
-
-	rbt->maxhashbits = rehash_bits(rbt, newsize);
-
-	maybe_rehash(rbt, newsize);
+	if (size > 0) {
+		/*
+		 * Setting a new, finite size limit was requested for the RBT.
+		 * Estimate how many hash table slots are needed for the
+		 * requested size and how many bits would be needed to index
+		 * those hash table slots, then rehash the RBT if necessary.
+		 * Note that the hash table can only grow, it is not shrunk if
+		 * the requested size limit is lower than the current one.
+		 */
+		size_t newsize = size / RBT_HASH_BUCKETSIZE;
+		rbt->maxhashbits = rehash_bits(rbt, newsize);
+		maybe_rehash(rbt, newsize);
+	} else {
+		/*
+		 * Setting an infinite size limit was requested for the RBT.
+		 * Increase the maximum allowed number of hash table slots to
+		 * 2^32, which enables the hash table to grow as nodes are
+		 * added to the RBT without immediately preallocating 2^32 hash
+		 * table slots.
+		 */
+		rbt->maxhashbits = RBT_HASH_MAX_BITS;
+	}
 
 	return (ISC_R_SUCCESS);
 }