Set a LMDB mapsize and also provide a config option to control it (#44954)

CHANGES

@@ -1,3 +1,10 @@
+4611.	[bug]		The default LMDB mapsize was too low and caused
+			errors after few thousand zones were added using
+			rndc addzone. A new config option "lmdb-mapsize"
+			has been introduced to configure the LMDB
+			mapsize depending on operational needs.
+			[RT #44954]
+
 4610.	[func]		The "new-zones-directory" option specifies the
 			location of NZF or NZD files for storing
 			configuration of zones added by "rndc addzone".

bin/named/config.c

@@ -176,6 +176,7 @@ options {\n\
 	zero-no-soa-ttl-cache no;\n\
 	nsec3-test-zone no;\n\
 	allow-new-zones no;\n\
+	lmdb-mapsize 32M;\n\
 	fetches-per-server 0;\n\
 	require-server-cookie no;\n\
 	v6-bias 50;\n\

bin/named/named.conf.docbook

@@ -1,5 +1,5 @@
 <!--
- - Copyright (C) 2004-2017  Internet Systems Consortium, Inc. ("ISC")
+ - Copyright (C) 2004-2017 Internet Systems Consortium, Inc. ("ISC")
  -
  - This Source Code Form is subject to the terms of the Mozilla Public
  - License, v. 2.0. If a copy of the MPL was not distributed with this
@@ -10,7 +10,7 @@
 
 <refentry xmlns:db="http://docbook.org/ns/docbook" version="5.0" xml:id="man.named.conf">
   <info>
-    <date>2017-03-08</date>
+    <date>2017-04-25</date>
   </info>
   <refentryinfo>
     <corpname>ISC</corpname>
@@ -320,6 +320,7 @@ options {
 	listen-on-v6 <optional> port <replaceable>integer</replaceable> </optional> <optional> dscp
 	    <replaceable>integer</replaceable> </optional> {
 	    <replaceable>address_match_element</replaceable>; ... };
+	lmdb-mapsize <replaceable>sizeval</replaceable>;
 	lock-file ( <replaceable>quoted_string</replaceable> | none );
 	managed-keys-directory <replaceable>quoted_string</replaceable>;
 	masterfile-format ( map | raw | text );
@@ -636,6 +637,7 @@ view <replaceable>string</replaceable> <optional> <replaceable>class</replaceabl
 	};
 	key-directory <replaceable>quoted_string</replaceable>;
 	lame-ttl <replaceable>ttlval</replaceable>;
+	lmdb-mapsize <replaceable>sizeval</replaceable>;
 	managed-keys { <replaceable>string</replaceable> <replaceable>string</replaceable>
 	    <replaceable>integer</replaceable> <replaceable>integer</replaceable> <replaceable>integer</replaceable>
 	    <replaceable>quoted_string</replaceable>; ... };

bin/named/server.c

@@ -6631,7 +6631,9 @@ setup_newzones(dns_view_t *view, cfg_obj_t *config, cfg_obj_t *vconfig,
 	const cfg_obj_t *nz = NULL;
 	const cfg_obj_t *nzdir = NULL;
 	const char *dir = NULL;
+	const cfg_obj_t *obj = NULL;
 	int i = 0;
+	isc_uint64_t mapsize = 0ULL;
 
 	REQUIRE (config != NULL);
 
@@ -6665,6 +6667,32 @@ setup_newzones(dns_view_t *view, cfg_obj_t *config, cfg_obj_t *vconfig,
 		dns_view_setnewzonedir(view, dir);
 	}
 
+#ifdef HAVE_LMDB
+	result = ns_config_get(maps, "lmdb-mapsize", &obj);
+	if (result == ISC_R_SUCCESS && obj != NULL) {
+		mapsize = cfg_obj_asuint64(obj);
+		if (mapsize < (1ULL << 20)) { /* 1 megabyte */
+			cfg_obj_log(obj, ns_g_lctx,
+				    ISC_LOG_ERROR,
+				    "'lmdb-mapsize "
+				    "%" ISC_PRINT_QUADFORMAT "d' "
+				    "is too small",
+				    mapsize);
+			return (ISC_R_FAILURE);
+		} else if (mapsize > (1ULL << 40)) { /* 1 terabyte */
+			cfg_obj_log(obj, ns_g_lctx,
+				    ISC_LOG_ERROR,
+				    "'lmdb-mapsize "
+				    "%" ISC_PRINT_QUADFORMAT "d' "
+				    "is too large",
+				    mapsize);
+			return (ISC_R_FAILURE);
+		}
+	}
+#else
+	UNUSED(obj);
+#endif /* HAVE_LMDB */
+
 	/*
 	 * A non-empty catalog-zones statement implies allow-new-zones
 	 */
@@ -6680,7 +6708,7 @@ setup_newzones(dns_view_t *view, cfg_obj_t *config, cfg_obj_t *vconfig,
 	}
 
 	if (!allow) {
-		dns_view_setnewzones(view, ISC_FALSE, NULL, NULL);
+		dns_view_setnewzones(view, ISC_FALSE, NULL, NULL, 0ULL);
 		if (num_zones != NULL)
 			*num_zones = 0;
 		return (ISC_R_SUCCESS);
@@ -6688,14 +6716,14 @@ setup_newzones(dns_view_t *view, cfg_obj_t *config, cfg_obj_t *vconfig,
 
 	nzcfg = isc_mem_get(view->mctx, sizeof(*nzcfg));
 	if (nzcfg == NULL) {
-		dns_view_setnewzones(view, ISC_FALSE, NULL, NULL);
+		dns_view_setnewzones(view, ISC_FALSE, NULL, NULL, 0ULL);
 		return (ISC_R_NOMEMORY);
 	}
 
 	memset(nzcfg, 0, sizeof(*nzcfg));
 
 	result = dns_view_setnewzones(view, ISC_TRUE, nzcfg,
-				      newzone_cfgctx_destroy);
+				      newzone_cfgctx_destroy, mapsize);
 	if (result != ISC_R_SUCCESS) {
 		isc_mem_free(view->mctx, nzcfg);
 		return (result);
@@ -11140,11 +11168,25 @@ nzd_save(MDB_txn **txnp, MDB_dbi dbi, dns_zone_t *zone,
 		const cfg_obj_t *zoptions;
 
 		result = isc_buffer_allocate(view->mctx, &text, 256);
-		if (result != ISC_R_SUCCESS)
+		if (result != ISC_R_SUCCESS) {
+			isc_log_write(ns_g_lctx,
+				      NS_LOGCATEGORY_GENERAL,
+				      NS_LOGMODULE_SERVER,
+				      ISC_LOG_ERROR,
+				      "Unable to allocate buffer in "
+				      "nzd_save(): %s",
+				      isc_result_totext(result));
 			goto cleanup;
+		}
 
 		zoptions = cfg_tuple_get(zconfig, "options");
 		if (zoptions == NULL) {
+			isc_log_write(ns_g_lctx,
+				      NS_LOGCATEGORY_GENERAL,
+				      NS_LOGMODULE_SERVER,
+				      ISC_LOG_ERROR,
+				      "Unable to get options from config in "
+				      "nzd_save()");
 			result = ISC_R_FAILURE;
 			goto cleanup;
 		}
@@ -11177,8 +11219,16 @@ nzd_save(MDB_txn **txnp, MDB_dbi dbi, dns_zone_t *zone,
 		(void) mdb_txn_abort(*txnp);
 	else {
 		status = mdb_txn_commit(*txnp);
-		if (status != 0)
+		if (status != 0) {
+			isc_log_write(ns_g_lctx,
+				      NS_LOGCATEGORY_GENERAL,
+				      NS_LOGMODULE_SERVER,
+				      ISC_LOG_ERROR,
+				      "Error committing "
+				      "NZD database: %s",
+				      mdb_strerror(status));
 			result = ISC_R_FAILURE;
+		}
 	}
 	*txnp = NULL;
 
@@ -11694,8 +11744,18 @@ do_addzone(ns_server_t *server, ns_cfgctx_t *cfg, dns_view_t *view,
 		if (view->redirect == NULL)
 			CHECK(ISC_R_NOTFOUND);
 		dns_zone_attach(view->redirect, &zone);
-	} else
+	} else {
-		CHECK(dns_zt_find(view->zonetable, name, 0, NULL, &zone));
+		result = dns_zt_find(view->zonetable, name, 0, NULL, &zone);
+		if (result != ISC_R_SUCCESS) {
+			isc_log_write(ns_g_lctx,
+				      NS_LOGCATEGORY_GENERAL,
+				      NS_LOGMODULE_SERVER,
+				      ISC_LOG_ERROR,
+				      "added new zone was not found: %s",
+				      isc_result_totext(result));
+			goto cleanup;
+		}
+	}
 
 #ifndef HAVE_LMDB
 	/*

bin/tests/system/checkconf/bad-lmdb-mapsize-bogus.conf

@@ -0,0 +1,11 @@
+/*
+ * Copyright (C) 2016  Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+options {
+	lmdb-mapsize bogusvalue;
+};

bin/tests/system/checkconf/bad-lmdb-mapsize-toolarge.conf

@@ -0,0 +1,11 @@
+/*
+ * Copyright (C) 2016  Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+options {
+	lmdb-mapsize 2048G;
+};

bin/tests/system/checkconf/bad-lmdb-mapsize-toosmall.conf

@@ -0,0 +1,11 @@
+/*
+ * Copyright (C) 2016  Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+options {
+	lmdb-mapsize 1;
+};

bin/tests/system/checkconf/bad-lmdb-mapsize-unlimited.conf

@@ -0,0 +1,11 @@
+/*
+ * Copyright (C) 2016  Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+options {
+	lmdb-mapsize unlimited;
+};

bin/tests/system/checkconf/good-lmdb-mapsize-largest.conf

@@ -0,0 +1,11 @@
+/*
+ * Copyright (C) 2016  Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+options {
+	lmdb-mapsize 1024G;
+};

bin/tests/system/checkconf/good-lmdb-mapsize-smallest.conf

@@ -0,0 +1,11 @@
+/*
+ * Copyright (C) 2016  Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+options {
+	lmdb-mapsize 1M;
+};

doc/arm/Bv9ARM-book.xml

@@ -5085,6 +5085,31 @@ badresp:1,adberr:0,findfail:0,valfail:0]
 	    </listitem>
 	  </varlistentry>
 
+	  <varlistentry>
+	    <term><command>lmdb-mapsize</command></term>
+	    <listitem>
+	      <para>
+		When <command>named</command> is built with liblmdb,
+		this option sets a maximum size for the memory map of
+		the new-zone database (NZD) in LMDB database format.
+		This database is used to store configuration information
+		for zones added using <command>rndc addzone</command>.
+		Note that this is not the NZD database file size, but
+		the largest size that the database may grow to.
+	      </para>
+	      <para>
+		Because the database file is memory mapped, its size is
+		limited by the address space of the named process.  The
+		default of 32 megabytes was chosen to be usable with
+		32-bit <command>named</command> builds.  The largest
+		permitted value is 1 terabyte. Given typical zone
+		configurations without elaborate ACLs, a 32 MB NZD file
+		ought to be able to hold configurations of about 100,000
+		zones.
+	      </para>
+	    </listitem>
+	  </varlistentry>
+
 	  <varlistentry>
 	    <term><command>managed-keys-directory</command></term>
 	    <listitem>
@@ -5954,6 +5979,15 @@ options {
 		  incompatible with use as a file name, in which case a
 		  cryptographic hash of the view name is used instead.
 		</para>
+		<para>
+		  Zones added at runtime will have their configuration
+		  stored either in a new-zone file (NZF) or a new-zone
+		  database (NZD) depending on whether
+		  <command>named</command> was linked with
+		  liblmdb at compile time.
+		  See <xref linkend="man.rndc"/> for further details
+		  about <command>rndc addzone</command>.
+		</para>
 	      </listitem>
 	    </varlistentry>
 

doc/misc/options

@@ -207,6 +207,7 @@ options {
         listen-on-v6 [ port <integer> ] [ dscp
             <integer> ] {
             <address_match_element>; ... }; // may occur multiple times
+        lmdb-mapsize <sizeval>;
         lock-file ( <quoted_string> | none );
         maintain-ixfr-base <boolean>; // obsolete
         managed-keys-directory <quoted_string>;
@@ -527,6 +528,7 @@ view <string> [ <class> ] {
         }; // may occur multiple times
         key-directory <quoted_string>;
         lame-ttl <ttlval>;
+        lmdb-mapsize <sizeval>;
         maintain-ixfr-base <boolean>; // obsolete
         managed-keys { <string> <string>
             <integer> <integer> <integer>

lib/bind9/check.c

@@ -22,6 +22,7 @@
 #include <isc/netaddr.h>
 #include <isc/parseint.h>
 #include <isc/platform.h>
+#include <isc/print.h>
 #include <isc/region.h>
 #include <isc/result.h>
 #include <isc/sha1.h>
@@ -1490,6 +1491,30 @@ check_options(const cfg_obj_t *options, isc_log_t *logctx, isc_mem_t *mctx,
 	}
 #endif
 
+	obj = NULL;
+	(void)cfg_map_get(options, "lmdb-mapsize", &obj);
+	if (obj != NULL) {
+		isc_uint64_t mapsize = cfg_obj_asuint64(obj);
+
+		if (mapsize < (1ULL << 20)) { /* 1 megabyte */
+			cfg_obj_log(obj, logctx,
+				    ISC_LOG_ERROR,
+				    "'lmdb-mapsize "
+				    "%" ISC_PRINT_QUADFORMAT "d' "
+				    "is too small",
+				    mapsize);
+			return (ISC_R_RANGE);
+		} else if (mapsize > (1ULL << 40)) { /* 1 terabyte */
+			cfg_obj_log(obj, logctx,
+				    ISC_LOG_ERROR,
+				    "'lmdb-mapsize "
+				    "%" ISC_PRINT_QUADFORMAT "d' "
+				    "is too large",
+				    mapsize);
+			return (ISC_R_RANGE);
+		}
+	}
+
 	return (result);
 }
 

lib/dns/include/dns/view.h

@@ -1216,7 +1216,7 @@ dns_view_untrust(dns_view_t *view, const dns_name_t *keyname,
 
 isc_result_t
 dns_view_setnewzones(dns_view_t *view, isc_boolean_t allow, void *cfgctx,
-		     void (*cfg_destroy)(void **));
+		     void (*cfg_destroy)(void **), isc_uint64_t mapsize);
 /*%<
  * Set whether or not to allow zones to be created or deleted at runtime.
  *

lib/dns/view.c

@@ -503,7 +503,7 @@ destroy(dns_view_t *view) {
 	if (view->dtenv != NULL)
 		dns_dt_detach(&view->dtenv);
 #endif /* HAVE_DNSTAP */
-	dns_view_setnewzones(view, ISC_FALSE, NULL, NULL);
+	dns_view_setnewzones(view, ISC_FALSE, NULL, NULL, 0ULL);
 	if (view->new_zone_file != NULL) {
 		isc_mem_free(view->mctx, view->new_zone_file);
 		view->new_zone_file = NULL;
@@ -2031,7 +2031,7 @@ nz_legacy(const char *directory, const char *viewname,
 
 isc_result_t
 dns_view_setnewzones(dns_view_t *view, isc_boolean_t allow, void *cfgctx,
-		     void (*cfg_destroy)(void **))
+		     void (*cfg_destroy)(void **), isc_uint64_t mapsize)
 {
 	isc_result_t result;
 	char buffer[1024];
@@ -2088,6 +2088,14 @@ dns_view_setnewzones(dns_view_t *view, isc_boolean_t allow, void *cfgctx,
 		goto out;
 	}
 
+	if (mapsize != 0ULL) {
+		status = mdb_env_set_mapsize(env, mapsize);
+		if (status != 0) {
+			result = ISC_R_FAILURE;
+			goto out;
+		}
+	}
+
 	status = mdb_env_open(env, view->new_zone_db,
 			      MDB_NOSUBDIR|MDB_CREATE, 0600);
 	if (status != 0) {

lib/isccfg/include/isccfg/grammar.h

@@ -48,6 +48,9 @@
 #define CFG_CLAUSEFLAG_NOTCONFIGURED	0x00000080
 /*% A option for a experimental feature. */
 #define CFG_CLAUSEFLAG_EXPERIMENTAL	0x00000100
+/*% A configuration option that is ineffective due to
+ * compile time options, but is harmless. */
+#define CFG_CLAUSEFLAG_NOOP		0x00000200
 
 typedef struct cfg_clausedef cfg_clausedef_t;
 typedef struct cfg_tuplefielddef cfg_tuplefielddef_t;

lib/isccfg/namedconf.c

@@ -134,6 +134,7 @@ static cfg_type_t cfg_type_server_key_kludge;
 static cfg_type_t cfg_type_size;
 static cfg_type_t cfg_type_sizenodefault;
 static cfg_type_t cfg_type_sizeorpercent;
+static cfg_type_t cfg_type_sizeval;
 static cfg_type_t cfg_type_sockaddr4wild;
 static cfg_type_t cfg_type_sockaddr6wild;
 static cfg_type_t cfg_type_statschannels;
@@ -1882,6 +1883,11 @@ view_clauses[] = {
 #endif
 	{ "ixfr-from-differences", &cfg_type_ixfrdifftype, 0 },
 	{ "lame-ttl", &cfg_type_ttlval, 0 },
+#ifdef HAVE_LMDB
+	{ "lmdb-mapsize", &cfg_type_sizeval, 0 },
+#else
+	{ "lmdb-mapsize", &cfg_type_sizeval, CFG_CLAUSEFLAG_NOOP },
+#endif
 	{ "nocookie-udp-size", &cfg_type_uint32, 0 },
 	{ "nosit-udp-size", &cfg_type_uint32, CFG_CLAUSEFLAG_OBSOLETE },
 	{ "max-acache-size", &cfg_type_sizenodefault,

lib/isccfg/parser.c

@@ -1717,6 +1717,12 @@ cfg_parse_mapbody(cfg_parser_t *pctx, const cfg_type_t *type, cfg_obj_t **ret)
 			cfg_parser_warning(pctx, 0, "option '%s' is "
 				       "not implemented", clause->name);
 
+		if ((clause->flags & CFG_CLAUSEFLAG_NOOP) != 0) {
+			cfg_parser_warning(pctx, 0, "option '%s' was not "
+					   "enabled at compile time "
+					   "(ignored)", clause->name);
+		}
+
 		if ((clause->flags & CFG_CLAUSEFLAG_NOTCONFIGURED) != 0) {
 			cfg_parser_warning(pctx, 0, "option '%s' was not "
 					   "enabled at compile time",
@@ -1973,6 +1979,7 @@ static struct flagtext {
 	{ CFG_CLAUSEFLAG_NOTCONFIGURED, "not configured" },
 	{ CFG_CLAUSEFLAG_MULTI, "may occur multiple times" },
 	{ CFG_CLAUSEFLAG_EXPERIMENTAL, "experimental" },
+	{ CFG_CLAUSEFLAG_NOOP, "non-operational" },
 	{ 0, NULL }
 };