1911. [func] Attempt to make the amount of work performed in a

iteration self tuning.  The covers nodes clean from
                        the cache per iteration, nodes written to disk when
                        rewriting a master file and nodes destroyed per
                        iteration when destroying a zone or a cache.
                        [RT #14996]

CHANGES

@@ -1,3 +1,10 @@
+1911.	[func]		Attempt to make the amount of work performed in a
+			iteration self tuning.  The covers nodes clean from
+			the cache per iteration, nodes written to disk when
+			rewriting a master file and nodes destroyed per
+			iteration when destroying a zone or a cache.
+			[RT #14996]
+
 1910.	[cleanup]	Don't add DNSKEY records to the additional section.
 
 1909.	[bug]		ixfr-from-differences failed to ensure that the

bin/named/client.c

@@ -15,7 +15,7 @@
  * PERFORMANCE OF THIS SOFTWARE.
  */
 
-/* $Id: client.c,v 1.229 2005/07/27 02:28:58 marka Exp $ */
+/* $Id: client.c,v 1.230 2005/08/15 01:21:03 marka Exp $ */
 
 #include <config.h>
 
@@ -168,6 +168,7 @@ struct ns_clientmgr {
  * Must be greater than any valid state.
  */
 
+unsigned int ns_client_requests;
 
 static void client_read(ns_client_t *client);
 static void client_accept(ns_client_t *client);
@@ -1213,6 +1214,8 @@ client_request(isc_task_t *task, isc_event_t *event) {
 	       NS_CLIENTSTATE_READING :
 	       NS_CLIENTSTATE_READY);
 
+	ns_client_requests++;
+
 	if (event->ev_type == ISC_SOCKEVENT_RECVDONE) {
 		INSIST(!TCP_CLIENT(client));
 		sevent = (isc_socketevent_t *)event;

bin/named/include/named/client.h

@@ -15,7 +15,7 @@
  * PERFORMANCE OF THIS SOFTWARE.
  */
 
-/* $Id: client.h,v 1.76 2005/07/27 02:28:59 marka Exp $ */
+/* $Id: client.h,v 1.77 2005/08/15 01:21:04 marka Exp $ */
 
 #ifndef NAMED_CLIENT_H
 #define NAMED_CLIENT_H 1
@@ -160,6 +160,7 @@ struct ns_client {
 #define NS_CLIENTATTR_MULTICAST		0x08 /*%< recv'd from multicast */
 #define NS_CLIENTATTR_WANTDNSSEC	0x10 /*%< include dnssec records */
 
+extern unsigned int ns_client_requests;
 
 /***
  *** Functions

bin/named/include/named/server.h

@@ -15,7 +15,7 @@
  * PERFORMANCE OF THIS SOFTWARE.
  */
 
-/* $Id: server.h,v 1.78 2005/04/29 00:22:32 marka Exp $ */
+/* $Id: server.h,v 1.79 2005/08/15 01:21:05 marka Exp $ */
 
 #ifndef NAMED_SERVER_H
 #define NAMED_SERVER_H 1
@@ -82,6 +82,8 @@ struct ns_server {
 
 	isc_timer_t *		interface_timer;
 	isc_timer_t *		heartbeat_timer;
+	isc_timer_t *		pps_timer;
+
 	isc_uint32_t		interface_interval;
 	isc_uint32_t		heartbeat_interval;
 

bin/named/server.c

@@ -15,7 +15,7 @@
  * PERFORMANCE OF THIS SOFTWARE.
  */
 
-/* $Id: server.c,v 1.446 2005/07/27 02:28:58 marka Exp $ */
+/* $Id: server.c,v 1.447 2005/08/15 01:21:04 marka Exp $ */
 
 /*! \file */
 
@@ -51,6 +51,7 @@
 #include <dns/forward.h>
 #include <dns/journal.h>
 #include <dns/keytable.h>
+#include <dns/lib.h>
 #include <dns/master.h>
 #include <dns/masterdump.h>
 #include <dns/order.h>
@@ -2099,6 +2100,21 @@ heartbeat_timer_tick(isc_task_t *task, isc_event_t *event) {
 	}
 }
 
+static void
+pps_timer_tick(isc_task_t *task, isc_event_t *event) {
+	static unsigned int oldrequests = 0;
+	unsigned int requests = ns_client_requests;
+
+	UNUSED(task);
+	isc_event_free(&event);
+
+	/*
+	 * Don't worry about wrapping as the overflow result will be right.
+	 */
+	dns_pps = (requests - oldrequests) / 1200;
+	oldrequests = requests;
+}
+
 /*
  * Replace the current value of '*field', a dynamically allocated
  * string or NULL, with a dynamically allocated copy of the
@@ -2204,6 +2220,7 @@ load_configuration(const char *filename, ns_server_t *server,
 		   isc_boolean_t first_time)
 {
 	isc_result_t result;
+	isc_interval_t interval;
 	cfg_parser_t *parser = NULL;
 	cfg_obj_t *config;
 	cfg_obj_t *options;
@@ -2479,7 +2496,6 @@ load_configuration(const char *filename, ns_server_t *server,
 				      isc_timertype_inactive,
 				      NULL, NULL, ISC_TRUE));
 	} else if (server->interface_interval != interface_interval) {
-		isc_interval_t interval;
 		isc_interval_set(&interval, interface_interval, 0);
 		CHECK(isc_timer_reset(server->interface_timer,
 				      isc_timertype_ticker,
@@ -2499,13 +2515,16 @@ load_configuration(const char *filename, ns_server_t *server,
 				      isc_timertype_inactive,
 				      NULL, NULL, ISC_TRUE));
 	} else if (server->heartbeat_interval != heartbeat_interval) {
-		isc_interval_t interval;
 		isc_interval_set(&interval, heartbeat_interval, 0);
 		CHECK(isc_timer_reset(server->heartbeat_timer,
 				      isc_timertype_ticker,
 				      NULL, &interval, ISC_FALSE));
 	}
 	server->heartbeat_interval = heartbeat_interval;
+	
+	isc_interval_set(&interval, 1200, 0);
+	CHECK(isc_timer_reset(server->pps_timer, isc_timertype_ticker, NULL,
+			      &interval, ISC_FALSE));
 
 	/*
 	 * Configure and freeze all explicit views.  Explicit
@@ -2938,6 +2957,12 @@ run_server(isc_task_t *task, isc_event_t *event) {
 				    server, &server->heartbeat_timer),
 		   "creating heartbeat timer");
 
+	CHECKFATAL(isc_timer_create(ns_g_timermgr, isc_timertype_inactive,
+				    NULL, NULL, server->task,
+				    heartbeat_timer_tick,
+				    server, &server->pps_timer),
+		   "creating pps timer");
+
 	CHECKFATAL(cfg_parser_create(ns_g_mctx, NULL, &ns_g_parser),
 		   "creating default configuration parser");
 
@@ -3002,6 +3027,7 @@ shutdown_server(isc_task_t *task, isc_event_t *event) {
 
 	isc_timer_detach(&server->interface_timer);
 	isc_timer_detach(&server->heartbeat_timer);
+	isc_timer_detach(&server->pps_timer);
 
 	ns_interfacemgr_shutdown(server->interfacemgr);
 	ns_interfacemgr_detach(&server->interfacemgr);
@@ -3090,6 +3116,7 @@ ns_server_create(isc_mem_t *mctx, ns_server_t **serverp) {
 
 	server->interface_timer = NULL;
 	server->heartbeat_timer = NULL;
+	server->pps_timer = NULL;
 	
 	server->interface_interval = 0;
 	server->heartbeat_interval = 0;

lib/dns/cache.c

@@ -15,7 +15,7 @@
  * PERFORMANCE OF THIS SOFTWARE.
  */
 
-/* $Id: cache.c,v 1.61 2005/07/12 01:00:14 marka Exp $ */
+/* $Id: cache.c,v 1.62 2005/08/15 01:21:05 marka Exp $ */
 
 /*! \file */
 
@@ -31,6 +31,7 @@
 #include <dns/db.h>
 #include <dns/dbiterator.h>
 #include <dns/events.h>
+#include <dns/lib.h>
 #include <dns/log.h>
 #include <dns/masterdump.h>
 #include <dns/rdata.h>
@@ -52,7 +53,7 @@
  * CLEANERINCREMENT is how many nodes are examined in one pass.
  * See also DNS_CACHE_MINSIZE 
  */
-#define DNS_CACHE_CLEANERINCREMENT	1000	/*%< Number of nodes. */
+#define DNS_CACHE_CLEANERINCREMENT	1000U	/*%< Number of nodes. */
 
 /***
  ***	Types
@@ -104,7 +105,7 @@ struct cache_cleaner {
 	isc_event_t	*overmem_event;
 
 	dns_dbiterator_t *iterator;
-	int 		 increment;	/*% Number of names to
+	unsigned int 	 increment;	/*% Number of names to
 					   clean in one increment */
 	cleaner_state_t  state;		/*% Idle/Busy. */
 	isc_boolean_t	 overmem;	/*% The cache is in an overmem state. */
@@ -156,6 +157,64 @@ cleaner_shutdown_action(isc_task_t *task, isc_event_t *event);
 static void
 overmem_cleaning_action(isc_task_t *task, isc_event_t *event);
 
+static void
+adjust_increment(cache_cleaner_t *cleaner, unsigned int remaining,
+		 isc_time_t *start)
+{
+	isc_time_t end;
+	isc_uint64_t usecs;
+	isc_uint64_t new;
+	unsigned int pps = dns_pps;
+	unsigned int interval;
+	unsigned int names;
+	
+	/*
+	 * Tune for minumum of 100 pps.
+	 */
+	if (pps < 100)
+		pps = 100;
+
+	isc_time_now(&end);
+
+	interval = 1000000 / pps;
+	if (interval == 0)
+		interval = 1;
+
+	INSIST(cleaner->increment >= remaining);
+	names = cleaner->increment - remaining;
+	usecs = isc_time_microdiff(&end, start);
+
+	isc_log_write(dns_lctx, DNS_LOGCATEGORY_DATABASE, DNS_LOGMODULE_CACHE,
+		      ISC_LOG_INFO, "adjust_increment interval=%u "
+		      "names=%u usec=%" ISC_PLATFORM_QUADFORMAT "u",
+		      interval, names, usecs);
+	
+	if (usecs == 0) {
+		if (names == cleaner->increment) {
+			cleaner->increment *= 2;
+			if (cleaner->increment > DNS_CACHE_CLEANERINCREMENT)
+				cleaner->increment = DNS_CACHE_CLEANERINCREMENT;
+			isc_log_write(dns_lctx, DNS_LOGCATEGORY_DATABASE,
+				      DNS_LOGMODULE_CACHE, ISC_LOG_INFO,
+				      "new clear->increment = %d\n",
+				      cleaner->increment);
+		}
+		return;
+	}
+
+	new = (names * interval);
+	new /= usecs;
+	if (new == 0)
+		new = 1;
+	else if (new > DNS_CACHE_CLEANERINCREMENT)
+		new = DNS_CACHE_CLEANERINCREMENT;
+
+	cleaner->increment = new;
+	isc_log_write(dns_lctx, DNS_LOGCATEGORY_DATABASE, DNS_LOGMODULE_CACHE,
+		      ISC_LOG_INFO, "new clear->increment = %u\n",
+		      cleaner->increment);
+}
+
 static inline isc_result_t
 cache_create_db(dns_cache_t *cache, dns_db_t **db) {
 	return (dns_db_create(cache->mctx, cache->db_type, dns_rootname,
@@ -717,7 +776,8 @@ static void
 incremental_cleaning_action(isc_task_t *task, isc_event_t *event) {
 	cache_cleaner_t *cleaner = event->ev_arg;
 	isc_result_t result;
-	int n_names;
+	unsigned int n_names;
+	isc_time_t start;
 
 	UNUSED(task);
 
@@ -736,6 +796,7 @@ incremental_cleaning_action(isc_task_t *task, isc_event_t *event) {
 
 	REQUIRE(DNS_DBITERATOR_VALID(cleaner->iterator));
 
+	isc_time_now(&start);
 	while (n_names-- > 0) {
 		dns_dbnode_t *node = NULL;
 
@@ -746,6 +807,7 @@ incremental_cleaning_action(isc_task_t *task, isc_event_t *event) {
 				 "cache cleaner: dns_dbiterator_current() "
 				 "failed: %s", dns_result_totext(result));
 
+			adjust_increment(cleaner, n_names, &start);
 			end_cleaning(cleaner, event);
 			return;
 		}
@@ -789,11 +851,14 @@ incremental_cleaning_action(isc_task_t *task, isc_event_t *event) {
 				}
 			}
 
+			adjust_increment(cleaner, n_names, &start);
 			end_cleaning(cleaner, event);
 			return;
 		}
 	}
 
+	adjust_increment(cleaner, 0U, &start);
+
 	/*
 	 * We have successfully performed a cleaning increment but have
 	 * not gone through the entire cache.  Free the iterator locks
@@ -804,7 +869,7 @@ incremental_cleaning_action(isc_task_t *task, isc_event_t *event) {
 	RUNTIME_CHECK(result == ISC_R_SUCCESS);
 
 	isc_log_write(dns_lctx, DNS_LOGCATEGORY_DATABASE, DNS_LOGMODULE_CACHE,
-		      ISC_LOG_DEBUG(1), "cache cleaner: checked %d nodes, "
+		      ISC_LOG_DEBUG(1), "cache cleaner: checked %u nodes, "
 		      "mem inuse %lu, sleeping", cleaner->increment,
 		      (unsigned long)isc_mem_inuse(cleaner->cache->mctx));
 

lib/dns/include/dns/lib.h

@@ -15,7 +15,7 @@
  * PERFORMANCE OF THIS SOFTWARE.
  */
 
-/* $Id: lib.h,v 1.10 2005/04/29 00:22:59 marka Exp $ */
+/* $Id: lib.h,v 1.11 2005/08/15 01:21:07 marka Exp $ */
 
 #ifndef DNS_LIB_H
 #define DNS_LIB_H 1
@@ -27,6 +27,7 @@
 
 ISC_LANG_BEGINDECLS
 
+LIBDNS_EXTERNAL_DATA extern unsigned int dns_pps;
 LIBDNS_EXTERNAL_DATA extern isc_msgcat_t *dns_msgcat;
 
 void

lib/dns/lib.c

@@ -15,7 +15,7 @@
  * PERFORMANCE OF THIS SOFTWARE.
  */
 
-/* $Id: lib.c,v 1.13 2005/04/29 00:22:48 marka Exp $ */
+/* $Id: lib.c,v 1.14 2005/08/15 01:21:06 marka Exp $ */
 
 /*! \file */
 
@@ -33,6 +33,7 @@
  *** Globals
  ***/
 
+LIBDNS_EXTERNAL_DATA unsigned int			dns_pps = 0U;
 LIBDNS_EXTERNAL_DATA isc_msgcat_t *			dns_msgcat = NULL;
 
 

lib/dns/masterdump.c

@@ -15,7 +15,7 @@
  * PERFORMANCE OF THIS SOFTWARE.
  */
 
-/* $Id: masterdump.c,v 1.78 2005/06/20 01:03:53 marka Exp $ */
+/* $Id: masterdump.c,v 1.79 2005/08/15 01:21:06 marka Exp $ */
 
 /*! \file */
 
@@ -31,12 +31,14 @@
 #include <isc/stdio.h>
 #include <isc/string.h>
 #include <isc/task.h>
+#include <isc/time.h>
 #include <isc/util.h>
 
 #include <dns/db.h>
 #include <dns/dbiterator.h>
 #include <dns/events.h>
 #include <dns/fixedname.h>
+#include <dns/lib.h>
 #include <dns/log.h>
 #include <dns/master.h>
 #include <dns/masterdump.h>
@@ -1261,6 +1263,7 @@ dumptostreaminc(dns_dumpctx_t *dctx) {
 	unsigned int nodes;
 	dns_masterrawheader_t rawheader;
 	isc_uint32_t now32;
+	isc_time_t start;
 
 	bufmem = isc_mem_get(dctx->mctx, initial_buffer_length);
 	if (bufmem == NULL)
@@ -1333,6 +1336,7 @@ dumptostreaminc(dns_dumpctx_t *dctx) {
 		result = ISC_R_SUCCESS;
 
 	nodes = dctx->nodes;
+	isc_time_now(&start);
 	while (result == ISC_R_SUCCESS && (dctx->nodes == 0 || nodes--)) {
 		dns_rdatasetiter_t *rdsiter = NULL;
 		dns_dbnode_t *node = NULL;
@@ -1367,6 +1371,30 @@ dumptostreaminc(dns_dumpctx_t *dctx) {
 	}
 
 	if (dctx->nodes != 0 && result == ISC_R_SUCCESS) {
+		unsigned int pps = dns_pps;
+		unsigned int interval;
+		isc_uint64_t usecs;
+		isc_time_t end;
+
+		isc_time_now(&end);
+		if (pps < 100)
+			pps = 100;
+		interval = 1000000 / pps;
+		if (interval == 0)
+			interval = 1;
+		usecs = isc_time_microdiff(&end, &start);
+		if (usecs == 0) {
+			dctx->nodes = dctx->nodes * 2;
+			if (dctx->nodes > 1000)
+				dctx->nodes = 1000;
+		} else {
+			dctx->nodes = dctx->nodes * interval;
+			dctx->nodes /= usecs;
+			if (dctx->nodes == 0)
+				dctx->nodes = 1;
+			else if (dctx->nodes > 1000)
+				dctx->nodes = 1000;
+		}
 		dns_dbiterator_pause(dctx->dbiter);
 		result = DNS_R_CONTINUE;
 	} else if (result == ISC_R_NOMORE)

lib/dns/rbtdb.c

@@ -15,7 +15,7 @@
  * PERFORMANCE OF THIS SOFTWARE.
  */
 
-/* $Id: rbtdb.c,v 1.214 2005/07/18 05:58:58 marka Exp $ */
+/* $Id: rbtdb.c,v 1.215 2005/08/15 01:21:06 marka Exp $ */
 
 /*! \file */
 
@@ -34,6 +34,7 @@
 #include <isc/rwlock.h>
 #include <isc/string.h>
 #include <isc/task.h>
+#include <isc/time.h>
 #include <isc/util.h>
 
 #include <dns/acache.h>
@@ -41,6 +42,7 @@
 #include <dns/dbiterator.h>
 #include <dns/events.h>
 #include <dns/fixedname.h>
+#include <dns/lib.h>
 #include <dns/log.h>
 #include <dns/masterdump.h>
 #include <dns/rbt.h>
@@ -346,6 +348,7 @@ typedef struct {
 
 	/* Unlocked */
 	isc_mem_t **			nodemctxs;
+	unsigned int			quantum;
 } dns_rbtdb_t;
 
 #define RBTDB_ATTR_LOADED		0x01
@@ -548,12 +551,43 @@ free_rbtdb_callback(isc_task_t *task, isc_event_t *event) {
 	free_rbtdb(rbtdb, ISC_TRUE, event);
 }
 
+static unsigned int
+adjust_quantum(unsigned int old, isc_time_t *start) {
+	unsigned int pps = dns_pps;
+	unsigned int interval;
+	isc_uint64_t usecs;
+	isc_time_t end;
+
+	if (pps < 100)
+		pps = 100;
+	isc_time_now(&end);
+
+	interval = 1000000 / pps;
+	if (interval == 0)
+		interval = 1;
+	usecs = isc_time_microdiff(&end, start);
+	if (usecs == 0) {
+		old *= 2;
+		if (old > 1000)
+			old = 1000;
+		return (old);
+	}
+	old = old * interval;
+	old /= usecs;
+	if (old == 0)
+		old = 1;
+	else if (old > 1000)
+		old = 1000;
+	return (old);
+}
+		
 static void
 free_rbtdb(dns_rbtdb_t *rbtdb, isc_boolean_t log, isc_event_t *event) {
 	unsigned int i;
 	isc_ondestroy_t ondest;
 	isc_result_t result;
 	char buf[DNS_NAME_FORMATSIZE];
+	isc_time_t start;
 
 	REQUIRE(rbtdb->current_version != NULL || EMPTY(rbtdb->open_versions));
 	REQUIRE(rbtdb->future_version == NULL);
@@ -569,12 +603,17 @@ free_rbtdb(dns_rbtdb_t *rbtdb, isc_boolean_t log, isc_event_t *event) {
 		isc_mem_put(rbtdb->common.mctx, rbtdb->current_version,
 			    sizeof(rbtdb_version_t));
 	}
+	if (event == NULL)
+		rbtdb->quantum = (rbtdb->task != NULL) ? 100 : 0;
  again:
 	if (rbtdb->tree != NULL) {
-		result = dns_rbt_destroy2(&rbtdb->tree,
+		isc_time_now(&start);
-					  (rbtdb->task != NULL) ? 1000 : 0);
+		result = dns_rbt_destroy2(&rbtdb->tree, rbtdb->quantum);
 		if (result == ISC_R_QUOTA) {
 			INSIST(rbtdb->task != NULL);
+			if (rbtdb->quantum != 0)
+				rbtdb->quantum = adjust_quantum(rbtdb->quantum,
+								&start);
 			if (event == NULL)
 				event = isc_event_allocate(rbtdb->common.mctx,
 							   NULL,