From 261a6a1f7d95eaf0cd882f3123dcfd775517a54f Mon Sep 17 00:00:00 2001 From: Mark Andrews Date: Mon, 15 Aug 2005 01:21:07 +0000 Subject: [PATCH] 1911. [func] Attempt to make the amount of work performed in a iteration self tuning. The covers nodes clean from the cache per iteration, nodes written to disk when rewriting a master file and nodes destroyed per iteration when destroying a zone or a cache. [RT #14996] --- CHANGES | 7 +++ bin/named/client.c | 5 ++- bin/named/include/named/client.h | 3 +- bin/named/include/named/server.h | 4 +- bin/named/server.c | 33 ++++++++++++-- lib/dns/cache.c | 75 +++++++++++++++++++++++++++++--- lib/dns/include/dns/lib.h | 3 +- lib/dns/lib.c | 3 +- lib/dns/masterdump.c | 30 ++++++++++++- lib/dns/rbtdb.c | 45 +++++++++++++++++-- 10 files changed, 191 insertions(+), 17 deletions(-) diff --git a/CHANGES b/CHANGES index 0c88cd717f..894c380405 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,10 @@ +1911. [func] Attempt to make the amount of work performed in a + iteration self tuning. The covers nodes clean from + the cache per iteration, nodes written to disk when + rewriting a master file and nodes destroyed per + iteration when destroying a zone or a cache. + [RT #14996] + 1910. [cleanup] Don't add DNSKEY records to the additional section. 1909. [bug] ixfr-from-differences failed to ensure that the diff --git a/bin/named/client.c b/bin/named/client.c index f999545beb..4b16228aac 100644 --- a/bin/named/client.c +++ b/bin/named/client.c @@ -15,7 +15,7 @@ * PERFORMANCE OF THIS SOFTWARE. */ -/* $Id: client.c,v 1.229 2005/07/27 02:28:58 marka Exp $ */ +/* $Id: client.c,v 1.230 2005/08/15 01:21:03 marka Exp $ */ #include @@ -168,6 +168,7 @@ struct ns_clientmgr { * Must be greater than any valid state. */ +unsigned int ns_client_requests; static void client_read(ns_client_t *client); static void client_accept(ns_client_t *client); @@ -1213,6 +1214,8 @@ client_request(isc_task_t *task, isc_event_t *event) { NS_CLIENTSTATE_READING : NS_CLIENTSTATE_READY); + ns_client_requests++; + if (event->ev_type == ISC_SOCKEVENT_RECVDONE) { INSIST(!TCP_CLIENT(client)); sevent = (isc_socketevent_t *)event; diff --git a/bin/named/include/named/client.h b/bin/named/include/named/client.h index 334c813cd3..44e25668a8 100644 --- a/bin/named/include/named/client.h +++ b/bin/named/include/named/client.h @@ -15,7 +15,7 @@ * PERFORMANCE OF THIS SOFTWARE. */ -/* $Id: client.h,v 1.76 2005/07/27 02:28:59 marka Exp $ */ +/* $Id: client.h,v 1.77 2005/08/15 01:21:04 marka Exp $ */ #ifndef NAMED_CLIENT_H #define NAMED_CLIENT_H 1 @@ -160,6 +160,7 @@ struct ns_client { #define NS_CLIENTATTR_MULTICAST 0x08 /*%< recv'd from multicast */ #define NS_CLIENTATTR_WANTDNSSEC 0x10 /*%< include dnssec records */ +extern unsigned int ns_client_requests; /*** *** Functions diff --git a/bin/named/include/named/server.h b/bin/named/include/named/server.h index 9ae6d9d110..fc1a0712e5 100644 --- a/bin/named/include/named/server.h +++ b/bin/named/include/named/server.h @@ -15,7 +15,7 @@ * PERFORMANCE OF THIS SOFTWARE. */ -/* $Id: server.h,v 1.78 2005/04/29 00:22:32 marka Exp $ */ +/* $Id: server.h,v 1.79 2005/08/15 01:21:05 marka Exp $ */ #ifndef NAMED_SERVER_H #define NAMED_SERVER_H 1 @@ -82,6 +82,8 @@ struct ns_server { isc_timer_t * interface_timer; isc_timer_t * heartbeat_timer; + isc_timer_t * pps_timer; + isc_uint32_t interface_interval; isc_uint32_t heartbeat_interval; diff --git a/bin/named/server.c b/bin/named/server.c index 39ff745f74..96435cb42d 100644 --- a/bin/named/server.c +++ b/bin/named/server.c @@ -15,7 +15,7 @@ * PERFORMANCE OF THIS SOFTWARE. */ -/* $Id: server.c,v 1.446 2005/07/27 02:28:58 marka Exp $ */ +/* $Id: server.c,v 1.447 2005/08/15 01:21:04 marka Exp $ */ /*! \file */ @@ -51,6 +51,7 @@ #include #include #include +#include #include #include #include @@ -2099,6 +2100,21 @@ heartbeat_timer_tick(isc_task_t *task, isc_event_t *event) { } } +static void +pps_timer_tick(isc_task_t *task, isc_event_t *event) { + static unsigned int oldrequests = 0; + unsigned int requests = ns_client_requests; + + UNUSED(task); + isc_event_free(&event); + + /* + * Don't worry about wrapping as the overflow result will be right. + */ + dns_pps = (requests - oldrequests) / 1200; + oldrequests = requests; +} + /* * Replace the current value of '*field', a dynamically allocated * string or NULL, with a dynamically allocated copy of the @@ -2204,6 +2220,7 @@ load_configuration(const char *filename, ns_server_t *server, isc_boolean_t first_time) { isc_result_t result; + isc_interval_t interval; cfg_parser_t *parser = NULL; cfg_obj_t *config; cfg_obj_t *options; @@ -2479,7 +2496,6 @@ load_configuration(const char *filename, ns_server_t *server, isc_timertype_inactive, NULL, NULL, ISC_TRUE)); } else if (server->interface_interval != interface_interval) { - isc_interval_t interval; isc_interval_set(&interval, interface_interval, 0); CHECK(isc_timer_reset(server->interface_timer, isc_timertype_ticker, @@ -2499,13 +2515,16 @@ load_configuration(const char *filename, ns_server_t *server, isc_timertype_inactive, NULL, NULL, ISC_TRUE)); } else if (server->heartbeat_interval != heartbeat_interval) { - isc_interval_t interval; isc_interval_set(&interval, heartbeat_interval, 0); CHECK(isc_timer_reset(server->heartbeat_timer, isc_timertype_ticker, NULL, &interval, ISC_FALSE)); } server->heartbeat_interval = heartbeat_interval; + + isc_interval_set(&interval, 1200, 0); + CHECK(isc_timer_reset(server->pps_timer, isc_timertype_ticker, NULL, + &interval, ISC_FALSE)); /* * Configure and freeze all explicit views. Explicit @@ -2938,6 +2957,12 @@ run_server(isc_task_t *task, isc_event_t *event) { server, &server->heartbeat_timer), "creating heartbeat timer"); + CHECKFATAL(isc_timer_create(ns_g_timermgr, isc_timertype_inactive, + NULL, NULL, server->task, + heartbeat_timer_tick, + server, &server->pps_timer), + "creating pps timer"); + CHECKFATAL(cfg_parser_create(ns_g_mctx, NULL, &ns_g_parser), "creating default configuration parser"); @@ -3002,6 +3027,7 @@ shutdown_server(isc_task_t *task, isc_event_t *event) { isc_timer_detach(&server->interface_timer); isc_timer_detach(&server->heartbeat_timer); + isc_timer_detach(&server->pps_timer); ns_interfacemgr_shutdown(server->interfacemgr); ns_interfacemgr_detach(&server->interfacemgr); @@ -3090,6 +3116,7 @@ ns_server_create(isc_mem_t *mctx, ns_server_t **serverp) { server->interface_timer = NULL; server->heartbeat_timer = NULL; + server->pps_timer = NULL; server->interface_interval = 0; server->heartbeat_interval = 0; diff --git a/lib/dns/cache.c b/lib/dns/cache.c index 16986d883f..0d3bd2a6b6 100644 --- a/lib/dns/cache.c +++ b/lib/dns/cache.c @@ -15,7 +15,7 @@ * PERFORMANCE OF THIS SOFTWARE. */ -/* $Id: cache.c,v 1.61 2005/07/12 01:00:14 marka Exp $ */ +/* $Id: cache.c,v 1.62 2005/08/15 01:21:05 marka Exp $ */ /*! \file */ @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -52,7 +53,7 @@ * CLEANERINCREMENT is how many nodes are examined in one pass. * See also DNS_CACHE_MINSIZE */ -#define DNS_CACHE_CLEANERINCREMENT 1000 /*%< Number of nodes. */ +#define DNS_CACHE_CLEANERINCREMENT 1000U /*%< Number of nodes. */ /*** *** Types @@ -104,7 +105,7 @@ struct cache_cleaner { isc_event_t *overmem_event; dns_dbiterator_t *iterator; - int increment; /*% Number of names to + unsigned int increment; /*% Number of names to clean in one increment */ cleaner_state_t state; /*% Idle/Busy. */ isc_boolean_t overmem; /*% The cache is in an overmem state. */ @@ -156,6 +157,64 @@ cleaner_shutdown_action(isc_task_t *task, isc_event_t *event); static void overmem_cleaning_action(isc_task_t *task, isc_event_t *event); +static void +adjust_increment(cache_cleaner_t *cleaner, unsigned int remaining, + isc_time_t *start) +{ + isc_time_t end; + isc_uint64_t usecs; + isc_uint64_t new; + unsigned int pps = dns_pps; + unsigned int interval; + unsigned int names; + + /* + * Tune for minumum of 100 pps. + */ + if (pps < 100) + pps = 100; + + isc_time_now(&end); + + interval = 1000000 / pps; + if (interval == 0) + interval = 1; + + INSIST(cleaner->increment >= remaining); + names = cleaner->increment - remaining; + usecs = isc_time_microdiff(&end, start); + + isc_log_write(dns_lctx, DNS_LOGCATEGORY_DATABASE, DNS_LOGMODULE_CACHE, + ISC_LOG_INFO, "adjust_increment interval=%u " + "names=%u usec=%" ISC_PLATFORM_QUADFORMAT "u", + interval, names, usecs); + + if (usecs == 0) { + if (names == cleaner->increment) { + cleaner->increment *= 2; + if (cleaner->increment > DNS_CACHE_CLEANERINCREMENT) + cleaner->increment = DNS_CACHE_CLEANERINCREMENT; + isc_log_write(dns_lctx, DNS_LOGCATEGORY_DATABASE, + DNS_LOGMODULE_CACHE, ISC_LOG_INFO, + "new clear->increment = %d\n", + cleaner->increment); + } + return; + } + + new = (names * interval); + new /= usecs; + if (new == 0) + new = 1; + else if (new > DNS_CACHE_CLEANERINCREMENT) + new = DNS_CACHE_CLEANERINCREMENT; + + cleaner->increment = new; + isc_log_write(dns_lctx, DNS_LOGCATEGORY_DATABASE, DNS_LOGMODULE_CACHE, + ISC_LOG_INFO, "new clear->increment = %u\n", + cleaner->increment); +} + static inline isc_result_t cache_create_db(dns_cache_t *cache, dns_db_t **db) { return (dns_db_create(cache->mctx, cache->db_type, dns_rootname, @@ -717,7 +776,8 @@ static void incremental_cleaning_action(isc_task_t *task, isc_event_t *event) { cache_cleaner_t *cleaner = event->ev_arg; isc_result_t result; - int n_names; + unsigned int n_names; + isc_time_t start; UNUSED(task); @@ -736,6 +796,7 @@ incremental_cleaning_action(isc_task_t *task, isc_event_t *event) { REQUIRE(DNS_DBITERATOR_VALID(cleaner->iterator)); + isc_time_now(&start); while (n_names-- > 0) { dns_dbnode_t *node = NULL; @@ -746,6 +807,7 @@ incremental_cleaning_action(isc_task_t *task, isc_event_t *event) { "cache cleaner: dns_dbiterator_current() " "failed: %s", dns_result_totext(result)); + adjust_increment(cleaner, n_names, &start); end_cleaning(cleaner, event); return; } @@ -789,11 +851,14 @@ incremental_cleaning_action(isc_task_t *task, isc_event_t *event) { } } + adjust_increment(cleaner, n_names, &start); end_cleaning(cleaner, event); return; } } + adjust_increment(cleaner, 0U, &start); + /* * We have successfully performed a cleaning increment but have * not gone through the entire cache. Free the iterator locks @@ -804,7 +869,7 @@ incremental_cleaning_action(isc_task_t *task, isc_event_t *event) { RUNTIME_CHECK(result == ISC_R_SUCCESS); isc_log_write(dns_lctx, DNS_LOGCATEGORY_DATABASE, DNS_LOGMODULE_CACHE, - ISC_LOG_DEBUG(1), "cache cleaner: checked %d nodes, " + ISC_LOG_DEBUG(1), "cache cleaner: checked %u nodes, " "mem inuse %lu, sleeping", cleaner->increment, (unsigned long)isc_mem_inuse(cleaner->cache->mctx)); diff --git a/lib/dns/include/dns/lib.h b/lib/dns/include/dns/lib.h index 9f2bd3a632..5f384fd692 100644 --- a/lib/dns/include/dns/lib.h +++ b/lib/dns/include/dns/lib.h @@ -15,7 +15,7 @@ * PERFORMANCE OF THIS SOFTWARE. */ -/* $Id: lib.h,v 1.10 2005/04/29 00:22:59 marka Exp $ */ +/* $Id: lib.h,v 1.11 2005/08/15 01:21:07 marka Exp $ */ #ifndef DNS_LIB_H #define DNS_LIB_H 1 @@ -27,6 +27,7 @@ ISC_LANG_BEGINDECLS +LIBDNS_EXTERNAL_DATA extern unsigned int dns_pps; LIBDNS_EXTERNAL_DATA extern isc_msgcat_t *dns_msgcat; void diff --git a/lib/dns/lib.c b/lib/dns/lib.c index db7a0f6a7f..255d9b2c39 100644 --- a/lib/dns/lib.c +++ b/lib/dns/lib.c @@ -15,7 +15,7 @@ * PERFORMANCE OF THIS SOFTWARE. */ -/* $Id: lib.c,v 1.13 2005/04/29 00:22:48 marka Exp $ */ +/* $Id: lib.c,v 1.14 2005/08/15 01:21:06 marka Exp $ */ /*! \file */ @@ -33,6 +33,7 @@ *** Globals ***/ +LIBDNS_EXTERNAL_DATA unsigned int dns_pps = 0U; LIBDNS_EXTERNAL_DATA isc_msgcat_t * dns_msgcat = NULL; diff --git a/lib/dns/masterdump.c b/lib/dns/masterdump.c index 72327d9558..0cb15f9486 100644 --- a/lib/dns/masterdump.c +++ b/lib/dns/masterdump.c @@ -15,7 +15,7 @@ * PERFORMANCE OF THIS SOFTWARE. */ -/* $Id: masterdump.c,v 1.78 2005/06/20 01:03:53 marka Exp $ */ +/* $Id: masterdump.c,v 1.79 2005/08/15 01:21:06 marka Exp $ */ /*! \file */ @@ -31,12 +31,14 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include #include @@ -1261,6 +1263,7 @@ dumptostreaminc(dns_dumpctx_t *dctx) { unsigned int nodes; dns_masterrawheader_t rawheader; isc_uint32_t now32; + isc_time_t start; bufmem = isc_mem_get(dctx->mctx, initial_buffer_length); if (bufmem == NULL) @@ -1333,6 +1336,7 @@ dumptostreaminc(dns_dumpctx_t *dctx) { result = ISC_R_SUCCESS; nodes = dctx->nodes; + isc_time_now(&start); while (result == ISC_R_SUCCESS && (dctx->nodes == 0 || nodes--)) { dns_rdatasetiter_t *rdsiter = NULL; dns_dbnode_t *node = NULL; @@ -1367,6 +1371,30 @@ dumptostreaminc(dns_dumpctx_t *dctx) { } if (dctx->nodes != 0 && result == ISC_R_SUCCESS) { + unsigned int pps = dns_pps; + unsigned int interval; + isc_uint64_t usecs; + isc_time_t end; + + isc_time_now(&end); + if (pps < 100) + pps = 100; + interval = 1000000 / pps; + if (interval == 0) + interval = 1; + usecs = isc_time_microdiff(&end, &start); + if (usecs == 0) { + dctx->nodes = dctx->nodes * 2; + if (dctx->nodes > 1000) + dctx->nodes = 1000; + } else { + dctx->nodes = dctx->nodes * interval; + dctx->nodes /= usecs; + if (dctx->nodes == 0) + dctx->nodes = 1; + else if (dctx->nodes > 1000) + dctx->nodes = 1000; + } dns_dbiterator_pause(dctx->dbiter); result = DNS_R_CONTINUE; } else if (result == ISC_R_NOMORE) diff --git a/lib/dns/rbtdb.c b/lib/dns/rbtdb.c index f48fbd322a..86420ec9c7 100644 --- a/lib/dns/rbtdb.c +++ b/lib/dns/rbtdb.c @@ -15,7 +15,7 @@ * PERFORMANCE OF THIS SOFTWARE. */ -/* $Id: rbtdb.c,v 1.214 2005/07/18 05:58:58 marka Exp $ */ +/* $Id: rbtdb.c,v 1.215 2005/08/15 01:21:06 marka Exp $ */ /*! \file */ @@ -34,6 +34,7 @@ #include #include #include +#include #include #include @@ -41,6 +42,7 @@ #include #include #include +#include #include #include #include @@ -346,6 +348,7 @@ typedef struct { /* Unlocked */ isc_mem_t ** nodemctxs; + unsigned int quantum; } dns_rbtdb_t; #define RBTDB_ATTR_LOADED 0x01 @@ -548,12 +551,43 @@ free_rbtdb_callback(isc_task_t *task, isc_event_t *event) { free_rbtdb(rbtdb, ISC_TRUE, event); } +static unsigned int +adjust_quantum(unsigned int old, isc_time_t *start) { + unsigned int pps = dns_pps; + unsigned int interval; + isc_uint64_t usecs; + isc_time_t end; + + if (pps < 100) + pps = 100; + isc_time_now(&end); + + interval = 1000000 / pps; + if (interval == 0) + interval = 1; + usecs = isc_time_microdiff(&end, start); + if (usecs == 0) { + old *= 2; + if (old > 1000) + old = 1000; + return (old); + } + old = old * interval; + old /= usecs; + if (old == 0) + old = 1; + else if (old > 1000) + old = 1000; + return (old); +} + static void free_rbtdb(dns_rbtdb_t *rbtdb, isc_boolean_t log, isc_event_t *event) { unsigned int i; isc_ondestroy_t ondest; isc_result_t result; char buf[DNS_NAME_FORMATSIZE]; + isc_time_t start; REQUIRE(rbtdb->current_version != NULL || EMPTY(rbtdb->open_versions)); REQUIRE(rbtdb->future_version == NULL); @@ -569,12 +603,17 @@ free_rbtdb(dns_rbtdb_t *rbtdb, isc_boolean_t log, isc_event_t *event) { isc_mem_put(rbtdb->common.mctx, rbtdb->current_version, sizeof(rbtdb_version_t)); } + if (event == NULL) + rbtdb->quantum = (rbtdb->task != NULL) ? 100 : 0; again: if (rbtdb->tree != NULL) { - result = dns_rbt_destroy2(&rbtdb->tree, - (rbtdb->task != NULL) ? 1000 : 0); + isc_time_now(&start); + result = dns_rbt_destroy2(&rbtdb->tree, rbtdb->quantum); if (result == ISC_R_QUOTA) { INSIST(rbtdb->task != NULL); + if (rbtdb->quantum != 0) + rbtdb->quantum = adjust_quantum(rbtdb->quantum, + &start); if (event == NULL) event = isc_event_allocate(rbtdb->common.mctx, NULL,