From b09106e93aaa61d195f9bffe1bb493764dd8da1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Sur=C3=BD?= Date: Thu, 4 Feb 2021 21:56:49 +0100 Subject: [PATCH] Make the memory and mempool counters to be stdatomic types This is yet another step into unlocking some parts of the memory contexts. All the regularly updated variables has been turned into atomic types, so we can later remove the locks when updating various counters. Also unlock as much code as possible without breaking anything. --- lib/isc/include/isc/mem.h | 64 +-- lib/isc/mem.c | 836 +++++++++++++++--------------------- lib/isc/win32/libisc.def.in | 2 + 3 files changed, 394 insertions(+), 508 deletions(-) diff --git a/lib/isc/include/isc/mem.h b/lib/isc/include/isc/mem.h index 01dac4c2e0..2d4f70dedd 100644 --- a/lib/isc/include/isc/mem.h +++ b/lib/isc/include/isc/mem.h @@ -250,6 +250,19 @@ isc_mem_total(isc_mem_t *mctx); * not yet used. */ +size_t +isc_mem_malloced(isc_mem_t *ctx); +/*%< + * Get an estimate of the amount of memory allocated in 'mctx', in bytes. + */ + +size_t +isc_mem_maxmalloced(isc_mem_t *ctx); +/*%< + * Get an estimate of the largest amount of memory that has been + * allocated in 'mctx' at any time. + */ + bool isc_mem_isovermem(isc_mem_t *mctx); /*%< @@ -265,13 +278,13 @@ isc_mem_setwater(isc_mem_t *mctx, isc_mem_water_t water, void *water_arg, * Set high and low water marks for this memory context. * * When the memory usage of 'mctx' exceeds 'hiwater', - * '(water)(water_arg, #ISC_MEM_HIWATER)' will be called. 'water' needs to - * call isc_mem_waterack() with #ISC_MEM_HIWATER to acknowledge the state - * change. 'water' may be called multiple times. + * '(water)(water_arg, #ISC_MEM_HIWATER)' will be called. 'water' needs + *to call isc_mem_waterack() with #ISC_MEM_HIWATER to acknowledge the + *state change. 'water' may be called multiple times. * - * When the usage drops below 'lowater', 'water' will again be called, this - * time with #ISC_MEM_LOWATER. 'water' need to calls isc_mem_waterack() with - * #ISC_MEM_LOWATER to acknowledge the change. + * When the usage drops below 'lowater', 'water' will again be called, + *this time with #ISC_MEM_LOWATER. 'water' need to calls + *isc_mem_waterack() with #ISC_MEM_LOWATER to acknowledge the change. * * static void * water(void *arg, int mark) { @@ -396,7 +409,8 @@ isc_mempool_destroy(isc_mempool_t **mpctxp); void isc_mempool_setname(isc_mempool_t *mpctx, const char *name); /*%< - * Associate a name with a memory pool. At most 15 characters may be used. + * Associate a name with a memory pool. At most 15 characters may be + *used. * * Requires: *\li mpctx is a valid pool. @@ -408,15 +422,15 @@ isc_mempool_associatelock(isc_mempool_t *mpctx, isc_mutex_t *lock); /*%< * Associate a lock with this memory pool. * - * This lock is used when getting or putting items using this memory pool, - * and it is also used to set or get internal state via the isc_mempool_get*() - * and isc_mempool_set*() set of functions. + * This lock is used when getting or putting items using this memory + *pool, and it is also used to set or get internal state via the + *isc_mempool_get*() and isc_mempool_set*() set of functions. * - * Multiple pools can each share a single lock. For instance, if "manager" - * type object contained pools for various sizes of events, and each of - * these pools used a common lock. Note that this lock must NEVER be used - * by other than mempool routines once it is given to a pool, since that can - * easily cause double locking. + * Multiple pools can each share a single lock. For instance, if + *"manager" type object contained pools for various sizes of events, and + *each of these pools used a common lock. Note that this lock must + *NEVER be used by other than mempool routines once it is given to a + *pool, since that can easily cause double locking. * * Requires: * @@ -426,19 +440,19 @@ isc_mempool_associatelock(isc_mempool_t *mpctx, isc_mutex_t *lock); * *\li No previous lock is assigned to this pool. * - *\li The lock is initialized before calling this function via the normal - * means of doing that. + *\li The lock is initialized before calling this function via the + *normal means of doing that. */ /* * The following functions get/set various parameters. Note that due to - * the unlocked nature of pools these are potentially random values unless - * the imposed externally provided locking protocols are followed. + * the unlocked nature of pools these are potentially random values + *unless the imposed externally provided locking protocols are followed. * - * Also note that the quota limits will not always take immediate effect. - * For instance, setting "maxalloc" to a number smaller than the currently - * allocated count is permitted. New allocations will be refused until - * the count drops below this threshold. + * Also note that the quota limits will not always take immediate + *effect. For instance, setting "maxalloc" to a number smaller than the + *currently allocated count is permitted. New allocations will be + *refused until the count drops below this threshold. * * All functions require (in addition to other requirements): * mpctx is a valid memory pool @@ -486,8 +500,8 @@ isc_mempool_getallocated(isc_mempool_t *mpctx); unsigned int isc_mempool_getfillcount(isc_mempool_t *mpctx); /*%< - * Returns the number of items allocated as a block from the parent memory - * context when the free list is empty. + * Returns the number of items allocated as a block from the parent + * memory context when the free list is empty. */ void diff --git a/lib/isc/mem.c b/lib/isc/mem.c index 7a0c650247..c3c4ac463a 100644 --- a/lib/isc/mem.c +++ b/lib/isc/mem.c @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -45,6 +46,14 @@ #define MCTXLOCK(m) LOCK(&m->lock) #define MCTXUNLOCK(m) UNLOCK(&m->lock) +#define MPCTXLOCK(mp) \ + if (mp->lock != NULL) { \ + LOCK(mp->lock); \ + } +#define MPCTXUNLOCK(mp) \ + if (mp->lock != NULL) { \ + UNLOCK(mp->lock); \ + } #ifndef ISC_MEM_DEBUGGING #define ISC_MEM_DEBUGGING 0 @@ -56,9 +65,11 @@ LIBISC_EXTERNAL_DATA unsigned int isc_mem_defaultflags = ISC_MEMFLAG_DEFAULT; * Constants. */ -#define DEF_MAX_SIZE 1100 -#define ALIGNMENT_SIZE 8U /*%< must be a power of 2 */ +#define ALIGNMENT 8U /*%< must be a power of 2 */ +#define ALIGNMENT_SIZE sizeof(size_info) #define DEBUG_TABLE_COUNT 512U +#define STATS_BUCKETS 512U +#define STATS_BUCKET_SIZE 32U /* * Types. @@ -88,19 +99,15 @@ struct element { }; typedef struct { - /*! - * This structure must be ALIGNMENT_SIZE bytes. - */ - union { + alignas(ALIGNMENT) union { size_t size; isc_mem_t *ctx; - char bytes[ALIGNMENT_SIZE]; - } u; + }; } size_info; struct stats { - unsigned long gets; - unsigned long totalgets; + atomic_size_t gets; + atomic_size_t totalgets; }; #define MEM_MAGIC ISC_MAGIC('M', 'e', 'm', 'C') @@ -123,20 +130,19 @@ struct isc_mem { unsigned int magic; unsigned int flags; isc_mutex_t lock; - size_t max_size; bool checkfree; - struct stats *stats; + struct stats stats[STATS_BUCKETS + 1]; isc_refcount_t references; char name[16]; - size_t total; - size_t inuse; - size_t maxinuse; - size_t malloced; - size_t maxmalloced; - size_t hi_water; - size_t lo_water; - bool hi_called; - bool is_overmem; + atomic_size_t total; + atomic_size_t inuse; + atomic_size_t maxinuse; + atomic_size_t malloced; + atomic_size_t maxmalloced; + atomic_size_t hi_water; + atomic_size_t lo_water; + atomic_bool hi_called; + atomic_bool is_overmem; isc_mem_water_t water; void *water_arg; ISC_LIST(isc_mempool_t) pools; @@ -161,16 +167,16 @@ struct isc_mempool { /*%< locked via the memory context's lock */ ISC_LINK(isc_mempool_t) link; /*%< next pool in this mem context */ /*%< optionally locked from here down */ - element *items; /*%< low water item list */ - size_t size; /*%< size of each item on this pool */ - unsigned int maxalloc; /*%< max number of items allowed */ - unsigned int allocated; /*%< # of items currently given out */ - unsigned int freecount; /*%< # of items on reserved list */ - unsigned int freemax; /*%< # of items allowed on free list */ - unsigned int fillcount; /*%< # of items to fetch on each fill */ + element *items; /*%< low water item list */ + size_t size; /*%< size of each item on this pool */ + atomic_size_t maxalloc; /*%< max number of items allowed */ + atomic_size_t allocated; /*%< # of items currently given out */ + atomic_size_t freecount; /*%< # of items on reserved list */ + atomic_size_t freemax; /*%< # of items allowed on free list */ + atomic_size_t fillcount; /*%< # of items to fetch on each fill */ /*%< Stats only. */ - unsigned int gets; /*%< # of requests to this pool */ - /*%< Debugging only. */ + atomic_size_t gets; /*%< # of requests to this pool */ + /*%< Debugging only. */ #if ISC_MEMPOOL_NAMES char name[16]; /*%< printed name in stats reports */ #endif /* if ISC_MEMPOOL_NAMES */ @@ -186,27 +192,48 @@ struct isc_mempool { #define ISC_MEMFUNC_SCOPE #else /* if !ISC_MEM_TRACKLINES */ #define TRACE_OR_RECORD (ISC_MEM_DEBUGTRACE | ISC_MEM_DEBUGRECORD) -#define ADD_TRACE(a, b, c, d, e) \ - do { \ - if (ISC_UNLIKELY((isc_mem_debugging & TRACE_OR_RECORD) != 0 && \ - b != NULL)) \ - add_trace_entry(a, b, c, d, e); \ - } while (0) -#define DELETE_TRACE(a, b, c, d, e) \ - do { \ - if (ISC_UNLIKELY((isc_mem_debugging & TRACE_OR_RECORD) != 0 && \ - b != NULL)) \ - delete_trace_entry(a, b, c, d, e); \ - } while (0) + +#define SHOULD_TRACE_OR_RECORD(ptr) \ + (ISC_UNLIKELY((isc_mem_debugging & TRACE_OR_RECORD) != 0) && \ + ptr != NULL) + +#define ADD_TRACE(a, b, c, d, e) \ + if (SHOULD_TRACE_OR_RECORD(b)) { \ + add_trace_entry(a, b, c, d, e); \ + } + +#define DELETE_TRACE(a, b, c, d, e) \ + if (SHOULD_TRACE_OR_RECORD(b)) { \ + delete_trace_entry(a, b, c, d, e); \ + } static void print_active(isc_mem_t *ctx, FILE *out); - #endif /* ISC_MEM_TRACKLINES */ +static inline size_t +increment_malloced(isc_mem_t *ctx, size_t size) { + size_t malloced = atomic_fetch_add_relaxed(&ctx->malloced, size) + size; + size_t maxmalloced = atomic_load_acquire(&ctx->maxmalloced); + if (malloced > maxmalloced) { + atomic_compare_exchange_strong(&ctx->maxmalloced, &maxmalloced, + malloced); + } + + return (malloced); +} + +static inline size_t +decrement_malloced(isc_mem_t *ctx, size_t size) { + size_t malloced = atomic_fetch_sub_release(&ctx->malloced, size) - size; + INSIST(size >= 0); + + return (malloced); +} + #if ISC_MEM_TRACKLINES /*! - * mctx must be locked. + * mctx must not be locked. */ static void add_trace_entry(isc_mem_t *mctx, const void *ptr, size_t size FLARG) { @@ -214,13 +241,15 @@ add_trace_entry(isc_mem_t *mctx, const void *ptr, size_t size FLARG) { uint32_t hash; uint32_t idx; + MCTXLOCK(mctx); + if ((isc_mem_debugging & ISC_MEM_DEBUGTRACE) != 0) { fprintf(stderr, "add %p size %zu file %s line %u mctx %p\n", ptr, size, file, line, mctx); } if (mctx->debuglist == NULL) { - return; + goto unlock; } #ifdef __COVERITY__ @@ -236,10 +265,7 @@ add_trace_entry(isc_mem_t *mctx, const void *ptr, size_t size FLARG) { dl = malloc(sizeof(debuglink_t)); INSIST(dl != NULL); - mctx->malloced += sizeof(debuglink_t); - if (mctx->malloced > mctx->maxmalloced) { - mctx->maxmalloced = mctx->malloced; - } + increment_malloced(mctx, sizeof(debuglink_t)); ISC_LINK_INIT(dl, link); dl->ptr = ptr; @@ -249,6 +275,8 @@ add_trace_entry(isc_mem_t *mctx, const void *ptr, size_t size FLARG) { ISC_LIST_PREPEND(mctx->debuglist[idx], dl, link); mctx->debuglistcnt++; +unlock: + MCTXUNLOCK(mctx); } static void @@ -258,13 +286,15 @@ delete_trace_entry(isc_mem_t *mctx, const void *ptr, size_t size, uint32_t hash; uint32_t idx; + MCTXLOCK(mctx); + if ((isc_mem_debugging & ISC_MEM_DEBUGTRACE) != 0) { fprintf(stderr, "del %p size %zu file %s line %u mctx %p\n", ptr, size, file, line, mctx); } if (mctx->debuglist == NULL) { - return; + goto unlock; } #ifdef __COVERITY__ @@ -282,9 +312,9 @@ delete_trace_entry(isc_mem_t *mctx, const void *ptr, size_t size, while (ISC_LIKELY(dl != NULL)) { if (ISC_UNLIKELY(dl->ptr == ptr)) { ISC_LIST_UNLINK(mctx->debuglist[idx], dl, link); - mctx->malloced -= sizeof(*dl); + decrement_malloced(mctx, sizeof(*dl)); free(dl); - return; + goto unlock; } dl = ISC_LIST_NEXT(dl, link); } @@ -295,6 +325,8 @@ delete_trace_entry(isc_mem_t *mctx, const void *ptr, size_t size, */ INSIST(0); ISC_UNREACHABLE(); +unlock: + MCTXUNLOCK(mctx); } #endif /* ISC_MEM_TRACKLINES */ @@ -348,29 +380,25 @@ mem_put(isc_mem_t *ctx, void *mem, size_t size) { default_memfree(mem); } +#define stats_bucket(ctx, size) \ + ((size / STATS_BUCKET_SIZE) >= STATS_BUCKETS \ + ? &ctx->stats[STATS_BUCKETS] \ + : &ctx->stats[size / STATS_BUCKET_SIZE]) + /*! * Update internal counters after a memory get. */ static inline void mem_getstats(isc_mem_t *ctx, size_t size) { - ctx->total += size; - ctx->inuse += size; + struct stats *stats = stats_bucket(ctx, size); - if (size > ctx->max_size) { - ctx->stats[ctx->max_size].gets++; - ctx->stats[ctx->max_size].totalgets++; - } else { - ctx->stats[size].gets++; - ctx->stats[size].totalgets++; - } + atomic_fetch_add_relaxed(&ctx->total, size); + atomic_fetch_add_release(&ctx->inuse, size); -#if ISC_MEM_CHECKOVERRUN - size += 1; -#endif /* if ISC_MEM_CHECKOVERRUN */ - ctx->malloced += size; - if (ctx->malloced > ctx->maxmalloced) { - ctx->maxmalloced = ctx->malloced; - } + atomic_fetch_add_relaxed(&stats->gets, 1); + atomic_fetch_add_relaxed(&stats->totalgets, 1); + + increment_malloced(ctx, size); } /*! @@ -378,22 +406,15 @@ mem_getstats(isc_mem_t *ctx, size_t size) { */ static inline void mem_putstats(isc_mem_t *ctx, void *ptr, size_t size) { + struct stats *stats = stats_bucket(ctx, size); + UNUSED(ptr); - INSIST(ctx->inuse >= size); - ctx->inuse -= size; + INSIST(atomic_fetch_sub_release(&ctx->inuse, size) >= size); - if (size > ctx->max_size) { - INSIST(ctx->stats[ctx->max_size].gets > 0U); - ctx->stats[ctx->max_size].gets--; - } else { - INSIST(ctx->stats[size].gets > 0U); - ctx->stats[size].gets--; - } -#if ISC_MEM_CHECKOVERRUN - size += 1; -#endif /* if ISC_MEM_CHECKOVERRUN */ - ctx->malloced -= size; + INSIST(atomic_fetch_sub_release(&stats->gets, 1) >= 1); + + decrement_malloced(ctx, size); } /* @@ -450,45 +471,30 @@ mem_create(isc_mem_t **ctxp, unsigned int flags) { isc_mem_t *ctx; STATIC_ASSERT((ALIGNMENT_SIZE & (ALIGNMENT_SIZE - 1)) == 0, - "wrong alignment size"); + "alignment size not power of 2"); + STATIC_ASSERT(ALIGNMENT_SIZE >= sizeof(size_info), + "alignment size too small"); RUNTIME_CHECK(isc_once_do(&once, initialize_action) == ISC_R_SUCCESS); - ctx = (default_memalloc)(sizeof(*ctx)); + ctx = default_memalloc(sizeof(*ctx)); + + *ctx = (isc_mem_t){ + .magic = MEM_MAGIC, + .flags = flags, + .checkfree = true, + }; isc_mutex_init(&ctx->lock); - - ctx->max_size = DEF_MAX_SIZE; - ctx->flags = flags; isc_refcount_init(&ctx->references, 1); - memset(ctx->name, 0, sizeof(ctx->name)); - ctx->total = 0; - ctx->inuse = 0; - ctx->maxinuse = 0; - ctx->malloced = sizeof(*ctx); - ctx->maxmalloced = sizeof(*ctx); - ctx->hi_water = 0; - ctx->lo_water = 0; - ctx->hi_called = false; - ctx->is_overmem = false; - ctx->water = NULL; - ctx->water_arg = NULL; - ctx->magic = MEM_MAGIC; - ctx->stats = NULL; - ctx->checkfree = true; -#if ISC_MEM_TRACKLINES - ctx->debuglist = NULL; - ctx->debuglistcnt = 0; -#endif /* if ISC_MEM_TRACKLINES */ + + atomic_init(&ctx->total, 0); + atomic_init(&ctx->inuse, 0); + atomic_init(&ctx->maxinuse, 0); + atomic_init(&ctx->malloced, sizeof(*ctx)); + atomic_init(&ctx->maxmalloced, sizeof(*ctx)); + ISC_LIST_INIT(ctx->pools); - ctx->poolcnt = 0; - - ctx->stats = - default_memalloc((ctx->max_size + 1) * sizeof(struct stats)); - - memset(ctx->stats, 0, (ctx->max_size + 1) * sizeof(struct stats)); - ctx->malloced += (ctx->max_size + 1) * sizeof(struct stats); - ctx->maxmalloced += (ctx->max_size + 1) * sizeof(struct stats); #if ISC_MEM_TRACKLINES if (ISC_UNLIKELY((isc_mem_debugging & ISC_MEM_DEBUGRECORD) != 0)) { @@ -499,8 +505,8 @@ mem_create(isc_mem_t **ctxp, unsigned int flags) { for (i = 0; i < DEBUG_TABLE_COUNT; i++) { ISC_LIST_INIT(ctx->debuglist[i]); } - ctx->malloced += DEBUG_TABLE_COUNT * sizeof(debuglist_t); - ctx->maxmalloced += DEBUG_TABLE_COUNT * sizeof(debuglist_t); + increment_malloced(ctx, + DEBUG_TABLE_COUNT * sizeof(debuglist_t)); } #endif /* if ISC_MEM_TRACKLINES */ @@ -518,10 +524,11 @@ mem_create(isc_mem_t **ctxp, unsigned int flags) { static void destroy(isc_mem_t *ctx) { unsigned int i; + size_t malloced; LOCK(&contextslock); ISC_LIST_UNLINK(contexts, ctx, link); - totallost += ctx->inuse; + totallost += isc_mem_inuse(ctx); UNLOCK(&contextslock); ctx->magic = 0; @@ -542,39 +549,40 @@ destroy(isc_mem_t *ctx) { ISC_LIST_UNLINK(ctx->debuglist[i], dl, link); free(dl); - ctx->malloced -= sizeof(*dl); + decrement_malloced(ctx, sizeof(*dl)); } } default_memfree(ctx->debuglist); - ctx->malloced -= DEBUG_TABLE_COUNT * sizeof(debuglist_t); + decrement_malloced(ctx, + DEBUG_TABLE_COUNT * sizeof(debuglist_t)); } #endif /* if ISC_MEM_TRACKLINES */ if (ctx->checkfree) { - for (i = 0; i <= ctx->max_size; i++) { - if (ctx->stats[i].gets != 0U) { + for (i = 0; i <= STATS_BUCKETS; i++) { + struct stats *stats = &ctx->stats[i]; + size_t gets = atomic_load_acquire(&stats->gets); + if (gets != 0U) { fprintf(stderr, "Failing assertion due to probable " "leaked memory in context %p (\"%s\") " - "(stats[%u].gets == %lu).\n", - ctx, ctx->name, i, ctx->stats[i].gets); + "(stats[%u].gets == %zu).\n", + ctx, ctx->name, i, gets); #if ISC_MEM_TRACKLINES print_active(ctx, stderr); #endif /* if ISC_MEM_TRACKLINES */ - INSIST(ctx->stats[i].gets == 0U); + INSIST(gets == 0U); } } } - default_memfree(ctx->stats); - ctx->malloced -= (ctx->max_size + 1) * sizeof(struct stats); - isc_mutex_destroy(&ctx->lock); - ctx->malloced -= sizeof(*ctx); + malloced = decrement_malloced(ctx, sizeof(*ctx)); + if (ctx->checkfree) { - INSIST(ctx->malloced == 0); + INSIST(malloced == 0); } default_memfree(ctx); } @@ -625,7 +633,7 @@ isc__mem_putanddetach(isc_mem_t **ctxp, void *ptr, size_t size FLARG) { { if ((isc_mem_debugging & ISC_MEM_DEBUGSIZE) != 0) { size_info *si = &(((size_info *)ptr)[-1]); - size_t oldsize = si->u.size - ALIGNMENT_SIZE; + size_t oldsize = si->size - ALIGNMENT_SIZE; if ((isc_mem_debugging & ISC_MEM_DEBUGCTX) != 0) { oldsize -= ALIGNMENT_SIZE; } @@ -636,13 +644,10 @@ isc__mem_putanddetach(isc_mem_t **ctxp, void *ptr, size_t size FLARG) { goto destroy; } - MCTXLOCK(ctx); - DELETE_TRACE(ctx, ptr, size, file, line); mem_putstats(ctx, ptr, size); mem_put(ctx, ptr, size); - MCTXUNLOCK(ctx); destroy: if (isc_refcount_decrement(&ctx->references) == 1) { @@ -675,6 +680,55 @@ isc_mem_destroy(isc_mem_t **ctxp) { *ctxp = NULL; } +static inline bool +hi_water(isc_mem_t *ctx) { + bool call_water = false; + size_t inuse = atomic_load_acquire(&ctx->inuse); + size_t maxinuse = atomic_load_acquire(&ctx->maxinuse); + size_t hi_water = atomic_load_acquire(&ctx->hi_water); + + if (hi_water != 0U && inuse > hi_water) { + atomic_store(&ctx->is_overmem, true); + if (!atomic_load_acquire(&ctx->hi_called)) { + call_water = true; + } + } + if (inuse > maxinuse) { + (void)atomic_compare_exchange_strong(&ctx->maxinuse, &maxinuse, + inuse); + + if (hi_water != 0U && inuse > hi_water && + (isc_mem_debugging & ISC_MEM_DEBUGUSAGE) != 0) + { + fprintf(stderr, "maxinuse = %lu\n", + (unsigned long)inuse); + } + } + + return (call_water); +} + +/* + * The check against ctx->lo_water == 0 is for the condition + * when the context was pushed over hi_water but then had + * isc_mem_setwater() called with 0 for hi_water and lo_water. + */ +static inline bool +lo_water(isc_mem_t *ctx) { + bool call_water = false; + size_t inuse = atomic_load_acquire(&ctx->inuse); + size_t lo_water = atomic_load_acquire(&ctx->lo_water); + + if ((inuse < lo_water) || (lo_water == 0U)) { + atomic_store(&ctx->is_overmem, false); + if (atomic_load_acquire(&ctx->hi_called)) { + call_water = true; + } + } + + return (call_water); +} + void * isc__mem_get(isc_mem_t *ctx, size_t size FLARG) { REQUIRE(VALID_CONTEXT(ctx)); @@ -689,27 +743,11 @@ isc__mem_get(isc_mem_t *ctx, size_t size FLARG) { } ptr = mem_get(ctx, size); - MCTXLOCK(ctx); mem_getstats(ctx, size); ADD_TRACE(ctx, ptr, size, file, line); - if (ctx->hi_water != 0U && ctx->inuse > ctx->hi_water) { - ctx->is_overmem = true; - if (!ctx->hi_called) { - call_water = true; - } - } - if (ctx->inuse > ctx->maxinuse) { - ctx->maxinuse = ctx->inuse; - if (ctx->hi_water != 0U && ctx->inuse > ctx->hi_water && - (isc_mem_debugging & ISC_MEM_DEBUGUSAGE) != 0) - { - fprintf(stderr, "maxinuse = %lu\n", - (unsigned long)ctx->inuse); - } - } - MCTXUNLOCK(ctx); + call_water = hi_water(ctx); if (call_water && (ctx->water != NULL)) { (ctx->water)(ctx->water_arg, ISC_MEM_HIWATER); @@ -725,14 +763,14 @@ isc__mem_put(isc_mem_t *ctx, void *ptr, size_t size FLARG) { bool call_water = false; size_info *si; - size_t oldsize; if (ISC_UNLIKELY((isc_mem_debugging & (ISC_MEM_DEBUGSIZE | ISC_MEM_DEBUGCTX)) != 0)) { if ((isc_mem_debugging & ISC_MEM_DEBUGSIZE) != 0) { + size_t oldsize; si = &(((size_info *)ptr)[-1]); - oldsize = si->u.size - ALIGNMENT_SIZE; + oldsize = si->size - ALIGNMENT_SIZE; if ((isc_mem_debugging & ISC_MEM_DEBUGCTX) != 0) { oldsize -= ALIGNMENT_SIZE; } @@ -742,26 +780,12 @@ isc__mem_put(isc_mem_t *ctx, void *ptr, size_t size FLARG) { return; } - MCTXLOCK(ctx); - DELETE_TRACE(ctx, ptr, size, file, line); mem_putstats(ctx, ptr, size); mem_put(ctx, ptr, size); - /* - * The check against ctx->lo_water == 0 is for the condition - * when the context was pushed over hi_water but then had - * isc_mem_setwater() called with 0 for hi_water and lo_water. - */ - if ((ctx->inuse < ctx->lo_water) || (ctx->lo_water == 0U)) { - ctx->is_overmem = false; - if (ctx->hi_called) { - call_water = true; - } - } - - MCTXUNLOCK(ctx); + call_water = lo_water(ctx); if (call_water && (ctx->water != NULL)) { (ctx->water)(ctx->water_arg, ISC_MEM_LOWATER); @@ -772,13 +796,11 @@ void isc_mem_waterack(isc_mem_t *ctx, int flag) { REQUIRE(VALID_CONTEXT(ctx)); - MCTXLOCK(ctx); if (flag == ISC_MEM_LOWATER) { - ctx->hi_called = false; + atomic_store(&ctx->hi_called, false); } else if (flag == ISC_MEM_HIWATER) { - ctx->hi_called = true; + atomic_store(&ctx->hi_called, true); } - MCTXUNLOCK(ctx); } #if ISC_MEM_TRACKLINES @@ -789,7 +811,8 @@ print_active(isc_mem_t *mctx, FILE *out) { unsigned int i; bool found; - fprintf(out, "Dump of all outstanding memory allocations:\n"); + fprintf(out, "Dump of all outstanding memory " + "allocations:\n"); found = false; for (i = 0; i < DEBUG_TABLE_COUNT; i++) { dl = ISC_LIST_HEAD(mctx->debuglist[i]); @@ -801,7 +824,8 @@ print_active(isc_mem_t *mctx, FILE *out) { while (dl != NULL) { if (dl->ptr != NULL) { fprintf(out, - "\tptr %p size %zu file %s " + "\tptr %p size %zu " + "file %s " "line %u\n", dl->ptr, dl->size, dl->file, dl->line); @@ -824,30 +848,32 @@ void isc_mem_stats(isc_mem_t *ctx, FILE *out) { REQUIRE(VALID_CONTEXT(ctx)); - size_t i; - const struct stats *s; - const isc_mempool_t *pool; + isc_mempool_t *pool; MCTXLOCK(ctx); - for (i = 0; i <= ctx->max_size; i++) { - s = &ctx->stats[i]; + for (size_t i = 0; i <= STATS_BUCKETS; i++) { + size_t totalgets; + size_t gets; + struct stats *stats = &ctx->stats[i]; - if (s->totalgets == 0U && s->gets == 0U) { - continue; + totalgets = atomic_load_acquire(&stats->totalgets); + gets = atomic_load_acquire(&stats->gets); + + if (totalgets != 0U && gets != 0U) { + fprintf(out, "%s%5zu: %11zu gets, %11zu rem", + (i == STATS_BUCKETS) ? ">=" : " ", i, + totalgets, gets); + fputc('\n', out); } - fprintf(out, "%s%5lu: %11lu gets, %11lu rem", - (i == ctx->max_size) ? ">=" : " ", (unsigned long)i, - s->totalgets, s->gets); - fputc('\n', out); } /* - * Note that since a pool can be locked now, these stats might be - * somewhat off if the pool is in active use at the time the stats - * are dumped. The link fields are protected by the isc_mem_t's - * lock, however, so walking this list and extracting integers from - * stats fields is always safe. + * Note that since a pool can be locked now, these stats might + * be somewhat off if the pool is in active use at the time the + * stats are dumped. The link fields are protected by the + * isc_mem_t's lock, however, so walking this list and + * extracting integers from stats fields is always safe. */ pool = ISC_LIST_HEAD(ctx->pools); if (pool != NULL) { @@ -857,15 +883,19 @@ isc_mem_stats(isc_mem_t *ctx, FILE *out) { "freemax", "fillcount", "gets", "L"); } while (pool != NULL) { - fprintf(out, "%15s %10lu %10u %10u %10u %10u %10u %10u %s\n", + fprintf(out, + "%15s %10zu %10zu %10zu %10zu %10zu %10zu %10zu %s\n", #if ISC_MEMPOOL_NAMES pool->name, #else /* if ISC_MEMPOOL_NAMES */ "(not tracked)", #endif /* if ISC_MEMPOOL_NAMES */ - (unsigned long)pool->size, pool->maxalloc, - pool->allocated, pool->freecount, pool->freemax, - pool->fillcount, pool->gets, + pool->size, atomic_load_relaxed(&pool->maxalloc), + atomic_load_relaxed(&pool->allocated), + atomic_load_relaxed(&pool->freecount), + atomic_load_relaxed(&pool->freemax), + atomic_load_relaxed(&pool->fillcount), + atomic_load_relaxed(&pool->gets), (pool->lock == NULL ? "N" : "Y")); pool = ISC_LIST_NEXT(pool, link); } @@ -894,10 +924,10 @@ mem_allocateunlocked(isc_mem_t *ctx, size_t size) { si = mem_get(ctx, size); if (ISC_UNLIKELY((isc_mem_debugging & ISC_MEM_DEBUGCTX) != 0)) { - si->u.ctx = ctx; + si->ctx = ctx; si++; } - si->u.size = size; + si->size = size; return (&si[1]); } @@ -908,34 +938,14 @@ isc__mem_allocate(isc_mem_t *ctx, size_t size FLARG) { size_info *si; bool call_water = false; - MCTXLOCK(ctx); si = mem_allocateunlocked(ctx, size); - mem_getstats(ctx, si[-1].u.size); + mem_getstats(ctx, si[-1].size); - ADD_TRACE(ctx, si, si[-1].u.size, file, line); - if (ctx->hi_water != 0U && ctx->inuse > ctx->hi_water && - !ctx->is_overmem) { - ctx->is_overmem = true; - } + ADD_TRACE(ctx, si, si[-1].size, file, line); - if (ctx->hi_water != 0U && !ctx->hi_called && - ctx->inuse > ctx->hi_water) { - ctx->hi_called = true; - call_water = true; - } - if (ctx->inuse > ctx->maxinuse) { - ctx->maxinuse = ctx->inuse; - if (ISC_UNLIKELY(ctx->hi_water != 0U && - ctx->inuse > ctx->hi_water && - (isc_mem_debugging & ISC_MEM_DEBUGUSAGE) != 0)) - { - fprintf(stderr, "maxinuse = %lu\n", - (unsigned long)ctx->inuse); - } - } - MCTXUNLOCK(ctx); + call_water = hi_water(ctx); - if (call_water) { + if (call_water && (ctx->water != NULL)) { (ctx->water)(ctx->water_arg, ISC_MEM_HIWATER); } @@ -947,15 +957,16 @@ isc__mem_reallocate(isc_mem_t *ctx, void *ptr, size_t size FLARG) { REQUIRE(VALID_CONTEXT(ctx)); void *new_ptr = NULL; - size_t oldsize, copysize; /* - * This function emulates the realloc(3) standard library function: - * - if size > 0, allocate new memory; and if ptr is non NULL, copy - * as much of the old contents to the new buffer and free the old one. - * Note that when allocation fails the original pointer is intact; - * the caller must free it. - * - if size is 0 and ptr is non NULL, simply free the given ptr. + * This function emulates the realloc(3) standard library + * function: + * - if size > 0, allocate new memory; and if ptr is non NULL, + * copy as much of the old contents to the new buffer and free + * the old one. Note that when allocation fails the original + * pointer is intact; the caller must free it. + * - if size is 0 and ptr is non NULL, simply free the given + * ptr. * - this function returns: * pointer to the newly allocated memory, or * NULL if allocation fails or doesn't happen. @@ -963,7 +974,8 @@ isc__mem_reallocate(isc_mem_t *ctx, void *ptr, size_t size FLARG) { if (size > 0U) { new_ptr = isc__mem_allocate(ctx, size FLARG_PASS); if (new_ptr != NULL && ptr != NULL) { - oldsize = (((size_info *)ptr)[-1]).u.size; + size_t oldsize, copysize; + oldsize = (((size_info *)ptr)[-1]).size; INSIST(oldsize >= ALIGNMENT_SIZE); oldsize -= ALIGNMENT_SIZE; if (ISC_UNLIKELY((isc_mem_debugging & @@ -993,41 +1005,21 @@ isc__mem_free(isc_mem_t *ctx, void *ptr FLARG) { if (ISC_UNLIKELY((isc_mem_debugging & ISC_MEM_DEBUGCTX) != 0)) { si = &(((size_info *)ptr)[-2]); - REQUIRE(si->u.ctx == ctx); - size = si[1].u.size; + REQUIRE(si->ctx == ctx); + size = si[1].size; } else { si = &(((size_info *)ptr)[-1]); - size = si->u.size; + size = si->size; } - MCTXLOCK(ctx); - DELETE_TRACE(ctx, ptr, size, file, line); mem_putstats(ctx, si, size); mem_put(ctx, si, size); - /* - * The check against ctx->lo_water == 0 is for the condition - * when the context was pushed over hi_water but then had - * isc_mem_setwater() called with 0 for hi_water and lo_water. - */ - if (ctx->is_overmem && - (ctx->inuse < ctx->lo_water || ctx->lo_water == 0U)) { - ctx->is_overmem = false; - } + call_water = lo_water(ctx); - if (ctx->hi_called && - (ctx->inuse < ctx->lo_water || ctx->lo_water == 0U)) { - ctx->hi_called = false; - - if (ctx->water != NULL) { - call_water = true; - } - } - MCTXUNLOCK(ctx); - - if (call_water) { + if (call_water && (ctx->water != NULL)) { (ctx->water)(ctx->water_arg, ISC_MEM_LOWATER); } } @@ -1092,45 +1084,35 @@ size_t isc_mem_inuse(isc_mem_t *ctx) { REQUIRE(VALID_CONTEXT(ctx)); - size_t inuse; - - MCTXLOCK(ctx); - - inuse = ctx->inuse; - - MCTXUNLOCK(ctx); - - return (inuse); + return (atomic_load_acquire(&ctx->inuse)); } size_t isc_mem_maxinuse(isc_mem_t *ctx) { REQUIRE(VALID_CONTEXT(ctx)); - size_t maxinuse; - - MCTXLOCK(ctx); - - maxinuse = ctx->maxinuse; - - MCTXUNLOCK(ctx); - - return (maxinuse); + return (atomic_load_acquire(&ctx->maxinuse)); } size_t isc_mem_total(isc_mem_t *ctx) { REQUIRE(VALID_CONTEXT(ctx)); - size_t total; + return (atomic_load_acquire(&ctx->total)); +} - MCTXLOCK(ctx); +size_t +isc_mem_malloced(isc_mem_t *ctx) { + REQUIRE(VALID_CONTEXT(ctx)); - total = ctx->total; + return (atomic_load_acquire(&ctx->malloced)); +} - MCTXUNLOCK(ctx); +size_t +isc_mem_maxmalloced(isc_mem_t *ctx) { + REQUIRE(VALID_CONTEXT(ctx)); - return (total); + return (atomic_load_acquire(&ctx->maxmalloced)); } void @@ -1147,22 +1129,23 @@ isc_mem_setwater(isc_mem_t *ctx, isc_mem_water_t water, void *water_arg, oldwater = ctx->water; oldwater_arg = ctx->water_arg; if (water == NULL) { - callwater = ctx->hi_called; + callwater = atomic_load(&ctx->hi_called); ctx->water = NULL; ctx->water_arg = NULL; - ctx->hi_water = 0; - ctx->lo_water = 0; + atomic_store_release(&ctx->hi_water, 0); + atomic_store_release(&ctx->lo_water, 0); } else { - if (ctx->hi_called && + if (atomic_load_acquire(&ctx->hi_called) && (ctx->water != water || ctx->water_arg != water_arg || - ctx->inuse < lowater || lowater == 0U)) + atomic_load_acquire(&ctx->inuse) < lowater || + lowater == 0U)) { callwater = true; } ctx->water = water; ctx->water_arg = water_arg; - ctx->hi_water = hiwater; - ctx->lo_water = lowater; + atomic_store_release(&ctx->hi_water, hiwater); + atomic_store_release(&ctx->lo_water, lowater); } MCTXUNLOCK(ctx); @@ -1171,16 +1154,11 @@ isc_mem_setwater(isc_mem_t *ctx, isc_mem_water_t water, void *water_arg, } } -ISC_NO_SANITIZE_THREAD bool +bool isc_mem_isovermem(isc_mem_t *ctx) { REQUIRE(VALID_CONTEXT(ctx)); - /* - * We don't bother to lock the context because 100% accuracy isn't - * necessary (and even if we locked the context the returned value - * could be different from the actual state when it's used anyway) - */ - return (ctx->is_overmem); + return (atomic_load_relaxed(&ctx->is_overmem)); } void @@ -1215,32 +1193,30 @@ isc_mempool_create(isc_mem_t *mctx, size_t size, isc_mempool_t **mpctxp) { isc_mempool_t *mpctx; - /* - * Allocate space for this pool, initialize values, and if all works - * well, attach to the memory context. - */ - mpctx = isc_mem_get(mctx, sizeof(isc_mempool_t)); - - mpctx->magic = MEMPOOL_MAGIC; - mpctx->lock = NULL; - mpctx->mctx = mctx; /* * Mempools are stored as a linked list of element. */ if (size < sizeof(element)) { size = sizeof(element); } - mpctx->size = size; - mpctx->maxalloc = UINT_MAX; - mpctx->allocated = 0; - mpctx->freecount = 0; - mpctx->freemax = 1; - mpctx->fillcount = 1; - mpctx->gets = 0; -#if ISC_MEMPOOL_NAMES - mpctx->name[0] = 0; -#endif /* if ISC_MEMPOOL_NAMES */ - mpctx->items = NULL; + + /* + * Allocate space for this pool, initialize values, and if all + * works well, attach to the memory context. + */ + mpctx = isc_mem_get(mctx, sizeof(isc_mempool_t)); + + *mpctx = (isc_mempool_t){ + .magic = MEMPOOL_MAGIC, + .mctx = mctx, + .size = size, + }; + + atomic_init(&mpctx->maxalloc, SIZE_MAX); + atomic_init(&mpctx->allocated, 0); + atomic_init(&mpctx->freecount, 0); + atomic_init(&mpctx->freemax, 1); + atomic_init(&mpctx->fillcount, 1); *mpctxp = (isc_mempool_t *)mpctx; @@ -1256,15 +1232,12 @@ isc_mempool_setname(isc_mempool_t *mpctx, const char *name) { REQUIRE(name != NULL); #if ISC_MEMPOOL_NAMES - if (mpctx->lock != NULL) { - LOCK(mpctx->lock); - } + MPCTXLOCK(mpctx); strlcpy(mpctx->name, name, sizeof(mpctx->name)); - if (mpctx->lock != NULL) { - UNLOCK(mpctx->lock); - } + MPCTXUNLOCK(mpctx); + #else /* if ISC_MEMPOOL_NAMES */ UNUSED(mpctx); UNUSED(name); @@ -1284,14 +1257,14 @@ isc_mempool_destroy(isc_mempool_t **mpctxp) { mpctx = *mpctxp; *mpctxp = NULL; #if ISC_MEMPOOL_NAMES - if (mpctx->allocated > 0) { + if (atomic_load_acquire(&mpctx->allocated) > 0) { UNEXPECTED_ERROR(__FILE__, __LINE__, "isc_mempool_destroy(): mempool %s " "leaked memory", mpctx->name); } #endif /* if ISC_MEMPOOL_NAMES */ - REQUIRE(mpctx->allocated == 0); + REQUIRE(atomic_load_acquire(&mpctx->allocated) == 0); mctx = mpctx->mctx; @@ -1304,17 +1277,15 @@ isc_mempool_destroy(isc_mempool_t **mpctxp) { /* * Return any items on the free list */ - MCTXLOCK(mctx); while (mpctx->items != NULL) { - INSIST(mpctx->freecount > 0); - mpctx->freecount--; + INSIST(atomic_fetch_sub_release(&mpctx->freecount, 1) > 0); + item = mpctx->items; mpctx->items = item->next; mem_putstats(mctx, item, mpctx->size); mem_put(mctx, item, mpctx->size); } - MCTXUNLOCK(mctx); /* * Remove our linked list entry from the memory context. @@ -1348,37 +1319,33 @@ isc__mempool_get(isc_mempool_t *mpctx FLARG) { REQUIRE(VALID_MEMPOOL(mpctx)); element *item; - isc_mem_t *mctx; unsigned int i; - - mctx = mpctx->mctx; - - if (mpctx->lock != NULL) { - LOCK(mpctx->lock); - } + size_t allocated = atomic_load_acquire(&mpctx->allocated); + size_t maxalloc = atomic_load_acquire(&mpctx->maxalloc); /* * Don't let the caller go over quota */ - if (ISC_UNLIKELY(mpctx->allocated >= mpctx->maxalloc)) { + if (ISC_UNLIKELY(allocated >= maxalloc)) { item = NULL; goto out; } + MPCTXLOCK(mpctx); if (ISC_UNLIKELY(mpctx->items == NULL)) { + isc_mem_t *mctx = mpctx->mctx; + size_t fillcount = atomic_load_acquire(&mpctx->fillcount); /* - * We need to dip into the well. Lock the memory context - * here and fill up our free list. + * We need to dip into the well. Lock the memory + * context here and fill up our free list. */ - MCTXLOCK(mctx); - for (i = 0; i < mpctx->fillcount; i++) { + for (i = 0; i < fillcount; i++) { item = mem_get(mctx, mpctx->size); mem_getstats(mctx, mpctx->size); item->next = mpctx->items; mpctx->items = item; - mpctx->freecount++; + atomic_fetch_add_relaxed(&mpctx->freecount, 1); } - MCTXUNLOCK(mctx); } /* @@ -1390,24 +1357,15 @@ isc__mempool_get(isc_mempool_t *mpctx FLARG) { } mpctx->items = item->next; - INSIST(mpctx->freecount > 0); - mpctx->freecount--; - mpctx->gets++; - mpctx->allocated++; + + INSIST(atomic_fetch_sub_release(&mpctx->freecount, 1) > 0); + atomic_fetch_add_relaxed(&mpctx->gets, 1); + atomic_fetch_add_relaxed(&mpctx->allocated, 1); out: - if (mpctx->lock != NULL) { - UNLOCK(mpctx->lock); - } + MPCTXUNLOCK(mpctx); -#if ISC_MEM_TRACKLINES - if (ISC_UNLIKELY(((isc_mem_debugging & TRACE_OR_RECORD) != 0) && - item != NULL)) { - MCTXLOCK(mctx); - ADD_TRACE(mctx, item, mpctx->size, file, line); - MCTXUNLOCK(mctx); - } -#endif /* ISC_MEM_TRACKLINES */ + ADD_TRACE(mpctx->mctx, item, mpctx->size, file, line); return (item); } @@ -1420,47 +1378,33 @@ isc__mempool_put(isc_mempool_t *mpctx, void *mem FLARG) { isc_mem_t *mctx = mpctx->mctx; element *item; + size_t freecount = atomic_load_acquire(&mpctx->freecount); + size_t freemax = atomic_load_acquire(&mpctx->freemax); - if (mpctx->lock != NULL) { - LOCK(mpctx->lock); - } + INSIST(atomic_fetch_sub_release(&mpctx->allocated, 1) > 0); - INSIST(mpctx->allocated > 0); - mpctx->allocated--; - -#if ISC_MEM_TRACKLINES - if (ISC_UNLIKELY((isc_mem_debugging & TRACE_OR_RECORD) != 0)) { - MCTXLOCK(mctx); - DELETE_TRACE(mctx, mem, mpctx->size, file, line); - MCTXUNLOCK(mctx); - } -#endif /* ISC_MEM_TRACKLINES */ + DELETE_TRACE(mctx, mem, mpctx->size, file, line); /* * If our free list is full, return this to the mctx directly. */ - if (mpctx->freecount >= mpctx->freemax) { - MCTXLOCK(mctx); + if (freecount >= freemax) { mem_putstats(mctx, mem, mpctx->size); mem_put(mctx, mem, mpctx->size); - MCTXUNLOCK(mctx); - if (mpctx->lock != NULL) { - UNLOCK(mpctx->lock); - } return; } /* * Otherwise, attach it to our free list and bump the counter. */ - mpctx->freecount++; + MPCTXLOCK(mpctx); + item = (element *)mem; item->next = mpctx->items; mpctx->items = item; + atomic_fetch_add_relaxed(&mpctx->freecount, 1); - if (mpctx->lock != NULL) { - UNLOCK(mpctx->lock); - } + MPCTXUNLOCK(mpctx); } /* @@ -1471,15 +1415,11 @@ void isc_mempool_setfreemax(isc_mempool_t *mpctx, unsigned int limit) { REQUIRE(VALID_MEMPOOL(mpctx)); - if (mpctx->lock != NULL) { - LOCK(mpctx->lock); - } + MPCTXLOCK(mpctx); mpctx->freemax = limit; - if (mpctx->lock != NULL) { - UNLOCK(mpctx->lock); - } + MPCTXUNLOCK(mpctx); } unsigned int @@ -1488,15 +1428,11 @@ isc_mempool_getfreemax(isc_mempool_t *mpctx) { unsigned int freemax; - if (mpctx->lock != NULL) { - LOCK(mpctx->lock); - } + MPCTXLOCK(mpctx); freemax = mpctx->freemax; - if (mpctx->lock != NULL) { - UNLOCK(mpctx->lock); - } + MPCTXUNLOCK(mpctx); return (freemax); } @@ -1505,19 +1441,7 @@ unsigned int isc_mempool_getfreecount(isc_mempool_t *mpctx) { REQUIRE(VALID_MEMPOOL(mpctx)); - unsigned int freecount; - - if (mpctx->lock != NULL) { - LOCK(mpctx->lock); - } - - freecount = mpctx->freecount; - - if (mpctx->lock != NULL) { - UNLOCK(mpctx->lock); - } - - return (freecount); + return (atomic_load_relaxed(&mpctx->freecount)); } void @@ -1525,53 +1449,21 @@ isc_mempool_setmaxalloc(isc_mempool_t *mpctx, unsigned int limit) { REQUIRE(VALID_MEMPOOL(mpctx)); REQUIRE(limit > 0); - if (mpctx->lock != NULL) { - LOCK(mpctx->lock); - } - - mpctx->maxalloc = limit; - - if (mpctx->lock != NULL) { - UNLOCK(mpctx->lock); - } + atomic_store_release(&mpctx->maxalloc, limit); } unsigned int isc_mempool_getmaxalloc(isc_mempool_t *mpctx) { REQUIRE(VALID_MEMPOOL(mpctx)); - unsigned int maxalloc; - - if (mpctx->lock != NULL) { - LOCK(mpctx->lock); - } - - maxalloc = mpctx->maxalloc; - - if (mpctx->lock != NULL) { - UNLOCK(mpctx->lock); - } - - return (maxalloc); + return (atomic_load_relaxed(&mpctx->maxalloc)); } unsigned int isc_mempool_getallocated(isc_mempool_t *mpctx) { REQUIRE(VALID_MEMPOOL(mpctx)); - unsigned int allocated; - - if (mpctx->lock != NULL) { - LOCK(mpctx->lock); - } - - allocated = mpctx->allocated; - - if (mpctx->lock != NULL) { - UNLOCK(mpctx->lock); - } - - return (allocated); + return (atomic_load_relaxed(&mpctx->allocated)); } void @@ -1579,34 +1471,14 @@ isc_mempool_setfillcount(isc_mempool_t *mpctx, unsigned int limit) { REQUIRE(VALID_MEMPOOL(mpctx)); REQUIRE(limit > 0); - if (mpctx->lock != NULL) { - LOCK(mpctx->lock); - } - - mpctx->fillcount = limit; - - if (mpctx->lock != NULL) { - UNLOCK(mpctx->lock); - } + atomic_store_release(&mpctx->fillcount, limit); } unsigned int isc_mempool_getfillcount(isc_mempool_t *mpctx) { REQUIRE(VALID_MEMPOOL(mpctx)); - unsigned int fillcount; - - if (mpctx->lock != NULL) { - LOCK(mpctx->lock); - } - - fillcount = mpctx->fillcount; - - if (mpctx->lock != NULL) { - UNLOCK(mpctx->lock); - } - - return (fillcount); + return (atomic_load_relaxed(&mpctx->fillcount)); } /* @@ -1686,9 +1558,7 @@ xml_renderctx(isc_mem_t *ctx, summarystat_t *summary, xmlTextWriterPtr writer) { TRY0(xmlTextWriterEndElement(writer)); /* name */ } - summary->contextsize += sizeof(*ctx) + - (ctx->max_size + 1) * sizeof(struct stats) + - ctx->max_size * sizeof(element *); + summary->contextsize += sizeof(*ctx); #if ISC_MEM_TRACKLINES if (ctx->debuglist != NULL) { summary->contextsize += DEBUG_TABLE_COUNT * @@ -1702,32 +1572,32 @@ xml_renderctx(isc_mem_t *ctx, summarystat_t *summary, xmlTextWriterPtr writer) { isc_refcount_current(&ctx->references))); TRY0(xmlTextWriterEndElement(writer)); /* references */ - summary->total += ctx->total; + summary->total += isc_mem_total(ctx); TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "total")); TRY0(xmlTextWriterWriteFormatString(writer, "%" PRIu64 "", - (uint64_t)ctx->total)); + (uint64_t)isc_mem_total(ctx))); TRY0(xmlTextWriterEndElement(writer)); /* total */ - summary->inuse += ctx->inuse; + summary->inuse += isc_mem_inuse(ctx); TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "inuse")); TRY0(xmlTextWriterWriteFormatString(writer, "%" PRIu64 "", - (uint64_t)ctx->inuse)); + (uint64_t)isc_mem_inuse(ctx))); TRY0(xmlTextWriterEndElement(writer)); /* inuse */ TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "maxinuse")); TRY0(xmlTextWriterWriteFormatString(writer, "%" PRIu64 "", - (uint64_t)ctx->maxinuse)); + (uint64_t)isc_mem_maxinuse(ctx))); TRY0(xmlTextWriterEndElement(writer)); /* maxinuse */ - summary->malloced += ctx->malloced; + summary->malloced += isc_mem_malloced(ctx); TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "malloced")); TRY0(xmlTextWriterWriteFormatString(writer, "%" PRIu64 "", - (uint64_t)ctx->malloced)); + (uint64_t)isc_mem_malloced(ctx))); TRY0(xmlTextWriterEndElement(writer)); /* malloced */ TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "maxmalloced")); - TRY0(xmlTextWriterWriteFormatString(writer, "%" PRIu64 "", - (uint64_t)ctx->maxmalloced)); + TRY0(xmlTextWriterWriteFormatString( + writer, "%" PRIu64 "", (uint64_t)isc_mem_maxmalloced(ctx))); TRY0(xmlTextWriterEndElement(writer)); /* maxmalloced */ TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "pools")); @@ -1736,13 +1606,15 @@ xml_renderctx(isc_mem_t *ctx, summarystat_t *summary, xmlTextWriterPtr writer) { summary->contextsize += ctx->poolcnt * sizeof(isc_mempool_t); TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "hiwater")); - TRY0(xmlTextWriterWriteFormatString(writer, "%" PRIu64 "", - (uint64_t)ctx->hi_water)); + TRY0(xmlTextWriterWriteFormatString( + writer, "%" PRIu64 "", + (uint64_t)atomic_load_relaxed(&ctx->hi_water))); TRY0(xmlTextWriterEndElement(writer)); /* hiwater */ TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "lowater")); - TRY0(xmlTextWriterWriteFormatString(writer, "%" PRIu64 "", - (uint64_t)ctx->lo_water)); + TRY0(xmlTextWriterWriteFormatString( + writer, "%" PRIu64 "", + (uint64_t)atomic_load_relaxed(&ctx->lo_water))); TRY0(xmlTextWriterEndElement(writer)); /* lowater */ TRY0(xmlTextWriterEndElement(writer)); /* context */ @@ -1828,12 +1700,10 @@ json_renderctx(isc_mem_t *ctx, summarystat_t *summary, json_object *array) { MCTXLOCK(ctx); - summary->contextsize += sizeof(*ctx) + - (ctx->max_size + 1) * sizeof(struct stats) + - ctx->max_size * sizeof(element *); - summary->total += ctx->total; - summary->inuse += ctx->inuse; - summary->malloced += ctx->malloced; + summary->contextsize += sizeof(*ctx); + summary->total += isc_mem_total(ctx); + summary->inuse += isc_mem_inuse(ctx); + summary->malloced += isc_mem_malloced(ctx); #if ISC_MEM_TRACKLINES if (ctx->debuglist != NULL) { summary->contextsize += DEBUG_TABLE_COUNT * @@ -1860,23 +1730,23 @@ json_renderctx(isc_mem_t *ctx, summarystat_t *summary, json_object *array) { CHECKMEM(obj); json_object_object_add(ctxobj, "references", obj); - obj = json_object_new_int64(ctx->total); + obj = json_object_new_int64(isc_mem_total(ctx)); CHECKMEM(obj); json_object_object_add(ctxobj, "total", obj); - obj = json_object_new_int64(ctx->inuse); + obj = json_object_new_int64(isc_mem_inuse(ctx)); CHECKMEM(obj); json_object_object_add(ctxobj, "inuse", obj); - obj = json_object_new_int64(ctx->maxinuse); + obj = json_object_new_int64(isc_mem_maxinuse(ctx)); CHECKMEM(obj); json_object_object_add(ctxobj, "maxinuse", obj); - obj = json_object_new_int64(ctx->malloced); + obj = json_object_new_int64(isc_mem_malloced(ctx)); CHECKMEM(obj); json_object_object_add(ctxobj, "malloced", obj); - obj = json_object_new_int64(ctx->maxmalloced); + obj = json_object_new_int64(isc_mem_maxmalloced(ctx)); CHECKMEM(obj); json_object_object_add(ctxobj, "maxmalloced", obj); @@ -1886,11 +1756,11 @@ json_renderctx(isc_mem_t *ctx, summarystat_t *summary, json_object *array) { summary->contextsize += ctx->poolcnt * sizeof(isc_mempool_t); - obj = json_object_new_int64(ctx->hi_water); + obj = json_object_new_int64(atomic_load_relaxed(&ctx->hi_water)); CHECKMEM(obj); json_object_object_add(ctxobj, "hiwater", obj); - obj = json_object_new_int64(ctx->lo_water); + obj = json_object_new_int64(atomic_load_relaxed(&ctx->lo_water)); CHECKMEM(obj); json_object_object_add(ctxobj, "lowater", obj); diff --git a/lib/isc/win32/libisc.def.in b/lib/isc/win32/libisc.def.in index 7bc4926a96..b02fb22bbb 100644 --- a/lib/isc/win32/libisc.def.in +++ b/lib/isc/win32/libisc.def.in @@ -370,7 +370,9 @@ isc_mem_detach isc_mem_getname isc_mem_inuse isc_mem_isovermem +isc_mem_malloced isc_mem_maxinuse +isc_mem_maxmalloced isc_mem_references @IF NOTYET isc_mem_renderjson