diff --git a/lib/isc/include/isc/mem.h b/lib/isc/include/isc/mem.h index 01dac4c2e0..2d4f70dedd 100644 --- a/lib/isc/include/isc/mem.h +++ b/lib/isc/include/isc/mem.h @@ -250,6 +250,19 @@ isc_mem_total(isc_mem_t *mctx); * not yet used. */ +size_t +isc_mem_malloced(isc_mem_t *ctx); +/*%< + * Get an estimate of the amount of memory allocated in 'mctx', in bytes. + */ + +size_t +isc_mem_maxmalloced(isc_mem_t *ctx); +/*%< + * Get an estimate of the largest amount of memory that has been + * allocated in 'mctx' at any time. + */ + bool isc_mem_isovermem(isc_mem_t *mctx); /*%< @@ -265,13 +278,13 @@ isc_mem_setwater(isc_mem_t *mctx, isc_mem_water_t water, void *water_arg, * Set high and low water marks for this memory context. * * When the memory usage of 'mctx' exceeds 'hiwater', - * '(water)(water_arg, #ISC_MEM_HIWATER)' will be called. 'water' needs to - * call isc_mem_waterack() with #ISC_MEM_HIWATER to acknowledge the state - * change. 'water' may be called multiple times. + * '(water)(water_arg, #ISC_MEM_HIWATER)' will be called. 'water' needs + *to call isc_mem_waterack() with #ISC_MEM_HIWATER to acknowledge the + *state change. 'water' may be called multiple times. * - * When the usage drops below 'lowater', 'water' will again be called, this - * time with #ISC_MEM_LOWATER. 'water' need to calls isc_mem_waterack() with - * #ISC_MEM_LOWATER to acknowledge the change. + * When the usage drops below 'lowater', 'water' will again be called, + *this time with #ISC_MEM_LOWATER. 'water' need to calls + *isc_mem_waterack() with #ISC_MEM_LOWATER to acknowledge the change. * * static void * water(void *arg, int mark) { @@ -396,7 +409,8 @@ isc_mempool_destroy(isc_mempool_t **mpctxp); void isc_mempool_setname(isc_mempool_t *mpctx, const char *name); /*%< - * Associate a name with a memory pool. At most 15 characters may be used. + * Associate a name with a memory pool. At most 15 characters may be + *used. * * Requires: *\li mpctx is a valid pool. @@ -408,15 +422,15 @@ isc_mempool_associatelock(isc_mempool_t *mpctx, isc_mutex_t *lock); /*%< * Associate a lock with this memory pool. * - * This lock is used when getting or putting items using this memory pool, - * and it is also used to set or get internal state via the isc_mempool_get*() - * and isc_mempool_set*() set of functions. + * This lock is used when getting or putting items using this memory + *pool, and it is also used to set or get internal state via the + *isc_mempool_get*() and isc_mempool_set*() set of functions. * - * Multiple pools can each share a single lock. For instance, if "manager" - * type object contained pools for various sizes of events, and each of - * these pools used a common lock. Note that this lock must NEVER be used - * by other than mempool routines once it is given to a pool, since that can - * easily cause double locking. + * Multiple pools can each share a single lock. For instance, if + *"manager" type object contained pools for various sizes of events, and + *each of these pools used a common lock. Note that this lock must + *NEVER be used by other than mempool routines once it is given to a + *pool, since that can easily cause double locking. * * Requires: * @@ -426,19 +440,19 @@ isc_mempool_associatelock(isc_mempool_t *mpctx, isc_mutex_t *lock); * *\li No previous lock is assigned to this pool. * - *\li The lock is initialized before calling this function via the normal - * means of doing that. + *\li The lock is initialized before calling this function via the + *normal means of doing that. */ /* * The following functions get/set various parameters. Note that due to - * the unlocked nature of pools these are potentially random values unless - * the imposed externally provided locking protocols are followed. + * the unlocked nature of pools these are potentially random values + *unless the imposed externally provided locking protocols are followed. * - * Also note that the quota limits will not always take immediate effect. - * For instance, setting "maxalloc" to a number smaller than the currently - * allocated count is permitted. New allocations will be refused until - * the count drops below this threshold. + * Also note that the quota limits will not always take immediate + *effect. For instance, setting "maxalloc" to a number smaller than the + *currently allocated count is permitted. New allocations will be + *refused until the count drops below this threshold. * * All functions require (in addition to other requirements): * mpctx is a valid memory pool @@ -486,8 +500,8 @@ isc_mempool_getallocated(isc_mempool_t *mpctx); unsigned int isc_mempool_getfillcount(isc_mempool_t *mpctx); /*%< - * Returns the number of items allocated as a block from the parent memory - * context when the free list is empty. + * Returns the number of items allocated as a block from the parent + * memory context when the free list is empty. */ void diff --git a/lib/isc/mem.c b/lib/isc/mem.c index 7a0c650247..c3c4ac463a 100644 --- a/lib/isc/mem.c +++ b/lib/isc/mem.c @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -45,6 +46,14 @@ #define MCTXLOCK(m) LOCK(&m->lock) #define MCTXUNLOCK(m) UNLOCK(&m->lock) +#define MPCTXLOCK(mp) \ + if (mp->lock != NULL) { \ + LOCK(mp->lock); \ + } +#define MPCTXUNLOCK(mp) \ + if (mp->lock != NULL) { \ + UNLOCK(mp->lock); \ + } #ifndef ISC_MEM_DEBUGGING #define ISC_MEM_DEBUGGING 0 @@ -56,9 +65,11 @@ LIBISC_EXTERNAL_DATA unsigned int isc_mem_defaultflags = ISC_MEMFLAG_DEFAULT; * Constants. */ -#define DEF_MAX_SIZE 1100 -#define ALIGNMENT_SIZE 8U /*%< must be a power of 2 */ +#define ALIGNMENT 8U /*%< must be a power of 2 */ +#define ALIGNMENT_SIZE sizeof(size_info) #define DEBUG_TABLE_COUNT 512U +#define STATS_BUCKETS 512U +#define STATS_BUCKET_SIZE 32U /* * Types. @@ -88,19 +99,15 @@ struct element { }; typedef struct { - /*! - * This structure must be ALIGNMENT_SIZE bytes. - */ - union { + alignas(ALIGNMENT) union { size_t size; isc_mem_t *ctx; - char bytes[ALIGNMENT_SIZE]; - } u; + }; } size_info; struct stats { - unsigned long gets; - unsigned long totalgets; + atomic_size_t gets; + atomic_size_t totalgets; }; #define MEM_MAGIC ISC_MAGIC('M', 'e', 'm', 'C') @@ -123,20 +130,19 @@ struct isc_mem { unsigned int magic; unsigned int flags; isc_mutex_t lock; - size_t max_size; bool checkfree; - struct stats *stats; + struct stats stats[STATS_BUCKETS + 1]; isc_refcount_t references; char name[16]; - size_t total; - size_t inuse; - size_t maxinuse; - size_t malloced; - size_t maxmalloced; - size_t hi_water; - size_t lo_water; - bool hi_called; - bool is_overmem; + atomic_size_t total; + atomic_size_t inuse; + atomic_size_t maxinuse; + atomic_size_t malloced; + atomic_size_t maxmalloced; + atomic_size_t hi_water; + atomic_size_t lo_water; + atomic_bool hi_called; + atomic_bool is_overmem; isc_mem_water_t water; void *water_arg; ISC_LIST(isc_mempool_t) pools; @@ -161,16 +167,16 @@ struct isc_mempool { /*%< locked via the memory context's lock */ ISC_LINK(isc_mempool_t) link; /*%< next pool in this mem context */ /*%< optionally locked from here down */ - element *items; /*%< low water item list */ - size_t size; /*%< size of each item on this pool */ - unsigned int maxalloc; /*%< max number of items allowed */ - unsigned int allocated; /*%< # of items currently given out */ - unsigned int freecount; /*%< # of items on reserved list */ - unsigned int freemax; /*%< # of items allowed on free list */ - unsigned int fillcount; /*%< # of items to fetch on each fill */ + element *items; /*%< low water item list */ + size_t size; /*%< size of each item on this pool */ + atomic_size_t maxalloc; /*%< max number of items allowed */ + atomic_size_t allocated; /*%< # of items currently given out */ + atomic_size_t freecount; /*%< # of items on reserved list */ + atomic_size_t freemax; /*%< # of items allowed on free list */ + atomic_size_t fillcount; /*%< # of items to fetch on each fill */ /*%< Stats only. */ - unsigned int gets; /*%< # of requests to this pool */ - /*%< Debugging only. */ + atomic_size_t gets; /*%< # of requests to this pool */ + /*%< Debugging only. */ #if ISC_MEMPOOL_NAMES char name[16]; /*%< printed name in stats reports */ #endif /* if ISC_MEMPOOL_NAMES */ @@ -186,27 +192,48 @@ struct isc_mempool { #define ISC_MEMFUNC_SCOPE #else /* if !ISC_MEM_TRACKLINES */ #define TRACE_OR_RECORD (ISC_MEM_DEBUGTRACE | ISC_MEM_DEBUGRECORD) -#define ADD_TRACE(a, b, c, d, e) \ - do { \ - if (ISC_UNLIKELY((isc_mem_debugging & TRACE_OR_RECORD) != 0 && \ - b != NULL)) \ - add_trace_entry(a, b, c, d, e); \ - } while (0) -#define DELETE_TRACE(a, b, c, d, e) \ - do { \ - if (ISC_UNLIKELY((isc_mem_debugging & TRACE_OR_RECORD) != 0 && \ - b != NULL)) \ - delete_trace_entry(a, b, c, d, e); \ - } while (0) + +#define SHOULD_TRACE_OR_RECORD(ptr) \ + (ISC_UNLIKELY((isc_mem_debugging & TRACE_OR_RECORD) != 0) && \ + ptr != NULL) + +#define ADD_TRACE(a, b, c, d, e) \ + if (SHOULD_TRACE_OR_RECORD(b)) { \ + add_trace_entry(a, b, c, d, e); \ + } + +#define DELETE_TRACE(a, b, c, d, e) \ + if (SHOULD_TRACE_OR_RECORD(b)) { \ + delete_trace_entry(a, b, c, d, e); \ + } static void print_active(isc_mem_t *ctx, FILE *out); - #endif /* ISC_MEM_TRACKLINES */ +static inline size_t +increment_malloced(isc_mem_t *ctx, size_t size) { + size_t malloced = atomic_fetch_add_relaxed(&ctx->malloced, size) + size; + size_t maxmalloced = atomic_load_acquire(&ctx->maxmalloced); + if (malloced > maxmalloced) { + atomic_compare_exchange_strong(&ctx->maxmalloced, &maxmalloced, + malloced); + } + + return (malloced); +} + +static inline size_t +decrement_malloced(isc_mem_t *ctx, size_t size) { + size_t malloced = atomic_fetch_sub_release(&ctx->malloced, size) - size; + INSIST(size >= 0); + + return (malloced); +} + #if ISC_MEM_TRACKLINES /*! - * mctx must be locked. + * mctx must not be locked. */ static void add_trace_entry(isc_mem_t *mctx, const void *ptr, size_t size FLARG) { @@ -214,13 +241,15 @@ add_trace_entry(isc_mem_t *mctx, const void *ptr, size_t size FLARG) { uint32_t hash; uint32_t idx; + MCTXLOCK(mctx); + if ((isc_mem_debugging & ISC_MEM_DEBUGTRACE) != 0) { fprintf(stderr, "add %p size %zu file %s line %u mctx %p\n", ptr, size, file, line, mctx); } if (mctx->debuglist == NULL) { - return; + goto unlock; } #ifdef __COVERITY__ @@ -236,10 +265,7 @@ add_trace_entry(isc_mem_t *mctx, const void *ptr, size_t size FLARG) { dl = malloc(sizeof(debuglink_t)); INSIST(dl != NULL); - mctx->malloced += sizeof(debuglink_t); - if (mctx->malloced > mctx->maxmalloced) { - mctx->maxmalloced = mctx->malloced; - } + increment_malloced(mctx, sizeof(debuglink_t)); ISC_LINK_INIT(dl, link); dl->ptr = ptr; @@ -249,6 +275,8 @@ add_trace_entry(isc_mem_t *mctx, const void *ptr, size_t size FLARG) { ISC_LIST_PREPEND(mctx->debuglist[idx], dl, link); mctx->debuglistcnt++; +unlock: + MCTXUNLOCK(mctx); } static void @@ -258,13 +286,15 @@ delete_trace_entry(isc_mem_t *mctx, const void *ptr, size_t size, uint32_t hash; uint32_t idx; + MCTXLOCK(mctx); + if ((isc_mem_debugging & ISC_MEM_DEBUGTRACE) != 0) { fprintf(stderr, "del %p size %zu file %s line %u mctx %p\n", ptr, size, file, line, mctx); } if (mctx->debuglist == NULL) { - return; + goto unlock; } #ifdef __COVERITY__ @@ -282,9 +312,9 @@ delete_trace_entry(isc_mem_t *mctx, const void *ptr, size_t size, while (ISC_LIKELY(dl != NULL)) { if (ISC_UNLIKELY(dl->ptr == ptr)) { ISC_LIST_UNLINK(mctx->debuglist[idx], dl, link); - mctx->malloced -= sizeof(*dl); + decrement_malloced(mctx, sizeof(*dl)); free(dl); - return; + goto unlock; } dl = ISC_LIST_NEXT(dl, link); } @@ -295,6 +325,8 @@ delete_trace_entry(isc_mem_t *mctx, const void *ptr, size_t size, */ INSIST(0); ISC_UNREACHABLE(); +unlock: + MCTXUNLOCK(mctx); } #endif /* ISC_MEM_TRACKLINES */ @@ -348,29 +380,25 @@ mem_put(isc_mem_t *ctx, void *mem, size_t size) { default_memfree(mem); } +#define stats_bucket(ctx, size) \ + ((size / STATS_BUCKET_SIZE) >= STATS_BUCKETS \ + ? &ctx->stats[STATS_BUCKETS] \ + : &ctx->stats[size / STATS_BUCKET_SIZE]) + /*! * Update internal counters after a memory get. */ static inline void mem_getstats(isc_mem_t *ctx, size_t size) { - ctx->total += size; - ctx->inuse += size; + struct stats *stats = stats_bucket(ctx, size); - if (size > ctx->max_size) { - ctx->stats[ctx->max_size].gets++; - ctx->stats[ctx->max_size].totalgets++; - } else { - ctx->stats[size].gets++; - ctx->stats[size].totalgets++; - } + atomic_fetch_add_relaxed(&ctx->total, size); + atomic_fetch_add_release(&ctx->inuse, size); -#if ISC_MEM_CHECKOVERRUN - size += 1; -#endif /* if ISC_MEM_CHECKOVERRUN */ - ctx->malloced += size; - if (ctx->malloced > ctx->maxmalloced) { - ctx->maxmalloced = ctx->malloced; - } + atomic_fetch_add_relaxed(&stats->gets, 1); + atomic_fetch_add_relaxed(&stats->totalgets, 1); + + increment_malloced(ctx, size); } /*! @@ -378,22 +406,15 @@ mem_getstats(isc_mem_t *ctx, size_t size) { */ static inline void mem_putstats(isc_mem_t *ctx, void *ptr, size_t size) { + struct stats *stats = stats_bucket(ctx, size); + UNUSED(ptr); - INSIST(ctx->inuse >= size); - ctx->inuse -= size; + INSIST(atomic_fetch_sub_release(&ctx->inuse, size) >= size); - if (size > ctx->max_size) { - INSIST(ctx->stats[ctx->max_size].gets > 0U); - ctx->stats[ctx->max_size].gets--; - } else { - INSIST(ctx->stats[size].gets > 0U); - ctx->stats[size].gets--; - } -#if ISC_MEM_CHECKOVERRUN - size += 1; -#endif /* if ISC_MEM_CHECKOVERRUN */ - ctx->malloced -= size; + INSIST(atomic_fetch_sub_release(&stats->gets, 1) >= 1); + + decrement_malloced(ctx, size); } /* @@ -450,45 +471,30 @@ mem_create(isc_mem_t **ctxp, unsigned int flags) { isc_mem_t *ctx; STATIC_ASSERT((ALIGNMENT_SIZE & (ALIGNMENT_SIZE - 1)) == 0, - "wrong alignment size"); + "alignment size not power of 2"); + STATIC_ASSERT(ALIGNMENT_SIZE >= sizeof(size_info), + "alignment size too small"); RUNTIME_CHECK(isc_once_do(&once, initialize_action) == ISC_R_SUCCESS); - ctx = (default_memalloc)(sizeof(*ctx)); + ctx = default_memalloc(sizeof(*ctx)); + + *ctx = (isc_mem_t){ + .magic = MEM_MAGIC, + .flags = flags, + .checkfree = true, + }; isc_mutex_init(&ctx->lock); - - ctx->max_size = DEF_MAX_SIZE; - ctx->flags = flags; isc_refcount_init(&ctx->references, 1); - memset(ctx->name, 0, sizeof(ctx->name)); - ctx->total = 0; - ctx->inuse = 0; - ctx->maxinuse = 0; - ctx->malloced = sizeof(*ctx); - ctx->maxmalloced = sizeof(*ctx); - ctx->hi_water = 0; - ctx->lo_water = 0; - ctx->hi_called = false; - ctx->is_overmem = false; - ctx->water = NULL; - ctx->water_arg = NULL; - ctx->magic = MEM_MAGIC; - ctx->stats = NULL; - ctx->checkfree = true; -#if ISC_MEM_TRACKLINES - ctx->debuglist = NULL; - ctx->debuglistcnt = 0; -#endif /* if ISC_MEM_TRACKLINES */ + + atomic_init(&ctx->total, 0); + atomic_init(&ctx->inuse, 0); + atomic_init(&ctx->maxinuse, 0); + atomic_init(&ctx->malloced, sizeof(*ctx)); + atomic_init(&ctx->maxmalloced, sizeof(*ctx)); + ISC_LIST_INIT(ctx->pools); - ctx->poolcnt = 0; - - ctx->stats = - default_memalloc((ctx->max_size + 1) * sizeof(struct stats)); - - memset(ctx->stats, 0, (ctx->max_size + 1) * sizeof(struct stats)); - ctx->malloced += (ctx->max_size + 1) * sizeof(struct stats); - ctx->maxmalloced += (ctx->max_size + 1) * sizeof(struct stats); #if ISC_MEM_TRACKLINES if (ISC_UNLIKELY((isc_mem_debugging & ISC_MEM_DEBUGRECORD) != 0)) { @@ -499,8 +505,8 @@ mem_create(isc_mem_t **ctxp, unsigned int flags) { for (i = 0; i < DEBUG_TABLE_COUNT; i++) { ISC_LIST_INIT(ctx->debuglist[i]); } - ctx->malloced += DEBUG_TABLE_COUNT * sizeof(debuglist_t); - ctx->maxmalloced += DEBUG_TABLE_COUNT * sizeof(debuglist_t); + increment_malloced(ctx, + DEBUG_TABLE_COUNT * sizeof(debuglist_t)); } #endif /* if ISC_MEM_TRACKLINES */ @@ -518,10 +524,11 @@ mem_create(isc_mem_t **ctxp, unsigned int flags) { static void destroy(isc_mem_t *ctx) { unsigned int i; + size_t malloced; LOCK(&contextslock); ISC_LIST_UNLINK(contexts, ctx, link); - totallost += ctx->inuse; + totallost += isc_mem_inuse(ctx); UNLOCK(&contextslock); ctx->magic = 0; @@ -542,39 +549,40 @@ destroy(isc_mem_t *ctx) { ISC_LIST_UNLINK(ctx->debuglist[i], dl, link); free(dl); - ctx->malloced -= sizeof(*dl); + decrement_malloced(ctx, sizeof(*dl)); } } default_memfree(ctx->debuglist); - ctx->malloced -= DEBUG_TABLE_COUNT * sizeof(debuglist_t); + decrement_malloced(ctx, + DEBUG_TABLE_COUNT * sizeof(debuglist_t)); } #endif /* if ISC_MEM_TRACKLINES */ if (ctx->checkfree) { - for (i = 0; i <= ctx->max_size; i++) { - if (ctx->stats[i].gets != 0U) { + for (i = 0; i <= STATS_BUCKETS; i++) { + struct stats *stats = &ctx->stats[i]; + size_t gets = atomic_load_acquire(&stats->gets); + if (gets != 0U) { fprintf(stderr, "Failing assertion due to probable " "leaked memory in context %p (\"%s\") " - "(stats[%u].gets == %lu).\n", - ctx, ctx->name, i, ctx->stats[i].gets); + "(stats[%u].gets == %zu).\n", + ctx, ctx->name, i, gets); #if ISC_MEM_TRACKLINES print_active(ctx, stderr); #endif /* if ISC_MEM_TRACKLINES */ - INSIST(ctx->stats[i].gets == 0U); + INSIST(gets == 0U); } } } - default_memfree(ctx->stats); - ctx->malloced -= (ctx->max_size + 1) * sizeof(struct stats); - isc_mutex_destroy(&ctx->lock); - ctx->malloced -= sizeof(*ctx); + malloced = decrement_malloced(ctx, sizeof(*ctx)); + if (ctx->checkfree) { - INSIST(ctx->malloced == 0); + INSIST(malloced == 0); } default_memfree(ctx); } @@ -625,7 +633,7 @@ isc__mem_putanddetach(isc_mem_t **ctxp, void *ptr, size_t size FLARG) { { if ((isc_mem_debugging & ISC_MEM_DEBUGSIZE) != 0) { size_info *si = &(((size_info *)ptr)[-1]); - size_t oldsize = si->u.size - ALIGNMENT_SIZE; + size_t oldsize = si->size - ALIGNMENT_SIZE; if ((isc_mem_debugging & ISC_MEM_DEBUGCTX) != 0) { oldsize -= ALIGNMENT_SIZE; } @@ -636,13 +644,10 @@ isc__mem_putanddetach(isc_mem_t **ctxp, void *ptr, size_t size FLARG) { goto destroy; } - MCTXLOCK(ctx); - DELETE_TRACE(ctx, ptr, size, file, line); mem_putstats(ctx, ptr, size); mem_put(ctx, ptr, size); - MCTXUNLOCK(ctx); destroy: if (isc_refcount_decrement(&ctx->references) == 1) { @@ -675,6 +680,55 @@ isc_mem_destroy(isc_mem_t **ctxp) { *ctxp = NULL; } +static inline bool +hi_water(isc_mem_t *ctx) { + bool call_water = false; + size_t inuse = atomic_load_acquire(&ctx->inuse); + size_t maxinuse = atomic_load_acquire(&ctx->maxinuse); + size_t hi_water = atomic_load_acquire(&ctx->hi_water); + + if (hi_water != 0U && inuse > hi_water) { + atomic_store(&ctx->is_overmem, true); + if (!atomic_load_acquire(&ctx->hi_called)) { + call_water = true; + } + } + if (inuse > maxinuse) { + (void)atomic_compare_exchange_strong(&ctx->maxinuse, &maxinuse, + inuse); + + if (hi_water != 0U && inuse > hi_water && + (isc_mem_debugging & ISC_MEM_DEBUGUSAGE) != 0) + { + fprintf(stderr, "maxinuse = %lu\n", + (unsigned long)inuse); + } + } + + return (call_water); +} + +/* + * The check against ctx->lo_water == 0 is for the condition + * when the context was pushed over hi_water but then had + * isc_mem_setwater() called with 0 for hi_water and lo_water. + */ +static inline bool +lo_water(isc_mem_t *ctx) { + bool call_water = false; + size_t inuse = atomic_load_acquire(&ctx->inuse); + size_t lo_water = atomic_load_acquire(&ctx->lo_water); + + if ((inuse < lo_water) || (lo_water == 0U)) { + atomic_store(&ctx->is_overmem, false); + if (atomic_load_acquire(&ctx->hi_called)) { + call_water = true; + } + } + + return (call_water); +} + void * isc__mem_get(isc_mem_t *ctx, size_t size FLARG) { REQUIRE(VALID_CONTEXT(ctx)); @@ -689,27 +743,11 @@ isc__mem_get(isc_mem_t *ctx, size_t size FLARG) { } ptr = mem_get(ctx, size); - MCTXLOCK(ctx); mem_getstats(ctx, size); ADD_TRACE(ctx, ptr, size, file, line); - if (ctx->hi_water != 0U && ctx->inuse > ctx->hi_water) { - ctx->is_overmem = true; - if (!ctx->hi_called) { - call_water = true; - } - } - if (ctx->inuse > ctx->maxinuse) { - ctx->maxinuse = ctx->inuse; - if (ctx->hi_water != 0U && ctx->inuse > ctx->hi_water && - (isc_mem_debugging & ISC_MEM_DEBUGUSAGE) != 0) - { - fprintf(stderr, "maxinuse = %lu\n", - (unsigned long)ctx->inuse); - } - } - MCTXUNLOCK(ctx); + call_water = hi_water(ctx); if (call_water && (ctx->water != NULL)) { (ctx->water)(ctx->water_arg, ISC_MEM_HIWATER); @@ -725,14 +763,14 @@ isc__mem_put(isc_mem_t *ctx, void *ptr, size_t size FLARG) { bool call_water = false; size_info *si; - size_t oldsize; if (ISC_UNLIKELY((isc_mem_debugging & (ISC_MEM_DEBUGSIZE | ISC_MEM_DEBUGCTX)) != 0)) { if ((isc_mem_debugging & ISC_MEM_DEBUGSIZE) != 0) { + size_t oldsize; si = &(((size_info *)ptr)[-1]); - oldsize = si->u.size - ALIGNMENT_SIZE; + oldsize = si->size - ALIGNMENT_SIZE; if ((isc_mem_debugging & ISC_MEM_DEBUGCTX) != 0) { oldsize -= ALIGNMENT_SIZE; } @@ -742,26 +780,12 @@ isc__mem_put(isc_mem_t *ctx, void *ptr, size_t size FLARG) { return; } - MCTXLOCK(ctx); - DELETE_TRACE(ctx, ptr, size, file, line); mem_putstats(ctx, ptr, size); mem_put(ctx, ptr, size); - /* - * The check against ctx->lo_water == 0 is for the condition - * when the context was pushed over hi_water but then had - * isc_mem_setwater() called with 0 for hi_water and lo_water. - */ - if ((ctx->inuse < ctx->lo_water) || (ctx->lo_water == 0U)) { - ctx->is_overmem = false; - if (ctx->hi_called) { - call_water = true; - } - } - - MCTXUNLOCK(ctx); + call_water = lo_water(ctx); if (call_water && (ctx->water != NULL)) { (ctx->water)(ctx->water_arg, ISC_MEM_LOWATER); @@ -772,13 +796,11 @@ void isc_mem_waterack(isc_mem_t *ctx, int flag) { REQUIRE(VALID_CONTEXT(ctx)); - MCTXLOCK(ctx); if (flag == ISC_MEM_LOWATER) { - ctx->hi_called = false; + atomic_store(&ctx->hi_called, false); } else if (flag == ISC_MEM_HIWATER) { - ctx->hi_called = true; + atomic_store(&ctx->hi_called, true); } - MCTXUNLOCK(ctx); } #if ISC_MEM_TRACKLINES @@ -789,7 +811,8 @@ print_active(isc_mem_t *mctx, FILE *out) { unsigned int i; bool found; - fprintf(out, "Dump of all outstanding memory allocations:\n"); + fprintf(out, "Dump of all outstanding memory " + "allocations:\n"); found = false; for (i = 0; i < DEBUG_TABLE_COUNT; i++) { dl = ISC_LIST_HEAD(mctx->debuglist[i]); @@ -801,7 +824,8 @@ print_active(isc_mem_t *mctx, FILE *out) { while (dl != NULL) { if (dl->ptr != NULL) { fprintf(out, - "\tptr %p size %zu file %s " + "\tptr %p size %zu " + "file %s " "line %u\n", dl->ptr, dl->size, dl->file, dl->line); @@ -824,30 +848,32 @@ void isc_mem_stats(isc_mem_t *ctx, FILE *out) { REQUIRE(VALID_CONTEXT(ctx)); - size_t i; - const struct stats *s; - const isc_mempool_t *pool; + isc_mempool_t *pool; MCTXLOCK(ctx); - for (i = 0; i <= ctx->max_size; i++) { - s = &ctx->stats[i]; + for (size_t i = 0; i <= STATS_BUCKETS; i++) { + size_t totalgets; + size_t gets; + struct stats *stats = &ctx->stats[i]; - if (s->totalgets == 0U && s->gets == 0U) { - continue; + totalgets = atomic_load_acquire(&stats->totalgets); + gets = atomic_load_acquire(&stats->gets); + + if (totalgets != 0U && gets != 0U) { + fprintf(out, "%s%5zu: %11zu gets, %11zu rem", + (i == STATS_BUCKETS) ? ">=" : " ", i, + totalgets, gets); + fputc('\n', out); } - fprintf(out, "%s%5lu: %11lu gets, %11lu rem", - (i == ctx->max_size) ? ">=" : " ", (unsigned long)i, - s->totalgets, s->gets); - fputc('\n', out); } /* - * Note that since a pool can be locked now, these stats might be - * somewhat off if the pool is in active use at the time the stats - * are dumped. The link fields are protected by the isc_mem_t's - * lock, however, so walking this list and extracting integers from - * stats fields is always safe. + * Note that since a pool can be locked now, these stats might + * be somewhat off if the pool is in active use at the time the + * stats are dumped. The link fields are protected by the + * isc_mem_t's lock, however, so walking this list and + * extracting integers from stats fields is always safe. */ pool = ISC_LIST_HEAD(ctx->pools); if (pool != NULL) { @@ -857,15 +883,19 @@ isc_mem_stats(isc_mem_t *ctx, FILE *out) { "freemax", "fillcount", "gets", "L"); } while (pool != NULL) { - fprintf(out, "%15s %10lu %10u %10u %10u %10u %10u %10u %s\n", + fprintf(out, + "%15s %10zu %10zu %10zu %10zu %10zu %10zu %10zu %s\n", #if ISC_MEMPOOL_NAMES pool->name, #else /* if ISC_MEMPOOL_NAMES */ "(not tracked)", #endif /* if ISC_MEMPOOL_NAMES */ - (unsigned long)pool->size, pool->maxalloc, - pool->allocated, pool->freecount, pool->freemax, - pool->fillcount, pool->gets, + pool->size, atomic_load_relaxed(&pool->maxalloc), + atomic_load_relaxed(&pool->allocated), + atomic_load_relaxed(&pool->freecount), + atomic_load_relaxed(&pool->freemax), + atomic_load_relaxed(&pool->fillcount), + atomic_load_relaxed(&pool->gets), (pool->lock == NULL ? "N" : "Y")); pool = ISC_LIST_NEXT(pool, link); } @@ -894,10 +924,10 @@ mem_allocateunlocked(isc_mem_t *ctx, size_t size) { si = mem_get(ctx, size); if (ISC_UNLIKELY((isc_mem_debugging & ISC_MEM_DEBUGCTX) != 0)) { - si->u.ctx = ctx; + si->ctx = ctx; si++; } - si->u.size = size; + si->size = size; return (&si[1]); } @@ -908,34 +938,14 @@ isc__mem_allocate(isc_mem_t *ctx, size_t size FLARG) { size_info *si; bool call_water = false; - MCTXLOCK(ctx); si = mem_allocateunlocked(ctx, size); - mem_getstats(ctx, si[-1].u.size); + mem_getstats(ctx, si[-1].size); - ADD_TRACE(ctx, si, si[-1].u.size, file, line); - if (ctx->hi_water != 0U && ctx->inuse > ctx->hi_water && - !ctx->is_overmem) { - ctx->is_overmem = true; - } + ADD_TRACE(ctx, si, si[-1].size, file, line); - if (ctx->hi_water != 0U && !ctx->hi_called && - ctx->inuse > ctx->hi_water) { - ctx->hi_called = true; - call_water = true; - } - if (ctx->inuse > ctx->maxinuse) { - ctx->maxinuse = ctx->inuse; - if (ISC_UNLIKELY(ctx->hi_water != 0U && - ctx->inuse > ctx->hi_water && - (isc_mem_debugging & ISC_MEM_DEBUGUSAGE) != 0)) - { - fprintf(stderr, "maxinuse = %lu\n", - (unsigned long)ctx->inuse); - } - } - MCTXUNLOCK(ctx); + call_water = hi_water(ctx); - if (call_water) { + if (call_water && (ctx->water != NULL)) { (ctx->water)(ctx->water_arg, ISC_MEM_HIWATER); } @@ -947,15 +957,16 @@ isc__mem_reallocate(isc_mem_t *ctx, void *ptr, size_t size FLARG) { REQUIRE(VALID_CONTEXT(ctx)); void *new_ptr = NULL; - size_t oldsize, copysize; /* - * This function emulates the realloc(3) standard library function: - * - if size > 0, allocate new memory; and if ptr is non NULL, copy - * as much of the old contents to the new buffer and free the old one. - * Note that when allocation fails the original pointer is intact; - * the caller must free it. - * - if size is 0 and ptr is non NULL, simply free the given ptr. + * This function emulates the realloc(3) standard library + * function: + * - if size > 0, allocate new memory; and if ptr is non NULL, + * copy as much of the old contents to the new buffer and free + * the old one. Note that when allocation fails the original + * pointer is intact; the caller must free it. + * - if size is 0 and ptr is non NULL, simply free the given + * ptr. * - this function returns: * pointer to the newly allocated memory, or * NULL if allocation fails or doesn't happen. @@ -963,7 +974,8 @@ isc__mem_reallocate(isc_mem_t *ctx, void *ptr, size_t size FLARG) { if (size > 0U) { new_ptr = isc__mem_allocate(ctx, size FLARG_PASS); if (new_ptr != NULL && ptr != NULL) { - oldsize = (((size_info *)ptr)[-1]).u.size; + size_t oldsize, copysize; + oldsize = (((size_info *)ptr)[-1]).size; INSIST(oldsize >= ALIGNMENT_SIZE); oldsize -= ALIGNMENT_SIZE; if (ISC_UNLIKELY((isc_mem_debugging & @@ -993,41 +1005,21 @@ isc__mem_free(isc_mem_t *ctx, void *ptr FLARG) { if (ISC_UNLIKELY((isc_mem_debugging & ISC_MEM_DEBUGCTX) != 0)) { si = &(((size_info *)ptr)[-2]); - REQUIRE(si->u.ctx == ctx); - size = si[1].u.size; + REQUIRE(si->ctx == ctx); + size = si[1].size; } else { si = &(((size_info *)ptr)[-1]); - size = si->u.size; + size = si->size; } - MCTXLOCK(ctx); - DELETE_TRACE(ctx, ptr, size, file, line); mem_putstats(ctx, si, size); mem_put(ctx, si, size); - /* - * The check against ctx->lo_water == 0 is for the condition - * when the context was pushed over hi_water but then had - * isc_mem_setwater() called with 0 for hi_water and lo_water. - */ - if (ctx->is_overmem && - (ctx->inuse < ctx->lo_water || ctx->lo_water == 0U)) { - ctx->is_overmem = false; - } + call_water = lo_water(ctx); - if (ctx->hi_called && - (ctx->inuse < ctx->lo_water || ctx->lo_water == 0U)) { - ctx->hi_called = false; - - if (ctx->water != NULL) { - call_water = true; - } - } - MCTXUNLOCK(ctx); - - if (call_water) { + if (call_water && (ctx->water != NULL)) { (ctx->water)(ctx->water_arg, ISC_MEM_LOWATER); } } @@ -1092,45 +1084,35 @@ size_t isc_mem_inuse(isc_mem_t *ctx) { REQUIRE(VALID_CONTEXT(ctx)); - size_t inuse; - - MCTXLOCK(ctx); - - inuse = ctx->inuse; - - MCTXUNLOCK(ctx); - - return (inuse); + return (atomic_load_acquire(&ctx->inuse)); } size_t isc_mem_maxinuse(isc_mem_t *ctx) { REQUIRE(VALID_CONTEXT(ctx)); - size_t maxinuse; - - MCTXLOCK(ctx); - - maxinuse = ctx->maxinuse; - - MCTXUNLOCK(ctx); - - return (maxinuse); + return (atomic_load_acquire(&ctx->maxinuse)); } size_t isc_mem_total(isc_mem_t *ctx) { REQUIRE(VALID_CONTEXT(ctx)); - size_t total; + return (atomic_load_acquire(&ctx->total)); +} - MCTXLOCK(ctx); +size_t +isc_mem_malloced(isc_mem_t *ctx) { + REQUIRE(VALID_CONTEXT(ctx)); - total = ctx->total; + return (atomic_load_acquire(&ctx->malloced)); +} - MCTXUNLOCK(ctx); +size_t +isc_mem_maxmalloced(isc_mem_t *ctx) { + REQUIRE(VALID_CONTEXT(ctx)); - return (total); + return (atomic_load_acquire(&ctx->maxmalloced)); } void @@ -1147,22 +1129,23 @@ isc_mem_setwater(isc_mem_t *ctx, isc_mem_water_t water, void *water_arg, oldwater = ctx->water; oldwater_arg = ctx->water_arg; if (water == NULL) { - callwater = ctx->hi_called; + callwater = atomic_load(&ctx->hi_called); ctx->water = NULL; ctx->water_arg = NULL; - ctx->hi_water = 0; - ctx->lo_water = 0; + atomic_store_release(&ctx->hi_water, 0); + atomic_store_release(&ctx->lo_water, 0); } else { - if (ctx->hi_called && + if (atomic_load_acquire(&ctx->hi_called) && (ctx->water != water || ctx->water_arg != water_arg || - ctx->inuse < lowater || lowater == 0U)) + atomic_load_acquire(&ctx->inuse) < lowater || + lowater == 0U)) { callwater = true; } ctx->water = water; ctx->water_arg = water_arg; - ctx->hi_water = hiwater; - ctx->lo_water = lowater; + atomic_store_release(&ctx->hi_water, hiwater); + atomic_store_release(&ctx->lo_water, lowater); } MCTXUNLOCK(ctx); @@ -1171,16 +1154,11 @@ isc_mem_setwater(isc_mem_t *ctx, isc_mem_water_t water, void *water_arg, } } -ISC_NO_SANITIZE_THREAD bool +bool isc_mem_isovermem(isc_mem_t *ctx) { REQUIRE(VALID_CONTEXT(ctx)); - /* - * We don't bother to lock the context because 100% accuracy isn't - * necessary (and even if we locked the context the returned value - * could be different from the actual state when it's used anyway) - */ - return (ctx->is_overmem); + return (atomic_load_relaxed(&ctx->is_overmem)); } void @@ -1215,32 +1193,30 @@ isc_mempool_create(isc_mem_t *mctx, size_t size, isc_mempool_t **mpctxp) { isc_mempool_t *mpctx; - /* - * Allocate space for this pool, initialize values, and if all works - * well, attach to the memory context. - */ - mpctx = isc_mem_get(mctx, sizeof(isc_mempool_t)); - - mpctx->magic = MEMPOOL_MAGIC; - mpctx->lock = NULL; - mpctx->mctx = mctx; /* * Mempools are stored as a linked list of element. */ if (size < sizeof(element)) { size = sizeof(element); } - mpctx->size = size; - mpctx->maxalloc = UINT_MAX; - mpctx->allocated = 0; - mpctx->freecount = 0; - mpctx->freemax = 1; - mpctx->fillcount = 1; - mpctx->gets = 0; -#if ISC_MEMPOOL_NAMES - mpctx->name[0] = 0; -#endif /* if ISC_MEMPOOL_NAMES */ - mpctx->items = NULL; + + /* + * Allocate space for this pool, initialize values, and if all + * works well, attach to the memory context. + */ + mpctx = isc_mem_get(mctx, sizeof(isc_mempool_t)); + + *mpctx = (isc_mempool_t){ + .magic = MEMPOOL_MAGIC, + .mctx = mctx, + .size = size, + }; + + atomic_init(&mpctx->maxalloc, SIZE_MAX); + atomic_init(&mpctx->allocated, 0); + atomic_init(&mpctx->freecount, 0); + atomic_init(&mpctx->freemax, 1); + atomic_init(&mpctx->fillcount, 1); *mpctxp = (isc_mempool_t *)mpctx; @@ -1256,15 +1232,12 @@ isc_mempool_setname(isc_mempool_t *mpctx, const char *name) { REQUIRE(name != NULL); #if ISC_MEMPOOL_NAMES - if (mpctx->lock != NULL) { - LOCK(mpctx->lock); - } + MPCTXLOCK(mpctx); strlcpy(mpctx->name, name, sizeof(mpctx->name)); - if (mpctx->lock != NULL) { - UNLOCK(mpctx->lock); - } + MPCTXUNLOCK(mpctx); + #else /* if ISC_MEMPOOL_NAMES */ UNUSED(mpctx); UNUSED(name); @@ -1284,14 +1257,14 @@ isc_mempool_destroy(isc_mempool_t **mpctxp) { mpctx = *mpctxp; *mpctxp = NULL; #if ISC_MEMPOOL_NAMES - if (mpctx->allocated > 0) { + if (atomic_load_acquire(&mpctx->allocated) > 0) { UNEXPECTED_ERROR(__FILE__, __LINE__, "isc_mempool_destroy(): mempool %s " "leaked memory", mpctx->name); } #endif /* if ISC_MEMPOOL_NAMES */ - REQUIRE(mpctx->allocated == 0); + REQUIRE(atomic_load_acquire(&mpctx->allocated) == 0); mctx = mpctx->mctx; @@ -1304,17 +1277,15 @@ isc_mempool_destroy(isc_mempool_t **mpctxp) { /* * Return any items on the free list */ - MCTXLOCK(mctx); while (mpctx->items != NULL) { - INSIST(mpctx->freecount > 0); - mpctx->freecount--; + INSIST(atomic_fetch_sub_release(&mpctx->freecount, 1) > 0); + item = mpctx->items; mpctx->items = item->next; mem_putstats(mctx, item, mpctx->size); mem_put(mctx, item, mpctx->size); } - MCTXUNLOCK(mctx); /* * Remove our linked list entry from the memory context. @@ -1348,37 +1319,33 @@ isc__mempool_get(isc_mempool_t *mpctx FLARG) { REQUIRE(VALID_MEMPOOL(mpctx)); element *item; - isc_mem_t *mctx; unsigned int i; - - mctx = mpctx->mctx; - - if (mpctx->lock != NULL) { - LOCK(mpctx->lock); - } + size_t allocated = atomic_load_acquire(&mpctx->allocated); + size_t maxalloc = atomic_load_acquire(&mpctx->maxalloc); /* * Don't let the caller go over quota */ - if (ISC_UNLIKELY(mpctx->allocated >= mpctx->maxalloc)) { + if (ISC_UNLIKELY(allocated >= maxalloc)) { item = NULL; goto out; } + MPCTXLOCK(mpctx); if (ISC_UNLIKELY(mpctx->items == NULL)) { + isc_mem_t *mctx = mpctx->mctx; + size_t fillcount = atomic_load_acquire(&mpctx->fillcount); /* - * We need to dip into the well. Lock the memory context - * here and fill up our free list. + * We need to dip into the well. Lock the memory + * context here and fill up our free list. */ - MCTXLOCK(mctx); - for (i = 0; i < mpctx->fillcount; i++) { + for (i = 0; i < fillcount; i++) { item = mem_get(mctx, mpctx->size); mem_getstats(mctx, mpctx->size); item->next = mpctx->items; mpctx->items = item; - mpctx->freecount++; + atomic_fetch_add_relaxed(&mpctx->freecount, 1); } - MCTXUNLOCK(mctx); } /* @@ -1390,24 +1357,15 @@ isc__mempool_get(isc_mempool_t *mpctx FLARG) { } mpctx->items = item->next; - INSIST(mpctx->freecount > 0); - mpctx->freecount--; - mpctx->gets++; - mpctx->allocated++; + + INSIST(atomic_fetch_sub_release(&mpctx->freecount, 1) > 0); + atomic_fetch_add_relaxed(&mpctx->gets, 1); + atomic_fetch_add_relaxed(&mpctx->allocated, 1); out: - if (mpctx->lock != NULL) { - UNLOCK(mpctx->lock); - } + MPCTXUNLOCK(mpctx); -#if ISC_MEM_TRACKLINES - if (ISC_UNLIKELY(((isc_mem_debugging & TRACE_OR_RECORD) != 0) && - item != NULL)) { - MCTXLOCK(mctx); - ADD_TRACE(mctx, item, mpctx->size, file, line); - MCTXUNLOCK(mctx); - } -#endif /* ISC_MEM_TRACKLINES */ + ADD_TRACE(mpctx->mctx, item, mpctx->size, file, line); return (item); } @@ -1420,47 +1378,33 @@ isc__mempool_put(isc_mempool_t *mpctx, void *mem FLARG) { isc_mem_t *mctx = mpctx->mctx; element *item; + size_t freecount = atomic_load_acquire(&mpctx->freecount); + size_t freemax = atomic_load_acquire(&mpctx->freemax); - if (mpctx->lock != NULL) { - LOCK(mpctx->lock); - } + INSIST(atomic_fetch_sub_release(&mpctx->allocated, 1) > 0); - INSIST(mpctx->allocated > 0); - mpctx->allocated--; - -#if ISC_MEM_TRACKLINES - if (ISC_UNLIKELY((isc_mem_debugging & TRACE_OR_RECORD) != 0)) { - MCTXLOCK(mctx); - DELETE_TRACE(mctx, mem, mpctx->size, file, line); - MCTXUNLOCK(mctx); - } -#endif /* ISC_MEM_TRACKLINES */ + DELETE_TRACE(mctx, mem, mpctx->size, file, line); /* * If our free list is full, return this to the mctx directly. */ - if (mpctx->freecount >= mpctx->freemax) { - MCTXLOCK(mctx); + if (freecount >= freemax) { mem_putstats(mctx, mem, mpctx->size); mem_put(mctx, mem, mpctx->size); - MCTXUNLOCK(mctx); - if (mpctx->lock != NULL) { - UNLOCK(mpctx->lock); - } return; } /* * Otherwise, attach it to our free list and bump the counter. */ - mpctx->freecount++; + MPCTXLOCK(mpctx); + item = (element *)mem; item->next = mpctx->items; mpctx->items = item; + atomic_fetch_add_relaxed(&mpctx->freecount, 1); - if (mpctx->lock != NULL) { - UNLOCK(mpctx->lock); - } + MPCTXUNLOCK(mpctx); } /* @@ -1471,15 +1415,11 @@ void isc_mempool_setfreemax(isc_mempool_t *mpctx, unsigned int limit) { REQUIRE(VALID_MEMPOOL(mpctx)); - if (mpctx->lock != NULL) { - LOCK(mpctx->lock); - } + MPCTXLOCK(mpctx); mpctx->freemax = limit; - if (mpctx->lock != NULL) { - UNLOCK(mpctx->lock); - } + MPCTXUNLOCK(mpctx); } unsigned int @@ -1488,15 +1428,11 @@ isc_mempool_getfreemax(isc_mempool_t *mpctx) { unsigned int freemax; - if (mpctx->lock != NULL) { - LOCK(mpctx->lock); - } + MPCTXLOCK(mpctx); freemax = mpctx->freemax; - if (mpctx->lock != NULL) { - UNLOCK(mpctx->lock); - } + MPCTXUNLOCK(mpctx); return (freemax); } @@ -1505,19 +1441,7 @@ unsigned int isc_mempool_getfreecount(isc_mempool_t *mpctx) { REQUIRE(VALID_MEMPOOL(mpctx)); - unsigned int freecount; - - if (mpctx->lock != NULL) { - LOCK(mpctx->lock); - } - - freecount = mpctx->freecount; - - if (mpctx->lock != NULL) { - UNLOCK(mpctx->lock); - } - - return (freecount); + return (atomic_load_relaxed(&mpctx->freecount)); } void @@ -1525,53 +1449,21 @@ isc_mempool_setmaxalloc(isc_mempool_t *mpctx, unsigned int limit) { REQUIRE(VALID_MEMPOOL(mpctx)); REQUIRE(limit > 0); - if (mpctx->lock != NULL) { - LOCK(mpctx->lock); - } - - mpctx->maxalloc = limit; - - if (mpctx->lock != NULL) { - UNLOCK(mpctx->lock); - } + atomic_store_release(&mpctx->maxalloc, limit); } unsigned int isc_mempool_getmaxalloc(isc_mempool_t *mpctx) { REQUIRE(VALID_MEMPOOL(mpctx)); - unsigned int maxalloc; - - if (mpctx->lock != NULL) { - LOCK(mpctx->lock); - } - - maxalloc = mpctx->maxalloc; - - if (mpctx->lock != NULL) { - UNLOCK(mpctx->lock); - } - - return (maxalloc); + return (atomic_load_relaxed(&mpctx->maxalloc)); } unsigned int isc_mempool_getallocated(isc_mempool_t *mpctx) { REQUIRE(VALID_MEMPOOL(mpctx)); - unsigned int allocated; - - if (mpctx->lock != NULL) { - LOCK(mpctx->lock); - } - - allocated = mpctx->allocated; - - if (mpctx->lock != NULL) { - UNLOCK(mpctx->lock); - } - - return (allocated); + return (atomic_load_relaxed(&mpctx->allocated)); } void @@ -1579,34 +1471,14 @@ isc_mempool_setfillcount(isc_mempool_t *mpctx, unsigned int limit) { REQUIRE(VALID_MEMPOOL(mpctx)); REQUIRE(limit > 0); - if (mpctx->lock != NULL) { - LOCK(mpctx->lock); - } - - mpctx->fillcount = limit; - - if (mpctx->lock != NULL) { - UNLOCK(mpctx->lock); - } + atomic_store_release(&mpctx->fillcount, limit); } unsigned int isc_mempool_getfillcount(isc_mempool_t *mpctx) { REQUIRE(VALID_MEMPOOL(mpctx)); - unsigned int fillcount; - - if (mpctx->lock != NULL) { - LOCK(mpctx->lock); - } - - fillcount = mpctx->fillcount; - - if (mpctx->lock != NULL) { - UNLOCK(mpctx->lock); - } - - return (fillcount); + return (atomic_load_relaxed(&mpctx->fillcount)); } /* @@ -1686,9 +1558,7 @@ xml_renderctx(isc_mem_t *ctx, summarystat_t *summary, xmlTextWriterPtr writer) { TRY0(xmlTextWriterEndElement(writer)); /* name */ } - summary->contextsize += sizeof(*ctx) + - (ctx->max_size + 1) * sizeof(struct stats) + - ctx->max_size * sizeof(element *); + summary->contextsize += sizeof(*ctx); #if ISC_MEM_TRACKLINES if (ctx->debuglist != NULL) { summary->contextsize += DEBUG_TABLE_COUNT * @@ -1702,32 +1572,32 @@ xml_renderctx(isc_mem_t *ctx, summarystat_t *summary, xmlTextWriterPtr writer) { isc_refcount_current(&ctx->references))); TRY0(xmlTextWriterEndElement(writer)); /* references */ - summary->total += ctx->total; + summary->total += isc_mem_total(ctx); TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "total")); TRY0(xmlTextWriterWriteFormatString(writer, "%" PRIu64 "", - (uint64_t)ctx->total)); + (uint64_t)isc_mem_total(ctx))); TRY0(xmlTextWriterEndElement(writer)); /* total */ - summary->inuse += ctx->inuse; + summary->inuse += isc_mem_inuse(ctx); TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "inuse")); TRY0(xmlTextWriterWriteFormatString(writer, "%" PRIu64 "", - (uint64_t)ctx->inuse)); + (uint64_t)isc_mem_inuse(ctx))); TRY0(xmlTextWriterEndElement(writer)); /* inuse */ TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "maxinuse")); TRY0(xmlTextWriterWriteFormatString(writer, "%" PRIu64 "", - (uint64_t)ctx->maxinuse)); + (uint64_t)isc_mem_maxinuse(ctx))); TRY0(xmlTextWriterEndElement(writer)); /* maxinuse */ - summary->malloced += ctx->malloced; + summary->malloced += isc_mem_malloced(ctx); TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "malloced")); TRY0(xmlTextWriterWriteFormatString(writer, "%" PRIu64 "", - (uint64_t)ctx->malloced)); + (uint64_t)isc_mem_malloced(ctx))); TRY0(xmlTextWriterEndElement(writer)); /* malloced */ TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "maxmalloced")); - TRY0(xmlTextWriterWriteFormatString(writer, "%" PRIu64 "", - (uint64_t)ctx->maxmalloced)); + TRY0(xmlTextWriterWriteFormatString( + writer, "%" PRIu64 "", (uint64_t)isc_mem_maxmalloced(ctx))); TRY0(xmlTextWriterEndElement(writer)); /* maxmalloced */ TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "pools")); @@ -1736,13 +1606,15 @@ xml_renderctx(isc_mem_t *ctx, summarystat_t *summary, xmlTextWriterPtr writer) { summary->contextsize += ctx->poolcnt * sizeof(isc_mempool_t); TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "hiwater")); - TRY0(xmlTextWriterWriteFormatString(writer, "%" PRIu64 "", - (uint64_t)ctx->hi_water)); + TRY0(xmlTextWriterWriteFormatString( + writer, "%" PRIu64 "", + (uint64_t)atomic_load_relaxed(&ctx->hi_water))); TRY0(xmlTextWriterEndElement(writer)); /* hiwater */ TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "lowater")); - TRY0(xmlTextWriterWriteFormatString(writer, "%" PRIu64 "", - (uint64_t)ctx->lo_water)); + TRY0(xmlTextWriterWriteFormatString( + writer, "%" PRIu64 "", + (uint64_t)atomic_load_relaxed(&ctx->lo_water))); TRY0(xmlTextWriterEndElement(writer)); /* lowater */ TRY0(xmlTextWriterEndElement(writer)); /* context */ @@ -1828,12 +1700,10 @@ json_renderctx(isc_mem_t *ctx, summarystat_t *summary, json_object *array) { MCTXLOCK(ctx); - summary->contextsize += sizeof(*ctx) + - (ctx->max_size + 1) * sizeof(struct stats) + - ctx->max_size * sizeof(element *); - summary->total += ctx->total; - summary->inuse += ctx->inuse; - summary->malloced += ctx->malloced; + summary->contextsize += sizeof(*ctx); + summary->total += isc_mem_total(ctx); + summary->inuse += isc_mem_inuse(ctx); + summary->malloced += isc_mem_malloced(ctx); #if ISC_MEM_TRACKLINES if (ctx->debuglist != NULL) { summary->contextsize += DEBUG_TABLE_COUNT * @@ -1860,23 +1730,23 @@ json_renderctx(isc_mem_t *ctx, summarystat_t *summary, json_object *array) { CHECKMEM(obj); json_object_object_add(ctxobj, "references", obj); - obj = json_object_new_int64(ctx->total); + obj = json_object_new_int64(isc_mem_total(ctx)); CHECKMEM(obj); json_object_object_add(ctxobj, "total", obj); - obj = json_object_new_int64(ctx->inuse); + obj = json_object_new_int64(isc_mem_inuse(ctx)); CHECKMEM(obj); json_object_object_add(ctxobj, "inuse", obj); - obj = json_object_new_int64(ctx->maxinuse); + obj = json_object_new_int64(isc_mem_maxinuse(ctx)); CHECKMEM(obj); json_object_object_add(ctxobj, "maxinuse", obj); - obj = json_object_new_int64(ctx->malloced); + obj = json_object_new_int64(isc_mem_malloced(ctx)); CHECKMEM(obj); json_object_object_add(ctxobj, "malloced", obj); - obj = json_object_new_int64(ctx->maxmalloced); + obj = json_object_new_int64(isc_mem_maxmalloced(ctx)); CHECKMEM(obj); json_object_object_add(ctxobj, "maxmalloced", obj); @@ -1886,11 +1756,11 @@ json_renderctx(isc_mem_t *ctx, summarystat_t *summary, json_object *array) { summary->contextsize += ctx->poolcnt * sizeof(isc_mempool_t); - obj = json_object_new_int64(ctx->hi_water); + obj = json_object_new_int64(atomic_load_relaxed(&ctx->hi_water)); CHECKMEM(obj); json_object_object_add(ctxobj, "hiwater", obj); - obj = json_object_new_int64(ctx->lo_water); + obj = json_object_new_int64(atomic_load_relaxed(&ctx->lo_water)); CHECKMEM(obj); json_object_object_add(ctxobj, "lowater", obj); diff --git a/lib/isc/win32/libisc.def.in b/lib/isc/win32/libisc.def.in index 7bc4926a96..b02fb22bbb 100644 --- a/lib/isc/win32/libisc.def.in +++ b/lib/isc/win32/libisc.def.in @@ -370,7 +370,9 @@ isc_mem_detach isc_mem_getname isc_mem_inuse isc_mem_isovermem +isc_mem_malloced isc_mem_maxinuse +isc_mem_maxmalloced isc_mem_references @IF NOTYET isc_mem_renderjson