2
0
mirror of https://gitlab.isc.org/isc-projects/bind9 synced 2025-09-05 09:05:40 +00:00
Files
bind/lib/isc/mem.c
alessio d21f63884a Adaptive memory allocation strategy for qp-tries
qp-tries allocate their nodes (twigs) in chunks to reduce allocator
pressure and improve memory locality. The choice of chunk size presents
a tradeoff: larger chunks benefit qp-tries with many values (as seen
in large zones and resolvers) but waste memory in smaller use cases.

Previously, our fixed chunk size of 2^10 twigs meant that even an
empty qp-trie would consume 12KB of memory, while reducing this size
would negatively impact resolver performance.

This commit implements an adaptive chunking strategy that:
 - Tracks the size of the most recently allocated chunk.
 - Doubles the chunk size for each new allocation until reaching a
   predefined maximum.

This approach effectively balances memory efficiency for small tries
while maintaining the performance benefits of larger chunk sizes for
bigger data structures.

This commit also splits the callback freeing qpmultis into two
phases, one that frees the underlying qptree, and one that reclaims
the qpmulti memory. In order to prevent races between the qpmulti
destructor and chunk garbage collection jobs, the second phase is
protected by reference counting.

(cherry picked from commit 70b1777d8a)
2025-08-05 12:48:19 +02:00

1606 lines
36 KiB
C

/*
* Copyright (C) Internet Systems Consortium, Inc. ("ISC")
*
* SPDX-License-Identifier: MPL-2.0
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, you can obtain one at https://mozilla.org/MPL/2.0/.
*
* See the COPYRIGHT file distributed with this work for additional
* information regarding copyright ownership.
*/
/*! \file */
#include <inttypes.h>
#include <limits.h>
#include <stdbool.h>
#include <stddef.h>
#include <stdio.h>
#include <stdlib.h>
#include <isc/hash.h>
#include <isc/magic.h>
#include <isc/mem.h>
#include <isc/mutex.h>
#include <isc/once.h>
#include <isc/os.h>
#include <isc/overflow.h>
#include <isc/refcount.h>
#include <isc/strerr.h>
#include <isc/string.h>
#include <isc/types.h>
#include <isc/urcu.h>
#include <isc/util.h>
#ifdef HAVE_LIBXML2
#include <libxml/xmlwriter.h>
#define ISC_XMLCHAR (const xmlChar *)
#endif /* HAVE_LIBXML2 */
#ifdef HAVE_JSON_C
#include <json_object.h>
#endif /* HAVE_JSON_C */
/* On DragonFly BSD the header does not provide jemalloc API */
#if defined(HAVE_MALLOC_NP_H) && !defined(__DragonFly__)
#include <malloc_np.h>
#define JEMALLOC_API_SUPPORTED 1
#elif defined(HAVE_JEMALLOC)
#include <jemalloc/jemalloc.h>
#define JEMALLOC_API_SUPPORTED 1
#else
#include "jemalloc_shim.h"
#endif
#include "mem_p.h"
#define MCTXLOCK(m) LOCK(&m->lock)
#define MCTXUNLOCK(m) UNLOCK(&m->lock)
#ifndef ISC_MEM_DEBUGGING
#define ISC_MEM_DEBUGGING 0
#endif /* ifndef ISC_MEM_DEBUGGING */
unsigned int isc_mem_debugging = ISC_MEM_DEBUGGING;
unsigned int isc_mem_defaultflags = ISC_MEMFLAG_DEFAULT;
#define ISC_MEM_ILLEGAL_ARENA (UINT_MAX)
volatile void *isc__mem_malloc = mallocx;
/*
* Constants.
*/
#define ZERO_ALLOCATION_SIZE sizeof(void *)
#define ALIGNMENT 8U /*%< must be a power of 2 */
#define ALIGNMENT_SIZE sizeof(size_info)
#define DEBUG_TABLE_COUNT 512U
/*
* Types.
*/
#if ISC_MEM_TRACKLINES
typedef struct debuglink debuglink_t;
struct debuglink {
size_t dlsize;
ISC_LINK(debuglink_t) link;
const void *ptr;
size_t size;
unsigned int line;
const char file[];
};
typedef ISC_LIST(debuglink_t) debuglist_t;
#define FLARG_PASS , file, line
#define FLARG , const char *file, unsigned int line
#else /* if ISC_MEM_TRACKLINES */
#define FLARG_PASS
#define FLARG
#endif /* if ISC_MEM_TRACKLINES */
typedef struct element element;
struct element {
element *next;
};
#define MEM_MAGIC ISC_MAGIC('M', 'e', 'm', 'C')
#define VALID_CONTEXT(c) ISC_MAGIC_VALID(c, MEM_MAGIC)
/* List of all active memory contexts. */
static ISC_LIST(isc_mem_t) contexts;
static isc_once_t init_once = ISC_ONCE_INIT;
static isc_once_t shut_once = ISC_ONCE_INIT;
static isc_mutex_t contextslock;
struct isc_mem {
unsigned int magic;
unsigned int flags;
unsigned int jemalloc_flags;
unsigned int jemalloc_arena;
unsigned int debugging;
isc_mutex_t lock;
bool checkfree;
isc_refcount_t references;
char name[16];
atomic_size_t inuse;
atomic_bool hi_called;
atomic_bool is_overmem;
atomic_size_t hi_water;
atomic_size_t lo_water;
ISC_LIST(isc_mempool_t) pools;
unsigned int poolcnt;
#if ISC_MEM_TRACKLINES
debuglist_t *debuglist;
size_t debuglistcnt;
#endif /* if ISC_MEM_TRACKLINES */
ISC_LINK(isc_mem_t) link;
};
#define MEMPOOL_MAGIC ISC_MAGIC('M', 'E', 'M', 'p')
#define VALID_MEMPOOL(c) ISC_MAGIC_VALID(c, MEMPOOL_MAGIC)
struct isc_mempool {
/* always unlocked */
unsigned int magic;
isc_mem_t *mctx; /*%< our memory context */
ISC_LINK(isc_mempool_t) link; /*%< next pool in this mem context */
element *items; /*%< low water item list */
size_t size; /*%< size of each item on this pool */
size_t allocated; /*%< # of items currently given out */
size_t freecount; /*%< # of items on reserved list */
size_t freemax; /*%< # of items allowed on free list */
size_t fillcount; /*%< # of items to fetch on each fill */
/*%< Stats only. */
size_t gets; /*%< # of requests to this pool */
/*%< Debugging only. */
char name[16]; /*%< printed name in stats reports */
};
/*
* Private Inline-able.
*/
#if !ISC_MEM_TRACKLINES
#define ADD_TRACE(mctx, ptr, size, file, line)
#define DELETE_TRACE(mctx, ptr, size, file, line)
#define ISC_MEMFUNC_SCOPE
#else /* if !ISC_MEM_TRACKLINES */
#define TRACE_OR_RECORD (ISC_MEM_DEBUGTRACE | ISC_MEM_DEBUGRECORD)
#define SHOULD_TRACE_OR_RECORD(mctx, ptr) \
(((mctx)->debugging & TRACE_OR_RECORD) != 0 && ptr != NULL)
#define ADD_TRACE(mctx, ptr, size, file, line) \
if (SHOULD_TRACE_OR_RECORD(mctx, ptr)) { \
add_trace_entry(mctx, ptr, size, file, line); \
}
#define DELETE_TRACE(mctx, ptr, size, file, line) \
if (SHOULD_TRACE_OR_RECORD(mctx, ptr)) { \
delete_trace_entry(mctx, ptr, size, file, line); \
}
static void
print_active(isc_mem_t *ctx, FILE *out);
#endif /* ISC_MEM_TRACKLINES */
#if ISC_MEM_TRACKLINES
/*!
* mctx must not be locked.
*/
static void
add_trace_entry(isc_mem_t *mctx, const void *ptr, size_t size FLARG) {
debuglink_t *dl = NULL;
uint32_t hash;
uint32_t idx;
/*
* "file" needs to be copied because it can be part of a dynamically
* loaded plugin which would be unloaded at the time the trace is
* dumped. Storing "file" pointer then leads to a dangling pointer
* dereference and a crash.
*/
size_t filelen = strlen(file) + 1;
size_t dlsize = STRUCT_FLEX_SIZE(dl, file, filelen);
MCTXLOCK(mctx);
if ((mctx->debugging & ISC_MEM_DEBUGTRACE) != 0) {
fprintf(stderr, "add %p size %zu file %s line %u mctx %p\n",
ptr, size, file, line, mctx);
}
if (mctx->debuglist == NULL) {
goto unlock;
}
#ifdef __COVERITY__
/*
* Use simple conversion from pointer to hash to avoid
* tainting 'ptr' due to byte swap in isc_hash32.
*/
hash = (uintptr_t)ptr >> 3;
#else
hash = isc_hash32(&ptr, sizeof(ptr), true);
#endif
idx = hash % DEBUG_TABLE_COUNT;
dl = mallocx(dlsize, mctx->jemalloc_flags);
INSIST(dl != NULL);
ISC_LINK_INIT(dl, link);
dl->ptr = ptr;
dl->size = size;
dl->line = line;
dl->dlsize = dlsize;
strlcpy((char *)dl->file, file, filelen);
ISC_LIST_PREPEND(mctx->debuglist[idx], dl, link);
mctx->debuglistcnt++;
unlock:
MCTXUNLOCK(mctx);
}
static void
delete_trace_entry(isc_mem_t *mctx, const void *ptr, size_t size,
const char *file, unsigned int line) {
debuglink_t *dl = NULL;
uint32_t hash;
uint32_t idx;
MCTXLOCK(mctx);
if ((mctx->debugging & ISC_MEM_DEBUGTRACE) != 0) {
fprintf(stderr, "del %p size %zu file %s line %u mctx %p\n",
ptr, size, file, line, mctx);
}
if (mctx->debuglist == NULL) {
goto unlock;
}
#ifdef __COVERITY__
/*
* Use simple conversion from pointer to hash to avoid
* tainting 'ptr' due to byte swap in isc_hash32.
*/
hash = (uintptr_t)ptr >> 3;
#else
hash = isc_hash32(&ptr, sizeof(ptr), true);
#endif
idx = hash % DEBUG_TABLE_COUNT;
dl = ISC_LIST_HEAD(mctx->debuglist[idx]);
while (dl != NULL) {
if (dl->ptr == ptr) {
ISC_LIST_UNLINK(mctx->debuglist[idx], dl, link);
sdallocx(dl, dl->dlsize, mctx->jemalloc_flags);
goto unlock;
}
dl = ISC_LIST_NEXT(dl, link);
}
/*
* If we get here, we didn't find the item on the list. We're
* screwed.
*/
UNREACHABLE();
unlock:
MCTXUNLOCK(mctx);
}
#endif /* ISC_MEM_TRACKLINES */
#define ADJUST_ZERO_ALLOCATION_SIZE(s) \
if (s == 0) { \
s = ZERO_ALLOCATION_SIZE; \
}
/*!
* Perform a malloc, doing memory filling and overrun detection as necessary.
*/
static void *
mem_get(isc_mem_t *ctx, size_t size, int flags) {
char *ret = NULL;
ADJUST_ZERO_ALLOCATION_SIZE(size);
ret = mallocx(size, flags | ctx->jemalloc_flags);
INSIST(ret != NULL);
if ((flags & ISC__MEM_ZERO) == 0 &&
(ctx->flags & ISC_MEMFLAG_FILL) != 0)
{
memset(ret, 0xbe, size); /* Mnemonic for "beef". */
}
return ret;
}
/*!
* Perform a free, doing memory filling and overrun detection as necessary.
*/
/* coverity[+free : arg-1] */
static void
mem_put(isc_mem_t *ctx, void *mem, size_t size, int flags) {
ADJUST_ZERO_ALLOCATION_SIZE(size);
if ((ctx->flags & ISC_MEMFLAG_FILL) != 0) {
memset(mem, 0xde, size); /* Mnemonic for "dead". */
}
sdallocx(mem, size, flags | ctx->jemalloc_flags);
}
static void *
mem_realloc(isc_mem_t *ctx, void *old_ptr, size_t old_size, size_t new_size,
int flags) {
void *new_ptr = NULL;
ADJUST_ZERO_ALLOCATION_SIZE(new_size);
new_ptr = rallocx(old_ptr, new_size, flags | ctx->jemalloc_flags);
INSIST(new_ptr != NULL);
if ((flags & ISC__MEM_ZERO) == 0 &&
(ctx->flags & ISC_MEMFLAG_FILL) != 0)
{
ssize_t diff_size = new_size - old_size;
void *diff_ptr = (uint8_t *)new_ptr + old_size;
if (diff_size > 0) {
/* Mnemonic for "beef". */
memset(diff_ptr, 0xbe, diff_size);
}
}
return new_ptr;
}
/*!
* Update internal counters after a memory get.
*/
static void
mem_getstats(isc_mem_t *ctx, size_t size) {
atomic_fetch_add_relaxed(&ctx->inuse, size);
}
/*!
* Update internal counters after a memory put.
*/
static void
mem_putstats(isc_mem_t *ctx, size_t size) {
atomic_size_t s = atomic_fetch_sub_relaxed(&ctx->inuse, size);
INSIST(s >= size);
}
/*
* Private.
*/
static bool
mem_jemalloc_arena_create(unsigned int *pnew_arenano) {
REQUIRE(pnew_arenano != NULL);
#ifdef JEMALLOC_API_SUPPORTED
unsigned int arenano = 0;
size_t len = sizeof(arenano);
int res = 0;
res = mallctl("arenas.create", &arenano, &len, NULL, 0);
if (res != 0) {
return false;
}
*pnew_arenano = arenano;
return true;
#else
*pnew_arenano = ISC_MEM_ILLEGAL_ARENA;
return true;
#endif /* JEMALLOC_API_SUPPORTED */
}
static bool
mem_jemalloc_arena_destroy(unsigned int arenano) {
#ifdef JEMALLOC_API_SUPPORTED
int res = 0;
char buf[256] = { 0 };
(void)snprintf(buf, sizeof(buf), "arena.%u.destroy", arenano);
res = mallctl(buf, NULL, NULL, NULL, 0);
if (res != 0) {
return false;
}
return true;
#else
UNUSED(arenano);
return true;
#endif /* JEMALLOC_API_SUPPORTED */
}
static void
mem_initialize(void) {
/*
* Check if the values copied from jemalloc still match
*/
#ifdef JEMALLOC_API_SUPPORTED
RUNTIME_CHECK(ISC__MEM_ZERO == MALLOCX_ZERO);
#endif /* JEMALLOC_API_SUPPORTED */
isc_mutex_init(&contextslock);
ISC_LIST_INIT(contexts);
}
void
isc__mem_initialize(void) {
isc_once_do(&init_once, mem_initialize);
}
static void
mem_shutdown(void) {
bool empty;
rcu_barrier();
isc__mem_checkdestroyed();
LOCK(&contextslock);
empty = ISC_LIST_EMPTY(contexts);
UNLOCK(&contextslock);
if (empty) {
isc_mutex_destroy(&contextslock);
}
}
void
isc__mem_shutdown(void) {
isc_once_do(&shut_once, mem_shutdown);
}
static void
mem_create(isc_mem_t **ctxp, unsigned int debugging, unsigned int flags,
unsigned int jemalloc_flags) {
isc_mem_t *ctx = NULL;
REQUIRE(ctxp != NULL && *ctxp == NULL);
ctx = mallocx(sizeof(*ctx), jemalloc_flags);
INSIST(ctx != NULL);
*ctx = (isc_mem_t){
.magic = MEM_MAGIC,
.debugging = debugging,
.flags = flags,
.jemalloc_flags = jemalloc_flags,
.jemalloc_arena = ISC_MEM_ILLEGAL_ARENA,
.checkfree = true,
};
isc_mutex_init(&ctx->lock);
isc_refcount_init(&ctx->references, 1);
atomic_init(&ctx->inuse, 0);
atomic_init(&ctx->hi_water, 0);
atomic_init(&ctx->lo_water, 0);
atomic_init(&ctx->hi_called, false);
atomic_init(&ctx->is_overmem, false);
ISC_LIST_INIT(ctx->pools);
#if ISC_MEM_TRACKLINES
if ((ctx->debugging & ISC_MEM_DEBUGRECORD) != 0) {
unsigned int i;
ctx->debuglist = mallocx(
ISC_CHECKED_MUL(DEBUG_TABLE_COUNT, sizeof(debuglist_t)),
jemalloc_flags);
INSIST(ctx->debuglist != NULL);
for (i = 0; i < DEBUG_TABLE_COUNT; i++) {
ISC_LIST_INIT(ctx->debuglist[i]);
}
}
#endif /* if ISC_MEM_TRACKLINES */
LOCK(&contextslock);
ISC_LIST_INITANDAPPEND(contexts, ctx, link);
UNLOCK(&contextslock);
*ctxp = ctx;
}
/*
* Public.
*/
static void
destroy(isc_mem_t *ctx) {
unsigned int arena_no;
LOCK(&contextslock);
ISC_LIST_UNLINK(contexts, ctx, link);
UNLOCK(&contextslock);
ctx->magic = 0;
arena_no = ctx->jemalloc_arena;
INSIST(ISC_LIST_EMPTY(ctx->pools));
#if ISC_MEM_TRACKLINES
if (ctx->debuglist != NULL) {
debuglink_t *dl;
for (size_t i = 0; i < DEBUG_TABLE_COUNT; i++) {
for (dl = ISC_LIST_HEAD(ctx->debuglist[i]); dl != NULL;
dl = ISC_LIST_HEAD(ctx->debuglist[i]))
{
if (ctx->checkfree && dl->ptr != NULL) {
print_active(ctx, stderr);
}
INSIST(!ctx->checkfree || dl->ptr == NULL);
ISC_LIST_UNLINK(ctx->debuglist[i], dl, link);
sdallocx(dl, sizeof(*dl), ctx->jemalloc_flags);
}
}
sdallocx(
ctx->debuglist,
ISC_CHECKED_MUL(DEBUG_TABLE_COUNT, sizeof(debuglist_t)),
ctx->jemalloc_flags);
}
#endif /* if ISC_MEM_TRACKLINES */
isc_mutex_destroy(&ctx->lock);
if (ctx->checkfree) {
INSIST(atomic_load(&ctx->inuse) == 0);
}
sdallocx(ctx, sizeof(*ctx), ctx->jemalloc_flags);
if (arena_no != ISC_MEM_ILLEGAL_ARENA) {
RUNTIME_CHECK(mem_jemalloc_arena_destroy(arena_no) == true);
}
}
void
isc_mem_attach(isc_mem_t *source, isc_mem_t **targetp) {
REQUIRE(VALID_CONTEXT(source));
REQUIRE(targetp != NULL && *targetp == NULL);
isc_refcount_increment(&source->references);
*targetp = source;
}
void
isc__mem_detach(isc_mem_t **ctxp FLARG) {
isc_mem_t *ctx = NULL;
REQUIRE(ctxp != NULL && VALID_CONTEXT(*ctxp));
ctx = *ctxp;
*ctxp = NULL;
if (isc_refcount_decrement(&ctx->references) == 1) {
isc_refcount_destroy(&ctx->references);
#if ISC_MEM_TRACKLINES
if ((ctx->debugging & ISC_MEM_DEBUGTRACE) != 0) {
fprintf(stderr, "destroy mctx %p file %s line %u\n",
ctx, file, line);
}
#endif
destroy(ctx);
}
}
/*
* isc_mem_putanddetach() is the equivalent of:
*
* mctx = NULL;
* isc_mem_attach(ptr->mctx, &mctx);
* isc_mem_detach(&ptr->mctx);
* isc_mem_put(mctx, ptr, sizeof(*ptr);
* isc_mem_detach(&mctx);
*/
void
isc__mem_putanddetach(isc_mem_t **ctxp, void *ptr, size_t size,
int flags FLARG) {
REQUIRE(ctxp != NULL && VALID_CONTEXT(*ctxp));
REQUIRE(ptr != NULL);
REQUIRE(size != 0);
isc_mem_t *ctx = *ctxp;
*ctxp = NULL;
isc__mem_put(ctx, ptr, size, flags FLARG_PASS);
isc__mem_detach(&ctx FLARG_PASS);
}
void
isc__mem_destroy(isc_mem_t **ctxp FLARG) {
isc_mem_t *ctx = NULL;
/*
* This routine provides legacy support for callers who use mctxs
* without attaching/detaching.
*/
REQUIRE(ctxp != NULL && VALID_CONTEXT(*ctxp));
ctx = *ctxp;
*ctxp = NULL;
rcu_barrier();
#if ISC_MEM_TRACKLINES
if ((ctx->debugging & ISC_MEM_DEBUGTRACE) != 0) {
fprintf(stderr, "destroy mctx %p file %s line %u\n", ctx, file,
line);
}
if (isc_refcount_decrement(&ctx->references) > 1) {
print_active(ctx, stderr);
}
#else /* if ISC_MEM_TRACKLINES */
isc_refcount_decrementz(&ctx->references);
#endif /* if ISC_MEM_TRACKLINES */
isc_refcount_destroy(&ctx->references);
destroy(ctx);
*ctxp = NULL;
}
void *
isc__mem_get(isc_mem_t *ctx, size_t size, int flags FLARG) {
void *ptr = NULL;
REQUIRE(VALID_CONTEXT(ctx));
ptr = mem_get(ctx, size, flags);
mem_getstats(ctx, size);
ADD_TRACE(ctx, ptr, size, file, line);
return ptr;
}
void
isc__mem_put(isc_mem_t *ctx, void *ptr, size_t size, int flags FLARG) {
REQUIRE(VALID_CONTEXT(ctx));
DELETE_TRACE(ctx, ptr, size, file, line);
mem_putstats(ctx, size);
mem_put(ctx, ptr, size, flags);
}
#if ISC_MEM_TRACKLINES
static void
print_active(isc_mem_t *mctx, FILE *out) {
if (mctx->debuglist != NULL) {
debuglink_t *dl;
unsigned int i;
bool found;
fprintf(out, "Dump of all outstanding memory "
"allocations:\n");
found = false;
for (i = 0; i < DEBUG_TABLE_COUNT; i++) {
dl = ISC_LIST_HEAD(mctx->debuglist[i]);
if (dl != NULL) {
found = true;
}
while (dl != NULL) {
if (dl->ptr != NULL) {
fprintf(out,
"\tptr %p size %zu "
"file %s "
"line %u\n",
dl->ptr, dl->size, dl->file,
dl->line);
}
dl = ISC_LIST_NEXT(dl, link);
}
}
if (!found) {
fprintf(out, "\tNone.\n");
}
}
}
#endif /* if ISC_MEM_TRACKLINES */
/*
* Print the stats[] on the stream "out" with suitable formatting.
*/
void
isc_mem_stats(isc_mem_t *ctx, FILE *out) {
isc_mempool_t *pool = NULL;
REQUIRE(VALID_CONTEXT(ctx));
MCTXLOCK(ctx);
/*
* Note that since a pool can be locked now, these stats might
* be somewhat off if the pool is in active use at the time the
* stats are dumped. The link fields are protected by the
* isc_mem_t's lock, however, so walking this list and
* extracting integers from stats fields is always safe.
*/
pool = ISC_LIST_HEAD(ctx->pools);
if (pool != NULL) {
fprintf(out, "[Pool statistics]\n");
fprintf(out, "%15s %10s %10s %10s %10s %10s %10s %1s\n", "name",
"size", "allocated", "freecount", "freemax",
"fillcount", "gets", "L");
}
while (pool != NULL) {
fprintf(out,
"%15s %10zu %10zu %10zu %10zu %10zu %10zu %10zu %s\n",
pool->name, pool->size, (size_t)0, pool->allocated,
pool->freecount, pool->freemax, pool->fillcount,
pool->gets, "N");
pool = ISC_LIST_NEXT(pool, link);
}
#if ISC_MEM_TRACKLINES
print_active(ctx, out);
#endif /* if ISC_MEM_TRACKLINES */
MCTXUNLOCK(ctx);
}
void *
isc__mem_allocate(isc_mem_t *ctx, size_t size, int flags FLARG) {
void *ptr = NULL;
REQUIRE(VALID_CONTEXT(ctx));
ptr = mem_get(ctx, size, flags);
/* Recalculate the real allocated size */
size = sallocx(ptr, flags | ctx->jemalloc_flags);
mem_getstats(ctx, size);
ADD_TRACE(ctx, ptr, size, file, line);
return ptr;
}
void *
isc__mem_reget(isc_mem_t *ctx, void *old_ptr, size_t old_size, size_t new_size,
int flags FLARG) {
void *new_ptr = NULL;
if (old_ptr == NULL) {
REQUIRE(old_size == 0);
new_ptr = isc__mem_get(ctx, new_size, flags FLARG_PASS);
} else if (new_size == 0) {
isc__mem_put(ctx, old_ptr, old_size, flags FLARG_PASS);
} else {
DELETE_TRACE(ctx, old_ptr, old_size, file, line);
mem_putstats(ctx, old_size);
new_ptr = mem_realloc(ctx, old_ptr, old_size, new_size, flags);
mem_getstats(ctx, new_size);
ADD_TRACE(ctx, new_ptr, new_size, file, line);
/*
* We want to postpone the call to water in edge case
* where the realloc will exactly hit on the boundary of
* the water and we would call water twice.
*/
}
return new_ptr;
}
void *
isc__mem_reallocate(isc_mem_t *ctx, void *old_ptr, size_t new_size,
int flags FLARG) {
void *new_ptr = NULL;
REQUIRE(VALID_CONTEXT(ctx));
if (old_ptr == NULL) {
new_ptr = isc__mem_allocate(ctx, new_size, flags FLARG_PASS);
} else if (new_size == 0) {
isc__mem_free(ctx, old_ptr, flags FLARG_PASS);
} else {
size_t old_size = sallocx(old_ptr, flags | ctx->jemalloc_flags);
DELETE_TRACE(ctx, old_ptr, old_size, file, line);
mem_putstats(ctx, old_size);
new_ptr = mem_realloc(ctx, old_ptr, old_size, new_size, flags);
/* Recalculate the real allocated size */
new_size = sallocx(new_ptr, flags | ctx->jemalloc_flags);
mem_getstats(ctx, new_size);
ADD_TRACE(ctx, new_ptr, new_size, file, line);
/*
* We want to postpone the call to water in edge case
* where the realloc will exactly hit on the boundary of
* the water and we would call water twice.
*/
}
return new_ptr;
}
void
isc__mem_free(isc_mem_t *ctx, void *ptr, int flags FLARG) {
size_t size = 0;
REQUIRE(VALID_CONTEXT(ctx));
REQUIRE(ptr != NULL);
size = sallocx(ptr, flags | ctx->jemalloc_flags);
DELETE_TRACE(ctx, ptr, size, file, line);
mem_putstats(ctx, size);
mem_put(ctx, ptr, size, flags);
}
/*
* Other useful things.
*/
char *
isc__mem_strdup(isc_mem_t *mctx, const char *s FLARG) {
size_t len;
char *ns = NULL;
REQUIRE(VALID_CONTEXT(mctx));
REQUIRE(s != NULL);
len = strlen(s) + 1;
ns = isc__mem_allocate(mctx, len, 0 FLARG_PASS);
strlcpy(ns, s, len);
return ns;
}
char *
isc__mem_strndup(isc_mem_t *mctx, const char *s, size_t size FLARG) {
size_t len;
char *ns = NULL;
REQUIRE(VALID_CONTEXT(mctx));
REQUIRE(s != NULL);
REQUIRE(size != 0);
len = strlen(s) + 1;
if (len > size) {
len = size;
}
ns = isc__mem_allocate(mctx, len, 0 FLARG_PASS);
strlcpy(ns, s, len);
return ns;
}
void
isc_mem_setdestroycheck(isc_mem_t *ctx, bool flag) {
REQUIRE(VALID_CONTEXT(ctx));
MCTXLOCK(ctx);
ctx->checkfree = flag;
MCTXUNLOCK(ctx);
}
size_t
isc_mem_inuse(isc_mem_t *ctx) {
REQUIRE(VALID_CONTEXT(ctx));
return atomic_load_relaxed(&ctx->inuse);
}
void
isc_mem_clearwater(isc_mem_t *mctx) {
isc_mem_setwater(mctx, 0, 0);
}
void
isc_mem_setwater(isc_mem_t *ctx, size_t hiwater, size_t lowater) {
REQUIRE(VALID_CONTEXT(ctx));
REQUIRE(hiwater >= lowater);
atomic_store_release(&ctx->hi_water, hiwater);
atomic_store_release(&ctx->lo_water, lowater);
return;
}
bool
isc_mem_isovermem(isc_mem_t *ctx) {
REQUIRE(VALID_CONTEXT(ctx));
bool is_overmem = atomic_load_relaxed(&ctx->is_overmem);
if (!is_overmem) {
/* We are not overmem, check whether we should be? */
size_t hiwater = atomic_load_relaxed(&ctx->hi_water);
if (hiwater == 0) {
return false;
}
size_t inuse = atomic_load_relaxed(&ctx->inuse);
if (inuse <= hiwater) {
return false;
}
if ((isc_mem_debugging & ISC_MEM_DEBUGUSAGE) != 0) {
fprintf(stderr,
"overmem mctx %p inuse %zu hi_water %zu\n", ctx,
inuse, hiwater);
}
atomic_store_relaxed(&ctx->is_overmem, true);
return true;
} else {
/* We are overmem, check whether we should not be? */
size_t lowater = atomic_load_relaxed(&ctx->lo_water);
if (lowater == 0) {
return false;
}
size_t inuse = atomic_load_relaxed(&ctx->inuse);
if (inuse >= lowater) {
return true;
}
if ((isc_mem_debugging & ISC_MEM_DEBUGUSAGE) != 0) {
fprintf(stderr,
"overmem mctx %p inuse %zu lo_water %zu\n", ctx,
inuse, lowater);
}
atomic_store_relaxed(&ctx->is_overmem, false);
return false;
}
}
void
isc_mem_setname(isc_mem_t *ctx, const char *name) {
REQUIRE(VALID_CONTEXT(ctx));
LOCK(&ctx->lock);
strlcpy(ctx->name, name, sizeof(ctx->name));
UNLOCK(&ctx->lock);
}
const char *
isc_mem_getname(isc_mem_t *ctx) {
REQUIRE(VALID_CONTEXT(ctx));
if (ctx->name[0] == 0) {
return "";
}
return ctx->name;
}
/*
* Memory pool stuff
*/
void
isc__mempool_create(isc_mem_t *restrict mctx, const size_t element_size,
isc_mempool_t **restrict mpctxp FLARG) {
isc_mempool_t *restrict mpctx = NULL;
size_t size = element_size;
REQUIRE(VALID_CONTEXT(mctx));
REQUIRE(size > 0U);
REQUIRE(mpctxp != NULL && *mpctxp == NULL);
/*
* Mempools are stored as a linked list of element.
*/
if (size < sizeof(element)) {
size = sizeof(element);
}
/*
* Allocate space for this pool, initialize values, and if all
* works well, attach to the memory context.
*/
mpctx = isc_mem_get(mctx, sizeof(isc_mempool_t));
*mpctx = (isc_mempool_t){
.size = size,
.freemax = 1,
.fillcount = 1,
};
#if ISC_MEM_TRACKLINES
if ((mctx->debugging & ISC_MEM_DEBUGTRACE) != 0) {
fprintf(stderr, "create pool %p file %s line %u mctx %p\n",
mpctx, file, line, mctx);
}
#endif /* ISC_MEM_TRACKLINES */
isc_mem_attach(mctx, &mpctx->mctx);
mpctx->magic = MEMPOOL_MAGIC;
*mpctxp = (isc_mempool_t *)mpctx;
MCTXLOCK(mctx);
ISC_LIST_INITANDAPPEND(mctx->pools, mpctx, link);
mctx->poolcnt++;
MCTXUNLOCK(mctx);
}
void
isc_mempool_setname(isc_mempool_t *restrict mpctx, const char *name) {
REQUIRE(VALID_MEMPOOL(mpctx));
REQUIRE(name != NULL);
strlcpy(mpctx->name, name, sizeof(mpctx->name));
}
void
isc__mempool_destroy(isc_mempool_t **restrict mpctxp FLARG) {
isc_mempool_t *restrict mpctx = NULL;
isc_mem_t *mctx = NULL;
element *restrict item = NULL;
REQUIRE(mpctxp != NULL);
REQUIRE(VALID_MEMPOOL(*mpctxp));
mpctx = *mpctxp;
*mpctxp = NULL;
mctx = mpctx->mctx;
#if ISC_MEM_TRACKLINES
if ((mctx->debugging & ISC_MEM_DEBUGTRACE) != 0) {
fprintf(stderr, "destroy pool %p file %s line %u mctx %p\n",
mpctx, file, line, mctx);
}
#endif
if (mpctx->allocated > 0) {
UNEXPECTED_ERROR("mempool %s leaked memory", mpctx->name);
}
REQUIRE(mpctx->allocated == 0);
/*
* Return any items on the free list
*/
while (mpctx->items != NULL) {
INSIST(mpctx->freecount > 0);
mpctx->freecount--;
item = mpctx->items;
mpctx->items = item->next;
mem_putstats(mctx, mpctx->size);
mem_put(mctx, item, mpctx->size, 0);
}
/*
* Remove our linked list entry from the memory context.
*/
MCTXLOCK(mctx);
ISC_LIST_UNLINK(mctx->pools, mpctx, link);
mctx->poolcnt--;
MCTXUNLOCK(mctx);
mpctx->magic = 0;
isc_mem_putanddetach(&mpctx->mctx, mpctx, sizeof(isc_mempool_t));
}
void *
isc__mempool_get(isc_mempool_t *restrict mpctx FLARG) {
element *restrict item = NULL;
REQUIRE(VALID_MEMPOOL(mpctx));
mpctx->allocated++;
if (mpctx->items == NULL) {
isc_mem_t *mctx = mpctx->mctx;
#if !__SANITIZE_ADDRESS__
const size_t fillcount = mpctx->fillcount;
#else
const size_t fillcount = 1;
#endif
/*
* We need to dip into the well. Fill up our free list.
*/
for (size_t i = 0; i < fillcount; i++) {
item = mem_get(mctx, mpctx->size, 0);
mem_getstats(mctx, mpctx->size);
item->next = mpctx->items;
mpctx->items = item;
mpctx->freecount++;
}
}
INSIST(mpctx->items != NULL);
item = mpctx->items;
mpctx->items = item->next;
INSIST(mpctx->freecount > 0);
mpctx->freecount--;
mpctx->gets++;
ADD_TRACE(mpctx->mctx, item, mpctx->size, file, line);
return item;
}
/* coverity[+free : arg-1] */
void
isc__mempool_put(isc_mempool_t *restrict mpctx, void *mem FLARG) {
element *restrict item = NULL;
REQUIRE(VALID_MEMPOOL(mpctx));
REQUIRE(mem != NULL);
isc_mem_t *mctx = mpctx->mctx;
const size_t freecount = mpctx->freecount;
#if !__SANITIZE_ADDRESS__
const size_t freemax = mpctx->freemax;
#else
const size_t freemax = 0;
#endif
INSIST(mpctx->allocated > 0);
mpctx->allocated--;
DELETE_TRACE(mctx, mem, mpctx->size, file, line);
/*
* If our free list is full, return this to the mctx directly.
*/
if (freecount >= freemax) {
mem_putstats(mctx, mpctx->size);
mem_put(mctx, mem, mpctx->size, 0);
return;
}
/*
* Otherwise, attach it to our free list and bump the counter.
*/
item = (element *)mem;
item->next = mpctx->items;
mpctx->items = item;
mpctx->freecount++;
}
/*
* Quotas
*/
void
isc_mempool_setfreemax(isc_mempool_t *restrict mpctx,
const unsigned int limit) {
REQUIRE(VALID_MEMPOOL(mpctx));
mpctx->freemax = limit;
}
unsigned int
isc_mempool_getfreemax(isc_mempool_t *restrict mpctx) {
REQUIRE(VALID_MEMPOOL(mpctx));
return mpctx->freemax;
}
unsigned int
isc_mempool_getfreecount(isc_mempool_t *restrict mpctx) {
REQUIRE(VALID_MEMPOOL(mpctx));
return mpctx->freecount;
}
unsigned int
isc_mempool_getallocated(isc_mempool_t *restrict mpctx) {
REQUIRE(VALID_MEMPOOL(mpctx));
return mpctx->allocated;
}
void
isc_mempool_setfillcount(isc_mempool_t *restrict mpctx,
unsigned int const limit) {
REQUIRE(VALID_MEMPOOL(mpctx));
REQUIRE(limit > 0);
mpctx->fillcount = limit;
}
unsigned int
isc_mempool_getfillcount(isc_mempool_t *restrict mpctx) {
REQUIRE(VALID_MEMPOOL(mpctx));
return mpctx->fillcount;
}
/*
* Requires contextslock to be held by caller.
*/
#if ISC_MEM_TRACKLINES
static void
print_contexts(FILE *file) {
isc_mem_t *ctx;
for (ctx = ISC_LIST_HEAD(contexts); ctx != NULL;
ctx = ISC_LIST_NEXT(ctx, link))
{
fprintf(file, "context: %p (%s): %" PRIuFAST32 " references\n",
ctx, ctx->name[0] == 0 ? "<unknown>" : ctx->name,
isc_refcount_current(&ctx->references));
print_active(ctx, file);
}
fflush(file);
}
#endif
static atomic_uintptr_t checkdestroyed = 0;
void
isc_mem_checkdestroyed(FILE *file) {
atomic_store_release(&checkdestroyed, (uintptr_t)file);
}
void
isc__mem_checkdestroyed(void) {
FILE *file = (FILE *)atomic_load_acquire(&checkdestroyed);
if (file == NULL) {
return;
}
LOCK(&contextslock);
if (!ISC_LIST_EMPTY(contexts)) {
#if ISC_MEM_TRACKLINES
if ((isc_mem_debugging & TRACE_OR_RECORD) != 0) {
print_contexts(file);
}
#endif /* if ISC_MEM_TRACKLINES */
UNREACHABLE();
}
UNLOCK(&contextslock);
}
unsigned int
isc_mem_references(isc_mem_t *ctx) {
return isc_refcount_current(&ctx->references);
}
#ifdef HAVE_LIBXML2
#define TRY0(a) \
do { \
xmlrc = (a); \
if (xmlrc < 0) \
goto error; \
} while (0)
static int
xml_renderctx(isc_mem_t *ctx, size_t *inuse, xmlTextWriterPtr writer) {
REQUIRE(VALID_CONTEXT(ctx));
int xmlrc;
MCTXLOCK(ctx);
TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "context"));
TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "id"));
TRY0(xmlTextWriterWriteFormatString(writer, "%p", ctx));
TRY0(xmlTextWriterEndElement(writer)); /* id */
if (ctx->name[0] != 0) {
TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "name"));
TRY0(xmlTextWriterWriteFormatString(writer, "%s", ctx->name));
TRY0(xmlTextWriterEndElement(writer)); /* name */
}
TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "references"));
TRY0(xmlTextWriterWriteFormatString(
writer, "%" PRIuFAST32,
isc_refcount_current(&ctx->references)));
TRY0(xmlTextWriterEndElement(writer)); /* references */
*inuse += isc_mem_inuse(ctx);
TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "inuse"));
TRY0(xmlTextWriterWriteFormatString(writer, "%" PRIu64 "",
(uint64_t)isc_mem_inuse(ctx)));
TRY0(xmlTextWriterEndElement(writer)); /* inuse */
TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "malloced"));
TRY0(xmlTextWriterWriteFormatString(writer, "%" PRIu64 "",
(uint64_t)isc_mem_inuse(ctx)));
TRY0(xmlTextWriterEndElement(writer)); /* malloced */
TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "pools"));
TRY0(xmlTextWriterWriteFormatString(writer, "%u", ctx->poolcnt));
TRY0(xmlTextWriterEndElement(writer)); /* pools */
TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "hiwater"));
TRY0(xmlTextWriterWriteFormatString(
writer, "%" PRIu64 "",
(uint64_t)atomic_load_relaxed(&ctx->hi_water)));
TRY0(xmlTextWriterEndElement(writer)); /* hiwater */
TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "lowater"));
TRY0(xmlTextWriterWriteFormatString(
writer, "%" PRIu64 "",
(uint64_t)atomic_load_relaxed(&ctx->lo_water)));
TRY0(xmlTextWriterEndElement(writer)); /* lowater */
TRY0(xmlTextWriterEndElement(writer)); /* context */
error:
MCTXUNLOCK(ctx);
return xmlrc;
}
int
isc_mem_renderxml(void *writer0) {
isc_mem_t *ctx;
size_t inuse = 0;
int xmlrc;
xmlTextWriterPtr writer = (xmlTextWriterPtr)writer0;
TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "contexts"));
LOCK(&contextslock);
for (ctx = ISC_LIST_HEAD(contexts); ctx != NULL;
ctx = ISC_LIST_NEXT(ctx, link))
{
xmlrc = xml_renderctx(ctx, &inuse, writer);
if (xmlrc < 0) {
UNLOCK(&contextslock);
goto error;
}
}
UNLOCK(&contextslock);
TRY0(xmlTextWriterEndElement(writer)); /* contexts */
TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "summary"));
TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "Malloced"));
TRY0(xmlTextWriterWriteFormatString(writer, "%" PRIu64 "",
(uint64_t)inuse));
TRY0(xmlTextWriterEndElement(writer)); /* malloced */
TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "InUse"));
TRY0(xmlTextWriterWriteFormatString(writer, "%" PRIu64 "",
(uint64_t)inuse));
TRY0(xmlTextWriterEndElement(writer)); /* InUse */
TRY0(xmlTextWriterEndElement(writer)); /* summary */
error:
return xmlrc;
}
#endif /* HAVE_LIBXML2 */
#ifdef HAVE_JSON_C
#define CHECKMEM(m) RUNTIME_CHECK(m != NULL)
static isc_result_t
json_renderctx(isc_mem_t *ctx, size_t *inuse, json_object *array) {
REQUIRE(VALID_CONTEXT(ctx));
REQUIRE(array != NULL);
json_object *ctxobj, *obj;
char buf[1024];
MCTXLOCK(ctx);
*inuse += isc_mem_inuse(ctx);
ctxobj = json_object_new_object();
CHECKMEM(ctxobj);
snprintf(buf, sizeof(buf), "%p", ctx);
obj = json_object_new_string(buf);
CHECKMEM(obj);
json_object_object_add(ctxobj, "id", obj);
if (ctx->name[0] != 0) {
obj = json_object_new_string(ctx->name);
CHECKMEM(obj);
json_object_object_add(ctxobj, "name", obj);
}
obj = json_object_new_int64(isc_refcount_current(&ctx->references));
CHECKMEM(obj);
json_object_object_add(ctxobj, "references", obj);
obj = json_object_new_int64(isc_mem_inuse(ctx));
CHECKMEM(obj);
json_object_object_add(ctxobj, "malloced", obj);
obj = json_object_new_int64(isc_mem_inuse(ctx));
CHECKMEM(obj);
json_object_object_add(ctxobj, "inuse", obj);
obj = json_object_new_int64(ctx->poolcnt);
CHECKMEM(obj);
json_object_object_add(ctxobj, "pools", obj);
obj = json_object_new_int64(atomic_load_relaxed(&ctx->hi_water));
CHECKMEM(obj);
json_object_object_add(ctxobj, "hiwater", obj);
obj = json_object_new_int64(atomic_load_relaxed(&ctx->lo_water));
CHECKMEM(obj);
json_object_object_add(ctxobj, "lowater", obj);
MCTXUNLOCK(ctx);
json_object_array_add(array, ctxobj);
return ISC_R_SUCCESS;
}
isc_result_t
isc_mem_renderjson(void *memobj0) {
isc_result_t result = ISC_R_SUCCESS;
isc_mem_t *ctx;
size_t inuse = 0;
json_object *ctxarray, *obj;
json_object *memobj = (json_object *)memobj0;
ctxarray = json_object_new_array();
CHECKMEM(ctxarray);
LOCK(&contextslock);
for (ctx = ISC_LIST_HEAD(contexts); ctx != NULL;
ctx = ISC_LIST_NEXT(ctx, link))
{
result = json_renderctx(ctx, &inuse, ctxarray);
if (result != ISC_R_SUCCESS) {
UNLOCK(&contextslock);
goto error;
}
}
UNLOCK(&contextslock);
obj = json_object_new_int64(inuse);
CHECKMEM(obj);
json_object_object_add(memobj, "InUse", obj);
obj = json_object_new_int64(inuse);
CHECKMEM(obj);
json_object_object_add(memobj, "Malloced", obj);
json_object_object_add(memobj, "contexts", ctxarray);
return ISC_R_SUCCESS;
error:
if (ctxarray != NULL) {
json_object_put(ctxarray);
}
return result;
}
#endif /* HAVE_JSON_C */
void
isc__mem_create(isc_mem_t **mctxp FLARG) {
mem_create(mctxp, isc_mem_debugging, isc_mem_defaultflags, 0);
#if ISC_MEM_TRACKLINES
if ((isc_mem_debugging & ISC_MEM_DEBUGTRACE) != 0) {
fprintf(stderr, "create mctx %p file %s line %u\n", *mctxp,
file, line);
}
#endif /* ISC_MEM_TRACKLINES */
}
void
isc__mem_create_arena(isc_mem_t **mctxp FLARG) {
unsigned int arena_no = ISC_MEM_ILLEGAL_ARENA;
RUNTIME_CHECK(mem_jemalloc_arena_create(&arena_no));
/*
* We use MALLOCX_TCACHE_NONE to bypass the tcache and route
* allocations directly to the arena. That is a recommendation
* from jemalloc developers:
*
* https://github.com/jemalloc/jemalloc/issues/2483#issuecomment-1698173849
*/
mem_create(mctxp, isc_mem_debugging, isc_mem_defaultflags,
arena_no == ISC_MEM_ILLEGAL_ARENA
? 0
: MALLOCX_ARENA(arena_no) | MALLOCX_TCACHE_NONE);
(*mctxp)->jemalloc_arena = arena_no;
#if ISC_MEM_TRACKLINES
if ((isc_mem_debugging & ISC_MEM_DEBUGTRACE) != 0) {
fprintf(stderr,
"create mctx %p file %s line %u for jemalloc arena "
"%u\n",
*mctxp, file, line, arena_no);
}
#endif /* ISC_MEM_TRACKLINES */
}
#ifdef JEMALLOC_API_SUPPORTED
static bool
jemalloc_set_ssize_value(const char *valname, ssize_t newval) {
int ret;
ret = mallctl(valname, NULL, NULL, &newval, sizeof(newval));
return ret == 0;
}
#endif /* JEMALLOC_API_SUPPORTED */
static isc_result_t
mem_set_arena_ssize_value(isc_mem_t *mctx, const char *arena_valname,
const ssize_t newval) {
REQUIRE(VALID_CONTEXT(mctx));
#ifdef JEMALLOC_API_SUPPORTED
bool ret;
char buf[256] = { 0 };
if (mctx->jemalloc_arena == ISC_MEM_ILLEGAL_ARENA) {
return ISC_R_UNEXPECTED;
}
(void)snprintf(buf, sizeof(buf), "arena.%u.%s", mctx->jemalloc_arena,
arena_valname);
ret = jemalloc_set_ssize_value(buf, newval);
if (!ret) {
return ISC_R_FAILURE;
}
return ISC_R_SUCCESS;
#else
UNUSED(arena_valname);
UNUSED(newval);
return ISC_R_NOTIMPLEMENTED;
#endif
}
isc_result_t
isc_mem_arena_set_muzzy_decay_ms(isc_mem_t *mctx, const ssize_t decay_ms) {
return mem_set_arena_ssize_value(mctx, "muzzy_decay_ms", decay_ms);
}
isc_result_t
isc_mem_arena_set_dirty_decay_ms(isc_mem_t *mctx, const ssize_t decay_ms) {
return mem_set_arena_ssize_value(mctx, "dirty_decay_ms", decay_ms);
}
void
isc__mem_printactive(isc_mem_t *ctx, FILE *file) {
#if ISC_MEM_TRACKLINES
REQUIRE(VALID_CONTEXT(ctx));
REQUIRE(file != NULL);
print_active(ctx, file);
#else /* if ISC_MEM_TRACKLINES */
UNUSED(ctx);
UNUSED(file);
#endif /* if ISC_MEM_TRACKLINES */
}