From 16a45837ca4321eaec9e665cc76539dcb2e2ef0e Mon Sep 17 00:00:00 2001 From: Artem Boldariev Date: Thu, 10 Aug 2023 17:02:43 +0300 Subject: [PATCH] Make it possible to create memory contexts backed by jemalloc arenas This commit extends the internal memory management middleware code in BIND so that memory contexts backed by dedicated jemalloc arenas can be created. A new function (isc_mem_create_arena()) is added for that. Moreover, it extends the existing code so that specialised memory contexts can be created easily, should we need that functionality for other future purposes. We have achieved that by passing the flags to the underlying jemalloc-related calls. See the above isc_mem_create_arena(), which can serve as an example of this. Having this opens up possibilities for creating memory contexts tuned for specific needs. (cherry picked from commit 8550c525881da757ac7db12990c2c1e0c72ec13c) --- lib/isc/include/isc/mem.h | 11 ++++ lib/isc/jemalloc_shim.h | 5 +- lib/isc/mem.c | 122 ++++++++++++++++++++++++++++++++------ 3 files changed, 120 insertions(+), 18 deletions(-) diff --git a/lib/isc/include/isc/mem.h b/lib/isc/include/isc/mem.h index 0121f560b9..72b965148f 100644 --- a/lib/isc/include/isc/mem.h +++ b/lib/isc/include/isc/mem.h @@ -194,6 +194,17 @@ void ISCMEMFUNC(create)(isc_mem_t **_ISC_MEM_FLARG); * mctxp != NULL && *mctxp == NULL */ /*@}*/ +#define isc_mem_create_arena(cp) isc__mem_create_arena((cp)_ISC_MEM_FILELINE) +void +isc__mem_create_arena(isc_mem_t **_ISC_MEM_FLARG); +/*!< + * \brief Create a memory context that routs all its operations to a dedicated + * jemalloc arena (when available). + * + * Requires: + * mctxp != NULL && *mctxp == NULL */ +/*@}*/ + /*@{*/ void isc_mem_attach(isc_mem_t *, isc_mem_t **); diff --git a/lib/isc/jemalloc_shim.h b/lib/isc/jemalloc_shim.h index 37e984ae1a..493bf5ffc0 100644 --- a/lib/isc/jemalloc_shim.h +++ b/lib/isc/jemalloc_shim.h @@ -22,7 +22,10 @@ const char *malloc_conf = NULL; /* Without jemalloc, isc_mem_get_align() is equal to isc_mem_get() */ -#define MALLOCX_ALIGN(a) (a & 0) /* Clear the flag */ +#define MALLOCX_ALIGN(a) (a & 0) /* Clear the flag */ +#define MALLOCX_ZERO ((int)0x40) +#define MALLOCX_TCACHE_NONE (0) +#define MALLOCX_ARENA(a) (0) #if defined(HAVE_MALLOC_SIZE) || defined(HAVE_MALLOC_USABLE_SIZE) diff --git a/lib/isc/mem.c b/lib/isc/mem.c index 2abb91c7a2..c2bd212474 100644 --- a/lib/isc/mem.c +++ b/lib/isc/mem.c @@ -53,6 +53,7 @@ #if JEMALLOC_VERSION_MAJOR < 4 #define sdallocx(ptr, size, flags) dallocx(ptr, flags) +#define MALLOCX_TCACHE_NONE (0) #endif /* JEMALLOC_VERSION_MAJOR < 4 */ #else @@ -70,6 +71,8 @@ unsigned int isc_mem_debugging = ISC_MEM_DEBUGGING; unsigned int isc_mem_defaultflags = ISC_MEMFLAG_DEFAULT; +#define ISC_MEM_ILLEGAL_ARENA (UINT_MAX) + /* * Constants. */ @@ -133,6 +136,8 @@ static uint64_t totallost; struct isc_mem { unsigned int magic; unsigned int flags; + unsigned int jemalloc_flags; + unsigned int jemalloc_arena; isc_mutex_t lock; bool checkfree; struct stats stats[STATS_BUCKETS + 1]; @@ -260,9 +265,9 @@ add_trace_entry(isc_mem_t *mctx, const void *ptr, size_t size FLARG) { #endif idx = hash % DEBUG_TABLE_COUNT; - dl = mallocx(sizeof(debuglink_t), 0); + dl = mallocx(sizeof(*dl), mctx->jemalloc_flags); INSIST(dl != NULL); - increment_malloced(mctx, sizeof(debuglink_t)); + increment_malloced(mctx, sizeof(*dl)); ISC_LINK_INIT(dl, link); dl->ptr = ptr; @@ -310,7 +315,7 @@ delete_trace_entry(isc_mem_t *mctx, const void *ptr, size_t size, if (dl->ptr == ptr) { ISC_LIST_UNLINK(mctx->debuglist[idx], dl, link); decrement_malloced(mctx, sizeof(*dl)); - sdallocx(dl, sizeof(*dl), 0); + sdallocx(dl, sizeof(*dl), mctx->jemalloc_flags); goto unlock; } dl = ISC_LIST_NEXT(dl, link); @@ -342,7 +347,7 @@ mem_get(isc_mem_t *ctx, size_t size, int flags) { ADJUST_ZERO_ALLOCATION_SIZE(size); - ret = mallocx(size, flags); + ret = mallocx(size, flags | ctx->jemalloc_flags); INSIST(ret != NULL); if ((ctx->flags & ISC_MEMFLAG_FILL) != 0) { @@ -363,7 +368,7 @@ mem_put(isc_mem_t *ctx, void *mem, size_t size, int flags) { if ((ctx->flags & ISC_MEMFLAG_FILL) != 0) { memset(mem, 0xde, size); /* Mnemonic for "dead". */ } - sdallocx(mem, size, flags); + sdallocx(mem, size, flags | ctx->jemalloc_flags); } static void * @@ -373,7 +378,7 @@ mem_realloc(isc_mem_t *ctx, void *old_ptr, size_t old_size, size_t new_size, ADJUST_ZERO_ALLOCATION_SIZE(new_size); - new_ptr = rallocx(old_ptr, new_size, flags); + new_ptr = rallocx(old_ptr, new_size, flags | ctx->jemalloc_flags); INSIST(new_ptr != NULL); if ((ctx->flags & ISC_MEMFLAG_FILL) != 0) { @@ -432,6 +437,48 @@ mem_putstats(isc_mem_t *ctx, void *ptr, size_t size) { * Private. */ +static bool +mem_jemalloc_arena_create(unsigned int *pnew_arenano) { + REQUIRE(pnew_arenano != NULL); + +#if defined(JEMALLOC_API_SUPPORTED) && JEMALLOC_VERSION_MAJOR >= 4 + unsigned int arenano = 0; + size_t len = sizeof(arenano); + int res = 0; + + res = mallctl("arenas.create", &arenano, &len, NULL, 0); + if (res != 0) { + return (false); + } + + *pnew_arenano = arenano; + + return (true); +#else + *pnew_arenano = ISC_MEM_ILLEGAL_ARENA; + return (true); +#endif /* defined(JEMALLOC_API_SUPPORTED) && JEMALLOC_VERSION_MAJOR >= 4 */ +} + +static bool +mem_jemalloc_arena_destroy(unsigned int arenano) { +#if defined(JEMALLOC_API_SUPPORTED) && JEMALLOC_VERSION_MAJOR >= 4 + int res = 0; + char buf[256] = { 0 }; + + (void)snprintf(buf, sizeof(buf), "arena.%u.destroy", arenano); + res = mallctl(buf, NULL, NULL, NULL, 0); + if (res != 0) { + return (false); + } + + return (true); +#else + UNUSED(arenano); + return (true); +#endif /* defined(JEMALLOC_API_SUPPORTED) && JEMALLOC_VERSION_MAJOR >= 4 */ +} + static void mem_initialize(void) { isc_mutex_init(&contextslock); @@ -457,17 +504,20 @@ isc__mem_shutdown(void) { } static void -mem_create(isc_mem_t **ctxp, unsigned int flags) { +mem_create(isc_mem_t **ctxp, unsigned int flags, unsigned int jemalloc_flags) { isc_mem_t *ctx = NULL; REQUIRE(ctxp != NULL && *ctxp == NULL); - ctx = mallocx(sizeof(*ctx), MALLOCX_ALIGN(isc_os_cacheline())); + ctx = mallocx(sizeof(*ctx), + MALLOCX_ALIGN(isc_os_cacheline()) | jemalloc_flags); INSIST(ctx != NULL); *ctx = (isc_mem_t){ .magic = MEM_MAGIC, .flags = flags, + .jemalloc_flags = jemalloc_flags, + .jemalloc_arena = ISC_MEM_ILLEGAL_ARENA, .checkfree = true, }; @@ -495,7 +545,8 @@ mem_create(isc_mem_t **ctxp, unsigned int flags) { unsigned int i; ctx->debuglist = - mallocx((DEBUG_TABLE_COUNT * sizeof(debuglist_t)), 0); + mallocx((DEBUG_TABLE_COUNT * sizeof(debuglist_t)), + ctx->jemalloc_flags); INSIST(ctx->debuglist != NULL); for (i = 0; i < DEBUG_TABLE_COUNT; i++) { @@ -521,6 +572,7 @@ static void destroy(isc_mem_t *ctx) { unsigned int i; size_t malloced; + unsigned int arena_no; LOCK(&contextslock); ISC_LIST_UNLINK(contexts, ctx, link); @@ -529,6 +581,8 @@ destroy(isc_mem_t *ctx) { ctx->magic = 0; + arena_no = ctx->jemalloc_arena; + INSIST(ISC_LIST_EMPTY(ctx->pools)); #if ISC_MEM_TRACKLINES @@ -544,13 +598,14 @@ destroy(isc_mem_t *ctx) { INSIST(!ctx->checkfree || dl->ptr == NULL); ISC_LIST_UNLINK(ctx->debuglist[i], dl, link); - sdallocx(dl, sizeof(*dl), 0); + sdallocx(dl, sizeof(*dl), ctx->jemalloc_flags); decrement_malloced(ctx, sizeof(*dl)); } } sdallocx(ctx->debuglist, - (DEBUG_TABLE_COUNT * sizeof(debuglist_t)), 0); + (DEBUG_TABLE_COUNT * sizeof(debuglist_t)), + ctx->jemalloc_flags); decrement_malloced(ctx, DEBUG_TABLE_COUNT * sizeof(debuglist_t)); } @@ -581,7 +636,12 @@ destroy(isc_mem_t *ctx) { if (ctx->checkfree) { INSIST(malloced == 0); } - sdallocx(ctx, sizeof(*ctx), MALLOCX_ALIGN(isc_os_cacheline())); + sdallocx(ctx, sizeof(*ctx), + MALLOCX_ALIGN(isc_os_cacheline()) | ctx->jemalloc_flags); + + if (arena_no != ISC_MEM_ILLEGAL_ARENA) { + RUNTIME_CHECK(mem_jemalloc_arena_destroy(arena_no) == true); + } } void @@ -897,7 +957,7 @@ isc__mem_allocate(isc_mem_t *ctx, size_t size FLARG) { ptr = mem_get(ctx, size, 0); /* Recalculate the real allocated size */ - size = sallocx(ptr, 0); + size = sallocx(ptr, ctx->jemalloc_flags); mem_getstats(ctx, size); ADD_TRACE(ctx, ptr, size, file, line); @@ -950,7 +1010,7 @@ isc__mem_reallocate(isc_mem_t *ctx, void *old_ptr, size_t new_size FLARG) { } else if (new_size == 0) { isc__mem_free(ctx, old_ptr FLARG_PASS); } else { - size_t old_size = sallocx(old_ptr, 0); + size_t old_size = sallocx(old_ptr, ctx->jemalloc_flags); DELETE_TRACE(ctx, old_ptr, old_size, file, line); mem_putstats(ctx, old_ptr, old_size); @@ -958,7 +1018,7 @@ isc__mem_reallocate(isc_mem_t *ctx, void *old_ptr, size_t new_size FLARG) { new_ptr = mem_realloc(ctx, old_ptr, old_size, new_size, 0); /* Recalculate the real allocated size */ - new_size = sallocx(new_ptr, 0); + new_size = sallocx(new_ptr, ctx->jemalloc_flags); mem_getstats(ctx, new_size); ADD_TRACE(ctx, new_ptr, new_size, file, line); @@ -981,7 +1041,7 @@ isc__mem_free(isc_mem_t *ctx, void *ptr FLARG) { REQUIRE(VALID_CONTEXT(ctx)); - size = sallocx(ptr, 0); + size = sallocx(ptr, ctx->jemalloc_flags); DELETE_TRACE(ctx, ptr, size, file, line); @@ -1748,7 +1808,7 @@ error: void isc__mem_create(isc_mem_t **mctxp FLARG) { - mem_create(mctxp, isc_mem_defaultflags); + mem_create(mctxp, isc_mem_defaultflags, 0); #if ISC_MEM_TRACKLINES if ((isc_mem_debugging & ISC_MEM_DEBUGTRACE) != 0) { fprintf(stderr, "create mctx %p file %s line %u\n", *mctxp, @@ -1757,6 +1817,34 @@ isc__mem_create(isc_mem_t **mctxp FLARG) { #endif /* ISC_MEM_TRACKLINES */ } +void +isc__mem_create_arena(isc_mem_t **mctxp FLARG) { + unsigned int arena_no = ISC_MEM_ILLEGAL_ARENA; + + RUNTIME_CHECK(mem_jemalloc_arena_create(&arena_no)); + + /* + * We use MALLOCX_TCACHE_NONE to bypass the tcache and route + * allocations directly to the arena. That is a recommendation + * from jemalloc developers: + * + * https://github.com/jemalloc/jemalloc/issues/2483#issuecomment-1698173849 + */ + mem_create(mctxp, isc_mem_defaultflags, + arena_no == ISC_MEM_ILLEGAL_ARENA + ? 0 + : MALLOCX_ARENA(arena_no) | MALLOCX_TCACHE_NONE); + (*mctxp)->jemalloc_arena = arena_no; +#if ISC_MEM_TRACKLINES + if ((isc_mem_debugging & ISC_MEM_DEBUGTRACE) != 0) { + fprintf(stderr, + "create mctx %p file %s line %u for jemalloc arena " + "%u\n", + *mctxp, file, line, arena_no); + } +#endif /* ISC_MEM_TRACKLINES */ +} + void isc__mem_printactive(isc_mem_t *ctx, FILE *file) { #if ISC_MEM_TRACKLINES