diff --git a/lib/isc/include/isc/mem.h b/lib/isc/include/isc/mem.h index 0121f560b9..72b965148f 100644 --- a/lib/isc/include/isc/mem.h +++ b/lib/isc/include/isc/mem.h @@ -194,6 +194,17 @@ void ISCMEMFUNC(create)(isc_mem_t **_ISC_MEM_FLARG); * mctxp != NULL && *mctxp == NULL */ /*@}*/ +#define isc_mem_create_arena(cp) isc__mem_create_arena((cp)_ISC_MEM_FILELINE) +void +isc__mem_create_arena(isc_mem_t **_ISC_MEM_FLARG); +/*!< + * \brief Create a memory context that routs all its operations to a dedicated + * jemalloc arena (when available). + * + * Requires: + * mctxp != NULL && *mctxp == NULL */ +/*@}*/ + /*@{*/ void isc_mem_attach(isc_mem_t *, isc_mem_t **); diff --git a/lib/isc/jemalloc_shim.h b/lib/isc/jemalloc_shim.h index 37e984ae1a..493bf5ffc0 100644 --- a/lib/isc/jemalloc_shim.h +++ b/lib/isc/jemalloc_shim.h @@ -22,7 +22,10 @@ const char *malloc_conf = NULL; /* Without jemalloc, isc_mem_get_align() is equal to isc_mem_get() */ -#define MALLOCX_ALIGN(a) (a & 0) /* Clear the flag */ +#define MALLOCX_ALIGN(a) (a & 0) /* Clear the flag */ +#define MALLOCX_ZERO ((int)0x40) +#define MALLOCX_TCACHE_NONE (0) +#define MALLOCX_ARENA(a) (0) #if defined(HAVE_MALLOC_SIZE) || defined(HAVE_MALLOC_USABLE_SIZE) diff --git a/lib/isc/mem.c b/lib/isc/mem.c index 2abb91c7a2..c2bd212474 100644 --- a/lib/isc/mem.c +++ b/lib/isc/mem.c @@ -53,6 +53,7 @@ #if JEMALLOC_VERSION_MAJOR < 4 #define sdallocx(ptr, size, flags) dallocx(ptr, flags) +#define MALLOCX_TCACHE_NONE (0) #endif /* JEMALLOC_VERSION_MAJOR < 4 */ #else @@ -70,6 +71,8 @@ unsigned int isc_mem_debugging = ISC_MEM_DEBUGGING; unsigned int isc_mem_defaultflags = ISC_MEMFLAG_DEFAULT; +#define ISC_MEM_ILLEGAL_ARENA (UINT_MAX) + /* * Constants. */ @@ -133,6 +136,8 @@ static uint64_t totallost; struct isc_mem { unsigned int magic; unsigned int flags; + unsigned int jemalloc_flags; + unsigned int jemalloc_arena; isc_mutex_t lock; bool checkfree; struct stats stats[STATS_BUCKETS + 1]; @@ -260,9 +265,9 @@ add_trace_entry(isc_mem_t *mctx, const void *ptr, size_t size FLARG) { #endif idx = hash % DEBUG_TABLE_COUNT; - dl = mallocx(sizeof(debuglink_t), 0); + dl = mallocx(sizeof(*dl), mctx->jemalloc_flags); INSIST(dl != NULL); - increment_malloced(mctx, sizeof(debuglink_t)); + increment_malloced(mctx, sizeof(*dl)); ISC_LINK_INIT(dl, link); dl->ptr = ptr; @@ -310,7 +315,7 @@ delete_trace_entry(isc_mem_t *mctx, const void *ptr, size_t size, if (dl->ptr == ptr) { ISC_LIST_UNLINK(mctx->debuglist[idx], dl, link); decrement_malloced(mctx, sizeof(*dl)); - sdallocx(dl, sizeof(*dl), 0); + sdallocx(dl, sizeof(*dl), mctx->jemalloc_flags); goto unlock; } dl = ISC_LIST_NEXT(dl, link); @@ -342,7 +347,7 @@ mem_get(isc_mem_t *ctx, size_t size, int flags) { ADJUST_ZERO_ALLOCATION_SIZE(size); - ret = mallocx(size, flags); + ret = mallocx(size, flags | ctx->jemalloc_flags); INSIST(ret != NULL); if ((ctx->flags & ISC_MEMFLAG_FILL) != 0) { @@ -363,7 +368,7 @@ mem_put(isc_mem_t *ctx, void *mem, size_t size, int flags) { if ((ctx->flags & ISC_MEMFLAG_FILL) != 0) { memset(mem, 0xde, size); /* Mnemonic for "dead". */ } - sdallocx(mem, size, flags); + sdallocx(mem, size, flags | ctx->jemalloc_flags); } static void * @@ -373,7 +378,7 @@ mem_realloc(isc_mem_t *ctx, void *old_ptr, size_t old_size, size_t new_size, ADJUST_ZERO_ALLOCATION_SIZE(new_size); - new_ptr = rallocx(old_ptr, new_size, flags); + new_ptr = rallocx(old_ptr, new_size, flags | ctx->jemalloc_flags); INSIST(new_ptr != NULL); if ((ctx->flags & ISC_MEMFLAG_FILL) != 0) { @@ -432,6 +437,48 @@ mem_putstats(isc_mem_t *ctx, void *ptr, size_t size) { * Private. */ +static bool +mem_jemalloc_arena_create(unsigned int *pnew_arenano) { + REQUIRE(pnew_arenano != NULL); + +#if defined(JEMALLOC_API_SUPPORTED) && JEMALLOC_VERSION_MAJOR >= 4 + unsigned int arenano = 0; + size_t len = sizeof(arenano); + int res = 0; + + res = mallctl("arenas.create", &arenano, &len, NULL, 0); + if (res != 0) { + return (false); + } + + *pnew_arenano = arenano; + + return (true); +#else + *pnew_arenano = ISC_MEM_ILLEGAL_ARENA; + return (true); +#endif /* defined(JEMALLOC_API_SUPPORTED) && JEMALLOC_VERSION_MAJOR >= 4 */ +} + +static bool +mem_jemalloc_arena_destroy(unsigned int arenano) { +#if defined(JEMALLOC_API_SUPPORTED) && JEMALLOC_VERSION_MAJOR >= 4 + int res = 0; + char buf[256] = { 0 }; + + (void)snprintf(buf, sizeof(buf), "arena.%u.destroy", arenano); + res = mallctl(buf, NULL, NULL, NULL, 0); + if (res != 0) { + return (false); + } + + return (true); +#else + UNUSED(arenano); + return (true); +#endif /* defined(JEMALLOC_API_SUPPORTED) && JEMALLOC_VERSION_MAJOR >= 4 */ +} + static void mem_initialize(void) { isc_mutex_init(&contextslock); @@ -457,17 +504,20 @@ isc__mem_shutdown(void) { } static void -mem_create(isc_mem_t **ctxp, unsigned int flags) { +mem_create(isc_mem_t **ctxp, unsigned int flags, unsigned int jemalloc_flags) { isc_mem_t *ctx = NULL; REQUIRE(ctxp != NULL && *ctxp == NULL); - ctx = mallocx(sizeof(*ctx), MALLOCX_ALIGN(isc_os_cacheline())); + ctx = mallocx(sizeof(*ctx), + MALLOCX_ALIGN(isc_os_cacheline()) | jemalloc_flags); INSIST(ctx != NULL); *ctx = (isc_mem_t){ .magic = MEM_MAGIC, .flags = flags, + .jemalloc_flags = jemalloc_flags, + .jemalloc_arena = ISC_MEM_ILLEGAL_ARENA, .checkfree = true, }; @@ -495,7 +545,8 @@ mem_create(isc_mem_t **ctxp, unsigned int flags) { unsigned int i; ctx->debuglist = - mallocx((DEBUG_TABLE_COUNT * sizeof(debuglist_t)), 0); + mallocx((DEBUG_TABLE_COUNT * sizeof(debuglist_t)), + ctx->jemalloc_flags); INSIST(ctx->debuglist != NULL); for (i = 0; i < DEBUG_TABLE_COUNT; i++) { @@ -521,6 +572,7 @@ static void destroy(isc_mem_t *ctx) { unsigned int i; size_t malloced; + unsigned int arena_no; LOCK(&contextslock); ISC_LIST_UNLINK(contexts, ctx, link); @@ -529,6 +581,8 @@ destroy(isc_mem_t *ctx) { ctx->magic = 0; + arena_no = ctx->jemalloc_arena; + INSIST(ISC_LIST_EMPTY(ctx->pools)); #if ISC_MEM_TRACKLINES @@ -544,13 +598,14 @@ destroy(isc_mem_t *ctx) { INSIST(!ctx->checkfree || dl->ptr == NULL); ISC_LIST_UNLINK(ctx->debuglist[i], dl, link); - sdallocx(dl, sizeof(*dl), 0); + sdallocx(dl, sizeof(*dl), ctx->jemalloc_flags); decrement_malloced(ctx, sizeof(*dl)); } } sdallocx(ctx->debuglist, - (DEBUG_TABLE_COUNT * sizeof(debuglist_t)), 0); + (DEBUG_TABLE_COUNT * sizeof(debuglist_t)), + ctx->jemalloc_flags); decrement_malloced(ctx, DEBUG_TABLE_COUNT * sizeof(debuglist_t)); } @@ -581,7 +636,12 @@ destroy(isc_mem_t *ctx) { if (ctx->checkfree) { INSIST(malloced == 0); } - sdallocx(ctx, sizeof(*ctx), MALLOCX_ALIGN(isc_os_cacheline())); + sdallocx(ctx, sizeof(*ctx), + MALLOCX_ALIGN(isc_os_cacheline()) | ctx->jemalloc_flags); + + if (arena_no != ISC_MEM_ILLEGAL_ARENA) { + RUNTIME_CHECK(mem_jemalloc_arena_destroy(arena_no) == true); + } } void @@ -897,7 +957,7 @@ isc__mem_allocate(isc_mem_t *ctx, size_t size FLARG) { ptr = mem_get(ctx, size, 0); /* Recalculate the real allocated size */ - size = sallocx(ptr, 0); + size = sallocx(ptr, ctx->jemalloc_flags); mem_getstats(ctx, size); ADD_TRACE(ctx, ptr, size, file, line); @@ -950,7 +1010,7 @@ isc__mem_reallocate(isc_mem_t *ctx, void *old_ptr, size_t new_size FLARG) { } else if (new_size == 0) { isc__mem_free(ctx, old_ptr FLARG_PASS); } else { - size_t old_size = sallocx(old_ptr, 0); + size_t old_size = sallocx(old_ptr, ctx->jemalloc_flags); DELETE_TRACE(ctx, old_ptr, old_size, file, line); mem_putstats(ctx, old_ptr, old_size); @@ -958,7 +1018,7 @@ isc__mem_reallocate(isc_mem_t *ctx, void *old_ptr, size_t new_size FLARG) { new_ptr = mem_realloc(ctx, old_ptr, old_size, new_size, 0); /* Recalculate the real allocated size */ - new_size = sallocx(new_ptr, 0); + new_size = sallocx(new_ptr, ctx->jemalloc_flags); mem_getstats(ctx, new_size); ADD_TRACE(ctx, new_ptr, new_size, file, line); @@ -981,7 +1041,7 @@ isc__mem_free(isc_mem_t *ctx, void *ptr FLARG) { REQUIRE(VALID_CONTEXT(ctx)); - size = sallocx(ptr, 0); + size = sallocx(ptr, ctx->jemalloc_flags); DELETE_TRACE(ctx, ptr, size, file, line); @@ -1748,7 +1808,7 @@ error: void isc__mem_create(isc_mem_t **mctxp FLARG) { - mem_create(mctxp, isc_mem_defaultflags); + mem_create(mctxp, isc_mem_defaultflags, 0); #if ISC_MEM_TRACKLINES if ((isc_mem_debugging & ISC_MEM_DEBUGTRACE) != 0) { fprintf(stderr, "create mctx %p file %s line %u\n", *mctxp, @@ -1757,6 +1817,34 @@ isc__mem_create(isc_mem_t **mctxp FLARG) { #endif /* ISC_MEM_TRACKLINES */ } +void +isc__mem_create_arena(isc_mem_t **mctxp FLARG) { + unsigned int arena_no = ISC_MEM_ILLEGAL_ARENA; + + RUNTIME_CHECK(mem_jemalloc_arena_create(&arena_no)); + + /* + * We use MALLOCX_TCACHE_NONE to bypass the tcache and route + * allocations directly to the arena. That is a recommendation + * from jemalloc developers: + * + * https://github.com/jemalloc/jemalloc/issues/2483#issuecomment-1698173849 + */ + mem_create(mctxp, isc_mem_defaultflags, + arena_no == ISC_MEM_ILLEGAL_ARENA + ? 0 + : MALLOCX_ARENA(arena_no) | MALLOCX_TCACHE_NONE); + (*mctxp)->jemalloc_arena = arena_no; +#if ISC_MEM_TRACKLINES + if ((isc_mem_debugging & ISC_MEM_DEBUGTRACE) != 0) { + fprintf(stderr, + "create mctx %p file %s line %u for jemalloc arena " + "%u\n", + *mctxp, file, line, arena_no); + } +#endif /* ISC_MEM_TRACKLINES */ +} + void isc__mem_printactive(isc_mem_t *ctx, FILE *file) { #if ISC_MEM_TRACKLINES