From 642b2e0852f713a89556193dfbe2fe954b21fd1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Sur=C3=BD?= Date: Wed, 15 Jul 2020 10:47:49 +0200 Subject: [PATCH] WIP: Needs split Part 1: Move the constructor/destructor to lib/isc/lib.c as there's a bug in Apple Clang that prevents the priority in __attribute__((constructor|destructor)) to work properly across the modules, e.g. all constructors and destructors needs to be listed in a single compilation unit. Part 2: Add proper support for compiling with different memory allocators, initially, we have these memory allocators available: 1. the current custom BIND 9 allocator, which is default 2. the system library memory allocator 3. jemalloc memory allocator 4. tcmalloc memory allocator from gperftools The work isn't 100% complete, as the non-default memory allocators are missing the accounting, so the memory limits don't work at the moment. The other suboptimal part is allocating chunks for the mempool, basically the mempool should be fixed separately, as allocating the memory as individual chunks is very suboptimal, it's slow and it leads to memory fragmentation. --- bin/named/main.c | 1 - configure.ac | 25 +++ lib/isc/Makefile.am | 2 + lib/isc/buffer.c | 16 +- lib/isc/include/isc/mem.h | 7 + lib/isc/include/isc/util.h | 10 +- lib/isc/lib.c | 72 ++++++ lib/isc/mem.c | 438 ++++++++++++++++++++++++++----------- lib/isc/tests/isctest.c | 8 +- lib/isc/tests/mem_test.c | 52 ++++- 10 files changed, 470 insertions(+), 161 deletions(-) diff --git a/bin/named/main.c b/bin/named/main.c index 6fa0a80b59..f5e5c37845 100644 --- a/bin/named/main.c +++ b/bin/named/main.c @@ -1614,7 +1614,6 @@ main(int argc, char *argv[]) { } } isc_mem_destroy(&named_g_mctx); - isc_mem_checkdestroyed(stderr); named_main_setmemstats(NULL); diff --git a/configure.ac b/configure.ac index 1ff23d73c1..c143182b0a 100644 --- a/configure.ac +++ b/configure.ac @@ -1580,6 +1580,31 @@ test -z "$with_dlz_stub" && with_dlz_stub=no AC_CHECK_HEADERS([glob.h]) +# +# Support for constructor and destructor attributes +# +AX_GCC_FUNC_ATTRIBUTE([constructor]) +AX_GCC_FUNC_ATTRIBUTE([destructor]) + +# +# Which memory allocator we should use? +# +AC_ARG_WITH([allocator], + [AS_HELP_STRING([--with-allocator=], + [Use different memory allocator (default=custom)])], + [], + [with_allocator=custom]) + +AS_CASE([$with_allocator], + [system], [AC_DEFINE([USE_ALLOCATOR_SYSTEM], [1], [Define if you want to use system memory allocator])], + [jemalloc], [PKG_CHECK_MODULES([ALLOCATOR], [jemalloc], + [AC_DEFINE([USE_ALLOCATOR_JEMALLOC], [1], [Define if you want to use jemalloc allocator])])], + [tcmalloc*], [PKG_CHECK_MODULES([ALLOCATOR], [lib$with_allocator], + [AC_DEFINE([USE_ALLOCATOR_TCMALLOC], [1], [Define if you want to use tcmalloc allocator])])] + [:]) + +AC_MSG_NOTICE([Using $with_allocator memory allocator]) + # # Files to configure. These are listed here because we used to # specify them as arguments to AC_OUTPUT. diff --git a/lib/isc/Makefile.am b/lib/isc/Makefile.am index 48ead7c17f..006bbb0bc0 100644 --- a/lib/isc/Makefile.am +++ b/lib/isc/Makefile.am @@ -226,6 +226,7 @@ libisc_la_SOURCES = \ libisc_la_CPPFLAGS = \ $(AM_CPPFLAGS) \ + $(ALLOCATOR_CFLAGS) \ $(LIBISC_CFLAGS) \ $(LIBUV_CFLAGS) \ $(OPENSSL_CFLAGS) \ @@ -236,6 +237,7 @@ libisc_la_LDFLAGS = \ -release "$(PACKAGE_VERSION)" libisc_la_LIBADD = \ + $(ALLOCATOR_LIBS) \ $(LIBUV_LIBS) \ $(OPENSSL_LIBS) \ $(ZLIB_LIBS) diff --git a/lib/isc/buffer.c b/lib/isc/buffer.c index 1db0eb026e..3d751f24ca 100644 --- a/lib/isc/buffer.c +++ b/lib/isc/buffer.c @@ -553,8 +553,7 @@ isc_buffer_allocate(isc_mem_t *mctx, isc_buffer_t **dynbuffer, isc_result_t isc_buffer_reserve(isc_buffer_t **dynbuffer, unsigned int size) { - unsigned char *bdata; - uint64_t len; + size_t len; REQUIRE(dynbuffer != NULL); REQUIRE(ISC_BUFFER_VALID(*dynbuffer)); @@ -581,18 +580,9 @@ isc_buffer_reserve(isc_buffer_t **dynbuffer, unsigned int size) { return (ISC_R_NOMEMORY); } - /* - * XXXMUKS: This is far more expensive than plain realloc() as - * it doesn't remap pages, but does ordinary copy. So is - * isc_mem_reallocate(), which has additional issues. - */ - bdata = isc_mem_get((*dynbuffer)->mctx, (unsigned int)len); + (*dynbuffer)->base = isc_mem_reallocate((*dynbuffer)->mctx, + (*dynbuffer)->base, len); - memmove(bdata, (*dynbuffer)->base, (*dynbuffer)->length); - isc_mem_put((*dynbuffer)->mctx, (*dynbuffer)->base, - (*dynbuffer)->length); - - (*dynbuffer)->base = bdata; (*dynbuffer)->length = (unsigned int)len; return (ISC_R_SUCCESS); diff --git a/lib/isc/include/isc/mem.h b/lib/isc/include/isc/mem.h index b4dc148c32..1c96a508f1 100644 --- a/lib/isc/include/isc/mem.h +++ b/lib/isc/include/isc/mem.h @@ -32,9 +32,13 @@ typedef void (*isc_mem_water_t)(void *, int); * Define ISC_MEM_TRACKLINES=1 to turn on detailed tracing of memory * allocation and freeing by file and line number. */ +#if !defined(USE_ALLOCATOR_SYSTEM) && !defined(USE_ALLOCATOR_JEMALLOC) && \ + !defined(USE_ALLOCATOR_TCMALLOC) #ifndef ISC_MEM_TRACKLINES #define ISC_MEM_TRACKLINES 1 #endif /* ifndef ISC_MEM_TRACKLINES */ +#endif /* !defined(USE_ALLOCATOR_SYSTEM) && !defined(USE_ALLOCATOR_JEMALLOC) \ + && !defined(USE_ALLOCATOR_TCMALLOC) */ /*% * Define ISC_MEM_CHECKOVERRUN=1 to turn on checks for using memory outside @@ -600,6 +604,9 @@ isc__strdup(const char *str _ISC_MEM_FLARG); char * isc__strndup(const char *str _ISC_MEM_FLARG); +isc_mem_t * +isc_get_default_mctx(void); + ISC_LANG_ENDDECLS #endif /* ISC_MEM_H */ diff --git a/lib/isc/include/isc/util.h b/lib/isc/include/isc/util.h index b88cc51aff..f8b4b1a5bd 100644 --- a/lib/isc/include/isc/util.h +++ b/lib/isc/include/isc/util.h @@ -48,12 +48,14 @@ #define ISC_NONSTRING #endif /* __GNUC__ */ -#if __GNUC__ +#if HAVE_FUNC_ATTRIBUTE_CONSTRUCTOR && HAVE_FUNC_ATTRIBUTE_DESTRUCTOR #define ISC_CONSTRUCTOR(priority) __attribute__((constructor(priority))) #define ISC_DESTRUCTOR(priority) __attribute__((destructor(priority))) -#else +#elif WIN32 #define ISC_CONSTRUCTOR(priority) #define ISC_DESTRUCTOR(priority) +#else +#error Either __attribute__((constructor|destructor))__ or DllMain support needed to compile BIND 9. #endif /*% @@ -208,6 +210,10 @@ #define __has_feature(x) 0 #endif /* if !defined(__has_feature) */ +#if !defined(__has_builtin) +#define __has_builtin(x) 0 +#endif /* if !defined(__has_attribute) */ + /* GCC defines __SANITIZE_ADDRESS__, so reuse the macro for clang */ #if __has_feature(address_sanitizer) #define __SANITIZE_ADDRESS__ 1 diff --git a/lib/isc/lib.c b/lib/isc/lib.c index 8bee4c21a4..9ff02cbd73 100644 --- a/lib/isc/lib.c +++ b/lib/isc/lib.c @@ -11,8 +11,12 @@ /*! \file */ +#include + #include #include +#include +#include /*** *** Functions @@ -22,3 +26,71 @@ void isc_lib_register(void) { isc_bind9 = false; } + +LIBISC_EXTERNAL_DATA extern isc_mem_t *isc__mem_mctx; + +void +isc__mem_initialize(void) ISC_CONSTRUCTOR(101); +void +isc__mem_shutdown(void) ISC_DESTRUCTOR(101); + +void +isc__mem_initialize(void) { + REQUIRE(isc__mem_mctx == NULL); + isc_mem_create(&isc__mem_mctx); + isc_mem_setname(isc__mem_mctx, "default"); +} + +void +isc__mem_shutdown(void) { + REQUIRE(isc__mem_mctx != NULL); + isc_mem_destroy(&isc__mem_mctx); + isc_mem_checkdestroyed(stderr); +} + +void +isc__nm_initialize(void) ISC_CONSTRUCTOR(102); +void +isc__nm_shutdown(void) ISC_DESTRUCTOR(102); + +#if UV_VERSION_MAJOR > 1 || (UV_VERSION_MAJOR == 1 && UV_VERSION_MINOR >= 38) + +static void * +_malloc(size_t size) { + return (isc_malloc(size)); +} + +static void * +_calloc(size_t num, size_t size) { + return (isc_calloc(num, size)); +} + +static void * +_realloc(void *ptr, size_t size) { + return (isc_realloc(ptr, size)); +} + +static void +_free(void *ptr) { + return (isc_free(ptr)); +} + +void +isc__nm_initialize(void) { + uv_replace_allocator(_malloc, _realloc, _calloc, _free); +} + +void +isc__nm_shutdown(void) { + uv_library_shutdown(); +} + +#else + +void +isc__nm_initialize(void) {} + +void +isc__nm_shutdown(void) {} + +#endif diff --git a/lib/isc/mem.c b/lib/isc/mem.c index cf12155952..269c15c577 100644 --- a/lib/isc/mem.c +++ b/lib/isc/mem.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -31,6 +32,14 @@ #include #include +#if USE_ALLOCATOR_TCMALLOC +#include +#endif + +#if USE_ALLOCATOR_JEMALLOC +#include +#endif + #ifdef HAVE_LIBXML2 #include #define ISC_XMLCHAR (const xmlChar *) @@ -57,7 +66,7 @@ LIBISC_EXTERNAL_DATA unsigned int isc_mem_defaultflags = ISC_MEMFLAG_DEFAULT; #define DEF_MAX_SIZE 1100 #define DEF_MEM_TARGET 4096 -#define ALIGNMENT_SIZE 8U /*%< must be a power of 2 */ +#define ALIGNMENT_SIZE sizeof(void *) /*%< must be a power of 2 */ #define NUM_BASIC_BLOCKS 64 /*%< must be > 1 */ #define TABLE_INCREMENT 1024 #define DEBUG_TABLE_COUNT 512U @@ -201,6 +210,7 @@ struct isc__mempool { #if ISC_MEMPOOL_NAMES char name[16]; /*%< printed name in stats reports */ #endif /* if ISC_MEMPOOL_NAMES */ + element *pages; }; /* @@ -325,14 +335,6 @@ delete_trace_entry(isc__mem_t *mctx, const void *ptr, size_t size, } #endif /* ISC_MEM_TRACKLINES */ -static inline size_t -rmsize(size_t size) { - /* - * round down to ALIGNMENT_SIZE - */ - return (size & (~(ALIGNMENT_SIZE - 1))); -} - static inline size_t quantize(size_t size) { /*! @@ -347,6 +349,15 @@ quantize(size_t size) { return ((size + ALIGNMENT_SIZE - 1) & (~(ALIGNMENT_SIZE - 1))); } +#if USE_ALLOCATOR_CUSTOM +static inline size_t +rmsize(size_t size) { + /* + * round down to ALIGNMENT_SIZE + */ + return (size & (~(ALIGNMENT_SIZE - 1))); +} + static inline void more_basic_blocks(isc__mem_t *ctx) { void *tmp; @@ -679,6 +690,7 @@ mem_putstats(isc__mem_t *ctx, void *ptr, size_t size) { #endif /* if ISC_MEM_CHECKOVERRUN */ ctx->malloced -= size; } +#endif /* * Private. @@ -700,8 +712,7 @@ default_memalloc(size_t size) { * * [ISO9899] * ISO/IEC WG 9899:2011: Programming languages - C. - * International Organization for Standardization, Geneva, - * Switzerland. + * International Organization for Standardization, Geneva, CH. * http://www.open-std.org/JTC1/SC22/WG14/www/docs/n1570.pdf */ @@ -912,7 +923,6 @@ isc_mem_attach(isc_mem_t *source0, isc_mem_t **targetp) { void isc_mem_detach(isc_mem_t **ctxp) { REQUIRE(ctxp != NULL && VALID_CONTEXT(*ctxp)); - isc__mem_t *ctx = (isc__mem_t *)*ctxp; *ctxp = NULL; @@ -940,6 +950,9 @@ isc__mem_putanddetach(isc_mem_t **ctxp, void *ptr, size_t size FLARG) { isc__mem_t *ctx = (isc__mem_t *)*ctxp; *ctxp = NULL; +#if USE_ALLOCATOR_CUSTOM + isc__mem_t *ctx = (isc__mem_t *)*ctxp; + if (ISC_UNLIKELY((isc_mem_debugging & (ISC_MEM_DEBUGSIZE | ISC_MEM_DEBUGCTX)) != 0)) { @@ -953,22 +966,30 @@ isc__mem_putanddetach(isc_mem_t **ctxp, void *ptr, size_t size FLARG) { } isc__mem_free((isc_mem_t *)ctx, ptr FLARG_PASS); - goto destroy; - } - - MCTXLOCK(ctx); - - DELETE_TRACE(ctx, ptr, size, file, line); - - if ((ctx->flags & ISC_MEMFLAG_INTERNAL) != 0) { - mem_putunlocked(ctx, ptr, size); } else { - mem_putstats(ctx, ptr, size); - mem_put(ctx, ptr, size); - } - MCTXUNLOCK(ctx); + MCTXLOCK(ctx); -destroy: + DELETE_TRACE(ctx, ptr, size, file, line); + + if ((ctx->flags & ISC_MEMFLAG_INTERNAL) != 0) { + mem_putunlocked(ctx, ptr, size); + } else { + mem_putstats(ctx, ptr, size); + mem_put(ctx, ptr, size); + } + MCTXUNLOCK(ctx); + } +#elif defined(USE_ALLOCATOR_JEMALLOC) + if (size == 0) { + size = ALIGNMENT_SIZE; + } + sdallocx(ptr, size, 0); +#elif defined(USE_ALLOCATOR_TCMALLOC) + tc_free_sized(ptr, size); +#else + UNUSED(size); + default_memfree(ptr); +#endif if (isc_refcount_decrement(&ctx->references) == 1) { isc_refcount_destroy(&ctx->references); destroy(ctx); @@ -1002,9 +1023,10 @@ isc_mem_destroy(isc_mem_t **ctxp) { void * isc__mem_get(isc_mem_t *ctx0, size_t size FLARG) { REQUIRE(VALID_CONTEXT(ctx0)); + void *ptr = NULL; +#if USE_ALLOCATOR_CUSTOM isc__mem_t *ctx = (isc__mem_t *)ctx0; - void *ptr; bool call_water = false; if (ISC_UNLIKELY((isc_mem_debugging & @@ -1023,7 +1045,6 @@ isc__mem_get(isc_mem_t *ctx0, size_t size FLARG) { mem_getstats(ctx, size); } } - ADD_TRACE(ctx, ptr, size, file, line); if (ctx->hi_water != 0U && ctx->inuse > ctx->hi_water) { @@ -1046,7 +1067,29 @@ isc__mem_get(isc_mem_t *ctx0, size_t size FLARG) { if (call_water && (ctx->water != NULL)) { (ctx->water)(ctx->water_arg, ISC_MEM_HIWATER); } +#elif USE_ALLOCATOR_JEMALLOC + if (size == 0) { + size = ALIGNMENT_SIZE; + } + ptr = mallocx(size, MALLOCX_ALIGN(ALIGNMENT_SIZE)); + if (ptr == NULL && size != 0) { + char strbuf[ISC_STRERRORSIZE]; + strerror_r(errno, strbuf, sizeof(strbuf)); + isc_error_fatal(__FILE__, __LINE__, + "mallocx(%zu, 0) failed: %s", size, strbuf); + } +#elif defined(USE_ALLOCATOR_TCMALLOC) + ptr = tc_malloc(size); + if (ptr == NULL && size != 0) { + char strbuf[ISC_STRERRORSIZE]; + strerror_r(errno, strbuf, sizeof(strbuf)); + isc_error_fatal(__FILE__, __LINE__, "tc_malloc(%zu) failed: %s", + size, strbuf); + } +#elif defined(USE_ALLOCATOR_SYSTEM) + ptr = default_memalloc(size); +#endif return (ptr); } @@ -1055,6 +1098,7 @@ isc__mem_put(isc_mem_t *ctx0, void *ptr, size_t size FLARG) { REQUIRE(VALID_CONTEXT(ctx0)); REQUIRE(ptr != NULL); +#if USE_ALLOCATOR_CUSTOM isc__mem_t *ctx = (isc__mem_t *)ctx0; bool call_water = false; size_info *si; @@ -1103,6 +1147,17 @@ isc__mem_put(isc_mem_t *ctx0, void *ptr, size_t size FLARG) { if (call_water && (ctx->water != NULL)) { (ctx->water)(ctx->water_arg, ISC_MEM_LOWATER); } +#elif defined(USE_ALLOCATOR_JEMALLOC) + if (size == 0) { + size = ALIGNMENT_SIZE; + } + sdallocx(ptr, size, 0); +#elif defined(USE_ALLOCATOR_TCMALLOC) + tc_free_sized(ptr, size); +#else + UNUSED(size); + default_memfree(ptr); +#endif } void @@ -1223,6 +1278,7 @@ isc_mem_stats(isc_mem_t *ctx0, FILE *out) { MCTXUNLOCK(ctx); } +#if USE_ALLOCATOR_CUSTOM /* * Replacements for malloc() and free() -- they implicitly remember the * size of the object allocated (with some additional overhead). @@ -1251,11 +1307,14 @@ mem_allocateunlocked(isc_mem_t *ctx0, size_t size) { si->u.size = size; return (&si[1]); } +#endif void * isc__mem_allocate(isc_mem_t *ctx0, size_t size FLARG) { REQUIRE(VALID_CONTEXT(ctx0)); + void *ptr = NULL; +#if USE_ALLOCATOR_CUSTOM isc__mem_t *ctx = (isc__mem_t *)ctx0; size_info *si; bool call_water = false; @@ -1293,17 +1352,86 @@ isc__mem_allocate(isc_mem_t *ctx0, size_t size FLARG) { (ctx->water)(ctx->water_arg, ISC_MEM_HIWATER); } - return (si); + ptr = si; +#elif USE_ALLOCATOR_JEMALLOC + if (size == 0U) { + size = ALIGNMENT_SIZE; + } + ptr = mallocx(size, MALLOCX_ALIGN(ALIGNMENT_SIZE)); + if (ptr == NULL && size != 0) { + char strbuf[ISC_STRERRORSIZE]; + strerror_r(errno, strbuf, sizeof(strbuf)); + isc_error_fatal(__FILE__, __LINE__, + "mallocx(%zu, 0) failed: %s", size, strbuf); + } +#elif USE_ALLOCATOR_TCMALLOC + ptr = tc_malloc(size); + if (ptr == NULL && size != 0) { + char strbuf[ISC_STRERRORSIZE]; + strerror_r(errno, strbuf, sizeof(strbuf)); + isc_error_fatal(__FILE__, __LINE__, "tc_malloc(%zu) failed: %s", + size, strbuf); + } +#else + ptr = default_memalloc(size); +#endif + return (ptr); } void * isc__mem_callocate(isc_mem_t *ctx0, size_t num, size_t size FLARG) { + void *ptr = NULL; +#if USE_ALLOCATOR_CUSTOM size_t numsize; ISC_MUL_OVERFLOW(num, size, &numsize); - void *ptr = isc__mem_allocate(ctx0, numsize FLARG_PASS); + ptr = isc__mem_allocate(ctx0, numsize FLARG_PASS); memset(ptr, 0, numsize); return (ptr); +#elif USE_ALLOCATOR_JEMALLOC + UNUSED(ctx0); + + size_t numsize; + + ISC_MUL_OVERFLOW(num, size, &numsize); + + if (numsize == 0U) { + numsize = ALIGNMENT_SIZE; + } + + ptr = mallocx(numsize, MALLOCX_ZERO | MALLOCX_ALIGN(ALIGNMENT_SIZE)); + if (ptr == NULL && size != 0) { + char strbuf[ISC_STRERRORSIZE]; + strerror_r(errno, strbuf, sizeof(strbuf)); + isc_error_fatal(__FILE__, __LINE__, + "mallocx(%zu, MALLOCX_ZERO) failed: %s", + numsize, strbuf); + } + +#elif USE_ALLOCATOR_TCMALLOC + UNUSED(ctx0); + + ptr = tc_calloc(num, size); + if (ptr == NULL && size != 0) { + char strbuf[ISC_STRERRORSIZE]; + strerror_r(errno, strbuf, sizeof(strbuf)); + isc_error_fatal(__FILE__, __LINE__, + "tc_calloc(%zu, %zu) failed: %s", num, size, + strbuf); + } +#else + UNUSED(ctx0); + + ptr = calloc(num, size); + + if (ptr == NULL && size != 0) { + char strbuf[ISC_STRERRORSIZE]; + strerror_r(errno, strbuf, sizeof(strbuf)); + isc_error_fatal(__FILE__, __LINE__, "calloc failed: %s", + strbuf); + } +#endif + return (ptr); } void * @@ -1319,8 +1447,9 @@ isc__mem_cget(isc_mem_t *ctx0, size_t num, size_t size FLARG) { void * isc__mem_reallocate(isc_mem_t *ctx0, void *ptr, size_t size FLARG) { REQUIRE(VALID_CONTEXT(ctx0)); - void *new_ptr = NULL; + +#if USE_ALLOCATOR_CUSTOM size_t oldsize, copysize; /* @@ -1353,6 +1482,51 @@ isc__mem_reallocate(isc_mem_t *ctx0, void *ptr, size_t size FLARG) { isc__mem_free(ctx0, ptr FLARG_PASS); } + return (new_ptr); +#elif USE_ALLOCATOR_JEMALLOC + if (size > 0U) { + if (ptr == NULL) { + new_ptr = mallocx(size, MALLOCX_ALIGN(ALIGNMENT_SIZE)); + } else { + new_ptr = rallocx(ptr, size, MALLOCX_ALIGN(ALIGNMENT_SIZE)); + } + if (new_ptr == NULL && size != 0) { + char strbuf[ISC_STRERRORSIZE]; + strerror_r(errno, strbuf, sizeof(strbuf)); + isc_error_fatal(__FILE__, __LINE__, + "rallocx(%p, %zu, 0) failed: %s", ptr, + size, strbuf); + } + } else if (ptr != NULL) { + dallocx(ptr, 0); + } +#elif USE_ALLOCATOR_TCMALLOC + if (size > 0U) { + new_ptr = tc_realloc(ptr, size); + if (new_ptr == NULL && size != 0) { + char strbuf[ISC_STRERRORSIZE]; + strerror_r(errno, strbuf, sizeof(strbuf)); + isc_error_fatal(__FILE__, __LINE__, + "tc_realloc(%p, %zu) failed: %s", ptr, size, + strbuf); + } + } else if (ptr != NULL) { + tc_free(ptr); + } +#else + if (size > 0U) { + new_ptr = realloc(ptr, size); + if (new_ptr == NULL && size != 0) { + char strbuf[ISC_STRERRORSIZE]; + strerror_r(errno, strbuf, sizeof(strbuf)); + isc_error_fatal(__FILE__, __LINE__, + "realloc(%p, %zu) failed: %s", ptr, size, + strbuf); + } + } else { + free(ptr); + } +#endif return (new_ptr); } @@ -1361,6 +1535,7 @@ isc__mem_free(isc_mem_t *ctx0, void *ptr FLARG) { REQUIRE(VALID_CONTEXT(ctx0)); REQUIRE(ptr != NULL); +#if USE_ALLOCATOR_CUSTOM isc__mem_t *ctx = (isc__mem_t *)ctx0; size_info *si; size_t size; @@ -1409,6 +1584,13 @@ isc__mem_free(isc_mem_t *ctx0, void *ptr FLARG) { if (call_water) { (ctx->water)(ctx->water_arg, ISC_MEM_LOWATER); } +#elif USE_ALLOCATOR_JEMALLOC + dallocx(ptr, 0); +#elif USE_ALLOCATOR_TCMALLOC + tc_free(ptr); +#else + free(ptr); +#endif } /* @@ -1475,8 +1657,10 @@ size_t isc_mem_inuse(isc_mem_t *ctx0) { REQUIRE(VALID_CONTEXT(ctx0)); + size_t inuse = 0; + +#if USE_ALLOCATOR_CUSTOM isc__mem_t *ctx = (isc__mem_t *)ctx0; - size_t inuse; MCTXLOCK(ctx); @@ -1484,6 +1668,10 @@ isc_mem_inuse(isc_mem_t *ctx0) { MCTXUNLOCK(ctx); +#elif USE_ALLOCATOR_JEMALLOC + size_t len = sizeof(inuse); + mallctl("stats.active", &inuse, &len, NULL, 0); +#endif return (inuse); } @@ -1507,14 +1695,19 @@ size_t isc_mem_total(isc_mem_t *ctx0) { REQUIRE(VALID_CONTEXT(ctx0)); + size_t total = 0; +#if USE_ALLOCATOR_CUSTOM isc__mem_t *ctx = (isc__mem_t *)ctx0; - size_t total; MCTXLOCK(ctx); total = ctx->total; MCTXUNLOCK(ctx); +#elif USE_ALLOCATOR_JEMALLOC + size_t len = sizeof(total); + mallctl("stats.allocated", &total, &len, NULL, 0); +#endif return (total); } @@ -1608,6 +1801,7 @@ isc_mempool_create(isc_mem_t *mctx0, size_t size, isc_mempool_t **mpctxp) { isc__mem_t *mctx = (isc__mem_t *)mctx0; isc__mempool_t *mpctx; + size_t pagesize = sysconf(_SC_PAGESIZE); /* * Allocate space for this pool, initialize values, and if all works @@ -1625,17 +1819,19 @@ isc_mempool_create(isc_mem_t *mctx0, size_t size, isc_mempool_t **mpctxp) { if (size < sizeof(element)) { size = sizeof(element); } + size = quantize(size); mpctx->size = size; mpctx->maxalloc = UINT_MAX; mpctx->allocated = 0; mpctx->freecount = 0; - mpctx->freemax = 1; - mpctx->fillcount = 1; + mpctx->freemax = UINT_MAX; + mpctx->fillcount = pagesize / size; mpctx->gets = 0; #if ISC_MEMPOOL_NAMES mpctx->name[0] = 0; #endif /* if ISC_MEMPOOL_NAMES */ mpctx->items = NULL; + mpctx->pages = NULL; *mpctxp = (isc_mempool_t *)mpctx; @@ -1676,7 +1872,6 @@ isc_mempool_destroy(isc_mempool_t **mpctxp) { isc__mempool_t *mpctx; isc__mem_t *mctx; isc_mutex_t *lock; - element *item; mpctx = (isc__mempool_t *)*mpctxp; #if ISC_MEMPOOL_NAMES @@ -1697,11 +1892,13 @@ isc_mempool_destroy(isc_mempool_t **mpctxp) { LOCK(lock); } +#if USE_ALLOCATOR_CUSTOM /* * Return any items on the free list */ MCTXLOCK(mctx); while (mpctx->items != NULL) { + element *item; INSIST(mpctx->freecount > 0); mpctx->freecount--; item = mpctx->items; @@ -1715,7 +1912,15 @@ isc_mempool_destroy(isc_mempool_t **mpctxp) { } } MCTXUNLOCK(mctx); - +#else + while (mpctx->pages != NULL) { + element *page; + page = mpctx->pages; + mpctx->pages = page->next; + isc_mem_put((isc_mem_t *)mpctx->mctx, page, mpctx->fillcount * mpctx->size); + mpctx->freecount -= mpctx->fillcount - 1; + } +#endif /* * Remove our linked list entry from the memory context. */ @@ -1754,10 +1959,6 @@ isc__mempool_get(isc_mempool_t *mpctx0 FLARG) { isc__mempool_t *mpctx = (isc__mempool_t *)mpctx0; element *item; - isc__mem_t *mctx; - unsigned int i; - - mctx = mpctx->mctx; if (mpctx->lock != NULL) { LOCK(mpctx->lock); @@ -1771,13 +1972,15 @@ isc__mempool_get(isc_mempool_t *mpctx0 FLARG) { goto out; } +#if USE_ALLOCATOR_CUSTOM + isc__mem_t *mctx = mpctx->mctx; if (ISC_UNLIKELY(mpctx->items == NULL)) { /* * We need to dip into the well. Lock the memory context * here and fill up our free list. */ MCTXLOCK(mctx); - for (i = 0; i < mpctx->fillcount; i++) { + for (size_t i = 0; i < mpctx->fillcount; i++) { if ((mctx->flags & ISC_MEMFLAG_INTERNAL) != 0) { item = mem_getunlocked(mctx, mpctx->size); } else { @@ -1796,6 +1999,21 @@ isc__mempool_get(isc_mempool_t *mpctx0 FLARG) { MCTXUNLOCK(mctx); } +#else + element *page = NULL; + if (ISC_UNLIKELY(mpctx->items == NULL)) { + page = isc__mem_get((isc_mem_t *)mpctx->mctx, mpctx->fillcount * mpctx->size FLARG_PASS); + page->next = mpctx->pages; + mpctx->pages = page; + item = (element *)((uintptr_t)page + mpctx->size); + for (size_t i = 1; i < mpctx->fillcount; i++) { + item->next = mpctx->items; + mpctx->items = item; + item = (element *)((uintptr_t)item + mpctx->size); + } + mpctx->freecount += mpctx->fillcount - 1; + } +#endif /* * If we didn't get any items, return NULL. */ @@ -1834,8 +2052,6 @@ isc__mempool_put(isc_mempool_t *mpctx0, void *mem FLARG) { REQUIRE(mem != NULL); isc__mempool_t *mpctx = (isc__mempool_t *)mpctx0; - isc__mem_t *mctx = mpctx->mctx; - element *item; if (mpctx->lock != NULL) { LOCK(mpctx->lock); @@ -1846,16 +2062,18 @@ isc__mempool_put(isc_mempool_t *mpctx0, void *mem FLARG) { #if ISC_MEM_TRACKLINES if (ISC_UNLIKELY((isc_mem_debugging & TRACE_OR_RECORD) != 0)) { - MCTXLOCK(mctx); - DELETE_TRACE(mctx, mem, mpctx->size, file, line); - MCTXUNLOCK(mctx); + MCTXLOCK(mpctx->mctx); + DELETE_TRACE(mpctx->mctx, mem, mpctx->size, file, line); + MCTXUNLOCK(mpctx->mctx); } #endif /* ISC_MEM_TRACKLINES */ +#if USE_ALLOCATOR_CUSTOM /* * If our free list is full, return this to the mctx directly. */ if (mpctx->freecount >= mpctx->freemax) { + isc__mem_t *mctx = mpctx->mctx; MCTXLOCK(mctx); if ((mctx->flags & ISC_MEMFLAG_INTERNAL) != 0) { mem_putunlocked(mctx, mem, mpctx->size); @@ -1869,7 +2087,8 @@ isc__mempool_put(isc_mempool_t *mpctx0, void *mem FLARG) { } return; } - +#endif + element *item; /* * Otherwise, attach it to our free list and bump the counter. */ @@ -1897,7 +2116,11 @@ isc_mempool_setfreemax(isc_mempool_t *mpctx0, unsigned int limit) { LOCK(mpctx->lock); } - mpctx->freemax = limit; + if (limit < mpctx->maxalloc) { + mpctx->freemax = mpctx->maxalloc; + } else { + mpctx->freemax = limit; + } if (mpctx->lock != NULL) { UNLOCK(mpctx->lock); @@ -1956,6 +2179,7 @@ isc_mempool_setmaxalloc(isc_mempool_t *mpctx0, unsigned int limit) { } mpctx->maxalloc = limit; + mpctx->freemax = limit; if (mpctx->lock != NULL) { UNLOCK(mpctx->lock); @@ -2008,11 +2232,15 @@ isc_mempool_setfillcount(isc_mempool_t *mpctx0, unsigned int limit) { REQUIRE(limit > 0); isc__mempool_t *mpctx = (isc__mempool_t *)mpctx0; + size_t pagesize = sysconf(_SC_PAGESIZE); if (mpctx->lock != NULL) { LOCK(mpctx->lock); } + if (limit < (pagesize / mpctx->size)) { + limit = (pagesize / mpctx->size) - 1; + } mpctx->fillcount = limit; if (mpctx->lock != NULL) { @@ -2041,6 +2269,7 @@ isc_mempool_getfillcount(isc_mempool_t *mpctx0) { return (fillcount); } +#if USE_ALLOCATOR_CUSTOM /* * Requires contextslock to be held by caller. */ @@ -2057,6 +2286,7 @@ print_contexts(FILE *file) { } fflush(file); } +#endif void isc_mem_checkdestroyed(FILE *file) { @@ -2434,65 +2664,6 @@ isc_mem_create(isc_mem_t **mctxp) { mem_create(mctxp, isc_mem_defaultflags); } -<<<<<<< HEAD -void * -isc__mem_get(isc_mem_t *mctx, size_t size FLARG) { - REQUIRE(ISCAPI_MCTX_VALID(mctx)); - - return (mctx->methods->memget(mctx, size FLARG_PASS)); -} - -void -isc__mem_put(isc_mem_t *mctx, void *ptr, size_t size FLARG) { - REQUIRE(ISCAPI_MCTX_VALID(mctx)); - - mctx->methods->memput(mctx, ptr, size FLARG_PASS); -} - -void -isc__mem_putanddetach(isc_mem_t **mctxp, void *ptr, size_t size FLARG) { - REQUIRE(mctxp != NULL && ISCAPI_MCTX_VALID(*mctxp)); - - (*mctxp)->methods->memputanddetach(mctxp, ptr, size FLARG_PASS); -} - -void * -isc__mem_allocate(isc_mem_t *mctx, size_t size FLARG) { - REQUIRE(ISCAPI_MCTX_VALID(mctx)); - - return (mctx->methods->memallocate(mctx, size FLARG_PASS)); -} - -void * -isc__mem_reallocate(isc_mem_t *mctx, void *ptr, size_t size FLARG) { - REQUIRE(ISCAPI_MCTX_VALID(mctx)); - - return (mctx->methods->memreallocate(mctx, ptr, size FLARG_PASS)); -} - -char * -isc__mem_strdup(isc_mem_t *mctx, const char *s FLARG) { - REQUIRE(ISCAPI_MCTX_VALID(mctx)); - - return (mctx->methods->memstrdup(mctx, s FLARG_PASS)); -} - -char * -isc__mem_strndup(isc_mem_t *mctx, const char *s, size_t size FLARG) { - REQUIRE(ISCAPI_MCTX_VALID(mctx)); - - return (mctx->methods->memstrndup(mctx, s, size FLARG_PASS)); -} - -void -isc__mem_free(isc_mem_t *mctx, void *ptr FLARG) { - REQUIRE(ISCAPI_MCTX_VALID(mctx)); - - mctx->methods->memfree(mctx, ptr FLARG_PASS); -} - -======= ->>>>>>> 532e5a3b4f (Remove an extra level of indirection in memory context routines) void isc__mem_printactive(isc_mem_t *ctx0, FILE *file) { #if ISC_MEM_TRACKLINES @@ -2512,51 +2683,52 @@ isc__mem_printactive(isc_mem_t *ctx0, FILE *file) { * Routines using default memory context */ -static isc_mem_t *isc__mem_mctx = NULL; - -void -isc__mem_initialize(void); -void -isc__mem_shutdown(void); - -ISC_CONSTRUCTOR(101) -void -isc__mem_initialize(void) { - REQUIRE(isc__mem_mctx == NULL); - isc_mem_create(&isc__mem_mctx); - isc_mem_setname(isc__mem_mctx, "default"); -} - -ISC_DESTRUCTOR(101) -void -isc__mem_shutdown(void) { - REQUIRE(isc__mem_mctx != NULL); - isc_mem_destroy(&isc__mem_mctx); - isc_mem_checkdestroyed(stderr); -} +LIBISC_EXTERNAL_DATA isc_mem_t *isc__mem_mctx = NULL; void * isc__malloc(size_t size FLARG) { + REQUIRE(isc__mem_mctx != NULL); return (isc__mem_allocate(isc__mem_mctx, size FLARG_PASS)); } void * isc__calloc(size_t num, size_t size FLARG) { + REQUIRE(isc__mem_mctx != NULL); return (isc__mem_callocate(isc__mem_mctx, num, size FLARG_PASS)); } void * isc__realloc(void *ptr, size_t size FLARG) { + REQUIRE(isc__mem_mctx != NULL); return (isc__mem_reallocate(isc__mem_mctx, ptr, size FLARG_PASS)); } void isc__free(void *ptr FLARG) { + REQUIRE(isc__mem_mctx != NULL); + /* + * The free function causes the space pointed to by ptr to be + * deallocated, that is, made available for further allocation. If ptr + * is a null pointer, no action occurs. Otherwise, if the argument does + * not match a pointer earlier returned by a memory management function, + * or if the space has been deallocated by a call to free or realloc, + * the behavior is undefined. + * [ISO9899 ยง 7.22.3.3] + * + * [ISO9899] + * ISO/IEC WG 9899:2011: Programming languages - C. + * International Organization for Standardization, Geneva, CH. + * http://www.open-std.org/jtc1/sc22/wg14/www/docs/n1548.pdf + */ + if (ptr == NULL) { + return; + } isc__mem_free(isc__mem_mctx, ptr FLARG_PASS); } char * isc__strdup(const char *s1 FLARG) { + REQUIRE(isc__mem_mctx != NULL); return (isc__mem_strdup(isc__mem_mctx, s1 FLARG_PASS)); } @@ -2564,3 +2736,9 @@ char * isc__strndup(const char *s1 FLARG) { return (isc__mem_strndup(isc__mem_mctx, s1 FLARG_PASS)); } + +isc_mem_t * +isc_get_default_mctx(void) { + REQUIRE(isc__mem_mctx != NULL); + return ((isc_mem_t *)isc__mem_mctx); +} diff --git a/lib/isc/tests/isctest.c b/lib/isc/tests/isctest.c index 66c70e5004..a333c16ed5 100644 --- a/lib/isc/tests/isctest.c +++ b/lib/isc/tests/isctest.c @@ -108,8 +108,8 @@ isc_test_begin(FILE *logfile, bool start_managers, unsigned int workers) { isc_mem_debugging |= ISC_MEM_DEBUGRECORD; - INSIST(test_mctx == NULL); - isc_mem_create(&test_mctx); + REQUIRE(test_mctx == NULL); + test_mctx = isc_get_default_mctx(); if (logfile != NULL) { isc_logdestination_t destination; @@ -156,9 +156,7 @@ isc_test_end(void) { if (test_lctx != NULL) { isc_log_destroy(&test_lctx); } - if (test_mctx != NULL) { - isc_mem_destroy(&test_mctx); - } + test_mctx = NULL; test_running = false; } diff --git a/lib/isc/tests/mem_test.c b/lib/isc/tests/mem_test.c index e096c54c67..708f08c549 100644 --- a/lib/isc/tests/mem_test.c +++ b/lib/isc/tests/mem_test.c @@ -64,6 +64,22 @@ _teardown(void **state) { #define MP2_FREEMAX 25 #define MP2_FILLCNT 25 +#define ALIGNMENT_SIZE sizeof(void *) + +static inline size_t +quantize(size_t size) { + /*! + * Round up the result in order to get a size big + * enough to satisfy the request and be aligned on ALIGNMENT_SIZE + * byte boundaries. + */ + + if (size == 0U) { + return (ALIGNMENT_SIZE); + } + return ((size + ALIGNMENT_SIZE - 1) & (~(ALIGNMENT_SIZE - 1))); +} + /* general memory system tests */ static void isc_mem_test(void **state) { @@ -72,7 +88,7 @@ isc_mem_test(void **state) { void *tmp; isc_mempool_t *mp1 = NULL, *mp2 = NULL; unsigned int i, j; - int rval; + int rval, mp1_freecount = 0; UNUSED(state); @@ -91,6 +107,12 @@ isc_mem_test(void **state) { assert_non_null(items1[i]); } + /* + * The number of free items on the list will be MAX(MP1_FREEMAX, + * MP1_MAXALLOC) aligned to page boundary. + */ + mp1_freecount = isc_mempool_getfreecount(mp1); + /* * Try to allocate one more. This should fail. */ @@ -107,7 +129,7 @@ isc_mem_test(void **state) { } rval = isc_mempool_getfreecount(mp1); - assert_int_equal(rval, 10); + assert_int_equal(rval, mp1_freecount + 11); rval = isc_mempool_getallocated(mp1); assert_int_equal(rval, 19); @@ -152,6 +174,9 @@ isc_mem_test(void **state) { isc_mempool_destroy(&mp1); } +#if !defined(USE_ALLOCATOR_SYSTEM) && !defined(USE_ALLOCATOR_JEMALLOC) && \ + !defined(USE_ALLOCATOR_TCMALLOC) + /* test TotalUse calculation */ static void isc_mem_total_test(void **state) { @@ -205,8 +230,7 @@ isc_mem_total_test(void **state) { static void isc_mem_inuse_test(void **state) { isc_mem_t *mctx2 = NULL; - size_t before, after; - ssize_t diff; + size_t before, meanwhile, after; void *ptr; UNUSED(state); @@ -216,16 +240,22 @@ isc_mem_inuse_test(void **state) { before = isc_mem_inuse(mctx2); ptr = isc_mem_allocate(mctx2, 1024000); + meanwhile = isc_mem_inuse(mctx2); isc_mem_free(mctx2, ptr); after = isc_mem_inuse(mctx2); - - diff = after - before; - - assert_int_equal(diff, 0); - isc_mem_destroy(&mctx2); + + fprintf(stderr, "%zu < %zu > %zu\n", before, meanwhile, after); + + assert_int_not_equal(before, meanwhile); + assert_true(before < meanwhile); + assert_int_not_equal(meanwhile, after); + assert_true(after < meanwhile); + assert_int_equal(after, before); } +#endif + #if ISC_MEM_TRACKLINES /* test mem with no flags */ @@ -497,11 +527,13 @@ main(void) { const struct CMUnitTest tests[] = { cmocka_unit_test_setup_teardown(isc_mem_test, _setup, _teardown), +#if !defined(USE_ALLOCATOR_SYSTEM) && !defined(USE_ALLOCATOR_JEMALLOC) && \ + !defined(USE_ALLOCATOR_TCMALLOC) cmocka_unit_test_setup_teardown(isc_mem_total_test, _setup, _teardown), cmocka_unit_test_setup_teardown(isc_mem_inuse_test, _setup, _teardown), - +#endif #if !defined(__SANITIZE_THREAD__) cmocka_unit_test_setup_teardown(isc_mem_benchmark, _setup, _teardown),