diff --git a/bin/named/main.c b/bin/named/main.c index 6fa0a80b59..f5e5c37845 100644 --- a/bin/named/main.c +++ b/bin/named/main.c @@ -1614,7 +1614,6 @@ main(int argc, char *argv[]) { } } isc_mem_destroy(&named_g_mctx); - isc_mem_checkdestroyed(stderr); named_main_setmemstats(NULL); diff --git a/configure.ac b/configure.ac index 1ff23d73c1..c143182b0a 100644 --- a/configure.ac +++ b/configure.ac @@ -1580,6 +1580,31 @@ test -z "$with_dlz_stub" && with_dlz_stub=no AC_CHECK_HEADERS([glob.h]) +# +# Support for constructor and destructor attributes +# +AX_GCC_FUNC_ATTRIBUTE([constructor]) +AX_GCC_FUNC_ATTRIBUTE([destructor]) + +# +# Which memory allocator we should use? +# +AC_ARG_WITH([allocator], + [AS_HELP_STRING([--with-allocator=], + [Use different memory allocator (default=custom)])], + [], + [with_allocator=custom]) + +AS_CASE([$with_allocator], + [system], [AC_DEFINE([USE_ALLOCATOR_SYSTEM], [1], [Define if you want to use system memory allocator])], + [jemalloc], [PKG_CHECK_MODULES([ALLOCATOR], [jemalloc], + [AC_DEFINE([USE_ALLOCATOR_JEMALLOC], [1], [Define if you want to use jemalloc allocator])])], + [tcmalloc*], [PKG_CHECK_MODULES([ALLOCATOR], [lib$with_allocator], + [AC_DEFINE([USE_ALLOCATOR_TCMALLOC], [1], [Define if you want to use tcmalloc allocator])])] + [:]) + +AC_MSG_NOTICE([Using $with_allocator memory allocator]) + # # Files to configure. These are listed here because we used to # specify them as arguments to AC_OUTPUT. diff --git a/lib/isc/Makefile.am b/lib/isc/Makefile.am index 48ead7c17f..006bbb0bc0 100644 --- a/lib/isc/Makefile.am +++ b/lib/isc/Makefile.am @@ -226,6 +226,7 @@ libisc_la_SOURCES = \ libisc_la_CPPFLAGS = \ $(AM_CPPFLAGS) \ + $(ALLOCATOR_CFLAGS) \ $(LIBISC_CFLAGS) \ $(LIBUV_CFLAGS) \ $(OPENSSL_CFLAGS) \ @@ -236,6 +237,7 @@ libisc_la_LDFLAGS = \ -release "$(PACKAGE_VERSION)" libisc_la_LIBADD = \ + $(ALLOCATOR_LIBS) \ $(LIBUV_LIBS) \ $(OPENSSL_LIBS) \ $(ZLIB_LIBS) diff --git a/lib/isc/buffer.c b/lib/isc/buffer.c index 1db0eb026e..3d751f24ca 100644 --- a/lib/isc/buffer.c +++ b/lib/isc/buffer.c @@ -553,8 +553,7 @@ isc_buffer_allocate(isc_mem_t *mctx, isc_buffer_t **dynbuffer, isc_result_t isc_buffer_reserve(isc_buffer_t **dynbuffer, unsigned int size) { - unsigned char *bdata; - uint64_t len; + size_t len; REQUIRE(dynbuffer != NULL); REQUIRE(ISC_BUFFER_VALID(*dynbuffer)); @@ -581,18 +580,9 @@ isc_buffer_reserve(isc_buffer_t **dynbuffer, unsigned int size) { return (ISC_R_NOMEMORY); } - /* - * XXXMUKS: This is far more expensive than plain realloc() as - * it doesn't remap pages, but does ordinary copy. So is - * isc_mem_reallocate(), which has additional issues. - */ - bdata = isc_mem_get((*dynbuffer)->mctx, (unsigned int)len); + (*dynbuffer)->base = isc_mem_reallocate((*dynbuffer)->mctx, + (*dynbuffer)->base, len); - memmove(bdata, (*dynbuffer)->base, (*dynbuffer)->length); - isc_mem_put((*dynbuffer)->mctx, (*dynbuffer)->base, - (*dynbuffer)->length); - - (*dynbuffer)->base = bdata; (*dynbuffer)->length = (unsigned int)len; return (ISC_R_SUCCESS); diff --git a/lib/isc/include/isc/mem.h b/lib/isc/include/isc/mem.h index b4dc148c32..1c96a508f1 100644 --- a/lib/isc/include/isc/mem.h +++ b/lib/isc/include/isc/mem.h @@ -32,9 +32,13 @@ typedef void (*isc_mem_water_t)(void *, int); * Define ISC_MEM_TRACKLINES=1 to turn on detailed tracing of memory * allocation and freeing by file and line number. */ +#if !defined(USE_ALLOCATOR_SYSTEM) && !defined(USE_ALLOCATOR_JEMALLOC) && \ + !defined(USE_ALLOCATOR_TCMALLOC) #ifndef ISC_MEM_TRACKLINES #define ISC_MEM_TRACKLINES 1 #endif /* ifndef ISC_MEM_TRACKLINES */ +#endif /* !defined(USE_ALLOCATOR_SYSTEM) && !defined(USE_ALLOCATOR_JEMALLOC) \ + && !defined(USE_ALLOCATOR_TCMALLOC) */ /*% * Define ISC_MEM_CHECKOVERRUN=1 to turn on checks for using memory outside @@ -600,6 +604,9 @@ isc__strdup(const char *str _ISC_MEM_FLARG); char * isc__strndup(const char *str _ISC_MEM_FLARG); +isc_mem_t * +isc_get_default_mctx(void); + ISC_LANG_ENDDECLS #endif /* ISC_MEM_H */ diff --git a/lib/isc/include/isc/util.h b/lib/isc/include/isc/util.h index b88cc51aff..f8b4b1a5bd 100644 --- a/lib/isc/include/isc/util.h +++ b/lib/isc/include/isc/util.h @@ -48,12 +48,14 @@ #define ISC_NONSTRING #endif /* __GNUC__ */ -#if __GNUC__ +#if HAVE_FUNC_ATTRIBUTE_CONSTRUCTOR && HAVE_FUNC_ATTRIBUTE_DESTRUCTOR #define ISC_CONSTRUCTOR(priority) __attribute__((constructor(priority))) #define ISC_DESTRUCTOR(priority) __attribute__((destructor(priority))) -#else +#elif WIN32 #define ISC_CONSTRUCTOR(priority) #define ISC_DESTRUCTOR(priority) +#else +#error Either __attribute__((constructor|destructor))__ or DllMain support needed to compile BIND 9. #endif /*% @@ -208,6 +210,10 @@ #define __has_feature(x) 0 #endif /* if !defined(__has_feature) */ +#if !defined(__has_builtin) +#define __has_builtin(x) 0 +#endif /* if !defined(__has_attribute) */ + /* GCC defines __SANITIZE_ADDRESS__, so reuse the macro for clang */ #if __has_feature(address_sanitizer) #define __SANITIZE_ADDRESS__ 1 diff --git a/lib/isc/lib.c b/lib/isc/lib.c index 8bee4c21a4..9ff02cbd73 100644 --- a/lib/isc/lib.c +++ b/lib/isc/lib.c @@ -11,8 +11,12 @@ /*! \file */ +#include + #include #include +#include +#include /*** *** Functions @@ -22,3 +26,71 @@ void isc_lib_register(void) { isc_bind9 = false; } + +LIBISC_EXTERNAL_DATA extern isc_mem_t *isc__mem_mctx; + +void +isc__mem_initialize(void) ISC_CONSTRUCTOR(101); +void +isc__mem_shutdown(void) ISC_DESTRUCTOR(101); + +void +isc__mem_initialize(void) { + REQUIRE(isc__mem_mctx == NULL); + isc_mem_create(&isc__mem_mctx); + isc_mem_setname(isc__mem_mctx, "default"); +} + +void +isc__mem_shutdown(void) { + REQUIRE(isc__mem_mctx != NULL); + isc_mem_destroy(&isc__mem_mctx); + isc_mem_checkdestroyed(stderr); +} + +void +isc__nm_initialize(void) ISC_CONSTRUCTOR(102); +void +isc__nm_shutdown(void) ISC_DESTRUCTOR(102); + +#if UV_VERSION_MAJOR > 1 || (UV_VERSION_MAJOR == 1 && UV_VERSION_MINOR >= 38) + +static void * +_malloc(size_t size) { + return (isc_malloc(size)); +} + +static void * +_calloc(size_t num, size_t size) { + return (isc_calloc(num, size)); +} + +static void * +_realloc(void *ptr, size_t size) { + return (isc_realloc(ptr, size)); +} + +static void +_free(void *ptr) { + return (isc_free(ptr)); +} + +void +isc__nm_initialize(void) { + uv_replace_allocator(_malloc, _realloc, _calloc, _free); +} + +void +isc__nm_shutdown(void) { + uv_library_shutdown(); +} + +#else + +void +isc__nm_initialize(void) {} + +void +isc__nm_shutdown(void) {} + +#endif diff --git a/lib/isc/mem.c b/lib/isc/mem.c index cf12155952..269c15c577 100644 --- a/lib/isc/mem.c +++ b/lib/isc/mem.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -31,6 +32,14 @@ #include #include +#if USE_ALLOCATOR_TCMALLOC +#include +#endif + +#if USE_ALLOCATOR_JEMALLOC +#include +#endif + #ifdef HAVE_LIBXML2 #include #define ISC_XMLCHAR (const xmlChar *) @@ -57,7 +66,7 @@ LIBISC_EXTERNAL_DATA unsigned int isc_mem_defaultflags = ISC_MEMFLAG_DEFAULT; #define DEF_MAX_SIZE 1100 #define DEF_MEM_TARGET 4096 -#define ALIGNMENT_SIZE 8U /*%< must be a power of 2 */ +#define ALIGNMENT_SIZE sizeof(void *) /*%< must be a power of 2 */ #define NUM_BASIC_BLOCKS 64 /*%< must be > 1 */ #define TABLE_INCREMENT 1024 #define DEBUG_TABLE_COUNT 512U @@ -201,6 +210,7 @@ struct isc__mempool { #if ISC_MEMPOOL_NAMES char name[16]; /*%< printed name in stats reports */ #endif /* if ISC_MEMPOOL_NAMES */ + element *pages; }; /* @@ -325,14 +335,6 @@ delete_trace_entry(isc__mem_t *mctx, const void *ptr, size_t size, } #endif /* ISC_MEM_TRACKLINES */ -static inline size_t -rmsize(size_t size) { - /* - * round down to ALIGNMENT_SIZE - */ - return (size & (~(ALIGNMENT_SIZE - 1))); -} - static inline size_t quantize(size_t size) { /*! @@ -347,6 +349,15 @@ quantize(size_t size) { return ((size + ALIGNMENT_SIZE - 1) & (~(ALIGNMENT_SIZE - 1))); } +#if USE_ALLOCATOR_CUSTOM +static inline size_t +rmsize(size_t size) { + /* + * round down to ALIGNMENT_SIZE + */ + return (size & (~(ALIGNMENT_SIZE - 1))); +} + static inline void more_basic_blocks(isc__mem_t *ctx) { void *tmp; @@ -679,6 +690,7 @@ mem_putstats(isc__mem_t *ctx, void *ptr, size_t size) { #endif /* if ISC_MEM_CHECKOVERRUN */ ctx->malloced -= size; } +#endif /* * Private. @@ -700,8 +712,7 @@ default_memalloc(size_t size) { * * [ISO9899] * ISO/IEC WG 9899:2011: Programming languages - C. - * International Organization for Standardization, Geneva, - * Switzerland. + * International Organization for Standardization, Geneva, CH. * http://www.open-std.org/JTC1/SC22/WG14/www/docs/n1570.pdf */ @@ -912,7 +923,6 @@ isc_mem_attach(isc_mem_t *source0, isc_mem_t **targetp) { void isc_mem_detach(isc_mem_t **ctxp) { REQUIRE(ctxp != NULL && VALID_CONTEXT(*ctxp)); - isc__mem_t *ctx = (isc__mem_t *)*ctxp; *ctxp = NULL; @@ -940,6 +950,9 @@ isc__mem_putanddetach(isc_mem_t **ctxp, void *ptr, size_t size FLARG) { isc__mem_t *ctx = (isc__mem_t *)*ctxp; *ctxp = NULL; +#if USE_ALLOCATOR_CUSTOM + isc__mem_t *ctx = (isc__mem_t *)*ctxp; + if (ISC_UNLIKELY((isc_mem_debugging & (ISC_MEM_DEBUGSIZE | ISC_MEM_DEBUGCTX)) != 0)) { @@ -953,22 +966,30 @@ isc__mem_putanddetach(isc_mem_t **ctxp, void *ptr, size_t size FLARG) { } isc__mem_free((isc_mem_t *)ctx, ptr FLARG_PASS); - goto destroy; - } - - MCTXLOCK(ctx); - - DELETE_TRACE(ctx, ptr, size, file, line); - - if ((ctx->flags & ISC_MEMFLAG_INTERNAL) != 0) { - mem_putunlocked(ctx, ptr, size); } else { - mem_putstats(ctx, ptr, size); - mem_put(ctx, ptr, size); - } - MCTXUNLOCK(ctx); + MCTXLOCK(ctx); -destroy: + DELETE_TRACE(ctx, ptr, size, file, line); + + if ((ctx->flags & ISC_MEMFLAG_INTERNAL) != 0) { + mem_putunlocked(ctx, ptr, size); + } else { + mem_putstats(ctx, ptr, size); + mem_put(ctx, ptr, size); + } + MCTXUNLOCK(ctx); + } +#elif defined(USE_ALLOCATOR_JEMALLOC) + if (size == 0) { + size = ALIGNMENT_SIZE; + } + sdallocx(ptr, size, 0); +#elif defined(USE_ALLOCATOR_TCMALLOC) + tc_free_sized(ptr, size); +#else + UNUSED(size); + default_memfree(ptr); +#endif if (isc_refcount_decrement(&ctx->references) == 1) { isc_refcount_destroy(&ctx->references); destroy(ctx); @@ -1002,9 +1023,10 @@ isc_mem_destroy(isc_mem_t **ctxp) { void * isc__mem_get(isc_mem_t *ctx0, size_t size FLARG) { REQUIRE(VALID_CONTEXT(ctx0)); + void *ptr = NULL; +#if USE_ALLOCATOR_CUSTOM isc__mem_t *ctx = (isc__mem_t *)ctx0; - void *ptr; bool call_water = false; if (ISC_UNLIKELY((isc_mem_debugging & @@ -1023,7 +1045,6 @@ isc__mem_get(isc_mem_t *ctx0, size_t size FLARG) { mem_getstats(ctx, size); } } - ADD_TRACE(ctx, ptr, size, file, line); if (ctx->hi_water != 0U && ctx->inuse > ctx->hi_water) { @@ -1046,7 +1067,29 @@ isc__mem_get(isc_mem_t *ctx0, size_t size FLARG) { if (call_water && (ctx->water != NULL)) { (ctx->water)(ctx->water_arg, ISC_MEM_HIWATER); } +#elif USE_ALLOCATOR_JEMALLOC + if (size == 0) { + size = ALIGNMENT_SIZE; + } + ptr = mallocx(size, MALLOCX_ALIGN(ALIGNMENT_SIZE)); + if (ptr == NULL && size != 0) { + char strbuf[ISC_STRERRORSIZE]; + strerror_r(errno, strbuf, sizeof(strbuf)); + isc_error_fatal(__FILE__, __LINE__, + "mallocx(%zu, 0) failed: %s", size, strbuf); + } +#elif defined(USE_ALLOCATOR_TCMALLOC) + ptr = tc_malloc(size); + if (ptr == NULL && size != 0) { + char strbuf[ISC_STRERRORSIZE]; + strerror_r(errno, strbuf, sizeof(strbuf)); + isc_error_fatal(__FILE__, __LINE__, "tc_malloc(%zu) failed: %s", + size, strbuf); + } +#elif defined(USE_ALLOCATOR_SYSTEM) + ptr = default_memalloc(size); +#endif return (ptr); } @@ -1055,6 +1098,7 @@ isc__mem_put(isc_mem_t *ctx0, void *ptr, size_t size FLARG) { REQUIRE(VALID_CONTEXT(ctx0)); REQUIRE(ptr != NULL); +#if USE_ALLOCATOR_CUSTOM isc__mem_t *ctx = (isc__mem_t *)ctx0; bool call_water = false; size_info *si; @@ -1103,6 +1147,17 @@ isc__mem_put(isc_mem_t *ctx0, void *ptr, size_t size FLARG) { if (call_water && (ctx->water != NULL)) { (ctx->water)(ctx->water_arg, ISC_MEM_LOWATER); } +#elif defined(USE_ALLOCATOR_JEMALLOC) + if (size == 0) { + size = ALIGNMENT_SIZE; + } + sdallocx(ptr, size, 0); +#elif defined(USE_ALLOCATOR_TCMALLOC) + tc_free_sized(ptr, size); +#else + UNUSED(size); + default_memfree(ptr); +#endif } void @@ -1223,6 +1278,7 @@ isc_mem_stats(isc_mem_t *ctx0, FILE *out) { MCTXUNLOCK(ctx); } +#if USE_ALLOCATOR_CUSTOM /* * Replacements for malloc() and free() -- they implicitly remember the * size of the object allocated (with some additional overhead). @@ -1251,11 +1307,14 @@ mem_allocateunlocked(isc_mem_t *ctx0, size_t size) { si->u.size = size; return (&si[1]); } +#endif void * isc__mem_allocate(isc_mem_t *ctx0, size_t size FLARG) { REQUIRE(VALID_CONTEXT(ctx0)); + void *ptr = NULL; +#if USE_ALLOCATOR_CUSTOM isc__mem_t *ctx = (isc__mem_t *)ctx0; size_info *si; bool call_water = false; @@ -1293,17 +1352,86 @@ isc__mem_allocate(isc_mem_t *ctx0, size_t size FLARG) { (ctx->water)(ctx->water_arg, ISC_MEM_HIWATER); } - return (si); + ptr = si; +#elif USE_ALLOCATOR_JEMALLOC + if (size == 0U) { + size = ALIGNMENT_SIZE; + } + ptr = mallocx(size, MALLOCX_ALIGN(ALIGNMENT_SIZE)); + if (ptr == NULL && size != 0) { + char strbuf[ISC_STRERRORSIZE]; + strerror_r(errno, strbuf, sizeof(strbuf)); + isc_error_fatal(__FILE__, __LINE__, + "mallocx(%zu, 0) failed: %s", size, strbuf); + } +#elif USE_ALLOCATOR_TCMALLOC + ptr = tc_malloc(size); + if (ptr == NULL && size != 0) { + char strbuf[ISC_STRERRORSIZE]; + strerror_r(errno, strbuf, sizeof(strbuf)); + isc_error_fatal(__FILE__, __LINE__, "tc_malloc(%zu) failed: %s", + size, strbuf); + } +#else + ptr = default_memalloc(size); +#endif + return (ptr); } void * isc__mem_callocate(isc_mem_t *ctx0, size_t num, size_t size FLARG) { + void *ptr = NULL; +#if USE_ALLOCATOR_CUSTOM size_t numsize; ISC_MUL_OVERFLOW(num, size, &numsize); - void *ptr = isc__mem_allocate(ctx0, numsize FLARG_PASS); + ptr = isc__mem_allocate(ctx0, numsize FLARG_PASS); memset(ptr, 0, numsize); return (ptr); +#elif USE_ALLOCATOR_JEMALLOC + UNUSED(ctx0); + + size_t numsize; + + ISC_MUL_OVERFLOW(num, size, &numsize); + + if (numsize == 0U) { + numsize = ALIGNMENT_SIZE; + } + + ptr = mallocx(numsize, MALLOCX_ZERO | MALLOCX_ALIGN(ALIGNMENT_SIZE)); + if (ptr == NULL && size != 0) { + char strbuf[ISC_STRERRORSIZE]; + strerror_r(errno, strbuf, sizeof(strbuf)); + isc_error_fatal(__FILE__, __LINE__, + "mallocx(%zu, MALLOCX_ZERO) failed: %s", + numsize, strbuf); + } + +#elif USE_ALLOCATOR_TCMALLOC + UNUSED(ctx0); + + ptr = tc_calloc(num, size); + if (ptr == NULL && size != 0) { + char strbuf[ISC_STRERRORSIZE]; + strerror_r(errno, strbuf, sizeof(strbuf)); + isc_error_fatal(__FILE__, __LINE__, + "tc_calloc(%zu, %zu) failed: %s", num, size, + strbuf); + } +#else + UNUSED(ctx0); + + ptr = calloc(num, size); + + if (ptr == NULL && size != 0) { + char strbuf[ISC_STRERRORSIZE]; + strerror_r(errno, strbuf, sizeof(strbuf)); + isc_error_fatal(__FILE__, __LINE__, "calloc failed: %s", + strbuf); + } +#endif + return (ptr); } void * @@ -1319,8 +1447,9 @@ isc__mem_cget(isc_mem_t *ctx0, size_t num, size_t size FLARG) { void * isc__mem_reallocate(isc_mem_t *ctx0, void *ptr, size_t size FLARG) { REQUIRE(VALID_CONTEXT(ctx0)); - void *new_ptr = NULL; + +#if USE_ALLOCATOR_CUSTOM size_t oldsize, copysize; /* @@ -1353,6 +1482,51 @@ isc__mem_reallocate(isc_mem_t *ctx0, void *ptr, size_t size FLARG) { isc__mem_free(ctx0, ptr FLARG_PASS); } + return (new_ptr); +#elif USE_ALLOCATOR_JEMALLOC + if (size > 0U) { + if (ptr == NULL) { + new_ptr = mallocx(size, MALLOCX_ALIGN(ALIGNMENT_SIZE)); + } else { + new_ptr = rallocx(ptr, size, MALLOCX_ALIGN(ALIGNMENT_SIZE)); + } + if (new_ptr == NULL && size != 0) { + char strbuf[ISC_STRERRORSIZE]; + strerror_r(errno, strbuf, sizeof(strbuf)); + isc_error_fatal(__FILE__, __LINE__, + "rallocx(%p, %zu, 0) failed: %s", ptr, + size, strbuf); + } + } else if (ptr != NULL) { + dallocx(ptr, 0); + } +#elif USE_ALLOCATOR_TCMALLOC + if (size > 0U) { + new_ptr = tc_realloc(ptr, size); + if (new_ptr == NULL && size != 0) { + char strbuf[ISC_STRERRORSIZE]; + strerror_r(errno, strbuf, sizeof(strbuf)); + isc_error_fatal(__FILE__, __LINE__, + "tc_realloc(%p, %zu) failed: %s", ptr, size, + strbuf); + } + } else if (ptr != NULL) { + tc_free(ptr); + } +#else + if (size > 0U) { + new_ptr = realloc(ptr, size); + if (new_ptr == NULL && size != 0) { + char strbuf[ISC_STRERRORSIZE]; + strerror_r(errno, strbuf, sizeof(strbuf)); + isc_error_fatal(__FILE__, __LINE__, + "realloc(%p, %zu) failed: %s", ptr, size, + strbuf); + } + } else { + free(ptr); + } +#endif return (new_ptr); } @@ -1361,6 +1535,7 @@ isc__mem_free(isc_mem_t *ctx0, void *ptr FLARG) { REQUIRE(VALID_CONTEXT(ctx0)); REQUIRE(ptr != NULL); +#if USE_ALLOCATOR_CUSTOM isc__mem_t *ctx = (isc__mem_t *)ctx0; size_info *si; size_t size; @@ -1409,6 +1584,13 @@ isc__mem_free(isc_mem_t *ctx0, void *ptr FLARG) { if (call_water) { (ctx->water)(ctx->water_arg, ISC_MEM_LOWATER); } +#elif USE_ALLOCATOR_JEMALLOC + dallocx(ptr, 0); +#elif USE_ALLOCATOR_TCMALLOC + tc_free(ptr); +#else + free(ptr); +#endif } /* @@ -1475,8 +1657,10 @@ size_t isc_mem_inuse(isc_mem_t *ctx0) { REQUIRE(VALID_CONTEXT(ctx0)); + size_t inuse = 0; + +#if USE_ALLOCATOR_CUSTOM isc__mem_t *ctx = (isc__mem_t *)ctx0; - size_t inuse; MCTXLOCK(ctx); @@ -1484,6 +1668,10 @@ isc_mem_inuse(isc_mem_t *ctx0) { MCTXUNLOCK(ctx); +#elif USE_ALLOCATOR_JEMALLOC + size_t len = sizeof(inuse); + mallctl("stats.active", &inuse, &len, NULL, 0); +#endif return (inuse); } @@ -1507,14 +1695,19 @@ size_t isc_mem_total(isc_mem_t *ctx0) { REQUIRE(VALID_CONTEXT(ctx0)); + size_t total = 0; +#if USE_ALLOCATOR_CUSTOM isc__mem_t *ctx = (isc__mem_t *)ctx0; - size_t total; MCTXLOCK(ctx); total = ctx->total; MCTXUNLOCK(ctx); +#elif USE_ALLOCATOR_JEMALLOC + size_t len = sizeof(total); + mallctl("stats.allocated", &total, &len, NULL, 0); +#endif return (total); } @@ -1608,6 +1801,7 @@ isc_mempool_create(isc_mem_t *mctx0, size_t size, isc_mempool_t **mpctxp) { isc__mem_t *mctx = (isc__mem_t *)mctx0; isc__mempool_t *mpctx; + size_t pagesize = sysconf(_SC_PAGESIZE); /* * Allocate space for this pool, initialize values, and if all works @@ -1625,17 +1819,19 @@ isc_mempool_create(isc_mem_t *mctx0, size_t size, isc_mempool_t **mpctxp) { if (size < sizeof(element)) { size = sizeof(element); } + size = quantize(size); mpctx->size = size; mpctx->maxalloc = UINT_MAX; mpctx->allocated = 0; mpctx->freecount = 0; - mpctx->freemax = 1; - mpctx->fillcount = 1; + mpctx->freemax = UINT_MAX; + mpctx->fillcount = pagesize / size; mpctx->gets = 0; #if ISC_MEMPOOL_NAMES mpctx->name[0] = 0; #endif /* if ISC_MEMPOOL_NAMES */ mpctx->items = NULL; + mpctx->pages = NULL; *mpctxp = (isc_mempool_t *)mpctx; @@ -1676,7 +1872,6 @@ isc_mempool_destroy(isc_mempool_t **mpctxp) { isc__mempool_t *mpctx; isc__mem_t *mctx; isc_mutex_t *lock; - element *item; mpctx = (isc__mempool_t *)*mpctxp; #if ISC_MEMPOOL_NAMES @@ -1697,11 +1892,13 @@ isc_mempool_destroy(isc_mempool_t **mpctxp) { LOCK(lock); } +#if USE_ALLOCATOR_CUSTOM /* * Return any items on the free list */ MCTXLOCK(mctx); while (mpctx->items != NULL) { + element *item; INSIST(mpctx->freecount > 0); mpctx->freecount--; item = mpctx->items; @@ -1715,7 +1912,15 @@ isc_mempool_destroy(isc_mempool_t **mpctxp) { } } MCTXUNLOCK(mctx); - +#else + while (mpctx->pages != NULL) { + element *page; + page = mpctx->pages; + mpctx->pages = page->next; + isc_mem_put((isc_mem_t *)mpctx->mctx, page, mpctx->fillcount * mpctx->size); + mpctx->freecount -= mpctx->fillcount - 1; + } +#endif /* * Remove our linked list entry from the memory context. */ @@ -1754,10 +1959,6 @@ isc__mempool_get(isc_mempool_t *mpctx0 FLARG) { isc__mempool_t *mpctx = (isc__mempool_t *)mpctx0; element *item; - isc__mem_t *mctx; - unsigned int i; - - mctx = mpctx->mctx; if (mpctx->lock != NULL) { LOCK(mpctx->lock); @@ -1771,13 +1972,15 @@ isc__mempool_get(isc_mempool_t *mpctx0 FLARG) { goto out; } +#if USE_ALLOCATOR_CUSTOM + isc__mem_t *mctx = mpctx->mctx; if (ISC_UNLIKELY(mpctx->items == NULL)) { /* * We need to dip into the well. Lock the memory context * here and fill up our free list. */ MCTXLOCK(mctx); - for (i = 0; i < mpctx->fillcount; i++) { + for (size_t i = 0; i < mpctx->fillcount; i++) { if ((mctx->flags & ISC_MEMFLAG_INTERNAL) != 0) { item = mem_getunlocked(mctx, mpctx->size); } else { @@ -1796,6 +1999,21 @@ isc__mempool_get(isc_mempool_t *mpctx0 FLARG) { MCTXUNLOCK(mctx); } +#else + element *page = NULL; + if (ISC_UNLIKELY(mpctx->items == NULL)) { + page = isc__mem_get((isc_mem_t *)mpctx->mctx, mpctx->fillcount * mpctx->size FLARG_PASS); + page->next = mpctx->pages; + mpctx->pages = page; + item = (element *)((uintptr_t)page + mpctx->size); + for (size_t i = 1; i < mpctx->fillcount; i++) { + item->next = mpctx->items; + mpctx->items = item; + item = (element *)((uintptr_t)item + mpctx->size); + } + mpctx->freecount += mpctx->fillcount - 1; + } +#endif /* * If we didn't get any items, return NULL. */ @@ -1834,8 +2052,6 @@ isc__mempool_put(isc_mempool_t *mpctx0, void *mem FLARG) { REQUIRE(mem != NULL); isc__mempool_t *mpctx = (isc__mempool_t *)mpctx0; - isc__mem_t *mctx = mpctx->mctx; - element *item; if (mpctx->lock != NULL) { LOCK(mpctx->lock); @@ -1846,16 +2062,18 @@ isc__mempool_put(isc_mempool_t *mpctx0, void *mem FLARG) { #if ISC_MEM_TRACKLINES if (ISC_UNLIKELY((isc_mem_debugging & TRACE_OR_RECORD) != 0)) { - MCTXLOCK(mctx); - DELETE_TRACE(mctx, mem, mpctx->size, file, line); - MCTXUNLOCK(mctx); + MCTXLOCK(mpctx->mctx); + DELETE_TRACE(mpctx->mctx, mem, mpctx->size, file, line); + MCTXUNLOCK(mpctx->mctx); } #endif /* ISC_MEM_TRACKLINES */ +#if USE_ALLOCATOR_CUSTOM /* * If our free list is full, return this to the mctx directly. */ if (mpctx->freecount >= mpctx->freemax) { + isc__mem_t *mctx = mpctx->mctx; MCTXLOCK(mctx); if ((mctx->flags & ISC_MEMFLAG_INTERNAL) != 0) { mem_putunlocked(mctx, mem, mpctx->size); @@ -1869,7 +2087,8 @@ isc__mempool_put(isc_mempool_t *mpctx0, void *mem FLARG) { } return; } - +#endif + element *item; /* * Otherwise, attach it to our free list and bump the counter. */ @@ -1897,7 +2116,11 @@ isc_mempool_setfreemax(isc_mempool_t *mpctx0, unsigned int limit) { LOCK(mpctx->lock); } - mpctx->freemax = limit; + if (limit < mpctx->maxalloc) { + mpctx->freemax = mpctx->maxalloc; + } else { + mpctx->freemax = limit; + } if (mpctx->lock != NULL) { UNLOCK(mpctx->lock); @@ -1956,6 +2179,7 @@ isc_mempool_setmaxalloc(isc_mempool_t *mpctx0, unsigned int limit) { } mpctx->maxalloc = limit; + mpctx->freemax = limit; if (mpctx->lock != NULL) { UNLOCK(mpctx->lock); @@ -2008,11 +2232,15 @@ isc_mempool_setfillcount(isc_mempool_t *mpctx0, unsigned int limit) { REQUIRE(limit > 0); isc__mempool_t *mpctx = (isc__mempool_t *)mpctx0; + size_t pagesize = sysconf(_SC_PAGESIZE); if (mpctx->lock != NULL) { LOCK(mpctx->lock); } + if (limit < (pagesize / mpctx->size)) { + limit = (pagesize / mpctx->size) - 1; + } mpctx->fillcount = limit; if (mpctx->lock != NULL) { @@ -2041,6 +2269,7 @@ isc_mempool_getfillcount(isc_mempool_t *mpctx0) { return (fillcount); } +#if USE_ALLOCATOR_CUSTOM /* * Requires contextslock to be held by caller. */ @@ -2057,6 +2286,7 @@ print_contexts(FILE *file) { } fflush(file); } +#endif void isc_mem_checkdestroyed(FILE *file) { @@ -2434,65 +2664,6 @@ isc_mem_create(isc_mem_t **mctxp) { mem_create(mctxp, isc_mem_defaultflags); } -<<<<<<< HEAD -void * -isc__mem_get(isc_mem_t *mctx, size_t size FLARG) { - REQUIRE(ISCAPI_MCTX_VALID(mctx)); - - return (mctx->methods->memget(mctx, size FLARG_PASS)); -} - -void -isc__mem_put(isc_mem_t *mctx, void *ptr, size_t size FLARG) { - REQUIRE(ISCAPI_MCTX_VALID(mctx)); - - mctx->methods->memput(mctx, ptr, size FLARG_PASS); -} - -void -isc__mem_putanddetach(isc_mem_t **mctxp, void *ptr, size_t size FLARG) { - REQUIRE(mctxp != NULL && ISCAPI_MCTX_VALID(*mctxp)); - - (*mctxp)->methods->memputanddetach(mctxp, ptr, size FLARG_PASS); -} - -void * -isc__mem_allocate(isc_mem_t *mctx, size_t size FLARG) { - REQUIRE(ISCAPI_MCTX_VALID(mctx)); - - return (mctx->methods->memallocate(mctx, size FLARG_PASS)); -} - -void * -isc__mem_reallocate(isc_mem_t *mctx, void *ptr, size_t size FLARG) { - REQUIRE(ISCAPI_MCTX_VALID(mctx)); - - return (mctx->methods->memreallocate(mctx, ptr, size FLARG_PASS)); -} - -char * -isc__mem_strdup(isc_mem_t *mctx, const char *s FLARG) { - REQUIRE(ISCAPI_MCTX_VALID(mctx)); - - return (mctx->methods->memstrdup(mctx, s FLARG_PASS)); -} - -char * -isc__mem_strndup(isc_mem_t *mctx, const char *s, size_t size FLARG) { - REQUIRE(ISCAPI_MCTX_VALID(mctx)); - - return (mctx->methods->memstrndup(mctx, s, size FLARG_PASS)); -} - -void -isc__mem_free(isc_mem_t *mctx, void *ptr FLARG) { - REQUIRE(ISCAPI_MCTX_VALID(mctx)); - - mctx->methods->memfree(mctx, ptr FLARG_PASS); -} - -======= ->>>>>>> 532e5a3b4f (Remove an extra level of indirection in memory context routines) void isc__mem_printactive(isc_mem_t *ctx0, FILE *file) { #if ISC_MEM_TRACKLINES @@ -2512,51 +2683,52 @@ isc__mem_printactive(isc_mem_t *ctx0, FILE *file) { * Routines using default memory context */ -static isc_mem_t *isc__mem_mctx = NULL; - -void -isc__mem_initialize(void); -void -isc__mem_shutdown(void); - -ISC_CONSTRUCTOR(101) -void -isc__mem_initialize(void) { - REQUIRE(isc__mem_mctx == NULL); - isc_mem_create(&isc__mem_mctx); - isc_mem_setname(isc__mem_mctx, "default"); -} - -ISC_DESTRUCTOR(101) -void -isc__mem_shutdown(void) { - REQUIRE(isc__mem_mctx != NULL); - isc_mem_destroy(&isc__mem_mctx); - isc_mem_checkdestroyed(stderr); -} +LIBISC_EXTERNAL_DATA isc_mem_t *isc__mem_mctx = NULL; void * isc__malloc(size_t size FLARG) { + REQUIRE(isc__mem_mctx != NULL); return (isc__mem_allocate(isc__mem_mctx, size FLARG_PASS)); } void * isc__calloc(size_t num, size_t size FLARG) { + REQUIRE(isc__mem_mctx != NULL); return (isc__mem_callocate(isc__mem_mctx, num, size FLARG_PASS)); } void * isc__realloc(void *ptr, size_t size FLARG) { + REQUIRE(isc__mem_mctx != NULL); return (isc__mem_reallocate(isc__mem_mctx, ptr, size FLARG_PASS)); } void isc__free(void *ptr FLARG) { + REQUIRE(isc__mem_mctx != NULL); + /* + * The free function causes the space pointed to by ptr to be + * deallocated, that is, made available for further allocation. If ptr + * is a null pointer, no action occurs. Otherwise, if the argument does + * not match a pointer earlier returned by a memory management function, + * or if the space has been deallocated by a call to free or realloc, + * the behavior is undefined. + * [ISO9899 ยง 7.22.3.3] + * + * [ISO9899] + * ISO/IEC WG 9899:2011: Programming languages - C. + * International Organization for Standardization, Geneva, CH. + * http://www.open-std.org/jtc1/sc22/wg14/www/docs/n1548.pdf + */ + if (ptr == NULL) { + return; + } isc__mem_free(isc__mem_mctx, ptr FLARG_PASS); } char * isc__strdup(const char *s1 FLARG) { + REQUIRE(isc__mem_mctx != NULL); return (isc__mem_strdup(isc__mem_mctx, s1 FLARG_PASS)); } @@ -2564,3 +2736,9 @@ char * isc__strndup(const char *s1 FLARG) { return (isc__mem_strndup(isc__mem_mctx, s1 FLARG_PASS)); } + +isc_mem_t * +isc_get_default_mctx(void) { + REQUIRE(isc__mem_mctx != NULL); + return ((isc_mem_t *)isc__mem_mctx); +} diff --git a/lib/isc/tests/isctest.c b/lib/isc/tests/isctest.c index 66c70e5004..a333c16ed5 100644 --- a/lib/isc/tests/isctest.c +++ b/lib/isc/tests/isctest.c @@ -108,8 +108,8 @@ isc_test_begin(FILE *logfile, bool start_managers, unsigned int workers) { isc_mem_debugging |= ISC_MEM_DEBUGRECORD; - INSIST(test_mctx == NULL); - isc_mem_create(&test_mctx); + REQUIRE(test_mctx == NULL); + test_mctx = isc_get_default_mctx(); if (logfile != NULL) { isc_logdestination_t destination; @@ -156,9 +156,7 @@ isc_test_end(void) { if (test_lctx != NULL) { isc_log_destroy(&test_lctx); } - if (test_mctx != NULL) { - isc_mem_destroy(&test_mctx); - } + test_mctx = NULL; test_running = false; } diff --git a/lib/isc/tests/mem_test.c b/lib/isc/tests/mem_test.c index e096c54c67..708f08c549 100644 --- a/lib/isc/tests/mem_test.c +++ b/lib/isc/tests/mem_test.c @@ -64,6 +64,22 @@ _teardown(void **state) { #define MP2_FREEMAX 25 #define MP2_FILLCNT 25 +#define ALIGNMENT_SIZE sizeof(void *) + +static inline size_t +quantize(size_t size) { + /*! + * Round up the result in order to get a size big + * enough to satisfy the request and be aligned on ALIGNMENT_SIZE + * byte boundaries. + */ + + if (size == 0U) { + return (ALIGNMENT_SIZE); + } + return ((size + ALIGNMENT_SIZE - 1) & (~(ALIGNMENT_SIZE - 1))); +} + /* general memory system tests */ static void isc_mem_test(void **state) { @@ -72,7 +88,7 @@ isc_mem_test(void **state) { void *tmp; isc_mempool_t *mp1 = NULL, *mp2 = NULL; unsigned int i, j; - int rval; + int rval, mp1_freecount = 0; UNUSED(state); @@ -91,6 +107,12 @@ isc_mem_test(void **state) { assert_non_null(items1[i]); } + /* + * The number of free items on the list will be MAX(MP1_FREEMAX, + * MP1_MAXALLOC) aligned to page boundary. + */ + mp1_freecount = isc_mempool_getfreecount(mp1); + /* * Try to allocate one more. This should fail. */ @@ -107,7 +129,7 @@ isc_mem_test(void **state) { } rval = isc_mempool_getfreecount(mp1); - assert_int_equal(rval, 10); + assert_int_equal(rval, mp1_freecount + 11); rval = isc_mempool_getallocated(mp1); assert_int_equal(rval, 19); @@ -152,6 +174,9 @@ isc_mem_test(void **state) { isc_mempool_destroy(&mp1); } +#if !defined(USE_ALLOCATOR_SYSTEM) && !defined(USE_ALLOCATOR_JEMALLOC) && \ + !defined(USE_ALLOCATOR_TCMALLOC) + /* test TotalUse calculation */ static void isc_mem_total_test(void **state) { @@ -205,8 +230,7 @@ isc_mem_total_test(void **state) { static void isc_mem_inuse_test(void **state) { isc_mem_t *mctx2 = NULL; - size_t before, after; - ssize_t diff; + size_t before, meanwhile, after; void *ptr; UNUSED(state); @@ -216,16 +240,22 @@ isc_mem_inuse_test(void **state) { before = isc_mem_inuse(mctx2); ptr = isc_mem_allocate(mctx2, 1024000); + meanwhile = isc_mem_inuse(mctx2); isc_mem_free(mctx2, ptr); after = isc_mem_inuse(mctx2); - - diff = after - before; - - assert_int_equal(diff, 0); - isc_mem_destroy(&mctx2); + + fprintf(stderr, "%zu < %zu > %zu\n", before, meanwhile, after); + + assert_int_not_equal(before, meanwhile); + assert_true(before < meanwhile); + assert_int_not_equal(meanwhile, after); + assert_true(after < meanwhile); + assert_int_equal(after, before); } +#endif + #if ISC_MEM_TRACKLINES /* test mem with no flags */ @@ -497,11 +527,13 @@ main(void) { const struct CMUnitTest tests[] = { cmocka_unit_test_setup_teardown(isc_mem_test, _setup, _teardown), +#if !defined(USE_ALLOCATOR_SYSTEM) && !defined(USE_ALLOCATOR_JEMALLOC) && \ + !defined(USE_ALLOCATOR_TCMALLOC) cmocka_unit_test_setup_teardown(isc_mem_total_test, _setup, _teardown), cmocka_unit_test_setup_teardown(isc_mem_inuse_test, _setup, _teardown), - +#endif #if !defined(__SANITIZE_THREAD__) cmocka_unit_test_setup_teardown(isc_mem_benchmark, _setup, _teardown),