Compare commits

...

5 Commits

Author SHA1 Message Date
Ondřej Surý
dddc8b0c94 WIP: Make the isc_astack growable and start low 2021-12-09 18:16:35 +01:00
Ondřej Surý
df8d084c05 Enforce linking with jemalloc 2021-12-09 18:15:17 +01:00
Ondřej Surý
26de85e4c6 WIP: Disable internal malloc 2021-12-09 18:15:17 +01:00
Ondřej Surý
4e5f51dc88 WIP: Disable tuning large, enable tuning small 2021-12-09 18:15:17 +01:00
Ondřej Surý
cf5c21f3ed WIP: Reduce the memory used by hazard pointers 2021-12-09 13:33:56 +01:00
7 changed files with 310 additions and 90 deletions

120
configure vendored
View File

@@ -780,6 +780,8 @@ PANDOC
W3M
LN
ARFLAGS
JEMALLOC_LIBS
JEMALLOC_CFLAGS
XTARGETS
PKG_CONFIG_LIBDIR
PKG_CONFIG_PATH
@@ -967,6 +969,8 @@ LT_SYS_LIBRARY_PATH
PKG_CONFIG
PKG_CONFIG_PATH
PKG_CONFIG_LIBDIR
JEMALLOC_CFLAGS
JEMALLOC_LIBS
PYTHON
MAXMINDDB_CFLAGS
MAXMINDDB_LIBS
@@ -1687,7 +1691,7 @@ Optional Packages:
--with-libidn2=PATH enable IDN support using GNU libidn2
[yes|no(default)|path]
--with-cmocka=detect enable CMocka based tests (default is detect)
--with-tuning=ARG Specify server tuning (default or small)
--with-tuning=ARG Specify server tuning (default or large)
--with-dlopen=ARG support dynamically loadable DLZ and DYNDB drivers
--with-dnsrps-libname DNSRPS provider library name (librpz.so)
--with-dnsrps-dir path to DNSRPS provider library
@@ -1725,6 +1729,10 @@ Some influential environment variables:
directories to add to pkg-config's search path
PKG_CONFIG_LIBDIR
path overriding pkg-config's built-in search path
JEMALLOC_CFLAGS
C compiler flags for JEMALLOC, overriding pkg-config
JEMALLOC_LIBS
linker flags for JEMALLOC, overriding pkg-config
PYTHON path to python executable
MAXMINDDB_CFLAGS
C compiler flags for MAXMINDDB, overriding pkg-config
@@ -12321,6 +12329,104 @@ else
fi
# Enforce jemalloc
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for jemalloc" >&5
$as_echo_n "checking for jemalloc... " >&6; }
pkg_failed=no
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for jemalloc" >&5
$as_echo_n "checking for jemalloc... " >&6; }
if test -n "$JEMALLOC_CFLAGS"; then
pkg_cv_JEMALLOC_CFLAGS="$JEMALLOC_CFLAGS"
elif test -n "$PKG_CONFIG"; then
if test -n "$PKG_CONFIG" && \
{ { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"jemalloc\""; } >&5
($PKG_CONFIG --exists --print-errors "jemalloc") 2>&5
ac_status=$?
$as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
test $ac_status = 0; }; then
pkg_cv_JEMALLOC_CFLAGS=`$PKG_CONFIG --cflags "jemalloc" 2>/dev/null`
test "x$?" != "x0" && pkg_failed=yes
else
pkg_failed=yes
fi
else
pkg_failed=untried
fi
if test -n "$JEMALLOC_LIBS"; then
pkg_cv_JEMALLOC_LIBS="$JEMALLOC_LIBS"
elif test -n "$PKG_CONFIG"; then
if test -n "$PKG_CONFIG" && \
{ { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"jemalloc\""; } >&5
($PKG_CONFIG --exists --print-errors "jemalloc") 2>&5
ac_status=$?
$as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
test $ac_status = 0; }; then
pkg_cv_JEMALLOC_LIBS=`$PKG_CONFIG --libs "jemalloc" 2>/dev/null`
test "x$?" != "x0" && pkg_failed=yes
else
pkg_failed=yes
fi
else
pkg_failed=untried
fi
if test $pkg_failed = yes; then
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
$as_echo "no" >&6; }
if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
_pkg_short_errors_supported=yes
else
_pkg_short_errors_supported=no
fi
if test $_pkg_short_errors_supported = yes; then
JEMALLOC_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "jemalloc" 2>&1`
else
JEMALLOC_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "jemalloc" 2>&1`
fi
# Put the nasty error message in config.log where it belongs
echo "$JEMALLOC_PKG_ERRORS" >&5
as_fn_error $? "Package requirements (jemalloc) were not met:
$JEMALLOC_PKG_ERRORS
Consider adjusting the PKG_CONFIG_PATH environment variable if you
installed software in a non-standard prefix.
Alternatively, you may set the environment variables JEMALLOC_CFLAGS
and JEMALLOC_LIBS to avoid the need to call pkg-config.
See the pkg-config man page for more details." "$LINENO" 5
elif test $pkg_failed = untried; then
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
$as_echo "no" >&6; }
{ { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
is in your PATH or set the PKG_CONFIG environment variable to the full
path to pkg-config.
Alternatively, you may set the environment variables JEMALLOC_CFLAGS
and JEMALLOC_LIBS to avoid the need to call pkg-config.
See the pkg-config man page for more details.
To get pkg-config, see <http://pkg-config.freedesktop.org/>.
See \`config.log' for more details" "$LINENO" 5; }
else
JEMALLOC_CFLAGS=$pkg_cv_JEMALLOC_CFLAGS
JEMALLOC_LIBS=$pkg_cv_JEMALLOC_LIBS
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
$as_echo "yes" >&6; }
fi
CFLAGS="$JEMALLOC_CFLAGS $CFLAGS"
LDFLAGS="$JEMALLOC_LDFLAGS $LDFLAGS"
LIBS="$JEMALLOC_LIBS $LIBS"
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to enable fuzzing mode" >&5
$as_echo_n "checking whether to enable fuzzing mode... " >&6; }
case $enable_fuzzing in #(
@@ -22310,19 +22416,19 @@ done
if test "${with_tuning+set}" = set; then :
withval=$with_tuning;
else
with_tuning=no
with_tuning=small
fi
case $with_tuning in #(
small) :
{ $as_echo "$as_me:${as_lineno-$LINENO}: using small system tuning" >&5
$as_echo "$as_me: using small system tuning" >&6;} ;; #(
*) :
large) :
$as_echo "#define TUNE_LARGE 1" >>confdefs.h
{ $as_echo "$as_me:${as_lineno-$LINENO}: using default system tuning" >&5
{ $as_echo "$as_me:${as_lineno-$LINENO}: using small system tuning" >&5
$as_echo "$as_me: using small system tuning" >&6;} ;; #(
*) :
{ $as_echo "$as_me:${as_lineno-$LINENO}: using default system tuning" >&5
$as_echo "$as_me: using default system tuning" >&6;} ;;
esac

View File

@@ -117,6 +117,13 @@ AC_ARG_ENABLE([fuzzing],
[],
[enable_fuzzing=no])
# Enforce jemalloc
AC_MSG_CHECKING([for jemalloc])
PKG_CHECK_MODULES([JEMALLOC], [jemalloc])
CFLAGS="$JEMALLOC_CFLAGS $CFLAGS"
LDFLAGS="$JEMALLOC_LDFLAGS $LDFLAGS"
LIBS="$JEMALLOC_LIBS $LIBS"
AC_MSG_CHECKING([whether to enable fuzzing mode])
AS_CASE([$enable_fuzzing],
[no],[AC_MSG_RESULT([no])],
@@ -2299,13 +2306,13 @@ AC_CHECK_FUNCS(setlocale)
# [pairwise: --with-tuning=small, --without-tuning]
AC_ARG_WITH([tuning],
AS_HELP_STRING([--with-tuning=ARG],
[Specify server tuning (default or small)]),
[],[with_tuning=no])
[Specify server tuning (default or large)]),
[],[with_tuning=small])
AS_CASE([$with_tuning],
[small],[AC_MSG_NOTICE(using small system tuning)],
[AC_DEFINE(TUNE_LARGE, 1, [Define to use default system tuning.])
AC_MSG_NOTICE(using default system tuning)])
[large],[AC_DEFINE(TUNE_LARGE, 1, [Define to use default system tuning.])
AC_MSG_NOTICE(using small system tuning)],
[AC_MSG_NOTICE(using default system tuning)])
#
# was --enable-querytrace specified?

View File

@@ -19,65 +19,123 @@
#include <isc/types.h>
#include <isc/util.h>
struct isc_astack {
isc_mem_t *mctx;
typedef struct nodes {
size_t size;
size_t pos;
void **nodes;
} nodes_t;
struct isc_astack {
isc_mem_t *mctx;
isc_mutex_t lock;
uintptr_t nodes[];
size_t min_size;
size_t max_size;
nodes_t cur;
nodes_t old;
};
isc_astack_t *
isc_astack_new(isc_mem_t *mctx, size_t size) {
isc_astack_t *stack = isc_mem_get(
mctx, sizeof(isc_astack_t) + size * sizeof(uintptr_t));
static void
nodes_new(isc_mem_t *mctx, nodes_t *cur, size_t size) {
*cur = (nodes_t){ .size = size };
*stack = (isc_astack_t){
.size = size,
};
if (size > 0) {
cur->nodes = isc_mem_get(mctx, cur->size * sizeof(void *));
memset(cur->nodes, 0, cur->size * sizeof(void *));
}
}
static void
nodes_free(isc_mem_t *mctx, nodes_t *cur) {
REQUIRE(cur->pos == 0);
if (cur->size > 0) {
REQUIRE(cur->nodes != NULL);
isc_mem_put(mctx, cur->nodes, cur->size * sizeof(void *));
}
*cur = (nodes_t){ .size = 0 };
}
#define IS_POWEROF2(bits) (bits && !(bits & (bits - 1)))
isc_astack_t *
isc_astack_new(isc_mem_t *mctx, size_t min_size, size_t max_size) {
isc_astack_t *stack = isc_mem_get(mctx, sizeof(isc_astack_t));
REQUIRE(IS_POWEROF2(min_size));
REQUIRE(IS_POWEROF2(max_size));
*stack = (isc_astack_t){ .min_size = min_size, .max_size = max_size };
isc_mem_attach(mctx, &stack->mctx);
memset(stack->nodes, 0, size * sizeof(uintptr_t));
isc_mutex_init(&stack->lock);
nodes_new(mctx, &stack->cur, stack->min_size);
nodes_new(mctx, &stack->old, 0);
return (stack);
}
bool
isc_astack_trypush(isc_astack_t *stack, void *obj) {
if (!isc_mutex_trylock(&stack->lock)) {
if (stack->pos >= stack->size) {
if (isc_mutex_trylock(&stack->lock) != ISC_R_SUCCESS) {
return (false);
}
if (stack->cur.pos >= stack->cur.size) {
if (stack->old.size > 0) {
UNLOCK(&stack->lock);
return (false);
}
stack->nodes[stack->pos++] = (uintptr_t)obj;
UNLOCK(&stack->lock);
return (true);
} else {
return (false);
if (stack->cur.size * 2 > stack->max_size) {
UNLOCK(&stack->lock);
return (false);
}
stack->old = stack->cur;
nodes_new(stack->mctx, &stack->cur, stack->old.size * 2);
}
stack->cur.nodes[stack->cur.pos++] = obj;
UNLOCK(&stack->lock);
return (true);
}
void *
isc_astack_pop(isc_astack_t *stack) {
LOCK(&stack->lock);
uintptr_t rv;
if (stack->pos == 0) {
rv = 0;
void *rv;
if (isc_mutex_trylock(&stack->lock) != ISC_R_SUCCESS) {
return (false);
}
if (stack->old.size > 0) {
REQUIRE(stack->old.pos > 0);
rv = stack->old.nodes[--stack->old.pos];
if (stack->old.pos == 0) {
nodes_free(stack->mctx, &stack->old);
}
} else if (stack->cur.pos > 0) {
rv = stack->cur.nodes[--stack->cur.pos];
} else {
rv = stack->nodes[--stack->pos];
rv = NULL;
}
UNLOCK(&stack->lock);
return ((void *)rv);
return (rv);
}
void
isc_astack_destroy(isc_astack_t *stack) {
LOCK(&stack->lock);
REQUIRE(stack->pos == 0);
REQUIRE(stack->cur.pos == 0);
REQUIRE(stack->old.size == 0);
UNLOCK(&stack->lock);
nodes_free(stack->mctx, &stack->cur);
isc_mutex_destroy(&stack->lock);
isc_mem_putanddetach(&stack->mctx, stack,
sizeof(struct isc_astack) +
stack->size * sizeof(uintptr_t));
isc_mem_putanddetach(&stack->mctx, stack, sizeof(struct isc_astack));
}

View File

@@ -45,6 +45,7 @@
#include <inttypes.h>
#include <isc/align.h>
#include <isc/atomic.h>
#include <isc/hp.h>
#include <isc/mem.h>
@@ -53,14 +54,12 @@
#include <isc/thread.h>
#include <isc/util.h>
#define HP_MAX_THREADS 128
static int isc__hp_max_threads = HP_MAX_THREADS;
#define HP_MAX_HPS 4 /* This is named 'K' in the HP paper */
#define CLPAD (128 / sizeof(uintptr_t))
#define HP_THRESHOLD_R 0 /* This is named 'R' in the HP paper */
#define CACHELINE_SIZE 64
/* Maximum number of retired objects per thread */
static int isc__hp_max_retired = HP_MAX_THREADS * HP_MAX_HPS;
static int isc__hp_max_threads = 1;
#define HP_MAX_HPS 4 /* This is named 'K' in the HP paper */
#define CLPAD (CACHELINE_SIZE / sizeof(uintptr_t))
#define HP_THRESHOLD_R 0 /* This is named 'R' in the HP paper */
typedef struct retirelist {
int size;
@@ -69,10 +68,11 @@ typedef struct retirelist {
struct isc_hp {
int max_hps;
int max_retired;
isc_mem_t *mctx;
atomic_uintptr_t **hp;
retirelist_t **rl;
isc_hp_deletefunc_t *deletefunc;
alignas(CACHELINE_SIZE) atomic_uintptr_t **hp;
alignas(CACHELINE_SIZE) retirelist_t **rl;
};
static inline int
@@ -82,35 +82,72 @@ tid(void) {
void
isc_hp_init(int max_threads) {
REQUIRE(max_threads > 0);
if (isc__hp_max_threads > max_threads) {
return;
}
isc__hp_max_threads = max_threads;
isc__hp_max_retired = max_threads * HP_MAX_HPS;
}
static size_t
hp_clpad(size_t max_hps) {
size_t hp_size = max_hps * sizeof(atomic_uintptr_t);
size_t hp_padding = 0;
while (hp_size > CACHELINE_SIZE) {
hp_size -= CACHELINE_SIZE;
}
if (hp_size > 0) {
hp_padding = CACHELINE_SIZE / hp_size;
}
return (hp_size + hp_padding);
}
isc_hp_t *
isc_hp_new(isc_mem_t *mctx, size_t max_hps, isc_hp_deletefunc_t *deletefunc) {
isc_hp_t *hp = isc_mem_get(mctx, sizeof(*hp));
REQUIRE(isc__hp_max_threads > 0);
REQUIRE(max_hps <= HP_MAX_HPS);
if (max_hps == 0) {
max_hps = HP_MAX_HPS;
}
*hp = (isc_hp_t){ .max_hps = max_hps, .deletefunc = deletefunc };
*hp = (isc_hp_t){
.max_hps = max_hps,
.max_retired = isc__hp_max_threads * max_hps,
.deletefunc = deletefunc,
};
isc_mem_attach(mctx, &hp->mctx);
hp->hp = isc_mem_get(mctx, isc__hp_max_threads * sizeof(hp->hp[0]));
hp->rl = isc_mem_get(mctx, isc__hp_max_threads * sizeof(hp->rl[0]));
for (int i = 0; i < isc__hp_max_threads; i++) {
hp->hp[i] = isc_mem_get(mctx, CLPAD * 2 * sizeof(hp->hp[i][0]));
hp->rl[i] = isc_mem_get(mctx, sizeof(*hp->rl[0]));
*hp->rl[i] = (retirelist_t){ .size = 0 };
hp->hp[i] = isc_mem_get(mctx, hp_clpad(hp->max_hps));
for (int j = 0; j < hp->max_hps; j++) {
atomic_init(&hp->hp[i][j], 0);
}
hp->rl[i]->list = isc_mem_get(
hp->mctx, isc__hp_max_retired * sizeof(uintptr_t));
}
/*
* It's not nice that we have a lot of empty space, but we need padding
* to avoid false sharing.
*/
hp->rl = isc_mem_get(mctx,
(isc__hp_max_threads * CLPAD) * sizeof(hp->rl[0]));
for (int i = 0; i < isc__hp_max_threads; i++) {
retirelist_t *rl;
rl = isc_mem_get(mctx, sizeof(*rl));
rl->size = 0;
rl->list = isc_mem_get(hp->mctx,
hp->max_retired * sizeof(uintptr_t));
hp->rl[i * CLPAD] = rl;
}
return (hp);
@@ -119,19 +156,22 @@ isc_hp_new(isc_mem_t *mctx, size_t max_hps, isc_hp_deletefunc_t *deletefunc) {
void
isc_hp_destroy(isc_hp_t *hp) {
for (int i = 0; i < isc__hp_max_threads; i++) {
isc_mem_put(hp->mctx, hp->hp[i],
CLPAD * 2 * sizeof(hp->hp[i][0]));
retirelist_t *rl = hp->rl[i * CLPAD];
for (int j = 0; j < hp->rl[i]->size; j++) {
void *data = (void *)hp->rl[i]->list[j];
for (int j = 0; j < rl->size; j++) {
void *data = (void *)rl->list[j];
hp->deletefunc(data);
}
isc_mem_put(hp->mctx, hp->rl[i]->list,
isc__hp_max_retired * sizeof(uintptr_t));
isc_mem_put(hp->mctx, hp->rl[i], sizeof(*hp->rl[0]));
isc_mem_put(hp->mctx, rl->list,
hp->max_retired * sizeof(uintptr_t));
isc_mem_put(hp->mctx, rl, sizeof(*rl));
}
for (int i = 0; i < isc__hp_max_threads; i++) {
isc_mem_put(hp->mctx, hp->hp[i], hp_clpad(hp->max_hps));
}
isc_mem_put(hp->mctx, hp->hp, isc__hp_max_threads * sizeof(hp->hp[0]));
isc_mem_put(hp->mctx, hp->rl, isc__hp_max_threads * sizeof(hp->rl[0]));
isc_mem_put(hp->mctx, hp->rl,
(isc__hp_max_threads * CLPAD) * sizeof(hp->rl[0]));
isc_mem_putanddetach(&hp->mctx, hp, sizeof(*hp));
}
@@ -173,15 +213,16 @@ isc_hp_protect_release(isc_hp_t *hp, int ihp, atomic_uintptr_t ptr) {
void
isc_hp_retire(isc_hp_t *hp, uintptr_t ptr) {
hp->rl[tid()]->list[hp->rl[tid()]->size++] = ptr;
INSIST(hp->rl[tid()]->size < isc__hp_max_retired);
retirelist_t *rl = hp->rl[tid() * CLPAD];
rl->list[rl->size++] = ptr;
INSIST(rl->size < hp->max_retired);
if (hp->rl[tid()]->size < HP_THRESHOLD_R) {
if (rl->size < HP_THRESHOLD_R) {
return;
}
for (int iret = 0; iret < hp->rl[tid()]->size; iret++) {
uintptr_t obj = hp->rl[tid()]->list[iret];
for (int iret = 0; iret < rl->size; iret++) {
uintptr_t obj = rl->list[iret];
bool can_delete = true;
for (int itid = 0; itid < isc__hp_max_threads && can_delete;
itid++) {
@@ -194,11 +235,9 @@ isc_hp_retire(isc_hp_t *hp, uintptr_t ptr) {
}
if (can_delete) {
size_t bytes = (hp->rl[tid()]->size - iret) *
sizeof(hp->rl[tid()]->list[0]);
memmove(&hp->rl[tid()]->list[iret],
&hp->rl[tid()]->list[iret + 1], bytes);
hp->rl[tid()]->size--;
size_t bytes = (rl->size - iret) * sizeof(rl->list[0]);
memmove(&rl->list[iret], &rl->list[iret + 1], bytes);
rl->size--;
hp->deletefunc((void *)obj);
}
}

View File

@@ -17,9 +17,10 @@
#include <isc/types.h>
isc_astack_t *
isc_astack_new(isc_mem_t *mctx, size_t size);
isc_astack_new(isc_mem_t *mctx, size_t min_size, size_t max_size);
/*%<
* Allocate and initialize a new array stack of size 'size'.
* Allocate and initialize a new bounded array stack. Initially, min_size
* entries will be allocated, but would grow up to max_size entries.
*/
void

View File

@@ -117,7 +117,7 @@ LIBISC_EXTERNAL_DATA extern unsigned int isc_mem_defaultflags;
*/
#if !defined(ISC_MEM_USE_INTERNAL_MALLOC) && !__SANITIZE_ADDRESS__
#define ISC_MEM_USE_INTERNAL_MALLOC 1
#define ISC_MEM_USE_INTERNAL_MALLOC 0
#endif /* ifndef ISC_MEM_USE_INTERNAL_MALLOC */
/*

View File

@@ -48,8 +48,11 @@
* How many isc_nmhandles and isc_nm_uvreqs will we be
* caching for reuse in a socket.
*/
#define ISC_NM_HANDLES_STACK_SIZE 600
#define ISC_NM_REQS_STACK_SIZE 600
#define ISC_NM_HANDLES_MIN_STACK_SIZE 32
#define ISC_NM_HANDLES_MAX_STACK_SIZE 1024
#define ISC_NM_REQS_MIN_STACK_SIZE 32
#define ISC_NM_REQS_MAX_STACK_SIZE 1024
/*%
* Shortcut index arrays to get access to statistics counters.
@@ -1441,14 +1444,20 @@ isc___nmsocket_init(isc_nmsocket_t *sock, isc_nm_t *mgr, isc_nmsocket_type type,
family = iface->type.sa.sa_family;
*sock = (isc_nmsocket_t){ .type = type,
.iface = *iface,
.fd = -1,
.ah_size = 32,
.inactivehandles = isc_astack_new(
mgr->mctx, ISC_NM_HANDLES_STACK_SIZE),
.inactivereqs = isc_astack_new(
mgr->mctx, ISC_NM_REQS_STACK_SIZE) };
*sock = (isc_nmsocket_t){
.type = type,
.iface = *iface,
.fd = -1,
.ah_size = 32,
};
sock->inactivehandles = isc_astack_new(mgr->mctx,
ISC_NM_HANDLES_MIN_STACK_SIZE,
ISC_NM_HANDLES_MAX_STACK_SIZE);
sock->inactivereqs = isc_astack_new(mgr->mctx,
ISC_NM_REQS_MIN_STACK_SIZE,
ISC_NM_REQS_MAX_STACK_SIZE);
#if NETMGR_TRACE
sock->backtrace_size = backtrace(sock->backtrace, TRACE_SIZE);