Revert "WIP: Reduce UDP mmsg buffer"

This reverts commit 88330e5d16.

lib/isc/astack.c

@@ -19,65 +19,123 @@
 #include <isc/types.h>
 #include <isc/util.h>
 
-struct isc_astack {
-	isc_mem_t *mctx;
+typedef struct nodes {
 	size_t size;
 	size_t pos;
+	void **nodes;
+} nodes_t;
+
+struct isc_astack {
+	isc_mem_t *mctx;
 	isc_mutex_t lock;
-	uintptr_t nodes[];
+	size_t min_size;
+	size_t max_size;
+	nodes_t cur;
+	nodes_t old;
 };
 
-isc_astack_t *
-isc_astack_new(isc_mem_t *mctx, size_t size) {
-	isc_astack_t *stack = isc_mem_get(
-		mctx, sizeof(isc_astack_t) + size * sizeof(uintptr_t));
+static void
+nodes_new(isc_mem_t *mctx, nodes_t *cur, size_t size) {
+	*cur = (nodes_t){ .size = size };
 
-	*stack = (isc_astack_t){
-		.size = size,
-	};
+	if (size > 0) {
+		cur->nodes = isc_mem_get(mctx, cur->size * sizeof(void *));
+		memset(cur->nodes, 0, cur->size * sizeof(void *));
+	}
+}
+
+static void
+nodes_free(isc_mem_t *mctx, nodes_t *cur) {
+	REQUIRE(cur->pos == 0);
+
+	if (cur->size > 0) {
+		REQUIRE(cur->nodes != NULL);
+		isc_mem_put(mctx, cur->nodes, cur->size * sizeof(void *));
+	}
+
+	*cur = (nodes_t){ .size = 0 };
+}
+
+#define IS_POWEROF2(bits) (bits && !(bits & (bits - 1)))
+
+isc_astack_t *
+isc_astack_new(isc_mem_t *mctx, size_t min_size, size_t max_size) {
+	isc_astack_t *stack = isc_mem_get(mctx, sizeof(isc_astack_t));
+
+	REQUIRE(IS_POWEROF2(min_size));
+	REQUIRE(IS_POWEROF2(max_size));
+
+	*stack = (isc_astack_t){ .min_size = min_size, .max_size = max_size };
 	isc_mem_attach(mctx, &stack->mctx);
-	memset(stack->nodes, 0, size * sizeof(uintptr_t));
 	isc_mutex_init(&stack->lock);
+
+	nodes_new(mctx, &stack->cur, stack->min_size);
+	nodes_new(mctx, &stack->old, 0);
+
 	return (stack);
 }
 
 bool
 isc_astack_trypush(isc_astack_t *stack, void *obj) {
-	if (!isc_mutex_trylock(&stack->lock)) {
-		if (stack->pos >= stack->size) {
+	if (isc_mutex_trylock(&stack->lock) != ISC_R_SUCCESS) {
+		return (false);
+	}
+
+	if (stack->cur.pos >= stack->cur.size) {
+		if (stack->old.size > 0) {
 			UNLOCK(&stack->lock);
 			return (false);
 		}
-		stack->nodes[stack->pos++] = (uintptr_t)obj;
-		UNLOCK(&stack->lock);
-		return (true);
-	} else {
-		return (false);
+
+		if (stack->cur.size * 2 > stack->max_size) {
+			UNLOCK(&stack->lock);
+			return (false);
+		}
+
+		stack->old = stack->cur;
+		nodes_new(stack->mctx, &stack->cur, stack->old.size * 2);
 	}
+
+	stack->cur.nodes[stack->cur.pos++] = obj;
+	UNLOCK(&stack->lock);
+	return (true);
 }
 
 void *
 isc_astack_pop(isc_astack_t *stack) {
-	LOCK(&stack->lock);
-	uintptr_t rv;
-	if (stack->pos == 0) {
-		rv = 0;
+	void *rv;
+
+	if (isc_mutex_trylock(&stack->lock) != ISC_R_SUCCESS) {
+		return (false);
+	}
+
+	if (stack->old.size > 0) {
+		REQUIRE(stack->old.pos > 0);
+
+		rv = stack->old.nodes[--stack->old.pos];
+
+		if (stack->old.pos == 0) {
+			nodes_free(stack->mctx, &stack->old);
+		}
+	} else if (stack->cur.pos > 0) {
+		rv = stack->cur.nodes[--stack->cur.pos];
 	} else {
-		rv = stack->nodes[--stack->pos];
+		rv = NULL;
 	}
 	UNLOCK(&stack->lock);
-	return ((void *)rv);
+	return (rv);
 }
 
 void
 isc_astack_destroy(isc_astack_t *stack) {
 	LOCK(&stack->lock);
-	REQUIRE(stack->pos == 0);
+	REQUIRE(stack->cur.pos == 0);
+	REQUIRE(stack->old.size == 0);
 	UNLOCK(&stack->lock);
 
+	nodes_free(stack->mctx, &stack->cur);
+
 	isc_mutex_destroy(&stack->lock);
 
-	isc_mem_putanddetach(&stack->mctx, stack,
-			     sizeof(struct isc_astack) +
-				     stack->size * sizeof(uintptr_t));
+	isc_mem_putanddetach(&stack->mctx, stack, sizeof(struct isc_astack));
 }

lib/isc/hp.c

@@ -45,6 +45,7 @@
 
 #include <inttypes.h>
 
+#include <isc/align.h>
 #include <isc/atomic.h>
 #include <isc/hp.h>
 #include <isc/mem.h>
@@ -53,15 +54,13 @@
 #include <isc/thread.h>
 #include <isc/util.h>
 
-#define HP_MAX_THREADS 128
+#define CACHELINE_SIZE 64
+
 static int isc__hp_max_threads = 1;
 #define HP_MAX_HPS     4 /* This is named 'K' in the HP paper */
-#define CLPAD	       (128 / sizeof(uintptr_t))
+#define CLPAD	       (CACHELINE_SIZE / sizeof(uintptr_t))
 #define HP_THRESHOLD_R 0 /* This is named 'R' in the HP paper */
 
-/* Maximum number of retired objects per thread */
-static int isc__hp_max_retired = HP_MAX_THREADS * HP_MAX_HPS;
-
 typedef struct retirelist {
 	int size;
 	uintptr_t *list;
@@ -69,10 +68,11 @@ typedef struct retirelist {
 
 struct isc_hp {
 	int max_hps;
+	int max_retired;
 	isc_mem_t *mctx;
-	atomic_uintptr_t **hp;
-	retirelist_t **rl;
 	isc_hp_deletefunc_t *deletefunc;
+	alignas(CACHELINE_SIZE) atomic_uintptr_t **hp;
+	alignas(CACHELINE_SIZE) retirelist_t **rl;
 };
 
 static inline int
@@ -89,7 +89,20 @@ isc_hp_init(int max_threads) {
 	}
 
 	isc__hp_max_threads = max_threads;
-	isc__hp_max_retired = max_threads * HP_MAX_HPS;
+}
+
+static size_t
+hp_clpad(size_t max_hps) {
+	size_t hp_size = max_hps * sizeof(atomic_uintptr_t);
+	size_t hp_padding = 0;
+	while (hp_size > CACHELINE_SIZE) {
+		hp_size -= CACHELINE_SIZE;
+	}
+	if (hp_size > 0) {
+		hp_padding = CACHELINE_SIZE / hp_size;
+	}
+
+	return (hp_size + hp_padding);
 }
 
 isc_hp_t *
@@ -103,23 +116,38 @@ isc_hp_new(isc_mem_t *mctx, size_t max_hps, isc_hp_deletefunc_t *deletefunc) {
 		max_hps = HP_MAX_HPS;
 	}
 
-	*hp = (isc_hp_t){ .max_hps = max_hps, .deletefunc = deletefunc };
+	*hp = (isc_hp_t){
+		.max_hps = max_hps,
+		.max_retired = isc__hp_max_threads * max_hps,
+		.deletefunc = deletefunc,
+	};
 
 	isc_mem_attach(mctx, &hp->mctx);
 
 	hp->hp = isc_mem_get(mctx, isc__hp_max_threads * sizeof(hp->hp[0]));
-	hp->rl = isc_mem_get(mctx, isc__hp_max_threads * sizeof(hp->rl[0]));
-
 	for (int i = 0; i < isc__hp_max_threads; i++) {
-		hp->hp[i] = isc_mem_get(mctx, CLPAD * 2 * sizeof(hp->hp[i][0]));
-		hp->rl[i] = isc_mem_get(mctx, sizeof(*hp->rl[0]));
-		*hp->rl[i] = (retirelist_t){ .size = 0 };
-
+		hp->hp[i] = isc_mem_get(mctx, hp_clpad(hp->max_hps));
 		for (int j = 0; j < hp->max_hps; j++) {
 			atomic_init(&hp->hp[i][j], 0);
 		}
-		hp->rl[i]->list = isc_mem_get(
-			hp->mctx, isc__hp_max_retired * sizeof(uintptr_t));
+	}
+
+	/*
+	 * It's not nice that we have a lot of empty space, but we need padding
+	 * to avoid false sharing.
+	 */
+	hp->rl = isc_mem_get(mctx,
+			     (isc__hp_max_threads * CLPAD) * sizeof(hp->rl[0]));
+
+	for (int i = 0; i < isc__hp_max_threads; i++) {
+		retirelist_t *rl;
+
+		rl = isc_mem_get(mctx, sizeof(*rl));
+		rl->size = 0;
+		rl->list = isc_mem_get(hp->mctx,
+				       hp->max_retired * sizeof(uintptr_t));
+
+		hp->rl[i * CLPAD] = rl;
 	}
 
 	return (hp);
@@ -128,19 +156,22 @@ isc_hp_new(isc_mem_t *mctx, size_t max_hps, isc_hp_deletefunc_t *deletefunc) {
 void
 isc_hp_destroy(isc_hp_t *hp) {
 	for (int i = 0; i < isc__hp_max_threads; i++) {
-		isc_mem_put(hp->mctx, hp->hp[i],
-			    CLPAD * 2 * sizeof(hp->hp[i][0]));
+		retirelist_t *rl = hp->rl[i * CLPAD];
 
-		for (int j = 0; j < hp->rl[i]->size; j++) {
-			void *data = (void *)hp->rl[i]->list[j];
+		for (int j = 0; j < rl->size; j++) {
+			void *data = (void *)rl->list[j];
 			hp->deletefunc(data);
 		}
-		isc_mem_put(hp->mctx, hp->rl[i]->list,
-			    isc__hp_max_retired * sizeof(uintptr_t));
-		isc_mem_put(hp->mctx, hp->rl[i], sizeof(*hp->rl[0]));
+		isc_mem_put(hp->mctx, rl->list,
+			    hp->max_retired * sizeof(uintptr_t));
+		isc_mem_put(hp->mctx, rl, sizeof(*rl));
+	}
+	for (int i = 0; i < isc__hp_max_threads; i++) {
+		isc_mem_put(hp->mctx, hp->hp[i], hp_clpad(hp->max_hps));
 	}
 	isc_mem_put(hp->mctx, hp->hp, isc__hp_max_threads * sizeof(hp->hp[0]));
-	isc_mem_put(hp->mctx, hp->rl, isc__hp_max_threads * sizeof(hp->rl[0]));
+	isc_mem_put(hp->mctx, hp->rl,
+		    (isc__hp_max_threads * CLPAD) * sizeof(hp->rl[0]));
 
 	isc_mem_putanddetach(&hp->mctx, hp, sizeof(*hp));
 }
@@ -182,15 +213,16 @@ isc_hp_protect_release(isc_hp_t *hp, int ihp, atomic_uintptr_t ptr) {
 
 void
 isc_hp_retire(isc_hp_t *hp, uintptr_t ptr) {
-	hp->rl[tid()]->list[hp->rl[tid()]->size++] = ptr;
-	INSIST(hp->rl[tid()]->size < isc__hp_max_retired);
+	retirelist_t *rl = hp->rl[tid() * CLPAD];
+	rl->list[rl->size++] = ptr;
+	INSIST(rl->size < hp->max_retired);
 
-	if (hp->rl[tid()]->size < HP_THRESHOLD_R) {
+	if (rl->size < HP_THRESHOLD_R) {
 		return;
 	}
 
-	for (int iret = 0; iret < hp->rl[tid()]->size; iret++) {
-		uintptr_t obj = hp->rl[tid()]->list[iret];
+	for (int iret = 0; iret < rl->size; iret++) {
+		uintptr_t obj = rl->list[iret];
 		bool can_delete = true;
 		for (int itid = 0; itid < isc__hp_max_threads && can_delete;
 		     itid++) {
@@ -203,11 +235,9 @@ isc_hp_retire(isc_hp_t *hp, uintptr_t ptr) {
 		}
 
 		if (can_delete) {
-			size_t bytes = (hp->rl[tid()]->size - iret) *
-				       sizeof(hp->rl[tid()]->list[0]);
-			memmove(&hp->rl[tid()]->list[iret],
-				&hp->rl[tid()]->list[iret + 1], bytes);
-			hp->rl[tid()]->size--;
+			size_t bytes = (rl->size - iret) * sizeof(rl->list[0]);
+			memmove(&rl->list[iret], &rl->list[iret + 1], bytes);
+			rl->size--;
 			hp->deletefunc((void *)obj);
 		}
 	}

lib/isc/include/isc/astack.h

@@ -17,9 +17,10 @@
 #include <isc/types.h>
 
 isc_astack_t *
-isc_astack_new(isc_mem_t *mctx, size_t size);
+isc_astack_new(isc_mem_t *mctx, size_t min_size, size_t max_size);
 /*%<
- * Allocate and initialize a new array stack of size 'size'.
+ * Allocate and initialize a new bounded array stack.  Initially, min_size
+ * entries will be allocated, but would grow up to max_size entries.
  */
 
 void

lib/isc/netmgr/netmgr.c

@@ -47,8 +47,11 @@
  * How many isc_nmhandles and isc_nm_uvreqs will we be
  * caching for reuse in a socket.
  */
-#define ISC_NM_HANDLES_STACK_SIZE 600
-#define ISC_NM_REQS_STACK_SIZE	  600
+#define ISC_NM_HANDLES_MIN_STACK_SIZE 32
+#define ISC_NM_HANDLES_MAX_STACK_SIZE 1024
+
+#define ISC_NM_REQS_MIN_STACK_SIZE 32
+#define ISC_NM_REQS_MAX_STACK_SIZE 1024
 
 /*%
  * Shortcut index arrays to get access to statistics counters.
@@ -1460,13 +1463,19 @@ isc___nmsocket_init(isc_nmsocket_t *sock, isc_nm_t *mgr, isc_nmsocket_type type,
 	REQUIRE(sock != NULL);
 	REQUIRE(mgr != NULL);
 
-	*sock = (isc_nmsocket_t){ .type = type,
-				  .fd = -1,
-				  .ah_size = 32,
-				  .inactivehandles = isc_astack_new(
-					  mgr->mctx, ISC_NM_HANDLES_STACK_SIZE),
-				  .inactivereqs = isc_astack_new(
-					  mgr->mctx, ISC_NM_REQS_STACK_SIZE) };
+	*sock = (isc_nmsocket_t){
+		.type = type,
+		.fd = -1,
+		.ah_size = 32,
+	};
+
+	sock->inactivehandles = isc_astack_new(mgr->mctx,
+					       ISC_NM_HANDLES_MIN_STACK_SIZE,
+					       ISC_NM_HANDLES_MAX_STACK_SIZE);
+
+	sock->inactivereqs = isc_astack_new(mgr->mctx,
+					    ISC_NM_REQS_MIN_STACK_SIZE,
+					    ISC_NM_REQS_MAX_STACK_SIZE);
 
 	if (iface != NULL) {
 		family = iface->type.sa.sa_family;