Hold on segmented indexes for fixing sanitizer issues

Add a simple implementation of skiplists for efficient numerical range
based operations. They are currently implemented naively for the
numerical key value with an exponential segmented array. Unlike a heap,
they do not have any cascading cost.

lib/dns/include/dns/rdataslab.h

@@ -61,9 +61,9 @@ ISC_LANG_BEGINDECLS
 #define DNS_RDATASLAB_OFFLINE 0x01 /* RRSIG is for offline DNSKEY */
 
 struct dns_slabheader_proof {
-	dns_name_t	name;
-	void	       *neg;
-	void	       *negsig;
+	dns_name_t name;
+	void *neg;
+	void *negsig;
 	dns_rdatatype_t type;
 };
 
@@ -71,19 +71,16 @@ struct dns_slabheader {
 	/*%
 	 * Locked by the owning node's lock.
 	 */
-	uint32_t	      serial;
-	dns_ttl_t	      ttl;
-	dns_typepair_t	      type;
+	uint32_t serial;
+	dns_ttl_t ttl;
+	dns_typepair_t type;
 	atomic_uint_least16_t attributes;
-	dns_trust_t	      trust;
+	dns_trust_t trust;
 
 	unsigned int heap_index;
-	/*%<
-	 * Used for TTL-based cache cleaning.
-	 */
 
 	isc_stdtime_t resign;
-	unsigned int  resign_lsb : 1;
+	unsigned int resign_lsb : 1;
 
 	atomic_uint_fast16_t count;
 	/*%<
@@ -115,7 +112,7 @@ struct dns_slabheader {
 	 * this rdataset.
 	 */
 
-	dns_db_t     *db;
+	dns_db_t *db;
 	dns_dbnode_t *node;
 	/*%<
 	 * The database and database node objects containing
@@ -132,8 +129,13 @@ struct dns_slabheader {
 	 */
 	unsigned char upper[32];
 
-	isc_heap_t	   *heap;
-	dns_glue_t	   *glue_list;
+	uint64_t ttl_index;
+	/*%<
+	 * Used for TTL-based cache cleaning.
+	 */
+
+	isc_heap_t *heap;
+	dns_glue_t *glue_list;
 	struct cds_wfs_node wfs_node;
 };
 

lib/dns/qpcache.c

@@ -24,7 +24,6 @@
 #include <isc/file.h>
 #include <isc/hash.h>
 #include <isc/hashmap.h>
-#include <isc/heap.h>
 #include <isc/hex.h>
 #include <isc/loop.h>
 #include <isc/mem.h>
@@ -34,6 +33,7 @@
 #include <isc/refcount.h>
 #include <isc/result.h>
 #include <isc/rwlock.h>
+#include <isc/skiplist.h>
 #include <isc/stdio.h>
 #include <isc/string.h>
 #include <isc/time.h>
@@ -153,7 +153,7 @@
 #define DNS_QPDB_LRUUPDATE_REGULAR 600
 
 /*%
- * Number of buckets for cache DB entries (locks, LRU lists, TTL heaps).
+ * Number of buckets for cache DB entries (locks, LRU lists).
  * There is a tradeoff issue about configuring this value: if this is too
  * small, it may cause heavier contention between threads; if this is too large,
  * LRU purge algorithm won't work well (entries tend to be purged prematurely).
@@ -269,13 +269,13 @@ struct qpcache {
 	qpcnodelist_t *deadnodes;
 
 	/*
-	 * Heaps.  These are used for TTL based expiry in a cache,
-	 * or for zone resigning in a zone DB.  hmctx is the memory
-	 * context to use for the heap (which differs from the main
-	 * database memory context in the case of a cache).
+	 * A skiplist is used for TTL based expiry in a cache, or for zone
+	 * resigning in a zone DB. sctx is the memory context to use for the
+	 * skiplist (which differs from the main database memory context in the
+	 * case of a cache).
 	 */
-	isc_mem_t *hmctx;
-	isc_heap_t **heaps;
+	isc_mem_t *smctx;
+	isc_skiplist_t *slist;
 
 	/* Locked by tree_lock. */
 	dns_qp_t *tree;
@@ -506,7 +506,7 @@ need_headerupdate(dns_slabheader_t *header, isc_stdtime_t now) {
  *
  * Caller must hold the node (write) lock.
  *
- * Note that the we do NOT touch the heap here, as the TTL has not changed.
+ * Note that the we do NOT touch the skiplist here, as the TTL has not changed.
  */
 static void
 update_header(qpcache_t *qpdb, dns_slabheader_t *header, isc_stdtime_t now) {
@@ -556,7 +556,7 @@ clean_cache_node(qpcache_t *qpdb, qpcnode_t *node) {
 	dns_slabheader_t *current = NULL, *top_prev = NULL, *top_next = NULL;
 
 	/*
-	 * Caller must be holding the node lock.
+	 * Caller must be holding the node lock.qpcac
 	 */
 
 	for (current = node->data; current != NULL; current = top_next) {
@@ -910,30 +910,35 @@ mark(dns_slabheader_t *header, uint_least16_t flag) {
 static void
 setttl(dns_slabheader_t *header, dns_ttl_t newttl) {
 	dns_ttl_t oldttl = header->ttl;
+	qpcache_t *qpdb;
+
+	/*
+	 * This is a cache. Adjust the skiplist if necessary.
+	 */
+	if (newttl == oldttl) {
+		return;
+	}
+
+	if (header->db == NULL || !dns_db_iscache(header->db) ||
+	    header->ttl_index == 0)
+	{
+		header->ttl = newttl;
+		return;
+	}
+
+	qpdb = (qpcache_t *)header->db;
+
+	if (newttl == 0) {
+		isc_skiplist_delete(qpdb->slist, header, header->ttl_index);
+		header->ttl = 0;
+		header->ttl_index = 0;
+		return;
+	}
 
 	header->ttl = newttl;
 
-	if (header->db == NULL || !dns_db_iscache(header->db)) {
-		return;
-	}
-
-	/*
-	 * This is a cache. Adjust the heaps if necessary.
-	 */
-	if (header->heap == NULL || header->heap_index == 0 || newttl == oldttl)
-	{
-		return;
-	}
-
-	if (newttl < oldttl) {
-		isc_heap_increased(header->heap, header->heap_index);
-	} else {
-		isc_heap_decreased(header->heap, header->heap_index);
-	}
-
-	if (newttl == 0) {
-		isc_heap_delete(header->heap, header->heap_index);
-	}
+	fprintf(stderr, "%%%% setttl!\n");
+	header->ttl_index = isc_skiplist_insert(qpdb->slist, header);
 }
 
 /*
@@ -942,13 +947,14 @@ setttl(dns_slabheader_t *header, dns_ttl_t newttl) {
 static void
 expireheader(dns_slabheader_t *header, isc_rwlocktype_t *nlocktypep,
 	     isc_rwlocktype_t *tlocktypep, dns_expire_t reason DNS__DB_FLARG) {
-	setttl(header, 0);
+	// setttl(header, 0);
+	header->ttl = 0;
+	qpcache_t *qpdb = (qpcache_t *)header->db;
+
 	mark(header, DNS_SLABHEADERATTR_ANCIENT);
 	HEADERNODE(header)->dirty = 1;
 
 	if (isc_refcount_current(&HEADERNODE(header)->erefs) == 0) {
-		qpcache_t *qpdb = (qpcache_t *)header->db;
-
 		/*
 		 * If no one else is using the node, we can clean it up now.
 		 * We first need to gain a new reference to the node to meet a
@@ -969,6 +975,9 @@ expireheader(dns_slabheader_t *header, isc_rwlocktype_t *nlocktypep,
 					    dns_cachestatscounter_deletettl);
 			break;
 		case dns_expire_lru:
+			// TODO check this
+			isc_skiplist_delete(qpdb->slist, header,
+					    header->ttl_index);
 			isc_stats_increment(qpdb->cachestats,
 					    dns_cachestatscounter_deletelru);
 			break;
@@ -2478,26 +2487,11 @@ prio_type(dns_typepair_t type) {
 	return (false);
 }
 
-/*%
- * These functions allow the heap code to rank the priority of each
- * element.  It returns true if v1 happens "sooner" than v2.
- */
-static bool
-ttl_sooner(void *v1, void *v2) {
-	dns_slabheader_t *h1 = v1;
-	dns_slabheader_t *h2 = v2;
+static uint32_t
+get_ttl(void *ptr) {
+	dns_slabheader_t *h = ptr;
 
-	return (h1->ttl < h2->ttl);
-}
-
-/*%
- * This function sets the heap index into the header.
- */
-static void
-set_index(void *what, unsigned int idx) {
-	dns_slabheader_t *h = what;
-
-	h->heap_index = idx;
+	return h->ttl;
 }
 
 static void
@@ -2576,14 +2570,10 @@ free_qpdb(qpcache_t *qpdb, bool log) {
 			     qpdb->node_lock_count, sizeof(qpcnodelist_t));
 	}
 	/*
-	 * Clean up heap objects.
+	 * Clean up skiplist.
 	 */
-	if (qpdb->heaps != NULL) {
-		for (i = 0; i < qpdb->node_lock_count; i++) {
-			isc_heap_destroy(&qpdb->heaps[i]);
-		}
-		isc_mem_cput(qpdb->hmctx, qpdb->heaps, qpdb->node_lock_count,
-			     sizeof(isc_heap_t *));
+	if (qpdb->slist != NULL) {
+		isc_skiplist_destroy(&qpdb->slist);
 	}
 
 	if (qpdb->rrsetstats != NULL) {
@@ -2607,7 +2597,7 @@ free_qpdb(qpcache_t *qpdb, bool log) {
 	isc_rwlock_destroy(&qpdb->lock);
 	qpdb->common.magic = 0;
 	qpdb->common.impmagic = 0;
-	isc_mem_detach(&qpdb->hmctx);
+	isc_mem_detach(&qpdb->smctx);
 
 	isc_mem_putanddetach(&qpdb->common.mctx, qpdb, sizeof(*qpdb));
 }
@@ -3236,9 +3226,11 @@ find_header:
 				ISC_LIST_PREPEND(qpdb->lru[idx], newheader,
 						 link);
 			}
-			INSIST(qpdb->heaps != NULL);
-			isc_heap_insert(qpdb->heaps[idx], newheader);
-			newheader->heap = qpdb->heaps[idx];
+
+			INSIST(qpdb->slist != NULL);
+			INSIST(newheader->ttl_index == 0);
+			newheader->ttl_index = isc_skiplist_insert(qpdb->slist,
+								   newheader);
 
 			/*
 			 * There are no other references to 'header' when
@@ -3255,9 +3247,12 @@ find_header:
 			dns_slabheader_destroy(&header);
 		} else {
 			idx = HEADERNODE(newheader)->locknum;
-			INSIST(qpdb->heaps != NULL);
-			isc_heap_insert(qpdb->heaps[idx], newheader);
-			newheader->heap = qpdb->heaps[idx];
+
+			INSIST(qpdb->slist != NULL);
+			INSIST(newheader->ttl_index == 0);
+			newheader->ttl_index = isc_skiplist_insert(qpdb->slist,
+								   newheader);
+
 			if (ZEROTTL(newheader)) {
 				newheader->last_used = qpdb->last_used + 1;
 				ISC_LIST_APPEND(qpdb->lru[idx], newheader,
@@ -3294,9 +3289,13 @@ find_header:
 			return (DNS_R_UNCHANGED);
 		}
 
+		// TODO: why there are entries with an index already?
+		if (newheader->ttl_index == 0) {
+			newheader->ttl_index = isc_skiplist_insert(qpdb->slist,
+								   newheader);
+		}
+
 		idx = HEADERNODE(newheader)->locknum;
-		isc_heap_insert(qpdb->heaps[idx], newheader);
-		newheader->heap = qpdb->heaps[idx];
 		if (ZEROTTL(newheader)) {
 			ISC_LIST_APPEND(qpdb->lru[idx], newheader, link);
 		} else {
@@ -3430,9 +3429,9 @@ cleanup:
 }
 
 static void
-expire_ttl_headers(qpcache_t *qpdb, unsigned int locknum,
-		   isc_rwlocktype_t *nlocktypep, isc_rwlocktype_t *tlocktypep,
-		   isc_stdtime_t now, bool cache_is_overmem DNS__DB_FLARG);
+expire_ttl_headers(qpcache_t *qpdb, isc_rwlocktype_t *nlocktypep,
+		   isc_rwlocktype_t *tlocktypep, isc_stdtime_t now,
+		   bool cache_is_overmem DNS__DB_FLARG);
 
 static isc_result_t
 addrdataset(dns_db_t *db, dns_dbnode_t *node, dns_dbversion_t *version,
@@ -3563,7 +3562,7 @@ addrdataset(dns_db_t *db, dns_dbnode_t *node, dns_dbversion_t *version,
 		cleanup_dead_nodes(qpdb, qpnode->locknum DNS__DB_FLARG_PASS);
 	}
 
-	expire_ttl_headers(qpdb, qpnode->locknum, &nlocktype, &tlocktype, now,
+	expire_ttl_headers(qpdb, &nlocktype, &tlocktype, now,
 			   cache_is_overmem DNS__DB_FLARG_PASS);
 
 	/*
@@ -3634,7 +3633,10 @@ deleterdataset(dns_db_t *db, dns_dbnode_t *node, dns_dbversion_t *version,
 
 	newheader = dns_slabheader_new(db, node);
 	newheader->type = DNS_TYPEPAIR_VALUE(type, covers);
-	setttl(newheader, 0);
+
+	isc_skiplist_delete(qpdb->slist, newheader, newheader->ttl_index);
+	newheader->ttl_index = 0;
+
 	atomic_init(&newheader->attributes, DNS_SLABHEADERATTR_NONEXISTENT);
 
 	NODE_WRLOCK(&qpdb->node_locks[qpnode->locknum].lock, &nlocktype);
@@ -3730,7 +3732,7 @@ dns__qpcache_create(isc_mem_t *mctx, const dns_name_t *origin,
 		    unsigned int argc, char *argv[],
 		    void *driverarg ISC_ATTR_UNUSED, dns_db_t **dbp) {
 	qpcache_t *qpdb = NULL;
-	isc_mem_t *hmctx = mctx;
+	isc_mem_t *smctx = mctx;
 	int i;
 
 	/* This database implementation only supports cache semantics */
@@ -3748,10 +3750,10 @@ dns__qpcache_create(isc_mem_t *mctx, const dns_name_t *origin,
 	isc_refcount_init(&qpdb->common.references, 1);
 
 	/*
-	 * If argv[0] exists, it points to a memory context to use for heap
+	 * If argv[0] exists, it points to a memory context to use for skiplist
 	 */
 	if (argc != 0) {
-		hmctx = (isc_mem_t *)argv[0];
+		smctx = (isc_mem_t *)argv[0];
 	}
 
 	isc_rwlock_init(&qpdb->lock);
@@ -3768,14 +3770,9 @@ dns__qpcache_create(isc_mem_t *mctx, const dns_name_t *origin,
 	}
 
 	/*
-	 * Create the heaps.
+	 * Create the skiplist.
 	 */
-	qpdb->heaps = isc_mem_cget(hmctx, qpdb->node_lock_count,
-				   sizeof(isc_heap_t *));
-	for (i = 0; i < (int)qpdb->node_lock_count; i++) {
-		isc_heap_create(hmctx, ttl_sooner, set_index, 0,
-				&qpdb->heaps[i]);
-	}
+	isc_skiplist_create(smctx, get_ttl, &qpdb->slist);
 
 	/*
 	 * Create deadnode lists.
@@ -3800,7 +3797,7 @@ dns__qpcache_create(isc_mem_t *mctx, const dns_name_t *origin,
 	 * mctx won't disappear out from under us.
 	 */
 	isc_mem_attach(mctx, &qpdb->common.mctx);
-	isc_mem_attach(hmctx, &qpdb->hmctx);
+	isc_mem_attach(smctx, &qpdb->smctx);
 
 	/*
 	 * Make a copy of the origin name.
@@ -4361,9 +4358,8 @@ deletedata(dns_db_t *db ISC_ATTR_UNUSED, dns_dbnode_t *node ISC_ATTR_UNUSED,
 	dns_slabheader_t *header = data;
 	qpcache_t *qpdb = (qpcache_t *)header->db;
 
-	if (header->heap != NULL && header->heap_index != 0) {
-		isc_heap_delete(header->heap, header->heap_index);
-	}
+	isc_skiplist_delete(qpdb->slist, header, header->ttl_index);
+	header->ttl_index = 0;
 
 	update_rrsetstats(qpdb->rrsetstats, header->type,
 			  atomic_load_acquire(&header->attributes), false);
@@ -4381,43 +4377,53 @@ deletedata(dns_db_t *db ISC_ATTR_UNUSED, dns_dbnode_t *node ISC_ATTR_UNUSED,
 	}
 }
 
+typedef struct lock_ctx {
+	bool cache_is_overmem;
+	qpcache_t *qpdb;
+	isc_rwlocktype_t *nlocktypep;
+	isc_rwlocktype_t *tlocktypep;
+} lock_ctx_t;
+
+static bool
+popaction_cb(void *user, void *header, uint32_t now) {
+	dns_slabheader_t *h = header;
+	lock_ctx_t *ctx = user;
+	uint32_t ttl;
+
+	INSIST(user != NULL);
+
+	ttl = h->ttl;
+
+	if (ctx->cache_is_overmem) {
+		ttl += STALE_TTL(h, ctx->qpdb);
+	}
+
+	if (ttl >= now - QPDB_VIRTUAL) {
+		expireheader(header, ctx->nlocktypep, ctx->tlocktypep,
+			     dns_expire_ttl DNS__DB_FLARG_PASS);
+
+		return true;
+	}
+
+	return false;
+}
+
 /*
  * Caller must be holding the node write lock.
  */
 static void
-expire_ttl_headers(qpcache_t *qpdb, unsigned int locknum,
-		   isc_rwlocktype_t *nlocktypep, isc_rwlocktype_t *tlocktypep,
-		   isc_stdtime_t now, bool cache_is_overmem DNS__DB_FLARG) {
-	isc_heap_t *heap = qpdb->heaps[locknum];
+expire_ttl_headers(qpcache_t *qpdb, isc_rwlocktype_t *nlocktypep,
+		   isc_rwlocktype_t *tlocktypep, isc_stdtime_t now,
+		   bool cache_is_overmem DNS__DB_FLARG) {
+	struct lock_ctx ctx = {
+		.cache_is_overmem = cache_is_overmem,
+		.qpdb = qpdb,
+		.nlocktypep = nlocktypep,
+		.tlocktypep = tlocktypep,
+	};
 
-	for (size_t i = 0; i < DNS_QPDB_EXPIRE_TTL_COUNT; i++) {
-		dns_slabheader_t *header = isc_heap_element(heap, 1);
-
-		if (header == NULL) {
-			/* No headers left on this TTL heap; exit cleaning */
-			return;
-		}
-
-		dns_ttl_t ttl = header->ttl;
-
-		if (!cache_is_overmem) {
-			/* Only account for stale TTL if cache is not overmem */
-			ttl += STALE_TTL(header, qpdb);
-		}
-
-		if (ttl >= now - QPDB_VIRTUAL) {
-			/*
-			 * The header at the top of this TTL heap is not yet
-			 * eligible for expiry, so none of the other headers on
-			 * the same heap can be eligible for expiry, either;
-			 * exit cleaning.
-			 */
-			return;
-		}
-
-		expireheader(header, nlocktypep, tlocktypep,
-			     dns_expire_ttl DNS__DB_FLARG_PASS);
-	}
+	isc_skiplist_poprange(qpdb->slist, now, DNS_QPDB_EXPIRE_TTL_COUNT, &ctx,
+			      popaction_cb);
 }
 
 static dns_dbmethods_t qpdb_cachemethods = {

lib/isc/Makefile.am

@@ -77,6 +77,7 @@ libisc_la_HEADERS =			\
 	include/isc/serial.h		\
 	include/isc/signal.h		\
 	include/isc/siphash.h		\
+	include/isc/skiplist.h		\
 	include/isc/sockaddr.h		\
 	include/isc/spinlock.h		\
 	include/isc/stats.h		\
@@ -92,6 +93,7 @@ libisc_la_HEADERS =			\
 	include/isc/timer.h		\
 	include/isc/tls.h		\
 	include/isc/tm.h		\
+	include/isc/ttlwheel.h		\
 	include/isc/types.h		\
 	include/isc/urcu.h		\
 	include/isc/url.h		\
@@ -183,6 +185,7 @@ libisc_la_SOURCES =		\
 	safe.c			\
 	serial.c		\
 	signal.c		\
+	skiplist.c		\
 	sockaddr.c		\
 	stats.c			\
 	stdio.c			\
@@ -196,6 +199,7 @@ libisc_la_SOURCES =		\
 	timer.c			\
 	tls.c			\
 	tm.c			\
+	ttlwheel.c		\
 	url.c			\
 	utf8.c			\
 	uv.c			\

lib/isc/include/isc/skiplist.h

@@ -0,0 +1,82 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * SPDX-License-Identifier: MPL-2.0
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, you can obtain one at https://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+/*! \file isc/skiplist.h */
+
+#pragma once
+
+/*
+ * Skiplist for items that keep track of their value.
+ */
+
+#include <isc/types.h>
+
+typedef struct isc_skiplist isc_skiplist_t;
+
+typedef void (*isc_skiplist_index_update_fn_t)(void *user, uint64_t new_index);
+typedef bool (*isc_skiplist_popaction_t)(void *user, void *value, uint32_t);
+typedef uint32_t (*isc_skiplist_key_fn_t)(void *user);
+
+ISC_LANG_BEGINDECLS
+
+void
+isc_skiplist_create(isc_mem_t *mctx, isc_skiplist_key_fn_t key_fn,
+		    isc_skiplist_t **slistp);
+/*%
+ * Create skiplist at *slistp, using memory context
+ *
+ * Requires:
+ * \li	'slistp' is not NULL and '*slistp' is NULL.
+ * \li	'mctx' is a valid memory context.
+ *
+ */
+
+void
+isc_skiplist_destroy(isc_skiplist_t **slistp);
+/*%
+ * Destroy skiplist, freeing everything
+ *
+ * Requires:
+ * \li	'*slist' is valid skiplist
+ */
+
+uint64_t
+isc_skiplist_insert(isc_skiplist_t *slist, void *value);
+/*%
+ * Insert
+ *
+ * Requires
+ * \li	`value` is not NULL and returns a non `UINT32_MAX` key from the
+ * function.
+ *
+ * Note:
+ * \li The index value can safetly be discarded if neither `isc_skiplist_delete`
+ * nor `isc_skiplist_update` is used and elements are interacted exclusively
+ * through `isc_skiplist_poprange`.
+ */
+
+uint64_t
+isc_skiplist_update(isc_skiplist_t *slist, void *value, uint32_t new_key,
+		    uint64_t index);
+/*%
+ *
+ */
+
+isc_result_t
+isc_skiplist_delete(isc_skiplist_t *slist, void *value, uint64_t index);
+
+size_t
+isc_skiplist_poprange(isc_skiplist_t *slist, uint32_t range, size_t limit,
+		      void *user, isc_skiplist_popaction_t action);
+
+ISC_LANG_ENDDECLS

lib/isc/include/isc/ttlwheel.h

@@ -0,0 +1,122 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. (`ISC`)
+ *
+ * SPDX-License-Identifier: MPL-2.0
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, you can obtain one at https://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+#pragma once
+
+#include <stdint.h>
+
+#include <isc/stdtime.h>
+#include <isc/types.h>
+
+ISC_LANG_BEGINDECLS
+
+typedef struct isc_ttlwheel isc_ttlwheel_t;
+
+typedef void (*isc_ttlwheel_popaction_t)(void *user, void *data);
+
+void
+isc_ttlwheel_create(isc_mem_t *mctx, isc_stdtime_t now,
+		    isc_ttlwheel_t **wheelp);
+/*%<
+ * Creates a new TTL wheel.
+ *
+ * Requires:
+ * \li	`mctx` is valid.
+ * \li	`now` is the starting point where TTL wheel will start expiring from.
+ * \li	`wheelp != NULL && *wheelp == NULL`
+ *
+ * Ensures:
+ * \li `*wheelp` is a pointer to a valid isc_ttlwheel_t.
+ */
+
+void
+isc_ttlwheel_destroy(isc_ttlwheel_t **wheelp);
+/*%<
+ * Destroys a TTL wheel.
+ *
+ * Requires:
+ * \li `wheelp != NULL` and `*wheel` points to a valid isc_ttlwheel_t.
+ */
+
+isc_stdtime_t
+isc_ttlwheel_epoch(isc_ttlwheel_t *wheel);
+/*%<
+ * Returns the epoch.
+ *
+ * Requires:
+ * \li	`wheel` is not NULL and `*wheel` points to a valid isc_ttlwheel_t.
+ */
+
+uint64_t
+isc_ttlwheel_insert(isc_ttlwheel_t *wheel, isc_stdtime_t ttl, void *data);
+/*%<
+ * Inserts a new element into the TTL wheel.
+ *
+ * Requires:
+ * \li	`wheel` is not NULL and `*wheel` points to a valid isc_ttlwheel_t.
+ * \li	`ttl` is not UINT32_MAX.
+ * \li	`data` is not NULL.
+ *
+ * Returns:
+ * \li	0 if the entry has already expired according to the TTL wheel.
+ * \li	The index of the entry otherwise.
+ */
+
+enum isc_result
+isc_ttlwheel_update(isc_ttlwheel_t *wheel, uint64_t index, isc_stdtime_t ttl);
+/*%<
+ * Deletes an entry from the TTL wheel, by element index.
+ *
+ * Requires:
+ * \li	`wheel` is not NULL and `*wheel` points to a valid isc_ttlwheel_t.
+ * \li	`index` is a valid element index, as provided by isc_ttlwheel_insert.
+ * \li	`ttl` is
+ *
+ * Returns:
+ * \li	#ISC_R_SUCCESS			on success
+ * \li	#ISC_R_IGNORE			the new ttl is already expired
+ *
+ * Note:
+ * \li	The index doesn't change.
+ */
+
+void
+isc_ttlwheel_delete(isc_ttlwheel_t *wheel, uint64_t index);
+/*%<
+ * Deletes an entry from the TTL wheel, by element index.
+ *
+ * Requires:
+ * \li	`wheel` is not NULL and `*wheel` points to a valid isc_ttlwheel_t.
+ * \li	`index` is a valid element index, as provided by isc_ttlwheel_insert.
+ */
+
+size_t
+isc_ttlwheel_poprange(isc_ttlwheel_t *wheel, isc_stdtime_t now, size_t limit,
+		      void *user, isc_ttlwheel_popaction_t action);
+/*%<
+ * Iterates over the TTL wheel, removing expired entries up to the
+ * specified limit.
+ *
+ * Requires:
+ * \li	`wheel` is not NULL and `*wheel` points to a valid isc_ttlwheel_t.
+ * \li	`limit` is the amount of entries to iterate at most.
+ * \li	`action` is not NULL, and is a function which takes two arguments.
+ *	The first is `user` as provided to isc_ttlwheel_poprange, and the second
+ *	is a void * that represents the element.
+ * \li	`user` is a caller-provided argument and may be NULL.
+ *
+ * Returns:
+ * \li	The amount of entries expired.
+ */
+
+ISC_LANG_ENDDECLS

lib/isc/skiplist.c

@@ -0,0 +1,400 @@
+#include <inttypes.h>
+#include <stdint.h>
+
+#include <isc/list.h>
+#include <isc/magic.h>
+#include <isc/mem.h>
+#include <isc/random.h>
+#include <isc/rwlock.h>
+#include <isc/skiplist.h>
+#include <isc/util.h>
+
+#define ISC_SKIPLIST_MAGIC ISC_MAGIC('S', 'k', 'i', 'p')
+#define SKIPLIST_VALID(s)  ISC_MAGIC_VALID(s, ISC_SKIPLIST_MAGIC)
+
+#define MAX_LEVEL 32
+#define MAX_INDEX (MAX_LEVEL - 1)
+
+#define SEGMENT_SATURATED_INDEX 26
+#define SEGMENT_SATURATED_SIZE	(32U << 26)
+
+#define UNSATURATED_INDEX(x) ISC_MIN(26, x)
+
+STATIC_ASSERT(sizeof(void *) <= sizeof(uint64_t),
+	      "pointers must fit in 64 bits");
+
+typedef struct skiplist_node skiplist_node_t;
+typedef struct skiplist_entry skiplist_entry_t;
+typedef ISC_LIST(skiplist_entry_t) skiplist_entrylist_t;
+
+struct skiplist_entry {
+	void *value;
+	ISC_LINK(skiplist_entry_t) link;
+};
+
+struct skiplist_node {
+	uint32_t level;
+	uint32_t key;
+
+	uint64_t count;
+
+	skiplist_entrylist_t entries;
+
+	skiplist_node_t *nodes[];
+};
+
+struct isc_skiplist {
+	uint32_t magic;
+	isc_mem_t *mctx;
+	isc_skiplist_key_fn_t key_fn;
+
+	uint64_t entry_chunk_span;
+	uint32_t *cursors;
+	skiplist_entry_t **entries;
+
+	skiplist_entrylist_t frees;
+
+	skiplist_node_t *head;
+};
+
+static skiplist_node_t *
+node_create_raw(isc_mem_t *mctx, uint32_t key) {
+	skiplist_node_t *node;
+	uint32_t level;
+
+	STATIC_ASSERT(MAX_LEVEL == 32, "fix 0x1f masking in level generation");
+
+	level = (isc_random32() & 0x1f) + 1;
+
+	node = isc_mem_get(mctx, STRUCT_FLEX_SIZE(node, nodes, level));
+	*node = (skiplist_node_t){
+		.level = level,
+		.key = key,
+		.entries = ISC_LIST_INITIALIZER,
+	};
+
+	return (node);
+}
+
+static uint64_t
+insert_value(isc_skiplist_t *slist, skiplist_node_t *node, void *value) {
+	skiplist_entry_t *entry;
+	uint32_t cursor;
+	size_t i;
+
+	if (!ISC_LIST_EMPTY(slist->frees)) {
+		fprintf(stderr, "===================== FREE!\n");
+		entry = ISC_LIST_HEAD(slist->frees);
+		ISC_LIST_UNLINK(slist->frees, entry, link);
+		goto found;
+	}
+
+	for (i = 0; i < UNSATURATED_INDEX(slist->entry_chunk_span - 1); i++) {
+		cursor = slist->cursors[i];
+		if (cursor < (32U << i)) {
+			fprintf(stderr, "======== EARLY %zu, %u\n", i, cursor);
+			slist->cursors[i] = 32U << i;
+			entry = &slist->entries[i][cursor];
+			goto found;
+		}
+	}
+
+	for (; i < slist->entry_chunk_span - 1; i++) {
+		cursor = slist->cursors[i];
+		if (cursor < SEGMENT_SATURATED_SIZE) {
+			fprintf(stderr, "======== SATURATED %zu, %u\n", i,
+				cursor);
+			slist->cursors[i] = SEGMENT_SATURATED_SIZE;
+			entry = &slist->entries[i][cursor];
+			goto found;
+		}
+	}
+
+	INSIST(i == slist->entry_chunk_span - 1);
+
+	cursor = slist->cursors[i];
+	if (cursor < SEGMENT_SATURATED_SIZE) {
+		fprintf(stderr, "======== CURSOR MOVE %zu,%u\n", i, cursor);
+		slist->cursors[i]++;
+		entry = &slist->entries[i][cursor];
+		goto found;
+	}
+
+	i++;
+
+	fprintf(stderr, "=================RESIZING TO %zu\n", i);
+
+	slist->cursors = isc_mem_creget(slist->mctx, slist->cursors, i, i + 1,
+					sizeof(uint32_t));
+
+	slist->cursors[i] = 0;
+
+	slist->entries = isc_mem_creget(slist->mctx, slist->entries, i, i + 1,
+					sizeof(skiplist_entry_t *));
+
+	slist->entries[i] = isc_mem_cget(slist->mctx, 32U << (i < 26 ? i : 26),
+					 sizeof(skiplist_entry_t));
+
+	slist->entry_chunk_span = i + 1;
+
+	entry = &slist->entries[i][0];
+
+found:
+	*entry = (skiplist_entry_t){
+		.value = value,
+		.link = ISC_LINK_INITIALIZER,
+	};
+
+	ISC_LIST_APPEND(node->entries, entry, link);
+
+	node->count++;
+
+	fprintf(stderr, "\n>>>>>\ninsert,ttl:%u,value:%p,node:%p\n<<<<<\n\n",
+		node->key, value, entry);
+
+	return ((uint64_t)(uintptr_t)entry);
+}
+
+void
+isc_skiplist_create(isc_mem_t *mctx, isc_skiplist_key_fn_t key_fn,
+		    isc_skiplist_t **slistp) {
+	skiplist_entry_t **entries;
+	skiplist_node_t *node;
+	isc_skiplist_t *slist;
+	uint32_t *cursors;
+
+	REQUIRE(slistp != NULL);
+	REQUIRE(*slistp == NULL);
+
+	node = isc_mem_get(mctx, STRUCT_FLEX_SIZE(node, nodes, MAX_LEVEL));
+	*node = (skiplist_node_t){
+		.level = MAX_LEVEL,
+		.key = UINT32_MAX,
+		.entries = ISC_LIST_INITIALIZER,
+	};
+
+	for (size_t i = 0; i < node->level; i++) {
+		node->nodes[i] = node;
+	}
+
+	cursors = isc_mem_cget(mctx, 1, sizeof(uint32_t));
+	cursors[0] = 0;
+
+	entries = isc_mem_cget(mctx, 1, sizeof(skiplist_node_t *));
+	entries[0] = isc_mem_cget(mctx, 32, sizeof(skiplist_node_t));
+
+	slist = isc_mem_get(mctx, sizeof(*slist));
+	*slist = (isc_skiplist_t){
+		.magic = ISC_SKIPLIST_MAGIC,
+		.key_fn = key_fn,
+		.entry_chunk_span = 1,
+		.cursors = cursors,
+		.entries = entries,
+		.frees = ISC_LIST_INITIALIZER,
+		.head = node,
+	};
+
+	isc_mem_attach(mctx, &slist->mctx);
+
+	*slistp = slist;
+}
+
+void
+isc_skiplist_destroy(isc_skiplist_t **slistp) {
+	skiplist_node_t *node, *next;
+	isc_skiplist_t *slist;
+
+	REQUIRE(slistp != NULL);
+	REQUIRE(SKIPLIST_VALID(*slistp));
+
+	slist = *slistp;
+	*slistp = NULL;
+
+	slist->magic = 0;
+
+	node = slist->head->nodes[0];
+	while (node != slist->head) {
+		next = node->nodes[0];
+		isc_mem_put(slist->mctx, node,
+			    STRUCT_FLEX_SIZE(node, nodes, node->level));
+		node = next;
+	}
+
+	isc_mem_put(slist->mctx, node,
+		    STRUCT_FLEX_SIZE(node, nodes, MAX_LEVEL));
+
+	isc_mem_cput(slist->mctx, slist->cursors, slist->entry_chunk_span,
+		     sizeof(uint32_t));
+
+	for (size_t i = 0; i < slist->entry_chunk_span; i++) {
+		isc_mem_cput(slist->mctx, slist->entries[i],
+			     32U << (i < 26 ? i : 26),
+			     sizeof(skiplist_entry_t));
+	}
+
+	isc_mem_cput(slist->mctx, slist->entries, slist->entry_chunk_span,
+		     sizeof(skiplist_entry_t *));
+
+	isc_mem_putanddetach(&slist->mctx, slist, sizeof(*slist));
+}
+
+uint64_t
+isc_skiplist_insert(isc_skiplist_t *slist, void *value) {
+	skiplist_node_t *updates[MAX_LEVEL];
+	skiplist_node_t *node;
+	uint32_t key;
+	int32_t level;
+
+	REQUIRE(SKIPLIST_VALID(slist));
+
+	key = slist->key_fn(value);
+
+	INSIST(key != UINT32_MAX);
+
+	node = slist->head;
+	for (level = MAX_INDEX; level >= 0; level--) {
+		while (node->nodes[level]->key < key) {
+			node = node->nodes[level];
+		}
+
+		if (node->nodes[level]->key == key) {
+			return (insert_value(slist, node->nodes[level], value));
+		}
+
+		updates[level] = node;
+	}
+
+	node = node_create_raw(slist->mctx, key);
+
+	for (uint32_t i = 0; i < node->level; i++) {
+		node->nodes[i] = updates[i]->nodes[i];
+		updates[i]->nodes[i] = node;
+	}
+
+	return (insert_value(slist, node, value));
+}
+
+isc_result_t
+isc_skiplist_delete(isc_skiplist_t *slist, void *value, uint64_t index) {
+	skiplist_entry_t *entry;
+	skiplist_node_t *node;
+	uint32_t key;
+	int32_t level;
+
+	REQUIRE(SKIPLIST_VALID(slist));
+	REQUIRE(index != 0 && index != UINT32_MAX);
+
+	key = slist->key_fn(value);
+
+	INSIST(key != UINT32_MAX);
+
+	entry = (void *)((uintptr_t)index);
+
+	node = slist->head;
+	for (level = MAX_INDEX; level >= 0; level--) {
+		while (node->nodes[level]->key < key) {
+			node = node->nodes[level];
+		}
+
+		if (node->nodes[level]->key == key) {
+			fprintf(stderr, ">>>>> del,ttl:%u,node:%p ...", key,
+				node);
+			INSIST(node->nodes[level]->count > 0);
+			INSIST(entry->value == value);
+			node->nodes[level]->count--;
+			ISC_LIST_UNLINK(node->nodes[level]->entries, entry,
+					link);
+			*entry = (skiplist_entry_t){
+				.value = NULL,
+				.link = ISC_LINK_INITIALIZER,
+			};
+			ISC_LIST_APPEND(slist->frees, entry, link);
+			fprintf(stderr, "ok\n");
+			return (ISC_R_SUCCESS);
+		}
+	}
+
+	return (ISC_R_NOTFOUND);
+}
+
+size_t
+isc_skiplist_poprange(isc_skiplist_t *slist, uint32_t range, size_t limit,
+		      void *user, isc_skiplist_popaction_t action) {
+	skiplist_entry_t *entry, *next_entry;
+	skiplist_node_t *updates[MAX_LEVEL];
+	size_t processed, removed;
+	skiplist_node_t *node, *next_node;
+	void *value;
+
+	REQUIRE(SKIPLIST_VALID(slist));
+	REQUIRE(action != NULL);
+
+	if (limit == 0) {
+		limit = SIZE_MAX;
+	}
+
+	memmove(updates, slist->head->nodes, sizeof(updates));
+
+	removed = 0;
+	processed = 0;
+
+	node = slist->head->nodes[0];
+	while (node->key < range) {
+		INSIST(node != slist->head);
+
+		fprintf(stderr, "\n!!!!!\nrange,ttl:%u\n!!!!!\n", node->key);
+
+		ISC_LIST_FOREACH_SAFE (node->entries, entry, link, next_entry) {
+			value = entry->value;
+
+			fprintf(stderr,
+				">>>>> range,ttl:%u,value:%p,node:%p ...",
+				node->key, value, entry);
+
+			INSIST(value != NULL);
+
+			if ((action)(user, value, range)) {
+				node->count--;
+				removed++;
+
+				ISC_LIST_UNLINK(node->entries, entry, link);
+				*entry = (skiplist_entry_t){
+					.value = NULL,
+					.link = ISC_LINK_INITIALIZER,
+				};
+				ISC_LIST_APPEND(slist->frees, entry, link);
+			}
+
+			fprintf(stderr, "ok\n");
+
+			processed++;
+			if (processed >= limit) {
+				goto out;
+			}
+		}
+
+		if (ISC_LIST_EMPTY(node->entries)) {
+			INSIST(node->count == 0);
+			next_node = node->nodes[0];
+			memmove(updates, node->nodes,
+				node->level * sizeof(skiplist_node_t *));
+			isc_mem_put(slist->mctx, node,
+				    STRUCT_FLEX_SIZE(node, nodes, node->level));
+			node = next_node;
+		} else {
+			node = node->nodes[0];
+		}
+	}
+
+out:
+	if (ISC_LIST_EMPTY(node->entries) && node != slist->head) {
+		memmove(updates, node->nodes,
+			node->level * sizeof(skiplist_node_t *));
+		isc_mem_put(slist->mctx, node,
+			    STRUCT_FLEX_SIZE(node, nodes, node->level));
+	}
+
+	memmove(slist->head->nodes, updates, sizeof(updates));
+
+	return (removed);
+}

lib/isc/ttlwheel.c

@@ -0,0 +1,210 @@
+#include <inttypes.h>
+#include <stdint.h>
+
+#include <isc/magic.h>
+#include <isc/mem.h>
+#include <isc/overflow.h>
+#include <isc/result.h>
+#include <isc/stdtime.h>
+#include <isc/ttlwheel.h>
+#include <isc/util.h>
+
+#define ISC_TTLWHEEL_MAGIC ISC_MAGIC('T', 'T', 'L', 'w')
+
+#define TTLWHEEL_BUCKET(time) (time & 0xFF)
+#define TTLWHEEL_ENTRIES      256
+#define TTLWHEEL_VALID(w)     ISC_MAGIC_VALID(w, ISC_TTLWHEEL_MAGIC)
+
+typedef struct ttl_entry ttl_entry_t;
+typedef ISC_LIST(ttl_entry_t) ttl_entrylist_t;
+
+struct ttl_entry {
+	isc_stdtime_t ttl;
+	void *data;
+	ISC_LINK(ttl_entry_t) link;
+};
+
+struct isc_ttlwheel {
+	uint32_t magic;
+
+	isc_stdtime_t epoch;
+
+	isc_mem_t *mctx;
+
+	ttl_entrylist_t slot[TTLWHEEL_ENTRIES];
+};
+
+void
+isc_ttlwheel_create(isc_mem_t *mctx, isc_stdtime_t now,
+		    isc_ttlwheel_t **wheelp) {
+	isc_ttlwheel_t *wheel;
+
+	REQUIRE(wheelp != NULL);
+	REQUIRE(*wheelp == NULL);
+
+	wheel = isc_mem_get(mctx, sizeof(*wheel));
+	*wheel = (isc_ttlwheel_t){
+		.magic = ISC_TTLWHEEL_MAGIC,
+		.epoch = now,
+	};
+
+	for (size_t i = 0; i < TTLWHEEL_ENTRIES; i++) {
+		ISC_LIST_INIT(wheel->slot[i]);
+	}
+
+	isc_mem_attach(mctx, &wheel->mctx);
+
+	*wheelp = wheel;
+}
+
+void
+isc_ttlwheel_destroy(isc_ttlwheel_t **wheelp) {
+	isc_ttlwheel_t *wheel;
+	ttl_entry_t *entry;
+
+	REQUIRE(wheelp != NULL);
+	REQUIRE(*wheelp != NULL);
+	REQUIRE(TTLWHEEL_VALID(*wheelp));
+
+	wheel = *wheelp;
+	*wheelp = NULL;
+
+	wheel->magic = 0;
+
+	for (size_t i = 0; i < TTLWHEEL_ENTRIES; i++) {
+		entry = ISC_LIST_HEAD(wheel->slot[i]);
+		while (entry != NULL) {
+			ISC_LIST_UNLINK(wheel->slot[i], entry, link);
+			isc_mem_put(wheel->mctx, entry, sizeof(*entry));
+			entry = ISC_LIST_HEAD(wheel->slot[i]);
+		}
+	}
+
+	isc_mem_putanddetach(&wheel->mctx, wheel, sizeof(*wheel));
+}
+
+isc_stdtime_t
+isc_ttlwheel_epoch(isc_ttlwheel_t *wheel) {
+	REQUIRE(TTLWHEEL_VALID(wheel));
+
+	return wheel->epoch;
+}
+
+uint64_t
+isc_ttlwheel_insert(isc_ttlwheel_t *wheel, isc_stdtime_t ttl, void *data) {
+	ttl_entry_t *entry;
+
+	REQUIRE(TTLWHEEL_VALID(wheel));
+	REQUIRE(data != NULL);
+
+	if (wheel->epoch >= ttl) {
+		return 0;
+	}
+
+	entry = isc_mem_get(wheel->mctx, sizeof(*entry));
+	*entry = (ttl_entry_t){
+		.ttl = ttl,
+		.data = data,
+		.link = ISC_LINK_INITIALIZER,
+	};
+
+	ISC_LIST_APPEND(wheel->slot[TTLWHEEL_BUCKET(ttl) & 0xFF], entry, link);
+
+	STATIC_ASSERT(sizeof(uintptr_t) <= sizeof(uint64_t),
+		      "pointers must fit in 64-bits");
+
+	return (uint64_t)((uintptr_t)(entry));
+}
+
+enum isc_result
+isc_ttlwheel_update(isc_ttlwheel_t *wheel, uint64_t index, isc_stdtime_t ttl) {
+	ttl_entry_t *entry;
+
+	REQUIRE(TTLWHEEL_VALID(wheel));
+
+	entry = ((void *)((uintptr_t)index));
+
+	INSIST(entry != NULL);
+	INSIST(ISC_LINK_LINKED(entry, link));
+
+	if (ttl < wheel->epoch) {
+		return ISC_R_IGNORE;
+	}
+
+	ISC_LIST_UNLINK(wheel->slot[TTLWHEEL_BUCKET(entry->ttl)], entry, link);
+
+	entry->ttl = ttl;
+
+	ISC_LIST_APPEND(wheel->slot[TTLWHEEL_BUCKET(entry->ttl)], entry, link);
+
+	return ISC_R_SUCCESS;
+}
+
+void
+isc_ttlwheel_delete(isc_ttlwheel_t *wheel, uint64_t index) {
+	ttl_entry_t *entry;
+
+	REQUIRE(TTLWHEEL_VALID(wheel));
+
+	entry = ((void *)((uintptr_t)index));
+
+	INSIST(entry != NULL);
+	INSIST(ISC_LINK_LINKED(entry, link));
+
+	ISC_LIST_UNLINK(wheel->slot[TTLWHEEL_BUCKET(entry->ttl)], entry, link);
+
+	isc_mem_put(wheel->mctx, entry, sizeof(*entry));
+}
+
+size_t
+isc_ttlwheel_poprange(isc_ttlwheel_t *wheel, isc_stdtime_t now, size_t limit,
+		      void *user, isc_ttlwheel_popaction_t action) {
+	ttl_entry_t *entry, *next;
+	isc_stdtime_t advance, diff;
+	uint32_t i;
+	size_t ctr;
+
+	REQUIRE(TTLWHEEL_VALID(wheel));
+	REQUIRE(action != NULL);
+
+	// 0 is short for maximum possible
+	if (limit == 0) {
+		limit = SIZE_MAX;
+	}
+
+	if (ISC_OVERFLOW_SUB(now, wheel->epoch, &diff)) {
+		return 0;
+	}
+
+	advance = 0;
+	ctr = 0;
+
+	for (i = wheel->epoch; i < wheel->epoch + ISC_MIN(diff, 255); i++) {
+		entry = ISC_LIST_HEAD(wheel->slot[TTLWHEEL_BUCKET(i)]);
+		while (entry != NULL) {
+			next = ISC_LIST_NEXT(entry, link);
+
+			if (entry->ttl < now) {
+				ISC_LIST_UNLINK(wheel->slot[TTLWHEEL_BUCKET(i)],
+						entry, link);
+
+				action(user, entry->data);
+
+				isc_mem_put(wheel->mctx, entry, sizeof(*entry));
+			}
+
+			advance++;
+
+			ctr++;
+			if (ctr == limit) {
+				goto finish_range;
+			}
+
+			entry = next;
+		}
+	}
+
+finish_range:
+	wheel->epoch += advance;
+	return ctr;
+}

tests/isc/Makefile.am

@@ -46,6 +46,7 @@ check_PROGRAMS =	\
 	rwlock_test	\
 	safe_test	\
 	siphash_test	\
+	skiplist_test	\
 	sockaddr_test	\
 	spinlock_test	\
 	stats_test	\
@@ -56,6 +57,7 @@ check_PROGRAMS =	\
 	timer_test	\
 	tls_test	\
 	tlsdns_test	\
+	ttlwheel_test	\
 	udp_test	\
 	work_test
 

tests/isc/skiplist_test.c

@@ -0,0 +1,224 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * SPDX-License-Identifier: MPL-2.0
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, you can obtain one at https://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+/* ! \file */
+
+#include <inttypes.h>
+#include <sched.h> /* IWYU pragma: keep */
+#include <setjmp.h>
+#include <stdarg.h>
+#include <stddef.h>
+#include <stdlib.h>
+#include <string.h>
+
+#define UNIT_TESTING
+#include <cmocka.h>
+
+#include <isc/mem.h>
+#include <isc/skiplist.h>
+#include <isc/util.h>
+
+#include <tests/isc.h>
+
+struct entry {
+	uint32_t drift;
+	uint32_t ttl;
+};
+
+static struct entry entries[] = {
+	{ .drift = 0, .ttl = 10 }, { .drift = 0, .ttl = 20 },
+	{ .drift = 0, .ttl = 30 }, { .drift = 0, .ttl = 40 },
+	{ .drift = 0, .ttl = 50 }, { .drift = 0, .ttl = 60 },
+	{ .drift = 0, .ttl = 70 }, { .drift = 0, .ttl = 80 },
+	{ .drift = 0, .ttl = 90 }, { .drift = 0, .ttl = 99 },
+};
+
+static struct entry entries_drift[] = {
+	{ .drift = 5, .ttl = 10 }, { .drift = 5, .ttl = 20 },
+	{ .drift = 5, .ttl = 30 }, { .drift = 5, .ttl = 40 },
+	{ .drift = 5, .ttl = 50 }, { .drift = 5, .ttl = 60 },
+	{ .drift = 5, .ttl = 70 }, { .drift = 5, .ttl = 80 },
+	{ .drift = 5, .ttl = 90 }, { .drift = 5, .ttl = 99 },
+};
+
+static void
+fill_entries_driftless(isc_skiplist_t *slist) {
+	uint64_t index;
+
+	for (size_t i = 0; i < ARRAY_SIZE(entries); i++) {
+		index = isc_skiplist_insert(slist, &entries[i]);
+		assert_int_not_equal(index, 0);
+	}
+}
+
+static void
+fill_entries_drift(isc_skiplist_t *slist) {
+	uint64_t index;
+
+	for (size_t i = 0; i < ARRAY_SIZE(entries_drift); i++) {
+		index = isc_skiplist_insert(slist, &entries_drift[i]);
+		assert_int_not_equal(index, 0);
+	}
+}
+
+static uint32_t
+get_key(void *value) {
+	REQUIRE(value != NULL);
+	return ((struct entry *)value)->ttl;
+}
+
+static bool
+remove_direct(void *user, void *value, uint32_t range) {
+	struct entry *e = value;
+
+	REQUIRE(user == NULL && value != NULL);
+	REQUIRE(value != NULL);
+
+	assert_in_range(e->ttl, 0, range);
+
+	return true;
+}
+
+static bool
+remove_drifting(void *user, void *value, uint32_t range) {
+	struct entry *e;
+
+	REQUIRE(value != NULL);
+	REQUIRE(user == NULL);
+
+	e = value;
+
+	assert_in_range(e->ttl, 0, range);
+
+	return e->ttl + e->drift < range;
+}
+
+ISC_RUN_TEST_IMPL(isc_skiplist_create) {
+	isc_skiplist_t *slist = NULL;
+
+	isc_skiplist_create(mctx, get_key, &slist);
+	assert_non_null(slist);
+
+	isc_skiplist_destroy(&slist);
+	assert_null(slist);
+}
+
+ISC_RUN_TEST_IMPL(isc_skiplist_insert_single) {
+	isc_skiplist_t *slist = NULL;
+
+	isc_skiplist_create(mctx, get_key, &slist);
+	assert_non_null(slist);
+
+	isc_skiplist_insert(slist, &entries[0]);
+
+	isc_skiplist_destroy(&slist);
+	assert_null(slist);
+}
+
+ISC_RUN_TEST_IMPL(isc_skiplist_insert) {
+	isc_skiplist_t *slist = NULL;
+	size_t i;
+
+	isc_skiplist_create(mctx, get_key, &slist);
+	assert_non_null(slist);
+
+	for (i = 0; i < ARRAY_SIZE(entries); i++) {
+		isc_skiplist_insert(slist, &entries[i]);
+	}
+
+	for (i = 0; i < ARRAY_SIZE(entries); i++) {
+		isc_skiplist_insert(slist, &entries_drift[i]);
+	}
+
+	isc_skiplist_destroy(&slist);
+	assert_null(slist);
+}
+
+ISC_RUN_TEST_IMPL(isc_skiplist_insert_make_duplicate) {
+	isc_skiplist_t *slist = NULL;
+	uint64_t index1, index2;
+
+	isc_skiplist_create(mctx, get_key, &slist);
+	assert_non_null(slist);
+
+	index1 = isc_skiplist_insert(slist, &entries[0]);
+	index2 = isc_skiplist_insert(slist, &entries[0]);
+	assert_int_not_equal(index1, index2);
+
+	isc_skiplist_destroy(&slist);
+	assert_null(slist);
+}
+
+ISC_RUN_TEST_IMPL(isc_skiplist_delete) {
+	isc_skiplist_t *slist = NULL;
+	isc_result_t result;
+	uint64_t index;
+
+	isc_skiplist_create(mctx, get_key, &slist);
+	assert_non_null(slist);
+
+	index = isc_skiplist_insert(slist, &entries[0]);
+	assert_int_not_equal(index, 0);
+
+	result = isc_skiplist_delete(slist, &entries[0], index);
+	assert_int_equal(result, ISC_R_SUCCESS);
+
+	result = isc_skiplist_delete(slist, &entries[1], index);
+	assert_int_equal(result, ISC_R_NOTFOUND);
+
+	isc_skiplist_destroy(&slist);
+	assert_null(slist);
+}
+
+ISC_RUN_TEST_IMPL(isc_skiplist_poprange) {
+	isc_skiplist_t *slist = NULL;
+	size_t removed;
+
+	isc_skiplist_create(mctx, get_key, &slist);
+	assert_non_null(slist);
+
+	fill_entries_drift(slist);
+	fill_entries_driftless(slist);
+
+	removed = isc_skiplist_poprange(slist, 51, 0, NULL, remove_direct);
+	assert_int_equal(removed, 10);
+
+	removed = isc_skiplist_poprange(slist, 51, 0, NULL, remove_drifting);
+	assert_int_equal(removed, 0);
+
+	fill_entries_drift(slist);
+	fill_entries_driftless(slist);
+
+	removed = isc_skiplist_poprange(slist, 51, 0, NULL, remove_drifting);
+	assert_int_equal(removed, 9);
+
+	removed = isc_skiplist_poprange(slist, 100, 15, NULL, remove_direct);
+	assert_int_equal(removed, 15);
+
+	removed = isc_skiplist_poprange(slist, 100, 0, NULL, remove_direct);
+	assert_int_equal(removed, 6);
+
+	isc_skiplist_destroy(&slist);
+	assert_null(slist);
+}
+
+ISC_TEST_LIST_START
+ISC_TEST_ENTRY(isc_skiplist_create)
+ISC_TEST_ENTRY(isc_skiplist_insert_single)
+ISC_TEST_ENTRY(isc_skiplist_insert_make_duplicate)
+ISC_TEST_ENTRY(isc_skiplist_insert)
+ISC_TEST_ENTRY(isc_skiplist_delete)
+ISC_TEST_ENTRY(isc_skiplist_poprange)
+ISC_TEST_LIST_END
+
+ISC_TEST_MAIN

tests/isc/ttlwheel_test.c

@@ -0,0 +1,150 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * SPDX-License-Identifier: MPL-2.0
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, you can obtain one at https://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+/* ! \file */
+
+#include <inttypes.h>
+#include <sched.h> /* IWYU pragma: keep */
+#include <setjmp.h>
+#include <stdarg.h>
+#include <stddef.h>
+#include <stdlib.h>
+#include <string.h>
+
+#define UNIT_TESTING
+#include <cmocka.h>
+
+#include <isc/mem.h>
+#include <isc/ttlwheel.h>
+#include <isc/util.h>
+
+#include <tests/isc.h>
+
+struct e {
+	isc_stdtime_t prev;
+};
+
+static void
+noop_action(void *user, void *data) {
+	UNUSED(user);
+
+	assert_non_null(data);
+}
+
+/* test isc_ttlwheel_create() */
+ISC_RUN_TEST_IMPL(isc_ttlwheel_create) {
+	isc_ttlwheel_t *wheel = NULL;
+
+	isc_ttlwheel_create(mctx, 10, &wheel);
+	assert_non_null(wheel);
+
+	isc_ttlwheel_destroy(&wheel);
+	assert_null(wheel);
+}
+
+/* test isc_ttlwheel_insert() */
+ISC_RUN_TEST_IMPL(isc_ttlwheel_insert) {
+	isc_ttlwheel_t *wheel = NULL;
+	struct e element;
+	uint64_t index;
+
+	isc_ttlwheel_create(mctx, 10, &wheel);
+	assert_non_null(wheel);
+
+	index = isc_ttlwheel_insert(wheel, 5, &element);
+	assert_int_equal(index, 0);
+
+	index = isc_ttlwheel_insert(wheel, 10, &element);
+	assert_int_equal(index, 0);
+
+	index = isc_ttlwheel_insert(wheel, 15, &element);
+	assert_int_not_equal(index, 0);
+
+	isc_ttlwheel_destroy(&wheel);
+	assert_null(wheel);
+}
+
+/* test isc_ttlwheel_poprange() */
+ISC_RUN_TEST_IMPL(isc_ttlwheel_poprange) {
+	isc_ttlwheel_t *wheel = NULL;
+	uint64_t index;
+	size_t removed;
+	struct e e;
+
+	e = (struct e){
+		.prev = 0,
+	};
+
+	isc_ttlwheel_create(mctx, 10, &wheel);
+	assert_non_null(wheel);
+
+	index = isc_ttlwheel_insert(wheel, 5, &e);
+	assert_int_equal(index, 0);
+
+	index = isc_ttlwheel_insert(wheel, 15, &e);
+	assert_int_not_equal(index, 0);
+
+	removed = isc_ttlwheel_poprange(wheel, 20, 1, NULL, noop_action);
+	assert_int_equal(removed, 1);
+
+	isc_ttlwheel_destroy(&wheel);
+	assert_null(wheel);
+}
+
+ISC_RUN_TEST_IMPL(isc_ttlwheel_epoch_move) {
+	isc_ttlwheel_t *wheel = NULL;
+	uint64_t index;
+	size_t cleaned;
+
+	isc_ttlwheel_create(mctx, 10, &wheel);
+	assert_non_null(wheel);
+
+	index = isc_ttlwheel_insert(wheel, 15, &index);
+	assert_int_not_equal(index, 0);
+
+	index = isc_ttlwheel_insert(wheel, 15, &index);
+	assert_int_not_equal(index, 0);
+
+	index = isc_ttlwheel_insert(wheel, 15, &index);
+	assert_int_not_equal(index, 0);
+
+	index = isc_ttlwheel_insert(wheel, 15, &index);
+	assert_int_not_equal(index, 0);
+
+	index = isc_ttlwheel_insert(wheel, 15, &index);
+	assert_int_not_equal(index, 0);
+
+	index = isc_ttlwheel_insert(wheel, 16, &index);
+	assert_int_not_equal(index, 0);
+
+	index = isc_ttlwheel_insert(wheel, 15, &index);
+	assert_int_not_equal(index, 0);
+
+	cleaned = isc_ttlwheel_poprange(wheel, 20, 5, NULL, noop_action);
+	assert_int_equal(cleaned, 5);
+
+	cleaned = isc_ttlwheel_poprange(wheel, 20, 0, NULL, noop_action);
+	assert_int_equal(cleaned, 2);
+
+	isc_ttlwheel_destroy(&wheel);
+	assert_null(wheel);
+}
+
+ISC_TEST_LIST_START
+ISC_TEST_ENTRY(isc_ttlwheel_create)
+ISC_TEST_ENTRY(isc_ttlwheel_insert)
+ISC_TEST_ENTRY(isc_ttlwheel_poprange)
+ISC_TEST_ENTRY(isc_ttlwheel_epoch_move)
+ISC_TEST_LIST_END
+
+ISC_TEST_MAIN