WIP: Disable object reuse in the netmgr

We already print the formatted zone name via the xfrin_log() function,
generate the text once and store it in xfr->info static buffer.  Then we
can reuse the string to pass it to the LIBDNS_XFRIN probes.

CHANGES

@@ -1,3 +1,6 @@
+6227.	[func]		Add basic USDT framework for adding static
+			tracing points. [GL #4041]
+
 6226.	[bug]		Attach dispatchmgr in the dns_view object to prevent
 			use-after-free when shutting down. [GL #4228]
 

Makefile.am

@@ -8,8 +8,8 @@ SUBDIRS += tests
 # run fuzz tests before system tests
 SUBDIRS += fuzz bin
 
-BUILT_SOURCES = bind.keys.h
-CLEANFILES = bind.keys.h
+BUILT_SOURCES += bind.keys.h
+CLEANFILES += bind.keys.h
 
 bind.keys.h: bind.keys Makefile
 	${PERL} ${top_srcdir}/util/bindkeys.pl ${top_srcdir}/bind.keys > $@
@@ -19,6 +19,7 @@ bind.keys.h: bind.keys Makefile
 EXTRA_DIST =			\
 	bind.keys		\
 	util/bindkeys.pl	\
+	util/dtrace.sh		\
 	contrib			\
 	CHANGES			\
 	COPYRIGHT		\

Makefile.dtrace

@@ -0,0 +1,18 @@
+# Hey Emacs, this is -*- makefile-automake -*- file!
+# vim: filetype=automake
+
+AM_V_DTRACE = $(AM_V_DTRACE_@AM_V@)
+AM_V_DTRACE_ = $(AM_V_DTRACE_@AM_DEFAULT_V@)
+AM_V_DTRACE_0 = @echo "  DTRACE   $@";
+
+BUILT_SOURCES += probes.h
+CLEANFILES += probes.h probes.o
+
+probes.h: probes.d
+	$(AM_V_DTRACE)$(DTRACE) -s $(srcdir)/probes.d -h -o $@
+probes.lo: probes.d $(DTRACE_DEPS)
+	$(AM_V_DTRACE)$(LIBTOOL) --mode=compile --tag=CC $(DTRACE) -s $(srcdir)/probes.d -G -o $@ $(DTRACE_OBJS)
+
+if HAVE_DTRACE
+DTRACE_LIBADD = probes.lo
+endif

Makefile.top

@@ -15,6 +15,9 @@ AM_LDFLAGS =					\
 	$(STD_LDFLAGS)
 LDADD =
 
+BUILT_SOURCES =
+CLEANFILES =
+
 if HOST_MACOS
 AM_LDFLAGS +=					\
 	-Wl,-flat_namespace
@@ -26,6 +29,9 @@ LIBISC_CFLAGS =						\
 	-I$(top_builddir)/lib/isc/include
 
 LIBISC_LIBS = $(top_builddir)/lib/isc/libisc.la
+if HAVE_DTRACE
+LIBISC_DTRACE = $(top_builddir)/lib/isc/probes.lo
+endif
 
 LIBDNS_CFLAGS = \
 	-I$(top_srcdir)/lib/dns/include			\
@@ -33,12 +39,18 @@ LIBDNS_CFLAGS = \
 
 LIBDNS_LIBS = \
 	$(top_builddir)/lib/dns/libdns.la
+if HAVE_DTRACE
+LIBDNS_DTRACE = $(top_builddir)/lib/dns/probes.lo
+endif
 
 LIBNS_CFLAGS = \
 	-I$(top_srcdir)/lib/ns/include
 
 LIBNS_LIBS = \
 	$(top_builddir)/lib/ns/libns.la
+if HAVE_DTRACE
+LIBNS_DTRACE = $(top_builddir)/lib/ns/probes.lo
+endif
 
 LIBISCCFG_CFLAGS = \
 	-I$(top_srcdir)/lib/isccfg/include

bin/named/Makefile.am

@@ -38,8 +38,8 @@ AM_CPPFLAGS +=						\
 sbin_PROGRAMS = named
 
 nodist_named_SOURCES = xsl.c
-BUILT_SOURCES = xsl.c
-CLEANFILES = xsl.c
+BUILT_SOURCES += xsl.c
+CLEANFILES += xsl.c
 
 EXTRA_DIST = bind9.xsl
 

configure.ac

@@ -1568,6 +1568,49 @@ AC_CHECK_HEADERS([glob.h])
 AX_GCC_FUNC_ATTRIBUTE([constructor])
 AX_GCC_FUNC_ATTRIBUTE([destructor])
 
+#
+# Check whether to enable User Statically Defined Tracing.
+#
+# This is supported only on Linux now and requires SystemTap libraries.
+#
+# [pairwise: --enable-tracing=auto, --enable-tracing=yes, --disable-tracing]
+AC_ARG_ENABLE([tracing],
+	      [AS_HELP_STRING([--enable-tracing],
+			      [enable User Statically Defined Tracing])],
+	      [],
+	      [enable_tracing=auto])
+
+AC_ARG_VAR([DTRACE], [path to dtrace binary used to build User Statically Defined Tracing probes])
+AC_PATH_PROGS([DTRACE], [dtrace])
+AC_CHECK_HEADERS([sys/sdt.h])
+
+AC_MSG_CHECKING([whether to enable User Statically Defined Tracing support (default is auto)])
+AS_CASE([$enable_tracing],
+	[auto],	[enable_tracing=yes
+		 AS_IF([test "$ac_cv_header_sys_sdt_h" != "yes"], [enable_tracing=no])
+		 AS_IF([test -z "$DTRACE"],[enable_tracing=no])])
+AS_CASE([$enable_tracing],
+	[yes], [AS_IF([test "$ac_cv_header_sys_sdt_h" != "yes"],
+		      [AC_MSG_ERROR([sys/sdt.h header not found])])
+		AS_IF([test -z "$DTRACE"],
+		      [AC_MSG_ERROR([The dtrace command not found])])
+		AC_MSG_RESULT([yes])],
+	[DTRACE=""
+	 AC_MSG_RESULT([no])])
+
+AS_CASE([$host],
+	[*-linux*],[have_systemtap=true])
+
+AM_CONDITIONAL([HAVE_SYSTEMTAP], [test -n "$have_systemtap"])
+AM_CONDITIONAL([HAVE_DTRACE], [test -n "$DTRACE"])
+
+# Use shim script if dtrace is not available
+AS_IF([test -z "$DTRACE"],
+      [DTRACE='$(top_srcdir)/util/dtrace.sh'])
+
+AC_SUBST([DTRACE])
+
+
 #
 # Files to configure.  These are listed here because we used to
 # specify them as arguments to AC_OUTPUT.

doc/man/Makefile.am

@@ -123,7 +123,7 @@ EXTRA_DIST =				\
 	$(MANPAGES_RST)			\
 	$(MANPAGES_IN)
 
-CLEANFILES =				\
+CLEANFILES +=				\
 	$(man_MANS)
 
 #

doc/misc/Makefile.am

@@ -37,7 +37,7 @@ cfg_test_LDADD =			\
 	$(LIBDNS_LIBS)			\
 	$(LIBISCCFG_LIBS)
 
-BUILT_SOURCES =				\
+BUILT_SOURCES +=				\
 	$(OPTIONS_FILES)
 
 rndc.grammar: cfg_test

doc/notes/notes-current.rst

@@ -20,7 +20,9 @@ Security Fixes
 New Features
 ~~~~~~~~~~~~
 
-- None.
+- Add support for User Statically Defined Tracing (USDT) probes - static tracing
+  points for user-level software.  This allows a fine-grained application
+  tracing with zero-overhead when the probes are not enabled. :gl:`#4041`
 
 Removed Features
 ~~~~~~~~~~~~~~~~

lib/dns/.gitignore

@@ -6,4 +6,5 @@ enumtype.h
 rdatastruct.h
 gen.dSYM
 dnstap.pb-c.c
-dnstap.pb-c.h
+dnstap.pb-c.h
+/probes.h

lib/dns/Makefile.am

@@ -12,10 +12,10 @@ nodist_libdns_la_SOURCES =		\
 	$(nodist_libdns_la_HEADERS)	\
 	code.h
 
-BUILT_SOURCES =				\
+BUILT_SOURCES +=			\
 	$(nodist_libdns_la_SOURCES)
 
-CLEANFILES =				\
+CLEANFILES +=				\
 	$(nodist_libdns_la_SOURCES)	\
 	gen$(BUILD_EXEEXT)
 
@@ -124,6 +124,7 @@ libdns_la_HEADERS =			\
 	include/dns/ssu.h		\
 	include/dns/stats.h		\
 	include/dns/time.h		\
+	include/dns/trace.h		\
 	include/dns/transport.h		\
 	include/dns/tkey.h		\
 	include/dns/trace.h		\
@@ -208,6 +209,7 @@ libdns_la_SOURCES =			\
 	order.c				\
 	peer.c				\
 	private.c			\
+	probes.d			\
 	qp.c				\
 	qp_p.h				\
 	rbt.c				\
@@ -330,3 +332,12 @@ if HAVE_LMDB
 libdns_la_CPPFLAGS += $(LMDB_CFLAGS)
 libdns_la_LIBADD += $(LMDB_LIBS)
 endif
+
+if !HAVE_SYSTEMTAP
+DTRACE_DEPS = libdns_la-xfrin.lo
+DTRACE_OBJS = .libs/libdns_la-xfrin.$(OBJEXT)
+endif
+
+include $(top_srcdir)/Makefile.dtrace
+
+libdns_la_LIBADD += $(DTRACE_LIBADD)

lib/dns/probes.d

@@ -0,0 +1,32 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * SPDX-License-Identifier: MPL-2.0
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, you can obtain one at https://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+provider libdns {
+	probe xfrin_axfr_finalize_begin(void *, char *);
+	probe xfrin_axfr_finalize_end(void *, char *, int);
+	probe xfrin_connected(void *, char *, int);
+	probe xfrin_done_callback_begin(void *, char *, int);
+	probe xfrin_done_callback_end(void *, char *, int);
+	probe xfrin_journal_destroy_begin(void *, char *, int);
+	probe xfrin_journal_destroy_end(void *, char *, int);
+	probe xfrin_read(void *, char *, int);
+	probe xfrin_recv_answer(void *, char *, void *);
+	probe xfrin_recv_done(void *, char *, int);
+	probe xfrin_recv_parsed(void *, char *, int);
+	probe xfrin_recv_question(void *, char *, void *);
+	probe xfrin_recv_send_request(void *, char *);
+	probe xfrin_recv_start(void *, char *, int);
+	probe xfrin_recv_try_axfr(void *, char *, int);
+	probe xfrin_sent(void *, char *, int);
+	probe xfrin_start(void *, char *);
+};

lib/dns/xfrin.c

@@ -35,6 +35,7 @@
 #include <dns/rdataset.h>
 #include <dns/result.h>
 #include <dns/soa.h>
+#include <dns/trace.h>
 #include <dns/transport.h>
 #include <dns/tsig.h>
 #include <dns/view.h>
@@ -43,6 +44,8 @@
 
 #include <dst/dst.h>
 
+#include "probes.h"
+
 /*
  * Incoming AXFR and IXFR.
  */
@@ -180,6 +183,8 @@ struct dns_xfrin {
 
 	isc_timer_t *max_time_timer;
 	isc_timer_t *max_idle_timer;
+
+	char info[DNS_NAME_MAXTEXT + 32];
 };
 
 #define XFRIN_MAGIC    ISC_MAGIC('X', 'f', 'r', 'I')
@@ -249,11 +254,6 @@ xfrin_fail(dns_xfrin_t *xfr, isc_result_t result, const char *msg);
 static isc_result_t
 render(dns_message_t *msg, isc_mem_t *mctx, isc_buffer_t *buf);
 
-static void
-xfrin_logv(dns_xfrin_t *xff, int level, const char *zonetext,
-	   const isc_sockaddr_t *primaryaddr, const char *fmt, va_list ap)
-	ISC_FORMAT_PRINTF(5, 0);
-
 static void
 xfrin_log(dns_xfrin_t *xfr, int level, const char *fmt, ...)
 	ISC_FORMAT_PRINTF(3, 4);
@@ -360,10 +360,10 @@ static isc_result_t
 axfr_finalize(dns_xfrin_t *xfr) {
 	isc_result_t result;
 
-	CHECK(dns_zone_replacedb(xfr->zone, xfr->db, true));
+	LIBDNS_XFRIN_AXFR_FINALIZE_BEGIN(xfr, xfr->info);
+	result = dns_zone_replacedb(xfr->zone, xfr->db, true);
+	LIBDNS_XFRIN_AXFR_FINALIZE_END(xfr, xfr->info, result);
 
-	result = ISC_R_SUCCESS;
-failure:
 	return (result);
 }
 
@@ -910,6 +910,8 @@ xfrin_create(isc_mem_t *mctx, dns_zone_t *zone, dns_db_t *db,
 	isc_timer_create(dns_zone_getloop(zone), xfrin_idledout, xfr,
 			 &xfr->max_idle_timer);
 
+	dns_zone_name(xfr->zone, xfr->info, sizeof(xfr->info));
+
 	*xfrp = xfr;
 }
 
@@ -956,6 +958,8 @@ xfrin_start(dns_xfrin_t *xfr) {
 		CHECK(result);
 	}
 
+	LIBDNS_XFRIN_START(xfr, xfr->info);
+
 	/* Set the maximum timer */
 	isc_interval_set(&interval, dns_zone_getmaxxfrin(xfr->zone), 0);
 	isc_timer_start(xfr->max_time_timer, isc_timertype_once, &interval);
@@ -1027,6 +1031,8 @@ xfrin_connect_done(isc_result_t result, isc_region_t *region ISC_ATTR_UNUSED,
 		result = ISC_R_SHUTTINGDOWN;
 	}
 
+	LIBDNS_XFRIN_CONNECTED(xfr, xfr->info, result);
+
 	if (result != ISC_R_SUCCESS) {
 		xfrin_fail(xfr, result, "failed to connect");
 		goto failure;
@@ -1155,6 +1161,8 @@ xfrin_send_request(dns_xfrin_t *xfr) {
 	dns_dbversion_t *ver = NULL;
 	dns_name_t *msgsoaname = NULL;
 
+	LIBDNS_XFRIN_RECV_SEND_REQUEST(xfr, xfr->info);
+
 	/* Create the request message */
 	dns_message_create(xfr->mctx, DNS_MESSAGE_INTENTRENDER, &msg);
 	CHECK(dns_message_settsigkey(msg, xfr->tsigkey));
@@ -1245,6 +1253,8 @@ xfrin_send_done(isc_result_t result, isc_region_t *region, void *arg) {
 		result = ISC_R_SHUTTINGDOWN;
 	}
 
+	LIBDNS_XFRIN_SENT(xfr, xfr->info, result);
+
 	CHECK(result);
 
 	xfrin_log(xfr, ISC_LOG_DEBUG(3), "sent request data");
@@ -1274,6 +1284,8 @@ xfrin_recv_done(isc_result_t result, isc_region_t *region, void *arg) {
 	/* Stop the idle timer */
 	isc_timer_stop(xfr->max_idle_timer);
 
+	LIBDNS_XFRIN_RECV_START(xfr, xfr->info, result);
+
 	CHECK(result);
 
 	xfrin_log(xfr, ISC_LOG_DEBUG(7), "received %u bytes", region->length);
@@ -1307,6 +1319,8 @@ xfrin_recv_done(isc_result_t result, isc_region_t *region, void *arg) {
 			  isc_result_totext(result));
 	}
 
+	LIBDNS_XFRIN_RECV_PARSED(xfr, xfr->info, result);
+
 	if (result != ISC_R_SUCCESS || msg->rcode != dns_rcode_noerror ||
 	    msg->opcode != dns_opcode_query || msg->rdclass != xfr->rdclass)
 	{
@@ -1334,6 +1348,7 @@ xfrin_recv_done(isc_result_t result, isc_region_t *region, void *arg) {
 		xfrin_log(xfr, ISC_LOG_DEBUG(3), "got %s, retrying with AXFR",
 			  isc_result_totext(result));
 	try_axfr:
+		LIBDNS_XFRIN_RECV_TRY_AXFR(xfr, xfr->info, result);
 		dns_message_detach(&msg);
 		xfrin_reset(xfr);
 		xfr->reqtype = dns_rdatatype_soa;
@@ -1374,6 +1389,8 @@ xfrin_recv_done(isc_result_t result, isc_region_t *region, void *arg) {
 	{
 		dns_rdataset_t *rds = NULL;
 
+		LIBDNS_XFRIN_RECV_QUESTION(xfr, xfr->info, msg);
+
 		name = NULL;
 		dns_message_currentname(msg, DNS_SECTION_QUESTION, &name);
 		if (!dns_name_equal(name, &xfr->name)) {
@@ -1435,6 +1452,8 @@ xfrin_recv_done(isc_result_t result, isc_region_t *region, void *arg) {
 	{
 		dns_rdataset_t *rds = NULL;
 
+		LIBDNS_XFRIN_RECV_ANSWER(xfr, xfr->info, msg);
+
 		name = NULL;
 		dns_message_currentname(msg, DNS_SECTION_ANSWER, &name);
 		for (rds = ISC_LIST_HEAD(name->list); rds != NULL;
@@ -1513,15 +1532,22 @@ xfrin_recv_done(isc_result_t result, isc_region_t *region, void *arg) {
 		 * Close the journal.
 		 */
 		if (xfr->ixfr.journal != NULL) {
+			LIBDNS_XFRIN_JOURNAL_DESTROY_BEGIN(xfr, xfr->info,
+							   result);
 			dns_journal_destroy(&xfr->ixfr.journal);
+			LIBDNS_XFRIN_JOURNAL_DESTROY_END(xfr, xfr->info,
+							 result);
 		}
 
 		/*
 		 * Inform the caller we succeeded.
 		 */
 		if (xfr->done != NULL) {
+			LIBDNS_XFRIN_DONE_CALLBACK_BEGIN(xfr, xfr->info,
+							 result);
 			(xfr->done)(xfr->zone, ISC_R_SUCCESS);
 			xfr->done = NULL;
+			LIBDNS_XFRIN_DONE_CALLBACK_END(xfr, xfr->info, result);
 		}
 
 		atomic_store(&xfr->shuttingdown, true);
@@ -1539,6 +1565,8 @@ xfrin_recv_done(isc_result_t result, isc_region_t *region, void *arg) {
 		isc_interval_set(&interval, dns_zone_getidlein(xfr->zone), 0);
 		isc_timer_start(xfr->max_idle_timer, isc_timertype_once,
 				&interval);
+
+		LIBDNS_XFRIN_READ(xfr, xfr->info, result);
 		return;
 	}
 
@@ -1551,6 +1579,7 @@ failure:
 		dns_message_detach(&msg);
 	}
 	dns_xfrin_detach(&xfr);
+	LIBDNS_XFRIN_RECV_DONE(xfr, xfr->info, result);
 }
 
 static void
@@ -1674,36 +1703,24 @@ xfrin_destroy(dns_xfrin_t *xfr) {
  * Log incoming zone transfer messages in a format like
  * transfer of <zone> from <address>: <message>
  */
-static void
-xfrin_logv(dns_xfrin_t *xfr, int level, const char *zonetext,
-	   const isc_sockaddr_t *primaryaddr, const char *fmt, va_list ap) {
-	char primarytext[ISC_SOCKADDR_FORMATSIZE];
-	char msgtext[2048];
-
-	isc_sockaddr_format(primaryaddr, primarytext, sizeof(primarytext));
-	vsnprintf(msgtext, sizeof(msgtext), fmt, ap);
-
-	isc_log_write(dns_lctx, DNS_LOGCATEGORY_XFER_IN, DNS_LOGMODULE_XFER_IN,
-		      level, "%p: transfer of '%s' from %s: %s", xfr, zonetext,
-		      primarytext, msgtext);
-}
-
-/*
- * Logging function for use when there is a xfrin_ctx_t.
- */
 
 static void
 xfrin_log(dns_xfrin_t *xfr, int level, const char *fmt, ...) {
 	va_list ap;
-	char zonetext[DNS_NAME_MAXTEXT + 32];
+	char primarytext[ISC_SOCKADDR_FORMATSIZE];
+	char msgtext[2048];
 
 	if (!isc_log_wouldlog(dns_lctx, level)) {
 		return;
 	}
 
-	dns_zone_name(xfr->zone, zonetext, sizeof(zonetext));
-
+	isc_sockaddr_format(&xfr->primaryaddr, primarytext,
+			    sizeof(primarytext));
 	va_start(ap, fmt);
-	xfrin_logv(xfr, level, zonetext, &xfr->primaryaddr, fmt, ap);
+	vsnprintf(msgtext, sizeof(msgtext), fmt, ap);
 	va_end(ap);
+
+	isc_log_write(dns_lctx, DNS_LOGCATEGORY_XFER_IN, DNS_LOGMODULE_XFER_IN,
+		      level, "%p: transfer of '%s' from %s: %s", xfr, xfr->info,
+		      primarytext, msgtext);
 }

lib/isc/.gitignore

@@ -1 +1,2 @@
 !netdb.h
+/probes.h

lib/isc/Makefile.am

@@ -171,6 +171,7 @@ libisc_la_SOURCES =		\
 	picohttpparser.c	\
 	picohttpparser.h	\
 	portset.c		\
+	probes.d		\
 	quota.c			\
 	radix.c			\
 	random.c		\
@@ -256,3 +257,12 @@ libisc_la_CPPFLAGS +=		\
 libisc_la_LIBADD +=		\
 	$(LIBXML2_LIBS)
 endif HAVE_LIBXML2
+
+if !HAVE_SYSTEMTAP
+DTRACE_DEPS = libisc_la-rwlock.lo libisc_la-job.lo
+DTRACE_OBJS = .libs/libisc_la-rwlock.$(OBJEXT) .libs/libisc_la-job.$(OBJEXT)
+endif
+
+include $(top_srcdir)/Makefile.dtrace
+
+libisc_la_LIBADD += $(DTRACE_LIBADD)

lib/isc/job.c

@@ -35,6 +35,7 @@
 
 #include "job_p.h"
 #include "loop_p.h"
+#include "probes.h"
 
 /*
  * Public: #include <isc/job.h>
@@ -70,8 +71,12 @@ isc__job_cb(uv_idle_t *handle) {
 	     job != NULL;
 	     job = next, next = job ? ISC_LIST_NEXT(job, link) : NULL)
 	{
+		isc_job_cb cb = job->cb;
+		void *cbarg = job->cbarg;
 		ISC_LIST_UNLINK(jobs, job, link);
-		job->cb(job->cbarg);
+		LIBISC_JOB_CB_BEFORE(job, cb, cbarg);
+		cb(cbarg);
+		LIBISC_JOB_CB_AFTER(job, cb, cbarg);
 	}
 
 	if (ISC_LIST_EMPTY(loop->run_jobs)) {

lib/isc/netmgr/netmgr-int.h

@@ -198,8 +198,6 @@ typedef struct isc__networker {
 	bool recvbuf_inuse;
 
 	ISC_LIST(isc_nmsocket_t) active_sockets;
-
-	isc_mempool_t *uvreq_pool;
 } isc__networker_t;
 
 ISC_REFCOUNT_DECL(isc__networker);
@@ -619,11 +617,6 @@ struct isc_nmsocket {
 	 */
 	bool keepalive;
 
-	/*%
-	 * 'spare' handles for that can be reused to avoid allocations, for UDP.
-	 */
-	ISC_LIST(isc_nmhandle_t) inactive_handles;
-
 	/*%
 	 * 'active' handles and uvreqs, mostly for debugging purposes.
 	 */

lib/isc/netmgr/netmgr.c

@@ -237,9 +237,6 @@ isc_netmgr_create(isc_mem_t *mctx, isc_loopmgr_t *loopmgr, isc_nm_t **netmgrp) {
 
 		isc_mem_attach(loop->mctx, &worker->mctx);
 
-		isc_mempool_create(worker->mctx, sizeof(isc__nm_uvreq_t),
-				   &worker->uvreq_pool);
-
 		isc_loop_attach(loop, &worker->loop);
 		isc_loop_teardown(loop, networker_teardown, worker);
 		isc_refcount_init(&worker->references, 1);
@@ -433,7 +430,6 @@ nmsocket_cleanup(void *arg) {
 	REQUIRE(VALID_NMSOCK(sock));
 	REQUIRE(!isc__nmsocket_active(sock));
 
-	isc_nmhandle_t *handle = NULL;
 	isc__networker_t *worker = sock->worker;
 
 	isc_refcount_destroy(&sock->references);
@@ -472,11 +468,6 @@ nmsocket_cleanup(void *arg) {
 		isc__nmsocket_detach(&sock->outer);
 	}
 
-	while ((handle = ISC_LIST_HEAD(sock->inactive_handles)) != NULL) {
-		ISC_LIST_DEQUEUE(sock->inactive_handles, handle, inactive_link);
-		nmhandle_free(sock, handle);
-	}
-
 	INSIST(sock->server == NULL);
 	sock->pquota = NULL;
 
@@ -679,7 +670,6 @@ isc___nmsocket_init(isc_nmsocket_t *sock, isc__networker_t *worker,
 		.type = type,
 		.tid = worker->loop->tid,
 		.fd = -1,
-		.inactive_handles = ISC_LIST_INITIALIZER,
 		.result = ISC_R_UNSET,
 		.active_handles = ISC_LIST_INITIALIZER,
 		.active_link = ISC_LINK_INITIALIZER,
@@ -799,32 +789,12 @@ alloc_handle(isc_nmsocket_t *sock) {
 	return (handle);
 }
 
-static isc_nmhandle_t *
-dequeue_handle(isc_nmsocket_t *sock) {
-#if !__SANITIZE_ADDRESS__ && !__SANITIZE_THREAD__
-	isc_nmhandle_t *handle = ISC_LIST_HEAD(sock->inactive_handles);
-	if (handle != NULL) {
-		ISC_LIST_DEQUEUE(sock->inactive_handles, handle, inactive_link);
-
-		isc_refcount_init(&handle->references, 1);
-		INSIST(VALID_NMHANDLE(handle));
-		return (handle);
-	}
-#else
-	INSIST(ISC_LIST_EMPTY(sock->inactive_handles));
-#endif /* !__SANITIZE_ADDRESS__ && !__SANITIZE_THREAD__ */
-	return (NULL);
-}
-
 isc_nmhandle_t *
 isc___nmhandle_get(isc_nmsocket_t *sock, isc_sockaddr_t const *peer,
 		   isc_sockaddr_t const *local FLARG) {
 	REQUIRE(VALID_NMSOCK(sock));
 
-	isc_nmhandle_t *handle = dequeue_handle(sock);
-	if (handle == NULL) {
-		handle = alloc_handle(sock);
-	}
+	isc_nmhandle_t *handle = alloc_handle(sock);
 
 	NETMGR_TRACE_LOG(
 		"isc__nmhandle_get():handle %p->references = %" PRIuFAST32 "\n",
@@ -910,15 +880,7 @@ nmhandle__destroy(isc_nmhandle_t *handle) {
 	isc_nmsocket_t *sock = handle->sock;
 	handle->sock = NULL;
 
-#if defined(__SANITIZE_ADDRESS__) || defined(__SANITIZE_THREAD__)
 	nmhandle_free(sock, handle);
-#else
-	if (sock->active) {
-		ISC_LIST_APPEND(sock->inactive_handles, handle, inactive_link);
-	} else {
-		nmhandle_free(sock, handle);
-	}
-#endif
 
 	isc__nmsocket_detach(&sock);
 }
@@ -1504,7 +1466,7 @@ isc___nm_uvreq_get(isc_nmsocket_t *sock FLARG) {
 
 	isc__networker_t *worker = sock->worker;
 
-	isc__nm_uvreq_t *req = isc_mempool_get(worker->uvreq_pool);
+	isc__nm_uvreq_t *req = isc_mem_get(worker->mctx, sizeof(*req));
 	*req = (isc__nm_uvreq_t){
 		.connect_tries = 3,
 		.link = ISC_LINK_INITIALIZER,
@@ -1543,7 +1505,7 @@ isc___nm_uvreq_put(isc__nm_uvreq_t **reqp FLARG) {
 #endif
 	}
 
-	isc_mempool_put(sock->worker->uvreq_pool, req);
+	isc_mem_put(sock->worker->mctx, req, sizeof(*req));
 
 	isc___nmsocket_detach(&sock FLARG_PASS);
 }
@@ -2304,8 +2266,6 @@ isc__networker_destroy(isc__networker_t *worker) {
 
 	isc_loop_detach(&worker->loop);
 
-	isc_mempool_destroy(&worker->uvreq_pool);
-
 	isc_mem_putanddetach(&worker->mctx, worker->recvbuf,
 			     ISC_NETMGR_RECVBUF_SIZE);
 	isc_nm_detach(&netmgr);

lib/isc/probes.d

@@ -0,0 +1,30 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * SPDX-License-Identifier: MPL-2.0
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, you can obtain one at https://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+provider libisc {
+	probe job_cb_after(void *, void *, void *);
+	probe job_cb_before(void *, void *, void *);
+
+	probe rwlock_destroy(void *);
+	probe rwlock_downgrade(void *);
+	probe rwlock_init(void *);
+	probe rwlock_rdlock_acq(void *);
+	probe rwlock_rdlock_req(void *);
+	probe rwlock_rdunlock(void *);
+	probe rwlock_tryrdlock(void *, int);
+	probe rwlock_tryupgrade(void *, int);
+	probe rwlock_trywrlock(void *, int);
+	probe rwlock_wrlock_acq(void *);
+	probe rwlock_wrlock_req(void *);
+	probe rwlock_wrunlock(void *);
+};

lib/isc/rwlock.c

@@ -62,6 +62,8 @@
 #include <isc/tid.h>
 #include <isc/util.h>
 
+#include "probes.h"
+
 static atomic_uint_fast16_t isc__crwlock_workers = 128;
 
 #define ISC_RWLOCK_UNLOCKED false
@@ -137,6 +139,8 @@ isc_rwlock_rdlock(isc_rwlock_t *rwl) {
 	uint32_t cnt = 0;
 	bool barrier_raised = false;
 
+	LIBISC_RWLOCK_RDLOCK_REQ(rwl);
+
 	while (true) {
 		read_indicator_arrive(rwl);
 		if (!writers_lock_islocked(rwl)) {
@@ -158,6 +162,8 @@ isc_rwlock_rdlock(isc_rwlock_t *rwl) {
 	if (barrier_raised) {
 		writers_barrier_lower(rwl);
 	}
+
+	LIBISC_RWLOCK_RDLOCK_ACQ(rwl);
 }
 
 isc_result_t
@@ -167,27 +173,32 @@ isc_rwlock_tryrdlock(isc_rwlock_t *rwl) {
 		/* Writer has acquired the lock, release the read lock */
 		read_indicator_depart(rwl);
 
+		LIBISC_RWLOCK_TRYRDLOCK(rwl, ISC_R_LOCKBUSY);
 		return (ISC_R_LOCKBUSY);
 	}
 
 	/* Acquired lock in read-only mode */
+	LIBISC_RWLOCK_TRYRDLOCK(rwl, ISC_R_SUCCESS);
 	return (ISC_R_SUCCESS);
 }
 
 void
 isc_rwlock_rdunlock(isc_rwlock_t *rwl) {
 	read_indicator_depart(rwl);
+	LIBISC_RWLOCK_RDUNLOCK(rwl);
 }
 
 isc_result_t
 isc_rwlock_tryupgrade(isc_rwlock_t *rwl) {
 	/* Write Barriers has been raised */
 	if (writers_barrier_israised(rwl)) {
+		LIBISC_RWLOCK_TRYUPGRADE(rwl, ISC_R_LOCKBUSY);
 		return (ISC_R_LOCKBUSY);
 	}
 
 	/* Try to acquire the write-lock */
 	if (!writers_lock_acquire(rwl)) {
+		LIBISC_RWLOCK_TRYUPGRADE(rwl, ISC_R_LOCKBUSY);
 		return (ISC_R_LOCKBUSY);
 	}
 
@@ -200,8 +211,10 @@ isc_rwlock_tryupgrade(isc_rwlock_t *rwl) {
 
 		/* Unlock the write-lock */
 		writers_lock_release(rwl);
+		LIBISC_RWLOCK_TRYUPGRADE(rwl, ISC_R_LOCKBUSY);
 		return (ISC_R_LOCKBUSY);
 	}
+	LIBISC_RWLOCK_TRYUPGRADE(rwl, ISC_R_SUCCESS);
 	return (ISC_R_SUCCESS);
 }
 
@@ -218,6 +231,8 @@ read_indicator_wait_until_empty(isc_rwlock_t *rwl) {
 
 void
 isc_rwlock_wrlock(isc_rwlock_t *rwl) {
+	LIBISC_RWLOCK_WRLOCK_REQ(rwl);
+
 	/* Write Barriers has been raised, wait */
 	while (writers_barrier_israised(rwl)) {
 		isc_pause();
@@ -229,22 +244,27 @@ isc_rwlock_wrlock(isc_rwlock_t *rwl) {
 	}
 
 	read_indicator_wait_until_empty(rwl);
+
+	LIBISC_RWLOCK_WRLOCK_ACQ(rwl);
 }
 
 void
 isc_rwlock_wrunlock(isc_rwlock_t *rwl) {
 	writers_lock_release(rwl);
+	LIBISC_RWLOCK_WRUNLOCK(rwl);
 }
 
 isc_result_t
 isc_rwlock_trywrlock(isc_rwlock_t *rwl) {
 	/* Write Barriers has been raised */
 	if (writers_barrier_israised(rwl)) {
+		LIBISC_RWLOCK_TRYWRLOCK(rwl, ISC_R_LOCKBUSY);
 		return (ISC_R_LOCKBUSY);
 	}
 
 	/* Try to acquire the write-lock */
 	if (!writers_lock_acquire(rwl)) {
+		LIBISC_RWLOCK_TRYWRLOCK(rwl, ISC_R_LOCKBUSY);
 		return (ISC_R_LOCKBUSY);
 	}
 
@@ -252,9 +272,11 @@ isc_rwlock_trywrlock(isc_rwlock_t *rwl) {
 		/* Unlock the write-lock */
 		writers_lock_release(rwl);
 
+		LIBISC_RWLOCK_TRYWRLOCK(rwl, ISC_R_LOCKBUSY);
 		return (ISC_R_LOCKBUSY);
 	}
 
+	LIBISC_RWLOCK_TRYWRLOCK(rwl, ISC_R_SUCCESS);
 	return (ISC_R_SUCCESS);
 }
 
@@ -263,6 +285,8 @@ isc_rwlock_downgrade(isc_rwlock_t *rwl) {
 	read_indicator_arrive(rwl);
 
 	writers_lock_release(rwl);
+
+	LIBISC_RWLOCK_DOWNGRADE(rwl);
 }
 
 void
@@ -273,10 +297,12 @@ isc_rwlock_init(isc_rwlock_t *rwl) {
 	atomic_init(&rwl->writers_barrier, 0);
 	atomic_init(&rwl->readers_ingress, 0);
 	atomic_init(&rwl->readers_egress, 0);
+	LIBISC_RWLOCK_INIT(rwl);
 }
 
 void
 isc_rwlock_destroy(isc_rwlock_t *rwl) {
+	LIBISC_RWLOCK_DESTROY(rwl);
 	/* Check whether write lock has been unlocked */
 	REQUIRE(atomic_load(&rwl->writers_lock) == ISC_RWLOCK_UNLOCKED);
 	REQUIRE(read_indicator_isempty(rwl));

lib/ns/.gitignore

@@ -0,0 +1 @@
+/probes.h

lib/ns/Makefile.am

@@ -30,6 +30,7 @@ libns_la_SOURCES =		\
 	listenlist.c		\
 	log.c			\
 	notify.c		\
+	probes.d		\
 	query.c			\
 	server.c		\
 	sortlist.c		\
@@ -54,3 +55,12 @@ libns_la_LIBADD =	\
 libns_la_LDFLAGS =		\
 	$(AM_LDFLAGS)		\
 	-release "$(PACKAGE_VERSION)"
+
+if !HAVE_SYSTEMTAP
+DTRACE_DEPS = libns_la-query.lo
+DTRACE_OBJS = .libs/libns_la-query.$(OBJEXT)
+endif
+
+include $(top_srcdir)/Makefile.dtrace
+
+libns_la_LIBADD += $(DTRACE_LIBADD)

lib/ns/probes.d

@@ -0,0 +1,16 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * SPDX-License-Identifier: MPL-2.0
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, you can obtain one at https://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+provider libns {
+	probe rrl_drop(const char *, const char *, const char *, int);
+};

lib/ns/query.c

@@ -73,6 +73,8 @@
 #include <ns/stats.h>
 #include <ns/xfrout.h>
 
+#include "probes.h"
+
 #if 0
 /*
  * It has been recommended that DNS64 be changed to return excluded
@@ -7051,6 +7053,25 @@ ns__query_sfcache(query_ctx_t *qctx) {
 	return (ISC_R_COMPLETE);
 }
 
+static void
+query_trace_rrldrop(query_ctx_t *qctx,
+		    dns_rrl_result_t rrl_result ISC_ATTR_UNUSED) {
+	if (!LIBNS_RRL_DROP_ENABLED()) {
+		return;
+	}
+
+	char peerbuf[ISC_SOCKADDR_FORMATSIZE];
+	isc_netaddr_t peer;
+	isc_netaddr_fromsockaddr(&peer, &qctx->client->peeraddr);
+	isc_netaddr_format(&peer, peerbuf, sizeof(peerbuf));
+
+	char qnamebuf[DNS_NAME_FORMATSIZE];
+	char fnamebuf[DNS_NAME_FORMATSIZE];
+	dns_name_format(qctx->client->query.qname, qnamebuf, sizeof(qnamebuf));
+	dns_name_format(qctx->fname, fnamebuf, sizeof(fnamebuf));
+	LIBNS_RRL_DROP(peerbuf, qnamebuf, fnamebuf, rrl_result);
+}
+
 /*%
  * Handle response rate limiting (RRL).
  */
@@ -7184,6 +7205,12 @@ query_checkrrl(query_ctx_t *qctx, isc_result_t result) {
 					      DNS_RRL_LOG_DROP, "%s", log_buf);
 			}
 
+			/*
+			 * If tracing is enabled, format some extra information
+			 * to pass along.
+			 */
+			query_trace_rrldrop(qctx, rrl_result);
+
 			if (!qctx->view->rrl->log_only) {
 				if (rrl_result == DNS_RRL_RESULT_DROP) {
 					/*

tests/dns/Makefile.am

@@ -48,8 +48,6 @@ check_PROGRAMS =		\
 	zonemgr_test		\
 	zt_test
 
-CLEANFILES =
-
 if HAVE_PERL
 
 check_PROGRAMS +=		\

tests/isc/job_test.c

@@ -31,8 +31,6 @@
 #include <isc/result.h>
 #include <isc/util.h>
 
-#include "job.c"
-
 #include <tests/isc.h>
 
 static atomic_uint scheduled;

util/dtrace.sh

@@ -0,0 +1,49 @@
+#!/bin/sh
+
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+USAGE="# Usage: ${0} [-h | -G] -s File.d [-o <File>]"
+
+mode=
+while getopts hGs:o: opt; do
+	case "${opt}" in
+		h) mode=header ;;
+		s) input=$OPTARG ;;
+		o) output=$OPTARG ;;
+		G) mode=object ;;
+		\?) echo $USAGE; exit 1;;
+	esac
+done
+shift $((OPTIND - 1))
+
+if test -z "${mode}" || test -z "${input}"; then
+	echo $USAGE; exit 1;
+fi
+
+case "${mode}" in
+	header)
+		if test -z "${output}"; then
+			output="$(basename "${input}" .d).h"
+		fi
+		PROVIDER=$(cat "${input}" | sed -ne 's/^provider \(.*\) {/\1/p' | tr "a-z" "A-Z")
+		sed -ne 's/.*probe \(.*\)(.*);/\1/p' "${input}" | tr "a-z" "A-Z" | while read PROBE; do
+			echo "#define ${PROVIDER}_${PROBE}_ENABLED() 0";
+			echo "#define ${PROVIDER}_${PROBE}(...)";
+		done > "${output}"
+		;;
+	object)
+		if test -z "${output}"; then
+			output="$(basename "${input}" .d).o"
+		fi
+		echo "extern int empty;" | gcc -xc -c - -fPIC -DPIC  -o "${output}"
+		;;
+esac