Improve the Userspace RCU integration

This commit allows BIND 9 to be compiled with different flavours of
Userspace RCU, and improves the integration between Userspace RCU and
our event loop:

- In the RCU QSBR, the thread is put offline when polling and online
  when rcu_dereference, rcu_assign_pointer (or friends) are called.

- In other RCU modes, we check that we are not reading when reaching the
  quiescent callback in the event loop.

- We register the thread before uv_work_run() callback is called and
  after it has finished.  The rcu_(un)register_thread() has a large
  overhead, but that's fine in this case.

configure.ac

@@ -201,6 +201,34 @@ PKG_PROG_PKG_CONFIG
 AS_IF([test -z "$PKG_CONFIG"],
       [AC_MSG_ERROR([The pkg-config script could not be found or is too old.])])
 
+AC_ARG_WITH([liburcu],
+	    [AS_HELP_STRING([--with-liburcu=FLAVOR],
+			     [Build with Userspace-RCU variant (membarrier|qsbr|mb|signal|bp) [default=membarrier]]))],
+	    [], [with_liburcu=membarrier])
+
+#
+# Pick Userspace-RCU flavor (developer only option)
+#
+
+AC_MSG_CHECKING([liburcu flavor])
+AS_CASE([$with_liburcu],
+	[membarrier],[AC_MSG_RESULT([membarrier])
+		      PKG_CHECK_MODULES([LIBURCU], [liburcu liburcu-cds])
+		      AC_DEFINE([RCU_MEMBARRIER], [1], [Build with membarrier Userspace-RCU flavor])],
+	[qsbr],[AC_MSG_RESULT([qsbr])
+		PKG_CHECK_MODULES([LIBURCU], [liburcu-qsbr liburcu-cds])
+		AC_DEFINE([RCU_QSBR], [1], [Build with QSBR Userspace-RCU flavor])],
+	[mb],[AC_MSG_RESULT([mb])
+	      PKG_CHECK_MODULES([LIBURCU], [liburcu-mb liburcu-cds])
+	      AC_DEFINE([RCU_MB], [1], [Build with mb Userspace-RCU flavor])],
+	[signal],[AC_MSG_RESULT([signal])
+		  PKG_CHECK_MODULES([LIBURCU], [liburcu-signal liburcu-cds])
+		  AC_DEFINE([RCU_SIGNAL], [1], [Build with signal Userspace-RCU flavor])],
+	[bp],[AC_MSG_RESULT([bulletproof])
+	      PKG_CHECK_MODULES([LIBURCU], [liburcu-bp liburcu-cds])
+	      AC_DEFINE([RCU_BP], [1], [Pick bulletproof Userspace-RCU flavor])],
+	[AC_MSG_ERROR([Invalid Userspace-RCU flavor picked])])
+
 # Fuzzing is not included in pairwise testing as fuzzing tools are
 # not present in the relevant Docker image.
 #
@@ -620,9 +648,6 @@ AC_CHECK_DECLS([UV_UDP_LINUX_RECVERR], [], [], [[#include <uv.h>]])
 
 AX_RESTORE_FLAGS([libuv])
 
-# userspace-rcu (QSBR variant)
-PKG_CHECK_MODULES([LIBURCU], [liburcu-qsbr liburcu-cds])
-
 # [pairwise: --enable-doh --with-libnghttp2=auto, --enable-doh --with-libnghttp2=yes, --disable-doh]
 AC_ARG_ENABLE([doh],
 	      [AS_HELP_STRING([--disable-doh], [disable DNS over HTTPS, removes dependency on libnghttp2 (default is --enable-doh)])],

lib/isc/Makefile.am

@@ -96,6 +96,7 @@ libisc_la_HEADERS =			\
 	include/isc/tls.h		\
 	include/isc/tm.h		\
 	include/isc/types.h		\
+	include/isc/urcu.h		\
 	include/isc/url.h		\
 	include/isc/utf8.h		\
 	include/isc/util.h		\

lib/isc/include/isc/urcu.h

@@ -0,0 +1,84 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * SPDX-License-Identifier: MPL-2.0
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, you can obtain one at https://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+#pragma once
+
+#include <isc/util.h>
+
+/* Inline small (less than 10 lines) functions */
+#define URCU_INLINE_SMALL_FUNCTIONS
+
+#if defined(RCU_MEMBARRIER) || defined(RCU_MB) || defined(RCU_SIGNAL)
+#include <urcu.h>
+#elif defined(RCU_QSBR)
+#include <urcu-qsbr.h>
+#elif defined(RCU_BP)
+#include <urcu-bp.h>
+#endif
+
+#include <urcu/compiler.h>
+#include <urcu/rculfhash.h>
+#include <urcu/rculist.h>
+
+#if defined(RCU_QSBR)
+
+/*
+ * Define wrappers that allows us to make the thread online without any extra
+ * heavy tooling around libuv callbacks.
+ */
+
+#define isc_qsbr_read_lock()                       \
+	{                                          \
+		if (!urcu_qsbr_read_ongoing()) {   \
+			urcu_qsbr_thread_online(); \
+		}                                  \
+		urcu_qsbr_read_lock();             \
+	}
+
+#undef rcu_read_lock
+#define rcu_read_lock() isc_qsbr_read_lock()
+
+#define isc_qsbr_call_rcu(rcu_head, func)           \
+	{                                           \
+		if (!urcu_qsbr_read_ongoing()) {    \
+			urcu_qsbr_thread_online();  \
+		}                                   \
+		urcu_qsbr_call_rcu(rcu_head, func); \
+	}
+
+#undef call_rcu
+#define call_rcu(rcu_head, func) isc_qsbr_call_rcu(rcu_head, func)
+
+#define isc_qsbr_synchronize_rcu()                 \
+	{                                          \
+		if (!urcu_qsbr_read_ongoing()) {   \
+			urcu_qsbr_thread_online(); \
+		}                                  \
+		urcu_qsbr_synchronize_rcu();       \
+	}
+
+#undef synchronize_rcu
+#define synchronize_rcu() isc_qsbr_syncronize_rcu()
+
+#define isc_qsbr_rcu_dereference(ptr)              \
+	{                                          \
+		if (!urcu_qsbr_read_ongoing()) {   \
+			urcu_qsbr_thread_online(); \
+		}                                  \
+		urcu_qsbr_dereference(ptr);        \
+	}
+
+#undef rcu_dereference
+#define rcu_dereference(ptr) isc_qsbr_rcu_dereference(ptr)
+
+#endif /* RCU_QSBR */

lib/isc/loop.c

@@ -34,6 +34,7 @@
 #include <isc/thread.h>
 #include <isc/tid.h>
 #include <isc/time.h>
+#include <isc/urcu.h>
 #include <isc/util.h>
 #include <isc/uv.h>
 #include <isc/work.h>
@@ -41,6 +42,7 @@
 #include "async_p.h"
 #include "job_p.h"
 #include "loop_p.h"
+#include "random_p.h"
 
 /**
  * Private
@@ -90,6 +92,8 @@ static void
 pause_loop(isc_loop_t *loop) {
 	isc_loopmgr_t *loopmgr = loop->loopmgr;
 
+	rcu_thread_offline();
+
 	loop->paused = true;
 	(void)isc_barrier_wait(&loopmgr->pausing);
 }
@@ -100,6 +104,7 @@ resume_loop(isc_loop_t *loop) {
 
 	(void)isc_barrier_wait(&loopmgr->resuming);
 	loop->paused = false;
+	rcu_thread_online();
 }
 
 static void
@@ -260,18 +265,55 @@ setup_jobs_cb(void *arg) {
 	}
 }
 
+static void
+quiescent_cb(uv_prepare_t *handle) {
+	isc__qsbr_quiescent_cb(handle);
+
+#ifndef RCU_QSBR
+	INSIST(!rcu_read_ongoing());
+#else
+	/* safe memory reclamation */
+	rcu_quiescent_state();
+
+	/* mark the thread offline when polling */
+	rcu_thread_offline();
+#endif
+}
+
+static void
+loop_call_rcu_init(struct rcu_head *rcu_head __attribute__((__unused__))) {
+	/* Work around the jemalloc bug, see trampoline.c for details */
+	void *ptr = malloc(8);
+	free(ptr);
+
+	/* Initialize the random generator in the call_rcu thread */
+	isc__random_initialize();
+}
+
 static void
 loop_run(isc_loop_t *loop) {
-	int r = uv_prepare_start(&loop->quiescent, isc__qsbr_quiescent_cb);
+	int r = uv_prepare_start(&loop->quiescent, quiescent_cb);
 	UV_RUNTIME_CHECK(uv_prepare_start, r);
 
 	isc_barrier_wait(&loop->loopmgr->starting);
 
 	isc_async_run(loop, setup_jobs_cb, loop);
 
+	rcu_register_thread();
+
+	struct call_rcu_data *crdp = create_call_rcu_data(0, -1);
+	set_thread_call_rcu_data(crdp);
+
+	call_rcu(&loop->rcu_head, loop_call_rcu_init);
+
 	r = uv_run(&loop->loop, UV_RUN_DEFAULT);
 	UV_RUNTIME_CHECK(uv_run, r);
 
+	rcu_unregister_thread();
+
+	set_thread_call_rcu_data(NULL);
+	call_rcu_data_free(crdp);
+
 	isc_barrier_wait(&loop->loopmgr->stopping);
 }
 

lib/isc/loop_p.h

@@ -28,6 +28,7 @@
 #include <isc/stack.h>
 #include <isc/thread.h>
 #include <isc/types.h>
+#include <isc/urcu.h>
 #include <isc/uv.h>
 #include <isc/work.h>
 
@@ -79,6 +80,8 @@ struct isc_loop {
 	uv_async_t wakeup_trigger;
 	uv_prepare_t quiescent;
 	isc_qsbr_phase_t qsbr_phase;
+
+	struct rcu_head rcu_head;
 };
 
 /*

lib/isc/work.c

@@ -15,6 +15,7 @@
 
 #include <isc/job.h>
 #include <isc/loop.h>
+#include <isc/urcu.h>
 #include <isc/uv.h>
 #include <isc/work.h>
 
@@ -24,7 +25,11 @@ static void
 isc__work_cb(uv_work_t *req) {
 	isc_work_t *work = uv_req_get_data((uv_req_t *)req);
 
+	rcu_register_thread();
+
 	work->work_cb(work->cbarg);
+
+	rcu_unregister_thread();
 }
 
 static void