Disable the all netmgr tests only in the CI

The full netmgr test suite is unstable when run in the CI due to various
timing reasons.  Previously, we enabled the full test suite only when
CI_ENABLE_ALL_TESTS environment variable was set, but that went against
original intent of running the full suite when an individual developer
would run it locally.

This change disables the full test suite only when running in the CI and
the CI_ENABLE_ALL_TESTS is not set.

lib/isc/netmgr/http.c

@@ -55,6 +55,9 @@
 	(((namelen) == sizeof(header) - 1) && \
 	 (strncasecmp((header), (const char *)(name), (namelen)) == 0))
 
+#define MIN_SUCCESSFUL_HTTP_STATUS (200)
+#define MAX_SUCCESSFUL_HTTP_STATUS (299)
+
 typedef struct isc_nm_http_response_status {
 	size_t code;
 	size_t content_length;
@@ -507,6 +510,19 @@ on_data_chunk_recv_callback(nghttp2_session *ngsession, uint8_t flags,
 	return (rv);
 }
 
+static void
+call_unlink_cstream_readcb(http_cstream_t *cstream,
+			   isc_nm_http_session_t *session,
+			   isc_result_t result) {
+	REQUIRE(VALID_HTTP2_SESSION(session));
+	REQUIRE(cstream != NULL);
+	cstream->read_cb(session->handle, result,
+			 &(isc_region_t){ cstream->rbuf, cstream->rbufsize },
+			 cstream->read_cbarg);
+	ISC_LIST_UNLINK(session->cstreams, cstream, link);
+	put_http_cstream(session->mctx, cstream);
+}
+
 static int
 on_client_stream_close_callback(int32_t stream_id,
 				isc_nm_http_session_t *session) {
@@ -514,16 +530,13 @@ on_client_stream_close_callback(int32_t stream_id,
 
 	if (cstream != NULL) {
 		isc_result_t result =
-			cstream->response_status.code >= 200 &&
-					cstream->response_status.code < 300
+			cstream->response_status.code >=
+						MIN_SUCCESSFUL_HTTP_STATUS &&
+					cstream->response_status.code <=
+						MAX_SUCCESSFUL_HTTP_STATUS
 				? ISC_R_SUCCESS
 				: ISC_R_FAILURE;
-		cstream->read_cb(
-			session->handle, result,
-			&(isc_region_t){ cstream->rbuf, cstream->rbufsize },
-			cstream->read_cbarg);
-		ISC_LIST_UNLINK(session->cstreams, cstream, link);
-		put_http_cstream(session->mctx, cstream);
+		call_unlink_cstream_readcb(cstream, session, result);
 		if (ISC_LIST_EMPTY(session->cstreams)) {
 			int rv = 0;
 			rv = nghttp2_session_terminate_session(
@@ -576,7 +589,7 @@ on_stream_close_callback(nghttp2_session *ngsession, int32_t stream_id,
 	return (rv);
 }
 
-static void
+static bool
 client_handle_status_header(http_cstream_t *cstream, const uint8_t *value,
 			    const size_t valuelen) {
 	char tmp[32] = { 0 };
@@ -584,9 +597,17 @@ client_handle_status_header(http_cstream_t *cstream, const uint8_t *value,
 
 	strncpy(tmp, (const char *)value, ISC_MIN(tmplen, valuelen));
 	cstream->response_status.code = strtoul(tmp, NULL, 10);
+
+	if (cstream->response_status.code >= MIN_SUCCESSFUL_HTTP_STATUS &&
+	    cstream->response_status.code <= MAX_SUCCESSFUL_HTTP_STATUS)
+	{
+		return (true);
+	}
+
+	return (false);
 }
 
-static void
+static bool
 client_handle_content_length_header(http_cstream_t *cstream,
 				    const uint8_t *value,
 				    const size_t valuelen) {
@@ -595,9 +616,17 @@ client_handle_content_length_header(http_cstream_t *cstream,
 
 	strncpy(tmp, (const char *)value, ISC_MIN(tmplen, valuelen));
 	cstream->response_status.content_length = strtoul(tmp, NULL, 10);
+
+	if (cstream->response_status.content_length == 0 ||
+	    cstream->response_status.content_length > MAX_DNS_MESSAGE_SIZE)
+	{
+		return (false);
+	}
+
+	return (true);
 }
 
-static void
+static bool
 client_handle_content_type_header(http_cstream_t *cstream, const uint8_t *value,
 				  const size_t valuelen) {
 	const char type_dns_message[] = DNS_MEDIA_TYPE;
@@ -607,7 +636,10 @@ client_handle_content_type_header(http_cstream_t *cstream, const uint8_t *value,
 
 	if (strncasecmp((const char *)value, type_dns_message, len) == 0) {
 		cstream->response_status.content_type_valid = true;
+		return (true);
 	}
+
+	return (false);
 }
 
 static int
@@ -620,6 +652,7 @@ client_on_header_callback(nghttp2_session *ngsession,
 	const char status[] = ":status";
 	const char content_length[] = "Content-Length";
 	const char content_type[] = "Content-Type";
+	bool header_ok = true;
 
 	REQUIRE(VALID_HTTP2_SESSION(session));
 	REQUIRE(session->client);
@@ -637,20 +670,22 @@ client_on_header_callback(nghttp2_session *ngsession,
 		}
 
 		if (HEADER_MATCH(status, name, namelen)) {
-			client_handle_status_header(cstream, value, valuelen);
+			header_ok = client_handle_status_header(cstream, value,
+								valuelen);
 		} else if (HEADER_MATCH(content_length, name, namelen)) {
-			client_handle_content_length_header(cstream, value,
-							    valuelen);
+			header_ok = client_handle_content_length_header(
+				cstream, value, valuelen);
 		} else if (HEADER_MATCH(content_type, name, namelen)) {
-			client_handle_content_type_header(cstream, value,
-							  valuelen);
-			if (!cstream->response_status.content_type_valid) {
-				return (NGHTTP2_ERR_HTTP_HEADER);
-			}
+			header_ok = client_handle_content_type_header(
+				cstream, value, valuelen);
 		}
 		break;
 	}
 
+	if (!header_ok) {
+		return (NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE);
+	}
+
 	return (0);
 }
 

lib/isc/netmgr/netmgr-int.h

@@ -816,23 +816,22 @@ struct isc_nmsocket {
 	/*% TLS stuff */
 	struct tlsstream {
 		bool server;
-		BIO *app_bio;
+		BIO *bio_in;
+		BIO *bio_out;
 		isc_tls_t *tls;
 		isc_tlsctx_t *ctx;
-		BIO *ssl_bio;
 		isc_nmsocket_t *tlslistener;
 		isc_nmiface_t server_iface;
 		isc_nmiface_t local_iface;
-		bool connect_from_networker;
 		atomic_bool result_updated;
 		enum {
 			TLS_INIT,
 			TLS_HANDSHAKE,
 			TLS_IO,
-			TLS_CLOSING,
 			TLS_CLOSED
 		} state; /*%< The order of these is significant */
 		size_t nsending;
+		bool reading;
 	} tlsstream;
 
 	isc_nmsocket_h2_t h2;

lib/isc/netmgr/tlsstream.c

@@ -34,10 +34,11 @@
 #include <isc/thread.h>
 #include <isc/util.h>
 
+#include "../openssl_shim.h"
 #include "netmgr-int.h"
 #include "uv-compat.h"
 
-#define TLS_BUF_SIZE 65536
+#define TLS_BUF_SIZE (UINT16_MAX)
 
 static isc_result_t
 tls_error_to_result(int tls_err) {
@@ -52,7 +53,15 @@ tls_error_to_result(int tls_err) {
 }
 
 static void
-tls_do_bio(isc_nmsocket_t *sock, isc__nm_uvreq_t *send_data, bool finish);
+tls_failed_read_cb(isc_nmsocket_t *sock, const isc_result_t result);
+
+static void
+tls_do_bio(isc_nmsocket_t *sock, isc_region_t *received_data,
+	   isc__nm_uvreq_t *send_data, bool finish);
+
+static void
+tls_readcb(isc_nmhandle_t *handle, isc_result_t result, isc_region_t *region,
+	   void *cbarg);
 
 static void
 tls_close_direct(isc_nmsocket_t *sock);
@@ -103,7 +112,6 @@ tls_call_connect_cb(isc_nmsocket_t *sock, isc_nmhandle_t *handle,
 		return;
 	}
 	sock->connect_cb(handle, result, sock->connect_cbarg);
-	update_result(sock, result);
 	if (result != ISC_R_SUCCESS) {
 		isc__nmsocket_clearcb(handle->sock);
 	}
@@ -134,17 +142,31 @@ tls_senddone(isc_nmhandle_t *handle, isc_result_t eresult, void *cbarg) {
 	tlssock->tlsstream.nsending--;
 
 	if (finish && eresult == ISC_R_SUCCESS) {
+		tlssock->tlsstream.reading = false;
 		isc_nm_cancelread(handle);
 	} else if (eresult == ISC_R_SUCCESS) {
-		tls_do_bio(tlssock, NULL, false);
+		tls_do_bio(tlssock, NULL, NULL, false);
+	} else if (eresult != ISC_R_SUCCESS &&
+		   tlssock->tlsstream.state <= TLS_HANDSHAKE &&
+		   !tlssock->tlsstream.server)
+	{
+		/*
+		 * We are still waiting for the handshake to complete, but
+		 * it isn't going to happen. Call the connect callback,
+		 * passing the error code there.
+		 *
+		 * (Note: tls_failed_read_cb() calls the connect
+		 * rather than the read callback in this case.
+		 * XXX: clarify?)
+		 */
+		tls_failed_read_cb(tlssock, eresult);
 	}
 
 	isc__nmsocket_detach(&tlssock);
 }
 
 static void
-tls_failed_read_cb(isc_nmsocket_t *sock, isc_nmhandle_t *handle,
-		   const isc_result_t result) {
+tls_failed_read_cb(isc_nmsocket_t *sock, const isc_result_t result) {
 	REQUIRE(VALID_NMSOCK(sock));
 
 	if (!sock->tlsstream.server &&
@@ -152,22 +174,21 @@ tls_failed_read_cb(isc_nmsocket_t *sock, isc_nmhandle_t *handle,
 	     sock->tlsstream.state == TLS_HANDSHAKE) &&
 	    sock->connect_cb != NULL)
 	{
-		INSIST(handle == NULL);
-		handle = isc__nmhandle_get(sock, NULL, NULL);
+		isc_nmhandle_t *handle = NULL;
+		INSIST(sock->statichandle == NULL);
+		handle = isc__nmhandle_get(sock, &sock->peer,
+					   &sock->iface->addr);
 		tls_call_connect_cb(sock, handle, result);
+		isc__nmsocket_clearcb(sock);
 		isc_nmhandle_detach(&handle);
-	} else if (sock->recv_cb != NULL) {
+	} else if (sock->recv_cb != NULL && sock->statichandle != NULL) {
 		isc__nm_uvreq_t *req = NULL;
+		INSIST(VALID_NMHANDLE(sock->statichandle));
 		req = isc__nm_uvreq_get(sock->mgr, sock);
 		req->cb.recv = sock->recv_cb;
 		req->cbarg = sock->recv_cbarg;
 		req->handle = NULL;
-		if (handle != NULL) {
-			REQUIRE(VALID_NMHANDLE(handle));
-			isc_nmhandle_attach(handle, &req->handle);
-		} else {
-			req->handle = isc__nmhandle_get(sock, NULL, NULL);
-		}
+		isc_nmhandle_attach(sock->statichandle, &req->handle);
 		isc__nmsocket_clearcb(sock);
 		isc__nm_readcb(sock, req, result);
 	}
@@ -190,6 +211,7 @@ tls_send_outgoing(isc_nmsocket_t *sock, bool finish, isc_nmhandle_t *tlshandle,
 	isc_nmsocket_tls_send_req_t *send_req = NULL;
 	int pending;
 	int rv;
+	size_t len = 0;
 
 	if (inactive(sock)) {
 		if (cb != NULL) {
@@ -205,7 +227,7 @@ tls_send_outgoing(isc_nmsocket_t *sock, bool finish, isc_nmhandle_t *tlshandle,
 		(void)SSL_shutdown(sock->tlsstream.tls);
 	}
 
-	pending = BIO_pending(sock->tlsstream.app_bio);
+	pending = BIO_pending(sock->tlsstream.bio_out);
 	if (pending <= 0) {
 		return (pending);
 	}
@@ -229,9 +251,10 @@ tls_send_outgoing(isc_nmsocket_t *sock, bool finish, isc_nmhandle_t *tlshandle,
 		isc_nmhandle_attach(tlshandle, &send_req->handle);
 	}
 
-	rv = BIO_read(sock->tlsstream.app_bio, send_req->data.base, pending);
+	rv = BIO_read_ex(sock->tlsstream.bio_out, send_req->data.base, pending,
+			 &len);
 	/* There's something pending, read must succeed */
-	RUNTIME_CHECK(rv == pending);
+	RUNTIME_CHECK(rv == 1);
 
 	INSIST(VALID_NMHANDLE(sock->outerhandle));
 
@@ -241,81 +264,10 @@ tls_send_outgoing(isc_nmsocket_t *sock, bool finish, isc_nmhandle_t *tlshandle,
 	return (pending);
 }
 
-static void
-tls_do_bio(isc_nmsocket_t *sock, isc__nm_uvreq_t *send_data, bool finish) {
-	isc_result_t result = ISC_R_SUCCESS;
-	int pending, tls_err = 0;
-	int rv;
-	char buf[1];
-	bool sent_shutdown, received_shutdown;
-
-	REQUIRE(VALID_NMSOCK(sock));
-	REQUIRE(sock->tid == isc_nm_tid());
-
-	/* We will resume read if TLS layer wants us to */
-	if (sock->outerhandle != NULL) {
-		REQUIRE(VALID_NMHANDLE(sock->outerhandle));
-		isc_nm_pauseread(sock->outerhandle);
-	}
-
-	if (sock->tlsstream.state == TLS_INIT) {
-		(void)SSL_do_handshake(sock->tlsstream.tls);
-		sock->tlsstream.state = TLS_HANDSHAKE;
-	} else if (sock->tlsstream.state == TLS_CLOSED) {
-		return;
-	}
-
-	received_shutdown = (SSL_get_shutdown(sock->tlsstream.tls) &
-			     SSL_RECEIVED_SHUTDOWN) == SSL_RECEIVED_SHUTDOWN;
-
-	/* Data from TLS to client */
-	if (sock->tlsstream.state >= TLS_IO && sock->recv_cb != NULL &&
-	    !atomic_load(&sock->readpaused))
-	{
-		(void)SSL_peek(sock->tlsstream.tls, buf, 1);
-		while ((pending = SSL_pending(sock->tlsstream.tls)) > 0) {
-			uint8_t recv_buf[TLS_BUF_SIZE];
-			isc_region_t region, dregion;
-
-			if (pending > TLS_BUF_SIZE) {
-				pending = TLS_BUF_SIZE;
-			}
-			region = (isc_region_t){ .base = &recv_buf[0],
-						 .length = pending };
-
-			rv = SSL_read(sock->tlsstream.tls, region.base,
-				      region.length);
-			/* Pending succeded, so should read */
-			RUNTIME_CHECK(rv == pending);
-
-			dregion = (isc_region_t){ region.base, rv };
-			sock->recv_cb(sock->statichandle, ISC_R_SUCCESS,
-				      &dregion, sock->recv_cbarg);
-		}
-	}
-
-	if (send_data != NULL) {
-		INSIST(sock->tlsstream.state > TLS_HANDSHAKE);
-		rv = SSL_write(sock->tlsstream.tls, send_data->uvbuf.base,
-			       send_data->uvbuf.len);
-		if (rv != (int)send_data->uvbuf.len) {
-			result = received_shutdown ? ISC_R_CANCELED
-						   : ISC_R_TLSERROR;
-			send_data->cb.send(send_data->handle, result,
-					   send_data->cbarg);
-			send_data = NULL;
-			if (!received_shutdown) {
-				isc__nmsocket_detach(&sock);
-				return;
-			}
-		}
-	}
-
-	sent_shutdown = (SSL_get_shutdown(sock->tlsstream.tls) &
-			 SSL_SENT_SHUTDOWN) == SSL_SENT_SHUTDOWN;
-
-	/* Peek to move the session forward */
-	(void)SSL_peek(sock->tlsstream.tls, buf, 1);
+static int
+tls_process_outgoing(isc_nmsocket_t *sock, bool finish,
+		     isc__nm_uvreq_t *send_data) {
+	int pending;
 
 	/* Data from TLS to network */
 	if (send_data != NULL) {
@@ -323,6 +275,12 @@ tls_do_bio(isc_nmsocket_t *sock, isc__nm_uvreq_t *send_data, bool finish) {
 					    send_data->cb.send,
 					    send_data->cbarg);
 	} else {
+		bool received_shutdown =
+			((SSL_get_shutdown(sock->tlsstream.tls) &
+			  SSL_RECEIVED_SHUTDOWN) != 0);
+		bool sent_shutdown = ((SSL_get_shutdown(sock->tlsstream.tls) &
+				       SSL_SENT_SHUTDOWN) != 0);
+
 		if (received_shutdown && !sent_shutdown) {
 			finish = true;
 			(void)SSL_shutdown(sock->tlsstream.tls);
@@ -330,44 +288,144 @@ tls_do_bio(isc_nmsocket_t *sock, isc__nm_uvreq_t *send_data, bool finish) {
 		pending = tls_send_outgoing(sock, finish, NULL, NULL, NULL);
 	}
 
-	if (pending > 0) {
-		/* We'll continue in tls_senddone */
-		return;
+	return (pending);
+}
+
+static int
+tls_try_handshake(isc_nmsocket_t *sock) {
+	int rv = 0;
+	isc_nmhandle_t *tlshandle = NULL;
+
+	REQUIRE(sock->tlsstream.state == TLS_HANDSHAKE);
+
+	if (SSL_is_init_finished(sock->tlsstream.tls) == 1) {
+		return (0);
 	}
 
-	/* Get the potential error code */
-	rv = SSL_peek(sock->tlsstream.tls, buf, 1);
-	if (rv < 0) {
-		tls_err = SSL_get_error(sock->tlsstream.tls, rv);
-	}
-
-	/* Only after doing the IO we can check if SSL handshake is done */
-	if (sock->tlsstream.state == TLS_HANDSHAKE &&
-	    SSL_is_init_finished(sock->tlsstream.tls) == 1)
-	{
-		isc_nmhandle_t *tlshandle = isc__nmhandle_get(sock, NULL, NULL);
+	rv = SSL_do_handshake(sock->tlsstream.tls);
+	if (rv == 1) {
+		INSIST(SSL_is_init_finished(sock->tlsstream.tls) == 1);
+		INSIST(sock->statichandle == NULL);
+		tlshandle = isc__nmhandle_get(sock, &sock->peer,
+					      &sock->iface->addr);
 		if (sock->tlsstream.server) {
-			sock->listener->accept_cb(sock->statichandle,
-						  ISC_R_SUCCESS,
+			sock->listener->accept_cb(tlshandle, ISC_R_SUCCESS,
 						  sock->listener->accept_cbarg);
 		} else {
 			tls_call_connect_cb(sock, tlshandle, ISC_R_SUCCESS);
 		}
 		isc_nmhandle_detach(&tlshandle);
 		sock->tlsstream.state = TLS_IO;
-		async_tls_do_bio(sock);
+	}
+
+	return (rv);
+}
+
+static void
+tls_do_bio(isc_nmsocket_t *sock, isc_region_t *received_data,
+	   isc__nm_uvreq_t *send_data, bool finish) {
+	isc_result_t result = ISC_R_SUCCESS;
+	int pending, tls_status = SSL_ERROR_NONE;
+	int rv = 0;
+	bool sent_shutdown = false, received_shutdown = false;
+	size_t len = 0;
+
+	REQUIRE(VALID_NMSOCK(sock));
+	REQUIRE(sock->tid == isc_nm_tid());
+
+	/* We will resume read if TLS layer wants us to */
+	if (sock->tlsstream.reading && sock->outerhandle) {
+		REQUIRE(VALID_NMHANDLE(sock->outerhandle));
+		isc_nm_pauseread(sock->outerhandle);
+	}
+
+	if (sock->tlsstream.state == TLS_INIT) {
+		INSIST(received_data == NULL && send_data == NULL);
+		if (sock->tlsstream.server) {
+			SSL_set_accept_state(sock->tlsstream.tls);
+		} else {
+			SSL_set_connect_state(sock->tlsstream.tls);
+		}
+		sock->tlsstream.state = TLS_HANDSHAKE;
+		rv = tls_try_handshake(sock);
+		INSIST(SSL_is_init_finished(sock->tlsstream.tls) == 0);
+	} else if (sock->tlsstream.state == TLS_CLOSED) {
+		return;
+	} else { /* initialised and doing I/O */
+		if (received_data != NULL) {
+			INSIST(send_data == NULL);
+			rv = BIO_write_ex(sock->tlsstream.bio_in,
+					  received_data->base,
+					  received_data->length, &len);
+			if (rv <= 0 || len != received_data->length) {
+				result = ISC_R_TLSERROR;
+				goto error;
+			}
+
+			/*
+			 * Only after doing the IO we can check whether SSL
+			 * handshake is done.
+			 */
+			if (sock->tlsstream.state == TLS_HANDSHAKE) {
+				rv = tls_try_handshake(sock);
+			}
+		} else if (send_data != NULL) {
+			INSIST(received_data == NULL);
+			INSIST(sock->tlsstream.state > TLS_HANDSHAKE);
+			received_shutdown =
+				((SSL_get_shutdown(sock->tlsstream.tls) &
+				  SSL_RECEIVED_SHUTDOWN) != 0);
+			rv = SSL_write_ex(sock->tlsstream.tls,
+					  send_data->uvbuf.base,
+					  send_data->uvbuf.len, &len);
+			if (rv != 1 || len != send_data->uvbuf.len) {
+				result = received_shutdown ? ISC_R_CANCELED
+							   : ISC_R_TLSERROR;
+				send_data->cb.send(send_data->handle, result,
+						   send_data->cbarg);
+				send_data = NULL;
+				if (!received_shutdown) {
+					isc__nmsocket_detach(&sock);
+					return;
+				}
+			}
+		}
+
+		/* Decrypt and pass data from network to client */
+		if (sock->tlsstream.state >= TLS_IO && sock->recv_cb != NULL &&
+		    !atomic_load(&sock->readpaused))
+		{
+			uint8_t recv_buf[TLS_BUF_SIZE];
+			INSIST(sock->tlsstream.state > TLS_HANDSHAKE);
+			while ((rv = SSL_read_ex(sock->tlsstream.tls, recv_buf,
+						 TLS_BUF_SIZE, &len)) == 1)
+			{
+				isc_region_t region;
+				region = (isc_region_t){ .base = &recv_buf[0],
+							 .length = len };
+
+				sock->recv_cb(sock->statichandle, ISC_R_SUCCESS,
+					      &region, sock->recv_cbarg);
+			}
+		}
+	}
+	tls_status = SSL_get_error(sock->tlsstream.tls, rv);
+
+	pending = tls_process_outgoing(sock, finish, send_data);
+	if (pending > 0) {
+		/* We'll continue in tls_senddone */
 		return;
 	}
 
-	switch (tls_err) {
+	switch (tls_status) {
 	case SSL_ERROR_NONE:
+	case SSL_ERROR_ZERO_RETURN:
 		if (sent_shutdown && received_shutdown) {
 			/* clean shutdown */
 			isc_nm_cancelread(sock->outerhandle);
 			isc__nm_tls_close(sock);
 		};
 		return;
-		break;
 	case SSL_ERROR_WANT_WRITE:
 		if (sock->tlsstream.nsending == 0) {
 			/*
@@ -375,79 +433,77 @@ tls_do_bio(isc_nmsocket_t *sock, isc__nm_uvreq_t *send_data, bool finish) {
 			 * already the send callback will call it.
 			 */
 			async_tls_do_bio(sock);
-			return;
-		} else {
-			return;
-		}
-		break;
-	case SSL_ERROR_WANT_READ:
-		if (sock->outerhandle != NULL) {
-			INSIST(VALID_NMHANDLE(sock->outerhandle));
-			isc_nm_resumeread(sock->outerhandle);
 		}
 		return;
-		break;
+	case SSL_ERROR_WANT_READ:
+		if (sock->tlsstream.reading) {
+			INSIST(VALID_NMHANDLE(sock->outerhandle));
+			isc_nm_resumeread(sock->outerhandle);
+		} else if (sock->tlsstream.state == TLS_HANDSHAKE) {
+			sock->tlsstream.reading = true;
+			isc_nm_read(sock->outerhandle, tls_readcb, sock);
+		}
+		return;
 	default:
-		result = tls_error_to_result(tls_err);
-		goto error;
+		result = tls_error_to_result(tls_status);
+		break;
 	}
 
-	return;
-
 error:
 	isc_log_write(isc_lctx, ISC_LOGCATEGORY_GENERAL, ISC_LOGMODULE_NETMGR,
-		      ISC_LOG_ERROR, "SSL error in BIO: %d %s", tls_err,
+		      ISC_LOG_ERROR, "SSL error in BIO: %d %s", tls_status,
 		      isc_result_totext(result));
-	tls_failed_read_cb(sock, sock->statichandle, result);
+	tls_failed_read_cb(sock, result);
 }
 
 static void
 tls_readcb(isc_nmhandle_t *handle, isc_result_t result, isc_region_t *region,
 	   void *cbarg) {
 	isc_nmsocket_t *tlssock = (isc_nmsocket_t *)cbarg;
-	int rv;
 
 	REQUIRE(VALID_NMSOCK(tlssock));
 	REQUIRE(VALID_NMHANDLE(handle));
 	REQUIRE(tlssock->tid == isc_nm_tid());
 
 	if (result != ISC_R_SUCCESS) {
-		tls_failed_read_cb(tlssock, tlssock->statichandle, result);
+		tls_failed_read_cb(tlssock, result);
 		return;
 	}
-	rv = BIO_write(tlssock->tlsstream.app_bio, region->base,
-		       region->length);
-	if (rv != (int)region->length) {
-		/* XXXWPK log it? */
-		tls_failed_read_cb(tlssock, tlssock->statichandle,
-				   ISC_R_TLSERROR);
-		return;
-	}
-	tls_do_bio(tlssock, NULL, false);
+	tls_do_bio(tlssock, region, NULL, false);
 }
 
 static isc_result_t
 initialize_tls(isc_nmsocket_t *sock, bool server) {
 	REQUIRE(sock->tid == isc_nm_tid());
 
-	if (BIO_new_bio_pair(&(sock->tlsstream.ssl_bio), TLS_BUF_SIZE,
-			     &(sock->tlsstream.app_bio), TLS_BUF_SIZE) != 1)
-	{
+	sock->tlsstream.bio_in = BIO_new(BIO_s_mem());
+	if (sock->tlsstream.bio_in == NULL) {
+		isc_tls_free(&sock->tlsstream.tls);
+		return (ISC_R_TLSERROR);
+	}
+	sock->tlsstream.bio_out = BIO_new(BIO_s_mem());
+	if (sock->tlsstream.bio_out == NULL) {
+		BIO_free_all(sock->tlsstream.bio_in);
+		sock->tlsstream.bio_in = NULL;
 		isc_tls_free(&sock->tlsstream.tls);
 		return (ISC_R_TLSERROR);
 	}
 
-	SSL_set_bio(sock->tlsstream.tls, sock->tlsstream.ssl_bio,
-		    sock->tlsstream.ssl_bio);
-	if (server) {
-		SSL_set_accept_state(sock->tlsstream.tls);
-	} else {
-		SSL_set_connect_state(sock->tlsstream.tls);
+	if (BIO_set_mem_eof_return(sock->tlsstream.bio_in, EOF) != 1 ||
+	    BIO_set_mem_eof_return(sock->tlsstream.bio_out, EOF) != 1)
+	{
+		goto error;
 	}
+
+	SSL_set_bio(sock->tlsstream.tls, sock->tlsstream.bio_in,
+		    sock->tlsstream.bio_out);
+	sock->tlsstream.server = server;
 	sock->tlsstream.nsending = 0;
-	isc_nm_read(sock->outerhandle, tls_readcb, sock);
-	tls_do_bio(sock, NULL, false);
 	return (ISC_R_SUCCESS);
+error:
+	isc_tls_free(&sock->tlsstream.tls);
+	sock->tlsstream.bio_out = sock->tlsstream.bio_in = NULL;
+	return (ISC_R_TLSERROR);
 }
 
 static isc_result_t
@@ -487,7 +543,6 @@ tlslisten_acceptcb(isc_nmhandle_t *handle, isc_result_t result, void *cbarg) {
 	tlssock->peer = handle->sock->peer;
 	tlssock->read_timeout = atomic_load(&handle->sock->mgr->init);
 	tlssock->tid = isc_nm_tid();
-	tlssock->tlsstream.server = true;
 	tlssock->tlsstream.state = TLS_INIT;
 
 	tlssock->tlsstream.ctx = tlslistensock->tlsstream.ctx;
@@ -496,6 +551,7 @@ tlslisten_acceptcb(isc_nmhandle_t *handle, isc_result_t result, void *cbarg) {
 	RUNTIME_CHECK(result == ISC_R_SUCCESS);
 	/* TODO: catch failure code, detach tlssock, and log the error */
 
+	tls_do_bio(tlssock, NULL, NULL, false);
 	return (result);
 }
 
@@ -573,11 +629,12 @@ isc__nm_async_tlssend(isc__networker_t *worker, isc__netievent_t *ev0) {
 
 	if (inactive(sock)) {
 		req->cb.send(req->handle, ISC_R_CANCELED, req->cbarg);
-		isc__nm_uvreq_put(&req, sock);
-		return;
+		goto done;
 	}
 
-	tls_do_bio(sock, req, false);
+	tls_do_bio(sock, NULL, req, false);
+done:
+	isc_mem_free(sock->mgr->mctx, req->uvbuf.base);
 	isc__nm_uvreq_put(&req, sock);
 	return;
 }
@@ -606,7 +663,8 @@ isc__nm_tls_send(isc_nmhandle_t *handle, const isc_region_t *region,
 	uvreq->cb.send = cb;
 	uvreq->cbarg = cbarg;
 
-	uvreq->uvbuf.base = (char *)region->base;
+	uvreq->uvbuf.base = isc_mem_allocate(sock->mgr->mctx, region->length);
+	memmove(uvreq->uvbuf.base, region->base, region->length);
 	uvreq->uvbuf.len = region->length;
 
 	/*
@@ -627,7 +685,7 @@ isc__nm_async_tlsstartread(isc__networker_t *worker, isc__netievent_t *ev0) {
 
 	UNUSED(worker);
 
-	tls_do_bio(sock, NULL, false);
+	tls_do_bio(sock, NULL, NULL, false);
 }
 
 void
@@ -638,10 +696,10 @@ isc__nm_tls_read(isc_nmhandle_t *handle, isc_nm_recv_cb_t cb, void *cbarg) {
 	REQUIRE(VALID_NMHANDLE(handle));
 
 	sock = handle->sock;
-
 	REQUIRE(VALID_NMSOCK(sock));
 	REQUIRE(sock->statichandle == handle);
 	REQUIRE(sock->tid == isc_nm_tid());
+	REQUIRE(sock->recv_cb == NULL);
 
 	if (inactive(sock)) {
 		cb(handle, ISC_R_NOTCONNECTED, NULL, cbarg);
@@ -661,9 +719,12 @@ isc__nm_tls_pauseread(isc_nmhandle_t *handle) {
 	REQUIRE(VALID_NMHANDLE(handle));
 	REQUIRE(VALID_NMSOCK(handle->sock));
 
-	atomic_store(&handle->sock->readpaused, true);
-	if (handle->sock->outerhandle != NULL) {
-		isc_nm_pauseread(handle->sock->outerhandle);
+	if (atomic_compare_exchange_strong(&handle->sock->readpaused,
+					   &(bool){ false }, true))
+	{
+		if (handle->sock->outerhandle != NULL) {
+			isc_nm_pauseread(handle->sock->outerhandle);
+		}
 	}
 }
 
@@ -672,8 +733,11 @@ isc__nm_tls_resumeread(isc_nmhandle_t *handle) {
 	REQUIRE(VALID_NMHANDLE(handle));
 	REQUIRE(VALID_NMSOCK(handle->sock));
 
-	atomic_store(&handle->sock->readpaused, false);
-	async_tls_do_bio(handle->sock);
+	if (!atomic_compare_exchange_strong(&handle->sock->readpaused,
+					    &(bool){ false }, false))
+	{
+		async_tls_do_bio(handle->sock);
+	}
 }
 
 static void
@@ -686,6 +750,7 @@ tls_close_direct(isc_nmsocket_t *sock) {
 	 */
 	if (sock->outerhandle != NULL) {
 		isc_nm_pauseread(sock->outerhandle);
+		isc__nmsocket_clearcb(sock->outerhandle->sock);
 		isc_nmhandle_detach(&sock->outerhandle);
 	}
 
@@ -694,7 +759,6 @@ tls_close_direct(isc_nmsocket_t *sock) {
 	}
 
 	/* further cleanup performed in isc__nm_tls_cleanup_data() */
-	atomic_store(&sock->active, false);
 	atomic_store(&sock->closed, true);
 	sock->tlsstream.state = TLS_CLOSED;
 }
@@ -773,7 +837,6 @@ isc_nm_tlsconnect(isc_nm_t *mgr, isc_nmiface_t *local, isc_nmiface_t *peer,
 	nsock->connect_cbarg = cbarg;
 	nsock->connect_timeout = timeout;
 	nsock->tlsstream.ctx = ctx;
-	nsock->tlsstream.connect_from_networker = isc__nm_in_netthread();
 
 	ievent = isc__nm_get_netievent_tlsconnect(mgr, nsock);
 	ievent->local = local->addr;
@@ -782,6 +845,7 @@ isc_nm_tlsconnect(isc_nm_t *mgr, isc_nmiface_t *local, isc_nmiface_t *peer,
 
 	isc__nmsocket_attach(nsock, &tsock);
 	if (isc__nm_in_netthread()) {
+		atomic_store(&nsock->active, true);
 		nsock->tid = isc_nm_tid();
 		isc__nm_async_tlsconnect(&mgr->workers[nsock->tid],
 					 (isc__netievent_t *)ievent);
@@ -821,13 +885,15 @@ tcp_connected(isc_nmhandle_t *handle, isc_result_t result, void *cbarg) {
 		goto error;
 	}
 
-	tlssock->peer = isc_nmhandle_peeraddr(handle);
-	isc_nmhandle_attach(handle, &tlssock->outerhandle);
 	result = initialize_tls(tlssock, false);
 	if (result != ISC_R_SUCCESS) {
 		goto error;
 	}
 
+	tlssock->peer = isc_nmhandle_peeraddr(handle);
+	isc_nmhandle_attach(handle, &tlssock->outerhandle);
+
+	tls_do_bio(tlssock, NULL, NULL, false);
 	return;
 error:
 	tlshandle = isc__nmhandle_get(tlssock, NULL, NULL);
@@ -863,15 +929,23 @@ isc__nm_async_tlsconnect(isc__networker_t *worker, isc__netievent_t *ev0) {
 				   (isc_nmiface_t *)&ievent->peer,
 				   tcp_connected, tlssock,
 				   tlssock->connect_timeout, 0);
-	if (tlssock->tlsstream.connect_from_networker) {
-		update_result(tlssock, result);
+
+	/*
+	 * Sometimes on Linux, socket creation might fail if there are
+	 * already too many socket descriptors. In such a case the
+	 * connect callback is not going to be called.
+	 */
+	if (result != ISC_R_SUCCESS) {
+		goto error;
 	}
+	update_result(tlssock, result);
 	return;
 
 error:
 	tlshandle = isc__nmhandle_get(tlssock, NULL, NULL);
 	atomic_store(&tlssock->closed, true);
 	tls_call_connect_cb(tlssock, tlshandle, result);
+	update_result(tlssock, result);
 	isc_nmhandle_detach(&tlshandle);
 	isc__nmsocket_detach(&tlssock);
 }
@@ -879,8 +953,9 @@ error:
 static void
 tls_cancelread(isc_nmsocket_t *sock) {
 	if (!inactive(sock) && sock->tlsstream.state == TLS_IO) {
-		tls_do_bio(sock, NULL, true);
+		tls_do_bio(sock, NULL, NULL, true);
 	} else if (sock->outerhandle != NULL) {
+		sock->tlsstream.reading = false;
 		isc_nm_cancelread(sock->outerhandle);
 	}
 }
@@ -925,7 +1000,7 @@ isc__nm_async_tlsdobio(isc__networker_t *worker, isc__netievent_t *ev0) {
 
 	UNUSED(worker);
 
-	tls_do_bio(ievent->sock, NULL, false);
+	tls_do_bio(ievent->sock, NULL, NULL, false);
 }
 
 void
@@ -939,11 +1014,8 @@ isc__nm_tls_cleanup_data(isc_nmsocket_t *sock) {
 			isc_tls_free(&sock->tlsstream.tls);
 			/* These are destroyed when we free SSL */
 			sock->tlsstream.ctx = NULL;
-			sock->tlsstream.ssl_bio = NULL;
-		}
-		if (sock->tlsstream.app_bio != NULL) {
-			BIO_free(sock->tlsstream.app_bio);
-			sock->tlsstream.app_bio = NULL;
+			sock->tlsstream.bio_out = NULL;
+			sock->tlsstream.bio_in = NULL;
 		}
 	}
 }

lib/isc/tests/doh_test.c

@@ -65,7 +65,7 @@ static atomic_uint_fast64_t creads;
 
 static atomic_bool was_error;
 
-static unsigned int workers = 1;
+static unsigned int workers = 0;
 
 static bool reuse_supported = true;
 
@@ -227,9 +227,18 @@ setup_ephemeral_port(isc_sockaddr_t *addr, sa_family_t family) {
 
 static int
 _setup(void **state) {
+	char *p = NULL;
+
 	UNUSED(state);
 
-	workers = isc_os_ncpus();
+	if (workers == 0) {
+		workers = isc_os_ncpus();
+	}
+	p = getenv("ISC_TASK_WORKERS");
+	if (p != NULL) {
+		workers = atoi(p);
+	}
+	INSIST(workers != 0);
 
 	if (isc_test_begin(NULL, false, workers) != ISC_R_SUCCESS) {
 		return (-1);
@@ -587,17 +596,25 @@ static isc_threadresult_t
 doh_connect_thread(isc_threadarg_t arg) {
 	isc_nm_t *connect_nm = (isc_nm_t *)arg;
 	char req_url[256];
+	isc_result_t result;
 
 	sockaddr_to_url(&tcp_listen_addr, atomic_load(&use_TLS), req_url,
 			sizeof(req_url), DOH_PATH);
 
 	while (atomic_load(&nsends) > 0) {
-		(void)connect_send_request(
+		result = connect_send_request(
 			connect_nm, req_url, atomic_load(&POST),
 			&(isc_region_t){ .base = (uint8_t *)send_msg.base,
 					 .length = send_msg.len },
 			doh_receive_send_reply_cb, NULL, atomic_load(&use_TLS),
 			30000);
+		/* protection against "too many open files" */
+#ifndef _WIN32
+		if (result != ISC_R_SUCCESS) {
+			INSIST(result == ISC_R_TOOMANYOPENFILES);
+			usleep(1000 * workers);
+		}
+#endif
 	}
 
 	return ((isc_threadresult_t)0);

lib/isc/tests/isctest.c

@@ -80,14 +80,13 @@ create_managers(unsigned int workers) {
 	if (workers == 0) {
 		workers = isc_os_ncpus();
 	}
-
 	p = getenv("ISC_TASK_WORKERS");
 	if (p != NULL) {
 		workers = atoi(p);
 	}
 	INSIST(workers != 0);
 
-	isc_hp_init(4 * workers);
+	isc_hp_init(6 * workers);
 
 	netmgr = isc_nm_start(test_mctx, workers);
 	CHECK(isc_taskmgr_create(test_mctx, workers, 0, netmgr, &taskmgr));

lib/isc/tests/netmgr_test.c

@@ -60,7 +60,7 @@ static uv_buf_t stop_msg = { .base = (char *)&stop_magic,
 			     .len = sizeof(stop_magic) };
 
 static atomic_bool do_send = ATOMIC_VAR_INIT(false);
-static unsigned int workers = 1;
+static unsigned int workers = 0;
 
 static atomic_int_fast64_t nsends;
 static int_fast64_t esends; /* expected sends */
@@ -172,9 +172,11 @@ static bool skip_long_tests = false;
 
 static int
 _setup(void **state __attribute__((unused))) {
-	char *p;
+	char *p = NULL;
 
-	workers = isc_os_ncpus();
+	if (workers == 0) {
+		workers = isc_os_ncpus();
+	}
 	p = getenv("ISC_TASK_WORKERS");
 	if (p != NULL) {
 		workers = atoi(p);
@@ -189,7 +191,7 @@ _setup(void **state __attribute__((unused))) {
 
 	signal(SIGPIPE, SIG_IGN);
 
-	if (getenv("CI_ENABLE_ALL_TESTS") != NULL) {
+	if (getenv("CI") == NULL || getenv("CI_ENABLE_ALL_TESTS") != NULL) {
 		esends = NSENDS * workers;
 	} else {
 		esends = workers;
@@ -1957,6 +1959,387 @@ tcpdns_half_recv_half_send(void **state __attribute__((unused))) {
 	CHECK_RANGE_HALF(ssends);
 }
 
+/* TLS */
+
+static isc_result_t
+tls_connect(isc_nm_t *nm) {
+	return (isc_nm_tlsconnect(nm, (isc_nmiface_t *)&tcp_connect_addr,
+				  (isc_nmiface_t *)&tcp_listen_addr,
+				  connect_connect_cb, NULL, tcp_connect_tlsctx,
+				  T_CONNECT, 0));
+}
+
+static void
+tls_noop(void **state) {
+	isc_result_t result = ISC_R_SUCCESS;
+	isc_nmsocket_t *listen_sock = NULL;
+
+	UNUSED(state);
+
+	result = isc_nm_listentls(listen_nm, (isc_nmiface_t *)&tcp_listen_addr,
+				  noop_accept_cb, NULL, 0, 0, NULL,
+				  tcp_listen_tlsctx, &listen_sock);
+	assert_int_equal(result, ISC_R_SUCCESS);
+
+	isc_nm_stoplistening(listen_sock);
+	isc_nmsocket_close(&listen_sock);
+	assert_null(listen_sock);
+
+	do {
+		isc_refcount_increment0(&active_cconnects);
+		result = isc_nm_tlsconnect(
+			connect_nm, (isc_nmiface_t *)&tcp_connect_addr,
+			(isc_nmiface_t *)&tcp_listen_addr, noop_connect_cb,
+			NULL, tcp_connect_tlsctx, T_CONNECT, 0);
+		if (result != ISC_R_SUCCESS) {
+			isc_refcount_decrement(&active_cconnects);
+			usleep(1000);
+		}
+	} while (result != ISC_R_SUCCESS);
+
+	isc_nm_closedown(connect_nm);
+
+	assert_int_equal(0, atomic_load(&cconnects));
+	assert_int_equal(0, atomic_load(&csends));
+	assert_int_equal(0, atomic_load(&creads));
+	assert_int_equal(0, atomic_load(&sreads));
+	assert_int_equal(0, atomic_load(&ssends));
+}
+
+static void
+tls_noresponse(void **state) {
+	isc_result_t result = ISC_R_SUCCESS;
+	isc_nmsocket_t *listen_sock = NULL;
+
+	UNUSED(state);
+
+	result = isc_nm_listentls(listen_nm, (isc_nmiface_t *)&tcp_listen_addr,
+				  noop_accept_cb, NULL, 0, 0, NULL,
+				  tcp_listen_tlsctx, &listen_sock);
+	assert_int_equal(result, ISC_R_SUCCESS);
+
+	do {
+		isc_refcount_increment0(&active_cconnects);
+		result = isc_nm_tlsconnect(
+			connect_nm, (isc_nmiface_t *)&tcp_connect_addr,
+			(isc_nmiface_t *)&tcp_listen_addr, noop_connect_cb,
+			NULL, tcp_connect_tlsctx, T_CONNECT, 0);
+		if (result != ISC_R_SUCCESS) {
+			isc_refcount_decrement(&active_cconnects);
+			usleep(1000);
+		}
+	} while (result != ISC_R_SUCCESS);
+
+	isc_nm_stoplistening(listen_sock);
+	isc_nmsocket_close(&listen_sock);
+	assert_null(listen_sock);
+	isc_nm_closedown(connect_nm);
+}
+
+static void
+tls_recv_one(void **state __attribute__((unused))) {
+	isc_result_t result = ISC_R_SUCCESS;
+	isc_nmsocket_t *listen_sock = NULL;
+	isc_quota_t *quotap = tcp_listener_init_quota(1);
+
+	atomic_store(&nsends, 1);
+
+	result = isc_nm_listentls(listen_nm, (isc_nmiface_t *)&tcp_listen_addr,
+				  stream_accept_cb, NULL, 0, 0, quotap,
+				  tcp_listen_tlsctx, &listen_sock);
+	assert_int_equal(result, ISC_R_SUCCESS);
+
+	do {
+		isc_refcount_increment0(&active_cconnects);
+		result = isc_nm_tlsconnect(
+			connect_nm, (isc_nmiface_t *)&tcp_connect_addr,
+			(isc_nmiface_t *)&tcp_listen_addr, connect_connect_cb,
+			NULL, tcp_connect_tlsctx, T_CONNECT, 0);
+		if (result != ISC_R_SUCCESS) {
+			isc_refcount_decrement(&active_cconnects);
+		}
+	} while (result != ISC_R_SUCCESS);
+
+	WAIT_FOR_EQ(cconnects, 1);
+	WAIT_FOR_LE(nsends, 0);
+	WAIT_FOR_EQ(csends, 1);
+	WAIT_FOR_EQ(sreads, 1);
+	WAIT_FOR_EQ(ssends, 0);
+	WAIT_FOR_EQ(creads, 0);
+
+	isc_nm_stoplistening(listen_sock);
+	isc_nmsocket_close(&listen_sock);
+	assert_null(listen_sock);
+	isc_nm_closedown(connect_nm);
+
+	X(cconnects);
+	X(csends);
+	X(creads);
+	X(sreads);
+	X(ssends);
+
+	atomic_assert_int_eq(cconnects, 1);
+	atomic_assert_int_eq(csends, 1);
+	atomic_assert_int_eq(creads, 0);
+	atomic_assert_int_eq(sreads, 1);
+	atomic_assert_int_eq(ssends, 0);
+}
+
+static void
+tls_recv_two(void **state __attribute__((unused))) {
+	isc_result_t result = ISC_R_SUCCESS;
+	isc_nmsocket_t *listen_sock = NULL;
+	isc_quota_t *quotap = tcp_listener_init_quota(1);
+
+	atomic_store(&nsends, 2);
+
+	result = isc_nm_listentls(listen_nm, (isc_nmiface_t *)&tcp_listen_addr,
+				  stream_accept_cb, NULL, 0, 0, quotap,
+				  tcp_listen_tlsctx, &listen_sock);
+	assert_int_equal(result, ISC_R_SUCCESS);
+
+	do {
+		isc_refcount_increment0(&active_cconnects);
+		result = isc_nm_tlsconnect(
+			connect_nm, (isc_nmiface_t *)&tcp_connect_addr,
+			(isc_nmiface_t *)&tcp_listen_addr, connect_connect_cb,
+			NULL, tcp_connect_tlsctx, T_CONNECT, 0);
+		if (result != ISC_R_SUCCESS) {
+			isc_refcount_decrement(&active_cconnects);
+		}
+	} while (result != ISC_R_SUCCESS);
+
+	WAIT_FOR_EQ(cconnects, 1);
+
+	do {
+		isc_refcount_increment0(&active_cconnects);
+		result = isc_nm_tlsconnect(
+			connect_nm, (isc_nmiface_t *)&tcp_connect_addr,
+			(isc_nmiface_t *)&tcp_listen_addr, connect_connect_cb,
+			NULL, tcp_connect_tlsctx, T_CONNECT, 0);
+		if (result != ISC_R_SUCCESS) {
+			isc_refcount_decrement(&active_cconnects);
+		}
+	} while (result != ISC_R_SUCCESS);
+
+	WAIT_FOR_EQ(cconnects, 2);
+	WAIT_FOR_LE(nsends, 0);
+	WAIT_FOR_EQ(csends, 2);
+	WAIT_FOR_EQ(sreads, 2);
+	WAIT_FOR_EQ(ssends, 1);
+	WAIT_FOR_EQ(creads, 1);
+
+	isc_nm_stoplistening(listen_sock);
+	isc_nmsocket_close(&listen_sock);
+	assert_null(listen_sock);
+	isc_nm_closedown(connect_nm);
+
+	X(cconnects);
+	X(csends);
+	X(creads);
+	X(sreads);
+	X(ssends);
+
+	atomic_assert_int_eq(cconnects, 2);
+	atomic_assert_int_eq(csends, 2);
+	atomic_assert_int_eq(creads, 1);
+	atomic_assert_int_eq(sreads, 2);
+	atomic_assert_int_eq(ssends, 1);
+}
+
+static void
+tls_recv_send(void **state __attribute__((unused))) {
+	isc_result_t result = ISC_R_SUCCESS;
+	isc_nmsocket_t *listen_sock = NULL;
+	isc_thread_t threads[workers];
+	isc_quota_t *quotap = tcp_listener_init_quota(workers);
+
+	SKIP_IN_CI;
+
+	result = isc_nm_listentls(listen_nm, (isc_nmiface_t *)&tcp_listen_addr,
+				  stream_accept_cb, NULL, 0, 0, quotap,
+				  tcp_listen_tlsctx, &listen_sock);
+	assert_int_equal(result, ISC_R_SUCCESS);
+
+	memset(threads, 0, sizeof(threads));
+	for (size_t i = 0; i < workers; i++) {
+		isc_thread_create(connect_thread, tls_connect, &threads[i]);
+	}
+
+	WAIT_FOR_GE(cconnects, esends);
+	WAIT_FOR_GE(csends, esends);
+	WAIT_FOR_GE(sreads, esends);
+	WAIT_FOR_GE(ssends, esends / 2);
+	WAIT_FOR_GE(creads, esends / 2);
+
+	DONE();
+	for (size_t i = 0; i < workers; i++) {
+		isc_thread_join(threads[i], NULL);
+	}
+
+	isc_nm_closedown(connect_nm);
+	isc_nm_stoplistening(listen_sock);
+	isc_nmsocket_close(&listen_sock);
+	assert_null(listen_sock);
+
+	X(cconnects);
+	X(csends);
+	X(creads);
+	X(sreads);
+	X(ssends);
+
+	CHECK_RANGE_FULL(csends);
+	CHECK_RANGE_FULL(creads);
+	CHECK_RANGE_FULL(sreads);
+	CHECK_RANGE_FULL(ssends);
+}
+
+static void
+tls_recv_half_send(void **state __attribute__((unused))) {
+	isc_result_t result = ISC_R_SUCCESS;
+	isc_nmsocket_t *listen_sock = NULL;
+	isc_thread_t threads[workers];
+	isc_quota_t *quotap = tcp_listener_init_quota(workers);
+
+	SKIP_IN_CI;
+
+	result = isc_nm_listentls(listen_nm, (isc_nmiface_t *)&tcp_listen_addr,
+				  stream_accept_cb, NULL, 0, 0, quotap,
+				  tcp_listen_tlsctx, &listen_sock);
+	assert_int_equal(result, ISC_R_SUCCESS);
+
+	memset(threads, 0, sizeof(threads));
+	for (size_t i = 0; i < workers; i++) {
+		isc_thread_create(connect_thread, tls_connect, &threads[i]);
+	}
+
+	WAIT_FOR_GE(cconnects, esends / 2);
+	WAIT_FOR_GE(csends, esends / 2);
+	WAIT_FOR_GE(sreads, esends / 2);
+	WAIT_FOR_GE(ssends, esends / 2);
+	WAIT_FOR_GE(creads, esends / 2);
+
+	isc_nm_closedown(connect_nm);
+
+	DONE();
+	for (size_t i = 0; i < workers; i++) {
+		isc_thread_join(threads[i], NULL);
+	}
+
+	isc_nm_stoplistening(listen_sock);
+	isc_nmsocket_close(&listen_sock);
+	assert_null(listen_sock);
+
+	X(cconnects);
+	X(csends);
+	X(creads);
+	X(sreads);
+	X(ssends);
+
+	CHECK_RANGE_HALF(csends);
+	CHECK_RANGE_HALF(creads);
+	CHECK_RANGE_HALF(sreads);
+	CHECK_RANGE_HALF(ssends);
+}
+
+static void
+tls_half_recv_send(void **state __attribute__((unused))) {
+	isc_result_t result = ISC_R_SUCCESS;
+	isc_nmsocket_t *listen_sock = NULL;
+	isc_thread_t threads[workers];
+	isc_quota_t *quotap = tcp_listener_init_quota(workers);
+
+	SKIP_IN_CI;
+
+	result = isc_nm_listentls(listen_nm, (isc_nmiface_t *)&tcp_listen_addr,
+				  stream_accept_cb, NULL, 0, 0, quotap,
+				  tcp_listen_tlsctx, &listen_sock);
+	assert_int_equal(result, ISC_R_SUCCESS);
+
+	memset(threads, 0, sizeof(threads));
+	for (size_t i = 0; i < workers; i++) {
+		isc_thread_create(connect_thread, tls_connect, &threads[i]);
+	}
+
+	WAIT_FOR_GE(cconnects, esends / 2);
+	WAIT_FOR_GE(csends, esends / 2);
+	WAIT_FOR_GE(sreads, esends / 2);
+	WAIT_FOR_GE(ssends, esends / 2);
+	WAIT_FOR_GE(creads, esends / 2);
+
+	isc_nm_stoplistening(listen_sock);
+	isc_nmsocket_close(&listen_sock);
+	assert_null(listen_sock);
+
+	/* Try to send a little while longer */
+	usleep((esends / 2) * 10000);
+
+	isc_nm_closedown(connect_nm);
+
+	DONE();
+	for (size_t i = 0; i < workers; i++) {
+		isc_thread_join(threads[i], NULL);
+	}
+
+	X(cconnects);
+	X(csends);
+	X(creads);
+	X(sreads);
+	X(ssends);
+
+	CHECK_RANGE_HALF(csends);
+	CHECK_RANGE_HALF(creads);
+	CHECK_RANGE_HALF(sreads);
+	CHECK_RANGE_HALF(ssends);
+}
+
+static void
+tls_half_recv_half_send(void **state __attribute__((unused))) {
+	isc_result_t result = ISC_R_SUCCESS;
+	isc_nmsocket_t *listen_sock = NULL;
+	isc_thread_t threads[workers];
+	isc_quota_t *quotap = tcp_listener_init_quota(workers);
+
+	SKIP_IN_CI;
+
+	result = isc_nm_listentls(listen_nm, (isc_nmiface_t *)&tcp_listen_addr,
+				  stream_accept_cb, NULL, 0, 0, quotap,
+				  tcp_listen_tlsctx, &listen_sock);
+	assert_int_equal(result, ISC_R_SUCCESS);
+
+	memset(threads, 0, sizeof(threads));
+	for (size_t i = 0; i < workers; i++) {
+		isc_thread_create(connect_thread, tls_connect, &threads[i]);
+	}
+
+	WAIT_FOR_GE(cconnects, esends / 2);
+	WAIT_FOR_GE(csends, esends / 2);
+	WAIT_FOR_GE(sreads, esends / 2);
+	WAIT_FOR_GE(ssends, esends / 2);
+	WAIT_FOR_GE(creads, esends / 2);
+
+	isc_nm_closedown(connect_nm);
+	isc_nm_stoplistening(listen_sock);
+	isc_nmsocket_close(&listen_sock);
+	assert_null(listen_sock);
+
+	DONE();
+	for (size_t i = 0; i < workers; i++) {
+		isc_thread_join(threads[i], NULL);
+	}
+
+	X(cconnects);
+	X(csends);
+	X(creads);
+	X(sreads);
+	X(ssends);
+
+	CHECK_RANGE_HALF(csends);
+	CHECK_RANGE_HALF(creads);
+	CHECK_RANGE_HALF(sreads);
+	CHECK_RANGE_HALF(ssends);
+}
+
 /* TLSDNS */
 
 static isc_result_t
@@ -2464,6 +2847,24 @@ main(void) {
 		cmocka_unit_test_setup_teardown(tcpdns_half_recv_half_send,
 						nm_setup, nm_teardown),
 
+		/* TLS */
+		cmocka_unit_test_setup_teardown(tls_noop, nm_setup,
+						nm_teardown),
+		cmocka_unit_test_setup_teardown(tls_noresponse, nm_setup,
+						nm_teardown),
+		cmocka_unit_test_setup_teardown(tls_recv_one, nm_setup,
+						nm_teardown),
+		cmocka_unit_test_setup_teardown(tls_recv_two, nm_setup,
+						nm_teardown),
+		cmocka_unit_test_setup_teardown(tls_recv_send, nm_setup,
+						nm_teardown),
+		cmocka_unit_test_setup_teardown(tls_recv_half_send, nm_setup,
+						nm_teardown),
+		cmocka_unit_test_setup_teardown(tls_half_recv_send, nm_setup,
+						nm_teardown),
+		cmocka_unit_test_setup_teardown(tls_half_recv_half_send,
+						nm_setup, nm_teardown),
+
 		/* TLSDNS */
 		cmocka_unit_test_setup_teardown(tlsdns_recv_one, nm_setup,
 						nm_teardown),

util/copyrights

@@ -2014,6 +2014,7 @@
 ./lib/isc/tests/testdata/file/keep		X	2014,2018,2019,2020,2021
 ./lib/isc/tests/time_test.c			C	2014,2015,2016,2018,2019,2020,2021
 ./lib/isc/tests/timer_test.c			C	2018,2019,2020,2021
+./lib/isc/tests/tls_test.c			C	2021
 ./lib/isc/tests/uv_wrap.h			C	2020,2021
 ./lib/isc/timer.c				C	1998,1999,2000,2001,2002,2004,2005,2007,2008,2009,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020,2021
 ./lib/isc/tls.c					C	2021