TLS: cancelread support and numerous stability fixes

lib/isc/Makefile.am

@@ -129,6 +129,7 @@ libisc_la_SOURCES =		\
 	netmgr/tcp.c		\
 	netmgr/tcpdns.c		\
 	netmgr/tlsdns.c		\
+	netmgr/tls.c		\
 	netmgr/udp.c		\
 	netmgr/uv-compat.c	\
 	netmgr/uv-compat.h	\

lib/isc/include/isc/netmgr.h

@@ -473,6 +473,17 @@ isc_nm_setstats(isc_nm_t *mgr, isc_stats_t *stats);
  *	full range of socket-related stats counter numbers.
  */
 
+isc_result_t
+isc_nm_listentls(isc_nm_t *mgr, isc_nmiface_t *iface,
+		 isc_nm_accept_cb_t accept_cb, void *accept_cbarg,
+		 size_t extrahandlesize, int backlog, isc_quota_t *quota,
+		 isc_tlsctx_t *sslctx, isc_nmsocket_t **sockp);
+
+isc_result_t
+isc_nm_tlsconnect(isc_nm_t *mgr, isc_nmiface_t *local, isc_nmiface_t *peer,
+		  isc_nm_cb_t cb, void *cbarg, isc_tlsctx_t *ctx,
+		  unsigned int timeout, size_t extrahandlesize);
+
 isc_result_t
 isc_nm_tcpdnsconnect(isc_nm_t *mgr, isc_nmiface_t *local, isc_nmiface_t *peer,
 		     isc_nm_cb_t cb, void *cbarg, unsigned int timeout,
@@ -494,3 +505,19 @@ isc_nm_tlsdnsconnect(isc_nm_t *mgr, isc_nmiface_t *local, isc_nmiface_t *peer,
  * The connected socket can only be accessed via the handle passed to
  * 'cb'.
  */
+
+isc_result_t
+isc_nm_tls_create_server_ctx(const char *keyfile, const char *certfile,
+			     isc_tlsctx_t **ctxp);
+
+void
+isc_nm_tls_initialize(void);
+/*%<
+ * Initialize OpenSSL library, idempotent.
+ */
+
+void
+isc_nm_tls_deinitialize(void);
+/*%<
+ * De-Initialize OpenSSL library, idempotent.
+ */

lib/isc/netmgr/netmgr-int.h

@@ -252,6 +252,13 @@ typedef enum isc__netievent_type {
 	netievent_tcpdnsclose,
 	netievent_tcpdnsstop,
 
+	netievent_tlsclose,
+	netievent_tlssend,
+	netievent_tlsstartread,
+	netievent_tlsconnect,
+	netievent_tlsdobio,
+	netievent_tlscancel,
+
 	netievent_tlsdnsaccept,
 	netievent_tlsdnsconnect,
 	netievent_tlsdnssend,
@@ -642,6 +649,8 @@ typedef enum isc_nmsocket_type {
 	isc_nm_tcplistener,
 	isc_nm_tcpdnslistener,
 	isc_nm_tcpdnssocket,
+	isc_nm_tlslistener,
+	isc_nm_tlssocket,
 	isc_nm_tlsdnslistener,
 	isc_nm_tlsdnssocket
 } isc_nmsocket_type;
@@ -670,6 +679,12 @@ enum {
 	STATID_ACTIVE = 10
 };
 
+
+typedef struct isc_nmsocket_tls_send_req {
+	isc_nmsocket_t *tlssock;
+	isc_region_t data;
+} isc_nmsocket_tls_send_req_t;
+
 struct isc_nmsocket {
 	/*% Unlocked, RO */
 	int magic;
@@ -705,6 +720,27 @@ struct isc_nmsocket {
 		isc__nm_uvreq_t *pending_req;
 	} tls;
 
+	/*% TLS stuff */
+	struct tlsstream {
+		bool server;
+		BIO *app_bio;
+		SSL *ssl;
+		SSL_CTX *ctx;
+		BIO *ssl_bio;
+		isc_nmsocket_t *tlslistener;
+		enum {
+			TLS_INIT,
+			TLS_HANDSHAKE,
+			TLS_IO,
+			TLS_ERROR,
+			TLS_CLOSING,
+			TLS_CLOSED
+		} state;
+		size_t nsending;
+		/* List of active send requests. */
+		ISC_LIST(isc__nm_uvreq_t) sends;
+	} tlsstream;
+
 	/*%
 	 * quota is the TCP client, attached when a TCP connection
 	 * is established. pquota is a non-attached pointer to the
@@ -1220,6 +1256,28 @@ isc__nm_async_tcpclose(isc__networker_t *worker, isc__netievent_t *ev0);
  * stoplisten, send, read, pause, close).
  */
 
+void
+isc__nm_async_tlsclose(isc__networker_t *worker, isc__netievent_t *ev0);
+
+void
+isc__nm_async_tlssend(isc__networker_t *worker, isc__netievent_t *ev0);
+
+void
+isc__nm_async_tlsconnect(isc__networker_t *worker, isc__netievent_t *ev0);
+
+void
+isc__nm_async_tlsstartread(isc__networker_t *worker, isc__netievent_t *ev0);
+
+void
+isc__nm_async_tlsdobio(isc__networker_t *worker, isc__netievent_t *ev0);
+
+void
+isc__nm_async_tlscancel(isc__networker_t *worker, isc__netievent_t *ev0);
+
+/*%<
+ * Callback handlers for asynchronouse TLS events.
+ */
+
 void
 isc__nm_async_tcpdnsaccept(isc__networker_t *worker, isc__netievent_t *ev0);
 void
@@ -1291,6 +1349,10 @@ isc__nm_async_tlsdnslisten(isc__networker_t *worker, isc__netievent_t *ev0);
 void
 isc__nm_tlsdns_send(isc_nmhandle_t *handle, isc_region_t *region,
 		    isc_nm_cb_t cb, void *cbarg);
+
+void
+isc__nm_tls_cancelread(isc_nmhandle_t *handle);
+
 /*%<
  * Back-end implementation of isc_nm_send() for TLSDNS handles.
  */
@@ -1344,6 +1406,38 @@ isc__nm_tlsdns_cancelread(isc_nmhandle_t *handle);
  * Stop reading on a connected TLSDNS handle.
  */
 
+void
+isc__nm_tls_send(isc_nmhandle_t *handle, isc_region_t *region, isc_nm_cb_t cb,
+		 void *cbarg);
+
+void
+isc__nm_tls_read(isc_nmhandle_t *handle, isc_nm_recv_cb_t cb, void *cbarg);
+
+void
+isc__nm_tls_close(isc_nmsocket_t *sock);
+/*%<
+ * Close a TLS socket.
+ */
+
+void
+isc__nm_tls_pauseread(isc_nmhandle_t *handle);
+/*%<
+ * Pause reading on this handle, while still remembering the callback.
+ */
+
+void
+isc__nm_tls_resumeread(isc_nmhandle_t *handle);
+/*%<
+ * Resume reading from the handle.
+ *
+ */
+
+void
+isc__nm_tls_cleanup_data(isc_nmsocket_t *sock);
+
+void
+isc__nm_tls_stoplistening(isc_nmsocket_t *sock);
+
 #define isc__nm_uverr2result(x) \
 	isc___nm_uverr2result(x, true, __FILE__, __LINE__, __func__)
 isc_result_t
@@ -1444,6 +1538,12 @@ NETIEVENT_SOCKET_TYPE(tcpclose);
 NETIEVENT_SOCKET_TYPE(tcplisten);
 NETIEVENT_SOCKET_TYPE(tcppauseread);
 NETIEVENT_SOCKET_TYPE(tcpstop);
+NETIEVENT_SOCKET_TYPE(tlsclose);
+/* NETIEVENT_SOCKET_TYPE(tlsconnect); */ /* unique type, defined independently
+					  */
+NETIEVENT_SOCKET_TYPE(tlsdobio);
+NETIEVENT_SOCKET_TYPE(tlsstartread);
+NETIEVENT_SOCKET_HANDLE_TYPE(tlscancel);
 NETIEVENT_SOCKET_TYPE(udpclose);
 NETIEVENT_SOCKET_TYPE(udplisten);
 NETIEVENT_SOCKET_TYPE(udpread);
@@ -1473,6 +1573,7 @@ NETIEVENT_SOCKET_TYPE(tlsdnscycle);
 NETIEVENT_SOCKET_REQ_TYPE(tcpconnect);
 NETIEVENT_SOCKET_REQ_TYPE(tcpsend);
 NETIEVENT_SOCKET_TYPE(tcpstartread);
+NETIEVENT_SOCKET_REQ_TYPE(tlssend);
 NETIEVENT_SOCKET_REQ_TYPE(udpconnect);
 
 NETIEVENT_SOCKET_REQ_RESULT_TYPE(connectcb);
@@ -1498,6 +1599,11 @@ NETIEVENT_SOCKET_DECL(tcplisten);
 NETIEVENT_SOCKET_DECL(tcppauseread);
 NETIEVENT_SOCKET_DECL(tcpstartread);
 NETIEVENT_SOCKET_DECL(tcpstop);
+NETIEVENT_SOCKET_DECL(tlsclose);
+NETIEVENT_SOCKET_DECL(tlsconnect);
+NETIEVENT_SOCKET_DECL(tlsdobio);
+NETIEVENT_SOCKET_DECL(tlsstartread);
+NETIEVENT_SOCKET_HANDLE_DECL(tlscancel);
 NETIEVENT_SOCKET_DECL(udpclose);
 NETIEVENT_SOCKET_DECL(udplisten);
 NETIEVENT_SOCKET_DECL(udpread);
@@ -1526,6 +1632,7 @@ NETIEVENT_SOCKET_DECL(tlsdnscycle);
 
 NETIEVENT_SOCKET_REQ_DECL(tcpconnect);
 NETIEVENT_SOCKET_REQ_DECL(tcpsend);
+NETIEVENT_SOCKET_REQ_DECL(tlssend);
 NETIEVENT_SOCKET_REQ_DECL(udpconnect);
 
 NETIEVENT_SOCKET_REQ_RESULT_DECL(connectcb);

lib/isc/netmgr/netmgr.c

@@ -716,6 +716,13 @@ process_netievent(isc__networker_t *worker, isc__netievent_t *ievent) {
 		NETIEVENT_CASE(tcpdnsread);
 		NETIEVENT_CASE(tcpdnsstop);
 
+		NETIEVENT_CASE(tlsstartread);
+		NETIEVENT_CASE(tlssend);
+		NETIEVENT_CASE(tlsclose);
+		NETIEVENT_CASE(tlsconnect);
+		NETIEVENT_CASE(tlsdobio);
+		NETIEVENT_CASE(tlscancel);
+
 		NETIEVENT_CASE(tlsdnscycle);
 		NETIEVENT_CASE(tlsdnsaccept);
 		NETIEVENT_CASE(tlsdnslisten);
@@ -776,6 +783,11 @@ NETIEVENT_SOCKET_DEF(tcplisten);
 NETIEVENT_SOCKET_DEF(tcppauseread);
 NETIEVENT_SOCKET_DEF(tcpstartread);
 NETIEVENT_SOCKET_DEF(tcpstop);
+NETIEVENT_SOCKET_DEF(tlsclose);
+NETIEVENT_SOCKET_DEF(tlsconnect);
+NETIEVENT_SOCKET_DEF(tlsdobio);
+NETIEVENT_SOCKET_DEF(tlsstartread);
+NETIEVENT_SOCKET_HANDLE_DEF(tlscancel);
 NETIEVENT_SOCKET_DEF(udpclose);
 NETIEVENT_SOCKET_DEF(udplisten);
 NETIEVENT_SOCKET_DEF(udpread);
@@ -804,6 +816,7 @@ NETIEVENT_SOCKET_DEF(tlsdnsshutdown);
 
 NETIEVENT_SOCKET_REQ_DEF(tcpconnect);
 NETIEVENT_SOCKET_REQ_DEF(tcpsend);
+NETIEVENT_SOCKET_REQ_DEF(tlssend);
 NETIEVENT_SOCKET_REQ_DEF(udpconnect);
 
 NETIEVENT_SOCKET_REQ_RESULT_DEF(connectcb);
@@ -986,6 +999,8 @@ nmsocket_cleanup(isc_nmsocket_t *sock, bool dofree FLARG) {
 	isc_mutex_destroy(&sock->lock);
 	isc_condition_destroy(&sock->cond);
 	isc_condition_destroy(&sock->scond);
+	isc__nm_tls_cleanup_data(sock);
+
 #ifdef NETMGR_TRACE
 	LOCK(&sock->mgr->lock);
 	ISC_LIST_UNLINK(sock->mgr->active_sockets, sock, active_link);
@@ -1094,6 +1109,9 @@ isc___nmsocket_prep_destroy(isc_nmsocket_t *sock FLARG) {
 		case isc_nm_tcpdnssocket:
 			isc__nm_tcpdns_close(sock);
 			return;
+		case isc_nm_tlssocket:
+			isc__nm_tls_close(sock);
+			break;
 		case isc_nm_tlsdnssocket:
 			isc__nm_tlsdns_close(sock);
 			return;
@@ -1139,7 +1157,8 @@ isc_nmsocket_close(isc_nmsocket_t **sockp) {
 	REQUIRE((*sockp)->type == isc_nm_udplistener ||
 		(*sockp)->type == isc_nm_tcplistener ||
 		(*sockp)->type == isc_nm_tcpdnslistener ||
-		(*sockp)->type == isc_nm_tlsdnslistener);
+		(*sockp)->type == isc_nm_tlsdnslistener ||
+		(*sockp)->type == isc_nm_tlslistener);
 
 	isc__nmsocket_detach(sockp);
 }
@@ -1218,6 +1237,9 @@ isc___nmsocket_init(isc_nmsocket_t *sock, isc_nm_t *mgr, isc_nmsocket_type type,
 	isc_condition_init(&sock->scond);
 	isc_refcount_init(&sock->references, 1);
 
+	memset(&sock->tls, 0, sizeof(sock->tls));
+	ISC_LIST_INIT(sock->tlsstream.sends);
+
 	NETMGR_TRACE_LOG("isc__nmsocket_init():%p->references = %lu\n", sock,
 			 isc_refcount_current(&sock->references));
 
@@ -1353,7 +1375,7 @@ isc___nmhandle_get(isc_nmsocket_t *sock, isc_sockaddr_t *peer,
 #endif
 	UNLOCK(&sock->lock);
 
-	if (sock->type == isc_nm_tcpsocket ||
+	if (sock->type == isc_nm_tcpsocket || sock->type == isc_nm_tlssocket ||
 	    (sock->type == isc_nm_udpsocket && atomic_load(&sock->client)) ||
 	    (sock->type == isc_nm_tcpdnssocket && atomic_load(&sock->client)) ||
 	    (sock->type == isc_nm_tlsdnssocket && atomic_load(&sock->client)))
@@ -1390,6 +1412,7 @@ isc_nmhandle_is_stream(isc_nmhandle_t *handle) {
 
 	return (handle->sock->type == isc_nm_tcpsocket ||
 		handle->sock->type == isc_nm_tcpdnssocket ||
+		handle->sock->type == isc_nm_tlssocket ||
 		handle->sock->type == isc_nm_tlsdnssocket);
 }
 
@@ -1667,6 +1690,9 @@ isc_nm_send(isc_nmhandle_t *handle, isc_region_t *region, isc_nm_cb_t cb,
 	case isc_nm_tcpdnssocket:
 		isc__nm_tcpdns_send(handle, region, cb, cbarg);
 		break;
+	case isc_nm_tlssocket:
+		isc__nm_tls_send(handle, region, cb, cbarg);
+		break;
 	case isc_nm_tlsdnssocket:
 		isc__nm_tlsdns_send(handle, region, cb, cbarg);
 		break;
@@ -1697,6 +1723,9 @@ isc_nm_read(isc_nmhandle_t *handle, isc_nm_recv_cb_t cb, void *cbarg) {
 	case isc_nm_tcpdnssocket:
 		isc__nm_tcpdns_read(handle, cb, cbarg);
 		break;
+	case isc_nm_tlssocket:
+		isc__nm_tls_read(handle, cb, cbarg);
+		break;
 	case isc_nm_tlsdnssocket:
 		isc__nm_tlsdns_read(handle, cb, cbarg);
 		break;
@@ -1723,6 +1752,9 @@ isc_nm_cancelread(isc_nmhandle_t *handle) {
 	case isc_nm_tlsdnssocket:
 		isc__nm_tlsdns_cancelread(handle);
 		break;
+	case isc_nm_tlssocket:
+		isc__nm_tls_cancelread(handle);
+		break;
 	default:
 		INSIST(0);
 		ISC_UNREACHABLE();
@@ -1739,6 +1771,9 @@ isc_nm_pauseread(isc_nmhandle_t *handle) {
 	case isc_nm_tcpsocket:
 		isc__nm_tcp_pauseread(handle);
 		break;
+	case isc_nm_tlssocket:
+		isc__nm_tls_pauseread(handle);
+		break;
 	default:
 		INSIST(0);
 		ISC_UNREACHABLE();
@@ -1755,6 +1790,9 @@ isc_nm_resumeread(isc_nmhandle_t *handle) {
 	case isc_nm_tcpsocket:
 		isc__nm_tcp_resumeread(handle);
 		break;
+	case isc_nm_tlssocket:
+		isc__nm_tls_resumeread(handle);
+		break;
 	default:
 		INSIST(0);
 		ISC_UNREACHABLE();
@@ -1775,6 +1813,9 @@ isc_nm_stoplistening(isc_nmsocket_t *sock) {
 	case isc_nm_tcplistener:
 		isc__nm_tcp_stoplistening(sock);
 		break;
+	case isc_nm_tlslistener:
+		isc__nm_tls_stoplistening(sock);
+		break;
 	case isc_nm_tlsdnslistener:
 		isc__nm_tlsdns_stoplistening(sock);
 		break;
@@ -2322,6 +2363,10 @@ nmsocket_type_totext(isc_nmsocket_type type) {
 		return ("isc_nm_tcpdnslistener");
 	case isc_nm_tcpdnssocket:
 		return ("isc_nm_tcpdnssocket");
+	case isc_nm_tlssocket:
+		return ("isc_nm_tlssocket");
+	case isc_nm_tlslistener:
+		return ("isc_nm_tlslistener");
 	case isc_nm_tlsdnslistener:
 		return ("isc_nm_tlsdnslistener");
 	case isc_nm_tlsdnssocket:

lib/isc/win32/libisc.def.in

@@ -451,6 +451,7 @@ isc_nm_detach
 isc_nm_listentcpdns
 isc_nm_listentlsdns
 isc_nm_listentcp
+isc_nm_listentls
 isc_nm_listenudp
 isc_nm_maxudp
 isc_nm_pauseread
@@ -467,6 +468,8 @@ isc_nm_settimeouts
 isc_nm_tcpdns_keepalive
 isc_nm_tcpdns_sequential
 isc_nm_tid
+isc_nm_tls_create_server_ctx
+isc_nm_tlsconnect
 isc_nm_tlsdnsconnect
 isc_nm_udpconnect
 isc_nmsocket_close

lib/isc/win32/libisc.vcxproj.in

@@ -414,6 +414,7 @@ copy InstallFiles ..\Build\Release\
     <ClCompile Include="..\netmgr\uverr2result.c" />
     <ClCompile Include="..\netmgr\uv-compat.c" />
     <ClCompile Include="..\netmgr\tcpdns.c" />
+	<ClCompile Include="..\netmgr\tls.c" />
     <ClCompile Include="..\netmgr\tlsdns.c" />
     <ClCompile Include="..\netscope.c" />
     <ClCompile Include="..\nonce.c" />