fix handling of tcp connection errors

- a bug that was already fixed in tcpdns.c and tlsdns.c had been
  left unfixed in tcp.c: we needed to check whether the socket() call
  succeeds before attempting any asynchronous connection code.
  this corrects a problem that occurred when uv__socket() failed
  with too many open files.
- also corrected the behavior of nm_connectcb().  previously, it was
  forcing synchronous execution when force_async was false. what we
  want is to force asynchronous execution if force_async is true. if
  force_async is false, we want it to decide based on whether we're in
  the network thread.

lib/isc/netmgr/netmgr.c

@@ -1868,21 +1868,20 @@ isc_nm_stoplistening(isc_nmsocket_t *sock) {
 static void
 nm_connectcb(isc_nmsocket_t *sock, isc__nm_uvreq_t *uvreq, isc_result_t eresult,
 	     bool force_async) {
+	isc__netievent_connectcb_t *ievent = NULL;
+
 	REQUIRE(VALID_NMSOCK(sock));
 	REQUIRE(VALID_UVREQ(uvreq));
 	REQUIRE(VALID_NMHANDLE(uvreq->handle));
 
-	if (eresult == ISC_R_SUCCESS && !force_async) {
-		isc__netievent_connectcb_t ievent = { .sock = sock,
-						      .req = uvreq,
-						      .result = eresult };
-		isc__nm_async_connectcb(NULL, (isc__netievent_t *)&ievent);
-	} else {
-		isc__netievent_connectcb_t *ievent =
-			isc__nm_get_netievent_connectcb(sock->mgr, sock, uvreq,
-							eresult);
+	ievent = isc__nm_get_netievent_connectcb(sock->mgr, sock, uvreq,
+						 eresult);
+	if (force_async) {
 		isc__nm_enqueue_ievent(&sock->mgr->workers[sock->tid],
 				       (isc__netievent_t *)ievent);
+	} else {
+		isc__nm_maybe_enqueue_ievent(&sock->mgr->workers[sock->tid],
+					     (isc__netievent_t *)ievent);
 	}
 }
 

lib/isc/netmgr/tcp.c

@@ -167,23 +167,6 @@ tcp_connect_direct(isc_nmsocket_t *sock, isc__nm_uvreq_t *req) {
 	REQUIRE(isc__nm_in_netthread());
 	REQUIRE(sock->tid == isc_nm_tid());
 
-	result = isc__nm_socket(req->peer.type.sa.sa_family, SOCK_STREAM, 0,
-				&sock->fd);
-	/*
-	 * The socket() call can fail spuriously on FreeBSD 12, so we need to
-	 * handle the failure early and gracefully.
-	 */
-	if (result != ISC_R_SUCCESS) {
-		atomic_store(&sock->closed, true);
-		isc__nm_uvreq_t *cbreq = NULL;
-		cbreq = isc__nm_uvreq_get(sock->mgr, sock);
-		cbreq->cb.connect = req->cb.connect;
-		cbreq->cbarg = req->cbarg;
-		isc_nmhandle_attach(req->handle, &cbreq->handle);
-		isc__nmsocket_clearcb(sock);
-		isc__nm_connectcb(sock, cbreq, result);
-		goto error;
-	}
 	result = isc__nm_socket_connectiontimeout(sock->fd,
 						  sock->connect_timeout);
 	RUNTIME_CHECK(result == ISC_R_SUCCESS);
@@ -231,7 +214,7 @@ tcp_connect_direct(isc_nmsocket_t *sock, isc__nm_uvreq_t *req) {
 
 done:
 	result = isc__nm_uverr2result(r);
-error:
+
 	LOCK(&sock->lock);
 	sock->result = result;
 	SIGNAL(&sock->cond);
@@ -260,7 +243,6 @@ isc__nm_async_tcpconnect(isc__networker_t *worker, isc__netievent_t *ev0) {
 	REQUIRE(sock->parent == NULL);
 	REQUIRE(sock->tid == isc_nm_tid());
 
-	sock->fd = (uv_os_sock_t)(-1);
 	result = tcp_connect_direct(sock, req);
 	if (result != ISC_R_SUCCESS) {
 		atomic_store(&sock->active, false);
@@ -333,17 +315,31 @@ isc_nm_tcpconnect(isc_nm_t *mgr, isc_nmiface_t *local, isc_nmiface_t *peer,
 	isc_nmsocket_t *sock = NULL;
 	isc__netievent_tcpconnect_t *ievent = NULL;
 	isc__nm_uvreq_t *req = NULL;
+	sa_family_t sa_family;
+	uv_os_sock_t fd;
 
 	REQUIRE(VALID_NM(mgr));
 	REQUIRE(local != NULL);
 	REQUIRE(peer != NULL);
 
+	sa_family = peer->addr.type.sa.sa_family;
+
+	/*
+	 * The socket() call can fail spuriously on FreeBSD 12, so we need to
+	 * handle the failure early and gracefully.
+	 */
+	result = isc__nm_socket(sa_family, SOCK_STREAM, 0, &fd);
+	if (result != ISC_R_SUCCESS) {
+		return (result);
+	}
+
 	sock = isc_mem_get(mgr->mctx, sizeof(*sock));
 	isc__nmsocket_init(sock, mgr, isc_nm_tcpsocket, local);
 
 	sock->extrahandlesize = extrahandlesize;
 	sock->connect_timeout = timeout;
 	sock->result = ISC_R_DEFAULT;
+	sock->fd = fd;
 	atomic_init(&sock->client, true);
 
 	req = isc__nm_uvreq_get(mgr, sock);

lib/isc/netmgr/tlsstream.c

@@ -845,6 +845,7 @@ tcp_connected(isc_nmhandle_t *handle, isc_result_t result, void *cbarg) {
 	}
 	return;
 error:
+	update_result(tlssock, result);
 	tlshandle = isc__nmhandle_get(tlssock, NULL, NULL);
 	atomic_store(&tlssock->closed, true);
 	tls_call_connect_cb(tlssock, tlshandle, result);
@@ -882,17 +883,14 @@ isc__nm_async_tlsconnect(isc__networker_t *worker, isc__netievent_t *ev0) {
 	tlssock->timer_initialized = true;
 	tlssock->tlsstream.state = TLS_INIT;
 
-	/*
-	 * We ignore the return code, because even in the case of
-	 * failure, the callback will be called and the error code
-	 * passed to it.
-	 */
 	result = isc_nm_tcpconnect(worker->mgr, (isc_nmiface_t *)&ievent->local,
 				   (isc_nmiface_t *)&ievent->peer,
 				   tcp_connected, tlssock,
 				   tlssock->connect_timeout, 0);
-	update_result(tlssock, result);
-	return;
+	if (result == ISC_R_SUCCESS) {
+		update_result(tlssock, result);
+		return;
+	}
 
 error:
 	tlshandle = isc__nmhandle_get(tlssock, NULL, NULL);