Handle the transient TCP connect() failures on FreeBSD

On FreeBSD (and perhaps other *BSD) systems, the TCP connect() call (via
uv_tcp_connect()) can fail with transient UV_EADDRINUSE error.  The UDP
code already handles this by trying three times (is a charm) before
giving up.  Add a code for the TCP, TCPDNS and TLSDNS layers to also try
three times before giving up by calling uv_tcp_connect() from the
callback two more time on UV_EADDRINUSE error.

Additionally, stop the timer only if we succeed or on hard error via
isc__nm_failed_connect_cb().
This commit is contained in:
Ondřej Surý
2022-07-13 09:34:47 +02:00
parent 08af14a475
commit a280855f7b
6 changed files with 72 additions and 20 deletions

View File

@@ -377,6 +377,7 @@ struct isc__nm_uvreq {
isc__nm_cb_t cb; /* callback */
void *cbarg; /* callback argument */
isc_nm_timer_t *timer; /* TCP write timer */
int connect_tries; /* connect retries */
union {
uv_handle_t handle;

View File

@@ -2465,7 +2465,10 @@ isc___nm_uvreq_get(isc_nm_t *mgr, isc_nmsocket_t *sock FLARG) {
req = isc_mem_get(mgr->mctx, sizeof(*req));
}
*req = (isc__nm_uvreq_t){ .magic = 0 };
*req = (isc__nm_uvreq_t){
.magic = 0,
.connect_tries = 3,
};
ISC_LINK_INIT(req, link);
req->uv_req.req.data = req;
isc___nmsocket_attach(sock, &req->sock FLARG_PASS);

View File

@@ -228,9 +228,6 @@ tcp_connect_cb(uv_connect_t *uvreq, int status) {
REQUIRE(VALID_NMSOCK(sock));
REQUIRE(sock->tid == isc_nm_tid());
isc__nmsocket_timer_stop(sock);
uv_handle_set_data((uv_handle_t *)&sock->read_timer, sock);
req = uv_handle_get_data((uv_handle_t *)uvreq);
REQUIRE(VALID_UVREQ(req));
@@ -239,9 +236,7 @@ tcp_connect_cb(uv_connect_t *uvreq, int status) {
if (atomic_load(&sock->timedout)) {
result = ISC_R_TIMEDOUT;
goto error;
}
if (!atomic_load(&sock->connecting)) {
} else if (!atomic_load(&sock->connecting)) {
/*
* The connect was cancelled from timeout; just clean up
* the req.
@@ -260,11 +255,33 @@ tcp_connect_cb(uv_connect_t *uvreq, int status) {
/* Timeout status code here indicates hard error */
result = ISC_R_TIMEDOUT;
goto error;
} else if (status == UV_EADDRINUSE) {
/*
* On FreeBSD the TCP connect() call sometimes results in a
* spurious transient EADDRINUSE. Try a few more times before
* giving up.
*/
if (--req->connect_tries > 0) {
r = uv_tcp_connect(&req->uv_req.connect,
&sock->uv_handle.tcp,
&req->peer.type.sa, tcp_connect_cb);
if (r != 0) {
isc__nm_incstats(sock, STATID_CONNECTFAIL);
result = isc_uverr2result(r);
goto error;
}
return;
}
result = isc_uverr2result(status);
goto error;
} else if (status != 0) {
result = isc_uverr2result(status);
goto error;
}
isc__nmsocket_timer_stop(sock);
uv_handle_set_data((uv_handle_t *)&sock->read_timer, sock);
isc__nm_incstats(sock, STATID_CONNECT);
r = uv_tcp_getpeername(&sock->uv_handle.tcp, (struct sockaddr *)&ss,
&(int){ sizeof(ss) });
@@ -281,7 +298,6 @@ tcp_connect_cb(uv_connect_t *uvreq, int status) {
isc__nm_connectcb(sock, req, ISC_R_SUCCESS, false);
return;
error:
isc__nm_failed_connect_cb(sock, req, result, false);
}

View File

@@ -198,9 +198,6 @@ tcpdns_connect_cb(uv_connect_t *uvreq, int status) {
REQUIRE(VALID_NMSOCK(sock));
REQUIRE(sock->tid == isc_nm_tid());
isc__nmsocket_timer_stop(sock);
uv_handle_set_data((uv_handle_t *)&sock->read_timer, sock);
req = uv_handle_get_data((uv_handle_t *)uvreq);
REQUIRE(VALID_UVREQ(req));
@@ -209,9 +206,7 @@ tcpdns_connect_cb(uv_connect_t *uvreq, int status) {
if (atomic_load(&sock->timedout)) {
result = ISC_R_TIMEDOUT;
goto error;
}
if (isc__nm_closing(sock)) {
} else if (isc__nm_closing(sock)) {
/* Network manager shutting down */
result = ISC_R_SHUTTINGDOWN;
goto error;
@@ -223,11 +218,33 @@ tcpdns_connect_cb(uv_connect_t *uvreq, int status) {
/* Timeout status code here indicates hard error */
result = ISC_R_TIMEDOUT;
goto error;
} else if (status == UV_EADDRINUSE) {
/*
* On FreeBSD the TCP connect() call sometimes results in a
* spurious transient EADDRINUSE. Try a few more times before
* giving up.
*/
if (--req->connect_tries > 0) {
r = uv_tcp_connect(
&req->uv_req.connect, &sock->uv_handle.tcp,
&req->peer.type.sa, tcpdns_connect_cb);
if (r != 0) {
isc__nm_incstats(sock, STATID_CONNECTFAIL);
result = isc_uverr2result(r);
goto error;
}
return;
}
result = isc_uverr2result(status);
goto error;
} else if (status != 0) {
result = isc_uverr2result(status);
goto error;
}
isc__nmsocket_timer_stop(sock);
uv_handle_set_data((uv_handle_t *)&sock->read_timer, sock);
isc__nm_incstats(sock, STATID_CONNECT);
r = uv_tcp_getpeername(&sock->uv_handle.tcp, (struct sockaddr *)&ss,
&(int){ sizeof(ss) });
@@ -244,7 +261,6 @@ tcpdns_connect_cb(uv_connect_t *uvreq, int status) {
isc__nm_connectcb(sock, req, ISC_R_SUCCESS, false);
return;
error:
isc__nm_failed_connect_cb(sock, req, result, false);
}

View File

@@ -246,9 +246,7 @@ tlsdns_connect_cb(uv_connect_t *uvreq, int status) {
if (atomic_load(&sock->timedout)) {
result = ISC_R_TIMEDOUT;
goto error;
}
if (isc__nm_closing(sock)) {
} else if (isc__nm_closing(sock)) {
/* Network manager shutting down */
result = ISC_R_SHUTTINGDOWN;
goto error;
@@ -260,6 +258,25 @@ tlsdns_connect_cb(uv_connect_t *uvreq, int status) {
/* Timeout status code here indicates hard error */
result = ISC_R_TIMEDOUT;
goto error;
} else if (status == UV_EADDRINUSE) {
/*
* On FreeBSD the TCP connect() call sometimes results in a
* spurious transient EADDRINUSE. Try a few more times before
* giving up.
*/
if (--req->connect_tries > 0) {
r = uv_tcp_connect(
&req->uv_req.connect, &sock->uv_handle.tcp,
&req->peer.type.sa, tlsdns_connect_cb);
if (r != 0) {
isc__nm_incstats(sock, STATID_CONNECTFAIL);
result = isc_uverr2result(r);
goto error;
}
return;
}
result = isc_uverr2result(status);
goto error;
} else if (status != 0) {
result = isc_uverr2result(status);
goto error;

View File

@@ -846,7 +846,6 @@ udp_connect_direct(isc_nmsocket_t *sock, isc__nm_uvreq_t *req) {
isc__networker_t *worker = NULL;
int uv_bind_flags = UV_UDP_REUSEADDR;
isc_result_t result = ISC_R_UNSET;
int tries = 3;
int r;
REQUIRE(isc__nm_in_netthread());
@@ -901,7 +900,7 @@ udp_connect_direct(isc_nmsocket_t *sock, isc__nm_uvreq_t *req) {
do {
r = isc_uv_udp_connect(&sock->uv_handle.udp,
&req->peer.type.sa);
} while (r == UV_EADDRINUSE && --tries > 0);
} while (r == UV_EADDRINUSE && --req->connect_tries > 0);
if (r != 0) {
isc__nm_incstats(sock, STATID_CONNECTFAIL);
goto done;