diff --git a/lib/isc/netmgr/http.c b/lib/isc/netmgr/http.c index f1595eee94..a77216000d 100644 --- a/lib/isc/netmgr/http.c +++ b/lib/isc/netmgr/http.c @@ -122,6 +122,7 @@ struct isc_nm_http_session { size_t nsstreams; isc_nmhandle_t *handle; + isc_nmhandle_t *client_httphandle; isc_nmsocket_t *serversocket; isc_nmiface_t server_iface; @@ -412,11 +413,13 @@ finish_http_session(isc_nm_http_session_t *session) { if (session->closed) { return; } + if (session->handle != NULL) { if (!session->closed) { session->closed = true; isc_nm_cancelread(session->handle); } + if (!ISC_LIST_EMPTY(session->cstreams)) { http_cstream_t *cstream = ISC_LIST_HEAD(session->cstreams); @@ -426,7 +429,8 @@ finish_http_session(isc_nm_http_session_t *session) { ISC_LIST_DEQUEUE(session->cstreams, cstream, link); cstream->read_cb( - session->handle, ISC_R_UNEXPECTED, + session->client_httphandle, + ISC_R_UNEXPECTED, &(isc_region_t){ cstream->rbuf, cstream->rbufsize }, cstream->read_cbarg); @@ -437,6 +441,10 @@ finish_http_session(isc_nm_http_session_t *session) { isc_nmhandle_detach(&session->handle); } + if (session->client_httphandle != NULL) { + isc_nmhandle_detach(&session->client_httphandle); + } + INSIST(ISC_LIST_EMPTY(session->cstreams)); /* detach from server socket */ @@ -867,7 +875,9 @@ http_readcb(isc_nmhandle_t *handle, isc_result_t result, isc_region_t *region, UNUSED(handle); if (result != ISC_R_SUCCESS) { - session->reading = false; + if (result != ISC_R_TIMEDOUT) { + session->reading = false; + } failed_read_cb(result, session); return; } @@ -1047,15 +1057,24 @@ get_http_cstream(isc_nmsocket_t *sock, http_cstream_t **streamp) { } static void -http_call_connect_cb(isc_nmsocket_t *sock, isc_result_t result) { +http_call_connect_cb(isc_nmsocket_t *sock, isc_nm_http_session_t *session, + isc_result_t result) { isc__nm_uvreq_t *req = NULL; + isc_nmhandle_t *httphandle = isc__nmhandle_get(sock, &sock->peer, + &sock->iface->addr); REQUIRE(sock->connect_cb != NULL); req = isc__nm_uvreq_get(sock->mgr, sock); req->cb.connect = sock->connect_cb; req->cbarg = sock->connect_cbarg; - req->handle = isc__nmhandle_get(sock, &sock->peer, &sock->iface->addr); + if (session != NULL) { + session->client_httphandle = httphandle; + req->handle = NULL; + isc_nmhandle_attach(httphandle, &req->handle); + } else { + req->handle = httphandle; + } isc__nmsocket_clearcb(sock); isc__nm_connectcb(sock, req, result, true); @@ -1064,8 +1083,8 @@ http_call_connect_cb(isc_nmsocket_t *sock, isc_result_t result) { static void transport_connect_cb(isc_nmhandle_t *handle, isc_result_t result, void *cbarg) { isc_nmsocket_t *http_sock = (isc_nmsocket_t *)cbarg; - isc_nm_http_session_t *session = NULL; isc_nmsocket_t *transp_sock = NULL; + isc_nm_http_session_t *session = NULL; http_cstream_t *cstream = NULL; isc_mem_t *mctx = NULL; @@ -1132,13 +1151,15 @@ transport_connect_cb(isc_nmhandle_t *handle, isc_result_t result, void *cbarg) { } http_transpost_tcp_nodelay(handle); - http_call_connect_cb(http_sock, result); + + http_call_connect_cb(http_sock, session, result); + http_do_bio(session, NULL, NULL, NULL); isc__nmsocket_detach(&http_sock); return; error: - http_call_connect_cb(http_sock, result); + http_call_connect_cb(http_sock, session, result); if (http_sock->h2.connect.uri != NULL) { isc_mem_free(mctx, http_sock->h2.connect.uri); @@ -2352,20 +2373,35 @@ failed_httpstream_read_cb(isc_nmsocket_t *sock, isc_result_t result, static void failed_read_cb(isc_result_t result, isc_nm_http_session_t *session) { REQUIRE(VALID_HTTP2_SESSION(session)); + REQUIRE(result != ISC_R_SUCCESS); if (session->client) { http_cstream_t *cstream = NULL; cstream = ISC_LIST_HEAD(session->cstreams); while (cstream != NULL) { http_cstream_t *next = ISC_LIST_NEXT(cstream, link); - ISC_LIST_DEQUEUE(session->cstreams, cstream, link); - cstream->read_cb(session->handle, result, + cstream->read_cb(session->client_httphandle, result, &(isc_region_t){ cstream->rbuf, cstream->rbufsize }, cstream->read_cbarg); - put_http_cstream(session->mctx, cstream); + if (result != ISC_R_TIMEDOUT || + !isc__nmsocket_timer_running(session->handle->sock)) + { + ISC_LIST_DEQUEUE(session->cstreams, cstream, + link); + put_http_cstream(session->mctx, cstream); + } cstream = next; } + + /* + * If result was ISC_R_TIMEDOUT and the timer was reset, + * then we still have active streams and should not close + * the session. + */ + if (ISC_LIST_EMPTY(session->cstreams)) { + finish_http_session(session); + } } else { isc_nmsocket_h2_t *h2data = NULL; /* stream socket */ for (h2data = ISC_LIST_HEAD(session->sstreams); h2data != NULL; @@ -2386,9 +2422,12 @@ failed_read_cb(isc_result_t result, isc_nm_http_session_t *session) { h2data = next; } - } - finish_http_session(session); + /* + * All streams are now destroyed; close the session. + */ + finish_http_session(session); + } } static const bool base64url_validation_table[256] = { diff --git a/lib/isc/netmgr/netmgr.c b/lib/isc/netmgr/netmgr.c index c2891f1a63..5052bd00f6 100644 --- a/lib/isc/netmgr/netmgr.c +++ b/lib/isc/netmgr/netmgr.c @@ -2550,7 +2550,7 @@ isc__nm_readcb(isc_nmsocket_t *sock, isc__nm_uvreq_t *uvreq, REQUIRE(VALID_UVREQ(uvreq)); REQUIRE(VALID_NMHANDLE(uvreq->handle)); - if (eresult == ISC_R_SUCCESS) { + if (eresult == ISC_R_SUCCESS || eresult == ISC_R_TIMEDOUT) { isc__netievent_readcb_t ievent = { .sock = sock, .req = uvreq, .result = eresult }; diff --git a/lib/isc/netmgr/tcp.c b/lib/isc/netmgr/tcp.c index d86e1866e5..80ce1193c2 100644 --- a/lib/isc/netmgr/tcp.c +++ b/lib/isc/netmgr/tcp.c @@ -654,10 +654,10 @@ isc__nm_async_tcpstop(isc__networker_t *worker, isc__netievent_t *ev0) { stop_tcp_parent(sock); } -static void -failed_read_cb(isc_nmsocket_t *sock, isc_result_t result) { +void +isc__nm_tcp_failed_read_cb(isc_nmsocket_t *sock, isc_result_t result) { REQUIRE(VALID_NMSOCK(sock)); - REQUIRE(sock->statichandle != NULL); + REQUIRE(result != ISC_R_SUCCESS); isc__nmsocket_timer_stop(sock); isc__nm_stop_reading(sock); @@ -676,18 +676,15 @@ failed_read_cb(isc_nmsocket_t *sock, isc_result_t result) { destroy: isc__nmsocket_prep_destroy(sock); - /* We need to detach from quota after the read callback function had a - * chance to be executed. */ - if (sock->quota) { + /* + * We need to detach from quota after the read callback function had a + * chance to be executed. + */ + if (sock->quota != NULL) { isc_quota_detach(&sock->quota); } } -void -isc__nm_tcp_failed_read_cb(isc_nmsocket_t *sock, isc_result_t result) { - failed_read_cb(sock, result); -} - static void failed_send_cb(isc_nmsocket_t *sock, isc__nm_uvreq_t *req, isc_result_t eresult) { @@ -750,7 +747,7 @@ isc__nm_async_tcpstartread(isc__networker_t *worker, isc__netievent_t *ev0) { if (isc__nmsocket_closing(sock)) { sock->reading = true; - failed_read_cb(sock, ISC_R_CANCELED); + isc__nm_tcp_failed_read_cb(sock, ISC_R_CANCELED); return; } @@ -760,11 +757,12 @@ isc__nm_async_tcpstartread(isc__networker_t *worker, isc__netievent_t *ev0) { void isc__nm_tcp_pauseread(isc_nmhandle_t *handle) { - REQUIRE(VALID_NMHANDLE(handle)); - REQUIRE(VALID_NMSOCK(handle->sock)); - isc__netievent_tcppauseread_t *ievent = NULL; - isc_nmsocket_t *sock = handle->sock; + isc_nmsocket_t *sock = NULL; + + REQUIRE(VALID_NMHANDLE(handle)); + + sock = handle->sock; REQUIRE(VALID_NMSOCK(sock)); @@ -812,7 +810,7 @@ isc__nm_tcp_resumeread(isc_nmhandle_t *handle) { if (!isc__nmsocket_active(sock)) { sock->reading = true; - failed_read_cb(sock, ISC_R_CANCELED); + isc__nm_tcp_failed_read_cb(sock, ISC_R_CANCELED); return; } @@ -838,7 +836,7 @@ isc__nm_tcp_read_cb(uv_stream_t *stream, ssize_t nread, const uv_buf_t *buf) { REQUIRE(buf != NULL); if (isc__nmsocket_closing(sock)) { - failed_read_cb(sock, ISC_R_CANCELED); + isc__nm_tcp_failed_read_cb(sock, ISC_R_CANCELED); goto free; } @@ -848,7 +846,7 @@ isc__nm_tcp_read_cb(uv_stream_t *stream, ssize_t nread, const uv_buf_t *buf) { sock->statsindex[STATID_RECVFAIL]); } - failed_read_cb(sock, isc__nm_uverr2result(nread)); + isc__nm_tcp_failed_read_cb(sock, isc__nm_uverr2result(nread)); goto free; } @@ -1337,7 +1335,7 @@ isc__nm_tcp_shutdown(isc_nmsocket_t *sock) { } if (sock->statichandle != NULL) { - failed_read_cb(sock, ISC_R_CANCELED); + isc__nm_tcp_failed_read_cb(sock, ISC_R_CANCELED); return; } @@ -1377,7 +1375,7 @@ isc__nm_async_tcpcancel(isc__networker_t *worker, isc__netievent_t *ev0) { uv_timer_stop(&sock->timer); - failed_read_cb(sock, ISC_R_EOF); + isc__nm_tcp_failed_read_cb(sock, ISC_R_EOF); } int_fast32_t diff --git a/lib/isc/netmgr/tcpdns.c b/lib/isc/netmgr/tcpdns.c index f8005bb4d9..9d2601b906 100644 --- a/lib/isc/netmgr/tcpdns.c +++ b/lib/isc/netmgr/tcpdns.c @@ -651,9 +651,11 @@ isc__nm_tcpdns_failed_read_cb(isc_nmsocket_t *sock, isc_result_t result) { destroy: isc__nmsocket_prep_destroy(sock); - /* We need to detach from quota after the read callback function had a - * chance to be executed. */ - if (sock->quota) { + /* + * We need to detach from quota after the read callback function had a + * chance to be executed. + */ + if (sock->quota != NULL) { isc_quota_detach(&sock->quota); } } diff --git a/lib/isc/netmgr/tlsdns.c b/lib/isc/netmgr/tlsdns.c index ff83fa4624..bb7ea76a3c 100644 --- a/lib/isc/netmgr/tlsdns.c +++ b/lib/isc/netmgr/tlsdns.c @@ -802,9 +802,11 @@ isc__nm_tlsdns_failed_read_cb(isc_nmsocket_t *sock, isc_result_t result, destroy: isc__nmsocket_prep_destroy(sock); - /* We need to detach from quota after the read callback function - * had a chance to be executed. */ - if (sock->quota) { + /* + * We need to detach from quota after the read callback function + * had a chance to be executed. + */ + if (sock->quota != NULL) { isc_quota_detach(&sock->quota); } } diff --git a/lib/isc/netmgr/tlsstream.c b/lib/isc/netmgr/tlsstream.c index a68838a775..07f3d26427 100644 --- a/lib/isc/netmgr/tlsstream.c +++ b/lib/isc/netmgr/tlsstream.c @@ -168,7 +168,10 @@ tls_senddone(isc_nmhandle_t *handle, isc_result_t eresult, void *cbarg) { static void tls_failed_read_cb(isc_nmsocket_t *sock, const isc_result_t result) { + bool destroy = true; + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(result != ISC_R_SUCCESS); if (!sock->tlsstream.server && (sock->tlsstream.state == TLS_INIT || @@ -188,14 +191,22 @@ tls_failed_read_cb(isc_nmsocket_t *sock, const isc_result_t result) { req = isc__nm_uvreq_get(sock->mgr, sock); req->cb.recv = sock->recv_cb; req->cbarg = sock->recv_cbarg; - req->handle = NULL; isc_nmhandle_attach(sock->statichandle, &req->handle); - isc__nmsocket_clearcb(sock); + if (result != ISC_R_TIMEDOUT) { + isc__nmsocket_clearcb(sock); + } isc__nm_readcb(sock, req, result); + if (result == ISC_R_TIMEDOUT && + isc__nmsocket_timer_running(sock->outerhandle->sock)) + { + destroy = false; + } } - isc__nmsocket_prep_destroy(sock); - isc__nmsocket_detach(&sock); + if (destroy) { + isc__nmsocket_prep_destroy(sock); + isc__nmsocket_detach(&sock); + } } static void @@ -520,6 +531,7 @@ tls_readcb(isc_nmhandle_t *handle, isc_result_t result, isc_region_t *region, tls_failed_read_cb(tlssock, result); return; } + tls_do_bio(tlssock, region, NULL, false); } diff --git a/lib/isc/netmgr/udp.c b/lib/isc/netmgr/udp.c index eb57c954be..ef404d93da 100644 --- a/lib/isc/netmgr/udp.c +++ b/lib/isc/netmgr/udp.c @@ -815,6 +815,7 @@ isc__nm_udp_failed_read_cb(isc_nmsocket_t *sock, isc_result_t result) { REQUIRE(result != ISC_R_SUCCESS); if (atomic_load(&sock->client)) { + isc__nmsocket_timer_stop(sock); isc__nm_stop_reading(sock); if (!sock->recv_read) { diff --git a/lib/isc/tests/doh_test.c b/lib/isc/tests/doh_test.c index d27b1f7b9b..71dedbad6a 100644 --- a/lib/isc/tests/doh_test.c +++ b/lib/isc/tests/doh_test.c @@ -55,19 +55,19 @@ static uint64_t stop_magic = 0; static uv_buf_t send_msg = { .base = (char *)&send_magic, .len = sizeof(send_magic) }; -static atomic_int_fast64_t nsends; - -static atomic_uint_fast64_t ssends; -static atomic_uint_fast64_t sreads; - -static atomic_uint_fast64_t csends; -static atomic_uint_fast64_t creads; +static atomic_int_fast64_t nsends = ATOMIC_VAR_INIT(0); +static atomic_int_fast64_t ssends = ATOMIC_VAR_INIT(0); +static atomic_int_fast64_t sreads = ATOMIC_VAR_INIT(0); +static atomic_int_fast64_t csends = ATOMIC_VAR_INIT(0); +static atomic_int_fast64_t creads = ATOMIC_VAR_INIT(0); +static atomic_int_fast64_t ctimeouts = ATOMIC_VAR_INIT(0); static atomic_bool was_error; static unsigned int workers = 0; static bool reuse_supported = true; +static bool noanswer = false; static atomic_bool POST = ATOMIC_VAR_INIT(true); @@ -77,6 +77,9 @@ static atomic_bool use_TLS = ATOMIC_VAR_INIT(false); static isc_tlsctx_t *server_tlsctx = NULL; static isc_tlsctx_t *client_tlsctx = NULL; +/* Timeout for soft-timeout tests (0.05 seconds) */ +#define T_SOFT 50 + #define NSENDS 100 #define NWRITES 10 @@ -292,12 +295,15 @@ nm_setup(void **state) { atomic_store(&creads, 0); atomic_store(&sreads, 0); atomic_store(&ssends, 0); + atomic_store(&ctimeouts, 0); atomic_store(&was_error, false); atomic_store(&POST, false); atomic_store(&use_TLS, false); + noanswer = false; + isc_nonce_buf(&send_magic, sizeof(send_magic)); isc_nonce_buf(&stop_magic, sizeof(stop_magic)); if (send_magic == stop_magic) { @@ -428,11 +434,16 @@ doh_receive_request_cb(isc_nmhandle_t *handle, isc_result_t eresult, tcp_buffer_length); if (magic == send_magic) { - isc_nm_send(handle, region, doh_reply_sent_cb, NULL); + if (!noanswer) { + isc_nm_send(handle, region, doh_reply_sent_cb, + NULL); + } return; } else if (magic == stop_magic) { - /* We are done, so we don't send anything back */ - /* There should be no more packets in the buffer */ + /* + * We are done, so we don't send anything back. + * There should be no more packets in the buffer. + */ assert_int_equal(tcp_buffer_length, 0); } } @@ -543,6 +554,127 @@ doh_noresponse_GET(void **state) { doh_noresponse(state); } +static void +timeout_query_sent_cb(isc_nmhandle_t *handle, isc_result_t eresult, + void *cbarg) { + UNUSED(eresult); + UNUSED(cbarg); + + assert_non_null(handle); + + if (eresult == ISC_R_SUCCESS) { + atomic_fetch_add(&csends, 1); + } + + isc_nmhandle_detach(&handle); +} + +static void +timeout_retry_cb(isc_nmhandle_t *handle, isc_result_t eresult, + isc_region_t *region, void *arg) { + UNUSED(region); + UNUSED(arg); + + assert_non_null(handle); + + atomic_fetch_add(&ctimeouts, 1); + + if (eresult == ISC_R_TIMEDOUT && atomic_load(&ctimeouts) < 5) { + isc_nmhandle_settimeout(handle, T_SOFT); + return; + } + + isc_nmhandle_detach(&handle); +} + +static void +timeout_request_cb(isc_nmhandle_t *handle, isc_result_t result, void *arg) { + isc_nmhandle_t *sendhandle = NULL; + isc_nmhandle_t *readhandle = NULL; + + REQUIRE(VALID_NMHANDLE(handle)); + + if (result != ISC_R_SUCCESS) { + goto error; + } + + isc_nmhandle_attach(handle, &sendhandle); + isc_nm_send(handle, + &(isc_region_t){ .base = (uint8_t *)send_msg.base, + .length = send_msg.len }, + timeout_query_sent_cb, arg); + + isc_nmhandle_attach(handle, &readhandle); + isc_nm_read(handle, timeout_retry_cb, NULL); + return; + +error: + atomic_store(&was_error, true); +} + +static void +doh_timeout_recovery(void **state) { + isc_nm_t **nm = (isc_nm_t **)*state; + isc_nm_t *listen_nm = nm[0]; + isc_nm_t *connect_nm = nm[1]; + isc_result_t result = ISC_R_SUCCESS; + isc_nmsocket_t *listen_sock = NULL; + isc_tlsctx_t *ctx = atomic_load(&use_TLS) ? server_tlsctx : NULL; + char req_url[256]; + + result = isc_nm_listenhttp(listen_nm, (isc_nmiface_t *)&tcp_listen_addr, + 0, NULL, NULL, &listen_sock); + assert_int_equal(result, ISC_R_SUCCESS); + + /* + * Accept connections but don't send responses, forcing client + * reads to time out. + */ + noanswer = true; + result = isc_nm_http_endpoint(listen_sock, DOH_PATH, + doh_receive_request_cb, NULL, 0); + assert_int_equal(result, ISC_R_SUCCESS); + + /* + * Shorten all the TCP client timeouts to 0.05 seconds. + * timeout_retry_cb() will give up after five timeouts. + */ + isc_nm_settimeouts(connect_nm, T_SOFT, T_SOFT, T_SOFT, T_SOFT); + sockaddr_to_url(&tcp_listen_addr, false, req_url, sizeof(req_url), + DOH_PATH); + isc_nm_httpconnect(connect_nm, NULL, (isc_nmiface_t *)&tcp_listen_addr, + req_url, atomic_load(&POST), timeout_request_cb, + NULL, ctx, T_SOFT, 0); + + /* + * Sleep until sends reaches 5. + */ + for (size_t i = 0; i < 1000; i++) { + if (atomic_load(&ctimeouts) == 5) { + break; + } + usleep(1000); + } + assert_true(atomic_load(&ctimeouts) == 5); + + isc_nm_stoplistening(listen_sock); + isc_nmsocket_close(&listen_sock); + assert_null(listen_sock); + isc_nm_closedown(connect_nm); +} + +static void +doh_timeout_recovery_POST(void **state) { + atomic_store(&POST, true); + doh_timeout_recovery(state); +} + +static void +doh_timeout_recovery_GET(void **state) { + atomic_store(&POST, false); + doh_timeout_recovery(state); +} + static void doh_receive_send_reply_cb(isc_nmhandle_t *handle, isc_result_t eresult, isc_region_t *region, void *cbarg) { @@ -1636,6 +1768,10 @@ main(void) { nm_teardown), cmocka_unit_test_setup_teardown(doh_noresponse_GET, nm_setup, nm_teardown), + cmocka_unit_test_setup_teardown(doh_timeout_recovery_POST, + nm_setup, nm_teardown), + cmocka_unit_test_setup_teardown(doh_timeout_recovery_GET, + nm_setup, nm_teardown), cmocka_unit_test_setup_teardown(doh_recv_one_POST, nm_setup, nm_teardown), cmocka_unit_test_setup_teardown(doh_recv_one_GET, nm_setup, diff --git a/lib/isc/tests/netmgr_test.c b/lib/isc/tests/netmgr_test.c index 805f7a6a58..8f1ae9ece7 100644 --- a/lib/isc/tests/netmgr_test.c +++ b/lib/isc/tests/netmgr_test.c @@ -41,6 +41,12 @@ typedef void (*stream_connect_function)(isc_nm_t *nm); +static void +connect_connect_cb(isc_nmhandle_t *handle, isc_result_t eresult, void *cbarg); +static void +connect_read_cb(isc_nmhandle_t *handle, isc_result_t eresult, + isc_region_t *region, void *cbarg); + isc_nm_t *listen_nm = NULL; isc_nm_t *connect_nm = NULL; @@ -67,13 +73,14 @@ static unsigned int workers = 0; static atomic_int_fast64_t nsends; static int_fast64_t esends; /* expected sends */ -static atomic_int_fast64_t ssends; -static atomic_int_fast64_t sreads; -static atomic_int_fast64_t saccepts; +static atomic_int_fast64_t ssends = ATOMIC_VAR_INIT(0); +static atomic_int_fast64_t sreads = ATOMIC_VAR_INIT(0); +static atomic_int_fast64_t saccepts = ATOMIC_VAR_INIT(0); -static atomic_int_fast64_t cconnects; -static atomic_int_fast64_t csends; -static atomic_int_fast64_t creads; +static atomic_int_fast64_t cconnects = ATOMIC_VAR_INIT(0); +static atomic_int_fast64_t csends = ATOMIC_VAR_INIT(0); +static atomic_int_fast64_t creads = ATOMIC_VAR_INIT(0); +static atomic_int_fast64_t ctimeouts = ATOMIC_VAR_INIT(0); static isc_refcount_t active_cconnects; static isc_refcount_t active_csends; @@ -87,8 +94,11 @@ static atomic_bool check_listener_quota; static bool skip_long_tests = false; static bool allow_send_back = false; +static bool noanswer = false; static bool stream_use_TLS = false; +static isc_nm_recv_cb_t connect_readcb = NULL; + #define SKIP_IN_CI \ if (skip_long_tests) { \ skip(); \ @@ -97,6 +107,9 @@ static bool stream_use_TLS = false; #define NSENDS 100 +/* Timeout for soft-timeout tests (0.05 seconds) */ +#define T_SOFT 50 + /* Timeouts in miliseconds */ #define T_INIT 120 * 1000 #define T_IDLE 120 * 1000 @@ -306,6 +319,7 @@ nm_setup(void **state __attribute__((unused))) { atomic_store(&cconnects, 0); atomic_store(&csends, 0); atomic_store(&creads, 0); + atomic_store(&ctimeouts, 0); allow_send_back = false; stream_use_TLS = false; @@ -334,6 +348,9 @@ nm_setup(void **state __attribute__((unused))) { isc_quota_init(&listener_quota, 0); atomic_store(&check_listener_quota, false); + connect_readcb = connect_read_cb; + noanswer = false; + return (0); } @@ -382,15 +399,6 @@ noop_accept_cb(isc_nmhandle_t *handle, unsigned int result, void *cbarg) { return (0); } -static void -noop_connect_cb(isc_nmhandle_t *handle, isc_result_t result, void *cbarg) { - UNUSED(handle); - UNUSED(result); - UNUSED(cbarg); - - isc_refcount_decrement(&active_cconnects); -} - static void connect_send_cb(isc_nmhandle_t *handle, isc_result_t eresult, void *cbarg); @@ -448,7 +456,7 @@ connect_read_cb(isc_nmhandle_t *handle, isc_result_t eresult, goto unref; } - assert_int_equal(region->length, sizeof(magic)); + assert_true(region->length >= sizeof(magic)); atomic_fetch_add(&creads, 1); @@ -462,7 +470,7 @@ connect_read_cb(isc_nmhandle_t *handle, isc_result_t eresult, } unref: - atomic_fetch_sub(&active_creads, 1); + isc_refcount_decrement(&active_creads); isc_nmhandle_detach(&handle); } @@ -476,7 +484,7 @@ connect_connect_cb(isc_nmhandle_t *handle, isc_result_t eresult, void *cbarg) { isc_refcount_decrement(&active_cconnects); - if (eresult != ISC_R_SUCCESS) { + if (eresult != ISC_R_SUCCESS || connect_readcb == NULL) { return; } @@ -484,7 +492,7 @@ connect_connect_cb(isc_nmhandle_t *handle, isc_result_t eresult, void *cbarg) { isc_refcount_increment0(&active_creads); isc_nmhandle_attach(handle, &readhandle); - isc_nm_read(handle, connect_read_cb, NULL); + isc_nm_read(handle, connect_readcb, NULL); connect_send(handle); } @@ -525,20 +533,22 @@ listen_read_cb(isc_nmhandle_t *handle, isc_result_t eresult, atomic_fetch_add(&sreads, 1); - assert_int_equal(region->length, sizeof(magic)); + assert_true(region->length >= sizeof(magic)); memmove(&magic, region->base, sizeof(magic)); assert_true(magic == stop_magic || magic == send_magic); if (magic == send_magic) { - isc_nmhandle_t *sendhandle = NULL; - isc_nmhandle_attach(handle, &sendhandle); - isc_refcount_increment0(&active_ssends); - isc_nm_send(sendhandle, (isc_region_t *)&send_msg, - listen_send_cb, cbarg); + if (!noanswer) { + isc_nmhandle_t *sendhandle = NULL; + isc_nmhandle_attach(handle, &sendhandle); + isc_refcount_increment0(&active_ssends); + isc_nm_send(sendhandle, (isc_region_t *)&send_msg, + listen_send_cb, cbarg); + } return; } - /* close the connection on stop_magic */ + unref: if (handle == cbarg) { isc_refcount_decrement(&active_sreads); @@ -662,9 +672,10 @@ static void mock_udpconnect_uv_udp_open(void **state __attribute__((unused))) { WILL_RETURN(uv_udp_open, UV_ENOMEM); + connect_readcb = NULL; isc_refcount_increment0(&active_cconnects); isc_nm_udpconnect(connect_nm, (isc_nmiface_t *)&udp_connect_addr, - (isc_nmiface_t *)&udp_listen_addr, noop_connect_cb, + (isc_nmiface_t *)&udp_listen_addr, connect_connect_cb, NULL, T_CONNECT, 0); isc_nm_closedown(connect_nm); @@ -675,9 +686,10 @@ static void mock_udpconnect_uv_udp_bind(void **state __attribute__((unused))) { WILL_RETURN(uv_udp_bind, UV_ENOMEM); + connect_readcb = NULL; isc_refcount_increment0(&active_cconnects); isc_nm_udpconnect(connect_nm, (isc_nmiface_t *)&udp_connect_addr, - (isc_nmiface_t *)&udp_listen_addr, noop_connect_cb, + (isc_nmiface_t *)&udp_listen_addr, connect_connect_cb, NULL, T_CONNECT, 0); isc_nm_closedown(connect_nm); @@ -689,9 +701,10 @@ static void mock_udpconnect_uv_udp_connect(void **state __attribute__((unused))) { WILL_RETURN(uv_udp_connect, UV_ENOMEM); + connect_readcb = NULL; isc_refcount_increment0(&active_cconnects); isc_nm_udpconnect(connect_nm, (isc_nmiface_t *)&udp_connect_addr, - (isc_nmiface_t *)&udp_listen_addr, noop_connect_cb, + (isc_nmiface_t *)&udp_listen_addr, connect_connect_cb, NULL, T_CONNECT, 0); isc_nm_closedown(connect_nm); @@ -703,9 +716,10 @@ static void mock_udpconnect_uv_recv_buffer_size(void **state __attribute__((unused))) { WILL_RETURN(uv_recv_buffer_size, UV_ENOMEM); + connect_readcb = NULL; isc_refcount_increment0(&active_cconnects); isc_nm_udpconnect(connect_nm, (isc_nmiface_t *)&udp_connect_addr, - (isc_nmiface_t *)&udp_listen_addr, noop_connect_cb, + (isc_nmiface_t *)&udp_listen_addr, connect_connect_cb, NULL, T_CONNECT, 0); isc_nm_closedown(connect_nm); @@ -716,9 +730,10 @@ static void mock_udpconnect_uv_send_buffer_size(void **state __attribute__((unused))) { WILL_RETURN(uv_send_buffer_size, UV_ENOMEM); + connect_readcb = NULL; isc_refcount_increment0(&active_cconnects); isc_nm_udpconnect(connect_nm, (isc_nmiface_t *)&udp_connect_addr, - (isc_nmiface_t *)&udp_listen_addr, noop_connect_cb, + (isc_nmiface_t *)&udp_listen_addr, connect_connect_cb, NULL, T_CONNECT, 0); isc_nm_closedown(connect_nm); @@ -738,9 +753,10 @@ udp_noop(void **state __attribute__((unused))) { isc_nmsocket_close(&listen_sock); assert_null(listen_sock); + connect_readcb = NULL; isc_refcount_increment0(&active_cconnects); isc_nm_udpconnect(connect_nm, (isc_nmiface_t *)&udp_connect_addr, - (isc_nmiface_t *)&udp_listen_addr, noop_connect_cb, + (isc_nmiface_t *)&udp_listen_addr, connect_connect_cb, NULL, T_CONNECT, 0); isc_nm_closedown(connect_nm); @@ -786,6 +802,67 @@ udp_noresponse(void **state __attribute__((unused))) { atomic_assert_int_eq(ssends, 0); } +static void +timeout_retry_cb(isc_nmhandle_t *handle, isc_result_t eresult, + isc_region_t *region, void *cbarg) { + UNUSED(region); + UNUSED(cbarg); + + assert_non_null(handle); + + F(); + + if (eresult == ISC_R_TIMEDOUT && atomic_load(&csends) < 5) { + isc_nmhandle_settimeout(handle, T_SOFT); + connect_send(handle); + return; + } + + atomic_fetch_add(&ctimeouts, 1); + + isc_refcount_decrement(&active_creads); + isc_nmhandle_detach(&handle); +} + +static void +udp_timeout_recovery(void **state __attribute__((unused))) { + isc_result_t result = ISC_R_SUCCESS; + isc_nmsocket_t *listen_sock = NULL; + + SKIP_IN_CI; + + /* + * Listen using the noop callback so that client reads will time out. + */ + result = isc_nm_listenudp(listen_nm, (isc_nmiface_t *)&udp_listen_addr, + noop_recv_cb, NULL, 0, &listen_sock); + assert_int_equal(result, ISC_R_SUCCESS); + + /* + * Connect with client timeout set to 0.05 seconds, then sleep for at + * least a second for each 'tick'. timeout_retry_cb() will give up + * after five timeouts. + */ + connect_readcb = timeout_retry_cb; + isc_refcount_increment0(&active_cconnects); + isc_nm_udpconnect(connect_nm, (isc_nmiface_t *)&udp_connect_addr, + (isc_nmiface_t *)&udp_listen_addr, connect_connect_cb, + NULL, T_SOFT, 0); + + WAIT_FOR_EQ(cconnects, 1); + WAIT_FOR_GE(csends, 1); + WAIT_FOR_GE(csends, 2); + WAIT_FOR_GE(csends, 3); + WAIT_FOR_GE(csends, 4); + WAIT_FOR_EQ(csends, 5); + WAIT_FOR_EQ(ctimeouts, 1); + + isc_nm_stoplistening(listen_sock); + isc_nmsocket_close(&listen_sock); + assert_null(listen_sock); + isc_nm_closedown(connect_nm); +} + static void udp_recv_one(void **state __attribute__((unused))) { isc_result_t result = ISC_R_SUCCESS; @@ -1118,12 +1195,12 @@ stream_connect(isc_nm_cb_t cb, void *cbarg, unsigned int timeout, (isc_nmiface_t *)&tcp_connect_addr, (isc_nmiface_t *)&tcp_listen_addr, cb, cbarg, tcp_connect_tlsctx, timeout, extrahandlesize); - return; + } else { + isc_nm_tcpconnect(connect_nm, + (isc_nmiface_t *)&tcp_connect_addr, + (isc_nmiface_t *)&tcp_listen_addr, cb, cbarg, + timeout, extrahandlesize); } - - isc_nm_tcpconnect(connect_nm, (isc_nmiface_t *)&tcp_connect_addr, - (isc_nmiface_t *)&tcp_listen_addr, cb, cbarg, timeout, - extrahandlesize); } static void @@ -1138,8 +1215,9 @@ stream_noop(void **state __attribute__((unused))) { isc_nmsocket_close(&listen_sock); assert_null(listen_sock); + connect_readcb = NULL; isc_refcount_increment0(&active_cconnects); - stream_connect(noop_connect_cb, NULL, T_CONNECT, 0); + stream_connect(connect_connect_cb, NULL, T_CONNECT, 0); isc_nm_closedown(connect_nm); atomic_assert_int_eq(cconnects, 0); @@ -1181,6 +1259,44 @@ stream_noresponse(void **state __attribute__((unused))) { atomic_assert_int_eq(ssends, 0); } +static void +stream_timeout_recovery(void **state __attribute__((unused))) { + isc_result_t result = ISC_R_SUCCESS; + isc_nmsocket_t *listen_sock = NULL; + + SKIP_IN_CI; + + /* + * Accept connections but don't send responses, forcing client + * reads to time out. + */ + noanswer = true; + result = stream_listen(stream_accept_cb, NULL, 0, 0, NULL, + &listen_sock); + assert_int_equal(result, ISC_R_SUCCESS); + + /* + * Shorten all the client timeouts to 0.05 seconds. + */ + isc_nm_settimeouts(connect_nm, T_SOFT, T_SOFT, T_SOFT, T_SOFT); + connect_readcb = timeout_retry_cb; + isc_refcount_increment0(&active_cconnects); + stream_connect(connect_connect_cb, NULL, T_SOFT, 0); + + WAIT_FOR_EQ(cconnects, 1); + WAIT_FOR_GE(csends, 1); + WAIT_FOR_GE(csends, 2); + WAIT_FOR_GE(csends, 3); + WAIT_FOR_GE(csends, 4); + WAIT_FOR_EQ(csends, 5); + WAIT_FOR_EQ(ctimeouts, 1); + + isc_nm_stoplistening(listen_sock); + isc_nmsocket_close(&listen_sock); + assert_null(listen_sock); + isc_nm_closedown(connect_nm); +} + static void stream_recv_one(void **state __attribute__((unused))) { isc_result_t result = ISC_R_SUCCESS; @@ -1486,6 +1602,11 @@ tcp_noresponse(void **state) { stream_noresponse(state); } +static void +tcp_timeout_recovery(void **state) { + stream_timeout_recovery(state); +} + static void tcp_recv_one(void **state) { stream_recv_one(state); @@ -1641,10 +1762,11 @@ tcpdns_noop(void **state __attribute__((unused))) { isc_nmsocket_close(&listen_sock); assert_null(listen_sock); + connect_readcb = NULL; isc_refcount_increment0(&active_cconnects); isc_nm_tcpdnsconnect(connect_nm, (isc_nmiface_t *)&tcp_connect_addr, - (isc_nmiface_t *)&tcp_listen_addr, noop_connect_cb, - NULL, T_CONNECT, 0); + (isc_nmiface_t *)&tcp_listen_addr, + connect_connect_cb, NULL, T_CONNECT, 0); isc_nm_closedown(connect_nm); atomic_assert_int_eq(cconnects, 0); @@ -1694,6 +1816,49 @@ tcpdns_noresponse(void **state __attribute__((unused))) { atomic_assert_int_eq(ssends, 0); } +static void +tcpdns_timeout_recovery(void **state __attribute__((unused))) { + isc_result_t result = ISC_R_SUCCESS; + isc_nmsocket_t *listen_sock = NULL; + + SKIP_IN_CI; + + /* + * Accept connections but don't send responses, forcing client + * reads to time out. + */ + noanswer = true; + result = isc_nm_listentcpdns( + listen_nm, (isc_nmiface_t *)&tcp_listen_addr, listen_read_cb, + NULL, listen_accept_cb, NULL, 0, 0, NULL, &listen_sock); + assert_int_equal(result, ISC_R_SUCCESS); + + /* + * Shorten all the TCP client timeouts to 0.05 seconds, connect, + * then sleep for at least a second for each 'tick'. + * timeout_retry_cb() will give up after five timeouts. + */ + connect_readcb = timeout_retry_cb; + isc_nm_settimeouts(connect_nm, T_SOFT, T_SOFT, T_SOFT, T_SOFT); + isc_refcount_increment0(&active_cconnects); + isc_nm_tcpdnsconnect(connect_nm, (isc_nmiface_t *)&tcp_connect_addr, + (isc_nmiface_t *)&tcp_listen_addr, + connect_connect_cb, NULL, T_SOFT, 0); + + WAIT_FOR_EQ(cconnects, 1); + WAIT_FOR_GE(csends, 1); + WAIT_FOR_GE(csends, 2); + WAIT_FOR_GE(csends, 3); + WAIT_FOR_GE(csends, 4); + WAIT_FOR_EQ(csends, 5); + WAIT_FOR_EQ(ctimeouts, 1); + + isc_nm_stoplistening(listen_sock); + isc_nmsocket_close(&listen_sock); + assert_null(listen_sock); + isc_nm_closedown(connect_nm); +} + static void tcpdns_recv_one(void **state __attribute__((unused))) { isc_result_t result = ISC_R_SUCCESS; @@ -1996,6 +2161,12 @@ tls_noresponse(void **state) { stream_noresponse(state); } +static void +tls_timeout_recovery(void **state) { + stream_use_TLS = true; + stream_timeout_recovery(state); +} + static void tls_recv_one(void **state) { stream_use_TLS = true; @@ -2175,10 +2346,12 @@ tlsdns_noop(void **state __attribute__((unused))) { isc_nmsocket_close(&listen_sock); assert_null(listen_sock); + connect_readcb = NULL; isc_refcount_increment0(&active_cconnects); isc_nm_tlsdnsconnect(connect_nm, (isc_nmiface_t *)&tcp_connect_addr, - (isc_nmiface_t *)&tcp_listen_addr, noop_connect_cb, - NULL, T_CONNECT, 0, tcp_connect_tlsctx); + (isc_nmiface_t *)&tcp_listen_addr, + connect_connect_cb, NULL, T_CONNECT, 0, + tcp_connect_tlsctx); isc_nm_closedown(connect_nm); @@ -2231,6 +2404,55 @@ tlsdns_noresponse(void **state __attribute__((unused))) { atomic_assert_int_eq(ssends, 0); } +static void +tlsdns_timeout_recovery(void **state __attribute__((unused))) { + isc_result_t result = ISC_R_SUCCESS; + isc_nmsocket_t *listen_sock = NULL; + isc_sockaddr_t connect_addr; + + SKIP_IN_CI; + + connect_addr = (isc_sockaddr_t){ .length = 0 }; + isc_sockaddr_fromin6(&connect_addr, &in6addr_loopback, 0); + + /* + * Accept connections but don't send responses, forcing client + * reads to time out. + */ + noanswer = true; + result = isc_nm_listentlsdns( + listen_nm, (isc_nmiface_t *)&tcp_listen_addr, listen_read_cb, + NULL, listen_accept_cb, NULL, 0, 0, NULL, tcp_listen_tlsctx, + &listen_sock); + assert_int_equal(result, ISC_R_SUCCESS); + + /* + * Shorten all the TCP client timeouts to 0.05 seconds, connect, + * then sleep for at least a second for each 'tick'. + * timeout_retry_cb() will give up after five timeouts. + */ + connect_readcb = timeout_retry_cb; + isc_nm_settimeouts(connect_nm, T_SOFT, T_SOFT, T_SOFT, T_SOFT); + isc_refcount_increment0(&active_cconnects); + isc_nm_tlsdnsconnect(connect_nm, (isc_nmiface_t *)&tcp_connect_addr, + (isc_nmiface_t *)&tcp_listen_addr, + connect_connect_cb, NULL, T_SOFT, 0, + tcp_connect_tlsctx); + + WAIT_FOR_EQ(cconnects, 1); + WAIT_FOR_GE(csends, 1); + WAIT_FOR_GE(csends, 2); + WAIT_FOR_GE(csends, 3); + WAIT_FOR_GE(csends, 4); + WAIT_FOR_EQ(csends, 5); + WAIT_FOR_EQ(ctimeouts, 1); + + isc_nm_stoplistening(listen_sock); + isc_nmsocket_close(&listen_sock); + assert_null(listen_sock); + isc_nm_closedown(connect_nm); +} + static void tlsdns_recv_one(void **state __attribute__((unused))) { isc_result_t result = ISC_R_SUCCESS; @@ -2550,6 +2772,8 @@ main(void) { nm_teardown), cmocka_unit_test_setup_teardown(udp_noresponse, nm_setup, nm_teardown), + cmocka_unit_test_setup_teardown(udp_timeout_recovery, nm_setup, + nm_teardown), cmocka_unit_test_setup_teardown(udp_recv_one, nm_setup, nm_teardown), cmocka_unit_test_setup_teardown(udp_recv_two, nm_setup, @@ -2564,26 +2788,12 @@ main(void) { nm_setup, nm_teardown), /* TCP */ - /* cmocka_unit_test_setup_teardown(mock_listentcp_uv_tcp_bind, - */ - /* nm_setup, nm_teardown), */ - /* cmocka_unit_test_setup_teardown(mock_listentcp_uv_fileno, */ - /* nm_setup, nm_teardown), */ - /* cmocka_unit_test_setup_teardown( */ - /* mock_listentcp_uv_tcp_getsockname, nm_setup, */ - /* nm_teardown), */ - /* cmocka_unit_test_setup_teardown(mock_listentcp_uv_listen, */ - /* nm_setup, nm_teardown), */ - /* cmocka_unit_test_setup_teardown(mock_tcpconnect_uv_tcp_bind, - */ - /* nm_setup, nm_teardown), */ - /* cmocka_unit_test_setup_teardown(mock_tcpconnect_uv_tcp_connect, - */ - /* nm_setup, nm_teardown), */ cmocka_unit_test_setup_teardown(tcp_noop, nm_setup, nm_teardown), cmocka_unit_test_setup_teardown(tcp_noresponse, nm_setup, nm_teardown), + cmocka_unit_test_setup_teardown(tcp_timeout_recovery, nm_setup, + nm_teardown), cmocka_unit_test_setup_teardown(tcp_recv_one, nm_setup, nm_teardown), cmocka_unit_test_setup_teardown(tcp_recv_two, nm_setup, @@ -2640,6 +2850,8 @@ main(void) { nm_teardown), cmocka_unit_test_setup_teardown(tcpdns_noresponse, nm_setup, nm_teardown), + cmocka_unit_test_setup_teardown(tcpdns_timeout_recovery, + nm_setup, nm_teardown), cmocka_unit_test_setup_teardown(tcpdns_recv_send, nm_setup, nm_teardown), cmocka_unit_test_setup_teardown(tcpdns_recv_half_send, nm_setup, @@ -2654,6 +2866,8 @@ main(void) { nm_teardown), cmocka_unit_test_setup_teardown(tls_noresponse, nm_setup, nm_teardown), + cmocka_unit_test_setup_teardown(tls_timeout_recovery, nm_setup, + nm_teardown), cmocka_unit_test_setup_teardown(tls_recv_one, nm_setup, nm_teardown), cmocka_unit_test_setup_teardown(tls_recv_two, nm_setup, @@ -2710,6 +2924,8 @@ main(void) { nm_teardown), cmocka_unit_test_setup_teardown(tlsdns_noresponse, nm_setup, nm_teardown), + cmocka_unit_test_setup_teardown(tlsdns_timeout_recovery, + nm_setup, nm_teardown), cmocka_unit_test_setup_teardown(tlsdns_recv_send, nm_setup, nm_teardown), cmocka_unit_test_setup_teardown(tlsdns_recv_half_send, nm_setup,