From 9846f395ad79bb50a5fa5ca6ab97ef904b3be35a Mon Sep 17 00:00:00 2001 From: Artem Boldariev Date: Thu, 4 Jul 2024 14:58:10 +0300 Subject: [PATCH 1/5] DoH: process data chunk by chunk instead of all at once Initially, our DNS-over-HTTP(S) implementation would try to process as much incoming data from the network as possible. However, that might be undesirable as we might create too many streams (each effectively backed by a ns_client_t object). That is too forgiving as it might overwhelm the server and trash its memory allocator, causing high CPU and memory usage. Instead of doing that, we resort to processing incoming data using a chunk-by-chunk processing strategy. That is, we split data into small chunks (currently 256 bytes) and process each of them asynchronously. However, we can process more than one chunk at once (up to 4 currently), given that the number of HTTP/2 streams has not increased while processing a chunk. That alone is not enough, though. In addition to the above, we should limit the number of active streams: these streams for which we have received a request and started processing it (the ones for which a read callback was called), as it is perfectly fine to have more opened streams than active ones. In the case we have reached or surpassed the limit of active streams, we stop reading AND processing the data from the remote peer. The number of active streams is effectively decreased only when responses associated with the active streams are sent to the remote peer. Overall, this strategy is very similar to the one used for other stream-based DNS transports like TCP and TLS. --- lib/isc/netmgr/http.c | 325 ++++++++++++++++++++++++++++++++---- lib/isc/netmgr/netmgr-int.h | 1 + 2 files changed, 290 insertions(+), 36 deletions(-) diff --git a/lib/isc/netmgr/http.c b/lib/isc/netmgr/http.c index 9fd1097f1f..7bf8dbd541 100644 --- a/lib/isc/netmgr/http.c +++ b/lib/isc/netmgr/http.c @@ -85,6 +85,21 @@ #define INITIAL_DNS_MESSAGE_BUFFER_SIZE (512) +/* + * The value should be small enough to not allow a server to open too + * many streams at once. It should not be too small either because + * the incoming data will be split into too many chunks with each of + * them processed asynchronously. + */ +#define INCOMING_DATA_CHUNK_SIZE (256) + +/* + * Often processing a chunk does not change the number of streams. In + * that case we can process more than once, but we still should have a + * hard limit on that. + */ +#define INCOMING_DATA_MAX_CHUNKS_AT_ONCE (4) + typedef struct isc_nm_http_response_status { size_t code; size_t content_length; @@ -155,6 +170,15 @@ struct isc_nm_http_session { isc__nm_http_pending_callbacks_t pending_write_callbacks; isc_buffer_t *pending_write_data; + + /* + * The statistical values below are for usage on server-side + * only. They are meant to detect clients that are taking too many + * resources from the server. + */ + uint64_t received; /* How many requests have been received. */ + uint64_t submitted; /* How many responses were submitted to send */ + uint64_t processed; /* How many responses were processed. */ }; typedef enum isc_http_error_responses { @@ -177,6 +201,7 @@ typedef struct isc_http_send_req { void *cbarg; isc_buffer_t *pending_write_data; isc__nm_http_pending_callbacks_t pending_write_callbacks; + uint64_t submitted; } isc_http_send_req_t; #define HTTP_ENDPOINTS_MAGIC ISC_MAGIC('H', 'T', 'E', 'P') @@ -189,10 +214,20 @@ static bool http_send_outgoing(isc_nm_http_session_t *session, isc_nmhandle_t *httphandle, isc_nm_cb_t cb, void *cbarg); +static ssize_t +http_process_input_data(isc_nm_http_session_t *session, + isc_buffer_t *input_data); + +static inline bool +http_too_many_active_streams(isc_nm_http_session_t *session); + static void http_do_bio(isc_nm_http_session_t *session, isc_nmhandle_t *send_httphandle, isc_nm_cb_t send_cb, void *send_cbarg); +static void +http_do_bio_async(isc_nm_http_session_t *session); + static void failed_httpstream_read_cb(isc_nmsocket_t *sock, isc_result_t result, isc_nm_http_session_t *session); @@ -502,6 +537,15 @@ finish_http_session(isc_nm_http_session_t *session) { isc_nmhandle_close(session->handle); } + /* + * Free any unprocessed incoming data in order to not process + * it during indirect calls to http_do_bio() that might happen + * when calling the failed callbacks. + */ + if (session->buf != NULL) { + isc_buffer_free(&session->buf); + } + if (session->client) { client_call_failed_read_cb(ISC_R_UNEXPECTED, session); } else { @@ -664,6 +708,9 @@ on_server_stream_close_callback(int32_t stream_id, ISC_LIST_UNLINK(session->sstreams, sock->h2, link); session->nsstreams--; + if (sock->h2->request_received) { + session->submitted++; + } /* * By making a call to isc__nmsocket_prep_destroy(), we ensure that @@ -975,6 +1022,103 @@ client_submit_request(isc_nm_http_session_t *session, http_cstream_t *stream) { return ISC_R_SUCCESS; } +static ssize_t +http_process_input_data(isc_nm_http_session_t *session, + isc_buffer_t *input_data) { + ssize_t readlen = 0; + ssize_t processed = 0; + isc_region_t chunk = { 0 }; + size_t before, after; + size_t i; + + REQUIRE(VALID_HTTP2_SESSION(session)); + REQUIRE(input_data != NULL); + + if (!http_session_active(session)) { + return 0; + } + + /* + * For clients that initiate request themselves just process + * everything. + */ + if (session->client) { + isc_buffer_remainingregion(input_data, &chunk); + if (chunk.length == 0) { + return 0; + } + + readlen = nghttp2_session_mem_recv(session->ngsession, + chunk.base, chunk.length); + + if (readlen >= 0) { + isc_buffer_forward(input_data, readlen); + } + + return readlen; + } + + /* + * If no streams are created during processing, we might process + * more than one chunk at a time. Still we should not overdo that + * to avoid processing too much data at once as such behaviour is + * known for trashing the memory allocator at times. + */ + for (before = after = session->nsstreams, i = 0; + after <= before && i < INCOMING_DATA_MAX_CHUNKS_AT_ONCE; + after = session->nsstreams, i++) + { + const uint64_t active_streams = + (session->received - session->processed); + + /* + * If there are non completed send requests in flight -let's + * not process any incoming data, as it could lead to piling + * up too much send data in send buffers. With many clients + * connected it can lead to excessive memory consumption on + * the server instance. + */ + if (session->sending > 0) { + break; + } + + /* + * If we have reached the maximum number of streams used, we + * might stop processing for now, as nghttp2 will happily + * consume as much data as possible. + */ + if (session->nsstreams >= session->max_concurrent_streams && + active_streams > 0) + { + break; + } + + if (http_too_many_active_streams(session)) { + break; + } + + isc_buffer_remainingregion(input_data, &chunk); + if (chunk.length == 0) { + break; + } + + chunk.length = ISC_MIN(chunk.length, INCOMING_DATA_CHUNK_SIZE); + + readlen = nghttp2_session_mem_recv(session->ngsession, + chunk.base, chunk.length); + + if (readlen >= 0) { + isc_buffer_forward(input_data, readlen); + processed += readlen; + } else { + isc_buffer_clear(input_data); + return readlen; + } + } + + return processed; +} + /* * Read callback from TLS socket. */ @@ -984,6 +1128,7 @@ http_readcb(isc_nmhandle_t *handle ISC_ATTR_UNUSED, isc_result_t result, isc_nm_http_session_t *session = (isc_nm_http_session_t *)data; isc_nm_http_session_t *tmpsess = NULL; ssize_t readlen; + isc_buffer_t input; REQUIRE(VALID_HTTP2_SESSION(session)); @@ -1001,8 +1146,10 @@ http_readcb(isc_nmhandle_t *handle ISC_ATTR_UNUSED, isc_result_t result, goto done; } - readlen = nghttp2_session_mem_recv(session->ngsession, region->base, - region->length); + isc_buffer_init(&input, region->base, region->length); + isc_buffer_add(&input, region->length); + + readlen = http_process_input_data(session, &input); if (readlen < 0) { failed_read_cb(ISC_R_UNEXPECTED, session); goto done; @@ -1017,11 +1164,12 @@ http_readcb(isc_nmhandle_t *handle ISC_ATTR_UNUSED, isc_result_t result, isc_buffer_putmem(session->buf, region->base + readlen, unread_size); isc_nm_read_stop(session->handle); + http_do_bio_async(session); + } else { + /* We might have something to receive or send, do IO */ + http_do_bio(session, NULL, NULL, NULL); } - /* We might have something to receive or send, do IO */ - http_do_bio(session, NULL, NULL, NULL); - done: isc__nm_httpsession_detach(&tmpsess); } @@ -1059,14 +1207,18 @@ http_writecb(isc_nmhandle_t *handle, isc_result_t result, void *arg) { } isc_buffer_free(&req->pending_write_data); + session->processed += req->submitted; isc_mem_put(session->mctx, req, sizeof(*req)); session->sending--; - http_do_bio(session, NULL, NULL, NULL); - isc_nmhandle_detach(&transphandle); - if (result != ISC_R_SUCCESS && session->sending == 0) { + + if (result == ISC_R_SUCCESS) { + http_do_bio(session, NULL, NULL, NULL); + } else { finish_http_session(session); } + isc_nmhandle_detach(&transphandle); + isc__nm_httpsession_detach(&session); } @@ -1227,7 +1379,9 @@ http_send_outgoing(isc_nm_http_session_t *session, isc_nmhandle_t *httphandle, *send = (isc_http_send_req_t){ .pending_write_data = session->pending_write_data, .cb = cb, - .cbarg = cbarg }; + .cbarg = cbarg, + .submitted = session->submitted }; + session->submitted = 0; session->pending_write_data = NULL; move_pending_send_callbacks(session, send); @@ -1249,6 +1403,28 @@ nothing_to_send: return false; } +static inline bool +http_too_many_active_streams(isc_nm_http_session_t *session) { + const uint64_t active_streams = session->received - session->processed; + const uint64_t max_active_streams = + ISC_MIN(ISC_NETMGR_MAX_STREAM_CLIENTS_PER_CONN, + session->max_concurrent_streams); + + if (session->client) { + return false; + } + + /* + * Do not process incoming data if there are too many active DNS + * clients (streams) per connection. + */ + if (active_streams >= max_active_streams) { + return true; + } + + return false; +} + static void http_do_bio(isc_nm_http_session_t *session, isc_nmhandle_t *send_httphandle, isc_nm_cb_t send_cb, void *send_cbarg) { @@ -1264,41 +1440,80 @@ http_do_bio(isc_nm_http_session_t *session, isc_nmhandle_t *send_httphandle, finish_http_session(session); } return; - } else if (nghttp2_session_want_read(session->ngsession) == 0 && - nghttp2_session_want_write(session->ngsession) == 0 && - session->pending_write_data == NULL) - { - session->closing = true; + } + + if (send_cb != NULL) { + INSIST(VALID_NMHANDLE(send_httphandle)); + (void)http_send_outgoing(session, send_httphandle, send_cb, + send_cbarg); + return; + } + + INSIST(send_httphandle == NULL); + INSIST(send_cb == NULL); + INSIST(send_cbarg == NULL); + + if (session->pending_write_data != NULL && session->sending == 0) { + (void)http_send_outgoing(session, NULL, NULL, NULL); return; } if (nghttp2_session_want_read(session->ngsession) != 0) { if (!session->reading) { - /* We have not yet started - * reading from this handle */ + /* We have not yet started reading from this handle */ isc_nm_read(session->handle, http_readcb, session); session->reading = true; } else if (session->buf != NULL) { size_t remaining = isc_buffer_remaininglength(session->buf); - /* Leftover data in the - * buffer, use it */ - size_t readlen = nghttp2_session_mem_recv( - session->ngsession, - isc_buffer_current(session->buf), remaining); + /* Leftover data in the buffer, use it */ + size_t remaining_after = 0; + ssize_t readlen = 0; + isc_nm_http_session_t *tmpsess = NULL; - if (readlen == remaining) { + /* + * Let's ensure that HTTP/2 session and its associated + * data will not go "out of scope" too early. + */ + isc__nm_httpsession_attach(session, &tmpsess); + + readlen = http_process_input_data(session, + session->buf); + + remaining_after = + isc_buffer_remaininglength(session->buf); + + if (readlen < 0) { + failed_read_cb(ISC_R_UNEXPECTED, session); + } else if ((size_t)readlen == remaining) { isc_buffer_free(&session->buf); + http_do_bio(session, NULL, NULL, NULL); + } else if (remaining_after > 0 && + remaining_after < remaining) + { + /* + * We have processed a part of the data, now + * let's delay processing of whatever is left + * here. We want it to be an async operation so + * that we will: + * + * a) let other things run; + * b) have finer grained control over how much + * data is processed at once, because nghttp2 + * would happily consume as much data we pass to + * it and that could overwhelm the server. + */ + http_do_bio_async(session); } else { - isc_buffer_forward(session->buf, readlen); + (void)http_send_outgoing(session, NULL, NULL, + NULL); } - http_do_bio(session, send_httphandle, send_cb, - send_cbarg); + isc__nm_httpsession_detach(&tmpsess); return; } else { - /* Resume reading, it's - * idempotent, wait for more + /* + * Resume reading, it's idempotent, wait for more */ isc_nm_read(session->handle, http_readcb, session); } @@ -1307,20 +1522,54 @@ http_do_bio(isc_nm_http_session_t *session, isc_nmhandle_t *send_httphandle, isc_nm_read_stop(session->handle); } - if (send_cb != NULL) { - INSIST(VALID_NMHANDLE(send_httphandle)); - (void)http_send_outgoing(session, send_httphandle, send_cb, - send_cbarg); - } else { - INSIST(send_httphandle == NULL); - INSIST(send_cb == NULL); - INSIST(send_cbarg == NULL); - (void)http_send_outgoing(session, NULL, NULL, NULL); + /* we might have some data to send after processing */ + (void)http_send_outgoing(session, NULL, NULL, NULL); + + if (nghttp2_session_want_read(session->ngsession) == 0 && + nghttp2_session_want_write(session->ngsession) == 0 && + session->pending_write_data == NULL) + { + session->closing = true; + isc_nm_read_stop(session->handle); + if (session->sending == 0) { + finish_http_session(session); + } } return; } +static void +http_do_bio_async_cb(void *arg) { + isc_nm_http_session_t *session = arg; + + REQUIRE(VALID_HTTP2_SESSION(session)); + + if (session->handle != NULL && + !isc__nmsocket_closing(session->handle->sock)) + { + http_do_bio(session, NULL, NULL, NULL); + } + + isc__nm_httpsession_detach(&session); +} + +static void +http_do_bio_async(isc_nm_http_session_t *session) { + isc_nm_http_session_t *tmpsess = NULL; + + REQUIRE(VALID_HTTP2_SESSION(session)); + + if (session->handle == NULL || + isc__nmsocket_closing(session->handle->sock)) + { + return; + } + isc__nm_httpsession_attach(session, &tmpsess); + isc_async_run(session->handle->sock->worker->loop, http_do_bio_async_cb, + tmpsess); +} + static isc_result_t get_http_cstream(isc_nmsocket_t *sock, http_cstream_t **streamp) { http_cstream_t *cstream = sock->h2->connect.cstream; @@ -2091,6 +2340,10 @@ server_call_cb(isc_nmsocket_t *socket, const isc_result_t result, if (result != ISC_R_SUCCESS) { data = NULL; } + if (result == ISC_R_SUCCESS) { + socket->h2->request_received = true; + socket->h2->session->received++; + } socket->h2->cb(handle, result, data, socket->h2->cbarg); isc_nmhandle_detach(&handle); } diff --git a/lib/isc/netmgr/netmgr-int.h b/lib/isc/netmgr/netmgr-int.h index d790590ec4..e6c6e82830 100644 --- a/lib/isc/netmgr/netmgr-int.h +++ b/lib/isc/netmgr/netmgr-int.h @@ -474,6 +474,7 @@ typedef struct isc_nmsocket_h2 { isc_nm_http_endpoints_t *peer_endpoints; + bool request_received; bool response_submitted; struct { char *uri; From 3425e4b1d04746520931e93ac7ef5979fd6b54fd Mon Sep 17 00:00:00 2001 From: Artem Boldariev Date: Mon, 8 Jul 2024 23:27:29 +0300 Subject: [PATCH 2/5] DoH: floodding clients detection This commit adds logic to make code better protected against clients that send valid HTTP/2 data that is useless from a DNS server perspective. Firstly, it adds logic that protects against clients who send too little useful (=DNS) data. We achieve that by adding a check that eventually detects such clients with a nonfavorable useful to processed data ratio after the initial grace period. The grace period is limited to processing 128 KiB of data, which should be enough for sending the largest possible DNS message in a GET request and then some. This is the main safety belt that would detect even flooding clients that initially behave well in order to fool the checks server. Secondly, in addition to the above, we introduce additional checks to detect outright misbehaving clients earlier: The code will treat clients that open too many streams (50) without sending any data for processing as flooding ones; The clients that managed to send 1.5 KiB of data without opening a single stream or submitting at least some DNS data will be treated as flooding ones. Of course, the behaviour described above is nothing else but heuristical checks, so they can never be perfect. At the same time, they should be reasonable enough not to drop any valid clients, realatively easy to implement, and have negligible computational overhead. --- lib/isc/netmgr/http.c | 115 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) diff --git a/lib/isc/netmgr/http.c b/lib/isc/netmgr/http.c index 7bf8dbd541..77ddfd59a8 100644 --- a/lib/isc/netmgr/http.c +++ b/lib/isc/netmgr/http.c @@ -100,6 +100,22 @@ */ #define INCOMING_DATA_MAX_CHUNKS_AT_ONCE (4) +/* + * These constants define the grace period to help detect flooding clients. + * + * The first one defines how much data can be processed before opening + * a first stream and received at least some useful (=DNS) data. + * + * The second one defines how much data from a client we read before + * trying to drop a clients who sends not enough useful data. + * + * The third constant defines how many streams we agree to process + * before checking if there was at least one DNS request received. + */ +#define INCOMING_DATA_INITIAL_STREAM_SIZE (1536) +#define INCOMING_DATA_GRACE_SIZE (MAX_ALLOWED_DATA_IN_HEADERS) +#define MAX_STREAMS_BEFORE_FIRST_REQUEST (50) + typedef struct isc_nm_http_response_status { size_t code; size_t content_length; @@ -158,6 +174,7 @@ struct isc_nm_http_session { ISC_LIST(http_cstream_t) cstreams; ISC_LIST(isc_nmsocket_h2_t) sstreams; size_t nsstreams; + uint64_t total_opened_sstreams; isc_nmhandle_t *handle; isc_nmhandle_t *client_httphandle; @@ -179,6 +196,9 @@ struct isc_nm_http_session { uint64_t received; /* How many requests have been received. */ uint64_t submitted; /* How many responses were submitted to send */ uint64_t processed; /* How many responses were processed. */ + + uint64_t processed_incoming_data; + uint64_t processed_useful_data; /* DNS data */ }; typedef enum isc_http_error_responses { @@ -214,6 +234,12 @@ static bool http_send_outgoing(isc_nm_http_session_t *session, isc_nmhandle_t *httphandle, isc_nm_cb_t cb, void *cbarg); +static void +http_log_flooding_peer(isc_nm_http_session_t *session); + +static bool +http_is_flooding_peer(isc_nm_http_session_t *session); + static ssize_t http_process_input_data(isc_nm_http_session_t *session, isc_buffer_t *input_data); @@ -619,6 +645,7 @@ on_server_data_chunk_recv_callback(int32_t stream_id, const uint8_t *data, if (new_bufsize <= MAX_DNS_MESSAGE_SIZE && new_bufsize <= h2->content_length) { + session->processed_useful_data += len; isc_buffer_putmem(&h2->rbuf, data, len); break; } @@ -1053,6 +1080,7 @@ http_process_input_data(isc_nm_http_session_t *session, if (readlen >= 0) { isc_buffer_forward(input_data, readlen); + session->processed_incoming_data += readlen; } return readlen; @@ -1109,6 +1137,7 @@ http_process_input_data(isc_nm_http_session_t *session, if (readlen >= 0) { isc_buffer_forward(input_data, readlen); + session->processed_incoming_data += readlen; processed += readlen; } else { isc_buffer_clear(input_data); @@ -1119,6 +1148,82 @@ http_process_input_data(isc_nm_http_session_t *session, return processed; } +static void +http_log_flooding_peer(isc_nm_http_session_t *session) { + const int log_level = ISC_LOG_DEBUG(1); + if (session->handle != NULL && isc_log_wouldlog(log_level)) { + char client_sabuf[ISC_SOCKADDR_FORMATSIZE]; + char local_sabuf[ISC_SOCKADDR_FORMATSIZE]; + + isc_sockaddr_format(&session->handle->sock->peer, client_sabuf, + sizeof(client_sabuf)); + isc_sockaddr_format(&session->handle->sock->iface, local_sabuf, + sizeof(local_sabuf)); + isc__nmsocket_log(session->handle->sock, log_level, + "Dropping a flooding HTTP/2 peer " + "%s (on %s) - processed: %" PRIu64 + " bytes, of them useful: %" PRIu64 "", + client_sabuf, local_sabuf, + session->processed_incoming_data, + session->processed_useful_data); + } +} + +static bool +http_is_flooding_peer(isc_nm_http_session_t *session) { + if (session->client) { + return false; + } + + /* + * A flooding client can try to open a lot of streams before + * submitting a request. Let's drop such clients. + */ + if (session->received == 0 && + session->total_opened_sstreams > MAX_STREAMS_BEFORE_FIRST_REQUEST) + { + return true; + } + + /* + * We have processed enough data to open at least one stream and + * get some useful data. + */ + if (session->processed_incoming_data > + INCOMING_DATA_INITIAL_STREAM_SIZE && + (session->total_opened_sstreams == 0 || + session->processed_useful_data == 0)) + { + return true; + } + + if (session->processed_incoming_data < INCOMING_DATA_GRACE_SIZE) { + return false; + } + + /* + * The overhead of DoH per DNS message can be minimum 160-180 + * bytes. We should allow more for extra information that can be + * included in headers, so let's use 256 bytes. Minimum DNS + * message size is 12 bytes. So, (256+12)/12=22. Even that can be + * too restricting for some edge cases, but should be good enough + * for any practical purposes. Not to mention that HTTP/2 may + * include legitimate data that is completely useless for DNS + * purposes... + * + * Anyway, at that point we should have processed enough requests + * for such clients (if any). + */ + if (session->processed_useful_data == 0 || + (session->processed_incoming_data / + session->processed_useful_data) > 22) + { + return true; + } + + return false; +} + /* * Read callback from TLS socket. */ @@ -1153,6 +1258,10 @@ http_readcb(isc_nmhandle_t *handle ISC_ATTR_UNUSED, isc_result_t result, if (readlen < 0) { failed_read_cb(ISC_R_UNEXPECTED, session); goto done; + } else if (http_is_flooding_peer(session)) { + http_log_flooding_peer(session); + failed_read_cb(ISC_R_RANGE, session); + goto done; } if ((size_t)readlen < region->length) { @@ -1485,6 +1594,9 @@ http_do_bio(isc_nm_http_session_t *session, isc_nmhandle_t *send_httphandle, if (readlen < 0) { failed_read_cb(ISC_R_UNEXPECTED, session); + } else if (http_is_flooding_peer(session)) { + http_log_flooding_peer(session); + failed_read_cb(ISC_R_RANGE, session); } else if ((size_t)readlen == remaining) { isc_buffer_free(&session->buf); http_do_bio(session, NULL, NULL, NULL); @@ -1972,6 +2084,7 @@ server_on_begin_headers_callback(nghttp2_session *ngsession, session->nsstreams++; isc__nm_httpsession_attach(session, &socket->h2->session); ISC_LIST_APPEND(session->sstreams, socket->h2, link); + session->total_opened_sstreams++; nghttp2_session_set_stream_user_data(ngsession, frame->hd.stream_id, socket); @@ -2031,6 +2144,8 @@ server_handle_path_header(isc_nmsocket_t *socket, const uint8_t *value, socket->worker->mctx, dns_value, dns_value_len, &socket->h2->query_data_len); + socket->h2->session->processed_useful_data += + dns_value_len; } else { socket->h2->query_too_large = true; return ISC_HTTP_ERROR_PAYLOAD_TOO_LARGE; From 609a41517b1631c320876a41c43c68c9a0ee0f9f Mon Sep 17 00:00:00 2001 From: Artem Boldariev Date: Tue, 9 Jul 2024 23:23:11 +0300 Subject: [PATCH 3/5] DoH: introduce manual read timer control This commit introduces manual read timer control as used by StreamDNS and its underlying transports. Before that, DoH code would rely on the timer control provided by TCP, which would reset the timer any time some data arrived. Now, the timer is restarted only when a full DNS message is processed in line with other DNS transports. That change is required because we should not stop the timer when reading from the network is paused due to throttling. We need a way to drop timed-out clients, particularly those who refuse to read the data we send. --- lib/isc/netmgr/http.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lib/isc/netmgr/http.c b/lib/isc/netmgr/http.c index 77ddfd59a8..6e2a3bbe46 100644 --- a/lib/isc/netmgr/http.c +++ b/lib/isc/netmgr/http.c @@ -560,6 +560,8 @@ finish_http_session(isc_nm_http_session_t *session) { if (!session->closed) { session->closed = true; session->reading = false; + isc_nm_read_stop(session->handle); + isc__nmsocket_timer_stop(session->handle->sock); isc_nmhandle_close(session->handle); } @@ -694,6 +696,9 @@ call_unlink_cstream_readcb(http_cstream_t *cstream, isc_buffer_usedregion(cstream->rbuf, &read_data); cstream->read_cb(session->client_httphandle, result, &read_data, cstream->read_cbarg); + if (result == ISC_R_SUCCESS) { + isc__nmsocket_timer_restart(session->handle->sock); + } put_http_cstream(session->mctx, cstream); } @@ -1570,6 +1575,7 @@ http_do_bio(isc_nm_http_session_t *session, isc_nmhandle_t *send_httphandle, if (nghttp2_session_want_read(session->ngsession) != 0) { if (!session->reading) { /* We have not yet started reading from this handle */ + isc__nmsocket_timer_start(session->handle->sock); isc_nm_read(session->handle, http_readcb, session); session->reading = true; } else if (session->buf != NULL) { @@ -1627,6 +1633,7 @@ http_do_bio(isc_nm_http_session_t *session, isc_nmhandle_t *send_httphandle, /* * Resume reading, it's idempotent, wait for more */ + isc__nmsocket_timer_start(session->handle->sock); isc_nm_read(session->handle, http_readcb, session); } } else { @@ -1803,6 +1810,7 @@ transport_connect_cb(isc_nmhandle_t *handle, isc_result_t result, void *cbarg) { } http_transpost_tcp_nodelay(handle); + isc__nmhandle_set_manual_timer(session->handle, true); http_call_connect_cb(http_sock, session, result); @@ -2454,6 +2462,8 @@ server_call_cb(isc_nmsocket_t *socket, const isc_result_t result, handle = isc__nmhandle_get(socket, NULL, NULL); if (result != ISC_R_SUCCESS) { data = NULL; + } else if (socket->h2->session->handle != NULL) { + isc__nmsocket_timer_restart(socket->h2->session->handle->sock); } if (result == ISC_R_SUCCESS) { socket->h2->request_received = true; @@ -2860,6 +2870,8 @@ httplisten_acceptcb(isc_nmhandle_t *handle, isc_result_t result, void *cbarg) { isc__nmsocket_attach(httpserver, &session->serversocket); server_send_connection_header(session); + isc__nmhandle_set_manual_timer(session->handle, true); + /* TODO H2 */ http_do_bio(session, NULL, NULL, NULL); return ISC_R_SUCCESS; From 4ae4e255cf0aef16d9b1b462e08ff7237352393e Mon Sep 17 00:00:00 2001 From: Artem Boldariev Date: Tue, 16 Jul 2024 16:38:56 +0300 Subject: [PATCH 4/5] Do not stop timer in isc_nm_read_stop() in manual timer mode A call to isc_nm_read_stop() would always stop reading timer even in manual timer control mode which was added with StreamDNS in mind. That looks like an omission that happened due to how timers are controlled in StreamDNS where we always stop the timer before pausing reading anyway (see streamdns_on_complete_dnsmessage()). That would not work well for HTTP, though, where we might want pause reading without stopping the timer in the case we want to split incoming data into multiple chunks to be processed independently. I suppose that it happened due to NM refactoring in the middle of StreamDNS development (at the time isc_nm_cancelread() and isc_nm_pauseread() were removed), as the StreamDNS code seems to be written as if timers are not stoping during a call to isc_nm_read_stop(). --- lib/isc/netmgr/proxystream.c | 2 ++ lib/isc/netmgr/streamdns.c | 1 + lib/isc/netmgr/tcp.c | 4 +++- lib/isc/netmgr/tlsstream.c | 6 ++++++ 4 files changed, 12 insertions(+), 1 deletion(-) diff --git a/lib/isc/netmgr/proxystream.c b/lib/isc/netmgr/proxystream.c index 883e8cf942..8d65ea5444 100644 --- a/lib/isc/netmgr/proxystream.c +++ b/lib/isc/netmgr/proxystream.c @@ -121,6 +121,7 @@ proxystream_on_header_data_cb(const isc_result_t result, * the case of TCP it is disabled by default */ proxystream_read_stop(sock); + isc__nmsocket_timer_stop(sock); isc__nmhandle_set_manual_timer(sock->outerhandle, false); sock->proxy.header_processed = true; @@ -775,6 +776,7 @@ isc__nm_proxystream_close(isc_nmsocket_t *sock) { * external references, we can close everything. */ proxystream_read_stop(sock); + isc__nmsocket_timer_stop(sock); if (sock->outerhandle != NULL) { sock->reading = false; isc_nm_read_stop(sock->outerhandle); diff --git a/lib/isc/netmgr/streamdns.c b/lib/isc/netmgr/streamdns.c index 30b3903986..c80b2285c9 100644 --- a/lib/isc/netmgr/streamdns.c +++ b/lib/isc/netmgr/streamdns.c @@ -1009,6 +1009,7 @@ streamdns_close_direct(isc_nmsocket_t *sock) { if (sock->outerhandle != NULL) { sock->streamdns.reading = false; + isc__nmsocket_timer_stop(sock); isc_nm_read_stop(sock->outerhandle); isc_nmhandle_close(sock->outerhandle); isc_nmhandle_detach(&sock->outerhandle); diff --git a/lib/isc/netmgr/tcp.c b/lib/isc/netmgr/tcp.c index ca3ed8b7f4..4f98b50862 100644 --- a/lib/isc/netmgr/tcp.c +++ b/lib/isc/netmgr/tcp.c @@ -759,7 +759,9 @@ isc__nm_tcp_read_stop(isc_nmhandle_t *handle) { isc_nmsocket_t *sock = handle->sock; - isc__nmsocket_timer_stop(sock); + if (!sock->manual_read_timer) { + isc__nmsocket_timer_stop(sock); + } isc__nm_stop_reading(sock); sock->reading = false; diff --git a/lib/isc/netmgr/tlsstream.c b/lib/isc/netmgr/tlsstream.c index c599600f10..8d5fe1fd37 100644 --- a/lib/isc/netmgr/tlsstream.c +++ b/lib/isc/netmgr/tlsstream.c @@ -465,6 +465,7 @@ tls_try_handshake(isc_nmsocket_t *sock, isc_result_t *presult) { isc__nmsocket_log_tls_session_reuse(sock, sock->tlsstream.tls); tlshandle = isc__nmhandle_get(sock, &sock->peer, &sock->iface); + isc__nmsocket_timer_stop(sock); tls_read_stop(sock); if (isc__nm_closing(sock->worker)) { @@ -1154,6 +1155,10 @@ isc__nm_tls_read_stop(isc_nmhandle_t *handle) { handle->sock->reading = false; + if (!handle->sock->manual_read_timer) { + isc__nmsocket_timer_stop(handle->sock); + } + tls_read_stop(handle->sock); } @@ -1174,6 +1179,7 @@ isc__nm_tls_close(isc_nmsocket_t *sock) { */ tls_read_stop(sock); if (sock->outerhandle != NULL) { + isc__nmsocket_timer_stop(sock); isc_nm_read_stop(sock->outerhandle); isc_nmhandle_close(sock->outerhandle); isc_nmhandle_detach(&sock->outerhandle); From 937b5f8349a6a5e15af254a53a659e39c7c1d179 Mon Sep 17 00:00:00 2001 From: Artem Boldariev Date: Thu, 18 Jul 2024 20:21:53 +0300 Subject: [PATCH 5/5] DoH: reduce excessive bad request logging We started using isc_nm_bad_request() more actively throughout codebase. In the case of HTTP/2 it can lead to a large count of useless "Bad Request" messages in the BIND log, as often we attempt to send such request over effectively finished HTTP/2 sessions. This commit fixes that. --- lib/isc/netmgr/http.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/isc/netmgr/http.c b/lib/isc/netmgr/http.c index 6e2a3bbe46..2287035921 100644 --- a/lib/isc/netmgr/http.c +++ b/lib/isc/netmgr/http.c @@ -2484,6 +2484,12 @@ isc__nm_http_bad_request(isc_nmhandle_t *handle) { REQUIRE(!sock->client); REQUIRE(VALID_HTTP2_SESSION(sock->h2->session)); + if (sock->h2->response_submitted || + !http_session_active(sock->h2->session)) + { + return; + } + (void)server_send_error_response(ISC_HTTP_ERROR_BAD_REQUEST, sock->h2->session->ngsession, sock); }