From f1d9e9ee3859976f403914d20ad2a10855343702 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Sur=C3=BD?= Date: Tue, 30 May 2023 08:46:17 +0200 Subject: [PATCH 1/9] Improve RBT overmem cache cleaning When cache memory usage is over the configured cache size (overmem) and we are cleaning unused entries, it might not be enough to clean just two entries if the entries to be expired are smaller than the newly added rdata. This could be abused by an attacker to cause a remote Denial of Service by possibly running out of the operating system memory. Currently, the addrdataset() tries to do a single TTL-based cleaning considering the serve-stale TTL and then optionally moves to overmem cleaning if we are in that condition. Then the overmem_purge() tries to do another single TTL based cleaning from the TTL heap and then continue with LRU-based cleaning up to 2 entries cleaned. Squash the TTL-cleaning mechanism into single call from addrdataset(), but ignore the serve-stale TTL if we are currently overmem. Then instead of having a fixed number of entries to clean, pass the size of newly added rdatasetheader to the overmem_purge() function and cleanup at least the size of the newly added data. This prevents the cache going over the configured memory limit (`max-cache-size`). Additionally, refactor the overmem_purge() function to reduce for-loop nesting for readability. --- lib/dns/rbtdb.c | 105 ++++++++++++++++++++++++++++++------------------ 1 file changed, 65 insertions(+), 40 deletions(-) diff --git a/lib/dns/rbtdb.c b/lib/dns/rbtdb.c index c32db44cad..ee06c51e92 100644 --- a/lib/dns/rbtdb.c +++ b/lib/dns/rbtdb.c @@ -605,7 +605,7 @@ static void expire_header(dns_rbtdb_t *rbtdb, rdatasetheader_t *header, bool tree_locked, expire_t reason); static void -overmem_purge(dns_rbtdb_t *rbtdb, unsigned int locknum_start, isc_stdtime_t now, +overmem_purge(dns_rbtdb_t *rbtdb, unsigned int locknum_start, size_t purgesize, bool tree_locked); static void resign_insert(dns_rbtdb_t *rbtdb, int idx, rdatasetheader_t *newheader); @@ -6823,6 +6823,16 @@ cleanup: static dns_dbmethods_t zone_methods; +static size_t +rdataset_size(rdatasetheader_t *header) { + if (!NONEXISTENT(header)) { + return (dns_rdataslab_size((unsigned char *)header, + sizeof(*header))); + } + + return (sizeof(*header)); +} + static isc_result_t addrdataset(dns_db_t *db, dns_dbnode_t *node, dns_dbversion_t *version, isc_stdtime_t now, dns_rdataset_t *rdataset, unsigned int options, @@ -6987,7 +6997,8 @@ addrdataset(dns_db_t *db, dns_dbnode_t *node, dns_dbversion_t *version, } if (cache_is_overmem) { - overmem_purge(rbtdb, rbtnode->locknum, now, tree_locked); + overmem_purge(rbtdb, rbtnode->locknum, rdataset_size(newheader), + tree_locked); } NODE_LOCK(&rbtdb->node_locks[rbtnode->locknum].lock, @@ -7006,10 +7017,18 @@ addrdataset(dns_db_t *db, dns_dbnode_t *node, dns_dbversion_t *version, } header = isc_heap_element(rbtdb->heaps[rbtnode->locknum], 1); - if (header != NULL && header->rdh_ttl + rbtdb->serve_stale_ttl < - now - RBTDB_VIRTUAL) - { - expire_header(rbtdb, header, tree_locked, expire_ttl); + if (header != NULL) { + dns_ttl_t rdh_ttl = header->rdh_ttl; + + /* Only account for stale TTL if cache is not overmem */ + if (!cache_is_overmem) { + rdh_ttl += rbtdb->serve_stale_ttl; + } + + if (rdh_ttl < now - RBTDB_VIRTUAL) { + expire_header(rbtdb, header, tree_locked, + expire_ttl); + } } /* @@ -10547,52 +10566,58 @@ update_header(dns_rbtdb_t *rbtdb, rdatasetheader_t *header, isc_stdtime_t now) { ISC_LIST_PREPEND(rbtdb->rdatasets[header->node->locknum], header, link); } +static size_t +expire_lru_headers(dns_rbtdb_t *rbtdb, unsigned int locknum, size_t purgesize, + bool tree_locked) { + rdatasetheader_t *header, *header_prev; + size_t purged = 0; + + for (header = ISC_LIST_TAIL(rbtdb->rdatasets[locknum]); + header != NULL && purged <= purgesize; header = header_prev) + { + header_prev = ISC_LIST_PREV(header, link); + /* + * Unlink the entry at this point to avoid checking it + * again even if it's currently used someone else and + * cannot be purged at this moment. This entry won't be + * referenced any more (so unlinking is safe) since the + * TTL was reset to 0. + */ + ISC_LIST_UNLINK(rbtdb->rdatasets[locknum], header, link); + size_t header_size = rdataset_size(header); + expire_header(rbtdb, header, tree_locked, expire_lru); + purged += header_size; + } + + return (purged); +} + /*% - * Purge some expired and/or stale (i.e. unused for some period) cache entries - * under an overmem condition. To recover from this condition quickly, up to - * 2 entries will be purged. This process is triggered while adding a new - * entry, and we specifically avoid purging entries in the same LRU bucket as - * the one to which the new entry will belong. Otherwise, we might purge - * entries of the same name of different RR types while adding RRsets from a - * single response (consider the case where we're adding A and AAAA glue records - * of the same NS name). + * Purge some stale (i.e. unused for some period - LRU based cleaning) cache + * entries under the overmem condition. To recover from this condition quickly, + * we cleanup entries up to the size of newly added rdata (passed as purgesize). + * + * This process is triggered while adding a new entry, and we specifically avoid + * purging entries in the same LRU bucket as the one to which the new entry will + * belong. Otherwise, we might purge entries of the same name of different RR + * types while adding RRsets from a single response (consider the case where + * we're adding A and AAAA glue records of the same NS name). */ static void -overmem_purge(dns_rbtdb_t *rbtdb, unsigned int locknum_start, isc_stdtime_t now, +overmem_purge(dns_rbtdb_t *rbtdb, unsigned int locknum_start, size_t purgesize, bool tree_locked) { - rdatasetheader_t *header, *header_prev; unsigned int locknum; - int purgecount = 2; + size_t purged = 0; for (locknum = (locknum_start + 1) % rbtdb->node_lock_count; - locknum != locknum_start && purgecount > 0; + locknum != locknum_start && purged <= purgesize; locknum = (locknum + 1) % rbtdb->node_lock_count) { NODE_LOCK(&rbtdb->node_locks[locknum].lock, isc_rwlocktype_write); - header = isc_heap_element(rbtdb->heaps[locknum], 1); - if (header && header->rdh_ttl < now - RBTDB_VIRTUAL) { - expire_header(rbtdb, header, tree_locked, expire_ttl); - purgecount--; - } - - for (header = ISC_LIST_TAIL(rbtdb->rdatasets[locknum]); - header != NULL && purgecount > 0; header = header_prev) - { - header_prev = ISC_LIST_PREV(header, link); - /* - * Unlink the entry at this point to avoid checking it - * again even if it's currently used someone else and - * cannot be purged at this moment. This entry won't be - * referenced any more (so unlinking is safe) since the - * TTL was reset to 0. - */ - ISC_LIST_UNLINK(rbtdb->rdatasets[locknum], header, - link); - expire_header(rbtdb, header, tree_locked, expire_lru); - purgecount--; - } + purged += expire_lru_headers(rbtdb, locknum, purgesize - purged, + tree_locked); NODE_UNLOCK(&rbtdb->node_locks[locknum].lock, isc_rwlocktype_write); From a295566683b035758772377e014b2367eb010323 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Sur=C3=BD?= Date: Thu, 1 Jun 2023 15:46:23 +0200 Subject: [PATCH 2/9] Add CHANGES and release note for [GL #4055] --- CHANGES | 4 ++++ doc/notes/notes-current.rst | 9 ++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 1d225c08ee..221e59f7a3 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,7 @@ +6190. [security] Improve the overmem cleaning process to prevent the + cache going over the configured limit. (CVE-2023-2828) + [GL #4055] + 6183. [bug] Fix a serve-stale bug where a delegation from cache could be returned to the client. [GL #3950] diff --git a/doc/notes/notes-current.rst b/doc/notes/notes-current.rst index 1cd8c68f6f..a44c0c516d 100644 --- a/doc/notes/notes-current.rst +++ b/doc/notes/notes-current.rst @@ -15,7 +15,14 @@ Notes for BIND 9.16.42 Security Fixes ~~~~~~~~~~~~~~ -- None. +- The overmem cleaning process has been improved, to prevent the cache from + significantly exceeding the configured ``max-cache-size`` limit. + (CVE-2023-2828) + + ISC would like to thank Shoham Danino from Reichman University, Anat + Bremler-Barr from Tel-Aviv University, Yehuda Afek from Tel-Aviv University, + and Yuval Shavitt from Tel-Aviv University for bringing this vulnerability to + our attention. :gl:`#4055` New Features ~~~~~~~~~~~~ From 0fa9acc2c0b93d1a917971d00986feb46e4f9b91 Mon Sep 17 00:00:00 2001 From: Michal Nowak Date: Wed, 7 Jun 2023 14:03:01 +0200 Subject: [PATCH 3/9] Set max-cache-size expectations for low values --- doc/arm/reference.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/arm/reference.rst b/doc/arm/reference.rst index 9a2f4af1ba..55af7b8b42 100644 --- a/doc/arm/reference.rst +++ b/doc/arm/reference.rst @@ -3095,6 +3095,11 @@ system. default value of that option (90% of physical memory for each individual cache) may lead to memory exhaustion over time. + .. note:: + + ``max-cache-size`` does not work reliably for the maximum + amount of memory of 100 MB or lower. + Upon startup and reconfiguration, caches with a limited size preallocate a small amount of memory (less than 1% of ``max-cache-size`` for a given view). This preallocation serves as an From 0101e28f91fb36b6a16a0049d3b3e2b7846f23f0 Mon Sep 17 00:00:00 2001 From: Evan Hunt Date: Thu, 25 May 2023 23:53:50 -0700 Subject: [PATCH 4/9] Stale answer lookups could loop when over recursion quota When a query was aborted because of the recursion quota being exceeded, but triggered a stale answer response and a stale data refresh query, it could cause named to loop back where we are iterating and following a delegation. Having no good answer in cache, we would fall back to using serve-stale again, use the stale data, try to refresh the RRset, and loop back again, without ever terminating until crashing due to stack overflow. This happens because in the functions 'query_notfound()' and 'query_delegation_recurse()', we check whether we can fall back to serving stale data. We shouldn't do so if we are already refreshing an RRset due to having prioritized stale data in cache. In other words, we need to add an extra check to 'query_usestale()' to disallow serving stale data if we are currently refreshing a stale RRset. As an additional mitigation to prevent looping, we now use the result code ISC_R_ALREADYRUNNING rather than ISC_R_FAILURE when a recursion loop is encountered, and we check for that condition in 'query_usestale()' as well. --- lib/ns/query.c | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/lib/ns/query.c b/lib/ns/query.c index 1d7f70ae1c..28bef2d99c 100644 --- a/lib/ns/query.c +++ b/lib/ns/query.c @@ -6324,7 +6324,7 @@ ns_query_recurse(ns_client_t *client, dns_rdatatype_t qtype, dns_name_t *qname, if (recparam_match(&client->query.recparam, qtype, qname, qdomain)) { ns_client_log(client, NS_LOGCATEGORY_CLIENT, NS_LOGMODULE_QUERY, ISC_LOG_INFO, "recursion loop detected"); - return (ISC_R_FAILURE); + return (ISC_R_ALREADYRUNNING); } recparam_update(&client->query.recparam, qtype, qname, qdomain); @@ -7288,10 +7288,21 @@ query_usestale(query_ctx_t *qctx, isc_result_t result) { return (false); } - if (result == DNS_R_DUPLICATE || result == DNS_R_DROP) { + if (qctx->refresh_rrset) { + /* + * This is a refreshing query, we have already prioritized + * stale data, so don't enable serve-stale again. + */ + return (false); + } + + if (result == DNS_R_DUPLICATE || result == DNS_R_DROP || + result == ISC_R_ALREADYRUNNING) + { /* * Don't enable serve-stale if the result signals a duplicate - * query or query that is being dropped. + * query or a query that is being dropped or can't proceed + * because of a recursion loop. */ return (false); } From c003c5bc3c68f3e513654b6689e1f60280d14844 Mon Sep 17 00:00:00 2001 From: Matthijs Mekking Date: Thu, 1 Jun 2023 10:03:48 +0200 Subject: [PATCH 5/9] Fix serve-stale hang at shutdown The 'refresh_rrset' variable is used to determine if we can detach from the client. This can cause a hang on shutdown. To fix this, move setting of the 'nodetach' variable up to where 'refresh_rrset' is set (in query_lookup(), and thus not in ns_query_done()), and set it to false when actually refreshing the RRset, so that when this lookup is completed, the client will be detached. --- lib/ns/query.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/lib/ns/query.c b/lib/ns/query.c index 28bef2d99c..5cf502eef0 100644 --- a/lib/ns/query.c +++ b/lib/ns/query.c @@ -5719,6 +5719,7 @@ query_refresh_rrset(query_ctx_t *orig_qctx) { qctx.client->query.dboptions &= ~(DNS_DBFIND_STALETIMEOUT | DNS_DBFIND_STALEOK | DNS_DBFIND_STALEENABLED); + qctx.client->nodetach = false; /* * We'll need some resources... @@ -5976,7 +5977,14 @@ query_lookup(query_ctx_t *qctx) { "%s stale answer used, an attempt to " "refresh the RRset will still be made", namebuf); + qctx->refresh_rrset = STALE(qctx->rdataset); + + /* + * If we are refreshing the RRSet, we must not + * detach from the client in query_send(). + */ + qctx->client->nodetach = qctx->refresh_rrset; } } else { /* @@ -11565,12 +11573,7 @@ ns_query_done(query_ctx_t *qctx) { /* * Client may have been detached after query_send(), so * we test and store the flag state here, for safety. - * If we are refreshing the RRSet, we must not detach from the client - * in the query_send(), so we need to override the flag. */ - if (qctx->refresh_rrset) { - qctx->client->nodetach = true; - } nodetach = qctx->client->nodetach; query_send(qctx->client); From b226745c0fd59571abb838359ae996716e57a943 Mon Sep 17 00:00:00 2001 From: Evan Hunt Date: Wed, 31 May 2023 12:47:31 -0700 Subject: [PATCH 6/9] CHANGES and release notes for [GL #4089] --- CHANGES | 7 +++++++ doc/notes/notes-current.rst | 6 ++++++ 2 files changed, 13 insertions(+) diff --git a/CHANGES b/CHANGES index 221e59f7a3..0f215ba9dc 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,10 @@ +6192. [security] A query that prioritizes stale data over lookup + triggers a fetch to refresh the stale data in cache. + If the fetch is aborted for exceeding the recursion + quota, it was possible for 'named' to enter an infinite + callback loop and crash due to stack overflow. This has + been fixed. (CVE-2023-2911) [GL #4089] + 6190. [security] Improve the overmem cleaning process to prevent the cache going over the configured limit. (CVE-2023-2828) [GL #4055] diff --git a/doc/notes/notes-current.rst b/doc/notes/notes-current.rst index a44c0c516d..7f388996c8 100644 --- a/doc/notes/notes-current.rst +++ b/doc/notes/notes-current.rst @@ -24,6 +24,12 @@ Security Fixes and Yuval Shavitt from Tel-Aviv University for bringing this vulnerability to our attention. :gl:`#4055` +- A query that prioritizes stale data over lookup triggers a fetch to refresh + the stale data in cache. If the fetch is aborted for exceeding the recursion + quota, it was possible for :iscman:`named` to enter an infinite callback + loop and crash due to stack overflow. This has been fixed. (CVE-2023-2911) + :gl:`#4089` + New Features ~~~~~~~~~~~~ From 35a43186a606c321ba5909019293740beda6b48d Mon Sep 17 00:00:00 2001 From: Michal Nowak Date: Fri, 9 Jun 2023 16:13:28 +0200 Subject: [PATCH 7/9] Prepare release notes for BIND 9.16.42 --- doc/arm/notes.rst | 2 +- .../{notes-current.rst => notes-9.16.42.rst} | 15 --------------- 2 files changed, 1 insertion(+), 16 deletions(-) rename doc/notes/{notes-current.rst => notes-9.16.42.rst} (93%) diff --git a/doc/arm/notes.rst b/doc/arm/notes.rst index 10dd7caedf..4eb912229a 100644 --- a/doc/arm/notes.rst +++ b/doc/arm/notes.rst @@ -46,7 +46,7 @@ for Microsoft Windows operating systems. .. include:: ../notes/notes-known-issues.rst -.. include:: ../notes/notes-current.rst +.. include:: ../notes/notes-9.16.42.rst .. include:: ../notes/notes-9.16.41.rst .. include:: ../notes/notes-9.16.40.rst .. include:: ../notes/notes-9.16.39.rst diff --git a/doc/notes/notes-current.rst b/doc/notes/notes-9.16.42.rst similarity index 93% rename from doc/notes/notes-current.rst rename to doc/notes/notes-9.16.42.rst index 7f388996c8..82db28cff5 100644 --- a/doc/notes/notes-current.rst +++ b/doc/notes/notes-9.16.42.rst @@ -30,21 +30,6 @@ Security Fixes loop and crash due to stack overflow. This has been fixed. (CVE-2023-2911) :gl:`#4089` -New Features -~~~~~~~~~~~~ - -- None. - -Removed Features -~~~~~~~~~~~~~~~~ - -- None. - -Feature Changes -~~~~~~~~~~~~~~~ - -- None. - Bug Fixes ~~~~~~~~~ From 606af41ceda5dddfaed0493e159d9428a3b2129f Mon Sep 17 00:00:00 2001 From: Michal Nowak Date: Fri, 9 Jun 2023 16:18:38 +0200 Subject: [PATCH 8/9] Tweak and reword release notes --- doc/notes/notes-9.16.42.rst | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/doc/notes/notes-9.16.42.rst b/doc/notes/notes-9.16.42.rst index 82db28cff5..85b0edeabd 100644 --- a/doc/notes/notes-9.16.42.rst +++ b/doc/notes/notes-9.16.42.rst @@ -15,27 +15,27 @@ Notes for BIND 9.16.42 Security Fixes ~~~~~~~~~~~~~~ -- The overmem cleaning process has been improved, to prevent the cache from - significantly exceeding the configured ``max-cache-size`` limit. +- The overmem cleaning process has been improved, to prevent the cache + from significantly exceeding the configured ``max-cache-size`` limit. (CVE-2023-2828) ISC would like to thank Shoham Danino from Reichman University, Anat - Bremler-Barr from Tel-Aviv University, Yehuda Afek from Tel-Aviv University, - and Yuval Shavitt from Tel-Aviv University for bringing this vulnerability to - our attention. :gl:`#4055` + Bremler-Barr from Tel-Aviv University, Yehuda Afek from Tel-Aviv + University, and Yuval Shavitt from Tel-Aviv University for bringing + this vulnerability to our attention. :gl:`#4055` -- A query that prioritizes stale data over lookup triggers a fetch to refresh - the stale data in cache. If the fetch is aborted for exceeding the recursion - quota, it was possible for :iscman:`named` to enter an infinite callback - loop and crash due to stack overflow. This has been fixed. (CVE-2023-2911) - :gl:`#4089` +- A query that prioritizes stale data over lookup triggers a fetch to + refresh the stale data in cache. If the fetch is aborted for exceeding + the recursion quota, it was possible for :iscman:`named` to enter an + infinite callback loop and crash due to stack overflow. This has been + fixed. (CVE-2023-2911) :gl:`#4089` Bug Fixes ~~~~~~~~~ -- It could happen that after the ``stale-answer-client-timeout`` duration, - a delegation from cache was returned to the client. This has now been fixed. - :gl:`#3950` +- Previously, it was possible for a delegation from cache to be returned + to the client after the ``stale-answer-client-timeout`` duration. + This has been fixed. :gl:`#3950` Known Issues ~~~~~~~~~~~~ From a62d1bd69a5a145daaad41aeb1f0d4d1f4f3f285 Mon Sep 17 00:00:00 2001 From: Michal Nowak Date: Fri, 9 Jun 2023 16:35:17 +0200 Subject: [PATCH 9/9] prep 9.16.42 --- CHANGES | 2 ++ version | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 0f215ba9dc..e04878f596 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,5 @@ + --- 9.16.42 released --- + 6192. [security] A query that prioritizes stale data over lookup triggers a fetch to refresh the stale data in cache. If the fetch is aborted for exceeding the recursion diff --git a/version b/version index d1d1bfe452..c8234484c2 100644 --- a/version +++ b/version @@ -8,4 +8,4 @@ MINORVER=16 PATCHVER=42 RELEASETYPE= RELEASEVER= -EXTENSIONS=-dev +EXTENSIONS=