Compare commits

...

4 Commits

Author SHA1 Message Date
Evan Hunt
44d1430725 CHANGES for [GL #3033] 2021-12-08 10:56:25 -08:00
Mark Andrews
8bfd197856 Test validation deadlock recovery
Check that the following scenarios do not cause fetches to hang:
- An undelegated unsigned zone with CNAME to apex where the parent
  zone is signed
- An NSEC3 zone with only DNSKEY correctly signed
2021-12-08 10:56:25 -08:00
Evan Hunt
69c74bf6bb check for validator/fetch dependency loops
It was possible for a deadlock to occur when a fetch started
a validator which started another fetch for the same name.
To address this, we now store pointers from the valdiator object
to the originating fetch, and from the fetch object to the originating
validator (if any), so that potential loops can be detected.
2021-12-08 10:56:25 -08:00
Evan Hunt
56ae95e79e prevent a possible validator deadlock
When processing a CNAME response, a validator could initiate a
query that matches the one waiting on the validator.
2021-12-08 10:56:25 -08:00
14 changed files with 267 additions and 37 deletions

View File

@@ -1,3 +1,6 @@
5778. [bug] Address some validator deadlocks that cause hung
fetches. [GL #3033]
5777. [bug] TCP connections could hang after receiving
non-matching responses. [GL #3042]

View File

@@ -25,7 +25,6 @@ rm -f ./canonical?.*
rm -f ./delv.out*
rm -f ./delve.out*
rm -f ./dig.out.*
rm -f ./ns2/too-many-iterations.db
rm -f ./dnssectools.out*
rm -f ./dsfromkey.out.*
rm -f ./keygen.err
@@ -52,7 +51,12 @@ rm -f ./ns2/cds.secure.db ./ns2/cds-x.secure.db
rm -f ./ns2/in-addr.arpa.db
rm -f ./ns2/nsec3chain-test.db
rm -f ./ns2/single-nsec3.db
rm -f ./ns2/too-many-iterations.db
rm -f ./ns2/updatecheck-kskonly.secure.*
rm -f ./ns3/NSEC ./ns3/NSEC3
rm -f ./ns3/all-but-dnskey-signed-with-nonexistent-key-nsec3.example.db not removed
rm -f ./ns3/all-but-dnskey-signed-with-nonexistent-key-nsec3.example.db.badsigs not removed
rm -f ./ns3/all-but-dnskey-signed-with-nonexistent-key-nsec3.example.db.goodsigs not removed
rm -f ./ns3/auto-nsec.example.db ./ns3/auto-nsec3.example.db
rm -f ./ns3/badds.example.db
rm -f ./ns3/dname-at-apex-nsec3.example.db
@@ -94,7 +98,6 @@ rm -f ./ns3/ttlpatch.example.db ./ns3/ttlpatch.example.db.signed
rm -f ./ns3/ttlpatch.example.db.patched
rm -f ./ns3/unsecure.example.db ./ns3/bogus.example.db ./ns3/keyless.example.db
rm -f ./ns3/unsupported.managed.db.tmp ./ns3/unsupported.trusted.db.tmp
rm -f ./ns3/NSEC ./ns3/NSEC3
rm -f ./ns4/managed-keys.bind*
rm -f ./ns4/named_dump.db*
rm -f ./ns6/optout-tld.db

View File

@@ -163,3 +163,6 @@ revkey NS ns.revkey
ns.revkey A 10.53.0.3
dname-at-apex-nsec3 NS ns3
all-but-dnskey-signed-with-nonexistent-key-nsec3 NS ns.all-but-dnskey-signed-with-nonexistent-key-nsec3
ns.all-but-dnskey-signed-with-nonexistent-key-nsec3 A 10.53.0.3

View File

@@ -56,6 +56,11 @@ zone "example" {
allow-update { any; };
};
zone "notdelegated.example" {
type primary;
file "notdelegated.example.db";
};
zone "insecure.secure.example" {
type primary;
file "insecure.secure.example.db";

View File

@@ -0,0 +1,17 @@
; Copyright (C) Internet Systems Consortium, Inc. ("ISC")
;
; This Source Code Form is subject to the terms of the Mozilla Public
; License, v. 2.0. If a copy of the MPL was not distributed with this
; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;
; See the COPYRIGHT file distributed with this work for additional
; information regarding copyright ownership.
$TTL 300
@ SOA ns2 hostmaster 0 0 0 0 0
@ NS ns2
@ NS ns3
@ A 10.53.0.100
ns2 A 10.53.0.2
ns3 A 10.53.0.3
www CNAME @

View File

@@ -61,7 +61,8 @@ for subdomain in secure badds bogus dynamic keyless nsec3 optout \
ttlpatch split-dnssec split-smart expired expiring upper lower \
dnskey-unknown dnskey-unsupported dnskey-unsupported-2 \
dnskey-nsec3-unknown managed-future revkey \
dname-at-apex-nsec3 occluded
dname-at-apex-nsec3 occluded \
all-but-dnskey-signed-with-nonexistent-key-nsec3
do
cp "../ns3/dsset-$subdomain.example$TP" .
done

View File

@@ -0,0 +1,38 @@
; Copyright (C) Internet Systems Consortium, Inc. ("ISC")
;
; This Source Code Form is subject to the terms of the Mozilla Public
; License, v. 2.0. If a copy of the MPL was not distributed with this
; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;
; See the COPYRIGHT file distributed with this work for additional
; information regarding copyright ownership.
$TTL 300 ; 5 minutes
@ IN SOA mname1. . (
2000042407 ; serial
20 ; refresh (20 seconds)
20 ; retry (20 seconds)
1814400 ; expire (3 weeks)
3600 ; minimum (1 hour)
)
NS ns3
ns3 A 10.53.0.3
a A 10.0.0.1
b A 10.0.0.2
c A 10.0.0.3
d A 10.0.0.4
e A 10.0.0.5
f A 10.0.0.6
g A 10.0.0.7
z A 10.0.0.26
a.a.a.a.a.a.a.a.a.a.e A 10.0.0.27
x CNAME a
zz DNSKEY 258 3 5 Cg==
normalthenrrsig A 10.0.0.28
rrsigonly A 10.0.0.29
cnameandkey CNAME @
cnamenokey CNAME @
dnameandkey DNAME @

View File

@@ -46,6 +46,12 @@ zone "example" {
file "example.bk";
};
zone "notdelegated.example" {
type secondary;
primaries { 10.53.0.2; };
file "notdelegated.example.bk";
};
zone "secure.example" {
type primary;
file "secure.example.db.signed";
@@ -370,6 +376,11 @@ zone "too-many-iterations" {
file "too-many-iterations.bk";
};
zone "all-but-dnskey-signed-with-nonexistent-key-nsec3.example" {
type primary;
file "all-but-dnskey-signed-with-nonexistent-key-nsec3.example.db";
};
include "siginterval.conf";
include "trusted.conf";

View File

@@ -670,3 +670,28 @@ $DSFROMKEY "$dnskeyname.key" > "dsset-delegation.${zone}$TP"
cat "$infile" "${kskname}.key" "${zskname}.key" "${keyname}.key" \
"${dnskeyname}.key" "dsset-delegation.${zone}$TP" >"$zonefile"
"$SIGNER" -P -o "$zone" "$zonefile" > /dev/null
#
# A NSEC3 zone which is signed but only has a valid DNSKEY RRSet.
#
zone=all-but-dnskey-signed-with-nonexistent-key-nsec3.example
infile=all-but-dnskey-signed-with-nonexistent-key-nsec3.example.db.in
zonefile=all-but-dnskey-signed-with-nonexistent-key-nsec3.example.db
# these keys will be discarded
kskname=$("$KEYGEN" -q -a "$DEFAULT_ALGORITHM" -3fk "$zone")
zskname=$("$KEYGEN" -q -a "$DEFAULT_ALGORITHM" -3 "$zone")
cat "$infile" "${kskname}.key" "${zskname}.key" >"$zonefile"
"$SIGNER" -P -D -O full -3 - -o "$zone" "$zonefile" > /dev/null
awk '$4 == "RRSIG" && $5 == "DNSKEY" { next }
{ print }' "$zonefile.signed" > "$zonefile.badsigs"
# these keys will be added to the final zone
kskname=$("$KEYGEN" -q -a "$DEFAULT_ALGORITHM" -3fk "$zone")
zskname=$("$KEYGEN" -q -a "$DEFAULT_ALGORITHM" -3 "$zone")
cat "$infile" "${kskname}.key" "${zskname}.key" >"$zonefile"
"$SIGNER" -P -D -O full -3 - -o "$zone" "$zonefile" > /dev/null
awk ' $4 == "RRSIG" && $5 == "DNSKEY" { print }
{ next }' "$zonefile.signed" > "$zonefile.goodsigs"
echo '$INCLUDE "'"$zonefile.goodsigs"'"' >> "$zonefile"
echo '$INCLUDE "'"$zonefile.badsigs"'"' >> "$zonefile"

View File

@@ -4421,5 +4421,31 @@ n=$((n+1))
test "$ret" -eq 0 || echo_i "failed"
status=$((status+ret))
echo_i "check for validation deadlock - NSEC3 zone with only DNSKEY correctly signed ($n)"
ret=0
nextpart ns4/named.run > /dev/null
zone=all-but-dnskey-signed-with-nonexistent-key-nsec3.example
pattern="validating [0123456789ABCDEFGHIJKLMNOPQRSTUV]*.$zone/DS:"
pattern="$pattern fetch/validator loop detected: aborting validation"
dig_with_opts xx.$zone A @10.53.0.4 > dig.out.ns4.test$n || ret=1
grep "status: SERVFAIL" dig.out.ns4.test$n > /dev/null || ret=1
nextpart ns4/named.run | grep "$pattern" > /dev/null || ret=1
n=$((n+1))
test "$ret" -eq 0 || echo_i "failed"
status=$((status+ret))
echo_i "check for validation deadlock - undelegated unsigned zone with CNAME to apex ($n)"
ret=0
zone=notdelegated.example
nextpart ns4/named.run > /dev/null
dig_with_opts www.$zone A @10.53.0.3 > dig.out.ns3.test$n || ret=1
grep "status: NOERROR" dig.out.ns3.test$n > /dev/null || ret=1
dig_with_opts www.$zone A @10.53.0.4 > dig.out.ns4.test$n || ret=1
grep "status: SERVFAIL" dig.out.ns4.test$n > /dev/null || ret=1
nextpart ns4/named.run | grep "aborting validation" || ret=1
n=$((n+1))
test "$ret" -eq 0 || echo_i "failed"
status=$((status+ret))
echo_i "exit status: $status"
[ $status -eq 0 ] || exit 1

View File

@@ -94,11 +94,16 @@ typedef enum { dns_quotatype_zone = 0, dns_quotatype_server } dns_quotatype_t;
#define DNS_FETCHOPT_NOEDNS0 0x00000008 /*%< Do not use EDNS. */
#define DNS_FETCHOPT_FORWARDONLY 0x00000010 /*%< Only use forwarders. */
#define DNS_FETCHOPT_NOVALIDATE 0x00000020 /*%< Disable validation. */
#define DNS_FETCHOPT_OBSOLETE1 0x00000040 /*%< Obsolete */
#define DNS_FETCHOPT_WANTNSID 0x00000080 /*%< Request NSID */
#define DNS_FETCHOPT_PREFETCH 0x00000100 /*%< Do prefetch */
#define DNS_FETCHOPT_NOCDFLAG 0x00000200 /*%< Don't set CD flag. */
#define DNS_FETCHOPT_NONTA 0x00000400 /*%< Ignore NTA table. */
#define DNS_FETCHOPT_VALIDATING \
0x00000040 /*%< This fetch was created by a \
* validator; its 'arg' is a \
* dns_validator_t object, and \
* can be stored and used for \
* validator loop detection. */
#define DNS_FETCHOPT_WANTNSID 0x00000080 /*%< Request NSID */
#define DNS_FETCHOPT_PREFETCH 0x00000100 /*%< Do prefetch */
#define DNS_FETCHOPT_NOCDFLAG 0x00000200 /*%< Don't set CD flag. */
#define DNS_FETCHOPT_NONTA 0x00000400 /*%< Ignore NTA table. */
/* RESERVED ECS 0x00000000 */
/* RESERVED ECS 0x00001000 */
/* RESERVED ECS 0x00002000 */
@@ -734,4 +739,19 @@ void
dns_resolver_setfuzzing(void);
#endif /* ifdef ENABLE_AFL */
dns_validator_t *
dns_fetch_validator(dns_fetch_t *fetch);
/*%<
* If a fetch was created with DNS_FETCHOPT_VALIDATING, then the
* validator that called it will have been passed in as the 'arg'
* parameter to dns_resolver_createfetch(). When another fetch is
* joined to the original fetch, this function allows the new caller
* to retrieve a pointer to the original caller; this is used in the
* validator to detect dependency loops (i.e., conditions in which
* a validator creates a fetch which cannot succeed because it's
* waiting on the same validator).
*
* Requires:
* \li 'fetch' is valid.
*/
ISC_LANG_ENDDECLS

View File

@@ -77,6 +77,18 @@ typedef struct dns_validatorevent {
*/
dns_name_t *name;
dns_rdatatype_t type;
/*
* Original query type (may differ from type in the case
* of CNAME responses).
*/
dns_rdatatype_t origtype;
/*
* Original fetch that triggered this validation.
*/
dns_fetch_t *fetch;
/*
* Rdata and RRSIG (if any) for positive responses.
*/
@@ -122,6 +134,7 @@ struct dns_validator {
unsigned int options;
unsigned int attributes;
dns_validatorevent_t *event;
dns_fetch_t *caller;
dns_fetch_t *fetch;
dns_validator_t *subvalidator;
dns_validator_t *parent;
@@ -161,17 +174,17 @@ ISC_LANG_BEGINDECLS
isc_result_t
dns_validator_create(dns_view_t *view, dns_name_t *name, dns_rdatatype_t type,
dns_rdataset_t *rdataset, dns_rdataset_t *sigrdataset,
dns_message_t *message, unsigned int options,
isc_task_t *task, isc_taskaction_t action, void *arg,
dns_rdatatype_t origtype, dns_rdataset_t *rdataset,
dns_rdataset_t *sigrdataset, dns_message_t *message,
unsigned int options, isc_task_t *task,
isc_taskaction_t action, void *arg, dns_fetch_t *fetch,
dns_validator_t **validatorp);
/*%<
* Start a DNSSEC validation.
*
* This validates a response to the question given by
* 'name' and 'type'.
* This validates a response RRset matching owner 'name' and type 'type'.
*
* To validate a positive response, the response data is
* When validating a positive response, the response data is
* given by 'rdataset' and 'sigrdataset'. If 'sigrdataset'
* is NULL, the data is presumed insecure and an attempt
* is made to prove its insecurity by finding the appropriate
@@ -184,13 +197,22 @@ dns_validator_create(dns_view_t *view, dns_name_t *name, dns_rdatatype_t type,
* is implemented yet). If the complete response message
* is not available, 'message' is NULL.
*
* To validate a negative response, the complete negative response
* message is given in 'message'. The 'rdataset', and
* When validating a negative response, the complete negative
* response message is given in 'message'. The 'rdataset', and
* 'sigrdataset' arguments must be NULL, but the 'name' and 'type'
* arguments must be provided.
*
* 'origtype' is the original query type that generated this
* response. This may differ from 'type', if 'type' is CNAME.
* This is used for loop detection, to prevent a case in
* which a validator attempts to restart the same fetch that was
* already waiting for the validator.
*
* The validation is performed in the context of 'view'.
*
* If set, 'fetch' references the resolver fetch that triggered
* this validation.
*
* When the validation finishes, a dns_validatorevent_t with
* the given 'action' and 'arg' are sent to 'task'.
* Its 'result' field will be ISC_R_SUCCESS iff the

View File

@@ -298,6 +298,7 @@ struct fetchctx {
/*% Not locked. */
unsigned int magic;
dns_resolver_t *res;
dns_fetch_t *fetch;
dns_fixedname_t fname;
dns_name_t *name;
dns_rdatatype_t type;
@@ -308,6 +309,7 @@ struct fetchctx {
isc_mem_t *mctx;
isc_stdtime_t now;
isc_task_t *task;
dns_validator_t *pval;
/* Atomic */
isc_refcount_t references;
@@ -919,9 +921,10 @@ valcreate(fetchctx_t *fctx, dns_message_t *message, dns_adbaddrinfo_t *addrinfo,
dns_name_t *name, dns_rdatatype_t type, dns_rdataset_t *rdataset,
dns_rdataset_t *sigrdataset, unsigned int valoptions,
isc_task_t *task) {
dns_validator_t *validator = NULL;
dns_valarg_t *valarg;
isc_result_t result;
dns_validator_t *validator = NULL;
dns_valarg_t *valarg = NULL;
dns_rdatatype_t origtype = dns_rdatatype_none;
valarg = isc_mem_get(fctx->mctx, sizeof(*valarg));
@@ -938,9 +941,13 @@ valcreate(fetchctx_t *fctx, dns_message_t *message, dns_adbaddrinfo_t *addrinfo,
valoptions &= ~DNS_VALIDATOR_DEFER;
}
result = dns_validator_create(fctx->res->view, name, type, rdataset,
sigrdataset, message, valoptions, task,
validated, valarg, &validator);
if (type == dns_rdatatype_cname) {
origtype = fctx->type;
}
result = dns_validator_create(fctx->res->view, name, type, origtype,
rdataset, sigrdataset, message,
valoptions, task, validated, valarg,
fctx->fetch, &validator);
RUNTIME_CHECK(result == ISC_R_SUCCESS);
if (result == ISC_R_SUCCESS) {
inc_stats(fctx->res, dns_resstatscounter_val);
@@ -4626,7 +4633,8 @@ fctx_create(dns_resolver_t *res, isc_task_t *task, const dns_name_t *name,
dns_rdatatype_t type, const dns_name_t *domain,
dns_rdataset_t *nameservers, const isc_sockaddr_t *client,
unsigned int options, unsigned int bucketnum, unsigned int depth,
isc_counter_t *qc, fetchctx_t **fctxp) {
isc_counter_t *qc, dns_fetch_t *fetch, dns_validator_t *pval,
fetchctx_t **fctxp) {
fetchctx_t *fctx = NULL;
isc_result_t result;
isc_result_t iresult;
@@ -4643,9 +4651,11 @@ fctx_create(dns_resolver_t *res, isc_task_t *task, const dns_name_t *name,
fctx = isc_mem_get(res->mctx, sizeof(*fctx));
*fctx = (fetchctx_t){
.fetch = fetch,
.type = type,
.qmintype = type,
.options = options,
.pval = pval,
.task = task,
.bucketnum = bucketnum,
.dbucketnum = RES_NOBUCKET,
@@ -10679,9 +10689,16 @@ dns_resolver_createfetch(dns_resolver_t *res, const dns_name_t *name,
}
if (fctx == NULL) {
void *validator = NULL;
if ((options & DNS_FETCHOPT_VALIDATING) != 0) {
options &= ~DNS_FETCHOPT_VALIDATING;
validator = arg;
}
result = fctx_create(res, task, name, type, domain, nameservers,
client, options, bucketnum, depth, qc,
&fctx);
fetch, validator, &fctx);
if (result != ISC_R_SUCCESS) {
goto unlock;
}
@@ -11459,3 +11476,14 @@ dns_resolver_setnonbackofftries(dns_resolver_t *resolver, unsigned int tries) {
resolver->nonbackofftries = tries;
}
dns_validator_t *
dns_fetch_validator(dns_fetch_t *fetch) {
fetchctx_t *fctx = NULL;
REQUIRE(DNS_FETCH_VALID(fetch));
fctx = fetch->private;
REQUIRE(VALID_FCTX(fctx));
return (fctx->pval);
}

View File

@@ -1001,19 +1001,21 @@ notfound:
static inline bool
check_deadlock(dns_validator_t *val, dns_name_t *name, dns_rdatatype_t type,
dns_rdataset_t *rdataset, dns_rdataset_t *sigrdataset) {
dns_validator_t *parent;
dns_validator_t *parent = NULL;
for (parent = val; parent != NULL; parent = parent->parent) {
if (parent->event != NULL && parent->event->type == type &&
if (parent->event != NULL &&
(parent->event->type == type ||
(parent->event->type == dns_rdatatype_cname &&
parent->event->origtype == type)) &&
dns_name_equal(parent->event->name, name) &&
/*
* As NSEC3 records are meta data you sometimes
* need to prove a NSEC3 record which says that
* itself doesn't exist.
*/
(parent->event->type != dns_rdatatype_nsec3 ||
rdataset == NULL || sigrdataset == NULL ||
parent->event->message == NULL ||
(type != dns_rdatatype_nsec3 || rdataset == NULL ||
sigrdataset == NULL || parent->event->message == NULL ||
parent->event->rdataset != NULL ||
parent->event->sigrdataset != NULL))
{
@@ -1022,6 +1024,26 @@ check_deadlock(dns_validator_t *val, dns_name_t *name, dns_rdatatype_t type,
"deadlock: aborting validation");
return (true);
}
if (parent->caller != NULL) {
dns_validator_t *pv =
dns_fetch_validator(parent->caller);
/*
* If the fetch that started this validator
* was started by another validator, we need to
* recursively check for a possible deadlock with
* that one (and its parents) too.
*/
if (pv != NULL &&
check_deadlock(pv, name, type, NULL, NULL)) {
validator_log(val, ISC_LOG_DEBUG(3),
"fetch/validator "
"loop detected: "
"aborting validation");
return (true);
}
}
}
return (false);
}
@@ -1032,7 +1054,8 @@ check_deadlock(dns_validator_t *val, dns_name_t *name, dns_rdatatype_t type,
static inline isc_result_t
create_fetch(dns_validator_t *val, dns_name_t *name, dns_rdatatype_t type,
isc_taskaction_t callback, const char *caller) {
unsigned int fopts = 0;
isc_result_t result;
unsigned int fopts = DNS_FETCHOPT_VALIDATING;
disassociate_rdatasets(val);
@@ -1051,10 +1074,12 @@ create_fetch(dns_validator_t *val, dns_name_t *name, dns_rdatatype_t type,
}
validator_logcreate(val, name, type, caller, "fetch");
return (dns_resolver_createfetch(
result = dns_resolver_createfetch(
val->view->resolver, name, type, NULL, NULL, NULL, NULL, 0,
fopts, 0, NULL, val->event->ev_sender, callback, val,
&val->frdataset, &val->fsigrdataset, &val->fetch));
&val->frdataset, &val->fsigrdataset, &val->fetch);
return (result);
}
/*%
@@ -1083,9 +1108,9 @@ create_validator(dns_validator_t *val, dns_name_t *name, dns_rdatatype_t type,
(DNS_VALIDATOR_NOCDFLAG | DNS_VALIDATOR_NONTA));
validator_logcreate(val, name, type, caller, "validator");
result = dns_validator_create(val->view, name, type, rdataset, sig,
NULL, vopts, val->task, action, val,
&val->subvalidator);
result = dns_validator_create(val->view, name, type, dns_rdatatype_none,
rdataset, sig, NULL, vopts, val->task,
action, val, NULL, &val->subvalidator);
if (result == ISC_R_SUCCESS) {
val->subvalidator->parent = val;
val->subvalidator->depth = val->depth + 1;
@@ -3100,9 +3125,10 @@ validator_start(isc_task_t *task, isc_event_t *event) {
isc_result_t
dns_validator_create(dns_view_t *view, dns_name_t *name, dns_rdatatype_t type,
dns_rdataset_t *rdataset, dns_rdataset_t *sigrdataset,
dns_message_t *message, unsigned int options,
isc_task_t *task, isc_taskaction_t action, void *arg,
dns_rdatatype_t origtype, dns_rdataset_t *rdataset,
dns_rdataset_t *sigrdataset, dns_message_t *message,
unsigned int options, isc_task_t *task,
isc_taskaction_t action, void *arg, dns_fetch_t *fetch,
dns_validator_t **validatorp) {
isc_result_t result = ISC_R_FAILURE;
dns_validator_t *val;
@@ -3122,6 +3148,7 @@ dns_validator_create(dns_view_t *view, dns_name_t *name, dns_rdatatype_t type,
event->result = ISC_R_FAILURE;
event->name = name;
event->type = type;
event->origtype = origtype;
event->rdataset = rdataset;
event->sigrdataset = sigrdataset;
event->message = message;
@@ -3132,6 +3159,7 @@ dns_validator_create(dns_view_t *view, dns_name_t *name, dns_rdatatype_t type,
val = isc_mem_get(view->mctx, sizeof(*val));
*val = (dns_validator_t){ .event = event,
.options = options,
.caller = fetch,
.task = task,
.action = action,
.arg = arg };