From 67551184938d35a11135a95f6fd44a99b08ac286 Mon Sep 17 00:00:00 2001
From: Stephen Morris <stephen@isc.org>
Date: Mon, 19 Mar 2018 16:12:47 +0000
Subject: [PATCH 1/6] Initial set of IDNA tests.

---
 bin/tests/system/Makefile.in            |   2 +-
 bin/tests/system/conf.sh.in             |   2 +-
 bin/tests/system/conf.sh.win32          |   2 +-
 bin/tests/system/idna/clean.sh          |  16 ++
 bin/tests/system/idna/ns1/named.conf.in |  29 +++
 bin/tests/system/idna/ns1/root.db       |  24 +++
 bin/tests/system/idna/prereq.sh         |  20 ++
 bin/tests/system/idna/setup.sh          |  16 ++
 bin/tests/system/idna/tests.sh          | 248 ++++++++++++++++++++++++
 util/copyrights                         |   7 +
 10 files changed, 363 insertions(+), 3 deletions(-)
 create mode 100644 bin/tests/system/idna/clean.sh
 create mode 100644 bin/tests/system/idna/ns1/named.conf.in
 create mode 100644 bin/tests/system/idna/ns1/root.db
 create mode 100644 bin/tests/system/idna/prereq.sh
 create mode 100644 bin/tests/system/idna/setup.sh
 create mode 100644 bin/tests/system/idna/tests.sh

diff --git a/bin/tests/system/Makefile.in b/bin/tests/system/Makefile.in
index ba3972939e..1b2fccbbe0 100644
--- a/bin/tests/system/Makefile.in
+++ b/bin/tests/system/Makefile.in
@@ -60,7 +60,7 @@ PARALLEL = rpzrecurse serve-stale dnssec \
 	   dns64 @DNSTAP@ dscp dsdigest dyndb \
 	   ednscompliance emptyzones \
 	   fetchlimit filter-aaaa formerr forward \
-	   geoip glue inline integrity ixfr keepalive \
+	   geoip glue idna inline integrity ixfr keepalive \
 	   legacy limits logfileconfig \
 	   masterfile masterformat metadata mkeys \
 	   names notify nslookup nsupdate nzd2nzf \
diff --git a/bin/tests/system/conf.sh.in b/bin/tests/system/conf.sh.in
index 32cec849d9..ecd01d677a 100644
--- a/bin/tests/system/conf.sh.in
+++ b/bin/tests/system/conf.sh.in
@@ -94,7 +94,7 @@ PARALLELDIRS="acl additional addzone allow-query auth autosign \
 	dns64 dnssec @DNSTAP@ dscp dsdigest dyndb \
 	ednscompliance emptyzones \
 	fetchlimit filter-aaaa formerr forward \
-	geoip glue inline integrity ixfr keepalive \
+	geoip glue idna inline integrity ixfr keepalive \
 	legacy limits logfileconfig \
         masterfile masterformat metadata mkeys \
         names notify nslookup nsupdate nzd2nzf \
diff --git a/bin/tests/system/conf.sh.win32 b/bin/tests/system/conf.sh.win32
index 146303af24..e46957e40b 100644
--- a/bin/tests/system/conf.sh.win32
+++ b/bin/tests/system/conf.sh.win32
@@ -91,7 +91,7 @@ SEQUENTIALDIRS="acl additional addzone autosign builtin \
 	 database digdelv dlv dlvauto dlz dlzexternal dname \
 	 dns64 dnssec @DNSTAP@ dscp dsdigest dyndb ecdsa eddsa \
 	 ednscompliance emptyzones \
-	 fetchlimit filter-aaaa formerr forward geoip glue gost inline ixfr \
+	 fetchlimit filter-aaaa formerr forward geoip glue gost idna inline ixfr \
 	 keepalive @KEYMGR@ legacy limits logfileconfig masterfile \
 	 masterformat metadata mkeys names notify nslookup nsupdate \
 	 nzd2nzf padding pending pipelined @PKCS11_TEST@ reclimit \
diff --git a/bin/tests/system/idna/clean.sh b/bin/tests/system/idna/clean.sh
new file mode 100644
index 0000000000..7a8e0d3aaf
--- /dev/null
+++ b/bin/tests/system/idna/clean.sh
@@ -0,0 +1,16 @@
+#!/bin/sh
+#
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+rm -f */named.memstats
+rm -f */named.run
+rm -f */named.conf
+rm -f dig.out.*
+rm -f ns*/named.lock
diff --git a/bin/tests/system/idna/ns1/named.conf.in b/bin/tests/system/idna/ns1/named.conf.in
new file mode 100644
index 0000000000..92acac9f4a
--- /dev/null
+++ b/bin/tests/system/idna/ns1/named.conf.in
@@ -0,0 +1,29 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+// NS1
+
+options {
+	query-source address 10.53.0.1;
+	port @PORT@;
+	pid-file "named.pid";
+	listen-on { 10.53.0.1; };
+	listen-on-v6 { fd92:7065:b8e:ffff::1; };
+	recursion no;
+	notify yes;
+	dnssec-enable no;
+	dnssec-validation no;
+};
+
+zone "." {
+	type master;
+	file "root.db";
+};
diff --git a/bin/tests/system/idna/ns1/root.db b/bin/tests/system/idna/ns1/root.db
new file mode 100644
index 0000000000..76ed2a0310
--- /dev/null
+++ b/bin/tests/system/idna/ns1/root.db
@@ -0,0 +1,24 @@
+; Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+;
+; This Source Code Form is subject to the terms of the Mozilla Public
+; License, v. 2.0. If a copy of the MPL was not distributed with this
+; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;
+; See the COPYRIGHT file distributed with this work for additional
+; information regarding copyright ownership.
+
+$TTL 300
+. 			IN SOA	gson.nominum.com. a.root.servers.nil. (
+				2000042100   	; serial
+				600         	; refresh
+				600         	; retry
+				1200    	; expire
+				600       	; minimum
+				)
+.			NS	a.root-servers.nil.
+a.root-servers.nil.	A	10.53.0.1
+a.root-servers.nil.	AAAA	fd92:7065:b8e:ffff::1
+
+example.		NS	ns2.example.
+ns2.example.		A	10.53.0.2
+ns2.example.		AAAA	fd92:7065:b8e:ffff::2
diff --git a/bin/tests/system/idna/prereq.sh b/bin/tests/system/idna/prereq.sh
new file mode 100644
index 0000000000..dad4c595d6
--- /dev/null
+++ b/bin/tests/system/idna/prereq.sh
@@ -0,0 +1,20 @@
+#!/bin/sh
+#
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+SYSTEMTESTTOP=..
+. $SYSTEMTESTTOP/conf.sh
+
+$FEATURETEST --with-idn
+if [ $? -ne 1 ]; then
+    echo_i "This test requires that BIND be built with IDN support"
+    exit 
+fi
+exit 0
diff --git a/bin/tests/system/idna/setup.sh b/bin/tests/system/idna/setup.sh
new file mode 100644
index 0000000000..7e606316f4
--- /dev/null
+++ b/bin/tests/system/idna/setup.sh
@@ -0,0 +1,16 @@
+#!/bin/sh
+#
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+SYSTEMTESTTOP=..
+. $SYSTEMTESTTOP/conf.sh
+
+$SHELL clean.sh
+copy_setports ns1/named.conf.in ns1/named.conf
diff --git a/bin/tests/system/idna/tests.sh b/bin/tests/system/idna/tests.sh
new file mode 100644
index 0000000000..15964de1c5
--- /dev/null
+++ b/bin/tests/system/idna/tests.sh
@@ -0,0 +1,248 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+SYSTEMTESTTOP=..
+. $SYSTEMTESTTOP/conf.sh
+
+# This set of tests check the behavior of the IDNA options in "dig".
+#
+# "dig" supports two IDNA-related options:
+#
+# +[no]idnin -  Translates a domain name into punycode format before sending
+#               the query to the server.
+# +[no]idnout - Translates the received punycode domain names into appropriate
+#               unicode characters before displaying.
+#
+# The tests run "dig" against an authoritative server configured with a minimal
+# root zone and nothing else.  As a result, all queries will result in an
+# NXDOMAIN.  The server will return the qname sent, which "dig" will display
+# according to the options selected.  This returned string is compared with
+# the qname originally sent.
+#
+# In the comments below, the following nomenclature (taken from RFC 5890) is
+# used:
+#
+# A-label: Label comprising ASCII characters that starts xn-- and whose
+#          characters after the xn-- are a valid output of the Punycode
+#          algorithm.
+#
+# Fake A-label: An A-label whose characters after the xn-- are not valid
+#          Punycode output.
+#
+# U-label: Unicode (native character) form of a label.
+#
+# For the purpose of this test script, U-labels do not include labels that
+# comprise purely ASCII characters, which are referred to as "ASCII-labels"
+# here. Valid ASCII-labels comprise letters, digits and hyphens and do not
+# start with a hyphen.
+#
+# References:
+# 1. http://www.unicode.org/reports/tr46/#Deviations
+# 2. http://www.unicode.org/reports/tr46/#IDNAComparison
+
+# Using dig insecure mode as we are not testing DNSSEC here
+DIGCMD="$DIG -i -p ${PORT} @10.53.0.1"
+
+# Initialize test count and status return
+n=0
+status=0
+
+
+# Function for extracting the question name reported by "dig".
+#
+# This is the first field after the line starting ";; QUESTION SECTION:".
+# The string returned includes the trailing period.
+
+qname() {
+    awk 'BEGIN { qs = 0; } \
+        /;; QUESTION SECTION:/ { qs = 1; next; } \
+        qs == 1 {sub(";", "", $1) ; print $1; exit 0; }' \
+        $1
+}
+
+# Function for performing the test where "dig" is expected to succeed.
+#
+#   $1 - Description of the test
+#   $2 - Dig command additional options
+#   $3 - Name being queried
+#   $4 - The name that should be displayed by "dig".  Note that names displayed
+#        by "dig" will always have a trailing period, so this parameter should
+#        have that period as well.
+
+idna_test() {
+    n=`expr $n + 1`
+    description=$1
+    if [ "$2" != "" ]; then
+        description="${description}: $2"
+    fi
+    echo_i "$description ($n)"
+
+    ret=0
+    $DIGCMD $2 $3 > dig.out.$n 2>&1
+    if [ $? -ne 0 ]; then
+        echo_i "failed: dig returned error status $?"
+        ret=1
+    else
+        actual=`qname dig.out.$n`
+        if [ "$4" != "$actual" ]; then
+            echo_i "failed: expected answer $4, actual result $actual"
+            ret=1
+        fi
+    fi
+    status=`expr $status + $ret`
+}
+
+# Function for performing test where "dig" is expected to fail
+#
+# $1 - $3: As for idna_test function
+
+idna_fail() {
+    n=`expr $n + 1`
+    description=$1
+    if [ "$2" != "" ]; then
+        description="${description}: $2"
+    fi
+    echo_i "$description ($n)"
+
+    ret=0
+    $DIGCMD $2 $3 > dig.out.$n 2>&1
+    if [ $? -eq 0 ]; then
+        echo_i "failed: dig command unexpectedly succeeded"
+        ret=1
+    fi
+    status=`expr $status + $ret`
+}
+
+# Tests of valid ASCII-label.
+#
+# +noidnin: The label is sent unchanged to the server.
+# +idnin:   The label is lower-cased and sent to the server.
+#
+# The +[no]idnout flag has no effect on the result.
+
+text="Checking valid ASCII label"
+idna_test "$text" ""                   LocalhosT localhost.
+idna_test "$text" "+noidnin +noidnout" LocalhosT LocalhosT.
+idna_test "$text" "+noidnin +idnout"   LocalhosT LocalhosT.
+idna_test "$text" "+idnin   +noidnout" LocalhosT localhost.
+idna_test "$text" "+idnin   +idnout"   LocalhosT localhost.
+
+
+
+# Tests of a valid U-label.
+#
+# +noidnin +noidnout: The label is sent as a unicode octet stream and dig will
+#                     display the string in the \nnn format.
+# +noidnin +idnout:   As for the previous case.
+# +idnin   +noidnout: The label is converted to the xn-- format.  "dig"
+#                     displays the returned xn-- text.
+# +idnin   +idnout:   The label is converted to the xn-- format.  "dig"
+#                     converts the returned xn-- string back to the original
+#                     unicode text.
+#
+# Note that ASCII characters are converted to lower-case.
+
+text="Checking valid non-ASCII label"
+idna_test "$text" ""                   "München" "münchen." 
+idna_test "$text" "+noidnin +noidnout" "München" "M\195\188nchen."
+idna_test "$text" "+noidnin +idnout"   "München" "M\195\188nchen."
+idna_test "$text" "+idnin   +noidnout" "München" "xn--mnchen-3ya."
+idna_test "$text" "+idnin   +idnout"   "München" "münchen." 
+
+
+
+# Tests of transitional processing of a valid U-label
+#
+# IDNA2003 introduced national character sets but, unfortunately, didn't
+# support several characters properly.  One of those was the German character
+# "ß" (the "Eszett" or "sharp s"), which was interpreted as "ss".  So the
+# domain “faß.de” domain (for example) was processed as “fass.de”.
+#
+# This was corrected in IDNA2008, although some vendors that adopted this
+# standard chose to keep the existing IDNA2003 translation for this character
+# to prevent problems (e.g. people visiting www.faß.example would, under
+# IDNA2003, go to www.fass.example but under IDNA2008 would end up at
+# www.fa\195\159.example - a different web site).
+#
+# BIND has adopted a hard transition, so this test checks that the transitional
+# mapping is not used.  The tests are essentially the same as for the valid
+# U-label.
+
+text="Checking that non-transitional IDNA processing is used"
+idna_test "$text" ""                   "faß.de" "faß.de."
+idna_test "$text" "+noidnin +noidnout" "faß.de" "fa\195\159.de."
+idna_test "$text" "+noidnin +idnout"   "faß.de" "fa\195\159.de."
+idna_test "$text" "+idnin   +noidnout" "faß.de" "xn--fa-hia.de."
+idna_test "$text" "+idnin   +idnout"   "faß.de" "faß.de."
+
+# Another problem character.  The final character in the first label mapped
+# onto the Greek sigma character ("σ") in IDNA2003.
+
+text="Second check that non-transitional IDNA processing is used"
+idna_test "$text" ""                   "βόλος.com" "βόλος.com." 
+idna_test "$text" "+noidnin +noidnout" "βόλος.com" "\206\178\207\140\206\187\206\191\207\130.com."
+idna_test "$text" "+noidnin +idnout"   "βόλος.com" "\206\178\207\140\206\187\206\191\207\130.com."
+idna_test "$text" "+idnin   +noidnout" "βόλος.com" "xn--nxasmm1c.com."
+idna_test "$text" "+idnin   +idnout"   "βόλος.com" "βόλος.com." 
+
+
+
+# Tests of a valid A-label (i.e. starting xn--)
+#
+# +noidnout: The string is sent as-is to the server and the returned qname is
+#            displayed in the same form.
+# +idnout:   The string is sent as-is to the server and the returned qname is
+#            displayed as the corresponding U-label.
+#
+# The "+[no]idnin" flag has no effect in these cases.
+
+text="Checking valid A-label"
+idna_test "$text" ""                   "xn--nxasmq6b.com" "βόλοσ.com." 
+idna_test "$text" "+noidnin +noidnout" "xn--nxasmq6b.com" "xn--nxasmq6b.com."
+idna_test "$text" "+noidnin +idnout"   "xn--nxasmq6b.com" "βόλοσ.com." 
+idna_test "$text" "+idnin +noidnout"   "xn--nxasmq6b.com" "xn--nxasmq6b.com."
+idna_test "$text" "+idnin +idnout"     "xn--nxasmq6b.com" "βόλοσ.com." A
+
+
+
+# Tests of a fake A-label
+#
+# +noidnin: The label is sent as-is to the server and dig will display the
+#           returned fake A-label in the same form.
+# +idnin:   "dig" should report that the label is not correct.
+#
+# The +[no]idnout options should not have any effect on the test.
+
+text="Checking invalid A-label"
+idna_fail "$text" ""                   "xn--ahahah"
+idna_test "$text" "+noidnin +noidnout" "xn--ahahah" "xn--ahahah."
+idna_test "$text" "+noidnin +idnout"   "xn--ahahah" "xn--ahahah."
+idna_fail "$text" "+idnin   +noidnout" "xn--ahahah"
+idna_fail "$text" "+idnin   +idnout"   "xn--ahahah"
+
+
+
+# Tests of a valid unicode string but an invalid U-label
+#
+# Symbols are not valid IDNA names.
+#
+# +noidnin: "dig" should send unicode octets to the server and display the
+#           returned qname in the same form.
+# +idnin:   "dig" should generate an error.
+#
+# The +[no]idnout options should not have any effect on the test.
+
+text="Checking invalid U-label"
+idna_fail "$text" ""                   "❤︎.com"
+idna_test "$text" "+noidnin +noidnout" "❤︎.com" "\226\157\164\239\184\142.com."
+idna_test "$text" "+noidnin +idnout"   "❤︎.com" "\226\157\164\239\184\142.com."
+idna_fail "$text" "+idnin   +noidnout" "❤︎.com"
+idna_fail "$text" "+idnin   +idnout"   "❤︎.com"
+
+exit $status
diff --git a/util/copyrights b/util/copyrights
index 786ae9661e..e32ce18611 100644
--- a/util/copyrights
+++ b/util/copyrights
@@ -1374,6 +1374,13 @@
 ./bin/tests/system/gost/prereq.sh		SH	2010,2012,2014,2016,2018
 ./bin/tests/system/gost/setup.sh		SH	2010,2012,2014,2016,2017,2018
 ./bin/tests/system/gost/tests.sh		SH	2010,2012,2013,2016,2018
+./bin/tests/system/idna/clean.sh		SH	2018
+./bin/tests/system/idna/ns1/named.conf		CONF-C	2018
+./bin/tests/system/idna/ns1/named.conf.in	CONF-C	2018
+./bin/tests/system/idna/ns1/root.db		ZONE	2018
+./bin/tests/system/idna/prereq.sh		SH	2018
+./bin/tests/system/idna/setup.sh		SH	2018
+./bin/tests/system/idna/tests.sh		SH	2018
 ./bin/tests/system/ifconfig.bat			BAT	2016,2018
 ./bin/tests/system/ifconfig.sh			SH	2000,2001,2002,2003,2004,2007,2008,2009,2010,2012,2013,2016,2018
 ./bin/tests/system/inline/.gitignore		X	2014,2018

From dc7b3d7597abc330e4cc3d5101b707418e405f38 Mon Sep 17 00:00:00 2001
From: Stephen Morris <stephen@isc.org>
Date: Thu, 22 Mar 2018 16:30:36 +0000
Subject: [PATCH 2/6] Added additional tests for invalid punycode strings.

---
 bin/tests/system/idna/tests.sh | 305 ++++++++++++++++++++-------------
 1 file changed, 190 insertions(+), 115 deletions(-)

diff --git a/bin/tests/system/idna/tests.sh b/bin/tests/system/idna/tests.sh
index 15964de1c5..7684c7a137 100644
--- a/bin/tests/system/idna/tests.sh
+++ b/bin/tests/system/idna/tests.sh
@@ -10,12 +10,18 @@
 SYSTEMTESTTOP=..
 . $SYSTEMTESTTOP/conf.sh
 
+
+
 # This set of tests check the behavior of the IDNA options in "dig".
 #
 # "dig" supports two IDNA-related options:
 #
 # +[no]idnin -  Translates a domain name into punycode format before sending
 #               the query to the server.
+#
+#               Should the input name be a punycode name, "dig +idnin" will also
+#               validate the punycode, rejecting it if it is invalid.
+#
 # +[no]idnout - Translates the received punycode domain names into appropriate
 #               unicode characters before displaying.
 #
@@ -54,9 +60,11 @@ n=0
 status=0
 
 
-# Function for extracting the question name reported by "dig".
+# Function for extracting the qname from the response
+#
+# This is the first field in the line after the line starting
+# ";; QUESTION SECTION:".
 #
-# This is the first field after the line starting ";; QUESTION SECTION:".
 # The string returned includes the trailing period.
 
 qname() {
@@ -66,14 +74,14 @@ qname() {
         $1
 }
 
-# Function for performing the test where "dig" is expected to succeed.
+# Function for performing a test where "dig" is expected to succeed.
 #
 #   $1 - Description of the test
 #   $2 - Dig command additional options
 #   $3 - Name being queried
-#   $4 - The name that should be displayed by "dig".  Note that names displayed
-#        by "dig" will always have a trailing period, so this parameter should
-#        have that period as well.
+#   $4 - The name that is expected to be displayed by "dig".  Note that names
+#        displayed by "dig" will always have a trailing period, so this
+#        parameter should have that period as well.
 
 idna_test() {
     n=`expr $n + 1`
@@ -86,7 +94,7 @@ idna_test() {
     ret=0
     $DIGCMD $2 $3 > dig.out.$n 2>&1
     if [ $? -ne 0 ]; then
-        echo_i "failed: dig returned error status $?"
+        echo_i "failed: dig command returned non-zero status"
         ret=1
     else
         actual=`qname dig.out.$n`
@@ -98,9 +106,11 @@ idna_test() {
     status=`expr $status + $ret`
 }
 
-# Function for performing test where "dig" is expected to fail
+# Function for performing a test where "dig" is expected to fail
 #
-# $1 - $3: As for idna_test function
+#   $1 - Description of the test
+#   $2 - Dig command additional options
+#   $3 - Name being queried
 
 idna_fail() {
     n=`expr $n + 1`
@@ -119,130 +129,195 @@ idna_fail() {
     status=`expr $status + $ret`
 }
 
-# Tests of valid ASCII-label.
+# Function to check that case is preserved for an all-ASCII label.
 #
-# +noidnin: The label is sent unchanged to the server.
-# +idnin:   The label is lower-cased and sent to the server.
+# Without IDNA support, case-preservation is the expected behavior.
 #
-# The +[no]idnout flag has no effect on the result.
-
-text="Checking valid ASCII label"
-idna_test "$text" ""                   LocalhosT localhost.
-idna_test "$text" "+noidnin +noidnout" LocalhosT LocalhosT.
-idna_test "$text" "+noidnin +idnout"   LocalhosT LocalhosT.
-idna_test "$text" "+idnin   +noidnout" LocalhosT localhost.
-idna_test "$text" "+idnin   +idnout"   LocalhosT localhost.
-
-
-
-# Tests of a valid U-label.
+# With IDNA support... not really.  IDNA maps uppercase ASCII characters to
+# their lower-case equivalent.  When IDNA support in "dig" was updated to
+# non-transitional IDNA 2008, the switch "+idnin" was added and made the default
+# behaviour. This meant that the command "dig LocalhosT" (no command switches)
+# sends the qname "localhost", a change in behavior from earlier versions.
 #
-# +noidnin +noidnout: The label is sent as a unicode octet stream and dig will
-#                     display the string in the \nnn format.
-# +noidnin +idnout:   As for the previous case.
-# +idnin   +noidnout: The label is converted to the xn-- format.  "dig"
-#                     displays the returned xn-- text.
-# +idnin   +idnout:   The label is converted to the xn-- format.  "dig"
-#                     converts the returned xn-- string back to the original
-#                     unicode text.
-#
-# Note that ASCII characters are converted to lower-case.
+# This was felt to be confusing to the significant number of users who are
+# not interested in IDNA. For this reason, after "dig" passes the input qname
+# through the IDNA conversion, is does a case-insensitive comparison with the
+# result.  If the two are the same, "dig" can conclude that the qname is
+# entirely ASCII and is uses the entered string instead of the converted string
+# as the qname.
 
-text="Checking valid non-ASCII label"
-idna_test "$text" ""                   "München" "münchen." 
-idna_test "$text" "+noidnin +noidnout" "München" "M\195\188nchen."
-idna_test "$text" "+noidnin +idnout"   "München" "M\195\188nchen."
-idna_test "$text" "+idnin   +noidnout" "München" "xn--mnchen-3ya."
-idna_test "$text" "+idnin   +idnout"   "München" "münchen." 
+ascii_case_preservation_test() {
+    text="Checking valid ASCII label"
+    idna_test "$text" ""                   LocalhosT LocalhosT.
+    idna_test "$text" "+noidnin +noidnout" LocalhosT LocalhosT.
+    idna_test "$text" "+noidnin +idnout"   LocalhosT LocalhosT.
+    idna_test "$text" "+idnin   +noidnout" LocalhosT LocalhosT.
+    idna_test "$text" "+idnin   +idnout"   LocalhosT LocalhosT.
+}
+
+# Function to perform the tests if IDNA is enabled.
+
+idna_enabled_test() {
+    echo_i "IDNA is enabled, all IDNA tests will be performed"
+    # Check that case is preserved on an ASCII label.
+
+    ascii_case_preservation_test
+
+
+    # Test of a valid U-label
+    #
+    # +noidnin +noidnout: The label is sent as a unicode octet stream and dig
+    #                     will display the string in the \nnn format.
+    # +noidnin +idnout:   As for the previous case.
+    # +idnin   +noidnout: The label is converted to the xn-- format.  "dig"
+    #                     displays the returned xn-- text.
+    # +idnin   +idnout:   The label is converted to the xn-- format.  "dig"
+    #                     converts the returned xn-- string back to the original
+    #                     unicode text.
+    #
+    # Note that ASCII characters are converted to lower-case.
+
+    text="Checking valid non-ASCII label"
+    idna_test "$text" ""                   "München" "münchen." 
+    idna_test "$text" "+noidnin +noidnout" "München" "M\195\188nchen."
+    idna_test "$text" "+noidnin +idnout"   "München" "M\195\188nchen."
+    idna_test "$text" "+idnin   +noidnout" "München" "xn--mnchen-3ya."
+    idna_test "$text" "+idnin   +idnout"   "München" "münchen." 
+
+
+    # Tests of transitional processing of a valid U-label
+    #
+    # IDNA2003 introduced national character sets but, unfortunately, didn't
+    # support several characters properly.  One of those was the German
+    # character "ß" (the "Eszett" or "sharp s"), which was interpreted as "ss".
+    # So the domain “faß.de” domain (for example) was processed as “fass.de”.
+    #
+    # This was corrected in IDNA2008, although some vendors that adopted this
+    # standard chose to keep the existing IDNA2003 translation for this
+    # character to prevent problems (e.g. people visiting www.faß.example would,
+    # under IDNA2003, go to www.fass.example but under IDNA2008 would end up at
+    # www.fa\195\159.example - a different web site).
+    #
+    # BIND has adopted a hard transition, so this test checks that these
+    # transitional mapping is not used.  The tests are essentially the same as
+    # for the valid U-label.
+
+    text="Checking that non-transitional IDNA processing is used"
+    idna_test "$text" ""                   "faß.de" "faß.de."
+    idna_test "$text" "+noidnin +noidnout" "faß.de" "fa\195\159.de."
+    idna_test "$text" "+noidnin +idnout"   "faß.de" "fa\195\159.de."
+    idna_test "$text" "+idnin   +noidnout" "faß.de" "xn--fa-hia.de."
+    idna_test "$text" "+idnin   +idnout"   "faß.de" "faß.de."
+
+    # Another problem character.  The final character in the first label mapped
+    # onto the Greek sigma character ("σ") in IDNA2003.
+
+    text="Second check that non-transitional IDNA processing is used"
+    idna_test "$text" ""                   "βόλος.com" "βόλος.com." 
+    idna_test "$text" "+noidnin +noidnout" "βόλος.com" "\206\178\207\140\206\187\206\191\207\130.com."
+    idna_test "$text" "+noidnin +idnout"   "βόλος.com" "\206\178\207\140\206\187\206\191\207\130.com."
+    idna_test "$text" "+idnin   +noidnout" "βόλος.com" "xn--nxasmm1c.com."
+    idna_test "$text" "+idnin   +idnout"   "βόλος.com" "βόλος.com." 
 
 
 
-# Tests of transitional processing of a valid U-label
-#
-# IDNA2003 introduced national character sets but, unfortunately, didn't
-# support several characters properly.  One of those was the German character
-# "ß" (the "Eszett" or "sharp s"), which was interpreted as "ss".  So the
-# domain “faß.de” domain (for example) was processed as “fass.de”.
-#
-# This was corrected in IDNA2008, although some vendors that adopted this
-# standard chose to keep the existing IDNA2003 translation for this character
-# to prevent problems (e.g. people visiting www.faß.example would, under
-# IDNA2003, go to www.fass.example but under IDNA2008 would end up at
-# www.fa\195\159.example - a different web site).
-#
-# BIND has adopted a hard transition, so this test checks that the transitional
-# mapping is not used.  The tests are essentially the same as for the valid
-# U-label.
+    # Tests of a valid A-label (i.e. starting xn--)
+    #
+    # +noidnout: The string is sent as-is to the server and the returned qname
+    #            is displayed in the same form.
+    # +idnout:   The string is sent as-is to the server and the returned qname
+    #            is displayed as the corresponding U-label.
+    #
+    # The "+[no]idnin" flag has no effect in these cases.
 
-text="Checking that non-transitional IDNA processing is used"
-idna_test "$text" ""                   "faß.de" "faß.de."
-idna_test "$text" "+noidnin +noidnout" "faß.de" "fa\195\159.de."
-idna_test "$text" "+noidnin +idnout"   "faß.de" "fa\195\159.de."
-idna_test "$text" "+idnin   +noidnout" "faß.de" "xn--fa-hia.de."
-idna_test "$text" "+idnin   +idnout"   "faß.de" "faß.de."
-
-# Another problem character.  The final character in the first label mapped
-# onto the Greek sigma character ("σ") in IDNA2003.
-
-text="Second check that non-transitional IDNA processing is used"
-idna_test "$text" ""                   "βόλος.com" "βόλος.com." 
-idna_test "$text" "+noidnin +noidnout" "βόλος.com" "\206\178\207\140\206\187\206\191\207\130.com."
-idna_test "$text" "+noidnin +idnout"   "βόλος.com" "\206\178\207\140\206\187\206\191\207\130.com."
-idna_test "$text" "+idnin   +noidnout" "βόλος.com" "xn--nxasmm1c.com."
-idna_test "$text" "+idnin   +idnout"   "βόλος.com" "βόλος.com." 
+    text="Checking valid A-label"
+    idna_test "$text" ""                   "xn--nxasmq6b.com" "βόλοσ.com." 
+    idna_test "$text" "+noidnin +noidnout" "xn--nxasmq6b.com" "xn--nxasmq6b.com."
+    idna_test "$text" "+noidnin +idnout"   "xn--nxasmq6b.com" "βόλοσ.com." 
+    idna_test "$text" "+idnin +noidnout"   "xn--nxasmq6b.com" "xn--nxasmq6b.com."
+    idna_test "$text" "+idnin +idnout"     "xn--nxasmq6b.com" "βόλοσ.com."
 
 
 
-# Tests of a valid A-label (i.e. starting xn--)
-#
-# +noidnout: The string is sent as-is to the server and the returned qname is
-#            displayed in the same form.
-# +idnout:   The string is sent as-is to the server and the returned qname is
-#            displayed as the corresponding U-label.
-#
-# The "+[no]idnin" flag has no effect in these cases.
+    # Tests of invalid A-labels
+    #
+    # +noidnin: The label is sent as-is to the server and dig will display the
+    #           returned fake A-label in the same form.
+    # +idnin:   "dig" should report that the label is not correct.
+    #
+    # +[no]idnout: If the label makes it to the server (via +noidnin), "dig"
+    #           should report an error if +idnout is specified.
 
-text="Checking valid A-label"
-idna_test "$text" ""                   "xn--nxasmq6b.com" "βόλοσ.com." 
-idna_test "$text" "+noidnin +noidnout" "xn--nxasmq6b.com" "xn--nxasmq6b.com."
-idna_test "$text" "+noidnin +idnout"   "xn--nxasmq6b.com" "βόλοσ.com." 
-idna_test "$text" "+idnin +noidnout"   "xn--nxasmq6b.com" "xn--nxasmq6b.com."
-idna_test "$text" "+idnin +idnout"     "xn--nxasmq6b.com" "βόλοσ.com." A
+    # The minimum length of a punycode A-label is 7 characters.  Check that
+    # a shorter label is detected and rejected.
+
+    text="Checking punycode label shorter than minimum valid length"
+    idna_fail "$text" ""                   "xn--xx"
+    idna_test "$text" "+noidnin +noidnout" "xn--xx" "xn--xx."
+    idna_fail "$text" "+noidnin   +idnout" "xn--xx"
+    idna_fail "$text" "+idnin   +noidnout" "xn--xx"
+    idna_fail "$text" "+idnin     +idnout" "xn--xx"
+
+    # Fake A-label - the string does not translate to anything.
+
+    text="Checking fake A-label"
+    idna_fail "$text" ""                   "xn--ahahah"
+    idna_test "$text" "+noidnin +noidnout" "xn--ahahah" "xn--ahahah."
+#   The next test is commented out as libidn2 gives invalid output for the
+#   invalid punycode name.  Until this is fixed, the test will be skipped.
+#   idna_test "$text" "+noidnin   +idnout" "xn--ahahah" "xn--ahahah."
+    echo_i "(fake A-label: +noidnin +idnout test skipped due to libidn2 issues)"
+    idna_fail "$text" "+idnin   +noidnout" "xn--ahahah"
+    idna_fail "$text" "+idnin     +idnout" "xn--ahahah"
+
+    # Too long a label. The punycode string is too long (at 64 characters).
+    # BIND rejects such labels: with +idnin
+
+    label="xn--xflod18hstflod18hstflod18hstflod18hstflod18hstflod18-1iejjjj"
+    text="Checking punycode label shorter than minimum valid length"
+    idna_fail "$text" ""                   "$label"
+    idna_fail "$text" "+noidnin +noidnout" "$label"
+    idna_fail "$text" "+noidnin   +idnout" "$label"
+    idna_fail "$text" "+idnin   +noidnout" "$label"
+    idna_fail "$text" "+idnin     +idnout" "$label"
+
+    
 
 
+    # Tests of a valid unicode string but an invalid U-label
+    #
+    # Symbols are not valid IDNA names.
+    #
+    # +noidnin: "dig" should send unicode octets to the server and display the
+    #           returned qname in the same form.
+    # +idnin:   "dig" should generate an error.
+    #
+    # The +[no]idnout options should not have any effect on the test.
 
-# Tests of a fake A-label
-#
-# +noidnin: The label is sent as-is to the server and dig will display the
-#           returned fake A-label in the same form.
-# +idnin:   "dig" should report that the label is not correct.
-#
-# The +[no]idnout options should not have any effect on the test.
-
-text="Checking invalid A-label"
-idna_fail "$text" ""                   "xn--ahahah"
-idna_test "$text" "+noidnin +noidnout" "xn--ahahah" "xn--ahahah."
-idna_test "$text" "+noidnin +idnout"   "xn--ahahah" "xn--ahahah."
-idna_fail "$text" "+idnin   +noidnout" "xn--ahahah"
-idna_fail "$text" "+idnin   +idnout"   "xn--ahahah"
+    text="Checking invalid U-label"
+    idna_fail "$text" ""                   "🧦.com"
+    idna_test "$text" "+noidnin +noidnout" "🧦.com" "\240\159\167\166.com."
+    idna_test "$text" "+noidnin +idnout"   "🧦.com" "\240\159\167\166.com."
+    idna_fail "$text" "+idnin   +noidnout" "🧦.com"
+    idna_fail "$text" "+idnin   +idnout"   "🧦.com"
+}
 
 
+# Function to perform tests if IDNA is not enabled.
 
-# Tests of a valid unicode string but an invalid U-label
-#
-# Symbols are not valid IDNA names.
-#
-# +noidnin: "dig" should send unicode octets to the server and display the
-#           returned qname in the same form.
-# +idnin:   "dig" should generate an error.
-#
-# The +[no]idnout options should not have any effect on the test.
+idna_disabled_test() {
+    echo_i "IDNA is disabled, only case mapping tests will be performed"
+    case_preservation_test
+}
 
-text="Checking invalid U-label"
-idna_fail "$text" ""                   "❤︎.com"
-idna_test "$text" "+noidnin +noidnout" "❤︎.com" "\226\157\164\239\184\142.com."
-idna_test "$text" "+noidnin +idnout"   "❤︎.com" "\226\157\164\239\184\142.com."
-idna_fail "$text" "+idnin   +noidnout" "❤︎.com"
-idna_fail "$text" "+idnin   +idnout"   "❤︎.com"
+
+# Main test begins here
+
+$FEATURETEST --with-idn
+if [ $? -eq 1 ]; then
+    idna_enabled_test
+else
+    idna_disabled_test
+fi
 
 exit $status

From 5009724f408c3080eddef0eb9b86a9d863780106 Mon Sep 17 00:00:00 2001
From: Stephen Morris <stephen@isc.org>
Date: Thu, 22 Mar 2018 19:04:12 +0000
Subject: [PATCH 3/6] Fix check for the presence of IDNA.

---
 bin/tests/system/feature-test.c |  2 +-
 bin/tests/system/idna/prereq.sh | 20 --------------------
 bin/tests/system/idna/tests.sh  |  4 ++--
 3 files changed, 3 insertions(+), 23 deletions(-)
 delete mode 100644 bin/tests/system/idna/prereq.sh

diff --git a/bin/tests/system/feature-test.c b/bin/tests/system/feature-test.c
index ed2a5b2fbe..0c20930cc4 100644
--- a/bin/tests/system/feature-test.c
+++ b/bin/tests/system/feature-test.c
@@ -153,7 +153,7 @@ main(int argc, char **argv) {
 	}
 
 	if (strcmp(argv[1], "--with-idn") == 0) {
-#ifdef WITH_IDN
+#ifdef WITH_LIBIDN2
 		return (0);
 #else
 		return (1);
diff --git a/bin/tests/system/idna/prereq.sh b/bin/tests/system/idna/prereq.sh
deleted file mode 100644
index dad4c595d6..0000000000
--- a/bin/tests/system/idna/prereq.sh
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/bin/sh
-#
-# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-#
-# See the COPYRIGHT file distributed with this work for additional
-# information regarding copyright ownership.
-
-SYSTEMTESTTOP=..
-. $SYSTEMTESTTOP/conf.sh
-
-$FEATURETEST --with-idn
-if [ $? -ne 1 ]; then
-    echo_i "This test requires that BIND be built with IDN support"
-    exit 
-fi
-exit 0
diff --git a/bin/tests/system/idna/tests.sh b/bin/tests/system/idna/tests.sh
index 7684c7a137..711a0250cb 100644
--- a/bin/tests/system/idna/tests.sh
+++ b/bin/tests/system/idna/tests.sh
@@ -307,14 +307,14 @@ idna_enabled_test() {
 
 idna_disabled_test() {
     echo_i "IDNA is disabled, only case mapping tests will be performed"
-    case_preservation_test
+    ascii_case_preservation_test
 }
 
 
 # Main test begins here
 
 $FEATURETEST --with-idn
-if [ $? -eq 1 ]; then
+if [ $? -eq 0 ]; then
     idna_enabled_test
 else
     idna_disabled_test

From 8cf29aafec012da0f65607cfcde54d0f5c758b64 Mon Sep 17 00:00:00 2001
From: Stephen Morris <stephen@isc.org>
Date: Fri, 23 Mar 2018 10:23:15 +0000
Subject: [PATCH 4/6] Set known locale prior to running tests.

---
 bin/tests/system/idna/tests.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/bin/tests/system/idna/tests.sh b/bin/tests/system/idna/tests.sh
index 711a0250cb..b4aaf6e71b 100644
--- a/bin/tests/system/idna/tests.sh
+++ b/bin/tests/system/idna/tests.sh
@@ -10,6 +10,9 @@
 SYSTEMTESTTOP=..
 . $SYSTEMTESTTOP/conf.sh
 
+# Set known locale for the tests
+LC_ALL="en_US.UTF-8"
+export LC_ALL
 
 
 # This set of tests check the behavior of the IDNA options in "dig".

From 4dad408ead18ea3202e3745a34b566a363d39d6c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ond=C5=99ej=20Sur=C3=BD?= <ondrej@sury.org>
Date: Fri, 23 Mar 2018 11:14:18 +0000
Subject: [PATCH 5/6] Use C.UTF-8 if available, and en_US.UTF-8 is available,
 otherwise do nothing and let the test break

---
 bin/tests/system/idna/tests.sh | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/bin/tests/system/idna/tests.sh b/bin/tests/system/idna/tests.sh
index b4aaf6e71b..19a086dff2 100644
--- a/bin/tests/system/idna/tests.sh
+++ b/bin/tests/system/idna/tests.sh
@@ -11,9 +11,13 @@ SYSTEMTESTTOP=..
 . $SYSTEMTESTTOP/conf.sh
 
 # Set known locale for the tests
-LC_ALL="en_US.UTF-8"
-export LC_ALL
 
+if locale -a | grep -qE "^C\\.(UTF-8|utf8)"; then
+    LC_ALL="C.UTF-8"
+elif locale -a | grep -qE "^en_US\\.(UTF-8|utf8)"; then
+    LC_ALL="en_US.UTF-8"
+fi
+export LC_ALL
 
 # This set of tests check the behavior of the IDNA options in "dig".
 #

From e0e81288cde88f04bc8996b98fe421f7365a8a2f Mon Sep 17 00:00:00 2001
From: Stephen Morris <stephen@isc.org>
Date: Tue, 3 Apr 2018 20:02:35 +0100
Subject: [PATCH 6/6] Check libidn2 version before proceding with one of the
 tests.

A known issue in libidn2 causes one of the tests of a fake A-label
to fail.  The problem should be corrected in version 2.0.5 of
libidn2.
---
 bin/tests/system/idna/tests.sh | 54 +++++++++++++++++++++++++++++++---
 1 file changed, 50 insertions(+), 4 deletions(-)

diff --git a/bin/tests/system/idna/tests.sh b/bin/tests/system/idna/tests.sh
index 19a086dff2..3a9b91b442 100644
--- a/bin/tests/system/idna/tests.sh
+++ b/bin/tests/system/idna/tests.sh
@@ -136,6 +136,46 @@ idna_fail() {
     status=`expr $status + $ret`
 }
 
+# Check if current version of libidn2 is >= a given version
+#
+# This requires that:
+# a) "pkg-config" exists on the system
+# b) The libidn2 installed has an associated ".pc" file
+# c) The system sort command supports "-V"
+#
+# $1 - Minimum version required
+#
+# Returns:
+# 0 - Version check is OK, libidn2 at required version or greater.
+# 1 - Version check was made, but libidn2 not at required version.
+# 2 - Could not carry out version check
+
+libidn_version_check() {
+    ret=2
+    if [ -n "`command -v pkg-config`" ]; then
+        version=`pkg-config --modversion --silence-errors libidn2`
+        if [ -n "$version" ]; then
+            # Does the sort command have a "-V" flag on this system?
+            sort -V 2>&1 > /dev/null << .
+.
+            if [ $? -eq 0 ]; then
+                # Sort -V exists.  Sort the IDN version and the minimum version
+                # required.  If the IDN version is greater than or equal to that
+                # version, it will appear last in the list.
+                last_version=`printf "%s\n" $version $1 | sort -V | tail -1`
+                if [ "$version" = "$last_version" ]; then
+                    ret=0
+                else
+                    ret=1
+                fi
+            fi
+        fi
+    fi
+
+    return $ret
+}
+
+
 # Function to check that case is preserved for an all-ASCII label.
 #
 # Without IDNA support, case-preservation is the expected behavior.
@@ -270,10 +310,16 @@ idna_enabled_test() {
     text="Checking fake A-label"
     idna_fail "$text" ""                   "xn--ahahah"
     idna_test "$text" "+noidnin +noidnout" "xn--ahahah" "xn--ahahah."
-#   The next test is commented out as libidn2 gives invalid output for the
-#   invalid punycode name.  Until this is fixed, the test will be skipped.
-#   idna_test "$text" "+noidnin   +idnout" "xn--ahahah" "xn--ahahah."
-    echo_i "(fake A-label: +noidnin +idnout test skipped due to libidn2 issues)"
+
+    # Owing to issues with libdns, the next test will fail for versions of
+    # libidn earlier than 2.0.5.  For this reason, get the version (if
+    # available) and compare with 2.0.5.
+    libidn_version_check 2.0.5
+    if [ $? -ne 0 ]; then
+        echo_i "Skipping fake A-label +noidnin +idnout test (libidn2 version issues)"
+    else
+        idna_test "$text" "+noidnin   +idnout" "xn--ahahah" "xn--ahahah."
+    fi
     idna_fail "$text" "+idnin   +noidnout" "xn--ahahah"
     idna_fail "$text" "+idnin     +idnout" "xn--ahahah"