From 82b03e949f34449a459f9f27b2dc724cfe8440b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20K=C4=99pie=C5=84?= Date: Thu, 12 Jul 2018 13:35:10 +0200 Subject: [PATCH 1/7] Add "-f" to command line arguments for autoreconf in autogen.sh Depending on tool versions being used, "autoreconf -i" may not update all Autoconf-generated files, which in turn may result in build errors. Make autogen.sh call autoreconf with the "-f" command line argument to ensure all Autoconf-generated files are updated when autogen.sh is run. (cherry picked from commit 45e77a3680c53c5f48657972a2d718079778fbb9) --- autogen.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autogen.sh b/autogen.sh index 7add8eabc1..583b00d870 100755 --- a/autogen.sh +++ b/autogen.sh @@ -10,4 +10,4 @@ # information regarding copyright ownership. # Run this script after modifying configure.in to generate configure -autoreconf -i +autoreconf -f -i From 0561d6bb0c459c9ffae03cb61a59216ef2b9c1d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20K=C4=99pie=C5=84?= Date: Thu, 12 Jul 2018 13:35:10 +0200 Subject: [PATCH 2/7] Update ATF path in the ./configure invocation used during CI The Docker images used for CI install ATF to /usr, not /usr/local. Update the ./configure invocation in .gitlab-ci.yml accordingly in order to prevent confusion. (cherry picked from commit 12df6829d17925f03ab752872cd93ad228f69e7e) --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 83092761e1..f73f8ddb4e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -100,7 +100,7 @@ stages: - test -w "${CCACHE_DIR}" && export PATH="/usr/lib/ccache:${PATH}" - autoreconf -fi script: - - ./configure --enable-developer --with-libtool --disable-static --with-atf=/usr/local --with-libidn2 + - ./configure --enable-developer --with-libtool --disable-static --with-atf=/usr --with-libidn2 - make -j${PARALLEL_JOBS_BUILD:-1} -k all V=1 artifacts: expire_in: '1 hour' From 4c7eea4437dbdf0027c64503441eb51572f35477 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20K=C4=99pie=C5=84?= Date: Thu, 12 Jul 2018 13:35:10 +0200 Subject: [PATCH 3/7] Include conf.sh from all prereq.sh scripts Every prereq.sh script must include bin/tests/system/conf.sh, otherwise if some prerequisite is not met, errors about echo_i not being found will be printed instead of actual error messages. (cherry picked from commit cc0e8cda71fb60dee58ff8f91d49bb851e648dc6) --- bin/tests/system/ixfr/prereq.sh | 3 +++ bin/tests/system/statistics/prereq.sh | 3 +++ bin/tests/system/statschannel/prereq.sh | 3 +++ bin/tests/system/upforwd/prereq.sh | 3 +++ bin/tests/system/xfer/prereq.sh | 3 +++ 5 files changed, 15 insertions(+) diff --git a/bin/tests/system/ixfr/prereq.sh b/bin/tests/system/ixfr/prereq.sh index affe54bfcf..0e299f4db8 100644 --- a/bin/tests/system/ixfr/prereq.sh +++ b/bin/tests/system/ixfr/prereq.sh @@ -9,6 +9,9 @@ # See the COPYRIGHT file distributed with this work for additional # information regarding copyright ownership. +SYSTEMTESTTOP=.. +. $SYSTEMTESTTOP/conf.sh + if $PERL -e 'use Net::DNS;' 2>/dev/null then : diff --git a/bin/tests/system/statistics/prereq.sh b/bin/tests/system/statistics/prereq.sh index 9bca4f871d..ec3fefe08a 100644 --- a/bin/tests/system/statistics/prereq.sh +++ b/bin/tests/system/statistics/prereq.sh @@ -9,6 +9,9 @@ # See the COPYRIGHT file distributed with this work for additional # information regarding copyright ownership. +SYSTEMTESTTOP=.. +. $SYSTEMTESTTOP/conf.sh + if $PERL -e 'use Net::DNS;' 2>/dev/null then if $PERL -e 'use Net::DNS; die if ($Net::DNS::VERSION >= 0.76 && $Net::DNS::VERSION <= 0.77);' 2>/dev/null diff --git a/bin/tests/system/statschannel/prereq.sh b/bin/tests/system/statschannel/prereq.sh index ef052f5580..f3d4fd17a6 100644 --- a/bin/tests/system/statschannel/prereq.sh +++ b/bin/tests/system/statschannel/prereq.sh @@ -9,6 +9,9 @@ # See the COPYRIGHT file distributed with this work for additional # information regarding copyright ownership. +SYSTEMTESTTOP=.. +. $SYSTEMTESTTOP/conf.sh + fail=0 if $PERL -e 'use File::Fetch;' 2>/dev/null diff --git a/bin/tests/system/upforwd/prereq.sh b/bin/tests/system/upforwd/prereq.sh index affe54bfcf..0e299f4db8 100644 --- a/bin/tests/system/upforwd/prereq.sh +++ b/bin/tests/system/upforwd/prereq.sh @@ -9,6 +9,9 @@ # See the COPYRIGHT file distributed with this work for additional # information regarding copyright ownership. +SYSTEMTESTTOP=.. +. $SYSTEMTESTTOP/conf.sh + if $PERL -e 'use Net::DNS;' 2>/dev/null then : diff --git a/bin/tests/system/xfer/prereq.sh b/bin/tests/system/xfer/prereq.sh index daa3d5d86c..b8982d610c 100644 --- a/bin/tests/system/xfer/prereq.sh +++ b/bin/tests/system/xfer/prereq.sh @@ -9,6 +9,9 @@ # See the COPYRIGHT file distributed with this work for additional # information regarding copyright ownership. +SYSTEMTESTTOP=.. +. $SYSTEMTESTTOP/conf.sh + if $PERL -e 'use Net::DNS;' 2>/dev/null then if $PERL -e 'use Net::DNS; die if ($Net::DNS::VERSION >= 0.69 && $Net::DNS::VERSION <= 0.74);' 2>/dev/null From 7fe0f00a3bf31dc26ee3dc8777e3076546b83990 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20K=C4=99pie=C5=84?= Date: Tue, 10 Jul 2018 14:34:35 +0200 Subject: [PATCH 4/7] Improve error handling in idn_ace_to_locale() While idn2_to_unicode_8zlz() takes a 'flags' argument, it is ignored and thus cannot be used to perform IDN checks on the output string. The bug in libidn2 versions before 2.0.5 was not that a call to idn2_to_unicode_8zlz() with certain flags set did not cause IDN checks to be performed. The bug was that idn2_to_unicode_8zlz() did not check whether a conversion can be performed between UTF-8 and the current locale's character encoding. In other words, with libidn2 version 2.0.5+, if the current locale's character encoding is ASCII, then idn2_to_unicode_8zlz() will fail when it is passed any Punycode string which decodes to a non-ASCII string, even if it is a valid IDNA2008 name. Rework idn_ace_to_locale() so that invalid IDNA2008 names are properly and consistently detected for all libidn2 versions and locales. Update the "idna" system test accordingly. Add checks for processing a server response containing Punycode which decodes to an invalid IDNA2008 name. Fix invalid subtest description. (cherry picked from commit b896fc497212e97cc36119adcf4fe9a8a406cbce) --- bin/dig/dighost.c | 78 ++++++++++++++++++++++++++-------- bin/tests/system/idna/tests.sh | 77 +++++++++++---------------------- 2 files changed, 85 insertions(+), 70 deletions(-) diff --git a/bin/dig/dighost.c b/bin/dig/dighost.c index bb21bb3c00..f4c561ec76 100644 --- a/bin/dig/dighost.c +++ b/bin/dig/dighost.c @@ -4389,26 +4389,70 @@ idn_locale_to_ace(const char *from, char *to, size_t tolen) { static isc_result_t idn_ace_to_locale(const char *from, char *to, size_t tolen) { int res; - char *tmp_str = NULL; + char *utf8_src, *tmp_str = NULL; - res = idn2_to_unicode_8zlz(from, &tmp_str, - IDN2_NONTRANSITIONAL|IDN2_NFC_INPUT); - - if (res == IDN2_OK) { - /* check the length */ - if (strlen(tmp_str) >= tolen) { - debug("encoded ASC string is too long"); - idn2_free(tmp_str); - return ISC_R_FAILURE; - } - - (void) strlcpy(to, tmp_str, tolen); - idn2_free(tmp_str); - return ISC_R_SUCCESS; + /* + * We need to: + * + * 1) check whether 'from' is a valid IDNA2008 name, + * 2) if it is, output it in the current locale's character encoding. + * + * Unlike idn2_to_ascii_*(), idn2_to_unicode_*() functions are unable + * to perform IDNA2008 validity checks. Thus, we need to decode any + * Punycode in 'from', check if the resulting name is a valid IDNA2008 + * name, and only once we ensure it is, output that name in the current + * locale's character encoding. + * + * We could just use idn2_to_unicode_8zlz() + idn2_to_ascii_lz(), but + * then we would not be able to universally tell invalid names and + * character encoding errors apart (if the current locale uses ASCII + * for character encoding, the former function would fail even for a + * valid IDNA2008 name, as long as it contained any non-ASCII + * character). Thus, we need to take a longer route. + * + * First, convert 'from' to UTF-8, ignoring the current locale. + */ + res = idn2_to_unicode_8z8z(from, &utf8_src, 0); + if (res != IDN2_OK) { + fatal("Bad ACE string '%s' (%s), use +noidnout", + from, idn2_strerror(res)); } - fatal("'%s' is not a legal IDN name (%s), use +noidnout", from, idn2_strerror(res)); - return ISC_R_FAILURE; + /* + * Then, check whether decoded 'from' is a valid IDNA2008 name. + */ + res = idn2_to_ascii_8z(utf8_src, NULL, IDN2_NONTRANSITIONAL); + if (res != IDN2_OK) { + fatal("'%s' is not a legal IDNA2008 name (%s), use +noidnout", + from, idn2_strerror(res)); + } + + /* + * Finally, try converting the decoded 'from' into the current locale's + * character encoding. + */ + res = idn2_to_unicode_8zlz(utf8_src, &tmp_str, 0); + if (res != IDN2_OK) { + fatal("Cannot represent '%s' in the current locale (%s), " + "use +noidnout or a different locale", + from, idn2_strerror(res)); + } + + /* + * Free the interim conversion result. + */ + idn2_free(utf8_src); + + /* check the length */ + if (strlen(tmp_str) >= tolen) { + debug("encoded ASC string is too long"); + idn2_free(tmp_str); + return (ISC_R_FAILURE); + } + + (void) strlcpy(to, tmp_str, tolen); + idn2_free(tmp_str); + return (ISC_R_SUCCESS); } #endif /* WITH_IDN_OUT_SUPPORT */ #endif /* WITH_LIBIDN2 */ diff --git a/bin/tests/system/idna/tests.sh b/bin/tests/system/idna/tests.sh index 3a9b91b442..6637bf6828 100644 --- a/bin/tests/system/idna/tests.sh +++ b/bin/tests/system/idna/tests.sh @@ -136,46 +136,6 @@ idna_fail() { status=`expr $status + $ret` } -# Check if current version of libidn2 is >= a given version -# -# This requires that: -# a) "pkg-config" exists on the system -# b) The libidn2 installed has an associated ".pc" file -# c) The system sort command supports "-V" -# -# $1 - Minimum version required -# -# Returns: -# 0 - Version check is OK, libidn2 at required version or greater. -# 1 - Version check was made, but libidn2 not at required version. -# 2 - Could not carry out version check - -libidn_version_check() { - ret=2 - if [ -n "`command -v pkg-config`" ]; then - version=`pkg-config --modversion --silence-errors libidn2` - if [ -n "$version" ]; then - # Does the sort command have a "-V" flag on this system? - sort -V 2>&1 > /dev/null << . -. - if [ $? -eq 0 ]; then - # Sort -V exists. Sort the IDN version and the minimum version - # required. If the IDN version is greater than or equal to that - # version, it will appear last in the list. - last_version=`printf "%s\n" $version $1 | sort -V | tail -1` - if [ "$version" = "$last_version" ]; then - ret=0 - else - ret=1 - fi - fi - fi - fi - - return $ret -} - - # Function to check that case is preserved for an all-ASCII label. # # Without IDNA support, case-preservation is the expected behavior. @@ -310,16 +270,7 @@ idna_enabled_test() { text="Checking fake A-label" idna_fail "$text" "" "xn--ahahah" idna_test "$text" "+noidnin +noidnout" "xn--ahahah" "xn--ahahah." - - # Owing to issues with libdns, the next test will fail for versions of - # libidn earlier than 2.0.5. For this reason, get the version (if - # available) and compare with 2.0.5. - libidn_version_check 2.0.5 - if [ $? -ne 0 ]; then - echo_i "Skipping fake A-label +noidnin +idnout test (libidn2 version issues)" - else - idna_test "$text" "+noidnin +idnout" "xn--ahahah" "xn--ahahah." - fi + idna_fail "$text" "+noidnin +idnout" "xn--ahahah" idna_fail "$text" "+idnin +noidnout" "xn--ahahah" idna_fail "$text" "+idnin +idnout" "xn--ahahah" @@ -327,7 +278,7 @@ idna_enabled_test() { # BIND rejects such labels: with +idnin label="xn--xflod18hstflod18hstflod18hstflod18hstflod18hstflod18-1iejjjj" - text="Checking punycode label shorter than minimum valid length" + text="Checking punycode label longer than maximum valid length" idna_fail "$text" "" "$label" idna_fail "$text" "+noidnin +noidnout" "$label" idna_fail "$text" "+noidnin +idnout" "$label" @@ -337,7 +288,7 @@ idna_enabled_test() { - # Tests of a valid unicode string but an invalid U-label + # Tests of a valid unicode string but an invalid U-label (input) # # Symbols are not valid IDNA names. # @@ -347,12 +298,32 @@ idna_enabled_test() { # # The +[no]idnout options should not have any effect on the test. - text="Checking invalid U-label" + text="Checking invalid input U-label" idna_fail "$text" "" "🧦.com" idna_test "$text" "+noidnin +noidnout" "🧦.com" "\240\159\167\166.com." idna_test "$text" "+noidnin +idnout" "🧦.com" "\240\159\167\166.com." idna_fail "$text" "+idnin +noidnout" "🧦.com" idna_fail "$text" "+idnin +idnout" "🧦.com" + + # Tests of a valid unicode string but an invalid U-label (output) + # + # Symbols are not valid IDNA names. + # + # Note that an invalid U-label is accepted even when +idnin is in effect + # because "xn--19g" is valid Punycode. + # + # +noidnout: "dig" should send the ACE string to the server and display the + # returned qname. + # +idnout: "dig" should generate an error. + # + # The +[no]idnin options should not have any effect on the test. + + text="Checking invalid output U-label" + idna_fail "$text" "" "xn--19g" + idna_test "$text" "+noidnin +noidnout" "xn--19g" "xn--19g." + idna_fail "$text" "+noidnin +idnout" "xn--19g" + idna_test "$text" "+idnin +noidnout" "xn--19g" "xn--19g." + idna_fail "$text" "+idnin +idnout" "xn--19g" } From 106b56d7c9c2684dece4dbd15ad370c044b32262 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20K=C4=99pie=C5=84?= Date: Thu, 12 Jul 2018 13:35:10 +0200 Subject: [PATCH 5/7] Remove IDN subtest from the "digdelv" system test The output of certain "dig +idnout" invocations may be locale-dependent. Remove the "dig +idnout" subtest from the "digdelv" system test as IDN support is already thoroughly tested by the "idna" system test. (cherry picked from commit fd30a03f2b3e7e96598aab3b28e011911fd8a679) --- bin/tests/system/digdelv/tests.sh | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/bin/tests/system/digdelv/tests.sh b/bin/tests/system/digdelv/tests.sh index 1260a104ae..25898b2276 100644 --- a/bin/tests/system/digdelv/tests.sh +++ b/bin/tests/system/digdelv/tests.sh @@ -424,23 +424,6 @@ if [ -x ${DIG} ] ; then if [ $ret != 0 ]; then echo_i "failed"; fi status=`expr $status + $ret` - n=`expr $n + 1` - if $FEATURETEST --with-idn - then - echo_i "checking dig +idnout ($n)" - ret=0 - $DIG $DIGOPTS @10.53.0.3 +noidnout xn--caf-dma.example. > dig.out.1.test$n 2>&1 || ret=1 - $DIG $DIGOPTS @10.53.0.3 +idnout xn--caf-dma.example. > dig.out.2.test$n 2>&1 || ret=1 - grep "^xn--caf-dma.example" dig.out.1.test$n > /dev/null || ret=1 - grep "^xn--caf-dma.example" dig.out.2.test$n > /dev/null && ret=1 - grep 10.1.2.3 dig.out.1.test$n > /dev/null || ret=1 - grep 10.1.2.3 dig.out.2.test$n > /dev/null || ret=1 - if [ $ret != 0 ]; then echo_i "failed"; fi - status=`expr $status + $ret` - else - echo_i "skipping 'dig +idnout' as IDN support is not enabled ($n)" - fi - n=`expr $n + 1` echo_i "checking that dig warns about .local queries ($n)" ret=0 From e56a528c70911276d3ac8fd2ebf9e991ed798753 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20K=C4=99pie=C5=84?= Date: Thu, 12 Jul 2018 13:35:10 +0200 Subject: [PATCH 6/7] Do not spam console if "git status --ignored" fails during tests The "git status" command in Git versions before 1.7.2 does not support the "--ignored" option. Prevent spamming the console when running system tests from a Git repository on a host with an ancient Git version installed. (cherry picked from commit 2be97feb4642746660dd793beed54b193579db0d) --- bin/tests/system/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/tests/system/run.sh b/bin/tests/system/run.sh index 600cacd5c8..0e7632ffbf 100644 --- a/bin/tests/system/run.sh +++ b/bin/tests/system/run.sh @@ -194,7 +194,7 @@ else $SHELL clean.sh $runall $systest "$@" if test -d ../../../.git then - git status -su --ignored $systest | \ + git status -su --ignored $systest 2>/dev/null | \ sed -n -e 's|^?? \(.*\)|I:file \1 not removed|p' \ -e 's|^!! \(.*/named.run\)$|I:file \1 not removed|p' \ -e 's|^!! \(.*/named.memstats\)$|I:file \1 not removed|p' From de00137f42e8d535e18c0941278779bcfc544aa7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20K=C4=99pie=C5=84?= Date: Thu, 12 Jul 2018 13:35:10 +0200 Subject: [PATCH 7/7] Add CentOS 6/7 to CI Modify .gitlab-ci.yml so that every CI pipeline also builds and tests BIND on CentOS versions 6 and 7. Use --disable-warn-error on CentOS 6 since it uses GCC 4.4.7 which suffers from bugs causing bogus warnings to be generated, e.g.: sigs_test.c: In function 'compare_tuples': sigs_test.c:75: warning: declaration of 'index' shadows a global declaration /usr/include/string.h:489: warning: shadowed declaration is here sigs_test.c: In function 'updatesigs_test': sigs_test.c:193: warning: declaration of 'index' shadows a global declaration /usr/include/string.h:489: warning: shadowed declaration is here (cherry picked from commit f0966d1485816c95bf312e41e7796a374b2b0fe6) --- .gitlab-ci.yml | 55 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f73f8ddb4e..bf4d1934d5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,6 +10,20 @@ stages: - build - test +.centos-centos6-amd64: ¢os_centos6_amd64_image + image: "$CI_REGISTRY_IMAGE:centos-centos6-amd64" + tags: + - linux + - docker + - amd64 + +.centos-centos7-amd64: ¢os_centos7_amd64_image + image: "$CI_REGISTRY_IMAGE:centos-centos7-amd64" + tags: + - linux + - docker + - amd64 + .debian-jessie-amd64: &debian_jessie_amd64_image image: "$CI_REGISTRY_IMAGE:debian-jessie-amd64" tags: @@ -100,7 +114,7 @@ stages: - test -w "${CCACHE_DIR}" && export PATH="/usr/lib/ccache:${PATH}" - autoreconf -fi script: - - ./configure --enable-developer --with-libtool --disable-static --with-atf=/usr --with-libidn2 + - ./configure --enable-developer --with-libtool --disable-static --with-atf=/usr --with-libidn2 "${EXTRA_CONFIGURE}" - make -j${PARALLEL_JOBS_BUILD:-1} -k all V=1 artifacts: expire_in: '1 hour' @@ -178,6 +192,21 @@ precheck:debian:sid:amd64: # <<: *ubuntu_xenial_i386_image # <<: *build_job +build:centos:centos6:amd64: + variables: + CC: gcc + CFLAGS: "-Wall -Wextra -O2 -g" + EXTRA_CONFIGURE: "--disable-warn-error" + <<: *centos_centos6_amd64_image + <<: *build_job + +build:centos:centos7:amd64: + variables: + CC: gcc + CFLAGS: "-Wall -Wextra -O2 -g" + <<: *centos_centos7_amd64_image + <<: *build_job + build:debian:sid:amd64: <<: *debian_sid_amd64_image <<: *build_job @@ -186,6 +215,18 @@ build:debian:sid:i386: <<: *debian_sid_i386_image <<: *build_job +unittest:centos:centos6:amd64: + <<: *centos_centos6_amd64_image + <<: *unit_test_job + dependencies: + - build:centos:centos6:amd64 + +unittest:centos:centos7:amd64: + <<: *centos_centos7_amd64_image + <<: *unit_test_job + dependencies: + - build:centos:centos7:amd64 + unittest:debian:sid:amd64: <<: *debian_sid_amd64_image <<: *unit_test_job @@ -198,6 +239,18 @@ unittest:debian:sid:i386: dependencies: - build:debian:sid:i386 +systemtest:centos:centos6:amd64: + <<: *centos_centos6_amd64_image + <<: *system_test_job + dependencies: + - build:centos:centos6:amd64 + +systemtest:centos:centos7:amd64: + <<: *centos_centos7_amd64_image + <<: *system_test_job + dependencies: + - build:centos:centos7:amd64 + systemtest:debian:sid:amd64: <<: *debian_sid_amd64_image <<: *system_test_job