ComputerSurge/bind9
3
0
Fork 0
Code Pull Requests 1 Activity

add geoip2 system test

Browse Source
This commit is contained in:
Evan Hunt
2019-06-22 12:52:26 -07:00
parent 6399a70cb4
commit 9a1caf99ef
37 changed files with 3095 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

506
bin/tests/system/geoip2/data/GeoIP2-City.json Normal file
View File

@@ -0,0 +1,506 @@
[
{
"::10.53.0.1/128" : {
"continent" : {
"code" : "NA",
"names" : {
"en" : "North America"
}
},
"country" : {
"iso_code" : "US",
"names" : {
"en" : "United States"
}
},
"city" : {
"names" : {
"en" : "Redwood City"
}
},
"location" : {
"metro_code" : "807",
"time_zone" : "America/Los_Angeles"
},
"postal" : {
"code" : "94063"
},
"subdivisions" : [
{
"iso_code" : "CA",
"names" : {
"en" : "California"
}
}
]
}
},
{
"::10.53.0.2/128" : {
"continent" : {
"code" : "NA",
"names" : {
"en" : "North America"
}
},
"country" : {
"iso_code" : "US",
"names" : {
"en" : "United States"
}
},
"city" : {
"names" : {
"en" : "Santa Cruz"
}
},
"location" : {
"metro_code" : "828",
"time_zone" : "America/Los_Angeles"
},
"postal" : {
"code" : "95060"
},
"subdivisions" : [
{
"iso_code" : "CA",
"names" : {
"en" : "California"
}
}
]
}
},
{
"::10.53.0.3/128" : {
"continent" : {
"code" : "NA",
"names" : {
"en" : "North America"
}
},
"country" : {
"iso_code" : "US",
"names" : {
"en" : "United States"
}
},
"city" : {
"names" : {
"en" : "Oklahoma City"
}
},
"location" : {
"metro_code" : "650",
"time_zone" : "America/Chicago"
},
"postal" : {
"code" : "73120"
},
"subdivisions" : [
{
"iso_code" : "OK",
"names" : {
"en" : "Oklahoma"
}
}
]
}
},
{
"::10.53.0.4/128" : {
"continent" : {
"code" : "NA",
"names" : {
"en" : "North America"
}
},
"country" : {
"iso_code" : "US",
"names" : {
"en" : "United States"
}
},
"city" : {
"names" : {
"en" : "Ashland"
}
},
"location" : {
"metro_code" : "556",
"time_zone" : "America/New_York"
},
"postal" : {
"code" : "23005"
},
"subdivisions" : [
{
"iso_code" : "VA",
"names" : {
"en" : "Virginia"
}
}
]
}
},
{
"::10.53.0.5/128" : {
"continent" : {
"code" : "NA",
"names" : {
"en" : "North America"
}
},
"country" : {
"iso_code" : "US",
"names" : {
"en" : "United States"
}
},
"city" : {
"names" : {
"en" : "Atlanta"
}
},
"location" : {
"metro_code" : "524",
"time_zone" : "America/New_York"
},
"postal" : {
"code" : "30345"
},
"subdivisions" : [
{
"iso_code" : "GA",
"names" : {
"en" : "Georgia"
}
}
]
}
},
{
"::10.53.0.6/128" : {
"continent" : {
"code" : "NA",
"names" : {
"en" : "North America"
}
},
"country" : {
"iso_code" : "US",
"names" : {
"en" : "United States"
}
},
"city" : {
"names" : {
"en" : "Morrison"
}
},
"location" : {
"metro_code" : "751",
"time_zone" : "America/Denver"
},
"postal" : {
"code" : "80465"
},
"subdivisions" : [
{
"iso_code" : "CO",
"names" : {
"en" : "Colorado"
}
}
]
}
},
{
"::10.53.0.7/128" : {
"continent" : {
"code" : "NA",
"names" : {
"en" : "North America"
}
},
"country" : {
"iso_code" : "US",
"names" : {
"en" : "United States"
}
},
"city" : {
"names" : {
"en" : "Ketchikan"
}
},
"location" : {
"metro_code" : "747",
"time_zone" : "America/Anchorage"
},
"postal" : {
"code" : "99901"
},
"subdivisions" : [
{
"iso_code" : "AK",
"names" : {
"en" : "Alaska"
}
}
]
}
},
{
"fd92:7065:b8e:ffff::1/128" : {
"continent" : {
"code" : "NA",
"names" : {
"en" : "North America"
}
},
"country" : {
"iso_code" : "US",
"names" : {
"en" : "United States"
}
},
"city" : {
"names" : {
"en" : "Redwood City"
}
},
"location" : {
"metro_code" : "807",
"time_zone" : "America/Los_Angeles"
},
"postal" : {
"code" : "94063"
},
"subdivisions" : [
{
"iso_code" : "CA",
"names" : {
"en" : "California"
}
}
]
}
},
{
"fd92:7065:b8e:ffff::2/128" : {
"continent" : {
"code" : "NA",
"names" : {
"en" : "North America"
}
},
"country" : {
"iso_code" : "US",
"names" : {
"en" : "United States"
}
},
"city" : {
"names" : {
"en" : "Santa Cruz"
}
},
"location" : {
"metro_code" : "828",
"time_zone" : "America/Los_Angeles"
},
"postal" : {
"code" : "95060"
},
"subdivisions" : [
{
"iso_code" : "CA",
"names" : {
"en" : "California"
}
}
]
}
},
{
"fd92:7065:b8e:ffff::3/128" : {
"continent" : {
"code" : "NA",
"names" : {
"en" : "North America"
}
},
"country" : {
"iso_code" : "US",
"names" : {
"en" : "United States"
}
},
"city" : {
"names" : {
"en" : "Oklahoma City"
}
},
"location" : {
"metro_code" : "650",
"time_zone" : "America/Chicago"
},
"postal" : {
"code" : "73120"
},
"subdivisions" : [
{
"iso_code" : "OK",
"names" : {
"en" : "Oklahoma"
}
}
]
}
},
{
"fd92:7065:b8e:ffff::4/128" : {
"continent" : {
"code" : "NA",
"names" : {
"en" : "North America"
}
},
"country" : {
"iso_code" : "US",
"names" : {
"en" : "United States"
}
},
"city" : {
"names" : {
"en" : "Ashland"
}
},
"location" : {
"metro_code" : "556",
"time_zone" : "America/New_York"
},
"postal" : {
"code" : "23005"
},
"subdivisions" : [
{
"iso_code" : "VA",
"names" : {
"en" : "Virginia"
}
}
]
}
},
{
"fd92:7065:b8e:ffff::5/128" : {
"continent" : {
"code" : "NA",
"names" : {
"en" : "North America"
}
},
"country" : {
"iso_code" : "US",
"names" : {
"en" : "United States"
}
},
"city" : {
"names" : {
"en" : "Atlanta"
}
},
"location" : {
"metro_code" : "524",
"time_zone" : "America/New_York"
},
"postal" : {
"code" : "30345"
},
"subdivisions" : [
{
"iso_code" : "GA",
"names" : {
"en" : "Georgia"
}
}
]
}
},
{
"fd92:7065:b8e:ffff::6/128" : {
"continent" : {
"code" : "NA",
"names" : {
"en" : "North America"
}
},
"country" : {
"iso_code" : "US",
"names" : {
"en" : "United States"
}
},
"city" : {
"names" : {
"en" : "Morrison"
}
},
"location" : {
"metro_code" : "751",
"time_zone" : "America/Denver"
},
"postal" : {
"code" : "80465"
},
"subdivisions" : [
{
"iso_code" : "CO",
"names" : {
"en" : "Colorado"
}
}
]
}
},
{
"fd92:7065:b8e:ffff::7/128" : {
"continent" : {
"code" : "NA",
"names" : {
"en" : "North America"
}
},
"country" : {
"iso_code" : "US",
"names" : {
"en" : "United States"
}
},
"city" : {
"names" : {
"en" : "Ketchikan"
}
},
"location" : {
"metro_code" : "747",
"time_zone" : "America/Anchorage"
},
"postal" : {
"code" : "99901"
},
"subdivisions" : [
{
"iso_code" : "AK",
"names" : {
"en" : "Alaska"
}
}
]
}
}
]

BIN
bin/tests/system/geoip2/data/GeoIP2-City.mmdb Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 3.1 KiB

242
bin/tests/system/geoip2/data/GeoIP2-Country.json Normal file
View File

@@ -0,0 +1,242 @@
[
{
"::10.53.0.1/128" : {
"continent" : {
"code" : "OC",
"names" : {
"en" : "Oceania"
}
},
"country" : {
"iso_code" : "AU",
"names" : {
"en" : "Australia"
}
}
}
},
{
"::10.53.0.2/128" : {
"continent" : {
"code" : "NA",
"names" : {
"en" : "North America"
}
},
"country" : {
"iso_code" : "US",
"names" : {
"en" : "United States"
}
}
}
},
{
"::10.53.0.3/128" : {
"continent" : {
"code" : "EU",
"names" : {
"en" : "Europe"
}
},
"country" : {
"iso_code" : "GB",
"names" : {
"en" : "United Kingdom"
}
}
}
},
{
"::10.53.0.4/128" : {
"continent" : {
"code" : "NA",
"names" : {
"en" : "North America"
}
},
"country" : {
"iso_code" : "CA",
"names" : {
"en" : "Canada"
}
}
}
},
{
"::10.53.0.5/128" : {
"continent" : {
"code" : "SA",
"names" : {
"en" : "South America"
}
},
"country" : {
"iso_code" : "CL",
"names" : {
"en" : "Chile"
}
}
}
},
{
"::10.53.0.6/128" : {
"continent" : {
"code" : "EU",
"names" : {
"en" : "Europe"
}
},
"country" : {
"iso_code" : "DE",
"names" : {
"en" : "Germany"
}
}
}
},
{
"::10.53.0.7/128" : {
"continent" : {
"code" : "AF",
"names" : {
"en" : "Africa"
}
},
"country" : {
"iso_code" : "EH",
"names" : {
"en" : "Western Sahara"
}
}
}
},
{
"::192.0.2.0/120" : {
"continent" : {
"code" : "O1",
"names" : {
"en" : "Other"
}
},
"country" : {
"iso_code" : "O1",
"names" : {
"en" : "Other"
}
}
}
},
{
"fd92:7065:b8e:ffff::1/128" : {
"continent" : {
"code" : "OC",
"names" : {
"en" : "Oceania"
}
},
"country" : {
"iso_code" : "AU",
"names" : {
"en" : "Australia"
}
}
}
},
{
"fd92:7065:b8e:ffff::2/128" : {
"continent" : {
"code" : "NA",
"names" : {
"en" : "North America"
}
},
"country" : {
"iso_code" : "US",
"names" : {
"en" : "United States"
}
}
}
},
{
"fd92:7065:b8e:ffff::3/128" : {
"continent" : {
"code" : "EU",
"names" : {
"en" : "Europe"
}
},
"country" : {
"iso_code" : "GB",
"names" : {
"en" : "United Kingdom"
}
}
}
},
{
"fd92:7065:b8e:ffff::4/128" : {
"continent" : {
"code" : "NA",
"names" : {
"en" : "North America"
}
},
"country" : {
"iso_code" : "CA",
"names" : {
"en" : "Canada"
}
}
}
},
{
"fd92:7065:b8e:ffff::5/128" : {
"continent" : {
"code" : "SA",
"names" : {
"en" : "South America"
}
},
"country" : {
"iso_code" : "CL",
"names" : {
"en" : "Chile"
}
}
}
},
{
"fd92:7065:b8e:ffff::6/128" : {
"continent" : {
"code" : "EU",
"names" : {
"en" : "Europe"
}
},
"country" : {
"iso_code" : "DE",
"names" : {
"en" : "Germany"
}
}
}
},
{
"fd92:7065:b8e:ffff::7/128" : {
"continent" : {
"code" : "AF",
"names" : {
"en" : "Africa"
}
},
"country" : {
"iso_code" : "EH",
"names" : {
"en" : "Western Sahara"
}
}
}
}
]

BIN
bin/tests/system/geoip2/data/GeoIP2-Country.mmdb Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 2.9 KiB

72
bin/tests/system/geoip2/data/GeoIP2-Domain.json Normal file
View File

@@ -0,0 +1,72 @@
[
{
"::10.53.0.1/128" : {
"domain" : "one.de"
}
},
{
"::10.53.0.2/128" : {
"domain" : "two.com"
}
},
{
"::10.53.0.3/128" : {
"domain" : "three.com"
}
},
{
"::10.53.0.4/128" : {
"domain" : "four.edu"
}
},
{
"::10.53.0.5/128" : {
"domain" : "five.es"
}
},
{
"::10.53.0.6/128" : {
"domain" : "six.it"
}
},
{
"::10.53.0.7/128" : {
"domain" : "seven.org"
}
},
{
"fd92:7065:b8e:ffff::1/128" : {
"domain" : "one.de"
}
},
{
"fd92:7065:b8e:ffff::2/128" : {
"domain" : "two.com"
}
},
{
"fd92:7065:b8e:ffff::3/128" : {
"domain" : "three.com"
}
},
{
"fd92:7065:b8e:ffff::4/128" : {
"domain" : "four.edu"
}
},
{
"fd92:7065:b8e:ffff::5/128" : {
"domain" : "five.es"
}
},
{
"fd92:7065:b8e:ffff::6/128" : {
"domain" : "six.it"
}
},
{
"fd92:7065:b8e:ffff::7/128" : {
"domain" : "seven.org"
}
}
]

BIN
bin/tests/system/geoip2/data/GeoIP2-Domain.mmdb Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 2.5 KiB

86
bin/tests/system/geoip2/data/GeoIP2-ISP.json Normal file
View File

@@ -0,0 +1,86 @@
[
{
"::10.53.0.1/128" : {
"isp" : "One Systems, Inc.",
"organization" : "One Systems, Inc."
}
},
{
"::10.53.0.2/128" : {
"isp" : "Two Technology Ltd.",
"organization" : "Two Technology Ltd."
}
},
{
"::10.53.0.3/128" : {
"isp" : "Three Network Labs",
"organization" : "Three Network Labs"
}
},
{
"::10.53.0.4/128" : {
"isp" : "Four University",
"organization" : "Four University"
}
},
{
"::10.53.0.5/128" : {
"isp" : "Five Telecom",
"organization" : "Five Telecom"
}
},
{
"::10.53.0.6/128" : {
"isp" : "Six Company",
"organization" : "Six Company"
}
},
{
"::10.53.0.7/128" : {
"isp" : "Seven Communications",
"organization" : "Seven Communications"
}
},
{
"fd92:7065:b8e:ffff::1/128" : {
"isp" : "One Systems, Inc.",
"organization" : "One Systems, Inc."
}
},
{
"fd92:7065:b8e:ffff::2/128" : {
"isp" : "Two Technology Ltd.",
"organization" : "Two Technology Ltd."
}
},
{
"fd92:7065:b8e:ffff::3/128" : {
"isp" : "Three Network Labs",
"organization" : "Three Network Labs"
}
},
{
"fd92:7065:b8e:ffff::4/128" : {
"isp" : "Four University",
"organization" : "Four University"
}
},
{
"fd92:7065:b8e:ffff::5/128" : {
"isp" : "Five Telecom",
"organization" : "Five Telecom"
}
},
{
"fd92:7065:b8e:ffff::6/128" : {
"isp" : "Six Company",
"organization" : "Six Company"
}
},
{
"fd92:7065:b8e:ffff::7/128" : {
"isp" : "Seven Communications",
"organization" : "Seven Communications"
}
}
]

BIN
bin/tests/system/geoip2/data/GeoIP2-ISP.mmdb Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 2.6 KiB

86
bin/tests/system/geoip2/data/GeoLite2-ASN.json Normal file
View File

@@ -0,0 +1,86 @@
[
{
"::10.53.0.1/128" : {
"autonomous_system_number" : 100001,
"autonomous_system_organization" : "One Systems, Inc."
}
},
{
"::10.53.0.2/128" : {
"autonomous_system_number" : 100002,
"autonomous_system_organization" : "Two Technology Ltd."
}
},
{
"::10.53.0.3/128" : {
"autonomous_system_number" : 100003,
"autonomous_system_organization" : "Three Network Labs"
}
},
{
"::10.53.0.4/128" : {
"autonomous_system_number" : 100004,
"autonomous_system_organization" : "Four University"
}
},
{
"::10.53.0.5/128" : {
"autonomous_system_number" : 100005,
"autonomous_system_organization" : "Five Telecom"
}
},
{
"::10.53.0.6/128" : {
"autonomous_system_number" : 100006,
"autonomous_system_organization" : "Six Company"
}
},
{
"::10.53.0.7/128" : {
"autonomous_system_number" : 100007,
"autonomous_system_organization" : "Seven Communications"
}
},
{
"fd92:7065:b8e:ffff::1/128" : {
"autonomous_system_number" : 100001,
"autonomous_system_organization" : "One Systems, Inc."
}
},
{
"fd92:7065:b8e:ffff::2/128" : {
"autonomous_system_number" : 100002,
"autonomous_system_organization" : "Two Technology Ltd."
}
},
{
"fd92:7065:b8e:ffff::3/128" : {
"autonomous_system_number" : 100003,
"autonomous_system_organization" : "Three Network Labs"
}
},
{
"fd92:7065:b8e:ffff::4/128" : {
"autonomous_system_number" : 100004,
"autonomous_system_organization" : "Four University"
}
},
{
"fd92:7065:b8e:ffff::5/128" : {
"autonomous_system_number" : 100005,
"autonomous_system_organization" : "Five Telecom"
}
},
{
"fd92:7065:b8e:ffff::6/128" : {
"autonomous_system_number" : 100006,
"autonomous_system_organization" : "Six Company"
}
},
{
"fd92:7065:b8e:ffff::7/128" : {
"autonomous_system_number" : 100007,
"autonomous_system_organization" : "Seven Communications"
}
}
]

BIN
bin/tests/system/geoip2/data/GeoLite2-ASN.mmdb Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 2.6 KiB

12
bin/tests/system/geoip2/data/README.md Normal file
View File

@@ -0,0 +1,12 @@
##### Test MMDB databases
This directory contains test versions of the GeoIP2/GeoLite2 CIty,
Country, Domain, ISP, and ASN databases. The `.mmdb` files are built
from the corresponding `.json` source files; to regenerate them, modify
the source files and run `perl write-test-data.pl`.
This script is adapted from one in
[https://github.com/maxmind/MaxMind-DB](https://github.com/maxmind/MaxMind-DB).
It depends on the MaxMind:DB:Writer module, which can be found in
CPAN or at
[https://github.com/maxmind/MaxMind-DB-Writer-perl](https://github.com/maxmind/MaxMind-DB-Writer-perl) .

192
bin/tests/system/geoip2/data/write-test-data.pl Executable file
View File

@@ -0,0 +1,192 @@
#!/usr/bin/env perl
#
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
use strict;
use warnings;
use autodie;
use utf8;
use Carp qw( croak );
use Cwd qw( abs_path );
use File::Basename qw( dirname );
use File::Slurper qw( read_binary write_binary );
use Cpanel::JSON::XS qw( decode_json );
use Math::Int128 qw( MAX_UINT128 string_to_uint128 uint128 );
use MaxMind::DB::Writer::Serializer 0.100004;
use MaxMind::DB::Writer::Tree 0.100004;
use MaxMind::DB::Writer::Util qw( key_for_data );
use Net::Works::Network;
use Test::MaxMind::DB::Common::Util qw( standard_test_metadata );
my $Dir = dirname( abs_path($0) );
sub main {
write_geoip2_dbs();
}
sub write_geoip2_dbs {
_write_geoip2_db( @{$_}[ 0, 1 ], 'Test' )
for (
['GeoIP2-City'],
['GeoIP2-Country'],
['GeoIP2-Domain'],
['GeoIP2-ISP'],
['GeoLite2-ASN'],
);
}
sub _universal_map_key_type_callback {
my $map = {
# languages
de => 'utf8_string',
en => 'utf8_string',
es => 'utf8_string',
fr => 'utf8_string',
ja => 'utf8_string',
'pt-BR' => 'utf8_string',
ru => 'utf8_string',
'zh-CN' => 'utf8_string',
# production
accuracy_radius => 'uint16',
autonomous_system_number => 'uint32',
autonomous_system_organization => 'utf8_string',
average_income => 'uint32',
city => 'map',
code => 'utf8_string',
confidence => 'uint16',
connection_type => 'utf8_string',
continent => 'map',
country => 'map',
domain => 'utf8_string',
geoname_id => 'uint32',
ipv4_24 => 'uint32',
ipv4_32 => 'uint32',
ipv6_32 => 'uint32',
ipv6_48 => 'uint32',
ipv6_64 => 'uint32',
is_anonymous => 'boolean',
is_anonymous_proxy => 'boolean',
is_anonymous_vpn => 'boolean',
is_hosting_provider => 'boolean',
is_in_european_union => 'boolean',
is_legitimate_proxy => 'boolean',
is_public_proxy => 'boolean',
is_satellite_provider => 'boolean',
is_tor_exit_node => 'boolean',
iso_code => 'utf8_string',
isp => 'utf8_string',
latitude => 'double',
location => 'map',
longitude => 'double',
metro_code => 'uint16',
names => 'map',
organization => 'utf8_string',
population_density => 'uint32',
postal => 'map',
registered_country => 'map',
represented_country => 'map',
subdivisions => [ 'array', 'map' ],
time_zone => 'utf8_string',
traits => 'map',
traits => 'map',
type => 'utf8_string',
user_type => 'utf8_string',
# for testing only
foo => 'utf8_string',
bar => 'utf8_string',
buzz => 'utf8_string',
our_value => 'utf8_string',
};
my $callback = sub {
my $key = shift;
return $map->{$key} || die <<"ERROR";
Unknown tree key '$key'.
The universal_map_key_type_callback doesn't know what type to use for the passed
key. If you are adding a new key that will be used in a frozen tree / mmdb then
you should update the mapping in both our internal code and here.
ERROR
};
return $callback;
}
sub _write_geoip2_db {
my $type = shift;
my $populate_all_networks_with_data = shift;
my $description = shift;
my $writer = MaxMind::DB::Writer::Tree->new(
ip_version => 6,
record_size => 28,
ip_version => 6,
database_type => $type,
languages => [ 'en', $type eq 'GeoIP2-City' ? ('zh') : () ],
description => {
en => ( $type =~ s/-/ /gr )
. " $description Database (fake GeoIP2 data, for example purposes only)",
$type eq 'GeoIP2-City' ? ( zh => '小型数据库' ) : (),
},
alias_ipv6_to_ipv4 => 1,
map_key_type_callback => _universal_map_key_type_callback(),
remove_reserved_networks => 0,
);
_populate_all_networks( $writer, $populate_all_networks_with_data )
if $populate_all_networks_with_data;
my $value = shift;
my $nodes
= decode_json( read_binary("$Dir/$type.json") );
for my $node (@$nodes) {
for my $network ( keys %$node ) {
$writer->insert_network(
Net::Works::Network->new_from_string( string => $network ),
$node->{$network}
);
}
}
open my $output_fh, '>', "$Dir/$type.mmdb";
$writer->write_tree($output_fh);
close $output_fh;
return;
}
sub _populate_all_networks {
my $writer = shift;
my $data = shift;
my $max_uint128 = uint128(0) - 1;
my @networks = Net::Works::Network->range_as_subnets(
Net::Works::Address->new_from_integer(
integer => 0,
version => 6,
),
Net::Works::Address->new_from_integer(
integer => $max_uint128,
version => 6,
),
);
for my $network (@networks) {
$writer->insert_network( $network => $data );
}
}
main();