Skip to content
Snippets Groups Projects
Unverified Commit a3b0c03d authored by Arne Øslebø's avatar Arne Øslebø
Browse files

added script for sanitizing geoip databases

parent 6e00797b
No related branches found
No related tags found
No related merge requests found
#!/usr/bin/perl
use strict;
use MaxMind::DB::Reader;
use MaxMind::DB::Writer;
use MaxMind::DB::Writer::Tree;
use Data::Dumper;
use Net::Works::Address;
use Try::Tiny;
my $reader = MaxMind::DB::Reader->new( file => $ARGV[0] );
my %types = (
names => 'map',
city => 'map',
continent => 'map',
registered_country => 'map',
represented_country => 'map',
country => 'map',
location => 'map',
postal => 'map',
traits => 'map',
geoname_id => 'uint32',
type => 'utf8_string',
en => 'utf8_string',
de => 'utf8_string',
es => 'utf8_string',
fr => 'utf8_string',
ja => 'utf8_string',
'pt-BR' => 'utf8_string',
ru => 'utf8_string',
'zh-CN' => 'utf8_string',
locales => [ 'array', 'utf8_string' ],
code => 'utf8_string',
geoname_id => 'uint32',
ip_address => 'utf8_string',
subdivisions => [ 'array' , 'map' ],
iso_code => 'utf8_string',
environments => [ 'array', 'utf8_string' ],
expires => 'uint32',
name => 'utf8_string',
time_zone => 'utf8_string',
accuracy_radius => 'uint32',
latitude => 'float',
longitude => 'float',
metro_code => 'uint32',
time_zone => 'utf8_string',
is_in_european_union => 'utf8_string',
is_satellite_provider => 'utf8_string',
is_anonymous_proxy => 'utf8_string',
);
my $tree = MaxMind::DB::Writer::Tree->new(
database_type => 'GeoLite2-City',
description => { en => 'GeoLite2 City database' },
ip_version => 6,
map_key_type_callback => sub { $types{ $_[0] } },
merge_strategy => 'recurse',
record_size => 28,
remove_reserved_networks => 0,
);
my $count=0;
$reader->iterate_search_tree(
sub {
my $ip_as_integer = shift;
my $mask_length = shift;
my $dataref = shift;
my $net_address;
my %data=%{$dataref};
if($count%100000==0) {
print "$count\n";
}
$count++;
my $address = Net::Works::Address->new_from_integer( integer => $ip_as_integer );
$net_address = join '/', $address->as_string, $mask_length;
my $newdata={
city => {
geoname_id => defined($data{'city'}{'geoname_id'})?$data{'city'}{'geoname_id'}:0,
names => {
en => defined($data{'city'}{'names'}{'en'})?$data{'city'}{'names'}{'en'}:"?",
},
},
country => {
geoname_id => defined($data{'country'}{'geoname_id'})?$data{'country'}{'geoname_id'}:0,
iso_code => defined($data{'country'}{'iso_code'})?$data{'country'}{'iso_code'}:"?",
names => {
en => defined($data{'country'}{'names'}{'en'})?$data{'country'}{'names'}{'en'}:"?",
},
},
location => {
accuracy_radius => defined($data{'location'}{'accuracy_radius'})?$data{'location'}{'accuracy_radius'}:0,
latitude => defined($data{'location'}{'latitude'})?$data{'location'}{'latitude'}:0.0,
longitude => defined($data{'location'}{'longitude'})?$data{'location'}{'longitude'}:0.0,
}
};
$tree->insert_network( $net_address, $newdata );
}
);
print "Saving..\n";
open my $fh, '>:raw', $ARGV[1];
$tree->write_tree( $fh );
close $fh;
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment