diff --git a/scripts/geoipsanitizer/geoipsanitizer.pl b/scripts/geoipsanitizer/geoipsanitizer.pl new file mode 100644 index 0000000000000000000000000000000000000000..4a50ad608fa78a803e3cac641681d8da9517eecd --- /dev/null +++ b/scripts/geoipsanitizer/geoipsanitizer.pl @@ -0,0 +1,117 @@ +#!/usr/bin/perl + +use strict; + +use MaxMind::DB::Reader; +use MaxMind::DB::Writer; +use MaxMind::DB::Writer::Tree; +use Data::Dumper; +use Net::Works::Address; +use Try::Tiny; + +my $reader = MaxMind::DB::Reader->new( file => $ARGV[0] ); + +my %types = ( + names => 'map', + city => 'map', + continent => 'map', + registered_country => 'map', + represented_country => 'map', + country => 'map', + location => 'map', + postal => 'map', + traits => 'map', + + geoname_id => 'uint32', + + type => 'utf8_string', + en => 'utf8_string', + de => 'utf8_string', + es => 'utf8_string', + fr => 'utf8_string', + ja => 'utf8_string', + 'pt-BR' => 'utf8_string', + ru => 'utf8_string', + 'zh-CN' => 'utf8_string', + + locales => [ 'array', 'utf8_string' ], + code => 'utf8_string', + geoname_id => 'uint32', + ip_address => 'utf8_string', + subdivisions => [ 'array' , 'map' ], + iso_code => 'utf8_string', + environments => [ 'array', 'utf8_string' ], + expires => 'uint32', + name => 'utf8_string', + time_zone => 'utf8_string', + accuracy_radius => 'uint32', + latitude => 'float', + longitude => 'float', + metro_code => 'uint32', + time_zone => 'utf8_string', + is_in_european_union => 'utf8_string', + is_satellite_provider => 'utf8_string', + is_anonymous_proxy => 'utf8_string', +); + +my $tree = MaxMind::DB::Writer::Tree->new( + + database_type => 'GeoLite2-City', + description => { en => 'GeoLite2 City database' }, + ip_version => 6, + map_key_type_callback => sub { $types{ $_[0] } }, + merge_strategy => 'recurse', + record_size => 28, + remove_reserved_networks => 0, +); + +my $count=0; + + +$reader->iterate_search_tree( + sub { + my $ip_as_integer = shift; + my $mask_length = shift; + my $dataref = shift; + my $net_address; + + my %data=%{$dataref}; + + if($count%100000==0) { + print "$count\n"; + } + $count++; + + my $address = Net::Works::Address->new_from_integer( integer => $ip_as_integer ); + $net_address = join '/', $address->as_string, $mask_length; + + my $newdata={ + city => { + geoname_id => defined($data{'city'}{'geoname_id'})?$data{'city'}{'geoname_id'}:0, + names => { + en => defined($data{'city'}{'names'}{'en'})?$data{'city'}{'names'}{'en'}:"?", + }, + }, + country => { + geoname_id => defined($data{'country'}{'geoname_id'})?$data{'country'}{'geoname_id'}:0, + iso_code => defined($data{'country'}{'iso_code'})?$data{'country'}{'iso_code'}:"?", + names => { + en => defined($data{'country'}{'names'}{'en'})?$data{'country'}{'names'}{'en'}:"?", + }, + }, + location => { + accuracy_radius => defined($data{'location'}{'accuracy_radius'})?$data{'location'}{'accuracy_radius'}:0, + latitude => defined($data{'location'}{'latitude'})?$data{'location'}{'latitude'}:0.0, + longitude => defined($data{'location'}{'longitude'})?$data{'location'}{'longitude'}:0.0, + } + }; + + $tree->insert_network( $net_address, $newdata ); + } + ); + +print "Saving..\n"; +open my $fh, '>:raw', $ARGV[1]; +$tree->write_tree( $fh ); +close $fh; +