Merge branch 'master' of scm.uninett.no:geant-wp8-t3.1/nifi-processors

scripts/geoipsanitizer/geoipsanitizer.pl

+#!/usr/bin/perl
+
+use strict;
+
+use MaxMind::DB::Reader;
+use MaxMind::DB::Writer;
+use MaxMind::DB::Writer::Tree;
+use Data::Dumper;
+use Net::Works::Address;
+use Try::Tiny;
+
+my $reader = MaxMind::DB::Reader->new( file => $ARGV[0] );
+
+my %types = (
+    names                  => 'map',
+    city                   => 'map',
+    continent              => 'map',
+    registered_country     => 'map',
+    represented_country    => 'map',
+    country                => 'map',
+    location               => 'map',
+    postal                 => 'map',
+    traits                 => 'map',
+ 
+    geoname_id             => 'uint32',
+ 
+    type                   => 'utf8_string',
+    en                     => 'utf8_string',
+    de                     => 'utf8_string',
+    es                     => 'utf8_string',
+    fr                     => 'utf8_string',
+    ja                     => 'utf8_string',
+    'pt-BR'                => 'utf8_string',
+    ru                     => 'utf8_string',
+    'zh-CN'                => 'utf8_string',
+ 
+    locales                => [ 'array', 'utf8_string' ],
+    code                   => 'utf8_string',
+    geoname_id             => 'uint32',
+    ip_address             => 'utf8_string',
+    subdivisions           => [ 'array' , 'map' ],
+    iso_code               => 'utf8_string',
+    environments           => [ 'array', 'utf8_string' ],
+    expires                => 'uint32',
+    name                   => 'utf8_string',
+    time_zone              => 'utf8_string',
+    accuracy_radius        => 'uint32',
+    latitude               => 'float',
+    longitude              => 'float',
+    metro_code             => 'uint32',
+    time_zone              => 'utf8_string',
+    is_in_european_union   => 'utf8_string',
+    is_satellite_provider   => 'utf8_string',
+    is_anonymous_proxy     => 'utf8_string',
+);
+
+my $tree = MaxMind::DB::Writer::Tree->new(
+ 
+    database_type => 'GeoLite2-City',
+    description => { en => 'GeoLite2 City database' },
+    ip_version => 6,
+    map_key_type_callback => sub { $types{ $_[0] } },
+    merge_strategy => 'recurse',
+    record_size => 28,
+    remove_reserved_networks => 0,
+);
+
+my $count=0;
+
+
+$reader->iterate_search_tree(
+    sub {
+	my $ip_as_integer = shift;
+	my $mask_length   = shift;
+	my $dataref          = shift;
+	my $net_address;
+
+	my %data=%{$dataref};
+	
+	if($count%100000==0) {
+	    print "$count\n";
+	}
+	$count++;
+	
+	my $address = Net::Works::Address->new_from_integer( integer => $ip_as_integer );
+	$net_address = join '/', $address->as_string, $mask_length;
+
+	my $newdata={
+	    city => {
+		    geoname_id => defined($data{'city'}{'geoname_id'})?$data{'city'}{'geoname_id'}:0,
+		    names => {
+			en => defined($data{'city'}{'names'}{'en'})?$data{'city'}{'names'}{'en'}:"?",
+		    },
+		},
+		country => {
+		    geoname_id => defined($data{'country'}{'geoname_id'})?$data{'country'}{'geoname_id'}:0,
+		    iso_code => defined($data{'country'}{'iso_code'})?$data{'country'}{'iso_code'}:"?",
+		    names => {
+			en => defined($data{'country'}{'names'}{'en'})?$data{'country'}{'names'}{'en'}:"?",
+		    },
+		},
+		location => {
+		    accuracy_radius => defined($data{'location'}{'accuracy_radius'})?$data{'location'}{'accuracy_radius'}:0,
+		    latitude => defined($data{'location'}{'latitude'})?$data{'location'}{'latitude'}:0.0,
+		    longitude => defined($data{'location'}{'longitude'})?$data{'location'}{'longitude'}:0.0,
+	    }
+	};
+
+	$tree->insert_network( $net_address, $newdata );
+    }
+    );
+
+print "Saving..\n";
+open my $fh, '>:raw', $ARGV[1];
+$tree->write_tree( $fh );
+close $fh;
+