Skip to content

Commit

Permalink
implemented geoliteDB.get(db), releasing 1.0.0
Browse files Browse the repository at this point in the history
various formatting improvements, added accuracy_radius because I found it was still relevant in the free data (my bad!)

updated tests to include and validate some data
  • Loading branch information
zikeji committed Apr 11, 2017
1 parent 26fb96a commit adf2b77
Show file tree
Hide file tree
Showing 13 changed files with 423 additions and 28 deletions.
107 changes: 98 additions & 9 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,22 +14,16 @@ I had originally wanted to return the data similar to the [node-maxmind](https:/
- [API](#api)
- [geolite.init(options)](#geoliteinitoptions)
- [geoliteDB.update()](#geolitedbupdate)
- [geoliteDB.get(ip)](#geolitedbgetip)
- [Performance](#performance)
- [Initializing & updating the database](#initializing--updating-the-database)
- [Queries](#queries)
- [License](#license)

## NOTICE

This module is in pre-pre-pre alpha and is not ready for use!

### TODO

- geolite geoname_id importing
- API for queries

## Requirements

- 1GB RAM
- This is the for the build process only, if you plan on building & updating a different machine you can still utilize this on a machine with less RAM.
- PostgreSQL >= 9.4

## Install
Expand Down Expand Up @@ -89,6 +83,101 @@ Forces an update on the PostgreSQL database using the options provided in `geoli

Resolves when finished, if any error occurs it rejects.

### `geoliteDB.get(ip)`

Gets all the information available on the provided IP.

#### String: `ip`

The IPv4 address or IPv6 address you are querying, as a string.

#### Returns: `Promise (result)`

The resulting lookup data from you query.

##### Example

```json
{
"city":{
"names":{
"de":"Mountain View",
"en":"Mountain View",
"fr":"Mountain View",
"ja":"マウンテンビュー",
"ru":"Маунтин-Вью",
"zh-CN":"芒廷维尤"
}
},
"continent":{
"code":"NA",
"names":{
"de":"Nordamerika",
"en":"North America",
"es":"Norteamérica",
"fr":"Amérique du Nord",
"ja":"北アメリカ",
"pt-BR":"América do Norte",
"ru":"Северная Америка",
"zh-CN":"北美洲"
}
},
"country":{
"iso_code":"US",
"names":{
"de":"USA",
"en":"United States",
"es":"Estados Unidos",
"fr":"États-Unis",
"ja":"アメリカ合衆国",
"pt-BR":"Estados Unidos",
"ru":"США",
"zh-CN":"美国"
}
},
"isp":{
"asn":"AS15169",
"org":"Google Inc."
},
"location":{
"latitude":"37.386",
"longitude":"-122.0838",
"time_zone":"America/Los_Angeles"
},
"postal":{
"code":"94035"
},
"registered_country":{
"iso_code":"US",
"names":{
"de":"USA",
"en":"United States",
"es":"Estados Unidos",
"fr":"États-Unis",
"ja":"アメリカ合衆国",
"pt-BR":"Estados Unidos",
"ru":"США",
"zh-CN":"美国"
}
},
"subdivisions":[
{
"iso_code":"CA",
"names":{
"de":"Kalifornien",
"en":"California",
"es":"California",
"fr":"Californie",
"ja":"カリフォルニア州",
"pt-BR":"Califórnia",
"ru":"Калифорния",
"zh-CN":"加利福尼亚州"
}
}
]
}
```

## Performance

If you clone the repo and run the tests on your target system you should get a good idea of expected performance from the test results.
Expand Down
4 changes: 2 additions & 2 deletions lib/db/importer/dbvalidator/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,12 @@ module.exports = (db) => new Promise((resolve, reject) => {

p.push(validator.validate(db, {
table_name: 'geolite2_city_ipv4',
columns: ['network', 'geoname_id', 'registered_country_geoname_id', 'represented_country_geoname_id', 'postal_code', 'latitude', 'longitude'],
columns: ['network', 'geoname_id', 'registered_country_geoname_id', 'represented_country_geoname_id', 'postal_code', 'latitude', 'longitude', 'accuracy_radius'],
indexes: ['gl2_ipv4_coords_index', 'gl2_ipv4_postal_code_index', 'gl2_ipv4_represented_country_geoname_index', 'gl2_ipv4_registered_county_geoname_index', 'gl2_ipv4_geoname_index', 'gl2_ipv4_network_index'],
}));
p.push(validator.validate(db, {
table_name: 'geolite2_city_ipv6',
columns: ['network', 'geoname_id', 'registered_country_geoname_id', 'represented_country_geoname_id', 'postal_code', 'latitude', 'longitude'],
columns: ['network', 'geoname_id', 'registered_country_geoname_id', 'represented_country_geoname_id', 'postal_code', 'latitude', 'longitude', 'accuracy_radius'],
indexes: ['gl2_ipv6_coords_index', 'gl2_ipv6_postal_code_index', 'gl2_ipv6_represented_country_geoname_index', 'gl2_ipv6_registered_county_geoname_index', 'gl2_ipv6_geoname_index', 'gl2_ipv6_network_index'],
}));
p.push(validator.validate(db, {
Expand Down
5 changes: 3 additions & 2 deletions lib/db/importer/handlers/geolite2_city/geolite2_city_x.js
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ const parse_file = (path, db, cs) => new Promise((resolve, reject) => {
postal_code: line.postal_code,
latitude: parseFloat(line.latitude),
longitude: parseFloat(line.longitude),
accuracy_radius: line.accuracy_radius ? parseInt(line.accuracy_radius, 10) : null,
};
acc.push(record);
if (acc.length > 10000) {
Expand All @@ -53,14 +54,14 @@ module.exports = (db, config, proto, path_ips) => new Promise((resolve, reject)
const dbname = `geolite2_city_${proto}`;
const keyname = `gl2_${proto}`;
db.none(`DROP TABLE IF EXISTS ${dbname}_new;
CREATE TABLE ${dbname}_new (network cidr NOT NULL, geoname_id int, registered_country_geoname_id int, represented_country_geoname_id int, postal_code varchar(10), latitude decimal, longitude decimal);
CREATE TABLE ${dbname}_new (network cidr NOT NULL, geoname_id int, registered_country_geoname_id int, represented_country_geoname_id int, postal_code varchar(10), latitude decimal, longitude decimal, accuracy_radius INT);
CREATE INDEX ${keyname}_network_index_new ON ${dbname}_new USING gist (network inet_ops);
CREATE INDEX ${keyname}_geoname_index_new ON ${dbname}_new (geoname_id);
CREATE INDEX ${keyname}_registered_county_geoname_index_new ON ${dbname}_new (registered_country_geoname_id);
CREATE INDEX ${keyname}_represented_country_geoname_index_new ON ${dbname}_new (represented_country_geoname_id);
CREATE INDEX ${keyname}_postal_code_index_new ON ${dbname}_new (postal_code);
CREATE INDEX ${keyname}_coords_index_new ON ${dbname}_new (latitude, longitude);`).then(() => {
const cs = new db.$config.pgp.helpers.ColumnSet(['network', 'geoname_id', 'registered_country_geoname_id', 'represented_country_geoname_id', 'postal_code', 'latitude', 'longitude'], {
const cs = new db.$config.pgp.helpers.ColumnSet(['network', 'geoname_id', 'registered_country_geoname_id', 'represented_country_geoname_id', 'postal_code', 'latitude', 'longitude', 'accuracy_radius'], {
table: `${dbname}_new`,
});

Expand Down
6 changes: 3 additions & 3 deletions lib/db/importer/handlers/geolite2_city/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,9 @@ module.exports = (db, config, files) => new Promise((resolve, reject) => {
}
}
const p = [];
p.push(require('./geolite2_city_x.js')(db, config, 'ipv4', path_ips_v4));
p.push(require('./geolite2_city_x.js')(db, config, 'ipv6', path_ips_v6));
p.push(require('./geolite2_city_locations.js')(db, config, path_locs));
p.push(require('./geolite2_city_x')(db, config, 'ipv4', path_ips_v4));
p.push(require('./geolite2_city_x')(db, config, 'ipv6', path_ips_v6));
p.push(require('./geolite2_city_locations')(db, config, path_locs));

Promise.all(p)
.then(resolve)
Expand Down
4 changes: 2 additions & 2 deletions lib/db/importer/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@ module.exports = (config, db) => {
const p = [];
if (config.downloads.local) {
p.push(decompress(config.downloads.resources.geolite2_city, config.downloads.location).then((files) => require('./handlers/geolite2_city')(db, config, files)));
p.push(decompress(config.downloads.resources.geolite_asn, config.downloads.location).then((files) => require('./handlers/geolite_asn.js')(db, config, files)));
p.push(decompress(config.downloads.resources.geolite_asn_ipv6, config.downloads.location).then((files) => require('./handlers/geolite_asn_ipv6.js')(db, config, files)));
p.push(decompress(config.downloads.resources.geolite_asn, config.downloads.location).then((files) => require('./handlers/geolite_asn')(db, config, files)));
p.push(decompress(config.downloads.resources.geolite_asn_ipv6, config.downloads.location).then((files) => require('./handlers/geolite_asn_ipv6')(db, config, files)));
} else {
p.push(download(config.downloads.resources.geolite2_city, config.downloads.location, {
extract: true,
Expand Down
2 changes: 2 additions & 0 deletions lib/db/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,14 @@ module.exports = (config) => new Promise((resolve, reject) => {
conn.done();

const importer = require('./importer')(config, db);
const queries = require('./queries')(db);

importer.init()
.then(() => {
// todo finish DB
const geoliteDB = {
update: importer.update,
get: queries.get,
};
resolve(geoliteDB);
})
Expand Down
8 changes: 8 additions & 0 deletions lib/db/queries/geonames.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
'use strict';

module.exports = {
get: (t, geoname_id) => t.one('SELECT continent_code, country_iso_code, subdivision_1_iso_code, subdivision_2_iso_code, metro_code, time_zone FROM geolite2_city_locations WHERE geoname_id = $1', geoname_id),
getCountry: (t, geoname_id) => t.one('SELECT country_iso_code FROM geolite2_city_locations WHERE geoname_id = $1', geoname_id),
getLocales: (t, geoname_id) => t.many('SELECT locale_code, continent_name, country_name, subdivision_1_name, subdivision_2_name, city_name FROM geolite2_city_locations_locales WHERE geoname_id = $1 ORDER BY locale_code ASC', geoname_id),
getCountryLocales: (t, geoname_id) => t.many('SELECT locale_code, country_name FROM geolite2_city_locations_locales WHERE geoname_id = $1 ORDER BY locale_code ASC', geoname_id),
};
78 changes: 78 additions & 0 deletions lib/db/queries/getip.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
'use strict';

const ip = require('ip-address');

const invalid_v4 = [
new ip.Address4('0.0.0.0/8'),
new ip.Address4('10.0.0.0/8'),
new ip.Address4('100.64.0.0/10'),
new ip.Address4('127.0.0.0/8'),
new ip.Address4('127.0.53.53'),
new ip.Address4('169.254.0.0/16'),
new ip.Address4('172.16.0.0/12'),
new ip.Address4('192.0.0.0/24'),
new ip.Address4('192.0.2.0/24'),
new ip.Address4('192.168.0.0/16'),
new ip.Address4('198.18.0.0/15'),
new ip.Address4('198.51.100.0/24'),
new ip.Address4('203.0.113.0/24'),
new ip.Address4('224.0.0.0/4'),
new ip.Address4('240.0.0.0/4'),
new ip.Address4('255.255.255.255/32'),
];

const invalid_v6 = [
new ip.Address6('::/128'),
new ip.Address6('::1/128'),
new ip.Address6('::ffff:0:0/96'),
new ip.Address6('::/96'),
new ip.Address6('100::/64'),
new ip.Address6('2001:10::/28'),
new ip.Address6('2001:db8::/32'),
new ip.Address6('fc00::/7'),
new ip.Address6('fe80::/10'),
new ip.Address6('fec0::/10'),
new ip.Address6('2002::/24'),
new ip.Address6('2002:a00::/24'),
new ip.Address6('2002:7f00::/24'),
new ip.Address6('2002:a9fe::/32'),
new ip.Address6('2002:ac10::/28'),
new ip.Address6('2002:c000::/40'),
new ip.Address6('2002:c000:200::/40'),
new ip.Address6('2002:c0a8::/32'),
new ip.Address6('2002:c612::/31'),
new ip.Address6('2002:c633:6400::/40'),
new ip.Address6('2002:cb00:7100::/40'),
new ip.Address6('2002:e000::/20'),
new ip.Address6('2002:f000::/20'),
new ip.Address6('2002:ffff:ffff::/48'),
new ip.Address6('2001::/40'),
new ip.Address6('2001:0:a00::/40'),
new ip.Address6('2001:0:7f00::/40'),
new ip.Address6('2001:0:a9fe::/48'),
new ip.Address6('2001:0:ac10::/44'),
new ip.Address6('2001:0:c000::/56'),
new ip.Address6('2001:0:c000:200::/56'),
new ip.Address6('2001:0:c0a8::/48'),
new ip.Address6('2001:0:c612::/47'),
new ip.Address6('2001:0:c633:6400::/56'),
new ip.Address6('2001:0:cb00:7100::/56'),
new ip.Address6('2001:0:e000::/36'),
new ip.Address6('2001:0:f000::/36'),
new ip.Address6('2001:0:ffff:ffff::/64'),
];

module.exports = (ipaddr) => {
const ipv4 = new ip.Address4(ipaddr);
const ipv6 = new ip.Address6(ipaddr);
if (ipv4.isValid()) {
if (invalid_v4.every((subnet) => !ipv4.isInSubnet(subnet))) {
return ipv4;
}
} else if (ipv6.isValid()) {
if (invalid_v6.every((subnet) => !ipv6.isInSubnet(subnet))) {
return ipv6;
}
}
return false;
};
66 changes: 66 additions & 0 deletions lib/db/queries/index.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
'use strict';

const getIP = require('./getip');
const pretty = require('./pretty');

module.exports = (db) => {
const ip = require('./ip');
const geonames = require('./geonames');

const queries = {
get: (ipaddr) => new Promise((resolve, reject) => {
ipaddr = getIP(ipaddr);
if (ipaddr) {
if (ipaddr.v4) {
ip.getv4(db, ipaddr.address)
.then(iploc => {
db.task(t => {
const ops = [ip.getASNv4(t, ipaddr.address), geonames.get(t, iploc.geoname_id), geonames.getLocales(t, iploc.geoname_id)];
if (iploc.registered_country_geoname_id !== null) {
ops.push(geonames.getCountry(t, iploc.registered_country_geoname_id));
ops.push(geonames.getCountryLocales(t, iploc.registered_country_geoname_id));
}
if (iploc.represented_country_geoname_id !== null) {
ops.push(geonames.getCountry(t, iploc.represented_country_geoname_id));
ops.push(geonames.getCountryLocales(t, iploc.represented_country_geoname_id));
}
return t.batch(ops);
})
.then(data => {
resolve(pretty(iploc, data));
})
.catch(reject);
})
.catch(reject);
} else {
ip.getv6(db, ipaddr.address)
.then(iploc => {
db.task(t => {
const ops = [ip.getASNv6(t, ipaddr.address), geonames.get(t, iploc.geoname_id), geonames.getLocales(t, iploc.geoname_id)];
if (iploc.registered_country_geoname_id !== null) {
ops.push(geonames.getCountry(t, iploc.registered_country_geoname_id));
ops.push(geonames.getCountryLocales(t, iploc.registered_country_geoname_id));
}
if (iploc.represented_country_geoname_id !== null) {
ops.push(geonames.getCountry(t, iploc.represented_country_geoname_id));
ops.push(geonames.getCountryLocales(t, iploc.represented_country_geoname_id));
}
return t.batch(ops);
})
.then(data => {
resolve(pretty(iploc, data));
})
.catch(reject);
})
.catch(reject);
}
} else {
reject({
error: 'invalid_ip',
});
}
}),
};

return queries;
};
8 changes: 8 additions & 0 deletions lib/db/queries/ip.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
'use strict';

module.exports = {
getv4: (t, ip) => t.one('SELECT geoname_id, registered_country_geoname_id, represented_country_geoname_id, postal_code, latitude, longitude, accuracy_radius FROM geolite2_city_ipv4 WHERE $1::inet <<= network', ip),
getv6: (t, ip) => t.one('SELECT geoname_id, registered_country_geoname_id, represented_country_geoname_id, postal_code, latitude, longitude, accuracy_radius FROM geolite2_city_ipv6 WHERE $1::inet <<= network', ip),
getASNv4: (t, ip) => t.one('SELECT asn, org FROM geolite_asn WHERE $1::inet BETWEEN ip_start AND ip_end', ip),
getASNv6: (t, ip) => t.one('SELECT asn, org FROM geolite_asn_ipv6 WHERE $1::inet <<= network', ip),
};
Loading

0 comments on commit adf2b77

Please sign in to comment.