-
Notifications
You must be signed in to change notification settings - Fork 0
/
app.js
53 lines (49 loc) · 2.23 KB
/
app.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
var scraper = require('./scraper'),
debug = require('debug')('app'),
jsonfile = require('jsonfile'),
path = require('path'),
readline = require('readline'),
pd = require('parse-domain'),
fork = require('child_process').fork,
spawn = require('child_process').spawn,
util = require('util');
var rl = readline.createInterface({
input: process.stdin,
output: process.stdout
});
console.log('Please enter the URL you wish to scan:');
rl.on('line', function(line) {
startingUrl = line;
console.log(line);
var parsedDomain = pd(line,{});
if(!parsedDomain) {
console.log('Invalid URL: ' + startingUrl + '. Exiting...');
process.exit(1);
} else {
domain = parsedDomain.domain + '.' + parsedDomain.tld;
console.log(domain);
console.log("^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^");
console.log("Give it a minute, but if this crashes try changing the depth limit from 3 to 2 in the line following this console.log statement in app.js.");
console.log("^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^");
scraper.scrape(startingUrl, domain, 3, function(err, res) {
if(err) {
debug('Unexpected Error: ' + err);
} else {
console.log(res);
var filename = path.resolve(__dirname, 'sitemaps/' + domain + '.json');
jsonfile.writeFile(filename, res, {spaces: 2}, function(err) {
console.log('Sitemap successfully written to ' + path.resolve(__dirname, 'sitemaps/' + domain + '.json'));
var child = fork('treeToGraph.js',[filename,domain]);
child.on('exit', (code) => {
console.log(`$$$$$$$$$$$$child process exited with code ${code}`);
console.log(`$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$`);
console.log("Open http:https://localhost:8080/gravity.html?file="+domain+".json to see a visualization of the site map.");
console.log("Also try http:https://localhost:8080/force4.html?file="+domain+".json to see clustering of the site map.");
var favProc = fork('favicon.js',[domain]);
favProc.on('exit', (code) => {var serveProc = fork('serve.js',[filename]);});
});
});
}
});
}
});