Start to fix getPiData.js

.gitignore

@@ -1,5 +1,8 @@
-update/repositories
 update/node_modules/
+update/repositories
+update/repositories/*
+update/repositories*
+
 doc/
 node_modules/
 npm-debug.log

bower.json

@@ -3,7 +3,7 @@
  "version": "0.1.0",
  "author": "(mostly)Arnon Eilat",
  "description": "A listing of open source efforts at The Public Knoweldge Workshop on GitHub.",
- "license": "gplv3",
+ "license": "GPLv3",
  "keywords": [
  "Hasadna",
  "Public Knoweldge Workshop"

editor/script.js

@@ -5,10 +5,9 @@ var App = angular.module('App', [
 ]);
 App.directive('ngBooleanRadio', ngBooleanRadio);
 
-/***
- * 
- * @param {type} param1
- * @param {type} param2
+/**
+ * @param {object} $scope
+ * @param {object} $http
  */
 App.controller('eKnightEditor', function($scope, $http) {
 
@@ -81,8 +80,7 @@ App.controller('eKnightEditor', function($scope, $http) {
 
 });
 
-/***
- * 
+/**
  * @param {ngRoute.$routeProvider} $routeProvider
  */
 App.config(function($routeProvider) {

editor/server.js

@@ -6,6 +6,8 @@ var removeHashKeys = require("./serverUtill").removeHashKeys;
 var mime = require("./serverUtill").mime;
 var port = process.argv[2] || 8888;
 var beautify = require('js-beautify').js_beautify;
+
+
 function server(request, response) {
  var uri = url.parse(request.url).pathname;
  var filename = path.join(process.cwd(), uri);
@@ -21,7 +23,7 @@ function server(request, response) {
  + '\n// If we are running from node let be a module.'
  + '\nif (typeof module !== \'undefined\' && module.exports)'
  + '\n\tmodule.exports.eKnightsData = eKnightsData;';
- fs.writeFile("../eKnightsData.js", fileContents, function(err) {
+ fs.writeFile("../src/eKnightsData.js", fileContents, function(err) {
  if (err)
  console.error(err);
  else
@@ -38,7 +40,7 @@ function server(request, response) {
  return;
  }
  else {
- path.exists(filename, function(exists) {
+ fs.exists(filename, function(exists) {
  if (!exists) {
  response.writeHead(404, {"Content-Type": "text/plain"});
  response.write("404 Not Found\n");
@@ -66,7 +68,7 @@ function server(request, response) {
 }
 
 // Copy eKnightsData.js to work with.
-fs.createReadStream('../eKnightsData.js').pipe(fs.createWriteStream('eKnightsData.js'));
+fs.createReadStream('../src/eKnightsData.js').pipe(fs.createWriteStream('eKnightsData.js'));
 // Remove eKnightsData.js
 function exitHandler(options, err) {
  fs.unlink('eKnightsData.js', function(err) {

editor/serverUtill.js

@@ -1,11 +1,11 @@
 
 
 
-/***
+/**
  * @see https://github.com/ry/node_chat/blob/master/fu.js
  */
-var mime = {
- /***
+module.exports.mime = {
+ /**
  * @description Returns MIME type for extension, or fallback, or octet-steam
  * @param {string} ext
  * @param {string} fallback
@@ -185,11 +185,11 @@ var mime = {
 };
 
 
-/***
+/**
  * @description Remove All angular $$HashKeys from obj and its childrens recursive
  * @param {Object} obj
  */
-function removeHashKeys(obj) {
+module.exports.removeHashKeys = function(obj) {
  var k;
  if (obj instanceof Object) {
  if (obj.$$hashKey)
@@ -200,8 +200,4 @@ function removeHashKeys(obj) {
  removeHashKeys(obj[k]); // Recursive call
  }
  }
-}
-
-
-module.exports.removeHashKeys = removeHashKeys;
-module.exports.mime = mime;
+};

src/data/open-pension-pi.json

@@ -1 +1 @@
-{"name":"OpenPension","size":1310901,"lines":69,"children":[{"name":"","size":0,"lines":0,"children":"no children","log":""},{"name":"","size":0,"lines":0,"children":"no children","log":""},{"name":"","size":0,"lines":0,"children":"no children","log":""},{"name":".travis.yml","size":35,"lines":1,"children":"","log":[]},{"name":"CSVWriter.js","size":327,"lines":1,"children":"","log":[]},{"name":"Cakefile","size":407,"lines":1,"children":"","log":[]},{"name":"LICENSE","size":1104,"lines":1,"children":"","log":[]},{"name":"LevDistance.js","size":1253,"lines":1,"children":"","log":[]},{"name":"README.md","size":418,"lines":1,"children":"","log":[]},{"name":"common","size":0,"lines":0,"children":[],"log":[]},{"name":"config.json","size":117,"lines":1,"children":"","log":[]},{"name":"csv_handlers","size":2670,"lines":3,"children":[{"name":"__init__.py","size":0,"lines":1,"children":"","log":[]},{"name":"db.py","size":671,"lines":1,"children":"","log":[]},{"name":"handler.py","size":1999,"lines":1,"children":"","log":[]}],"log":[]},{"name":"customFetcher.coffee","size":3096,"lines":1,"children":"","log":[]},{"name":"db.js","size":6686,"lines":1,"children":"","log":[]},{"name":"dbServer.coffee","size":1052,"lines":1,"children":"","log":[]},{"name":"db_runner.coffee","size":219,"lines":1,"children":"","log":[]},{"name":"fetcher.common.js","size":3597,"lines":1,"children":"","log":[]},{"name":"fetcher.harel.js","size":3437,"lines":1,"children":"","log":[]},{"name":"fetcher.js","size":3077,"lines":1,"children":"","log":[]},{"name":"files.csv","size":11123,"lines":1,"children":"","log":[]},{"name":"genericImporter.js","size":13196,"lines":1,"children":"","log":[]},{"name":"index.js","size":1697,"lines":1,"children":"","log":[]},{"name":"package.json","size":724,"lines":1,"children":"","log":[]},{"name":"specs","size":1218763,"lines":38,"children":[{"name":"CheckSumsSpec.coffee","size":0,"lines":1,"children":"","log":[]},{"name":"Helpers.coffee","size":672,"lines":1,"children":"","log":[]},{"name":"ParseXMLSpec.coffee","size":4418,"lines":1,"children":"","log":[]},{"name":"ValidatorSpec.coffee","size":7975,"lines":1,"children":"","log":[]},{"name":"assets","size":1205698,"lines":34,"children":[{"name":"Migdal_2012_4_1161.xlsx","size":248111,"lines":6,"children":"","log":[]},{"name":"Migdal_2013_3_414.xlsx","size":184266,"lines":3,"children":"","log":[]},{"name":"Migdal_2013_3_579.xlsx","size":176028,"lines":5,"children":"","log":[]},{"name":"Migdal_2013_3_99118.xlsx","size":260026,"lines":8,"children":"","log":[]},{"name":"harel_2010_3_739.xlsx","size":98101,"lines":5,"children":"","log":[]},{"name":"harel_2013_3_153.xlsx","size":239166,"lines":7,"children":"","log":[]}],"log":[]}],"log":[]},{"name":"test.js","size":905,"lines":1,"children":"","log":[]},{"name":"tests","size":7410,"lines":5,"children":[{"name":"dbtest.js","size":1686,"lines":1,"children":"","log":[]},{"name":"dbtest1.js","size":1549,"lines":1,"children":"","log":[]},{"name":"dbtest2.js","size":1379,"lines":1,"children":"","log":[]},{"name":"dbtest3.js","size":1449,"lines":1,"children":"","log":[]},{"name":"dbtest4.js","size":1347,"lines":1,"children":"","log":[]}],"log":[]},{"name":"validator.js","size":22313,"lines":1,"children":"","log":[]},{"name":"xlsxparser.js","size":6953,"lines":1,"children":"","log":[]}],"log":[]}
+{"name":"OpenPension","size":1429488,"lines":76,"children":[{"name":"","size":0,"lines":0,"children":"no children","log":""},{"name":"","size":0,"lines":0,"children":"no children","log":""},{"name":"","size":0,"lines":0,"children":"no children","log":""},{"name":".travis.yml","size":35,"lines":1,"children":"","log":[]},{"name":"CSVWriter.js","size":327,"lines":1,"children":"","log":[]},{"name":"Cakefile","size":399,"lines":1,"children":"","log":[]},{"name":"LICENSE","size":1172,"lines":1,"children":"","log":[]},{"name":"LevDistance.js","size":1253,"lines":1,"children":"","log":[]},{"name":"README.md","size":418,"lines":1,"children":"","log":[]},{"name":"common","size":0,"lines":0,"children":[],"log":[]},{"name":"config.json","size":117,"lines":1,"children":"","log":[]},{"name":"csv_handlers","size":2670,"lines":3,"children":[{"name":"__init__.py","size":0,"lines":1,"children":"","log":[]},{"name":"db.py","size":671,"lines":1,"children":"","log":[]},{"name":"handler.py","size":1999,"lines":1,"children":"","log":[]}],"log":[]},{"name":"customFetcher.coffee","size":3614,"lines":1,"children":"","log":[]},{"name":"db.js","size":6686,"lines":1,"children":"","log":[]},{"name":"dbServer.coffee","size":1052,"lines":1,"children":"","log":[]},{"name":"db_runner.coffee","size":219,"lines":1,"children":"","log":[]},{"name":"fetcher.common.js","size":3849,"lines":1,"children":"","log":[]},{"name":"fetcher.harel.js","size":4504,"lines":1,"children":"","log":[]},{"name":"fetcher.js","size":5009,"lines":1,"children":"","log":[]},{"name":"files.csv","size":11123,"lines":1,"children":"","log":[]},{"name":"files_loader.js","size":794,"lines":1,"children":"","log":[]},{"name":"genericImporter.js","size":13289,"lines":1,"children":"","log":[]},{"name":"index.js","size":2147,"lines":1,"children":"","log":[]},{"name":"package.json","size":724,"lines":1,"children":"","log":[]},{"name":"specs","size":1331663,"lines":44,"children":[{"name":"CheckSumsSpec.coffee","size":0,"lines":1,"children":"","log":[]},{"name":"Helpers.coffee","size":672,"lines":1,"children":"","log":[]},{"name":"ParseXMLSpec.coffee","size":4418,"lines":1,"children":"","log":[]},{"name":"ValidatorSpec.coffee","size":8853,"lines":1,"children":"","log":[]},{"name":"assets","size":1317720,"lines":40,"children":[{"name":"Migdal_2012_4_1161.xlsx","size":248111,"lines":6,"children":"","log":[]},{"name":"Migdal_2013_3_414.xlsx","size":184266,"lines":3,"children":"","log":[]},{"name":"Migdal_2013_3_579.xlsx","size":176028,"lines":5,"children":"","log":[]},{"name":"Migdal_2013_3_99118.xlsx","size":260026,"lines":8,"children":"","log":[]},{"name":"harel_2010_3_739.xlsx","size":98101,"lines":5,"children":"","log":[]},{"name":"harel_2013_3_153.xlsx","size":239166,"lines":7,"children":"","log":[]},{"name":"psagot_2013_3_15312070.xlsx","size":112022,"lines":6,"children":"","log":[]}],"log":[]}],"log":[]},{"name":"test.js","size":905,"lines":1,"children":"","log":[]},{"name":"tests","size":7410,"lines":5,"children":[{"name":"dbtest.js","size":1686,"lines":1,"children":"","log":[]},{"name":"dbtest1.js","size":1549,"lines":1,"children":"","log":[]},{"name":"dbtest2.js","size":1379,"lines":1,"children":"","log":[]},{"name":"dbtest3.js","size":1449,"lines":1,"children":"","log":[]},{"name":"dbtest4.js","size":1347,"lines":1,"children":"","log":[]}],"log":[]},{"name":"validator.js","size":22718,"lines":1,"children":"","log":[]},{"name":"xlsxparser.js","size":7069,"lines":1,"children":"","log":[]}],"log":[]}

src/data/openlaw-bot-pi.json

@@ -1 +1 @@
-{"name":"openlaw-bot","size":54370,"lines":10,"children":[{"name":"","size":0,"lines":0,"children":"no children","log":""},{"name":"","size":0,"lines":0,"children":"no children","log":""},{"name":"README","size":0,"lines":1,"children":"","log":[]},{"name":"README.md","size":46,"lines":1,"children":"","log":[]},{"name":"bot.pl","size":2220,"lines":1,"children":"","log":[]},{"name":"bot.py","size":5499,"lines":1,"children":"","log":[]},{"name":"editnotice-bot.pl","size":1216,"lines":1,"children":"","log":[]},{"name":"format-wiki.pl","size":30242,"lines":1,"children":"","log":[]},{"name":"syntax-wiki.pl","size":6610,"lines":1,"children":"","log":[]},{"name":"unformat.pl","size":3855,"lines":1,"children":"","log":[]},{"name":"wikiconnect.py","size":4544,"lines":1,"children":"","log":[]}],"log":[]}
+{"name":"openlaw-bot","size":58841,"lines":10,"children":[{"name":"","size":0,"lines":0,"children":"no children","log":""},{"name":"","size":0,"lines":0,"children":"no children","log":""},{"name":"README","size":0,"lines":1,"children":"","log":[]},{"name":"README.md","size":46,"lines":1,"children":"","log":[]},{"name":"bot.pl","size":4332,"lines":1,"children":"","log":[]},{"name":"bot.py","size":5499,"lines":1,"children":"","log":[]},{"name":"editnotice-bot.pl","size":1543,"lines":1,"children":"","log":[]},{"name":"format-wiki.pl","size":31554,"lines":1,"children":"","log":[]},{"name":"syntax-wiki.pl","size":7313,"lines":1,"children":"","log":[]},{"name":"unformat.pl","size":3855,"lines":1,"children":"","log":[]},{"name":"wikiconnect.py","size":4544,"lines":1,"children":"","log":[]}],"log":[]}

src/data/opentaba-server-pi.json

@@ -1 +1 @@
-{"name":"opentaba-server","size":494902,"lines":121,"children":[{"name":"","size":0,"lines":0,"children":"no children","log":""},{"name":"","size":0,"lines":0,"children":"no children","log":""},{"name":".travis.yml","size":426,"lines":1,"children":"","log":[]},{"name":"CONTRIBUTING.md","size":412,"lines":3,"children":"","log":[]},{"name":"LICENSE","size":1596,"lines":1,"children":"","log":[]},{"name":"Procfile","size":51,"lines":1,"children":"","log":[]},{"name":"Tests","size":458015,"lines":92,"children":[{"name":"functional_tests","size":4424,"lines":2,"children":[{"name":"test_feed.py","size":1868,"lines":1,"children":"","log":[]},{"name":"test_return_json.py","size":2556,"lines":1,"children":"","log":[]}],"log":[]},{"name":"unit_test","size":453591,"lines":90,"children":[{"name":"filecache","size":450987,"lines":88,"children":[{"name":"30649.json","size":52867,"lines":9,"children":"","log":[]},{"name":"bad.quote.mark.html","size":66354,"lines":13,"children":"","log":[]},{"name":"current.fixed.html","size":66353,"lines":13,"children":"","log":[]},{"name":"empty.json","size":3,"lines":1,"children":"","log":[]},{"name":"non.standard.attribute.html","size":66359,"lines":13,"children":"","log":[]},{"name":"non.standard.tag.html","size":66354,"lines":13,"children":"","log":[]},{"name":"unclosed.tr.html","size":66348,"lines":13,"children":"","log":[]},{"name":"unopened.tr.html","size":66349,"lines":13,"children":"","log":[]}],"log":[]},{"name":"test_createdb.py","size":45,"lines":1,"children":"","log":[]},{"name":"test_scrape.py","size":2559,"lines":1,"children":"","log":[]}],"log":[]}],"log":[]},{"name":"app.py","size":6189,"lines":2,"children":"","log":[]},{"name":"clean_db.py","size":1635,"lines":1,"children":"","log":[]},{"name":"create_db.py","size":1172,"lines":1,"children":"","log":[]},{"name":"readme.md","size":6794,"lines":11,"children":"","log":[]},{"name":"requirements.txt","size":447,"lines":1,"children":"","log":[]},{"name":"scrape.py","size":1788,"lines":1,"children":"","log":[]},{"name":"test_requirements.txt","size":348,"lines":1,"children":"","log":[]},{"name":"tools","size":15600,"lines":3,"children":[{"name":"__init__.py","size":0,"lines":1,"children":"","log":[]},{"name":"gushim.py","size":6336,"lines":1,"children":"","log":[]},{"name":"scrapelib.py","size":9264,"lines":1,"children":"","log":[]}],"log":[]},{"name":"worker.py","size":356,"lines":1,"children":"","log":[]}],"log":[]}
+{"name":"opentaba-server","size":513866,"lines":125,"children":[{"name":"","size":0,"lines":0,"children":"no children","log":""},{"name":"","size":0,"lines":0,"children":"no children","log":""},{"name":".travis.yml","size":426,"lines":1,"children":"","log":[]},{"name":"CONTRIBUTING.md","size":412,"lines":3,"children":"","log":[]},{"name":"LICENSE","size":1596,"lines":1,"children":"","log":[]},{"name":"Procfile","size":51,"lines":1,"children":"","log":[]},{"name":"Tests","size":474936,"lines":96,"children":[{"name":"functional_tests","size":4671,"lines":2,"children":[{"name":"test_feed.py","size":1868,"lines":1,"children":"","log":[]},{"name":"test_return_json.py","size":2803,"lines":1,"children":"","log":[]}],"log":[]},{"name":"unit_test","size":470265,"lines":94,"children":[{"name":"filecache","size":467661,"lines":92,"children":[{"name":"30649.json","size":69541,"lines":13,"children":"","log":[]},{"name":"bad.quote.mark.html","size":66354,"lines":13,"children":"","log":[]},{"name":"current.fixed.html","size":66353,"lines":13,"children":"","log":[]},{"name":"empty.json","size":3,"lines":1,"children":"","log":[]},{"name":"non.standard.attribute.html","size":66359,"lines":13,"children":"","log":[]},{"name":"non.standard.tag.html","size":66354,"lines":13,"children":"","log":[]},{"name":"unclosed.tr.html","size":66348,"lines":13,"children":"","log":[]},{"name":"unopened.tr.html","size":66349,"lines":13,"children":"","log":[]}],"log":[]},{"name":"test_createdb.py","size":45,"lines":1,"children":"","log":[]},{"name":"test_scrape.py","size":2559,"lines":1,"children":"","log":[]}],"log":[]}],"log":[]},{"name":"app.py","size":6189,"lines":2,"children":"","log":[]},{"name":"clean_db.py","size":1635,"lines":1,"children":"","log":[]},{"name":"create_db.py","size":1172,"lines":1,"children":"","log":[]},{"name":"readme.md","size":6794,"lines":11,"children":"","log":[]},{"name":"requirements.txt","size":447,"lines":1,"children":"","log":[]},{"name":"scrape.py","size":1788,"lines":1,"children":"","log":[]},{"name":"test_requirements.txt","size":348,"lines":1,"children":"","log":[]},{"name":"tools","size":17643,"lines":3,"children":[{"name":"__init__.py","size":0,"lines":1,"children":"","log":[]},{"name":"gushim.py","size":6336,"lines":1,"children":"","log":[]},{"name":"scrapelib.py","size":11307,"lines":1,"children":"","log":[]}],"log":[]},{"name":"worker.py","size":356,"lines":1,"children":"","log":[]}],"log":[]}

update/getPiData.js

@@ -1,12 +1,12 @@
-var fs = require('fs');
-var path = require('path');
-var posix = require('posix');
-//var colors = require('colors');
+var fs  = require('fs');
+var path  = require('path');
+var posix  = require('posix');
+var colors  = require('colors');
 var rootChecker = require('./rootChecker');
-var lnCounter = require('./lineCounter').createLineCounter();
-var github = require('./github');
-var eKnights = require('./repositories').repositories;
-var index = eKnights.length - 1;
+var lnCounter  = require('./lineCounter').createLineCounter();
+var github  = require('./github');
+var eKnights  = require('./repositories').repositories;
+var index  = eKnights.length - 1;
 
 rootChecker.isRootUser(function() {
  /**
@@ -77,10 +77,10 @@ var mapData = function(folderToMap, cb) {
 
  var obj = dirTree(__dirname + "/repositories/" + folderToMap);
  var fileWriter = function() {
- fs.writeFile("../data/" + eKnights[index].slug + "-pi.json", JSON.stringify(obj), function(err) {
+ fs.writeFile("../src/data/" + eKnights[index].slug + "-pi.json", JSON.stringify(obj), function(err) {
  cb(--index);
  if (err)
- console.log(err);
+ throw new Error(err);
  else
  console.log("../data/" + folderToMap + "-pi.json was saved!");
  });
@@ -96,11 +96,12 @@ var repositoriesPath = __dirname + "/repositories/";
 
 function main() {
  if (index !== -1) {
- var folderName = eKnights[index].getMainRepository().getFolderName();
- var dotGitFolder = eKnights[index].getMainRepository().getDotGitFolder();
+ var mr = eKnights[index].getMainRepository();
+ var folderName = mr.getFolderName();
+ var dotGitFolder = mr.getDotGitFolder();
  github.setDotGitPath(dotGitFolder);
 
- github.cloneOrPull(repositoriesPath + folderName, function() {
+ github.cloneOrPull(mr.url, function() {
  mapData(folderName, main);
  }, true);
  }

update/github.js

@@ -74,7 +74,7 @@ parentPath = findParentPath(parentPath);
  * @param {string} str Repository URL.
  * @returns {string} Folder name
  */
-module.exports.getFolderName = function(str) {
+var getFolderName = function(str) {
  var lastSlash = str.lastIndexOf("/");
  var end = str.length;
  if (str.indexOf('.git') === -1)
@@ -101,14 +101,15 @@ module.exports.cloneOrPull = function(repoURL, callBack, verbose) {
  else {
  var process = spawn('git', ['clone', repoURL]);
  clone = true;
+ console.log('repoURL: ' + repoURL);
  }
 
  process.on('exit', function(code) { // When exit run the callback.
  if (clone === true) {
  var src = parentPath + '/' + containingFolder;
  var des = parentPath + '/repositories/' + containingFolder;
  fs.renameSync(src, des);
- console.log(containingFolder + ' moved from: \n' + parentPath + '\nTo:\n' + parentPath + '/repositories/');
+ console.log(containingFolder + " moved from: \n\t" + parentPath + "\nTo:\n\t" + parentPath + '/repositories/');
  }
  callBack(code, containingFolder);
  });
@@ -229,3 +230,6 @@ module.exports.getLogForFile = function(fullPath, callBack) {
  console.log(buff.toString('utf8').red);
  });
 };
+
+
+module.exports.getFolderName = getFolderName;