Isolate tokenizer, create head module

Remove the tokenizer's dependency on the default grammar.
Set up entry module and export primary classes / functions.

.travis.yml

@@ -4,7 +4,11 @@ node_js:
  - "5.1"
  - "6"
  - "stable"
+
+script:
+ - npm run lint
 after_script:
  - npm run coveralls
+
 notifications:
  email: false

lib/parser.js

@@ -6,7 +6,7 @@ const tokenize = require('./tokenizer');
 
 
 function parse(sent, grammar, tokenizer) {
- let tokens = (tokenizer || tokenize)(sent);
+ let tokens = (tokenizer || tokenize)(sent, grammar || rules.rules);
  let states = earley(tokens, grammar || rules.rules);
  return dfs(states, tokens);
 }
@@ -220,4 +220,6 @@ let tree = parse(sentence, rules.rules);
 
 //console.log(JSON.stringify(tree, null, 2));
 
-module.exports.parse = parse;
+module.exports.parse = parse;
+module.exports.earley = earley;
+module.exports.dfs = dfs;

lib/rules.js

@@ -82,18 +82,9 @@ let rules = [
  new Rule(factor , [/\d+/] , (n) => parseFloat(n))
 ];
 
-function getTokens() {
- return rules.reduce(
- (tokens, rule) => tokens.concat(
- rule.filter((sym) => typeof sym === 'string' || sym instanceof RegExp)
- )
- , []);
-}
-
 module.exports = {
- rules : rules,
- Rule : Rule,
- Sym : Sym,
- getTokens : getTokens
+ rules : rules,
+ Rule : Rule,
+ Sym : Sym
 };
 

lib/tokenizer.js

@@ -1,17 +1,16 @@
 
-let terminals = require('./rules').getTokens();
-
-
-module.exports = (sent, terms) => {
- terms = terms || terminals;
-
- let tokens = terms
- .map((token) => {
- if (typeof token === 'string') {
- return token.replace(/[\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|]/g, '\\$&');
- }
- return token.source;
- })
+module.exports = (sent, grammar) => {
+ let terms = grammar.reduce(
+ (tokens, rule) => tokens.concat(
+ rule.filter((sym) => typeof sym === 'string' || sym instanceof RegExp)
+ ), [])
+ , tokens = terms
+ .map((token) => {
+ if (typeof token === 'string') {
+ return token.replace(/[\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|]/g, '\\$&');
+ }
+ return token.source;
+ })
  , delims = RegExp('(' + tokens.join('|') + ')');
 
  return sent

package.json

@@ -2,7 +2,7 @@
  "name": "parsey",
  "version": "0.0.0",
  "description": "Parser for context-free grammars",
- "main": "parser.js",
+ "main": "parsey.js",
  "scripts": {
  "lint": "eslint . --ignore-pattern coverage",
  "test": "jasmine",

parsey.js

@@ -0,0 +1,8 @@
+
+module.exports = {
+ parse : require('./lib/parser').parse,
+ tokenize : require('./lib/tokenizer'),
+ Rule : require('./lib/rules').Rule,
+ Sym : require('./lib/rules').Sym
+};
+