Separate earley from parse() | add valuator to Rule

The earley function takes several parameters and spits out a set of
states (one for each token). This should be its own function since it
must be tested apart from the DFS tree constructor.

The Rule constructor takes an optional third parameter, the valuator,
which is the function used to evaluate the expression given values for
each symbol.

lib/parser.js

@@ -2,15 +2,21 @@
 'use strict';
 
 const rules     = require('./rules');
-const interp    = require('./interp');
 const tokenize  = require('./tokenizer');
 
-function parse(sent, tokenizer) {
-  let tokens  = (tokenizer || tokenize)(sent);
+
+function parse(sent, grammar, tokenizer) {
+  let tokens = (tokenizer || tokenize)(sent);
+  let states = earley(tokens, grammar || rules.rules);
+  return dfs(states, tokens);
+}
+
+
+function earley(tokens, grammar) {
   let states  = Array.apply(null, Array(tokens.length + 1)).map(() => []);
   var i, j;
 
-  let rulePairs = rules.rules.map((rule) => ({
+  let rulePairs = grammar.map((rule) => ({
     name    : rule.lhs.name,
     rule    : rule,
     position: 0,
@@ -21,22 +27,22 @@ function parse(sent, tokenizer) {
 
   for (i = 0; i <= tokens.length; i += 1) {
     for (j = 0; j < states[i].length; j += 1) {
-      predict(tokens, states, i, j);
+      predict(tokens, states, i, j, grammar);
       scan(tokens, states, i, j);
       complete(tokens, states, i, j);
     }
   }
 
-  return dfs(swap(removeUnfinishedItems(states)), tokens);
+  return swap(removeUnfinishedItems(states));
 }
 
 
-function predict(tokens, states, i, j) {
+function predict(tokens, states, i, j, grammar) {
   let curr = states[i][j];
 
   // prediction
   if (curr.rule[curr.position] instanceof rules.Sym) {
-    rules.rules.forEach((rule) => {
+    grammar.forEach((rule) => {
       let stateHasItem = states[i].filter((earleyItem) => {
         return earleyItem.rule === rule &&
                curr.position === earleyItem.position;
@@ -137,7 +143,7 @@ function dfs(states, tokens) {
   }
 
   return {
-    item    : root.name,
+    item    : root.rule,
     children: dfsHelper(states, root, 0, 0, tokens)
   };
 }
@@ -180,7 +186,7 @@ function dfsHelper(states, root, state, depth, tokens) {
 
       if (subMatch) {
         return [{
-          item    : item,
+          item    : item.rule,
           children: dfsHelper(states, item, state, 0, tokens)
         }].concat(subMatch);
       }
@@ -194,52 +200,24 @@ function dfsHelper(states, root, state, depth, tokens) {
     );
 
     if (diffs.length > 0) {
-      console.log('Ambiguity\n' + JSON.stringify(edges, null, 2));
+      //console.log('Ambiguity\n' + JSON.stringify(edges, null, 2));
+      console.log('Ambiguous rules');
     }
   }
 
   return edges[0];
 }
 
 
-function interpret(parseTree) {
-  if (typeof parseTree === 'string' || parseTree == null) {
-    return parseTree;
-  }
-
-  let values = parseTree.children
-    .map((tree) => interpret(tree))
-    .filter((value) => value != null);
-
-  return interp.valueOf(parseTree.item, values);
-}
-
-
-
 //let sentence = '23 + ( 32 * 46 )';
 //let sentence = '( 23 + 32 ) * 46';
 //let sentence  = '23 + 32 * 46';
 //let sentence = '( ( 12 ) )';
 //let sentence = '1 * 2 + 3 * 4 + 5';
 //let sentence = '1 + 2 + 3';
 let sentence = '1^3 + 2 * 3(3)';
+let tree = parse(sentence, rules.rules);
 
-let states = parse(sentence);
-
-console.log('\n\n--FINAL--');
-//printStates(states, sentence);
-console.log('\n');
-console.log('input:', sentence);
-console.log('=================');
-
-
-console.log('\n~~ dfs ~~');
-console.log(JSON.stringify(
-  dfs(states, tokenize(sentence)),
-  null, 2
-));
-
-console.log(interpret(dfs(states, tokenize(sentence))));
+//console.log(JSON.stringify(tree, null, 2));
 
-module.exports.parse        = parse;
-module.exports.interpret    = interpret;
+module.exports.parse = parse;

lib/rules.js

@@ -1,12 +1,18 @@
 
-function Rule(lhs, rhs) {
+function Rule(lhs, rhs, valuator) {
   let arr = [];
 
   if (!rhs || rhs.length === 0) {
     throw new Error('Rule does not produce anything');
   }
   arr.push.apply(arr, rhs);
   arr.lhs = lhs;
+
+  Object.defineProperty(arr, 'lhs', { value: lhs });
+  Object.defineProperty(arr, 'evaluate', {
+    value: (values) => valuator.apply(null, values)
+  });
+
   arr.__proto__ = Rule.prototype;
 
   return arr;

lib/tokenizer.js

@@ -1,6 +1,11 @@
 
-module.exports.tokenize = (sent, terminals) => {
-  let tokens = terminals
+let terminals = require('./rules').getTokens();
+
+
+module.exports = (sent, terms) => {
+  terms = terms || terminals;
+
+  let tokens = terms
     .map((token) => {
       if (typeof token === 'string') {
         return token.replace(/[\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|]/g, '\\$&');

test.js

@@ -0,0 +1,58 @@
+/*eslint no-console:0*/
+
+
+const parser    = require('./lib/parser');
+//const interp    = require('./interp');
+const readline  = require('readline');
+
+const rl        = readline.createInterface(process.stdin, process.stdout);
+
+
+function printTree(tree, level) {
+  if (tree === undefined) { return; }
+  if (typeof tree === 'object') {
+    console.log(`${Array(level).join(' ')}(${tree.item.lhs.name}`);
+    tree.children.forEach(t => printTree(t, level + 2));
+    console.log(`${Array(level).join(' ')})`);
+    return;
+  }
+  return console.log(`${Array(level).join(' ')}${tree}`);
+}
+
+
+function interpret(parseTree) {
+  if (typeof parseTree === 'string' || parseTree == null) {
+    return parseTree;
+  }
+
+  let values = parseTree.children
+    .map((tree) => interpret(tree))
+    .filter((value) => value != null);
+
+  return parseTree.item.evaluate(values);
+}
+
+
+
+rl.setPrompt('expr> ');
+rl.prompt();
+
+rl.on('line', (line) => {
+  if (line.trim() === '') {
+    return rl.prompt();
+  }
+
+  try {
+    let tree = parser.parse(line.trim());
+    //console.log(JSON.stringify(tree, null, 2));
+    printTree(tree, 1);
+    console.log('=>', interpret(tree));
+  } catch (e) {
+    console.error(e.message);
+  }
+  rl.prompt();
+
+}).on('close', () => {
+  console.log();
+  process.exit(0);
+});