Add Lexer option

Author: tjvr

lib/nearley.js

@@ -101,7 +101,7 @@ Column.prototype.process = function(nextColumn) {
 
                 // special-case nullables
                 if (state.reference === this.index) {
-                    // make sure future predictors of this rule get completed. 
+                    // make sure future predictors of this rule get completed.
                     var exp = state.rule.name;
                     (this.completed[exp] = this.completed[exp] || []).push(state);
                 }
@@ -177,6 +177,35 @@ Grammar.fromCompiled = function(rules, start) {
 }
 
 
+function ChunkLexer() {
+    this.reset();
+}
+ChunkLexer.clone = function() { return new ChunkLexer(); }
+
+ChunkLexer.prototype.save = function() {
+    return {};
+}
+
+ChunkLexer.prototype.reset = function(data, state) {
+    this.buffer = data || [];
+    this.index = 0;
+}
+
+ChunkLexer.prototype.feed = function(data) {
+    if (this.index < this.buffer.length) {
+        throw new Error("ChunkLexer doesn't support feed() before chunk finished")
+    }
+    this.index = 0;
+    this.buffer = data;
+}
+
+ChunkLexer.prototype.next = function() {
+    if (this.index < this.buffer.length) {
+        return {value: this.buffer[this.index++]}
+    }
+}
+
+
 function Parser(rules, start, options) {
     if (rules instanceof Grammar) {
         var grammar = rules;
@@ -189,12 +218,14 @@ function Parser(rules, start, options) {
     // Read options
     this.options = {
         keepHistory: false,
-        // rewindable: false,
+        lexer: ChunkLexer,
     };
     for (var key in (options || {})) {
         this.options[key] = options[key];
     }
-    // if (this.options.rewindable) { this.options.keepHistory = true; }
+
+    // Setup lexer
+    this.lexer = this.options.lexer.clone();
 
     // Setup a table
     var column = new Column(grammar, 0);
@@ -212,7 +243,9 @@ function Parser(rules, start, options) {
 Parser.fail = {};
 
 Parser.prototype.feed = function(chunk) {
-    for (var chunkPos = 0; chunkPos < chunk.length; chunkPos++) {
+    var lexer = this.lexer;
+    lexer.feed(chunk);
+    while (token = lexer.next()) {
         // We add new states to table[current+1]
         var column = this.table[this.current];
 
@@ -228,16 +261,16 @@ Parser.prototype.feed = function(chunk) {
         // Advance all tokens that expect the symbol
         // So for each state in the previous row,
 
-        var token = chunk[chunkPos];
+        var value = token.value;
         var scannable = column.scannable;
         for (var w = scannable.length; w--; ) {
             var state = scannable[w];
             var expect = state.rule.symbols[state.dot];
             // Try to consume the token
             // either regex or literal
-            if (expect.test ? expect.test(token) : expect.literal === token) {
+            if (expect.test ? expect.test(value) : expect.literal === value) {
                 // Add it
-                var next = state.nextState(token);
+                var next = state.nextState(value);
                 nextColumn.states.push(next);
             }
         }
@@ -289,6 +322,9 @@ Parser.prototype.rewind = function(index) {
     this.table.splice(index + 1);
     this.current = index;
 
+    // Rewind lexer too
+    this.lexer.reset();
+
     // Incrementally keep track of results
     this.results = this.finish();
 };