Add @lexer directive

Author: tjvr

lib/generate.js

@@ -112,6 +112,7 @@ generate.js = generate._default = generate.javascript = function (parser, export
     output += "function id(x) {return x[0]; }\n";
     output += parser.body.join('\n');
     output += "var grammar = {\n";
+    output += "    Lexer: " + parser.config.lexer + ",\n";
     output += "    ParserRules: " +
         serializeRules(parser.rules, generate.javascript.builtinPostprocessors)
         + "\n";
@@ -141,6 +142,7 @@ generate.cs = generate.coffee = generate.coffeescript = function (parser, export
     output += "  id = (d)->d[0]\n";
     output += tabulateString(dedentFunc(parser.body.join('\n')), '  ') + '\n';
     output += "  grammar = {\n";
+    output += "    Lexer: " + parser.config.lexer + ",\n";
     output += "    ParserRules: " +
         tabulateString(
                 serializeRules(parser.rules, generate.coffeescript.builtinPostprocessors),
@@ -173,6 +175,7 @@ generate.ts = generate.typescript = function (parser, exportName) {
     output += "interface NearleyRule {name:string; symbols:NearleySymbol[]; postprocess?:(d:any[],loc?:number,reject?:{})=>any};\n";
     output += "type NearleySymbol = string | {literal:any} | {test:(token:any) => boolean};\n";
     output += "export var grammar : NearleyGrammar = {\n";
+    output += "    Lexer: " + parser.config.lexer + ",\n";
     output += "    ParserRules: " + serializeRules(parser.rules, generate.typescript.builtinPostprocessors) + "\n";
     output += "  , ParserStart: " + JSON.stringify(parser.start) + "\n";
     output += "}\n";

lib/nearley-language-bootstrapped.js

@@ -20,6 +20,7 @@ function $(o) {
     };
 }
 var grammar = {
+    Lexer: undefined,
     ParserRules: [
     {"name": "dqstring$ebnf$1", "symbols": []},
     {"name": "dqstring$ebnf$1", "symbols": ["dqstring$ebnf$1", "dstrchar"], "postprocess": function arrpush(d) {return d[0].concat([d[1]]);}},

lib/nearley.js

@@ -168,12 +168,15 @@ function Grammar(rules, start) {
 
 // So we can allow passing (rules, start) directly to Parser for backwards compatibility
 Grammar.fromCompiled = function(rules, start) {
+    var lexer = rules.Lexer;
     if (rules.ParserStart) {
       start = rules.ParserStart;
       rules = rules.ParserRules;
     }
     var rules = rules.map(function (r) { return (new Rule(r.name, r.symbols, r.postprocess)); });
-    return new Grammar(rules, start);
+    var g = new Grammar(rules, start);
+    g.lexer = lexer; // TODO storing lexer onto Grammar seems iffy!
+    return g;
 }
 
 
@@ -218,7 +221,7 @@ function Parser(rules, start, options) {
     // Read options
     this.options = {
         keepHistory: false,
-        lexer: ChunkLexer,
+        lexer: grammar.lexer || ChunkLexer,
     };
     for (var key in (options || {})) {
         this.options[key] = options[key];
@@ -261,14 +264,17 @@ Parser.prototype.feed = function(chunk) {
         // Advance all tokens that expect the symbol
         // So for each state in the previous row,
 
-        var value = token.value;
+        var literal = token.value;
+        var value = lexer.constructor === ChunkLexer ? token.value : token;
         var scannable = column.scannable;
         for (var w = scannable.length; w--; ) {
             var state = scannable[w];
             var expect = state.rule.symbols[state.dot];
             // Try to consume the token
             // either regex or literal
-            if (expect.test ? expect.test(value) : expect.literal === value) {
+            if (expect.test ? expect.test(value) :
+                expect.type ? expect.type === token.type
+                            : expect.literal === literal) {
                 // Add it
                 var next = state.nextState(value);
                 nextColumn.states.push(next);
@@ -290,7 +296,7 @@ Parser.prototype.feed = function(chunk) {
             // No states at all! This is not good.
             var err = new Error(
                 "nearley: No possible parsings (@" + (this.current)
-                    + ": '" + token + "')."
+                    + ": '" + JSON.stringify(token) + "')."
             );
             err.offset = this.current;
             throw err;