Merge pull request #42 from lucacome/fix-quotes-take-2

Treat quotes inside a token like any other char

Author: aluttik

crossplane/lexer.py

@@ -28,7 +28,6 @@ def _lex_file_object(file_obj):
     token = ''  # the token buffer
     token_line = 0  # the line the token starts on
     next_token_is_directive = True
-    quote_inside_token = False
 
     it = itertools.chain.from_iterable(file_obj)
     it = _iterescape(it)  # treat escaped characters differently
@@ -73,21 +72,17 @@ def _lex_file_object(file_obj):
 
         # if a quote is found, add the whole string to the token buffer
         if char in ('"', "'"):
+            # if a quote is inside a token, treat it like any other char
             if token:
                 token += char
-                quote_inside_token = True
+                continue
 
             quote = char
             char, line = next(it)
             while char != quote:
                 token += quote if char == '\\' + quote else char
                 char, line = next(it)
 
-            if quote_inside_token:
-                token += quote
-                quote_inside_token = False
-                continue
-
             yield (token, token_line)
             if next_token_is_directive and token in EXTERNAL_LEXERS:
                 for custom_lexer_token in EXTERNAL_LEXERS[token](it, token):

tests/configs/quote-behavior/nginx.conf

@@ -1,2 +1,2 @@
 "outer-quote" "left"-quote right-"quote" inner"-"quote;
-"" ""left-empty right-empty"" inner""empty;
+"" ""left-empty right-empty"" inner""empty right-empty-single";

tests/test_lex.py

@@ -75,5 +75,5 @@ def test_quote_behavior():
     tokens = crossplane.lex(config)
     assert list(token for token, line in tokens) == [
         'outer-quote', 'left', '-quote', 'right-"quote"', 'inner"-"quote', ';',
-        '', '', 'left-empty', 'right-empty""', 'inner""empty', ';',
+        '', '', 'left-empty', 'right-empty""', 'inner""empty', 'right-empty-single"', ';',
     ]