#18 Add relevant tests from Kolasu, lint

Author: alessiostalla

.github/workflows/pythonapp.yml

@@ -23,7 +23,7 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install -r Requirements.txt
-          curl -O https://www.antlr.org/download/antlr-4.11.1-complete.jar
+          pip install antlr4-cli
       - name: Lint with flake8
         run: |
           pip install flake8

pylasu/parsing/antlr.py

@@ -1,6 +1,6 @@
 import time
 from abc import abstractmethod
-from typing import Optional, List
+from typing import Optional, List, Union
 
 from antlr4 import CommonTokenStream, InputStream, Lexer, Parser, ParserATNSimulator, ParserRuleContext, \
     PredictionContextCache, Recognizer, Token, TokenStream
@@ -22,8 +22,8 @@ class PylasuANTLRParser:
     def __init__(self):
         self.prediction_context_cache = PredictionContextCache()
 
-    def parse(self, input_stream: InputStream, consider_range: bool = True, measure_lexing_time: bool = False,
-              source: Optional[Source] = None):
+    def parse(self, input_stream: Union[InputStream, str], consider_range: bool = True,
+              measure_lexing_time: bool = False, source: Optional[Source] = None):
         """Parses source code, returning a result that includes an AST and a collection of parse issues
         (errors, warnings).
         The parsing is done in accordance to the StarLasu methodology i.e. a first-stage parser builds a parse tree
@@ -34,6 +34,8 @@ def parse(self, input_stream: InputStream, consider_range: bool = True, measure_
         @param measureLexingTime if true, the result will include a measurement of the time spent in lexing i.e.
         breaking the input stream into tokens."""
         start = time.time_ns()
+        if type(input_stream) is str:
+            input_stream = InputStream(input_stream)
         first_stage = self.parse_first_stage(input_stream, measure_lexing_time)
         issues = first_stage.issues
         ast = self.parse_tree_to_ast(first_stage.root, consider_range, issues, source)
@@ -120,12 +122,12 @@ def create_token_stream(self, lexer: Lexer) -> TokenStream:
         return CommonTokenStream(lexer)
 
     @abstractmethod
-    def create_antlr_lexer(self, input_stream: InputStream):
+    def create_antlr_lexer(self, input_stream: InputStream) -> Lexer:
         """Creates the lexer."""
         pass
 
     @abstractmethod
-    def create_antlr_parser(self, token_stream: TokenStream):
+    def create_antlr_parser(self, token_stream: TokenStream) -> Parser:
         """Creates the first-stage parser."""
         pass
 
@@ -147,4 +149,5 @@ def syntaxError(self, recognizer, offending_symbol, line, column, msg, e):
         end_point = start_point
         if isinstance(offending_symbol, Token):
             end_point = token_end_point(offending_symbol)
-        self.issues.append(Issue(IssueType.SYNTACTIC, msg or "unspecified", position=Position(start_point, end_point)))
+        msg = (msg or "unspecified").capitalize()
+        self.issues.append(Issue(IssueType.SYNTACTIC, msg, position=Position(start_point, end_point)))

tests/SimpleLangParser.g4

@@ -3,7 +3,7 @@ parser grammar SimpleLangParser;
 options { tokenVocab = SimpleLangLexer; }
 
 compilationUnit:
-    statement+;
+    statement+ EOF;
 
 statement:
       DISPLAY expression #displayStmt

tests/generate-test-parsers.sh

@@ -1,3 +1,3 @@
-java -cp ../antlr-4.11.1-complete.jar org.antlr.v4.Tool -Dlanguage=Python3 -visitor -o simple_lang SimpleLangLexer.g4 SimpleLangParser.g4
-java -cp ../antlr-4.11.1-complete.jar org.antlr.v4.Tool -Dlanguage=Python3 -visitor -o antlr_entity AntlrEntityLexer.g4 AntlrEntityParser.g4
-java -cp ../antlr-4.11.1-complete.jar org.antlr.v4.Tool -Dlanguage=Python3 -visitor -o antlr_script AntlrScriptLexer.g4 AntlrScriptParser.g4
+antlr4 -Dlanguage=Python3 -visitor -o simple_lang SimpleLangLexer.g4 SimpleLangParser.g4
+antlr4 -Dlanguage=Python3 -visitor -o antlr_entity AntlrEntityLexer.g4 AntlrEntityParser.g4
+antlr4 -Dlanguage=Python3 -visitor -o antlr_script AntlrScriptLexer.g4 AntlrScriptParser.g4

tests/parsing/__init__.py

@@ -0,0 +1,53 @@
+import unittest
+from typing import List, Optional
+
+from antlr4 import TokenStream, InputStream
+
+from pylasu.model import Source, Node, Position, Point
+from pylasu.parsing.antlr import PylasuANTLRParser
+from pylasu.validation import Issue
+from tests.simple_lang.SimpleLangLexer import SimpleLangLexer
+from tests.simple_lang.SimpleLangParser import SimpleLangParser
+
+
+class SimpleLangPylasuParser(PylasuANTLRParser):
+
+    def create_antlr_lexer(self, input_stream: InputStream):
+        return SimpleLangLexer(input_stream)
+
+    def create_antlr_parser(self, token_stream: TokenStream):
+        return SimpleLangParser(token_stream)
+
+    def parse_tree_to_ast(self, root, consider_range: bool, issues: List[Issue], source: Source) -> Optional[Node]:
+        return None
+
+
+class KolasuParserTest(unittest.TestCase):
+    def test_lexing(self):
+        parser = SimpleLangPylasuParser()
+        result = parser.parse("""set a = 10
+set b = ""
+display c
+""")
+        self.assertIsNotNone(result)
+        # TODO we don't have Pylasu Tokens yet
+
+    def test_issues_are_capitalized(self):
+        parser = SimpleLangPylasuParser()
+        result = parser.parse("""set set a = 10
+display c
+""")
+        self.assertTrue(result.issues)
+        self.assertTrue([i for i in result.issues if i.message.startswith("Extraneous input 'set'")])
+        self.assertTrue([i for i in result.issues if i.message.startswith("Mismatched input 'c'")])
+
+    def test_issues_have_not_flat_position(self):
+        parser = SimpleLangPylasuParser()
+        result = parser.parse("""set set a = 10
+display c
+""")
+        self.assertTrue(result.issues)
+        extraneous_input = [i for i in result.issues if i.message.startswith("Extraneous input 'set'")][0]
+        self.assertEqual(Position(Point(1, 4), Point(1, 7)), extraneous_input.position)
+        mismatched_input = [i for i in result.issues if i.message.startswith("Mismatched input 'c'")][0]
+        self.assertEqual(Position(Point(2, 8), Point(2, 9)), mismatched_input.position)