added WordLexer support for txt files

Author: ivanharvard

compare50/_data.py

@@ -8,6 +8,8 @@
 import pygments
 import pygments.lexers
 
+from .lexers import WordLexer
+
 
 __all__ = ["Pass", "Comparator", "File", "Submission",
            "Pass", "Span", "Score", "Comparison", "Token"]
@@ -191,14 +193,18 @@ def lexer(self):
 
         # get lexer for this file type
         try:
-            lexer = pygments.lexers.get_lexer_for_filename(self.name.name)
+            if ext == ".txt":
+                lexer = WordLexer()
+            else:
+                lexer = pygments.lexers.get_lexer_for_filename(self.name.name)
+    
             self._lexer_cache[ext] = lexer
             return lexer
         except pygments.util.ClassNotFound:
             try:
                 return pygments.lexers.guess_lexer(self.read())
             except pygments.util.ClassNotFound:
-                return pygments.lexers.special.TextLexer()
+                return WordLexer()
 
     @classmethod
     def get(cls, id):

compare50/lexers.py

@@ -0,0 +1,16 @@
+from pygments.lexer import RegexLexer
+from pygments.token import Text, Name
+
+class WordLexer(RegexLexer):
+    """Custom compare50 lexer that creates a token based on each 'word'."""
+    name = "WordLexer"
+    aliases = ["word"]
+    filenames = ["*.txt"]
+
+    tokens = {
+        "root": [
+            (r"\s+", Text),           # whitespace
+            (r"\w+", Name),           # word (alphanumeric)
+            (r"\W", Text),            # punctuation or other
+        ]
+    }