parsing a jitlog can abort the parsing and return the half read forest + tests. this is useful when the jitlog has not been generated fully (vm termination)

Author: planrich

jitlog/parser.py

@@ -7,6 +7,9 @@
 JITLOG_MIN_VERSION = 1
 JITLOG_VERSION = 1
 
+class ParseException(Exception):
+    pass
+
 def read_jitlog_data(filename):
     with open(str(filename), 'rb') as fileobj:
         return fileobj.read()
@@ -20,29 +23,49 @@ def parse_jitlog(filename, data=None):
     return f
 
 def _parse_jitlog(fileobj):
+    """ JitLog is parsed. if an exception is raised after
+    the header has been read, it is saved in the field 'exc' on
+    the forest returned.
+
+    This allows to parse an incomplete log file an still display
+    everything that was readable!
+    If a ParseException is directly raised, this is an error that cannot
+    be recovered from!
+    """
     is_jit_log = fileobj.read(1) == const.MARK_JITLOG_HEADER
     version = ord(fileobj.read(1)) | (ord(fileobj.read(1)) << 8)
     is_32bit = ord(fileobj.read(1))
     machine = read_string(fileobj, True)
     forest = TraceForest(version, is_32bit, machine)
-    assert is_jit_log, "Missing header. Data might not be a jitlog!"
-    assert version >= JITLOG_MIN_VERSION, \
-            "Version does not match. Log is version %d%d is not satisfied" % \
-                (version, JITLOG_VERSION)
+    if not is_jit_log:
+        raise ParseException("Missing header. Provided input might not be a jitlog!")
+    if version < JITLOG_MIN_VERSION:
+        raise ParseException("Version %d is not supported" % version)
     while True:
         marker = fileobj.read(1)
         if len(marker) == 0:
             break # end of file!
-        assert forest.is_jitlog_marker(marker), \
-                "marker unkown: 0x%x at pos 0x%x" % (ord(marker), fileobj.tell())
+        if not forest.is_jitlog_marker(marker):
+            msg = "marker unknown: 0x%x at pos 0x%x" % \
+                      (ord(marker), fileobj.tell())
+            forest.exc = ParseException(msg)
+            break
         trace = forest.last_trace
         try:
             read = marks.get_reader(version, marker)
             read(forest, trace, fileobj)
             forest.time_tick()
-        except KeyError:
-            print("failed at", hex(fileobj.tell()), "with marker", marker)
-            raise
+        except KeyError as e:
+            forest.exc = e
+            break
+        except ParseException as e:
+            forest.exc = e
+            break
+        except Exception as e:
+            msg = "failed at %s with marker %s with exc %s" %\
+                    (hex(fileobj.tell()), marker, str(e))
+            forest.exc = ParseException(msg)
+            break
 
     return forest
 

jitlog/test/test_jitlog.py

@@ -1,7 +1,9 @@
 import struct, py, sys
+import pytest
 from vmprof import reader
 from jitlog import constants as const
 from jitlog import marks
+from jitlog.parser import ParseException
 from jitlog.parser import _parse_jitlog
 from jitlog.objects import (FlatOp, TraceForest, Trace,
         MergePoint, PointInTrace, iter_ranges)
@@ -278,3 +280,31 @@ def test_v2_start_trace():
     fw = FileObjWrapper(fobj)
     forest = construct_forest(fw, version=2)
     assert forest.get_trace(0x15).jd_name == 'jd_is_a_hippy'
+
+def test_exception_recover():
+    # weird file provided, fails without returning anything
+    fobj = FileObj([0x0])
+    with pytest.raises(ParseException):
+        _parse_jitlog(fobj)
+
+    # incomplete log, bails and adds exception
+    fobj = FileObj([const.MARK_JITLOG_HEADER,
+                    b'\x01\x00\x00', encode_str('x86_64'),
+                   b'\x00'
+                   ])
+    f = _parse_jitlog(fobj)
+    assert hasattr(f, 'exc')
+    assert "marker unknown" in f.exc.args[0]
+
+    # some valid data, but most of it missing
+    fobj = FileObj([const.MARK_JITLOG_HEADER,
+                    b'\x01\x00\x00', encode_str('x86_64'),
+                    const.MARK_START_TRACE, encode_le_u64(0xffaa), encode_str('loop'), encode_le_u64(0),
+                    const.MARK_TRACE, encode_le_u64(0xffaa),
+                    const.MARK_START_TRACE # uff, trace ends here, data missing
+                   ])
+    f = _parse_jitlog(fobj)
+    assert len(f.traces) == 1
+    assert hasattr(f, 'exc')
+
+

setup.py

@@ -33,7 +33,7 @@
     name='vmprof',
     author='vmprof team',
     author_email='[email protected]',
-    version="0.3.5",
+    version="0.3.6.dev0",
     packages=find_packages(),
     description="Python's vmprof client",
     long_description='See https://vmprof.readthedocs.org/',

vmprof/test/test_reader.py

@@ -24,6 +24,9 @@ def read(self, count):
     def write(self, s):
         self.s += s
 
+    def tell(self):
+        return 0
+
 def test_fileobj():
     f = FileObj()
     f.write(b'foo')