Merge pull request #39 from juanjux/feature/remove_msgpack

Remove msgpack dependency

Author: juanjux

native/python_package/python_driver/cli.py

@@ -31,7 +31,7 @@ def get_processor_instance(format_: str, custom_inbuffer: InBuffer=None,
     """
     conf = ProcessorConfigs.get(format_)
     if not conf:
-        raise RequestInstantiationException('No RequestProcessor found for format %s' % format)
+        raise RequestInstantiationException('No RequestProcessor found for format %s' % format_)
 
     inbuffer  = custom_inbuffer if custom_inbuffer else conf['inbuffer']
     outbuffer = custom_outbuffer if custom_outbuffer else conf['outbuffer']
@@ -41,24 +41,15 @@ def get_processor_instance(format_: str, custom_inbuffer: InBuffer=None,
 
 
 def main() -> None:
-    """
-    If you pass the --json command line parameter the replies will be
-    printed using pprint without wrapping them in the msgpack format which
-    can be handy when debugging.
-    """
-
-    if len(sys.argv) > 1 and sys.argv[1] == '--msgpack':
-        format_ = 'msgpack'
-    else:
-        format_ = 'json'
+    format_ = 'json'
 
     processor, inbuffer = get_processor_instance(format_)
     try:
         processor.process_requests(inbuffer)
     except UnicodeDecodeError:
         print_exc()
         print('Error while trying to decode the message, are you sure you are not '
-              'using a different input format that the currently configured (%s)?' % format)
+              'using a different input format that the currently configured (%s)?' % format_)
 
 
 if __name__ == '__main__':

native/python_package/python_driver/processor_configs.py

@@ -1,16 +1,10 @@
 import sys
-from python_driver.requestprocessor import RequestProcessorJSON, RequestProcessorMSGPack
+from python_driver.requestprocessor import RequestProcessorJSON
 
 ProcessorConfigs = {
         'json': {
             'class': RequestProcessorJSON,
             'inbuffer': sys.stdin,
             'outbuffer': sys.stdout
         },
-
-        'msgpack': {
-            'class': RequestProcessorMSGPack,
-            'inbuffer': sys.stdin.buffer,
-            'outbuffer': sys.stdout.buffer
-        }
 }

native/python_package/python_driver/requestprocessor.py

@@ -251,54 +251,3 @@ def process_requests(self, inbuffer: InStrBuffer) -> None:
         """
         for doc in self._extract_docs(inbuffer):
             self.process_request(doc)
-
-
-class RequestProcessorMSGPack(RequestProcessor):
-    """
-    RequestProcessor subclass that operates deserializing and serializing responses
-    using the MSGPACK format. Input and output packages
-    """
-    def __init__(self, outbuffer: OutBytesBuffer) -> None:
-        """
-        :param outbuffer: the output buffer. This must be a bytes-based file like object
-        supporting the write(bytes) and flush() methods.
-        """
-        super().__init__(outbuffer)
-
-    def _send_response(self, response: Response) -> None:
-        from msgpack import dumps
-        self.outbuffer.write(dumps(response))
-        self.outbuffer.flush()
-
-    def _tostr_request(self, request: RawRequest) -> Request:
-        """
-        Convert all byte-string keys and values to normal strings (non recursively since
-        we only have one level)
-
-        :param request: dictionary that potentially can contain bytestring keys and values.
-        :return: the converted dictionary
-        """
-        try:
-            newrequest = Request({})
-            for key, value in request.items():
-                if isinstance(value, bytes):
-                    value = value.decode()
-
-                if isinstance(key, bytes):
-                    key = key.decode()
-
-                newrequest[key] = value
-        except AttributeError as exc:
-            self.errors.append('Error trying to decode message, are you sure that the input ' +
-                               'format is msgpack?')
-            raise AttributeError from exc
-        return newrequest
-
-    def process_requests(self, inbuffer: InStrBuffer) -> None:
-        """
-        :param inbuffer: file-like object based on bytes supporting the read() and
-        iteration by lines
-        """
-        from msgpack import Unpacker
-        for request in Unpacker(inbuffer):
-            self.process_request(request)

native/python_package/requirements.txt

@@ -1,4 +1,3 @@
-msgpack-python==0.4.8
 pydetector==0.8.1
 -e git+git://github.com/python/mypy.git@0bb2d1680e8b9522108b38d203cb73021a617e64#egg=mypy-lang
 typed-ast==1.0.1

native/python_package/setup.py

@@ -31,7 +31,6 @@
         ]
     },
     install_requires=[
-        "msgpack-python==0.4.8",
         "pydetector==0.8.1"
     ],
     classifiers=[

native/python_package/test/test_python_driver.py

@@ -1,56 +1,20 @@
 import io
+import json
 import os
+import subprocess
 import sys
-import json
 import unittest
-import subprocess
 from os.path import join, abspath, dirname
 
-# msgpack is an optional dependency
-try:
-    import msgpack
-except ImportError:
-    TEST_MSGPACK = False
-else:
-    TEST_MSGPACK = True
-
 sys.path.append('..')
 from python_driver import __version__, get_processor_instance
 from python_driver.requestprocessor import (
     Request, Response, RequestProcessorJSON, InBuffer, EmptyCodeException)
 
-if TEST_MSGPACK:
-    from python_driver.requestprocessor import RequestProcessorMSGPack
 from typing import Dict, Any, List, AnyStr, Optional, Iterator, cast
 
 CURDIR = abspath(dirname(__file__))
 
-
-def convert_bytes(data: Any, to_bytes: bool=False) -> Any:
-    """
-    Both in normal operation and with this tests data comes trough a bytestream so this is needed to
-    recursively convert the msgpack incoming data (the pprint data is converted just decoding and using
-    literal_eval)
-    """
-    if type(data) in (list, tuple):
-        newlist: List[Any] = []
-        for item in data:
-            newlist.append(convert_bytes(item, to_bytes))
-        return newlist
-    elif isinstance(data, dict):
-        newdict: Dict[str, Any] = {}
-        for key, value in data.items():
-            newvalue = convert_bytes(value, to_bytes)
-            newkey = convert_bytes(key, to_bytes)
-            newdict[newkey] = newvalue
-        return newdict
-    elif isinstance(data, bytes) and not to_bytes:
-        return data.decode()
-    elif isinstance(data, str) and to_bytes:
-        return data.encode()
-    return data
-
-
 class TestTypeCheck(unittest.TestCase):
     def test_10_check(self) -> None:
         prevdir = os.getcwd()
@@ -64,7 +28,7 @@ def test_10_check(self) -> None:
 
 class TestPythonDriverBase(unittest.TestCase):
     def _restart_data(self, format_: str='json') -> None:
-        assert format_ in ('json', 'msgpack')
+        assert format_ == 'json'
 
         with open(join(CURDIR, 'data', 'helloworld.py')) as f:
             testcode = f.read()
@@ -94,27 +58,21 @@ def _extract_docs(inbuffer: InBuffer) -> Iterator[Response]:
             yield json.loads(line)
 
     def _loadResults(self, format_: str) -> List[Response]:
-        """Read all msgpacks from the recvbuffer"""
+        """Read all msgs from the recvbuffer"""
         self.recvbuffer.seek(0)
 
         res: List[Response] = []
-        if format_ == 'json':
-            res = [doc for doc in self._extract_docs(self.recvbuffer)]
-        elif TEST_MSGPACK and format_ == 'msgpack':
-            res = [convert_bytes(msg) for msg in msgpack.Unpacker(self.recvbuffer)]
+        res = [doc for doc in self._extract_docs(self.recvbuffer)]
         return res
 
 
 class Test10ProcessRequestFunc(TestPythonDriverBase):
 
     def _add_to_buffer(self, count: int, format_: str) -> None:
-        """Add count test msgpacks to the sendbuffer"""
+        """Add count test msgs to the sendbuffer"""
         for i in range(count):
             msg = ''
-            if TEST_MSGPACK and format_ == 'msgpack':
-                msg = msgpack.dumps(self.data)
-            elif format_ == 'json':
-                msg = json.dumps(self.data, ensure_ascii=False) + '\n'
+            msg = json.dumps(self.data, ensure_ascii=False) + '\n'
             self.sendbuffer.write(msg)
 
         self.sendbuffer.flush()
@@ -176,26 +134,13 @@ def test_010_normal_json(self) -> None:
         self.assertEqual(len(replies), 1)
         self._check_reply_dict(replies[0])
 
-    if TEST_MSGPACK:
-        def test_020_normal_msgpack(self) -> None:
-            replies = self._send_receive(1, 'msgpack')
-            self.assertEqual(len(replies), 1)
-            self._check_reply_dict(replies[0])
-
-    def test_030_normal_json_many(self) -> None:
+    def test_020_normal_json_many(self) -> None:
         replies = self._send_receive(100, 'json')
         self.assertEqual(len(replies), 100)
         for reply in replies:
             self._check_reply_dict(reply)
 
-    if TEST_MSGPACK:
-        def test_040_normal_msgpack_many(self) -> None:
-            replies = self._send_receive(100, 'msgpack')
-            self.assertEqual(len(replies), 100)
-            for reply in replies:
-                self._check_reply_dict(reply)
-
-    def test_050_error_print(self) -> None:
+    def test_030_error_print(self) -> None:
         wrongcode = 'wtf lol'
 
         replies = self._send_receive(1, 'json', {'content': wrongcode})
@@ -209,31 +154,7 @@ def test_050_error_print(self) -> None:
         replies = self._send_receive(1, 'json')
         self.assertEqual(len(replies), 1)
 
-    if TEST_MSGPACK:
-        def test_060_error_msgpack(self) -> None:
-            wrongcode = 'wtf lol'
-
-            replies = self._send_receive(1, 'msgpack', {'content': wrongcode})
-            self.assertEqual(len(replies), 1)
-            ast = replies[0].get('ast')
-            self.assertIsNone(ast)
-            self._check_reply_dict(replies[0], has_errors=True)
-
-            # Check that it still alive
-            self._restart_data()
-            replies = self._send_receive(1, 'json')
-            self.assertEqual(len(replies), 1)
-
-        def test_070_broken_msgpack(self) -> None:
-            self._restart_data('msgpack')
-            brokendata = msgpack.dumps(self.data)[:-30]
-            self.sendbuffer.write(brokendata)
-            self.sendbuffer.flush()
-            reply = self._send_receive(1, 'msgpack', restart_data=False)[0]
-            self.assertEqual(reply['status'], 'fatal')
-            self.assertEqual(len(reply['errors']), 1)
-
-    def test_080_broken_json(self) -> None:
+    def test_040_broken_json(self) -> None:
         self._restart_data('json')
         brokendata = json.dumps(self.data, ensure_ascii=False)[:-30]
         self.sendbuffer.write(brokendata)
@@ -244,31 +165,8 @@ def test_080_broken_json(self) -> None:
 
 
 class Test20ReqProcMethods(TestPythonDriverBase):
-    if TEST_MSGPACK:
-        def test_10_check_input(self) -> None:
-            self._restart_data('json')
-            brequest = convert_bytes(self.data, to_bytes=True)
-            processor = RequestProcessorMSGPack(self.recvbuffer)
-            res = processor._parse_input_request(brequest)
-            self.assertEqual(res[1], 'test.py')
-
-        def test_20_check_input_bad(self) -> None:
-            self._restart_data('msgpack')
-            del self.data['content']
-            brequest = convert_bytes(self.data, to_bytes=True)
-            processor = RequestProcessorMSGPack(self.recvbuffer)
-            with self.assertRaises(EmptyCodeException) as _:  # noqa: F841
-                processor._parse_input_request(brequest)
-
-        def test_30_send_response_msgpack(self) -> None:
-            self._restart_data('msgpack')
-            processor = RequestProcessorMSGPack(self.recvbuffer)
-            processor._send_response(cast(Response, self.data))
-            res = self._loadResults('msgpack')
-            self.assertEqual(len(res), 1)
-            self.assertDictEqual(self.data, res[0])
-
-    def test_40_send_response_json(self) -> None:
+
+    def test_10_send_response_json(self) -> None:
         self._restart_data('json')
         processor = RequestProcessorJSON(self.recvbuffer)
         processor._send_response(cast(Response, self.data))
@@ -278,7 +176,7 @@ def test_40_send_response_json(self) -> None:
 
     # process request already tested with TestPythonDriverBase
 
-    def test_50_return_error(self) -> None:
+    def test_20_return_error(self) -> None:
         self._restart_data('json')
         processor = RequestProcessorJSON(self.recvbuffer)
         processor.errors = ['test error']