Make empty input code a non fatal error

- Return an AST for the module for empty files
- Dont lose the AST on errors that produced it anyway
- Updated tests, improved typing

Author: juanjux

native/python_package/python_driver/requestprocessor.py

@@ -4,7 +4,7 @@
 from pydetector import detector
 from traceback import format_exc
 from python_driver.version import __version__
-from typing import (Any, IO, NewType, Tuple, cast, List, Iterator, Dict)
+from typing import (Any, IO, NewType, Tuple, cast, List, Iterator, Dict, Optional)
 
 # typing.AnyStr is bugged on this version of MyPy, so...:
 AnyStr = Any
@@ -30,6 +30,14 @@ class RequestCheckException(Exception):
     """
     pass
 
+class EmptyCodeException(Exception):
+    """
+    Exception produced when the input code is empty. This should
+    generate an error (because we can't parse anything) but not
+    a fatal one (e. g. empty __init__.py files)
+    """
+    pass
+
 
 def asstr(txt: AnyStr) -> str:
     """
@@ -67,7 +75,7 @@ def _tostr_request(self, request: RawRequest) -> Request:
         """
         pass
 
-    def _check_input_request(self, request: RawRequest) -> Tuple[str, str]:
+    def _parse_input_request(self, request: RawRequest) -> Tuple[str, str]:
         """
         Check the incoming request package and validate that the 'content' and
         'language' keys are not missing and that 'language' and 'language_version'
@@ -77,13 +85,13 @@ def _check_input_request(self, request: RawRequest) -> Tuple[str, str]:
         :param request: The incoming request, already deserialized.
 
         .. raises::
-            RequestCheckException if the request failed to validate.
+            EmptyCodeException if the code was empty.
         """
         str_request = self._tostr_request(request)
         code = asstr(str_request.get('content', ''))
 
         if not code:
-            raise RequestCheckException('Bad input message, missing content')
+            raise EmptyCodeException('Bad input message, missing content')
 
         return code, asstr(str_request.get('filepath', ''))
 
@@ -99,7 +107,8 @@ def _send_response(self, response: Response) -> None:
         """
         pass
 
-    def _return_error(self, filepath: AnyStr='', status: AnyStr='error') -> None:
+    def _return_error(self, filepath: AnyStr='', status: AnyStr='error',
+            ast: Optional[Dict[Any, Any]] = None) -> None:
         """
         Build and send to stdout and error response. Also log
         the errors to the python_driver.log.
@@ -109,11 +118,24 @@ def _return_error(self, filepath: AnyStr='', status: AnyStr='error') -> None:
         :param status: error type, 'error' or 'fatal'
         """
 
+        if status == 'fatal':
+            ret_ast = None
+        elif ast:
+            ret_ast = ast
+        else:
+            # Empty modules are still modules
+            ret_ast = {"PY3AST": {
+                    "ast_type"   : "Module",
+                    "lineno"     : 1,
+                    "col_offset" : 1,
+                  }}
+
         logging.error('Filepath: {}, Errors: {}'.format(filepath, self.errors))
         response = Response({
             'status': status,
             'errors': self.errors,
             'driver': 'python23:%s' % __version__,
+            'ast': ret_ast,
         })
         if filepath:
             response['filepath'] = filepath
@@ -134,7 +156,7 @@ def process_request(self, request: RawRequest) -> None:
         self.errors = []
 
         try:
-            code, filepath = self._check_input_request(request)
+            code, filepath = self._parse_input_request(request)
 
             # We want the code detection to be fast and we prefer Python3 AST so using
             # the stop_on_ok_ast will avoid running a Python2 subprocess to check the
@@ -168,6 +190,10 @@ def process_request(self, request: RawRequest) -> None:
 
             self._send_response(response)
 
+        except EmptyCodeException:
+            self.errors.append('Code field empty')
+            self._return_error(filepath, status='error', ast=ast)
+
         except:
             status = 'fatal' if ast is None else 'error'
             self.errors.append(format_exc())

native/python_package/test/test_python_driver.py

@@ -18,7 +18,7 @@
 from python_driver import __version__, get_processor_instance
 from python_driver.requestprocessor import (
     Request, Response, RequestProcessorJSON,
-    InBuffer, RequestCheckException
+    InBuffer, RequestCheckException, EmptyCodeException
 )
 
 if TEST_MSGPACK:
@@ -251,16 +251,16 @@ def test_10_check_input(self) -> None:
             self._restart_data('json')
             brequest = convert_bytes(self.data, to_bytes=True)
             processor = RequestProcessorMSGPack(self.recvbuffer)
-            res = processor._check_input_request(brequest)
+            res = processor._parse_input_request(brequest)
             self.assertEqual(res[1], 'test.py')
 
         def test_20_check_input_bad(self) -> None:
             self._restart_data('msgpack')
             del self.data['content']
             brequest = convert_bytes(self.data, to_bytes=True)
             processor = RequestProcessorMSGPack(self.recvbuffer)
-            with self.assertRaises(RequestCheckException) as _:  # noqa: F841
-                processor._check_input_request(brequest)
+            with self.assertRaises(EmptyCodeException) as _:  # noqa: F841
+                processor._parse_input_request(brequest)
 
         def test_30_send_response_msgpack(self) -> None:
             self._restart_data('msgpack')
@@ -290,6 +290,7 @@ def test_50_return_error(self) -> None:
         self.assertDictEqual(res[0] , {'driver': 'python23:%s' % __version__,
                                        'errors': ['test error'],
                                        'filepath': 'test.py',
+                                       'ast': None,
                                        'status': 'fatal'})