Merge pull request #13 from marselester/json-best-effort

Make best effort to serialize json, e.g., WSGIRequest

Author: marselester

.travis.yml

@@ -7,7 +7,7 @@ python:
   - 3.7
   - 3.8
 install:
-  - pip install ujson simplejson
+  - pip install ujson simplejson django
   - python setup.py install
 script: nosetests tests
 notifications:

README.rst

@@ -51,7 +51,6 @@ JSON libraries
 --------------
 
 You can use **ujson** or **simplejson** instead of built-in **json** library.
-They are faster and can serialize ``Decimal`` values.
 
 .. code-block:: python
 
@@ -61,6 +60,9 @@ They are faster and can serialize ``Decimal`` values.
     formatter = json_log_formatter.JSONFormatter()
     formatter.json_lib = ujson
 
+Note, **ujson** doesn't support `dumps(default=f)` argument:
+if it can't serialize an attribute, it might fail with `TypeError` or skip an attribute.
+
 Django integration
 ------------------
 
@@ -111,7 +113,7 @@ To do so you should override ``JSONFormatter.json_record()``.
             return extra
 
 Let's say you want ``datetime`` to be serialized as timestamp.
-Then you should use **ujson** (which does it by default) and disable
+You can use **ujson** (which does it by default) and disable
 ISO8601 date mutation.
 
 .. code-block:: python

json_log_formatter/__init__.py

@@ -73,10 +73,18 @@ def format(self, record):
     def to_json(self, record):
         """Converts record dict to a JSON string.
 
+        It makes best effort to serialize a record (represents an object as a string)
+        instead of raising TypeError if json library supports default argument.
+        Note, ujson doesn't support it.
+
         Override this method to change the way dict is converted to JSON.
 
         """
-        return self.json_lib.dumps(record)
+        try:
+            return self.json_lib.dumps(record, default=_json_serializable)
+        # ujson doesn't support default argument and raises TypeError.
+        except TypeError:
+            return self.json_lib.dumps(record)
 
     def extra_from_record(self, record):
         """Returns `extra` dict you passed to logger.
@@ -123,3 +131,10 @@ def mutate_json_record(self, json_record):
             if isinstance(attr, datetime):
                 json_record[attr_name] = attr.isoformat()
         return json_record
+
+
+def _json_serializable(obj):
+    try:
+        return obj.__dict__
+    except AttributeError:
+        return str(obj)

tests.py

@@ -1,9 +1,13 @@
-import unittest
-import logging
 from datetime import datetime
 from decimal import Decimal
-
+from io import BytesIO
+import unittest
+import logging
 import json
+
+
+from django.core.handlers.wsgi import WSGIRequest
+from django.conf import settings
 import ujson
 import simplejson
 
@@ -25,6 +29,8 @@
 DATETIME = datetime(2015, 9, 1, 6, 9, 42, 797203)
 DATETIME_ISO = u'2015-09-01T06:09:42.797203'
 
+settings.configure(DEBUG=True)
+
 
 class TestCase(unittest.TestCase):
     def tearDown(self):
@@ -100,11 +106,49 @@ class JsonLibTest(TestCase):
     def setUp(self):
         json_handler.setFormatter(JSONFormatter())
 
-    def test_error_when_decimal_is_passed(self):
-        with self.assertRaises(TypeError):
-            logger.log(lvl=0, msg='Payment was sent', extra={
-                'amount': Decimal('0.00497265')
-            })
+    def test_builtin_types_are_serialized(self):
+        logger.log(level=logging.ERROR, msg='Payment was sent', extra={
+            'first_name': 'bob',
+            'amount': 0.00497265,
+            'context': {
+                'tags': ['fizz', 'bazz'],
+            },
+            'things': ('a', 'b'),
+            'ok': True,
+            'none': None,
+        })
+
+        json_record = json.loads(log_buffer.getvalue())
+        self.assertEqual(json_record['first_name'], 'bob')
+        self.assertEqual(json_record['amount'], 0.00497265)
+        self.assertEqual(json_record['context'], {'tags': ['fizz', 'bazz']})
+        self.assertEqual(json_record['things'], ['a', 'b'])
+        self.assertEqual(json_record['ok'], True)
+        self.assertEqual(json_record['none'], None)
+
+    def test_decimal_is_serialized_as_string(self):
+        logger.log(level=logging.ERROR, msg='Payment was sent', extra={
+            'amount': Decimal('0.00497265')
+        })
+        expected_amount = '"amount": "0.00497265"'
+        self.assertIn(expected_amount, log_buffer.getvalue())
+
+    def test_django_wsgi_request_is_serialized_as_dict(self):
+        request = WSGIRequest({
+            'PATH_INFO': 'bogus',
+            'REQUEST_METHOD': 'bogus',
+            'CONTENT_TYPE': 'text/html; charset=utf8',
+            'wsgi.input': BytesIO(b''),
+        })
+
+        logger.log(level=logging.ERROR, msg='Django response error', extra={
+            'status_code': 500,
+            'request': request
+        })
+        json_record = json.loads(log_buffer.getvalue())
+        self.assertEqual(json_record['status_code'], 500)
+        self.assertEqual(json_record['request']['path'], '/bogus')
+        self.assertEqual(json_record['request']['method'], 'BOGUS')
 
 
 class UjsonLibTest(TestCase):
@@ -113,6 +157,26 @@ def setUp(self):
         formatter.json_lib = ujson
         json_handler.setFormatter(formatter)
 
+    def test_builtin_types_are_serialized(self):
+        logger.log(level=logging.ERROR, msg='Payment was sent', extra={
+            'first_name': 'bob',
+            'amount': 0.00497265,
+            'context': {
+                'tags': ['fizz', 'bazz'],
+            },
+            'things': ('a', 'b'),
+            'ok': True,
+            'none': None,
+        })
+
+        json_record = json.loads(log_buffer.getvalue())
+        self.assertEqual(json_record['first_name'], 'bob')
+        self.assertEqual(json_record['amount'], 0.00497265)
+        self.assertEqual(json_record['context'], {'tags': ['fizz', 'bazz']})
+        self.assertEqual(json_record['things'], ['a', 'b'])
+        self.assertEqual(json_record['ok'], True)
+        self.assertEqual(json_record['none'], None)
+
     def test_decimal_is_serialized_as_number(self):
         logger.info('Payment was sent', extra={
             'amount': Decimal('0.00497265')
@@ -127,13 +191,49 @@ def test_zero_expected_when_decimal_is_in_scientific_notation(self):
         expected_amount = '"amount":0.0'
         self.assertIn(expected_amount, log_buffer.getvalue())
 
+    def test_django_wsgi_request_is_serialized_as_empty_list(self):
+        request = WSGIRequest({
+            'PATH_INFO': 'bogus',
+            'REQUEST_METHOD': 'bogus',
+            'CONTENT_TYPE': 'text/html; charset=utf8',
+            'wsgi.input': BytesIO(b''),
+        })
+
+        logger.log(level=logging.ERROR, msg='Django response error', extra={
+            'status_code': 500,
+            'request': request
+        })
+        json_record = json.loads(log_buffer.getvalue())
+        self.assertEqual(json_record['status_code'], 500)
+        self.assertEqual(json_record['request'], [])
+
 
 class SimplejsonLibTest(TestCase):
     def setUp(self):
         formatter = JSONFormatter()
         formatter.json_lib = simplejson
         json_handler.setFormatter(formatter)
 
+    def test_builtin_types_are_serialized(self):
+        logger.log(level=logging.ERROR, msg='Payment was sent', extra={
+            'first_name': 'bob',
+            'amount': 0.00497265,
+            'context': {
+                'tags': ['fizz', 'bazz'],
+            },
+            'things': ('a', 'b'),
+            'ok': True,
+            'none': None,
+        })
+
+        json_record = json.loads(log_buffer.getvalue())
+        self.assertEqual(json_record['first_name'], 'bob')
+        self.assertEqual(json_record['amount'], 0.00497265)
+        self.assertEqual(json_record['context'], {'tags': ['fizz', 'bazz']})
+        self.assertEqual(json_record['things'], ['a', 'b'])
+        self.assertEqual(json_record['ok'], True)
+        self.assertEqual(json_record['none'], None)
+
     def test_decimal_is_serialized_as_number(self):
         logger.info('Payment was sent', extra={
             'amount': Decimal('0.00497265')
@@ -147,3 +247,20 @@ def test_decimal_is_serialized_as_it_is_when_it_is_in_scientific_notation(self):
         })
         expected_amount = '"amount": 0E-8'
         self.assertIn(expected_amount, log_buffer.getvalue())
+
+    def test_django_wsgi_request_is_serialized_as_dict(self):
+        request = WSGIRequest({
+            'PATH_INFO': 'bogus',
+            'REQUEST_METHOD': 'bogus',
+            'CONTENT_TYPE': 'text/html; charset=utf8',
+            'wsgi.input': BytesIO(b''),
+        })
+
+        logger.log(level=logging.ERROR, msg='Django response error', extra={
+            'status_code': 500,
+            'request': request
+        })
+        json_record = json.loads(log_buffer.getvalue())
+        self.assertEqual(json_record['status_code'], 500)
+        self.assertEqual(json_record['request']['path'], '/bogus')
+        self.assertEqual(json_record['request']['method'], 'BOGUS')

tox.ini

@@ -6,5 +6,6 @@ deps=
     pytest
     ujson
     simplejson
+    django
 commands=
     pytest -s tests.py