Add DataConverter backed by msgspec as the default DataConverter

Author: natemort

cadence/data_converter.py

@@ -0,0 +1,80 @@
+from abc import abstractmethod
+from typing import Protocol, List, Type, Any
+
+from cadence.api.v1.common_pb2 import Payload
+from json import JSONDecoder
+from msgspec import json, convert
+
+
+class DataConverter(Protocol):
+
+    @abstractmethod
+    async def from_data(self, payload: Payload, type_hints: List[Type]) -> List[Any]:
+        raise NotImplementedError()
+
+    @abstractmethod
+    async def to_data(self, values: List[Any]) -> Payload:
+        raise NotImplementedError()
+
+class DefaultDataConverter(DataConverter):
+    def __init__(self):
+        self._encoder = json.Encoder()
+        self._decoder = json.Decoder()
+        self._fallback_decoder = JSONDecoder(strict=False)
+
+
+    async def from_data(self, payload: Payload, type_hints: List[Type]) -> List[Any]:
+        if len(type_hints) > 1:
+            payload_str = payload.data.decode()
+            # Handle payloads from the Go client, which are a series of json objects rather than a json array
+            if not payload_str.startswith("["):
+                return self._decode_whitespace_delimited(payload_str, type_hints)
+            else:
+                as_list = self._decoder.decode(payload_str)
+                return DefaultDataConverter._convert_into(as_list, type_hints)
+
+        as_value = self._decoder.decode(payload.data)
+        return DefaultDataConverter._convert_into([as_value], type_hints)
+
+
+    def _decode_whitespace_delimited(self, payload: str, type_hints: List[Type]) -> List[Any]:
+        results = []
+        start, end = 0, len(payload)
+        while start < end and len(results) < len(type_hints):
+            remaining = payload[start:end]
+            (value, value_end) = self._fallback_decoder.raw_decode(remaining)
+            start += value_end + 1
+            results.append(value)
+
+        return DefaultDataConverter._convert_into(results, type_hints)
+
+    @staticmethod
+    def _convert_into(values: List[Any], type_hints: List[Type]) -> List[Any]:
+        results = []
+        for i, type_hint in enumerate(type_hints):
+            if i < len(values):
+                value = convert(values[i], type_hint)
+            else:
+                value = DefaultDataConverter._get_default(type_hint)
+
+            results.append(value)
+
+        return results
+
+    @staticmethod
+    def _get_default(type_hint: Type) -> Any:
+        if type_hint in (int, float):
+            return 0
+        if type_hint == bool:
+            return False
+        return None
+
+
+    async def to_data(self, values: List[Any]) -> Payload:
+        data_value = values
+        # Don't wrap single values in a json array
+        if len(values) == 1:
+            data_value = values[0]
+
+        return Payload(data=self._encoder.encode(data_value))
+

pyproject.toml

@@ -28,6 +28,7 @@ requires-python = ">=3.11,<3.14"
 dependencies = [
     "grpcio>=1.50.0",
     "grpcio-tools>=1.50.0",
+    "msgspec>=0.19.0",
     "protobuf==5.29.1",
     "typing-extensions>=4.0.0",
 ]

tests/cadence/__init__.py

@@ -0,0 +1,88 @@
+import dataclasses
+from typing import Any, Type
+
+import pytest
+
+from cadence.api.v1.common_pb2 import Payload
+from cadence.data_converter import DefaultDataConverter
+from msgspec import json
+
+@dataclasses.dataclass
+class TestDataClass:
+    foo: str = "foo"
+    bar: int = -1
+    baz: 'TestDataClass' = None
+
+@pytest.mark.parametrize(
+    "json,types,expected",
+    [
+        pytest.param(
+            '"Hello world"', [str], ["Hello world"], id="happy path"
+        ),
+        pytest.param(
+            '"Hello" "world"', [str, str], ["Hello", "world"], id="space delimited"
+        ),
+        pytest.param(
+            '["Hello", "world"]', [str, str], ["Hello", "world"], id="json array"
+        ),
+        pytest.param(
+            "[1]", [int, int], [1, 0], id="ints"
+        ),
+        pytest.param(
+            "[1.5]", [float, float], [1.5, 0.0], id="floats"
+        ),
+        pytest.param(
+            "[true]", [bool, bool], [True, False], id="bools"
+        ),
+        pytest.param(
+            '[{"foo": "hello world", "bar": 42, "baz": {"bar": 43}}]', [TestDataClass, TestDataClass], [TestDataClass("hello world", 42, TestDataClass(bar=43)), None], id="data classes"
+        ),
+        pytest.param(
+            '[{"foo": "hello world"}]', [dict, dict], [{"foo": "hello world"}, None], id="dicts"
+        ),
+        pytest.param(
+            '[{"foo": 52}]', [dict[str, int], dict], [{"foo": 52}, None], id="generic dicts"
+        ),
+        pytest.param(
+            '[["hello"]]', [list[str], list[str]], [["hello"], None], id="lists"
+        ),
+        pytest.param(
+            '[["hello"]]', [set[str], set[str]], [{"hello"}, None], id="sets"
+        ),
+        pytest.param(
+            '["hello", "world"]', [list[str]], [["hello", "world"]], id="list"
+        ),
+        pytest.param(
+            '{"foo": "bar"} {"bar": 100} ["hello"] "world"', [TestDataClass, TestDataClass, list[str], str],
+            [TestDataClass(foo="bar"), TestDataClass(bar=100), ["hello"], "world"], id="space delimited mix"
+        ),
+    ]
+)
+@pytest.mark.asyncio
+async def test_data_converter_from_data(json: str, types: list[Type], expected: list[Any]):
+    converter = DefaultDataConverter()
+    actual = await converter.from_data(Payload(data=json.encode()), types)
+    assert expected == actual
+
+@pytest.mark.parametrize(
+    "values,expected",
+    [
+        pytest.param(
+            ["hello world"], '"hello world"', id="happy path"
+        ),
+        pytest.param(
+            ["hello", "world"], '["hello", "world"]', id="multiple values"
+        ),
+        pytest.param(
+            [TestDataClass()], '{"foo": "foo", "bar": -1, "baz": null}', id="data classes"
+        ),
+    ]
+)
+@pytest.mark.asyncio
+async def test_data_converter_to_data(values: list[Any], expected: str):
+    converter = DefaultDataConverter()
+    actual = await converter.to_data(values)
+    # Parse both rather than trying to compare strings
+    actual_parsed = json.decode(actual.data)
+    expected_parsed = json.decode(expected)
+    assert expected_parsed == actual_parsed