From 85caa4e076e18f6b2841cbc32cfa6a7d73514a77 Mon Sep 17 00:00:00 2001 From: Nate Mortensen Date: Thu, 21 Aug 2025 15:59:36 -0700 Subject: [PATCH] Add DataConverter backed by msgspec as the default DataConverter --- cadence/data_converter.py | 80 +++++++++++++++++++++++++ pyproject.toml | 1 + tests/cadence/__init__.py | 0 tests/cadence/data_converter_test.py | 88 ++++++++++++++++++++++++++++ uv.lock | 31 ++++++++++ 5 files changed, 200 insertions(+) create mode 100644 cadence/data_converter.py create mode 100644 tests/cadence/__init__.py create mode 100644 tests/cadence/data_converter_test.py diff --git a/cadence/data_converter.py b/cadence/data_converter.py new file mode 100644 index 0000000..819fd90 --- /dev/null +++ b/cadence/data_converter.py @@ -0,0 +1,80 @@ +from abc import abstractmethod +from typing import Protocol, List, Type, Any + +from cadence.api.v1.common_pb2 import Payload +from json import JSONDecoder +from msgspec import json, convert + + +class DataConverter(Protocol): + + @abstractmethod + async def from_data(self, payload: Payload, type_hints: List[Type]) -> List[Any]: + raise NotImplementedError() + + @abstractmethod + async def to_data(self, values: List[Any]) -> Payload: + raise NotImplementedError() + +class DefaultDataConverter(DataConverter): + def __init__(self): + self._encoder = json.Encoder() + self._decoder = json.Decoder() + self._fallback_decoder = JSONDecoder(strict=False) + + + async def from_data(self, payload: Payload, type_hints: List[Type]) -> List[Any]: + if len(type_hints) > 1: + payload_str = payload.data.decode() + # Handle payloads from the Go client, which are a series of json objects rather than a json array + if not payload_str.startswith("["): + return self._decode_whitespace_delimited(payload_str, type_hints) + else: + as_list = self._decoder.decode(payload_str) + return DefaultDataConverter._convert_into(as_list, type_hints) + + as_value = self._decoder.decode(payload.data) + return DefaultDataConverter._convert_into([as_value], type_hints) + + + def _decode_whitespace_delimited(self, payload: str, type_hints: List[Type]) -> List[Any]: + results = [] + start, end = 0, len(payload) + while start < end and len(results) < len(type_hints): + remaining = payload[start:end] + (value, value_end) = self._fallback_decoder.raw_decode(remaining) + start += value_end + 1 + results.append(value) + + return DefaultDataConverter._convert_into(results, type_hints) + + @staticmethod + def _convert_into(values: List[Any], type_hints: List[Type]) -> List[Any]: + results = [] + for i, type_hint in enumerate(type_hints): + if i < len(values): + value = convert(values[i], type_hint) + else: + value = DefaultDataConverter._get_default(type_hint) + + results.append(value) + + return results + + @staticmethod + def _get_default(type_hint: Type) -> Any: + if type_hint in (int, float): + return 0 + if type_hint is bool: + return False + return None + + + async def to_data(self, values: List[Any]) -> Payload: + data_value = values + # Don't wrap single values in a json array + if len(values) == 1: + data_value = values[0] + + return Payload(data=self._encoder.encode(data_value)) + diff --git a/pyproject.toml b/pyproject.toml index b3a1dba..e9be17c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ requires-python = ">=3.11,<3.14" dependencies = [ "grpcio>=1.50.0", "grpcio-tools>=1.50.0", + "msgspec>=0.19.0", "protobuf==5.29.1", "typing-extensions>=4.0.0", ] diff --git a/tests/cadence/__init__.py b/tests/cadence/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/cadence/data_converter_test.py b/tests/cadence/data_converter_test.py new file mode 100644 index 0000000..cf60299 --- /dev/null +++ b/tests/cadence/data_converter_test.py @@ -0,0 +1,88 @@ +import dataclasses +from typing import Any, Type + +import pytest + +from cadence.api.v1.common_pb2 import Payload +from cadence.data_converter import DefaultDataConverter +from msgspec import json + +@dataclasses.dataclass +class TestDataClass: + foo: str = "foo" + bar: int = -1 + baz: 'TestDataClass' = None + +@pytest.mark.parametrize( + "json,types,expected", + [ + pytest.param( + '"Hello world"', [str], ["Hello world"], id="happy path" + ), + pytest.param( + '"Hello" "world"', [str, str], ["Hello", "world"], id="space delimited" + ), + pytest.param( + '["Hello", "world"]', [str, str], ["Hello", "world"], id="json array" + ), + pytest.param( + "[1]", [int, int], [1, 0], id="ints" + ), + pytest.param( + "[1.5]", [float, float], [1.5, 0.0], id="floats" + ), + pytest.param( + "[true]", [bool, bool], [True, False], id="bools" + ), + pytest.param( + '[{"foo": "hello world", "bar": 42, "baz": {"bar": 43}}]', [TestDataClass, TestDataClass], [TestDataClass("hello world", 42, TestDataClass(bar=43)), None], id="data classes" + ), + pytest.param( + '[{"foo": "hello world"}]', [dict, dict], [{"foo": "hello world"}, None], id="dicts" + ), + pytest.param( + '[{"foo": 52}]', [dict[str, int], dict], [{"foo": 52}, None], id="generic dicts" + ), + pytest.param( + '[["hello"]]', [list[str], list[str]], [["hello"], None], id="lists" + ), + pytest.param( + '[["hello"]]', [set[str], set[str]], [{"hello"}, None], id="sets" + ), + pytest.param( + '["hello", "world"]', [list[str]], [["hello", "world"]], id="list" + ), + pytest.param( + '{"foo": "bar"} {"bar": 100} ["hello"] "world"', [TestDataClass, TestDataClass, list[str], str], + [TestDataClass(foo="bar"), TestDataClass(bar=100), ["hello"], "world"], id="space delimited mix" + ), + ] +) +@pytest.mark.asyncio +async def test_data_converter_from_data(json: str, types: list[Type], expected: list[Any]): + converter = DefaultDataConverter() + actual = await converter.from_data(Payload(data=json.encode()), types) + assert expected == actual + +@pytest.mark.parametrize( + "values,expected", + [ + pytest.param( + ["hello world"], '"hello world"', id="happy path" + ), + pytest.param( + ["hello", "world"], '["hello", "world"]', id="multiple values" + ), + pytest.param( + [TestDataClass()], '{"foo": "foo", "bar": -1, "baz": null}', id="data classes" + ), + ] +) +@pytest.mark.asyncio +async def test_data_converter_to_data(values: list[Any], expected: str): + converter = DefaultDataConverter() + actual = await converter.to_data(values) + # Parse both rather than trying to compare strings + actual_parsed = json.decode(actual.data) + expected_parsed = json.decode(expected) + assert expected_parsed == actual_parsed \ No newline at end of file diff --git a/uv.lock b/uv.lock index 4fb2f1a..feb34bf 100644 --- a/uv.lock +++ b/uv.lock @@ -154,6 +154,7 @@ source = { editable = "." } dependencies = [ { name = "grpcio" }, { name = "grpcio-tools" }, + { name = "msgspec" }, { name = "protobuf" }, { name = "typing-extensions" }, ] @@ -187,6 +188,7 @@ requires-dist = [ { name = "grpcio", specifier = ">=1.50.0" }, { name = "grpcio-tools", specifier = ">=1.50.0" }, { name = "isort", marker = "extra == 'dev'", specifier = ">=5.12.0" }, + { name = "msgspec", specifier = ">=0.19.0" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.0.0" }, { name = "myst-parser", marker = "extra == 'docs'", specifier = ">=1.0.0" }, { name = "pre-commit", marker = "extra == 'dev'", specifier = ">=3.0.0" }, @@ -692,6 +694,35 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, ] +[[package]] +name = "msgspec" +version = "0.19.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cf/9b/95d8ce458462b8b71b8a70fa94563b2498b89933689f3a7b8911edfae3d7/msgspec-0.19.0.tar.gz", hash = "sha256:604037e7cd475345848116e89c553aa9a233259733ab51986ac924ab1b976f8e", size = 216934, upload-time = "2024-12-27T17:40:28.597Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/d4/2ec2567ac30dab072cce3e91fb17803c52f0a37aab6b0c24375d2b20a581/msgspec-0.19.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:aa77046904db764b0462036bc63ef71f02b75b8f72e9c9dd4c447d6da1ed8f8e", size = 187939, upload-time = "2024-12-27T17:39:32.347Z" }, + { url = "https://files.pythonhosted.org/packages/2b/c0/18226e4328897f4f19875cb62bb9259fe47e901eade9d9376ab5f251a929/msgspec-0.19.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:047cfa8675eb3bad68722cfe95c60e7afabf84d1bd8938979dd2b92e9e4a9551", size = 182202, upload-time = "2024-12-27T17:39:33.633Z" }, + { url = "https://files.pythonhosted.org/packages/81/25/3a4b24d468203d8af90d1d351b77ea3cffb96b29492855cf83078f16bfe4/msgspec-0.19.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e78f46ff39a427e10b4a61614a2777ad69559cc8d603a7c05681f5a595ea98f7", size = 209029, upload-time = "2024-12-27T17:39:35.023Z" }, + { url = "https://files.pythonhosted.org/packages/85/2e/db7e189b57901955239f7689b5dcd6ae9458637a9c66747326726c650523/msgspec-0.19.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c7adf191e4bd3be0e9231c3b6dc20cf1199ada2af523885efc2ed218eafd011", size = 210682, upload-time = "2024-12-27T17:39:36.384Z" }, + { url = "https://files.pythonhosted.org/packages/03/97/7c8895c9074a97052d7e4a1cc1230b7b6e2ca2486714eb12c3f08bb9d284/msgspec-0.19.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:f04cad4385e20be7c7176bb8ae3dca54a08e9756cfc97bcdb4f18560c3042063", size = 214003, upload-time = "2024-12-27T17:39:39.097Z" }, + { url = "https://files.pythonhosted.org/packages/61/61/e892997bcaa289559b4d5869f066a8021b79f4bf8e955f831b095f47a4cd/msgspec-0.19.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:45c8fb410670b3b7eb884d44a75589377c341ec1392b778311acdbfa55187716", size = 216833, upload-time = "2024-12-27T17:39:41.203Z" }, + { url = "https://files.pythonhosted.org/packages/ce/3d/71b2dffd3a1c743ffe13296ff701ee503feaebc3f04d0e75613b6563c374/msgspec-0.19.0-cp311-cp311-win_amd64.whl", hash = "sha256:70eaef4934b87193a27d802534dc466778ad8d536e296ae2f9334e182ac27b6c", size = 186184, upload-time = "2024-12-27T17:39:43.702Z" }, + { url = "https://files.pythonhosted.org/packages/b2/5f/a70c24f075e3e7af2fae5414c7048b0e11389685b7f717bb55ba282a34a7/msgspec-0.19.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f98bd8962ad549c27d63845b50af3f53ec468b6318400c9f1adfe8b092d7b62f", size = 190485, upload-time = "2024-12-27T17:39:44.974Z" }, + { url = "https://files.pythonhosted.org/packages/89/b0/1b9763938cfae12acf14b682fcf05c92855974d921a5a985ecc197d1c672/msgspec-0.19.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:43bbb237feab761b815ed9df43b266114203f53596f9b6e6f00ebd79d178cdf2", size = 183910, upload-time = "2024-12-27T17:39:46.401Z" }, + { url = "https://files.pythonhosted.org/packages/87/81/0c8c93f0b92c97e326b279795f9c5b956c5a97af28ca0fbb9fd86c83737a/msgspec-0.19.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4cfc033c02c3e0aec52b71710d7f84cb3ca5eb407ab2ad23d75631153fdb1f12", size = 210633, upload-time = "2024-12-27T17:39:49.099Z" }, + { url = "https://files.pythonhosted.org/packages/d0/ef/c5422ce8af73928d194a6606f8ae36e93a52fd5e8df5abd366903a5ca8da/msgspec-0.19.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d911c442571605e17658ca2b416fd8579c5050ac9adc5e00c2cb3126c97f73bc", size = 213594, upload-time = "2024-12-27T17:39:51.204Z" }, + { url = "https://files.pythonhosted.org/packages/19/2b/4137bc2ed45660444842d042be2cf5b18aa06efd2cda107cff18253b9653/msgspec-0.19.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:757b501fa57e24896cf40a831442b19a864f56d253679f34f260dcb002524a6c", size = 214053, upload-time = "2024-12-27T17:39:52.866Z" }, + { url = "https://files.pythonhosted.org/packages/9d/e6/8ad51bdc806aac1dc501e8fe43f759f9ed7284043d722b53323ea421c360/msgspec-0.19.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5f0f65f29b45e2816d8bded36e6b837a4bf5fb60ec4bc3c625fa2c6da4124537", size = 219081, upload-time = "2024-12-27T17:39:55.142Z" }, + { url = "https://files.pythonhosted.org/packages/b1/ef/27dd35a7049c9a4f4211c6cd6a8c9db0a50647546f003a5867827ec45391/msgspec-0.19.0-cp312-cp312-win_amd64.whl", hash = "sha256:067f0de1c33cfa0b6a8206562efdf6be5985b988b53dd244a8e06f993f27c8c0", size = 187467, upload-time = "2024-12-27T17:39:56.531Z" }, + { url = "https://files.pythonhosted.org/packages/3c/cb/2842c312bbe618d8fefc8b9cedce37f773cdc8fa453306546dba2c21fd98/msgspec-0.19.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f12d30dd6266557aaaf0aa0f9580a9a8fbeadfa83699c487713e355ec5f0bd86", size = 190498, upload-time = "2024-12-27T17:40:00.427Z" }, + { url = "https://files.pythonhosted.org/packages/58/95/c40b01b93465e1a5f3b6c7d91b10fb574818163740cc3acbe722d1e0e7e4/msgspec-0.19.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:82b2c42c1b9ebc89e822e7e13bbe9d17ede0c23c187469fdd9505afd5a481314", size = 183950, upload-time = "2024-12-27T17:40:04.219Z" }, + { url = "https://files.pythonhosted.org/packages/e8/f0/5b764e066ce9aba4b70d1db8b087ea66098c7c27d59b9dd8a3532774d48f/msgspec-0.19.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:19746b50be214a54239aab822964f2ac81e38b0055cca94808359d779338c10e", size = 210647, upload-time = "2024-12-27T17:40:05.606Z" }, + { url = "https://files.pythonhosted.org/packages/9d/87/bc14f49bc95c4cb0dd0a8c56028a67c014ee7e6818ccdce74a4862af259b/msgspec-0.19.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:60ef4bdb0ec8e4ad62e5a1f95230c08efb1f64f32e6e8dd2ced685bcc73858b5", size = 213563, upload-time = "2024-12-27T17:40:10.516Z" }, + { url = "https://files.pythonhosted.org/packages/53/2f/2b1c2b056894fbaa975f68f81e3014bb447516a8b010f1bed3fb0e016ed7/msgspec-0.19.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ac7f7c377c122b649f7545810c6cd1b47586e3aa3059126ce3516ac7ccc6a6a9", size = 213996, upload-time = "2024-12-27T17:40:12.244Z" }, + { url = "https://files.pythonhosted.org/packages/aa/5a/4cd408d90d1417e8d2ce6a22b98a6853c1b4d7cb7669153e4424d60087f6/msgspec-0.19.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a5bc1472223a643f5ffb5bf46ccdede7f9795078194f14edd69e3aab7020d327", size = 219087, upload-time = "2024-12-27T17:40:14.881Z" }, + { url = "https://files.pythonhosted.org/packages/23/d8/f15b40611c2d5753d1abb0ca0da0c75348daf1252220e5dda2867bd81062/msgspec-0.19.0-cp313-cp313-win_amd64.whl", hash = "sha256:317050bc0f7739cb30d257ff09152ca309bf5a369854bbf1e57dffc310c1f20f", size = 187432, upload-time = "2024-12-27T17:40:16.256Z" }, +] + [[package]] name = "multidict" version = "6.6.3"