From 1d53e13ee29b1514be11a59396a236df0de1ade0 Mon Sep 17 00:00:00 2001 From: Kushal J Date: Wed, 23 Jul 2025 20:29:22 +0530 Subject: [PATCH 1/7] added default values for field decoding --- python/cocoindex/convert.py | 111 +++++++++++++++++++++++++++++++----- 1 file changed, 97 insertions(+), 14 deletions(-) diff --git a/python/cocoindex/convert.py b/python/cocoindex/convert.py index 946153e0..de960cab 100644 --- a/python/cocoindex/convert.py +++ b/python/cocoindex/convert.py @@ -5,6 +5,7 @@ import dataclasses import datetime import inspect +import warnings from enum import Enum from typing import Any, Callable, Mapping, get_origin @@ -67,6 +68,7 @@ def make_engine_value_decoder( field_path: list[str], src_type: dict[str, Any], dst_annotation: Any, + auto_default_missing_fields: bool = False, ) -> Callable[[Any], Any]: """ Make a decoder from an engine value to a Python value. @@ -90,15 +92,17 @@ def make_engine_value_decoder( if src_type_kind == "Union": return lambda value: value[1] if src_type_kind == "Struct": - return _make_engine_struct_to_dict_decoder(field_path, src_type["fields"]) + return _make_engine_struct_to_dict_decoder( + field_path, src_type["fields"], auto_default_missing_fields + ) if src_type_kind in TABLE_TYPES: if src_type_kind == "LTable": return _make_engine_ltable_to_list_dict_decoder( - field_path, src_type["row"]["fields"] + field_path, src_type["row"]["fields"], auto_default_missing_fields ) elif src_type_kind == "KTable": return _make_engine_ktable_to_dict_dict_decoder( - field_path, src_type["row"]["fields"] + field_path, src_type["row"]["fields"], auto_default_missing_fields ) return lambda value: value @@ -111,7 +115,9 @@ def make_engine_value_decoder( if args == (str, Any): is_dict_annotation = True if is_dict_annotation and src_type_kind == "Struct": - return _make_engine_struct_to_dict_decoder(field_path, src_type["fields"]) + return _make_engine_struct_to_dict_decoder( + field_path, src_type["fields"], auto_default_missing_fields + ) dst_type_info = analyze_type_info(dst_annotation) @@ -129,7 +135,10 @@ def make_engine_value_decoder( for dst_type_variant in dst_type_variants: try: decoder = make_engine_value_decoder( - src_field_path, src_type_variant, dst_type_variant + src_field_path, + src_type_variant, + dst_type_variant, + auto_default_missing_fields, ) break except ValueError: @@ -175,6 +184,7 @@ def decode_scalar(value: Any) -> Any | None: field_path + ["[*]"], src_type["element_type"], dst_type_info.elem_type, + auto_default_missing_fields, ) else: # for NDArray vector scalar_dtype = extract_ndarray_scalar_dtype(dst_type_info.np_number_type) @@ -206,7 +216,10 @@ def decode_vector(value: Any) -> Any | None: if dst_type_info.struct_type is not None: return _make_engine_struct_value_decoder( - field_path, src_type["fields"], dst_type_info.struct_type + field_path, + src_type["fields"], + dst_type_info.struct_type, + auto_default_missing_fields, ) if src_type_kind in TABLE_TYPES: @@ -222,11 +235,17 @@ def decode_vector(value: Any) -> Any | None: key_field_schema = engine_fields_schema[0] field_path.append(f".{key_field_schema.get('name', KEY_FIELD_NAME)}") key_decoder = make_engine_value_decoder( - field_path, key_field_schema["type"], elem_type_info.key_type + field_path, + key_field_schema["type"], + elem_type_info.key_type, + auto_default_missing_fields, ) field_path.pop() value_decoder = _make_engine_struct_value_decoder( - field_path, engine_fields_schema[1:], elem_type_info.struct_type + field_path, + engine_fields_schema[1:], + elem_type_info.struct_type, + auto_default_missing_fields, ) def decode(value: Any) -> Any | None: @@ -235,7 +254,10 @@ def decode(value: Any) -> Any | None: return {key_decoder(v[0]): value_decoder(v[1:]) for v in value} else: elem_decoder = _make_engine_struct_value_decoder( - field_path, engine_fields_schema, elem_type_info.struct_type + field_path, + engine_fields_schema, + elem_type_info.struct_type, + auto_default_missing_fields, ) def decode(value: Any) -> Any | None: @@ -249,10 +271,45 @@ def decode(value: Any) -> Any | None: return lambda value: value +def _get_auto_default_for_type( + annotation: Any, field_name: str, field_path: list[str] +) -> Any: + """ + Get an auto-default value for a type annotation if it's safe to do so. + + Returns: + The default value if auto-defaulting is safe, None otherwise. + """ + if annotation is None or annotation is inspect.Parameter.empty or annotation is Any: + return None + + try: + type_info = analyze_type_info(annotation) + + # Case 1: Nullable types (Optional[T] or T | None) + if type_info.nullable: + return None + + # Case 2: Table types (KTable or LTable) + if type_info.kind in TABLE_TYPES: + if type_info.kind == "LTable": + return [] # Empty list for LTable + elif type_info.kind == "KTable": + return {} # Empty dict for KTable + + # For all other types, don't auto-default to avoid ambiguity + return None + + except (ValueError, TypeError): + # If we can't analyze the type, don't auto-default + return None + + def _make_engine_struct_value_decoder( field_path: list[str], src_fields: list[dict[str, Any]], dst_struct_type: type, + auto_default_missing_fields: bool = False, ) -> Callable[[list[Any]], Any]: """Make a decoder from an engine field values to a Python value.""" @@ -285,7 +342,10 @@ def make_closure_for_value( if src_idx is not None: field_path.append(f".{name}") field_decoder = make_engine_value_decoder( - field_path, src_fields[src_idx]["type"], param.annotation + field_path, + src_fields[src_idx]["type"], + param.annotation, + auto_default_missing_fields, ) field_path.pop() return ( @@ -296,8 +356,21 @@ def make_closure_for_value( default_value = param.default if default_value is inspect.Parameter.empty: + if auto_default_missing_fields: + auto_default = _get_auto_default_for_type( + param.annotation, name, field_path + ) + if auto_default is not None: + warnings.warn( + f"Field '{name}' (type {param.annotation}) without default value is missing in input: " + f"{''.join(field_path)}. Auto-assigning default value: {auto_default}", + UserWarning, + stacklevel=3, + ) + return lambda _: auto_default + raise ValueError( - f"Field without default value is missing in input: {''.join(field_path)}" + f"Field '{name}' (type {param.annotation}) without default value is missing in input: {''.join(field_path)}" ) return lambda _: default_value @@ -314,6 +387,7 @@ def make_closure_for_value( def _make_engine_struct_to_dict_decoder( field_path: list[str], src_fields: list[dict[str, Any]], + auto_default_missing_fields: bool = False, ) -> Callable[[list[Any] | None], dict[str, Any] | None]: """Make a decoder from engine field values to a Python dict.""" @@ -325,6 +399,7 @@ def _make_engine_struct_to_dict_decoder( field_path, field_schema["type"], Any, # Use Any for recursive decoding + auto_default_missing_fields, ) field_path.pop() field_decoders.append((field_name, field_decoder)) @@ -347,11 +422,14 @@ def decode_to_dict(values: list[Any] | None) -> dict[str, Any] | None: def _make_engine_ltable_to_list_dict_decoder( field_path: list[str], src_fields: list[dict[str, Any]], + auto_default_missing_fields: bool = False, ) -> Callable[[list[Any] | None], list[dict[str, Any]] | None]: """Make a decoder from engine LTable values to a list of dicts.""" # Create a decoder for each row (struct) to dict - row_decoder = _make_engine_struct_to_dict_decoder(field_path, src_fields) + row_decoder = _make_engine_struct_to_dict_decoder( + field_path, src_fields, auto_default_missing_fields + ) def decode_to_list_dict(values: list[Any] | None) -> list[dict[str, Any]] | None: if values is None: @@ -372,6 +450,7 @@ def decode_to_list_dict(values: list[Any] | None) -> list[dict[str, Any]] | None def _make_engine_ktable_to_dict_dict_decoder( field_path: list[str], src_fields: list[dict[str, Any]], + auto_default_missing_fields: bool = False, ) -> Callable[[list[Any] | None], dict[Any, dict[str, Any]] | None]: """Make a decoder from engine KTable values to a dict of dicts.""" @@ -384,10 +463,14 @@ def _make_engine_ktable_to_dict_dict_decoder( # Create decoders field_path.append(f".{key_field_schema.get('name', KEY_FIELD_NAME)}") - key_decoder = make_engine_value_decoder(field_path, key_field_schema["type"], Any) + key_decoder = make_engine_value_decoder( + field_path, key_field_schema["type"], Any, auto_default_missing_fields + ) field_path.pop() - value_decoder = _make_engine_struct_to_dict_decoder(field_path, value_fields_schema) + value_decoder = _make_engine_struct_to_dict_decoder( + field_path, value_fields_schema, auto_default_missing_fields + ) def decode_to_dict_dict( values: list[Any] | None, From 66ca70c0a97e83475983167b6f8716b32125d78d Mon Sep 17 00:00:00 2001 From: Kushal J Date: Wed, 23 Jul 2025 20:35:46 +0530 Subject: [PATCH 2/7] removed few comments --- python/cocoindex/convert.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/python/cocoindex/convert.py b/python/cocoindex/convert.py index de960cab..6780f899 100644 --- a/python/cocoindex/convert.py +++ b/python/cocoindex/convert.py @@ -293,15 +293,14 @@ def _get_auto_default_for_type( # Case 2: Table types (KTable or LTable) if type_info.kind in TABLE_TYPES: if type_info.kind == "LTable": - return [] # Empty list for LTable + return [] elif type_info.kind == "KTable": - return {} # Empty dict for KTable + return {} # For all other types, don't auto-default to avoid ambiguity return None except (ValueError, TypeError): - # If we can't analyze the type, don't auto-default return None From 0c262c26c5ed2c1a524af6e28d0557694d8ebdee Mon Sep 17 00:00:00 2001 From: Kushal J Date: Thu, 24 Jul 2025 21:09:33 +0530 Subject: [PATCH 3/7] made suggested changes --- python/cocoindex/convert.py | 76 +++++++++----------------- python/cocoindex/tests/test_convert.py | 38 ++++++++++++- 2 files changed, 64 insertions(+), 50 deletions(-) diff --git a/python/cocoindex/convert.py b/python/cocoindex/convert.py index 6780f899..ca38d60c 100644 --- a/python/cocoindex/convert.py +++ b/python/cocoindex/convert.py @@ -68,7 +68,6 @@ def make_engine_value_decoder( field_path: list[str], src_type: dict[str, Any], dst_annotation: Any, - auto_default_missing_fields: bool = False, ) -> Callable[[Any], Any]: """ Make a decoder from an engine value to a Python value. @@ -92,17 +91,15 @@ def make_engine_value_decoder( if src_type_kind == "Union": return lambda value: value[1] if src_type_kind == "Struct": - return _make_engine_struct_to_dict_decoder( - field_path, src_type["fields"], auto_default_missing_fields - ) + return _make_engine_struct_to_dict_decoder(field_path, src_type["fields"]) if src_type_kind in TABLE_TYPES: if src_type_kind == "LTable": return _make_engine_ltable_to_list_dict_decoder( - field_path, src_type["row"]["fields"], auto_default_missing_fields + field_path, src_type["row"]["fields"] ) elif src_type_kind == "KTable": return _make_engine_ktable_to_dict_dict_decoder( - field_path, src_type["row"]["fields"], auto_default_missing_fields + field_path, src_type["row"]["fields"] ) return lambda value: value @@ -115,9 +112,7 @@ def make_engine_value_decoder( if args == (str, Any): is_dict_annotation = True if is_dict_annotation and src_type_kind == "Struct": - return _make_engine_struct_to_dict_decoder( - field_path, src_type["fields"], auto_default_missing_fields - ) + return _make_engine_struct_to_dict_decoder(field_path, src_type["fields"]) dst_type_info = analyze_type_info(dst_annotation) @@ -138,7 +133,6 @@ def make_engine_value_decoder( src_field_path, src_type_variant, dst_type_variant, - auto_default_missing_fields, ) break except ValueError: @@ -184,7 +178,6 @@ def decode_scalar(value: Any) -> Any | None: field_path + ["[*]"], src_type["element_type"], dst_type_info.elem_type, - auto_default_missing_fields, ) else: # for NDArray vector scalar_dtype = extract_ndarray_scalar_dtype(dst_type_info.np_number_type) @@ -219,7 +212,6 @@ def decode_vector(value: Any) -> Any | None: field_path, src_type["fields"], dst_type_info.struct_type, - auto_default_missing_fields, ) if src_type_kind in TABLE_TYPES: @@ -238,14 +230,12 @@ def decode_vector(value: Any) -> Any | None: field_path, key_field_schema["type"], elem_type_info.key_type, - auto_default_missing_fields, ) field_path.pop() value_decoder = _make_engine_struct_value_decoder( field_path, engine_fields_schema[1:], elem_type_info.struct_type, - auto_default_missing_fields, ) def decode(value: Any) -> Any | None: @@ -257,7 +247,6 @@ def decode(value: Any) -> Any | None: field_path, engine_fields_schema, elem_type_info.struct_type, - auto_default_missing_fields, ) def decode(value: Any) -> Any | None: @@ -273,42 +262,43 @@ def decode(value: Any) -> Any | None: def _get_auto_default_for_type( annotation: Any, field_name: str, field_path: list[str] -) -> Any: +) -> tuple[Any, bool]: """ Get an auto-default value for a type annotation if it's safe to do so. Returns: - The default value if auto-defaulting is safe, None otherwise. + A tuple of (default_value, is_supported) where: + - default_value: The default value if auto-defaulting is supported + - is_supported: True if auto-defaulting is supported for this type """ if annotation is None or annotation is inspect.Parameter.empty or annotation is Any: - return None + return None, False try: type_info = analyze_type_info(annotation) # Case 1: Nullable types (Optional[T] or T | None) if type_info.nullable: - return None + return None, True # Case 2: Table types (KTable or LTable) if type_info.kind in TABLE_TYPES: if type_info.kind == "LTable": - return [] + return [], True elif type_info.kind == "KTable": - return {} + return {}, True # For all other types, don't auto-default to avoid ambiguity - return None + return None, False except (ValueError, TypeError): - return None + return None, False def _make_engine_struct_value_decoder( field_path: list[str], src_fields: list[dict[str, Any]], dst_struct_type: type, - auto_default_missing_fields: bool = False, ) -> Callable[[list[Any]], Any]: """Make a decoder from an engine field values to a Python value.""" @@ -344,7 +334,6 @@ def make_closure_for_value( field_path, src_fields[src_idx]["type"], param.annotation, - auto_default_missing_fields, ) field_path.pop() return ( @@ -355,18 +344,17 @@ def make_closure_for_value( default_value = param.default if default_value is inspect.Parameter.empty: - if auto_default_missing_fields: - auto_default = _get_auto_default_for_type( - param.annotation, name, field_path + auto_default, is_supported = _get_auto_default_for_type( + param.annotation, name, field_path + ) + if is_supported: + warnings.warn( + f"Field '{name}' (type {param.annotation}) without default value is missing in input: " + f"{''.join(field_path)}. Auto-assigning default value: {auto_default}", + UserWarning, + stacklevel=3, ) - if auto_default is not None: - warnings.warn( - f"Field '{name}' (type {param.annotation}) without default value is missing in input: " - f"{''.join(field_path)}. Auto-assigning default value: {auto_default}", - UserWarning, - stacklevel=3, - ) - return lambda _: auto_default + return lambda _: auto_default raise ValueError( f"Field '{name}' (type {param.annotation}) without default value is missing in input: {''.join(field_path)}" @@ -386,7 +374,6 @@ def make_closure_for_value( def _make_engine_struct_to_dict_decoder( field_path: list[str], src_fields: list[dict[str, Any]], - auto_default_missing_fields: bool = False, ) -> Callable[[list[Any] | None], dict[str, Any] | None]: """Make a decoder from engine field values to a Python dict.""" @@ -398,7 +385,6 @@ def _make_engine_struct_to_dict_decoder( field_path, field_schema["type"], Any, # Use Any for recursive decoding - auto_default_missing_fields, ) field_path.pop() field_decoders.append((field_name, field_decoder)) @@ -421,14 +407,11 @@ def decode_to_dict(values: list[Any] | None) -> dict[str, Any] | None: def _make_engine_ltable_to_list_dict_decoder( field_path: list[str], src_fields: list[dict[str, Any]], - auto_default_missing_fields: bool = False, ) -> Callable[[list[Any] | None], list[dict[str, Any]] | None]: """Make a decoder from engine LTable values to a list of dicts.""" # Create a decoder for each row (struct) to dict - row_decoder = _make_engine_struct_to_dict_decoder( - field_path, src_fields, auto_default_missing_fields - ) + row_decoder = _make_engine_struct_to_dict_decoder(field_path, src_fields) def decode_to_list_dict(values: list[Any] | None) -> list[dict[str, Any]] | None: if values is None: @@ -449,7 +432,6 @@ def decode_to_list_dict(values: list[Any] | None) -> list[dict[str, Any]] | None def _make_engine_ktable_to_dict_dict_decoder( field_path: list[str], src_fields: list[dict[str, Any]], - auto_default_missing_fields: bool = False, ) -> Callable[[list[Any] | None], dict[Any, dict[str, Any]] | None]: """Make a decoder from engine KTable values to a dict of dicts.""" @@ -462,14 +444,10 @@ def _make_engine_ktable_to_dict_dict_decoder( # Create decoders field_path.append(f".{key_field_schema.get('name', KEY_FIELD_NAME)}") - key_decoder = make_engine_value_decoder( - field_path, key_field_schema["type"], Any, auto_default_missing_fields - ) + key_decoder = make_engine_value_decoder(field_path, key_field_schema["type"], Any) field_path.pop() - value_decoder = _make_engine_struct_to_dict_decoder( - field_path, value_fields_schema, auto_default_missing_fields - ) + value_decoder = _make_engine_struct_to_dict_decoder(field_path, value_fields_schema) def decode_to_dict_dict( values: list[Any] | None, diff --git a/python/cocoindex/tests/test_convert.py b/python/cocoindex/tests/test_convert.py index 57260a55..4a253db0 100644 --- a/python/cocoindex/tests/test_convert.py +++ b/python/cocoindex/tests/test_convert.py @@ -1,6 +1,6 @@ import datetime import uuid -from dataclasses import dataclass, make_dataclass +from dataclasses import dataclass, make_dataclass, field from typing import Annotated, Any, Callable, Literal, NamedTuple import numpy as np @@ -1468,3 +1468,39 @@ class Team: # Test Any annotation validate_full_roundtrip(teams, dict[str, Team], (expected_dict_dict, Any)) + + +def test_auto_default_supported_and_unsupported() -> None: + from dataclasses import dataclass, field + + @dataclass + class Base: + a: int + b: int + + @dataclass + class ExtraFieldSupported: + a: int + b: int + c: list[int] = field(default_factory=list) + + @dataclass + class ExtraFieldUnsupported: + a: int + b: int + c: int + + engine_val = [1, 2] + + # Should succeed: c is a list (LTable), auto-defaults to [] + validate_full_roundtrip( + Base(1, 2), Base, (ExtraFieldSupported(1, 2, []), ExtraFieldSupported) + ) + + # Should fail: c is a non-nullable int, no default, not supported + with pytest.raises( + ValueError, + match=r"Field 'c' \(type \) without default value is missing in input: ", + ): + decoder = build_engine_value_decoder(Base, ExtraFieldUnsupported) + decoder(engine_val) From 3259610c25e54d1163d9b95ee1b87bcb62610076 Mon Sep 17 00:00:00 2001 From: Kushal J Date: Sat, 26 Jul 2025 13:06:57 +0530 Subject: [PATCH 4/7] added new test case and changed existing lambda function to better handle errors --- python/cocoindex/convert.py | 58 +++++++++++++++++--------- python/cocoindex/tests/test_convert.py | 41 ++++++++++-------- 2 files changed, 63 insertions(+), 36 deletions(-) diff --git a/python/cocoindex/convert.py b/python/cocoindex/convert.py index ca38d60c..b9165693 100644 --- a/python/cocoindex/convert.py +++ b/python/cocoindex/convert.py @@ -295,6 +295,31 @@ def _get_auto_default_for_type( return None, False +def _handle_missing_field_with_auto_default( + param: inspect.Parameter, name: str, field_path: list[str] +) -> Any: + """ + Handle missing field by trying auto-default or raising an error. + + Returns the auto-default value if supported, otherwise raises ValueError. + """ + auto_default, is_supported = _get_auto_default_for_type( + param.annotation, name, field_path + ) + if is_supported: + warnings.warn( + f"Field '{name}' (type {param.annotation}) without default value is missing in input: " + f"{''.join(field_path)}. Auto-assigning default value: {auto_default}", + UserWarning, + stacklevel=4, + ) + return auto_default + + raise ValueError( + f"Field '{name}' (type {param.annotation}) without default value is missing in input: {''.join(field_path)}" + ) + + def _make_engine_struct_value_decoder( field_path: list[str], src_fields: list[dict[str, Any]], @@ -336,29 +361,24 @@ def make_closure_for_value( param.annotation, ) field_path.pop() - return ( - lambda values: field_decoder(values[src_idx]) - if len(values) > src_idx - else param.default - ) + + def field_value_getter(values: list[Any]) -> Any: + if len(values) > src_idx: + return field_decoder(values[src_idx]) + default_value = param.default + if default_value is not inspect.Parameter.empty: + return default_value + + return _handle_missing_field_with_auto_default(param, name, field_path) + + return field_value_getter default_value = param.default if default_value is inspect.Parameter.empty: - auto_default, is_supported = _get_auto_default_for_type( - param.annotation, name, field_path - ) - if is_supported: - warnings.warn( - f"Field '{name}' (type {param.annotation}) without default value is missing in input: " - f"{''.join(field_path)}. Auto-assigning default value: {auto_default}", - UserWarning, - stacklevel=3, - ) - return lambda _: auto_default - - raise ValueError( - f"Field '{name}' (type {param.annotation}) without default value is missing in input: {''.join(field_path)}" + auto_default = _handle_missing_field_with_auto_default( + param, name, field_path ) + return lambda _: auto_default return lambda _: default_value diff --git a/python/cocoindex/tests/test_convert.py b/python/cocoindex/tests/test_convert.py index 4a253db0..979cb49b 100644 --- a/python/cocoindex/tests/test_convert.py +++ b/python/cocoindex/tests/test_convert.py @@ -1470,37 +1470,44 @@ class Team: validate_full_roundtrip(teams, dict[str, Team], (expected_dict_dict, Any)) -def test_auto_default_supported_and_unsupported() -> None: - from dataclasses import dataclass, field - +def test_auto_default_for_supported_and_unsupported_types() -> None: @dataclass class Base: a: int - b: int @dataclass - class ExtraFieldSupported: + class NullableField: a: int - b: int - c: list[int] = field(default_factory=list) + b: int | None @dataclass - class ExtraFieldUnsupported: + class LTableField: + a: int + b: list[Base] + + @dataclass + class KTableField: + a: int + b: dict[str, Base] + + @dataclass + class UnsupportedField: a: int b: int - c: int - engine_val = [1, 2] + engine_val = [1] - # Should succeed: c is a list (LTable), auto-defaults to [] - validate_full_roundtrip( - Base(1, 2), Base, (ExtraFieldSupported(1, 2, []), ExtraFieldSupported) - ) + validate_full_roundtrip(NullableField(1, None), NullableField) + + validate_full_roundtrip(LTableField(1, []), LTableField) + + decoder = build_engine_value_decoder(KTableField) + result = decoder(engine_val) + assert result == KTableField(1, {}) - # Should fail: c is a non-nullable int, no default, not supported with pytest.raises( ValueError, - match=r"Field 'c' \(type \) without default value is missing in input: ", + match=r"Field 'b' \(type \) without default value is missing in input: ", ): - decoder = build_engine_value_decoder(Base, ExtraFieldUnsupported) + decoder = build_engine_value_decoder(Base, UnsupportedField) decoder(engine_val) From a736a414b6c322d2f65c43169515949e7ac59a6e Mon Sep 17 00:00:00 2001 From: Kushal J Date: Sun, 27 Jul 2025 10:23:28 +0530 Subject: [PATCH 5/7] commented full_roundtrip test --- python/cocoindex/tests/test_convert.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/cocoindex/tests/test_convert.py b/python/cocoindex/tests/test_convert.py index 979cb49b..cf3a7d13 100644 --- a/python/cocoindex/tests/test_convert.py +++ b/python/cocoindex/tests/test_convert.py @@ -1505,6 +1505,8 @@ class UnsupportedField: result = decoder(engine_val) assert result == KTableField(1, {}) + # validate_full_roundtrip(KTableField(1, {}), KTableField) + with pytest.raises( ValueError, match=r"Field 'b' \(type \) without default value is missing in input: ", From ea4fb88b0b5f360be15eb2201dc48b3fbadbe475 Mon Sep 17 00:00:00 2001 From: Kushal J Date: Sun, 27 Jul 2025 11:44:48 +0530 Subject: [PATCH 6/7] fixed errors after merging --- python/cocoindex/convert.py | 42 ++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/python/cocoindex/convert.py b/python/cocoindex/convert.py index 162e0e07..12539b8c 100644 --- a/python/cocoindex/convert.py +++ b/python/cocoindex/convert.py @@ -286,6 +286,7 @@ def decode_scalar(value: Any) -> Any | None: return lambda value: value + def _get_auto_default_for_type( annotation: Any, field_name: str, field_path: list[str] ) -> tuple[Any, bool]: @@ -307,12 +308,11 @@ def _get_auto_default_for_type( if type_info.nullable: return None, True - # Case 2: Table types (KTable or LTable) - if type_info.kind in TABLE_TYPES: - if type_info.kind == "LTable": - return [], True - elif type_info.kind == "KTable": - return {}, True + # Case 2: Table types (KTable or LTable) - check if it's a list or dict type + if isinstance(type_info.variant, AnalyzedListType): + return [], True + elif isinstance(type_info.variant, AnalyzedDictType): + return {}, True # For all other types, don't auto-default to avoid ambiguity return None, False @@ -345,6 +345,7 @@ def _handle_missing_field_with_auto_default( f"Field '{name}' (type {param.annotation}) without default value is missing in input: {''.join(field_path)}" ) + def make_engine_struct_decoder( field_path: list[str], src_fields: list[dict[str, Any]], @@ -408,23 +409,26 @@ def make_closure_for_value( field_decoder = make_engine_value_decoder( field_path, src_fields[src_idx]["type"], param.annotation ) - def field_value_getter(values: list[Any]) -> Any: - if len(values) > src_idx: - return field_decoder(values[src_idx]) - default_value = param.default - if default_value is not inspect.Parameter.empty: - return default_value - return _handle_missing_field_with_auto_default(param, name, field_path) + def field_value_getter(values: list[Any]) -> Any: + if src_idx is not None and len(values) > src_idx: + return field_decoder(values[src_idx]) + default_value = param.default + if default_value is not inspect.Parameter.empty: + return default_value + + return _handle_missing_field_with_auto_default( + param, name, field_path + ) - return field_value_getter + return field_value_getter default_value = param.default - if default_value is inspect.Parameter.empty: - auto_default = _handle_missing_field_with_auto_default( - param, name, field_path - ) - return lambda _: auto_default + if default_value is not inspect.Parameter.empty: + return lambda _: default_value + + auto_default = _handle_missing_field_with_auto_default(param, name, field_path) + return lambda _: auto_default field_value_decoder = [ make_closure_for_value(name, param) for (name, param) in parameters.items() From 279ab931d4e59293d947f5e72a1ca3eb3cfd8d57 Mon Sep 17 00:00:00 2001 From: Kushal J Date: Sun, 27 Jul 2025 21:48:08 +0530 Subject: [PATCH 7/7] Addressed new comments --- python/cocoindex/convert.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/python/cocoindex/convert.py b/python/cocoindex/convert.py index 12539b8c..a60bf9ec 100644 --- a/python/cocoindex/convert.py +++ b/python/cocoindex/convert.py @@ -423,12 +423,14 @@ def field_value_getter(values: list[Any]) -> Any: return field_value_getter - default_value = param.default - if default_value is not inspect.Parameter.empty: - return lambda _: default_value + default_value = param.default + if default_value is not inspect.Parameter.empty: + return lambda _: default_value - auto_default = _handle_missing_field_with_auto_default(param, name, field_path) - return lambda _: auto_default + auto_default = _handle_missing_field_with_auto_default( + param, name, field_path + ) + return lambda _: auto_default field_value_decoder = [ make_closure_for_value(name, param) for (name, param) in parameters.items()