feat: Transform strings with respect to property schema when conforming properties

pyproject.toml

@@ -33,6 +33,7 @@ requires-python = ">=3.9"
 dependencies = [
     'backoff>=2.0.0; python_version<"4"',
     'backports-datetime-fromisoformat>=2.0.1; python_version<"3.11"',
+    "cachetools>=5.5.2",
     "click~=8.0",
     "fsspec>=2024.9.0",
     'importlib-metadata>=5.0; python_version<"3.12"',

singer_sdk/helpers/_typing.py

@@ -5,13 +5,17 @@
 import copy
 import datetime
 import decimal
+import functools
+import json
 import logging
 import math
 import typing as t
 import uuid
 from enum import Enum
 from functools import lru_cache
 
+from cachetools import cached
+
 _MAX_TIMESTAMP = "9999-12-31 23:59:59.999999"
 _MAX_TIME = "23:59:59.999999"
 JSONSCHEMA_ANNOTATION_SECRET = "secret"  # noqa: S105
@@ -21,6 +25,11 @@
 logger = logging.getLogger(__name__)
 
 
+@functools.wraps
+def cached_schema():  # noqa: ANN202
+    return cached({}, key=lambda schema: json.dumps(schema, sort_keys=True))
+
+
 class DatetimeErrorTreatmentEnum(Enum):
     """Enum for treatment options for date parsing error."""
 
@@ -78,6 +87,7 @@
     return result
 
 
+@cached_schema
 def is_secret_type(type_dict: dict) -> bool:
     """Return True if JSON Schema type definition appears to be a secret.
 
@@ -108,6 +118,7 @@
     return False
 
 
+@cached_schema
 def is_object_type(property_schema: dict) -> bool | None:
     """Return true if the JSON Schema type is an object or None if detection fails."""
     if "anyOf" not in property_schema and "type" not in property_schema:
@@ -121,6 +132,7 @@
     )
 
 
+@cached_schema
 def is_uniform_list(property_schema: dict) -> bool | None:
     """Return true if the JSON Schema type is an array with a single schema.
 
@@ -135,6 +147,7 @@
     )
 
 
+@cached_schema
 def is_datetime_type(type_dict: dict) -> bool:
     """Return True if JSON Schema type definition is a 'date-time' type.
 
@@ -154,6 +167,7 @@
     raise ValueError(msg)
 
 
+@cached_schema
 def is_date_or_datetime_type(type_dict: dict) -> bool:
     """Return True if JSON Schema type definition is a 'date'/'date-time' type.
 
@@ -185,6 +199,7 @@
     raise ValueError(msg)
 
 
+@cached_schema
 def get_datelike_property_type(property_schema: dict) -> str | None:
     """Return one of 'date-time', 'time', or 'date' if property is date-like.
 
@@ -199,6 +214,7 @@
     return None
 
 
+@cached_schema
 def _is_string_with_format(type_dict: dict[str, t.Any]) -> bool | None:
     if "string" in type_dict.get("type", []) and type_dict.get("format") in {
         "date-time",
@@ -235,6 +251,7 @@
     raise ValueError(msg)
 
 
+@cached_schema
 def is_string_array_type(type_dict: dict) -> bool:
     """Return True if JSON Schema type definition is a string array."""
     if not type_dict:
@@ -253,6 +270,7 @@
     return "array" in type_dict["type"] and bool(is_string_type(type_dict["items"]))
 
 
+@cached_schema
 def is_array_type(type_dict: dict) -> bool:
     """Return True if JSON Schema type is an array."""
     if not type_dict:
@@ -271,6 +289,7 @@
     return "array" in type_dict["type"]
 
 
+@cached_schema
 def is_boolean_type(property_schema: dict) -> bool | None:
     """Return true if the JSON Schema type is a boolean or None if detection fails."""
     if "anyOf" not in property_schema and "type" not in property_schema:
@@ -286,6 +305,7 @@
     return False
 
 
+@cached_schema
 def _is_exclusive_boolean_type(property_schema: dict) -> bool:
     if "type" not in property_schema:
         return False
@@ -297,6 +317,7 @@
     )
 
 
+@cached_schema
 def is_integer_type(property_schema: dict) -> bool | None:
     """Return true if the JSON Schema type is an integer or None if detection fails."""
     if "anyOf" not in property_schema and "type" not in property_schema:
@@ -312,6 +333,7 @@
     return False
 
 
+@cached_schema
 def is_string_type(property_schema: dict) -> bool | None:
     """Return true if the JSON Schema type is a string or None if detection fails."""
     if "anyOf" not in property_schema and "type" not in property_schema:
@@ -327,6 +349,7 @@
     return False
 
 
+@cached_schema
 def is_null_type(property_schema: dict) -> bool | None:
     """Return true if the JSON Schema type is a null or None if detection fails."""
     if "anyOf" not in property_schema and "type" not in property_schema:
@@ -342,6 +365,7 @@
     return False
 
 
+@cached_schema
 def is_number_type(property_schema: dict) -> bool | None:
     """Return true if the JSON Schema type is a number or None if detection fails."""
     if "anyOf" not in property_schema and "type" not in property_schema:
@@ -542,9 +566,32 @@
         # for BIT value, treat 0 as False and anything else as True
         return elem != b"\x00" if is_boolean_type(property_schema) else elem.hex()
     if isinstance(elem, (float, decimal.Decimal)):
-        if math.isnan(elem) or math.isinf(elem):
-            return None
-        return elem
+        return elem if math.isfinite(elem) else None
+    if isinstance(elem, str) and not is_string_type(property_schema):
+        return _transform_string_property(elem, property_schema)
     if _is_exclusive_boolean_type(property_schema):
         return None if elem is None else elem != 0
     return elem
+
+
+def _transform_string_property(  # noqa: PLR0911
+    elem: str,
+    property_schema: dict,
+) -> t.Any:  # noqa: ANN401
+    if not elem and is_null_type(property_schema):
+        return None  # if nullable, None for empty string
+
+    if is_boolean_type(property_schema):
+        return (
+            elem.lower() == "true"
+        )  # false for any non-"true" string (case-insensitive), including empty string
+    if is_integer_type(property_schema):
+        return int(elem or 0)  # 0 for empty string
+    if is_number_type(property_schema):
+        d = decimal.Decimal(elem or 0)  # 0 for empty string
+        return d if d.is_finite() else None
+    if is_array_type(property_schema):
+        return json.loads(elem) if elem else []  # empty array for empty string
+    if is_object_type(property_schema):
+        return json.loads(elem) if elem else {}  # empty object for empty string
+    return elem

tests/core/test_typing.py

@@ -14,6 +14,7 @@
 from singer_sdk.helpers._typing import (
     TypeConformanceLevel,
     _conform_primitive_property,
+    _transform_string_property,
     conform_record_data_types,
 )
 from singer_sdk.typing import (
@@ -357,12 +358,68 @@ def test_conform_object_additional_properties():
         pytest.param(
             decimal.Decimal("nan"), {"type": "number"}, None, id="decimal_nan_to_number"
         ),
+        pytest.param("", {"type": "string"}, "", id="string_empty_to_string"),
+        pytest.param(
+            "",
+            {"type": ["boolean", "null"]},
+            None,
+            id="string_empty_to_any_nullable_non_string",
+        ),
+        pytest.param("true", {"type": "boolean"}, True, id="string_true_to_boolean"),
+        pytest.param(
+            "TRUE",
+            {"type": "boolean"},
+            True,
+            id="string_true_uppercase_to_boolean",
+        ),
+        pytest.param("false", {"type": "boolean"}, False, id="string_false_to_boolean"),
+        pytest.param(
+            "something else",
+            {"type": "boolean"},
+            False,
+            id="string_not_true_to_boolean",
+        ),
+        pytest.param("", {"type": "boolean"}, False, id="string_empty_to_boolean"),
+        pytest.param("3", {"type": "integer"}, 3, id="string_integer_to_integer"),
+        pytest.param("", {"type": "integer"}, 0, id="string_empty_to_integer"),
+        pytest.param(
+            "3.14",
+            {"type": "number"},
+            decimal.Decimal("3.14"),
+            id="string_float_to_number",
+        ),
+        pytest.param("inf", {"type": "number"}, None, id="string_inf_to_number"),
+        pytest.param("nan", {"type": "number"}, None, id="string_nan_to_number"),
+        pytest.param(
+            "",
+            {"type": "number"},
+            decimal.Decimal(0),
+            id="string_empty_to_number",
+        ),
+        pytest.param(
+            "[1, 2, 3]",
+            {"type": "array"},
+            [1, 2, 3],
+            id="string_json_array_to_array",
+        ),
+        pytest.param("", {"type": "array"}, [], id="string_empty_to_array"),
+        pytest.param(
+            '{"a": 1, "b": true, "c": 3.14}',
+            {"type": "object"},
+            {"a": 1, "b": True, "c": 3.14},
+            id="string_json_object_to_object",
+        ),
+        pytest.param("", {"type": "object"}, {}, id="string_empty_to_object"),
     ],
 )
 def test_conform_primitives(value: t.Any, type_dict: dict, expected: t.Any):
     assert _conform_primitive_property(value, type_dict) == expected
 
 
+def test_transform_string_to_string():
+    assert _transform_string_property("test", {"type": "string"}) == "test"
+
+
 @pytest.mark.filterwarnings("ignore:Use `JSONSchemaToSQL` instead.:DeprecationWarning")
 @pytest.mark.parametrize(
     "jsonschema_type,expected",