Merge pull request #222 from bmsuisse/updates

fixes byte and short

Author: aersam

bmsdna/lakeapi/context/df_base.py

@@ -255,7 +255,6 @@ def get_pyarrow_dataset(
                 import pyarrow.dataset as ds
 
                 return ds.dataset(files, filesystem=spec_fs, format="parquet")  # type: ignore
-
             case _:
                 raise FileTypeNotSupportedError(
                     f"Not supported file type {file_type}",

bmsdna/lakeapi/core/datasource.py

@@ -64,6 +64,8 @@ def to_pyarrow_schema(schema: DataType) -> pa.DataType:
         type_map: dict[PrimitiveType, pa.DataType] = {
             "integer": pa.int64(),
             "long": pa.int64(),
+            "short": pa.int16(),
+            "byte": pa.int8(),
             "string": pa.string(),
             "float": pa.float32(),
             "double": pa.float64(),

bmsdna/lakeapi/core/types.py

@@ -74,10 +74,12 @@
     "struct": dict,
     "string": str,
     "integer": int,
+    "short": int,
+    "byte": int,
     "float": float,
     "array": list,
     "double": Decimal,
-    "timesamp": datetime,
+    "timestamp": datetime,
     "date": date,
     "time": time,
     "duration": timedelta,

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "bmsdna-lakeapi"
-version = "0.24.2"
+version = "0.24.3"
 description = ""
 authors = [{ name = "DWH Team", email = "you@example.com" }]
 dependencies = [
@@ -19,7 +19,7 @@ dependencies = [
     "pandas >=2.1.0,<3",
     "fsspec >=2024.2.0,<2025",
     "adlfs >=2024.2.0,<2025",
-    "deltalake2db >=1.0.5",
+    "deltalake2db >=1.0.7",
 ]
 requires-python = "~=3.10"