diff --git a/build-logic/src/main/kotlin/dfbuild.buildConfig.gradle.kts b/build-logic/src/main/kotlin/dfbuild.buildConfig.gradle.kts index b30578afab..40a80480cb 100644 --- a/build-logic/src/main/kotlin/dfbuild.buildConfig.gradle.kts +++ b/build-logic/src/main/kotlin/dfbuild.buildConfig.gradle.kts @@ -9,7 +9,7 @@ plugins { } buildConfig { - packageName = "org.jetbrains.kotlinx.dataframe" + packageName = "org.jetbrains.kotlinx.dataframe.`${project.name}`" className = "BuildConfig" buildConfigField("KOTLIN_VERSION", libs.versions.kotlin.asProvider().get()) buildConfigField("KOTLIN_COMPILER_VERSION", kotlin.compilerVersion.get()) diff --git a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/columns/DataColumnImpl.kt b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/columns/DataColumnImpl.kt index b511009259..250955ccae 100644 --- a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/columns/DataColumnImpl.kt +++ b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/columns/DataColumnImpl.kt @@ -1,9 +1,9 @@ package org.jetbrains.kotlinx.dataframe.impl.columns -import org.jetbrains.kotlinx.dataframe.BuildConfig import org.jetbrains.kotlinx.dataframe.DataColumn import org.jetbrains.kotlinx.dataframe.annotations.RequiredByIntellijPlugin import org.jetbrains.kotlinx.dataframe.api.dataFrameOf +import org.jetbrains.kotlinx.dataframe.core.BuildConfig import org.jetbrains.kotlinx.dataframe.impl.isArray import org.jetbrains.kotlinx.dataframe.impl.isPrimitiveArray import org.jetbrains.kotlinx.dataframe.kind diff --git a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/columns/FrameColumnImpl.kt b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/columns/FrameColumnImpl.kt index 930ab72fbf..70336a40dd 100644 --- a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/columns/FrameColumnImpl.kt +++ b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/columns/FrameColumnImpl.kt @@ -1,13 +1,13 @@ package org.jetbrains.kotlinx.dataframe.impl.columns import org.jetbrains.kotlinx.dataframe.AnyRow -import org.jetbrains.kotlinx.dataframe.BuildConfig import org.jetbrains.kotlinx.dataframe.DataColumn import org.jetbrains.kotlinx.dataframe.DataFrame import org.jetbrains.kotlinx.dataframe.api.schema import org.jetbrains.kotlinx.dataframe.columns.ColumnGroup import org.jetbrains.kotlinx.dataframe.columns.ColumnResolutionContext import org.jetbrains.kotlinx.dataframe.columns.FrameColumn +import org.jetbrains.kotlinx.dataframe.core.BuildConfig import org.jetbrains.kotlinx.dataframe.impl.anyNull import org.jetbrains.kotlinx.dataframe.impl.createStarProjectedType import org.jetbrains.kotlinx.dataframe.impl.schema.intersectSchemas diff --git a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/JupyterConfiguration.kt b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/JupyterConfiguration.kt index 8859cbef4e..92e6846ed3 100644 --- a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/JupyterConfiguration.kt +++ b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/JupyterConfiguration.kt @@ -1,6 +1,6 @@ package org.jetbrains.kotlinx.dataframe.jupyter -import org.jetbrains.kotlinx.dataframe.BuildConfig +import org.jetbrains.kotlinx.dataframe.core.BuildConfig import org.jetbrains.kotlinx.dataframe.io.DisplayConfiguration public class JupyterConfiguration( diff --git a/core/src/test/kotlin/org/jetbrains/kotlinx/dataframe/columns/DataColumns.kt b/core/src/test/kotlin/org/jetbrains/kotlinx/dataframe/columns/DataColumns.kt index 25fc0de21c..b857606369 100644 --- a/core/src/test/kotlin/org/jetbrains/kotlinx/dataframe/columns/DataColumns.kt +++ b/core/src/test/kotlin/org/jetbrains/kotlinx/dataframe/columns/DataColumns.kt @@ -3,11 +3,11 @@ package org.jetbrains.kotlinx.dataframe.columns import io.kotest.assertions.throwables.shouldThrow import io.kotest.matchers.shouldBe import org.jetbrains.kotlinx.dataframe.AnyFrame -import org.jetbrains.kotlinx.dataframe.BuildConfig import org.jetbrains.kotlinx.dataframe.DataColumn import org.jetbrains.kotlinx.dataframe.api.dataFrameOf import org.jetbrains.kotlinx.dataframe.api.toColumn import org.jetbrains.kotlinx.dataframe.api.toDataFrame +import org.jetbrains.kotlinx.dataframe.core.BuildConfig import org.junit.Test import java.net.URI diff --git a/dataframe-jdbc/api/dataframe-jdbc.api b/dataframe-jdbc/api/dataframe-jdbc.api index bb142cba42..92cb14b833 100644 --- a/dataframe-jdbc/api/dataframe-jdbc.api +++ b/dataframe-jdbc/api/dataframe-jdbc.api @@ -94,34 +94,59 @@ public final class org/jetbrains/kotlinx/dataframe/io/ReadJdbcKt { public static synthetic fun readSqlTable$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Ljava/lang/String;Ljava/lang/Integer;ZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;ZLkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataFrame; } +public abstract class org/jetbrains/kotlinx/dataframe/io/db/AdvancedDbType : org/jetbrains/kotlinx/dataframe/io/db/DbType { + public fun (Ljava/lang/String;)V + public final fun buildDataColumn (Ljava/lang/String;Ljava/util/List;Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;Lorg/jetbrains/kotlinx/dataframe/schema/ColumnSchema;Z)Lorg/jetbrains/kotlinx/dataframe/DataColumn; + protected abstract fun generateConverter (Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;)Lorg/jetbrains/kotlinx/dataframe/io/db/JdbcToDataFrameConverter; + protected final fun getConverter (Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;)Lorg/jetbrains/kotlinx/dataframe/io/db/JdbcToDataFrameConverter; + public final fun getExpectedJdbcType (Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;)Lkotlin/reflect/KType; + public final fun getPreprocessedValueType (Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;Lkotlin/reflect/KType;)Lkotlin/reflect/KType; + public final fun getTargetColumnSchema (Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;Lkotlin/reflect/KType;)Lorg/jetbrains/kotlinx/dataframe/schema/ColumnSchema; + public final fun getValueFromResultSet (Ljava/sql/ResultSet;ILorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;Lkotlin/reflect/KType;)Ljava/lang/Object; + public final fun preprocessValue (Ljava/lang/Object;Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;Lkotlin/reflect/KType;Lkotlin/reflect/KType;)Ljava/lang/Object; +} + +public abstract interface class org/jetbrains/kotlinx/dataframe/io/db/DbColumnBuilder { + public abstract fun buildDataColumn (Ljava/lang/String;Ljava/util/List;Z)Lorg/jetbrains/kotlinx/dataframe/DataColumn; +} + +public abstract interface class org/jetbrains/kotlinx/dataframe/io/db/DbResultSetReader { + public abstract fun getValue (Ljava/sql/ResultSet;I)Ljava/lang/Object; +} + public abstract class org/jetbrains/kotlinx/dataframe/io/db/DbType { public fun (Ljava/lang/String;)V - public fun buildDataColumn (Ljava/lang/String;Ljava/util/List;Lkotlin/reflect/KType;Z)Lorg/jetbrains/kotlinx/dataframe/DataColumn; + public fun buildDataColumn (Ljava/lang/String;Ljava/util/List;Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;Lorg/jetbrains/kotlinx/dataframe/schema/ColumnSchema;Z)Lorg/jetbrains/kotlinx/dataframe/DataColumn; public fun buildSelectTableQueryWithLimit (Ljava/lang/String;Ljava/lang/Integer;)Ljava/lang/String; public fun buildSqlQueryWithLimit (Ljava/lang/String;I)Ljava/lang/String; public static synthetic fun buildSqlQueryWithLimit$default (Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;Ljava/lang/String;IILjava/lang/Object;)Ljava/lang/String; public abstract fun buildTableMetadata (Ljava/sql/ResultSet;)Lorg/jetbrains/kotlinx/dataframe/io/db/TableMetadata; public fun configureReadStatement (Ljava/sql/PreparedStatement;)V - public abstract fun convertSqlTypeToColumnSchemaValue (Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;)Lorg/jetbrains/kotlinx/dataframe/schema/ColumnSchema; - public abstract fun convertSqlTypeToKType (Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;)Lkotlin/reflect/KType; public fun createConnection (Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;)Ljava/sql/Connection; - public fun extractValueFromResultSet (Ljava/sql/ResultSet;ILorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;Lkotlin/reflect/KType;)Ljava/lang/Object; public final fun getDbTypeInJdbcUrl ()Ljava/lang/String; public fun getDefaultFetchSize ()I + protected final fun getDefaultJdbcTypeToKTypeMapping ()Ljava/util/Map; public fun getDefaultQueryTimeout ()Ljava/lang/Integer; public abstract fun getDriverClassName ()Ljava/lang/String; + public fun getExpectedJdbcType (Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;)Lkotlin/reflect/KType; + public fun getPreprocessedValueType (Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;Lkotlin/reflect/KType;)Lkotlin/reflect/KType; public fun getTableColumnsMetadata (Ljava/sql/ResultSet;)Ljava/util/List; public fun getTableTypes ()Ljava/util/List; + public fun getTargetColumnSchema (Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;Lkotlin/reflect/KType;)Lorg/jetbrains/kotlinx/dataframe/schema/ColumnSchema; + public fun getValueFromResultSet (Ljava/sql/ResultSet;ILorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;Lkotlin/reflect/KType;)Ljava/lang/Object; public abstract fun isSystemTable (Lorg/jetbrains/kotlinx/dataframe/io/db/TableMetadata;)Z - public fun makeCommonSqlToKTypeMapping (Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;)Lkotlin/reflect/KType; + public fun preprocessValue (Ljava/lang/Object;Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;Lkotlin/reflect/KType;Lkotlin/reflect/KType;)Ljava/lang/Object; public fun quoteIdentifier (Ljava/lang/String;)Ljava/lang/String; + protected final fun toDataColumn (Ljava/util/List;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/schema/ColumnSchema;Z)Lorg/jetbrains/kotlinx/dataframe/DataColumn; } -public final class org/jetbrains/kotlinx/dataframe/io/db/DuckDb : org/jetbrains/kotlinx/dataframe/io/db/DbType { +public abstract interface class org/jetbrains/kotlinx/dataframe/io/db/DbValuePreprocessor { + public abstract fun preprocess (Ljava/lang/Object;)Ljava/lang/Object; +} + +public final class org/jetbrains/kotlinx/dataframe/io/db/DuckDb : org/jetbrains/kotlinx/dataframe/io/db/AdvancedDbType { public static final field INSTANCE Lorg/jetbrains/kotlinx/dataframe/io/db/DuckDb; public fun buildTableMetadata (Ljava/sql/ResultSet;)Lorg/jetbrains/kotlinx/dataframe/io/db/TableMetadata; - public fun convertSqlTypeToColumnSchemaValue (Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;)Lorg/jetbrains/kotlinx/dataframe/schema/ColumnSchema; - public fun convertSqlTypeToKType (Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;)Lkotlin/reflect/KType; public fun createConnection (Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;)Ljava/sql/Connection; public fun getDriverClassName ()Ljava/lang/String; public fun isSystemTable (Lorg/jetbrains/kotlinx/dataframe/io/db/TableMetadata;)Z @@ -139,9 +164,8 @@ public class org/jetbrains/kotlinx/dataframe/io/db/H2 : org/jetbrains/kotlinx/da public synthetic fun (Lorg/jetbrains/kotlinx/dataframe/io/db/H2$Mode;ILkotlin/jvm/internal/DefaultConstructorMarker;)V public fun buildSqlQueryWithLimit (Ljava/lang/String;I)Ljava/lang/String; public fun buildTableMetadata (Ljava/sql/ResultSet;)Lorg/jetbrains/kotlinx/dataframe/io/db/TableMetadata; - public fun convertSqlTypeToColumnSchemaValue (Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;)Lorg/jetbrains/kotlinx/dataframe/schema/ColumnSchema; - public fun convertSqlTypeToKType (Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;)Lkotlin/reflect/KType; public fun getDriverClassName ()Ljava/lang/String; + public fun getExpectedJdbcType (Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;)Lkotlin/reflect/KType; public final fun getMode ()Lorg/jetbrains/kotlinx/dataframe/io/db/H2$Mode; public fun isSystemTable (Lorg/jetbrains/kotlinx/dataframe/io/db/TableMetadata;)Z } @@ -168,12 +192,42 @@ public final class org/jetbrains/kotlinx/dataframe/io/db/H2$Mode$Companion { public final fun fromValue (Ljava/lang/String;)Lorg/jetbrains/kotlinx/dataframe/io/db/H2$Mode; } +public final class org/jetbrains/kotlinx/dataframe/io/db/JdbcToDataFrameConverter { + public fun (Lkotlin/reflect/KType;Lorg/jetbrains/kotlinx/dataframe/io/db/DbResultSetReader;Lkotlin/reflect/KType;Lorg/jetbrains/kotlinx/dataframe/io/db/DbValuePreprocessor;Lorg/jetbrains/kotlinx/dataframe/schema/ColumnSchema;Lorg/jetbrains/kotlinx/dataframe/io/db/DbColumnBuilder;)V + public final fun buildDataColumnOrNull (Ljava/lang/String;Ljava/util/List;Z)Lorg/jetbrains/kotlinx/dataframe/DataColumn; + public final fun copy (Lkotlin/reflect/KType;Lorg/jetbrains/kotlinx/dataframe/io/db/DbResultSetReader;Lkotlin/reflect/KType;Lorg/jetbrains/kotlinx/dataframe/io/db/DbValuePreprocessor;Lorg/jetbrains/kotlinx/dataframe/schema/ColumnSchema;Lorg/jetbrains/kotlinx/dataframe/io/db/DbColumnBuilder;)Lorg/jetbrains/kotlinx/dataframe/io/db/JdbcToDataFrameConverter; + public static synthetic fun copy$default (Lorg/jetbrains/kotlinx/dataframe/io/db/JdbcToDataFrameConverter;Lkotlin/reflect/KType;Lorg/jetbrains/kotlinx/dataframe/io/db/DbResultSetReader;Lkotlin/reflect/KType;Lorg/jetbrains/kotlinx/dataframe/io/db/DbValuePreprocessor;Lorg/jetbrains/kotlinx/dataframe/schema/ColumnSchema;Lorg/jetbrains/kotlinx/dataframe/io/db/DbColumnBuilder;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/io/db/JdbcToDataFrameConverter; + public final fun getColumnBuilder ()Lorg/jetbrains/kotlinx/dataframe/io/db/DbColumnBuilder; + public final fun getExpectedJdbcType ()Lkotlin/reflect/KType; + public final fun getPreprocessedValueType ()Lkotlin/reflect/KType; + public final fun getResultSetReader ()Lorg/jetbrains/kotlinx/dataframe/io/db/DbResultSetReader; + public final fun getTargetSchema ()Lorg/jetbrains/kotlinx/dataframe/schema/ColumnSchema; + public final fun getValueFromResultSetOrElse (Ljava/sql/ResultSet;ILkotlin/jvm/functions/Function0;)Ljava/lang/Object; + public final fun getValuePreprocessor ()Lorg/jetbrains/kotlinx/dataframe/io/db/DbValuePreprocessor; + public final fun preprocessOrCast (Ljava/lang/Object;)Ljava/lang/Object; +} + +public final class org/jetbrains/kotlinx/dataframe/io/db/JdbcToDataFrameConverterKt { + public static final fun cast (Lorg/jetbrains/kotlinx/dataframe/io/db/DbColumnBuilder;)Lorg/jetbrains/kotlinx/dataframe/io/db/DbColumnBuilder; + public static final fun cast (Lorg/jetbrains/kotlinx/dataframe/io/db/DbValuePreprocessor;)Lorg/jetbrains/kotlinx/dataframe/io/db/DbValuePreprocessor; + public static final fun cast (Lorg/jetbrains/kotlinx/dataframe/io/db/JdbcToDataFrameConverter;)Lorg/jetbrains/kotlinx/dataframe/io/db/JdbcToDataFrameConverter; + public static final fun castToAny (Lorg/jetbrains/kotlinx/dataframe/io/db/DbColumnBuilder;)Lorg/jetbrains/kotlinx/dataframe/io/db/DbColumnBuilder; + public static final fun castToAny (Lorg/jetbrains/kotlinx/dataframe/io/db/DbValuePreprocessor;)Lorg/jetbrains/kotlinx/dataframe/io/db/DbValuePreprocessor; + public static final fun castToAny (Lorg/jetbrains/kotlinx/dataframe/io/db/JdbcToDataFrameConverter;)Lorg/jetbrains/kotlinx/dataframe/io/db/JdbcToDataFrameConverter; + public static final fun jdbcToDfConverterFor (Lkotlin/reflect/KType;)Lorg/jetbrains/kotlinx/dataframe/io/db/JdbcToDataFrameConverter; + public static final fun withColumnBuilder (Lorg/jetbrains/kotlinx/dataframe/io/db/JdbcToDataFrameConverter;Lorg/jetbrains/kotlinx/dataframe/schema/ColumnSchema;ZLorg/jetbrains/kotlinx/dataframe/io/db/DbColumnBuilder;)Lorg/jetbrains/kotlinx/dataframe/io/db/JdbcToDataFrameConverter; + public static synthetic fun withColumnBuilder$default (Lorg/jetbrains/kotlinx/dataframe/io/db/JdbcToDataFrameConverter;Lorg/jetbrains/kotlinx/dataframe/schema/ColumnSchema;ZLorg/jetbrains/kotlinx/dataframe/io/db/DbColumnBuilder;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/io/db/JdbcToDataFrameConverter; + public static final fun withPreprocessor (Lorg/jetbrains/kotlinx/dataframe/io/db/JdbcToDataFrameConverter;Lkotlin/reflect/KType;ZLorg/jetbrains/kotlinx/dataframe/io/db/DbValuePreprocessor;)Lorg/jetbrains/kotlinx/dataframe/io/db/JdbcToDataFrameConverter; + public static synthetic fun withPreprocessor$default (Lorg/jetbrains/kotlinx/dataframe/io/db/JdbcToDataFrameConverter;Lkotlin/reflect/KType;ZLorg/jetbrains/kotlinx/dataframe/io/db/DbValuePreprocessor;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/io/db/JdbcToDataFrameConverter; + public static final fun withResultReader (Lorg/jetbrains/kotlinx/dataframe/io/db/JdbcToDataFrameConverter;Lorg/jetbrains/kotlinx/dataframe/io/db/DbResultSetReader;)Lorg/jetbrains/kotlinx/dataframe/io/db/JdbcToDataFrameConverter; + public static final fun withTargetSchema (Lorg/jetbrains/kotlinx/dataframe/io/db/JdbcToDataFrameConverter;Lorg/jetbrains/kotlinx/dataframe/schema/ColumnSchema;)Lorg/jetbrains/kotlinx/dataframe/io/db/JdbcToDataFrameConverter; +} + public final class org/jetbrains/kotlinx/dataframe/io/db/MariaDb : org/jetbrains/kotlinx/dataframe/io/db/DbType { public static final field INSTANCE Lorg/jetbrains/kotlinx/dataframe/io/db/MariaDb; public fun buildTableMetadata (Ljava/sql/ResultSet;)Lorg/jetbrains/kotlinx/dataframe/io/db/TableMetadata; - public fun convertSqlTypeToColumnSchemaValue (Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;)Lorg/jetbrains/kotlinx/dataframe/schema/ColumnSchema; - public fun convertSqlTypeToKType (Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;)Lkotlin/reflect/KType; public fun getDriverClassName ()Ljava/lang/String; + public fun getExpectedJdbcType (Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;)Lkotlin/reflect/KType; public fun isSystemTable (Lorg/jetbrains/kotlinx/dataframe/io/db/TableMetadata;)Z public fun quoteIdentifier (Ljava/lang/String;)Ljava/lang/String; } @@ -182,8 +236,6 @@ public final class org/jetbrains/kotlinx/dataframe/io/db/MsSql : org/jetbrains/k public static final field INSTANCE Lorg/jetbrains/kotlinx/dataframe/io/db/MsSql; public fun buildSqlQueryWithLimit (Ljava/lang/String;I)Ljava/lang/String; public fun buildTableMetadata (Ljava/sql/ResultSet;)Lorg/jetbrains/kotlinx/dataframe/io/db/TableMetadata; - public fun convertSqlTypeToColumnSchemaValue (Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;)Lorg/jetbrains/kotlinx/dataframe/schema/ColumnSchema; - public fun convertSqlTypeToKType (Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;)Lkotlin/reflect/KType; public fun getDriverClassName ()Ljava/lang/String; public fun isSystemTable (Lorg/jetbrains/kotlinx/dataframe/io/db/TableMetadata;)Z public fun quoteIdentifier (Ljava/lang/String;)Ljava/lang/String; @@ -192,9 +244,8 @@ public final class org/jetbrains/kotlinx/dataframe/io/db/MsSql : org/jetbrains/k public final class org/jetbrains/kotlinx/dataframe/io/db/MySql : org/jetbrains/kotlinx/dataframe/io/db/DbType { public static final field INSTANCE Lorg/jetbrains/kotlinx/dataframe/io/db/MySql; public fun buildTableMetadata (Ljava/sql/ResultSet;)Lorg/jetbrains/kotlinx/dataframe/io/db/TableMetadata; - public fun convertSqlTypeToColumnSchemaValue (Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;)Lorg/jetbrains/kotlinx/dataframe/schema/ColumnSchema; - public fun convertSqlTypeToKType (Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;)Lkotlin/reflect/KType; public fun getDriverClassName ()Ljava/lang/String; + public fun getExpectedJdbcType (Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;)Lkotlin/reflect/KType; public fun isSystemTable (Lorg/jetbrains/kotlinx/dataframe/io/db/TableMetadata;)Z public fun quoteIdentifier (Ljava/lang/String;)Ljava/lang/String; } @@ -202,9 +253,9 @@ public final class org/jetbrains/kotlinx/dataframe/io/db/MySql : org/jetbrains/k public final class org/jetbrains/kotlinx/dataframe/io/db/PostgreSql : org/jetbrains/kotlinx/dataframe/io/db/DbType { public static final field INSTANCE Lorg/jetbrains/kotlinx/dataframe/io/db/PostgreSql; public fun buildTableMetadata (Ljava/sql/ResultSet;)Lorg/jetbrains/kotlinx/dataframe/io/db/TableMetadata; - public fun convertSqlTypeToColumnSchemaValue (Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;)Lorg/jetbrains/kotlinx/dataframe/schema/ColumnSchema; - public fun convertSqlTypeToKType (Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;)Lkotlin/reflect/KType; public fun getDriverClassName ()Ljava/lang/String; + public fun getExpectedJdbcType (Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;)Lkotlin/reflect/KType; + public fun getValueFromResultSet (Ljava/sql/ResultSet;ILorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;Lkotlin/reflect/KType;)Ljava/lang/Object; public fun isSystemTable (Lorg/jetbrains/kotlinx/dataframe/io/db/TableMetadata;)Z public fun quoteIdentifier (Ljava/lang/String;)Ljava/lang/String; } @@ -212,8 +263,6 @@ public final class org/jetbrains/kotlinx/dataframe/io/db/PostgreSql : org/jetbra public final class org/jetbrains/kotlinx/dataframe/io/db/Sqlite : org/jetbrains/kotlinx/dataframe/io/db/DbType { public static final field INSTANCE Lorg/jetbrains/kotlinx/dataframe/io/db/Sqlite; public fun buildTableMetadata (Ljava/sql/ResultSet;)Lorg/jetbrains/kotlinx/dataframe/io/db/TableMetadata; - public fun convertSqlTypeToColumnSchemaValue (Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;)Lorg/jetbrains/kotlinx/dataframe/schema/ColumnSchema; - public fun convertSqlTypeToKType (Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;)Lkotlin/reflect/KType; public fun createConnection (Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;)Ljava/sql/Connection; public fun getDriverClassName ()Ljava/lang/String; public fun isSystemTable (Lorg/jetbrains/kotlinx/dataframe/io/db/TableMetadata;)Z diff --git a/dataframe-jdbc/build.gradle.kts b/dataframe-jdbc/build.gradle.kts index 9e96d2a9ef..b6db360066 100644 --- a/dataframe-jdbc/build.gradle.kts +++ b/dataframe-jdbc/build.gradle.kts @@ -1,6 +1,7 @@ plugins { with(convention.plugins) { alias(kotlinJvm8) + alias(buildConfig) } with(libs.plugins) { alias(publisher) @@ -14,6 +15,7 @@ dependencies { api(projects.core) compileOnly(libs.duckdb.jdbc) compileOnly(libs.sqlite) + compileOnly(libs.postgresql) implementation(libs.kotlinLogging) testImplementation(libs.mariadb) testImplementation(libs.sqlite) diff --git a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/AdvancedDbType.kt b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/AdvancedDbType.kt new file mode 100644 index 0000000000..30616a2801 --- /dev/null +++ b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/AdvancedDbType.kt @@ -0,0 +1,93 @@ +package org.jetbrains.kotlinx.dataframe.io.db + +import org.jetbrains.kotlinx.dataframe.DataColumn +import org.jetbrains.kotlinx.dataframe.schema.ColumnSchema +import java.sql.ResultSet +import kotlin.reflect.KType + +/** + * Alternative version of [DbType] that allows to customize type mapping + * by initializing a [JdbcToDataFrameConverter] instance for each JDBC type. + * + * This can be helpful for JDBC databases that support structured data, like [DuckDb] + * or that need to a lot of type mapping. + * + * This API is experimental and subject to change. + */ +public abstract class AdvancedDbType(dbTypeInJdbcUrl: String) : DbType(dbTypeInJdbcUrl) { + + protected abstract fun generateConverter(tableColumnMetadata: TableColumnMetadata): AnyJdbcToDataFrameConverter + + private data class CacheKey( + val sqlTypeName: String, + val jdbcType: Int, + val javaClassName: String, + val isNullable: Boolean, + ) + + private fun TableColumnMetadata.cacheKey(): CacheKey = CacheKey(sqlTypeName, jdbcType, javaClassName, isNullable) + + private val converterCache = mutableMapOf() + + protected fun getConverter(tableColumnMetadata: TableColumnMetadata): AnyJdbcToDataFrameConverter = + converterCache.getOrPut(tableColumnMetadata.cacheKey()) { + generateConverter(tableColumnMetadata) + } + + final override fun getExpectedJdbcType(tableColumnMetadata: TableColumnMetadata): KType = + getConverter(tableColumnMetadata).expectedJdbcType + + final override fun getPreprocessedValueType( + tableColumnMetadata: TableColumnMetadata, + expectedJdbcType: KType, + ): KType = getConverter(tableColumnMetadata).preprocessedValueType + + final override fun getTargetColumnSchema( + tableColumnMetadata: TableColumnMetadata, + expectedValueType: KType, + ): ColumnSchema? = getConverter(tableColumnMetadata).targetSchema + + @Suppress("UNCHECKED_CAST") + final override fun getValueFromResultSet( + rs: ResultSet, + columnIndex: Int, + tableColumnMetadata: TableColumnMetadata, + expectedJdbcType: KType, + ): J = + getConverter(tableColumnMetadata) + .cast() + .getValueFromResultSetOrElse(rs, columnIndex) { + try { + rs.getObject(columnIndex + 1) + } catch (_: Throwable) { + // TODO? + rs.getString(columnIndex + 1) + } as J + } + + final override fun preprocessValue( + value: J, + tableColumnMetadata: TableColumnMetadata, + expectedJdbcType: KType, + expectedPreprocessedValueType: KType, + ): D = + getConverter(tableColumnMetadata) + .cast() + .preprocessOrCast(value) + + final override fun buildDataColumn( + name: String, + values: List, + tableColumnMetadata: TableColumnMetadata, + targetColumnSchema: ColumnSchema?, + inferNullability: Boolean, + ): DataColumn

= + getConverter(tableColumnMetadata) + .cast() + .buildDataColumnOrNull(name, values, inferNullability) + ?: values.toDataColumn( + name = name, + targetColumnSchema = targetColumnSchema, + inferNullability = inferNullability, + ) +} diff --git a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/DbType.kt b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/DbType.kt index e4fb482d8f..8b8c84295d 100644 --- a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/DbType.kt +++ b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/DbType.kt @@ -1,7 +1,18 @@ package org.jetbrains.kotlinx.dataframe.io.db +import kotlinx.datetime.LocalDateTime +import kotlinx.datetime.toKotlinLocalDateTime +import org.jetbrains.kotlinx.dataframe.AnyFrame +import org.jetbrains.kotlinx.dataframe.AnyRow import org.jetbrains.kotlinx.dataframe.DataColumn +import org.jetbrains.kotlinx.dataframe.DataFrame +import org.jetbrains.kotlinx.dataframe.DataRow import org.jetbrains.kotlinx.dataframe.api.Infer +import org.jetbrains.kotlinx.dataframe.api.asDataColumn +import org.jetbrains.kotlinx.dataframe.api.cast +import org.jetbrains.kotlinx.dataframe.api.schema +import org.jetbrains.kotlinx.dataframe.api.toDataFrame +import org.jetbrains.kotlinx.dataframe.impl.ColumnNameGenerator import org.jetbrains.kotlinx.dataframe.io.DbConnectionConfig import org.jetbrains.kotlinx.dataframe.io.readAllSqlTables import org.jetbrains.kotlinx.dataframe.schema.ColumnSchema @@ -21,17 +32,20 @@ import java.sql.SQLXML import java.sql.Time import java.sql.Timestamp import java.sql.Types -import java.time.LocalDateTime import java.time.OffsetDateTime import java.time.OffsetTime -import java.util.Date import java.util.UUID import kotlin.reflect.KClass import kotlin.reflect.KType -import kotlin.reflect.full.createType -import kotlin.reflect.full.isSupertypeOf import kotlin.reflect.full.safeCast -import kotlin.reflect.full.starProjectedType +import kotlin.reflect.full.withNullability +import kotlin.reflect.typeOf +import kotlin.time.Instant +import kotlin.time.toKotlinInstant +import kotlin.uuid.Uuid +import kotlin.uuid.toKotlinUuid +import java.time.LocalDateTime as JavaLocalDateTime +import java.util.Date as JavaDate /** * The `DbType` class represents a database type used for reading dataframe from the database. @@ -39,6 +53,7 @@ import kotlin.reflect.full.starProjectedType * @property [dbTypeInJdbcUrl] The name of the database as specified in the JDBC URL. */ public abstract class DbType(public val dbTypeInJdbcUrl: String) { + /** * Represents the JDBC driver class name for a given database type. * @@ -82,10 +97,445 @@ public abstract class DbType(public val dbTypeInJdbcUrl: String) { */ public open val defaultQueryTimeout: Int? = null // null = no timeout + /** Default mapping of [Java SQL Types][Types] to [KType]. */ + protected val defaultJdbcTypeToKTypeMapping: Map = mapOf( + Types.BIT to typeOf(), + Types.TINYINT to typeOf(), + Types.SMALLINT to typeOf(), + Types.INTEGER to typeOf(), + Types.BIGINT to typeOf(), + Types.FLOAT to typeOf(), + Types.REAL to typeOf(), + Types.DOUBLE to typeOf(), + Types.NUMERIC to typeOf(), + Types.DECIMAL to typeOf(), + Types.CHAR to typeOf(), + Types.VARCHAR to typeOf(), + Types.LONGVARCHAR to typeOf(), + Types.DATE to typeOf(), + Types.TIME to typeOf

`. + * + * If `null` is returned, the [schema][ColumnSchema] cannot be determined before looking at the actual data. + * + * @param [tableColumnMetadata] all information we have about the column + * @param [expectedValueType] the type of the values after preprocessing, `D` + * @return the target [schema][ColumnSchema] of the given column, + * or `null` if it cannot be determined from the types alone. + */ + public open fun getTargetColumnSchema( + tableColumnMetadata: TableColumnMetadata, + expectedValueType: KType, + ): ColumnSchema? = + when (tableColumnMetadata.jdbcType) { + // buildDataColumn post-processes java.sql.Array -> Kotlin arrays, making the result type `Array<*>` + Types.ARRAY -> ColumnSchema.Value(typeOf>().withNullability(expectedValueType.isMarkedNullable)) + + else -> ColumnSchema.Value(expectedValueType) + } + + /** + * Builds a [DataColumn] from the given ([preprocessed][preprocessValue]) [values], + * adhering to [targetColumnSchema]. + * + * @param [D] the type of the values after preprocessing + * @param [P] the type of the resulting [DataColumn][DataColumn]`<`[P][P]`>`, [targetColumnSchema] + * @param [name] the name of the column + * @param [values] the ([preprocessed][preprocessValue]) values to put in the column + * @param [tableColumnMetadata] all information we have about the column + * @param [targetColumnSchema] the schema of the column [DataColumn][DataColumn]`<`[P][P]`>`, + * as determined by [getTargetColumnSchema] + * @param [inferNullability] whether to infer nullability from the runtime values (this is more expensive), + * as opposed to using the nullability information from the [targetColumnSchema] + * @return the built [DataColumn] + */ + public open fun buildDataColumn( + name: String, + values: List, + tableColumnMetadata: TableColumnMetadata, + targetColumnSchema: ColumnSchema?, + inferNullability: Boolean, + ): DataColumn

{ + val postProcessedValues = when (tableColumnMetadata.jdbcType) { + // Special case which post-processes java.sql.Array -> Kotlin arrays + Types.ARRAY -> handleArrayValues(values) + + else -> values + } + return postProcessedValues.toDataColumn( + name = name, + targetColumnSchema = targetColumnSchema, + inferNullability = inferNullability, + ) + } + + /** + * Helper function to convert [this] list of values to a [DataColumn][DataColumn]`<`[P][P]`>`. + * + * **NOTE:** While this function can handle + * [targetColumnSchema][targetColumnSchema]` = `[ColumnSchema.Group][ColumnSchema.Group], + * and [this] being a [List][List]`<`[`DataRow<*>`][DataRow]`>`, + * this should generally be avoided to circumvent creating `n` [data rows][DataRow], + * (which essentially are `n` single-row [dataframes][DataFrame]). + * + * Instead, use [preprocessValue][preprocessValue] to convert to [Map][Map]`<`[String][String]`, `[Any?][Any]`>` + * and then use the more efficient [Iterable>.toDataFrame()][Iterable.toDataFrame] in [buildDataColumn]: + * ```kt + * (values as List>) + * .toDataFrame() + * .asColumnGroup(name) + * .asDataColumn() + * ``` + */ + protected fun List.toDataColumn( + name: String, + targetColumnSchema: ColumnSchema?, + inferNullability: Boolean, + ): DataColumn

= + when (targetColumnSchema) { + is ColumnSchema.Value -> + DataColumn.createValueColumn( + name = name, + values = this, + infer = if (inferNullability) Infer.Nulls else Infer.None, + type = targetColumnSchema.type, + ).cast() + + // NOTE: this case should be avoided. + // Creating `n` DataRows is heavy! + is ColumnSchema.Group -> + DataColumn.createColumnGroup( + name = name, + df = (this as List).toDataFrame(), + ).asDataColumn().cast() + + is ColumnSchema.Frame -> + DataColumn.createFrameColumn( + name = name, + groups = this as List, + schema = lazy { targetColumnSchema.schema }, + ).cast() + + null -> + DataColumn.createByInference( + name = name, + values = this, + ).cast() + } /** * Checks if the given table name is a system table for the specified database type. @@ -103,14 +553,6 @@ public abstract class DbType(public val dbTypeInJdbcUrl: String) { */ public abstract fun buildTableMetadata(tables: ResultSet): TableMetadata - /** - * Converts SQL data type to a Kotlin data type. - * - * @param [tableColumnMetadata] The metadata of the table column. - * @return The corresponding Kotlin data type, or null if no mapping is found. - */ - public abstract fun convertSqlTypeToKType(tableColumnMetadata: TableColumnMetadata): KType? - /** * Builds a SELECT query for reading from a table. * @@ -197,78 +639,7 @@ public abstract class DbType(public val dbTypeInJdbcUrl: String) { } /** - * Extracts a value from the ResultSet for the given column. - * This method can be overridden by custom database types to provide specialized parsing logic. - * - * @param [rs] the ResultSet to read from - * @param [columnIndex] zero-based column index - * @param [columnMetadata] metadata for the column - * @param [kType] the Kotlin type for this column - * @return the extracted value, or null - */ - public open fun extractValueFromResultSet( - rs: ResultSet, - columnIndex: Int, - columnMetadata: TableColumnMetadata, - kType: KType, - ): Any? = - try { - rs.getObject(columnIndex + 1) - // TODO: add a special handler for Blob via Streams - } catch (_: Throwable) { - // TODO: expand for all the types like in generateKType function - if (kType.isSupertypeOf(String::class.starProjectedType)) { - rs.getString(columnIndex + 1) - } else { - rs.getString(columnIndex + 1) - } - } - - /** - * Builds a single DataColumn with proper type handling. - * Accepts a mutable list to allow efficient post-processing. - */ - public open fun buildDataColumn( - name: String, - values: MutableList, - kType: KType, - inferNullability: Boolean, - ): DataColumn<*> { - val correctedValues = postProcessColumnValues(values, kType) - - return DataColumn.createValueColumn( - name = name, - values = correctedValues, - infer = convertNullabilityInference(inferNullability), - type = kType, - ) - } - - private fun convertNullabilityInference(inferNullability: Boolean) = - if (inferNullability) Infer.Nulls else Infer.None - - /** - * Processes the column values retrieved from the database and performs transformations based on the provided - * Kotlin type and column metadata. The method allows for custom post-processing logic, such as handling - * specific database column types, including arrays. - * - * @param values the list of raw values retrieved from the database for the column. - * @param kType the Kotlin type that the column values should be transformed to. - * @return a list of processed column values, with transformations applied where necessary, or the original list if no transformation is needed. - */ - private fun postProcessColumnValues(values: MutableList, kType: KType): List = - when { - /* EXAMPLE: columnMetadata.sqlTypeName == "MY_CUSTOM_ARRAY" -> { - values.map { /* custom transformation */ } - } */ - kType.classifier == Array::class -> { - handleArrayValues(values) - } - - else -> values - } - - /** + * todo? * Converts SQL Array objects to strongly-typed arrays. * * Extracts arrays from SQL Array objects and converts them to a consistent type @@ -277,7 +648,7 @@ public abstract class DbType(public val dbTypeInJdbcUrl: String) { * @param values raw values containing SQL Array objects * @return list of consistently typed arrays, or original arrays if no common type exists */ - private fun handleArrayValues(values: MutableList): List { + private fun handleArrayValues(values: List): List?> { // Intermediate variable for the first mapping val sqlArrays = values.mapNotNull { (it as? java.sql.Array)?.array?.let { array -> array as? Array<*> } @@ -305,257 +676,7 @@ public abstract class DbType(public val dbTypeInJdbcUrl: String) { /** Utility function to cast arrays based on the type of elements */ private fun castArray(array: Array<*>, elementType: KClass): List = array.mapNotNull { elementType.safeCast(it) } - - /** - * Creates a mapping between common SQL types and their corresponding KTypes. - * - * @param tableColumnMetadata The metadata of the table column. - * @return The KType associated with the SQL type or a default type if no mapping is found. - */ - public open fun makeCommonSqlToKTypeMapping(tableColumnMetadata: TableColumnMetadata): KType { - val jdbcTypeToKTypeMapping = mapOf( - Types.BIT to Boolean::class, - Types.TINYINT to Int::class, - Types.SMALLINT to Int::class, - Types.INTEGER to Int::class, - Types.BIGINT to Long::class, - Types.FLOAT to Float::class, - Types.REAL to Float::class, - Types.DOUBLE to Double::class, - Types.NUMERIC to BigDecimal::class, - Types.DECIMAL to BigDecimal::class, - Types.CHAR to String::class, - Types.VARCHAR to String::class, - Types.LONGVARCHAR to String::class, - Types.DATE to Date::class, - Types.TIME to Time::class, - Types.TIMESTAMP to Timestamp::class, - Types.BINARY to ByteArray::class, - Types.VARBINARY to ByteArray::class, - Types.LONGVARBINARY to ByteArray::class, - Types.NULL to String::class, - Types.JAVA_OBJECT to Any::class, - Types.DISTINCT to Any::class, - Types.STRUCT to Any::class, - Types.ARRAY to Array::class, - Types.BLOB to ByteArray::class, - Types.CLOB to Clob::class, - Types.REF to Ref::class, - Types.DATALINK to Any::class, - Types.BOOLEAN to Boolean::class, - Types.ROWID to RowId::class, - Types.NCHAR to String::class, - Types.NVARCHAR to String::class, - Types.LONGNVARCHAR to String::class, - Types.NCLOB to NClob::class, - Types.SQLXML to SQLXML::class, - Types.REF_CURSOR to Ref::class, - Types.TIME_WITH_TIMEZONE to OffsetTime::class, - Types.TIMESTAMP_WITH_TIMEZONE to OffsetDateTime::class, - ) - - fun determineKotlinClass(tableColumnMetadata: TableColumnMetadata): KClass<*> = - when { - tableColumnMetadata.jdbcType == Types.OTHER -> when (tableColumnMetadata.javaClassName) { - "[B" -> ByteArray::class - else -> Any::class - } - - tableColumnMetadata.javaClassName == "[B" -> ByteArray::class - - tableColumnMetadata.javaClassName == "java.sql.Blob" -> Blob::class - - tableColumnMetadata.jdbcType == Types.TIMESTAMP && - tableColumnMetadata.javaClassName == "java.time.LocalDateTime" -> LocalDateTime::class - - tableColumnMetadata.jdbcType == Types.BINARY && - tableColumnMetadata.javaClassName == "java.util.UUID" -> UUID::class - - tableColumnMetadata.jdbcType == Types.REAL && - tableColumnMetadata.javaClassName == "java.lang.Double" -> Double::class - - tableColumnMetadata.jdbcType == Types.FLOAT && - tableColumnMetadata.javaClassName == "java.lang.Double" -> Double::class - - tableColumnMetadata.jdbcType == Types.NUMERIC && - tableColumnMetadata.javaClassName == "java.lang.Double" -> Double::class - - // Force BIGINT to always be Long, regardless of javaClassName - // Some JDBC drivers (e.g., MariaDB) may report Integer for small BIGINT values - // TODO: tableColumnMetadata.jdbcType == Types.BIGINT -> Long::class - - else -> jdbcTypeToKTypeMapping[tableColumnMetadata.jdbcType] ?: String::class - } - - fun createArrayTypeIfNeeded(kClass: KClass<*>, isNullable: Boolean): KType = - if (kClass == Array::class) { - val typeParam = kClass.typeParameters[0].createType() - kClass.createType( - arguments = listOf(kotlin.reflect.KTypeProjection.invariant(typeParam)), - nullable = isNullable, - ) - } else { - kClass.createType(nullable = isNullable) - } - - val kClass: KClass<*> = determineKotlinClass(tableColumnMetadata) - val kType = createArrayTypeIfNeeded(kClass, tableColumnMetadata.isNullable) - return kType - } - - /** - * Retrieves column metadata from a JDBC ResultSet. - * - * This method reads column metadata from [ResultSetMetaData] with graceful fallbacks - * for JDBC drivers that throw [java.sql.SQLFeatureNotSupportedException] for certain methods - * (e.g., Apache Hive). - * - * Fallback behavior for unsupported methods: - * - `getColumnName()` → `getColumnLabel()` → `"column_N"` - * - `getTableName()` → extract from column name if contains '.' → `null` - * - `isNullable()` → [DatabaseMetaData.getColumns] → `true` (assume nullable) - * - `getColumnTypeName()` → `"OTHER"` - * - `getColumnType()` → [java.sql.Types.OTHER] - * - `getColumnDisplaySize()` → `0` - * - `getColumnClassName()` → `"java.lang.Object"` - * - * Override this method in subclasses to provide database-specific behavior - * (for example, to disable fallback for databases like Teradata or Oracle - * where [DatabaseMetaData.getColumns] is known to be slow). - * - * @param resultSet The [ResultSet] containing query results. - * @return A list of [TableColumnMetadata] objects. - */ - public open fun getTableColumnsMetadata(resultSet: ResultSet): List { - val rsMetaData = resultSet.metaData - val connection = resultSet.statement.connection - val dbMetaData = connection.metaData - - // Some JDBC drivers (e.g., Hive) throw SQLFeatureNotSupportedException - val catalog = try { - connection.catalog.takeUnless { it.isNullOrBlank() } - } catch (_: Exception) { - null - } - - val schema = try { - connection.schema.takeUnless { it.isNullOrBlank() } - } catch (_: Exception) { - null - } - - val columnCount = rsMetaData.columnCount - val columns = mutableListOf() - val nameCounter = mutableMapOf() - - for (index in 1..columnCount) { - // Try to getColumnName, fallback to getColumnLabel, then generate name - val columnName = try { - rsMetaData.getColumnName(index) - } catch (_: Exception) { - try { - rsMetaData.getColumnLabel(index) - } catch (_: Exception) { - "column$index" - } - } - - // Some JDBC drivers (e.g., Apache Hive) throw SQLFeatureNotSupportedException - val tableName = try { - rsMetaData.getTableName(index).takeUnless { it.isBlank() } - } catch (_: Exception) { - // Fallback: try to extract table name from column name if it contains '.' - val dotIndex = columnName.lastIndexOf('.') - if (dotIndex > 0) columnName.take(dotIndex) else null - } - - // Try to detect nullability from ResultSetMetaData - val isNullable = try { - when (rsMetaData.isNullable(index)) { - ResultSetMetaData.columnNoNulls -> false - - ResultSetMetaData.columnNullable -> true - - // Unknown nullability: assume it nullable, may trigger fallback - ResultSetMetaData.columnNullableUnknown -> true - - else -> true - } - } catch (_: Exception) { - // Some drivers may throw for unsupported features - // Try fallback to DatabaseMetaData, with additional safety - try { - dbMetaData.getColumns(catalog, schema, tableName, columnName).use { cols -> - if (cols.next()) !cols.getString("IS_NULLABLE").equals("NO", ignoreCase = true) else true - } - } catch (_: Exception) { - // Fallback failed, assume nullable as the safest default - true - } - } - - // adding fallbacks to avoid SQLException - val columnType = try { - rsMetaData.getColumnTypeName(index) - } catch (_: Exception) { - "OTHER" - } - - val jdbcType = try { - rsMetaData.getColumnType(index) - } catch (_: Exception) { - Types.OTHER - } - - val displaySize = try { - rsMetaData.getColumnDisplaySize(index) - } catch (_: Exception) { - 0 - } - - val javaClassName = try { - rsMetaData.getColumnClassName(index) - } catch (_: Exception) { - "java.lang.Object" - } - - val uniqueName = manageColumnNameDuplication(nameCounter, columnName) - - columns += TableColumnMetadata( - uniqueName, - columnType, - jdbcType, - displaySize, - javaClassName, - isNullable, - ) - } - - return columns - } - - /** - * Manages the duplication of column names by appending a unique identifier to the original name if necessary. - * - * @param columnNameCounter a mutable map that keeps track of the count for each column name. - * @param originalName the original name of the column to be managed. - * @return the modified column name that is free from duplication. - */ - internal fun manageColumnNameDuplication(columnNameCounter: MutableMap, originalName: String): String { - var name = originalName - val count = columnNameCounter[originalName] - - if (count != null) { - var incrementedCount = count + 1 - while (columnNameCounter.containsKey("${originalName}_$incrementedCount")) { - incrementedCount++ - } - columnNameCounter[originalName] = incrementedCount - name = "${originalName}_$incrementedCount" - } else { - columnNameCounter[originalName] = 0 - } - - return name - } } + +// same as org.jetbrains.kotlinx.dataframe.impl.UNNAMED_COLUMN_PREFIX +internal const val UNNAMED_COLUMN_PREFIX = "untitled" diff --git a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/DuckDb.kt b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/DuckDb.kt index d6882133e7..e454dec406 100644 --- a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/DuckDb.kt +++ b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/DuckDb.kt @@ -1,7 +1,8 @@ package org.jetbrains.kotlinx.dataframe.io.db import io.github.oshai.kotlinlogging.KotlinLogging -import org.duckdb.DuckDBColumnType +import kotlinx.datetime.toKotlinLocalDate +import kotlinx.datetime.toKotlinLocalTime import org.duckdb.DuckDBColumnType.ARRAY import org.duckdb.DuckDBColumnType.BIGINT import org.duckdb.DuckDBColumnType.BIT @@ -39,32 +40,43 @@ import org.duckdb.DuckDBColumnType.UUID import org.duckdb.DuckDBColumnType.VARCHAR import org.duckdb.DuckDBResultSetMetaData import org.duckdb.JsonNode +import org.jetbrains.kotlinx.dataframe.AnyRow import org.jetbrains.kotlinx.dataframe.DataFrame +import org.jetbrains.kotlinx.dataframe.api.Infer +import org.jetbrains.kotlinx.dataframe.api.asColumnGroup +import org.jetbrains.kotlinx.dataframe.api.asDataColumn +import org.jetbrains.kotlinx.dataframe.api.inferType +import org.jetbrains.kotlinx.dataframe.api.toColumn +import org.jetbrains.kotlinx.dataframe.api.toDataFrame +import org.jetbrains.kotlinx.dataframe.api.tryParse +import org.jetbrains.kotlinx.dataframe.impl.schema.DataFrameSchemaImpl import org.jetbrains.kotlinx.dataframe.io.DbConnectionConfig -import org.jetbrains.kotlinx.dataframe.io.db.DuckDb.convertSqlTypeToKType import org.jetbrains.kotlinx.dataframe.io.readAllSqlTables import org.jetbrains.kotlinx.dataframe.schema.ColumnSchema import java.math.BigDecimal import java.math.BigInteger -import java.sql.Array import java.sql.Blob import java.sql.Connection import java.sql.DatabaseMetaData import java.sql.DriverManager import java.sql.ResultSet import java.sql.Struct -import java.sql.Timestamp -import java.time.LocalDate -import java.time.LocalTime -import java.time.OffsetDateTime -import java.time.OffsetTime import java.util.Properties -import java.util.UUID -import kotlin.reflect.KType +import kotlin.reflect.KClass import kotlin.reflect.KTypeProjection import kotlin.reflect.full.createType +import kotlin.reflect.full.isSubtypeOf import kotlin.reflect.full.withNullability import kotlin.reflect.typeOf +import kotlin.time.toKotlinInstant +import kotlin.uuid.toKotlinUuid +import java.sql.Array as SqlArray +import java.sql.Timestamp as SqlTimestamp +import java.time.LocalDate as JavaLocalDate +import java.time.LocalTime as JavaLocalTime +import java.time.OffsetDateTime as JavaOffsetDateTime +import java.time.OffsetTime as JavaOffsetTime +import java.util.UUID as JavaUUID private val logger = KotlinLogging.logger {} @@ -74,88 +86,223 @@ private val logger = KotlinLogging.logger {} * This class provides methods to convert data from a [ResultSet] to the appropriate type for DuckDB, * and to generate the corresponding [column schema][ColumnSchema]. */ -public object DuckDb : DbType("duckdb") { +public object DuckDb : AdvancedDbType("duckdb") { /** the name of the class of the DuckDB JDBC driver */ override val driverClassName: String = "org.duckdb.DuckDBDriver" + override fun generateConverter(tableColumnMetadata: TableColumnMetadata): AnyJdbcToDataFrameConverter = + parseDuckDbType(tableColumnMetadata.sqlTypeName, tableColumnMetadata.isNullable) + + private val duckDbTypeCache = mutableMapOf, AnyJdbcToDataFrameConverter>() + /** - * How a column type from JDBC, [tableColumnMetadata], is read in Java/Kotlin. + * How a column type from JDBC, [sqlTypeName], is read in Java/Kotlin. * The returned type must exactly follow [ResultSet.getObject] of your specific database's JDBC driver. * Returning `null` defer the implementation to the default one (which may not always be correct). * - * Following [org.duckdb.DuckDBVector.getObject]. + * Following [org.duckdb.DuckDBVector.getObject] and converting the result to + * */ - override fun convertSqlTypeToKType(tableColumnMetadata: TableColumnMetadata): KType = - tableColumnMetadata.sqlTypeName.toKType(tableColumnMetadata.isNullable) + internal fun parseDuckDbType(sqlTypeName: String, isNullable: Boolean): AnyJdbcToDataFrameConverter = + duckDbTypeCache.getOrPut(Pair(sqlTypeName, isNullable)) { + when (DuckDBResultSetMetaData.TypeNameToType(sqlTypeName)) { + BOOLEAN -> jdbcToDfConverterFor(isNullable) - /** - * How a column from JDBC should be represented as DataFrame (value) column - * See [convertSqlTypeToKType]. - */ - override fun convertSqlTypeToColumnSchemaValue(tableColumnMetadata: TableColumnMetadata): ColumnSchema { - val type = convertSqlTypeToKType(tableColumnMetadata) - return ColumnSchema.Value(type) - } + TINYINT -> jdbcToDfConverterFor(isNullable) - /** - * Follows exactly [org.duckdb.DuckDBVector.getObject]. - * - * "// dataframe-jdbc" is added for all types that are covered correctly by - * [org.jetbrains.kotlinx.dataframe.io.db.DbType.makeCommonSqlToKTypeMapping] at the moment, however, to cover - * all nested types, we'll use a full type-map for all [DuckDB types][DuckDBColumnType] exactly. - */ - @Suppress("ktlint:standard:blank-line-between-when-conditions") - internal fun String.toKType(isNullable: Boolean): KType { - val sqlTypeName = this - return when (DuckDBResultSetMetaData.TypeNameToType(sqlTypeName)) { - BOOLEAN -> typeOf() // dataframe-jdbc - TINYINT -> typeOf() - SMALLINT -> typeOf() - INTEGER -> typeOf() // dataframe-jdbc - BIGINT -> typeOf() // dataframe-jdbc - HUGEINT -> typeOf() - UHUGEINT -> typeOf() - UTINYINT -> typeOf() - USMALLINT -> typeOf() - UINTEGER -> typeOf() - UBIGINT -> typeOf() - FLOAT -> typeOf() // dataframe-jdbc - DOUBLE -> typeOf() // dataframe-jdbc - DECIMAL -> typeOf() // dataframe-jdbc - TIME -> typeOf() - TIME_WITH_TIME_ZONE -> typeOf() // dataframe-jdbc - DATE -> typeOf() - TIMESTAMP, TIMESTAMP_MS, TIMESTAMP_NS, TIMESTAMP_S -> typeOf() // dataframe-jdbc - TIMESTAMP_WITH_TIME_ZONE -> typeOf() // dataframe-jdbc - JSON -> typeOf() - BLOB -> typeOf() - UUID -> typeOf() - MAP -> { - val (key, value) = parseMapTypes(sqlTypeName) - Map::class.createType( - listOf( - KTypeProjection.invariant(key.toKType(false)), - KTypeProjection.invariant(value.toKType(true)), - ), - ) - } + SMALLINT -> jdbcToDfConverterFor(isNullable) + + INTEGER -> jdbcToDfConverterFor(isNullable) + + BIGINT -> jdbcToDfConverterFor(isNullable) + + HUGEINT -> jdbcToDfConverterFor(isNullable) + + UHUGEINT -> jdbcToDfConverterFor(isNullable) + + UTINYINT -> jdbcToDfConverterFor(isNullable) + + USMALLINT -> jdbcToDfConverterFor(isNullable) + + UINTEGER -> jdbcToDfConverterFor(isNullable) + + UBIGINT -> jdbcToDfConverterFor(isNullable) + + FLOAT -> jdbcToDfConverterFor(isNullable) + + DOUBLE -> jdbcToDfConverterFor(isNullable) + + DECIMAL -> jdbcToDfConverterFor(isNullable) + + TIME -> + jdbcToDfConverterFor(isNullable) + .withPreprocessor { it?.toKotlinLocalTime() } + + // todo? + TIME_WITH_TIME_ZONE -> jdbcToDfConverterFor(isNullable) + + DATE -> jdbcToDfConverterFor(isNullable) + .withPreprocessor { it?.toKotlinLocalDate() } + + TIMESTAMP, TIMESTAMP_MS, TIMESTAMP_NS, TIMESTAMP_S -> + jdbcToDfConverterFor(isNullable) + .withPreprocessor { it?.toInstant()?.toKotlinInstant() } + + // todo? + TIMESTAMP_WITH_TIME_ZONE -> jdbcToDfConverterFor(isNullable) + + JSON -> + jdbcToDfConverterFor(isNullable) + .withPreprocessor { it?.toString() } + .withColumnBuilder(targetSchema = null) { name, values, inferNullability -> + values + .toColumn(name, if (inferNullability) Infer.Nulls else Infer.None) + .tryParse() + .inferType() + } + + BLOB -> jdbcToDfConverterFor(isNullable) + + UUID -> jdbcToDfConverterFor(isNullable) + .withPreprocessor { it?.toKotlinUuid() } + + MAP -> { + val (key, value) = parseMapTypes(sqlTypeName) + + val parsedKeyType = parseDuckDbType(key, false) + val parsedValueType = parseDuckDbType(value, true).castToAny() + + val targetMapType = Map::class.createType( + listOf( + KTypeProjection.invariant(parsedKeyType.targetSchema?.type ?: typeOf()), + KTypeProjection.invariant(parsedValueType.targetSchema?.type ?: typeOf()), + ), + ).withNullability(isNullable) + + jdbcToDfConverterFor>(isNullable) + .withPreprocessor(preprocessedValueType = targetMapType) { map -> + // only need to preprocess the values, as the keys are just Strings + map?.mapValues { (_, value) -> + parsedValueType.preprocessOrCast(value) + } + } + } + + LIST, ARRAY -> { + val listType = parseListType(sqlTypeName) + val parsedListType = + parseDuckDbType(listType, true).castToAny() + + val targetListType = List::class + .createType( + listOf( + KTypeProjection.invariant( + parsedListType.targetSchema?.type ?: typeOf(), + ), + ), + ) + .withNullability(isNullable) + + when (val listTargetSchema = parsedListType.targetSchema) { + // convert STRUCT[] -> DataFrame<*> to create a FrameColumn + is ColumnSchema.Group if parsedListType.expectedJdbcType.isSubtypeOf(typeOf()) -> + jdbcToDfConverterFor(isNullable) + .withPreprocessor { sqlArray -> + sqlArray + ?.toList() + ?.mapNotNull { + parsedListType.cast?, AnyRow?>() + .preprocessOrCast(it) + }?.toDataFrame() + } + .withTargetSchema( + targetSchema = with(listTargetSchema) { + ColumnSchema.Frame(schema, nullable, contentType) + }, + ) + + else -> + jdbcToDfConverterFor(isNullable) + .withPreprocessor(preprocessedValueType = targetListType) { sqlArray -> + sqlArray + ?.toList() + ?.map { parsedListType.preprocessOrCast(it) } // recursively preprocess + } + } + } + + STRUCT -> { + val structEntries = parseStructType(sqlTypeName) + val parsedStructEntries = structEntries.mapValues { (_, type) -> + parseDuckDbType(sqlTypeName = type, isNullable = true) + } + + val targetSchema = ColumnSchema.Group( + schema = DataFrameSchemaImpl( + parsedStructEntries.mapValues { + it.value.targetSchema ?: ColumnSchema.Value(typeOf()) + }, + ), + contentType = typeOf(), + ) + + jdbcToDfConverterFor(isNullable) + .withPreprocessor { struct -> + // NOTE DataRows cannot be `null` in DataFrame, instead, all its fields become `null` + if (struct == null) { + parsedStructEntries.mapValues { null } + } else { + // read data from the struct + val attrs = struct.getAttributes( + parsedStructEntries.mapValues { (fieldName, entry) -> + val expectedType = entry.expectedJdbcType + val classifier = expectedType.classifier as? KClass<*> + ?: error( + "DuckDB STRUCT field '$fieldName' has expected JDBC type '$expectedType' with no classifier; This is an incorrect KType.", + ) + classifier.java + }, + ) - LIST, ARRAY -> { - // TODO requires #1266 and #1273 for specific types - // val listType = parseListType(sqlTypeName) - // Array::class.createType( - // listOf(KTypeProjection.invariant(listType.toKType(true))), - // ) - typeOf() + // and potentially, preprocess each value individually + parsedStructEntries.entries.withIndex().associate { (i, entry) -> + entry.key to entry.value.castToAny().preprocessOrCast(attrs[i]) + } + } + } + .withColumnBuilder(targetSchema) { name, values, _ -> + values + .toDataFrame() + .asColumnGroup(name) + .asDataColumn() + } + } + + // Cannot handle this in Kotlin + UNION -> jdbcToDfConverterFor(isNullable) + + VARCHAR -> jdbcToDfConverterFor(isNullable) + + UNKNOWN, BIT, INTERVAL, ENUM -> jdbcToDfConverterFor(isNullable) } + } - STRUCT -> typeOf() // TODO requires #1266 for specific types - UNION -> typeOf() // Cannot handle this in Kotlin - VARCHAR -> typeOf() - UNKNOWN, BIT, INTERVAL, ENUM -> typeOf() - }.withNullability(isNullable) - } + @Suppress("UNCHECKED_CAST") + private fun SqlArray.toList(): List = + when (val array = this.array) { + is IntArray -> array.toList() + is LongArray -> array.toList() + is ShortArray -> array.toList() + is ByteArray -> array.toList() + is FloatArray -> array.toList() + is DoubleArray -> array.toList() + is BooleanArray -> array.toList() + is CharArray -> array.toList() + is Array<*> -> array.toList() + is SqlArray -> array.toList() + else -> error("unknown array type $array") + } as List /** Parses "MAP(X, Y)" into "X" and "Y", taking parentheses into account */ internal fun parseMapTypes(typeString: String): Pair { @@ -196,6 +343,60 @@ public object DuckDb : DbType("duckdb") { return typeString.take(typeString.indexOfLast { it == '[' }) } + /** + * Parses + * - `"STRUCT(v VARCHAR, i INTEGER)"` into `[("v", "VARCHAR"), ("i", "INTEGER")]`; + * - `"STRUCT(col1 STRUCT(i INTEGER, j VARCHAR), col2 INTEGER)"` + * into `[("col1", "STRUCT(i INTEGER, j VARCHAR)"), ("col2", "INTEGER")]`; + * - etc. + */ + internal fun parseStructType(typeString: String): Map { + if (!typeString.startsWith("STRUCT(") || !typeString.endsWith(")")) { + error("invalid STRUCT type: $typeString") + } + + val content = typeString.removeSurrounding("STRUCT(", ")") + + // Split the struct into entries, taking parentheses and spaces into account + val entries = buildMap { + var parenCount = 0 + var entryPart = "" + var spaceIndex = -1 + + fun yieldEntryPart() { + require(spaceIndex > 0) { + "Invalid struct entry format: '$entryPart' in DuckDB Struct type: '$typeString'" + } + val key = entryPart.take(spaceIndex).trim() + val value = entryPart.substring(spaceIndex + 1).trim() + this += key to value + entryPart = "" + spaceIndex = -1 + } + + for (i in content.indices) { + when (content[i]) { + '(' -> parenCount++ + + ')' -> parenCount-- + + ' ' if (parenCount == 0) -> { + spaceIndex = entryPart.length + } + + ',' if (parenCount == 0) -> { + yieldEntryPart() + continue + } + } + entryPart += content[i] + } + yieldEntryPart() + } + + return entries + } + /** * How to filter out system tables from user-created ones when using * [DataFrame.readAllSqlTables][DataFrame.Companion.readAllSqlTables] and @@ -228,7 +429,7 @@ public object DuckDb : DbType("duckdb") { * but supports read-only mode through connection parameters. * * @param [dbConfig] The database configuration containing URL, credentials, and read-only flag. - * @return A configured [java.sql.Connection] instance. + * @return A configured [Connection] instance. */ override fun createConnection(dbConfig: DbConnectionConfig): Connection { val properties = Properties().apply { diff --git a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/H2.kt b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/H2.kt index 96cea43724..7d7b40f7a5 100644 --- a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/H2.kt +++ b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/H2.kt @@ -1,6 +1,5 @@ package org.jetbrains.kotlinx.dataframe.io.db -import org.jetbrains.kotlinx.dataframe.schema.ColumnSchema import java.sql.ResultSet import java.util.Locale import kotlin.reflect.KType @@ -17,7 +16,6 @@ import org.jetbrains.kotlinx.dataframe.io.db.PostgreSql as PostgreSqlType * * NOTE: All date and timestamp-related types are converted to String to avoid java.sql.* types. */ - public open class H2(public val mode: Mode = Mode.Regular) : DbType("h2") { @Deprecated("Use H2(mode = Mode.XXX) instead", ReplaceWith("H2(H2.Mode.MySql)")) public constructor(dialect: DbType) : this( @@ -119,8 +117,12 @@ public open class H2(public val mode: Mode = Mode.Regular) : DbType("h2") { override val driverClassName: String get() = "org.h2.Driver" - override fun convertSqlTypeToColumnSchemaValue(tableColumnMetadata: TableColumnMetadata): ColumnSchema? = - delegate?.convertSqlTypeToColumnSchemaValue(tableColumnMetadata) + /** + * TODO check and map all types from https://www.h2database.com/html/datatypes.html + */ + override fun getExpectedJdbcType(tableColumnMetadata: TableColumnMetadata): KType = + delegate?.getExpectedJdbcType(tableColumnMetadata) + ?: super.getExpectedJdbcType(tableColumnMetadata) override fun isSystemTable(tableMetadata: TableMetadata): Boolean { val locale = Locale.getDefault() @@ -146,9 +148,6 @@ public open class H2(public val mode: Mode = Mode.Regular) : DbType("h2") { tables.getString("table_cat"), ) - override fun convertSqlTypeToKType(tableColumnMetadata: TableColumnMetadata): KType? = - delegate?.convertSqlTypeToKType(tableColumnMetadata) - public override fun buildSqlQueryWithLimit(sqlQuery: String, limit: Int): String = delegate?.buildSqlQueryWithLimit(sqlQuery, limit) ?: super.buildSqlQueryWithLimit(sqlQuery, limit) } diff --git a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/JdbcToDataFrameConverter.kt b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/JdbcToDataFrameConverter.kt new file mode 100644 index 0000000000..8443d7ed2e --- /dev/null +++ b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/JdbcToDataFrameConverter.kt @@ -0,0 +1,232 @@ +package org.jetbrains.kotlinx.dataframe.io.db + +import org.jetbrains.kotlinx.dataframe.DataColumn +import org.jetbrains.kotlinx.dataframe.schema.ColumnSchema +import java.sql.ResultSet +import kotlin.reflect.KType +import kotlin.reflect.full.withNullability +import kotlin.reflect.typeOf + +public typealias AnyJdbcToDataFrameConverter = JdbcToDataFrameConverter<*, *, *> + +/** + * Represents all type information that can be retrieved from an SQL column. + * This can be extended for your specific [DbType] if you need extra information. + * + * This class needs to be stateless, so it can be memoized in [AdvancedDbType.generateConverter]. + * + * Use [jdbcToDfConverterFor] to create an instance. + * + * This API is experimental and subject to change. + * + * @param J the type of the value coming from the JDBC driver, [expectedJdbcType]. + * @param D the type of the column values after preprocessing, [preprocessedValueType]. Will be equal to [J] if [valuePreprocessor] is `null`. + * @param P the type of the column values after postprocessing, comparable to [DataColumn][DataColumn]`<`[P][P]`>` of [targetSchema]. + * Will be equal to [D] if [columnBuilder] is `null`. + * + * @property targetSchema the target schema of the column after running the optional + * [valuePreprocessor] and [columnBuilder]. Can be `null` if the target schema is dependent on the runtime input + * and thus cannot be determined from input types alone. + * @property valuePreprocessor an optional function that converts values from [ResultSet.getObject] + * to a cell/row suitable to be put into a [DataColumn]. + * @property columnBuilder an optional function that converts a [List] with values of type [D] + * to a [DataColumn] of with values of type [P]. + */ +public class JdbcToDataFrameConverter( + public val expectedJdbcType: KType, + public val resultSetReader: DbResultSetReader?, + public val preprocessedValueType: KType, + public val valuePreprocessor: DbValuePreprocessor?, + public val targetSchema: ColumnSchema?, + public val columnBuilder: DbColumnBuilder?, +) { + public fun getValueFromResultSetOrElse(rs: ResultSet, columnIndex: Int, default: () -> J): J { + resultSetReader?.let { reader -> + return reader.getValue(rs, columnIndex) + } + return default() + } + + @Suppress("UNCHECKED_CAST") + public fun preprocessOrCast(value: J): D { + valuePreprocessor?.let { valuePreprocessor -> + return valuePreprocessor.preprocess(value) + } + return value as D + } + + public fun buildDataColumnOrNull(name: String, values: List, inferNullability: Boolean): DataColumn

? { + columnBuilder?.let { columnPostprocessor -> + return columnPostprocessor.buildDataColumn(name, values, inferNullability) + } + return null + } + + public fun copy( + expectedJdbcType: KType = this.expectedJdbcType, + resultSetReader: DbResultSetReader? = this.resultSetReader, + preprocessedValueType: KType = this.preprocessedValueType, + valuePreprocessor: DbValuePreprocessor? = this.valuePreprocessor, + targetSchema: ColumnSchema? = this.targetSchema, + columnBuilder: DbColumnBuilder? = this.columnBuilder, + ): JdbcToDataFrameConverter = + JdbcToDataFrameConverter( + expectedJdbcType = expectedJdbcType, + resultSetReader = resultSetReader, + preprocessedValueType = preprocessedValueType, + valuePreprocessor = valuePreprocessor, + targetSchema = targetSchema, + columnBuilder = columnBuilder, + ) +} + +@Suppress("UNCHECKED_CAST") +public fun JdbcToDataFrameConverter<*, *, *>.cast(): JdbcToDataFrameConverter = + this as JdbcToDataFrameConverter + +public fun JdbcToDataFrameConverter<*, *, *>.castToAny(): JdbcToDataFrameConverter = cast() + +// region builders + +public fun jdbcToDfConverterFor(jdbcSourceType: KType): JdbcToDataFrameConverter = + JdbcToDataFrameConverter( + expectedJdbcType = jdbcSourceType, + resultSetReader = null, + preprocessedValueType = jdbcSourceType, + valuePreprocessor = null, + targetSchema = ColumnSchema.Value(jdbcSourceType), + columnBuilder = null, + ) + +public inline fun jdbcToDfConverterFor( + isNullable: Boolean = typeOf().isMarkedNullable, +): JdbcToDataFrameConverter = + jdbcToDfConverterFor( + jdbcSourceType = typeOf().withNullability(isNullable), + ) + +public fun JdbcToDataFrameConverter.withResultReader( + resultSetReader: DbResultSetReader?, +): JdbcToDataFrameConverter = copy(resultSetReader = resultSetReader) + +public fun JdbcToDataFrameConverter.withPreprocessor( + preprocessedValueType: KType, + canIntroduceNulls: Boolean = false, + valuePreprocessor: DbValuePreprocessor?, +): JdbcToDataFrameConverter { + // carry definitely-not-null knowledge from expectedJdbcType to preprocessedValueType + // if the valuePreprocessor does not introduce nulls itself + val preprocessedValueType = + if (!expectedJdbcType.isMarkedNullable && !canIntroduceNulls) { + preprocessedValueType.withNullability(false) + } else { + preprocessedValueType + } + return cast() + .copy( + preprocessedValueType = preprocessedValueType, + valuePreprocessor = valuePreprocessor, + targetSchema = ColumnSchema.Value(preprocessedValueType), + columnBuilder = null, + ) +} + +public inline fun JdbcToDataFrameConverter.withPreprocessor( + canIntroduceNulls: Boolean = false, + valuePreprocessor: DbValuePreprocessor?, +): JdbcToDataFrameConverter = + withPreprocessor( + canIntroduceNulls = canIntroduceNulls, + preprocessedValueType = typeOf(), + valuePreprocessor = valuePreprocessor, + ) + +public fun JdbcToDataFrameConverter.withTargetSchema( + targetSchema: ColumnSchema?, +): JdbcToDataFrameConverter = + withColumnBuilder( + targetSchema = targetSchema, + canIntroduceNulls = false, + columnBuilder = null, + ) + +public fun JdbcToDataFrameConverter.withColumnBuilder( + targetSchema: ColumnSchema?, + canIntroduceNulls: Boolean = false, + columnBuilder: DbColumnBuilder?, +): JdbcToDataFrameConverter { + // carry definitely-not-null knowledge from preprocessedValueType to targetSchema + // if the columnBuilder does not introduce nulls itself + val targetSchema = + if (targetSchema != null && !preprocessedValueType.isMarkedNullable && !canIntroduceNulls) { + when (targetSchema) { + is ColumnSchema.Value -> + ColumnSchema.Value(targetSchema.type.withNullability(false)) + + is ColumnSchema.Group -> targetSchema + + is ColumnSchema.Frame -> ColumnSchema.Frame( + schema = targetSchema.schema, + nullable = false, + contentType = targetSchema.contentType, + ) + } + } else { + targetSchema + } + return cast() + .copy( + targetSchema = targetSchema, + columnBuilder = columnBuilder, + ) +} + +// endregion + +public fun interface DbResultSetReader { + + public fun getValue(rs: ResultSet, columnIndex: Int): J +} + +/** + * This preprocessor can be created for types where you want to convert the values + * coming from [ResultSet.getObject] to a different type more suitable to be put in a [DataColumn] + * + * @param J the type of the value coming from the JDBC driver. + * @param D the type of the column values after preprocessing. + */ +public fun interface DbValuePreprocessor { + + /** + * Converts the given [jdbcValue]: [J] to a [D]. + * + * If you intend to create a [org.jetbrains.kotlinx.dataframe.columns.ColumnGroup], + * return a [org.jetbrains.kotlinx.dataframe.DataRow] here. + * + * If you intend to create a [org.jetbrains.kotlinx.dataframe.columns.FrameColumn], + * return a [org.jetbrains.kotlinx.dataframe.DataFrame] here. + */ + public fun preprocess(jdbcValue: J): D +} + +@Suppress("UNCHECKED_CAST") +public fun DbValuePreprocessor<*, *>.cast(): DbValuePreprocessor = this as DbValuePreprocessor + +public fun DbValuePreprocessor<*, *>.castToAny(): DbValuePreprocessor = cast() + +/** + * @param D the type of the column values before entering the column. + * @param P the type of the column values after entering the column. + */ +public fun interface DbColumnBuilder { + + /** + * Converts the given [values]: [DataColumn] with values of type [D] to a [DataColumn] of with values of type [P]. + */ + public fun buildDataColumn(name: String, values: List, inferNullability: Boolean): DataColumn

+} + +@Suppress("UNCHECKED_CAST") +public fun DbColumnBuilder<*, *>.cast(): DbColumnBuilder = this as DbColumnBuilder + +public fun DbColumnBuilder<*, *>.castToAny(): DbColumnBuilder = cast() diff --git a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/MariaDb.kt b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/MariaDb.kt index d91b60a0b2..9060004e62 100644 --- a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/MariaDb.kt +++ b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/MariaDb.kt @@ -1,11 +1,9 @@ package org.jetbrains.kotlinx.dataframe.io.db -import org.jetbrains.kotlinx.dataframe.io.db.TableColumnMetadata -import org.jetbrains.kotlinx.dataframe.io.db.TableMetadata -import org.jetbrains.kotlinx.dataframe.schema.ColumnSchema import java.sql.ResultSet import kotlin.reflect.KType -import kotlin.reflect.full.createType +import kotlin.reflect.full.withNullability +import kotlin.reflect.typeOf /** * Represents the MariaDb database type. @@ -17,7 +15,7 @@ public object MariaDb : DbType("mariadb") { override val driverClassName: String get() = "org.mariadb.jdbc.Driver" - override fun convertSqlTypeToColumnSchemaValue(tableColumnMetadata: TableColumnMetadata): ColumnSchema? { + override fun getExpectedJdbcType(tableColumnMetadata: TableColumnMetadata): KType { // Force BIGINT to always be Long, regardless of javaClassName // MariaDB JDBC driver may report Integer for small BIGINT values // TODO: investigate the corner case @@ -30,15 +28,13 @@ public object MariaDb : DbType("mariadb") { if (tableColumnMetadata.sqlTypeName == "INTEGER UNSIGNED" || tableColumnMetadata.sqlTypeName == "INT UNSIGNED" ) { - val kType = Long::class.createType(nullable = tableColumnMetadata.isNullable) - return ColumnSchema.Value(kType) + return typeOf().withNullability(tableColumnMetadata.isNullable) } if (tableColumnMetadata.sqlTypeName == "SMALLINT" && tableColumnMetadata.javaClassName == "java.lang.Short") { - val kType = Short::class.createType(nullable = tableColumnMetadata.isNullable) - return ColumnSchema.Value(kType) + return typeOf().withNullability(tableColumnMetadata.isNullable) } - return null + return super.getExpectedJdbcType(tableColumnMetadata) } override fun isSystemTable(tableMetadata: TableMetadata): Boolean = MySql.isSystemTable(tableMetadata) @@ -50,26 +46,6 @@ public object MariaDb : DbType("mariadb") { tables.getString("table_cat"), ) - override fun convertSqlTypeToKType(tableColumnMetadata: TableColumnMetadata): KType? { - // Force BIGINT to always be Long, regardless of javaClassName - // MariaDB JDBC driver may report Integer for small BIGINT values - // TODO: investigate the corner case - // if (tableColumnMetadata.jdbcType == java.sql.Types.BIGINT) { - // return Long::class.createType(nullable = tableColumnMetadata.isNullable) - // } - - if (tableColumnMetadata.sqlTypeName == "INTEGER UNSIGNED" || - tableColumnMetadata.sqlTypeName == "INT UNSIGNED" - ) { - return Long::class.createType(nullable = tableColumnMetadata.isNullable) - } - - if (tableColumnMetadata.sqlTypeName == "SMALLINT" && tableColumnMetadata.javaClassName == "java.lang.Short") { - return Short::class.createType(nullable = tableColumnMetadata.isNullable) - } - return null - } - override fun quoteIdentifier(name: String): String { // schema.table -> `schema`.`table` return name.split(".").joinToString(".") { "`$it`" } diff --git a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/MsSql.kt b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/MsSql.kt index 2709e04b6a..78f3c9c5a7 100644 --- a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/MsSql.kt +++ b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/MsSql.kt @@ -17,8 +17,6 @@ public object MsSql : DbType("sqlserver") { override val driverClassName: String get() = "com.microsoft.sqlserver.jdbc.SQLServerDriver" - override fun convertSqlTypeToColumnSchemaValue(tableColumnMetadata: TableColumnMetadata): ColumnSchema? = null - override fun isSystemTable(tableMetadata: TableMetadata): Boolean { val locale = Locale.getDefault() @@ -47,8 +45,6 @@ public object MsSql : DbType("sqlserver") { tables.getString("table_cat"), ) - override fun convertSqlTypeToKType(tableColumnMetadata: TableColumnMetadata): KType? = null - public override fun buildSqlQueryWithLimit(sqlQuery: String, limit: Int): String = sqlQuery.replace("SELECT", "SELECT TOP $limit", ignoreCase = true) diff --git a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/MySql.kt b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/MySql.kt index e411345879..7c4f74486c 100644 --- a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/MySql.kt +++ b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/MySql.kt @@ -1,12 +1,10 @@ package org.jetbrains.kotlinx.dataframe.io.db -import org.jetbrains.kotlinx.dataframe.io.db.TableColumnMetadata -import org.jetbrains.kotlinx.dataframe.io.db.TableMetadata -import org.jetbrains.kotlinx.dataframe.schema.ColumnSchema import java.sql.ResultSet import java.util.Locale import kotlin.reflect.KType -import kotlin.reflect.full.createType +import kotlin.reflect.full.withNullability +import kotlin.reflect.typeOf /** * Represents the MySql database type. @@ -18,12 +16,11 @@ public object MySql : DbType("mysql") { override val driverClassName: String get() = "com.mysql.jdbc.Driver" - override fun convertSqlTypeToColumnSchemaValue(tableColumnMetadata: TableColumnMetadata): ColumnSchema? { + override fun getExpectedJdbcType(tableColumnMetadata: TableColumnMetadata): KType { if (tableColumnMetadata.sqlTypeName == "INT UNSIGNED") { - val kType = Long::class.createType(nullable = tableColumnMetadata.isNullable) - return ColumnSchema.Value(kType) + return typeOf().withNullability(tableColumnMetadata.isNullable) } - return null + return super.getExpectedJdbcType(tableColumnMetadata) } override fun isSystemTable(tableMetadata: TableMetadata): Boolean { @@ -49,13 +46,6 @@ public object MySql : DbType("mysql") { tables.getString("table_cat"), ) - override fun convertSqlTypeToKType(tableColumnMetadata: TableColumnMetadata): KType? { - if (tableColumnMetadata.sqlTypeName == "INT UNSIGNED") { - return Long::class.createType(nullable = tableColumnMetadata.isNullable) - } - return null - } - override fun quoteIdentifier(name: String): String { // schema.table -> `schema`.`table` return name.split(".").joinToString(".") { "`$it`" } diff --git a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/PostgreSql.kt b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/PostgreSql.kt index 8da1a66833..cff1af63ed 100644 --- a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/PostgreSql.kt +++ b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/PostgreSql.kt @@ -1,12 +1,20 @@ package org.jetbrains.kotlinx.dataframe.io.db -import org.jetbrains.kotlinx.dataframe.io.db.TableColumnMetadata -import org.jetbrains.kotlinx.dataframe.io.db.TableMetadata -import org.jetbrains.kotlinx.dataframe.schema.ColumnSchema +import org.postgresql.geometric.PGbox +import org.postgresql.geometric.PGcircle +import org.postgresql.geometric.PGline +import org.postgresql.geometric.PGlseg +import org.postgresql.geometric.PGpath +import org.postgresql.geometric.PGpoint +import org.postgresql.geometric.PGpolygon +import org.postgresql.util.PGInterval +import org.postgresql.util.PGmoney +import org.postgresql.util.PGobject import java.sql.ResultSet import java.util.Locale import kotlin.reflect.KType -import kotlin.reflect.full.createType +import kotlin.reflect.full.starProjectedType +import kotlin.reflect.full.withNullability /** * Represents the PostgreSql database type. @@ -18,14 +26,53 @@ public object PostgreSql : DbType("postgresql") { override val driverClassName: String get() = "org.postgresql.Driver" - override fun convertSqlTypeToColumnSchemaValue(tableColumnMetadata: TableColumnMetadata): ColumnSchema? { - // TODO: could be a wrapper of convertSqlTypeToKType - // because of https://github.com/pgjdbc/pgjdbc/issues/425 - if (tableColumnMetadata.sqlTypeName == "money") { - val kType = String::class.createType(nullable = tableColumnMetadata.isNullable) - return ColumnSchema.Value(kType) + /** + * Map of [PostgreSQL object][PGobject] types by name + * containing both their [Java class][Class] and [Kotlin type][KType]. + * + * These types need to be retrieved explicitly with [Java class][Class] in [ResultSet.getObject], else + * their return type is unpredictable. + */ + private val pgObjectTypes = + listOf( + PGbox(), + PGcircle(), + PGline(), + PGlseg(), + PGpath(), + PGpoint(), + PGpolygon(), + PGmoney(), + PGInterval(), + ).map(::PgObjectType) + .associateBy { it.typeName } + + // TODO: Composite types like tableColumnMetadata.sqlTypeName = ROW("a" INTEGER, "b" CHARACTER VARYING(10)) + override fun getExpectedJdbcType(tableColumnMetadata: TableColumnMetadata): KType { + val typeName = tableColumnMetadata.sqlTypeName.lowercase() + return if (typeName in pgObjectTypes) { + pgObjectTypes[typeName]!!.kType.withNullability(tableColumnMetadata.isNullable) + } else { + super.getExpectedJdbcType(tableColumnMetadata) + } + } + + /** + * Overridden so [PGobject] types are retrieved explicitly with [Java class][Class], + * else their return type is unpredictable. + */ + override fun getValueFromResultSet( + rs: ResultSet, + columnIndex: Int, + tableColumnMetadata: TableColumnMetadata, + expectedJdbcType: KType, + ): J { + val typeName = tableColumnMetadata.sqlTypeName.lowercase() + return if (typeName in pgObjectTypes) { + rs.getObject(columnIndex + 1, pgObjectTypes[typeName]!!.javaClass) as J + } else { + super.getValueFromResultSet(rs, columnIndex, tableColumnMetadata, expectedJdbcType) } - return null } override fun isSystemTable(tableMetadata: TableMetadata): Boolean = @@ -39,17 +86,16 @@ public object PostgreSql : DbType("postgresql") { tables.getString("table_cat"), ) - override fun convertSqlTypeToKType(tableColumnMetadata: TableColumnMetadata): KType? { - // because of https://github.com/pgjdbc/pgjdbc/issues/425 - if (tableColumnMetadata.sqlTypeName == "money") { - return String::class.createType(nullable = tableColumnMetadata.isNullable) - } - - return null - } - override fun quoteIdentifier(name: String): String { // schema.table -> "schema"."table" return name.split(".").joinToString(".") { "\"$it\"" } } + + private data class PgObjectType(val typeName: String, val kType: KType, val javaClass: Class<*>) { + constructor(pgObject: PGobject) : this( + typeName = pgObject.type, + kType = pgObject::class.starProjectedType, + javaClass = pgObject::class.java, + ) + } } diff --git a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/Sqlite.kt b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/Sqlite.kt index adb03b2753..1073e12be5 100644 --- a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/Sqlite.kt +++ b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/Sqlite.kt @@ -20,8 +20,6 @@ public object Sqlite : DbType("sqlite") { override val driverClassName: String get() = "org.sqlite.JDBC" - override fun convertSqlTypeToColumnSchemaValue(tableColumnMetadata: TableColumnMetadata): ColumnSchema? = null - override fun isSystemTable(tableMetadata: TableMetadata): Boolean = tableMetadata.name.startsWith("sqlite_") override fun buildTableMetadata(tables: ResultSet): TableMetadata = @@ -31,8 +29,6 @@ public object Sqlite : DbType("sqlite") { tables.getString("TABLE_CAT"), ) - override fun convertSqlTypeToKType(tableColumnMetadata: TableColumnMetadata): KType? = null - override fun createConnection(dbConfig: DbConnectionConfig): Connection = if (dbConfig.readOnly) { val config = SQLiteConfig() diff --git a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/readDataFrameSchema.kt b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/readDataFrameSchema.kt index acce35f9fc..aa94cadb89 100644 --- a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/readDataFrameSchema.kt +++ b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/readDataFrameSchema.kt @@ -9,10 +9,13 @@ import org.jetbrains.kotlinx.dataframe.io.db.extractDBTypeFromConnection import org.jetbrains.kotlinx.dataframe.schema.ColumnSchema import org.jetbrains.kotlinx.dataframe.schema.DataFrameSchema import java.sql.Connection +import java.sql.DatabaseMetaData import java.sql.DriverManager import java.sql.ResultSet +import java.sql.ResultSetMetaData +import java.sql.SQLException import javax.sql.DataSource -import kotlin.use +import kotlin.reflect.typeOf /** * Retrieves the schema for an SQL table using the provided database configuration. @@ -98,6 +101,8 @@ public fun DataFrameSchema.Companion.readSqlTable( ): DataFrameSchema { val determinedDbType = dbType ?: extractDBTypeFromConnection(connection) + // TODO don't need to read 1 row, take it just from TableColumnMetadatas + // Read just 1 row to get the schema val singleRowDataFrame = DataFrame.readSqlTable( connection = connection, @@ -322,7 +327,15 @@ public fun Connection.readDataFrameSchema(sqlQueryOrTableName: String, dbType: D */ public fun DataFrameSchema.Companion.readResultSet(resultSet: ResultSet, dbType: DbType): DataFrameSchema { val tableColumns = getTableColumnsMetadata(resultSet, dbType) - return buildSchemaByTableColumns(tableColumns, dbType) + val expectedJdbcTypes = getExpectedJdbcTypes(dbType, tableColumns) + val preprocessedValueTypes = getPreprocessedValueTypes(dbType, tableColumns, expectedJdbcTypes) + val targetColumnSchemas = getTargetColumnSchemas(dbType, tableColumns, preprocessedValueTypes) + .withIndex() + .associate { (index, it) -> + tableColumns[index].name to (it ?: ColumnSchema.Value(typeOf())) + } + + return DataFrameSchemaImpl(targetColumnSchemas) } /** @@ -437,28 +450,3 @@ public fun DataFrameSchema.Companion.readAllSqlTables( return dataFrameSchemas } - -/** - * Builds a DataFrame schema based on the given table columns. - * - * @param [tableColumns] a mutable map containing the table columns, where the key represents the column name - * and the value represents the metadata of the column - * @param [dbType] the type of database. - * @return a [DataFrameSchema] object representing the schema built from the table columns. - */ -internal fun buildSchemaByTableColumns( - tableColumns: MutableList, - dbType: DbType, -): DataFrameSchema { - val schemaColumns = tableColumns.associate { - Pair(it.name, generateColumnSchemaValue(dbType, it)) - } - - return DataFrameSchemaImpl( - columns = schemaColumns, - ) -} - -internal fun generateColumnSchemaValue(dbType: DbType, tableColumnMetadata: TableColumnMetadata): ColumnSchema = - dbType.convertSqlTypeToColumnSchemaValue(tableColumnMetadata) - ?: ColumnSchema.Value(dbType.makeCommonSqlToKTypeMapping(tableColumnMetadata)) diff --git a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/readJdbc.kt b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/readJdbc.kt index b565ad3d5b..9af8492626 100644 --- a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/readJdbc.kt +++ b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/readJdbc.kt @@ -1,19 +1,30 @@ package org.jetbrains.kotlinx.dataframe.io import io.github.oshai.kotlinlogging.KotlinLogging +import org.jetbrains.kotlinx.dataframe.AnyCol import org.jetbrains.kotlinx.dataframe.AnyFrame +import org.jetbrains.kotlinx.dataframe.DataColumn import org.jetbrains.kotlinx.dataframe.DataFrame +import org.jetbrains.kotlinx.dataframe.api.isColumnGroup +import org.jetbrains.kotlinx.dataframe.api.isFrameColumn +import org.jetbrains.kotlinx.dataframe.api.isValueColumn +import org.jetbrains.kotlinx.dataframe.api.schema import org.jetbrains.kotlinx.dataframe.api.toDataFrame +import org.jetbrains.kotlinx.dataframe.`dataframe-jdbc`.BuildConfig +import org.jetbrains.kotlinx.dataframe.impl.ColumnNameGenerator import org.jetbrains.kotlinx.dataframe.io.db.DbType import org.jetbrains.kotlinx.dataframe.io.db.TableColumnMetadata import org.jetbrains.kotlinx.dataframe.io.db.extractDBTypeFromConnection +import org.jetbrains.kotlinx.dataframe.schema.ColumnSchema import java.sql.Connection import java.sql.DatabaseMetaData import java.sql.DriverManager import java.sql.PreparedStatement import java.sql.ResultSet import javax.sql.DataSource +import kotlin.reflect.KClass import kotlin.reflect.KType +import kotlin.reflect.full.isSubclassOf private val logger = KotlinLogging.logger {} @@ -181,7 +192,7 @@ private fun executeQueryAndBuildDataFrame( logger.debug { "Executing query: $sqlQuery" } statement.executeQuery().use { rs -> val tableColumns = getTableColumnsMetadata(rs, determinedDbType) - fetchAndConvertDataFromResultSet(tableColumns, rs, determinedDbType, limit, inferNullability) + fetchAndConvertDataFromResultSet(determinedDbType, tableColumns, rs, limit, inferNullability) } } } catch (e: java.sql.SQLException) { @@ -562,7 +573,7 @@ public fun DataFrame.Companion.readResultSet( ): AnyFrame { validateLimit(limit) val tableColumns = getTableColumnsMetadata(resultSet, dbType) - return fetchAndConvertDataFromResultSet(tableColumns, resultSet, dbType, limit, inferNullability) + return fetchAndConvertDataFromResultSet(dbType, tableColumns, resultSet, limit, inferNullability) } /** @@ -851,11 +862,23 @@ private fun readTableAsDataFrame( return dataFrame } -internal fun getTableColumnsMetadata(resultSet: ResultSet, dbType: DbType): MutableList = - dbType.getTableColumnsMetadata(resultSet).toMutableList() +internal fun getTableColumnsMetadata(resultSet: ResultSet, dbType: DbType): List = + dbType.getTableColumnsMetadata(resultSet) /** - * Fetches and converts data from a ResultSet into a mutable map. + * Fetches and converts data from a ResultSet into a [DataFrame]. + * + * Will handle data in the following order: + * + * - For each column (inside [readAndPreprocessRowsFromResultSet]): + * - Fetches each individual value from the [ResultSet] using [DbType.getValueFromResultSet] + * with the return type defined by [DbType.getExpectedJdbcType]. + * - Potentially preprocesses the value using [DbType.preprocessValue] + * with the return type defined by [DbType.getPreprocessedValueType]. + * - From the resulting list of columns,`List>`, (inside [buildDataFrameFromColumnData]): + * - Uses [DbType.buildDataColumn] to turn each list of values into a [DataColumn] + * with the correct structure defined by [DbType.getTargetColumnSchema]. + * - Turns the result into a [DataFrame]. * * @param [tableColumns] a list containing the column metadata for the table. * @param [rs] the ResultSet object containing the data to be fetched and converted. @@ -863,18 +886,46 @@ internal fun getTableColumnsMetadata(resultSet: ResultSet, dbType: DbType): Muta * @param [limit] the maximum number of rows to retrieve from the table. * `null` (default) means no limit - all available rows will be fetched. * @param [inferNullability] indicates how the column nullability should be inferred. - * @return A mutable map containing the fetched and converted data. + * @return A [DataFrame] containing the fetched and converted data. */ internal fun fetchAndConvertDataFromResultSet( - tableColumns: MutableList, - rs: ResultSet, dbType: DbType, + tableColumns: List, + rs: ResultSet, limit: Int?, inferNullability: Boolean, ): AnyFrame { - val columnKTypes = buildColumnKTypes(tableColumns, dbType) - val columnData = readAllRowsFromResultSet(rs, tableColumns, columnKTypes, dbType, limit) - val dataFrame = buildDataFrameFromColumnData(columnData, tableColumns, columnKTypes, dbType, inferNullability) + val expectedJdbcTypes = getExpectedJdbcTypes( + dbType = dbType, + tableColumns = tableColumns, + ) + val preprocessedValueTypes = getPreprocessedValueTypes( + dbType = dbType, + tableColumns = tableColumns, + expectedJdbcTypes = expectedJdbcTypes, + ) + val targetColumnSchemas = getTargetColumnSchemas( + dbType = dbType, + tableColumns = tableColumns, + preprocessedValueTypes = preprocessedValueTypes, + ) + + val columnData = readAndPreprocessRowsFromResultSet( + rs = rs, + tableColumns = tableColumns, + expectedJdbcTypes = expectedJdbcTypes, + preprocessedValueTypes = preprocessedValueTypes, + dbType = dbType, + limit = limit, + ) + + val dataFrame = buildDataFrameFromColumnData( + dbType = dbType, + tableColumns = tableColumns, + columnData = columnData, + targetColumnSchemas = targetColumnSchemas, + inferNullability = inferNullability, + ) logger.debug { "DataFrame with ${dataFrame.rowsCount()} rows and ${dataFrame.columnsCount()} columns created as a result of SQL query." @@ -883,38 +934,67 @@ internal fun fetchAndConvertDataFromResultSet( return dataFrame } -/** - * Builds a map of column indices to their Kotlin types. - */ -private fun buildColumnKTypes(tableColumns: List, dbType: DbType): Map = - tableColumns.indices.associateWith { index -> - generateKType(dbType, tableColumns[index]) +internal fun getExpectedJdbcTypes(dbType: DbType, tableColumns: List): List = + tableColumns.map { + dbType.getExpectedJdbcType(tableColumnMetadata = it) + } + +internal fun getPreprocessedValueTypes( + dbType: DbType, + tableColumns: List, + expectedJdbcTypes: List, +): List = + tableColumns.mapIndexed { index, it -> + dbType.getPreprocessedValueType( + tableColumnMetadata = it, + expectedJdbcType = expectedJdbcTypes[index], + ) + } + +internal fun getTargetColumnSchemas( + dbType: DbType, + tableColumns: List, + preprocessedValueTypes: List, +): List = + tableColumns.mapIndexed { index, it -> + dbType.getTargetColumnSchema( + tableColumnMetadata = it, + expectedValueType = preprocessedValueTypes[index], + ) } /** * Reads all rows from ResultSet and returns a column-oriented data structure. - * Returns mutable lists to allow efficient post-processing without copying. */ -private fun readAllRowsFromResultSet( +private fun readAndPreprocessRowsFromResultSet( + dbType: DbType, rs: ResultSet, tableColumns: List, - columnKTypes: Map, - dbType: DbType, + expectedJdbcTypes: List, + preprocessedValueTypes: List, limit: Int?, -): List> { - val columnsCount = tableColumns.size - val columnData = List(columnsCount) { mutableListOf() } +): List> { + val columnData = tableColumns.map { mutableListOf() }.toMutableList() var rowsRead = 0 while (rs.next() && (limit == null || rowsRead < limit)) { - repeat(columnsCount) { columnIndex -> - val value = dbType.extractValueFromResultSet( + tableColumns.forEachIndexed { index, tableColumnMetadata -> + val expectedJdbcType = expectedJdbcTypes[index] + val preprocessedValueType = preprocessedValueTypes[index] + + val value = dbType.getValueFromResultSet( rs = rs, - columnIndex = columnIndex, - columnMetadata = tableColumns[columnIndex], - kType = columnKTypes.getValue(columnIndex), + columnIndex = index, + tableColumnMetadata = tableColumnMetadata, + expectedJdbcType = expectedJdbcType, + ) + val preprocessedValue = dbType.preprocessValue( + value = value, + tableColumnMetadata = tableColumnMetadata, + expectedJdbcType = expectedJdbcType, + expectedPreprocessedValueType = preprocessedValueType, ) - columnData[columnIndex].add(value) + columnData[index] += preprocessedValue } rowsRead++ // if (rowsRead % 1000 == 0) logger.debug { "Loaded $rowsRead rows." } // TODO: https://github.com/Kotlin/dataframe/issues/455 @@ -928,29 +1008,91 @@ private fun readAllRowsFromResultSet( * Accepts mutable lists to enable efficient in-place transformations. */ private fun buildDataFrameFromColumnData( - columnData: List>, - tableColumns: List, - columnKTypes: Map, dbType: DbType, + tableColumns: List, + columnData: List>, + targetColumnSchemas: List, inferNullability: Boolean, + checkSchema: Boolean = BuildConfig.DEBUG, ): AnyFrame = - columnData.mapIndexed { index, values -> - dbType.buildDataColumn( - name = tableColumns[index].name, - values = values, - kType = columnKTypes.getValue(index), + tableColumns.mapIndexed { index, it -> + val column = dbType.buildDataColumn( + name = it.name, + values = columnData[index], + tableColumnMetadata = it, + targetColumnSchema = targetColumnSchemas[index], inferNullability = inferNullability, ) + + if (checkSchema) { + column.checkSchema(targetColumnSchemas[index]) + } + + column }.toDataFrame() -/** - * Generates a KType based on the given database type and table column metadata. - * - * @param dbType The database type. - * @param tableColumnMetadata The table column metadata. - * - * @return The generated KType. - */ -internal fun generateKType(dbType: DbType, tableColumnMetadata: TableColumnMetadata): KType = - dbType.convertSqlTypeToKType(tableColumnMetadata) - ?: dbType.makeCommonSqlToKTypeMapping(tableColumnMetadata) +private fun AnyCol.checkSchema(expected: ColumnSchema?) { + when (expected) { + null -> { + // nothing to check + } + + is ColumnSchema.Value -> { + require(this.isValueColumn()) { + """ + Found mismatching schema for column '${this.name()}'. + Column ${this.name()} is expected to be a value column of type ${expected.type} but it is ${this.type()}. + """.trimIndent() + } + require(values().all { it == null || it::class.isSubclassOf(expected.type.classifier as KClass<*>) }) { + """ + Found mismatching type for value column '${this.name()}'. + Expected type: ${expected.type} + Actual types: ${values().map { it?.javaClass?.name ?: "null" }.distinct()} + """.trimIndent() + } + } + + is ColumnSchema.Group -> { + require(this.isColumnGroup()) { + """ + Found mismatching schema for column '${name()}'. + Column ${this.name()} is expected to be a column group but it is ${this.type()}. + """.trimIndent() + } + require(expected.schema.compare(this.schema()).isSuperOrMatches()) { + """ + Found mismatching schema for column group '${name()}'. + Expected schema: + ${expected.schema} + + Actual schema: + ${this.schema()} + """.trimIndent() + } + } + + is ColumnSchema.Frame -> { + require(this.isFrameColumn()) { + """ + Found mismatching schema for column '${this.name()}'. + Column ${this.name()} is expected to be a frame column but it is ${this.type()}. + """.trimIndent() + } + require(values().all { expected.schema.compare(it.schema()).isSuperOrMatches() }) { + """ + Found mismatching schema for frame column '${this.name()}'. + Expected schema: + ${expected.schema} + + Actual (deviating) schemas: + ${ + values().map { it.schema() } + .distinct() + .filterNot { expected.schema.compare(it).isSuperOrMatches() } + } + """.trimIndent() + } + } + } +} diff --git a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/commonTestScenarios.kt b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/commonTestScenarios.kt index 2669f7e944..0c9e83aaa2 100644 --- a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/commonTestScenarios.kt +++ b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/commonTestScenarios.kt @@ -1,5 +1,11 @@ package org.jetbrains.kotlinx.dataframe.io +import io.kotest.assertions.Actual +import io.kotest.assertions.AssertionFailedError +import io.kotest.assertions.Exceptions +import io.kotest.assertions.Expected +import io.kotest.assertions.failure +import io.kotest.assertions.print.printed import io.kotest.assertions.withClue import io.kotest.matchers.shouldBe import org.intellij.lang.annotations.Language @@ -144,17 +150,21 @@ internal fun inferNullability(connection: Connection) { */ @Suppress("INVISIBLE_REFERENCE") fun AnyFrame.assertInferredTypesMatchSchema() { - withClue({ - """ - |Inferred schema must be <: Provided schema - | - |Inferred Schema: - |${inferType().schema().toString().lines().joinToString("\n|")} - | - |Provided Schema: - |${schema().toString().lines().joinToString("\n|")} - """.trimMargin() - }) { - schema().compare(inferType().schema()).isSuperOrMatches() shouldBe true + if (!schema().compare(inferType().schema()).isSuperOrMatches()) { + throw failure( + expected = Expected(inferType().schema().toString().lines().sorted().joinToString("\n").printed()), + actual = Actual(schema().toString().lines().sorted().joinToString("\n").printed()), + prependMessage = "Inferred schema must be <: Provided schema", + ) + } +} + +fun DataFrameSchema.assertMatches(other: DataFrameSchema) { + if (!this.compare(other).isSuperOrMatches()) { + throw failure( + expected = Expected(other.toString().lines().sorted().joinToString("\n").printed()), + actual = Actual(this.toString().lines().sorted().joinToString("\n").printed()), + prependMessage = "Schemas must be <:", + ) } } diff --git a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/h2/h2Test.kt b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/h2/h2Test.kt index 40c09fd04a..ee277e74af 100644 --- a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/h2/h2Test.kt +++ b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/h2/h2Test.kt @@ -933,7 +933,7 @@ class JdbcTest { val schema = DataFrameSchema.readSqlQuery(connection, sqlQuery) schema.columns.size shouldBe 2 schema.columns.toList()[0].first shouldBe "name" - schema.columns.toList()[1].first shouldBe "name_1" + schema.columns.toList()[1].first shouldBe "name1" } @Test @@ -949,8 +949,8 @@ class JdbcTest { val schema = DataFrameSchema.readSqlQuery(connection, sqlQuery) schema.columns.size shouldBe 3 schema.columns.toList()[0].first shouldBe "name" - schema.columns.toList()[1].first shouldBe "name_1" - schema.columns.toList()[2].first shouldBe "name_2" + schema.columns.toList()[1].first shouldBe "name1" + schema.columns.toList()[2].first shouldBe "name2" } @Test diff --git a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/h2/mariadbH2Test.kt b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/h2/mariadbH2Test.kt index 0707000ef6..b4c99468f5 100644 --- a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/h2/mariadbH2Test.kt +++ b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/h2/mariadbH2Test.kt @@ -21,8 +21,10 @@ import java.sql.Blob import java.sql.Connection import java.sql.DriverManager import java.sql.SQLException +import java.sql.Timestamp import java.util.Date import kotlin.reflect.typeOf +import kotlin.time.Instant private const val URL = "jdbc:h2:mem:test1;DB_CLOSE_DELAY=-1;MODE=MariaDB;DATABASE_TO_LOWER=TRUE" @@ -230,8 +232,8 @@ class MariadbH2Test { st.setDouble(11, i * 10.0) st.setBigDecimal(12, BigDecimal(i * 10)) st.setDate(13, java.sql.Date(System.currentTimeMillis())) - st.setTimestamp(14, java.sql.Timestamp(System.currentTimeMillis())) - st.setTimestamp(15, java.sql.Timestamp(System.currentTimeMillis())) + st.setTimestamp(14, Timestamp(System.currentTimeMillis())) + st.setTimestamp(15, Timestamp(System.currentTimeMillis())) st.setTime(16, java.sql.Time(System.currentTimeMillis())) st.setInt(17, 2023) st.setString(18, "varcharValue$i") @@ -268,8 +270,8 @@ class MariadbH2Test { st.setDouble(11, i * 20.0) st.setBigDecimal(12, BigDecimal(i * 20)) st.setDate(13, java.sql.Date(System.currentTimeMillis())) - st.setTimestamp(14, java.sql.Timestamp(System.currentTimeMillis())) - st.setTimestamp(15, java.sql.Timestamp(System.currentTimeMillis())) + st.setTimestamp(14, Timestamp(System.currentTimeMillis())) + st.setTimestamp(15, Timestamp(System.currentTimeMillis())) st.setTime(16, java.sql.Time(System.currentTimeMillis())) st.setInt(17, 2023) st.setString(18, "varcharValue$i") @@ -316,8 +318,8 @@ class MariadbH2Test { schema.columns["longblobcol"]!!.type shouldBe typeOf() schema.columns["tinyblobcol"]!!.type shouldBe typeOf() schema.columns["datecol"]!!.type shouldBe typeOf() - schema.columns["datetimecol"]!!.type shouldBe typeOf() - schema.columns["timestampcol"]!!.type shouldBe typeOf() + schema.columns["datetimecol"]!!.type shouldBe typeOf() + schema.columns["timestampcol"]!!.type shouldBe typeOf() schema.columns["timecol"]!!.type shouldBe typeOf() schema.columns["yearcol"]!!.type shouldBe typeOf() diff --git a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/h2/mssqlH2Test.kt b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/h2/mssqlH2Test.kt index 3d37005b85..c937402545 100644 --- a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/h2/mssqlH2Test.kt +++ b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/h2/mssqlH2Test.kt @@ -21,6 +21,7 @@ import java.sql.SQLException import java.util.Date import java.util.UUID import kotlin.reflect.typeOf +import kotlin.time.Instant private const val URL = "jdbc:h2:mem:testmssql;DB_CLOSE_DELAY=-1;MODE=MSSQLServer;DATABASE_TO_UPPER=FALSE;CASE_INSENSITIVE_IDENTIFIERS=TRUE" @@ -31,34 +32,34 @@ interface Table1MSSSQL { val bigintColumn: Long val binaryColumn: ByteArray val bitColumn: Boolean - val charColumn: Char + val charColumn: String val dateColumn: Date - val datetime3Column: java.sql.Timestamp - val datetime2Column: java.sql.Timestamp + val datetime3Column: Instant + val datetime2Column: Instant val decimalColumn: BigDecimal val floatColumn: Double val imageColumn: ByteArray? val intColumn: Int val moneyColumn: BigDecimal - val ncharColumn: Char + val ncharColumn: String val ntextColumn: String val numericColumn: BigDecimal val nvarcharColumn: String val nvarcharMaxColumn: String val realColumn: Float - val smalldatetimeColumn: java.sql.Timestamp + val smalldatetimeColumn: Instant val smallintColumn: Int val smallmoneyColumn: BigDecimal val timeColumn: java.sql.Time - val timestampColumn: java.sql.Timestamp + val timestampColumn: Instant val tinyintColumn: Int val uniqueidentifierColumn: Char val varbinaryColumn: ByteArray val varbinaryMaxColumn: ByteArray val varcharColumn: String val varcharMaxColumn: String - val geometryColumn: String - val geographyColumn: String + val geometryColumn: ByteArray + val geographyColumn: ByteArray } class MSSQLH2Test { @@ -188,8 +189,8 @@ class MSSQLH2Test { schema.columns["bitColumn"]!!.type shouldBe typeOf() schema.columns["charColumn"]!!.type shouldBe typeOf() schema.columns["dateColumn"]!!.type shouldBe typeOf() - schema.columns["datetime3Column"]!!.type shouldBe typeOf() - schema.columns["datetime2Column"]!!.type shouldBe typeOf() + schema.columns["datetime3Column"]!!.type shouldBe typeOf() + schema.columns["datetime2Column"]!!.type shouldBe typeOf() schema.columns["decimalColumn"]!!.type shouldBe typeOf() schema.columns["floatColumn"]!!.type shouldBe typeOf() schema.columns["intColumn"]!!.type shouldBe typeOf() @@ -200,11 +201,11 @@ class MSSQLH2Test { schema.columns["nvarcharColumn"]!!.type shouldBe typeOf() schema.columns["nvarcharMaxColumn"]!!.type shouldBe typeOf() schema.columns["realColumn"]!!.type shouldBe typeOf() - schema.columns["smalldatetimeColumn"]!!.type shouldBe typeOf() + schema.columns["smalldatetimeColumn"]!!.type shouldBe typeOf() schema.columns["smallintColumn"]!!.type shouldBe typeOf() schema.columns["smallmoneyColumn"]!!.type shouldBe typeOf() schema.columns["timeColumn"]!!.type shouldBe typeOf() - schema.columns["timestampColumn"]!!.type shouldBe typeOf() + schema.columns["timestampColumn"]!!.type shouldBe typeOf() schema.columns["tinyintColumn"]!!.type shouldBe typeOf() schema.columns["varbinaryColumn"]!!.type shouldBe typeOf() schema.columns["varbinaryMaxColumn"]!!.type shouldBe typeOf() diff --git a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/h2/mysqlH2Test.kt b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/h2/mysqlH2Test.kt index c0a6cdbe14..86d6ebc5b2 100644 --- a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/h2/mysqlH2Test.kt +++ b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/h2/mysqlH2Test.kt @@ -20,8 +20,10 @@ import java.math.BigDecimal import java.sql.Connection import java.sql.DriverManager import java.sql.SQLException +import java.sql.Timestamp import java.util.Date import kotlin.reflect.typeOf +import kotlin.time.Instant // NOTE: the names of testing databases should be different to avoid collisions and should not contain the system names itself private const val URL = "jdbc:h2:mem:test2;DB_CLOSE_DELAY=-1;MODE=MySQL;DATABASE_TO_LOWER=TRUE" @@ -228,8 +230,8 @@ class MySqlH2Test { st.setDouble(11, i * 10.0) st.setBigDecimal(12, BigDecimal(i * 10)) st.setDate(13, java.sql.Date(System.currentTimeMillis())) - st.setTimestamp(14, java.sql.Timestamp(System.currentTimeMillis())) - st.setTimestamp(15, java.sql.Timestamp(System.currentTimeMillis())) + st.setTimestamp(14, Timestamp(System.currentTimeMillis())) + st.setTimestamp(15, Timestamp(System.currentTimeMillis())) st.setTime(16, java.sql.Time(System.currentTimeMillis())) st.setInt(17, 2023) st.setString(18, "varcharValue$i") @@ -265,8 +267,8 @@ class MySqlH2Test { st.setDouble(11, i * 20.0) st.setBigDecimal(12, BigDecimal(i * 20)) st.setDate(13, java.sql.Date(System.currentTimeMillis())) - st.setTimestamp(14, java.sql.Timestamp(System.currentTimeMillis())) - st.setTimestamp(15, java.sql.Timestamp(System.currentTimeMillis())) + st.setTimestamp(14, Timestamp(System.currentTimeMillis())) + st.setTimestamp(15, Timestamp(System.currentTimeMillis())) st.setTime(16, java.sql.Time(System.currentTimeMillis())) st.setInt(17, 2023) st.setString(18, "varcharValue$i") @@ -308,8 +310,8 @@ class MySqlH2Test { schema.columns["id"]!!.type shouldBe typeOf() schema.columns["textcol"]!!.type shouldBe typeOf() schema.columns["datecol"]!!.type shouldBe typeOf() - schema.columns["datetimecol"]!!.type shouldBe typeOf() - schema.columns["timestampcol"]!!.type shouldBe typeOf() + schema.columns["datetimecol"]!!.type shouldBe typeOf() + schema.columns["timestampcol"]!!.type shouldBe typeOf() schema.columns["timecol"]!!.type shouldBe typeOf() schema.columns["yearcol"]!!.type shouldBe typeOf() schema.columns["varbinarycol"]!!.type shouldBe typeOf() diff --git a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/h2/postgresH2Test.kt b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/h2/postgresH2Test.kt index d2b4bef65d..efd1425555 100644 --- a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/h2/postgresH2Test.kt +++ b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/h2/postgresH2Test.kt @@ -123,6 +123,18 @@ class PostgresH2Test { connection.createStatement().execute(createTableQuery.trimIndent()) + // Table with STRUCT/ROW column to verify JDBC Types.STRUCT handling + @Language("SQL") + val createTableWithStruct = + """ + CREATE TABLE IF NOT EXISTS table3 ( + id serial PRIMARY KEY, + structCol ROW(a INT, b VARCHAR(10)) not null + ) + """.trimIndent() + + connection.createStatement().execute(createTableWithStruct) + @Language("SQL") val insertData1 = """ @@ -196,6 +208,17 @@ class PostgresH2Test { st.executeUpdate() } } + + // Insert data into table3 with ROW/STRUCT literals + @Language("SQL") + val insertStructs = + """ + INSERT INTO table3 (structCol) VALUES + (ROW(1, 'X')), + (ROW(2, 'Y')), + (ROW(3, 'Z')) + """.trimIndent() + connection.createStatement().execute(insertStructs) } @AfterClass @@ -286,6 +309,19 @@ class PostgresH2Test { table2Df[0][4] shouldBe 1001 } + @Test + fun `read composite column from table`() { + val tableName3 = "table3" + val df3 = DataFrame.readSqlTable(connection, tableName3) + + // Validate row count + df3.rowsCount() shouldBe 3 + + // Validate schema type stays as Any for STRUCT (no special mapping yet) + val schema3 = DataFrameSchema.readSqlTable(connection, tableName3) + schema3.columns["structcol"]!!.type shouldBe typeOf() + } + @Test fun `read columns of different types to check type mapping`() { val tableName1 = "table1" diff --git a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/local/duckDbTest.kt b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/local/duckDbTest.kt index 3cc00e15a3..d1b68318b6 100644 --- a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/local/duckDbTest.kt +++ b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/local/duckDbTest.kt @@ -2,17 +2,21 @@ package org.jetbrains.kotlinx.dataframe.io.local +import io.kotest.assertions.withClue import io.kotest.matchers.shouldBe +import kotlinx.datetime.LocalDate +import kotlinx.datetime.LocalTime import org.duckdb.DuckDBConnection import org.duckdb.DuckDBResultSet -import org.duckdb.JsonNode import org.jetbrains.kotlinx.dataframe.AnyFrame import org.jetbrains.kotlinx.dataframe.DataFrame import org.jetbrains.kotlinx.dataframe.annotations.ColumnName import org.jetbrains.kotlinx.dataframe.annotations.DataSchema +import org.jetbrains.kotlinx.dataframe.api.DataRowSchema import org.jetbrains.kotlinx.dataframe.api.cast import org.jetbrains.kotlinx.dataframe.api.colsOf import org.jetbrains.kotlinx.dataframe.api.convert +import org.jetbrains.kotlinx.dataframe.api.dataFrameOf import org.jetbrains.kotlinx.dataframe.api.reorderColumnsByName import org.jetbrains.kotlinx.dataframe.api.schema import org.jetbrains.kotlinx.dataframe.api.single @@ -21,6 +25,7 @@ import org.jetbrains.kotlinx.dataframe.api.values import org.jetbrains.kotlinx.dataframe.api.with import org.jetbrains.kotlinx.dataframe.io.DbConnectionConfig import org.jetbrains.kotlinx.dataframe.io.assertInferredTypesMatchSchema +import org.jetbrains.kotlinx.dataframe.io.assertMatches import org.jetbrains.kotlinx.dataframe.io.db.DuckDb import org.jetbrains.kotlinx.dataframe.io.readAllSqlTables import org.jetbrains.kotlinx.dataframe.io.readDataFrame @@ -36,11 +41,11 @@ import java.nio.file.Files import java.sql.Blob import java.sql.DriverManager import java.sql.Timestamp -import java.time.LocalDate -import java.time.LocalTime -import java.time.OffsetDateTime -import java.util.UUID import kotlin.io.path.createTempDirectory +import kotlin.time.Instant +import kotlin.time.toKotlinInstant +import kotlin.uuid.Uuid +import java.time.OffsetDateTime as JavaOffsetDateTime private const val URL = "jdbc:duckdb:" @@ -58,14 +63,17 @@ class DuckDbTest { ) { companion object { val expected = listOf( - Person(1, "John Doe", 30, 50000.0, LocalDate.of(2020, 1, 15)), - Person(2, "Jane Smith", 28, 55000.0, LocalDate.of(2021, 3, 20)), - Person(3, "Bob Johnson", 35, 65000.0, LocalDate.of(2019, 11, 10)), - Person(4, "Alice Brown", 32, 60000.0, LocalDate.of(2020, 7, 1)), + Person(1, "John Doe", 30, 50000.0, LocalDate(2020, 1, 15)), + Person(2, "Jane Smith", 28, 55000.0, LocalDate(2021, 3, 20)), + Person(3, "Bob Johnson", 35, 65000.0, LocalDate(2019, 11, 10)), + Person(4, "Alice Brown", 32, 60000.0, LocalDate(2020, 7, 1)), ).toDataFrame() } } + @DataSchema + data class SomeJson(val key: String) + @DataSchema data class GeneralPurposeTypes( @ColumnName("bigint_col") @@ -91,7 +99,7 @@ class DuckDbTest { @ColumnName("date_col") val dateCol: LocalDate, @ColumnName("datetime_col") - val datetimeCol: Timestamp, + val datetimeCol: Instant, @ColumnName("decimal_col") val decimalCol: BigDecimal, @ColumnName("double_col") @@ -129,7 +137,7 @@ class DuckDbTest { @ColumnName("interval_col") val intervalCol: String, @ColumnName("json_col") - val jsonCol: JsonNode, + val jsonCol: SomeJson, @ColumnName("logical_col") val logicalCol: Boolean, @ColumnName("long_col") @@ -151,11 +159,11 @@ class DuckDbTest { @ColumnName("time_col") val timeCol: LocalTime, @ColumnName("timestamp_col") - val timestampCol: Timestamp, + val timestampCol: Instant, @ColumnName("timestamptz_col") - val timestamptzCol: OffsetDateTime, + val timestamptzCol: JavaOffsetDateTime, @ColumnName("timestampwtz_col") - val timestampwtzCol: OffsetDateTime, + val timestampwtzCol: JavaOffsetDateTime, @ColumnName("tinyint_col") val tinyintCol: Byte, @ColumnName("ubigint_col") @@ -179,7 +187,7 @@ class DuckDbTest { @ColumnName("utinyint_col") val utinyintCol: Short, @ColumnName("uuid_col") - val uuidCol: UUID, + val uuidCol: Uuid, @ColumnName("varbinary_col") val varbinaryCol: Blob, @ColumnName("varchar_col") @@ -199,7 +207,7 @@ class DuckDbTest { byteaCol = DuckDBResultSet.DuckDBBlobResult(ByteBuffer.wrap("DEADBEEF".toByteArray())), charCol = "test", dateCol = LocalDate.parse("2025-06-19"), - datetimeCol = Timestamp.valueOf("2025-06-19 12:34:56"), + datetimeCol = Timestamp.valueOf("2025-06-19 12:34:56").toInstant().toKotlinInstant(), decimalCol = BigDecimal("123.45"), doubleCol = 3.14159, enumCol = "female", @@ -218,7 +226,7 @@ class DuckDbTest { intCol = 2147483647, integerCol = 2147483647, intervalCol = "1 year", - jsonCol = JsonNode("{\"key\": \"value\"}"), + jsonCol = SomeJson("value"), logicalCol = true, longCol = 9223372036854775807L, numericCol = BigDecimal("123.45"), @@ -229,9 +237,9 @@ class DuckDbTest { stringCol = "test string", textCol = "test text", timeCol = LocalTime.parse("12:34:56"), - timestampCol = Timestamp.valueOf("2025-06-19 12:34:56"), - timestamptzCol = OffsetDateTime.parse("2025-06-19T12:34:56+02:00"), - timestampwtzCol = OffsetDateTime.parse("2025-06-19T12:34:56+02:00"), + timestampCol = Timestamp.valueOf("2025-06-19 12:34:56").toInstant().toKotlinInstant(), + timestamptzCol = JavaOffsetDateTime.parse("2025-06-19T12:34:56+02:00"), + timestampwtzCol = JavaOffsetDateTime.parse("2025-06-19T12:34:56+02:00"), tinyintCol = 127, ubigintCol = BigInteger("18446744073709551615"), uhugeintCol = BigInteger("340282366920938463463374607431768211455"), @@ -243,7 +251,7 @@ class DuckDbTest { uintCol = 4294967295L, usmallintCol = 65535, utinyintCol = 255, - uuidCol = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"), + uuidCol = Uuid.parse("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"), varbinaryCol = DuckDBResultSet.DuckDBBlobResult(ByteBuffer.wrap("DEADBEEF".toByteArray())), varcharCol = "test string", ), @@ -251,24 +259,34 @@ class DuckDbTest { } } + @DataSchema + data class NestedEntry(val i: Int, val j: String) : DataRowSchema + + @DataSchema + data class NullableNestedEntry(val i: Int?, val j: String?) : DataRowSchema + @DataSchema data class NestedTypes( + @ColumnName("test_col") + val testCol: Int, @ColumnName("ijstruct_col") - val ijstructCol: java.sql.Struct, + val ijstructCol: NestedEntry, + @ColumnName("ijstructlist_col") + val ijstructlistCol: DataFrame, @ColumnName("intarray_col") - val intarrayCol: java.sql.Array, + val intarrayCol: List, @ColumnName("intlist_col") - val intlistCol: java.sql.Array, + val intlistCol: List, @ColumnName("intstringmap_col") val intstringmapCol: Map, @ColumnName("intstrinstinggmap_col") val intstrinstinggmapCol: Map?>, @ColumnName("stringarray_col") - val stringarrayCol: java.sql.Array, + val stringarrayCol: List, @ColumnName("stringlist_col") - val stringlistCol: java.sql.Array, + val stringlistCol: List, @ColumnName("stringlistlist_col") - val stringlistlistCol: java.sql.Array, + val stringlistlistCol: List?>, @ColumnName("union_col") val unionCol: Any, ) @@ -310,7 +328,19 @@ class DuckDbTest { subset = DataFrame.readSqlQuery(connection, """SELECT test_table.name, test_table.age FROM test_table""") } - schema.compare(Person.expected.schema()).isSuperOrMatches() shouldBe true + withClue({ + """ + |Read schema must be <: expected schema + | + |Read Schema: + |${schema.toString().lines().joinToString("\n|")} + | + |expected Schema: + |${Person.expected.schema().toString().lines().joinToString("\n|")} + """.trimMargin() + }) { + schema.compare(Person.expected.schema()).isSuperOrMatches() shouldBe true + } df.cast(verify = true) shouldBe Person.expected df.assertInferredTypesMatchSchema() @@ -545,10 +575,12 @@ class DuckDbTest { df = DataFrame.readSqlTable(connection, "table1").reorderColumnsByName() } - schema.compare(GeneralPurposeTypes.expected.schema()).isSuperOrMatches() shouldBe true + GeneralPurposeTypes.expected.schema().toString().lines().sorted().joinToString("\n") + + schema.assertMatches(GeneralPurposeTypes.expected.schema()) // on some systems OffsetDateTime's get converted to UTC sometimes, let's compare them as Instant instead - fun AnyFrame.fixOffsetDateTime() = convert { colsOf() }.with { it.toInstant() } + fun AnyFrame.fixOffsetDateTime() = convert { colsOf() }.with { it.toInstant() } df.cast(verify = true).fixOffsetDateTime() shouldBe GeneralPurposeTypes.expected.fixOffsetDateTime() @@ -567,6 +599,7 @@ class DuckDbTest { connection.prepareStatement( """ CREATE TABLE IF NOT EXISTS table2 ( + test_col INTEGER, intarray_col INTEGER[3], stringarray_col VARCHAR[3], intlist_col INTEGER[], @@ -575,6 +608,7 @@ class DuckDbTest { intstringmap_col MAP(INTEGER, VARCHAR), intstrinstinggmap_col MAP(INTEGER, MAP(VARCHAR, VARCHAR)), ijstruct_col STRUCT(i INTEGER, j VARCHAR), + ijstructlist_col STRUCT(i INTEGER, j VARCHAR)[], union_col UNION(num INTEGER, text VARCHAR), ) """.trimIndent(), @@ -583,6 +617,7 @@ class DuckDbTest { connection.prepareStatement( """ INSERT INTO table2 VALUES ( + 1, -- int array_value(1, 2, NULL), -- int array array_value('a', 'ab', 'abc'), -- string array list_value(1, 2, 3), -- int list @@ -591,6 +626,7 @@ class DuckDbTest { MAP { 1: 'value1', 200: 'value2' }, -- int string map MAP { 1: MAP { 'value1': 'a', 'value2': 'b' }, 200: MAP { 'value1': 'c', 'value2': 'd' } }, -- int string string map { 'i': 42, 'j': 'answer' }, -- struct + list_value({ 'i': 42, 'j': 'answer' }, { 'i': 44, 'j': 'answer' }), -- struct list union_value(num := 2), -- union ) """.trimIndent(), @@ -606,19 +642,24 @@ class DuckDbTest { df as DataFrame df.single().let { - it[{ "intarray_col"() }].array shouldBe arrayOf(1, 2, null) - it[{ "stringarray_col"() }].array shouldBe arrayOf("a", "ab", "abc") - it[{ "intlist_col"() }].array shouldBe arrayOf(1, 2, 3) - it[{ "stringlist_col"() }].array shouldBe arrayOf("a", "ab", "abc") - (it[{ "stringlistlist_col"() }].array as Array<*>) - .map { (it as java.sql.Array?)?.array } shouldBe listOf(arrayOf("a", "ab"), arrayOf("abc"), null) - it[{ "intstringmap_col">() }] shouldBe mapOf(1 to "value1", 200 to "value2") - it[{ "intstrinstinggmap_col">>() }] shouldBe mapOf( + it["intarray_col"] shouldBe listOf(1, 2, null) + it["stringarray_col"] shouldBe listOf("a", "ab", "abc") + it["intlist_col"] shouldBe listOf(1, 2, 3) + it["stringlist_col"] shouldBe listOf("a", "ab", "abc") + it["stringlistlist_col"] shouldBe listOf(listOf("a", "ab"), listOf("abc"), null) + it["intstringmap_col"] shouldBe mapOf(1 to "value1", 200 to "value2") + it["intstrinstinggmap_col"] shouldBe mapOf( 1 to mapOf("value1" to "a", "value2" to "b"), 200 to mapOf("value1" to "c", "value2" to "d"), ) - it[{ "ijstruct_col"() }].attributes shouldBe arrayOf(42, "answer") - it[{ "union_col"() }] shouldBe 2 + it[{ "ijstruct_col"["i"]() }] shouldBe 42 + it[{ "ijstruct_col"["j"]() }] shouldBe "answer" + it[{ "ijstructlist_col">() }] shouldBe + dataFrameOf( + NestedEntry(42, "answer"), + NestedEntry(44, "answer"), + ) + it["union_col"] shouldBe 2 } } @@ -653,4 +694,41 @@ class DuckDbTest { Files.deleteIfExists(tempDir) } } + + @Test + fun `parse Struct types`() { + DuckDb.parseStructType( + "STRUCT(v VARCHAR, i INTEGER)", + ) shouldBe mapOf("v" to "VARCHAR", "i" to "INTEGER") + DuckDb.parseStructType( + "STRUCT(col1 MAP(INTEGER, VARCHAR), col2 INTEGER)", + ) shouldBe mapOf("col1" to "MAP(INTEGER, VARCHAR)", "col2" to "INTEGER") + DuckDb.parseStructType( + "STRUCT(col1 STRUCT(i INTEGER, j VARCHAR), col2 INTEGER)", + ) shouldBe mapOf("col1" to "STRUCT(i INTEGER, j VARCHAR)", "col2" to "INTEGER") + DuckDb.parseStructType( + "STRUCT(col1 INTEGER[], col2 VARCHAR[][])", + ) shouldBe mapOf("col1" to "INTEGER[]", "col2" to "VARCHAR[][]") + } + + @Test + fun `parse Map types`() { + DuckDb.parseMapTypes( + "MAP(INTEGER, VARCHAR)", + ) shouldBe ("INTEGER" to "VARCHAR") + DuckDb.parseMapTypes( + "MAP(INTEGER, MAP(VARCHAR, VARCHAR))", + ) shouldBe ("INTEGER" to "MAP(VARCHAR, VARCHAR)") + DuckDb.parseMapTypes( + "MAP(STRUCT(i INTEGER, j VARCHAR), VARCHAR)", + ) shouldBe ("STRUCT(i INTEGER, j VARCHAR)" to "VARCHAR") + } + + @Test + fun `parse List types`() { + DuckDb.parseListType("INTEGER[]") shouldBe "INTEGER" + DuckDb.parseListType("INTEGER[3]") shouldBe "INTEGER" + DuckDb.parseListType("VARCHAR[][]") shouldBe "VARCHAR[]" + DuckDb.parseListType("STRUCT(i INTEGER, j VARCHAR)[]") shouldBe "STRUCT(i INTEGER, j VARCHAR)" + } } diff --git a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/local/mariadbTest.kt b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/local/mariadbTest.kt index 4b5d3b13f5..a46ee11f11 100644 --- a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/local/mariadbTest.kt +++ b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/local/mariadbTest.kt @@ -24,6 +24,7 @@ import java.sql.DriverManager import java.sql.SQLException import java.util.Date import kotlin.reflect.typeOf +import kotlin.time.Instant private const val URL = "jdbc:mariadb://localhost:3307" private const val USER_NAME = "root" @@ -348,8 +349,8 @@ class MariadbTest { schema.columns["longblobCol"]!!.type shouldBe typeOf() schema.columns["tinyblobCol"]!!.type shouldBe typeOf() schema.columns["dateCol"]!!.type shouldBe typeOf() - schema.columns["datetimeCol"]!!.type shouldBe typeOf() - schema.columns["timestampCol"]!!.type shouldBe typeOf() + schema.columns["datetimeCol"]!!.type shouldBe typeOf() + schema.columns["timestampCol"]!!.type shouldBe typeOf() schema.columns["timeCol"]!!.type shouldBe typeOf() schema.columns["yearCol"]!!.type shouldBe typeOf() diff --git a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/local/mssqlTest.kt b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/local/mssqlTest.kt index 4c586f0e0f..b37d45ebc7 100644 --- a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/local/mssqlTest.kt +++ b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/local/mssqlTest.kt @@ -22,6 +22,7 @@ import java.sql.SQLException import java.util.Date import java.util.UUID import kotlin.reflect.typeOf +import kotlin.time.Instant private const val URL = "jdbc:sqlserver://localhost:1433;encrypt=true;trustServerCertificate=true" private const val USER_NAME = "root" @@ -36,8 +37,8 @@ interface Table1MSSSQL { val bitColumn: Boolean val charColumn: Char val dateColumn: Date - val datetime3Column: java.sql.Timestamp - val datetime2Column: java.sql.Timestamp + val datetime3Column: Instant + val datetime2Column: Instant val datetimeoffset2Column: String val decimalColumn: BigDecimal val floatColumn: Double @@ -50,11 +51,11 @@ interface Table1MSSSQL { val nvarcharColumn: String val nvarcharMaxColumn: String val realColumn: Float - val smalldatetimeColumn: java.sql.Timestamp + val smalldatetimeColumn: Instant val smallintColumn: Int val smallmoneyColumn: BigDecimal val timeColumn: java.sql.Time - val timestampColumn: java.sql.Timestamp + val timestampColumn: Instant val tinyintColumn: Int val uniqueidentifierColumn: Char val varbinaryColumn: ByteArray @@ -63,8 +64,8 @@ interface Table1MSSSQL { val varcharMaxColumn: String val xmlColumn: String val sqlvariantColumn: String - val geometryColumn: String - val geographyColumn: String + val geometryColumn: ByteArray + val geographyColumn: ByteArray } @Ignore @@ -227,37 +228,37 @@ class MSSQLTest { schema.columns["bigintColumn"]!!.type shouldBe typeOf() schema.columns["binaryColumn"]!!.type shouldBe typeOf() schema.columns["bitColumn"]!!.type shouldBe typeOf() - schema.columns["charColumn"]!!.type shouldBe typeOf() + schema.columns["charColumn"]!!.type shouldBe typeOf() schema.columns["dateColumn"]!!.type shouldBe typeOf() - schema.columns["datetime3Column"]!!.type shouldBe typeOf() - schema.columns["datetime2Column"]!!.type shouldBe typeOf() + schema.columns["datetime3Column"]!!.type shouldBe typeOf() + schema.columns["datetime2Column"]!!.type shouldBe typeOf() schema.columns["datetimeoffset2Column"]!!.type shouldBe typeOf() schema.columns["decimalColumn"]!!.type shouldBe typeOf() schema.columns["floatColumn"]!!.type shouldBe typeOf() schema.columns["imageColumn"]!!.type shouldBe typeOf() schema.columns["intColumn"]!!.type shouldBe typeOf() schema.columns["moneyColumn"]!!.type shouldBe typeOf() - schema.columns["ncharColumn"]!!.type shouldBe typeOf() + schema.columns["ncharColumn"]!!.type shouldBe typeOf() schema.columns["ntextColumn"]!!.type shouldBe typeOf() schema.columns["numericColumn"]!!.type shouldBe typeOf() schema.columns["nvarcharColumn"]!!.type shouldBe typeOf() schema.columns["nvarcharMaxColumn"]!!.type shouldBe typeOf() schema.columns["realColumn"]!!.type shouldBe typeOf() - schema.columns["smalldatetimeColumn"]!!.type shouldBe typeOf() + schema.columns["smalldatetimeColumn"]!!.type shouldBe typeOf() schema.columns["smallintColumn"]!!.type shouldBe typeOf() schema.columns["smallmoneyColumn"]!!.type shouldBe typeOf() schema.columns["timeColumn"]!!.type shouldBe typeOf() - schema.columns["timestampColumn"]!!.type shouldBe typeOf() + schema.columns["timestampColumn"]!!.type shouldBe typeOf() schema.columns["tinyintColumn"]!!.type shouldBe typeOf() - schema.columns["uniqueidentifierColumn"]!!.type shouldBe typeOf() + schema.columns["uniqueidentifierColumn"]!!.type shouldBe typeOf() schema.columns["varbinaryColumn"]!!.type shouldBe typeOf() schema.columns["varbinaryMaxColumn"]!!.type shouldBe typeOf() schema.columns["varcharColumn"]!!.type shouldBe typeOf() schema.columns["varcharMaxColumn"]!!.type shouldBe typeOf() schema.columns["xmlColumn"]!!.type shouldBe typeOf() schema.columns["sqlvariantColumn"]!!.type shouldBe typeOf() - schema.columns["geometryColumn"]!!.type shouldBe typeOf() - schema.columns["geographyColumn"]!!.type shouldBe typeOf() + schema.columns["geometryColumn"]!!.type shouldBe typeOf() + schema.columns["geographyColumn"]!!.type shouldBe typeOf() } @Test diff --git a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/local/mysqlTest.kt b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/local/mysqlTest.kt index cee90eb4a6..adc06bb33d 100644 --- a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/local/mysqlTest.kt +++ b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/local/mysqlTest.kt @@ -1,6 +1,7 @@ package org.jetbrains.kotlinx.dataframe.io.local import io.kotest.matchers.shouldBe +import kotlinx.datetime.LocalDateTime import org.intellij.lang.annotations.Language import org.jetbrains.kotlinx.dataframe.DataFrame import org.jetbrains.kotlinx.dataframe.annotations.DataSchema @@ -23,6 +24,7 @@ import java.sql.DriverManager import java.sql.SQLException import java.util.Date import kotlin.reflect.typeOf +import kotlin.time.Instant private const val URL = "jdbc:mysql://localhost:3306" private const val USER_NAME = "root" @@ -343,8 +345,8 @@ class MySqlTest { schema.columns["id"]!!.type shouldBe typeOf() schema.columns["textCol"]!!.type shouldBe typeOf() schema.columns["dateCol"]!!.type shouldBe typeOf() - schema.columns["datetimeCol"]!!.type shouldBe typeOf() - schema.columns["timestampCol"]!!.type shouldBe typeOf() + schema.columns["datetimeCol"]!!.type shouldBe typeOf() + schema.columns["timestampCol"]!!.type shouldBe typeOf() schema.columns["timeCol"]!!.type shouldBe typeOf() schema.columns["yearCol"]!!.type shouldBe typeOf() schema.columns["textCol"]!!.type shouldBe typeOf() diff --git a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/local/postgresTest.kt b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/local/postgresTest.kt index 024b0d781f..c1776ac75f 100644 --- a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/local/postgresTest.kt +++ b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/local/postgresTest.kt @@ -25,6 +25,7 @@ import org.postgresql.geometric.PGpath import org.postgresql.geometric.PGpoint import org.postgresql.geometric.PGpolygon import org.postgresql.util.PGInterval +import org.postgresql.util.PGmoney import org.postgresql.util.PGobject import java.math.BigDecimal import java.sql.Connection @@ -48,12 +49,12 @@ interface Table1 { val smallintcol: Int val bigserialcol: Long val booleancol: Boolean - val boxcol: String + val boxcol: PGbox val byteacol: ByteArray val charactercol: String val characterncol: String val charcol: String - val circlecol: String + val circlecol: PGcircle val datecol: java.sql.Date val doublecol: Double val integercol: Int? @@ -65,14 +66,14 @@ interface Table1 { @DataSchema interface Table2 { val id: Int - val linecol: org.postgresql.geometric.PGline - val lsegcol: String + val linecol: PGline + val lsegcol: PGlseg val macaddrcol: String - val moneycol: String + val moneycol: PGmoney val numericcol: BigDecimal - val pathcol: org.postgresql.geometric.PGpath - val pointcol: String - val polygoncol: String + val pathcol: PGpath + val pointcol: PGpoint + val polygoncol: PGpolygon val realcol: Float val smallintcol: Int val smallserialcol: Int @@ -298,7 +299,7 @@ class PostgresTest { schema.columns["id"]!!.type shouldBe typeOf() schema.columns["integercol"]!!.type shouldBe typeOf() schema.columns["smallintcol"]!!.type shouldBe typeOf() - schema.columns["circlecol"]!!.type shouldBe typeOf() + schema.columns["circlecol"]!!.type shouldBe typeOf() schema.columns["intarraycol"]!!.type.classifier shouldBe kotlin.Array::class schema.columns["doublearraycol"]!!.type.classifier shouldBe kotlin.Array::class schema.columns["datearraycol"]!!.type.classifier shouldBe kotlin.Array::class @@ -313,9 +314,9 @@ class PostgresTest { val schema2 = DataFrameSchema.readSqlTable(connection, tableName2) schema2.columns["id"]!!.type shouldBe typeOf() - schema2.columns["pathcol"]!!.type shouldBe typeOf() // TODO: https://github.com/Kotlin/dataframe/issues/537 + schema2.columns["pathcol"]!!.type shouldBe typeOf() schema2.columns["textcol"]!!.type shouldBe typeOf() - schema2.columns["linecol"]!!.type shouldBe typeOf() // TODO: https://github.com/Kotlin/dataframe/issues/537 + schema2.columns["linecol"]!!.type shouldBe typeOf() } @Test @@ -357,7 +358,7 @@ class PostgresTest { table2Df.rowsCount() shouldBe 3 table2Df.filter { - it[Table2::pathcol] == org.postgresql.geometric.PGpath("((1,2),(3,1))") + it[Table2::pathcol] == PGpath("((1,2),(3,1))") }.rowsCount() shouldBe 1 table2Df[0][11] shouldBe 1001 } @@ -383,7 +384,7 @@ class PostgresTest { val result3 = df2.select("moneycol") .add("moneycol2") { it[Table2::moneycol] } - result3[0][1] shouldBe "123,45 ?" // TODO: weird mapping + (result3[0][1] as PGmoney).`val` shouldBe 123.45 val result4 = df2.select("numericcol") .add("numericcol2") { it[Table2::numericcol] } @@ -407,7 +408,7 @@ class PostgresTest { schema.columns["doublecol"]!!.type shouldBe typeOf() val schema1 = DataFrameSchema.readSqlTable(connection, tableName2) - schema1.columns["moneycol"]!!.type shouldBe typeOf() + schema1.columns["moneycol"]!!.type shouldBe typeOf() schema1.columns["numericcol"]!!.type shouldBe typeOf() schema1.columns["realcol"]!!.type shouldBe typeOf() schema1.columns["smallserialcol"]!!.type shouldBe typeOf() diff --git a/dataframe-jupyter/build.gradle.kts b/dataframe-jupyter/build.gradle.kts index b147a82aa1..c07d260ef2 100644 --- a/dataframe-jupyter/build.gradle.kts +++ b/dataframe-jupyter/build.gradle.kts @@ -3,6 +3,7 @@ import org.jetbrains.kotlin.gradle.tasks.KotlinCompile plugins { with(convention.plugins) { alias(kotlinJvm11) + alias(buildConfig) } with(libs.plugins) { alias(publisher) diff --git a/dataframe-jupyter/src/test/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/SampleNotebooksTests.kt b/dataframe-jupyter/src/test/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/SampleNotebooksTests.kt index d935473f3f..225ee6d8db 100644 --- a/dataframe-jupyter/src/test/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/SampleNotebooksTests.kt +++ b/dataframe-jupyter/src/test/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/SampleNotebooksTests.kt @@ -3,7 +3,7 @@ package org.jetbrains.kotlinx.dataframe.jupyter import org.jetbrains.jupyter.parser.JupyterParser import org.jetbrains.jupyter.parser.notebook.CodeCell import org.jetbrains.jupyter.parser.notebook.Output -import org.jetbrains.kotlinx.dataframe.BuildConfig +import org.jetbrains.kotlinx.dataframe.`dataframe-jupyter`.BuildConfig import org.jetbrains.kotlinx.jupyter.exceptions.causesSequence import org.jetbrains.kotlinx.jupyter.repl.result.EvalResultEx import org.junit.Assume diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index dea4f54dad..70f1307677 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -55,7 +55,7 @@ android-gradle-api = "7.3.1" # need to revise our tests to update ktor = "3.0.1" # needs jupyter compatibility with Kotlin 2.1 to update kotlin-compile-testing = "0.7.1" hikari = "7.0.2" -duckdb = "1.3.1.0" +duckdb = "1.4.2.0" buildconfig = "5.6.7" benchmark = "0.4.12" diff --git a/plugins/keywords-generator/src/main/kotlin/org/jetbrains/dataframe/keywords/KeywordsGeneratorPlugin.kt b/plugins/keywords-generator/src/main/kotlin/org/jetbrains/dataframe/keywords/KeywordsGeneratorPlugin.kt index d5f0a747d6..981f9338f2 100644 --- a/plugins/keywords-generator/src/main/kotlin/org/jetbrains/dataframe/keywords/KeywordsGeneratorPlugin.kt +++ b/plugins/keywords-generator/src/main/kotlin/org/jetbrains/dataframe/keywords/KeywordsGeneratorPlugin.kt @@ -8,7 +8,7 @@ import org.gradle.api.tasks.SourceSet import org.gradle.api.tasks.SourceSetContainer import org.gradle.kotlin.dsl.get import org.gradle.kotlin.dsl.register -import org.jetbrains.kotlinx.dataframe.BuildConfig +import org.jetbrains.kotlinx.dataframe.`keywords-generator`.BuildConfig import java.io.File @Suppress("UnstableApiUsage")