diff --git a/core/api/core.api b/core/api/core.api index 9a0e9adbe6..3b232abcee 100644 --- a/core/api/core.api +++ b/core/api/core.api @@ -6808,11 +6808,15 @@ public final class org/jetbrains/kotlinx/dataframe/schema/ComparisonMode : java/ } public abstract interface class org/jetbrains/kotlinx/dataframe/schema/DataFrameSchema { + public static final field Companion Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema$Companion; public abstract fun compare (Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema;Lorg/jetbrains/kotlinx/dataframe/schema/ComparisonMode;)Lorg/jetbrains/kotlinx/dataframe/schema/CompareResult; public static synthetic fun compare$default (Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema;Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema;Lorg/jetbrains/kotlinx/dataframe/schema/ComparisonMode;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/schema/CompareResult; public abstract fun getColumns ()Ljava/util/Map; } +public final class org/jetbrains/kotlinx/dataframe/schema/DataFrameSchema$Companion { +} + public final class org/jetbrains/kotlinx/dataframe/util/DeprecationMessagesKt { public static final field DF_READ_EXCEL Ljava/lang/String; } diff --git a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/schema/DataFrameSchema.kt b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/schema/DataFrameSchema.kt index e0e112eef0..e0bd4248fc 100644 --- a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/schema/DataFrameSchema.kt +++ b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/schema/DataFrameSchema.kt @@ -1,6 +1,7 @@ package org.jetbrains.kotlinx.dataframe.schema public interface DataFrameSchema { + public companion object; public val columns: Map diff --git a/dataframe-jdbc/api/dataframe-jdbc.api b/dataframe-jdbc/api/dataframe-jdbc.api index 0a0fbf50f5..96edb36662 100644 --- a/dataframe-jdbc/api/dataframe-jdbc.api +++ b/dataframe-jdbc/api/dataframe-jdbc.api @@ -1,10 +1,6 @@ public final class org/jetbrains/kotlinx/dataframe/io/DbConnectionConfig { public fun (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Z)V public synthetic fun (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;ZILkotlin/jvm/internal/DefaultConstructorMarker;)V - public final fun component1 ()Ljava/lang/String; - public final fun component2 ()Ljava/lang/String; - public final fun component3 ()Ljava/lang/String; - public final fun component4 ()Z public final fun copy (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Z)Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig; public static synthetic fun copy$default (Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;ZILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig; public fun equals (Ljava/lang/Object;)Z @@ -33,110 +29,100 @@ public final class org/jetbrains/kotlinx/dataframe/io/JdbcSchemaKt { public static final fun getDatabaseCodeGenReader (Lorg/jetbrains/kotlinx/dataframe/codeGen/CodeGenerator$Companion;)Lkotlin/jvm/functions/Function2; } -public final class org/jetbrains/kotlinx/dataframe/io/ReadJdbcKt { - public static final fun getDataFrameSchema (Ljava/sql/Connection;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema; - public static final fun getDataFrameSchema (Ljava/sql/ResultSet;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema; - public static final fun getDataFrameSchema (Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema; - public static synthetic fun getDataFrameSchema$default (Ljava/sql/Connection;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema; - public static synthetic fun getDataFrameSchema$default (Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema; - public static final fun getSchemaForAllSqlTables (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/sql/Connection;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;)Ljava/util/Map; - public static final fun getSchemaForAllSqlTables (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;)Ljava/util/Map; - public static synthetic fun getSchemaForAllSqlTables$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/sql/Connection;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;ILjava/lang/Object;)Ljava/util/Map; - public static synthetic fun getSchemaForAllSqlTables$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;ILjava/lang/Object;)Ljava/util/Map; - public static final fun getSchemaForResultSet (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/sql/ResultSet;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema; - public static final fun getSchemaForSqlQuery (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/sql/Connection;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema; - public static final fun getSchemaForSqlQuery (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema; - public static synthetic fun getSchemaForSqlQuery$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/sql/Connection;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema; - public static synthetic fun getSchemaForSqlQuery$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema; - public static final fun getSchemaForSqlTable (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/sql/Connection;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema; - public static final fun getSchemaForSqlTable (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema; - public static synthetic fun getSchemaForSqlTable$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/sql/Connection;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema; - public static synthetic fun getSchemaForSqlTable$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema; - public static final fun readAllSqlTables (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/sql/Connection;Ljava/lang/String;IZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;)Ljava/util/Map; - public static final fun readAllSqlTables (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Ljava/lang/String;IZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;)Ljava/util/Map; - public static synthetic fun readAllSqlTables$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/sql/Connection;Ljava/lang/String;IZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;ILjava/lang/Object;)Ljava/util/Map; - public static synthetic fun readAllSqlTables$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Ljava/lang/String;IZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;ILjava/lang/Object;)Ljava/util/Map; - public static final fun readDataFrame (Ljava/sql/Connection;Ljava/lang/String;IZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;Z)Lorg/jetbrains/kotlinx/dataframe/DataFrame; - public static final fun readDataFrame (Ljava/sql/ResultSet;Ljava/sql/Connection;IZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;)Lorg/jetbrains/kotlinx/dataframe/DataFrame; - public static final fun readDataFrame (Ljava/sql/ResultSet;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;IZ)Lorg/jetbrains/kotlinx/dataframe/DataFrame; - public static final fun readDataFrame (Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Ljava/lang/String;IZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;Z)Lorg/jetbrains/kotlinx/dataframe/DataFrame; - public static synthetic fun readDataFrame$default (Ljava/sql/Connection;Ljava/lang/String;IZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;ZILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataFrame; - public static synthetic fun readDataFrame$default (Ljava/sql/ResultSet;Ljava/sql/Connection;IZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataFrame; - public static synthetic fun readDataFrame$default (Ljava/sql/ResultSet;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;IZILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataFrame; - public static synthetic fun readDataFrame$default (Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Ljava/lang/String;IZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;ZILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataFrame; - public static final fun readResultSet (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/sql/ResultSet;Ljava/sql/Connection;IZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;)Lorg/jetbrains/kotlinx/dataframe/DataFrame; - public static final fun readResultSet (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/sql/ResultSet;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;IZ)Lorg/jetbrains/kotlinx/dataframe/DataFrame; - public static synthetic fun readResultSet$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/sql/ResultSet;Ljava/sql/Connection;IZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataFrame; - public static synthetic fun readResultSet$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/sql/ResultSet;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;IZILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataFrame; - public static final fun readSqlQuery (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/sql/Connection;Ljava/lang/String;IZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;Z)Lorg/jetbrains/kotlinx/dataframe/DataFrame; - public static final fun readSqlQuery (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Ljava/lang/String;IZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;Z)Lorg/jetbrains/kotlinx/dataframe/DataFrame; - public static synthetic fun readSqlQuery$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/sql/Connection;Ljava/lang/String;IZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;ZILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataFrame; - public static synthetic fun readSqlQuery$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Ljava/lang/String;IZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;ZILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataFrame; - public static final fun readSqlTable (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/sql/Connection;Ljava/lang/String;IZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;Z)Lorg/jetbrains/kotlinx/dataframe/DataFrame; - public static final fun readSqlTable (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Ljava/lang/String;IZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;Z)Lorg/jetbrains/kotlinx/dataframe/DataFrame; - public static synthetic fun readSqlTable$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/sql/Connection;Ljava/lang/String;IZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;ZILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataFrame; - public static synthetic fun readSqlTable$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Ljava/lang/String;IZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;ZILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataFrame; -} - -public final class org/jetbrains/kotlinx/dataframe/io/TableColumnMetadata { - public fun (Ljava/lang/String;Ljava/lang/String;IILjava/lang/String;Z)V - public synthetic fun (Ljava/lang/String;Ljava/lang/String;IILjava/lang/String;ZILkotlin/jvm/internal/DefaultConstructorMarker;)V - public final fun component1 ()Ljava/lang/String; - public final fun component2 ()Ljava/lang/String; - public final fun component3 ()I - public final fun component4 ()I - public final fun component5 ()Ljava/lang/String; - public final fun component6 ()Z - public final fun copy (Ljava/lang/String;Ljava/lang/String;IILjava/lang/String;Z)Lorg/jetbrains/kotlinx/dataframe/io/TableColumnMetadata; - public static synthetic fun copy$default (Lorg/jetbrains/kotlinx/dataframe/io/TableColumnMetadata;Ljava/lang/String;Ljava/lang/String;IILjava/lang/String;ZILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/io/TableColumnMetadata; - public fun equals (Ljava/lang/Object;)Z - public final fun getJavaClassName ()Ljava/lang/String; - public final fun getJdbcType ()I - public final fun getName ()Ljava/lang/String; - public final fun getSize ()I - public final fun getSqlTypeName ()Ljava/lang/String; - public fun hashCode ()I - public final fun isNullable ()Z - public fun toString ()Ljava/lang/String; +public final class org/jetbrains/kotlinx/dataframe/io/ReadDataFrameSchemaKt { + public static final fun readAllSqlTables (Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema$Companion;Ljava/sql/Connection;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;)Ljava/util/Map; + public static final fun readAllSqlTables (Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema$Companion;Ljavax/sql/DataSource;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;)Ljava/util/Map; + public static final fun readAllSqlTables (Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema$Companion;Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;)Ljava/util/Map; + public static synthetic fun readAllSqlTables$default (Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema$Companion;Ljava/sql/Connection;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;ILjava/lang/Object;)Ljava/util/Map; + public static synthetic fun readAllSqlTables$default (Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema$Companion;Ljavax/sql/DataSource;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;ILjava/lang/Object;)Ljava/util/Map; + public static synthetic fun readAllSqlTables$default (Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema$Companion;Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;ILjava/lang/Object;)Ljava/util/Map; + public static final fun readDataFrameSchema (Ljava/sql/Connection;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema; + public static final fun readDataFrameSchema (Ljava/sql/ResultSet;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema; + public static final fun readDataFrameSchema (Ljavax/sql/DataSource;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema; + public static final fun readDataFrameSchema (Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema; + public static synthetic fun readDataFrameSchema$default (Ljava/sql/Connection;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema; + public static synthetic fun readDataFrameSchema$default (Ljavax/sql/DataSource;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema; + public static synthetic fun readDataFrameSchema$default (Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema; + public static final fun readResultSet (Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema$Companion;Ljava/sql/ResultSet;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema; + public static final fun readSqlQuery (Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema$Companion;Ljava/sql/Connection;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema; + public static final fun readSqlQuery (Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema$Companion;Ljavax/sql/DataSource;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema; + public static final fun readSqlQuery (Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema$Companion;Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema; + public static synthetic fun readSqlQuery$default (Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema$Companion;Ljava/sql/Connection;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema; + public static synthetic fun readSqlQuery$default (Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema$Companion;Ljavax/sql/DataSource;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema; + public static synthetic fun readSqlQuery$default (Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema$Companion;Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema; + public static final fun readSqlTable (Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema$Companion;Ljava/sql/Connection;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema; + public static final fun readSqlTable (Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema$Companion;Ljavax/sql/DataSource;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema; + public static final fun readSqlTable (Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema$Companion;Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema; + public static synthetic fun readSqlTable$default (Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema$Companion;Ljava/sql/Connection;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema; + public static synthetic fun readSqlTable$default (Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema$Companion;Ljavax/sql/DataSource;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema; + public static synthetic fun readSqlTable$default (Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema$Companion;Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema; } -public final class org/jetbrains/kotlinx/dataframe/io/TableMetadata { - public fun (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)V - public final fun component1 ()Ljava/lang/String; - public final fun component2 ()Ljava/lang/String; - public final fun component3 ()Ljava/lang/String; - public final fun copy (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)Lorg/jetbrains/kotlinx/dataframe/io/TableMetadata; - public static synthetic fun copy$default (Lorg/jetbrains/kotlinx/dataframe/io/TableMetadata;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/io/TableMetadata; - public fun equals (Ljava/lang/Object;)Z - public final fun getCatalogue ()Ljava/lang/String; - public final fun getName ()Ljava/lang/String; - public final fun getSchemaName ()Ljava/lang/String; - public fun hashCode ()I - public fun toString ()Ljava/lang/String; +public final class org/jetbrains/kotlinx/dataframe/io/ReadJdbcKt { + public static final fun readAllSqlTables (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/sql/Connection;Ljava/lang/String;Ljava/lang/Integer;ZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;Lkotlin/jvm/functions/Function1;)Ljava/util/Map; + public static final fun readAllSqlTables (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljavax/sql/DataSource;Ljava/lang/String;Ljava/lang/Integer;ZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;Lkotlin/jvm/functions/Function1;)Ljava/util/Map; + public static final fun readAllSqlTables (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Ljava/lang/String;Ljava/lang/Integer;ZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;Lkotlin/jvm/functions/Function1;)Ljava/util/Map; + public static synthetic fun readAllSqlTables$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/sql/Connection;Ljava/lang/String;Ljava/lang/Integer;ZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)Ljava/util/Map; + public static synthetic fun readAllSqlTables$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljavax/sql/DataSource;Ljava/lang/String;Ljava/lang/Integer;ZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)Ljava/util/Map; + public static synthetic fun readAllSqlTables$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Ljava/lang/String;Ljava/lang/Integer;ZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)Ljava/util/Map; + public static final fun readDataFrame (Ljava/sql/Connection;Ljava/lang/String;Ljava/lang/Integer;ZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;ZLkotlin/jvm/functions/Function1;)Lorg/jetbrains/kotlinx/dataframe/DataFrame; + public static final fun readDataFrame (Ljava/sql/ResultSet;Ljava/sql/Connection;Ljava/lang/Integer;ZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;)Lorg/jetbrains/kotlinx/dataframe/DataFrame; + public static final fun readDataFrame (Ljava/sql/ResultSet;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;Ljava/lang/Integer;Z)Lorg/jetbrains/kotlinx/dataframe/DataFrame; + public static final fun readDataFrame (Ljavax/sql/DataSource;Ljava/lang/String;Ljava/lang/Integer;ZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;ZLkotlin/jvm/functions/Function1;)Lorg/jetbrains/kotlinx/dataframe/DataFrame; + public static final fun readDataFrame (Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Ljava/lang/String;Ljava/lang/Integer;ZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;ZLkotlin/jvm/functions/Function1;)Lorg/jetbrains/kotlinx/dataframe/DataFrame; + public static synthetic fun readDataFrame$default (Ljava/sql/Connection;Ljava/lang/String;Ljava/lang/Integer;ZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;ZLkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataFrame; + public static synthetic fun readDataFrame$default (Ljava/sql/ResultSet;Ljava/sql/Connection;Ljava/lang/Integer;ZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataFrame; + public static synthetic fun readDataFrame$default (Ljava/sql/ResultSet;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;Ljava/lang/Integer;ZILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataFrame; + public static synthetic fun readDataFrame$default (Ljavax/sql/DataSource;Ljava/lang/String;Ljava/lang/Integer;ZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;ZLkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataFrame; + public static synthetic fun readDataFrame$default (Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Ljava/lang/String;Ljava/lang/Integer;ZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;ZLkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataFrame; + public static final fun readResultSet (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/sql/ResultSet;Ljava/sql/Connection;Ljava/lang/Integer;ZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;)Lorg/jetbrains/kotlinx/dataframe/DataFrame; + public static final fun readResultSet (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/sql/ResultSet;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;Ljava/lang/Integer;Z)Lorg/jetbrains/kotlinx/dataframe/DataFrame; + public static synthetic fun readResultSet$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/sql/ResultSet;Ljava/sql/Connection;Ljava/lang/Integer;ZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataFrame; + public static synthetic fun readResultSet$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/sql/ResultSet;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;Ljava/lang/Integer;ZILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataFrame; + public static final fun readSqlQuery (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/sql/Connection;Ljava/lang/String;Ljava/lang/Integer;ZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;ZLkotlin/jvm/functions/Function1;)Lorg/jetbrains/kotlinx/dataframe/DataFrame; + public static final fun readSqlQuery (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljavax/sql/DataSource;Ljava/lang/String;Ljava/lang/Integer;ZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;ZLkotlin/jvm/functions/Function1;)Lorg/jetbrains/kotlinx/dataframe/DataFrame; + public static final fun readSqlQuery (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Ljava/lang/String;Ljava/lang/Integer;ZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;ZLkotlin/jvm/functions/Function1;)Lorg/jetbrains/kotlinx/dataframe/DataFrame; + public static synthetic fun readSqlQuery$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/sql/Connection;Ljava/lang/String;Ljava/lang/Integer;ZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;ZLkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataFrame; + public static synthetic fun readSqlQuery$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljavax/sql/DataSource;Ljava/lang/String;Ljava/lang/Integer;ZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;ZLkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataFrame; + public static synthetic fun readSqlQuery$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Ljava/lang/String;Ljava/lang/Integer;ZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;ZLkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataFrame; + public static final fun readSqlTable (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/sql/Connection;Ljava/lang/String;Ljava/lang/Integer;ZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;ZLkotlin/jvm/functions/Function1;)Lorg/jetbrains/kotlinx/dataframe/DataFrame; + public static final fun readSqlTable (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljavax/sql/DataSource;Ljava/lang/String;Ljava/lang/Integer;ZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;ZLkotlin/jvm/functions/Function1;)Lorg/jetbrains/kotlinx/dataframe/DataFrame; + public static final fun readSqlTable (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Ljava/lang/String;Ljava/lang/Integer;ZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;ZLkotlin/jvm/functions/Function1;)Lorg/jetbrains/kotlinx/dataframe/DataFrame; + public static synthetic fun readSqlTable$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/sql/Connection;Ljava/lang/String;Ljava/lang/Integer;ZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;ZLkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataFrame; + public static synthetic fun readSqlTable$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljavax/sql/DataSource;Ljava/lang/String;Ljava/lang/Integer;ZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;ZLkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataFrame; + public static synthetic fun readSqlTable$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Ljava/lang/String;Ljava/lang/Integer;ZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;ZLkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataFrame; } public abstract class org/jetbrains/kotlinx/dataframe/io/db/DbType { public fun (Ljava/lang/String;)V - public abstract fun buildTableMetadata (Ljava/sql/ResultSet;)Lorg/jetbrains/kotlinx/dataframe/io/TableMetadata; - public abstract fun convertSqlTypeToColumnSchemaValue (Lorg/jetbrains/kotlinx/dataframe/io/TableColumnMetadata;)Lorg/jetbrains/kotlinx/dataframe/schema/ColumnSchema; - public abstract fun convertSqlTypeToKType (Lorg/jetbrains/kotlinx/dataframe/io/TableColumnMetadata;)Lkotlin/reflect/KType; + public fun buildDataColumn (Ljava/lang/String;Ljava/util/List;Lkotlin/reflect/KType;Z)Lorg/jetbrains/kotlinx/dataframe/DataColumn; + public fun buildSelectTableQueryWithLimit (Ljava/lang/String;Ljava/lang/Integer;)Ljava/lang/String; + public fun buildSqlQueryWithLimit (Ljava/lang/String;I)Ljava/lang/String; + public static synthetic fun buildSqlQueryWithLimit$default (Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;Ljava/lang/String;IILjava/lang/Object;)Ljava/lang/String; + public abstract fun buildTableMetadata (Ljava/sql/ResultSet;)Lorg/jetbrains/kotlinx/dataframe/io/db/TableMetadata; + public fun configureReadStatement (Ljava/sql/PreparedStatement;)V + public abstract fun convertSqlTypeToColumnSchemaValue (Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;)Lorg/jetbrains/kotlinx/dataframe/schema/ColumnSchema; + public abstract fun convertSqlTypeToKType (Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;)Lkotlin/reflect/KType; public fun createConnection (Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;)Ljava/sql/Connection; + public fun extractValueFromResultSet (Ljava/sql/ResultSet;ILorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;Lkotlin/reflect/KType;)Ljava/lang/Object; public final fun getDbTypeInJdbcUrl ()Ljava/lang/String; + public fun getDefaultFetchSize ()I + public fun getDefaultQueryTimeout ()Ljava/lang/Integer; public abstract fun getDriverClassName ()Ljava/lang/String; public fun getTableTypes ()Ljava/util/List; - public abstract fun isSystemTable (Lorg/jetbrains/kotlinx/dataframe/io/TableMetadata;)Z - public fun sqlQueryLimit (Ljava/lang/String;I)Ljava/lang/String; - public static synthetic fun sqlQueryLimit$default (Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;Ljava/lang/String;IILjava/lang/Object;)Ljava/lang/String; + public abstract fun isSystemTable (Lorg/jetbrains/kotlinx/dataframe/io/db/TableMetadata;)Z + public fun makeCommonSqlToKTypeMapping (Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;)Lkotlin/reflect/KType; + public fun quoteIdentifier (Ljava/lang/String;)Ljava/lang/String; } public final class org/jetbrains/kotlinx/dataframe/io/db/DuckDb : org/jetbrains/kotlinx/dataframe/io/db/DbType { public static final field INSTANCE Lorg/jetbrains/kotlinx/dataframe/io/db/DuckDb; - public fun buildTableMetadata (Ljava/sql/ResultSet;)Lorg/jetbrains/kotlinx/dataframe/io/TableMetadata; - public fun convertSqlTypeToColumnSchemaValue (Lorg/jetbrains/kotlinx/dataframe/io/TableColumnMetadata;)Lorg/jetbrains/kotlinx/dataframe/schema/ColumnSchema; - public fun convertSqlTypeToKType (Lorg/jetbrains/kotlinx/dataframe/io/TableColumnMetadata;)Lkotlin/reflect/KType; + public fun buildTableMetadata (Ljava/sql/ResultSet;)Lorg/jetbrains/kotlinx/dataframe/io/db/TableMetadata; + public fun convertSqlTypeToColumnSchemaValue (Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;)Lorg/jetbrains/kotlinx/dataframe/schema/ColumnSchema; + public fun convertSqlTypeToKType (Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;)Lkotlin/reflect/KType; public fun createConnection (Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;)Ljava/sql/Connection; public fun getDriverClassName ()Ljava/lang/String; - public fun isSystemTable (Lorg/jetbrains/kotlinx/dataframe/io/TableMetadata;)Z + public fun isSystemTable (Lorg/jetbrains/kotlinx/dataframe/io/db/TableMetadata;)Z } public class org/jetbrains/kotlinx/dataframe/io/db/H2 : org/jetbrains/kotlinx/dataframe/io/db/DbType { @@ -148,13 +134,13 @@ public class org/jetbrains/kotlinx/dataframe/io/db/H2 : org/jetbrains/kotlinx/da public fun ()V public fun (Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;)V public synthetic fun (Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;ILkotlin/jvm/internal/DefaultConstructorMarker;)V - public fun buildTableMetadata (Ljava/sql/ResultSet;)Lorg/jetbrains/kotlinx/dataframe/io/TableMetadata; - public fun convertSqlTypeToColumnSchemaValue (Lorg/jetbrains/kotlinx/dataframe/io/TableColumnMetadata;)Lorg/jetbrains/kotlinx/dataframe/schema/ColumnSchema; - public fun convertSqlTypeToKType (Lorg/jetbrains/kotlinx/dataframe/io/TableColumnMetadata;)Lkotlin/reflect/KType; + public fun buildSqlQueryWithLimit (Ljava/lang/String;I)Ljava/lang/String; + public fun buildTableMetadata (Ljava/sql/ResultSet;)Lorg/jetbrains/kotlinx/dataframe/io/db/TableMetadata; + public fun convertSqlTypeToColumnSchemaValue (Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;)Lorg/jetbrains/kotlinx/dataframe/schema/ColumnSchema; + public fun convertSqlTypeToKType (Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;)Lkotlin/reflect/KType; public final fun getDialect ()Lorg/jetbrains/kotlinx/dataframe/io/db/DbType; public fun getDriverClassName ()Ljava/lang/String; - public fun isSystemTable (Lorg/jetbrains/kotlinx/dataframe/io/TableMetadata;)Z - public fun sqlQueryLimit (Ljava/lang/String;I)Ljava/lang/String; + public fun isSystemTable (Lorg/jetbrains/kotlinx/dataframe/io/db/TableMetadata;)Z } public final class org/jetbrains/kotlinx/dataframe/io/db/H2$Companion { @@ -162,49 +148,81 @@ public final class org/jetbrains/kotlinx/dataframe/io/db/H2$Companion { public final class org/jetbrains/kotlinx/dataframe/io/db/MariaDb : org/jetbrains/kotlinx/dataframe/io/db/DbType { public static final field INSTANCE Lorg/jetbrains/kotlinx/dataframe/io/db/MariaDb; - public fun buildTableMetadata (Ljava/sql/ResultSet;)Lorg/jetbrains/kotlinx/dataframe/io/TableMetadata; - public fun convertSqlTypeToColumnSchemaValue (Lorg/jetbrains/kotlinx/dataframe/io/TableColumnMetadata;)Lorg/jetbrains/kotlinx/dataframe/schema/ColumnSchema; - public fun convertSqlTypeToKType (Lorg/jetbrains/kotlinx/dataframe/io/TableColumnMetadata;)Lkotlin/reflect/KType; + public fun buildTableMetadata (Ljava/sql/ResultSet;)Lorg/jetbrains/kotlinx/dataframe/io/db/TableMetadata; + public fun convertSqlTypeToColumnSchemaValue (Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;)Lorg/jetbrains/kotlinx/dataframe/schema/ColumnSchema; + public fun convertSqlTypeToKType (Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;)Lkotlin/reflect/KType; public fun getDriverClassName ()Ljava/lang/String; - public fun isSystemTable (Lorg/jetbrains/kotlinx/dataframe/io/TableMetadata;)Z + public fun isSystemTable (Lorg/jetbrains/kotlinx/dataframe/io/db/TableMetadata;)Z + public fun quoteIdentifier (Ljava/lang/String;)Ljava/lang/String; } public final class org/jetbrains/kotlinx/dataframe/io/db/MsSql : org/jetbrains/kotlinx/dataframe/io/db/DbType { public static final field INSTANCE Lorg/jetbrains/kotlinx/dataframe/io/db/MsSql; - public fun buildTableMetadata (Ljava/sql/ResultSet;)Lorg/jetbrains/kotlinx/dataframe/io/TableMetadata; - public fun convertSqlTypeToColumnSchemaValue (Lorg/jetbrains/kotlinx/dataframe/io/TableColumnMetadata;)Lorg/jetbrains/kotlinx/dataframe/schema/ColumnSchema; - public fun convertSqlTypeToKType (Lorg/jetbrains/kotlinx/dataframe/io/TableColumnMetadata;)Lkotlin/reflect/KType; + public fun buildSqlQueryWithLimit (Ljava/lang/String;I)Ljava/lang/String; + public fun buildTableMetadata (Ljava/sql/ResultSet;)Lorg/jetbrains/kotlinx/dataframe/io/db/TableMetadata; + public fun convertSqlTypeToColumnSchemaValue (Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;)Lorg/jetbrains/kotlinx/dataframe/schema/ColumnSchema; + public fun convertSqlTypeToKType (Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;)Lkotlin/reflect/KType; public fun getDriverClassName ()Ljava/lang/String; - public fun isSystemTable (Lorg/jetbrains/kotlinx/dataframe/io/TableMetadata;)Z - public fun sqlQueryLimit (Ljava/lang/String;I)Ljava/lang/String; + public fun isSystemTable (Lorg/jetbrains/kotlinx/dataframe/io/db/TableMetadata;)Z + public fun quoteIdentifier (Ljava/lang/String;)Ljava/lang/String; } public final class org/jetbrains/kotlinx/dataframe/io/db/MySql : org/jetbrains/kotlinx/dataframe/io/db/DbType { public static final field INSTANCE Lorg/jetbrains/kotlinx/dataframe/io/db/MySql; - public fun buildTableMetadata (Ljava/sql/ResultSet;)Lorg/jetbrains/kotlinx/dataframe/io/TableMetadata; - public fun convertSqlTypeToColumnSchemaValue (Lorg/jetbrains/kotlinx/dataframe/io/TableColumnMetadata;)Lorg/jetbrains/kotlinx/dataframe/schema/ColumnSchema; - public fun convertSqlTypeToKType (Lorg/jetbrains/kotlinx/dataframe/io/TableColumnMetadata;)Lkotlin/reflect/KType; + public fun buildTableMetadata (Ljava/sql/ResultSet;)Lorg/jetbrains/kotlinx/dataframe/io/db/TableMetadata; + public fun convertSqlTypeToColumnSchemaValue (Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;)Lorg/jetbrains/kotlinx/dataframe/schema/ColumnSchema; + public fun convertSqlTypeToKType (Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;)Lkotlin/reflect/KType; public fun getDriverClassName ()Ljava/lang/String; - public fun isSystemTable (Lorg/jetbrains/kotlinx/dataframe/io/TableMetadata;)Z + public fun isSystemTable (Lorg/jetbrains/kotlinx/dataframe/io/db/TableMetadata;)Z + public fun quoteIdentifier (Ljava/lang/String;)Ljava/lang/String; } public final class org/jetbrains/kotlinx/dataframe/io/db/PostgreSql : org/jetbrains/kotlinx/dataframe/io/db/DbType { public static final field INSTANCE Lorg/jetbrains/kotlinx/dataframe/io/db/PostgreSql; - public fun buildTableMetadata (Ljava/sql/ResultSet;)Lorg/jetbrains/kotlinx/dataframe/io/TableMetadata; - public fun convertSqlTypeToColumnSchemaValue (Lorg/jetbrains/kotlinx/dataframe/io/TableColumnMetadata;)Lorg/jetbrains/kotlinx/dataframe/schema/ColumnSchema; - public fun convertSqlTypeToKType (Lorg/jetbrains/kotlinx/dataframe/io/TableColumnMetadata;)Lkotlin/reflect/KType; + public fun buildTableMetadata (Ljava/sql/ResultSet;)Lorg/jetbrains/kotlinx/dataframe/io/db/TableMetadata; + public fun convertSqlTypeToColumnSchemaValue (Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;)Lorg/jetbrains/kotlinx/dataframe/schema/ColumnSchema; + public fun convertSqlTypeToKType (Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;)Lkotlin/reflect/KType; public fun getDriverClassName ()Ljava/lang/String; - public fun isSystemTable (Lorg/jetbrains/kotlinx/dataframe/io/TableMetadata;)Z + public fun isSystemTable (Lorg/jetbrains/kotlinx/dataframe/io/db/TableMetadata;)Z + public fun quoteIdentifier (Ljava/lang/String;)Ljava/lang/String; } public final class org/jetbrains/kotlinx/dataframe/io/db/Sqlite : org/jetbrains/kotlinx/dataframe/io/db/DbType { public static final field INSTANCE Lorg/jetbrains/kotlinx/dataframe/io/db/Sqlite; - public fun buildTableMetadata (Ljava/sql/ResultSet;)Lorg/jetbrains/kotlinx/dataframe/io/TableMetadata; - public fun convertSqlTypeToColumnSchemaValue (Lorg/jetbrains/kotlinx/dataframe/io/TableColumnMetadata;)Lorg/jetbrains/kotlinx/dataframe/schema/ColumnSchema; - public fun convertSqlTypeToKType (Lorg/jetbrains/kotlinx/dataframe/io/TableColumnMetadata;)Lkotlin/reflect/KType; + public fun buildTableMetadata (Ljava/sql/ResultSet;)Lorg/jetbrains/kotlinx/dataframe/io/db/TableMetadata; + public fun convertSqlTypeToColumnSchemaValue (Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;)Lorg/jetbrains/kotlinx/dataframe/schema/ColumnSchema; + public fun convertSqlTypeToKType (Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;)Lkotlin/reflect/KType; public fun createConnection (Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;)Ljava/sql/Connection; public fun getDriverClassName ()Ljava/lang/String; - public fun isSystemTable (Lorg/jetbrains/kotlinx/dataframe/io/TableMetadata;)Z + public fun isSystemTable (Lorg/jetbrains/kotlinx/dataframe/io/db/TableMetadata;)Z +} + +public final class org/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata { + public fun (Ljava/lang/String;Ljava/lang/String;IILjava/lang/String;Z)V + public synthetic fun (Ljava/lang/String;Ljava/lang/String;IILjava/lang/String;ZILkotlin/jvm/internal/DefaultConstructorMarker;)V + public final fun copy (Ljava/lang/String;Ljava/lang/String;IILjava/lang/String;Z)Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata; + public static synthetic fun copy$default (Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;Ljava/lang/String;Ljava/lang/String;IILjava/lang/String;ZILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata; + public fun equals (Ljava/lang/Object;)Z + public final fun getJavaClassName ()Ljava/lang/String; + public final fun getJdbcType ()I + public final fun getName ()Ljava/lang/String; + public final fun getSize ()I + public final fun getSqlTypeName ()Ljava/lang/String; + public fun hashCode ()I + public final fun isNullable ()Z + public fun toString ()Ljava/lang/String; +} + +public final class org/jetbrains/kotlinx/dataframe/io/db/TableMetadata { + public fun (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)V + public final fun copy (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)Lorg/jetbrains/kotlinx/dataframe/io/db/TableMetadata; + public static synthetic fun copy$default (Lorg/jetbrains/kotlinx/dataframe/io/db/TableMetadata;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/io/db/TableMetadata; + public fun equals (Ljava/lang/Object;)Z + public final fun getCatalogue ()Ljava/lang/String; + public final fun getName ()Ljava/lang/String; + public final fun getSchemaName ()Ljava/lang/String; + public fun hashCode ()I + public fun toString ()Ljava/lang/String; } public final class org/jetbrains/kotlinx/dataframe/io/db/UtilKt { diff --git a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/DbConnectionConfig.kt b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/DbConnectionConfig.kt new file mode 100644 index 0000000000..602ff38cb9 --- /dev/null +++ b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/DbConnectionConfig.kt @@ -0,0 +1,93 @@ +package org.jetbrains.kotlinx.dataframe.io + +/** + * Represents the configuration for an internally managed JDBC database connection. + * + * This class defines connection parameters used by the library to create a `Connection` + * when the user does not provide one explicitly. + * It is designed for safe, read-only access by default. + * + * __NOTE:__ Connections created using this configuration are managed entirely by the library. + * Users do not have access to the underlying `Connection` instance and cannot commit or close it manually. + * + * ### Read-Only Mode Behavior: + * + * When [readOnly] is `true` (default), the connection operates in read-only mode with: + * - `Connection.setReadOnly(true)` + * - `Connection.setAutoCommit(false)` + * - automatic `rollback()` at the end of execution + * + * When [readOnly] is `false`, the connection uses JDBC defaults (usually read-write), + * but the library still rejects any queries that appear to modify data + * (e.g. contain `INSERT`, `UPDATE`, `DELETE`, etc.). + * + * ### Examples: + * + * ```kotlin + * // Safe read-only connection (default) + * val config = DbConnectionConfig("jdbc:sqlite::memory:") + * val df = DataFrame.readSqlQuery(config, "SELECT * FROM books") + * + * // Use default JDBC connection settings (still protected against mutations) + * val config = DbConnectionConfig( + * url = "jdbc:sqlite::memory:", + * readOnly = false + * ) + * ``` + * + * @property [url] The JDBC URL of the database, e.g., `"jdbc:postgresql://localhost:5432/mydb"`. + * Must follow the standard format: `jdbc:subprotocol:subname`. + * + * @property [user] The username used for authentication. + * Optional, default is an empty string. + * + * @property [password] The password used for authentication. + * Optional, default is an empty string. + * + * @property [readOnly] If `true` (default), enables read-only mode. If `false`, uses JDBC defaults + * but still prevents data-modifying queries. See class documentation for details. + */ +public class DbConnectionConfig( + public val url: String, + public val user: String = "", + public val password: String = "", + public val readOnly: Boolean = true, +) { + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other !is DbConnectionConfig) return false + + if (url != other.url) return false + if (user != other.user) return false + if (password != other.password) return false + if (readOnly != other.readOnly) return false + + return true + } + + override fun hashCode(): Int { + var result = url.hashCode() + result = 31 * result + user.hashCode() + result = 31 * result + password.hashCode() + result = 31 * result + readOnly.hashCode() + return result + } + + override fun toString(): String = "DbConnectionConfig(url='$url', user='$user', password='***', readOnly=$readOnly)" + + /** + * Creates a copy of this configuration with the option to override specific properties. + * + * @param url The JDBC URL. If not specified, uses the current value. + * @param user The username. If not specified, uses the current value. + * @param password The password. If not specified, uses the current value. + * @param readOnly The read-only flag. If not specified, uses the current value. + * @return A new [DbConnectionConfig] instance with the specified changes. + */ + public fun copy( + url: String = this.url, + user: String = this.user, + password: String = this.password, + readOnly: Boolean = this.readOnly, + ): DbConnectionConfig = DbConnectionConfig(url, user, password, readOnly) +} diff --git a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/DbType.kt b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/DbType.kt index 1598f4f73d..859f284317 100644 --- a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/DbType.kt +++ b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/DbType.kt @@ -1,16 +1,36 @@ package org.jetbrains.kotlinx.dataframe.io.db +import org.jetbrains.kotlinx.dataframe.DataColumn +import org.jetbrains.kotlinx.dataframe.api.Infer import org.jetbrains.kotlinx.dataframe.io.DbConnectionConfig -import org.jetbrains.kotlinx.dataframe.io.TableColumnMetadata -import org.jetbrains.kotlinx.dataframe.io.TableMetadata -import org.jetbrains.kotlinx.dataframe.io.getSchemaForAllSqlTables import org.jetbrains.kotlinx.dataframe.io.readAllSqlTables import org.jetbrains.kotlinx.dataframe.schema.ColumnSchema +import java.math.BigDecimal +import java.sql.Blob +import java.sql.Clob import java.sql.Connection import java.sql.DatabaseMetaData import java.sql.DriverManager +import java.sql.NClob +import java.sql.PreparedStatement +import java.sql.Ref import java.sql.ResultSet +import java.sql.RowId +import java.sql.SQLXML +import java.sql.Time +import java.sql.Timestamp +import java.sql.Types +import java.time.LocalDateTime +import java.time.OffsetDateTime +import java.time.OffsetTime +import java.util.Date +import java.util.UUID +import kotlin.reflect.KClass import kotlin.reflect.KType +import kotlin.reflect.full.createType +import kotlin.reflect.full.isSupertypeOf +import kotlin.reflect.full.safeCast +import kotlin.reflect.full.starProjectedType /** * The `DbType` class represents a database type used for reading dataframe from the database. @@ -29,7 +49,7 @@ public abstract class DbType(public val dbTypeInJdbcUrl: String) { /** * The table type(s) (`TABLE_TYPE`) of ordinary tables in the SQL database, used by - * [getSchemaForAllSqlTables], and [readAllSqlTables] as a filter when querying the database + * [readAllSqlTables], and [readAllSqlTables] as a filter when querying the database * for all the tables it has using [DatabaseMetaData.getTables]. * * This is usually "TABLE" or "BASE TABLE", which is what [tableTypes] is set to by default, @@ -40,6 +60,27 @@ public abstract class DbType(public val dbTypeInJdbcUrl: String) { */ public open val tableTypes: List? = listOf("TABLE", "BASE TABLE") + /** + * Specifies the default batch size for fetching rows from the database during query execution. + * + * This property determines how many rows are fetched in a single batch from the database. + * A proper fetch size can improve performance by reducing the number of network round-trips required + * when handling large result sets. + * + * Value is set to 1000 by default, but it can be overridden based on database-specific requirements + * or performance considerations. + */ + public open val defaultFetchSize: Int = 1000 + + /** + * Specifies the default timeout in seconds for database queries. + * + * If set to `null`, no timeout is applied, allowing queries to run indefinitely. + * This property can be used to set a default query timeout for the database type, + * which can help manage long-running queries. + */ + public open val defaultQueryTimeout: Int? = null // null = no timeout + /** * Returns a [ColumnSchema] produced from [tableColumnMetadata]. */ @@ -49,7 +90,6 @@ public abstract class DbType(public val dbTypeInJdbcUrl: String) { * Checks if the given table name is a system table for the specified database type. * * @param [tableMetadata] the table object representing the table from the database. - * @param [dbType] the database type to check against. * @return True if the table is a system table for the specified database type, false otherwise. */ public abstract fun isSystemTable(tableMetadata: TableMetadata): Boolean @@ -70,6 +110,64 @@ public abstract class DbType(public val dbTypeInJdbcUrl: String) { */ public abstract fun convertSqlTypeToKType(tableColumnMetadata: TableColumnMetadata): KType? + /** + * Builds a SELECT query for reading from a table. + * + * @param [tableName] the name of the table to query. + * @param [limit] the maximum number of rows to retrieve. If 0 or negative, no limit is applied. + * @return the SQL query string. + */ + public open fun buildSelectTableQueryWithLimit(tableName: String, limit: Int?): String { + require(tableName.isNotBlank()) { "Table name cannot be blank" } + + val quotedTableName = quoteIdentifier(tableName) + + return if (limit != null && limit > 0) { + buildSqlQueryWithLimit("SELECT * FROM $quotedTableName", limit) + } else { + "SELECT * FROM $quotedTableName" + } + } + + /** + * Configures the provided `PreparedStatement` for optimized read operations. + * + * This method sets the fetch size for efficient streaming, applies a query timeout if specified, + * and configures the fetch direction to forward-only for better performance in read-only operations. + * + * @param statement the `PreparedStatement` to be configured + */ + public open fun configureReadStatement(statement: PreparedStatement) { + // Set fetch size for better streaming performance + statement.fetchSize = defaultFetchSize + + defaultQueryTimeout?.let { + statement.queryTimeout = it + } + + // Set the fetch direction (forward-only for read-only operations) + statement.fetchDirection = ResultSet.FETCH_FORWARD + } + + /** + * Quotes an identifier (table or column name) according to database-specific rules. + * + * Examples: + * - PostgreSQL: "tableName" or "schema"."table" + * - MySQL: `tableName` or `schema`.`table` + * - MS SQL: `[tableName]` or `[schema].[table]` + * - SQLite/H2: no quotes for simple names + * + * @param [name] the identifier to quote (can contain dots for schema.table). + * @return the quoted identifier. + */ + public open fun quoteIdentifier(name: String): String { + require(name.isNotBlank()) { "Identifier cannot be blank" } + + // Default: no quoting (works for SQLite, H2, simple names) + return name + } + /** * Constructs a SQL query with a limit clause. * @@ -77,7 +175,7 @@ public abstract class DbType(public val dbTypeInJdbcUrl: String) { * @param limit The maximum number of rows to retrieve from the query. Default is 1. * @return A new SQL query with the limit clause added. */ - public open fun sqlQueryLimit(sqlQuery: String, limit: Int = 1): String = "$sqlQuery LIMIT $limit" + public open fun buildSqlQueryWithLimit(sqlQuery: String, limit: Int = 1): String = "$sqlQuery LIMIT $limit" /** * Creates a database connection using the provided configuration. @@ -96,4 +194,207 @@ public abstract class DbType(public val dbTypeInJdbcUrl: String) { } return connection } + + /** + * Extracts a value from the ResultSet for the given column. + * This method can be overridden by custom database types to provide specialized parsing logic. + * + * @param [rs] the ResultSet to read from + * @param [columnIndex] zero-based column index + * @param [columnMetadata] metadata for the column + * @param [kType] the Kotlin type for this column + * @return the extracted value, or null + */ + public open fun extractValueFromResultSet( + rs: ResultSet, + columnIndex: Int, + columnMetadata: TableColumnMetadata, + kType: KType, + ): Any? = + try { + rs.getObject(columnIndex + 1) + // TODO: add a special handler for Blob via Streams + } catch (_: Throwable) { + // TODO: expand for all the types like in generateKType function + if (kType.isSupertypeOf(String::class.starProjectedType)) { + rs.getString(columnIndex + 1) + } else { + rs.getString(columnIndex + 1) + } + } + + /** + * Builds a single DataColumn with proper type handling. + * Accepts a mutable list to allow efficient post-processing. + */ + public open fun buildDataColumn( + name: String, + values: MutableList, + kType: KType, + inferNullability: Boolean, + ): DataColumn<*> { + val correctedValues = postProcessColumnValues(values, kType) + + return DataColumn.createValueColumn( + name = name, + values = correctedValues, + infer = convertNullabilityInference(inferNullability), + type = kType, + ) + } + + private fun convertNullabilityInference(inferNullability: Boolean) = + if (inferNullability) Infer.Nulls else Infer.None + + /** + * Processes the column values retrieved from the database and performs transformations based on the provided + * Kotlin type and column metadata. The method allows for custom post-processing logic, such as handling + * specific database column types, including arrays. + * + * @param values the list of raw values retrieved from the database for the column. + * @param kType the Kotlin type that the column values should be transformed to. + * @return a list of processed column values, with transformations applied where necessary, or the original list if no transformation is needed. + */ + private fun postProcessColumnValues(values: MutableList, kType: KType): List = + when { + /* EXAMPLE: columnMetadata.sqlTypeName == "MY_CUSTOM_ARRAY" -> { + values.map { /* custom transformation */ } + } */ + kType.classifier == Array::class -> { + handleArrayValues(values) + } + + else -> values + } + + /** + * Converts SQL Array objects to strongly-typed arrays. + * + * Extracts arrays from SQL Array objects and converts them to a consistent type + * if all elements share the same type. Returns original arrays if types vary. + * + * @param values raw values containing SQL Array objects + * @return list of consistently typed arrays, or original arrays if no common type exists + */ + private fun handleArrayValues(values: MutableList): List { + // Intermediate variable for the first mapping + val sqlArrays = values.mapNotNull { + (it as? java.sql.Array)?.array?.let { array -> array as? Array<*> } + } + + // Flatten the arrays to iterate through all elements and filter out null values, then map to component types + val allElementTypes = sqlArrays + .flatMap { array -> + (array.javaClass.componentType?.kotlin?.let { listOf(it) } ?: emptyList()) + } // Get the component type of each array and convert it to a Kotlin class, if available + + // Find distinct types and ensure there's only one distinct type + val commonElementType = allElementTypes + .distinct() // Get unique element types + .singleOrNull() // Ensure there's only one unique element type, otherwise return null + ?: Any::class // Fallback to Any::class if multiple distinct types or no elements found + + return if (commonElementType != Any::class) { + sqlArrays.map { castArray(it, commonElementType).toTypedArray() } + } else { + sqlArrays + } + } + + /** Utility function to cast arrays based on the type of elements */ + private fun castArray(array: Array<*>, elementType: KClass): List = + array.mapNotNull { elementType.safeCast(it) } + + /** + * Creates a mapping between common SQL types and their corresponding KTypes. + * + * @param tableColumnMetadata The metadata of the table column. + * @return The KType associated with the SQL type or a default type if no mapping is found. + */ + public open fun makeCommonSqlToKTypeMapping(tableColumnMetadata: TableColumnMetadata): KType { + val jdbcTypeToKTypeMapping = mapOf( + Types.BIT to Boolean::class, + Types.TINYINT to Int::class, + Types.SMALLINT to Int::class, + Types.INTEGER to Int::class, + Types.BIGINT to Long::class, + Types.FLOAT to Float::class, + Types.REAL to Float::class, + Types.DOUBLE to Double::class, + Types.NUMERIC to BigDecimal::class, + Types.DECIMAL to BigDecimal::class, + Types.CHAR to String::class, + Types.VARCHAR to String::class, + Types.LONGVARCHAR to String::class, + Types.DATE to Date::class, + Types.TIME to Time::class, + Types.TIMESTAMP to Timestamp::class, + Types.BINARY to ByteArray::class, + Types.VARBINARY to ByteArray::class, + Types.LONGVARBINARY to ByteArray::class, + Types.NULL to String::class, + Types.JAVA_OBJECT to Any::class, + Types.DISTINCT to Any::class, + Types.STRUCT to Any::class, + Types.ARRAY to Array::class, + Types.BLOB to ByteArray::class, + Types.CLOB to Clob::class, + Types.REF to Ref::class, + Types.DATALINK to Any::class, + Types.BOOLEAN to Boolean::class, + Types.ROWID to RowId::class, + Types.NCHAR to String::class, + Types.NVARCHAR to String::class, + Types.LONGNVARCHAR to String::class, + Types.NCLOB to NClob::class, + Types.SQLXML to SQLXML::class, + Types.REF_CURSOR to Ref::class, + Types.TIME_WITH_TIMEZONE to OffsetTime::class, + Types.TIMESTAMP_WITH_TIMEZONE to OffsetDateTime::class, + ) + + fun determineKotlinClass(tableColumnMetadata: TableColumnMetadata): KClass<*> = + when { + tableColumnMetadata.jdbcType == Types.OTHER -> when (tableColumnMetadata.javaClassName) { + "[B" -> ByteArray::class + else -> Any::class + } + + tableColumnMetadata.javaClassName == "[B" -> ByteArray::class + + tableColumnMetadata.javaClassName == "java.sql.Blob" -> Blob::class + + tableColumnMetadata.jdbcType == Types.TIMESTAMP && + tableColumnMetadata.javaClassName == "java.time.LocalDateTime" -> LocalDateTime::class + + tableColumnMetadata.jdbcType == Types.BINARY && + tableColumnMetadata.javaClassName == "java.util.UUID" -> UUID::class + + tableColumnMetadata.jdbcType == Types.REAL && + tableColumnMetadata.javaClassName == "java.lang.Double" -> Double::class + + tableColumnMetadata.jdbcType == Types.FLOAT && + tableColumnMetadata.javaClassName == "java.lang.Double" -> Double::class + + tableColumnMetadata.jdbcType == Types.NUMERIC && + tableColumnMetadata.javaClassName == "java.lang.Double" -> Double::class + + else -> jdbcTypeToKTypeMapping[tableColumnMetadata.jdbcType] ?: String::class + } + + fun createArrayTypeIfNeeded(kClass: KClass<*>, isNullable: Boolean): KType = + if (kClass == Array::class) { + val typeParam = kClass.typeParameters[0].createType() + kClass.createType( + arguments = listOf(kotlin.reflect.KTypeProjection.invariant(typeParam)), + nullable = isNullable, + ) + } else { + kClass.createType(nullable = isNullable) + } + + val kClass: KClass<*> = determineKotlinClass(tableColumnMetadata) + val kType = createArrayTypeIfNeeded(kClass, tableColumnMetadata.isNullable) + return kType + } } diff --git a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/DuckDb.kt b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/DuckDb.kt index dfcb129e1d..d6882133e7 100644 --- a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/DuckDb.kt +++ b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/DuckDb.kt @@ -41,10 +41,7 @@ import org.duckdb.DuckDBResultSetMetaData import org.duckdb.JsonNode import org.jetbrains.kotlinx.dataframe.DataFrame import org.jetbrains.kotlinx.dataframe.io.DbConnectionConfig -import org.jetbrains.kotlinx.dataframe.io.TableColumnMetadata -import org.jetbrains.kotlinx.dataframe.io.TableMetadata import org.jetbrains.kotlinx.dataframe.io.db.DuckDb.convertSqlTypeToKType -import org.jetbrains.kotlinx.dataframe.io.getSchemaForAllSqlTables import org.jetbrains.kotlinx.dataframe.io.readAllSqlTables import org.jetbrains.kotlinx.dataframe.schema.ColumnSchema import java.math.BigDecimal @@ -105,7 +102,7 @@ public object DuckDb : DbType("duckdb") { * Follows exactly [org.duckdb.DuckDBVector.getObject]. * * "// dataframe-jdbc" is added for all types that are covered correctly by - * [org.jetbrains.kotlinx.dataframe.io.makeCommonSqlToKTypeMapping] at the moment, however, to cover + * [org.jetbrains.kotlinx.dataframe.io.db.DbType.makeCommonSqlToKTypeMapping] at the moment, however, to cover * all nested types, we'll use a full type-map for all [DuckDB types][DuckDBColumnType] exactly. */ @Suppress("ktlint:standard:blank-line-between-when-conditions") @@ -202,7 +199,7 @@ public object DuckDb : DbType("duckdb") { /** * How to filter out system tables from user-created ones when using * [DataFrame.readAllSqlTables][DataFrame.Companion.readAllSqlTables] and - * [DataFrame.getSchemaForAllSqlTables][DataFrame.Companion.getSchemaForAllSqlTables]. + * [DataFrameSchema.readAllSqlTables][org.jetbrains.kotlinx.dataframe.schema.DataFrameSchema.Companion.readAllSqlTables]. * * The names of these can sometimes be found in the specific JDBC integration. */ @@ -214,7 +211,7 @@ public object DuckDb : DbType("duckdb") { /** * How to retrieve the correct table metadata when using * [DataFrame.readAllSqlTables][DataFrame.Companion.readAllSqlTables] and - * [DataFrame.getSchemaForAllSqlTables][DataFrame.Companion.getSchemaForAllSqlTables]. + * [DataFrameSchema.readAllSqlTables][org.jetbrains.kotlinx.dataframe.schema.DataFrameSchema.Companion.readAllSqlTables]. * The names of these can be found in the [DatabaseMetaData] implementation of the DuckDB JDBC integration. */ override fun buildTableMetadata(tables: ResultSet): TableMetadata = diff --git a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/H2.kt b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/H2.kt index bd623046fe..4a7b47481d 100644 --- a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/H2.kt +++ b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/H2.kt @@ -1,7 +1,5 @@ package org.jetbrains.kotlinx.dataframe.io.db -import org.jetbrains.kotlinx.dataframe.io.TableColumnMetadata -import org.jetbrains.kotlinx.dataframe.io.TableMetadata import org.jetbrains.kotlinx.dataframe.schema.ColumnSchema import java.sql.ResultSet import java.util.Locale @@ -67,5 +65,6 @@ public open class H2(public val dialect: DbType = MySql) : DbType("h2") { override fun convertSqlTypeToKType(tableColumnMetadata: TableColumnMetadata): KType? = dialect.convertSqlTypeToKType(tableColumnMetadata) - public override fun sqlQueryLimit(sqlQuery: String, limit: Int): String = dialect.sqlQueryLimit(sqlQuery, limit) + public override fun buildSqlQueryWithLimit(sqlQuery: String, limit: Int): String = + dialect.buildSqlQueryWithLimit(sqlQuery, limit) } diff --git a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/MariaDb.kt b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/MariaDb.kt index b73941665e..7ce8e4d681 100644 --- a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/MariaDb.kt +++ b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/MariaDb.kt @@ -1,7 +1,7 @@ package org.jetbrains.kotlinx.dataframe.io.db -import org.jetbrains.kotlinx.dataframe.io.TableColumnMetadata -import org.jetbrains.kotlinx.dataframe.io.TableMetadata +import org.jetbrains.kotlinx.dataframe.io.db.TableColumnMetadata +import org.jetbrains.kotlinx.dataframe.io.db.TableMetadata import org.jetbrains.kotlinx.dataframe.schema.ColumnSchema import java.sql.ResultSet import kotlin.reflect.KType @@ -40,4 +40,9 @@ public object MariaDb : DbType("mariadb") { } return null } + + override fun quoteIdentifier(name: String): String { + // schema.table -> `schema`.`table` + return name.split(".").joinToString(".") { "`$it`" } + } } diff --git a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/MsSql.kt b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/MsSql.kt index c0c76d50eb..2709e04b6a 100644 --- a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/MsSql.kt +++ b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/MsSql.kt @@ -1,7 +1,7 @@ package org.jetbrains.kotlinx.dataframe.io.db -import org.jetbrains.kotlinx.dataframe.io.TableColumnMetadata -import org.jetbrains.kotlinx.dataframe.io.TableMetadata +import org.jetbrains.kotlinx.dataframe.io.db.TableColumnMetadata +import org.jetbrains.kotlinx.dataframe.io.db.TableMetadata import org.jetbrains.kotlinx.dataframe.schema.ColumnSchema import java.sql.ResultSet import java.util.Locale @@ -49,6 +49,11 @@ public object MsSql : DbType("sqlserver") { override fun convertSqlTypeToKType(tableColumnMetadata: TableColumnMetadata): KType? = null - public override fun sqlQueryLimit(sqlQuery: String, limit: Int): String = + public override fun buildSqlQueryWithLimit(sqlQuery: String, limit: Int): String = sqlQuery.replace("SELECT", "SELECT TOP $limit", ignoreCase = true) + + override fun quoteIdentifier(name: String): String { + // schema.table -> [schema].[table] + return name.split(".").joinToString(".") { "[$it]" } + } } diff --git a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/MySql.kt b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/MySql.kt index 5f8286823a..e411345879 100644 --- a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/MySql.kt +++ b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/MySql.kt @@ -1,7 +1,7 @@ package org.jetbrains.kotlinx.dataframe.io.db -import org.jetbrains.kotlinx.dataframe.io.TableColumnMetadata -import org.jetbrains.kotlinx.dataframe.io.TableMetadata +import org.jetbrains.kotlinx.dataframe.io.db.TableColumnMetadata +import org.jetbrains.kotlinx.dataframe.io.db.TableMetadata import org.jetbrains.kotlinx.dataframe.schema.ColumnSchema import java.sql.ResultSet import java.util.Locale @@ -55,4 +55,9 @@ public object MySql : DbType("mysql") { } return null } + + override fun quoteIdentifier(name: String): String { + // schema.table -> `schema`.`table` + return name.split(".").joinToString(".") { "`$it`" } + } } diff --git a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/PostgreSql.kt b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/PostgreSql.kt index ed645cbe2d..8da1a66833 100644 --- a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/PostgreSql.kt +++ b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/PostgreSql.kt @@ -1,7 +1,7 @@ package org.jetbrains.kotlinx.dataframe.io.db -import org.jetbrains.kotlinx.dataframe.io.TableColumnMetadata -import org.jetbrains.kotlinx.dataframe.io.TableMetadata +import org.jetbrains.kotlinx.dataframe.io.db.TableColumnMetadata +import org.jetbrains.kotlinx.dataframe.io.db.TableMetadata import org.jetbrains.kotlinx.dataframe.schema.ColumnSchema import java.sql.ResultSet import java.util.Locale @@ -47,4 +47,9 @@ public object PostgreSql : DbType("postgresql") { return null } + + override fun quoteIdentifier(name: String): String { + // schema.table -> "schema"."table" + return name.split(".").joinToString(".") { "\"$it\"" } + } } diff --git a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/Sqlite.kt b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/Sqlite.kt index 8d9b59ab82..adb03b2753 100644 --- a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/Sqlite.kt +++ b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/Sqlite.kt @@ -1,8 +1,8 @@ package org.jetbrains.kotlinx.dataframe.io.db import org.jetbrains.kotlinx.dataframe.io.DbConnectionConfig -import org.jetbrains.kotlinx.dataframe.io.TableColumnMetadata -import org.jetbrains.kotlinx.dataframe.io.TableMetadata +import org.jetbrains.kotlinx.dataframe.io.db.TableColumnMetadata +import org.jetbrains.kotlinx.dataframe.io.db.TableMetadata import org.jetbrains.kotlinx.dataframe.schema.ColumnSchema import org.sqlite.SQLiteConfig import java.sql.Connection diff --git a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata.kt b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata.kt new file mode 100644 index 0000000000..bfeaa34a49 --- /dev/null +++ b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata.kt @@ -0,0 +1,68 @@ +package org.jetbrains.kotlinx.dataframe.io.db + +/** + * Represents a column in a database table to keep all required meta-information. + * + * @property [name] the name of the column. + * @property [sqlTypeName] the SQL data type of the column. + * @property [jdbcType] the JDBC data type of the column produced from [java.sql.Types]. + * @property [size] the size of the column. + * @property [javaClassName] the class name in Java. + * @property [isNullable] true if column could contain nulls. + */ +public class TableColumnMetadata( + public val name: String, + public val sqlTypeName: String, + public val jdbcType: Int, + public val size: Int, + public val javaClassName: String, + public val isNullable: Boolean = false, +) { + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other !is TableColumnMetadata) return false + + if (name != other.name) return false + if (sqlTypeName != other.sqlTypeName) return false + if (jdbcType != other.jdbcType) return false + if (size != other.size) return false + if (javaClassName != other.javaClassName) return false + if (isNullable != other.isNullable) return false + + return true + } + + override fun hashCode(): Int { + var result = name.hashCode() + result = 31 * result + sqlTypeName.hashCode() + result = 31 * result + jdbcType + result = 31 * result + size + result = 31 * result + javaClassName.hashCode() + result = 31 * result + isNullable.hashCode() + return result + } + + override fun toString(): String = + "TableColumnMetadata(name='$name', sqlTypeName='$sqlTypeName', jdbcType=$jdbcType, " + + "size=$size, javaClassName='$javaClassName', isNullable=$isNullable)" + + /** + * Creates a copy of the current `TableColumnMetadata` instance with optionally modified attributes. + * + * @param name The name of the table column. Defaults to the current instance's `name`. + * @param sqlTypeName The SQL type name of the column. Defaults to the current instance's `sqlTypeName`. + * @param jdbcType The JDBC type of the column, represented as an integer. Defaults to the current instance's `jdbcType`. + * @param size The size of the column. Defaults to the current instance's `size`. + * @param javaClassName The fully qualified name of the Java class representing the column type. Defaults to the current instance's `javaClassName`. + * @param isNullable Indicates whether the column is nullable. Defaults to the current instance's `isNullable`. + * @return A new `TableColumnMetadata` instance with the specified attribute values. + */ + public fun copy( + name: String = this.name, + sqlTypeName: String = this.sqlTypeName, + jdbcType: Int = this.jdbcType, + size: Int = this.size, + javaClassName: String = this.javaClassName, + isNullable: Boolean = this.isNullable, + ): TableColumnMetadata = TableColumnMetadata(name, sqlTypeName, jdbcType, size, javaClassName, isNullable) +} diff --git a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/TableMetadata.kt b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/TableMetadata.kt new file mode 100644 index 0000000000..56b36a3099 --- /dev/null +++ b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/TableMetadata.kt @@ -0,0 +1,48 @@ +package org.jetbrains.kotlinx.dataframe.io.db + +/** + * Represents a table metadata to store information about a database table, + * including its name, schema name, and catalogue name. + * + * NOTE: we need to extract both, [schemaName] and [catalogue] + * because the different databases have different implementations of metadata. + * + * @property [name] the name of the table. + * @property [schemaName] the name of the schema the table belongs to (optional). + * @property [catalogue] the name of the catalogue the table belongs to (optional). + */ +public class TableMetadata(public val name: String, public val schemaName: String?, public val catalogue: String?) { + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other !is TableMetadata) return false + + if (name != other.name) return false + if (schemaName != other.schemaName) return false + if (catalogue != other.catalogue) return false + + return true + } + + override fun hashCode(): Int { + var result = name.hashCode() + result = 31 * result + (schemaName?.hashCode() ?: 0) + result = 31 * result + (catalogue?.hashCode() ?: 0) + return result + } + + override fun toString(): String = "TableMetadata(name='$name', schemaName=$schemaName, catalogue=$catalogue)" + + /** + * Creates a copy of the `TableMetadata` instance with optional modifications. + * + * @param name the name of the table; defaults to the current name of the instance. + * @param schemaName the name of the schema the table belongs to; defaults to the current schema name of the instance. + * @param catalogue the name of the catalogue the table belongs to; defaults to the current catalogue of the instance. + * @return a new `TableMetadata` instance with the specified or default values. + */ + public fun copy( + name: String = this.name, + schemaName: String? = this.schemaName, + catalogue: String? = this.catalogue, + ): TableMetadata = TableMetadata(name, schemaName, catalogue) +} diff --git a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/readDataFrameSchema.kt b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/readDataFrameSchema.kt new file mode 100644 index 0000000000..9667ec9540 --- /dev/null +++ b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/readDataFrameSchema.kt @@ -0,0 +1,464 @@ +package org.jetbrains.kotlinx.dataframe.io + +import org.jetbrains.kotlinx.dataframe.DataFrame +import org.jetbrains.kotlinx.dataframe.api.schema +import org.jetbrains.kotlinx.dataframe.impl.schema.DataFrameSchemaImpl +import org.jetbrains.kotlinx.dataframe.io.db.DbType +import org.jetbrains.kotlinx.dataframe.io.db.TableColumnMetadata +import org.jetbrains.kotlinx.dataframe.io.db.extractDBTypeFromConnection +import org.jetbrains.kotlinx.dataframe.schema.ColumnSchema +import org.jetbrains.kotlinx.dataframe.schema.DataFrameSchema +import java.sql.Connection +import java.sql.DriverManager +import java.sql.ResultSet +import javax.sql.DataSource +import kotlin.use + +/** + * Retrieves the schema for an SQL table using the provided database configuration. + * + * ### Default Behavior: + * If [DbConnectionConfig.readOnly] is `true` (which is the default), the connection will be: + * - explicitly set as read-only via `Connection.setReadOnly(true)` + * - used with `autoCommit = false` + * - automatically rolled back after reading, ensuring no changes to the database + * + * Even if [DbConnectionConfig.readOnly] is set to `false`, the library still prevents data-modifying queries + * and only permits safe `SELECT` operations internally. + * + * @param [dbConfig] the database configuration to connect to the database, including URL, user, and password. + * @param [tableName] the name of the SQL table for which to retrieve the schema. + * @param [dbType] the type of database, could be a custom object, provided by user, optional, default is `null`, + * in that case the [dbType] will be recognized from the [dbConfig]. + * @return the [DataFrameSchema] object representing the schema of the SQL table + */ +public fun DataFrameSchema.Companion.readSqlTable( + dbConfig: DbConnectionConfig, + tableName: String, + dbType: DbType? = null, +): DataFrameSchema = + withReadOnlyConnection(dbConfig, dbType) { connection -> + readSqlTable(connection, tableName, dbType) + } + +/** + * Retrieves the schema for an SQL table using the provided [DataSource]. + * + * ### Example with HikariCP: + * ```kotlin + * import com.zaxxer.hikari.HikariConfig + * import com.zaxxer.hikari.HikariDataSource + * + * val config = HikariConfig().apply { + * jdbcUrl = "jdbc:postgresql://localhost:5432/mydb" + * username = "user" + * password = "password" + * } + * val dataSource = HikariDataSource(config) + * + * // Get the schema for a specific table + * val customersSchema = DataFrame.getSchemaForSqlTable(dataSource, "customers") + * + * // Inspect the schema + * println(customersSchema.columns) + * ``` + * + * @param [dataSource] the [DataSource] to get a database connection from. + * @param [tableName] the name of the SQL table for which to retrieve the schema. + * @param [dbType] the type of database, could be a custom object, provided by user, optional, default is `null`, + * in that case the [dbType] will be recognized from the [dataSource]. + * @return the schema of the SQL table as a [DataFrameSchema] object. + * @see [DataSource.getConnection] + */ +public fun DataFrameSchema.Companion.readSqlTable( + dataSource: DataSource, + tableName: String, + dbType: DbType? = null, +): DataFrameSchema { + dataSource.connection.use { connection -> + return readSqlTable(connection, tableName, dbType) + } +} + +/** + * Retrieves the schema for an SQL table using the provided database connection. + * + * @param [connection] the database connection. + * @param [tableName] the name of the SQL table for which to retrieve the schema. + * @param [dbType] the type of database, could be a custom object, provided by user, optional, default is `null`, + * in that case the [dbType] will be recognized from the [connection]. + * @return the schema of the SQL table as a [DataFrameSchema] object. + * + * @see DriverManager.getConnection + */ +public fun DataFrameSchema.Companion.readSqlTable( + connection: Connection, + tableName: String, + dbType: DbType? = null, +): DataFrameSchema { + val determinedDbType = dbType ?: extractDBTypeFromConnection(connection) + + // Read just 1 row to get the schema + val singleRowDataFrame = DataFrame.readSqlTable( + connection = connection, + tableName = tableName, + limit = 1, + inferNullability = false, // Schema extraction doesn't need nullability inference + dbType = determinedDbType, + strictValidation = true, + ) + + return singleRowDataFrame.schema() +} + +/** + * Retrieves the schema of an SQL query result using the provided database configuration. + * + * ### Default Behavior: + * If [DbConnectionConfig.readOnly] is `true` (which is the default), the connection will be: + * - explicitly set as read-only via `Connection.setReadOnly(true)` + * - used with `autoCommit = false` + * - automatically rolled back after reading, ensuring no changes to the database + * + * Even if [DbConnectionConfig.readOnly] is set to `false`, the library still prevents data-modifying queries + * and only permits safe `SELECT` operations internally. + * + * @param [dbConfig] the database configuration to connect to the database, including URL, user, and password. + * @param [sqlQuery] the SQL query to execute and retrieve the schema from. + * @param [dbType] the type of database, could be a custom object, provided by user, optional, default is `null`, + * in that case the [dbType] will be recognized from the [dbConfig]. + * @return the schema of the SQL query as a [DataFrameSchema] object. + */ +public fun DataFrameSchema.Companion.readSqlQuery( + dbConfig: DbConnectionConfig, + sqlQuery: String, + dbType: DbType? = null, +): DataFrameSchema = + withReadOnlyConnection(dbConfig, dbType) { connection -> + readSqlQuery(connection, sqlQuery, dbType) + } + +/** + * Retrieves the schema of an SQL query result using the provided [DataSource]. + * + * ### Example with HikariCP: + * ```kotlin + * import com.zaxxer.hikari.HikariConfig + * import com.zaxxer.hikari.HikariDataSource + * + * val config = HikariConfig().apply { + * jdbcUrl = "jdbc:postgresql://localhost:5432/mydb" + * username = "user" + * password = "password" + * } + * val dataSource = HikariDataSource(config) + * + * // Get the schema for a SQL query + * val querySchema = DataFrame.getSchemaForSqlQuery( + * dataSource, + * "SELECT name, age, city FROM customers WHERE age > 25" + * ) + * + * // Inspect the schema + * println(querySchema.columns) + * ``` + * + * @param [dataSource] the [DataSource] to get a database connection from. + * @param [sqlQuery] the SQL query to execute and retrieve the schema from. + * @param [dbType] the type of database, could be a custom object, provided by user, optional, default is `null`, + * in that case the [dbType] will be recognized from the [dataSource]. + * @return the schema of the SQL query as a [DataFrameSchema] object. + * + * @see [DataSource.getConnection] + */ +public fun DataFrameSchema.Companion.readSqlQuery( + dataSource: DataSource, + sqlQuery: String, + dbType: DbType? = null, +): DataFrameSchema { + dataSource.connection.use { connection -> + return readSqlQuery(connection, sqlQuery, dbType) + } +} + +/** + * Retrieves the schema of an SQL query result using the provided database connection. + * + * @param [connection] the database connection. + * @param [sqlQuery] the SQL query to execute and retrieve the schema from. + * @param [dbType] the type of database, could be a custom object, provided by user, optional, default is `null`, + * in that case the [dbType] will be recognized from the [connection]. + * @return the schema of the SQL query as a [DataFrameSchema] object. + * + * @see DriverManager.getConnection + */ +public fun DataFrameSchema.Companion.readSqlQuery( + connection: Connection, + sqlQuery: String, + dbType: DbType? = null, +): DataFrameSchema { + val determinedDbType = dbType ?: extractDBTypeFromConnection(connection) + + // Read just 1 row to get the schema + val singleRowDataFrame = DataFrame.readSqlQuery( + connection = connection, + sqlQuery = sqlQuery, + limit = 1, + inferNullability = false, // Schema extraction doesn't need nullability inference + dbType = determinedDbType, + strictValidation = true, + ) + + return singleRowDataFrame.schema() +} + +/** + * Retrieves the schema of an SQL query result or the SQL table using the provided database configuration. + * + * ### Default Behavior: + * If [DbConnectionConfig.readOnly] is `true` (which is the default), the connection will be: + * - explicitly set as read-only via `Connection.setReadOnly(true)` + * - used with `autoCommit = false` + * - automatically rolled back after reading, ensuring no changes to the database + * + * Even if [DbConnectionConfig.readOnly] is set to `false`, the library still prevents data-modifying queries + * and only permits safe `SELECT` operations internally. + * + * @param [sqlQueryOrTableName] the SQL query to execute and retrieve the schema from. + * @param [dbType] the type of database, could be a custom object, provided by user, optional, default is `null`, + * in that case the [dbType] will be recognized from the [DbConnectionConfig]. + * @return the schema of the SQL query as a [DataFrameSchema] object. + */ +public fun DbConnectionConfig.readDataFrameSchema( + sqlQueryOrTableName: String, + dbType: DbType? = null, +): DataFrameSchema = + when { + isSqlQuery(sqlQueryOrTableName) -> DataFrameSchema.readSqlQuery(this, sqlQueryOrTableName, dbType) + + isSqlTableName(sqlQueryOrTableName) -> DataFrameSchema.readSqlTable(this, sqlQueryOrTableName, dbType) + + else -> throw IllegalArgumentException( + "$sqlQueryOrTableName should be SQL query or name of one of the existing SQL tables!", + ) + } + +/** + * Retrieves the schema of an SQL query result or the SQL table using the provided [DataSource]. + * + * ### Example with HikariCP: + * ```kotlin + * import com.zaxxer.hikari.HikariConfig + * import com.zaxxer.hikari.HikariDataSource + * + * val config = HikariConfig().apply { + * jdbcUrl = "jdbc:postgresql://localhost:5432/mydb" + * username = "user" + * password = "password" + * } + * val dataSource = HikariDataSource(config) + * + * // Get schema for a table + * val tableSchema = dataSource.getDataFrameSchema("customers") + * + * // Or get schema for a query + * val querySchema = dataSource.getDataFrameSchema("SELECT name, age FROM customers WHERE age > 25") + * + * // Inspect the schema + * println(tableSchema.columns) + * ``` + * + * @param [sqlQueryOrTableName] the SQL query to execute or name of the SQL table. + * It should be a name of one of the existing SQL tables, + * or the SQL query should start from SELECT and contain one query for reading data without any manipulation. + * It should not contain `;` symbol. + * @param [dbType] the type of database, could be a custom object, provided by user, optional, default is `null`, + * in that case the [dbType] will be recognized from the [DataSource]. + * @return the schema of the SQL query as a [DataFrameSchema] object. + * + * @see [DataSource.getConnection] + */ +public fun DataSource.readDataFrameSchema(sqlQueryOrTableName: String, dbType: DbType? = null): DataFrameSchema { + connection.use { conn -> + return when { + isSqlQuery(sqlQueryOrTableName) -> DataFrameSchema.readSqlQuery(conn, sqlQueryOrTableName, dbType) + + isSqlTableName(sqlQueryOrTableName) -> DataFrameSchema.readSqlTable(conn, sqlQueryOrTableName, dbType) + + else -> throw IllegalArgumentException( + "$sqlQueryOrTableName should be SQL query or name of one of the existing SQL tables!", + ) + } + } +} + +/** + * Retrieves the schema of an SQL query result or the SQL table using the provided database configuration. + * + * @param [sqlQueryOrTableName] the SQL query to execute and retrieve the schema from. + * @param [dbType] the type of database, could be a custom object, provided by user, optional, default is `null`, + * in that case the [dbType] will be recognized from the [Connection]. + * @return the schema of the SQL query as a [DataFrameSchema] object. + */ +public fun Connection.readDataFrameSchema(sqlQueryOrTableName: String, dbType: DbType? = null): DataFrameSchema = + when { + isSqlQuery(sqlQueryOrTableName) -> DataFrameSchema.readSqlQuery(this, sqlQueryOrTableName, dbType) + + isSqlTableName(sqlQueryOrTableName) -> DataFrameSchema.readSqlTable(this, sqlQueryOrTableName, dbType) + + else -> throw IllegalArgumentException( + "$sqlQueryOrTableName should be SQL query or name of one of the existing SQL tables!", + ) + } + +/** + * Retrieves the schema from [ResultSet]. + * + * NOTE: This function will not close connection and result set and not retrieve data from the result set. + * + * @param [resultSet] the [ResultSet] obtained from executing a database query. + * @param [dbType] the type of database that the [ResultSet] belongs to, could be a custom object, provided by user. + * @return the schema of the [ResultSet] as a [DataFrameSchema] object. + */ +public fun DataFrameSchema.Companion.readResultSet(resultSet: ResultSet, dbType: DbType): DataFrameSchema { + val tableColumns = getTableColumnsMetadata(resultSet) + return buildSchemaByTableColumns(tableColumns, dbType) +} + +/** + * Retrieves the schema from [ResultSet]. + * + * NOTE: This function will not close connection and result set and not retrieve data from the result set. + * + * @param [dbType] the type of database that the [ResultSet] belongs to, could be a custom object, provided by user. + * @return the schema of the [ResultSet] as a [DataFrameSchema] object. + */ +public fun ResultSet.readDataFrameSchema(dbType: DbType): DataFrameSchema = DataFrameSchema.readResultSet(this, dbType) + +/** + * Retrieves the schemas of all non-system tables in the database using the provided database configuration. + * + * ### Default Behavior: + * If [DbConnectionConfig.readOnly] is `true` (which is the default), the connection will be: + * - explicitly set as read-only via `Connection.setReadOnly(true)` + * - used with `autoCommit = false` + * - automatically rolled back after reading, ensuring no changes to the database + * + * Even if [DbConnectionConfig.readOnly] is set to `false`, the library still prevents data-modifying queries + * and only permits safe `SELECT` operations internally. + * + * @param [dbConfig] the database configuration to connect to the database, including URL, user, and password. + * @param [dbType] the type of database, could be a custom object, provided by user, optional, default is `null`, + * in that case the [dbType] will be recognized from the [dbConfig]. + * @return a map of [String, DataFrameSchema] objects representing the table name and its schema for each non-system table. + */ +public fun DataFrameSchema.Companion.readAllSqlTables( + dbConfig: DbConnectionConfig, + dbType: DbType? = null, +): Map = + withReadOnlyConnection(dbConfig, dbType) { connection -> + readAllSqlTables(connection, dbType) + } + +/** + * Retrieves the schemas of all non-system tables in the database using the provided [DataSource]. + * + * ### Example with HikariCP: + * ```kotlin + * import com.zaxxer.hikari.HikariConfig + * import com.zaxxer.hikari.HikariDataSource + * + * val config = HikariConfig().apply { + * jdbcUrl = "jdbc:postgresql://localhost:5432/mydb" + * username = "user" + * password = "password" + * } + * val dataSource = HikariDataSource(config) + * + * // Get schemas for all tables + * val allSchemas = DataFrame.getSchemaForAllSqlTables(dataSource) + * + * // Access individual table schemas + * val customersSchema = allSchemas["customers"] + * val ordersSchema = allSchemas["orders"] + * + * // Iterate through all schemas + * allSchemas.forEach { (tableName, schema) -> + * println("Table: \$tableName, Columns: \${schema.columns.keys}") + * } + * ``` + * + * @param [dataSource] the DataSource to get a database connection from. + * @param [dbType] the type of database, could be a custom object, provided by user, optional, default is `null`, + * in that case the [dbType] will be recognized from the [dataSource]. + * @return a map of [String, DataFrameSchema] objects representing the table name and its schema for each non-system table. + * + * @see [DataSource.getConnection] + */ +public fun DataFrameSchema.Companion.readAllSqlTables( + dataSource: DataSource, + dbType: DbType? = null, +): Map { + dataSource.connection.use { connection -> + return readAllSqlTables(connection, dbType) + } +} + +/** + * Retrieves the schemas of all non-system tables in the database using the provided database connection. + * + * @param [connection] the database connection. + * @param [dbType] the type of database, could be a custom object, provided by user, optional, default is `null`, + * in that case the [dbType] will be recognized from the [connection]. + * @return a map of [String, DataFrameSchema] objects representing the table name and its schema for each non-system table. + */ +public fun DataFrameSchema.Companion.readAllSqlTables( + connection: Connection, + dbType: DbType? = null, +): Map { + val metaData = connection.metaData + val determinedDbType = dbType ?: extractDBTypeFromConnection(connection) + + // exclude system- and other tables without data + val tableTypes = determinedDbType.tableTypes?.toTypedArray() + val tables = metaData.getTables(null, null, null, tableTypes) + + val dataFrameSchemas = mutableMapOf() + + while (tables.next()) { + val jdbcTable = determinedDbType.buildTableMetadata(tables) + if (!determinedDbType.isSystemTable(jdbcTable)) { + // we filter her a second time because of specific logic with SQLite and possible issues with future databases + val tableName = jdbcTable.name + val dataFrameSchema = readSqlTable(connection, tableName, determinedDbType) + dataFrameSchemas += tableName to dataFrameSchema + } + } + + return dataFrameSchemas +} + +/** + * Builds a DataFrame schema based on the given table columns. + * + * @param [tableColumns] a mutable map containing the table columns, where the key represents the column name + * and the value represents the metadata of the column + * @param [dbType] the type of database. + * @return a [DataFrameSchema] object representing the schema built from the table columns. + */ +internal fun buildSchemaByTableColumns( + tableColumns: MutableList, + dbType: DbType, +): DataFrameSchema { + val schemaColumns = tableColumns.associate { + Pair(it.name, generateColumnSchemaValue(dbType, it)) + } + + return DataFrameSchemaImpl( + columns = schemaColumns, + ) +} + +internal fun generateColumnSchemaValue(dbType: DbType, tableColumnMetadata: TableColumnMetadata): ColumnSchema = + dbType.convertSqlTypeToColumnSchemaValue(tableColumnMetadata) + ?: ColumnSchema.Value(dbType.makeCommonSqlToKTypeMapping(tableColumnMetadata)) diff --git a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/readJdbc.kt b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/readJdbc.kt index 3a738583c9..a2594ce131 100644 --- a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/readJdbc.kt +++ b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/readJdbc.kt @@ -2,236 +2,104 @@ package org.jetbrains.kotlinx.dataframe.io import io.github.oshai.kotlinlogging.KotlinLogging import org.jetbrains.kotlinx.dataframe.AnyFrame -import org.jetbrains.kotlinx.dataframe.DataColumn import org.jetbrains.kotlinx.dataframe.DataFrame -import org.jetbrains.kotlinx.dataframe.api.Infer import org.jetbrains.kotlinx.dataframe.api.toDataFrame -import org.jetbrains.kotlinx.dataframe.impl.schema.DataFrameSchemaImpl import org.jetbrains.kotlinx.dataframe.io.db.DbType +import org.jetbrains.kotlinx.dataframe.io.db.TableColumnMetadata import org.jetbrains.kotlinx.dataframe.io.db.extractDBTypeFromConnection -import org.jetbrains.kotlinx.dataframe.io.db.extractDBTypeFromUrl -import org.jetbrains.kotlinx.dataframe.schema.ColumnSchema -import org.jetbrains.kotlinx.dataframe.schema.DataFrameSchema -import java.math.BigDecimal -import java.sql.Blob -import java.sql.Clob import java.sql.Connection import java.sql.DatabaseMetaData import java.sql.DriverManager -import java.sql.NClob -import java.sql.Ref +import java.sql.PreparedStatement import java.sql.ResultSet import java.sql.ResultSetMetaData -import java.sql.RowId -import java.sql.SQLException -import java.sql.SQLXML -import java.sql.Time -import java.sql.Timestamp -import java.sql.Types -import java.time.LocalDateTime -import java.time.OffsetDateTime -import java.time.OffsetTime -import java.util.Date -import java.util.UUID -import kotlin.reflect.KClass +import javax.sql.DataSource import kotlin.reflect.KType -import kotlin.reflect.full.createType -import kotlin.reflect.full.isSupertypeOf -import kotlin.reflect.full.safeCast -import kotlin.reflect.full.starProjectedType private val logger = KotlinLogging.logger {} /** - * The default limit value. - * - * This constant represents the default limit value to be used in cases where no specific limit - * is provided. - * - * @see Int.MIN_VALUE - */ -private const val DEFAULT_LIMIT = Int.MIN_VALUE - -/** - * A regular expression defining the valid pattern for SQL table names. - * - * This pattern enforces that table names must: - * - Contain only Unicode letters, Unicode digits, or underscores. - * - Optionally be segmented by dots to indicate schema and table separation. - * - * It ensures compatibility with most SQL database naming conventions, thus minimizing risks of invalid names - * or injection vulnerabilities. - * - * Example of valid table names: - * - `my_table` - * - `schema1.table2` - * - * Example of invalid table names: - * - `my-table` (contains a dash) - * - `table!name` (contains special characters) - * - `.startWithDot` (cannot start with a dot) - */ -private const val TABLE_NAME_VALID_PATTERN = "^[\\p{L}\\p{N}_]+(\\.[\\p{L}\\p{N}_]+)*$" - -/** - * Represents a column in a database table to keep all required meta-information. - * - * @property [name] the name of the column. - * @property [sqlTypeName] the SQL data type of the column. - * @property [jdbcType] the JDBC data type of the column produced from [java.sql.Types]. - * @property [size] the size of the column. - * @property [javaClassName] the class name in Java. - * @property [isNullable] true if column could contain nulls. - */ -public data class TableColumnMetadata( - val name: String, - val sqlTypeName: String, - val jdbcType: Int, - val size: Int, - val javaClassName: String, - val isNullable: Boolean = false, -) - -/** - * Represents a table metadata to store information about a database table, - * including its name, schema name, and catalogue name. - * - * NOTE: we need to extract both, [schemaName] and [catalogue] - * because the different databases have different implementations of metadata. - * - * @property [name] the name of the table. - * @property [schemaName] the name of the schema the table belongs to (optional). - * @property [catalogue] the name of the catalogue the table belongs to (optional). - */ -public data class TableMetadata(val name: String, val schemaName: String?, val catalogue: String?) - -/** - * Represents the configuration for an internally managed JDBC database connection. - * - * This class defines connection parameters used by the library to create a `Connection` - * when the user does not provide one explicitly. It is designed for safe, read-only access by default. - * - * @property url The JDBC URL of the database, e.g., `"jdbc:postgresql://localhost:5432/mydb"`. - * Must follow the standard format: `jdbc:subprotocol:subname`. - * - * @property user The username used for authentication. - * Optional, default is an empty string. - * - * @property password The password used for authentication. - * Optional, default is an empty string. - * - * @property readOnly If `true` (default), the library will create the connection in read-only mode. - * This enables the following behavior: - * - `Connection.setReadOnly(true)` - * - `Connection.setAutoCommit(false)` - * - automatic `rollback()` at the end of execution - * - * If `false`, the connection will be created with JDBC defaults (usually read-write), - * but the library will still reject any queries that appear to modify data - * (e.g. contain `INSERT`, `UPDATE`, `DELETE`, etc.). - * - * Note: Connections created using this configuration are managed entirely by the library. - * Users do not have access to the underlying `Connection` instance and cannot commit or close it manually. - * - * ### Examples: - * - * ```kotlin - * // Safe read-only connection (default) - * val config = DbConnectionConfig("jdbc:sqlite::memory:") - * val df = DataFrame.readSqlQuery(config, "SELECT * FROM books") - * - * // Use default JDBC connection settings (still protected against mutations) - * val config = DbConnectionConfig( - * url = "jdbc:sqlite::memory:", - * readOnly = false - * ) - * ``` - */ -public data class DbConnectionConfig( - val url: String, - val user: String = "", - val password: String = "", - val readOnly: Boolean = true, -) - -/** - * Executes the given block with a managed JDBC connection created from [DbConnectionConfig]. + * Reads data from an SQL table and converts it into a DataFrame. * - * If [DbConnectionConfig.readOnly] is `true` (default), the connection will be: - * - explicitly marked as read-only - * - used with auto-commit disabled - * - rolled back after execution to prevent unintended modifications + * ### Default Behavior: + * If [DbConnectionConfig.readOnly] is `true` (which is the default), the connection will be: + * - explicitly set as read-only via `Connection.setReadOnly(true)` + * - used with `autoCommit = false` + * - automatically rolled back after reading, ensuring no changes to the database * - * This utility guarantees proper closing of the connection and safe rollback in read-only mode. - * It should be used when the user does not manually manage JDBC connections. + * Even if [DbConnectionConfig.readOnly] is set to `false`, the library still prevents data-modifying queries + * and only permits safe `SELECT` operations internally. * - * @param [dbConfig] The configuration used to create the connection. - * @param [dbType] Optional database type (not used here but can be passed through for logging or future extensions). - * @param [block] A lambda with receiver that runs with an open and managed [Connection]. - * @return The result of the [block] execution. + * @param [dbConfig] the configuration for the database, including URL, user, and password. + * @param [tableName] the name of the table to read data from. + * @param [limit] the maximum number of rows to retrieve from the table. + * `null` (default) means no limit - all available rows will be fetched + * or positive integer (e.g., `100`) - fetch at most that many rows + * @param [inferNullability] indicates how the column nullability should be inferred. + * @param [dbType] the type of database, could be a custom object, provided by user, optional, default is `null`, + * in that case the [dbType] will be recognized from the [dbConfig]. + * @param [strictValidation] if `true`, the method validates that the provided table name is in a valid format. + * Default is `true` for strict validation. + * @param [configureStatement] optional lambda to configure the [PreparedStatement] before execution. + * This allows for custom tuning of fetch size, query timeout, and other JDBC parameters. + * @return the DataFrame containing the data from the SQL table. */ -internal inline fun withReadOnlyConnection( +public fun DataFrame.Companion.readSqlTable( dbConfig: DbConnectionConfig, + tableName: String, + limit: Int? = null, + inferNullability: Boolean = true, dbType: DbType? = null, - block: (Connection) -> T, -): T { - val actualDbType = dbType ?: extractDBTypeFromUrl(dbConfig.url) - val connection = actualDbType.createConnection(dbConfig) - - return connection.use { conn -> - try { - if (dbConfig.readOnly) { - conn.autoCommit = false - } - - block(conn) - } finally { - if (dbConfig.readOnly) { - try { - conn.rollback() - } catch (e: SQLException) { - logger.warn(e) { - "Failed to rollback read-only transaction (url=${dbConfig.url})" - } - } - } - } + strictValidation: Boolean = true, + configureStatement: (PreparedStatement) -> Unit = {}, +): AnyFrame { + validateLimit(limit) + return withReadOnlyConnection(dbConfig, dbType) { conn -> + readSqlTable(conn, tableName, limit, inferNullability, dbType, strictValidation, configureStatement) } } /** * Reads data from an SQL table and converts it into a DataFrame. * - * @param [dbConfig] the configuration for the database, including URL, user, and password. + * @param [dataSource] the [DataSource] to get a database connection from. * @param [tableName] the name of the table to read data from. * @param [limit] the maximum number of rows to retrieve from the table. + * `null` (default) means no limit - all available rows will be fetched + * or positive integer (e.g., `100`) - fetch at most that many rows * @param [inferNullability] indicates how the column nullability should be inferred. * @param [dbType] the type of database, could be a custom object, provided by user, optional, default is `null`, - * in that case the [dbType] will be recognized from the [dbConfig]. + * in that case the [dbType] will be recognized from the [dataSource]. * @param [strictValidation] if `true`, the method validates that the provided table name is in a valid format. * Default is `true` for strict validation. + * @param [configureStatement] optional lambda to configure the [PreparedStatement] before execution. + * This allows for custom tuning of fetch size, query timeout, and other JDBC parameters. * @return the DataFrame containing the data from the SQL table. * - * ### Default Behavior: - * If [DbConnectionConfig.readOnly] is `true` (which is the default), the connection will be: - * - explicitly set as read-only via `Connection.setReadOnly(true)` - * - used with `autoCommit = false` - * - automatically rolled back after reading, ensuring no changes to the database - * - * Even if [DbConnectionConfig.readOnly] is set to `false`, the library still prevents data-modifying queries - * and only permits safe `SELECT` operations internally. + * @see [DataSource.getConnection] */ public fun DataFrame.Companion.readSqlTable( - dbConfig: DbConnectionConfig, + dataSource: DataSource, tableName: String, - limit: Int = DEFAULT_LIMIT, + limit: Int? = null, inferNullability: Boolean = true, dbType: DbType? = null, strictValidation: Boolean = true, -): AnyFrame = - withReadOnlyConnection(dbConfig, dbType) { conn -> - readSqlTable(conn, tableName, limit, inferNullability, dbType, strictValidation) + configureStatement: (PreparedStatement) -> Unit = {}, +): AnyFrame { + validateLimit(limit) + dataSource.connection.use { connection -> + return readSqlTable( + connection, + tableName, + limit, + inferNullability, + dbType, + strictValidation, + configureStatement, + ) } +} /** * Reads data from an SQL table and converts it into a DataFrame. @@ -239,23 +107,29 @@ public fun DataFrame.Companion.readSqlTable( * @param [connection] the database connection to read tables from. * @param [tableName] the name of the table to read data from. * @param [limit] the maximum number of rows to retrieve from the table. + * `null` (default) means no limit - all available rows will be fetched + * or positive integer (e.g., `100`) - fetch at most that many rows * @param [inferNullability] indicates how the column nullability should be inferred. * @param [dbType] the type of database, could be a custom object, provided by user, optional, default is `null`, * in that case the [dbType] will be recognized from the [connection]. * @param [strictValidation] if `true`, the method validates that the provided table name is in a valid format. * Default is `true` for strict validation. + * @param [configureStatement] optional lambda to configure the [PreparedStatement] before execution. + * This allows for custom tuning of fetch size, query timeout, and other JDBC parameters. * @return the DataFrame containing the data from the SQL table. * - * @see DriverManager.getConnection + * @see [DriverManager.getConnection] */ public fun DataFrame.Companion.readSqlTable( connection: Connection, tableName: String, - limit: Int = DEFAULT_LIMIT, + limit: Int? = null, inferNullability: Boolean = true, dbType: DbType? = null, strictValidation: Boolean = true, + configureStatement: (PreparedStatement) -> Unit = {}, ): AnyFrame { + validateLimit(limit) if (strictValidation) { require(isValidTableName(tableName)) { "The provided table name '$tableName' is invalid. Please ensure it matches a valid table name in the database schema." @@ -264,89 +138,179 @@ public fun DataFrame.Companion.readSqlTable( logger.warn { "Strict validation is disabled. Make sure the table name '$tableName' is correct." } } - val url = connection.metaData.url val determinedDbType = dbType ?: extractDBTypeFromConnection(connection) - val selectAllQuery = if (limit > 0) { - determinedDbType.sqlQueryLimit("SELECT * FROM $tableName", limit) - } else { - "SELECT * FROM $tableName" - } + // Build SQL query using DbType + val sqlQuery = determinedDbType.buildSelectTableQueryWithLimit(tableName, limit) - connection.prepareStatement(selectAllQuery).use { st -> - logger.debug { "Connection with url:$url is established successfully." } + return executeQueryAndBuildDataFrame( + connection, + sqlQuery, + determinedDbType, + configureStatement, + limit, + inferNullability, + ) +} - st.executeQuery().use { rs -> - val tableColumns = getTableColumnsMetadata(rs) - return fetchAndConvertDataFromResultSet(tableColumns, rs, determinedDbType, limit, inferNullability) +/** + * Reads a data frame from the specified database using the provided SQL query and configurations. + * + * @param [connection] The database connection to be used for executing the query. + * @param [sqlQuery] The SQL query string to be executed. + * @param [determinedDbType] The type of database being accessed, which determines specific configurations. + * @param [configureStatement] A lambda function to configure the prepared statement before execution. + * @param [limit] the maximum number of rows to retrieve from the table. + * `null` (default) means no limit - all available rows will be fetched. + * @param [inferNullability] A flag to determine whether to infer nullability for result set fields. + * @return The data frame constructed from the database query results. + * @throws [IllegalStateException] If an error occurs while reading from the database or processing the data. + */ +private fun executeQueryAndBuildDataFrame( + connection: Connection, + sqlQuery: String, + determinedDbType: DbType, + configureStatement: (PreparedStatement) -> Unit, + limit: Int?, + inferNullability: Boolean, +): AnyFrame = + try { + connection.prepareStatement(sqlQuery).use { statement -> + logger.debug { "Connection established successfully (${connection.metaData.databaseProductName})" } + determinedDbType.configureReadStatement(statement) + configureStatement(statement) + logger.debug { "Executing query: $sqlQuery" } + statement.executeQuery().use { rs -> + val tableColumns = getTableColumnsMetadata(rs) + fetchAndConvertDataFromResultSet(tableColumns, rs, determinedDbType, limit, inferNullability) + } } + } catch (e: java.sql.SQLException) { + // Provide the same type for all SQLExceptions from JDBC and enrich with additional information + logger.error(e) { "Database operation failed: $sqlQuery" } + throw IllegalStateException( + "Failed to read from database. Query: $sqlQuery, Database: ${determinedDbType.dbTypeInJdbcUrl}", + e, + ) + } catch (e: Exception) { + // Provide the same type for all unexpected errors from JDBC + logger.error(e) { "Unexpected error: ${e.message}" } + throw IllegalStateException("Unexpected error while reading from database", e) } -} /** * Converts the result of an SQL query to the DataFrame. * - * NOTE: SQL query should start from SELECT and contain one query for reading data without any manipulation. + * __NOTE:__ SQL query should start from SELECT and contain one query for reading data without any manipulation. * It should not contain `;` symbol. * + * ### Default Behavior: + * If [DbConnectionConfig.readOnly] is `true` (which is the default), the connection will be: + * - explicitly set as read-only via `Connection.setReadOnly(true)` + * - used with `autoCommit = false` + * - automatically rolled back after reading, ensuring no changes to the database + * + * Even if [DbConnectionConfig.readOnly] is set to `false`, the library still prevents data-modifying queries + * and only permits safe `SELECT` operations internally. + * * @param [dbConfig] the database configuration to connect to the database, including URL, user, and password. * @param [sqlQuery] the SQL query to execute. * @param [limit] the maximum number of rows to retrieve from the result of the SQL query execution. + * `null` (default) means no limit - all available rows will be fetched + * or positive integer (e.g., `100`) - fetch at most that many rows * @param [inferNullability] indicates how the column nullability should be inferred. * @param [dbType] the type of database, could be a custom object, provided by user, optional, default is `null`, * in that case the [dbType] will be recognized from the [dbConfig]. * @param [strictValidation] if `true`, the method validates that the provided query is in a valid format. * Default is `true` for strict validation. + * @param [configureStatement] optional lambda to configure the [PreparedStatement] before execution. + * This allows for custom tuning of fetch size, query timeout, and other JDBC parameters. * @return the DataFrame containing the result of the SQL query. + */ +public fun DataFrame.Companion.readSqlQuery( + dbConfig: DbConnectionConfig, + sqlQuery: String, + limit: Int? = null, + inferNullability: Boolean = true, + dbType: DbType? = null, + strictValidation: Boolean = true, + configureStatement: (PreparedStatement) -> Unit = {}, +): AnyFrame { + validateLimit(limit) + return withReadOnlyConnection(dbConfig, dbType) { conn -> + readSqlQuery(conn, sqlQuery, limit, inferNullability, dbType, strictValidation, configureStatement) + } +} + +/** + * Converts the result of an SQL query to the DataFrame. * - * ### Default Behavior: - * If [DbConnectionConfig.readOnly] is `true` (which is the default), the connection will be: - * - explicitly set as read-only via `Connection.setReadOnly(true)` - * - used with `autoCommit = false` - * - automatically rolled back after reading, ensuring no changes to the database + * __NOTE:__ SQL query should start from SELECT and contain one query for reading data without any manipulation. + * It should not contain `;` symbol. * - * Even if [DbConnectionConfig.readOnly] is set to `false`, the library still prevents data-modifying queries - * and only permits safe `SELECT` operations internally. + * @param [dataSource] the [DataSource] to obtain a database connection from. + * @param [sqlQuery] the SQL query to execute. + * @param [limit] the maximum number of rows to retrieve from the result of the SQL query execution. + * `null` (default) means no limit - all available rows will be fetched + * or positive integer (e.g., `100`) - fetch at most that many rows + * @param [inferNullability] indicates how the column nullability should be inferred. + * @param [dbType] the type of database, could be a custom object, provided by user, optional, default is `null`, + * in that case the [dbType] will be recognized from the [dataSource]. + * @param [strictValidation] if `true`, the method validates that the provided query is in a valid format. + * Default is `true` for strict validation. + * @param [configureStatement] optional lambda to configure the [PreparedStatement] before execution. + * This allows for custom tuning of fetch size, query timeout, and other JDBC parameters. + * @return the DataFrame containing the result of the SQL query. + * + * @see [DataSource.getConnection] */ - public fun DataFrame.Companion.readSqlQuery( - dbConfig: DbConnectionConfig, + dataSource: DataSource, sqlQuery: String, - limit: Int = DEFAULT_LIMIT, + limit: Int? = null, inferNullability: Boolean = true, dbType: DbType? = null, strictValidation: Boolean = true, -): AnyFrame = - withReadOnlyConnection(dbConfig, dbType) { conn -> - readSqlQuery(conn, sqlQuery, limit, inferNullability, dbType, strictValidation) + configureStatement: (PreparedStatement) -> Unit = {}, +): AnyFrame { + validateLimit(limit) + dataSource.connection.use { connection -> + return readSqlQuery(connection, sqlQuery, limit, inferNullability, dbType, strictValidation, configureStatement) } +} /** * Converts the result of an SQL query to the DataFrame. * - * NOTE: SQL query should start from SELECT and contain one query for reading data without any manipulation. + * __NOTE:__ SQL query should start from SELECT and contain one query for reading data without any manipulation. * It should not contain `;` symbol. * * @param [connection] the database connection to execute the SQL query. * @param [sqlQuery] the SQL query to execute. * @param [limit] the maximum number of rows to retrieve from the result of the SQL query execution. + * `null` (default) means no limit - all available rows will be fetched + * or positive integer (e.g., `100`) - fetch at most that many rows * @param [inferNullability] indicates how the column nullability should be inferred. * @param [dbType] the type of database, could be a custom object, provided by user, optional, default is `null`, * in that case the [dbType] will be recognized from the [connection]. * @param [strictValidation] if `true`, the method validates that the provided query is in a valid format. * Default is `true` for strict validation. + * @param [configureStatement] optional lambda to configure the [PreparedStatement] before execution. + * This allows for custom tuning of fetch size, query timeout, and other JDBC parameters. * @return the DataFrame containing the result of the SQL query. * - * @see DriverManager.getConnection + * @see [DriverManager.getConnection] */ public fun DataFrame.Companion.readSqlQuery( connection: Connection, sqlQuery: String, - limit: Int = DEFAULT_LIMIT, + limit: Int? = null, inferNullability: Boolean = true, dbType: DbType? = null, strictValidation: Boolean = true, + configureStatement: (PreparedStatement) -> Unit = {}, ): AnyFrame { + validateLimit(limit) if (strictValidation) { require(isValidSqlQuery(sqlQuery)) { "SQL query should start from SELECT and contain one query for reading data without any manipulation. " + @@ -358,50 +322,58 @@ public fun DataFrame.Companion.readSqlQuery( val determinedDbType = dbType ?: extractDBTypeFromConnection(connection) - val internalSqlQuery = if (limit > 0) determinedDbType.sqlQueryLimit(sqlQuery, limit) else sqlQuery - - logger.debug { "Executing SQL query: $internalSqlQuery" } - - connection.prepareStatement(internalSqlQuery).use { st -> - st.executeQuery().use { rs -> - val tableColumns = getTableColumnsMetadata(rs) - return fetchAndConvertDataFromResultSet(tableColumns, rs, determinedDbType, limit, inferNullability) - } - } + val internalSqlQuery = limit?.let { + determinedDbType.buildSqlQueryWithLimit(sqlQuery, it) + } ?: sqlQuery + + return executeQueryAndBuildDataFrame( + connection, + internalSqlQuery, + determinedDbType, + configureStatement, + limit, + inferNullability, + ) } /** * Converts the result of an SQL query or SQL table (by name) to the DataFrame. * + * ### Default Behavior: + * If [DbConnectionConfig.readOnly] is `true` (which is the default), the connection will be: + * - explicitly set as read-only via `Connection.setReadOnly(true)` + * - used with `autoCommit = false` + * - automatically rolled back after reading, ensuring no changes to the database + * + * Even if [DbConnectionConfig.readOnly] is set to `false`, the library still prevents data-modifying queries + * and only permits safe `SELECT` operations internally. + * * @param [sqlQueryOrTableName] the SQL query to execute or name of the SQL table. * It should be a name of one of the existing SQL tables, * or the SQL query should start from SELECT and contain one query for reading data without any manipulation. * It should not contain `;` symbol. * @param [limit] the maximum number of rows to retrieve from the result of the SQL query execution. + * `null` (default) means no limit - all available rows will be fetched + * or positive integer (e.g., `100`) - fetch at most that many rows * @param [inferNullability] indicates how the column nullability should be inferred. * @param [dbType] the type of database, could be a custom object, provided by user, optional, default is `null`, * in that case the [dbType] will be recognized from the [DbConnectionConfig]. * @param [strictValidation] if `true`, the method validates that the provided query or table name is in a valid format. * Default is `true` for strict validation. + * @param [configureStatement] optional lambda to configure the [PreparedStatement] before execution. + * This allows for custom tuning of fetch size, query timeout, and other JDBC parameters. * @return the DataFrame containing the result of the SQL query. - * - * ### Default Behavior: - * If [DbConnectionConfig.readOnly] is `true` (which is the default), the connection will be: - * - explicitly set as read-only via `Connection.setReadOnly(true)` - * - used with `autoCommit = false` - * - automatically rolled back after reading, ensuring no changes to the database - * - * Even if [DbConnectionConfig.readOnly] is set to `false`, the library still prevents data-modifying queries - * and only permits safe `SELECT` operations internally. */ public fun DbConnectionConfig.readDataFrame( sqlQueryOrTableName: String, - limit: Int = DEFAULT_LIMIT, + limit: Int? = null, inferNullability: Boolean = true, dbType: DbType? = null, strictValidation: Boolean = true, -): AnyFrame = - when { + configureStatement: (PreparedStatement) -> Unit = {}, +): AnyFrame { + validateLimit(limit) + return when { isSqlQuery(sqlQueryOrTableName) -> DataFrame.readSqlQuery( this, sqlQueryOrTableName, @@ -409,6 +381,7 @@ public fun DbConnectionConfig.readDataFrame( inferNullability, dbType, strictValidation, + configureStatement, ) isSqlTableName(sqlQueryOrTableName) -> DataFrame.readSqlTable( @@ -418,22 +391,13 @@ public fun DbConnectionConfig.readDataFrame( inferNullability, dbType, strictValidation, + configureStatement, ) else -> throw IllegalArgumentException( "$sqlQueryOrTableName should be SQL query or name of one of the existing SQL tables!", ) } - -private fun isSqlQuery(sqlQueryOrTableName: String): Boolean { - val queryPattern = Regex("(?i)\\b(SELECT)\\b") - return queryPattern.containsMatchIn(sqlQueryOrTableName.trim()) -} - -private fun isSqlTableName(sqlQueryOrTableName: String): Boolean { - // Match table names with optional schema and catalog (e.g., catalog.schema.table) - val tableNamePattern = Regex("^[a-zA-Z_][a-zA-Z0-9_]*(\\.[a-zA-Z_][a-zA-Z0-9_]*){0,2}$") - return tableNamePattern.matches(sqlQueryOrTableName.trim()) } /** @@ -444,21 +408,27 @@ private fun isSqlTableName(sqlQueryOrTableName: String): Boolean { * or the SQL query should start from SELECT and contain one query for reading data without any manipulation. * It should not contain `;` symbol. * @param [limit] the maximum number of rows to retrieve from the result of the SQL query execution. + * `null` (default) means no limit - all available rows will be fetched + * or positive integer (e.g., `100`) - fetch at most that many rows * @param [inferNullability] indicates how the column nullability should be inferred. * @param [dbType] the type of database, could be a custom object, provided by user, optional, default is `null`, * in that case the [dbType] will be recognized from the [Connection]. * @param [strictValidation] if `true`, the method validates that the provided query or table name is in a valid format. * Default is `true` for strict validation. + * @param [configureStatement] optional lambda to configure the [PreparedStatement] before execution. + * This allows for custom tuning of fetch size, query timeout, and other JDBC parameters. * @return the DataFrame containing the result of the SQL query. */ public fun Connection.readDataFrame( sqlQueryOrTableName: String, - limit: Int = DEFAULT_LIMIT, + limit: Int? = null, inferNullability: Boolean = true, dbType: DbType? = null, strictValidation: Boolean = true, -): AnyFrame = - when { + configureStatement: (PreparedStatement) -> Unit = {}, +): AnyFrame { + validateLimit(limit) + return when { isSqlQuery(sqlQueryOrTableName) -> DataFrame.readSqlQuery( this, sqlQueryOrTableName, @@ -466,6 +436,7 @@ public fun Connection.readDataFrame( inferNullability, dbType, strictValidation, + configureStatement, ) isSqlTableName(sqlQueryOrTableName) -> DataFrame.readSqlTable( @@ -475,158 +446,91 @@ public fun Connection.readDataFrame( inferNullability, dbType, strictValidation, + configureStatement, ) else -> throw IllegalArgumentException( "$sqlQueryOrTableName should be SQL query or name of one of the existing SQL tables!", ) } - -private val FORBIDDEN_PATTERNS_REGEX = listOf( - ";", // Separator for SQL statements - "--", // Single-line comments - "/\\*", // Start of multi-line comments - "\\*/", // End of multi-line comments - "\\bDROP\\b", // DROP as a full word - "\\bDELETE\\b", // DELETE as a full word - "\\bINSERT\\b", // INSERT as a full word - "\\bUPDATE\\b", // UPDATE as a full word - "\\bEXEC\\b", // EXEC as a full word - "\\bEXECUTE\\b", // EXECUTE as a full word - "\\bCREATE\\b", // CREATE as a full word - "\\bALTER\\b", // ALTER as a full word - "\\bGRANT\\b", // GRANT as a full word - "\\bREVOKE\\b", // REVOKE as a full word - "\\bMERGE\\b", // MERGE as a full word -).map { Regex(it, RegexOption.IGNORE_CASE) } +} /** - * Checks if a given string contains forbidden patterns or keywords. - * Logs a clear and friendly message if any forbidden pattern is found. - * - * ### Forbidden SQL Examples: - * 1. **Single-line comment** (using `--`): - * - `SELECT * FROM Sale WHERE amount = 100.0 -- AND id = 5` - * - * 2. **Multi-line comment** (using `/* */`): - * - `SELECT * FROM Customer /* Possible malicious comment */ WHERE id = 1` - * - * 3. **Multiple statements separated by semicolon (`;`)**: - * - `SELECT * FROM Sale WHERE amount = 500.0; DROP TABLE Customer` - * - * 4. **Potentially malicious SQL with single quotes for injection**: - * - `SELECT * FROM Sale WHERE id = 1 AND amount = 100.0 OR '1'='1` + * Converts the result of an SQL query or SQL table (by name) to the DataFrame. * - * 5. **Usage of dangerous commands like `DROP`, `DELETE`, `ALTER`, etc.**: - * - `DROP TABLE Customer; SELECT * FROM Sale` + * ### Example with HikariCP: + * ```kotlin + * import com.zaxxer.hikari.HikariConfig + * import com.zaxxer.hikari.HikariDataSource * - * ### Allowed SQL Examples: - * 1. Query with names containing reserved words as parts of identifiers: - * - `SELECT last_update FROM HELLO_ALTER` + * val config = HikariConfig().apply { + * jdbcUrl = "jdbc:postgresql://localhost:5432/mydb" + * username = "user" + * password = "password" + * } + * val dataSource = HikariDataSource(config) * - * 2. Query with fully valid syntax: - * - `SELECT id, name FROM Customers WHERE age > 25` + * // Read from a table + * val customersDF = dataSource.readDataFrame("customers", limit = 100) * - * 3. Query with identifiers resembling commands but not in forbidden contexts: - * - `SELECT id, amount FROM TRANSACTION_DROP` + * // Or execute a query + * val queryDF = dataSource.readDataFrame("SELECT * FROM orders WHERE amount > 100") + * ``` * - * 4. Query with case-insensitive identifiers: - * - `select Id, Name from Hello_Table` + * @param [sqlQueryOrTableName] the SQL query to execute or name of the SQL table. + * It should be a name of one of the existing SQL tables, + * or the SQL query should start from SELECT and contain one query for reading data without any manipulation. + * It should not contain `;` symbol. + * @param [limit] the maximum number of rows to retrieve from the result of the SQL query execution. + * `null` (default) means no limit - all available rows will be fetched + * or positive integer (e.g., `100`) - fetch at most that many rows + * @param [inferNullability] indicates how the column nullability should be inferred. + * @param [dbType] the type of database, could be a custom object, provided by user, optional, default is `null`, + * in that case the [dbType] will be recognized from the [DataSource]. + * @param [strictValidation] if `true`, the method validates that the provided query or table name is in a valid format. + * Default is `true` for strict validation. + * @param [configureStatement] optional lambda to configure the [PreparedStatement] before execution. + * This allows for custom tuning of fetch size, query timeout, and other JDBC parameters. + * @return the DataFrame containing the result of the SQL query. * - * ### Key Notes: - * - Reserved keywords like `DROP`, `DELETE`, `ALTER`, etc., are forbidden **only when they appear as standalone commands**. - * - Reserved words as parts of table or column names (e.g., `HELLO_ALTER`, `myDropTable`) **are allowed**. - * - Inline or multi-line comments (`--` or `/* */`) are restricted to prevent potential SQL injection attacks. - * - Multiple SQL statements separated by semicolons (`;`) are not allowed to prevent the execution of unintended commands. - */ -private fun hasForbiddenPatterns(input: String): Boolean { - for (regex in FORBIDDEN_PATTERNS_REGEX) { - if (regex.containsMatchIn(input)) { - logger.error { - "Validation failed: The input contains a forbidden element matching '${regex.pattern}'. Please review the input: '$input'." - } - return true - } - } - return false -} - -/** - * Allowed list of SQL operators - */ -private val ALLOWED_SQL_OPERATORS = listOf("SELECT", "WITH", "VALUES", "TABLE") - -/** - * Validates if the SQL query is safe and starts with SELECT. - * Ensures a proper syntax structure, checks for balanced quotes, and disallows dangerous commands or patterns. - */ -private fun isValidSqlQuery(sqlQuery: String): Boolean { - val normalizedSqlQuery = sqlQuery.trim().uppercase() - - // Log the query being validated - logger.warn { "Validating SQL query: '$sqlQuery'" } - - // Ensure the query starts from one of the allowed SQL operators - if (ALLOWED_SQL_OPERATORS.none { normalizedSqlQuery.startsWith(it) }) { - logger.error { - "Validation failed: The SQL query must start with one of: $ALLOWED_SQL_OPERATORS. Given query: '$sqlQuery'." - } - return false - } - - // Validate against forbidden patterns - if (hasForbiddenPatterns(normalizedSqlQuery)) { - return false - } - - // Check if there are balanced quotes (single and double) - val singleQuotes = sqlQuery.count { it == '\'' } - val doubleQuotes = sqlQuery.count { it == '"' } - if (singleQuotes % 2 != 0) { - logger.error { - "Validation failed: Unbalanced single quotes in the SQL query. " + - "Please correct the query: '$sqlQuery'." - } - return false - } - if (doubleQuotes % 2 != 0) { - logger.error { - "Validation failed: Unbalanced double quotes in the SQL query. " + - "Please correct the query: '$sqlQuery'." - } - return false - } - - logger.warn { "SQL query validation succeeded for query: '$sqlQuery'." } - return true -} - -/** - * Validates if the given SQL table name is safe and logs any validation violations. + * @see [DataSource.getConnection] */ -private fun isValidTableName(tableName: String): Boolean { - val normalizedTableName = tableName.trim().uppercase() - - // Log the table name being validated - logger.warn { "Validating SQL table name: '$tableName'" } +public fun DataSource.readDataFrame( + sqlQueryOrTableName: String, + limit: Int? = null, + inferNullability: Boolean = true, + dbType: DbType? = null, + strictValidation: Boolean = true, + configureStatement: (PreparedStatement) -> Unit = {}, +): AnyFrame { + validateLimit(limit) + connection.use { conn -> + return when { + isSqlQuery(sqlQueryOrTableName) -> DataFrame.readSqlQuery( + conn, + sqlQueryOrTableName, + limit, + inferNullability, + dbType, + strictValidation, + configureStatement, + ) - // Validate against forbidden patterns - if (hasForbiddenPatterns(normalizedTableName)) { - return false - } + isSqlTableName(sqlQueryOrTableName) -> DataFrame.readSqlTable( + conn, + sqlQueryOrTableName, + limit, + inferNullability, + dbType, + strictValidation, + configureStatement, + ) - // Validate the table name structure: letters, numbers, underscores, and dots are allowed - val tableNameRegex = Regex(TABLE_NAME_VALID_PATTERN) - if (!tableNameRegex.matches(normalizedTableName)) { - logger.error { - "Validation failed: The table name contains invalid characters. " + - "Only letters, numbers, underscores, and dots are allowed. Provided name: '$tableName'." + else -> throw IllegalArgumentException( + "$sqlQueryOrTableName should be SQL query or name of one of the existing SQL tables!", + ) } - return false } - - logger.warn { "Table name validation passed for table: '$tableName'." } - return true } /** @@ -644,17 +548,20 @@ private fun isValidTableName(tableName: String): Boolean { * Its state may be altered after the read operation. * @param [dbType] the type of database that the [ResultSet] belongs to. * @param [limit] the maximum number of rows to read from the [ResultSet][java.sql.ResultSet]. + * `null` (default) means no limit - all available rows will be fetched + * or positive integer (e.g., `100`) - fetch at most that many rows * @param [inferNullability] indicates how the column nullability should be inferred. * @return the DataFrame generated from the [ResultSet][java.sql.ResultSet] data. * - * [java.sql.ResultSet]: https://docs.oracle.com/javase/8/docs/api/java/sql/ResultSet.html + * @see [java.sql.ResultSet] */ public fun DataFrame.Companion.readResultSet( resultSet: ResultSet, dbType: DbType, - limit: Int = DEFAULT_LIMIT, + limit: Int? = null, inferNullability: Boolean = true, ): AnyFrame { + validateLimit(limit) val tableColumns = getTableColumnsMetadata(resultSet) return fetchAndConvertDataFromResultSet(tableColumns, resultSet, dbType, limit, inferNullability) } @@ -672,16 +579,17 @@ public fun DataFrame.Companion.readResultSet( * * @param [dbType] the type of database that the [ResultSet] belongs to. * @param [limit] the maximum number of rows to read from the [ResultSet][java.sql.ResultSet]. + * `null` (default) means no limit - all available rows will be fetched + * or positive integer (e.g., `100`) - fetch at most that many rows * @param [inferNullability] indicates how the column nullability should be inferred. * @return the DataFrame generated from the [ResultSet][java.sql.ResultSet] data. * - * [java.sql.ResultSet]: https://docs.oracle.com/javase/8/docs/api/java/sql/ResultSet.html + * @see [java.sql.ResultSet] */ -public fun ResultSet.readDataFrame( - dbType: DbType, - limit: Int = DEFAULT_LIMIT, - inferNullability: Boolean = true, -): AnyFrame = DataFrame.Companion.readResultSet(this, dbType, limit, inferNullability) +public fun ResultSet.readDataFrame(dbType: DbType, limit: Int? = null, inferNullability: Boolean = true): AnyFrame { + validateLimit(limit) + return DataFrame.readResultSet(this, dbType, limit, inferNullability) +} /** * Reads the data from a [ResultSet][java.sql.ResultSet] and converts it into a DataFrame. @@ -692,27 +600,30 @@ public fun ResultSet.readDataFrame( * * For more details, refer to the official Java documentation on [ResultSet][java.sql.ResultSet]. * - * NOTE: Reading from the [ResultSet][java.sql.ResultSet] could potentially change its state. + * __NOTE:__ Reading from the [ResultSet][java.sql.ResultSet] could potentially change its state. * * @param [resultSet] the [ResultSet][java.sql.ResultSet] containing the data to read. * Its state may be altered after the read operation. * @param [connection] the connection to the database (it's required to extract the database type) * that the [ResultSet] belongs to. * @param [limit] the maximum number of rows to read from the [ResultSet][java.sql.ResultSet]. + * `null` (default) means no limit - all available rows will be fetched + * or positive integer (e.g., `100`) - fetch at most that many rows * @param [inferNullability] indicates how the column nullability should be inferred. * @param [dbType] the type of database, could be a custom object, provided by user, optional, default is `null`, * in that case the [dbType] will be recognized from the [resultSet]. * @return the DataFrame generated from the [ResultSet][java.sql.ResultSet] data. * - * [java.sql.ResultSet]: https://docs.oracle.com/javase/8/docs/api/java/sql/ResultSet.html + * @see [java.sql.ResultSet] */ public fun DataFrame.Companion.readResultSet( resultSet: ResultSet, connection: Connection, - limit: Int = DEFAULT_LIMIT, + limit: Int? = null, inferNullability: Boolean = true, dbType: DbType? = null, ): AnyFrame { + validateLimit(limit) val determinedDbType = dbType ?: extractDBTypeFromConnection(connection) return readResultSet(resultSet, determinedDbType, limit, inferNullability) @@ -727,37 +638,34 @@ public fun DataFrame.Companion.readResultSet( * * For more details, refer to the official Java documentation on [ResultSet][java.sql.ResultSet]. * - * NOTE: Reading from the [ResultSet][java.sql.ResultSet] could potentially change its state. + * __NOTE:__ Reading from the [ResultSet][java.sql.ResultSet] could potentially change its state. * * @param [connection] the connection to the database (it's required to extract the database type) * that the [ResultSet] belongs to. * @param [limit] the maximum number of rows to read from the [ResultSet][java.sql.ResultSet]. + * `null` (default) means no limit - all available rows will be fetched + * or positive integer (e.g., `100`) - fetch at most that many rows * @param [inferNullability] indicates how the column nullability should be inferred. * @param [dbType] the type of database, could be a custom object, provided by user, optional, default is `null`, * in that case the [dbType] will be recognized from the [ResultSet]. * @return the DataFrame generated from the [ResultSet][java.sql.ResultSet] data. * - * [java.sql.ResultSet]: https://docs.oracle.com/javase/8/docs/api/java/sql/ResultSet.html + * @see [java.sql.ResultSet] */ public fun ResultSet.readDataFrame( connection: Connection, - limit: Int = DEFAULT_LIMIT, + limit: Int? = null, inferNullability: Boolean = true, dbType: DbType? = null, -): AnyFrame = DataFrame.Companion.readResultSet(this, connection, limit, inferNullability, dbType) +): AnyFrame { + validateLimit(limit) + return DataFrame.readResultSet(this, connection, limit, inferNullability, dbType) +} /** * Reads all non-system tables from a database and returns them * as a map of SQL tables and corresponding dataframes using the provided database configuration and limit. * - * @param [dbConfig] the database configuration to connect to the database, including URL, user, and password. - * @param [limit] the maximum number of rows to read from each table. - * @param [catalogue] a name of the catalog from which tables will be retrieved. A null value retrieves tables from all catalogs. - * @param [inferNullability] indicates how the column nullability should be inferred. - * @param [dbType] the type of database, could be a custom object, provided by user, optional, default is `null`, - * in that case the [dbType] will be recognized from the [dbConfig]. - * @return a map of [String] to [AnyFrame] objects representing the non-system tables from the database. - * * ### Default Behavior: * If [DbConnectionConfig.readOnly] is `true` (which is the default), the connection will be: * - explicitly set as read-only via `Connection.setReadOnly(true)` @@ -766,336 +674,184 @@ public fun ResultSet.readDataFrame( * * Even if [DbConnectionConfig.readOnly] is set to `false`, the library still prevents data-modifying queries * and only permits safe `SELECT` operations internally. + * + * @param [dbConfig] the database configuration to connect to the database, including URL, user, and password. + * @param [limit] the maximum number of rows to read from each table. + * `null` (default) means no limit - all available rows will be fetched + * or positive integer (e.g., `100`) - fetch at most that many rows + * @param [catalogue] a name of the catalog from which tables will be retrieved. A null value retrieves tables from all catalogs. + * @param [inferNullability] indicates how the column nullability should be inferred. + * @param [dbType] the type of database, could be a custom object, provided by user, optional, default is `null`, + * in that case the [dbType] will be recognized from the [dbConfig]. + * @param [configureStatement] optional lambda to configure the [PreparedStatement] before execution. + * This allows for custom tuning of fetch size, query timeout, and other JDBC parameters. + * @return a map of [String] to [AnyFrame] objects representing the non-system tables from the database. */ public fun DataFrame.Companion.readAllSqlTables( dbConfig: DbConnectionConfig, catalogue: String? = null, - limit: Int = DEFAULT_LIMIT, + limit: Int? = null, inferNullability: Boolean = true, dbType: DbType? = null, -): Map = - withReadOnlyConnection(dbConfig, dbType) { connection -> - readAllSqlTables(connection, catalogue, limit, inferNullability, dbType) + configureStatement: (PreparedStatement) -> Unit = {}, +): Map { + validateLimit(limit) + return withReadOnlyConnection(dbConfig, dbType) { connection -> + readAllSqlTables(connection, catalogue, limit, inferNullability, dbType, configureStatement) } +} /** * Reads all non-system tables from a database and returns them * as a map of SQL tables and corresponding dataframes. * - * @param [connection] the database connection to read tables from. - * @param [limit] the maximum number of rows to read from each table. + * ### Example with HikariCP: + * ```kotlin + * import com.zaxxer.hikari.HikariConfig + * import com.zaxxer.hikari.HikariDataSource + * + * val config = HikariConfig().apply { + * jdbcUrl = "jdbc:postgresql://localhost:5432/mydb" + * username = "user" + * password = "password" + * } + * val dataSource = HikariDataSource(config) + * + * // Read all tables from the database + * val allTables = DataFrame.readAllSqlTables(dataSource, limit = 100) + * + * // Access individual tables + * val customersDF = allTables["customers"] + * val ordersDF = allTables["orders"] + * ``` + * + * @param [dataSource] the [DataSource] to get a database connection from. * @param [catalogue] a name of the catalog from which tables will be retrieved. A null value retrieves tables from all catalogs. + * @param [limit] the maximum number of rows to read from each table. + * `null` (default) means no limit - all available rows will be fetched + * or positive integer (e.g., `100`) - fetch at most that many rows * @param [inferNullability] indicates how the column nullability should be inferred. * @param [dbType] the type of database, could be a custom object, provided by user, optional, default is `null`, - * in that case the [dbType] will be recognized from the [connection]. + * in that case the [dbType] will be recognized from the [dataSource]. + * @param [configureStatement] optional lambda to configure the [PreparedStatement] before execution. + * This allows for custom tuning of fetch size, query timeout, and other JDBC parameters. * @return a map of [String] to [AnyFrame] objects representing the non-system tables from the database. * - * @see DriverManager.getConnection + * @see [DataSource.getConnection] */ public fun DataFrame.Companion.readAllSqlTables( - connection: Connection, + dataSource: DataSource, catalogue: String? = null, - limit: Int = DEFAULT_LIMIT, + limit: Int? = null, inferNullability: Boolean = true, dbType: DbType? = null, + configureStatement: (PreparedStatement) -> Unit = {}, ): Map { - val metaData = connection.metaData - val determinedDbType = dbType ?: extractDBTypeFromConnection(connection) - - // exclude system- and other tables without data (it looks like it is supported badly for many databases) - val tableTypes = determinedDbType.tableTypes?.toTypedArray() - val tables = metaData.getTables(catalogue, null, null, tableTypes) - - val dataFrames = mutableMapOf() - - while (tables.next()) { - val table = determinedDbType.buildTableMetadata(tables) - if (!determinedDbType.isSystemTable(table)) { - // we filter here a second time because of specific logic with SQLite and possible issues with future databases - val tableName = when { - catalogue != null && table.schemaName != null -> "$catalogue.${table.schemaName}.${table.name}" - catalogue != null && table.schemaName == null -> "$catalogue.${table.name}" - else -> table.name - } - // TODO: both cases is schema specified or not in URL - // in h2 database name is recognized as a schema name https://www.h2database.com/html/features.html#database_url - // https://stackoverflow.com/questions/20896935/spring-hibernate-h2-database-schema-not-found - // could be Dialect/Database specific - logger.debug { "Reading table: $tableName" } - - val dataFrame = readSqlTable(connection, tableName, limit, inferNullability, dbType) - dataFrames += tableName to dataFrame - logger.debug { "Finished reading table: $tableName" } - } + validateLimit(limit) + dataSource.connection.use { connection -> + return readAllSqlTables(connection, catalogue, limit, inferNullability, dbType, configureStatement) } - - return dataFrames } /** - * Retrieves the schema for an SQL table using the provided database configuration. - * - * @param [dbConfig] the database configuration to connect to the database, including URL, user, and password. - * @param [tableName] the name of the SQL table for which to retrieve the schema. - * @param [dbType] the type of database, could be a custom object, provided by user, optional, default is `null`, - * in that case the [dbType] will be recognized from the [dbConfig]. - * @return the [DataFrameSchema] object representing the schema of the SQL table - * - * ### Default Behavior: - * If [DbConnectionConfig.readOnly] is `true` (which is the default), the connection will be: - * - explicitly set as read-only via `Connection.setReadOnly(true)` - * - used with `autoCommit = false` - * - automatically rolled back after reading, ensuring no changes to the database - * - * Even if [DbConnectionConfig.readOnly] is set to `false`, the library still prevents data-modifying queries - * and only permits safe `SELECT` operations internally. - */ -public fun DataFrame.Companion.getSchemaForSqlTable( - dbConfig: DbConnectionConfig, - tableName: String, - dbType: DbType? = null, -): DataFrameSchema = - withReadOnlyConnection(dbConfig, dbType) { connection -> - getSchemaForSqlTable(connection, tableName, dbType) - } - -/** - * Retrieves the schema for an SQL table using the provided database connection. + * Reads all non-system tables from a database and returns them + * as a map of SQL tables and corresponding dataframes. * - * @param [connection] the database connection. - * @param [tableName] the name of the SQL table for which to retrieve the schema. + * @param [connection] the database connection to read tables from. + * @param [limit] the maximum number of rows to read from each table. + * `null` (default) means no limit - all available rows will be fetched + * or positive integer (e.g., `100`) - fetch at most that many rows + * @param [catalogue] a name of the catalog from which tables will be retrieved. A null value retrieves tables from all catalogs. + * @param [inferNullability] indicates how the column nullability should be inferred. * @param [dbType] the type of database, could be a custom object, provided by user, optional, default is `null`, * in that case the [dbType] will be recognized from the [connection]. - * @return the schema of the SQL table as a [DataFrameSchema] object. + * @param [configureStatement] optional lambda to configure the [PreparedStatement] before execution. + * This allows for custom tuning of fetch size, query timeout, and other JDBC parameters. + * @return a map of [String] to [AnyFrame] objects representing the non-system tables from the database. * - * @see DriverManager.getConnection + * @see [DriverManager.getConnection] */ -public fun DataFrame.Companion.getSchemaForSqlTable( +public fun DataFrame.Companion.readAllSqlTables( connection: Connection, - tableName: String, + catalogue: String? = null, + limit: Int? = null, + inferNullability: Boolean = true, dbType: DbType? = null, -): DataFrameSchema { + configureStatement: (PreparedStatement) -> Unit = {}, +): Map { + validateLimit(limit) val determinedDbType = dbType ?: extractDBTypeFromConnection(connection) + val metaData = connection.metaData + val tablesResultSet = retrieveTableMetadata(metaData, catalogue, determinedDbType) - val sqlQuery = "SELECT * FROM $tableName" - val selectFirstRowQuery = determinedDbType.sqlQueryLimit(sqlQuery, limit = 1) + return buildMap { + while (tablesResultSet.next()) { + val tableMetadata = determinedDbType.buildTableMetadata(tablesResultSet) - connection.prepareStatement(selectFirstRowQuery).use { st -> - st.executeQuery().use { rs -> - val tableColumns = getTableColumnsMetadata(rs) - return buildSchemaByTableColumns(tableColumns, determinedDbType) - } - } -} + // We filter here a second time because of specific logic with SQLite and possible issues with future databases + if (determinedDbType.isSystemTable(tableMetadata)) { + continue + } -/** - * Retrieves the schema of an SQL query result using the provided database configuration. - * - * @param [dbConfig] the database configuration to connect to the database, including URL, user, and password. - * @param [sqlQuery] the SQL query to execute and retrieve the schema from. - * @param [dbType] the type of database, could be a custom object, provided by user, optional, default is `null`, - * in that case the [dbType] will be recognized from the [dbConfig]. - * @return the schema of the SQL query as a [DataFrameSchema] object. - * - * ### Default Behavior: - * If [DbConnectionConfig.readOnly] is `true` (which is the default), the connection will be: - * - explicitly set as read-only via `Connection.setReadOnly(true)` - * - used with `autoCommit = false` - * - automatically rolled back after reading, ensuring no changes to the database - * - * Even if [DbConnectionConfig.readOnly] is set to `false`, the library still prevents data-modifying queries - * and only permits safe `SELECT` operations internally. - */ -public fun DataFrame.Companion.getSchemaForSqlQuery( - dbConfig: DbConnectionConfig, - sqlQuery: String, - dbType: DbType? = null, -): DataFrameSchema = - withReadOnlyConnection(dbConfig, dbType) { connection -> - getSchemaForSqlQuery(connection, sqlQuery, dbType) - } + val fullTableName = buildFullTableName(catalogue, tableMetadata.schemaName, tableMetadata.name) -/** - * Retrieves the schema of an SQL query result using the provided database connection. - * - * @param [connection] the database connection. - * @param [sqlQuery] the SQL query to execute and retrieve the schema from. - * @param [dbType] the type of database, could be a custom object, provided by user, optional, default is `null`, - * in that case the [dbType] will be recognized from the [connection]. - * @return the schema of the SQL query as a [DataFrameSchema] object. - * - * @see DriverManager.getConnection - */ -public fun DataFrame.Companion.getSchemaForSqlQuery( - connection: Connection, - sqlQuery: String, - dbType: DbType? = null, -): DataFrameSchema { - val determinedDbType = dbType ?: extractDBTypeFromConnection(connection) + val dataFrame = readTableAsDataFrame( + connection, + fullTableName, + limit, + inferNullability, + dbType, + configureStatement, + ) - connection.prepareStatement(sqlQuery).use { st -> - st.executeQuery().use { rs -> - val tableColumns = getTableColumnsMetadata(rs) - return buildSchemaByTableColumns(tableColumns, determinedDbType) + put(fullTableName, dataFrame) } } } -/** - * Retrieves the schema of an SQL query result or the SQL table using the provided database configuration. - * - * @param [sqlQueryOrTableName] the SQL query to execute and retrieve the schema from. - * @param [dbType] the type of database, could be a custom object, provided by user, optional, default is `null`, - * in that case the [dbType] will be recognized from the [DbConnectionConfig]. - * @return the schema of the SQL query as a [DataFrameSchema] object. - * - * ### Default Behavior: - * If [DbConnectionConfig.readOnly] is `true` (which is the default), the connection will be: - * - explicitly set as read-only via `Connection.setReadOnly(true)` - * - used with `autoCommit = false` - * - automatically rolled back after reading, ensuring no changes to the database - * - * Even if [DbConnectionConfig.readOnly] is set to `false`, the library still prevents data-modifying queries - * and only permits safe `SELECT` operations internally. - */ -public fun DbConnectionConfig.getDataFrameSchema( - sqlQueryOrTableName: String, - dbType: DbType? = null, -): DataFrameSchema = - when { - isSqlQuery(sqlQueryOrTableName) -> DataFrame.getSchemaForSqlQuery(this, sqlQueryOrTableName, dbType) - - isSqlTableName(sqlQueryOrTableName) -> DataFrame.getSchemaForSqlTable(this, sqlQueryOrTableName, dbType) - - else -> throw IllegalArgumentException( - "$sqlQueryOrTableName should be SQL query or name of one of the existing SQL tables!", - ) - } - -/** - * Retrieves the schema of an SQL query result or the SQL table using the provided database configuration. - * - * @param [sqlQueryOrTableName] the SQL query to execute and retrieve the schema from. - * @param [dbType] the type of database, could be a custom object, provided by user, optional, default is `null`, - * in that case the [dbType] will be recognized from the [Connection]. - * @return the schema of the SQL query as a [DataFrameSchema] object. - */ -public fun Connection.getDataFrameSchema(sqlQueryOrTableName: String, dbType: DbType? = null): DataFrameSchema = - when { - isSqlQuery(sqlQueryOrTableName) -> DataFrame.getSchemaForSqlQuery(this, sqlQueryOrTableName, dbType) - - isSqlTableName(sqlQueryOrTableName) -> DataFrame.getSchemaForSqlTable(this, sqlQueryOrTableName, dbType) - - else -> throw IllegalArgumentException( - "$sqlQueryOrTableName should be SQL query or name of one of the existing SQL tables!", - ) - } - -/** - * Retrieves the schema from [ResultSet]. - * - * NOTE: This function will not close connection and result set and not retrieve data from the result set. - * - * @param [resultSet] the [ResultSet] obtained from executing a database query. - * @param [dbType] the type of database that the [ResultSet] belongs to, could be a custom object, provided by user. - * @return the schema of the [ResultSet] as a [DataFrameSchema] object. - */ -public fun DataFrame.Companion.getSchemaForResultSet(resultSet: ResultSet, dbType: DbType): DataFrameSchema { - val tableColumns = getTableColumnsMetadata(resultSet) - return buildSchemaByTableColumns(tableColumns, dbType) +private fun retrieveTableMetadata(metaData: DatabaseMetaData, catalogue: String?, dbType: DbType): ResultSet { + // Exclude system- and other tables without data (it looks like it is supported badly for many databases) + val tableTypes = dbType.tableTypes?.toTypedArray() + return metaData.getTables(catalogue, null, null, tableTypes) } -/** - * Retrieves the schema from [ResultSet]. - * - * NOTE: This function will not close connection and result set and not retrieve data from the result set. - * - * @param [dbType] the type of database that the [ResultSet] belongs to, could be a custom object, provided by user. - * @return the schema of the [ResultSet] as a [DataFrameSchema] object. - */ -public fun ResultSet.getDataFrameSchema(dbType: DbType): DataFrameSchema = DataFrame.getSchemaForResultSet(this, dbType) - -/** - * Retrieves the schemas of all non-system tables in the database using the provided database configuration. - * - * @param [dbConfig] the database configuration to connect to the database, including URL, user, and password. - * @param [dbType] the type of database, could be a custom object, provided by user, optional, default is `null`, - * in that case the [dbType] will be recognized from the [dbConfig]. - * @return a map of [String, DataFrameSchema] objects representing the table name and its schema for each non-system table. - * - * ### Default Behavior: - * If [DbConnectionConfig.readOnly] is `true` (which is the default), the connection will be: - * - explicitly set as read-only via `Connection.setReadOnly(true)` - * - used with `autoCommit = false` - * - automatically rolled back after reading, ensuring no changes to the database - * - * Even if [DbConnectionConfig.readOnly] is set to `false`, the library still prevents data-modifying queries - * and only permits safe `SELECT` operations internally. - */ -public fun DataFrame.Companion.getSchemaForAllSqlTables( - dbConfig: DbConnectionConfig, - dbType: DbType? = null, -): Map = - withReadOnlyConnection(dbConfig, dbType) { connection -> - getSchemaForAllSqlTables(connection, dbType) +private fun buildFullTableName(catalogue: String?, schemaName: String?, tableName: String): String { + // TODO: both cases is schema specified or not in URL + // in h2 database name is recognized as a schema name https://www.h2database.com/html/features.html#database_url + // https://stackoverflow.com/questions/20896935/spring-hibernate-h2-database-schema-not-found + // could be Dialect/Database specific + return when { + catalogue != null && schemaName != null -> "$catalogue.$schemaName.$tableName" + catalogue != null -> "$catalogue.$tableName" + else -> tableName } +} -/** - * Retrieves the schemas of all non-system tables in the database using the provided database connection. - * - * @param [connection] the database connection. - * @param [dbType] the type of database, could be a custom object, provided by user, optional, default is `null`, - * in that case the [dbType] will be recognized from the [connection]. - * @return a map of [String, DataFrameSchema] objects representing the table name and its schema for each non-system table. - */ -public fun DataFrame.Companion.getSchemaForAllSqlTables( +private fun readTableAsDataFrame( connection: Connection, - dbType: DbType? = null, -): Map { - val metaData = connection.metaData - val determinedDbType = dbType ?: extractDBTypeFromConnection(connection) - - // exclude system- and other tables without data - val tableTypes = determinedDbType.tableTypes?.toTypedArray() - val tables = metaData.getTables(null, null, null, tableTypes) - - val dataFrameSchemas = mutableMapOf() - - while (tables.next()) { - val jdbcTable = determinedDbType.buildTableMetadata(tables) - if (!determinedDbType.isSystemTable(jdbcTable)) { - // we filter her a second time because of specific logic with SQLite and possible issues with future databases - val tableName = jdbcTable.name - val dataFrameSchema = getSchemaForSqlTable(connection, tableName, determinedDbType) - dataFrameSchemas += tableName to dataFrameSchema - } - } - - return dataFrameSchemas -} + tableName: String, + limit: Int?, + inferNullability: Boolean, + dbType: DbType?, + configureStatement: (PreparedStatement) -> Unit = {}, +): AnyFrame { + logger.debug { "Reading table: $tableName" } + + val dataFrame = DataFrame.readSqlTable( + connection, + tableName, + limit, + inferNullability, + dbType, + true, + configureStatement, + ) -/** - * Builds a DataFrame schema based on the given table columns. - * - * @param [tableColumns] a mutable map containing the table columns, where the key represents the column name - * and the value represents the metadata of the column - * @param [dbType] the type of database. - * @return a [DataFrameSchema] object representing the schema built from the table columns. - */ -private fun buildSchemaByTableColumns(tableColumns: MutableList, dbType: DbType): DataFrameSchema { - val schemaColumns = tableColumns.associate { - Pair(it.name, generateColumnSchemaValue(dbType, it)) - } + logger.debug { "Finished reading table: $tableName" } - return DataFrameSchemaImpl( - columns = schemaColumns, - ) + return dataFrame } -private fun generateColumnSchemaValue(dbType: DbType, tableColumnMetadata: TableColumnMetadata): ColumnSchema = - dbType.convertSqlTypeToColumnSchemaValue(tableColumnMetadata) - ?: ColumnSchema.Value(makeCommonSqlToKTypeMapping(tableColumnMetadata)) - /** * Retrieves the metadata of the columns in the result set. * @@ -1104,7 +860,7 @@ private fun generateColumnSchemaValue(dbType: DbType, tableColumnMetadata: Table * where each TableColumnMetadata object contains information such as the column type, * JDBC type, size, and name. */ -private fun getTableColumnsMetadata(rs: ResultSet): MutableList { +internal fun getTableColumnsMetadata(rs: ResultSet): MutableList { val metaData: ResultSetMetaData = rs.metaData val numberOfColumns: Int = metaData.columnCount val tableColumns = mutableListOf() @@ -1144,7 +900,7 @@ private fun getTableColumnsMetadata(rs: ResultSet): MutableList, originalName: String): String { +internal fun manageColumnNameDuplication(columnNameCounter: MutableMap, originalName: String): String { var name = originalName val count = columnNameCounter[originalName] @@ -1162,65 +918,27 @@ private fun manageColumnNameDuplication(columnNameCounter: MutableMap castArray(array: Array<*>, elementType: KClass): List = - array.mapNotNull { elementType.safeCast(it) } - /** * Fetches and converts data from a ResultSet into a mutable map. * * @param [tableColumns] a list containing the column metadata for the table. * @param [rs] the ResultSet object containing the data to be fetched and converted. * @param [dbType] the type of the database. - * @param [limit] the maximum number of rows to fetch and convert. + * @param [limit] the maximum number of rows to retrieve from the table. + * `null` (default) means no limit - all available rows will be fetched. * @param [inferNullability] indicates how the column nullability should be inferred. * @return A mutable map containing the fetched and converted data. */ -private fun fetchAndConvertDataFromResultSet( +internal fun fetchAndConvertDataFromResultSet( tableColumns: MutableList, rs: ResultSet, dbType: DbType, - limit: Int, + limit: Int?, inferNullability: Boolean, ): AnyFrame { - val data = List(tableColumns.size) { mutableListOf() } - - val kotlinTypesForSqlColumns = mutableMapOf() - List(tableColumns.size) { index -> - kotlinTypesForSqlColumns[index] = generateKType(dbType, tableColumns[index]) - } - - var counter = 0 - - if (limit > 0) { - while (counter < limit && rs.next()) { - extractNewRowFromResultSetAndAddToData(tableColumns, data, rs, kotlinTypesForSqlColumns) - counter++ - // if (counter % 1000 == 0) logger.debug { "Loaded $counter rows." } // TODO: https://github.com/Kotlin/dataframe/issues/455 - } - } else { - while (rs.next()) { - extractNewRowFromResultSetAndAddToData(tableColumns, data, rs, kotlinTypesForSqlColumns) - counter++ - // if (counter % 1000 == 0) logger.debug { "Loaded $counter rows." } // TODO: https://github.com/Kotlin/dataframe/issues/455 - } - } - - val dataFrame = data.mapIndexed { index, values -> - // TODO: add override handlers from dbType to intercept the final parcing before column creation - val correctedValues = if (kotlinTypesForSqlColumns[index]!!.classifier == Array::class) { - handleArrayValues(values) - } else { - values - } - - DataColumn.createValueColumn( - name = tableColumns[index].name, - values = correctedValues, - infer = convertNullabilityInference(inferNullability), - type = kotlinTypesForSqlColumns[index]!!, - ) - }.toDataFrame() + val columnKTypes = buildColumnKTypes(tableColumns, dbType) + val columnData = readAllRowsFromResultSet(rs, tableColumns, columnKTypes, dbType, limit) + val dataFrame = buildDataFrameFromColumnData(columnData, tableColumns, columnKTypes, dbType, inferNullability) logger.debug { "DataFrame with ${dataFrame.rowsCount()} rows and ${dataFrame.columnsCount()} columns created as a result of SQL query." @@ -1229,52 +947,65 @@ private fun fetchAndConvertDataFromResultSet( return dataFrame } -private fun handleArrayValues(values: MutableList): List { - // Intermediate variable for the first mapping - val sqlArrays = values.mapNotNull { - (it as? java.sql.Array)?.array?.let { array -> array as? Array<*> } +/** + * Builds a map of column indices to their Kotlin types. + */ +private fun buildColumnKTypes(tableColumns: List, dbType: DbType): Map = + tableColumns.indices.associateWith { index -> + generateKType(dbType, tableColumns[index]) } - // Flatten the arrays to iterate through all elements and filter out null values, then map to component types - val allElementTypes = sqlArrays - .flatMap { array -> - (array.javaClass.componentType?.kotlin?.let { listOf(it) } ?: emptyList()) - } // Get the component type of each array and convert it to a Kotlin class, if available - - // Find distinct types and ensure there's only one distinct type - val commonElementType = allElementTypes - .distinct() // Get unique element types - .singleOrNull() // Ensure there's only one unique element type, otherwise return null - ?: Any::class // Fallback to Any::class if multiple distinct types or no elements found - - return if (commonElementType != Any::class) { - sqlArrays.map { castArray(it, commonElementType).toTypedArray() } - } else { - sqlArrays +/** + * Reads all rows from ResultSet and returns a column-oriented data structure. + * Returns mutable lists to allow efficient post-processing without copying. + */ +private fun readAllRowsFromResultSet( + rs: ResultSet, + tableColumns: List, + columnKTypes: Map, + dbType: DbType, + limit: Int?, +): List> { + val columnsCount = tableColumns.size + val columnData = List(columnsCount) { mutableListOf() } + var rowsRead = 0 + + while (rs.next() && (limit == null || rowsRead < limit)) { + repeat(columnsCount) { columnIndex -> + val value = dbType.extractValueFromResultSet( + rs = rs, + columnIndex = columnIndex, + columnMetadata = tableColumns[columnIndex], + kType = columnKTypes.getValue(columnIndex), + ) + columnData[columnIndex].add(value) + } + rowsRead++ + // if (rowsRead % 1000 == 0) logger.debug { "Loaded $rowsRead rows." } // TODO: https://github.com/Kotlin/dataframe/issues/455 } -} -private fun convertNullabilityInference(inferNullability: Boolean) = if (inferNullability) Infer.Nulls else Infer.None + return columnData +} -private fun extractNewRowFromResultSetAndAddToData( - tableColumns: MutableList, - data: List>, - rs: ResultSet, - kotlinTypesForSqlColumns: MutableMap, -) { - repeat(tableColumns.size) { i -> - data[i].add( - try { - rs.getObject(i + 1) - // TODO: add a special handler for Blob via Streams - } catch (_: Throwable) { - val kType = kotlinTypesForSqlColumns[i]!! - // TODO: expand for all the types like in generateKType function - if (kType.isSupertypeOf(String::class.starProjectedType)) rs.getString(i + 1) else rs.getString(i + 1) - }, +/** + * Builds DataFrame from column-oriented data. + * Accepts mutable lists to enable efficient in-place transformations. + */ +private fun buildDataFrameFromColumnData( + columnData: List>, + tableColumns: List, + columnKTypes: Map, + dbType: DbType, + inferNullability: Boolean, +): AnyFrame = + columnData.mapIndexed { index, values -> + dbType.buildDataColumn( + name = tableColumns[index].name, + values = values, + kType = columnKTypes.getValue(index), + inferNullability = inferNullability, ) - } -} + }.toDataFrame() /** * Generates a KType based on the given database type and table column metadata. @@ -1284,99 +1015,6 @@ private fun extractNewRowFromResultSetAndAddToData( * * @return The generated KType. */ -private fun generateKType(dbType: DbType, tableColumnMetadata: TableColumnMetadata): KType = +internal fun generateKType(dbType: DbType, tableColumnMetadata: TableColumnMetadata): KType = dbType.convertSqlTypeToKType(tableColumnMetadata) - ?: makeCommonSqlToKTypeMapping(tableColumnMetadata) - -/** - * Creates a mapping between common SQL types and their corresponding KTypes. - * - * @param tableColumnMetadata The metadata of the table column. - * @return The KType associated with the SQL type or a default type if no mapping is found. - */ -private fun makeCommonSqlToKTypeMapping(tableColumnMetadata: TableColumnMetadata): KType { - val jdbcTypeToKTypeMapping = mapOf( - Types.BIT to Boolean::class, - Types.TINYINT to Int::class, - Types.SMALLINT to Int::class, - Types.INTEGER to Int::class, - Types.BIGINT to Long::class, - Types.FLOAT to Float::class, - Types.REAL to Float::class, - Types.DOUBLE to Double::class, - Types.NUMERIC to BigDecimal::class, - Types.DECIMAL to BigDecimal::class, - Types.CHAR to String::class, - Types.VARCHAR to String::class, - Types.LONGVARCHAR to String::class, - Types.DATE to Date::class, - Types.TIME to Time::class, - Types.TIMESTAMP to Timestamp::class, - Types.BINARY to ByteArray::class, - Types.VARBINARY to ByteArray::class, - Types.LONGVARBINARY to ByteArray::class, - Types.NULL to String::class, - Types.JAVA_OBJECT to Any::class, - Types.DISTINCT to Any::class, - Types.STRUCT to Any::class, - Types.ARRAY to Array::class, - Types.BLOB to ByteArray::class, - Types.CLOB to Clob::class, - Types.REF to Ref::class, - Types.DATALINK to Any::class, - Types.BOOLEAN to Boolean::class, - Types.ROWID to RowId::class, - Types.NCHAR to String::class, - Types.NVARCHAR to String::class, - Types.LONGNVARCHAR to String::class, - Types.NCLOB to NClob::class, - Types.SQLXML to SQLXML::class, - Types.REF_CURSOR to Ref::class, - Types.TIME_WITH_TIMEZONE to OffsetTime::class, - Types.TIMESTAMP_WITH_TIMEZONE to OffsetDateTime::class, - ) - - fun determineKotlinClass(tableColumnMetadata: TableColumnMetadata): KClass<*> = - when { - tableColumnMetadata.jdbcType == Types.OTHER -> when (tableColumnMetadata.javaClassName) { - "[B" -> ByteArray::class - else -> Any::class - } - - tableColumnMetadata.javaClassName == "[B" -> ByteArray::class - - tableColumnMetadata.javaClassName == "java.sql.Blob" -> Blob::class - - tableColumnMetadata.jdbcType == Types.TIMESTAMP && - tableColumnMetadata.javaClassName == "java.time.LocalDateTime" -> LocalDateTime::class - - tableColumnMetadata.jdbcType == Types.BINARY && - tableColumnMetadata.javaClassName == "java.util.UUID" -> UUID::class - - tableColumnMetadata.jdbcType == Types.REAL && - tableColumnMetadata.javaClassName == "java.lang.Double" -> Double::class - - tableColumnMetadata.jdbcType == Types.FLOAT && - tableColumnMetadata.javaClassName == "java.lang.Double" -> Double::class - - tableColumnMetadata.jdbcType == Types.NUMERIC && - tableColumnMetadata.javaClassName == "java.lang.Double" -> Double::class - - else -> jdbcTypeToKTypeMapping[tableColumnMetadata.jdbcType] ?: String::class - } - - fun createArrayTypeIfNeeded(kClass: KClass<*>, isNullable: Boolean): KType = - if (kClass == Array::class) { - val typeParam = kClass.typeParameters[0].createType() - kClass.createType( - arguments = listOf(kotlin.reflect.KTypeProjection.invariant(typeParam)), - nullable = isNullable, - ) - } else { - kClass.createType(nullable = isNullable) - } - - val kClass: KClass<*> = determineKotlinClass(tableColumnMetadata) - val kType = createArrayTypeIfNeeded(kClass, tableColumnMetadata.isNullable) - return kType -} + ?: dbType.makeCommonSqlToKTypeMapping(tableColumnMetadata) diff --git a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/validationUtil.kt b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/validationUtil.kt new file mode 100644 index 0000000000..03a64d53dc --- /dev/null +++ b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/validationUtil.kt @@ -0,0 +1,250 @@ +package org.jetbrains.kotlinx.dataframe.io + +import io.github.oshai.kotlinlogging.KotlinLogging +import org.jetbrains.kotlinx.dataframe.io.db.DbType +import org.jetbrains.kotlinx.dataframe.io.db.extractDBTypeFromUrl +import java.sql.Connection +import java.sql.SQLException + +private val logger = KotlinLogging.logger {} + +/** + * Validates the provided limit to ensure it is either null or a positive integer. + * Throws an [IllegalArgumentException] if the limit is negative or zero. + * + * @param limit The maximum allowed number of rows. Use `null` for unlimited rows. + */ +internal fun validateLimit(limit: Int?) { + require(limit == null || limit > 0) { + "Parameter 'limit' must be positive, but was: $limit. Use null for unlimited rows." + } +} + +/** + * Executes the given block with a managed JDBC connection created from [DbConnectionConfig]. + * + * If [DbConnectionConfig.readOnly] is `true` (default), the connection will be: + * - explicitly marked as read-only + * - used with auto-commit disabled + * - rolled back after execution to prevent unintended modifications + * + * This utility guarantees proper closing of the connection and safe rollback in read-only mode. + * It should be used when the user does not manually manage JDBC connections. + * + * @param [dbConfig] The configuration used to create the connection. + * @param [dbType] Optional database type (not used here but can be passed through for logging or future extensions). + * @param [block] A lambda with receiver that runs with an open and managed [java.sql.Connection]. + * @return The result of the [block] execution. + */ +internal inline fun withReadOnlyConnection( + dbConfig: DbConnectionConfig, + dbType: DbType? = null, + block: (Connection) -> T, +): T { + val actualDbType = dbType ?: extractDBTypeFromUrl(dbConfig.url) + val connection = actualDbType.createConnection(dbConfig) + + return connection.use { conn -> + try { + if (dbConfig.readOnly) { + conn.autoCommit = false + } + + block(conn) + } finally { + if (dbConfig.readOnly) { + try { + conn.rollback() + } catch (e: SQLException) { + logger.warn(e) { + "Failed to rollback read-only transaction (url=${dbConfig.url})" + } + } + } + } + } +} + +/** + * A regular expression defining the valid pattern for SQL table names. + * + * This pattern enforces that table names must: + * - Contain only Unicode letters, Unicode digits, or underscores. + * - Optionally be segmented by dots to indicate schema and table separation. + * + * It ensures compatibility with most SQL database naming conventions, thus minimizing risks of invalid names + * or injection vulnerabilities. + * + * Example of valid table names: + * - `my_table` + * - `schema1.table2` + * + * Example of invalid table names: + * - `my-table` (contains a dash) + * - `table!name` (contains special characters) + * - `.startWithDot` (cannot start with a dot) + */ +internal const val TABLE_NAME_VALID_PATTERN = "^[\\p{L}\\p{N}_]+(\\.[\\p{L}\\p{N}_]+)*$" + +internal fun isSqlQuery(sqlQueryOrTableName: String): Boolean { + val queryPattern = Regex("(?i)\\b(SELECT)\\b") + return queryPattern.containsMatchIn(sqlQueryOrTableName.trim()) +} + +/** + * SQL table name pattern matching: __catalog.schema.table__ + * Allows alphanumeric characters and underscores, must start with letter or underscore + */ +private val SQL_TABLE_NAME_PATTERN = Regex("^[a-zA-Z_][a-zA-Z0-9_]*(\\.[a-zA-Z_][a-zA-Z0-9_]*){0,2}$") + +internal fun isSqlTableName(sqlQueryOrTableName: String): Boolean { + // Match table names with optional schema and catalog (e.g., catalog.schema.table) + return SQL_TABLE_NAME_PATTERN.matches(sqlQueryOrTableName.trim()) +} + +internal val FORBIDDEN_PATTERNS_REGEX = listOf( + ";", // Separator for SQL statements + "--", // Single-line comments + "/\\*", // Start of multi-line comments + "\\*/", // End of multi-line comments + "\\bDROP\\b", // DROP as a full word + "\\bDELETE\\b", // DELETE as a full word + "\\bINSERT\\b", // INSERT as a full word + "\\bUPDATE\\b", // UPDATE as a full word + "\\bEXEC\\b", // EXEC as a full word + "\\bEXECUTE\\b", // EXECUTE as a full word + "\\bCREATE\\b", // CREATE as a full word + "\\bALTER\\b", // ALTER as a full word + "\\bGRANT\\b", // GRANT as a full word + "\\bREVOKE\\b", // REVOKE as a full word + "\\bMERGE\\b", // MERGE as a full word +).map { Regex(it, RegexOption.IGNORE_CASE) } + +/** + * Checks if a given string contains forbidden patterns or keywords. + * Logs a clear and friendly message if any forbidden pattern is found. + * + * ### Forbidden SQL Examples: + * 1. **Single-line comment** (using `--`): + * - `SELECT * FROM Sale WHERE amount = 100.0 -- AND id = 5` + * + * 2. **Multi-line comment** (using `/* */`): + * - `SELECT * FROM Customer /* Possible malicious comment */ WHERE id = 1` + * + * 3. **Multiple statements separated by semicolon (`;`)**: + * - `SELECT * FROM Sale WHERE amount = 500.0; DROP TABLE Customer` + * + * 4. **Potentially malicious SQL with single quotes for injection**: + * - `SELECT * FROM Sale WHERE id = 1 AND amount = 100.0 OR '1'='1` + * + * 5. **Usage of dangerous commands like `DROP`, `DELETE`, `ALTER`, etc.**: + * - `DROP TABLE Customer; SELECT * FROM Sale` + * + * ### Allowed SQL Examples: + * 1. Query with names containing reserved words as parts of identifiers: + * - `SELECT last_update FROM HELLO_ALTER` + * + * 2. Query with fully valid syntax: + * - `SELECT id, name FROM Customers WHERE age > 25` + * + * 3. Query with identifiers resembling commands but not in forbidden contexts: + * - `SELECT id, amount FROM TRANSACTION_DROP` + * + * 4. Query with case-insensitive identifiers: + * - `select Id, Name from Hello_Table` + * + * ### Key Notes: + * - Reserved keywords like `DROP`, `DELETE`, `ALTER`, etc., are forbidden **only when they appear as standalone commands**. + * - Reserved words as parts of table or column names (e.g., `HELLO_ALTER`, `myDropTable`) **are allowed**. + * - Inline or multi-line comments (`--` or `/* */`) are restricted to prevent potential SQL injection attacks. + * - Multiple SQL statements separated by semicolons (`;`) are not allowed to prevent the execution of unintended commands. + */ +internal fun hasForbiddenPatterns(input: String): Boolean { + for (regex in FORBIDDEN_PATTERNS_REGEX) { + if (regex.containsMatchIn(input)) { + logger.error { + "Validation failed: The input contains a forbidden element matching '${regex.pattern}'. Please review the input: '$input'." + } + return true + } + } + return false +} + +/** + * Allowed list of SQL operators + */ +internal val ALLOWED_SQL_OPERATORS = listOf("SELECT", "WITH", "VALUES", "TABLE") + +/** + * Validates if the SQL query is safe and starts with SELECT. + * Ensures a proper syntax structure, checks for balanced quotes, and disallows dangerous commands or patterns. + */ +internal fun isValidSqlQuery(sqlQuery: String): Boolean { + val normalizedSqlQuery = sqlQuery.trim().uppercase() + + // Log the query being validated + logger.debug { "Validating SQL query: '$sqlQuery'" } + + // Ensure the query starts from one of the allowed SQL operators + if (ALLOWED_SQL_OPERATORS.none { normalizedSqlQuery.startsWith(it) }) { + logger.error { + "Validation failed: The SQL query must start with one of: $ALLOWED_SQL_OPERATORS. Given query: '$sqlQuery'." + } + return false + } + + // Validate against forbidden patterns + if (hasForbiddenPatterns(normalizedSqlQuery)) { + return false + } + + // Check if there are balanced quotes (single and double) + val singleQuotes = sqlQuery.count { it == '\'' } + val doubleQuotes = sqlQuery.count { it == '"' } + if (singleQuotes % 2 != 0) { + logger.error { + "Validation failed: Unbalanced single quotes in the SQL query. " + + "Please correct the query: '$sqlQuery'." + } + return false + } + if (doubleQuotes % 2 != 0) { + logger.error { + "Validation failed: Unbalanced double quotes in the SQL query. " + + "Please correct the query: '$sqlQuery'." + } + return false + } + + logger.debug { "SQL query validation succeeded for query: '$sqlQuery'." } + return true +} + +/** + * Validates if the given SQL table name is safe and logs any validation violations. + */ +internal fun isValidTableName(tableName: String): Boolean { + val normalizedTableName = tableName.trim().uppercase() + + // Log the table name being validated + logger.debug { "Validating SQL table name: '$tableName'" } + + // Validate against forbidden patterns + if (hasForbiddenPatterns(normalizedTableName)) { + return false + } + + // Validate the table name structure: letters, numbers, underscores, and dots are allowed + val tableNameRegex = Regex(TABLE_NAME_VALID_PATTERN) + if (!tableNameRegex.matches(normalizedTableName)) { + logger.error { + "Validation failed: The table name contains invalid characters. " + + "Only letters, numbers, underscores, and dots are allowed. Provided name: '$tableName'." + } + return false + } + + logger.debug { "Table name validation passed for table: '$tableName'." } + return true +} diff --git a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/commonTestScenarios.kt b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/commonTestScenarios.kt index 35435cf664..0f5fd81080 100644 --- a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/commonTestScenarios.kt +++ b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/commonTestScenarios.kt @@ -8,6 +8,7 @@ import org.jetbrains.kotlinx.dataframe.DataFrame import org.jetbrains.kotlinx.dataframe.api.inferType import org.jetbrains.kotlinx.dataframe.api.schema import org.jetbrains.kotlinx.dataframe.io.db.MsSql +import org.jetbrains.kotlinx.dataframe.schema.DataFrameSchema import java.sql.Connection import java.sql.ResultSet import kotlin.reflect.typeOf @@ -45,7 +46,7 @@ internal fun inferNullability(connection: Connection) { df.schema().columns["surname"]!!.type shouldBe typeOf() df.schema().columns["age"]!!.type shouldBe typeOf() - val dataSchema = DataFrame.getSchemaForSqlTable(connection, tableName) + val dataSchema = DataFrameSchema.readSqlTable(connection, tableName) dataSchema.columns.size shouldBe 4 dataSchema.columns["id"]!!.type shouldBe typeOf() dataSchema.columns["name"]!!.type shouldBe typeOf() @@ -77,7 +78,7 @@ internal fun inferNullability(connection: Connection) { df2.schema().columns["surname"]!!.type shouldBe typeOf() df2.schema().columns["age"]!!.type shouldBe typeOf() - val dataSchema2 = DataFrame.getSchemaForSqlQuery(connection, sqlQuery) + val dataSchema2 = DataFrameSchema.readSqlQuery(connection, sqlQuery) dataSchema2.columns.size shouldBe 3 dataSchema2.columns["name"]!!.type shouldBe typeOf() dataSchema2.columns["surname"]!!.type shouldBe typeOf() @@ -108,7 +109,7 @@ internal fun inferNullability(connection: Connection) { rs.beforeFirst() - val dataSchema3 = DataFrame.getSchemaForResultSet(rs, MsSql) + val dataSchema3 = DataFrameSchema.readResultSet(rs, MsSql) dataSchema3.columns.size shouldBe 4 dataSchema3.columns["id"]!!.type shouldBe typeOf() dataSchema3.columns["name"]!!.type shouldBe typeOf() diff --git a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/h2/h2Test.kt b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/h2/h2Test.kt index 148b8061cd..a95ec1e3db 100644 --- a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/h2/h2Test.kt +++ b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/h2/h2Test.kt @@ -3,7 +3,6 @@ package org.jetbrains.kotlinx.dataframe.io.h2 import io.kotest.assertions.throwables.shouldThrow import io.kotest.assertions.throwables.shouldThrowExactly import io.kotest.matchers.shouldBe -import org.h2.jdbc.JdbcSQLSyntaxErrorException import org.intellij.lang.annotations.Language import org.jetbrains.kotlinx.dataframe.DataFrame import org.jetbrains.kotlinx.dataframe.annotations.DataSchema @@ -14,18 +13,15 @@ import org.jetbrains.kotlinx.dataframe.api.select import org.jetbrains.kotlinx.dataframe.io.DbConnectionConfig import org.jetbrains.kotlinx.dataframe.io.db.H2 import org.jetbrains.kotlinx.dataframe.io.db.MySql -import org.jetbrains.kotlinx.dataframe.io.getDataFrameSchema -import org.jetbrains.kotlinx.dataframe.io.getSchemaForAllSqlTables -import org.jetbrains.kotlinx.dataframe.io.getSchemaForResultSet -import org.jetbrains.kotlinx.dataframe.io.getSchemaForSqlQuery -import org.jetbrains.kotlinx.dataframe.io.getSchemaForSqlTable import org.jetbrains.kotlinx.dataframe.io.inferNullability import org.jetbrains.kotlinx.dataframe.io.readAllSqlTables import org.jetbrains.kotlinx.dataframe.io.readDataFrame +import org.jetbrains.kotlinx.dataframe.io.readDataFrameSchema import org.jetbrains.kotlinx.dataframe.io.readResultSet import org.jetbrains.kotlinx.dataframe.io.readSqlQuery import org.jetbrains.kotlinx.dataframe.io.readSqlTable import org.jetbrains.kotlinx.dataframe.io.withReadOnlyConnection +import org.jetbrains.kotlinx.dataframe.schema.DataFrameSchema import org.junit.AfterClass import org.junit.BeforeClass import org.junit.Test @@ -164,7 +160,7 @@ class JdbcTest { val df = DataFrame.readSqlTable(connection, tableName) df.rowsCount() shouldBe 0 - val dataSchema = DataFrame.getSchemaForSqlTable(connection, tableName) + val dataSchema = DataFrameSchema.readSqlTable(connection, tableName) dataSchema.columns.size shouldBe 2 dataSchema.columns["characterCol"]!!.type shouldBe typeOf() @@ -292,7 +288,7 @@ class JdbcTest { BigDecimal("2.71").compareTo(result6[0][1] as BigDecimal) shouldBe 0 - val schema = DataFrame.getSchemaForSqlTable(connection, tableName) + val schema = DataFrameSchema.readSqlTable(connection, tableName) schema.columns["characterCol"]!!.type shouldBe typeOf() schema.columns["tinyIntCol"]!!.type shouldBe typeOf() @@ -321,7 +317,7 @@ class JdbcTest { df1.filter { it[Customer::age] != null && it[Customer::age]!! > 30 }.rowsCount() shouldBe 1 df1[0][1] shouldBe "John" - val dataSchema = DataFrame.getSchemaForSqlTable(connection, tableName) + val dataSchema = DataFrameSchema.readSqlTable(connection, tableName) dataSchema.columns.size shouldBe 3 dataSchema.columns["name"]!!.type shouldBe typeOf() @@ -338,7 +334,7 @@ class JdbcTest { df3.filter { it[Customer::age] != null && it[Customer::age]!! > 30 }.rowsCount() shouldBe 1 df3[0][1] shouldBe "John" - val dataSchema1 = DataFrame.getSchemaForSqlTable(dbConfig, tableName) + val dataSchema1 = DataFrameSchema.readSqlTable(dbConfig, tableName) dataSchema1.columns.size shouldBe 3 dataSchema1.columns["name"]!!.type shouldBe typeOf() } @@ -358,7 +354,7 @@ class JdbcTest { df1.filter { it[Customer::age] != null && it[Customer::age]!! > 30 }.rowsCount() shouldBe 1 df1[0][1] shouldBe "John" - val dataSchema = connection.getDataFrameSchema(tableName) + val dataSchema = connection.readDataFrameSchema(tableName) dataSchema.columns.size shouldBe 3 dataSchema.columns["name"]!!.type shouldBe typeOf() @@ -375,7 +371,7 @@ class JdbcTest { df3.filter { it[Customer::age] != null && it[Customer::age]!! > 30 }.rowsCount() shouldBe 1 df3[0][1] shouldBe "John" - val dataSchema1 = dbConfig.getDataFrameSchema(tableName) + val dataSchema1 = dbConfig.readDataFrameSchema(tableName) dataSchema1.columns.size shouldBe 3 dataSchema1.columns["name"]!!.type shouldBe typeOf() } @@ -424,7 +420,7 @@ class JdbcTest { rs.beforeFirst() - val dataSchema = DataFrame.getSchemaForResultSet(rs, H2(MySql)) + val dataSchema = DataFrameSchema.readResultSet(rs, H2(MySql)) dataSchema.columns.size shouldBe 3 dataSchema.columns["name"]!!.type shouldBe typeOf() @@ -446,7 +442,7 @@ class JdbcTest { rs.beforeFirst() - val dataSchema1 = DataFrame.getSchemaForResultSet(rs, H2(MySql)) + val dataSchema1 = DataFrameSchema.readResultSet(rs, H2(MySql)) dataSchema1.columns.size shouldBe 3 dataSchema1.columns["name"]!!.type shouldBe typeOf() } @@ -476,7 +472,7 @@ class JdbcTest { rs.beforeFirst() - val dataSchema = rs.getDataFrameSchema(H2(MySql)) + val dataSchema = rs.readDataFrameSchema(H2(MySql)) dataSchema.columns.size shouldBe 3 dataSchema.columns["name"]!!.type shouldBe typeOf() @@ -498,7 +494,7 @@ class JdbcTest { rs.beforeFirst() - val dataSchema1 = rs.getDataFrameSchema(H2(MySql)) + val dataSchema1 = rs.readDataFrameSchema(H2(MySql)) dataSchema1.columns.size shouldBe 3 dataSchema1.columns["name"]!!.type shouldBe typeOf() } @@ -536,7 +532,7 @@ class JdbcTest { @Test fun `read from non-existing table`() { - shouldThrow { + shouldThrow { DataFrame.readSqlTable(connection, "WrongTableName").cast() } } @@ -845,7 +841,7 @@ class JdbcTest { df1.filter { it[CustomerSales::totalSalesAmount]!! > 100 }.rowsCount() shouldBe 1 df1[0][0] shouldBe "John" - val dataSchema = DataFrame.getSchemaForSqlQuery(connection, sqlQuery) + val dataSchema = DataFrameSchema.readSqlQuery(connection, sqlQuery) dataSchema.columns.size shouldBe 2 dataSchema.columns["name"]!!.type shouldBe typeOf() @@ -862,7 +858,7 @@ class JdbcTest { df3.filter { it[CustomerSales::totalSalesAmount]!! > 100 }.rowsCount() shouldBe 1 df3[0][0] shouldBe "John" - val dataSchema1 = DataFrame.getSchemaForSqlQuery(dbConfig, sqlQuery) + val dataSchema1 = DataFrameSchema.readSqlQuery(dbConfig, sqlQuery) dataSchema1.columns.size shouldBe 2 dataSchema1.columns["name"]!!.type shouldBe typeOf() } @@ -891,7 +887,7 @@ class JdbcTest { df1.filter { it[CustomerSales::totalSalesAmount]!! > 100 }.rowsCount() shouldBe 1 df1[0][0] shouldBe "John" - val dataSchema = connection.getDataFrameSchema(sqlQuery) + val dataSchema = connection.readDataFrameSchema(sqlQuery) dataSchema.columns.size shouldBe 2 dataSchema.columns["name"]!!.type shouldBe typeOf() @@ -908,7 +904,7 @@ class JdbcTest { df3.filter { it[CustomerSales::totalSalesAmount]!! > 100 }.rowsCount() shouldBe 1 df3[0][0] shouldBe "John" - val dataSchema1 = dbConfig.getDataFrameSchema(sqlQuery) + val dataSchema1 = dbConfig.readDataFrameSchema(sqlQuery) dataSchema1.columns.size shouldBe 2 dataSchema1.columns["name"]!!.type shouldBe typeOf() } @@ -923,7 +919,7 @@ class JdbcTest { INNER JOIN Customer c2 ON c1.id = c2.id """.trimIndent() - val schema = DataFrame.getSchemaForSqlQuery(connection, sqlQuery) + val schema = DataFrameSchema.readSqlQuery(connection, sqlQuery) schema.columns.size shouldBe 2 schema.columns.toList()[0].first shouldBe "name" schema.columns.toList()[1].first shouldBe "name_1" @@ -939,7 +935,7 @@ class JdbcTest { INNER JOIN Customer c2 ON c1.id = c2.id """.trimIndent() - val schema = DataFrame.getSchemaForSqlQuery(connection, sqlQuery) + val schema = DataFrameSchema.readSqlQuery(connection, sqlQuery) schema.columns.size shouldBe 3 schema.columns.toList()[0].first shouldBe "name" schema.columns.toList()[1].first shouldBe "name_1" @@ -980,7 +976,7 @@ class JdbcTest { saleDf1.filter { it[Sale::amount] > 40 }.rowsCount() shouldBe 1 (saleDf[0][2] as BigDecimal).compareTo(BigDecimal(100.50)) shouldBe 0 - val dataFrameSchemaMap = DataFrame.getSchemaForAllSqlTables(connection) + val dataFrameSchemaMap = DataFrameSchema.readAllSqlTables(connection) dataFrameSchemaMap.containsKey("Customer") shouldBe true dataFrameSchemaMap.containsKey("Sale") shouldBe true @@ -1024,7 +1020,7 @@ class JdbcTest { saleDf3.filter { it[Sale::amount] > 40 }.rowsCount() shouldBe 1 (saleDf[0][2] as BigDecimal).compareTo(BigDecimal(100.50)) shouldBe 0 - val dataSchemas1 = DataFrame.getSchemaForAllSqlTables(dbConfig).values.toList() + val dataSchemas1 = DataFrameSchema.readAllSqlTables(dbConfig).values.toList() val customerDataSchema1 = dataSchemas1[0] customerDataSchema1.columns.size shouldBe 3 @@ -1060,7 +1056,7 @@ class JdbcTest { df.filter { it[Customer::age] != null && it[Customer::age]!! > 30 }.rowsCount() shouldBe 2 df[0][1] shouldBe "John" - val dataSchema = DataFrame.getSchemaForSqlTable(connection, tableName, dbType = CustomDB) + val dataSchema = DataFrameSchema.readSqlTable(connection, tableName, dbType = CustomDB) dataSchema.columns.size shouldBe 3 dataSchema.columns["name"]!!.type shouldBe typeOf() @@ -1071,7 +1067,7 @@ class JdbcTest { df2.filter { it[Customer::age] != null && it[Customer::age]!! > 30 }.rowsCount() shouldBe 2 df2[0][1] shouldBe "John" - val dataSchema1 = DataFrame.getSchemaForSqlTable(dbConfig, tableName, dbType = CustomDB) + val dataSchema1 = DataFrameSchema.readSqlTable(dbConfig, tableName, dbType = CustomDB) dataSchema1.columns.size shouldBe 3 dataSchema1.columns["name"]!!.type shouldBe typeOf() } @@ -1094,7 +1090,7 @@ class JdbcTest { df.filter { it[CustomerSales::totalSalesAmount]!! > 100 }.rowsCount() shouldBe 1 df[0][0] shouldBe "John" - val dataSchema = DataFrame.getSchemaForSqlQuery(connection, sqlQuery, dbType = CustomDB) + val dataSchema = DataFrameSchema.readSqlQuery(connection, sqlQuery, dbType = CustomDB) dataSchema.columns.size shouldBe 2 dataSchema.columns["name"]!!.type shouldBe typeOf() @@ -1105,7 +1101,7 @@ class JdbcTest { df2.filter { it[CustomerSales::totalSalesAmount]!! > 100 }.rowsCount() shouldBe 1 df2[0][0] shouldBe "John" - val dataSchema1 = DataFrame.getSchemaForSqlQuery(dbConfig, sqlQuery, dbType = CustomDB) + val dataSchema1 = DataFrameSchema.readSqlQuery(dbConfig, sqlQuery, dbType = CustomDB) dataSchema1.columns.size shouldBe 2 dataSchema1.columns["name"]!!.type shouldBe typeOf() } @@ -1130,7 +1126,7 @@ class JdbcTest { saleDf.filter { it[Sale::amount] > 40 }.rowsCount() shouldBe 3 (saleDf[0][2] as BigDecimal).compareTo(BigDecimal(100.50)) shouldBe 0 - val dataFrameSchemaMap = DataFrame.getSchemaForAllSqlTables(connection, dbType = CustomDB) + val dataFrameSchemaMap = DataFrameSchema.readAllSqlTables(connection, dbType = CustomDB) dataFrameSchemaMap.containsKey("Customer") shouldBe true dataFrameSchemaMap.containsKey("Sale") shouldBe true @@ -1160,7 +1156,7 @@ class JdbcTest { saleDf2.filter { it[Sale::amount] > 40 }.rowsCount() shouldBe 3 (saleDf[0][2] as BigDecimal).compareTo(BigDecimal(100.50)) shouldBe 0 - val dataSchemas1 = DataFrame.getSchemaForAllSqlTables(dbConfig, dbType = CustomDB).values.toList() + val dataSchemas1 = DataFrameSchema.readAllSqlTables(dbConfig, dbType = CustomDB).values.toList() val customerDataSchema1 = dataSchemas1[0] customerDataSchema1.columns.size shouldBe 3 diff --git a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/h2/mariadbH2Test.kt b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/h2/mariadbH2Test.kt index 637e4307fc..0707000ef6 100644 --- a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/h2/mariadbH2Test.kt +++ b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/h2/mariadbH2Test.kt @@ -8,12 +8,11 @@ import org.jetbrains.kotlinx.dataframe.api.add import org.jetbrains.kotlinx.dataframe.api.cast import org.jetbrains.kotlinx.dataframe.api.filter import org.jetbrains.kotlinx.dataframe.api.select -import org.jetbrains.kotlinx.dataframe.io.getSchemaForSqlQuery -import org.jetbrains.kotlinx.dataframe.io.getSchemaForSqlTable import org.jetbrains.kotlinx.dataframe.io.inferNullability import org.jetbrains.kotlinx.dataframe.io.readAllSqlTables import org.jetbrains.kotlinx.dataframe.io.readSqlQuery import org.jetbrains.kotlinx.dataframe.io.readSqlTable +import org.jetbrains.kotlinx.dataframe.schema.DataFrameSchema import org.junit.AfterClass import org.junit.BeforeClass import org.junit.Test @@ -309,7 +308,7 @@ class MariadbH2Test { val byteArray = "tinyblobValue".toByteArray() (result[0][22] as Blob).getBytes(1, byteArray.size) contentEquals byteArray - val schema = DataFrame.getSchemaForSqlTable(connection, "table1") + val schema = DataFrameSchema.readSqlTable(connection, "table1") schema.columns["id"]!!.type shouldBe typeOf() schema.columns["textcol"]!!.type shouldBe typeOf() schema.columns["varbinarycol"]!!.type shouldBe typeOf() @@ -326,7 +325,7 @@ class MariadbH2Test { val result2 = df2.filter { it[Table2MariaDb::id] == 1 } result2[0][26] shouldBe null - val schema2 = DataFrame.getSchemaForSqlTable(connection, "table2") + val schema2 = DataFrameSchema.readSqlTable(connection, "table2") schema2.columns["id"]!!.type shouldBe typeOf() schema2.columns["textcol"]!!.type shouldBe typeOf() } @@ -347,7 +346,7 @@ class MariadbH2Test { val result = df.filter { it[Table3MariaDb::id] == 1 } result[0][1] shouldBe "Value1" - val schema = DataFrame.getSchemaForSqlQuery(connection, sqlQuery = sqlQuery) + val schema = DataFrameSchema.readSqlQuery(connection, sqlQuery = sqlQuery) schema.columns["id"]!!.type shouldBe typeOf() schema.columns["enumcol"]!!.type shouldBe typeOf() } @@ -407,7 +406,7 @@ class MariadbH2Test { result8[0][1] shouldBe BigDecimal("10") - val schema = DataFrame.getSchemaForSqlTable(connection, "table1") + val schema = DataFrameSchema.readSqlTable(connection, "table1") schema.columns["tinyintcol"]!!.type shouldBe typeOf() schema.columns["smallintcol"]!!.type shouldBe typeOf() diff --git a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/h2/mssqlH2Test.kt b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/h2/mssqlH2Test.kt index 62bc40f231..3d37005b85 100644 --- a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/h2/mssqlH2Test.kt +++ b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/h2/mssqlH2Test.kt @@ -6,12 +6,11 @@ import org.jetbrains.kotlinx.dataframe.DataFrame import org.jetbrains.kotlinx.dataframe.annotations.DataSchema import org.jetbrains.kotlinx.dataframe.api.cast import org.jetbrains.kotlinx.dataframe.api.filter -import org.jetbrains.kotlinx.dataframe.io.getSchemaForSqlQuery -import org.jetbrains.kotlinx.dataframe.io.getSchemaForSqlTable import org.jetbrains.kotlinx.dataframe.io.inferNullability import org.jetbrains.kotlinx.dataframe.io.readAllSqlTables import org.jetbrains.kotlinx.dataframe.io.readSqlQuery import org.jetbrains.kotlinx.dataframe.io.readSqlTable +import org.jetbrains.kotlinx.dataframe.schema.DataFrameSchema import org.junit.AfterClass import org.junit.BeforeClass import org.junit.Test @@ -182,7 +181,7 @@ class MSSQLH2Test { result[0][Table1MSSSQL::intColumn] shouldBe 123456 result[0][Table1MSSSQL::ntextColumn] shouldBe "Sample1 text" - val schema = DataFrame.getSchemaForSqlTable(connection, "table1") + val schema = DataFrameSchema.readSqlTable(connection, "table1") schema.columns["id"]!!.type shouldBe typeOf() schema.columns["bigintColumn"]!!.type shouldBe typeOf() schema.columns["binaryColumn"]!!.type shouldBe typeOf() @@ -228,7 +227,7 @@ class MSSQLH2Test { val result = df.filter { it[Table1MSSSQL::id] == 1 } result[0][Table1MSSSQL::bigintColumn] shouldBe 123456789012345L - val schema = DataFrame.getSchemaForSqlQuery(connection, sqlQuery = sqlQuery) + val schema = DataFrameSchema.readSqlQuery(connection, sqlQuery = sqlQuery) schema.columns["id"]!!.type shouldBe typeOf() schema.columns["bigintColumn"]!!.type shouldBe typeOf() } diff --git a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/h2/mysqlH2Test.kt b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/h2/mysqlH2Test.kt index 921faae2ac..c0a6cdbe14 100644 --- a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/h2/mysqlH2Test.kt +++ b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/h2/mysqlH2Test.kt @@ -8,12 +8,11 @@ import org.jetbrains.kotlinx.dataframe.api.add import org.jetbrains.kotlinx.dataframe.api.cast import org.jetbrains.kotlinx.dataframe.api.filter import org.jetbrains.kotlinx.dataframe.api.select -import org.jetbrains.kotlinx.dataframe.io.getSchemaForSqlQuery -import org.jetbrains.kotlinx.dataframe.io.getSchemaForSqlTable import org.jetbrains.kotlinx.dataframe.io.inferNullability import org.jetbrains.kotlinx.dataframe.io.readAllSqlTables import org.jetbrains.kotlinx.dataframe.io.readSqlQuery import org.jetbrains.kotlinx.dataframe.io.readSqlTable +import org.jetbrains.kotlinx.dataframe.schema.DataFrameSchema import org.junit.AfterClass import org.junit.BeforeClass import org.junit.Test @@ -305,7 +304,7 @@ class MySqlH2Test { val result = df1.filter { it[Table1MySql::id] == 1 } result[0][26] shouldBe "textValue1" - val schema = DataFrame.getSchemaForSqlTable(connection, "table1") + val schema = DataFrameSchema.readSqlTable(connection, "table1") schema.columns["id"]!!.type shouldBe typeOf() schema.columns["textcol"]!!.type shouldBe typeOf() schema.columns["datecol"]!!.type shouldBe typeOf() @@ -322,7 +321,7 @@ class MySqlH2Test { val result2 = df2.filter { it[Table2MySql::id] == 1 } result2[0][26] shouldBe null - val schema2 = DataFrame.getSchemaForSqlTable(connection, "table2") + val schema2 = DataFrameSchema.readSqlTable(connection, "table2") schema2.columns["id"]!!.type shouldBe typeOf() schema2.columns["textcol"]!!.type shouldBe typeOf() } @@ -343,7 +342,7 @@ class MySqlH2Test { val result = df.filter { it[Table3MySql::id] == 1 } result[0][1] shouldBe "Value1" - val schema = DataFrame.getSchemaForSqlQuery(connection, sqlQuery = sqlQuery) + val schema = DataFrameSchema.readSqlQuery(connection, sqlQuery = sqlQuery) schema.columns["id"]!!.type shouldBe typeOf() schema.columns["enumcol"]!!.type shouldBe typeOf() } @@ -407,7 +406,7 @@ class MySqlH2Test { result8[0][1] shouldBe BigDecimal("10") - val schema = DataFrame.getSchemaForSqlTable(connection, "table1") + val schema = DataFrameSchema.readSqlTable(connection, "table1") schema.columns["tinyintcol"]!!.type shouldBe typeOf() schema.columns["smallintcol"]!!.type shouldBe typeOf() diff --git a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/h2/postgresH2Test.kt b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/h2/postgresH2Test.kt index 8b2572f397..d2b4bef65d 100644 --- a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/h2/postgresH2Test.kt +++ b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/h2/postgresH2Test.kt @@ -8,12 +8,11 @@ import org.jetbrains.kotlinx.dataframe.api.add import org.jetbrains.kotlinx.dataframe.api.cast import org.jetbrains.kotlinx.dataframe.api.filter import org.jetbrains.kotlinx.dataframe.api.select -import org.jetbrains.kotlinx.dataframe.io.getSchemaForSqlQuery -import org.jetbrains.kotlinx.dataframe.io.getSchemaForSqlTable import org.jetbrains.kotlinx.dataframe.io.inferNullability import org.jetbrains.kotlinx.dataframe.io.readAllSqlTables import org.jetbrains.kotlinx.dataframe.io.readSqlQuery import org.jetbrains.kotlinx.dataframe.io.readSqlTable +import org.jetbrains.kotlinx.dataframe.schema.DataFrameSchema import org.junit.AfterClass import org.junit.BeforeClass import org.junit.Test @@ -224,7 +223,7 @@ class PostgresH2Test { result[0][15] shouldBe arrayOf("Hello", "World") result[0][16] shouldBe arrayOf(true, false, true) - val schema = DataFrame.getSchemaForSqlTable(connection, tableName1) + val schema = DataFrameSchema.readSqlTable(connection, tableName1) schema.columns["id"]!!.type shouldBe typeOf() schema.columns["integercol"]!!.type shouldBe typeOf() schema.columns["smallintcol"]!!.type shouldBe typeOf() @@ -239,7 +238,7 @@ class PostgresH2Test { val result2 = df2.filter { it[Table2::id] == 1 } result2[0][4] shouldBe 1001 - val schema2 = DataFrame.getSchemaForSqlTable(connection, tableName2) + val schema2 = DataFrameSchema.readSqlTable(connection, tableName2) schema2.columns["id"]!!.type shouldBe typeOf() schema2.columns["textcol"]!!.type shouldBe typeOf() } @@ -261,7 +260,7 @@ class PostgresH2Test { val result = df.filter { it[ViewTable::id] == 1 } result[0][2] shouldBe null - val schema = DataFrame.getSchemaForSqlQuery(connection, sqlQuery = sqlQuery) + val schema = DataFrameSchema.readSqlQuery(connection, sqlQuery = sqlQuery) schema.columns["id"]!!.type shouldBe typeOf() schema.columns["bigintcol"]!!.type shouldBe typeOf() schema.columns["textcol"]!!.type shouldBe typeOf() @@ -318,12 +317,12 @@ class PostgresH2Test { .add("serialcol2") { it[Table2::serialcol] } result8[0][1] shouldBe 1000001 - val schema = DataFrame.getSchemaForSqlTable(connection, tableName1) + val schema = DataFrameSchema.readSqlTable(connection, tableName1) schema.columns["smallintcol"]!!.type shouldBe typeOf() schema.columns["bigserialcol"]!!.type shouldBe typeOf() schema.columns["doublecol"]!!.type shouldBe typeOf() - val schema1 = DataFrame.getSchemaForSqlTable(connection, tableName2) + val schema1 = DataFrameSchema.readSqlTable(connection, tableName2) schema1.columns["numericcol"]!!.type shouldBe typeOf() schema1.columns["realcol"]!!.type shouldBe typeOf() schema1.columns["serialcol"]!!.type shouldBe typeOf() diff --git a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/local/duckDbTest.kt b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/local/duckDbTest.kt index a5ad981ba2..10972696c4 100644 --- a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/local/duckDbTest.kt +++ b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/local/duckDbTest.kt @@ -22,9 +22,6 @@ import org.jetbrains.kotlinx.dataframe.api.with import org.jetbrains.kotlinx.dataframe.io.DbConnectionConfig import org.jetbrains.kotlinx.dataframe.io.assertInferredTypesMatchSchema import org.jetbrains.kotlinx.dataframe.io.db.DuckDb -import org.jetbrains.kotlinx.dataframe.io.getSchemaForAllSqlTables -import org.jetbrains.kotlinx.dataframe.io.getSchemaForResultSet -import org.jetbrains.kotlinx.dataframe.io.getSchemaForSqlTable import org.jetbrains.kotlinx.dataframe.io.readAllSqlTables import org.jetbrains.kotlinx.dataframe.io.readDataFrame import org.jetbrains.kotlinx.dataframe.io.readResultSet @@ -308,7 +305,7 @@ class DuckDbTest { ).executeUpdate() df = DataFrame.readSqlTable(connection, "test_table") - schema = DataFrame.getSchemaForSqlTable(connection, "test_table") + schema = DataFrameSchema.readSqlTable(connection, "test_table") subset = DataFrame.readSqlQuery(connection, """SELECT test_table.name, test_table.age FROM test_table""") } @@ -354,7 +351,7 @@ class DuckDbTest { connection.prepareStatement("SELECT * FROM test_table").executeQuery().use { rs -> df = DataFrame.readResultSet(rs, DuckDb) - schema = DataFrame.getSchemaForResultSet(rs, DuckDb) + schema = DataFrameSchema.readResultSet(rs, DuckDb) } } @@ -393,7 +390,7 @@ class DuckDbTest { ).executeUpdate() dfs = DataFrame.readAllSqlTables(connection = connection) - schemas = DataFrame.getSchemaForAllSqlTables(connection = connection) + schemas = DataFrameSchema.readAllSqlTables(connection = connection) } val df = dfs["test_table"]!! @@ -544,7 +541,7 @@ class DuckDbTest { """.trimIndent(), ).executeUpdate() - schema = DataFrame.getSchemaForSqlTable(connection, "table1") + schema = DataFrameSchema.readSqlTable(connection, "table1") df = DataFrame.readSqlTable(connection, "table1").reorderColumnsByName() } @@ -599,7 +596,7 @@ class DuckDbTest { """.trimIndent(), ).executeUpdate() - schema = DataFrame.getSchemaForSqlTable(connection, "table2") + schema = DataFrameSchema.readSqlTable(connection, "table2") df = DataFrame.readSqlTable(connection, "table2") } diff --git a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/local/imdbTest.kt b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/local/imdbTest.kt index 1d25648e67..11f09fe717 100644 --- a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/local/imdbTest.kt +++ b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/local/imdbTest.kt @@ -6,10 +6,9 @@ import org.jetbrains.kotlinx.dataframe.DataFrame import org.jetbrains.kotlinx.dataframe.annotations.DataSchema import org.jetbrains.kotlinx.dataframe.api.cast import org.jetbrains.kotlinx.dataframe.api.filter -import org.jetbrains.kotlinx.dataframe.io.getSchemaForSqlQuery -import org.jetbrains.kotlinx.dataframe.io.getSchemaForSqlTable import org.jetbrains.kotlinx.dataframe.io.readSqlQuery import org.jetbrains.kotlinx.dataframe.io.readSqlTable +import org.jetbrains.kotlinx.dataframe.schema.DataFrameSchema import org.junit.Ignore import org.junit.Test import java.sql.DriverManager @@ -55,7 +54,7 @@ class ImdbTestTest { val result = df.filter { it[ActorKDF::id] in 11..19 } result[0][1] shouldBe "Víctor" - val schema = DataFrame.getSchemaForSqlTable(connection, tableName) + val schema = DataFrameSchema.readSqlTable(connection, tableName) schema.columns["id"]!!.type shouldBe typeOf() schema.columns["first_name"]!!.type shouldBe typeOf() } @@ -76,7 +75,7 @@ class ImdbTestTest { val result = df.filter { it[ActorKDF::id] in 11..19 } result[0][1] shouldBe "Víctor" - val schema = DataFrame.getSchemaForSqlTable(connection, imdbTableName) + val schema = DataFrameSchema.readSqlTable(connection, imdbTableName) schema.columns["id"]!!.type shouldBe typeOf() schema.columns["first_name"]!!.type shouldBe typeOf() } @@ -109,7 +108,7 @@ class ImdbTestTest { df.filter { it[RankedMoviesWithGenres::year] != null && it[RankedMoviesWithGenres::year]!! > 2000 } result[0][1] shouldBe 2003 - val schema = DataFrame.getSchemaForSqlQuery(connection, sqlQuery) + val schema = DataFrameSchema.readSqlQuery(connection, sqlQuery) schema.columns["name"]!!.type shouldBe typeOf() schema.columns["year"]!!.type shouldBe typeOf() } diff --git a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/local/mariadbTest.kt b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/local/mariadbTest.kt index c79c7f0be8..be98d56ecf 100644 --- a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/local/mariadbTest.kt +++ b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/local/mariadbTest.kt @@ -8,12 +8,11 @@ import org.jetbrains.kotlinx.dataframe.api.add import org.jetbrains.kotlinx.dataframe.api.cast import org.jetbrains.kotlinx.dataframe.api.filter import org.jetbrains.kotlinx.dataframe.api.select -import org.jetbrains.kotlinx.dataframe.io.getSchemaForSqlQuery -import org.jetbrains.kotlinx.dataframe.io.getSchemaForSqlTable import org.jetbrains.kotlinx.dataframe.io.inferNullability import org.jetbrains.kotlinx.dataframe.io.readAllSqlTables import org.jetbrains.kotlinx.dataframe.io.readSqlQuery import org.jetbrains.kotlinx.dataframe.io.readSqlTable +import org.jetbrains.kotlinx.dataframe.schema.DataFrameSchema import org.junit.AfterClass import org.junit.BeforeClass import org.junit.Ignore @@ -341,7 +340,7 @@ class MariadbTest { val byteArray = "tinyblobValue".toByteArray() (result[0][22] as Blob).getBytes(1, byteArray.size) contentEquals byteArray - val schema = DataFrame.getSchemaForSqlTable(connection, "table1") + val schema = DataFrameSchema.readSqlTable(connection, "table1") schema.columns["id"]!!.type shouldBe typeOf() schema.columns["textCol"]!!.type shouldBe typeOf() schema.columns["varbinaryCol"]!!.type shouldBe typeOf() @@ -358,7 +357,7 @@ class MariadbTest { val result2 = df2.filter { it[Table2MariaDb::id] == 1 } result2[0][26] shouldBe null - val schema2 = DataFrame.getSchemaForSqlTable(connection, "table2") + val schema2 = DataFrameSchema.readSqlTable(connection, "table2") schema2.columns["id"]!!.type shouldBe typeOf() schema2.columns["textCol"]!!.type shouldBe typeOf() } @@ -380,7 +379,7 @@ class MariadbTest { val result = df.filter { it[Table3MariaDb::id] == 1 } result[0][2] shouldBe "Option1" - val schema = DataFrame.getSchemaForSqlQuery(connection, sqlQuery = sqlQuery) + val schema = DataFrameSchema.readSqlQuery(connection, sqlQuery = sqlQuery) schema.columns["id"]!!.type shouldBe typeOf() schema.columns["enumCol"]!!.type shouldBe typeOf() schema.columns["setCol"]!!.type shouldBe typeOf() @@ -457,7 +456,7 @@ class MariadbTest { result8[0][1] shouldBe BigDecimal("10") - val schema = DataFrame.getSchemaForSqlTable(connection, "table1") + val schema = DataFrameSchema.readSqlTable(connection, "table1") schema.columns["tinyintCol"]!!.type shouldBe typeOf() schema.columns["smallintCol"]!!.type shouldBe typeOf() diff --git a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/local/mssqlTest.kt b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/local/mssqlTest.kt index a92ff9fd22..4c586f0e0f 100644 --- a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/local/mssqlTest.kt +++ b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/local/mssqlTest.kt @@ -6,12 +6,11 @@ import org.jetbrains.kotlinx.dataframe.DataFrame import org.jetbrains.kotlinx.dataframe.annotations.DataSchema import org.jetbrains.kotlinx.dataframe.api.cast import org.jetbrains.kotlinx.dataframe.api.filter -import org.jetbrains.kotlinx.dataframe.io.getSchemaForSqlQuery -import org.jetbrains.kotlinx.dataframe.io.getSchemaForSqlTable import org.jetbrains.kotlinx.dataframe.io.inferNullability import org.jetbrains.kotlinx.dataframe.io.readAllSqlTables import org.jetbrains.kotlinx.dataframe.io.readSqlQuery import org.jetbrains.kotlinx.dataframe.io.readSqlTable +import org.jetbrains.kotlinx.dataframe.schema.DataFrameSchema import org.junit.AfterClass import org.junit.BeforeClass import org.junit.Ignore @@ -223,7 +222,7 @@ class MSSQLTest { result[0][Table1MSSSQL::intColumn] shouldBe 123456 result[0][Table1MSSSQL::ntextColumn] shouldBe "Sample1 text" - val schema = DataFrame.getSchemaForSqlTable(connection, "table1") + val schema = DataFrameSchema.readSqlTable(connection, "table1") schema.columns["id"]!!.type shouldBe typeOf() schema.columns["bigintColumn"]!!.type shouldBe typeOf() schema.columns["binaryColumn"]!!.type shouldBe typeOf() @@ -276,7 +275,7 @@ class MSSQLTest { val result = df.filter { it[Table1MSSSQL::id] == 1 } result[0][Table1MSSSQL::bigintColumn] shouldBe 123456789012345L - val schema = DataFrame.getSchemaForSqlQuery(connection, sqlQuery = sqlQuery) + val schema = DataFrameSchema.readSqlQuery(connection, sqlQuery = sqlQuery) schema.columns["id"]!!.type shouldBe typeOf() schema.columns["bigintColumn"]!!.type shouldBe typeOf() } diff --git a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/local/mysqlTest.kt b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/local/mysqlTest.kt index 868cfdd884..cee90eb4a6 100644 --- a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/local/mysqlTest.kt +++ b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/local/mysqlTest.kt @@ -8,12 +8,11 @@ import org.jetbrains.kotlinx.dataframe.api.add import org.jetbrains.kotlinx.dataframe.api.cast import org.jetbrains.kotlinx.dataframe.api.filter import org.jetbrains.kotlinx.dataframe.api.select -import org.jetbrains.kotlinx.dataframe.io.getSchemaForSqlQuery -import org.jetbrains.kotlinx.dataframe.io.getSchemaForSqlTable import org.jetbrains.kotlinx.dataframe.io.inferNullability import org.jetbrains.kotlinx.dataframe.io.readAllSqlTables import org.jetbrains.kotlinx.dataframe.io.readSqlQuery import org.jetbrains.kotlinx.dataframe.io.readSqlTable +import org.jetbrains.kotlinx.dataframe.schema.DataFrameSchema import org.junit.AfterClass import org.junit.BeforeClass import org.junit.Ignore @@ -340,7 +339,7 @@ class MySqlTest { result[0][26] shouldBe "textValue1" result[0][22] shouldBe "tinyblobValue".toByteArray() - val schema = DataFrame.getSchemaForSqlTable(connection, "table1") + val schema = DataFrameSchema.readSqlTable(connection, "table1") schema.columns["id"]!!.type shouldBe typeOf() schema.columns["textCol"]!!.type shouldBe typeOf() schema.columns["dateCol"]!!.type shouldBe typeOf() @@ -358,7 +357,7 @@ class MySqlTest { val result2 = df2.filter { it[Table2MySql::id] == 1 } result2[0][26] shouldBe null - val schema2 = DataFrame.getSchemaForSqlTable(connection, "table2") + val schema2 = DataFrameSchema.readSqlTable(connection, "table2") schema2.columns["id"]!!.type shouldBe typeOf() schema2.columns["textCol"]!!.type shouldBe typeOf() } @@ -380,7 +379,7 @@ class MySqlTest { val result = df.filter { it[Table3MySql::id] == 1 } result[0][2] shouldBe "Option1" - val schema = DataFrame.getSchemaForSqlQuery(connection, sqlQuery = sqlQuery) + val schema = DataFrameSchema.readSqlQuery(connection, sqlQuery = sqlQuery) schema.columns["id"]!!.type shouldBe typeOf() schema.columns["enumCol"]!!.type shouldBe typeOf() schema.columns["setCol"]!!.type shouldBe typeOf() @@ -455,7 +454,7 @@ class MySqlTest { result8[0][1] shouldBe BigDecimal("10") - val schema = DataFrame.getSchemaForSqlTable(connection, "table1") + val schema = DataFrameSchema.readSqlTable(connection, "table1") schema.columns["tinyintCol"]!!.type shouldBe typeOf() schema.columns["smallintCol"]!!.type shouldBe typeOf() diff --git a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/local/postgresTest.kt b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/local/postgresTest.kt index 9dddd6521f..024b0d781f 100644 --- a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/local/postgresTest.kt +++ b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/local/postgresTest.kt @@ -8,12 +8,11 @@ import org.jetbrains.kotlinx.dataframe.api.add import org.jetbrains.kotlinx.dataframe.api.cast import org.jetbrains.kotlinx.dataframe.api.filter import org.jetbrains.kotlinx.dataframe.api.select -import org.jetbrains.kotlinx.dataframe.io.getSchemaForSqlQuery -import org.jetbrains.kotlinx.dataframe.io.getSchemaForSqlTable import org.jetbrains.kotlinx.dataframe.io.inferNullability import org.jetbrains.kotlinx.dataframe.io.readAllSqlTables import org.jetbrains.kotlinx.dataframe.io.readSqlQuery import org.jetbrains.kotlinx.dataframe.io.readSqlTable +import org.jetbrains.kotlinx.dataframe.schema.DataFrameSchema import org.junit.AfterClass import org.junit.BeforeClass import org.junit.Ignore @@ -295,7 +294,7 @@ class PostgresTest { result[0][20] shouldBe arrayOf("Hello", "World") result[0][21] shouldBe arrayOf(true, false, true) - val schema = DataFrame.getSchemaForSqlTable(connection, tableName1) + val schema = DataFrameSchema.readSqlTable(connection, tableName1) schema.columns["id"]!!.type shouldBe typeOf() schema.columns["integercol"]!!.type shouldBe typeOf() schema.columns["smallintcol"]!!.type shouldBe typeOf() @@ -312,7 +311,7 @@ class PostgresTest { result2[0][11] shouldBe 1001 result2[0][13] shouldBe null - val schema2 = DataFrame.getSchemaForSqlTable(connection, tableName2) + val schema2 = DataFrameSchema.readSqlTable(connection, tableName2) schema2.columns["id"]!!.type shouldBe typeOf() schema2.columns["pathcol"]!!.type shouldBe typeOf() // TODO: https://github.com/Kotlin/dataframe/issues/537 schema2.columns["textcol"]!!.type shouldBe typeOf() @@ -337,7 +336,7 @@ class PostgresTest { val result = df.filter { it[ViewTable::id] == 1 } result[0][3] shouldBe null - val schema = DataFrame.getSchemaForSqlQuery(connection, sqlQuery = sqlQuery) + val schema = DataFrameSchema.readSqlQuery(connection, sqlQuery = sqlQuery) schema.columns["id"]!!.type shouldBe typeOf() schema.columns["bigintcol"]!!.type shouldBe typeOf() schema.columns["textcol"]!!.type shouldBe typeOf() @@ -402,12 +401,12 @@ class PostgresTest { .add("serialcol2") { it[Table2::serialcol] } result8[0][1] shouldBe 1000001 - val schema = DataFrame.getSchemaForSqlTable(connection, tableName1) + val schema = DataFrameSchema.readSqlTable(connection, tableName1) schema.columns["smallintcol"]!!.type shouldBe typeOf() schema.columns["bigserialcol"]!!.type shouldBe typeOf() schema.columns["doublecol"]!!.type shouldBe typeOf() - val schema1 = DataFrame.getSchemaForSqlTable(connection, tableName2) + val schema1 = DataFrameSchema.readSqlTable(connection, tableName2) schema1.columns["moneycol"]!!.type shouldBe typeOf() schema1.columns["numericcol"]!!.type shouldBe typeOf() schema1.columns["realcol"]!!.type shouldBe typeOf() diff --git a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/mssqlTest.kt b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/mssqlTest.kt index 8973c22ee2..c1657b8056 100644 --- a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/mssqlTest.kt +++ b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/mssqlTest.kt @@ -21,6 +21,6 @@ class MsSqlTest { @Test fun `test SQL Server TOP limit functionality`() { - MsSql.sqlQueryLimit("SELECT * FROM TestTable1", 1) shouldBe "SELECT TOP 1 * FROM TestTable1" + MsSql.buildSqlQueryWithLimit("SELECT * FROM TestTable1", 1) shouldBe "SELECT TOP 1 * FROM TestTable1" } } diff --git a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/sqliteTest.kt b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/sqliteTest.kt index f86d7883e7..7bbc18c295 100644 --- a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/sqliteTest.kt +++ b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/sqliteTest.kt @@ -6,6 +6,7 @@ import org.jetbrains.kotlinx.dataframe.DataFrame import org.jetbrains.kotlinx.dataframe.annotations.DataSchema import org.jetbrains.kotlinx.dataframe.api.cast import org.jetbrains.kotlinx.dataframe.api.filter +import org.jetbrains.kotlinx.dataframe.schema.DataFrameSchema import org.junit.AfterClass import org.junit.BeforeClass import org.junit.Test @@ -156,7 +157,7 @@ class SqliteTest { val result = df.filter { it[CustomerSQLite::name] == "John Doe" } result[0][2] shouldBe 30 - val schema = DataFrame.getSchemaForSqlTable(connection, customerTableName) + val schema = DataFrameSchema.readSqlTable(connection, customerTableName) schema.columns["id"]!!.type shouldBe typeOf() schema.columns["name"]!!.type shouldBe typeOf() schema.columns["salary"]!!.type shouldBe typeOf() @@ -167,7 +168,7 @@ class SqliteTest { val result2 = df2.filter { it[OrderSQLite::totalAmount] > 10 } result2[0][2] shouldBe "2023-07-21" - val schema2 = DataFrame.getSchemaForSqlTable(connection, orderTableName) + val schema2 = DataFrameSchema.readSqlTable(connection, orderTableName) schema2.columns["id"]!!.type shouldBe typeOf() schema2.columns["customerName"]!!.type shouldBe typeOf() schema2.columns["totalAmount"]!!.type shouldBe typeOf() @@ -183,7 +184,7 @@ class SqliteTest { val result = df.filter { it[CustomerSQLite::name] == "John Doe" } result[0][2] shouldBe 30 - val schema = DataFrame.getSchemaForSqlTable(dbConnectionConfig, customerTableName) + val schema = DataFrameSchema.readSqlTable(dbConnectionConfig, customerTableName) schema.columns["id"]!!.type shouldBe typeOf() schema.columns["name"]!!.type shouldBe typeOf() schema.columns["salary"]!!.type shouldBe typeOf() @@ -194,7 +195,7 @@ class SqliteTest { val result2 = df2.filter { it[OrderSQLite::totalAmount] > 10 } result2[0][2] shouldBe "2023-07-21" - val schema2 = DataFrame.getSchemaForSqlTable(dbConnectionConfig, orderTableName) + val schema2 = DataFrameSchema.readSqlTable(dbConnectionConfig, orderTableName) schema2.columns["id"]!!.type shouldBe typeOf() schema2.columns["customerName"]!!.type shouldBe typeOf() schema2.columns["totalAmount"]!!.type shouldBe typeOf() @@ -222,7 +223,7 @@ class SqliteTest { val result = df.filter { it[CustomerOrderSQLite::customerSalary] > 1 } result[0][3] shouldBe 2500.5 - val schema = DataFrame.getSchemaForSqlQuery(connection, sqlQuery = sqlQuery) + val schema = DataFrameSchema.readSqlQuery(connection, sqlQuery = sqlQuery) schema.columns["customerId"]!!.type shouldBe typeOf() schema.columns["customerName"]!!.type shouldBe typeOf() schema.columns["customerAge"]!!.type shouldBe typeOf() @@ -237,7 +238,7 @@ class SqliteTest { val result = df.filter { it[CustomerOrderSQLite::customerSalary] > 1 } result[0][3] shouldBe 2500.5 - val schema = DataFrame.getSchemaForSqlQuery(dbConnectionConfig, sqlQuery = sqlQuery) + val schema = DataFrameSchema.readSqlQuery(dbConnectionConfig, sqlQuery = sqlQuery) schema.columns["customerId"]!!.type shouldBe typeOf() schema.columns["customerName"]!!.type shouldBe typeOf() schema.columns["customerAge"]!!.type shouldBe typeOf() diff --git a/plugins/dataframe-gradle-plugin/src/main/kotlin/org/jetbrains/dataframe/gradle/GenerateDataSchemaTask.kt b/plugins/dataframe-gradle-plugin/src/main/kotlin/org/jetbrains/dataframe/gradle/GenerateDataSchemaTask.kt index 0bd69e6736..5ac74695cd 100644 --- a/plugins/dataframe-gradle-plugin/src/main/kotlin/org/jetbrains/dataframe/gradle/GenerateDataSchemaTask.kt +++ b/plugins/dataframe-gradle-plugin/src/main/kotlin/org/jetbrains/dataframe/gradle/GenerateDataSchemaTask.kt @@ -7,7 +7,6 @@ import org.gradle.api.provider.SetProperty import org.gradle.api.tasks.Input import org.gradle.api.tasks.OutputFile import org.gradle.api.tasks.TaskAction -import org.jetbrains.kotlinx.dataframe.DataFrame import org.jetbrains.kotlinx.dataframe.codeGen.CodeGenerator import org.jetbrains.kotlinx.dataframe.codeGen.MarkerVisibility import org.jetbrains.kotlinx.dataframe.codeGen.NameNormalizer @@ -23,9 +22,9 @@ import org.jetbrains.kotlinx.dataframe.io.Excel import org.jetbrains.kotlinx.dataframe.io.JSON import org.jetbrains.kotlinx.dataframe.io.OpenApi import org.jetbrains.kotlinx.dataframe.io.TsvDeephaven -import org.jetbrains.kotlinx.dataframe.io.getSchemaForSqlQuery -import org.jetbrains.kotlinx.dataframe.io.getSchemaForSqlTable import org.jetbrains.kotlinx.dataframe.io.isUrl +import org.jetbrains.kotlinx.dataframe.io.readSqlQuery +import org.jetbrains.kotlinx.dataframe.io.readSqlTable import org.jetbrains.kotlinx.dataframe.schema.DataFrameSchema import java.io.File import java.net.URL @@ -204,10 +203,10 @@ abstract class GenerateDataSchemaTask : DefaultTask() { private fun areBothNotBlank(tableName: String, sqlQuery: String) = sqlQuery.isNotBlank() && tableName.isNotBlank() private fun generateSchemaForTable(connection: Connection, tableName: String) = - DataFrame.getSchemaForSqlTable(connection, tableName) + DataFrameSchema.readSqlTable(connection, tableName) private fun generateSchemaForQuery(connection: Connection, sqlQuery: String) = - DataFrame.getSchemaForSqlQuery(connection, sqlQuery) + DataFrameSchema.readSqlQuery(connection, sqlQuery) private fun throwBothFieldsFilledException(tableName: String, sqlQuery: String): Nothing = throw RuntimeException( diff --git a/plugins/symbol-processor/src/main/kotlin/org/jetbrains/dataframe/ksp/DataSchemaGenerator.kt b/plugins/symbol-processor/src/main/kotlin/org/jetbrains/dataframe/ksp/DataSchemaGenerator.kt index 81d62dc7d3..33ed0e1905 100644 --- a/plugins/symbol-processor/src/main/kotlin/org/jetbrains/dataframe/ksp/DataSchemaGenerator.kt +++ b/plugins/symbol-processor/src/main/kotlin/org/jetbrains/dataframe/ksp/DataSchemaGenerator.kt @@ -6,7 +6,6 @@ import com.google.devtools.ksp.processing.Dependencies import com.google.devtools.ksp.processing.KSPLogger import com.google.devtools.ksp.processing.Resolver import com.google.devtools.ksp.symbol.KSFile -import org.jetbrains.kotlinx.dataframe.DataFrame import org.jetbrains.kotlinx.dataframe.annotations.CsvOptions import org.jetbrains.kotlinx.dataframe.annotations.DataSchemaVisibility import org.jetbrains.kotlinx.dataframe.annotations.ImportDataSchema @@ -30,9 +29,9 @@ import org.jetbrains.kotlinx.dataframe.io.OpenApi import org.jetbrains.kotlinx.dataframe.io.TsvDeephaven import org.jetbrains.kotlinx.dataframe.io.databaseCodeGenReader import org.jetbrains.kotlinx.dataframe.io.db.driverClassNameFromUrl -import org.jetbrains.kotlinx.dataframe.io.getSchemaForSqlQuery -import org.jetbrains.kotlinx.dataframe.io.getSchemaForSqlTable import org.jetbrains.kotlinx.dataframe.io.isUrl +import org.jetbrains.kotlinx.dataframe.io.readSqlQuery +import org.jetbrains.kotlinx.dataframe.io.readSqlTable import org.jetbrains.kotlinx.dataframe.schema.DataFrameSchema import java.io.File import java.net.MalformedURLException @@ -315,10 +314,10 @@ class DataSchemaGenerator( private fun areBothNotBlank(tableName: String, sqlQuery: String) = sqlQuery.isNotBlank() && tableName.isNotBlank() private fun generateSchemaForTable(connection: Connection, tableName: String) = - DataFrame.getSchemaForSqlTable(connection, tableName) + DataFrameSchema.readSqlTable(connection, tableName) private fun generateSchemaForQuery(connection: Connection, sqlQuery: String) = - DataFrame.getSchemaForSqlQuery(connection, sqlQuery) + DataFrameSchema.readSqlQuery(connection, sqlQuery) private fun throwBothFieldsFilledException(tableName: String, sqlQuery: String): Nothing = throw RuntimeException(