Skip to content

Add support for DateTime64(precision[, timezone]) in ClickHouse #26905

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

Add support for DateTime64(precision[, timezone]) in ClickHouse #26905

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions docs/src/main/sphinx/connector/clickhouse.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -233,6 +233,9 @@ to the following table:
* - `DateTime[(timezone)]`
- `TIMESTAMP(0) [WITH TIME ZONE]`
-
* - `DateTime64(p[, timezone])`
- `TIMESTAMP(p) [WITH TIME ZONE]`
-
* - `IPv4`
- `IPADDRESS`
-
Expand Down Expand Up @@ -304,6 +307,9 @@ to the following table:
* - `TIMESTAMP(0)`
- `DateTime`
-
* - `TIMESTAMP(p)`
- `DateTime64(p)`
-
* - `UUID`
- `UUID`
-
Expand Down
130 changes: 122 additions & 8 deletions plugin/trino-clickhouse/src/main/java/io/trino/plugin/clickhouse/ClickHouseClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
import io.trino.plugin.jdbc.JdbcTypeHandle;
import io.trino.plugin.jdbc.LongReadFunction;
import io.trino.plugin.jdbc.LongWriteFunction;
import io.trino.plugin.jdbc.ObjectReadFunction;
import io.trino.plugin.jdbc.ObjectWriteFunction;
import io.trino.plugin.jdbc.QueryBuilder;
import io.trino.plugin.jdbc.RemoteTableName;
Expand Down Expand Up @@ -74,7 +75,11 @@
import io.trino.spi.type.DecimalType;
import io.trino.spi.type.Decimals;
import io.trino.spi.type.Int128;
import io.trino.spi.type.LongTimestamp;
import io.trino.spi.type.LongTimestampWithTimeZone;
import io.trino.spi.type.StandardTypes;
import io.trino.spi.type.TimestampType;
import io.trino.spi.type.TimestampWithTimeZoneType;
import io.trino.spi.type.Type;
import io.trino.spi.type.TypeManager;
import io.trino.spi.type.TypeSignature;
Expand All @@ -97,6 +102,7 @@
import java.time.Instant;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.util.Collection;
import java.util.List;
Expand Down Expand Up @@ -124,6 +130,7 @@
import static io.trino.plugin.clickhouse.ClickHouseTableProperties.PRIMARY_KEY_PROPERTY;
import static io.trino.plugin.clickhouse.ClickHouseTableProperties.SAMPLE_BY_PROPERTY;
import static io.trino.plugin.clickhouse.TrinoToClickHouseWriteChecker.DATETIME;
import static io.trino.plugin.clickhouse.TrinoToClickHouseWriteChecker.DATETIME64;
import static io.trino.plugin.clickhouse.TrinoToClickHouseWriteChecker.UINT16;
import static io.trino.plugin.clickhouse.TrinoToClickHouseWriteChecker.UINT32;
import static io.trino.plugin.clickhouse.TrinoToClickHouseWriteChecker.UINT64;
Expand All @@ -147,6 +154,7 @@
import static io.trino.plugin.jdbc.StandardColumnMappings.integerWriteFunction;
import static io.trino.plugin.jdbc.StandardColumnMappings.longDecimalReadFunction;
import static io.trino.plugin.jdbc.StandardColumnMappings.longDecimalWriteFunction;
import static io.trino.plugin.jdbc.StandardColumnMappings.longTimestampReadFunction;
import static io.trino.plugin.jdbc.StandardColumnMappings.realWriteFunction;
import static io.trino.plugin.jdbc.StandardColumnMappings.shortDecimalWriteFunction;
import static io.trino.plugin.jdbc.StandardColumnMappings.smallintColumnMapping;
Expand Down Expand Up @@ -175,11 +183,18 @@
import static io.trino.spi.type.IntegerType.INTEGER;
import static io.trino.spi.type.RealType.REAL;
import static io.trino.spi.type.SmallintType.SMALLINT;
import static io.trino.spi.type.TimeZoneKey.getTimeZoneKey;
import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS;
import static io.trino.spi.type.TimestampType.TIMESTAMP_SECONDS;
import static io.trino.spi.type.TimestampType.createTimestampType;
import static io.trino.spi.type.TimestampWithTimeZoneType.TIMESTAMP_TZ_SECONDS;
import static io.trino.spi.type.TimestampWithTimeZoneType.createTimestampWithTimeZoneType;
import static io.trino.spi.type.Timestamps.MICROSECONDS_PER_SECOND;
import static io.trino.spi.type.Timestamps.MILLISECONDS_PER_SECOND;
import static io.trino.spi.type.Timestamps.NANOSECONDS_PER_MICROSECOND;
import static io.trino.spi.type.Timestamps.NANOSECONDS_PER_MILLISECOND;
import static io.trino.spi.type.Timestamps.PICOSECONDS_PER_NANOSECOND;
import static io.trino.spi.type.Timestamps.round;
import static io.trino.spi.type.TinyintType.TINYINT;
import static io.trino.spi.type.UuidType.javaUuidToTrinoUuid;
import static io.trino.spi.type.UuidType.trinoUuidToJavaUuid;
Expand All @@ -188,6 +203,7 @@
import static java.lang.Math.floorDiv;
import static java.lang.Math.floorMod;
import static java.lang.Math.max;
import static java.lang.Math.min;
import static java.lang.Math.toIntExact;
import static java.lang.String.format;
import static java.lang.String.join;
Expand All @@ -209,6 +225,7 @@ public class ClickHouseClient
private static final String NO_COMMENT = "";

public static final int DEFAULT_DOMAIN_COMPACTION_THRESHOLD = 1_000;
public static final int CLICKHOUSE_MAX_SUPPORTED_DATETIME64_PRECISION = 9;

private final ConnectorExpressionRewriter<ParameterizedExpression> connectorExpressionRewriter;
private final AggregateFunctionRewriter<JdbcExpression, ?> aggregateFunctionRewriter;
Expand Down Expand Up @@ -711,7 +728,7 @@ public Optional<ColumnMapping> toColumnMapping(ConnectorSession session, Connect

ColumnMapping decimalColumnMapping;
if (getDecimalRounding(session) == ALLOW_OVERFLOW && precision > Decimals.MAX_PRECISION) {
int scale = Math.min(decimalDigits, getDecimalDefaultScale(session));
int scale = min(decimalDigits, getDecimalDefaultScale(session));
decimalColumnMapping = decimalColumnMapping(createDecimalType(Decimals.MAX_PRECISION, scale), getDecimalRoundingMode(session));
}
else {
Expand All @@ -736,8 +753,11 @@ public Optional<ColumnMapping> toColumnMapping(ConnectorSession session, Connect
timestampReadFunction(TIMESTAMP_SECONDS),
timestampSecondsWriteFunction(version)));
}
// TODO (https://github.com/trinodb/trino/issues/10537) Add support for Datetime64 type
return Optional.of(timestampColumnMapping(TIMESTAMP_MILLIS));
if (columnDataType == ClickHouseDataType.DateTime64) {
return Optional.of(timestampColumnMapping(createTimestampType(typeHandle.requiredDecimalDigits()), getClickHouseServerVersion(session)));
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

issue (bug_risk): Check for requiredDecimalDigits exceeding supported precision.

Validate requiredDecimalDigits against CLICKHOUSE_MAX_SUPPORTED_DATETIME64_PRECISION before using it to avoid errors or data loss.

}
// TODO Add support for Datetime32 type
return Optional.of(timestampColumnMapping(TIMESTAMP_MILLIS, version));

case Types.TIMESTAMP_WITH_TIMEZONE:
if (columnDataType == ClickHouseDataType.DateTime) {
Expand All @@ -746,7 +766,10 @@ public Optional<ColumnMapping> toColumnMapping(ConnectorSession session, Connect
return Optional.of(ColumnMapping.longMapping(
TIMESTAMP_TZ_SECONDS,
shortTimestampWithTimeZoneReadFunction(),
shortTimestampWithTimeZoneWriteFunction(version, column.getTimeZone())));
shortTimestampWithTimeZoneWriteFunction(DATETIME, version, column.getTimeZone())));
}
if (columnDataType == ClickHouseDataType.DateTime64) {
return Optional.of(timestampWithTimeZoneColumnMapping(column, typeHandle.requiredDecimalDigits(), version));
}
}

Expand Down Expand Up @@ -799,8 +822,12 @@ public WriteMapping toWriteMapping(ConnectorSession session, Type type)
if (type == DATE) {
return WriteMapping.longMapping("Date", dateWriteFunctionUsingLocalDate(getClickHouseServerVersion(session)));
}
if (type == TIMESTAMP_SECONDS) {
return WriteMapping.longMapping("DateTime", timestampSecondsWriteFunction(getClickHouseServerVersion(session)));
if (type instanceof TimestampType timestampType) {
int precision = min(timestampType.getPrecision(), CLICKHOUSE_MAX_SUPPORTED_DATETIME64_PRECISION);
if (precision <= TimestampType.MAX_SHORT_PRECISION) {
return WriteMapping.longMapping(format("DateTime64(%d)", precision), shortTimestampWriteFunction(getClickHouseServerVersion(session)));
}
return WriteMapping.objectMapping(format("DateTime64(%d)", precision), longTimestampWriteFunction(getClickHouseServerVersion(session), precision));
Comment on lines +826 to +830
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

issue (bug_risk): Consider handling precision values below 0.

If timestampType.getPrecision() returns a negative value, min will not prevent invalid precision. Please add a check to ensure precision is non-negative before using it.

}
if (type.equals(uuidType)) {
return WriteMapping.sliceMapping("UUID", uuidWriteFunction());
Expand Down Expand Up @@ -906,6 +933,21 @@ private static LongWriteFunction dateWriteFunctionUsingLocalDate(ClickHouseVersi
};
}

private static ColumnMapping timestampColumnMapping(TimestampType timestampType, ClickHouseVersion version)
{
if (timestampType.getPrecision() <= TimestampType.MAX_SHORT_PRECISION) {
return ColumnMapping.longMapping(
timestampType,
timestampReadFunction(timestampType),
shortTimestampWriteFunction(version));
}
checkArgument(timestampType.getPrecision() <= CLICKHOUSE_MAX_SUPPORTED_DATETIME64_PRECISION, "Precision is out of range: %s", timestampType.getPrecision());
return ColumnMapping.objectMapping(
timestampType,
longTimestampReadFunction(timestampType),
longTimestampWriteFunction(version, timestampType.getPrecision()));
}

private static LongWriteFunction timestampSecondsWriteFunction(ClickHouseVersion version)
{
return (statement, index, value) -> {
Expand All @@ -919,6 +961,52 @@ private static LongWriteFunction timestampSecondsWriteFunction(ClickHouseVersion
};
}

private static LongWriteFunction shortTimestampWriteFunction(ClickHouseVersion version)
{
return (statement, index, value) -> {
long epochSecond = floorDiv(value, MICROSECONDS_PER_SECOND);
int nanoFraction = floorMod(value, MICROSECONDS_PER_SECOND) * NANOSECONDS_PER_MICROSECOND;
Instant instant = Instant.ofEpochSecond(epochSecond, nanoFraction);
LocalDateTime timestamp = LocalDateTime.ofInstant(instant, UTC);
// ClickHouse stores incorrect results when the values are out of supported range.
DATETIME64.validate(version, timestamp);
statement.setObject(index, timestamp);
};
}

private static ObjectWriteFunction longTimestampWriteFunction(ClickHouseVersion version, int precision)
{
checkArgument(precision > TimestampType.MAX_SHORT_PRECISION && precision <= CLICKHOUSE_MAX_SUPPORTED_DATETIME64_PRECISION,
"Precision is out of range: %s", precision);
return ObjectWriteFunction.of(
LongTimestamp.class,
(statement, index, value) -> {
long epochSeconds = floorDiv(value.getEpochMicros(), MICROSECONDS_PER_SECOND);
int microsOfSecond = floorMod(value.getEpochMicros(), MICROSECONDS_PER_SECOND);
long picosOfMicro = round(value.getPicosOfMicro(), TimestampType.MAX_PRECISION - precision);
int nanosOfSecond = (microsOfSecond * NANOSECONDS_PER_MICROSECOND) + toIntExact(picosOfMicro / PICOSECONDS_PER_NANOSECOND);
LocalDateTime timestamp = LocalDateTime.ofInstant(Instant.ofEpochSecond(epochSeconds, nanosOfSecond), UTC);
// ClickHouse stores incorrect results when the values are out of supported range.
DATETIME64.validate(version, timestamp);
statement.setObject(index, timestamp);
});
}

private static ColumnMapping timestampWithTimeZoneColumnMapping(ClickHouseColumn column, int precision, ClickHouseVersion version)
{
checkArgument(precision <= CLICKHOUSE_MAX_SUPPORTED_DATETIME64_PRECISION, "Unsupported precision %s", precision);
if (precision <= TimestampWithTimeZoneType.MAX_SHORT_PRECISION) {
return ColumnMapping.longMapping(
createTimestampWithTimeZoneType(precision),
shortTimestampWithTimeZoneReadFunction(),
shortTimestampWithTimeZoneWriteFunction(DATETIME64, version, column.getTimeZone()));
}
return ColumnMapping.objectMapping(
createTimestampWithTimeZoneType(precision),
longTimestampWithTimeZoneReadFunction(),
longTimestampWithTimeZoneWriteFunction());
}

private static LongReadFunction shortTimestampWithTimeZoneReadFunction()
{
return (resultSet, columnIndex) -> {
Expand All @@ -927,18 +1015,44 @@ private static LongReadFunction shortTimestampWithTimeZoneReadFunction()
};
}

private static LongWriteFunction shortTimestampWithTimeZoneWriteFunction(ClickHouseVersion version, TimeZone columnTimeZone)
private static LongWriteFunction shortTimestampWithTimeZoneWriteFunction(TrinoToClickHouseWriteChecker<LocalDateTime> valueChecker, ClickHouseVersion version, TimeZone columnTimeZone)
{
return (statement, index, value) -> {
long millisUtc = unpackMillisUtc(value);
// Clickhouse JDBC driver inserts datetime as string value as yyyy-MM-dd HH:mm:ss and zone from the Column metadata would be used.
Instant instant = Instant.ofEpochMilli(millisUtc);
// ClickHouse stores incorrect results when the values are out of supported range.
DATETIME.validate(version, instant.atZone(UTC).toLocalDateTime());
valueChecker.validate(version, instant.atZone(UTC).toLocalDateTime());
statement.setObject(index, instant.atZone(columnTimeZone.toZoneId()));
};
}

private static ObjectReadFunction longTimestampWithTimeZoneReadFunction()
{
return ObjectReadFunction.of(
LongTimestampWithTimeZone.class,
(resultSet, columnIndex) -> {
ZonedDateTime zonedDateTime = resultSet.getObject(columnIndex, ZonedDateTime.class);
return LongTimestampWithTimeZone.fromEpochSecondsAndFraction(
zonedDateTime.toEpochSecond(),
(long) zonedDateTime.getNano() * PICOSECONDS_PER_NANOSECOND,
getTimeZoneKey(zonedDateTime.getZone().getId()));
});
}

private static ObjectWriteFunction longTimestampWithTimeZoneWriteFunction()
{
return ObjectWriteFunction.of(
LongTimestampWithTimeZone.class,
(statement, index, value) -> {
long epochMillis = value.getEpochMillis();
long epochSeconds = floorDiv(epochMillis, MILLISECONDS_PER_SECOND);
int nanoAdjustment = floorMod(epochMillis, MILLISECONDS_PER_SECOND) * NANOSECONDS_PER_MILLISECOND + value.getPicosOfMilli() / PICOSECONDS_PER_NANOSECOND;
ZoneId zoneId = getTimeZoneKey(value.getTimeZoneKey()).getZoneId();
statement.setObject(index, Instant.ofEpochSecond(epochSeconds, nanoAdjustment).atZone(zoneId));
});
}

private ColumnMapping ipAddressColumnMapping(String clickhouseType)
{
return ColumnMapping.sliceMapping(
Expand Down
30 changes: 20 additions & 10 deletions ...no-clickhouse/src/main/java/io/trino/plugin/clickhouse/TrinoToClickHouseWriteChecker.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,16 +36,18 @@ public class TrinoToClickHouseWriteChecker<T>
// Different versions of ClickHouse may support different min/max values for the
// same data type, you can refer to the table below:
//
// | version | column type | min value | max value |
// |---------|-------------|---------------------|----------------------|
// | any | UInt8 | 0 | 255 |
// | any | UInt16 | 0 | 65535 |
// | any | UInt32 | 0 | 4294967295 |
// | any | UInt64 | 0 | 18446744073709551615 |
// | < 21.4 | Date | 1970-01-01 | 2106-02-07 |
// | < 21.4 | DateTime | 1970-01-01 00:00:00 | 2106-02-06 06:28:15 |
// | >= 21.4 | Date | 1970-01-01 | 2149-06-06 |
// | >= 21.4 | DateTime | 1970-01-01 00:00:00 | 2106-02-07 06:28:15 |
// | version | column type | min value | max value |
// |---------+-------------+---------------------+-------------------------------|
// | any | UInt8 | 0 | 255 |
// | any | UInt16 | 0 | 65535 |
// | any | UInt32 | 0 | 4294967295 |
// | any | UInt64 | 0 | 18446744073709551615 |
// | < 21.4 | Date | 1970-01-01 | 2106-02-07 |
// | < 21.4 | DateTime | 1970-01-01 00:00:00 | 2106-02-06 06:28:15 |
// | < 21.4 | DateTime64 | 1970-01-01 00:00:00 | 2106-02-06 06:28:15.999999999 |
// | >= 21.4 | Date | 1970-01-01 | 2149-06-06 |
// | >= 21.4 | DateTime | 1970-01-01 00:00:00 | 2106-02-07 06:28:15 |
// | >= 21.4 | DateTime64 | 1925-01-01 00:00:00 | 2283-11-11 23:59:59.99999999 |
//
// And when the value written to ClickHouse is out of range, ClickHouse will store
// the incorrect result, so we need to check the range of the written value to
Expand All @@ -68,6 +70,14 @@ public class TrinoToClickHouseWriteChecker<T>
new TimestampWriteValueChecker(
version -> version.isNewerOrEqualTo("21.4"),
new Range<>(LocalDateTime.parse("1970-01-01T00:00:00"), LocalDateTime.parse("2106-02-07T06:28:15")))));
public static final TrinoToClickHouseWriteChecker<LocalDateTime> DATETIME64 = new TrinoToClickHouseWriteChecker<>(
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could probably dedup this, not sure why it was split into pre/post 21.4. The values are the same.

I did widen these ranges from the original PR based on the documentation I found. https://clickhouse.com/docs/sql-reference/data-types/datetime64

I also went with the lower upper range value. Technically it could be higher when using a precision < 9.

ImmutableList.of(
new TimestampWriteValueChecker(
version -> version.isOlderThan("21.4"),
new Range<>(LocalDateTime.parse("1900-01-01T00:00:00"), LocalDateTime.parse("2262-04-11T23:47:16"))),
new TimestampWriteValueChecker(
version -> version.isNewerOrEqualTo("21.4"),
new Range<>(LocalDateTime.parse("1900-01-01T00:00:00"), LocalDateTime.parse("2262-04-11T23:47:16")))));
Comment on lines +73 to +80
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

issue: Consider aligning DateTime64 range with documented values.

Please verify that the DateTime64 range matches the documented values for each version, and update the code or comments to ensure consistency.


private final List<Checker<T>> checkers;

Expand Down
Loading