diff --git a/examples/scratch/duckdb.md b/examples/scratch/duckdb.md new file mode 100644 index 0000000..c6ef036 --- /dev/null +++ b/examples/scratch/duckdb.md @@ -0,0 +1,23 @@ +## Testing ### +DuckDB iceberg extension can also be used to validate ice created tables. +https://blog.min.io/duckdb-and-minio-for-a-modern-data-stack/ + +`duckdb` + +`install iceberg` + +`load iceberg` + +`update extensions` + +`create secret iceberg_secret(type iceberg, token 'foo');` + + +``` +set s3_use_ssl=false; +set s3_endpoint='localhost:9000'; +create secret(type s3, key_id 'miniouser', secret 'miniopassword'); + +attach 'warehouse' as iceberg_catalog(type iceberg, secret iceberg_secret, endpoint 'http://localhost:5000'); +show all tables; +``` diff --git a/ice/src/main/java/org/apache/iceberg/data/parquet/BaseParquetReaders.java b/ice/src/main/java/org/apache/iceberg/data/parquet/BaseParquetReaders.java deleted file mode 100644 index 7f237b7..0000000 --- a/ice/src/main/java/org/apache/iceberg/data/parquet/BaseParquetReaders.java +++ /dev/null @@ -1,439 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iceberg.data.parquet; - -import java.util.List; -import java.util.Map; -import java.util.Optional; -import org.apache.iceberg.MetadataColumns; -import org.apache.iceberg.Schema; -import org.apache.iceberg.parquet.ParquetSchemaUtil; -import org.apache.iceberg.parquet.ParquetValueReader; -import org.apache.iceberg.parquet.ParquetValueReaders; -import org.apache.iceberg.parquet.TypeWithSchemaVisitor; -import org.apache.iceberg.relocated.com.google.common.base.Preconditions; -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; -import org.apache.iceberg.relocated.com.google.common.collect.Lists; -import org.apache.iceberg.relocated.com.google.common.collect.Maps; -import org.apache.iceberg.types.Type.TypeID; -import org.apache.iceberg.types.Types; -import org.apache.parquet.column.ColumnDescriptor; -import org.apache.parquet.schema.GroupType; -import org.apache.parquet.schema.LogicalTypeAnnotation; -import org.apache.parquet.schema.LogicalTypeAnnotation.DateLogicalTypeAnnotation; -import org.apache.parquet.schema.LogicalTypeAnnotation.DecimalLogicalTypeAnnotation; -import org.apache.parquet.schema.LogicalTypeAnnotation.EnumLogicalTypeAnnotation; -import org.apache.parquet.schema.LogicalTypeAnnotation.IntLogicalTypeAnnotation; -import org.apache.parquet.schema.LogicalTypeAnnotation.JsonLogicalTypeAnnotation; -import org.apache.parquet.schema.LogicalTypeAnnotation.LogicalTypeAnnotationVisitor; -import org.apache.parquet.schema.LogicalTypeAnnotation.StringLogicalTypeAnnotation; -import org.apache.parquet.schema.LogicalTypeAnnotation.TimeLogicalTypeAnnotation; -import org.apache.parquet.schema.LogicalTypeAnnotation.TimestampLogicalTypeAnnotation; -import org.apache.parquet.schema.MessageType; -import org.apache.parquet.schema.PrimitiveType; -import org.apache.parquet.schema.Type; - -/** - * @deprecated since 1.8.0, will be made package-private in 1.9.0 - */ -@Deprecated -public abstract class BaseParquetReaders { - protected BaseParquetReaders() {} - - protected ParquetValueReader createReader(Schema expectedSchema, MessageType fileSchema) { - return createReader(expectedSchema, fileSchema, ImmutableMap.of()); - } - - @SuppressWarnings("unchecked") - protected ParquetValueReader createReader( - Schema expectedSchema, MessageType fileSchema, Map idToConstant) { - if (ParquetSchemaUtil.hasIds(fileSchema)) { - return (ParquetValueReader) - TypeWithSchemaVisitor.visit( - expectedSchema.asStruct(), fileSchema, new ReadBuilder(fileSchema, idToConstant)); - } else { - return (ParquetValueReader) - TypeWithSchemaVisitor.visit( - expectedSchema.asStruct(), - fileSchema, - new FallbackReadBuilder(fileSchema, idToConstant)); - } - } - - protected abstract ParquetValueReader createStructReader( - List types, List> fieldReaders, Types.StructType structType); - - protected ParquetValueReader fixedReader(ColumnDescriptor desc) { - return new GenericParquetReaders.FixedReader(desc); - } - - protected ParquetValueReader dateReader(ColumnDescriptor desc) { - return new GenericParquetReaders.DateReader(desc); - } - - protected ParquetValueReader timeReader(ColumnDescriptor desc) { - LogicalTypeAnnotation time = desc.getPrimitiveType().getLogicalTypeAnnotation(); - Preconditions.checkArgument( - time instanceof TimeLogicalTypeAnnotation, "Invalid time logical type: " + time); - - LogicalTypeAnnotation.TimeUnit unit = ((TimeLogicalTypeAnnotation) time).getUnit(); - switch (unit) { - case MICROS: - return new GenericParquetReaders.TimeReader(desc); - case MILLIS: - return new GenericParquetReaders.TimeMillisReader(desc); - default: - throw new UnsupportedOperationException("Unsupported unit for time: " + unit); - } - } - - protected ParquetValueReader timestampReader(ColumnDescriptor desc, boolean isAdjustedToUTC) { - if (desc.getPrimitiveType().getPrimitiveTypeName() == PrimitiveType.PrimitiveTypeName.INT96) { - return new GenericParquetReaders.TimestampInt96Reader(desc); - } - - LogicalTypeAnnotation timestamp = desc.getPrimitiveType().getLogicalTypeAnnotation(); - Preconditions.checkArgument( - timestamp instanceof TimestampLogicalTypeAnnotation, - "Invalid timestamp logical type: " + timestamp); - - LogicalTypeAnnotation.TimeUnit unit = ((TimestampLogicalTypeAnnotation) timestamp).getUnit(); - switch (unit) { - case MICROS: - return isAdjustedToUTC - ? new GenericParquetReaders.TimestamptzReader(desc) - : new GenericParquetReaders.TimestampReader(desc); - case MILLIS: - return isAdjustedToUTC - ? new GenericParquetReaders.TimestamptzMillisReader(desc) - : new GenericParquetReaders.TimestampMillisReader(desc); - default: - throw new UnsupportedOperationException("Unsupported unit for timestamp: " + unit); - } - } - - protected Object convertConstant(org.apache.iceberg.types.Type type, Object value) { - return value; - } - - private class FallbackReadBuilder extends ReadBuilder { - private FallbackReadBuilder(MessageType type, Map idToConstant) { - super(type, idToConstant); - } - - @Override - public ParquetValueReader message( - Types.StructType expected, MessageType message, List> fieldReaders) { - // the top level matches by ID, but the remaining IDs are missing - return super.struct(expected, message, fieldReaders); - } - - @Override - public ParquetValueReader struct( - Types.StructType expected, GroupType struct, List> fieldReaders) { - // the expected struct is ignored because nested fields are never found when the - List> newFields = - Lists.newArrayListWithExpectedSize(fieldReaders.size()); - List types = Lists.newArrayListWithExpectedSize(fieldReaders.size()); - List fields = struct.getFields(); - for (int i = 0; i < fields.size(); i += 1) { - ParquetValueReader fieldReader = fieldReaders.get(i); - if (fieldReader != null) { - Type fieldType = fields.get(i); - int fieldD = type().getMaxDefinitionLevel(path(fieldType.getName())) - 1; - newFields.add(ParquetValueReaders.option(fieldType, fieldD, fieldReader)); - types.add(fieldType); - } - } - - return createStructReader(types, newFields, expected); - } - } - - private class LogicalTypeReadBuilder - implements LogicalTypeAnnotationVisitor> { - - private final ColumnDescriptor desc; - private final org.apache.iceberg.types.Type.PrimitiveType expected; - - LogicalTypeReadBuilder( - ColumnDescriptor desc, org.apache.iceberg.types.Type.PrimitiveType expected) { - this.desc = desc; - this.expected = expected; - } - - @Override - public Optional> visit(StringLogicalTypeAnnotation stringLogicalType) { - return Optional.of(ParquetValueReaders.strings(desc)); - } - - @Override - public Optional> visit(EnumLogicalTypeAnnotation enumLogicalType) { - return Optional.of(ParquetValueReaders.strings(desc)); - } - - @Override - public Optional> visit(DecimalLogicalTypeAnnotation decimalLogicalType) { - return Optional.of(ParquetValueReaders.bigDecimals(desc)); - } - - @Override - public Optional> visit(DateLogicalTypeAnnotation dateLogicalType) { - return Optional.of(dateReader(desc)); - } - - @Override - public Optional> visit(TimeLogicalTypeAnnotation timeLogicalType) { - return Optional.of(timeReader(desc)); - } - - @Override - public Optional> visit( - TimestampLogicalTypeAnnotation timestampLogicalType) { - return Optional.of( - timestampReader(desc, ((Types.TimestampType) expected).shouldAdjustToUTC())); - } - - @Override - public Optional> visit(IntLogicalTypeAnnotation intLogicalType) { - if (intLogicalType.getBitWidth() == 64) { - /* - Preconditions.checkArgument( - intLogicalType.isSigned(), "Cannot read UINT64 as a long value"); - */ - return Optional.of(new ParquetValueReaders.UnboxedReader<>(desc)); - } - - if (expected.typeId() == TypeID.LONG) { - return Optional.of(new ParquetValueReaders.IntAsLongReader(desc)); - } - - Preconditions.checkArgument( - intLogicalType.isSigned() || intLogicalType.getBitWidth() < 32, - "Cannot read UINT32 as an int value"); - - return Optional.of(new ParquetValueReaders.UnboxedReader<>(desc)); - } - - @Override - public Optional> visit(JsonLogicalTypeAnnotation jsonLogicalType) { - return Optional.of(ParquetValueReaders.strings(desc)); - } - - @Override - public Optional> visit( - LogicalTypeAnnotation.BsonLogicalTypeAnnotation bsonLogicalType) { - return Optional.of(ParquetValueReaders.byteBuffers(desc)); - } - - @Override - public Optional> visit( - LogicalTypeAnnotation.UUIDLogicalTypeAnnotation uuidLogicalType) { - return Optional.of(ParquetValueReaders.uuids(desc)); - } - } - - private class ReadBuilder extends TypeWithSchemaVisitor> { - private final MessageType type; - private final Map idToConstant; - - private ReadBuilder(MessageType type, Map idToConstant) { - this.type = type; - this.idToConstant = idToConstant; - } - - @Override - public ParquetValueReader message( - Types.StructType expected, MessageType message, List> fieldReaders) { - return struct(expected, message.asGroupType(), fieldReaders); - } - - @Override - public ParquetValueReader struct( - Types.StructType expected, GroupType struct, List> fieldReaders) { - // match the expected struct's order - Map> readersById = Maps.newHashMap(); - Map typesById = Maps.newHashMap(); - Map maxDefinitionLevelsById = Maps.newHashMap(); - List fields = struct.getFields(); - for (int i = 0; i < fields.size(); i += 1) { - ParquetValueReader fieldReader = fieldReaders.get(i); - if (fieldReader != null) { - Type fieldType = fields.get(i); - int fieldD = type.getMaxDefinitionLevel(path(fieldType.getName())) - 1; - int id = fieldType.getId().intValue(); - readersById.put(id, ParquetValueReaders.option(fieldType, fieldD, fieldReader)); - typesById.put(id, fieldType); - if (idToConstant.containsKey(id)) { - maxDefinitionLevelsById.put(id, fieldD); - } - } - } - - List expectedFields = - expected != null ? expected.fields() : ImmutableList.of(); - List> reorderedFields = - Lists.newArrayListWithExpectedSize(expectedFields.size()); - List types = Lists.newArrayListWithExpectedSize(expectedFields.size()); - // Defaulting to parent max definition level - int defaultMaxDefinitionLevel = type.getMaxDefinitionLevel(currentPath()); - for (Types.NestedField field : expectedFields) { - int id = field.fieldId(); - ParquetValueReader reader = readersById.get(id); - if (idToConstant.containsKey(id)) { - // containsKey is used because the constant may be null - int fieldMaxDefinitionLevel = - maxDefinitionLevelsById.getOrDefault(id, defaultMaxDefinitionLevel); - reorderedFields.add( - ParquetValueReaders.constant(idToConstant.get(id), fieldMaxDefinitionLevel)); - types.add(null); - } else if (id == MetadataColumns.ROW_POSITION.fieldId()) { - reorderedFields.add(ParquetValueReaders.position()); - types.add(null); - } else if (id == MetadataColumns.IS_DELETED.fieldId()) { - reorderedFields.add(ParquetValueReaders.constant(false)); - types.add(null); - } else if (reader != null) { - reorderedFields.add(reader); - types.add(typesById.get(id)); - } else if (field.initialDefault() != null) { - reorderedFields.add( - ParquetValueReaders.constant( - convertConstant(field.type(), field.initialDefault()), - maxDefinitionLevelsById.getOrDefault(id, defaultMaxDefinitionLevel))); - types.add(typesById.get(id)); - } else if (field.isOptional()) { - reorderedFields.add(ParquetValueReaders.nulls()); - types.add(null); - } else { - throw new IllegalArgumentException( - String.format("Missing required field: %s", field.name())); - } - } - - return createStructReader(types, reorderedFields, expected); - } - - @Override - public ParquetValueReader list( - Types.ListType expectedList, GroupType array, ParquetValueReader elementReader) { - if (expectedList == null) { - return null; - } - - String[] repeatedPath = currentPath(); - - int repeatedD = type.getMaxDefinitionLevel(repeatedPath) - 1; - int repeatedR = type.getMaxRepetitionLevel(repeatedPath) - 1; - - Type elementType = ParquetSchemaUtil.determineListElementType(array); - int elementD = type.getMaxDefinitionLevel(path(elementType.getName())) - 1; - - return new ParquetValueReaders.ListReader<>( - repeatedD, repeatedR, ParquetValueReaders.option(elementType, elementD, elementReader)); - } - - @Override - public ParquetValueReader map( - Types.MapType expectedMap, - GroupType map, - ParquetValueReader keyReader, - ParquetValueReader valueReader) { - if (expectedMap == null) { - return null; - } - - GroupType repeatedKeyValue = map.getFields().get(0).asGroupType(); - String[] repeatedPath = currentPath(); - - int repeatedD = type.getMaxDefinitionLevel(repeatedPath) - 1; - int repeatedR = type.getMaxRepetitionLevel(repeatedPath) - 1; - - Type keyType = repeatedKeyValue.getType(0); - int keyD = type.getMaxDefinitionLevel(path(keyType.getName())) - 1; - Type valueType = repeatedKeyValue.getType(1); - int valueD = type.getMaxDefinitionLevel(path(valueType.getName())) - 1; - - return new ParquetValueReaders.MapReader<>( - repeatedD, - repeatedR, - ParquetValueReaders.option(keyType, keyD, keyReader), - ParquetValueReaders.option(valueType, valueD, valueReader)); - } - - @Override - @SuppressWarnings("checkstyle:CyclomaticComplexity") - public ParquetValueReader primitive( - org.apache.iceberg.types.Type.PrimitiveType expected, PrimitiveType primitive) { - if (expected == null) { - return null; - } - - ColumnDescriptor desc = type.getColumnDescription(currentPath()); - - if (primitive.getLogicalTypeAnnotation() != null) { - return primitive - .getLogicalTypeAnnotation() - .accept(new LogicalTypeReadBuilder(desc, expected)) - .orElseThrow( - () -> - new UnsupportedOperationException( - "Unsupported logical type: " + primitive.getLogicalTypeAnnotation())); - } - - switch (primitive.getPrimitiveTypeName()) { - case FIXED_LEN_BYTE_ARRAY: - return fixedReader(desc); - case BINARY: - if (expected.typeId() == TypeID.STRING) { - return ParquetValueReaders.strings(desc); - } else { - return ParquetValueReaders.byteBuffers(desc); - } - case INT32: - if (expected.typeId() == TypeID.LONG) { - return ParquetValueReaders.intsAsLongs(desc); - } else { - return ParquetValueReaders.unboxed(desc); - } - case FLOAT: - if (expected.typeId() == TypeID.DOUBLE) { - return ParquetValueReaders.floatsAsDoubles(desc); - } else { - return ParquetValueReaders.unboxed(desc); - } - case BOOLEAN: - case INT64: - case DOUBLE: - return ParquetValueReaders.unboxed(desc); - case INT96: - // Impala & Spark used to write timestamps as INT96 without a logical type. For backwards - // compatibility we try to read INT96 as timestamps. - return timestampReader(desc, true); - default: - throw new UnsupportedOperationException("Unsupported type: " + primitive); - } - } - - MessageType type() { - return type; - } - } -} diff --git a/ice/src/main/java/org/apache/iceberg/data/parquet/BaseParquetWriter.java b/ice/src/main/java/org/apache/iceberg/data/parquet/BaseParquetWriter.java deleted file mode 100644 index 8199f69..0000000 --- a/ice/src/main/java/org/apache/iceberg/data/parquet/BaseParquetWriter.java +++ /dev/null @@ -1,266 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iceberg.data.parquet; - -import java.util.List; -import java.util.Optional; -import org.apache.iceberg.parquet.ParquetTypeVisitor; -import org.apache.iceberg.parquet.ParquetValueWriter; -import org.apache.iceberg.parquet.ParquetValueWriters; -import org.apache.iceberg.relocated.com.google.common.base.Preconditions; -import org.apache.iceberg.relocated.com.google.common.collect.Lists; -import org.apache.parquet.column.ColumnDescriptor; -import org.apache.parquet.schema.GroupType; -import org.apache.parquet.schema.LogicalTypeAnnotation; -import org.apache.parquet.schema.MessageType; -import org.apache.parquet.schema.PrimitiveType; -import org.apache.parquet.schema.Type; - -/** - * @deprecated since 1.8.0, will be made package-private in 1.9.0 - */ -@Deprecated -public abstract class BaseParquetWriter { - - @SuppressWarnings("unchecked") - protected ParquetValueWriter createWriter(MessageType type) { - return (ParquetValueWriter) ParquetTypeVisitor.visit(type, new WriteBuilder(type)); - } - - protected abstract ParquetValueWriters.StructWriter createStructWriter( - List> writers); - - protected ParquetValueWriter fixedWriter(ColumnDescriptor desc) { - return new GenericParquetWriter.FixedWriter(desc); - } - - protected ParquetValueWriter dateWriter(ColumnDescriptor desc) { - return new GenericParquetWriter.DateWriter(desc); - } - - protected ParquetValueWriter timeWriter(ColumnDescriptor desc) { - return new GenericParquetWriter.TimeWriter(desc); - } - - protected ParquetValueWriter timestampWriter(ColumnDescriptor desc, boolean isAdjustedToUTC) { - if (isAdjustedToUTC) { - return new GenericParquetWriter.TimestamptzWriter(desc); - } else { - return new GenericParquetWriter.TimestampWriter(desc); - } - } - - private class WriteBuilder extends ParquetTypeVisitor> { - private final MessageType type; - - private WriteBuilder(MessageType type) { - this.type = type; - } - - @Override - public ParquetValueWriter message( - MessageType message, List> fieldWriters) { - - return struct(message.asGroupType(), fieldWriters); - } - - @Override - public ParquetValueWriter struct( - GroupType struct, List> fieldWriters) { - List fields = struct.getFields(); - List> writers = Lists.newArrayListWithExpectedSize(fieldWriters.size()); - for (int i = 0; i < fields.size(); i += 1) { - Type fieldType = struct.getType(i); - int fieldD = type.getMaxDefinitionLevel(path(fieldType.getName())); - writers.add(ParquetValueWriters.option(fieldType, fieldD, fieldWriters.get(i))); - } - - return createStructWriter(writers); - } - - @Override - public ParquetValueWriter list(GroupType array, ParquetValueWriter elementWriter) { - GroupType repeated = array.getFields().get(0).asGroupType(); - String[] repeatedPath = currentPath(); - - int repeatedD = type.getMaxDefinitionLevel(repeatedPath); - int repeatedR = type.getMaxRepetitionLevel(repeatedPath); - - Type elementType = repeated.getType(0); - int elementD = type.getMaxDefinitionLevel(path(elementType.getName())); - - return ParquetValueWriters.collections( - repeatedD, repeatedR, ParquetValueWriters.option(elementType, elementD, elementWriter)); - } - - @Override - public ParquetValueWriter map( - GroupType map, ParquetValueWriter keyWriter, ParquetValueWriter valueWriter) { - GroupType repeatedKeyValue = map.getFields().get(0).asGroupType(); - String[] repeatedPath = currentPath(); - - int repeatedD = type.getMaxDefinitionLevel(repeatedPath); - int repeatedR = type.getMaxRepetitionLevel(repeatedPath); - - Type keyType = repeatedKeyValue.getType(0); - int keyD = type.getMaxDefinitionLevel(path(keyType.getName())); - Type valueType = repeatedKeyValue.getType(1); - int valueD = type.getMaxDefinitionLevel(path(valueType.getName())); - - return ParquetValueWriters.maps( - repeatedD, - repeatedR, - ParquetValueWriters.option(keyType, keyD, keyWriter), - ParquetValueWriters.option(valueType, valueD, valueWriter)); - } - - @Override - public ParquetValueWriter primitive(PrimitiveType primitive) { - ColumnDescriptor desc = type.getColumnDescription(currentPath()); - LogicalTypeAnnotation logicalType = primitive.getLogicalTypeAnnotation(); - if (logicalType != null) { - Optional> writer = - logicalType.accept(new LogicalTypeWriterVisitor(desc)); - if (writer.isPresent()) { - return writer.get(); - } - } - - switch (primitive.getPrimitiveTypeName()) { - case FIXED_LEN_BYTE_ARRAY: - return fixedWriter(desc); - case BINARY: - return ParquetValueWriters.byteBuffers(desc); - case BOOLEAN: - return ParquetValueWriters.booleans(desc); - case INT32: - return ParquetValueWriters.ints(desc); - case INT64: - return ParquetValueWriters.longs(desc); - case FLOAT: - return ParquetValueWriters.floats(desc); - case DOUBLE: - return ParquetValueWriters.doubles(desc); - default: - throw new UnsupportedOperationException("Unsupported type: " + primitive); - } - } - } - - private class LogicalTypeWriterVisitor - implements LogicalTypeAnnotation.LogicalTypeAnnotationVisitor> { - private final ColumnDescriptor desc; - - private LogicalTypeWriterVisitor(ColumnDescriptor desc) { - this.desc = desc; - } - - @Override - public Optional> visit( - LogicalTypeAnnotation.StringLogicalTypeAnnotation stringType) { - return Optional.of(ParquetValueWriters.strings(desc)); - } - - @Override - public Optional> visit( - LogicalTypeAnnotation.EnumLogicalTypeAnnotation enumType) { - return Optional.of(ParquetValueWriters.strings(desc)); - } - - @Override - public Optional> visit( - LogicalTypeAnnotation.DecimalLogicalTypeAnnotation decimalType) { - switch (desc.getPrimitiveType().getPrimitiveTypeName()) { - case INT32: - return Optional.of( - ParquetValueWriters.decimalAsInteger( - desc, decimalType.getPrecision(), decimalType.getScale())); - case INT64: - return Optional.of( - ParquetValueWriters.decimalAsLong( - desc, decimalType.getPrecision(), decimalType.getScale())); - case BINARY: - case FIXED_LEN_BYTE_ARRAY: - return Optional.of( - ParquetValueWriters.decimalAsFixed( - desc, decimalType.getPrecision(), decimalType.getScale())); - } - return Optional.empty(); - } - - @Override - public Optional> visit( - LogicalTypeAnnotation.DateLogicalTypeAnnotation dateType) { - return Optional.of(dateWriter(desc)); - } - - @Override - public Optional> visit( - LogicalTypeAnnotation.TimeLogicalTypeAnnotation timeType) { - Preconditions.checkArgument( - LogicalTypeAnnotation.TimeUnit.MICROS.equals(timeType.getUnit()), - "Cannot write time in %s, only MICROS is supported", - timeType.getUnit()); - return Optional.of(timeWriter(desc)); - } - - @Override - public Optional> visit( - LogicalTypeAnnotation.TimestampLogicalTypeAnnotation timestampType) { - Preconditions.checkArgument( - LogicalTypeAnnotation.TimeUnit.MICROS.equals(timestampType.getUnit()), - "Cannot write timestamp in %s, only MICROS is supported", - timestampType.getUnit()); - return Optional.of(timestampWriter(desc, timestampType.isAdjustedToUTC())); - } - - @Override - public Optional> visit( - LogicalTypeAnnotation.IntLogicalTypeAnnotation intType) { - /* - Preconditions.checkArgument( - intType.isSigned() || intType.getBitWidth() < 64, - "Cannot read uint64: not a supported Java type"); - */ - if (intType.getBitWidth() < 64) { - return Optional.of(ParquetValueWriters.ints(desc)); - } else { - return Optional.of(ParquetValueWriters.longs(desc)); - } - } - - @Override - public Optional> visit( - LogicalTypeAnnotation.JsonLogicalTypeAnnotation jsonLogicalType) { - return Optional.of(ParquetValueWriters.strings(desc)); - } - - @Override - public Optional> visit( - LogicalTypeAnnotation.BsonLogicalTypeAnnotation bsonType) { - return Optional.of(ParquetValueWriters.byteBuffers(desc)); - } - - @Override - public Optional> visit( - LogicalTypeAnnotation.UUIDLogicalTypeAnnotation uuidLogicalType) { - return Optional.of(ParquetValueWriters.uuids(desc)); - } - } -} diff --git a/pom.xml b/pom.xml index 98b6a01..14daac7 100644 --- a/pom.xml +++ b/pom.xml @@ -14,12 +14,12 @@ 21 21 - 1.8.1 + 1.9.0 3.4.1 1.15.1 2.31.13 4.7.6 - 2.18.2 + 2.18.3 2.0.17 1.5.18