Merge pull request #530 from IyeOnline/topic/expanded-low-cardinality-support

Support writing generic LowCardinality(T)

Author: slabko

clickhouse/columns/factory.cpp

@@ -258,8 +258,16 @@ static ColumnRef CreateColumnFromAst(const TypeAst& ast, CreateColumnByTypeSetti
                                 std::make_shared<ColumnUInt8>()
                             )
                         );
-                    default:
-                        throw UnimplementedError("LowCardinality(" + nested.name + ") is not supported");
+                    default: {
+                        // Generic LowCardinality(T): build the inner column and
+                        // wrap it. Works for any fixed-size dictionary type that
+                        // AppendToDictionary supports.
+                        auto inner = CreateColumnFromAst(nested, settings);
+                        if (!inner) {
+                            throw UnimplementedError("LowCardinality(" + nested.name + ") is not supported");
+                        }
+                        return std::make_shared<ColumnLowCardinality>(std::move(inner));
+                    }
                 }
             }
         }

clickhouse/columns/lowcardinality.cpp

@@ -2,6 +2,13 @@
 
 #include "string.h"
 #include "nullable.h"
+#include "numeric.h"
+#include "enum.h"
+#include "date.h"
+#include "ip4.h"
+#include "ip6.h"
+#include "uuid.h"
+#include "../base/socket.h" // for htonl/ntohl and in_addr/in6_addr
 #include "../base/wire_format.h"
 
 #include <city.h>
@@ -10,6 +17,7 @@
 #include <string_view>
 #include <type_traits>
 #include <cmath>
+#include <cstring>
 
 #include <cassert>
 
@@ -78,30 +86,60 @@ inline ResultColumnType & column_down_cast(ColumnType & c) {
     return dynamic_cast<ResultColumnType &>(c);
 }
 
-// std::visit-ish function to avoid including <variant> header, which is not present in older version of XCode.
-template <typename Vizitor, typename ColumnType>
-inline auto VisitIndexColumn(Vizitor && vizitor, ColumnType && col) {
-    switch (col.Type()->GetCode()) {
+// Number of bytes an ItemView holds for a fixed-size dictionary type, or 0 for
+// variable-size (String/FixedString) or unsupported types. Used to build a
+// correctly-sized zero value for the default/null dictionary item.
+inline size_t FixedSizeForDictionaryType(Type::Code code) {
+    switch (code) {
+        case Type::Int8:
         case Type::UInt8:
-            return vizitor(column_down_cast<ColumnUInt8>(col));
+            return 1;
+        case Type::Int16:
         case Type::UInt16:
-            return vizitor(column_down_cast<ColumnUInt16>(col));
+        case Type::Date:
+            return 2;
+        case Type::Int32:
         case Type::UInt32:
-            return vizitor(column_down_cast<ColumnUInt32>(col));
+        case Type::Float32:
+        case Type::DateTime:
+        case Type::Date32:
+        case Type::IPv4:
+            return 4;
+        case Type::Int64:
         case Type::UInt64:
-            return vizitor(column_down_cast<ColumnUInt64>(col));
+        case Type::Float64:
+            return 8;
+        case Type::Int128:
+        case Type::UInt128:
+        case Type::IPv6:
+        case Type::UUID:
+            return 16;
         default:
-            throw ValidationError("Invalid index column type " + col.GetType().GetName());
+            return 0;
     }
 }
 
+// A zero-filled, correctly-sized ItemView for a fixed-size dictionary type. The
+// backing buffer is static so the non-owning view stays valid.
+inline ItemView ZeroItemForDictionary(Type::Code code) {
+    if (const auto size = FixedSizeForDictionaryType(code)) {
+        static const char zeros[16] = {};
+        if (size > sizeof(zeros)) {
+            throw AssertionError("The size of item view for ColumnLowCardinality exceeds the buffer size");
+        }
+        return ItemView{code, std::string_view{zeros, size}};
+    }
+    // Variable-size types (String/FixedString) accept an empty value.
+    return ItemView{code, std::string_view{}};
+}
+
 // A special NULL-item, which is expected at pos(0) in dictionary,
 // note that we distinguish empty string from NULL-value.
 inline auto GetNullItemForDictionary(const ColumnRef dictionary) {
     if (auto n = dictionary->As<ColumnNullable>()) {
         return ItemView {};
     } else {
-        return ItemView{dictionary->Type()->GetCode(), std::string_view{}};
+        return ZeroItemForDictionary(dictionary->Type()->GetCode());
     }
 }
 
@@ -111,7 +149,7 @@ inline ItemView GetDefaultItemForDictionary(const ColumnRef dictionary) {
     if (auto n = dictionary->As<ColumnNullable>()) {
         return GetDefaultItemForDictionary(n->Nested());
     } else {
-        return ItemView{dictionary->Type()->GetCode(), std::string_view{}};
+        return ZeroItemForDictionary(dictionary->Type()->GetCode());
     }
 }
 
@@ -147,6 +185,72 @@ inline void AppendToDictionary(Column& dictionary, const ItemView & item) {
         case Type::Nullable:
             AppendNullableToDictionary(column_down_cast<ColumnNullable>(dictionary), item);
             return;
+        // Fixed-size dictionary types. The ItemView holds the raw stored bytes
+        // (see the matching ColumnXxx::GetItem), so we re-append the raw value.
+        case Type::Int8:
+            column_down_cast<ColumnInt8>(dictionary).Append(item.get<int8_t>());
+            return;
+        case Type::Int16:
+            column_down_cast<ColumnInt16>(dictionary).Append(item.get<int16_t>());
+            return;
+        case Type::Int32:
+            column_down_cast<ColumnInt32>(dictionary).Append(item.get<int32_t>());
+            return;
+        case Type::Int64:
+            column_down_cast<ColumnInt64>(dictionary).Append(item.get<int64_t>());
+            return;
+        case Type::UInt8:
+            column_down_cast<ColumnUInt8>(dictionary).Append(item.get<uint8_t>());
+            return;
+        case Type::UInt16:
+            column_down_cast<ColumnUInt16>(dictionary).Append(item.get<uint16_t>());
+            return;
+        case Type::UInt32:
+            column_down_cast<ColumnUInt32>(dictionary).Append(item.get<uint32_t>());
+            return;
+        case Type::UInt64:
+            column_down_cast<ColumnUInt64>(dictionary).Append(item.get<uint64_t>());
+            return;
+        case Type::Int128:
+            column_down_cast<ColumnInt128>(dictionary).Append(item.get<Int128>());
+            return;
+        case Type::UInt128:
+            column_down_cast<ColumnUInt128>(dictionary).Append(item.get<UInt128>());
+            return;
+        case Type::Float32:
+            column_down_cast<ColumnFloat32>(dictionary).Append(item.get<float>());
+            return;
+        case Type::Float64:
+            column_down_cast<ColumnFloat64>(dictionary).Append(item.get<double>());
+            return;
+        case Type::Date:
+            column_down_cast<ColumnDate>(dictionary).AppendRaw(item.get<uint16_t>());
+            return;
+        case Type::Date32:
+            column_down_cast<ColumnDate32>(dictionary).AppendRaw(item.get<int32_t>());
+            return;
+        case Type::DateTime:
+            column_down_cast<ColumnDateTime>(dictionary).AppendRaw(item.get<uint32_t>());
+            return;
+        case Type::IPv4:
+            // ColumnIPv4::Append applies htonl, and GetItem returns the stored
+            // (already byte-swapped) value, so undo the swap to re-store as-is.
+            column_down_cast<ColumnIPv4>(dictionary).Append(ntohl(item.get<uint32_t>()));
+            return;
+        case Type::IPv6: {
+            in6_addr addr;
+            std::memcpy(&addr, item.data.data(), sizeof(addr));
+            column_down_cast<ColumnIPv6>(dictionary).Append(addr);
+            return;
+        }
+        case Type::UUID: {
+            UUID value;
+            std::memcpy(&value.first, item.data.data(), sizeof(value.first));
+            std::memcpy(&value.second, item.data.data() + sizeof(value.first),
+                        sizeof(value.second));
+            column_down_cast<ColumnUUID>(dictionary).Append(value);
+            return;
+        }
         default:
             throw ValidationError("Unexpected dictionary column type: " + dictionary.GetType().GetName());
     }

clickhouse/columns/uuid.h

@@ -12,6 +12,8 @@ namespace clickhouse {
  */
 class ColumnUUID : public Column {
 public:
+    using ValueType = UUID;
+
     ColumnUUID();
 
     explicit ColumnUUID(ColumnRef data);

ut/BUILD.bazel

@@ -105,6 +105,7 @@ cc_test(
         "connection_failed_client_test.cpp",
         "connection_failed_client_test.h",
         "low_cardinality_nullable_tests.cpp",
+        "low_cardinality_types_ut.cpp",
         "performance_tests.cpp",
         "readonly_client_test.cpp",
         "readonly_client_test.h",

ut/CMakeLists.txt

@@ -10,6 +10,7 @@ SET ( clickhouse-cpp-ut-src
     column_as_ut.cpp
     column_array_ut.cpp
     itemview_ut.cpp
+    low_cardinality_types_ut.cpp
     socket_ut.cpp
     stream_ut.cpp
     type_parser_ut.cpp

ut/CreateColumnByType_ut.cpp

@@ -75,6 +75,28 @@ TEST(CreateColumnByType, LowCardinalityAsWrappedColumn) {
     ASSERT_EQ(Type::FixedString, CreateColumnByType("LowCardinality(FixedString(10000))", create_column_settings)->As<ColumnFixedString>()->GetType().GetCode());
 }
 
+TEST(CreateColumnByType, LowCardinalityGeneralInnerTypes) {
+    // LowCardinality used to be supported only over String/FixedString. The
+    // factory now builds a generic ColumnLowCardinality for any fixed-size inner
+    // type.
+    for (const auto* type_name : {
+             "LowCardinality(Int8)",
+             "LowCardinality(Int64)",
+             "LowCardinality(UInt64)",
+             "LowCardinality(Float64)",
+             "LowCardinality(Date)",
+             "LowCardinality(DateTime)",
+             "LowCardinality(Nullable(Int64))",
+             "LowCardinality(Nullable(Float64))",
+         }) {
+        auto col = CreateColumnByType(type_name);
+        ASSERT_NE(nullptr, col) << type_name;
+        ASSERT_EQ(Type::LowCardinality, col->GetType().GetCode()) << type_name;
+        ASSERT_NE(nullptr, col->As<ColumnLowCardinality>()) << type_name;
+        EXPECT_EQ(std::string{type_name}, col->GetType().GetName()) << type_name;
+    }
+}
+
 TEST(CreateColumnByType, LowCardinality) {
     // In the default (non-wrapped) mode, LowCardinality(String)/LowCardinality(FixedString) map to
     // the base ColumnLowCardinality (like Array/Nullable/Tuple/Map do), and the strongly-typed

ut/columns_ut.cpp

@@ -1993,6 +1993,76 @@ TEST(ColumnsCase, ColumnMapT_Wrap) {
     EXPECT_EQ("abc", map_view.At(2));
 }
 
+// Regression tests for general LowCardinality support over non-String inner
+// types (previously only String/FixedString were supported).
+TEST(ColumnLowCardinality, AppendAndReadNumeric) {
+    auto col = std::make_shared<ColumnLowCardinalityT<ColumnInt64>>();
+    col->Append(7);
+    col->Append(7);
+    col->Append(9);
+    col->Append(7);
+
+    ASSERT_EQ(4u, col->Size());
+    EXPECT_EQ(7, col->At(0));
+    EXPECT_EQ(7, col->At(1));
+    EXPECT_EQ(9, col->At(2));
+    EXPECT_EQ(7, col->At(3));
+    // Dictionary holds the default item plus the two distinct values {7, 9}.
+    EXPECT_EQ(3u, col->GetDictionarySize());
+
+    // GetItem returns the raw value with the correct type code.
+    const auto item = col->GetItem(2);
+    EXPECT_EQ(Type::Int64, item.type);
+    EXPECT_EQ(9, item.get<int64_t>());
+}
+
+TEST(ColumnLowCardinality, AppendAndReadNullableNumeric) {
+    auto col
+        = std::make_shared<ColumnLowCardinalityT<ColumnNullableT<ColumnInt64>>>();
+    col->Append(7);
+    col->Append(std::nullopt);
+    col->Append(7);
+    col->Append(9);
+    col->Append(std::nullopt);
+
+    ASSERT_EQ(5u, col->Size());
+    EXPECT_EQ(std::optional<int64_t>{7}, col->At(0));
+    EXPECT_EQ(std::nullopt, col->At(1));
+    EXPECT_EQ(std::optional<int64_t>{7}, col->At(2));
+    EXPECT_EQ(std::optional<int64_t>{9}, col->At(3));
+    EXPECT_EQ(std::nullopt, col->At(4));
+
+    // Null rows are represented by a Void ItemView.
+    EXPECT_EQ(Type::Void, col->GetItem(1).type);
+    EXPECT_EQ(Type::Int64, col->GetItem(0).type);
+}
+
+TEST(ColumnLowCardinality, NumericLoadAndSave) {
+    auto column_A = std::make_shared<ColumnLowCardinalityT<ColumnUInt64>>();
+    for (auto v : {1u, 2u, 1u, 3u, 2u, 1u}) {
+        column_A->Append(v);
+    }
+
+    const auto BufferSize = 64 * 1024;
+    std::unique_ptr<char[]> buffer = std::make_unique<char[]>(BufferSize);
+    memset(buffer.get(), 0, BufferSize);
+    {
+        ArrayOutput output(buffer.get(), BufferSize);
+        ASSERT_NO_THROW(column_A->Save(&output));
+    }
+
+    auto column_B = std::make_shared<ColumnLowCardinalityT<ColumnUInt64>>();
+    {
+        ArrayInput input(buffer.get(), BufferSize);
+        ASSERT_TRUE(column_B->Load(&input, column_A->Size()));
+    }
+
+    ASSERT_EQ(column_A->Size(), column_B->Size());
+    for (size_t i = 0; i < column_A->Size(); ++i) {
+        EXPECT_EQ(column_A->At(i), column_B->At(i)) << "row " << i;
+    }
+}
+
 TEST(ColumnsCase, ColumnMapT_Wrap_AcceptsLvalue) {
     auto tupls = std::make_shared<ColumnTuple>(std::vector<ColumnRef>{
             std::make_shared<ColumnUInt64>(),