to 3.0: refactor(objectio): keep zonemap for hidden rowid and ts columns

pkg/objectio/ioutil/write_index.go

@@ -17,6 +17,7 @@ package ioutil
 import (
 	hll "github.com/axiomhq/hyperloglog"
 	"github.com/cespare/xxhash/v2"
+	"github.com/matrixorigin/matrixone/pkg/container/types"
 	"github.com/matrixorigin/matrixone/pkg/objectio"
 	"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/containers"
 	"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/index"
@@ -85,6 +86,9 @@ func (b *ObjectColumnMetasBuilder) UpdateZm(idx int, zm index.ZM) {
 	if !zm.IsInited() {
 		return
 	}
+	if b.zms[idx] == nil {
+		b.zms[idx] = index.NewZM(types.T(zm.GetType()), zm.GetScale())
+	}
 	index.UpdateZM(b.zms[idx], zm.GetMinBuf())
 	index.UpdateZM(b.zms[idx], zm.GetMaxBuf())
 	if zm.IsString() && zm.MaxTruncated() {

pkg/objectio/ioutil/write_index_test.go

@@ -0,0 +1,110 @@
+// Copyright 2021 Matrix Origin
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ioutil
+
+import (
+	"context"
+	"testing"
+
+	"github.com/matrixorigin/matrixone/pkg/common/mpool"
+	"github.com/matrixorigin/matrixone/pkg/container/batch"
+	"github.com/matrixorigin/matrixone/pkg/container/types"
+	"github.com/matrixorigin/matrixone/pkg/container/vector"
+	"github.com/matrixorigin/matrixone/pkg/objectio"
+	"github.com/matrixorigin/matrixone/pkg/testutil"
+	"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/index"
+	"github.com/stretchr/testify/require"
+)
+
+func TestObjectColumnMetasBuilderUpdateZmWithoutInspect(t *testing.T) {
+	mp := mpool.MustNewZero()
+	vec := vector.NewVec(types.T_TS.ToType())
+	for i := 0; i < 4; i++ {
+		require.NoError(t, vector.AppendFixed(vec, types.BuildTS(int64(i+1), 0), false, mp))
+	}
+
+	builder := NewObjectColumnMetasBuilder(1)
+	zm := index.NewZM(vec.GetType().Oid, vec.GetType().Scale)
+	require.NoError(t, index.BatchUpdateZM(zm, vec))
+	index.SetZMSum(zm, vec)
+	builder.UpdateZm(0, zm)
+
+	_, metas := builder.Build()
+	require.True(t, metas[0].ZoneMap().IsInited())
+	require.Zero(t, metas[0].Ndv())
+	require.Zero(t, metas[0].NullCnt())
+}
+
+func TestHiddenColumnsZoneMapPersistedAfterWriteAndRead(t *testing.T) {
+	ctx := context.Background()
+	fs := testutil.NewSharedFS()
+	mp := mpool.MustNewZero()
+
+	writer := ConstructWriter(0, []uint16{0, objectio.SEQNUM_ROWID, objectio.SEQNUM_COMMITTS}, -1, false, false, fs)
+	bat := batchForHiddenColumnsTest(t, mp)
+	blocks, _, err := func() ([]objectio.BlockObject, objectio.Extent, error) {
+		_, err := writer.WriteBatch(bat)
+		require.NoError(t, err)
+		return writer.Sync(ctx)
+	}()
+	require.NoError(t, err)
+	require.Len(t, blocks, 1)
+
+	loc := objectio.BuildLocation(writer.GetName(), blocks[0].GetExtent(), uint32(bat.RowCount()), blocks[0].GetID())
+	reader, err := NewObjectReader(fs, loc)
+	require.NoError(t, err)
+	meta, err := reader.LoadObjectMeta(ctx, mp)
+	require.NoError(t, err)
+
+	expectedMinRowid := types.NewRowid(new(types.Blockid), 1)
+	expectedMaxRowid := types.NewRowid(new(types.Blockid), 9)
+	expectedMinTS := types.BuildTS(10, 0)
+	expectedMaxTS := types.BuildTS(30, 0)
+
+	rowidBlkZM := meta.GetColumnMeta(0, 1).ZoneMap()
+	require.True(t, rowidBlkZM.IsInited())
+	require.Equal(t, expectedMinRowid, types.DecodeFixed[types.Rowid](rowidBlkZM.GetMinBuf()))
+	require.Equal(t, expectedMaxRowid, types.DecodeFixed[types.Rowid](rowidBlkZM.GetMaxBuf()))
+
+	tsBlkZM := meta.GetColumnMeta(0, 2).ZoneMap()
+	require.True(t, tsBlkZM.IsInited())
+	require.Equal(t, expectedMinTS, types.DecodeFixed[types.TS](tsBlkZM.GetMinBuf()))
+	require.Equal(t, expectedMaxTS, types.DecodeFixed[types.TS](tsBlkZM.GetMaxBuf()))
+}
+
+func batchForHiddenColumnsTest(t *testing.T, mp *mpool.MPool) *batch.Batch {
+	t.Helper()
+	bat := batch.NewWithSize(3)
+	bat.Vecs[0] = vector.NewVec(types.T_int32.ToType())
+	bat.Vecs[1] = vector.NewVec(types.T_Rowid.ToType())
+	bat.Vecs[2] = vector.NewVec(types.T_TS.ToType())
+
+	values := []int32{11, 22, 33}
+	rowOffsets := []uint32{7, 1, 9}
+	commitTS := []types.TS{
+		types.BuildTS(20, 0),
+		types.BuildTS(10, 0),
+		types.BuildTS(30, 0),
+	}
+
+	var blk types.Blockid
+	for i := range values {
+		require.NoError(t, vector.AppendFixed(bat.Vecs[0], values[i], false, mp))
+		require.NoError(t, vector.AppendFixed(bat.Vecs[1], types.NewRowid(&blk, rowOffsets[i]), false, mp))
+		require.NoError(t, vector.AppendFixed(bat.Vecs[2], commitTS[i], false, mp))
+	}
+	bat.SetRowCount(len(values))
+	return bat
+}

pkg/objectio/ioutil/writer.go

@@ -233,20 +233,17 @@ func (w *BlockWriter) WriteBatch(batch *batch.Batch) (objectio.BlockObject, erro
 			w.objMetaBuilder.AddRowCnt(vec.Length())
 		}
 
-		// PXU TODO: change this logic
-		if !w.isTombstone {
-			// only skip SchemaData type
-			if vec.GetType().Oid == types.T_Rowid || vec.GetType().Oid == types.T_TS {
-				continue
-			}
-		}
+		zmOnlyHiddenColumn := !w.isTombstone &&
+			(vec.GetType().Oid == types.T_Rowid || vec.GetType().Oid == types.T_TS)
 
 		if w.isSetPK && w.pk == uint16(i) {
 			isPK = true
 		}
 		columnData := containers.ToTNVector(vec, common.DefaultAllocator)
-		// update null count and distinct value
-		w.objMetaBuilder.InspectVector(i, columnData, isPK)
+		if !zmOnlyHiddenColumn {
+			// update null count and distinct value
+			w.objMetaBuilder.InspectVector(i, columnData, isPK)
+		}
 
 		// Build ZM
 		zm := index.NewZM(vec.GetType().Oid, vec.GetType().Scale)
@@ -259,6 +256,9 @@ func (w *BlockWriter) WriteBatch(batch *batch.Batch) (objectio.BlockObject, erro
 		w.writer.UpdateBlockZM(objectio.SchemaData, int(block.GetID()), seqnums[i], zm)
 		// update object zonemap
 		w.objMetaBuilder.UpdateZm(i, zm)
+		if zmOnlyHiddenColumn {
+			continue
+		}
 
 		if !w.isSetPK || w.pk != uint16(i) {
 			continue

pkg/objectio/ioutil/writer_bench_test.go

@@ -0,0 +1,106 @@
+// Copyright 2021 Matrix Origin
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ioutil
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/matrixorigin/matrixone/pkg/common/mpool"
+	"github.com/matrixorigin/matrixone/pkg/container/types"
+	"github.com/matrixorigin/matrixone/pkg/container/vector"
+	"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/common"
+	"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/containers"
+	"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/index"
+)
+
+func BenchmarkHiddenColumnOverhead(b *testing.B) {
+	benchmarkHiddenColumnOverheadRows(b, 8192)
+}
+
+func benchmarkHiddenColumnOverheadRows(b *testing.B, rows int) {
+	b.Run(fmt.Sprintf("rows=%d", rows), func(b *testing.B) {
+		for _, tc := range []struct {
+			name string
+			vec  *vector.Vector
+		}{
+			{name: "rowid", vec: buildBenchmarkRowIDVector(b, rows)},
+			{name: "ts", vec: buildBenchmarkTSVector(b, rows)},
+		} {
+			columnData := containers.ToTNVector(tc.vec, common.DefaultAllocator)
+			b.Run(tc.name+"/inspect", func(b *testing.B) {
+				b.ReportAllocs()
+				b.ResetTimer()
+				for i := 0; i < b.N; i++ {
+					builder := NewObjectColumnMetasBuilder(1)
+					builder.InspectVector(0, columnData, false)
+				}
+			})
+			b.Run(tc.name+"/zm", func(b *testing.B) {
+				b.ReportAllocs()
+				b.ResetTimer()
+				for i := 0; i < b.N; i++ {
+					zm := index.NewZM(tc.vec.GetType().Oid, tc.vec.GetType().Scale)
+					if err := index.BatchUpdateZM(zm, columnData.GetDownstreamVector()); err != nil {
+						b.Fatal(err)
+					}
+					index.SetZMSum(zm, columnData.GetDownstreamVector())
+				}
+			})
+			b.Run(tc.name+"/full", func(b *testing.B) {
+				b.ReportAllocs()
+				b.ResetTimer()
+				for i := 0; i < b.N; i++ {
+					builder := NewObjectColumnMetasBuilder(1)
+					builder.InspectVector(0, columnData, false)
+					zm := index.NewZM(tc.vec.GetType().Oid, tc.vec.GetType().Scale)
+					if err := index.BatchUpdateZM(zm, columnData.GetDownstreamVector()); err != nil {
+						b.Fatal(err)
+					}
+					index.SetZMSum(zm, columnData.GetDownstreamVector())
+					builder.UpdateZm(0, zm)
+					_, _ = builder.Build()
+				}
+			})
+		}
+	})
+}
+
+func buildBenchmarkRowIDVector(b *testing.B, rows int) *vector.Vector {
+	b.Helper()
+	mp := mpool.MustNewZero()
+	vec := vector.NewVec(types.T_Rowid.ToType())
+	var blk types.Blockid
+	for i := 0; i < rows; i++ {
+		rowid := types.NewRowid(&blk, uint32(i))
+		if err := vector.AppendFixed(vec, rowid, false, mp); err != nil {
+			b.Fatal(err)
+		}
+	}
+	return vec
+}
+
+func buildBenchmarkTSVector(b *testing.B, rows int) *vector.Vector {
+	b.Helper()
+	mp := mpool.MustNewZero()
+	vec := vector.NewVec(types.T_TS.ToType())
+	for i := 0; i < rows; i++ {
+		ts := types.BuildTS(int64(i+1), 0)
+		if err := vector.AppendFixed(vec, ts, false, mp); err != nil {
+			b.Fatal(err)
+		}
+	}
+	return vec
+}