Skip to content

Commit d42f4b0

Browse files
tangentaexpxiaoli
tangenta
authored and
expxiaoli
committed
ddl: support pre-split index regions before creating index (pingcap#57553)
close pingcap#57551, close pingcap#57552
1 parent 3e70ace commit d42f4b0

20 files changed

Lines changed: 7151 additions & 6342 deletions

‎pkg/ddl/BUILD.bazel‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ go_library(
4343
"index_cop.go",
4444
"index_merge_tmp.go",
4545
"job_table.go",
46+
"index_presplit.go",
4647
"mock.go",
4748
"multi_schema_change.go",
4849
"options.go",
@@ -136,6 +137,7 @@ go_library(
136137
"//pkg/util/collate",
137138
"//pkg/util/dbterror",
138139
"//pkg/util/dbterror/exeerrors",
140+
"//pkg/util/dbterror/plannererrors",
139141
"//pkg/util/domainutil",
140142
"//pkg/util/engine",
141143
"//pkg/util/filter",

‎pkg/ddl/backfilling.go‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -213,7 +213,6 @@ type reorgBackfillTask struct {
213213
startKey kv.Key
214214
endKey kv.Key
215215
jobID int64
216-
sqlQuery string
217216
priority int
218217
}
219218

‎pkg/ddl/ddl_api.go‎

Lines changed: 81 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,75 @@ import (
7878
"go.uber.org/zap"
7979
)
8080

81+
type indexPresplitOpt struct {
82+
Lower []string
83+
Upper []string
84+
Num int64
85+
ValueLists [][]string
86+
}
87+
88+
func buildIndexPresplitOpt(indexOpt *ast.IndexOption) (*indexPresplitOpt, error) {
89+
if indexOpt == nil {
90+
return nil, nil
91+
}
92+
opt := indexOpt.SplitOpt
93+
if opt == nil {
94+
return nil, nil
95+
}
96+
if len(opt.ValueLists) > 0 {
97+
valLists := make([][]string, 0, len(opt.ValueLists))
98+
for _, lst := range opt.ValueLists {
99+
values := make([]string, 0, len(lst))
100+
for _, exp := range lst {
101+
var sb strings.Builder
102+
rCtx := format.NewRestoreCtx(format.DefaultRestoreFlags, &sb)
103+
err := exp.Restore(rCtx)
104+
if err != nil {
105+
return nil, errors.Trace(err)
106+
}
107+
values = append(values, sb.String())
108+
}
109+
valLists = append(valLists, values)
110+
}
111+
return &indexPresplitOpt{
112+
Num: opt.Num,
113+
ValueLists: valLists,
114+
}, nil
115+
}
116+
117+
lowers := make([]string, 0, len(opt.Lower))
118+
for _, expL := range opt.Lower {
119+
var sb strings.Builder
120+
rCtx := format.NewRestoreCtx(format.DefaultRestoreFlags, &sb)
121+
err := expL.Restore(rCtx)
122+
if err != nil {
123+
return nil, errors.Trace(err)
124+
}
125+
lowers = append(lowers, sb.String())
126+
}
127+
uppers := make([]string, 0, len(opt.Upper))
128+
for _, expU := range opt.Upper {
129+
var sb strings.Builder
130+
rCtx := format.NewRestoreCtx(format.DefaultRestoreFlags, &sb)
131+
err := expU.Restore(rCtx)
132+
if err != nil {
133+
return nil, errors.Trace(err)
134+
}
135+
uppers = append(uppers, sb.String())
136+
}
137+
maxSplitRegionNum := int64(config.GetGlobalConfig().SplitRegionMaxNum)
138+
if opt.Num > maxSplitRegionNum {
139+
return nil, errors.Errorf("Split index region num exceeded the limit %v", maxSplitRegionNum)
140+
} else if opt.Num < 1 {
141+
return nil, errors.Errorf("Split index region num should be greater than 0")
142+
}
143+
return &indexPresplitOpt{
144+
Lower: lowers,
145+
Upper: uppers,
146+
Num: opt.Num,
147+
}, nil
148+
}
149+
81150
const (
82151
expressionIndexPrefix = "_V$"
83152
changingColumnPrefix = "_Col$_"
@@ -7472,6 +7541,11 @@ func (d *ddl) CreatePrimaryKey(ctx sessionctx.Context, ti ast.Ident, indexName m
74727541
}
74737542
}
74747543

7544+
splitOpt, err := buildIndexPresplitOpt(indexOption)
7545+
if err != nil {
7546+
return errors.Trace(err)
7547+
}
7548+
74757549
unique := true
74767550
sqlMode := ctx.GetSessionVars().SQLMode
74777551
job := &model.Job{
@@ -7482,7 +7556,7 @@ func (d *ddl) CreatePrimaryKey(ctx sessionctx.Context, ti ast.Ident, indexName m
74827556
Type: model.ActionAddPrimaryKey,
74837557
BinlogInfo: &model.HistoryInfo{},
74847558
ReorgMeta: nil,
7485-
Args: []any{unique, indexName, indexPartSpecifications, indexOption, sqlMode, nil, global},
7559+
Args: []any{unique, indexName, indexPartSpecifications, indexOption, sqlMode, nil, global, splitOpt},
74867560
Priority: ctx.GetSessionVars().DDLReorgPriority,
74877561
CDCWriteSource: ctx.GetSessionVars().CDCWriteSource,
74887562
SQLMode: ctx.GetSessionVars().SQLMode,
@@ -7725,6 +7799,11 @@ func (d *ddl) createIndex(ctx sessionctx.Context, ti ast.Ident, keyType ast.Inde
77257799
}
77267800
}
77277801

7802+
splitOpt, err := buildIndexPresplitOpt(indexOption)
7803+
if err != nil {
7804+
return errors.Trace(err)
7805+
}
7806+
77287807
if indexOption != nil && indexOption.Tp == model.IndexTypeHypo { // for hypo-index
77297808
indexInfo, err := BuildIndexInfo(ctx, tblInfo.Columns, indexName, false, unique, global,
77307809
indexPartSpecifications, indexOption, model.StatePublic)
@@ -7743,7 +7822,7 @@ func (d *ddl) createIndex(ctx sessionctx.Context, ti ast.Ident, keyType ast.Inde
77437822
Type: model.ActionAddIndex,
77447823
BinlogInfo: &model.HistoryInfo{},
77457824
ReorgMeta: nil,
7746-
Args: []any{unique, indexName, indexPartSpecifications, indexOption, hiddenCols, global},
7825+
Args: []any{unique, indexName, indexPartSpecifications, indexOption, hiddenCols, global, splitOpt},
77477826
Priority: ctx.GetSessionVars().DDLReorgPriority,
77487827
Charset: chs,
77497828
Collate: coll,

‎pkg/ddl/index.go‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -565,15 +565,25 @@ func (w *worker) onCreateIndex(d *ddlCtx, t *meta.Meta, job *model.Job, isPK boo
565565
indexNames := make([]model.CIStr, 1)
566566
indexPartSpecifications := make([][]*ast.IndexPartSpecification, 1)
567567
indexOption := make([]*ast.IndexOption, 1)
568+
splitOpt := make([]*indexPresplitOpt, 1)
568569
var sqlMode mysql.SQLMode
569570
var warnings []string
570571
hiddenCols := make([][]*model.ColumnInfo, 1)
571572

572573
if isPK {
573574
// Notice: sqlMode and warnings is used to support non-strict mode.
574-
err = job.DecodeArgs(&unique[0], &indexNames[0], &indexPartSpecifications[0], &indexOption[0], &sqlMode, &warnings, &global[0])
575+
err = job.DecodeArgs(&unique[0], &indexNames[0], &indexPartSpecifications[0], &indexOption[0], &sqlMode, &warnings, &global[0], &splitOpt[0])
576+
if err != nil {
577+
err = job.DecodeArgs(&unique[0], &indexNames[0], &indexPartSpecifications[0], &indexOption[0], &sqlMode, &warnings, &global[0])
578+
}
575579
} else {
576-
err = job.DecodeArgs(&unique[0], &indexNames[0], &indexPartSpecifications[0], &indexOption[0], &hiddenCols[0], &global[0])
580+
err = job.DecodeArgs(&unique[0], &indexNames[0], &indexPartSpecifications[0], &indexOption[0], &hiddenCols[0], &global[0], &splitOpt[0])
581+
if err != nil {
582+
err = job.DecodeArgs(&unique[0], &indexNames[0], &indexPartSpecifications[0], &indexOption[0], &hiddenCols[0], &global[0])
583+
}
584+
if err != nil {
585+
err = job.DecodeArgs(&unique, &indexNames, &indexPartSpecifications, &indexOption, &hiddenCols, &global, &splitOpt)
586+
}
577587
if err != nil {
578588
err = job.DecodeArgs(&unique, &indexNames, &indexPartSpecifications, &indexOption, &hiddenCols, &global)
579589
}
@@ -674,6 +684,13 @@ SwitchIndexState:
674684
indexInfo.BackfillState = model.BackfillStateRunning
675685
}
676686
}
687+
err = preSplitIndexRegions(w.ctx, w.sess.Context, d.store, tblInfo, allIndexInfos, job.ReorgMeta, indexPartSpecifications, splitOpt)
688+
if err != nil {
689+
if !errorIsRetryable(err, job) {
690+
job.State = model.JobStateCancelled
691+
}
692+
return ver, err
693+
}
677694
for _, indexInfo := range allIndexInfos {
678695
indexInfo.State = model.StateDeleteOnly
679696
moveAndUpdateHiddenColumnsToPublic(tblInfo, indexInfo)

0 commit comments

Comments
 (0)