Skip to content

Commit 599d1ce

Browse files
yumkamyumkam
authored
Merge pull request #348 from buhtr/feature/YQ-4193-add-between
PostgreSQL: pushdown `BETWEEN`
2 parents be7cd9f + a5f7247 commit 599d1ce

22 files changed

Lines changed: 464 additions & 80 deletions

File tree

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ __pycache__
1212
*.test
1313
.vscode
1414
.idea
15+
.tmp
1516
.DS_Store
1617

1718
.ipynb_checkpoints

‎app/server/datasource/rdbms/clickhouse/sql_formatter.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package clickhouse
22

33
import (
4+
"fmt"
45
"strings"
56

67
"github.com/ydb-platform/ydb-go-genproto/protos/Ydb"
@@ -102,6 +103,10 @@ func (f sqlFormatter) FormatFrom(tableName string) string {
102103
return f.SanitiseIdentifier(tableName)
103104
}
104105

106+
func (sqlFormatter) RenderBetween(value, least, greatest string) (string, error) {
107+
return fmt.Sprintf("%s BETWEEN %s AND %s", value, least, greatest), nil
108+
}
109+
105110
func NewSQLFormatter(cfg *config.TPushdownConfig) rdbms_utils.SQLFormatter {
106111
return sqlFormatter{cfg: cfg}
107112
}

‎app/server/datasource/rdbms/clickhouse/sql_formatter_test.go‎

Lines changed: 27 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,33 @@ func TestMakeSelectQuery(t *testing.T) {
138138
outputYdbTypes: []*ydb.Type{common.MakePrimitiveType(ydb.Type_INT32), common.MakePrimitiveType(ydb.Type_STRING)},
139139
err: nil,
140140
},
141+
{
142+
testName: "between",
143+
selectReq: &api_service_protos.TSelect{
144+
From: &api_service_protos.TSelect_TFrom{
145+
Table: "tab",
146+
},
147+
What: rdbms_utils.NewDefaultWhat(),
148+
Where: &api_service_protos.TSelect_TWhere{
149+
FilterTyped: &api_service_protos.TPredicate{
150+
Payload: &api_service_protos.TPredicate_Between{
151+
Between: &api_service_protos.TPredicate_TBetween{
152+
Value: rdbms_utils.NewColumnExpression("col2"),
153+
Least: rdbms_utils.NewColumnExpression("col1"),
154+
Greatest: rdbms_utils.NewColumnExpression("col3"),
155+
},
156+
},
157+
},
158+
},
159+
DataSourceInstance: &api_common.TGenericDataSourceInstance{
160+
Kind: api_common.EGenericDataSourceKind_CLICKHOUSE,
161+
},
162+
},
163+
outputQuery: `SELECT "col0", "col1" FROM "tab" WHERE "col2" BETWEEN "col1" AND "col3"`,
164+
outputArgs: []any{},
165+
outputYdbTypes: []*ydb.Type{common.MakePrimitiveType(ydb.Type_INT32), common.MakePrimitiveType(ydb.Type_STRING)},
166+
err: nil,
167+
},
141168
{
142169
testName: "bool_column",
143170
selectReq: &api_service_protos.TSelect{
@@ -228,33 +255,6 @@ func TestMakeSelectQuery(t *testing.T) {
228255
outputYdbTypes: []*ydb.Type{common.MakePrimitiveType(ydb.Type_INT32), common.MakePrimitiveType(ydb.Type_STRING)},
229256
err: nil,
230257
},
231-
{
232-
testName: "unsupported_predicate",
233-
selectReq: &api_service_protos.TSelect{
234-
From: &api_service_protos.TSelect_TFrom{
235-
Table: "tab",
236-
},
237-
What: rdbms_utils.NewDefaultWhat(),
238-
Where: &api_service_protos.TSelect_TWhere{
239-
FilterTyped: &api_service_protos.TPredicate{
240-
Payload: &api_service_protos.TPredicate_Between{
241-
Between: &api_service_protos.TPredicate_TBetween{
242-
Value: rdbms_utils.NewColumnExpression("col2"),
243-
Least: rdbms_utils.NewColumnExpression("col1"),
244-
Greatest: rdbms_utils.NewColumnExpression("col3"),
245-
},
246-
},
247-
},
248-
},
249-
DataSourceInstance: &api_common.TGenericDataSourceInstance{
250-
Kind: api_common.EGenericDataSourceKind_CLICKHOUSE,
251-
},
252-
},
253-
outputQuery: `SELECT "col0", "col1" FROM "tab"`,
254-
outputArgs: []any{},
255-
outputYdbTypes: []*ydb.Type{common.MakePrimitiveType(ydb.Type_INT32), common.MakePrimitiveType(ydb.Type_STRING)},
256-
err: nil,
257-
},
258258
{
259259
testName: "unsupported_type",
260260
selectReq: &api_service_protos.TSelect{

‎app/server/datasource/rdbms/ms_sql_server/sql_formatter.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,10 @@ func (f sqlFormatter) FormatFrom(tableName string) string {
9898
return f.SanitiseIdentifier(tableName)
9999
}
100100

101+
func (sqlFormatter) RenderBetween(value, least, greatest string) (string, error) {
102+
return fmt.Sprintf("%s BETWEEN %s AND %s", value, least, greatest), nil
103+
}
104+
101105
func NewSQLFormatter(cfg *config.TPushdownConfig) rdbms_utils.SQLFormatter {
102106
return sqlFormatter{cfg: cfg}
103107
}

‎app/server/datasource/rdbms/mysql/connection.go‎

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
package mysql
22

33
import (
4+
"errors"
45
"fmt"
6+
"reflect"
57
"sync/atomic"
68
"time"
79

@@ -25,7 +27,7 @@ type connection struct {
2527
tableName string
2628
}
2729

28-
func transformArgs(src *rdbms_utils.QueryArgs) []any {
30+
func transformArgs(src *rdbms_utils.QueryArgs) ([]any, error) {
2931
dst := make([]any, len(src.Values()))
3032

3133
for i, v := range src.Values() {
@@ -35,11 +37,21 @@ func transformArgs(src *rdbms_utils.QueryArgs) []any {
3537
// TODO: check if time.RFC3339 (without Nano) would be enough
3638
dst[i] = t.Format(time.RFC3339Nano)
3739
default:
38-
dst[i] = v
40+
rv := reflect.ValueOf(v)
41+
42+
if rv.Kind() == reflect.Ptr {
43+
if rv.IsNil() {
44+
return dst, errors.New("nil pointer does not supported")
45+
}
46+
47+
dst[i] = rv.Elem().Interface()
48+
} else {
49+
dst[i] = v
50+
}
3951
}
4052
}
4153

42-
return dst
54+
return dst, nil
4355
}
4456

4557
func (c *connection) Query(params *rdbms_utils.QueryParams) (*rdbms_utils.QueryResult, error) {
@@ -65,6 +77,11 @@ func (c *connection) Query(params *rdbms_utils.QueryParams) (*rdbms_utils.QueryR
6577
return &rdbms_utils.QueryResult{Rows: r}, fmt.Errorf("mysql: failed to prepare query: %w", err)
6678
}
6779

80+
args, err := transformArgs(params.QueryArgs)
81+
if err != nil {
82+
return nil, fmt.Errorf("transform args: %w", err)
83+
}
84+
6885
go func() {
6986
defer close(r.rowChan)
7087
defer close(r.errChan)
@@ -102,7 +119,7 @@ func (c *connection) Query(params *rdbms_utils.QueryParams) (*rdbms_utils.QueryR
102119
return nil
103120
},
104121
nil,
105-
transformArgs(params.QueryArgs)...,
122+
args...,
106123
)
107124
}()
108125

‎app/server/datasource/rdbms/mysql/sql_formatter.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,10 @@ func (f sqlFormatter) FormatFrom(tableName string) string {
9292
return f.SanitiseIdentifier(tableName)
9393
}
9494

95+
func (sqlFormatter) RenderBetween(value, least, greatest string) (string, error) {
96+
return fmt.Sprintf("%s BETWEEN %s AND %s", value, least, greatest), nil
97+
}
98+
9599
func NewSQLFormatter(cfg *config.TPushdownConfig) rdbms_utils.SQLFormatter {
96100
return sqlFormatter{cfg: cfg}
97101
}

‎app/server/datasource/rdbms/oracle/sql_formatter.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,10 @@ func (f sqlFormatter) FormatFrom(tableName string) string {
9898
return f.SanitiseIdentifier(tableName)
9999
}
100100

101+
func (sqlFormatter) RenderBetween(value, least, greatest string) (string, error) {
102+
return fmt.Sprintf("%s BETWEEN %s AND %s", value, least, greatest), nil
103+
}
104+
101105
func NewSQLFormatter(cfg *config.TPushdownConfig) rdbms_utils.SQLFormatter {
102106
return sqlFormatter{cfg: cfg}
103107
}

‎app/server/datasource/rdbms/postgresql/sql_formatter.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -248,6 +248,10 @@ func (f sqlFormatter) renderSelectQueryTextWithBoundsHelper(
248248
return sb.String(), nil
249249
}
250250

251+
func (sqlFormatter) RenderBetween(value, least, greatest string) (string, error) {
252+
return fmt.Sprintf("%s BETWEEN %s AND %s", value, least, greatest), nil
253+
}
254+
251255
func NewSQLFormatter(cfg *config.TPushdownConfig) rdbms_utils.SQLFormatter {
252256
return sqlFormatter{cfg: cfg}
253257
}

‎app/server/datasource/rdbms/postgresql/sql_formatter_test.go‎

Lines changed: 28 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,34 @@ func TestMakeSelectQuery(t *testing.T) {
146146
outputYdbTypes: []*ydb.Type{common.MakePrimitiveType(ydb.Type_INT32), common.MakePrimitiveType(ydb.Type_STRING)},
147147
err: nil,
148148
},
149+
{
150+
testName: "between",
151+
selectReq: &api_service_protos.TSelect{
152+
From: &api_service_protos.TSelect_TFrom{
153+
Table: "tab",
154+
},
155+
What: rdbms_utils.NewDefaultWhat(),
156+
Where: &api_service_protos.TSelect_TWhere{
157+
FilterTyped: &api_service_protos.TPredicate{
158+
Payload: &api_service_protos.TPredicate_Between{
159+
Between: &api_service_protos.TPredicate_TBetween{
160+
Value: rdbms_utils.NewColumnExpression("col2"),
161+
Least: rdbms_utils.NewColumnExpression("col1"),
162+
Greatest: rdbms_utils.NewColumnExpression("col3"),
163+
},
164+
},
165+
},
166+
},
167+
DataSourceInstance: &api_common.TGenericDataSourceInstance{
168+
Kind: api_common.EGenericDataSourceKind_POSTGRESQL,
169+
},
170+
},
171+
splitDescription: singleSplit,
172+
outputQuery: `SELECT "col0", "col1" FROM "tab" WHERE "col2" BETWEEN "col1" AND "col3"`,
173+
outputArgs: []any{},
174+
outputYdbTypes: []*ydb.Type{common.MakePrimitiveType(ydb.Type_INT32), common.MakePrimitiveType(ydb.Type_STRING)},
175+
err: nil,
176+
},
149177
{
150178
testName: "bool_column",
151179
selectReq: &api_service_protos.TSelect{
@@ -238,34 +266,6 @@ func TestMakeSelectQuery(t *testing.T) {
238266
outputYdbTypes: []*ydb.Type{common.MakePrimitiveType(ydb.Type_INT32), common.MakePrimitiveType(ydb.Type_STRING)},
239267
err: nil,
240268
},
241-
{
242-
testName: "unsupported_predicate",
243-
selectReq: &api_service_protos.TSelect{
244-
From: &api_service_protos.TSelect_TFrom{
245-
Table: "tab",
246-
},
247-
What: rdbms_utils.NewDefaultWhat(),
248-
Where: &api_service_protos.TSelect_TWhere{
249-
FilterTyped: &api_service_protos.TPredicate{
250-
Payload: &api_service_protos.TPredicate_Between{
251-
Between: &api_service_protos.TPredicate_TBetween{
252-
Value: rdbms_utils.NewColumnExpression("col2"),
253-
Least: rdbms_utils.NewColumnExpression("col1"),
254-
Greatest: rdbms_utils.NewColumnExpression("col3"),
255-
},
256-
},
257-
},
258-
},
259-
DataSourceInstance: &api_common.TGenericDataSourceInstance{
260-
Kind: api_common.EGenericDataSourceKind_POSTGRESQL,
261-
},
262-
},
263-
splitDescription: singleSplit,
264-
outputQuery: `SELECT "col0", "col1" FROM "tab"`,
265-
outputArgs: []any{},
266-
outputYdbTypes: []*ydb.Type{common.MakePrimitiveType(ydb.Type_INT32), common.MakePrimitiveType(ydb.Type_STRING)},
267-
err: nil,
268-
},
269269
{
270270
testName: "unsupported_type",
271271
selectReq: &api_service_protos.TSelect{

‎app/server/datasource/rdbms/utils/interface.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,8 @@ type SQLFormatter interface {
152152
*api_service_protos.TPredicate_TComparison, error)
153153
// ValidateWhere validates the query predicate
154154
ValidateWhere(where *api_service_protos.TSelect_TWhere) error
155+
// Renders `value BETWEEN least AND greatest`
156+
RenderBetween(value, least, greatest string) (string, error)
155157
}
156158

157159
type SchemaProvider interface {

0 commit comments

Comments
 (0)