Skip to content

SQL: Use specialized virtual columns for JSON_OBJECT, JSON_MERGE. #18553

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Oct 16, 2025
Merged

SQL: Use specialized virtual columns for JSON_OBJECT, JSON_MERGE. #18553

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
d786c7e
SQL: Use specialized virtual columns for JSON_OBJECT, JSON_MERGE.
gianm Sep 19, 2025
4bf0dd4
Correct javadoc.
gianm Sep 20, 2025
1efd55e
Update expectations.
gianm Sep 20, 2025
f44ed5c
NestedFieldVirtualColumn: Process arrays when making object selectors…
gianm Sep 21, 2025
b204a3b
Update test.
gianm Sep 21, 2025
ca35847
Merge branch 'master' into sql-specialize-recursively
gianm Sep 30, 2025
c8b1dca
Merge branch 'master' into sql-specialize-recursively
gianm Oct 15, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 14 additions & 6 deletions processing/src/main/java/org/apache/druid/segment/virtual/NestedFieldVirtualColumn.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -414,8 +414,14 @@ public ColumnValueSelector<?> makeColumnValueSelector(
return new ArrayElementColumnValueSelector(arraySelector, elementNumber);
}

// we are not a nested column and are being asked for a path that will never exist, so we are nil selector
return NilColumnValueSelector.instance();
if (holder.getCapabilities().isArray() || ColumnType.NESTED_DATA.equals(holder.getCapabilities().toColumnType())) {
// Not a root access and no specialized path available. But the underlying column is array or nested typed,
// so we may still be able to walk it using exprs. Try that.
return makeColumnValueSelectorUsingColumnSelectorFactory(selectorFactory);
} else {
// we are not a nested or array column, and are being asked for a path that will never exist, so nil selector
return NilColumnValueSelector.instance();
}
}

@Override
Expand Down Expand Up @@ -486,11 +492,11 @@ public VectorObjectSelector makeVectorObjectSelector(
final NestedVectorColumnSelectorFactory nestedColumnSelectorFactory =
column.as(NestedVectorColumnSelectorFactory.class);

if (isNestedColumn(holder)) {
if (isNestedColumn(holder) || holder.getCapabilities().isArray()) {
if (fieldSpec.processFromRaw || nestedTypeInspector == null || nestedColumnSelectorFactory == null) {
// 1) If processFromRaw is true, that means JSON_QUERY.
// 2) If no nestedTypeInspector, nestedColumnSelectorFactory then that means this is a nested type that is
// not exposed as a nested column.
// 2) If no nestedTypeInspector, nestedColumnSelectorFactory then that means this is a nested or array
// type that is not exposed as a nested column.
// Either way, we read and process raw objects.
return new RawFieldVectorObjectSelector(
selectorFactory.makeObjectSelector(fieldSpec.columnName),
Expand All @@ -499,7 +505,9 @@ public VectorObjectSelector makeVectorObjectSelector(
);
}
final ColumnType leastRestrictiveType = nestedTypeInspector.getFieldLogicalType(fieldSpec.parts);
if (leastRestrictiveType != null && leastRestrictiveType.isNumeric() && !Types.isNumeric(fieldSpec.expectedType)) {
if (leastRestrictiveType != null
&& leastRestrictiveType.isNumeric()
&& !Types.isNumeric(fieldSpec.expectedType)) {
return ExpressionVectorSelectors.castValueSelectorToObject(
offset,
columnName,
Expand Down
11 changes: 6 additions & 5 deletions processing/src/main/java/org/apache/druid/segment/virtual/NestedObjectVirtualColumn.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,11 +88,12 @@ public NestedObjectVirtualColumn(
StringUtils.format(
"%s(%s)",
NestedDataExpressions.JsonObjectExprMacro.NAME,
keyExprMap.entrySet().stream().sorted(Map.Entry.comparingByKey()).map(entry -> {
final String key = entry.getKey();
final TypedExpression valueExpr = entry.getValue();
return Parser.constant(key).stringify() + ',' + valueExpr.expression;
}).collect(Collectors.joining(","))
Preconditions.checkNotNull(keyExprMap, "object")
.entrySet().stream().map(entry -> {
final String key = entry.getKey();
final TypedExpression valueExpr = entry.getValue();
return Parser.constant(key).stringify() + ',' + valueExpr.expression;
}).collect(Collectors.joining(","))
),
ColumnType.NESTED_DATA,
macroTable
Expand Down
8 changes: 4 additions & 4 deletions processing/src/test/java/org/apache/druid/segment/virtual/NestedObjectVirtualColumnTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@
import org.junit.Assert;
import org.junit.Test;

import java.util.HashMap;
import java.util.Map;

public class NestedObjectVirtualColumnTest
Expand All @@ -56,9 +55,10 @@ public void testSerde() throws JsonProcessingException
@Test
public void testGetKeyExprMap()
{
Map<String, NestedObjectVirtualColumn.TypedExpression> keyExprMap = new HashMap<>();
keyExprMap.put("key1", new NestedObjectVirtualColumn.TypedExpression("expr1", ColumnType.STRING));
keyExprMap.put("key2", new NestedObjectVirtualColumn.TypedExpression("expr2", ColumnType.DOUBLE));
Map<String, NestedObjectVirtualColumn.TypedExpression> keyExprMap = ImmutableMap.of(
"key1", new NestedObjectVirtualColumn.TypedExpression("expr1", ColumnType.STRING),
"key2", new NestedObjectVirtualColumn.TypedExpression("expr2", ColumnType.DOUBLE)
);

NestedObjectVirtualColumn column = new NestedObjectVirtualColumn(
"test_obj",
Expand Down
207 changes: 175 additions & 32 deletions quidem-ut/src/test/quidem/org.apache.druid.quidem.QTest/qaArray/ops_funcs_mv_funcs.08.all.iq
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,16 +119,34 @@ FROM test_array;
#-------------------------------------------------------------------------
SELECT mv_to_array(json_value(a_nested, '$[0]' RETURNING boolean array)) AS col
FROM test_array;
should be an identifier expression. Use array() instead
!error
+--------------+
| col |
+--------------+
| [1, 0, null] |
| [1, 0, null] |
| [null, 0, 1] |
| [null, 0, 1] |
+--------------+
(4 rows)

!ok

#-------------------------------------------------------------------------
# TESTCASE: test_mv_funcs TEST_ID: A1_B16_C3_D1
#-------------------------------------------------------------------------
SELECT mv_to_array(json_value(a_nested, '$[7][0]' RETURNING boolean array)) AS col
FROM test_array;
should be an identifier expression. Use array() instead
!error
+--------------+
| col |
+--------------+
| [1, 0, null] |
| [1, 0, null] |
| [null, 0, 1] |
| [null, 0, 1] |
+--------------+
(4 rows)

!ok

#-------------------------------------------------------------------------
# TESTCASE: test_mv_funcs TEST_ID: A1_B16_C4_D1
Expand All @@ -152,16 +170,34 @@ FROM test_array;
#-------------------------------------------------------------------------
SELECT mv_to_array(json_value(a_nested, '$[1]' RETURNING bigint array)) AS col
FROM test_array;
should be an identifier expression. Use array() instead
!error
+--------------+
| col |
+--------------+
| [1, 2, null] |
| [1, 2, null] |
| [null, 2, 1] |
| [null, 2, 1] |
+--------------+
(4 rows)

!ok

#-------------------------------------------------------------------------
# TESTCASE: test_mv_funcs TEST_ID: A1_B16_C6_D1
#-------------------------------------------------------------------------
SELECT mv_to_array(json_value(a_nested, '$[7][1]' RETURNING bigint array)) AS col
FROM test_array;
should be an identifier expression. Use array() instead
!error
+--------------+
| col |
+--------------+
| [1, 2, null] |
| [1, 2, null] |
| [null, 2, 1] |
| [null, 2, 1] |
+--------------+
(4 rows)

!ok

#-------------------------------------------------------------------------
# TESTCASE: test_mv_funcs TEST_ID: A1_B16_C7_D1
Expand All @@ -185,16 +221,34 @@ FROM test_array;
#-------------------------------------------------------------------------
SELECT mv_to_array(json_value(a_nested, '$[2]' RETURNING decimal array)) AS col
FROM test_array;
should be an identifier expression. Use array() instead
!error
+------------------+
| col |
+------------------+
| [0.1, 0.2, null] |
| [0.1, 0.2, null] |
| [null, 0.2, 0.1] |
| [null, 0.2, 0.1] |
+------------------+
(4 rows)

!ok

#-------------------------------------------------------------------------
# TESTCASE: test_mv_funcs TEST_ID: A1_B16_C9_D1
#-------------------------------------------------------------------------
SELECT mv_to_array(json_value(a_nested, '$[7][2]' RETURNING decimal array)) AS col
FROM test_array;
should be an identifier expression. Use array() instead
!error
+------------------+
| col |
+------------------+
| [0.1, 0.2, null] |
| [0.1, 0.2, null] |
| [null, 0.2, 0.1] |
| [null, 0.2, 0.1] |
+------------------+
(4 rows)

!ok

#-------------------------------------------------------------------------
# TESTCASE: test_mv_funcs TEST_ID: A1_B16_C10_D1
Expand All @@ -218,16 +272,34 @@ FROM test_array;
#-------------------------------------------------------------------------
SELECT mv_to_array(json_value(a_nested, '$[3]' RETURNING varchar array)) AS col
FROM test_array;
should be an identifier expression. Use array() instead
!error
+----------------+
| col |
+----------------+
| [S1, S2, null] |
| [S1, S2, null] |
| [null, S2, S1] |
| [null, S2, S1] |
+----------------+
(4 rows)

!ok

#-------------------------------------------------------------------------
# TESTCASE: test_mv_funcs TEST_ID: A1_B16_C12_D1
#-------------------------------------------------------------------------
SELECT mv_to_array(json_value(a_nested, '$[7][3]' RETURNING varchar array)) AS col
FROM test_array;
should be an identifier expression. Use array() instead
!error
+----------------+
| col |
+----------------+
| [S1, S2, null] |
| [S1, S2, null] |
| [null, S2, S1] |
| [null, S2, S1] |
+----------------+
(4 rows)

!ok

#-------------------------------------------------------------------------
# TESTCASE: test_mv_funcs TEST_ID: A1_B16_C13_D1
Expand All @@ -251,16 +323,34 @@ FROM test_array;
#-------------------------------------------------------------------------
SELECT mv_to_array(json_value(a_nested, '$[4]' RETURNING varchar array)) AS col
FROM test_array;
should be an identifier expression. Use array() instead
!error
+--------------------+
| col |
+--------------------+
| [null, null, null] |
| [null, null, null] |
| [null, null, null] |
| [null, null, null] |
+--------------------+
(4 rows)

!ok

#-------------------------------------------------------------------------
# TESTCASE: test_mv_funcs TEST_ID: A1_B16_C15_D1
#-------------------------------------------------------------------------
SELECT mv_to_array(json_value(a_nested, '$[7][4]' RETURNING varchar array)) AS col
FROM test_array;
should be an identifier expression. Use array() instead
!error
+--------------------+
| col |
+--------------------+
| [null, null, null] |
| [null, null, null] |
| [null, null, null] |
| [null, null, null] |
+--------------------+
(4 rows)

!ok

#-------------------------------------------------------------------------
# TESTCASE: test_mv_funcs TEST_ID: A1_B16_C16_D1
Expand All @@ -284,16 +374,34 @@ FROM test_array;
#-------------------------------------------------------------------------
SELECT mv_to_array(json_value(a_nested, '$[5]' RETURNING varchar array)) AS col
FROM test_array;
should be an identifier expression. Use array() instead
!error
+-----+
| col |
+-----+
| [] |
| [] |
| [] |
| [] |
+-----+
(4 rows)

!ok

#-------------------------------------------------------------------------
# TESTCASE: test_mv_funcs TEST_ID: A1_B16_C18_D1
#-------------------------------------------------------------------------
SELECT mv_to_array(json_value(a_nested, '$[7][5]' RETURNING varchar array)) AS col
FROM test_array;
should be an identifier expression. Use array() instead
!error
+-----+
| col |
+-----+
| [] |
| [] |
| [] |
| [] |
+-----+
(4 rows)

!ok

#-------------------------------------------------------------------------
# TESTCASE: test_mv_funcs TEST_ID: A1_B16_C19_D1
Expand All @@ -317,16 +425,34 @@ FROM test_array;
#-------------------------------------------------------------------------
SELECT mv_to_array(json_value(a_nested, '$[6]' RETURNING varchar array)) AS col
FROM test_array;
should be an identifier expression. Use array() instead
!error
+--------------------------+
| col |
+--------------------------+
| [null, S1, 0.1, 1, true] |
| [null, S1, 0.1, 1, true] |
| [true, 1, 0.1, S1, null] |
| [true, 1, 0.1, S1, null] |
+--------------------------+
(4 rows)

!ok

#-------------------------------------------------------------------------
# TESTCASE: test_mv_funcs TEST_ID: A1_B16_C21_D1
#-------------------------------------------------------------------------
SELECT mv_to_array(json_value(a_nested, '$[7][6]' RETURNING varchar array)) AS col
FROM test_array;
should be an identifier expression. Use array() instead
!error
+--------------------------+
| col |
+--------------------------+
| [null, S1, 0.1, 1, true] |
| [null, S1, 0.1, 1, true] |
| [true, 1, 0.1, S1, null] |
| [true, 1, 0.1, S1, null] |
+--------------------------+
(4 rows)

!ok

#-------------------------------------------------------------------------
# TESTCASE: test_mv_funcs TEST_ID: A1_B16_C22_D1
Expand All @@ -341,14 +467,31 @@ Cannot apply 'MV_TO_ARRAY' to arguments of type 'MV_TO_ARRAY(
#-------------------------------------------------------------------------
SELECT mv_to_array(json_value(a_nested, '$[7]' RETURNING varchar array)) AS col
FROM test_array;
should be an identifier expression. Use array() instead
!error
+-----+
| col |
+-----+
| |
| |
| |
| |
+-----+
(4 rows)

!ok

#-------------------------------------------------------------------------
# TESTCASE: test_mv_funcs TEST_ID: A1_B16_C24_D1
#-------------------------------------------------------------------------
SELECT mv_to_array(json_value(a_nested, '$[7][7]' RETURNING varchar array)) AS col
FROM test_array;
should be an identifier expression. Use array() instead
!error
+-----+
| col |
+-----+
| |
| |
| |
| |
+-----+
(4 rows)

!ok
Loading