Skip to content

Commit 94561b4

Browse files
pdabre12Pratik Joseph Dabre
pdabre12
authored and
Pratik Joseph Dabre
committed
[native] Introduce presto-native-sql-invoked-functions-plugin
1 parent 6d69b41 commit 94561b4

File tree

17 files changed

+392
-16
lines changed

17 files changed

+392
-16
lines changed

‎.github/workflows/prestocpp-linux-build-and-unit-test.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -370,7 +370,7 @@ jobs:
370370
# Use different Maven options to install.
371371
MAVEN_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError"
372372
run: |
373-
for i in $(seq 1 3); do ./mvnw clean install $MAVEN_FAST_INSTALL -pl 'presto-native-execution' -am && s=0 && break || s=$? && sleep 10; done; (exit $s)
373+
for i in $(seq 1 3); do ./mvnw clean install $MAVEN_FAST_INSTALL -pl 'presto-native-sidecar-plugin' -am && s=0 && break || s=$? && sleep 10; done; (exit $s)
374374
375375
- name: Run presto-native sidecar tests
376376
if: |

‎pom.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -222,6 +222,7 @@
222222
<module>presto-router-example-plugin-scheduler</module>
223223
<module>presto-plan-checker-router-plugin</module>
224224
<module>presto-sql-invoked-functions-plugin</module>
225+
<module>presto-native-sql-invoked-functions-plugin</module>
225226
<module>presto-spark-classloader-spark${dep.pos.classloader.module-name.suffix}</module>
226227
</modules>
227228

‎presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/KeyBasedSampler.java‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@
1414
package com.facebook.presto.sql.planner.optimizations;
1515

1616
import com.facebook.presto.Session;
17-
import com.facebook.presto.common.QualifiedObjectName;
1817
import com.facebook.presto.common.function.OperatorType;
1918
import com.facebook.presto.common.type.Type;
2019
import com.facebook.presto.common.type.Varchars;
@@ -55,7 +54,6 @@
5554
import static com.facebook.presto.common.type.BooleanType.BOOLEAN;
5655
import static com.facebook.presto.common.type.DoubleType.DOUBLE;
5756
import static com.facebook.presto.common.type.VarcharType.VARCHAR;
58-
import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE;
5957
import static com.facebook.presto.metadata.CastType.CAST;
6058
import static com.facebook.presto.spi.StandardErrorCode.FUNCTION_NOT_FOUND;
6159
import static com.facebook.presto.spi.StandardWarningCode.SAMPLED_FIELDS;
@@ -150,7 +148,7 @@ private PlanNode addSamplingFilter(PlanNode tableScanNode, Optional<VariableRefe
150148
try {
151149
sampledArg = call(
152150
functionAndTypeManager,
153-
QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, getKeyBasedSamplingFunction(session)),
151+
getKeyBasedSamplingFunction(session),
154152
DOUBLE,
155153
ImmutableList.of(arg));
156154
}

‎presto-main-base/src/main/java/com/facebook/presto/sql/relational/Expressions.java‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@
1313
*/
1414
package com.facebook.presto.sql.relational;
1515

16-
import com.facebook.presto.common.QualifiedObjectName;
1716
import com.facebook.presto.common.function.OperatorType;
1817
import com.facebook.presto.common.type.Type;
1918
import com.facebook.presto.metadata.CastType;
@@ -154,12 +153,6 @@ public static CallExpression call(FunctionAndTypeManager functionAndTypeManager,
154153
return call(name, functionHandle, returnType, arguments);
155154
}
156155

157-
public static CallExpression call(FunctionAndTypeManager functionAndTypeManager, QualifiedObjectName qualifiedObjectName, Type returnType, List<RowExpression> arguments)
158-
{
159-
FunctionHandle functionHandle = functionAndTypeManager.lookupFunction(qualifiedObjectName, fromTypes(arguments.stream().map(RowExpression::getType).collect(toImmutableList())));
160-
return call(String.valueOf(qualifiedObjectName), functionHandle, returnType, arguments);
161-
}
162-
163156
public static CallExpression call(FunctionAndTypeResolver functionAndTypeResolver, String name, Type returnType, RowExpression... arguments)
164157
{
165158
FunctionHandle functionHandle = functionAndTypeResolver.lookupFunction(name, fromTypes(Arrays.stream(arguments).map(RowExpression::getType).collect(toImmutableList())));

‎presto-native-execution/src/test/java/com/facebook/presto/nativeworker/NativeQueryRunnerUtils.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,8 @@ public static Map<String, String> getNativeSidecarProperties()
5959
.put("coordinator-sidecar-enabled", "true")
6060
.put("exclude-invalid-worker-session-properties", "true")
6161
.put("presto.default-namespace", "native.default")
62+
// We override the inline-sql-functions config to true
63+
.put("inline-sql-functions", "true")
6264
.build();
6365
}
6466

‎presto-native-sidecar-plugin/pom.xml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -260,9 +260,25 @@
260260
</exclusion>
261261
</exclusions>
262262
</dependency>
263+
263264
<dependency>
264265
<groupId>com.facebook.presto</groupId>
265266
<artifactId>presto-built-in-worker-function-tools</artifactId>
267+
<version>${project.version}</version>
268+
</dependency>
269+
270+
<dependency>
271+
<groupId>com.facebook.presto</groupId>
272+
<artifactId>presto-native-sql-invoked-functions-plugin</artifactId>
273+
<version>${project.version}</version>
274+
<scope>test</scope>
275+
</dependency>
276+
277+
<dependency>
278+
<groupId>com.facebook.presto</groupId>
279+
<artifactId>presto-sql-invoked-functions-plugin</artifactId>
280+
<version>${project.version}</version>
281+
<scope>test</scope>
266282
</dependency>
267283
</dependencies>
268284

‎presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/NativeSidecarPluginQueryRunnerUtils.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
*/
1414
package com.facebook.presto.sidecar;
1515

16+
import com.facebook.presto.scalar.sql.NativeSqlInvokedFunctionsPlugin;
1617
import com.facebook.presto.sidecar.functionNamespace.NativeFunctionNamespaceManagerFactory;
1718
import com.facebook.presto.sidecar.sessionpropertyproviders.NativeSystemSessionPropertyProviderFactory;
1819
import com.facebook.presto.sidecar.typemanager.NativeTypeManagerFactory;
@@ -37,5 +38,6 @@ public static void setupNativeSidecarPlugin(QueryRunner queryRunner)
3738
"function-implementation-type", "CPP"));
3839
queryRunner.loadTypeManager(NativeTypeManagerFactory.NAME);
3940
queryRunner.loadPlanCheckerProviderManager("native", ImmutableMap.of());
41+
queryRunner.installPlugin(new NativeSqlInvokedFunctionsPlugin());
4042
}
4143
}

‎presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/TestNativeSidecarPlugin.java‎

Lines changed: 122 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,10 @@
1414
package com.facebook.presto.sidecar;
1515

1616
import com.facebook.airlift.units.DataSize;
17+
import com.facebook.presto.Session;
1718
import com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils;
19+
import com.facebook.presto.scalar.sql.NativeSqlInvokedFunctionsPlugin;
20+
import com.facebook.presto.scalar.sql.SqlInvokedFunctionsPlugin;
1821
import com.facebook.presto.sidecar.functionNamespace.FunctionDefinitionProvider;
1922
import com.facebook.presto.sidecar.functionNamespace.NativeFunctionDefinitionProvider;
2023
import com.facebook.presto.sidecar.functionNamespace.NativeFunctionNamespaceManager;
@@ -44,8 +47,11 @@
4447
import java.util.stream.Collectors;
4548

4649
import static com.facebook.airlift.units.DataSize.Unit.MEGABYTE;
50+
import static com.facebook.presto.SystemSessionProperties.INLINE_SQL_FUNCTIONS;
51+
import static com.facebook.presto.SystemSessionProperties.KEY_BASED_SAMPLING_ENABLED;
4752
import static com.facebook.presto.common.Utils.checkArgument;
4853
import static com.facebook.presto.common.type.BigintType.BIGINT;
54+
import static com.facebook.presto.nativeworker.NativeQueryRunnerUtils.createCustomer;
4955
import static com.facebook.presto.nativeworker.NativeQueryRunnerUtils.createLineitem;
5056
import static com.facebook.presto.nativeworker.NativeQueryRunnerUtils.createNation;
5157
import static com.facebook.presto.nativeworker.NativeQueryRunnerUtils.createOrders;
@@ -63,6 +69,7 @@ public class TestNativeSidecarPlugin
6369
private static final String REGEX_FUNCTION_NAMESPACE = "native.default.*";
6470
private static final String REGEX_SESSION_NAMESPACE = "Native Execution only.*";
6571
private static final long SIDECAR_HTTP_CLIENT_MAX_CONTENT_SIZE_MB = 128;
72+
private static final int INLINED_SQL_FUNCTIONS_COUNT = 7;
6673

6774
@Override
6875
protected void createTables()
@@ -73,6 +80,7 @@ protected void createTables()
7380
createOrders(queryRunner);
7481
createOrdersEx(queryRunner);
7582
createRegion(queryRunner);
83+
createCustomer(queryRunner);
7684
}
7785

7886
@Override
@@ -91,9 +99,11 @@ protected QueryRunner createQueryRunner()
9199
protected QueryRunner createExpectedQueryRunner()
92100
throws Exception
93101
{
94-
return PrestoNativeQueryRunnerUtils.javaHiveQueryRunnerBuilder()
102+
QueryRunner queryRunner = PrestoNativeQueryRunnerUtils.javaHiveQueryRunnerBuilder()
95103
.setAddStorageFormatToPath(true)
96104
.build();
105+
queryRunner.installPlugin(new SqlInvokedFunctionsPlugin());
106+
return queryRunner;
97107
}
98108

99109
public static void setupNativeSidecarPlugin(QueryRunner queryRunner)
@@ -111,6 +121,7 @@ public static void setupNativeSidecarPlugin(QueryRunner queryRunner)
111121
"sidecar.http-client.max-content-length", SIDECAR_HTTP_CLIENT_MAX_CONTENT_SIZE_MB + "MB"));
112122
queryRunner.loadTypeManager(NativeTypeManagerFactory.NAME);
113123
queryRunner.loadPlanCheckerProviderManager("native", ImmutableMap.of());
124+
queryRunner.installPlugin(new NativeSqlInvokedFunctionsPlugin());
114125
}
115126

116127
@Test
@@ -161,6 +172,7 @@ public void testSetNativeWorkerSessionProperty()
161172
@Test
162173
public void testShowFunctions()
163174
{
175+
int inlinedSQLFunctionsCount = 0;
164176
@Language("SQL") String sql = "SHOW FUNCTIONS";
165177
MaterializedResult actualResult = computeActual(sql);
166178
List<MaterializedRow> actualRows = actualResult.getMaterializedRows();
@@ -174,11 +186,17 @@ public void testShowFunctions()
174186

175187
// function namespace should be present.
176188
String fullFunctionName = row.get(5).toString();
177-
if (Pattern.matches(REGEX_FUNCTION_NAMESPACE, fullFunctionName)) {
178-
continue;
189+
if (!Pattern.matches(REGEX_FUNCTION_NAMESPACE, fullFunctionName)) {
190+
// If no namespace match found, check if it's an inlined SQL Invoked function.
191+
String language = row.get(9).toString();
192+
if (language.equalsIgnoreCase("SQL")) {
193+
inlinedSQLFunctionsCount++;
194+
continue;
195+
}
196+
fail(format("No namespace match found for row: %s", row));
179197
}
180-
fail(format("No namespace match found for row: %s", row));
181198
}
199+
assertEquals(inlinedSQLFunctionsCount, INLINED_SQL_FUNCTIONS_COUNT);
182200
}
183201

184202
@Test
@@ -319,7 +337,7 @@ public void testApproxPercentile()
319337
public void testInformationSchemaTables()
320338
{
321339
assertQuery("select lower(table_name) from information_schema.tables "
322-
+ "where table_name = 'lineitem' or table_name = 'LINEITEM' ");
340+
+ "where table_name = 'lineitem' or table_name = 'LINEITEM' ");
323341
}
324342

325343
@Test
@@ -405,6 +423,105 @@ public void testGeometryQueries()
405423
"Error from native plan checker: .SpatialJoinNode no abstract type PlanNode ");
406424
}
407425

426+
@Test
427+
public void testOverriddenInlinedSqlInvokedFunctions()
428+
{
429+
// String functions
430+
assertQuery("SELECT trail(comment, cast(nationkey as integer)) FROM nation");
431+
assertQuery("SELECT name, comment, replace_first(comment, 'iron', 'gold') from nation");
432+
433+
// Array functions
434+
assertQuery("SELECT array_intersect(ARRAY['apple', 'banana', 'cherry'], ARRAY['apple', 'mango', 'fig'])");
435+
assertQuery("SELECT array_frequency(split(comment, '')) from nation");
436+
assertQuery("SELECT array_duplicates(ARRAY[regionkey]), array_duplicates(ARRAY[comment]) from nation");
437+
assertQuery("SELECT array_has_duplicates(ARRAY[custkey]) from orders");
438+
assertQuery("SELECT array_max_by(ARRAY[comment], x -> length(x)) from orders");
439+
assertQuery("SELECT array_min_by(ARRAY[ROW('USA', 1), ROW('INDIA', 2), ROW('UK', 3)], x -> x[2])");
440+
assertQuery("SELECT array_sort_desc(map_keys(map_union(quantity_by_linenumber))) FROM orders_ex");
441+
assertQuery("SELECT remove_nulls(ARRAY[CAST(regionkey AS VARCHAR), comment, NULL]) from nation");
442+
assertQuery("SELECT array_top_n(ARRAY[CAST(nationkey AS VARCHAR)], 3) from nation");
443+
assertQuerySucceeds("SELECT array_sort_desc(quantities, x -> abs(x)) FROM orders_ex");
444+
445+
// Map functions
446+
assertQuery("SELECT map_normalize(MAP(ARRAY['a', 'b', 'c'], ARRAY[1, 4, 5]))");
447+
assertQuery("SELECT map_normalize(MAP(ARRAY['a', 'b', 'c'], ARRAY[1, 0, -1]))");
448+
assertQuery("SELECT name, map_normalize(MAP(ARRAY['regionkey', 'length'], ARRAY[regionkey, length(comment)])) from nation");
449+
assertQuery("SELECT name, map_remove_null_values(map(ARRAY['region', 'comment', 'nullable'], " +
450+
"ARRAY[CAST(regionkey AS VARCHAR), comment, NULL])) from nation");
451+
assertQuery("SELECT name, map_key_exists(map(ARRAY['nation', 'comment'], ARRAY[CAST(nationkey AS VARCHAR), comment]), 'comment') from nation");
452+
assertQuery("SELECT map_keys_by_top_n_values(MAP(ARRAY[orderkey], ARRAY[custkey]), 2) from orders");
453+
assertQuery("SELECT map_top_n(MAP(ARRAY[CAST(nationkey AS VARCHAR)], ARRAY[comment]), 3) from nation");
454+
assertQuery("SELECT map_top_n_keys(MAP(ARRAY[orderkey], ARRAY[custkey]), 3) from orders");
455+
assertQuery("SELECT map_top_n_values(MAP(ARRAY[orderkey], ARRAY[custkey]), 3) from orders");
456+
assertQuery("SELECT all_keys_match(MAP(ARRAY[comment], ARRAY[custkey]), k -> length(k) > 5) from orders");
457+
assertQuery("SELECT any_keys_match(MAP(ARRAY[comment], ARRAY[custkey]), k -> starts_with(k, 'abc')) from orders");
458+
assertQuery("SELECT any_values_match(MAP(ARRAY[orderkey], ARRAY[totalprice]), k -> abs(k) > 20) from orders");
459+
assertQuery("SELECT no_values_match(MAP(ARRAY[orderkey], ARRAY[comment]), k -> length(k) > 2) from orders");
460+
assertQuery("SELECT no_keys_match(MAP(ARRAY[comment], ARRAY[custkey]), k -> ends_with(k, 'a')) from orders");
461+
}
462+
463+
@Test
464+
public void testNonOverriddenInlinedSqlInvokedFunctionsWhenConfigEnabled()
465+
{
466+
// Array functions
467+
assertQuery("SELECT array_split_into_chunks(split(comment, ''), 2) from nation");
468+
assertQuery("SELECT array_least_frequent(quantities) from orders_ex");
469+
assertQuery("SELECT array_least_frequent(split(comment, ''), 5) from nation");
470+
assertQuerySucceeds("SELECT array_top_n(ARRAY[orderkey], 25, (x, y) -> if (x < y, cast(1 as bigint), if (x > y, cast(-1 as bigint), cast(0 as bigint)))) from orders");
471+
472+
// Map functions
473+
assertQuerySucceeds("SELECT map_top_n_values(MAP(ARRAY[comment], ARRAY[nationkey]), 2, (x, y) -> if (x < y, cast(1 as bigint), if (x > y, cast(-1 as bigint), cast(0 as bigint)))) from nation");
474+
assertQuerySucceeds("SELECT map_top_n_keys(MAP(ARRAY[regionkey], ARRAY[nationkey]), 5, (x, y) -> if (x < y, cast(1 as bigint), if (x > y, cast(-1 as bigint), cast(0 as bigint)))) from nation");
475+
476+
Session sessionWithKeyBasedSampling = Session.builder(getSession())
477+
.setSystemProperty(KEY_BASED_SAMPLING_ENABLED, "true")
478+
.build();
479+
480+
@Language("SQL") String query = "select count(1) FROM lineitem l left JOIN orders o ON l.orderkey = o.orderkey JOIN customer c ON o.custkey = c.custkey";
481+
482+
assertQuery(query, "select cast(60175 as bigint)");
483+
assertQuery(sessionWithKeyBasedSampling, query, "select cast(16185 as bigint)");
484+
}
485+
486+
@Test
487+
public void testNonOverriddenInlinedSqlInvokedFunctionsWhenConfigDisabled()
488+
{
489+
// When inline_sql_functions is set to false, the below queries should fail as the implementations don't exist on the native worker
490+
Session session = Session.builder(getSession())
491+
.setSystemProperty(KEY_BASED_SAMPLING_ENABLED, "true")
492+
.setSystemProperty(INLINE_SQL_FUNCTIONS, "false")
493+
.build();
494+
495+
// Array functions
496+
assertQueryFails(session,
497+
"SELECT array_split_into_chunks(split(comment, ''), 2) from nation",
498+
".*Scalar function name not registered: native.default.array_split_into_chunks.*");
499+
assertQueryFails(session,
500+
"SELECT array_least_frequent(quantities) from orders_ex",
501+
".*Scalar function name not registered: native.default.array_least_frequent.*");
502+
assertQueryFails(session,
503+
"SELECT array_least_frequent(split(comment, ''), 2) from nation",
504+
".*Scalar function name not registered: native.default.array_least_frequent.*");
505+
assertQueryFails(session,
506+
"SELECT array_top_n(ARRAY[orderkey], 25, (x, y) -> if (x < y, cast(1 as bigint), if (x > y, cast(-1 as bigint), cast(0 as bigint)))) from orders",
507+
" Scalar function native\\.default\\.array_top_n not registered with arguments.*",
508+
true);
509+
510+
// Map functions
511+
assertQueryFails(session,
512+
"SELECT map_top_n_values(MAP(ARRAY[comment], ARRAY[nationkey]), 2, (x, y) -> if (x < y, cast(1 as bigint), if (x > y, cast(-1 as bigint), cast(0 as bigint)))) from nation",
513+
".*Scalar function native\\.default\\.map_top_n_values not registered with arguments.*",
514+
true);
515+
assertQueryFails(session,
516+
"SELECT map_top_n_keys(MAP(ARRAY[regionkey], ARRAY[nationkey]), 5, (x, y) -> if (x < y, cast(1 as bigint), if (x > y, cast(-1 as bigint), cast(0 as bigint)))) from nation",
517+
".*Scalar function native\\.default\\.map_top_n_keys not registered with arguments.*",
518+
true);
519+
520+
assertQueryFails(session,
521+
"select count(1) FROM lineitem l left JOIN orders o ON l.orderkey = o.orderkey JOIN customer c ON o.custkey = c.custkey",
522+
".*Scalar function name not registered: native.default.key_sampling_percent.*");
523+
}
524+
408525
private String generateRandomTableName()
409526
{
410527
String tableName = "tmp_presto_" + UUID.randomUUID().toString().replace("-", "");

‎presto-native-sql-invoked-functions-plugin/pom.xml‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
2+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
3+
<modelVersion>4.0.0</modelVersion>
4+
<parent>
5+
<groupId>com.facebook.presto</groupId>
6+
<artifactId>presto-root</artifactId>
7+
<version>0.295-SNAPSHOT</version>
8+
</parent>
9+
10+
<artifactId>presto-native-sql-invoked-functions-plugin</artifactId>
11+
<description>Presto Native - Sql invoked functions plugin</description>
12+
<packaging>presto-plugin</packaging>
13+
14+
<properties>
15+
<air.main.basedir>${project.parent.basedir}</air.main.basedir>
16+
</properties>
17+
18+
<dependencies>
19+
<dependency>
20+
<groupId>com.facebook.presto</groupId>
21+
<artifactId>presto-spi</artifactId>
22+
<scope>provided</scope>
23+
</dependency>
24+
<dependency>
25+
<groupId>com.google.guava</groupId>
26+
<artifactId>guava</artifactId>
27+
</dependency>
28+
</dependencies>
29+
</project>

0 commit comments

Comments
 (0)