Skip to content

(feat) add support for ArrayMin scalar function #1944

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Aug 21, 2025
Merged

(feat) add support for ArrayMin scalar function #1944

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
acf3e92
feat: add support for array_min expression and corresponding unit tests
dharanad Jun 27, 2025
469e0f9
Merge branch 'main' into feat/1938
dharanad Aug 6, 2025
cadcaa6
formatting
dharanad Aug 6, 2025
9890531
update docs
dharanad Aug 9, 2025
074d013
Merge branch 'main' into feat/1938
andygrove Aug 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions docs/source/user-guide/expressions.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,7 @@ The following Spark expressions are currently available. Any known compatibility
| ArrayIntersect | Experimental |
| ArrayJoin | Experimental |
| ArrayMax | Experimental |
| ArrayMin | |
| ArrayRemove | |
| ArrayRepeat | Experimental |
| ArraysOverlap | Experimental |
Expand Down
1 change: 1 addition & 0 deletions spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@ object QueryPlanSerde extends Logging with CometExprShim {
classOf[ArrayIntersect] -> CometArrayIntersect,
classOf[ArrayJoin] -> CometArrayJoin,
classOf[ArrayMax] -> CometArrayMax,
classOf[ArrayMin] -> CometArrayMin,
classOf[ArrayRemove] -> CometArrayRemove,
classOf[ArrayRepeat] -> CometArrayRepeat,
classOf[ArraysOverlap] -> CometArraysOverlap,
Expand Down
14 changes: 13 additions & 1 deletion spark/src/main/scala/org/apache/comet/serde/arrays.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ package org.apache.comet.serde

import scala.annotation.tailrec

import org.apache.spark.sql.catalyst.expressions.{ArrayAppend, ArrayContains, ArrayDistinct, ArrayExcept, ArrayInsert, ArrayIntersect, ArrayJoin, ArrayMax, ArrayRemove, ArrayRepeat, ArraysOverlap, ArrayUnion, Attribute, CreateArray, Expression, Literal}
import org.apache.spark.sql.catalyst.expressions.{ArrayAppend, ArrayContains, ArrayDistinct, ArrayExcept, ArrayInsert, ArrayIntersect, ArrayJoin, ArrayMax, ArrayMin, ArrayRemove, ArrayRepeat, ArraysOverlap, ArrayUnion, Attribute, CreateArray, Expression, Literal}
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types._

Expand Down Expand Up @@ -189,6 +189,18 @@ object CometArrayMax extends CometExpressionSerde[ArrayMax] {
}
}

object CometArrayMin extends CometExpressionSerde[ArrayMin] {
override def convert(
expr: ArrayMin,
inputs: Seq[Attribute],
binding: Boolean): Option[ExprOuterClass.Expr] = {
val arrayExprProto = exprToProto(expr.children.head, inputs, binding)

val arrayMinScalarExpr = scalarFunctionExprToProto("array_min", arrayExprProto)
optExprWithInfo(arrayMinScalarExpr, expr)
}
}

object CometArraysOverlap extends CometExpressionSerde[ArraysOverlap] with IncompatExpr {
override def convert(
expr: ArraysOverlap,
Expand Down
23 changes: 23 additions & 0 deletions spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -409,6 +409,29 @@ class CometArrayExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelp
}
}

test("array_min") {
Seq(true, false).foreach { dictionaryEnabled =>
withTempDir { dir =>
val path = new Path(dir.toURI.toString, "test.parquet")
makeParquetFileAllPrimitiveTypes(path, dictionaryEnabled, n = 10000)
spark.read.parquet(path.toString).createOrReplaceTempView("t1");
checkSparkAnswerAndOperator(spark.sql("SELECT array_min(array(_2, _3, _4)) FROM t1"))
checkSparkAnswerAndOperator(
spark.sql("SELECT array_min((CASE WHEN _2 =_3 THEN array(_4) END)) FROM t1"))
checkSparkAnswerAndOperator(
spark.sql("SELECT array_min((CASE WHEN _2 =_3 THEN array(_2, _4) END)) FROM t1"))
checkSparkAnswerAndOperator(
spark.sql("SELECT array_min(array(CAST(NULL AS INT), CAST(NULL AS INT))) FROM t1"))
checkSparkAnswerAndOperator(
spark.sql("SELECT array_min(array(_2, CAST(NULL AS INT))) FROM t1"))
checkSparkAnswerAndOperator(spark.sql("SELECT array_min(array()) FROM t1"))
checkSparkAnswerAndOperator(
spark.sql(
"SELECT array_min(array(double('-Infinity'), 0.0, double('Infinity'))) FROM t1"))
}
}
}

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Minor nitpick : Could improve our test coverage by adding tests on rust side as well ? Apart from that the changes look good to me

test("array_intersect") {
// TODO test fails if scan is auto
// https://github.com/apache/datafusion-comet/issues/2174
Expand Down
Loading