perf: compare nested ScalarValues by type before converting to ArrayData (#26066)

## Which issue does this PR close?

- Closes #26065.

## Rationale for this change

Physical planning slows down, faster than the query grows, when
projections carry many typed NULL struct literals: a `UNION ALL` of 35
branches with 35 struct columns of 20 fields takes 862 ms in
`create_physical_plan` on `main`. `EquivalenceGroup::add_constant`
compares each new constant with every existing class, and
`ScalarValue::eq` compares nested values with arrow's array equality,
which converts both arrays to `ArrayData` before it compares anything,
even when the two values have different types.

## What changes are included in this PR?

`ScalarValue::eq` compares the nested variants (`List`, `LargeList`,
`FixedSizeList`, `ListView`, `LargeListView`, `Struct`, `Map`) through a
helper that returns early for the same `Arc`, and compares lengths and
data types before arrow's array equality. Results are unchanged; only
values of the same type and length reach the `ArrayData` comparison.

With the reproducer from the issue (release build, Apple M1 Max),
`create_physical_plan`:

| struct columns x fields | main | this PR |
|---|---:|---:|
| 10 x 20 | 31 ms | 17 ms |
| 20 x 20 | 179 ms | 47 ms |
| 35 x 20 | 862 ms | 147 ms |
| 35 x 40 | 1,766 ms | 253 ms |

## What is the testing strategy for this PR?

`test_nested_scalar_eq` covers typed NULL structs of the same and of
different types, struct values that are equal and unequal, and lists of
different element types. The scalar tests in `datafusion-common` pass.

## Are there any user-facing changes?

No.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com>

Author: 3 people

datafusion/common/src/scalar/mod.rs

@@ -481,6 +481,19 @@ impl Hash for Fl<f16> {
     }
 }
 
+/// Equality of the arrays inside nested scalars.
+///
+/// Arrow's array equality converts both arrays to `ArrayData` before it
+/// compares anything, which is costly for wide nested types. Values of
+/// different lengths or types are unequal without that, and an array is
+/// equal to itself.
+fn nested_eq<T: Array + PartialEq>(a: &Arc<T>, b: &Arc<T>) -> bool {
+    Arc::ptr_eq(a, b)
+        || (a.len() == b.len()
+            && a.data_type() == b.data_type()
+            && a.as_ref() == b.as_ref())
+}
+
 // manual implementation of `PartialEq`
 impl PartialEq for ScalarValue {
     fn eq(&self, other: &Self) -> bool {
@@ -552,19 +565,19 @@ impl PartialEq for ScalarValue {
             (FixedSizeBinary(_, _), _) => false,
             (LargeBinary(v1), LargeBinary(v2)) => v1.eq(v2),
             (LargeBinary(_), _) => false,
-            (FixedSizeList(v1), FixedSizeList(v2)) => v1.eq(v2),
+            (FixedSizeList(v1), FixedSizeList(v2)) => nested_eq(v1, v2),
             (FixedSizeList(_), _) => false,
-            (List(v1), List(v2)) => v1.eq(v2),
+            (List(v1), List(v2)) => nested_eq(v1, v2),
             (List(_), _) => false,
-            (LargeList(v1), LargeList(v2)) => v1.eq(v2),
+            (LargeList(v1), LargeList(v2)) => nested_eq(v1, v2),
             (LargeList(_), _) => false,
-            (ListView(v1), ListView(v2)) => v1.eq(v2),
+            (ListView(v1), ListView(v2)) => nested_eq(v1, v2),
             (ListView(_), _) => false,
-            (LargeListView(v1), LargeListView(v2)) => v1.eq(v2),
+            (LargeListView(v1), LargeListView(v2)) => nested_eq(v1, v2),
             (LargeListView(_), _) => false,
-            (Struct(v1), Struct(v2)) => v1.eq(v2),
+            (Struct(v1), Struct(v2)) => nested_eq(v1, v2),
             (Struct(_), _) => false,
-            (Map(v1), Map(v2)) => v1.eq(v2),
+            (Map(v1), Map(v2)) => nested_eq(v1, v2),
             (Map(_), _) => false,
             (Date32(v1), Date32(v2)) => v1.eq(v2),
             (Date32(_), _) => false,
@@ -9929,6 +9942,49 @@ mod tests {
         }
     }
 
+    #[test]
+    fn test_nested_scalar_eq() {
+        let struct_type = |name: &str| {
+            DataType::Struct(Fields::from(vec![
+                Field::new(format!("{name}_a"), DataType::Int32, true),
+                Field::new(format!("{name}_b"), DataType::Utf8, true),
+            ]))
+        };
+        let null_x = ScalarValue::try_from(&struct_type("x")).unwrap();
+        let null_x_again = ScalarValue::try_from(&struct_type("x")).unwrap();
+        let null_y = ScalarValue::try_from(&struct_type("y")).unwrap();
+        assert_eq!(null_x, null_x.clone());
+        assert_eq!(null_x, null_x_again);
+        assert_ne!(null_x, null_y);
+
+        let value = |a: i32| {
+            ScalarValue::Struct(Arc::new(StructArray::from(vec![
+                (
+                    Arc::new(Field::new("x_a", DataType::Int32, true)),
+                    Arc::new(Int32Array::from(vec![Some(a)])) as ArrayRef,
+                ),
+                (
+                    Arc::new(Field::new("x_b", DataType::Utf8, true)),
+                    Arc::new(StringArray::from(vec![Some("b")])) as ArrayRef,
+                ),
+            ])))
+        };
+        assert_eq!(value(1), value(1));
+        assert_ne!(value(1), value(2));
+        assert_ne!(value(1), null_x);
+
+        let list_i32 = ScalarValue::List(ScalarValue::new_list_nullable(
+            &[ScalarValue::Int32(Some(1))],
+            &DataType::Int32,
+        ));
+        let list_i64 = ScalarValue::List(ScalarValue::new_list_nullable(
+            &[ScalarValue::Int64(Some(1))],
+            &DataType::Int64,
+        ));
+        assert_eq!(list_i32, list_i32.clone());
+        assert_ne!(list_i32, list_i64);
+    }
+
     #[test]
     fn test_struct_nulls() {
         let fields_b = Fields::from(vec![