[SPARK-52815][CORE] Improve SparkClassUtils to support getAllInterfaces

### What changes were proposed in this pull request?

This PR aims to improve `SparkClassUtils` to support `getAllInterfaces`.

### Why are the changes needed?

To provide a more API in `SparkClassUtils` instead of depending on `org.apache.commons.lang3.ClassUtils.getAllInterfaces`

### Does this PR introduce _any_ user-facing change?

No. `SparkClassUtils` are a private trait.

### How was this patch tested?

Pass the CIs.

### Was this patch authored or co-authored using generative AI tooling?

No.

Closes #51509 from dongjoon-hyun/SPARK-52815.

Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Peter Toth <[email protected]>

Author: dongjoon-hyun

common/utils/src/main/scala/org/apache/spark/util/ClosureCleaner.scala

@@ -24,7 +24,6 @@ import java.lang.reflect.{Field, Modifier}
 import scala.collection.mutable.{Map, Queue, Set, Stack}
 import scala.jdk.CollectionConverters._
 
-import org.apache.commons.lang3.ClassUtils
 import org.apache.xbean.asm9.{ClassReader, ClassVisitor, Handle, MethodVisitor, Type}
 import org.apache.xbean.asm9.Opcodes._
 import org.apache.xbean.asm9.tree.{ClassNode, MethodNode}
@@ -619,7 +618,7 @@ private[spark] object IndylambdaScalaClosures extends Logging {
   def getSerializationProxy(maybeClosure: AnyRef): Option[SerializedLambda] = {
     def isClosureCandidate(cls: Class[_]): Boolean = {
       // TODO: maybe lift this restriction to support other functional interfaces in the future
-      val implementedInterfaces = ClassUtils.getAllInterfaces(cls).asScala
+      val implementedInterfaces = SparkClassUtils.getAllInterfaces(cls)
       implementedInterfaces.exists(_.getName.startsWith("scala.Function"))
     }
 

common/utils/src/main/scala/org/apache/spark/util/SparkClassUtils.scala

@@ -18,6 +18,7 @@ package org.apache.spark.util
 
 import java.util.Random
 
+import scala.collection.mutable.LinkedHashSet
 import scala.util.Try
 
 private[spark] trait SparkClassUtils {
@@ -136,6 +137,33 @@ private[spark] trait SparkClassUtils {
       }
     }
   }
+
+  /**
+   * Gets a list of all interfaces implemented by the given class and its superclasses.
+   */
+  def getAllInterfaces(cls: Class[_]): List[Class[_]] = {
+    if (cls == null) {
+      return null
+    }
+    val interfacesFound = LinkedHashSet[Class[_]]()
+    getAllInterfacesHelper(cls, interfacesFound)
+    interfacesFound.toList
+  }
+
+  private def getAllInterfacesHelper(
+      clazz: Class[_],
+      interfacesFound: LinkedHashSet[Class[_]]): Unit = {
+    var currentClass = clazz
+    while (currentClass != null) {
+      val interfaces = currentClass.getInterfaces
+      for (i <- interfaces) {
+        if (interfacesFound.add(i)) {
+          getAllInterfacesHelper(i, interfacesFound)
+        }
+      }
+      currentClass = currentClass.getSuperclass
+    }
+  }
 }
 
 private[spark] object SparkClassUtils extends SparkClassUtils