netlib.allowNativeBlas

Author: pan3793

common/utils/src/main/scala/org/apache/spark/util/SparkEnvUtils.scala

@@ -28,9 +28,9 @@ private[spark] trait SparkEnvUtils {
   }
 
   /**
-   * Whether allow using native BLAS library if available.
+   * Whether allow using native BLAS/LAPACK/ARPACK libraries if available.
    */
-  val allowNativeBlas = "true".equals(System.getProperty("spark.ml.allowNativeBlas", "true"))
+  val allowNativeBlas = "true".equals(System.getProperty("netlib.allowNativeBlas", "true"))
 }
 
 object SparkEnvUtils extends SparkEnvUtils

core/src/main/scala/org/apache/spark/SparkContext.scala

@@ -3435,7 +3435,7 @@ object SparkContext extends Logging {
   }
 
   private def supplementBlasOptions(conf: SparkConf): Unit = {
-    val opts = s"-Dspark.ml.allowNativeBlas=${Utils.allowNativeBlas}"
+    val opts = s"-Dnetlib.allowNativeBlas=${Utils.allowNativeBlas}"
     supplementJavaOpts(conf, DRIVER_JAVA_OPTIONS, opts)
     supplementJavaOpts(conf, EXECUTOR_JAVA_OPTIONS, opts)
   }

core/src/main/scala/org/apache/spark/internal/config/package.scala

@@ -2893,4 +2893,12 @@ package object config {
       .checkValue(v => v.forall(Set("stdout", "stderr").contains),
         "The value only can be one or more of 'stdout, stderr'.")
       .createWithDefault(Seq("stdout", "stderr"))
+
+  private[spark] val SPARK_ML_ALLOW_NATIVE_BLAS =
+    ConfigBuilder("spark.ml.allowNativeBlas")
+      .doc("Whether allow using native BLAS/LAPACK/ARPACK implementations when native " +
+        "libraries are available. If disabled, always use Java implementations.")
+      .version("4.1.0")
+      .booleanConf
+      .createWithDefault(true)
 }

docs/ml-linalg-guide.md

@@ -46,8 +46,7 @@ The installation should be done on all nodes of the cluster. Generic version of
 
 For Debian / Ubuntu:
 ```
-sudo apt-get install libopenblas-base
-sudo update-alternatives --config libblas.so.3
+sudo apt-get install libopenblas-dev
 ```
 For CentOS / RHEL:
 ```
@@ -76,7 +75,7 @@ You can also point `dev.ludovic.netlib` to specific libraries names and paths. F
 
 If native libraries are not properly configured in the system, the Java implementation (javaBLAS) will be used as fallback option.
 
-You can also set `spark.ml.allowNativeBlas` to `false` to disable native BLAS and always use the Java implementation.
+You can also set spark conf `spark.ml.allowNativeBlas` or Java system property `netlib.allowNativeBlas` to `false` to disable native BLAS and always use the Java implementation.
 
 ## Spark Configuration
 

launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java

@@ -340,7 +340,7 @@ private List<String> buildSparkSubmitCommand(Map<String, String> env)
 
     if (config.containsKey("spark.ml.allowNativeBlas")) {
       String allowNativeBlas = config.get("spark.ml.allowNativeBlas");
-      addOptionString(cmd, "-Dspark.ml.allowNativeBlas=" + allowNativeBlas);
+      addOptionString(cmd, "-Dnetlib.allowNativeBlas=" + allowNativeBlas);
     }
 
     // SPARK-36796: Always add some JVM runtime default options to submit command

mllib-local/src/main/scala/org/apache/spark/ml/linalg/BLAS.scala

@@ -45,7 +45,7 @@ private[spark] object BLAS extends Serializable with Logging {
       _nativeBLAS = if (SparkEnvUtils.allowNativeBlas) {
         try { NetlibNativeBLAS.getInstance } catch { case _: Throwable => javaBLAS }
       } else {
-        logInfo("Disable native BLAS because spark.ml.allowNativeBlas is false.")
+        logInfo("Disable native BLAS because netlib.allowNativeBlas is false.")
         javaBLAS
       }
     }

mllib/src/main/scala/org/apache/spark/mllib/linalg/ARPACK.scala

@@ -19,10 +19,13 @@ package org.apache.spark.mllib.linalg
 
 import dev.ludovic.netlib.arpack.{ARPACK => NetlibARPACK, JavaARPACK => NetlibJavaARPACK, NativeARPACK => NetlibNativeARPACK}
 
+import org.apache.spark.internal.Logging
+import org.apache.spark.util.SparkEnvUtils
+
 /**
  * ARPACK routines for MLlib's vectors and matrices.
  */
-private[spark] object ARPACK extends Serializable {
+private[spark] object ARPACK extends Serializable with Logging {
 
   @transient private var _javaARPACK: NetlibARPACK = _
   @transient private var _nativeARPACK: NetlibARPACK = _
@@ -36,8 +39,12 @@ private[spark] object ARPACK extends Serializable {
 
   private[spark] def nativeARPACK: NetlibARPACK = {
     if (_nativeARPACK == null) {
-      _nativeARPACK =
+      _nativeARPACK = if (SparkEnvUtils.allowNativeBlas) {
         try { NetlibNativeARPACK.getInstance } catch { case _: Throwable => javaARPACK }
+      } else {
+        logInfo("Disable native ARPACK because netlib.allowNativeBlas is false.")
+        javaARPACK
+      }
     }
     _nativeARPACK
   }

mllib/src/main/scala/org/apache/spark/mllib/linalg/BLAS.scala

@@ -17,44 +17,14 @@
 
 package org.apache.spark.mllib.linalg
 
-import dev.ludovic.netlib.blas.{BLAS => NetlibBLAS, JavaBLAS => NetlibJavaBLAS, NativeBLAS => NetlibNativeBLAS}
-
 import org.apache.spark.internal.Logging
+import org.apache.spark.ml.linalg.BLAS.{getBLAS, nativeBLAS}
 
 /**
  * BLAS routines for MLlib's vectors and matrices.
  */
 private[spark] object BLAS extends Serializable with Logging {
 
-  @transient private var _javaBLAS: NetlibBLAS = _
-  @transient private var _nativeBLAS: NetlibBLAS = _
-  private val nativeL1Threshold: Int = 256
-
-  // For level-1 function dspmv, use javaBLAS for better performance.
-  private[spark] def javaBLAS: NetlibBLAS = {
-    if (_javaBLAS == null) {
-      _javaBLAS = NetlibJavaBLAS.getInstance
-    }
-    _javaBLAS
-  }
-
-  // For level-3 routines, we use the native BLAS.
-  private[spark] def nativeBLAS: NetlibBLAS = {
-    if (_nativeBLAS == null) {
-      _nativeBLAS =
-        try { NetlibNativeBLAS.getInstance } catch { case _: Throwable => javaBLAS }
-    }
-    _nativeBLAS
-  }
-
-  private[spark] def getBLAS(vectorSize: Int): NetlibBLAS = {
-    if (vectorSize < nativeL1Threshold) {
-      javaBLAS
-    } else {
-      nativeBLAS
-    }
-  }
-
   /**
    * y += a * x
    */

mllib/src/main/scala/org/apache/spark/mllib/linalg/LAPACK.scala

@@ -19,10 +19,13 @@ package org.apache.spark.mllib.linalg
 
 import dev.ludovic.netlib.lapack.{JavaLAPACK => NetlibJavaLAPACK, LAPACK => NetlibLAPACK, NativeLAPACK => NetlibNativeLAPACK}
 
+import org.apache.spark.internal.Logging
+import org.apache.spark.util.SparkEnvUtils
+
 /**
  * LAPACK routines for MLlib's vectors and matrices.
  */
-private[spark] object LAPACK extends Serializable {
+private[spark] object LAPACK extends Serializable with Logging {
 
   @transient private var _javaLAPACK: NetlibLAPACK = _
   @transient private var _nativeLAPACK: NetlibLAPACK = _
@@ -36,8 +39,12 @@ private[spark] object LAPACK extends Serializable {
 
   private[spark] def nativeLAPACK: NetlibLAPACK = {
     if (_nativeLAPACK == null) {
-      _nativeLAPACK =
+      _nativeLAPACK = if (SparkEnvUtils.allowNativeBlas) {
         try { NetlibNativeLAPACK.getInstance } catch { case _: Throwable => javaLAPACK }
+      } else {
+        logInfo("Disable native LAPACK because netlib.allowNativeBlas is false.")
+        javaLAPACK
+      }
     }
     _nativeLAPACK
   }

mllib/src/main/scala/org/apache/spark/mllib/linalg/Matrices.scala

@@ -427,7 +427,7 @@ class DenseMatrix @Since("1.3.0") (
     if (isTransposed) {
       Iterator.tabulate(numCols) { j =>
         val col = new Array[Double](numRows)
-        BLAS.nativeBLAS.dcopy(numRows, values, j, numCols, col, 0, 1)
+        newlinalg.BLAS.nativeBLAS.dcopy(numRows, values, j, numCols, col, 0, 1)
         new DenseVector(col)
       }
     } else {