feat: Add ambi be ent-luminance stream for measured-dark torch decisions (fixes #693)

README.md

@@ -37,6 +37,7 @@ See the example app for detailed implementation information.
 | lensType     | :heavy_check_mark: | :heavy_check_mark: | :x:                | :x: |
 | getSupportedLenses(facing:) | :heavy_check_mark: | :heavy_check_mark: | :x: | :x: |
 | getBestCloseRangeScanningLens | :heavy_check_mark: (always normal) | :heavy_check_mark: (requires iOS 15, falls back to normal) | :x: (always normal) | :x: (always normal) |
+| luminanceStream | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :x: (never emits) |
 
 ### Querying supported lens types with facing filter
 
@@ -82,6 +83,28 @@ This returns:
 - **Web**: `CameraLensType.normal` when the device has a camera. The MediaDevices API has no concept of lens type, and its `focusDistance` capability, where available at all, is limited to Chrome on Android
 - **All platforms**: `null` if there is no camera for the requested facing direction
 
+### Ambient-luminance sampling (e.g. for auto-enabling the torch in the dark)
+
+`analyzeImage`/`returnImage` only surface a frame once a barcode decodes, so they can't measure a scene that's too dark to decode at all — which is exactly the scene you'd want to detect to offer a "turn on the torch?" prompt. `luminanceStream` fixes that gap: it emits an ambient-brightness sample (`0.0` = black, `255.0` = white) roughly every 500ms, on every analyzed frame, regardless of whether anything decodes.
+
+It's opt-in and off by default — call `setLuminanceEnabled(true)` to start sampling, and `setLuminanceEnabled(false)` (e.g. on pause/stop) to turn it back off, so apps that don't need a brightness signal pay no extra cost:
+
+```dart
+await controller.setLuminanceEnabled(enabled: true);
+
+final subscription = controller.luminanceStream.listen((luminance) {
+  if (luminance < 50 && controller.value.torchState == TorchState.off) {
+    controller.toggleTorch();
+  }
+});
+
+// ...later, e.g. when the scanner is paused or disposed:
+await subscription.cancel();
+await controller.setLuminanceEnabled(enabled: false);
+```
+
+This only decides *when to measure* darkness — whether and how to react (a one-shot auto-enable, a debounce, a threshold) is left to the app, since that policy varies (e.g. avoiding repeated toggles once the torch is already on and lighting the scene).
+
 ## Installation
 
 Add the dependency in your `pubspec.yaml` file:

android/src/main/kotlin/dev/steenbakker/mobile_scanner/MobileScanner.kt

@@ -61,6 +61,7 @@ class MobileScanner(
     private val deviceOrientationListener: DeviceOrientationListener,
     private val barcodeScannerFactory: (options: BarcodeScannerOptions?) -> BarcodeScanner = ::defaultBarcodeScannerFactory,
     @VisibleForTesting internal val inputImageFactory: (image: Image, rotationDegrees: Int) -> InputImage = InputImage::fromMediaImage,
+    private val luminanceCallback: LuminanceCallback = {},
 ) {
 
     init {
@@ -80,6 +81,15 @@ class MobileScanner(
     private var imageAnalysis: ImageAnalysis? = null
     private var analysisExecutor = Executors.newSingleThreadExecutor()
 
+    /// Wall-clock (ms) of the last emitted luminance sample, for the ~500ms throttle.
+    private var lastLuminanceTimestamp: Long = 0L
+
+    /// Whether ambient-luminance sampling is active. Off by default: computing and
+    /// emitting a sample on every analyzed frame is wasted work for consumers who
+    /// never look at [MobileScannerController.luminanceStream], so it only runs
+    /// once a consumer opts in via [setLuminanceEnabled].
+    var luminanceEnabled: Boolean = false
+
     /// Configurable variables
     var scanWindow: List<Float>? = null
     private var invertImage: Boolean = false
@@ -122,6 +132,53 @@ class MobileScanner(
             return@Analyzer
         }
 
+        // Emit a throttled ambient-luminance sample from the Y plane on every
+        // frame, independent of whether a barcode decodes — this is what lets a
+        // consumer auto-enable a torch when the scene is genuinely dark, even
+        // when nothing decodes (see #693). The Y plane *is* luminance, so this is
+        // just a cheap subsample; runs on the analyzer executor, off the main
+        // thread, and never touches the buffer MLKit scans below.
+        //
+        // Indexed through rowStride/pixelStride rather than the plane's raw
+        // buffer offset: when rowStride > width the Y plane carries per-row
+        // padding bytes (commonly 0) that a linear scan would average in,
+        // biasing the sample darker than the actual scene.
+        if (luminanceEnabled) {
+            val luminanceNow = System.currentTimeMillis()
+            if (luminanceNow - lastLuminanceTimestamp >= 500L) {
+                lastLuminanceTimestamp = luminanceNow
+                try {
+                    val plane = mediaImage.planes[0]
+                    val yBuffer = plane.buffer
+                    val rowStride = plane.rowStride
+                    val pixelStride = plane.pixelStride
+                    val width = mediaImage.width
+                    val height = mediaImage.height
+                    if (width > 0 && height > 0) {
+                        val cols = 16
+                        val rows = 16
+                        var sum = 0L
+                        var count = 0
+                        for (r in 0 until rows) {
+                            val y = r * height / rows
+                            val rowStart = y * rowStride
+                            for (c in 0 until cols) {
+                                val x = c * width / cols
+                                val idx = rowStart + x * pixelStride
+                                if (idx < yBuffer.limit()) {
+                                    sum += (yBuffer.get(idx).toInt() and 0xFF)
+                                    count++
+                                }
+                            }
+                        }
+                        if (count > 0) luminanceCallback(sum.toDouble() / count)
+                    }
+                } catch (_: Exception) {
+                    // Best-effort only; a sampling failure must never affect scanning.
+                }
+            }
+        }
+
         if (detectionSpeed == DetectionSpeed.NORMAL && scannerTimeout) {
             imageProxy.close()
             return@Analyzer

android/src/main/kotlin/dev/steenbakker/mobile_scanner/MobileScannerCallbacks.kt

@@ -8,4 +8,5 @@ typealias AnalyzerSuccessCallback = (barcodes: List<Map<String, Any?>>) -> Unit
 typealias MobileScannerErrorCallback = (error: String) -> Unit
 typealias TorchStateCallback = (state: Int) -> Unit
 typealias ZoomScaleStateCallback = (zoomScale: Double) -> Unit
-typealias MobileScannerStartedCallback = (parameters: MobileScannerStartParameters) -> Unit
+typealias MobileScannerStartedCallback = (parameters: MobileScannerStartParameters) -> Unit
+typealias LuminanceCallback = (luminance: Double) -> Unit

android/src/main/kotlin/dev/steenbakker/mobile_scanner/MobileScannerHandler.kt

@@ -97,6 +97,14 @@ class MobileScannerHandler(
         barcodeHandler.publishEvent(mapOf("name" to "zoomScaleState", "data" to zoomScale))
     }
 
+    /**
+     * Ambient-luminance samples (0-255), emitted while [MobileScanner.luminanceEnabled]
+     * is on. See [MobileScannerController.luminanceStream].
+     */
+    private val luminanceCallback: LuminanceCallback = {luminance: Double ->
+        barcodeHandler.publishEvent(mapOf("name" to "luminance", "data" to luminance))
+    }
+
     init {
         methodChannel = MethodChannel(binaryMessenger,
             "dev.steenbakker.mobile_scanner/scanner/method")
@@ -109,7 +117,8 @@ class MobileScannerHandler(
         deviceOrientationChannel!!.setStreamHandler(deviceOrientationListener)
 
         mobileScanner = MobileScanner(
-            activity, textureRegistry, callback, errorCallback, deviceOrientationListener)
+            activity, textureRegistry, callback, errorCallback, deviceOrientationListener,
+            luminanceCallback = luminanceCallback)
     }
 
     fun dispose(activityPluginBinding: ActivityPluginBinding) {
@@ -153,6 +162,7 @@ class MobileScannerHandler(
             "pause" -> pause(call, result)
             "stop" -> stop(call, result)
             "toggleTorch" -> toggleTorch(result)
+            "setLuminanceEnabled" -> setLuminanceEnabled(call, result)
             "getSupportedLenses" -> getSupportedLenses(call, result)
             "getBestCloseRangeScanningLens" -> getBestCloseRangeScanningLens(result)
             "analyzeImage" -> analyzeImage(call, result)
@@ -298,6 +308,15 @@ class MobileScannerHandler(
         result.success(null)
     }
 
+    /**
+     * Turns the native ambient-luminance sampler on or off. Off by default, so a
+     * consumer that never calls this pays no sampling cost.
+     */
+    private fun setLuminanceEnabled(call: MethodCall, result: MethodChannel.Result) {
+        mobileScanner?.luminanceEnabled = (call.arguments as? Boolean) ?: false
+        result.success(null)
+    }
+
     /**
      * Get the list of supported lens types on this device for a given facing direction.
      *

darwin/mobile_scanner/Sources/mobile_scanner/MobileScannerPlugin.swift

@@ -52,6 +52,15 @@ public class MobileScannerPlugin: NSObject, FlutterPlugin, FlutterStreamHandler,
 
     var standardZoomFactor: CGFloat = 1
 
+    /// Whether ambient-luminance sampling is active. Off by default: computing
+    /// and emitting a sample on every frame is wasted work for consumers who
+    /// never listen to `MobileScannerController.luminanceStream`, so it only
+    /// runs once a consumer opts in via `setLuminanceEnabled`.
+    var luminanceEnabled: Bool = false
+
+    /// Wall-clock (seconds) of the last emitted luminance sample, for the ~500ms throttle.
+    var nextLuminanceTime: Double = 0
+
 #if os(iOS)
     var interfaceOrientationObserver: NSObjectProtocol?
 #endif
@@ -149,6 +158,8 @@ public class MobileScannerPlugin: NSObject, FlutterPlugin, FlutterStreamHandler,
             start(call, result)
         case "toggleTorch":
             toggleTorch(result)
+        case "setLuminanceEnabled":
+            setLuminanceEnabled(call, result)
         case "getSupportedLenses":
             getSupportedLenses(call, result)
         case "getBestCloseRangeScanningLens":
@@ -235,7 +246,63 @@ public class MobileScannerPlugin: NSObject, FlutterPlugin, FlutterStreamHandler,
 
     var nextScanTime = 0.0
     var imagesCurrentlyBeingProcessed = false
-
+
+    /// Cheap average luminance (0-255) of a frame, sampled on a 16x16 grid.
+    /// Handles both planar (YUV — plane 0 is luma) and packed BGRA pixel
+    /// buffers, since `videoSettings` can negotiate either depending on the
+    /// device/format. Indexed via `bytesPerRow`, so per-row padding (when the
+    /// stride exceeds the pixel width) is never averaged in. A video-range
+    /// planar buffer carries luma in [16, 235], so it is expanded to the full
+    /// [0, 255] range to match the packed-BGRA path and the documented scale.
+    private static func averageLuminance(_ buffer: CVPixelBuffer) -> Double {
+        CVPixelBufferLockBaseAddress(buffer, .readOnly)
+        defer { CVPixelBufferUnlockBaseAddress(buffer, .readOnly) }
+        let samples = 16
+        var sum = 0.0
+        var count = 0
+        if CVPixelBufferIsPlanar(buffer) {
+            guard let base = CVPixelBufferGetBaseAddressOfPlane(buffer, 0) else { return 255.0 }
+            let w = CVPixelBufferGetWidthOfPlane(buffer, 0)
+            let h = CVPixelBufferGetHeightOfPlane(buffer, 0)
+            let bpr = CVPixelBufferGetBytesPerRowOfPlane(buffer, 0)
+            let ptr = base.assumingMemoryBound(to: UInt8.self)
+            let sx = max(1, w / samples), sy = max(1, h / samples)
+            var y = 0
+            while y < h {
+                var x = 0
+                while x < w { sum += Double(ptr[y * bpr + x]); count += 1; x += sx }
+                y += sy
+            }
+        } else {
+            guard let base = CVPixelBufferGetBaseAddress(buffer) else { return 255.0 }
+            let w = CVPixelBufferGetWidth(buffer)
+            let h = CVPixelBufferGetHeight(buffer)
+            let bpr = CVPixelBufferGetBytesPerRow(buffer)
+            let ptr = base.assumingMemoryBound(to: UInt8.self)
+            let sx = max(1, w / samples), sy = max(1, h / samples)
+            var y = 0
+            while y < h {
+                var x = 0
+                while x < w {
+                    let p = y * bpr + x * 4
+                    let b = Double(ptr[p]), g = Double(ptr[p + 1]), r = Double(ptr[p + 2])
+                    sum += 0.299 * r + 0.587 * g + 0.114 * b
+                    count += 1
+                    x += sx
+                }
+                y += sy
+            }
+        }
+        guard count > 0 else { return 255.0 }
+        let mean = sum / Double(count)
+        if CVPixelBufferIsPlanar(buffer),
+           CVPixelBufferGetPixelFormatType(buffer)
+               == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange {
+            return min(255.0, max(0.0, (mean - 16.0) * 255.0 / 219.0))
+        }
+        return mean
+    }
+
     // Gets called when a new image is added to the buffer
     public func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
         // Ignore invalid texture id.
@@ -248,6 +315,23 @@ public class MobileScannerPlugin: NSObject, FlutterPlugin, FlutterStreamHandler,
         latestBuffer = imageBuffer
         notifyFrameAvailable(for: textureId)
 
+        // Emit a throttled ambient-luminance sample on every frame — independent
+        // of barcode detection — so a consumer can auto-enable the torch when the
+        // scene is genuinely dark, even when nothing decodes (see #693). Sampled
+        // cheaply on this background queue; the sink call is hopped to the main
+        // thread. Gated by `luminanceEnabled` so it costs nothing unless a
+        // consumer opts in via `setLuminanceEnabled`.
+        if luminanceEnabled {
+            let luminanceNow = Date().timeIntervalSince1970
+            if luminanceNow >= nextLuminanceTime {
+                nextLuminanceTime = luminanceNow + 0.5
+                let luma = MobileScannerPlugin.averageLuminance(imageBuffer)
+                DispatchQueue.main.async { [weak self] in
+                    self?.sink?(["name": "luminance", "data": luma])
+                }
+            }
+        }
+
         let currentTime = Date().timeIntervalSince1970
         let eligibleForScan = currentTime > nextScanTime && !imagesCurrentlyBeingProcessed
         if ((detectionSpeed == DetectionSpeed.normal || detectionSpeed == DetectionSpeed.noDuplicates) && eligibleForScan || detectionSpeed == DetectionSpeed.unrestricted) {
@@ -867,6 +951,13 @@ public class MobileScannerPlugin: NSObject, FlutterPlugin, FlutterStreamHandler,
         return (actualScale - 1) / 4
     }
 
+    /// Turns the native ambient-luminance sampler on or off. Off by default, so
+    /// a consumer that never calls this pays no sampling cost.
+    private func setLuminanceEnabled(_ call: FlutterMethodCall, _ result: @escaping FlutterResult) {
+        luminanceEnabled = (call.arguments as? Bool) ?? false
+        result(nil)
+    }
+
     private func toggleTorch(_ result: @escaping FlutterResult) {
         guard let device = self.device else {
             result(nil)

lib/src/method_channel/mobile_scanner_method_channel.dart

@@ -43,6 +43,14 @@ class MethodChannelMobileScanner extends MobileScannerPlatform {
   @visibleForTesting
   static const String kZoomScaleStateEventName = 'zoomScaleState';
 
+  /// The name of the ambient-luminance event.
+  ///
+  /// Emitted while luminance sampling is enabled (see
+  /// [kSetLuminanceEnabledMethodName]), throttled to roughly once every
+  /// 500ms, independent of barcode detection.
+  @visibleForTesting
+  static const String kLuminanceEventName = 'luminance';
+
   /// The name of the method that gets the camera authorization state.
   @visibleForTesting
   static const String kAuthorizationStateMethodName = 'state';
@@ -83,6 +91,11 @@ class MethodChannelMobileScanner extends MobileScannerPlatform {
   @visibleForTesting
   static const String kToggleTorchMethodName = 'toggleTorch';
 
+  /// The name of the method that enables or disables ambient-luminance
+  /// sampling.
+  @visibleForTesting
+  static const String kSetLuminanceEnabledMethodName = 'setLuminanceEnabled';
+
   /// The name of the method that updates the scan window.
   @visibleForTesting
   static const String kUpdateScanWindowMethodName = 'updateScanWindow';
@@ -266,6 +279,22 @@ class MethodChannelMobileScanner extends MobileScannerPlatform {
         .map((event) => event['data'] as double? ?? 0.0);
   }
 
+  /// The stream of ambient-luminance samples (0-255, where 0 is black and 255
+  /// is white), throttled to roughly once every 500ms.
+  ///
+  /// Unlike [barcodesStream], these samples are emitted on every analyzed
+  /// frame regardless of whether a barcode decodes, so a fully dark scene
+  /// (where nothing decodes) can still be measured — see [setLuminanceEnabled].
+  ///
+  /// Defaults a malformed event to `255.0` (bright), so a bad payload can
+  /// never be mistaken for a dark reading.
+  @override
+  Stream<double> get luminanceStream {
+    return eventsStream
+        .where((event) => event['name'] == kLuminanceEventName)
+        .map((event) => (event['data'] as num?)?.toDouble() ?? 255.0);
+  }
+
   @override
   Future<BarcodeCapture?> analyzeImage(
     String path, {
@@ -499,6 +528,20 @@ class MethodChannelMobileScanner extends MobileScannerPlatform {
     await methodChannel.invokeMethod<void>(kToggleTorchMethodName);
   }
 
+  /// Enables or disables the native ambient-luminance sampler that feeds
+  /// [luminanceStream].
+  ///
+  /// Off by default: a consumer that never calls this pays no native sampling
+  /// cost, so it is safe to leave disabled for the common case where the app
+  /// does not need a brightness signal.
+  @override
+  Future<void> setLuminanceEnabled({required bool enabled}) async {
+    await methodChannel.invokeMethod<void>(
+      kSetLuminanceEnabledMethodName,
+      enabled,
+    );
+  }
+
   @override
   Future<void> updateScanWindow(Rect? window) async {
     if (_textureId == null) {