From 271342f6984c35556221f251f406075ea39616fa Mon Sep 17 00:00:00 2001 From: Benson Arafat Date: Wed, 25 Sep 2024 11:38:52 +0100 Subject: [PATCH 1/4] Update subject segmentation native code --- .../subject_segmenter_view.dart | 3 +- .../android/build.gradle | 1 + .../SubjectSegmenterProcess.java | 201 ++++++++++++++++-- .../lib/src/subject_segmenter.dart | 85 +++++++- 4 files changed, 265 insertions(+), 25 deletions(-) diff --git a/packages/example/lib/vision_detector_views/subject_segmenter_view.dart b/packages/example/lib/vision_detector_views/subject_segmenter_view.dart index 060cb73e..9e50a0a7 100644 --- a/packages/example/lib/vision_detector_views/subject_segmenter_view.dart +++ b/packages/example/lib/vision_detector_views/subject_segmenter_view.dart @@ -11,7 +11,8 @@ class SubjectSegmenterView extends StatefulWidget { } class _SubjectSegmenterViewState extends State { - final SubjectSegmenter _segmenter = SubjectSegmenter(); + final SubjectSegmenter _segmenter = SubjectSegmenter( + options: SubjectSegmenterOptions(enableForegroundBitmap: true)); bool _canProcess = true; bool _isBusy = false; CustomPaint? _customPaint; diff --git a/packages/google_mlkit_subject_segmentation/android/build.gradle b/packages/google_mlkit_subject_segmentation/android/build.gradle index ef6545ab..d32869c8 100644 --- a/packages/google_mlkit_subject_segmentation/android/build.gradle +++ b/packages/google_mlkit_subject_segmentation/android/build.gradle @@ -41,4 +41,5 @@ android { dependencies { implementation 'com.google.android.gms:play-services-mlkit-subject-segmentation:16.0.0-beta1' + implementation files('/Users/bensonarafat/development/flutter/bin/cache/artifacts/engine/android-arm/flutter.jar') } diff --git a/packages/google_mlkit_subject_segmentation/android/src/main/java/com/google_mlkit_subject_segmentation/SubjectSegmenterProcess.java b/packages/google_mlkit_subject_segmentation/android/src/main/java/com/google_mlkit_subject_segmentation/SubjectSegmenterProcess.java index bab91b18..b75d73a1 100644 --- a/packages/google_mlkit_subject_segmentation/android/src/main/java/com/google_mlkit_subject_segmentation/SubjectSegmenterProcess.java +++ b/packages/google_mlkit_subject_segmentation/android/src/main/java/com/google_mlkit_subject_segmentation/SubjectSegmenterProcess.java @@ -1,19 +1,27 @@ package com.google_mlkit_subject_segmentation; import android.content.Context; +import android.graphics.Bitmap; import androidx.annotation.NonNull; import com.google.mlkit.vision.common.InputImage; import com.google.mlkit.vision.segmentation.subject.Subject; import com.google.mlkit.vision.segmentation.subject.SubjectSegmentation; +import com.google.mlkit.vision.segmentation.subject.SubjectSegmentationResult; import com.google.mlkit.vision.segmentation.subject.SubjectSegmenter; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.lang.reflect.Method; import java.util.ArrayList; import java.util.List; import java.nio.FloatBuffer; import java.util.HashMap; import java.util.Map; +import java.util.Objects; import io.flutter.Log; import io.flutter.plugin.common.MethodCall; @@ -55,16 +63,138 @@ public void onMethodCall(@NonNull MethodCall call, @NonNull MethodChannel.Result } } + private void handleDetection(MethodCall call, MethodChannel.Result result) { + InputImage inputImage = InputImageConverter.getInputImageFromData(call.argument("imageDate"), context, result); + if(inputImage == null) return; + + String id = call.argument("id"); + SubjectSegmenter subjectSegmenter = getOrCreateSegmenter(id, call); + + subjectSegmenter.process(inputImage) + .addOnSuccessListener(subjectSegmentationResult -> processResult(subjectSegmentationResult, call, result)) + .addOnFailureListener(e -> result.error("Subject segmentation failure!", e.getMessage(), e)); + + } + + private SubjectSegmenter getOrCreateSegmenter(String id, MethodCall call) { + return instances.computeIfAbsent(id, k -> initialize(call)); + } private SubjectSegmenter initialize(MethodCall call) { - SubjectSegmenterOptions.Builder builder = new SubjectSegmenterOptions.Builder() - .enableMultipleSubjects(new SubjectSegmenterOptions.SubjectResultOptions.Builder() - .enableConfidenceMask().build()); - SubjectSegmenterOptions options = builder.build(); - return SubjectSegmentation.getClient(options); + Map options = call.argument("options"); + SubjectSegmenterOptions.Builder builder = new SubjectSegmenterOptions.Builder(); + assert options != null; + configureBuilder(builder, options); + return SubjectSegmentation.getClient(builder.build()) ; +// Map options = call.argument("options"); +// SubjectSegmenterOptions.Builder builder = new SubjectSegmenterOptions.Builder(); +// Boolean enableForegroundBitmap = (boolean) options.get("enableForegroundBitmap"); +// Boolean enableForegroundConfidenceMask = (boolean) options.get("enableForegroundConfidenceMask"); +// Boolean enableMultiConfidenceMask = (boolean) options.get("enableMultiConfidenceMask"); +// Boolean enableMultiSubjectBitmap = (boolean) options.get("enableMultiSubjectBitmap"); +// +// if(Boolean.TRUE.equals(enableForegroundConfidenceMask)){ +// builder.enableForegroundConfidenceMask(); +// } +// if(Boolean.TRUE.equals(enableForegroundBitmap)){ +// builder.enableForegroundBitmap(); +// } +// if(Boolean.TRUE.equals(enableMultiConfidenceMask) || Boolean.TRUE.equals(enableMultiSubjectBitmap)){ +// SubjectSegmenterOptions.SubjectResultOptions.Builder subjectBuilder = +// new SubjectSegmenterOptions.SubjectResultOptions.Builder(); +// +// if(Boolean.TRUE.equals(enableMultiConfidenceMask)){ +// subjectBuilder.enableConfidenceMask(); +// } +// +// if(Boolean.TRUE.equals(enableMultiSubjectBitmap)) { +// subjectBuilder.enableSubjectBitmap(); +// } +// builder.enableMultipleSubjects(subjectBuilder.build()); +// } +// SubjectSegmenterOptions subjectSegmenterOptions = builder.build(); +// return SubjectSegmentation.getClient(subjectSegmenterOptions); + } + + private void configureBuilder(SubjectSegmenterOptions.Builder builder, Map options) { + if(Boolean.TRUE.equals(options.get("enableForegroundBitmap"))){ + builder.enableForegroundBitmap(); + } + if(Boolean.TRUE.equals(options.get("enableForegroundConfidenceMask"))){ + builder.enableForegroundConfidenceMask(); + } + configureMultipleSubjects(builder, options); + } + + private void configureMultipleSubjects(SubjectSegmenterOptions.Builder builder, Map options) { + boolean enableMultiConfidenceMask = Boolean.TRUE.equals(options.get("enableMultiConfidenceMask")) ; + boolean enableMultiSubjectBitmap = Boolean.TRUE.equals(options.get("enableMultiSubjectBitmap")); + + if(enableMultiConfidenceMask || enableMultiSubjectBitmap) { + SubjectSegmenterOptions.SubjectResultOptions.Builder subjectBuilder = new SubjectSegmenterOptions.SubjectResultOptions.Builder(); + if(enableMultiConfidenceMask) subjectBuilder.enableConfidenceMask(); + if(enableMultiSubjectBitmap) subjectBuilder.enableSubjectBitmap(); + builder.enableMultipleSubjects(subjectBuilder.build()); + } + } + + private void processResult(SubjectSegmentationResult subjectSegmentationResult, MethodCall call, MethodChannel.Result result) { + Map resultMap = new HashMap<>(); + Map options = call.argument("options"); + + assert options != null; + if(Boolean.TRUE.equals(options.get("enableForegroundBitmap"))) { + addForegroundBitmap(resultMap, subjectSegmentationResult.getForegroundBitmap()); + } + + if(Boolean.TRUE.equals(options.get("enableForegroundConfidenceMask"))){ + addConfidenceMask(resultMap, subjectSegmentationResult.getForegroundConfidenceMask()); + } + if(Boolean.TRUE.equals(options.get("enableMultiConfidenceMask")) || Boolean.TRUE.equals(options.get("enableMultiSubjectBitmap"))) { + + List> subjectsData = new ArrayList<>(); + for(Subject subject: subjectSegmentationResult.getSubjects()){ + Map subjectData = getStringObjectMap(subject, options); + subjectsData.add(subjectData); + } + resultMap.put("subjects", subjectsData); + } + addImageDimensions(resultMap, call); + + result.success(resultMap); + } + + private void addForegroundBitmap(Map map, Bitmap bitmap) { + if(bitmap != null) { + map.put("bitmap", getBitmapBytes(bitmap)); + } + } + + private void addConfidenceMask(Map map, FloatBuffer mask) { + if(mask != null) { + map.put("confidences", getConfidences(mask)); + } + } + + private void addImageDimensions(Map map, MethodCall call) { + Map imageData = call.argument("imageData"); + assert imageData != null; + map.put("width", imageData.get("width")); + map.put("height", imageData.get("height")); + } + private static float[] getConfidences(FloatBuffer floatBuffer) { + float[] confidences = new float[floatBuffer.remaining()]; + floatBuffer.get(confidences); + return confidences; + } + + private static byte[] getBitmapBytes(Bitmap bitmap) { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + bitmap.compress(Bitmap.CompressFormat.PNG, 100, outputStream); + return outputStream.toByteArray(); } private void handleDetection(MethodCall call, MethodChannel.Result result){ - Map imageData = (Map) call.argument("imageData"); + Map imageData = call.argument("imageData"); InputImage inputImage = InputImageConverter.getInputImageFromData(imageData, context, result); if (inputImage == null) return; imageHeight = inputImage.getHeight(); @@ -78,35 +208,62 @@ private void handleDetection(MethodCall call, MethodChannel.Result result){ subjectSegmenter.process(inputImage) .addOnSuccessListener( subjectSegmentationResult -> { - List> subjectsData = new ArrayList<>(); - for(Subject subject : subjectSegmentationResult.getSubjects()){ - Map subjectData = getStringObjectMap(subject); - subjectsData.add(subjectData); - } - Map map = new HashMap<>(); - map.put("subjects", subjectsData); - map.put("width", imageWidth); - map.put("height", imageHeight); + + Bitmap foregroundBitmap = subjectSegmentationResult.getForegroundBitmap(); + + FloatBuffer foregroundConfidenceMask = subjectSegmentationResult.getForegroundConfidenceMask(); + + List> subjectsData = new ArrayList<>(); + for(Subject subject : subjectSegmentationResult.getSubjects()){ + Map subjectData = getStringObjectMap(subject); + subjectsData.add(subjectData); + } + Map map = new HashMap<>(); + map.put("subjects", subjectsData); + + assert foregroundBitmap != null; + map.put("bitmap", getBitMap(foregroundBitmap)); + + assert foregroundConfidenceMask != null; + map.put("confidences", getConfidences(foregroundConfidenceMask)); + + map.put("width", imageWidth); + map.put("height", imageHeight); + result.success(map); }).addOnFailureListener( e -> result.error("Subject segmentation failed!", e.getMessage(), e) ); } @NonNull - private static Map getStringObjectMap(Subject subject) { + private static Map getStringObjectMap(Subject subject, Map options) { Map subjectData = new HashMap<>(); subjectData.put("startX", subject.getStartX()); subjectData.put("startY", subject.getStartY()); subjectData.put("width", subject.getWidth()); subjectData.put("height", subject.getHeight()); - - FloatBuffer confidenceMask = subject.getConfidenceMask(); - assert confidenceMask != null; - float[] confidences = new float[confidenceMask.remaining()]; - confidenceMask.get(confidences); - subjectData.put("confidences", confidences); + if(Boolean.TRUE.equals(options.get("enableMultiConfidenceMask"))){ + subjectData.put("confidences", getConfidences(Objects.requireNonNull(subject.getConfidenceMask()))); + } + if(Boolean.TRUE.equals(options.get("enableMultiSubjectBitmap"))) { + subjectData.put("bitmap", getBitmapBytes(Objects.requireNonNull(subject.getBitmap()))); + } return subjectData; } +// private static float[] getConfidences(FloatBuffer floatBuffer) { +// assert floatBuffer != null; +// float[] confidences = new float[floatBuffer.remaining()]; +// floatBuffer.get(confidences); +// return confidences; +// } +// private static byte[] getBitMap(Bitmap bitmap){ +// ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); +// byte[] imageBytes = null; +// assert bitmap != null; +// bitmap.compress(Bitmap.CompressFormat.PNG, 100, outputStream); +// imageBytes = outputStream.toByteArray(); +// return imageBytes; +// } private void closeDetector(MethodCall call) { String id = call.argument("id"); SubjectSegmenter subjectSegmenter = instances.get(id); diff --git a/packages/google_mlkit_subject_segmentation/lib/src/subject_segmenter.dart b/packages/google_mlkit_subject_segmentation/lib/src/subject_segmenter.dart index c5e6e534..41f6778d 100644 --- a/packages/google_mlkit_subject_segmentation/lib/src/subject_segmenter.dart +++ b/packages/google_mlkit_subject_segmentation/lib/src/subject_segmenter.dart @@ -1,6 +1,10 @@ +// ignore_for_file: unnecessary_lambdas + import 'package:flutter/services.dart'; import 'package:google_mlkit_commons/google_mlkit_commons.dart'; +import '../google_mlkit_subject_segmentation.dart'; + /// A detector that performs segmentation on a given [InputImage]. class SubjectSegmenter { /// A platform channel used to communicate with native code for segmentation @@ -10,6 +14,12 @@ class SubjectSegmenter { /// A unique identifier for the segmentation session, generated using the current timestamp final id = DateTime.now().microsecondsSinceEpoch.toString(); + /// The options for the subject segmenter + final SubjectSegmenterOptions options; + + /// Constructor to create an instance of [FaceDetector]. + SubjectSegmenter({required this.options}); + /// Processes the given [InputImage] for segmentation. /// /// Sends the [InputImage] data to the natvie platform via the method channel @@ -19,6 +29,7 @@ class SubjectSegmenter { .invokeMethod('vision#startSubjectSegmenter', { 'id': id, 'imageData': inputImage.toJson(), + 'options': options.toJson(), }); // Convert the JSON response from the platform into a SubjectSegmenterMask instance. final SubjectSegmenterMask masks = SubjectSegmenterMask.fromJson(results); @@ -33,6 +44,75 @@ class SubjectSegmenter { _channel.invokeMethod('vision#closeSubjectSegmenter', {'id': id}); } +/// Immutable options for configuring features of [FaceDetector]. +/// +/// Used to configure features such as classification, face tracking, speed, +/// etc. +class SubjectSegmenterOptions { + /// Constructor for [FaceDetectorOptions]. + /// + /// The parameter [minFaceSize] must be between 0.0 and 1.0, inclusive. + SubjectSegmenterOptions({ + this.enableForegroundConfidenceMask = false, + this.enableForegroundBitmap = false, + this.enableMultiConfidenceMask = false, + this.enableMultiSubjectBitmap = false, + }) : assert( + (enableForegroundConfidenceMask ? 1 : 0) + + (enableForegroundBitmap ? 1 : 0) + + (enableMultiConfidenceMask ? 1 : 0) + + (enableMultiSubjectBitmap ? 1 : 0) == + 1, + 'Exactly one option must be true'); + + /// + /// TODO: Comment here + /// + final bool enableForegroundConfidenceMask; + + /// + /// TODO: comment here + /// + final bool enableForegroundBitmap; + + /// + /// TODO: Comment here + /// + final bool enableMultiConfidenceMask; + + /// + /// + /// + final bool enableMultiSubjectBitmap; + + /// Returns a json representation of an instance of [SubjectSegmenterOptions]. + Map toJson() => { + 'enableForegroundConfidenceMask': enableForegroundConfidenceMask, + 'enableForegroundBitmap': enableForegroundBitmap, + 'enableMultiConfidenceMask': enableMultiConfidenceMask, + 'enableMultiSubjectBitmap': enableMultiSubjectBitmap, + }; + + // Factory constructor to ensure one option is selected if none are provided + factory SubjectSegmenterOptions.withDefaultOption() { + return SubjectSegmenterOptions(enableForegroundConfidenceMask: true); + } + + // Method to validate options + static bool areOptionsValid({ + bool enableForegroundConfidenceMask = false, + bool enableForegroundBitmap = false, + bool enableMultiConfidenceMask = false, + bool enableMultiSubjectBitmap = false, + }) { + return (enableForegroundConfidenceMask ? 1 : 0) + + (enableForegroundBitmap ? 1 : 0) + + (enableMultiConfidenceMask ? 1 : 0) + + (enableMultiSubjectBitmap ? 1 : 0) == + 1; + } +} + /// A data class that represents the segmentation mask returned by the [SubjectSegmenterMask] class SubjectSegmenterMask { /// The width of the segmentation mask @@ -56,8 +136,9 @@ class SubjectSegmenterMask { /// Returns an instance of [SubjectSegmenterMask] from json factory SubjectSegmenterMask.fromJson(Map json) { - final List> list = json['subjects']; - final List subjects = list.map(Subject.fromJson).toList(); + final List list = json['subjects']; + final List subjects = + list.map((json) => Subject.fromJson(json)).toList(); return SubjectSegmenterMask( width: json['width'] as int, height: json['height'] as int, From 162ca650a12df222a84a01a3c1a28b2aaee018b1 Mon Sep 17 00:00:00 2001 From: Benson Arafat Date: Wed, 25 Sep 2024 11:51:52 +0100 Subject: [PATCH 2/4] Remove commented code --- .../SubjectSegmenterProcess.java | 84 +------------------ 1 file changed, 1 insertion(+), 83 deletions(-) diff --git a/packages/google_mlkit_subject_segmentation/android/src/main/java/com/google_mlkit_subject_segmentation/SubjectSegmenterProcess.java b/packages/google_mlkit_subject_segmentation/android/src/main/java/com/google_mlkit_subject_segmentation/SubjectSegmenterProcess.java index b75d73a1..10e8c94d 100644 --- a/packages/google_mlkit_subject_segmentation/android/src/main/java/com/google_mlkit_subject_segmentation/SubjectSegmenterProcess.java +++ b/packages/google_mlkit_subject_segmentation/android/src/main/java/com/google_mlkit_subject_segmentation/SubjectSegmenterProcess.java @@ -84,35 +84,7 @@ private SubjectSegmenter initialize(MethodCall call) { SubjectSegmenterOptions.Builder builder = new SubjectSegmenterOptions.Builder(); assert options != null; configureBuilder(builder, options); - return SubjectSegmentation.getClient(builder.build()) ; -// Map options = call.argument("options"); -// SubjectSegmenterOptions.Builder builder = new SubjectSegmenterOptions.Builder(); -// Boolean enableForegroundBitmap = (boolean) options.get("enableForegroundBitmap"); -// Boolean enableForegroundConfidenceMask = (boolean) options.get("enableForegroundConfidenceMask"); -// Boolean enableMultiConfidenceMask = (boolean) options.get("enableMultiConfidenceMask"); -// Boolean enableMultiSubjectBitmap = (boolean) options.get("enableMultiSubjectBitmap"); -// -// if(Boolean.TRUE.equals(enableForegroundConfidenceMask)){ -// builder.enableForegroundConfidenceMask(); -// } -// if(Boolean.TRUE.equals(enableForegroundBitmap)){ -// builder.enableForegroundBitmap(); -// } -// if(Boolean.TRUE.equals(enableMultiConfidenceMask) || Boolean.TRUE.equals(enableMultiSubjectBitmap)){ -// SubjectSegmenterOptions.SubjectResultOptions.Builder subjectBuilder = -// new SubjectSegmenterOptions.SubjectResultOptions.Builder(); -// -// if(Boolean.TRUE.equals(enableMultiConfidenceMask)){ -// subjectBuilder.enableConfidenceMask(); -// } -// -// if(Boolean.TRUE.equals(enableMultiSubjectBitmap)) { -// subjectBuilder.enableSubjectBitmap(); -// } -// builder.enableMultipleSubjects(subjectBuilder.build()); -// } -// SubjectSegmenterOptions subjectSegmenterOptions = builder.build(); -// return SubjectSegmentation.getClient(subjectSegmenterOptions); + return SubjectSegmentation.getClient(builder.build()); } private void configureBuilder(SubjectSegmenterOptions.Builder builder, Map options) { @@ -193,46 +165,6 @@ private static byte[] getBitmapBytes(Bitmap bitmap) { return outputStream.toByteArray(); } - private void handleDetection(MethodCall call, MethodChannel.Result result){ - Map imageData = call.argument("imageData"); - InputImage inputImage = InputImageConverter.getInputImageFromData(imageData, context, result); - if (inputImage == null) return; - imageHeight = inputImage.getHeight(); - imageWidth = inputImage.getWidth(); - String id = call.argument("id"); - SubjectSegmenter subjectSegmenter = instances.get(id); - if (subjectSegmenter == null) { - subjectSegmenter = initialize(call); - instances.put(id, subjectSegmenter); - } - - subjectSegmenter.process(inputImage) - .addOnSuccessListener( subjectSegmentationResult -> { - - Bitmap foregroundBitmap = subjectSegmentationResult.getForegroundBitmap(); - - FloatBuffer foregroundConfidenceMask = subjectSegmentationResult.getForegroundConfidenceMask(); - - List> subjectsData = new ArrayList<>(); - for(Subject subject : subjectSegmentationResult.getSubjects()){ - Map subjectData = getStringObjectMap(subject); - subjectsData.add(subjectData); - } - Map map = new HashMap<>(); - map.put("subjects", subjectsData); - - assert foregroundBitmap != null; - map.put("bitmap", getBitMap(foregroundBitmap)); - - assert foregroundConfidenceMask != null; - map.put("confidences", getConfidences(foregroundConfidenceMask)); - - map.put("width", imageWidth); - map.put("height", imageHeight); - - result.success(map); - }).addOnFailureListener( e -> result.error("Subject segmentation failed!", e.getMessage(), e) ); - } @NonNull private static Map getStringObjectMap(Subject subject, Map options) { @@ -250,20 +182,6 @@ private static Map getStringObjectMap(Subject subject, Map Date: Fri, 27 Sep 2024 21:36:20 +0100 Subject: [PATCH 3/4] Update SubjectSegmenterOptions --- .../subject_segmentation_painter.dart | 4 +- .../subject_segmenter_view.dart | 5 +- .../android/build.gradle | 2 +- .../SubjectSegmenterProcess.java | 15 +-- .../lib/src/subject_segmenter.dart | 115 ++++++++---------- 5 files changed, 61 insertions(+), 80 deletions(-) diff --git a/packages/example/lib/vision_detector_views/painters/subject_segmentation_painter.dart b/packages/example/lib/vision_detector_views/painters/subject_segmentation_painter.dart index 5f801165..78023c99 100644 --- a/packages/example/lib/vision_detector_views/painters/subject_segmentation_painter.dart +++ b/packages/example/lib/vision_detector_views/painters/subject_segmentation_painter.dart @@ -21,7 +21,7 @@ class SubjectSegmentationPainter extends CustomPainter { void paint(Canvas canvas, Size size) { final int width = mask.width; final int height = mask.height; - final List subjects = mask.subjects; + final List subjects = mask.subjects ?? []; final paint = Paint()..style = PaintingStyle.fill; @@ -30,7 +30,7 @@ class SubjectSegmentationPainter extends CustomPainter { final int startY = subject.startY; final int subjectWidth = subject.subjectWidth; final int subjectHeight = subject.subjectHeight; - final List confidences = subject.confidences; + final List confidences = subject.confidences ?? []; for (int y = 0; y < subjectHeight; y++) { for (int x = 0; y < subjectWidth; x++) { diff --git a/packages/example/lib/vision_detector_views/subject_segmenter_view.dart b/packages/example/lib/vision_detector_views/subject_segmenter_view.dart index 9e50a0a7..4977e65d 100644 --- a/packages/example/lib/vision_detector_views/subject_segmenter_view.dart +++ b/packages/example/lib/vision_detector_views/subject_segmenter_view.dart @@ -12,7 +12,7 @@ class SubjectSegmenterView extends StatefulWidget { class _SubjectSegmenterViewState extends State { final SubjectSegmenter _segmenter = SubjectSegmenter( - options: SubjectSegmenterOptions(enableForegroundBitmap: true)); + options: SubjectSegmenterOptions(enableForegroundConfidenceMask: true)); bool _canProcess = true; bool _isBusy = false; CustomPaint? _customPaint; @@ -57,8 +57,7 @@ class _SubjectSegmenterViewState extends State { _customPaint = CustomPaint(painter: painter); } else { // TODO: set _customPaint to draw on top of image - _text = 'There is a mask with ${mask.subjects.length} subjects'; - + _text = 'There is a mask with ${mask.subjects?.length} subjects'; _customPaint = null; } _isBusy = false; diff --git a/packages/google_mlkit_subject_segmentation/android/build.gradle b/packages/google_mlkit_subject_segmentation/android/build.gradle index d32869c8..ddb79afb 100644 --- a/packages/google_mlkit_subject_segmentation/android/build.gradle +++ b/packages/google_mlkit_subject_segmentation/android/build.gradle @@ -41,5 +41,5 @@ android { dependencies { implementation 'com.google.android.gms:play-services-mlkit-subject-segmentation:16.0.0-beta1' - implementation files('/Users/bensonarafat/development/flutter/bin/cache/artifacts/engine/android-arm/flutter.jar') +// implementation files('/Users/bensonarafat/development/flutter/bin/cache/artifacts/engine/android-arm/flutter.jar') } diff --git a/packages/google_mlkit_subject_segmentation/android/src/main/java/com/google_mlkit_subject_segmentation/SubjectSegmenterProcess.java b/packages/google_mlkit_subject_segmentation/android/src/main/java/com/google_mlkit_subject_segmentation/SubjectSegmenterProcess.java index 10e8c94d..eee52a71 100644 --- a/packages/google_mlkit_subject_segmentation/android/src/main/java/com/google_mlkit_subject_segmentation/SubjectSegmenterProcess.java +++ b/packages/google_mlkit_subject_segmentation/android/src/main/java/com/google_mlkit_subject_segmentation/SubjectSegmenterProcess.java @@ -35,8 +35,6 @@ public class SubjectSegmenterProcess implements MethodChannel.MethodCallHandler private static final String CLOSE = "vision#closeSubjectSegmenter"; private final Context context; - - private static final String TAG = "Logger"; private int imageWidth; private int imageHeight; @@ -64,8 +62,10 @@ public void onMethodCall(@NonNull MethodCall call, @NonNull MethodChannel.Result } private void handleDetection(MethodCall call, MethodChannel.Result result) { - InputImage inputImage = InputImageConverter.getInputImageFromData(call.argument("imageDate"), context, result); + InputImage inputImage = InputImageConverter.getInputImageFromData(call.argument("imageData"), context, result); if(inputImage == null) return; + imageHeight = inputImage.getHeight(); + imageWidth = inputImage.getWidth(); String id = call.argument("id"); SubjectSegmenter subjectSegmenter = getOrCreateSegmenter(id, call); @@ -130,7 +130,8 @@ private void processResult(SubjectSegmentationResult subjectSegmentationResult, } resultMap.put("subjects", subjectsData); } - addImageDimensions(resultMap, call); + resultMap.put("width", imageWidth); + resultMap.put("height", imageHeight); result.success(resultMap); } @@ -147,12 +148,6 @@ private void addConfidenceMask(Map map, FloatBuffer mask) { } } - private void addImageDimensions(Map map, MethodCall call) { - Map imageData = call.argument("imageData"); - assert imageData != null; - map.put("width", imageData.get("width")); - map.put("height", imageData.get("height")); - } private static float[] getConfidences(FloatBuffer floatBuffer) { float[] confidences = new float[floatBuffer.remaining()]; floatBuffer.get(confidences); diff --git a/packages/google_mlkit_subject_segmentation/lib/src/subject_segmenter.dart b/packages/google_mlkit_subject_segmentation/lib/src/subject_segmenter.dart index 41f6778d..07ca2d94 100644 --- a/packages/google_mlkit_subject_segmentation/lib/src/subject_segmenter.dart +++ b/packages/google_mlkit_subject_segmentation/lib/src/subject_segmenter.dart @@ -17,13 +17,13 @@ class SubjectSegmenter { /// The options for the subject segmenter final SubjectSegmenterOptions options; - /// Constructor to create an instance of [FaceDetector]. + /// Constructor to create an instance of [SubjectSegmention]. SubjectSegmenter({required this.options}); /// Processes the given [InputImage] for segmentation. /// /// Sends the [InputImage] data to the natvie platform via the method channel - /// Returns the segmentation mask in the given image or nil if there was an error. + /// Returns the segmentation mask in the given image. Future processImage(InputImage inputImage) async { final results = await _channel .invokeMethod('vision#startSubjectSegmenter', { @@ -44,45 +44,36 @@ class SubjectSegmenter { _channel.invokeMethod('vision#closeSubjectSegmenter', {'id': id}); } -/// Immutable options for configuring features of [FaceDetector]. +/// Immutable options for configuring features of [SubjectSegmention]. /// -/// Used to configure features such as classification, face tracking, speed, -/// etc. +/// Used to configure features such as foreground confidence mask, foreground bitmap, multi confidence mask +/// or multi subject bitmap class SubjectSegmenterOptions { - /// Constructor for [FaceDetectorOptions]. + /// Constructor for [SubjectSegmenterOptions]. /// - /// The parameter [minFaceSize] must be between 0.0 and 1.0, inclusive. + /// The parameter to enable options + /// NOTE: To improve memory efficiency, it is recommended to only enable the necessary options. SubjectSegmenterOptions({ - this.enableForegroundConfidenceMask = false, + this.enableForegroundConfidenceMask = true, this.enableForegroundBitmap = false, this.enableMultiConfidenceMask = false, this.enableMultiSubjectBitmap = false, - }) : assert( - (enableForegroundConfidenceMask ? 1 : 0) + - (enableForegroundBitmap ? 1 : 0) + - (enableMultiConfidenceMask ? 1 : 0) + - (enableMultiSubjectBitmap ? 1 : 0) == - 1, - 'Exactly one option must be true'); + }); /// - /// TODO: Comment here - /// + /// Enables foreground confidence mask. final bool enableForegroundConfidenceMask; /// - /// TODO: comment here - /// + /// Enables foreground bitmap final bool enableForegroundBitmap; /// - /// TODO: Comment here - /// + /// Enables confidence mask for segmented Subjects final bool enableMultiConfidenceMask; /// - /// - /// + /// Enables subject bitmap for segmented Subjects. final bool enableMultiSubjectBitmap; /// Returns a json representation of an instance of [SubjectSegmenterOptions]. @@ -92,25 +83,6 @@ class SubjectSegmenterOptions { 'enableMultiConfidenceMask': enableMultiConfidenceMask, 'enableMultiSubjectBitmap': enableMultiSubjectBitmap, }; - - // Factory constructor to ensure one option is selected if none are provided - factory SubjectSegmenterOptions.withDefaultOption() { - return SubjectSegmenterOptions(enableForegroundConfidenceMask: true); - } - - // Method to validate options - static bool areOptionsValid({ - bool enableForegroundConfidenceMask = false, - bool enableForegroundBitmap = false, - bool enableMultiConfidenceMask = false, - bool enableMultiSubjectBitmap = false, - }) { - return (enableForegroundConfidenceMask ? 1 : 0) + - (enableForegroundBitmap ? 1 : 0) + - (enableMultiConfidenceMask ? 1 : 0) + - (enableMultiSubjectBitmap ? 1 : 0) == - 1; - } } /// A data class that represents the segmentation mask returned by the [SubjectSegmenterMask] @@ -121,28 +93,37 @@ class SubjectSegmenterMask { /// The height of the segmentation mask final int height; + /// The masked bitmap for the input image + final Uint8List? bitmap; + + /// A list of forground confidence mask for the input image + final List? confidences; + /// A list of subjects detected in the image, each respresented by a [Subject] instance - final List subjects; + final List? subjects; /// Constructor to create a instance of [SubjectSegmenterMask]. - /// - /// The [width] and [height] represent the dimensions of the mark, - /// and [subjects] is a list of detected subjects SubjectSegmenterMask({ required this.width, required this.height, - required this.subjects, + this.subjects, + this.bitmap, + this.confidences, }); /// Returns an instance of [SubjectSegmenterMask] from json factory SubjectSegmenterMask.fromJson(Map json) { - final List list = json['subjects']; - final List subjects = - list.map((json) => Subject.fromJson(json)).toList(); + List? subjects; + if (json['subjects'] != null) { + subjects = + json['subjects'].map((json) => Subject.fromJson(json)).toList(); + } return SubjectSegmenterMask( width: json['width'] as int, height: json['height'] as int, subjects: subjects, + confidences: json['confidences'], + bitmap: json['bitmap'], ); } } @@ -162,25 +143,31 @@ class Subject { final int subjectHeight; /// A list of confidence values for the detected subject. - final List confidences; - - Subject( - {required this.startX, - required this.startY, - required this.subjectWidth, - required this.subjectHeight, - required this.confidences}); + final List? confidences; + + /// The masked bitmap of the subject + final Uint8List? bitmap; + + Subject({ + required this.startX, + required this.startY, + required this.subjectWidth, + required this.subjectHeight, + this.confidences, + this.bitmap, + }); /// Creates an instance of [Subject] from a JSON object. /// /// This factory constructor is used to convert JSON data into a [Subject] object. - factory Subject.fromJson(Map json) { return Subject( - startX: json['startX'] as int, - startY: json['startY'] as int, - subjectWidth: json['width'] as int, - subjectHeight: json['height'] as int, - confidences: json['confidences']); + startX: json['startX'] as int, + startY: json['startY'] as int, + subjectWidth: json['width'] as int, + subjectHeight: json['height'] as int, + confidences: json['confidences'], + bitmap: json['bitmap'], + ); } } From 1b5cc5653f4112f321bfadcf7d915ced2254edeb Mon Sep 17 00:00:00 2001 From: Benson Arafat Date: Fri, 27 Sep 2024 21:50:09 +0100 Subject: [PATCH 4/4] Update readme --- packages/google_mlkit_subject_segmentation/README.md | 3 ++- .../google_mlkit_subject_segmentation/android/build.gradle | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/google_mlkit_subject_segmentation/README.md b/packages/google_mlkit_subject_segmentation/README.md index a0eea991..34364349 100644 --- a/packages/google_mlkit_subject_segmentation/README.md +++ b/packages/google_mlkit_subject_segmentation/README.md @@ -81,7 +81,8 @@ final InputImage inputImage; #### Create an instance of `SubjectSegmenter` ```dart -final segmenter = SubjectSegmenter(); +final options = SubjectSegmenterOptions(); +final segmenter = SubjectSegmenter(options: options); ``` #### Process image diff --git a/packages/google_mlkit_subject_segmentation/android/build.gradle b/packages/google_mlkit_subject_segmentation/android/build.gradle index ddb79afb..ef6545ab 100644 --- a/packages/google_mlkit_subject_segmentation/android/build.gradle +++ b/packages/google_mlkit_subject_segmentation/android/build.gradle @@ -41,5 +41,4 @@ android { dependencies { implementation 'com.google.android.gms:play-services-mlkit-subject-segmentation:16.0.0-beta1' -// implementation files('/Users/bensonarafat/development/flutter/bin/cache/artifacts/engine/android-arm/flutter.jar') }