feat: Subject Segmentation (#666)

Author: bensonarafat

.github/workflows/code-analysis.yml

@@ -91,6 +91,10 @@ jobs:
       working-directory: ./packages/google_mlkit_selfie_segmentation
       run: flutter pub get
 
+    - name: Install dependencies for google_mlkit_subject_segmentation
+      working-directory: ./packages/google_mlkit_subject_segmentation
+      run: flutter pub get
+
     - name: Install dependencies for example
       working-directory: ./packages/example
       run: flutter pub get

README.md

@@ -17,7 +17,7 @@ Google's ML Kit for Flutter is a set of [Flutter plugins](https://flutter.io/pla
 | [Digital Ink Recognition](https://developers.google.com/ml-kit/vision/digital-ink-recognition) | [google\_mlkit\_digital\_ink\_recognition](https://pub.dev/packages/google_mlkit_digital_ink_recognition) [![Pub Version](https://img.shields.io/pub/v/google_mlkit_digital_ink_recognition)](https://pub.dev/packages/google_mlkit_digital_ink_recognition) | [![GitHub](https://img.shields.io/badge/github-%23121011.svg?style=for-the-badge&logo=github&logoColor=white)](https://github.com/flutter-ml/google_ml_kit_flutter/tree/master/packages/google_mlkit_digital_ink_recognition) | ✅       | ✅   |
 | [Pose Detection](https://developers.google.com/ml-kit/vision/pose-detection)                   | [google\_mlkit\_pose\_detection](https://pub.dev/packages/google_mlkit_pose_detection) [![Pub Version](https://img.shields.io/pub/v/google_mlkit_pose_detection)](https://pub.dev/packages/google_mlkit_pose_detection)                                      | [![GitHub](https://img.shields.io/badge/github-%23121011.svg?style=for-the-badge&logo=github&logoColor=white)](https://github.com/flutter-ml/google_ml_kit_flutter/tree/master/packages/google_mlkit_pose_detection)          | ✅       | ✅   |
 | [Selfie Segmentation](https://developers.google.com/ml-kit/vision/selfie-segmentation)         | [google\_mlkit\_selfie\_segmentation](https://pub.dev/packages/google_mlkit_selfie_segmentation) [![Pub Version](https://img.shields.io/pub/v/google_mlkit_selfie_segmentation)](https://pub.dev/packages/google_mlkit_selfie_segmentation)                  | [![GitHub](https://img.shields.io/badge/github-%23121011.svg?style=for-the-badge&logo=github&logoColor=white)](https://github.com/flutter-ml/google_ml_kit_flutter/tree/master/packages/google_mlkit_selfie_segmentation)     | ✅       | ✅   |
-| [Subject Segmentation](https://developers.google.com/ml-kit/vision/subject-segmentation)       | [google\_mlkit\_subject\_segemtation](https://pub.dev/packages/google_mlkit_subject_segemtation) [![Pub Version](https://img.shields.io/pub/v/google_mlkit_subject_segemtation)](https://pub.dev/packages/google_mlkit_subject_segemtation)                  | [![GitHub](https://img.shields.io/badge/github-%23121011.svg?style=for-the-badge&logo=github&logoColor=white)](https://github.com/flutter-ml/google_ml_kit_flutter/tree/master/packages/google_mlkit_subject_segemtation)     | ❌       | ❌   |
+| [Subject Segmentation](https://developers.google.com/ml-kit/vision/subject-segmentation)       | [google\_mlkit\_subject\_segemtation](https://pub.dev/packages/google_mlkit_subject_segemtation) [![Pub Version](https://img.shields.io/pub/v/google_mlkit_subject_segemtation)](https://pub.dev/packages/google_mlkit_subject_segemtation)                  | [![GitHub](https://img.shields.io/badge/github-%23121011.svg?style=for-the-badge&logo=github&logoColor=white)](https://github.com/flutter-ml/google_ml_kit_flutter/tree/master/packages/google_mlkit_subject_segemtation)     |  ✅         | ❌   |
 | [Document Scanner](https://developers.google.com/ml-kit/vision/doc-scanner)                    | [google\_mlkit\_document\_scanner](https://pub.dev/packages/google_mlkit_document_scanner) [![Pub Version](https://img.shields.io/pub/v/google_mlkit_document_scanner)](https://pub.dev/packages/google_mlkit_document_scanner)                              | [![GitHub](https://img.shields.io/badge/github-%23121011.svg?style=for-the-badge&logo=github&logoColor=white)](https://github.com/flutter-ml/google_ml_kit_flutter/tree/master/packages/google_mlkit_document_scanner)        | ✅       | ❌   |
 
 ### Natural Language APIs

packages/example/android/app/build.gradle

@@ -42,7 +42,7 @@ android {
 
     defaultConfig {
         applicationId "com.google.ml.kit.flutter.example"
-        minSdkVersion 21
+        minSdkVersion 24
         targetSdkVersion 34
         versionCode flutterVersionCode.toInteger()
         versionName flutterVersionName

packages/example/android/app/src/main/AndroidManifest.xml

@@ -30,7 +30,7 @@
         </activity>
         <meta-data
           android:name="com.google.mlkit.vision.DEPENDENCIES"
-          android:value="ica,ocr,face" />
+          android:value="ica,ocr,face,subject_segment" />
         <!-- Don't delete the meta-data below.
              This is used by the Flutter tool to generate GeneratedPluginRegistrant.java -->
         <meta-data

packages/example/assets/images/child_dog.jpg

@@ -1,7 +1,6 @@
 import 'dart:io';
 
 import 'package:flutter/material.dart';
-
 import 'nlp_detector_views/entity_extraction_view.dart';
 import 'nlp_detector_views/language_identifier_view.dart';
 import 'nlp_detector_views/language_translator_view.dart';
@@ -15,6 +14,7 @@ import 'vision_detector_views/label_detector_view.dart';
 import 'vision_detector_views/object_detector_view.dart';
 import 'vision_detector_views/pose_detector_view.dart';
 import 'vision_detector_views/selfie_segmenter_view.dart';
+import 'vision_detector_views/subject_segmenter_view.dart';
 import 'vision_detector_views/text_detector_view.dart';
 
 Future<void> main() async {
@@ -54,7 +54,9 @@ class Home extends StatelessWidget {
                     children: [
                       CustomCard('Barcode Scanning', BarcodeScannerView()),
                       CustomCard('Face Detection', FaceDetectorView()),
-                      CustomCard('Face Mesh Detection', FaceMeshDetectorView()),
+                      if (Platform.isAndroid)
+                        CustomCard(
+                            'Face Mesh Detection', FaceMeshDetectorView()),
                       CustomCard('Image Labeling', ImageLabelView()),
                       CustomCard('Object Detection', ObjectDetectorView()),
                       CustomCard('Text Recognition', TextRecognizerView()),
@@ -63,6 +65,9 @@ class Home extends StatelessWidget {
                       CustomCard('Selfie Segmentation', SelfieSegmenterView()),
                       if (Platform.isAndroid)
                         CustomCard('Document Scanner', DocumentScannerView()),
+                      if (Platform.isAndroid)
+                        CustomCard(
+                            'Subject Segmentation', SubjectSegmenterView())
                     ],
                   ),
                   SizedBox(

packages/example/assets/images/friends.jpg

@@ -0,0 +1,68 @@
+import 'package:camera/camera.dart';
+import 'package:flutter/material.dart';
+import 'package:google_mlkit_subject_segmentation/google_mlkit_subject_segmentation.dart';
+import 'coordinates_translator.dart';
+
+class SubjectSegmentationPainter extends CustomPainter {
+  final SubjectSegmenterMask mask;
+  final Size imageSize;
+  final Color color = Colors.red;
+  final InputImageRotation rotation;
+  final CameraLensDirection cameraLensDirection;
+
+  SubjectSegmentationPainter(
+    this.mask,
+    this.imageSize,
+    this.rotation,
+    this.cameraLensDirection,
+  );
+
+  @override
+  void paint(Canvas canvas, Size size) {
+    final int width = mask.width;
+    final int height = mask.height;
+    final List<Subject> subjects = mask.subjects;
+
+    final paint = Paint()..style = PaintingStyle.fill;
+
+    for (final Subject subject in subjects) {
+      final int startX = subject.startX;
+      final int startY = subject.startY;
+      final int subjectWidth = subject.subjectWidth;
+      final int subjectHeight = subject.subjectHeight;
+      final List<double> confidences = subject.confidences;
+
+      for (int y = 0; y < subjectHeight; y++) {
+        for (int x = 0; y < subjectWidth; x++) {
+          final int absoluteX = startX;
+          final int absoluteY = startY;
+
+          final int tx = translateX(
+                  absoluteX.toDouble(),
+                  size,
+                  Size(width.toDouble(), height.toDouble()),
+                  rotation,
+                  cameraLensDirection)
+              .round();
+
+          final int ty = translateY(
+                  absoluteY.toDouble(),
+                  size,
+                  Size(width.toDouble(), height.toDouble()),
+                  rotation,
+                  cameraLensDirection)
+              .round();
+
+          final double opacity = confidences[(y * subjectWidth) + x] * 0.5;
+          paint.color = color.withOpacity(opacity);
+          canvas.drawCircle(Offset(tx.toDouble(), ty.toDouble()), 2, paint);
+        }
+      }
+    }
+  }
+
+  @override
+  bool shouldRepaint(SubjectSegmentationPainter oldDelegate) {
+    return oldDelegate.mask != mask;
+  }
+}

packages/example/lib/main.dart

@@ -0,0 +1,68 @@
+import 'package:camera/camera.dart';
+import 'package:flutter/material.dart';
+import 'package:google_mlkit_subject_segmentation/google_mlkit_subject_segmentation.dart';
+
+import 'detector_view.dart';
+import 'painters/subject_segmentation_painter.dart';
+
+class SubjectSegmenterView extends StatefulWidget {
+  @override
+  State<SubjectSegmenterView> createState() => _SubjectSegmenterViewState();
+}
+
+class _SubjectSegmenterViewState extends State<SubjectSegmenterView> {
+  final SubjectSegmenter _segmenter = SubjectSegmenter();
+  bool _canProcess = true;
+  bool _isBusy = false;
+  CustomPaint? _customPaint;
+  String? _text;
+  var _cameraLensDirection = CameraLensDirection.back;
+
+  @override
+  void dispose() async {
+    _canProcess = false;
+    _segmenter.close();
+    super.dispose();
+  }
+
+  @override
+  Widget build(BuildContext context) {
+    return DetectorView(
+      title: 'Subject Segmenter',
+      customPaint: _customPaint,
+      text: _text,
+      onImage: _processImage,
+      initialCameraLensDirection: _cameraLensDirection,
+      onCameraLensDirectionChanged: (value) => _cameraLensDirection = value,
+    );
+  }
+
+  Future<void> _processImage(InputImage inputImage) async {
+    if (!_canProcess) return;
+    if (_isBusy) return;
+    _isBusy = true;
+    setState(() {
+      _text = '';
+    });
+    final SubjectSegmenterMask mask = await _segmenter.processImage(inputImage);
+    if (inputImage.metadata?.size != null &&
+        inputImage.metadata?.rotation != null) {
+      final painter = SubjectSegmentationPainter(
+        mask,
+        inputImage.metadata!.size,
+        inputImage.metadata!.rotation,
+        _cameraLensDirection,
+      );
+      _customPaint = CustomPaint(painter: painter);
+    } else {
+      // TODO: set _customPaint to draw on top of image
+      _text = 'There is a mask with ${mask.subjects.length} subjects';
+
+      _customPaint = null;
+    }
+    _isBusy = false;
+    if (mounted) {
+      setState(() {});
+    }
+  }
+}

packages/example/lib/vision_detector_views/painters/subject_segmentation_painter.dart

@@ -267,6 +267,13 @@ packages:
       relative: true
     source: path
     version: "0.11.0"
+  google_mlkit_subject_segmentation:
+    dependency: "direct main"
+    description:
+      path: "../google_mlkit_subject_segmentation"
+      relative: true
+    source: path
+    version: "0.0.1"
   google_mlkit_text_recognition:
     dependency: "direct main"
     description: