Skip to content

Commit a241191

Browse files
feinsteinfeinstein
authored
feat: Add bitmap to input image for text recognition (#752)
1 parent f10bd5f commit a241191

File tree

10 files changed

+377
-8
lines changed

10 files changed

+377
-8
lines changed

‎packages/example/lib/main.dart‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import 'vision_detector_views/pose_detector_view.dart';
1616
import 'vision_detector_views/selfie_segmenter_view.dart';
1717
import 'vision_detector_views/subject_segmenter_view.dart';
1818
import 'vision_detector_views/text_detector_view.dart';
19+
import 'vision_detector_views/text_from_widget_view.dart';
1920

2021
Future<void> main() async {
2122
WidgetsFlutterBinding.ensureInitialized();
@@ -60,6 +61,7 @@ class Home extends StatelessWidget {
6061
CustomCard('Image Labeling', ImageLabelView()),
6162
CustomCard('Object Detection', ObjectDetectorView()),
6263
CustomCard('Text Recognition', TextRecognizerView()),
64+
CustomCard('Text From Widget', TextFromWidgetView()),
6365
CustomCard('Digital Ink Recognition', DigitalInkView()),
6466
CustomCard('Pose Detection', PoseDetectorView()),
6567
CustomCard('Selfie Segmentation', SelfieSegmenterView()),

‎packages/example/lib/vision_detector_views/text_from_widget_view.dart‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
import 'dart:async';
2+
import 'dart:typed_data';
3+
import 'dart:ui' as ui;
4+
5+
import 'package:flutter/material.dart';
6+
import 'package:flutter/rendering.dart';
7+
import 'package:google_mlkit_text_recognition/google_mlkit_text_recognition.dart';
8+
9+
class TextFromWidgetView extends StatefulWidget {
10+
const TextFromWidgetView({Key? key}) : super(key: key);
11+
12+
@override
13+
State<TextFromWidgetView> createState() => _TextFromWidgetViewState();
14+
}
15+
16+
class _TextFromWidgetViewState extends State<TextFromWidgetView> {
17+
final _textRecognizer = TextRecognizer(script: TextRecognitionScript.latin);
18+
final _widgetKey = GlobalKey();
19+
String _extractedText = 'Recognized text will appear here';
20+
bool _isBusy = false;
21+
22+
@override
23+
void dispose() {
24+
_textRecognizer.close();
25+
super.dispose();
26+
}
27+
28+
@override
29+
Widget build(BuildContext context) {
30+
return Scaffold(
31+
appBar: AppBar(
32+
title: const Text('Text From Widget Example'),
33+
),
34+
body: Column(
35+
children: [
36+
Expanded(
37+
child: Center(
38+
child: RepaintBoundary(
39+
key: _widgetKey,
40+
child: Container(
41+
padding: const EdgeInsets.all(16.0),
42+
decoration: BoxDecoration(
43+
color: Colors.white,
44+
border: Border.all(color: Colors.blue, width: 2),
45+
borderRadius: BorderRadius.circular(12),
46+
),
47+
child: const Text(
48+
'This is sample text\nthat will be captured\nand processed using\nthe ML Kit Text Recognizer.\n\nTry different fonts\nand styles to test\nthe recognition capabilities!',
49+
style: TextStyle(fontSize: 24, fontWeight: FontWeight.bold),
50+
),
51+
),
52+
),
53+
),
54+
),
55+
Padding(
56+
padding: const EdgeInsets.all(16.0),
57+
child: ElevatedButton(
58+
onPressed: _isBusy ? null : _extractTextFromWidget,
59+
child: const Text('Capture and Recognize Text'),
60+
),
61+
),
62+
Expanded(
63+
child: SingleChildScrollView(
64+
padding: const EdgeInsets.all(16.0),
65+
child: Column(
66+
crossAxisAlignment: CrossAxisAlignment.start,
67+
children: [
68+
const Text(
69+
'Recognition Result:',
70+
style: TextStyle(fontWeight: FontWeight.bold, fontSize: 18),
71+
),
72+
const SizedBox(height: 8),
73+
Container(
74+
padding: const EdgeInsets.all(12),
75+
decoration: BoxDecoration(
76+
color: Colors.grey[200],
77+
borderRadius: BorderRadius.circular(8),
78+
),
79+
width: double.infinity,
80+
child: Text(_extractedText),
81+
),
82+
],
83+
),
84+
),
85+
),
86+
],
87+
),
88+
);
89+
}
90+
91+
Future<void> _extractTextFromWidget() async {
92+
if (_isBusy) return;
93+
94+
setState(() {
95+
_isBusy = true;
96+
_extractedText = 'Processing...';
97+
});
98+
99+
try {
100+
// Get the RenderObject from the GlobalKey
101+
final RenderRepaintBoundary? boundary =
102+
_widgetKey.currentContext?.findRenderObject() as RenderRepaintBoundary?;
103+
104+
if (boundary == null) {
105+
setState(() {
106+
_extractedText = 'Error: Unable to find widget render object';
107+
_isBusy = false;
108+
});
109+
return;
110+
}
111+
112+
// Capture the widget as an image
113+
final ui.Image image = await boundary.toImage(pixelRatio: 3.0);
114+
115+
// Convert to byte data in raw RGBA format
116+
final ByteData? byteData =
117+
await image.toByteData(format: ui.ImageByteFormat.rawRgba);
118+
119+
if (byteData == null) {
120+
setState(() {
121+
_extractedText = 'Error: Failed to get image byte data';
122+
_isBusy = false;
123+
});
124+
return;
125+
}
126+
127+
final Uint8List bytes = byteData.buffer.asUint8List();
128+
129+
// Create InputImage from bitmap data with dimensions
130+
final inputImage = InputImage.fromBitmap(
131+
bitmap: bytes,
132+
width: image.width,
133+
height: image.height,
134+
);
135+
136+
// Process the image with the text recognizer
137+
final RecognizedText recognizedText =
138+
await _textRecognizer.processImage(inputImage);
139+
140+
setState(() {
141+
_extractedText = recognizedText.text.isNotEmpty
142+
? recognizedText.text
143+
: 'No text recognized';
144+
_isBusy = false;
145+
});
146+
} catch (e) {
147+
setState(() {
148+
_extractedText = 'Error processing image: $e';
149+
_isBusy = false;
150+
});
151+
}
152+
}
153+
}

‎packages/google_mlkit_commons/CHANGELOG.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,7 @@
1+
## 0.10.0
2+
3+
* Add support for bitmap data with `InputImage.fromBitmap()` constructor.
4+
15
## 0.9.0
26

37
* Update README.

‎packages/google_mlkit_commons/android/src/main/java/com/google_mlkit_commons/InputImageConverter.java‎

Lines changed: 65 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,73 @@ public class InputImageConverter {
2020
public static InputImage getInputImageFromData(Map<String, Object> imageData,
2121
Context context,
2222
MethodChannel.Result result) {
23-
//Differentiates whether the image data is a path for a image file or contains image data in form of bytes
23+
//Differentiates whether the image data is a path for a image file, contains image data in form of bytes, or a bitmap
2424
String model = (String) imageData.get("type");
2525
InputImage inputImage;
26-
if (model != null && model.equals("file")) {
26+
if (model != null && model.equals("bitmap")) {
27+
try {
28+
byte[] bitmapData = (byte[]) imageData.get("bitmapData");
29+
if (bitmapData == null) {
30+
result.error("InputImageConverterError", "Bitmap data is null", null);
31+
return null;
32+
}
33+
34+
// Extract the rotation
35+
int rotation = 0;
36+
Object rotationObj = imageData.get("rotation");
37+
if (rotationObj != null) {
38+
rotation = (int) rotationObj;
39+
}
40+
41+
try {
42+
// Get metadata from the InputImage object if available
43+
Map<String, Object> metadataMap = (Map<String, Object>) imageData.get("metadata");
44+
if (metadataMap != null) {
45+
int width = Double.valueOf(Objects.requireNonNull(metadataMap.get("width")).toString()).intValue();
46+
int height = Double.valueOf(Objects.requireNonNull(metadataMap.get("height")).toString()).intValue();
47+
48+
// Create bitmap from the Flutter UI raw RGBA bytes
49+
android.graphics.Bitmap bitmap = android.graphics.Bitmap.createBitmap(width, height, android.graphics.Bitmap.Config.ARGB_8888);
50+
java.nio.IntBuffer intBuffer = java.nio.IntBuffer.allocate(bitmapData.length / 4);
51+
52+
// Convert RGBA bytes to int pixels
53+
for (int i = 0; i < bitmapData.length; i += 4) {
54+
int r = bitmapData[i] & 0xFF;
55+
int g = bitmapData[i + 1] & 0xFF;
56+
int b = bitmapData[i + 2] & 0xFF;
57+
int a = bitmapData[i + 3] & 0xFF;
58+
intBuffer.put((a << 24) | (r << 16) | (g << 8) | b);
59+
}
60+
intBuffer.rewind();
61+
62+
// Copy pixel data to bitmap
63+
bitmap.copyPixelsFromBuffer(intBuffer);
64+
return InputImage.fromBitmap(bitmap, rotation);
65+
}
66+
} catch (Exception e) {
67+
Log.e("ImageError", "Error creating bitmap from raw data", e);
68+
}
69+
70+
// Fallback: Try to decode as standard image format (JPEG, PNG)
71+
try {
72+
android.graphics.Bitmap bitmap = android.graphics.BitmapFactory.decodeByteArray(bitmapData, 0, bitmapData.length);
73+
if (bitmap == null) {
74+
result.error("InputImageConverterError", "Failed to decode bitmap from the provided data", null);
75+
return null;
76+
}
77+
return InputImage.fromBitmap(bitmap, rotation);
78+
} catch (Exception e) {
79+
Log.e("ImageError", "Getting Bitmap failed", e);
80+
result.error("InputImageConverterError", e.toString(), e);
81+
return null;
82+
}
83+
} catch (Exception e) {
84+
Log.e("ImageError", "Getting Bitmap failed");
85+
Log.e("ImageError", e.toString());
86+
result.error("InputImageConverterError", e.toString(), e);
87+
return null;
88+
}
89+
} else if (model != null && model.equals("file")) {
2790
try {
2891
inputImage = InputImage.fromFilePath(context, Uri.fromFile(new File(((String) imageData.get("path")))));
2992
return inputImage;

‎packages/google_mlkit_commons/ios/Classes/MLKVisionImage+FlutterPlugin.m‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ + (MLKVisionImage *)visionImageFromData:(NSDictionary *)imageData {
99
return [self filePathToVisionImage:imageData[@"path"]];
1010
} else if ([@"bytes" isEqualToString:imageType]) {
1111
return [self bytesToVisionImage:imageData];
12+
} else if ([@"bitmap" isEqualToString:imageType]) {
13+
return [self bitmapToVisionImage:imageData];
1214
} else {
1315
NSString *errorReason = [NSString stringWithFormat:@"No image type for: %@", imageType];
1416
@throw [NSException exceptionWithName:NSInvalidArgumentException
@@ -66,4 +68,67 @@ + (MLKVisionImage *)pixelBufferToVisionImage:(CVPixelBufferRef)pixelBufferRef {
6668
return [[MLKVisionImage alloc] initWithImage:uiImage];
6769
}
6870

71+
+ (MLKVisionImage *)bitmapToVisionImage:(NSDictionary *)imageDict {
72+
// Get the bitmap data
73+
FlutterStandardTypedData *bitmapData = imageDict[@"bitmapData"];
74+
75+
if (bitmapData == nil) {
76+
NSString *errorReason = @"Bitmap data is nil";
77+
@throw [NSException exceptionWithName:NSInvalidArgumentException
78+
reason:errorReason
79+
userInfo:nil];
80+
}
81+
82+
// Try to get metadata if available
83+
NSDictionary *metadata = imageDict[@"metadata"];
84+
if (metadata != nil) {
85+
NSNumber *width = metadata[@"width"];
86+
NSNumber *height = metadata[@"height"];
87+
88+
if (width != nil && height != nil) {
89+
// Create bitmap context from raw RGBA data
90+
CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB();
91+
uint8_t *rawData = (uint8_t*)[bitmapData.data bytes];
92+
size_t bytesPerPixel = 4;
93+
size_t bytesPerRow = bytesPerPixel * width.intValue;
94+
size_t bitsPerComponent = 8;
95+
96+
CGContextRef context = CGBitmapContextCreate(rawData, width.intValue, height.intValue,
97+
bitsPerComponent, bytesPerRow, colorSpace,
98+
kCGImageAlphaPremultipliedLast | kCGBitmapByteOrder32Big);
99+
100+
if (context) {
101+
CGImageRef imageRef = CGBitmapContextCreateImage(context);
102+
UIImage *image = [UIImage imageWithCGImage:imageRef];
103+
104+
CGImageRelease(imageRef);
105+
CGContextRelease(context);
106+
CGColorSpaceRelease(colorSpace);
107+
108+
if (image) {
109+
MLKVisionImage *visionImage = [[MLKVisionImage alloc] initWithImage:image];
110+
visionImage.orientation = image.imageOrientation;
111+
return visionImage;
112+
}
113+
}
114+
115+
CGColorSpaceRelease(colorSpace);
116+
}
117+
}
118+
119+
// Fallback: try to create UIImage directly from data
120+
UIImage *image = [UIImage imageWithData:bitmapData.data];
121+
122+
if (image == nil) {
123+
NSString *errorReason = @"Failed to create UIImage from bitmap data";
124+
@throw [NSException exceptionWithName:NSInvalidArgumentException
125+
reason:errorReason
126+
userInfo:nil];
127+
}
128+
129+
MLKVisionImage *visionImage = [[MLKVisionImage alloc] initWithImage:image];
130+
visionImage.orientation = image.imageOrientation;
131+
return visionImage;
132+
}
133+
69134
@end

0 commit comments

Comments
 (0)