Skip to content

Commit a86b6c9

Browse files
kwikwagkwikwag
authored
[camera_android] Use WeakReference to prevent startImageStream OOM error when main thread hangs (flutter#166533) (#9571)
This is an alternative solution to the pull request #8998 When streaming images using `CameraController.startImageStream()`, images are being `post()`-ed to the main looper in the Android implementation, even if it's hanging or paused. This will happen, for instance, when the Flutter debugger is paused or when the main thread is very busy. This can quickly result in an OOM (out-of-memory) error due to many images pending in queue, and the Android OS will kill the app abruptly. This version of the fix is done by keep the frames (except for the latest one) as `WeakReference`s. When memory pressur is on, the runtime can free weak references, therefore dropping frames. A log message is emitted for each dropped frame. As opposed to the back-pressure solution, there is no hard limit to how many frames are kept in queue - this means that if the main thread is delayed in processing frames for some reason, they will pile up as long as the memory can sustain them. Fixes flutter/flutter#166533 That issue also provides a demo program for testing the behavior. Some more data and considerations are listed in the [original pull request](#8998). ## Pre-Review Checklist [^1]: Regular contributors who have demonstrated familiarity with the repository guidelines only need to comment if the PR is not auto-exempted by repo tooling.
1 parent 84eeff1 commit a86b6c9

6 files changed

Lines changed: 210 additions & 135 deletions

File tree

‎packages/camera/camera_android/CHANGELOG.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,7 @@
1+
## 0.10.10+4
2+
3+
* Fix flutter#166533 - prevent startImageStream OOM error when main thread paused.
4+
15
## 0.10.10+3
26

37
* Waits for the creation of the capture session when initializing the camera to avoid thread race conditions.

‎packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java‎

Lines changed: 61 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,14 @@
99
import android.media.ImageReader;
1010
import android.os.Handler;
1111
import android.os.Looper;
12+
import android.util.Log;
1213
import android.view.Surface;
1314
import androidx.annotation.NonNull;
15+
import androidx.annotation.Nullable;
1416
import androidx.annotation.VisibleForTesting;
1517
import io.flutter.plugin.common.EventChannel;
1618
import io.flutter.plugins.camera.types.CameraCaptureProperties;
19+
import java.lang.ref.WeakReference;
1720
import java.nio.ByteBuffer;
1821
import java.util.ArrayList;
1922
import java.util.HashMap;
@@ -22,6 +25,7 @@
2225

2326
// Wraps an ImageReader to allow for testing of the image handler.
2427
public class ImageStreamReader {
28+
private static final String TAG = "ImageStreamReader";
2529

2630
/**
2731
* The image format we are going to send back to dart. Usually it's the same as streamImageFormat
@@ -33,6 +37,16 @@ public class ImageStreamReader {
3337
private final ImageReader imageReader;
3438
private final ImageStreamReaderUtils imageStreamReaderUtils;
3539

40+
@VisibleForTesting(otherwise = VisibleForTesting.NONE)
41+
@Nullable
42+
public Handler handler;
43+
44+
/**
45+
* This hard reference is required so frames don't get randomly dropped before reaching the main
46+
* looper.
47+
*/
48+
private Map<String, Object> latestImageBufferHardReference = null;
49+
3650
/**
3751
* Creates a new instance of the {@link ImageStreamReader}.
3852
*
@@ -95,40 +109,69 @@ public void onImageAvailable(
95109
@NonNull Image image,
96110
@NonNull CameraCaptureProperties captureProps,
97111
@NonNull EventChannel.EventSink imageStreamSink) {
98-
try {
99-
Map<String, Object> imageBuffer = new HashMap<>();
112+
Map<String, Object> imageBuffer = new HashMap<>();
100113

114+
imageBuffer.put("width", image.getWidth());
115+
imageBuffer.put("height", image.getHeight());
116+
try {
101117
// Get plane data ready
102118
if (dartImageFormat == ImageFormat.NV21) {
103119
imageBuffer.put("planes", parsePlanesForNv21(image));
104120
} else {
105121
imageBuffer.put("planes", parsePlanesForYuvOrJpeg(image));
106122
}
107-
108-
imageBuffer.put("width", image.getWidth());
109-
imageBuffer.put("height", image.getHeight());
110-
imageBuffer.put("format", dartImageFormat);
111-
imageBuffer.put("lensAperture", captureProps.getLastLensAperture());
112-
imageBuffer.put("sensorExposureTime", captureProps.getLastSensorExposureTime());
113-
Integer sensorSensitivity = captureProps.getLastSensorSensitivity();
114-
imageBuffer.put(
115-
"sensorSensitivity", sensorSensitivity == null ? null : (double) sensorSensitivity);
116-
117-
final Handler handler = new Handler(Looper.getMainLooper());
118-
handler.post(() -> imageStreamSink.success(imageBuffer));
119-
image.close();
120-
121123
} catch (IllegalStateException e) {
122-
// Handle "buffer is inaccessible" errors that can happen on some devices from ImageStreamReaderUtils.yuv420ThreePlanesToNV21()
123-
final Handler handler = new Handler(Looper.getMainLooper());
124+
// Handle "buffer is inaccessible" errors that can happen on some devices from
125+
// ImageStreamReaderUtils.yuv420ThreePlanesToNV21()
126+
final Handler handler =
127+
this.handler != null ? this.handler : new Handler(Looper.getMainLooper());
124128
handler.post(
125129
() ->
126130
imageStreamSink.error(
127131
"IllegalStateException",
128132
"Caught IllegalStateException: " + e.getMessage(),
129133
null));
134+
} finally {
130135
image.close();
131136
}
137+
138+
imageBuffer.put("format", dartImageFormat);
139+
imageBuffer.put("lensAperture", captureProps.getLastLensAperture());
140+
imageBuffer.put("sensorExposureTime", captureProps.getLastSensorExposureTime());
141+
Integer sensorSensitivity = captureProps.getLastSensorSensitivity();
142+
imageBuffer.put(
143+
"sensorSensitivity", sensorSensitivity == null ? null : (double) sensorSensitivity);
144+
145+
final Handler handler =
146+
this.handler != null ? this.handler : new Handler(Looper.getMainLooper());
147+
148+
// Keep a hard reference to the latest frame, so it isn't dropped before it reaches the main
149+
// looper
150+
latestImageBufferHardReference = imageBuffer;
151+
152+
boolean postResult =
153+
handler.post(
154+
new Runnable() {
155+
@VisibleForTesting public WeakReference<Map<String, Object>> weakImageBuffer;
156+
157+
public Runnable withImageBuffer(Map<String, Object> imageBuffer) {
158+
weakImageBuffer = new WeakReference<>(imageBuffer);
159+
return this;
160+
}
161+
162+
@Override
163+
public void run() {
164+
final Map<String, Object> imageBuffer = weakImageBuffer.get();
165+
if (imageBuffer == null) {
166+
// The memory was freed by the runtime, most likely due to a memory build-up
167+
// while the main thread was lagging. Frames are silently dropped in this
168+
// case.
169+
Log.d(TAG, "Image buffer was dropped by garbage collector.");
170+
return;
171+
}
172+
imageStreamSink.success(imageBuffer);
173+
}
174+
}.withImageBuffer(imageBuffer));
132175
}
133176

134177
/**

‎packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderTest.java‎

Lines changed: 83 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -9,15 +9,22 @@
99
import static org.mockito.ArgumentMatchers.anyInt;
1010
import static org.mockito.Mockito.mock;
1111
import static org.mockito.Mockito.never;
12+
import static org.mockito.Mockito.times;
1213
import static org.mockito.Mockito.verify;
1314
import static org.mockito.Mockito.when;
1415

1516
import android.graphics.ImageFormat;
1617
import android.media.Image;
1718
import android.media.ImageReader;
19+
import android.os.Handler;
1820
import io.flutter.plugin.common.EventChannel;
1921
import io.flutter.plugins.camera.types.CameraCaptureProperties;
22+
import java.lang.ref.WeakReference;
23+
import java.lang.reflect.Field;
2024
import java.nio.ByteBuffer;
25+
import java.util.ArrayList;
26+
import java.util.List;
27+
import java.util.Map;
2128
import org.junit.Test;
2229
import org.junit.runner.RunWith;
2330
import org.robolectric.RobolectricTestRunner;
@@ -61,48 +68,18 @@ public void onImageAvailable_parsesPlanesForNv21() {
6168
when(mockImageStreamReaderUtils.yuv420ThreePlanesToNV21(any(), anyInt(), anyInt()))
6269
.thenReturn(mockBytes);
6370

64-
// The image format as streamed from the camera
65-
int imageFormat = ImageFormat.YUV_420_888;
66-
67-
// Mock YUV image
68-
Image mockImage = mock(Image.class);
69-
when(mockImage.getWidth()).thenReturn(1280);
70-
when(mockImage.getHeight()).thenReturn(720);
71-
when(mockImage.getFormat()).thenReturn(imageFormat);
72-
73-
// Mock planes. YUV images have 3 planes (Y, U, V).
74-
Image.Plane planeY = mock(Image.Plane.class);
75-
Image.Plane planeU = mock(Image.Plane.class);
76-
Image.Plane planeV = mock(Image.Plane.class);
77-
78-
// Y plane is width*height
79-
// Row stride is generally == width but when there is padding it will
80-
// be larger. The numbers in this example are from a Vivo V2135 on 'high'
81-
// setting (1280x720).
82-
when(planeY.getBuffer()).thenReturn(ByteBuffer.allocate(1105664));
83-
when(planeY.getRowStride()).thenReturn(1536);
84-
when(planeY.getPixelStride()).thenReturn(1);
85-
86-
// U and V planes are always the same sizes/values.
87-
// https://developer.android.com/reference/android/graphics/ImageFormat#YUV_420_888
88-
when(planeU.getBuffer()).thenReturn(ByteBuffer.allocate(552703));
89-
when(planeV.getBuffer()).thenReturn(ByteBuffer.allocate(552703));
90-
when(planeU.getRowStride()).thenReturn(1536);
91-
when(planeV.getRowStride()).thenReturn(1536);
92-
when(planeU.getPixelStride()).thenReturn(2);
93-
when(planeV.getPixelStride()).thenReturn(2);
94-
95-
// Add planes to image
96-
Image.Plane[] planes = {planeY, planeU, planeV};
97-
when(mockImage.getPlanes()).thenReturn(planes);
71+
// Note: the code for getImage() was previously inlined, with uSize set to one less than
72+
// getImage() calculates (see function implementation)
73+
Image mockImage = ImageStreamReaderTestUtils.getImage(1280, 720, 256, ImageFormat.YUV_420_888);
9874

9975
CameraCaptureProperties mockCaptureProps = mock(CameraCaptureProperties.class);
10076
EventChannel.EventSink mockEventSink = mock(EventChannel.EventSink.class);
10177
imageStreamReader.onImageAvailable(mockImage, mockCaptureProps, mockEventSink);
10278

10379
// Make sure we processed the frame with parsePlanesForNv21
10480
verify(mockImageStreamReaderUtils)
105-
.yuv420ThreePlanesToNV21(planes, mockImage.getWidth(), mockImage.getHeight());
81+
.yuv420ThreePlanesToNV21(
82+
mockImage.getPlanes(), mockImage.getWidth(), mockImage.getHeight());
10683
}
10784

10885
/** If we are requesting YUV420, then we should send the 3-plane image as it is. */
@@ -120,40 +97,9 @@ public void onImageAvailable_parsesPlanesForYuv420() {
12097
when(mockImageStreamReaderUtils.yuv420ThreePlanesToNV21(any(), anyInt(), anyInt()))
12198
.thenReturn(mockBytes);
12299

123-
// The image format as streamed from the camera
124-
int imageFormat = ImageFormat.YUV_420_888;
125-
126-
// Mock YUV image
127-
Image mockImage = mock(Image.class);
128-
when(mockImage.getWidth()).thenReturn(1280);
129-
when(mockImage.getHeight()).thenReturn(720);
130-
when(mockImage.getFormat()).thenReturn(imageFormat);
131-
132-
// Mock planes. YUV images have 3 planes (Y, U, V).
133-
Image.Plane planeY = mock(Image.Plane.class);
134-
Image.Plane planeU = mock(Image.Plane.class);
135-
Image.Plane planeV = mock(Image.Plane.class);
136-
137-
// Y plane is width*height
138-
// Row stride is generally == width but when there is padding it will
139-
// be larger. The numbers in this example are from a Vivo V2135 on 'high'
140-
// setting (1280x720).
141-
when(planeY.getBuffer()).thenReturn(ByteBuffer.allocate(1105664));
142-
when(planeY.getRowStride()).thenReturn(1536);
143-
when(planeY.getPixelStride()).thenReturn(1);
144-
145-
// U and V planes are always the same sizes/values.
146-
// https://developer.android.com/reference/android/graphics/ImageFormat#YUV_420_888
147-
when(planeU.getBuffer()).thenReturn(ByteBuffer.allocate(552703));
148-
when(planeV.getBuffer()).thenReturn(ByteBuffer.allocate(552703));
149-
when(planeU.getRowStride()).thenReturn(1536);
150-
when(planeV.getRowStride()).thenReturn(1536);
151-
when(planeU.getPixelStride()).thenReturn(2);
152-
when(planeV.getPixelStride()).thenReturn(2);
153-
154-
// Add planes to image
155-
Image.Plane[] planes = {planeY, planeU, planeV};
156-
when(mockImage.getPlanes()).thenReturn(planes);
100+
// Note: the code for getImage() was previously inlined, with uSize set to one less than
101+
// getImage() calculates (see function implementation)
102+
Image mockImage = ImageStreamReaderTestUtils.getImage(1280, 720, 256, ImageFormat.YUV_420_888);
157103

158104
CameraCaptureProperties mockCaptureProps = mock(CameraCaptureProperties.class);
159105
EventChannel.EventSink mockEventSink = mock(EventChannel.EventSink.class);
@@ -162,4 +108,72 @@ public void onImageAvailable_parsesPlanesForYuv420() {
162108
// Make sure we processed the frame with parsePlanesForYuvOrJpeg
163109
verify(mockImageStreamReaderUtils, never()).yuv420ThreePlanesToNV21(any(), anyInt(), anyInt());
164110
}
111+
112+
@Test
113+
public void onImageAvailable_dropFramesWhenHandlerHalted() {
114+
int dartImageFormat = ImageFormat.YUV_420_888;
115+
116+
ImageReader mockImageReader = mock(ImageReader.class);
117+
ImageStreamReaderUtils mockImageStreamReaderUtils = mock(ImageStreamReaderUtils.class);
118+
ImageStreamReader imageStreamReader =
119+
new ImageStreamReader(mockImageReader, dartImageFormat, mockImageStreamReaderUtils);
120+
121+
for (boolean invalidateWeakReference : new boolean[] {true, false}) {
122+
final List<Runnable> runnables = new ArrayList<Runnable>();
123+
124+
Handler mockHandler = mock(Handler.class);
125+
imageStreamReader.handler = mockHandler;
126+
127+
// initially, handler will simulate a hanging main looper, that only queues inputs
128+
when(mockHandler.post(any(Runnable.class)))
129+
.thenAnswer(
130+
inputs -> {
131+
Runnable r = inputs.getArgument(0, Runnable.class);
132+
runnables.add(r);
133+
return true;
134+
});
135+
136+
CameraCaptureProperties mockCaptureProps = mock(CameraCaptureProperties.class);
137+
EventChannel.EventSink mockEventSink = mock(EventChannel.EventSink.class);
138+
139+
Image mockImage =
140+
ImageStreamReaderTestUtils.getImage(1280, 720, 256, ImageFormat.YUV_420_888);
141+
imageStreamReader.onImageAvailable(mockImage, mockCaptureProps, mockEventSink);
142+
143+
// make sure the image was closed, even when skipping frames
144+
verify(mockImage, times(1)).close();
145+
146+
// check that we collected all runnables in this method
147+
assertEquals(runnables.size(), 1);
148+
149+
// verify post() was not called more times than it should have
150+
verify(mockHandler, times(1)).post(any(Runnable.class));
151+
152+
// make sure callback was not yet invoked
153+
verify(mockEventSink, never()).success(any(Map.class));
154+
155+
// simulate frame processing
156+
for (Runnable r : runnables) {
157+
if (invalidateWeakReference) {
158+
// Replace the captured WeakReference with one pointing to null.
159+
Field[] fields = r.getClass().getDeclaredFields();
160+
for (Field field : fields) {
161+
if (field.getType().equals(WeakReference.class)) {
162+
// Remove the `final` modifier
163+
try {
164+
field.set(r, new WeakReference<Map<String, Object>>(null));
165+
} catch (IllegalAccessException e) {
166+
throw new RuntimeException("Failed to inject null WeakReference", e);
167+
}
168+
}
169+
}
170+
}
171+
172+
r.run();
173+
}
174+
175+
// make sure all callbacks were invoked so far
176+
verify(mockEventSink, invalidateWeakReference ? never() : times(1)).success(any(Map.class));
177+
}
178+
}
165179
}

‎packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderTestUtils.java‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
// Copyright 2013 The Flutter Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style license that can be
3+
// found in the LICENSE file.
4+
5+
package io.flutter.plugins.camera.media;
6+
7+
import static org.mockito.Mockito.mock;
8+
import static org.mockito.Mockito.when;
9+
10+
import android.media.Image;
11+
import java.nio.ByteBuffer;
12+
13+
public class ImageStreamReaderTestUtils {
14+
/**
15+
* Creates a mock {@link android.media.Image} object for use in tests, simulating the specified
16+
* dimensions, padding, and image format.
17+
*/
18+
public static Image getImage(int imageWidth, int imageHeight, int padding, int imageFormat) {
19+
int rowStride = imageWidth + padding;
20+
21+
int ySize = (rowStride * imageHeight) - padding;
22+
int uSize = (ySize / 2) - (padding / 2);
23+
int vSize = uSize;
24+
25+
// Mock YUV image
26+
Image mockImage = mock(Image.class);
27+
when(mockImage.getWidth()).thenReturn(imageWidth);
28+
when(mockImage.getHeight()).thenReturn(imageHeight);
29+
when(mockImage.getFormat()).thenReturn(imageFormat);
30+
31+
// Mock planes. YUV images have 3 planes (Y, U, V).
32+
Image.Plane planeY = mock(Image.Plane.class);
33+
Image.Plane planeU = mock(Image.Plane.class);
34+
Image.Plane planeV = mock(Image.Plane.class);
35+
36+
// Y plane is width*height
37+
// Row stride is generally == width but when there is padding it will
38+
// be larger.
39+
// Here we are adding 256 padding.
40+
when(planeY.getBuffer()).thenReturn(ByteBuffer.allocate(ySize));
41+
when(planeY.getRowStride()).thenReturn(rowStride);
42+
when(planeY.getPixelStride()).thenReturn(1);
43+
44+
// U and V planes are always the same sizes/values.
45+
// https://developer.android.com/reference/android/graphics/ImageFormat#YUV_420_888
46+
when(planeU.getBuffer()).thenReturn(ByteBuffer.allocate(uSize));
47+
when(planeV.getBuffer()).thenReturn(ByteBuffer.allocate(vSize));
48+
when(planeU.getRowStride()).thenReturn(rowStride);
49+
when(planeV.getRowStride()).thenReturn(rowStride);
50+
when(planeU.getPixelStride()).thenReturn(2);
51+
when(planeV.getPixelStride()).thenReturn(2);
52+
53+
// Add planes to image
54+
Image.Plane[] planes = {planeY, planeU, planeV};
55+
when(mockImage.getPlanes()).thenReturn(planes);
56+
57+
return mockImage;
58+
}
59+
}

0 commit comments

Comments
 (0)