[camera_android] Use WeakReference to prevent startImageStream OOM error when main thread hangs (flutter#166533)

packages/camera/camera_android/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.10.10+3
+
+* Fix flutter#166533 - prevent startImageStream OOM error when main thread paused.
+
 ## 0.10.10+2
 
 * Don't set the FPS range unless video recording. It can cause dark image previews on some devices becuse the auto exposure algorithm is more constrained after fixing the min/max FPS range to the same value. This change has the side effect that providing the `fps` parameter will not affect the camera preview when not video recording. And if you need a lower frame rate in your image streaming handler, you can skip frames by checking the time it passed since the last frame.

packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java

@@ -9,11 +9,14 @@
 import android.media.ImageReader;
 import android.os.Handler;
 import android.os.Looper;
+import android.util.Log;
 import android.view.Surface;
 import androidx.annotation.NonNull;
+import androidx.annotation.Nullable;
 import androidx.annotation.VisibleForTesting;
 import io.flutter.plugin.common.EventChannel;
 import io.flutter.plugins.camera.types.CameraCaptureProperties;
+import java.lang.ref.WeakReference;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -22,6 +25,7 @@
 
 // Wraps an ImageReader to allow for testing of the image handler.
 public class ImageStreamReader {
+  private static final String TAG = "ImageStreamReader";
 
   /**
    * The image format we are going to send back to dart. Usually it's the same as streamImageFormat
@@ -33,6 +37,16 @@ public class ImageStreamReader {
   private final ImageReader imageReader;
   private final ImageStreamReaderUtils imageStreamReaderUtils;
 
+  @VisibleForTesting(otherwise = VisibleForTesting.NONE)
+  @Nullable
+  public Handler handler;
+
+  /**
+   * This hard reference is required so frames don't get randomly dropped before reaching the main
+   * looper.
+   */
+  private Map<String, Object> latestImageBufferHardReference = null;
+
   /**
    * Creates a new instance of the {@link ImageStreamReader}.
    *
@@ -95,40 +109,67 @@ public void onImageAvailable(
       @NonNull Image image,
       @NonNull CameraCaptureProperties captureProps,
       @NonNull EventChannel.EventSink imageStreamSink) {
-    try {
-      Map<String, Object> imageBuffer = new HashMap<>();
+    Map<String, Object> imageBuffer = new HashMap<>();
 
+    imageBuffer.put("width", image.getWidth());
+    imageBuffer.put("height", image.getHeight());
+    try {
       // Get plane data ready
       if (dartImageFormat == ImageFormat.NV21) {
         imageBuffer.put("planes", parsePlanesForNv21(image));
       } else {
         imageBuffer.put("planes", parsePlanesForYuvOrJpeg(image));
       }
-
-      imageBuffer.put("width", image.getWidth());
-      imageBuffer.put("height", image.getHeight());
-      imageBuffer.put("format", dartImageFormat);
-      imageBuffer.put("lensAperture", captureProps.getLastLensAperture());
-      imageBuffer.put("sensorExposureTime", captureProps.getLastSensorExposureTime());
-      Integer sensorSensitivity = captureProps.getLastSensorSensitivity();
-      imageBuffer.put(
-          "sensorSensitivity", sensorSensitivity == null ? null : (double) sensorSensitivity);
-
-      final Handler handler = new Handler(Looper.getMainLooper());
-      handler.post(() -> imageStreamSink.success(imageBuffer));
-      image.close();
-
     } catch (IllegalStateException e) {
-      // Handle "buffer is inaccessible" errors that can happen on some devices from ImageStreamReaderUtils.yuv420ThreePlanesToNV21()
-      final Handler handler = new Handler(Looper.getMainLooper());
+      // Handle "buffer is inaccessible" errors that can happen on some devices from
+      // ImageStreamReaderUtils.yuv420ThreePlanesToNV21()
+      final Handler handler =
+          this.handler != null ? this.handler : new Handler(Looper.getMainLooper());
       handler.post(
           () ->
               imageStreamSink.error(
                   "IllegalStateException",
                   "Caught IllegalStateException: " + e.getMessage(),
                   null));
+    } finally {
       image.close();
     }
+
+    imageBuffer.put("format", dartImageFormat);
+    imageBuffer.put("lensAperture", captureProps.getLastLensAperture());
+    imageBuffer.put("sensorExposureTime", captureProps.getLastSensorExposureTime());
+    Integer sensorSensitivity = captureProps.getLastSensorSensitivity();
+    imageBuffer.put(
+        "sensorSensitivity", sensorSensitivity == null ? null : (double) sensorSensitivity);
+
+    final Handler handler =
+        this.handler != null ? this.handler : new Handler(Looper.getMainLooper());
+
+    // Keep a hard reference to the latest frame, so it isn't dropped before it reaches the main
+    // looper
+    latestImageBufferHardReference = imageBuffer;
+
+    boolean postResult =
+        handler.post(
+            new Runnable() {
+              // a public field is required in order to test this code
+              @VisibleForTesting public WeakReference<Map<String, Object>> weakImageBuffer;
+
+              public Runnable withImageBuffer(Map<String, Object> imageBuffer) {
+                weakImageBuffer = new WeakReference<>(imageBuffer);
+                return this;
+              }
+
+              @Override
+              public void run() {
+                final Map<String, Object> imageBuffer = weakImageBuffer.get();
+                if (imageBuffer == null) {
+                  Log.d(TAG, "Image buffer was dropped by garbage collector.");
+                  return;
+                }
+                imageStreamSink.success(imageBuffer);
+              }
+            }.withImageBuffer(imageBuffer));
   }
 
   /**

packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderTest.java

@@ -9,15 +9,22 @@
 import static org.mockito.ArgumentMatchers.anyInt;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 import android.graphics.ImageFormat;
 import android.media.Image;
 import android.media.ImageReader;
+import android.os.Handler;
 import io.flutter.plugin.common.EventChannel;
 import io.flutter.plugins.camera.types.CameraCaptureProperties;
+import java.lang.ref.WeakReference;
+import java.lang.reflect.Field;
 import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.robolectric.RobolectricTestRunner;
@@ -61,48 +68,18 @@ public void onImageAvailable_parsesPlanesForNv21() {
     when(mockImageStreamReaderUtils.yuv420ThreePlanesToNV21(any(), anyInt(), anyInt()))
         .thenReturn(mockBytes);
 
-    // The image format as streamed from the camera
-    int imageFormat = ImageFormat.YUV_420_888;
-
-    // Mock YUV image
-    Image mockImage = mock(Image.class);
-    when(mockImage.getWidth()).thenReturn(1280);
-    when(mockImage.getHeight()).thenReturn(720);
-    when(mockImage.getFormat()).thenReturn(imageFormat);
-
-    // Mock planes. YUV images have 3 planes (Y, U, V).
-    Image.Plane planeY = mock(Image.Plane.class);
-    Image.Plane planeU = mock(Image.Plane.class);
-    Image.Plane planeV = mock(Image.Plane.class);
-
-    // Y plane is width*height
-    // Row stride is generally == width but when there is padding it will
-    // be larger. The numbers in this example are from a Vivo V2135 on 'high'
-    // setting (1280x720).
-    when(planeY.getBuffer()).thenReturn(ByteBuffer.allocate(1105664));
-    when(planeY.getRowStride()).thenReturn(1536);
-    when(planeY.getPixelStride()).thenReturn(1);
-
-    // U and V planes are always the same sizes/values.
-    // https://developer.android.com/reference/android/graphics/ImageFormat#YUV_420_888
-    when(planeU.getBuffer()).thenReturn(ByteBuffer.allocate(552703));
-    when(planeV.getBuffer()).thenReturn(ByteBuffer.allocate(552703));
-    when(planeU.getRowStride()).thenReturn(1536);
-    when(planeV.getRowStride()).thenReturn(1536);
-    when(planeU.getPixelStride()).thenReturn(2);
-    when(planeV.getPixelStride()).thenReturn(2);
-
-    // Add planes to image
-    Image.Plane[] planes = {planeY, planeU, planeV};
-    when(mockImage.getPlanes()).thenReturn(planes);
+    // Note: the code for getImage() was previously inlined, with uSize set to one less than
+    // getImage() calculates (see function implementation)
+    Image mockImage = ImageStreamReaderTestUtils.getImage(1280, 720, 256, ImageFormat.YUV_420_888);
 
     CameraCaptureProperties mockCaptureProps = mock(CameraCaptureProperties.class);
     EventChannel.EventSink mockEventSink = mock(EventChannel.EventSink.class);
     imageStreamReader.onImageAvailable(mockImage, mockCaptureProps, mockEventSink);
 
     // Make sure we processed the frame with parsePlanesForNv21
     verify(mockImageStreamReaderUtils)
-        .yuv420ThreePlanesToNV21(planes, mockImage.getWidth(), mockImage.getHeight());
+        .yuv420ThreePlanesToNV21(
+            mockImage.getPlanes(), mockImage.getWidth(), mockImage.getHeight());
   }
 
   /** If we are requesting YUV420, then we should send the 3-plane image as it is. */
@@ -120,40 +97,9 @@ public void onImageAvailable_parsesPlanesForYuv420() {
     when(mockImageStreamReaderUtils.yuv420ThreePlanesToNV21(any(), anyInt(), anyInt()))
         .thenReturn(mockBytes);
 
-    // The image format as streamed from the camera
-    int imageFormat = ImageFormat.YUV_420_888;
-
-    // Mock YUV image
-    Image mockImage = mock(Image.class);
-    when(mockImage.getWidth()).thenReturn(1280);
-    when(mockImage.getHeight()).thenReturn(720);
-    when(mockImage.getFormat()).thenReturn(imageFormat);
-
-    // Mock planes. YUV images have 3 planes (Y, U, V).
-    Image.Plane planeY = mock(Image.Plane.class);
-    Image.Plane planeU = mock(Image.Plane.class);
-    Image.Plane planeV = mock(Image.Plane.class);
-
-    // Y plane is width*height
-    // Row stride is generally == width but when there is padding it will
-    // be larger. The numbers in this example are from a Vivo V2135 on 'high'
-    // setting (1280x720).
-    when(planeY.getBuffer()).thenReturn(ByteBuffer.allocate(1105664));
-    when(planeY.getRowStride()).thenReturn(1536);
-    when(planeY.getPixelStride()).thenReturn(1);
-
-    // U and V planes are always the same sizes/values.
-    // https://developer.android.com/reference/android/graphics/ImageFormat#YUV_420_888
-    when(planeU.getBuffer()).thenReturn(ByteBuffer.allocate(552703));
-    when(planeV.getBuffer()).thenReturn(ByteBuffer.allocate(552703));
-    when(planeU.getRowStride()).thenReturn(1536);
-    when(planeV.getRowStride()).thenReturn(1536);
-    when(planeU.getPixelStride()).thenReturn(2);
-    when(planeV.getPixelStride()).thenReturn(2);
-
-    // Add planes to image
-    Image.Plane[] planes = {planeY, planeU, planeV};
-    when(mockImage.getPlanes()).thenReturn(planes);
+    // Note: the code for getImage() was previously inlined, with uSize set to one less than
+    // getImage() calculates (see function implementation)
+    Image mockImage = ImageStreamReaderTestUtils.getImage(1280, 720, 256, ImageFormat.YUV_420_888);
 
     CameraCaptureProperties mockCaptureProps = mock(CameraCaptureProperties.class);
     EventChannel.EventSink mockEventSink = mock(EventChannel.EventSink.class);
@@ -162,4 +108,72 @@ public void onImageAvailable_parsesPlanesForYuv420() {
     // Make sure we processed the frame with parsePlanesForYuvOrJpeg
     verify(mockImageStreamReaderUtils, never()).yuv420ThreePlanesToNV21(any(), anyInt(), anyInt());
   }
+
+  @Test
+  public void onImageAvailable_dropFramesWhenHandlerHalted() {
+    int dartImageFormat = ImageFormat.YUV_420_888;
+
+    ImageReader mockImageReader = mock(ImageReader.class);
+    ImageStreamReaderUtils mockImageStreamReaderUtils = mock(ImageStreamReaderUtils.class);
+    ImageStreamReader imageStreamReader =
+        new ImageStreamReader(mockImageReader, dartImageFormat, mockImageStreamReaderUtils);
+
+    for (boolean invalidateWeakReference : new boolean[] {true, false}) {
+      final List<Runnable> runnables = new ArrayList<Runnable>();
+
+      Handler mockHandler = mock(Handler.class);
+      imageStreamReader.handler = mockHandler;
+
+      // initially, handler will simulate a hanging main looper, that only queues inputs
+      when(mockHandler.post(any(Runnable.class)))
+          .thenAnswer(
+              inputs -> {
+                Runnable r = inputs.getArgument(0, Runnable.class);
+                runnables.add(r);
+                return true;
+              });
+
+      CameraCaptureProperties mockCaptureProps = mock(CameraCaptureProperties.class);
+      EventChannel.EventSink mockEventSink = mock(EventChannel.EventSink.class);
+
+      Image mockImage =
+          ImageStreamReaderTestUtils.getImage(1280, 720, 256, ImageFormat.YUV_420_888);
+      imageStreamReader.onImageAvailable(mockImage, mockCaptureProps, mockEventSink);
+
+      // make sure the image was closed, even when skipping frames
+      verify(mockImage, times(1)).close();
+
+      // check that we collected all runnables in this method
+      assertEquals(runnables.size(), 1);
+
+      // verify post() was not called more times than it should have
+      verify(mockHandler, times(1)).post(any(Runnable.class));
+
+      // make sure callback was not yet invoked
+      verify(mockEventSink, never()).success(any(Map.class));
+
+      // simulate frame processing
+      for (Runnable r : runnables) {
+        if (invalidateWeakReference) {
+          // Replace the captured WeakReference with one pointing to null.
+          Field[] fields = r.getClass().getDeclaredFields();
+          for (Field field : fields) {
+            if (field.getType().equals(WeakReference.class)) {
+              // Remove the `final` modifier
+              try {
+                field.set(r, new WeakReference<Map<String, Object>>(null));
+              } catch (IllegalAccessException e) {
+                throw new RuntimeException("Failed to inject null WeakReference", e);
+              }
+            }
+          }
+        }
+
+        r.run();
+      }
+
+      // make sure all callbacks were invoked so far
+      verify(mockEventSink, invalidateWeakReference ? never() : times(1)).success(any(Map.class));
+    }
+  }
 }

packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderTestUtils.java

@@ -0,0 +1,55 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package io.flutter.plugins.camera.media;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import android.media.Image;
+import java.nio.ByteBuffer;
+
+public class ImageStreamReaderTestUtils {
+  public static Image getImage(int imageWidth, int imageHeight, int padding, int imageFormat) {
+    int rowStride = imageWidth + padding;
+
+    int ySize = (rowStride * imageHeight) - padding;
+    int uSize = (ySize / 2) - (padding / 2);
+    int vSize = uSize;
+
+    // Mock YUV image
+    Image mockImage = mock(Image.class);
+    when(mockImage.getWidth()).thenReturn(imageWidth);
+    when(mockImage.getHeight()).thenReturn(imageHeight);
+    when(mockImage.getFormat()).thenReturn(imageFormat);
+
+    // Mock planes. YUV images have 3 planes (Y, U, V).
+    Image.Plane planeY = mock(Image.Plane.class);
+    Image.Plane planeU = mock(Image.Plane.class);
+    Image.Plane planeV = mock(Image.Plane.class);
+
+    // Y plane is width*height
+    // Row stride is generally == width but when there is padding it will
+    // be larger.
+    // Here we are adding 256 padding.
+    when(planeY.getBuffer()).thenReturn(ByteBuffer.allocate(ySize));
+    when(planeY.getRowStride()).thenReturn(rowStride);
+    when(planeY.getPixelStride()).thenReturn(1);
+
+    // U and V planes are always the same sizes/values.
+    // https://developer.android.com/reference/android/graphics/ImageFormat#YUV_420_888
+    when(planeU.getBuffer()).thenReturn(ByteBuffer.allocate(uSize));
+    when(planeV.getBuffer()).thenReturn(ByteBuffer.allocate(vSize));
+    when(planeU.getRowStride()).thenReturn(rowStride);
+    when(planeV.getRowStride()).thenReturn(rowStride);
+    when(planeU.getPixelStride()).thenReturn(2);
+    when(planeV.getPixelStride()).thenReturn(2);
+
+    // Add planes to image
+    Image.Plane[] planes = {planeY, planeU, planeV};
+    when(mockImage.getPlanes()).thenReturn(planes);
+
+    return mockImage;
+  }
+}