1 /* 2 * Copyright (C) 2021 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.hardware.camera2; 18 19 import android.annotation.CallbackExecutor; 20 import android.annotation.IntRange; 21 import android.annotation.NonNull; 22 import android.annotation.Nullable; 23 import android.annotation.SuppressLint; 24 import android.graphics.ImageFormat; 25 import android.graphics.ImageFormat.Format; 26 import android.hardware.HardwareBuffer; 27 import android.hardware.HardwareBuffer.Usage; 28 import android.media.Image; 29 import android.media.ImageReader; 30 import android.hardware.camera2.params.MultiResolutionStreamInfo; 31 import android.os.Looper; 32 import android.os.Message; 33 import android.util.Log; 34 import android.view.Surface; 35 36 37 import java.nio.NioUtils; 38 import java.util.ArrayList; 39 import java.util.Collection; 40 import java.util.List; 41 import java.util.concurrent.Executor; 42 43 /** 44 * <p>The MultiResolutionImageReader class wraps a group of {@link ImageReader ImageReaders} with 45 * the same format and different sizes, source camera Id, or camera sensor modes.</p> 46 * 47 * <p>The main use case of this class is for a 48 * {@link CameraCharacteristics#REQUEST_AVAILABLE_CAPABILITIES_LOGICAL_MULTI_CAMERA logical 49 * multi-camera} or an ultra high resolution sensor camera to output variable-size images. For a 50 * logical multi-camera which implements optical zoom, different physical cameras may have different 51 * maximum resolutions. As a result, when the camera device switches between physical cameras 52 * depending on zoom ratio, the maximum resolution for a particular format may change. For an 53 * ultra high resolution sensor camera, the camera device may deem it better or worse to run in 54 * maximum resolution mode / default mode depending on lighting conditions. So the application may 55 * choose to let the camera device decide on its behalf.</p> 56 * 57 * <p>MultiResolutionImageReader should be used for a camera device only if the camera device 58 * supports multi-resolution output stream by advertising the specified output format in {@link 59 * CameraCharacteristics#SCALER_MULTI_RESOLUTION_STREAM_CONFIGURATION_MAP}.</p> 60 * 61 * <p>To acquire images from the MultiResolutionImageReader, the application must use the 62 * {@link ImageReader} object passed by 63 * {@link ImageReader.OnImageAvailableListener#onImageAvailable} callback to call 64 * {@link ImageReader#acquireNextImage} or {@link ImageReader#acquireLatestImage}. The application 65 * must not use the {@link ImageReader} passed by an {@link 66 * ImageReader.OnImageAvailableListener#onImageAvailable} callback to acquire future images 67 * because future images may originate from a different {@link ImageReader} contained within the 68 * {@code MultiResolutionImageReader}.</p> 69 * 70 * 71 * @see ImageReader 72 * @see android.hardware.camera2.CameraCharacteristics#SCALER_MULTI_RESOLUTION_STREAM_CONFIGURATION_MAP 73 */ 74 public class MultiResolutionImageReader implements AutoCloseable { 75 76 private static final String TAG = "MultiResolutionImageReader"; 77 78 /** 79 * <p> 80 * Create a new multi-resolution reader based on a group of camera stream properties returned 81 * by a camera device. 82 * </p> 83 * <p> 84 * The valid size and formats depend on the camera characteristics. 85 * {@code MultiResolutionImageReader} for an image format is supported by the camera device if 86 * the format is in the supported multi-resolution output stream formats returned by 87 * {@link android.hardware.camera2.params.MultiResolutionStreamConfigurationMap#getOutputFormats}. 88 * If the image format is supported, the {@code MultiResolutionImageReader} object can be 89 * created with the {@code streams} objects returned by 90 * {@link android.hardware.camera2.params.MultiResolutionStreamConfigurationMap#getOutputInfo}. 91 * </p> 92 * <p> 93 * The {@code maxImages} parameter determines the maximum number of 94 * {@link Image} objects that can be acquired from each of the {@code ImageReader} 95 * within the {@code MultiResolutionImageReader}. However, requesting more buffers will 96 * use up more memory, so it is important to use only the minimum number necessary. The 97 * application is strongly recommended to acquire no more than {@code maxImages} images 98 * from all of the internal ImageReader objects combined. By keeping track of the number of 99 * acquired images for the MultiResolutionImageReader, the application doesn't need to do the 100 * bookkeeping for each internal ImageReader returned from {@link 101 * ImageReader.OnImageAvailableListener#onImageAvailable onImageAvailable} callback. 102 * </p> 103 * <p> 104 * Unlike the normal ImageReader, the MultiResolutionImageReader has a more complex 105 * configuration sequence. Instead of passing the same surface to OutputConfiguration and 106 * CaptureRequest, the 107 * {@link android.hardware.camera2.params.OutputConfiguration#createInstancesForMultiResolutionOutput} 108 * call needs to be used to create the OutputConfigurations for session creation, and then 109 * {@link #getSurface} is used to get {@link CaptureRequest.Builder#addTarget the target for 110 * CaptureRequest}. 111 * </p> 112 * @param streams The group of multi-resolution stream info, which is used to create 113 * a multi-resolution reader containing a number of ImageReader objects. Each 114 * ImageReader object represents a multi-resolution stream in the group. 115 * @param format The format of the Image that this multi-resolution reader will produce. 116 * This must be one of the {@link android.graphics.ImageFormat} or 117 * {@link android.graphics.PixelFormat} constants. Note that not all formats are 118 * supported, like ImageFormat.NV21. The supported multi-resolution 119 * reader format can be queried by {@link 120 * android.hardware.camera2.params.MultiResolutionStreamConfigurationMap#getOutputFormats}. 121 * @param maxImages The maximum number of images the user will want to 122 * access simultaneously. This should be as small as possible to 123 * limit memory use. Once maxImages images are obtained by the 124 * user from any given internal ImageReader, one of them has to be released before 125 * a new Image will become available for access through the ImageReader's 126 * {@link ImageReader#acquireLatestImage()} or 127 * {@link ImageReader#acquireNextImage()}. Must be greater than 0. 128 * @see Image 129 * @see 130 * android.hardware.camera2.CameraCharacteristics#SCALER_MULTI_RESOLUTION_STREAM_CONFIGURATION_MAP 131 * @see 132 * android.hardware.camera2.params.MultiResolutionStreamConfigurationMap 133 */ MultiResolutionImageReader( @onNull Collection<MultiResolutionStreamInfo> streams, @Format int format, @IntRange(from = 1) int maxImages)134 public MultiResolutionImageReader( 135 @NonNull Collection<MultiResolutionStreamInfo> streams, 136 @Format int format, 137 @IntRange(from = 1) int maxImages) { 138 mFormat = format; 139 mMaxImages = maxImages; 140 141 if (streams == null || streams.size() <= 1) { 142 throw new IllegalArgumentException( 143 "The streams info collection must contain at least 2 entries"); 144 } 145 if (mMaxImages < 1) { 146 throw new IllegalArgumentException( 147 "Maximum outstanding image count must be at least 1"); 148 } 149 150 if (format == ImageFormat.NV21) { 151 throw new IllegalArgumentException( 152 "NV21 format is not supported"); 153 } 154 155 int numImageReaders = streams.size(); 156 mReaders = new ImageReader[numImageReaders]; 157 mStreamInfo = new MultiResolutionStreamInfo[numImageReaders]; 158 int index = 0; 159 for (MultiResolutionStreamInfo streamInfo : streams) { 160 mReaders[index] = ImageReader.newInstance(streamInfo.getWidth(), 161 streamInfo.getHeight(), format, maxImages); 162 mStreamInfo[index] = streamInfo; 163 index++; 164 } 165 } 166 167 /** 168 * Set onImageAvailableListener callback. 169 * 170 * <p>This function sets the onImageAvailableListener for all the internal 171 * {@link ImageReader} objects.</p> 172 * 173 * <p>For a multi-resolution ImageReader, the timestamps of images acquired in 174 * onImageAvailable callback from different internal ImageReaders may become 175 * out-of-order due to the asynchronous callbacks between the different resolution 176 * image queues.</p> 177 * 178 * @param listener 179 * The listener that will be run. 180 * @param executor 181 * The executor which will be used when invoking the callback. 182 */ 183 @SuppressLint({"ExecutorRegistration", "SamShouldBeLast"}) setOnImageAvailableListener( @ullable ImageReader.OnImageAvailableListener listener, @Nullable @CallbackExecutor Executor executor)184 public void setOnImageAvailableListener( 185 @Nullable ImageReader.OnImageAvailableListener listener, 186 @Nullable @CallbackExecutor Executor executor) { 187 for (int i = 0; i < mReaders.length; i++) { 188 mReaders[i].setOnImageAvailableListenerWithExecutor(listener, executor); 189 } 190 } 191 192 @Override close()193 public void close() { 194 flush(); 195 196 for (int i = 0; i < mReaders.length; i++) { 197 mReaders[i].close(); 198 } 199 } 200 201 @Override finalize()202 protected void finalize() { 203 close(); 204 } 205 206 /** 207 * Flush pending images from all internal ImageReaders 208 * 209 * <p>Acquire and close pending images from all internal ImageReaders. This has the same 210 * effect as calling acquireLatestImage() on all internal ImageReaders, and closing all 211 * latest images.</p> 212 */ flush()213 public void flush() { 214 flushOther(null); 215 } 216 217 /** 218 * Flush pending images from other internal ImageReaders 219 * 220 * <p>Acquire and close pending images from all internal ImageReaders except for the 221 * one specified.</p> 222 * 223 * @param reader The ImageReader object that won't be flushed. 224 * 225 * @hide 226 */ flushOther(ImageReader reader)227 public void flushOther(ImageReader reader) { 228 for (int i = 0; i < mReaders.length; i++) { 229 if (reader != null && reader == mReaders[i]) { 230 continue; 231 } 232 233 while (true) { 234 Image image = mReaders[i].acquireNextImageNoThrowISE(); 235 if (image == null) { 236 break; 237 } else { 238 image.close(); 239 } 240 } 241 } 242 } 243 244 /** 245 * Get the internal ImageReader objects 246 * 247 * @hide 248 */ getReaders()249 public @NonNull ImageReader[] getReaders() { 250 return mReaders; 251 } 252 253 /** 254 * Get the surface that is used as a target for {@link CaptureRequest} 255 * 256 * <p>The application must use the surface returned by this function as a target for 257 * {@link CaptureRequest}. The camera device makes the decision on which internal 258 * {@code ImageReader} will receive the output image.</p> 259 * 260 * <p>Please note that holding on to the Surface objects returned by this method is not enough 261 * to keep their parent MultiResolutionImageReaders from being reclaimed. In that sense, a 262 * Surface acts like a {@link java.lang.ref.WeakReference weak reference} to the 263 * MultiResolutionImageReader that provides it.</p> 264 * 265 * @return a {@link Surface} to use as the target for a capture request. 266 */ getSurface()267 public @NonNull Surface getSurface() { 268 // Pick the surface of smallest size. This is necessary for an ultra high resolution 269 // camera not to default to maximum resolution pixel mode. 270 int minReaderSize = mReaders[0].getWidth() * mReaders[0].getHeight(); 271 Surface candidateSurface = mReaders[0].getSurface(); 272 for (int i = 1; i < mReaders.length; i++) { 273 int readerSize = mReaders[i].getWidth() * mReaders[i].getHeight(); 274 if (readerSize < minReaderSize) { 275 minReaderSize = readerSize; 276 candidateSurface = mReaders[i].getSurface(); 277 } 278 } 279 return candidateSurface; 280 } 281 282 /** 283 * Get the MultiResolutionStreamInfo describing the ImageReader an image originates from 284 * 285 *<p>An image from a {@code MultiResolutionImageReader} is produced from one of the underlying 286 *{@code ImageReader}s. This function returns the {@link MultiResolutionStreamInfo} to describe 287 *the property for that {@code ImageReader}, such as width, height, and physical camera Id.</p> 288 * 289 * @param reader An internal ImageReader within {@code MultiResolutionImageReader}. 290 * 291 * @return The stream info describing the internal {@code ImageReader}. 292 */ getStreamInfoForImageReader( @onNull ImageReader reader)293 public @NonNull MultiResolutionStreamInfo getStreamInfoForImageReader( 294 @NonNull ImageReader reader) { 295 for (int i = 0; i < mReaders.length; i++) { 296 if (reader == mReaders[i]) { 297 return mStreamInfo[i]; 298 } 299 } 300 301 throw new IllegalArgumentException("ImageReader doesn't belong to this multi-resolution " 302 + "imagereader"); 303 } 304 305 // mReaders and mStreamInfo has the same length, and their entries are 1:1 mapped. 306 private final ImageReader[] mReaders; 307 private final MultiResolutionStreamInfo[] mStreamInfo; 308 309 private final int mFormat; 310 private final int mMaxImages; 311 } 312