1 /* 2 * Copyright (C) 2021 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.hardware.camera2.impl; 18 19 import static android.hardware.camera2.impl.CameraExtensionUtils.JPEG_DEFAULT_QUALITY; 20 import static android.hardware.camera2.impl.CameraExtensionUtils.JPEG_DEFAULT_ROTATION; 21 22 import android.annotation.NonNull; 23 import android.graphics.ImageFormat; 24 import android.hardware.camera2.CaptureResult; 25 import android.hardware.camera2.extension.CaptureBundle; 26 import android.hardware.camera2.extension.ICaptureProcessorImpl; 27 import android.hardware.camera2.extension.IProcessResultImpl; 28 import android.media.Image; 29 import android.media.Image.Plane; 30 import android.media.ImageReader; 31 import android.media.ImageWriter; 32 import android.os.Handler; 33 import android.os.HandlerThread; 34 import android.os.IBinder; 35 import android.os.RemoteException; 36 import android.util.Log; 37 import android.view.Surface; 38 39 import java.nio.ByteBuffer; 40 import java.util.HashSet; 41 import java.util.Iterator; 42 import java.util.List; 43 import java.util.concurrent.ConcurrentLinkedQueue; 44 45 // Jpeg compress input YUV and queue back in the client target surface. 46 public class CameraExtensionJpegProcessor implements ICaptureProcessorImpl { 47 public final static String TAG = "CameraExtensionJpeg"; 48 private final static int JPEG_QUEUE_SIZE = 1; 49 50 private final Handler mHandler; 51 private final HandlerThread mHandlerThread; 52 private final ICaptureProcessorImpl mProcessor; 53 54 private ImageReader mYuvReader = null; 55 private android.hardware.camera2.extension.Size mResolution = null; 56 private int mFormat = -1; 57 private Surface mOutputSurface = null; 58 private ImageWriter mOutputWriter = null; 59 60 private static final class JpegParameters { 61 public HashSet<Long> mTimeStamps = new HashSet<>(); 62 public int mRotation = JPEG_DEFAULT_ROTATION; // CW multiple of 90 degrees 63 public int mQuality = JPEG_DEFAULT_QUALITY; // [0..100] 64 } 65 66 private ConcurrentLinkedQueue<JpegParameters> mJpegParameters = new ConcurrentLinkedQueue<>(); 67 CameraExtensionJpegProcessor(@onNull ICaptureProcessorImpl processor)68 public CameraExtensionJpegProcessor(@NonNull ICaptureProcessorImpl processor) { 69 mProcessor = processor; 70 mHandlerThread = new HandlerThread(TAG); 71 mHandlerThread.start(); 72 mHandler = new Handler(mHandlerThread.getLooper()); 73 } 74 close()75 public void close() { 76 mHandlerThread.quitSafely(); 77 78 if (mOutputWriter != null) { 79 mOutputWriter.close(); 80 mOutputWriter = null; 81 } 82 83 if (mYuvReader != null) { 84 mYuvReader.close(); 85 mYuvReader = null; 86 } 87 } 88 getJpegParameters(List<CaptureBundle> captureBundles)89 private static JpegParameters getJpegParameters(List<CaptureBundle> captureBundles) { 90 JpegParameters ret = new JpegParameters(); 91 if (!captureBundles.isEmpty()) { 92 // The quality and orientation settings must be equal for requests in a burst 93 94 Byte jpegQuality = captureBundles.get(0).captureResult.get(CaptureResult.JPEG_QUALITY); 95 if (jpegQuality != null) { 96 ret.mQuality = jpegQuality; 97 } else { 98 Log.w(TAG, "No jpeg quality set, using default: " + JPEG_DEFAULT_QUALITY); 99 } 100 101 Integer orientation = captureBundles.get(0).captureResult.get( 102 CaptureResult.JPEG_ORIENTATION); 103 if (orientation != null) { 104 // The jpeg encoder expects CCW rotation, convert from CW 105 ret.mRotation = (360 - (orientation % 360)) / 90; 106 } else { 107 Log.w(TAG, "No jpeg rotation set, using default: " + JPEG_DEFAULT_ROTATION); 108 } 109 110 for (CaptureBundle bundle : captureBundles) { 111 Long timeStamp = bundle.captureResult.get(CaptureResult.SENSOR_TIMESTAMP); 112 if (timeStamp != null) { 113 ret.mTimeStamps.add(timeStamp); 114 } else { 115 Log.e(TAG, "Capture bundle without valid sensor timestamp!"); 116 } 117 } 118 } 119 120 return ret; 121 } 122 123 /** 124 * Compresses a YCbCr image to jpeg, applying a crop and rotation. 125 * <p> 126 * The input is defined as a set of 3 planes of 8-bit samples, one plane for 127 * each channel of Y, Cb, Cr.<br> 128 * The Y plane is assumed to have the same width and height of the entire 129 * image.<br> 130 * The Cb and Cr planes are assumed to be downsampled by a factor of 2, to 131 * have dimensions (floor(width / 2), floor(height / 2)).<br> 132 * Each plane is specified by a direct java.nio.ByteBuffer, a pixel-stride, 133 * and a row-stride. So, the sample at coordinate (x, y) can be retrieved 134 * from byteBuffer[x * pixel_stride + y * row_stride]. 135 * <p> 136 * The pre-compression transformation is applied as follows: 137 * <ol> 138 * <li>The image is cropped to the rectangle from (cropLeft, cropTop) to 139 * (cropRight - 1, cropBottom - 1). So, a cropping-rectangle of (0, 0) - 140 * (width, height) is a no-op.</li> 141 * <li>The rotation is applied counter-clockwise relative to the coordinate 142 * space of the image, so a CCW rotation will appear CW when the image is 143 * rendered in scanline order. Only rotations which are multiples of 144 * 90-degrees are suppored, so the parameter 'rot90' specifies which 145 * multiple of 90 to rotate the image.</li> 146 * </ol> 147 * 148 * @param width the width of the image to compress 149 * @param height the height of the image to compress 150 * @param yBuf the buffer containing the Y component of the image 151 * @param yPStride the stride between adjacent pixels in the same row in 152 * yBuf 153 * @param yRStride the stride between adjacent rows in yBuf 154 * @param cbBuf the buffer containing the Cb component of the image 155 * @param cbPStride the stride between adjacent pixels in the same row in 156 * cbBuf 157 * @param cbRStride the stride between adjacent rows in cbBuf 158 * @param crBuf the buffer containing the Cr component of the image 159 * @param crPStride the stride between adjacent pixels in the same row in 160 * crBuf 161 * @param crRStride the stride between adjacent rows in crBuf 162 * @param outBuf a direct java.nio.ByteBuffer to hold the compressed jpeg. 163 * This must have enough capacity to store the result, or an 164 * error code will be returned. 165 * @param outBufCapacity the capacity of outBuf 166 * @param quality the jpeg-quality (1-100) to use 167 * @param cropLeft left-edge of the bounds of the image to crop to before 168 * rotation 169 * @param cropTop top-edge of the bounds of the image to crop to before 170 * rotation 171 * @param cropRight right-edge of the bounds of the image to crop to before 172 * rotation 173 * @param cropBottom bottom-edge of the bounds of the image to crop to 174 * before rotation 175 * @param rot90 the multiple of 90 to rotate the image CCW (after cropping) 176 */ compressJpegFromYUV420pNative( int width, int height, ByteBuffer yBuf, int yPStride, int yRStride, ByteBuffer cbBuf, int cbPStride, int cbRStride, ByteBuffer crBuf, int crPStride, int crRStride, ByteBuffer outBuf, int outBufCapacity, int quality, int cropLeft, int cropTop, int cropRight, int cropBottom, int rot90)177 private static native int compressJpegFromYUV420pNative( 178 int width, int height, 179 ByteBuffer yBuf, int yPStride, int yRStride, 180 ByteBuffer cbBuf, int cbPStride, int cbRStride, 181 ByteBuffer crBuf, int crPStride, int crRStride, 182 ByteBuffer outBuf, int outBufCapacity, 183 int quality, 184 int cropLeft, int cropTop, int cropRight, int cropBottom, 185 int rot90); 186 187 @Override process(List<CaptureBundle> captureBundle, IProcessResultImpl captureCallback)188 public void process(List<CaptureBundle> captureBundle, IProcessResultImpl captureCallback) 189 throws RemoteException { 190 JpegParameters jpegParams = getJpegParameters(captureBundle); 191 try { 192 mJpegParameters.add(jpegParams); 193 mProcessor.process(captureBundle, captureCallback); 194 } catch (Exception e) { 195 mJpegParameters.remove(jpegParams); 196 throw e; 197 } 198 } 199 onOutputSurface(Surface surface, int format)200 public void onOutputSurface(Surface surface, int format) throws RemoteException { 201 if (format != ImageFormat.JPEG) { 202 Log.e(TAG, "Unsupported output format: " + format); 203 return; 204 } 205 mOutputSurface = surface; 206 initializePipeline(); 207 } 208 209 @Override onResolutionUpdate(android.hardware.camera2.extension.Size size)210 public void onResolutionUpdate(android.hardware.camera2.extension.Size size) 211 throws RemoteException { 212 mResolution = size; 213 initializePipeline(); 214 } 215 onImageFormatUpdate(int format)216 public void onImageFormatUpdate(int format) throws RemoteException { 217 if (format != ImageFormat.YUV_420_888) { 218 Log.e(TAG, "Unsupported input format: " + format); 219 return; 220 } 221 mFormat = format; 222 initializePipeline(); 223 } 224 initializePipeline()225 private void initializePipeline() throws RemoteException { 226 if ((mFormat != -1) && (mOutputSurface != null) && (mResolution != null) && 227 (mYuvReader == null)) { 228 // Jpeg/blobs are expected to be configured with (w*h)x1 229 mOutputWriter = ImageWriter.newInstance(mOutputSurface, 1 /*maxImages*/, 230 ImageFormat.JPEG, mResolution.width * mResolution.height, 1); 231 mYuvReader = ImageReader.newInstance(mResolution.width, mResolution.height, mFormat, 232 JPEG_QUEUE_SIZE); 233 mYuvReader.setOnImageAvailableListener(new YuvCallback(), mHandler); 234 mProcessor.onOutputSurface(mYuvReader.getSurface(), mFormat); 235 mProcessor.onResolutionUpdate(mResolution); 236 mProcessor.onImageFormatUpdate(mFormat); 237 } 238 } 239 240 @Override asBinder()241 public IBinder asBinder() { 242 throw new UnsupportedOperationException("Binder IPC not supported!"); 243 } 244 245 private class YuvCallback implements ImageReader.OnImageAvailableListener { 246 @Override onImageAvailable(ImageReader reader)247 public void onImageAvailable(ImageReader reader) { 248 Image yuvImage = null; 249 Image jpegImage = null; 250 try { 251 yuvImage = mYuvReader.acquireNextImage(); 252 jpegImage = mOutputWriter.dequeueInputImage(); 253 } catch (IllegalStateException e) { 254 if (yuvImage != null) { 255 yuvImage.close(); 256 } 257 if (jpegImage != null) { 258 jpegImage.close(); 259 } 260 Log.e(TAG, "Failed to acquire processed yuv image or jpeg image!"); 261 return; 262 } 263 264 ByteBuffer jpegBuffer = jpegImage.getPlanes()[0].getBuffer(); 265 jpegBuffer.clear(); 266 // Jpeg/blobs are expected to be configured with (w*h)x1 267 int jpegCapacity = jpegImage.getWidth(); 268 269 Plane lumaPlane = yuvImage.getPlanes()[0]; 270 Plane crPlane = yuvImage.getPlanes()[1]; 271 Plane cbPlane = yuvImage.getPlanes()[2]; 272 273 Iterator<JpegParameters> jpegIter = mJpegParameters.iterator(); 274 JpegParameters jpegParams = null; 275 while(jpegIter.hasNext()) { 276 JpegParameters currentParams = jpegIter.next(); 277 if (currentParams.mTimeStamps.contains(yuvImage.getTimestamp())) { 278 jpegParams = currentParams; 279 jpegIter.remove(); 280 break; 281 } 282 } 283 if (jpegParams == null) { 284 if (mJpegParameters.isEmpty()) { 285 Log.w(TAG, "Empty jpeg settings queue! Using default jpeg orientation" 286 + " and quality!"); 287 jpegParams = new JpegParameters(); 288 jpegParams.mRotation = JPEG_DEFAULT_ROTATION; 289 jpegParams.mQuality = JPEG_DEFAULT_QUALITY; 290 } else { 291 Log.w(TAG, "No jpeg settings found with matching timestamp for current" 292 + " processed input!"); 293 Log.w(TAG, "Using values from the top of the queue!"); 294 jpegParams = mJpegParameters.poll(); 295 } 296 } 297 298 compressJpegFromYUV420pNative( 299 yuvImage.getWidth(), yuvImage.getHeight(), 300 lumaPlane.getBuffer(), lumaPlane.getPixelStride(), lumaPlane.getRowStride(), 301 crPlane.getBuffer(), crPlane.getPixelStride(), crPlane.getRowStride(), 302 cbPlane.getBuffer(), cbPlane.getPixelStride(), cbPlane.getRowStride(), 303 jpegBuffer, jpegCapacity, jpegParams.mQuality, 304 0, 0, yuvImage.getWidth(), yuvImage.getHeight(), 305 jpegParams.mRotation); 306 jpegImage.setTimestamp(yuvImage.getTimestamp()); 307 yuvImage.close(); 308 309 try { 310 mOutputWriter.queueInputImage(jpegImage); 311 } catch (IllegalStateException e) { 312 Log.e(TAG, "Failed to queue encoded result!"); 313 } finally { 314 jpegImage.close(); 315 } 316 } 317 } 318 } 319