1 /* 2 * Copyright (C) 2021 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.hardware.camera2.impl; 18 19 import static android.hardware.camera2.impl.CameraExtensionUtils.JPEG_DEFAULT_QUALITY; 20 import static android.hardware.camera2.impl.CameraExtensionUtils.JPEG_DEFAULT_ROTATION; 21 22 import android.annotation.NonNull; 23 import android.graphics.ImageFormat; 24 import android.hardware.camera2.CaptureResult; 25 import android.hardware.camera2.extension.CaptureBundle; 26 import android.hardware.camera2.extension.ICaptureProcessorImpl; 27 import android.media.Image; 28 import android.media.Image.Plane; 29 import android.media.ImageReader; 30 import android.media.ImageWriter; 31 import android.os.Handler; 32 import android.os.HandlerThread; 33 import android.os.IBinder; 34 import android.os.RemoteException; 35 import android.util.Log; 36 import android.view.Surface; 37 38 import java.nio.ByteBuffer; 39 import java.util.HashSet; 40 import java.util.Iterator; 41 import java.util.List; 42 import java.util.concurrent.ConcurrentLinkedQueue; 43 44 // Jpeg compress input YUV and queue back in the client target surface. 45 public class CameraExtensionJpegProcessor implements ICaptureProcessorImpl { 46 public final static String TAG = "CameraExtensionJpeg"; 47 private final static int JPEG_QUEUE_SIZE = 1; 48 49 private final Handler mHandler; 50 private final HandlerThread mHandlerThread; 51 private final ICaptureProcessorImpl mProcessor; 52 53 private ImageReader mYuvReader = null; 54 private android.hardware.camera2.extension.Size mResolution = null; 55 private int mFormat = -1; 56 private Surface mOutputSurface = null; 57 private ImageWriter mOutputWriter = null; 58 59 private static final class JpegParameters { 60 public HashSet<Long> mTimeStamps = new HashSet<>(); 61 public int mRotation = JPEG_DEFAULT_ROTATION; // CCW multiple of 90 degrees 62 public int mQuality = JPEG_DEFAULT_QUALITY; // [0..100] 63 } 64 65 private ConcurrentLinkedQueue<JpegParameters> mJpegParameters = new ConcurrentLinkedQueue<>(); 66 CameraExtensionJpegProcessor(@onNull ICaptureProcessorImpl processor)67 public CameraExtensionJpegProcessor(@NonNull ICaptureProcessorImpl processor) { 68 mProcessor = processor; 69 mHandlerThread = new HandlerThread(TAG); 70 mHandlerThread.start(); 71 mHandler = new Handler(mHandlerThread.getLooper()); 72 } 73 close()74 public void close() { 75 mHandlerThread.quitSafely(); 76 77 if (mOutputWriter != null) { 78 mOutputWriter.close(); 79 mOutputWriter = null; 80 } 81 82 if (mYuvReader != null) { 83 mYuvReader.close(); 84 mYuvReader = null; 85 } 86 } 87 getJpegParameters(List<CaptureBundle> captureBundles)88 private static JpegParameters getJpegParameters(List<CaptureBundle> captureBundles) { 89 JpegParameters ret = new JpegParameters(); 90 if (!captureBundles.isEmpty()) { 91 // The quality and orientation settings must be equal for requests in a burst 92 93 Byte jpegQuality = captureBundles.get(0).captureResult.get(CaptureResult.JPEG_QUALITY); 94 if (jpegQuality != null) { 95 ret.mQuality = jpegQuality; 96 } else { 97 Log.w(TAG, "No jpeg quality set, using default: " + JPEG_DEFAULT_QUALITY); 98 } 99 100 Integer orientation = captureBundles.get(0).captureResult.get( 101 CaptureResult.JPEG_ORIENTATION); 102 if (orientation != null) { 103 ret.mRotation = orientation / 90; 104 } else { 105 Log.w(TAG, "No jpeg rotation set, using default: " + JPEG_DEFAULT_ROTATION); 106 } 107 108 for (CaptureBundle bundle : captureBundles) { 109 Long timeStamp = bundle.captureResult.get(CaptureResult.SENSOR_TIMESTAMP); 110 if (timeStamp != null) { 111 ret.mTimeStamps.add(timeStamp); 112 } else { 113 Log.e(TAG, "Capture bundle without valid sensor timestamp!"); 114 } 115 } 116 } 117 118 return ret; 119 } 120 121 /** 122 * Compresses a YCbCr image to jpeg, applying a crop and rotation. 123 * <p> 124 * The input is defined as a set of 3 planes of 8-bit samples, one plane for 125 * each channel of Y, Cb, Cr.<br> 126 * The Y plane is assumed to have the same width and height of the entire 127 * image.<br> 128 * The Cb and Cr planes are assumed to be downsampled by a factor of 2, to 129 * have dimensions (floor(width / 2), floor(height / 2)).<br> 130 * Each plane is specified by a direct java.nio.ByteBuffer, a pixel-stride, 131 * and a row-stride. So, the sample at coordinate (x, y) can be retrieved 132 * from byteBuffer[x * pixel_stride + y * row_stride]. 133 * <p> 134 * The pre-compression transformation is applied as follows: 135 * <ol> 136 * <li>The image is cropped to the rectangle from (cropLeft, cropTop) to 137 * (cropRight - 1, cropBottom - 1). So, a cropping-rectangle of (0, 0) - 138 * (width, height) is a no-op.</li> 139 * <li>The rotation is applied counter-clockwise relative to the coordinate 140 * space of the image, so a CCW rotation will appear CW when the image is 141 * rendered in scanline order. Only rotations which are multiples of 142 * 90-degrees are suppored, so the parameter 'rot90' specifies which 143 * multiple of 90 to rotate the image.</li> 144 * </ol> 145 * 146 * @param width the width of the image to compress 147 * @param height the height of the image to compress 148 * @param yBuf the buffer containing the Y component of the image 149 * @param yPStride the stride between adjacent pixels in the same row in 150 * yBuf 151 * @param yRStride the stride between adjacent rows in yBuf 152 * @param cbBuf the buffer containing the Cb component of the image 153 * @param cbPStride the stride between adjacent pixels in the same row in 154 * cbBuf 155 * @param cbRStride the stride between adjacent rows in cbBuf 156 * @param crBuf the buffer containing the Cr component of the image 157 * @param crPStride the stride between adjacent pixels in the same row in 158 * crBuf 159 * @param crRStride the stride between adjacent rows in crBuf 160 * @param outBuf a direct java.nio.ByteBuffer to hold the compressed jpeg. 161 * This must have enough capacity to store the result, or an 162 * error code will be returned. 163 * @param outBufCapacity the capacity of outBuf 164 * @param quality the jpeg-quality (1-100) to use 165 * @param cropLeft left-edge of the bounds of the image to crop to before 166 * rotation 167 * @param cropTop top-edge of the bounds of the image to crop to before 168 * rotation 169 * @param cropRight right-edge of the bounds of the image to crop to before 170 * rotation 171 * @param cropBottom bottom-edge of the bounds of the image to crop to 172 * before rotation 173 * @param rot90 the multiple of 90 to rotate the image CCW (after cropping) 174 */ compressJpegFromYUV420pNative( int width, int height, ByteBuffer yBuf, int yPStride, int yRStride, ByteBuffer cbBuf, int cbPStride, int cbRStride, ByteBuffer crBuf, int crPStride, int crRStride, ByteBuffer outBuf, int outBufCapacity, int quality, int cropLeft, int cropTop, int cropRight, int cropBottom, int rot90)175 private static native int compressJpegFromYUV420pNative( 176 int width, int height, 177 ByteBuffer yBuf, int yPStride, int yRStride, 178 ByteBuffer cbBuf, int cbPStride, int cbRStride, 179 ByteBuffer crBuf, int crPStride, int crRStride, 180 ByteBuffer outBuf, int outBufCapacity, 181 int quality, 182 int cropLeft, int cropTop, int cropRight, int cropBottom, 183 int rot90); 184 process(List<CaptureBundle> captureBundle)185 public void process(List<CaptureBundle> captureBundle) throws RemoteException { 186 JpegParameters jpegParams = getJpegParameters(captureBundle); 187 try { 188 mJpegParameters.add(jpegParams); 189 mProcessor.process(captureBundle); 190 } catch (Exception e) { 191 mJpegParameters.remove(jpegParams); 192 throw e; 193 } 194 } 195 onOutputSurface(Surface surface, int format)196 public void onOutputSurface(Surface surface, int format) throws RemoteException { 197 if (format != ImageFormat.JPEG) { 198 Log.e(TAG, "Unsupported output format: " + format); 199 return; 200 } 201 mOutputSurface = surface; 202 initializePipeline(); 203 } 204 205 @Override onResolutionUpdate(android.hardware.camera2.extension.Size size)206 public void onResolutionUpdate(android.hardware.camera2.extension.Size size) 207 throws RemoteException { 208 mResolution = size; 209 initializePipeline(); 210 } 211 onImageFormatUpdate(int format)212 public void onImageFormatUpdate(int format) throws RemoteException { 213 if (format != ImageFormat.YUV_420_888) { 214 Log.e(TAG, "Unsupported input format: " + format); 215 return; 216 } 217 mFormat = format; 218 initializePipeline(); 219 } 220 initializePipeline()221 private void initializePipeline() throws RemoteException { 222 if ((mFormat != -1) && (mOutputSurface != null) && (mResolution != null) && 223 (mYuvReader == null)) { 224 // Jpeg/blobs are expected to be configured with (w*h)x1 225 mOutputWriter = ImageWriter.newInstance(mOutputSurface, 1 /*maxImages*/, 226 ImageFormat.JPEG, mResolution.width * mResolution.height, 1); 227 mYuvReader = ImageReader.newInstance(mResolution.width, mResolution.height, mFormat, 228 JPEG_QUEUE_SIZE); 229 mYuvReader.setOnImageAvailableListener(new YuvCallback(), mHandler); 230 mProcessor.onOutputSurface(mYuvReader.getSurface(), mFormat); 231 mProcessor.onResolutionUpdate(mResolution); 232 mProcessor.onImageFormatUpdate(mFormat); 233 } 234 } 235 236 @Override asBinder()237 public IBinder asBinder() { 238 throw new UnsupportedOperationException("Binder IPC not supported!"); 239 } 240 241 private class YuvCallback implements ImageReader.OnImageAvailableListener { 242 @Override onImageAvailable(ImageReader reader)243 public void onImageAvailable(ImageReader reader) { 244 Image yuvImage = null; 245 Image jpegImage = null; 246 try { 247 yuvImage = mYuvReader.acquireNextImage(); 248 jpegImage = mOutputWriter.dequeueInputImage(); 249 } catch (IllegalStateException e) { 250 if (yuvImage != null) { 251 yuvImage.close(); 252 } 253 if (jpegImage != null) { 254 jpegImage.close(); 255 } 256 Log.e(TAG, "Failed to acquire processed yuv image or jpeg image!"); 257 return; 258 } 259 260 ByteBuffer jpegBuffer = jpegImage.getPlanes()[0].getBuffer(); 261 jpegBuffer.clear(); 262 // Jpeg/blobs are expected to be configured with (w*h)x1 263 int jpegCapacity = jpegImage.getWidth(); 264 265 Plane lumaPlane = yuvImage.getPlanes()[0]; 266 Plane crPlane = yuvImage.getPlanes()[1]; 267 Plane cbPlane = yuvImage.getPlanes()[2]; 268 269 Iterator<JpegParameters> jpegIter = mJpegParameters.iterator(); 270 JpegParameters jpegParams = null; 271 while(jpegIter.hasNext()) { 272 JpegParameters currentParams = jpegIter.next(); 273 if (currentParams.mTimeStamps.contains(yuvImage.getTimestamp())) { 274 jpegParams = currentParams; 275 jpegIter.remove(); 276 break; 277 } 278 } 279 if (jpegParams == null) { 280 if (mJpegParameters.isEmpty()) { 281 Log.w(TAG, "Empty jpeg settings queue! Using default jpeg orientation" 282 + " and quality!"); 283 jpegParams = new JpegParameters(); 284 jpegParams.mRotation = JPEG_DEFAULT_ROTATION; 285 jpegParams.mQuality = JPEG_DEFAULT_QUALITY; 286 } else { 287 Log.w(TAG, "No jpeg settings found with matching timestamp for current" 288 + " processed input!"); 289 Log.w(TAG, "Using values from the top of the queue!"); 290 jpegParams = mJpegParameters.poll(); 291 } 292 } 293 294 compressJpegFromYUV420pNative( 295 yuvImage.getWidth(), yuvImage.getHeight(), 296 lumaPlane.getBuffer(), lumaPlane.getPixelStride(), lumaPlane.getRowStride(), 297 crPlane.getBuffer(), crPlane.getPixelStride(), crPlane.getRowStride(), 298 cbPlane.getBuffer(), cbPlane.getPixelStride(), cbPlane.getRowStride(), 299 jpegBuffer, jpegCapacity, jpegParams.mQuality, 300 0, 0, yuvImage.getWidth(), yuvImage.getHeight(), 301 jpegParams.mRotation); 302 yuvImage.close(); 303 304 try { 305 mOutputWriter.queueInputImage(jpegImage); 306 } catch (IllegalStateException e) { 307 Log.e(TAG, "Failed to queue encoded result!"); 308 } finally { 309 jpegImage.close(); 310 } 311 } 312 } 313 } 314