1 /*
2 * Copyright 2013 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 //#define LOG_NDEBUG 0
18 #define LOG_TAG "ImageReader_JNI"
19 #include "android_media_Utils.h"
20 #include <cutils/atomic.h>
21 #include <utils/Log.h>
22 #include <utils/misc.h>
23 #include <utils/List.h>
24 #include <utils/String8.h>
25
26 #include <cstdio>
27
28 #include <gui/BufferItemConsumer.h>
29 #include <gui/Surface.h>
30 #include <camera3.h>
31
32 #include <android_runtime/AndroidRuntime.h>
33 #include <android_runtime/android_view_Surface.h>
34 #include <android_runtime/android_hardware_HardwareBuffer.h>
35 #include <grallocusage/GrallocUsageConversion.h>
36
37 #include <jni.h>
38 #include <JNIHelp.h>
39
40 #include <stdint.h>
41 #include <inttypes.h>
42
43 #define ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID "mNativeContext"
44 #define ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID "mNativeBuffer"
45 #define ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID "mTimestamp"
46
47 #define CONSUMER_BUFFER_USAGE_UNKNOWN 0;
48 // ----------------------------------------------------------------------------
49
50 using namespace android;
51
52
53 enum {
54 ACQUIRE_SUCCESS = 0,
55 ACQUIRE_NO_BUFFERS = 1,
56 ACQUIRE_MAX_IMAGES = 2,
57 };
58
59 static struct {
60 jfieldID mNativeContext;
61 jmethodID postEventFromNative;
62 } gImageReaderClassInfo;
63
64 static struct {
65 jfieldID mNativeBuffer;
66 jfieldID mTimestamp;
67 jfieldID mPlanes;
68 } gSurfaceImageClassInfo;
69
70 static struct {
71 jclass clazz;
72 jmethodID ctor;
73 } gSurfacePlaneClassInfo;
74
75 // Get an ID that's unique within this process.
createProcessUniqueId()76 static int32_t createProcessUniqueId() {
77 static volatile int32_t globalCounter = 0;
78 return android_atomic_inc(&globalCounter);
79 }
80
81 // ----------------------------------------------------------------------------
82
83 class JNIImageReaderContext : public ConsumerBase::FrameAvailableListener
84 {
85 public:
86 JNIImageReaderContext(JNIEnv* env, jobject weakThiz, jclass clazz, int maxImages);
87
88 virtual ~JNIImageReaderContext();
89
90 virtual void onFrameAvailable(const BufferItem& item);
91
92 BufferItem* getBufferItem();
93 void returnBufferItem(BufferItem* buffer);
94
95
setBufferConsumer(const sp<BufferItemConsumer> & consumer)96 void setBufferConsumer(const sp<BufferItemConsumer>& consumer) { mConsumer = consumer; }
getBufferConsumer()97 BufferItemConsumer* getBufferConsumer() { return mConsumer.get(); }
98
setProducer(const sp<IGraphicBufferProducer> & producer)99 void setProducer(const sp<IGraphicBufferProducer>& producer) { mProducer = producer; }
getProducer()100 IGraphicBufferProducer* getProducer() { return mProducer.get(); }
101
setBufferFormat(int format)102 void setBufferFormat(int format) { mFormat = format; }
getBufferFormat()103 int getBufferFormat() { return mFormat; }
104
setBufferDataspace(android_dataspace dataSpace)105 void setBufferDataspace(android_dataspace dataSpace) { mDataSpace = dataSpace; }
getBufferDataspace()106 android_dataspace getBufferDataspace() { return mDataSpace; }
107
setBufferWidth(int width)108 void setBufferWidth(int width) { mWidth = width; }
getBufferWidth()109 int getBufferWidth() { return mWidth; }
110
setBufferHeight(int height)111 void setBufferHeight(int height) { mHeight = height; }
getBufferHeight()112 int getBufferHeight() { return mHeight; }
113
114 private:
115 static JNIEnv* getJNIEnv(bool* needsDetach);
116 static void detachJNI();
117
118 List<BufferItem*> mBuffers;
119 sp<BufferItemConsumer> mConsumer;
120 sp<IGraphicBufferProducer> mProducer;
121 jobject mWeakThiz;
122 jclass mClazz;
123 int mFormat;
124 android_dataspace mDataSpace;
125 int mWidth;
126 int mHeight;
127 };
128
JNIImageReaderContext(JNIEnv * env,jobject weakThiz,jclass clazz,int maxImages)129 JNIImageReaderContext::JNIImageReaderContext(JNIEnv* env,
130 jobject weakThiz, jclass clazz, int maxImages) :
131 mWeakThiz(env->NewGlobalRef(weakThiz)),
132 mClazz((jclass)env->NewGlobalRef(clazz)),
133 mFormat(0),
134 mDataSpace(HAL_DATASPACE_UNKNOWN),
135 mWidth(-1),
136 mHeight(-1) {
137 for (int i = 0; i < maxImages; i++) {
138 BufferItem* buffer = new BufferItem;
139 mBuffers.push_back(buffer);
140 }
141 }
142
getJNIEnv(bool * needsDetach)143 JNIEnv* JNIImageReaderContext::getJNIEnv(bool* needsDetach) {
144 LOG_ALWAYS_FATAL_IF(needsDetach == NULL, "needsDetach is null!!!");
145 *needsDetach = false;
146 JNIEnv* env = AndroidRuntime::getJNIEnv();
147 if (env == NULL) {
148 JavaVMAttachArgs args = {JNI_VERSION_1_4, NULL, NULL};
149 JavaVM* vm = AndroidRuntime::getJavaVM();
150 int result = vm->AttachCurrentThread(&env, (void*) &args);
151 if (result != JNI_OK) {
152 ALOGE("thread attach failed: %#x", result);
153 return NULL;
154 }
155 *needsDetach = true;
156 }
157 return env;
158 }
159
detachJNI()160 void JNIImageReaderContext::detachJNI() {
161 JavaVM* vm = AndroidRuntime::getJavaVM();
162 int result = vm->DetachCurrentThread();
163 if (result != JNI_OK) {
164 ALOGE("thread detach failed: %#x", result);
165 }
166 }
167
getBufferItem()168 BufferItem* JNIImageReaderContext::getBufferItem() {
169 if (mBuffers.empty()) {
170 return NULL;
171 }
172 // Return a BufferItem pointer and remove it from the list
173 List<BufferItem*>::iterator it = mBuffers.begin();
174 BufferItem* buffer = *it;
175 mBuffers.erase(it);
176 return buffer;
177 }
178
returnBufferItem(BufferItem * buffer)179 void JNIImageReaderContext::returnBufferItem(BufferItem* buffer) {
180 buffer->mGraphicBuffer = nullptr;
181 mBuffers.push_back(buffer);
182 }
183
~JNIImageReaderContext()184 JNIImageReaderContext::~JNIImageReaderContext() {
185 bool needsDetach = false;
186 JNIEnv* env = getJNIEnv(&needsDetach);
187 if (env != NULL) {
188 env->DeleteGlobalRef(mWeakThiz);
189 env->DeleteGlobalRef(mClazz);
190 } else {
191 ALOGW("leaking JNI object references");
192 }
193 if (needsDetach) {
194 detachJNI();
195 }
196
197 // Delete buffer items.
198 for (List<BufferItem *>::iterator it = mBuffers.begin();
199 it != mBuffers.end(); it++) {
200 delete *it;
201 }
202
203 if (mConsumer != 0) {
204 mConsumer.clear();
205 }
206 }
207
onFrameAvailable(const BufferItem &)208 void JNIImageReaderContext::onFrameAvailable(const BufferItem& /*item*/)
209 {
210 ALOGV("%s: frame available", __FUNCTION__);
211 bool needsDetach = false;
212 JNIEnv* env = getJNIEnv(&needsDetach);
213 if (env != NULL) {
214 env->CallStaticVoidMethod(mClazz, gImageReaderClassInfo.postEventFromNative, mWeakThiz);
215 } else {
216 ALOGW("onFrameAvailable event will not posted");
217 }
218 if (needsDetach) {
219 detachJNI();
220 }
221 }
222
223 // ----------------------------------------------------------------------------
224
225 extern "C" {
226
ImageReader_getContext(JNIEnv * env,jobject thiz)227 static JNIImageReaderContext* ImageReader_getContext(JNIEnv* env, jobject thiz)
228 {
229 JNIImageReaderContext *ctx;
230 ctx = reinterpret_cast<JNIImageReaderContext *>
231 (env->GetLongField(thiz, gImageReaderClassInfo.mNativeContext));
232 return ctx;
233 }
234
ImageReader_getProducer(JNIEnv * env,jobject thiz)235 static IGraphicBufferProducer* ImageReader_getProducer(JNIEnv* env, jobject thiz)
236 {
237 ALOGV("%s:", __FUNCTION__);
238 JNIImageReaderContext* const ctx = ImageReader_getContext(env, thiz);
239 if (ctx == NULL) {
240 jniThrowRuntimeException(env, "ImageReaderContext is not initialized");
241 return NULL;
242 }
243
244 return ctx->getProducer();
245 }
246
ImageReader_setNativeContext(JNIEnv * env,jobject thiz,sp<JNIImageReaderContext> ctx)247 static void ImageReader_setNativeContext(JNIEnv* env,
248 jobject thiz, sp<JNIImageReaderContext> ctx)
249 {
250 ALOGV("%s:", __FUNCTION__);
251 JNIImageReaderContext* const p = ImageReader_getContext(env, thiz);
252 if (ctx != 0) {
253 ctx->incStrong((void*)ImageReader_setNativeContext);
254 }
255 if (p) {
256 p->decStrong((void*)ImageReader_setNativeContext);
257 }
258 env->SetLongField(thiz, gImageReaderClassInfo.mNativeContext,
259 reinterpret_cast<jlong>(ctx.get()));
260 }
261
ImageReader_getBufferConsumer(JNIEnv * env,jobject thiz)262 static BufferItemConsumer* ImageReader_getBufferConsumer(JNIEnv* env, jobject thiz)
263 {
264 ALOGV("%s:", __FUNCTION__);
265 JNIImageReaderContext* const ctx = ImageReader_getContext(env, thiz);
266 if (ctx == NULL) {
267 jniThrowRuntimeException(env, "ImageReaderContext is not initialized");
268 return NULL;
269 }
270
271 return ctx->getBufferConsumer();
272 }
273
Image_setBufferItem(JNIEnv * env,jobject thiz,const BufferItem * buffer)274 static void Image_setBufferItem(JNIEnv* env, jobject thiz,
275 const BufferItem* buffer)
276 {
277 env->SetLongField(thiz, gSurfaceImageClassInfo.mNativeBuffer, reinterpret_cast<jlong>(buffer));
278 }
279
Image_getBufferItem(JNIEnv * env,jobject image)280 static BufferItem* Image_getBufferItem(JNIEnv* env, jobject image)
281 {
282 return reinterpret_cast<BufferItem*>(
283 env->GetLongField(image, gSurfaceImageClassInfo.mNativeBuffer));
284 }
285
286
287 // ----------------------------------------------------------------------------
288
ImageReader_classInit(JNIEnv * env,jclass clazz)289 static void ImageReader_classInit(JNIEnv* env, jclass clazz)
290 {
291 ALOGV("%s:", __FUNCTION__);
292
293 jclass imageClazz = env->FindClass("android/media/ImageReader$SurfaceImage");
294 LOG_ALWAYS_FATAL_IF(imageClazz == NULL,
295 "can't find android/graphics/ImageReader$SurfaceImage");
296 gSurfaceImageClassInfo.mNativeBuffer = env->GetFieldID(
297 imageClazz, ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID, "J");
298 LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mNativeBuffer == NULL,
299 "can't find android/graphics/ImageReader.%s",
300 ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID);
301
302 gSurfaceImageClassInfo.mTimestamp = env->GetFieldID(
303 imageClazz, ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID, "J");
304 LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mTimestamp == NULL,
305 "can't find android/graphics/ImageReader.%s",
306 ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID);
307
308 gSurfaceImageClassInfo.mPlanes = env->GetFieldID(
309 imageClazz, "mPlanes", "[Landroid/media/ImageReader$SurfaceImage$SurfacePlane;");
310 LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mPlanes == NULL,
311 "can't find android/media/ImageReader$ReaderSurfaceImage.mPlanes");
312
313 gImageReaderClassInfo.mNativeContext = env->GetFieldID(
314 clazz, ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID, "J");
315 LOG_ALWAYS_FATAL_IF(gImageReaderClassInfo.mNativeContext == NULL,
316 "can't find android/graphics/ImageReader.%s",
317 ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID);
318
319 gImageReaderClassInfo.postEventFromNative = env->GetStaticMethodID(
320 clazz, "postEventFromNative", "(Ljava/lang/Object;)V");
321 LOG_ALWAYS_FATAL_IF(gImageReaderClassInfo.postEventFromNative == NULL,
322 "can't find android/graphics/ImageReader.postEventFromNative");
323
324 jclass planeClazz = env->FindClass("android/media/ImageReader$SurfaceImage$SurfacePlane");
325 LOG_ALWAYS_FATAL_IF(planeClazz == NULL, "Can not find SurfacePlane class");
326 // FindClass only gives a local reference of jclass object.
327 gSurfacePlaneClassInfo.clazz = (jclass) env->NewGlobalRef(planeClazz);
328 gSurfacePlaneClassInfo.ctor = env->GetMethodID(gSurfacePlaneClassInfo.clazz, "<init>",
329 "(Landroid/media/ImageReader$SurfaceImage;IILjava/nio/ByteBuffer;)V");
330 LOG_ALWAYS_FATAL_IF(gSurfacePlaneClassInfo.ctor == NULL,
331 "Can not find SurfacePlane constructor");
332 }
333
ImageReader_init(JNIEnv * env,jobject thiz,jobject weakThiz,jint width,jint height,jint format,jint maxImages,jlong ndkUsage)334 static void ImageReader_init(JNIEnv* env, jobject thiz, jobject weakThiz, jint width, jint height,
335 jint format, jint maxImages, jlong ndkUsage)
336 {
337 status_t res;
338 int nativeFormat;
339 android_dataspace nativeDataspace;
340
341 ALOGV("%s: width:%d, height: %d, format: 0x%x, maxImages:%d",
342 __FUNCTION__, width, height, format, maxImages);
343
344 PublicFormat publicFormat = static_cast<PublicFormat>(format);
345 nativeFormat = android_view_Surface_mapPublicFormatToHalFormat(
346 publicFormat);
347 nativeDataspace = android_view_Surface_mapPublicFormatToHalDataspace(
348 publicFormat);
349
350 jclass clazz = env->GetObjectClass(thiz);
351 if (clazz == NULL) {
352 jniThrowRuntimeException(env, "Can't find android/graphics/ImageReader");
353 return;
354 }
355 sp<JNIImageReaderContext> ctx(new JNIImageReaderContext(env, weakThiz, clazz, maxImages));
356
357 sp<IGraphicBufferProducer> gbProducer;
358 sp<IGraphicBufferConsumer> gbConsumer;
359 BufferQueue::createBufferQueue(&gbProducer, &gbConsumer);
360 sp<BufferItemConsumer> bufferConsumer;
361 String8 consumerName = String8::format("ImageReader-%dx%df%xm%d-%d-%d",
362 width, height, format, maxImages, getpid(),
363 createProcessUniqueId());
364 uint32_t consumerUsage = GRALLOC_USAGE_SW_READ_OFTEN;
365 bool needUsageOverride = ndkUsage != CONSUMER_BUFFER_USAGE_UNKNOWN;
366 uint64_t outProducerUsage = 0;
367 uint64_t outConsumerUsage = 0;
368 android_hardware_HardwareBuffer_convertToGrallocUsageBits(&outProducerUsage, &outConsumerUsage,
369 ndkUsage, 0);
370
371 if (isFormatOpaque(nativeFormat)) {
372 // Use the SW_READ_NEVER usage to tell producer that this format is not for preview or video
373 // encoding. The only possibility will be ZSL output.
374 consumerUsage = GRALLOC_USAGE_SW_READ_NEVER;
375 if (needUsageOverride) {
376 consumerUsage = android_convertGralloc1To0Usage(0, outConsumerUsage);
377 }
378 } else if (needUsageOverride) {
379 ALOGW("Consumer usage override for non-opaque format is not implemented yet, "
380 "ignore the provided usage from the application");
381 }
382 bufferConsumer = new BufferItemConsumer(gbConsumer, consumerUsage, maxImages,
383 /*controlledByApp*/true);
384 if (bufferConsumer == nullptr) {
385 jniThrowExceptionFmt(env, "java/lang/RuntimeException",
386 "Failed to allocate native buffer consumer for format 0x%x and usage 0x%x",
387 nativeFormat, consumerUsage);
388 return;
389 }
390 ctx->setBufferConsumer(bufferConsumer);
391 bufferConsumer->setName(consumerName);
392
393 ctx->setProducer(gbProducer);
394 bufferConsumer->setFrameAvailableListener(ctx);
395 ImageReader_setNativeContext(env, thiz, ctx);
396 ctx->setBufferFormat(nativeFormat);
397 ctx->setBufferDataspace(nativeDataspace);
398 ctx->setBufferWidth(width);
399 ctx->setBufferHeight(height);
400
401 // Set the width/height/format/dataspace to the bufferConsumer.
402 res = bufferConsumer->setDefaultBufferSize(width, height);
403 if (res != OK) {
404 jniThrowExceptionFmt(env, "java/lang/IllegalStateException",
405 "Failed to set buffer consumer default size (%dx%d) for format 0x%x",
406 width, height, nativeFormat);
407 return;
408 }
409 res = bufferConsumer->setDefaultBufferFormat(nativeFormat);
410 if (res != OK) {
411 jniThrowExceptionFmt(env, "java/lang/IllegalStateException",
412 "Failed to set buffer consumer default format 0x%x", nativeFormat);
413 }
414 res = bufferConsumer->setDefaultBufferDataSpace(nativeDataspace);
415 if (res != OK) {
416 jniThrowExceptionFmt(env, "java/lang/IllegalStateException",
417 "Failed to set buffer consumer default dataSpace 0x%x", nativeDataspace);
418 }
419 }
420
ImageReader_close(JNIEnv * env,jobject thiz)421 static void ImageReader_close(JNIEnv* env, jobject thiz)
422 {
423 ALOGV("%s:", __FUNCTION__);
424
425 JNIImageReaderContext* const ctx = ImageReader_getContext(env, thiz);
426 if (ctx == NULL) {
427 // ImageReader is already closed.
428 return;
429 }
430
431 BufferItemConsumer* consumer = NULL;
432 consumer = ImageReader_getBufferConsumer(env, thiz);
433
434 if (consumer != NULL) {
435 consumer->abandon();
436 consumer->setFrameAvailableListener(NULL);
437 }
438 ImageReader_setNativeContext(env, thiz, NULL);
439 }
440
Image_unlockIfLocked(JNIEnv * env,jobject image)441 static sp<Fence> Image_unlockIfLocked(JNIEnv* env, jobject image) {
442 ALOGV("%s", __FUNCTION__);
443 BufferItem* buffer = Image_getBufferItem(env, image);
444 if (buffer == NULL) {
445 jniThrowException(env, "java/lang/IllegalStateException",
446 "Image is not initialized");
447 return Fence::NO_FENCE;
448 }
449
450 // Is locked?
451 bool wasBufferLocked = false;
452 jobject planes = NULL;
453 if (!isFormatOpaque(buffer->mGraphicBuffer->getPixelFormat())) {
454 planes = env->GetObjectField(image, gSurfaceImageClassInfo.mPlanes);
455 }
456 wasBufferLocked = (planes != NULL);
457 if (wasBufferLocked) {
458 status_t res = OK;
459 int fenceFd = -1;
460 if (wasBufferLocked) {
461 res = buffer->mGraphicBuffer->unlockAsync(&fenceFd);
462 if (res != OK) {
463 jniThrowRuntimeException(env, "unlock buffer failed");
464 return Fence::NO_FENCE;
465 }
466 }
467 sp<Fence> releaseFence = new Fence(fenceFd);
468 return releaseFence;
469 ALOGV("Successfully unlocked the image");
470 }
471 return Fence::NO_FENCE;
472 }
473
ImageReader_imageRelease(JNIEnv * env,jobject thiz,jobject image)474 static void ImageReader_imageRelease(JNIEnv* env, jobject thiz, jobject image)
475 {
476 ALOGV("%s:", __FUNCTION__);
477 JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz);
478 if (ctx == NULL) {
479 ALOGW("ImageReader#close called before Image#close, consider calling Image#close first");
480 return;
481 }
482
483 BufferItemConsumer* bufferConsumer = ctx->getBufferConsumer();
484 BufferItem* buffer = Image_getBufferItem(env, image);
485 if (buffer == nullptr) {
486 // Release an already closed image is harmless.
487 return;
488 }
489
490 sp<Fence> releaseFence = Image_unlockIfLocked(env, image);
491 bufferConsumer->releaseBuffer(*buffer, releaseFence);
492 Image_setBufferItem(env, image, NULL);
493 ctx->returnBufferItem(buffer);
494 ALOGV("%s: Image (format: 0x%x) has been released", __FUNCTION__, ctx->getBufferFormat());
495 }
496
ImageReader_imageSetup(JNIEnv * env,jobject thiz,jobject image)497 static jint ImageReader_imageSetup(JNIEnv* env, jobject thiz, jobject image) {
498 ALOGV("%s:", __FUNCTION__);
499 JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz);
500 if (ctx == NULL) {
501 jniThrowException(env, "java/lang/IllegalStateException",
502 "ImageReader is not initialized or was already closed");
503 return -1;
504 }
505
506 BufferItemConsumer* bufferConsumer = ctx->getBufferConsumer();
507 BufferItem* buffer = ctx->getBufferItem();
508 if (buffer == NULL) {
509 ALOGW("Unable to acquire a buffer item, very likely client tried to acquire more than"
510 " maxImages buffers");
511 return ACQUIRE_MAX_IMAGES;
512 }
513
514 status_t res = bufferConsumer->acquireBuffer(buffer, 0);
515 if (res != OK) {
516 ctx->returnBufferItem(buffer);
517 if (res != BufferQueue::NO_BUFFER_AVAILABLE) {
518 if (res == INVALID_OPERATION) {
519 // Max number of images were already acquired.
520 ALOGE("%s: Max number of buffers allowed are already acquired : %s (%d)",
521 __FUNCTION__, strerror(-res), res);
522 return ACQUIRE_MAX_IMAGES;
523 } else {
524 ALOGE("%s: Acquire image failed with some unknown error: %s (%d)",
525 __FUNCTION__, strerror(-res), res);
526 jniThrowExceptionFmt(env, "java/lang/IllegalStateException",
527 "Unknown error (%d) when we tried to acquire an image.",
528 res);
529 return ACQUIRE_NO_BUFFERS;
530 }
531 }
532 // This isn't really an error case, as the application may acquire buffer at any time.
533 return ACQUIRE_NO_BUFFERS;
534 }
535
536 // Add some extra checks for non-opaque formats.
537 if (!isFormatOpaque(ctx->getBufferFormat())) {
538 // Check if the left-top corner of the crop rect is origin, we currently assume this point is
539 // zero, will revisit this once this assumption turns out problematic.
540 Point lt = buffer->mCrop.leftTop();
541 if (lt.x != 0 || lt.y != 0) {
542 jniThrowExceptionFmt(env, "java/lang/UnsupportedOperationException",
543 "crop left top corner [%d, %d] need to be at origin", lt.x, lt.y);
544 return -1;
545 }
546
547 // Check if the producer buffer configurations match what ImageReader configured.
548 int outputWidth = getBufferWidth(buffer);
549 int outputHeight = getBufferHeight(buffer);
550
551 int imgReaderFmt = ctx->getBufferFormat();
552 int imageReaderWidth = ctx->getBufferWidth();
553 int imageReaderHeight = ctx->getBufferHeight();
554 int bufferFormat = buffer->mGraphicBuffer->getPixelFormat();
555 if ((bufferFormat != HAL_PIXEL_FORMAT_BLOB) && (imgReaderFmt != HAL_PIXEL_FORMAT_BLOB) &&
556 (imageReaderWidth != outputWidth || imageReaderHeight != outputHeight)) {
557 ALOGV("%s: Producer buffer size: %dx%d, doesn't match ImageReader configured size: %dx%d",
558 __FUNCTION__, outputWidth, outputHeight, imageReaderWidth, imageReaderHeight);
559 }
560 if (imgReaderFmt != bufferFormat) {
561 if (imgReaderFmt == HAL_PIXEL_FORMAT_YCbCr_420_888 &&
562 isPossiblyYUV(bufferFormat)) {
563 // Treat formats that are compatible with flexible YUV
564 // (HAL_PIXEL_FORMAT_YCbCr_420_888) as HAL_PIXEL_FORMAT_YCbCr_420_888.
565 ALOGV("%s: Treat buffer format to 0x%x as HAL_PIXEL_FORMAT_YCbCr_420_888",
566 __FUNCTION__, bufferFormat);
567 } else if (imgReaderFmt == HAL_PIXEL_FORMAT_BLOB &&
568 bufferFormat == HAL_PIXEL_FORMAT_RGBA_8888) {
569 // Using HAL_PIXEL_FORMAT_RGBA_8888 Gralloc buffers containing JPEGs to get around
570 // SW write limitations for (b/17379185).
571 ALOGV("%s: Receiving JPEG in HAL_PIXEL_FORMAT_RGBA_8888 buffer.", __FUNCTION__);
572 } else {
573 // Return the buffer to the queue. No need to provide fence, as this buffer wasn't
574 // used anywhere yet.
575 bufferConsumer->releaseBuffer(*buffer);
576 ctx->returnBufferItem(buffer);
577
578 // Throw exception
579 ALOGE("Producer output buffer format: 0x%x, ImageReader configured format: 0x%x",
580 bufferFormat, ctx->getBufferFormat());
581 String8 msg;
582 msg.appendFormat("The producer output buffer format 0x%x doesn't "
583 "match the ImageReader's configured buffer format 0x%x.",
584 bufferFormat, ctx->getBufferFormat());
585 jniThrowException(env, "java/lang/UnsupportedOperationException",
586 msg.string());
587 return -1;
588 }
589 }
590
591 }
592
593 // Set SurfaceImage instance member variables
594 Image_setBufferItem(env, image, buffer);
595 env->SetLongField(image, gSurfaceImageClassInfo.mTimestamp,
596 static_cast<jlong>(buffer->mTimestamp));
597
598 return ACQUIRE_SUCCESS;
599 }
600
ImageReader_detachImage(JNIEnv * env,jobject thiz,jobject image)601 static jint ImageReader_detachImage(JNIEnv* env, jobject thiz, jobject image) {
602 ALOGV("%s:", __FUNCTION__);
603 JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz);
604 if (ctx == NULL) {
605 jniThrowException(env, "java/lang/IllegalStateException", "ImageReader was already closed");
606 return -1;
607 }
608
609 BufferItemConsumer* bufferConsumer = ctx->getBufferConsumer();
610 BufferItem* buffer = Image_getBufferItem(env, image);
611 if (!buffer) {
612 ALOGE(
613 "Image already released and can not be detached from ImageReader!!!");
614 jniThrowException(env, "java/lang/IllegalStateException",
615 "Image detach from ImageReader failed: buffer was already released");
616 return -1;
617 }
618
619 status_t res = OK;
620 Image_unlockIfLocked(env, image);
621 res = bufferConsumer->detachBuffer(buffer->mSlot);
622 if (res != OK) {
623 ALOGE("Image detach failed: %s (%d)!!!", strerror(-res), res);
624 jniThrowRuntimeException(env,
625 "nativeDetachImage failed for image!!!");
626 return res;
627 }
628 return OK;
629 }
630
ImageReader_discardFreeBuffers(JNIEnv * env,jobject thiz)631 static void ImageReader_discardFreeBuffers(JNIEnv* env, jobject thiz) {
632 ALOGV("%s:", __FUNCTION__);
633 JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz);
634 if (ctx == NULL) {
635 jniThrowException(env, "java/lang/IllegalStateException", "ImageReader was already closed");
636 return;
637 }
638
639 BufferItemConsumer* bufferConsumer = ctx->getBufferConsumer();
640 status_t res = bufferConsumer->discardFreeBuffers();
641 if (res != OK) {
642 ALOGE("Buffer discard failed: %s (%d)", strerror(-res), res);
643 jniThrowRuntimeException(env,
644 "nativeDicardFreebuffers failed");
645 }
646 }
647
ImageReader_getSurface(JNIEnv * env,jobject thiz)648 static jobject ImageReader_getSurface(JNIEnv* env, jobject thiz)
649 {
650 ALOGV("%s: ", __FUNCTION__);
651
652 IGraphicBufferProducer* gbp = ImageReader_getProducer(env, thiz);
653 if (gbp == NULL) {
654 jniThrowRuntimeException(env, "Buffer consumer is uninitialized");
655 return NULL;
656 }
657
658 // Wrap the IGBP in a Java-language Surface.
659 return android_view_Surface_createFromIGraphicBufferProducer(env, gbp);
660 }
661
Image_getLockedImage(JNIEnv * env,jobject thiz,LockedImage * image)662 static void Image_getLockedImage(JNIEnv* env, jobject thiz, LockedImage *image) {
663 ALOGV("%s", __FUNCTION__);
664 BufferItem* buffer = Image_getBufferItem(env, thiz);
665 if (buffer == NULL) {
666 jniThrowException(env, "java/lang/IllegalStateException",
667 "Image is not initialized");
668 return;
669 }
670
671 status_t res = lockImageFromBuffer(buffer,
672 GRALLOC_USAGE_SW_READ_OFTEN, buffer->mFence->dup(), image);
673 if (res != OK) {
674 jniThrowExceptionFmt(env, "java/lang/RuntimeException",
675 "lock buffer failed for format 0x%x",
676 buffer->mGraphicBuffer->getPixelFormat());
677 return;
678 }
679
680 // Carry over some fields from BufferItem.
681 image->crop = buffer->mCrop;
682 image->transform = buffer->mTransform;
683 image->scalingMode = buffer->mScalingMode;
684 image->timestamp = buffer->mTimestamp;
685 image->dataSpace = buffer->mDataSpace;
686 image->frameNumber = buffer->mFrameNumber;
687
688 ALOGV("%s: Successfully locked the image", __FUNCTION__);
689 // crop, transform, scalingMode, timestamp, and frameNumber should be set by producer,
690 // and we don't set them here.
691 }
692
Image_getLockedImageInfo(JNIEnv * env,LockedImage * buffer,int idx,int32_t writerFormat,uint8_t ** base,uint32_t * size,int * pixelStride,int * rowStride)693 static void Image_getLockedImageInfo(JNIEnv* env, LockedImage* buffer, int idx,
694 int32_t writerFormat, uint8_t **base, uint32_t *size, int *pixelStride, int *rowStride) {
695 ALOGV("%s", __FUNCTION__);
696
697 status_t res = getLockedImageInfo(buffer, idx, writerFormat, base, size,
698 pixelStride, rowStride);
699 if (res != OK) {
700 jniThrowExceptionFmt(env, "java/lang/UnsupportedOperationException",
701 "Pixel format: 0x%x is unsupported", buffer->flexFormat);
702 }
703 }
704
Image_createSurfacePlanes(JNIEnv * env,jobject thiz,int numPlanes,int readerFormat)705 static jobjectArray Image_createSurfacePlanes(JNIEnv* env, jobject thiz,
706 int numPlanes, int readerFormat)
707 {
708 ALOGV("%s: create SurfacePlane array with size %d", __FUNCTION__, numPlanes);
709 int rowStride = 0;
710 int pixelStride = 0;
711 uint8_t *pData = NULL;
712 uint32_t dataSize = 0;
713 jobject byteBuffer = NULL;
714
715 PublicFormat publicReaderFormat = static_cast<PublicFormat>(readerFormat);
716 int halReaderFormat = android_view_Surface_mapPublicFormatToHalFormat(
717 publicReaderFormat);
718
719 if (isFormatOpaque(halReaderFormat) && numPlanes > 0) {
720 String8 msg;
721 msg.appendFormat("Format 0x%x is opaque, thus not writable, the number of planes (%d)"
722 " must be 0", halReaderFormat, numPlanes);
723 jniThrowException(env, "java/lang/IllegalArgumentException", msg.string());
724 return NULL;
725 }
726
727 jobjectArray surfacePlanes = env->NewObjectArray(numPlanes, gSurfacePlaneClassInfo.clazz,
728 /*initial_element*/NULL);
729 if (surfacePlanes == NULL) {
730 jniThrowRuntimeException(env, "Failed to create SurfacePlane arrays,"
731 " probably out of memory");
732 return NULL;
733 }
734 if (isFormatOpaque(halReaderFormat)) {
735 // Return 0 element surface array.
736 return surfacePlanes;
737 }
738
739 LockedImage lockedImg = LockedImage();
740 Image_getLockedImage(env, thiz, &lockedImg);
741 if (env->ExceptionCheck()) {
742 return NULL;
743 }
744 // Create all SurfacePlanes
745 for (int i = 0; i < numPlanes; i++) {
746 Image_getLockedImageInfo(env, &lockedImg, i, halReaderFormat,
747 &pData, &dataSize, &pixelStride, &rowStride);
748 byteBuffer = env->NewDirectByteBuffer(pData, dataSize);
749 if ((byteBuffer == NULL) && (env->ExceptionCheck() == false)) {
750 jniThrowException(env, "java/lang/IllegalStateException",
751 "Failed to allocate ByteBuffer");
752 return NULL;
753 }
754
755 // Finally, create this SurfacePlane.
756 jobject surfacePlane = env->NewObject(gSurfacePlaneClassInfo.clazz,
757 gSurfacePlaneClassInfo.ctor, thiz, rowStride, pixelStride, byteBuffer);
758 env->SetObjectArrayElement(surfacePlanes, i, surfacePlane);
759 }
760
761 return surfacePlanes;
762 }
763
Image_getWidth(JNIEnv * env,jobject thiz)764 static jint Image_getWidth(JNIEnv* env, jobject thiz)
765 {
766 BufferItem* buffer = Image_getBufferItem(env, thiz);
767 return getBufferWidth(buffer);
768 }
769
Image_getHeight(JNIEnv * env,jobject thiz)770 static jint Image_getHeight(JNIEnv* env, jobject thiz)
771 {
772 BufferItem* buffer = Image_getBufferItem(env, thiz);
773 return getBufferHeight(buffer);
774 }
775
Image_getFormat(JNIEnv * env,jobject thiz,jint readerFormat)776 static jint Image_getFormat(JNIEnv* env, jobject thiz, jint readerFormat)
777 {
778 if (isFormatOpaque(readerFormat)) {
779 // Assuming opaque reader produce opaque images.
780 return static_cast<jint>(PublicFormat::PRIVATE);
781 } else {
782 BufferItem* buffer = Image_getBufferItem(env, thiz);
783 int readerHalFormat = android_view_Surface_mapPublicFormatToHalFormat(
784 static_cast<PublicFormat>(readerFormat));
785 int32_t fmt = applyFormatOverrides(
786 buffer->mGraphicBuffer->getPixelFormat(), readerHalFormat);
787 // Override the image format to HAL_PIXEL_FORMAT_YCbCr_420_888 if the actual format is
788 // NV21 or YV12. This could only happen when the Gralloc HAL version is v0.1 thus doesn't
789 // support lockycbcr(), the CpuConsumer need to use the lock() method in the
790 // lockNextBuffer() call. For Gralloc HAL v0.2 or newer, this format should already be
791 // overridden to HAL_PIXEL_FORMAT_YCbCr_420_888 for the flexible YUV compatible formats.
792 if (isPossiblyYUV(fmt)) {
793 fmt = HAL_PIXEL_FORMAT_YCbCr_420_888;
794 }
795 PublicFormat publicFmt = android_view_Surface_mapHalFormatDataspaceToPublicFormat(
796 fmt, buffer->mDataSpace);
797 return static_cast<jint>(publicFmt);
798 }
799 }
800
801 } // extern "C"
802
803 // ----------------------------------------------------------------------------
804
805 static const JNINativeMethod gImageReaderMethods[] = {
806 {"nativeClassInit", "()V", (void*)ImageReader_classInit },
807 {"nativeInit", "(Ljava/lang/Object;IIIIJ)V", (void*)ImageReader_init },
808 {"nativeClose", "()V", (void*)ImageReader_close },
809 {"nativeReleaseImage", "(Landroid/media/Image;)V", (void*)ImageReader_imageRelease },
810 {"nativeImageSetup", "(Landroid/media/Image;)I", (void*)ImageReader_imageSetup },
811 {"nativeGetSurface", "()Landroid/view/Surface;", (void*)ImageReader_getSurface },
812 {"nativeDetachImage", "(Landroid/media/Image;)I", (void*)ImageReader_detachImage },
813 {"nativeDiscardFreeBuffers", "()V", (void*)ImageReader_discardFreeBuffers }
814 };
815
816 static const JNINativeMethod gImageMethods[] = {
817 {"nativeCreatePlanes", "(II)[Landroid/media/ImageReader$SurfaceImage$SurfacePlane;",
818 (void*)Image_createSurfacePlanes },
819 {"nativeGetWidth", "()I", (void*)Image_getWidth },
820 {"nativeGetHeight", "()I", (void*)Image_getHeight },
821 {"nativeGetFormat", "(I)I", (void*)Image_getFormat },
822 };
823
register_android_media_ImageReader(JNIEnv * env)824 int register_android_media_ImageReader(JNIEnv *env) {
825
826 int ret1 = AndroidRuntime::registerNativeMethods(env,
827 "android/media/ImageReader", gImageReaderMethods, NELEM(gImageReaderMethods));
828
829 int ret2 = AndroidRuntime::registerNativeMethods(env,
830 "android/media/ImageReader$SurfaceImage", gImageMethods, NELEM(gImageMethods));
831
832 return (ret1 || ret2);
833 }
834