1 /*
2 * Copyright 2013 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 //#define LOG_NDEBUG 0
18 #define LOG_TAG "ImageReader_JNI"
19 #include "android_media_Utils.h"
20 #include <cutils/atomic.h>
21 #include <utils/Log.h>
22 #include <utils/misc.h>
23 #include <utils/List.h>
24 #include <utils/String8.h>
25
26 #include <cstdio>
27
28 #include <gui/BufferItemConsumer.h>
29 #include <gui/Surface.h>
30
31 #include <android_runtime/AndroidRuntime.h>
32 #include <android_runtime/android_view_Surface.h>
33 #include <android_runtime/android_hardware_HardwareBuffer.h>
34 #include <grallocusage/GrallocUsageConversion.h>
35
36 #include <private/android/AHardwareBufferHelpers.h>
37
38 #include <jni.h>
39 #include <nativehelper/JNIHelp.h>
40
41 #include <stdint.h>
42 #include <inttypes.h>
43 #include <android/hardware_buffer_jni.h>
44
45 #define ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID "mNativeContext"
46 #define ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID "mNativeBuffer"
47 #define ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID "mTimestamp"
48 #define ANDROID_MEDIA_SURFACEIMAGE_TF_JNI_ID "mTransform"
49 #define ANDROID_MEDIA_SURFACEIMAGE_SM_JNI_ID "mScalingMode"
50
51 #define CONSUMER_BUFFER_USAGE_UNKNOWN 0;
52 // ----------------------------------------------------------------------------
53
54 using namespace android;
55
56
57 enum {
58 ACQUIRE_SUCCESS = 0,
59 ACQUIRE_NO_BUFFERS = 1,
60 ACQUIRE_MAX_IMAGES = 2,
61 };
62
63 static struct {
64 jfieldID mNativeContext;
65 jmethodID postEventFromNative;
66 } gImageReaderClassInfo;
67
68 static struct {
69 jfieldID mNativeBuffer;
70 jfieldID mTimestamp;
71 jfieldID mTransform;
72 jfieldID mScalingMode;
73 jfieldID mPlanes;
74 } gSurfaceImageClassInfo;
75
76 static struct {
77 jclass clazz;
78 jmethodID ctor;
79 } gSurfacePlaneClassInfo;
80
81 // Get an ID that's unique within this process.
createProcessUniqueId()82 static int32_t createProcessUniqueId() {
83 static volatile int32_t globalCounter = 0;
84 return android_atomic_inc(&globalCounter);
85 }
86
87 // ----------------------------------------------------------------------------
88
89 class JNIImageReaderContext : public ConsumerBase::FrameAvailableListener
90 {
91 public:
92 JNIImageReaderContext(JNIEnv* env, jobject weakThiz, jclass clazz, int maxImages);
93
94 virtual ~JNIImageReaderContext();
95
96 virtual void onFrameAvailable(const BufferItem& item);
97
98 BufferItem* getBufferItem();
99 void returnBufferItem(BufferItem* buffer);
100
101
setBufferConsumer(const sp<BufferItemConsumer> & consumer)102 void setBufferConsumer(const sp<BufferItemConsumer>& consumer) { mConsumer = consumer; }
getBufferConsumer()103 BufferItemConsumer* getBufferConsumer() { return mConsumer.get(); }
104
setProducer(const sp<IGraphicBufferProducer> & producer)105 void setProducer(const sp<IGraphicBufferProducer>& producer) { mProducer = producer; }
getProducer()106 IGraphicBufferProducer* getProducer() { return mProducer.get(); }
107
setBufferFormat(int format)108 void setBufferFormat(int format) { mFormat = format; }
getBufferFormat()109 int getBufferFormat() { return mFormat; }
110
setBufferDataspace(android_dataspace dataSpace)111 void setBufferDataspace(android_dataspace dataSpace) { mDataSpace = dataSpace; }
getBufferDataspace()112 android_dataspace getBufferDataspace() { return mDataSpace; }
113
setBufferWidth(int width)114 void setBufferWidth(int width) { mWidth = width; }
getBufferWidth()115 int getBufferWidth() { return mWidth; }
116
setBufferHeight(int height)117 void setBufferHeight(int height) { mHeight = height; }
getBufferHeight()118 int getBufferHeight() { return mHeight; }
119
120 private:
121 static JNIEnv* getJNIEnv(bool* needsDetach);
122 static void detachJNI();
123
124 List<BufferItem*> mBuffers;
125 sp<BufferItemConsumer> mConsumer;
126 sp<IGraphicBufferProducer> mProducer;
127 jobject mWeakThiz;
128 jclass mClazz;
129 int mFormat;
130 android_dataspace mDataSpace;
131 int mWidth;
132 int mHeight;
133 };
134
JNIImageReaderContext(JNIEnv * env,jobject weakThiz,jclass clazz,int maxImages)135 JNIImageReaderContext::JNIImageReaderContext(JNIEnv* env,
136 jobject weakThiz, jclass clazz, int maxImages) :
137 mWeakThiz(env->NewGlobalRef(weakThiz)),
138 mClazz((jclass)env->NewGlobalRef(clazz)),
139 mFormat(0),
140 mDataSpace(HAL_DATASPACE_UNKNOWN),
141 mWidth(-1),
142 mHeight(-1) {
143 for (int i = 0; i < maxImages; i++) {
144 BufferItem* buffer = new BufferItem;
145 mBuffers.push_back(buffer);
146 }
147 }
148
getJNIEnv(bool * needsDetach)149 JNIEnv* JNIImageReaderContext::getJNIEnv(bool* needsDetach) {
150 LOG_ALWAYS_FATAL_IF(needsDetach == NULL, "needsDetach is null!!!");
151 *needsDetach = false;
152 JNIEnv* env = AndroidRuntime::getJNIEnv();
153 if (env == NULL) {
154 JavaVMAttachArgs args = {JNI_VERSION_1_4, NULL, NULL};
155 JavaVM* vm = AndroidRuntime::getJavaVM();
156 int result = vm->AttachCurrentThread(&env, (void*) &args);
157 if (result != JNI_OK) {
158 ALOGE("thread attach failed: %#x", result);
159 return NULL;
160 }
161 *needsDetach = true;
162 }
163 return env;
164 }
165
detachJNI()166 void JNIImageReaderContext::detachJNI() {
167 JavaVM* vm = AndroidRuntime::getJavaVM();
168 int result = vm->DetachCurrentThread();
169 if (result != JNI_OK) {
170 ALOGE("thread detach failed: %#x", result);
171 }
172 }
173
getBufferItem()174 BufferItem* JNIImageReaderContext::getBufferItem() {
175 if (mBuffers.empty()) {
176 return NULL;
177 }
178 // Return a BufferItem pointer and remove it from the list
179 List<BufferItem*>::iterator it = mBuffers.begin();
180 BufferItem* buffer = *it;
181 mBuffers.erase(it);
182 return buffer;
183 }
184
returnBufferItem(BufferItem * buffer)185 void JNIImageReaderContext::returnBufferItem(BufferItem* buffer) {
186 buffer->mGraphicBuffer = nullptr;
187 mBuffers.push_back(buffer);
188 }
189
~JNIImageReaderContext()190 JNIImageReaderContext::~JNIImageReaderContext() {
191 bool needsDetach = false;
192 JNIEnv* env = getJNIEnv(&needsDetach);
193 if (env != NULL) {
194 env->DeleteGlobalRef(mWeakThiz);
195 env->DeleteGlobalRef(mClazz);
196 } else {
197 ALOGW("leaking JNI object references");
198 }
199 if (needsDetach) {
200 detachJNI();
201 }
202
203 // Delete buffer items.
204 for (List<BufferItem *>::iterator it = mBuffers.begin();
205 it != mBuffers.end(); it++) {
206 delete *it;
207 }
208
209 if (mConsumer != 0) {
210 mConsumer.clear();
211 }
212 }
213
onFrameAvailable(const BufferItem &)214 void JNIImageReaderContext::onFrameAvailable(const BufferItem& /*item*/)
215 {
216 ALOGV("%s: frame available", __FUNCTION__);
217 bool needsDetach = false;
218 JNIEnv* env = getJNIEnv(&needsDetach);
219 if (env != NULL) {
220 env->CallStaticVoidMethod(mClazz, gImageReaderClassInfo.postEventFromNative, mWeakThiz);
221 } else {
222 ALOGW("onFrameAvailable event will not posted");
223 }
224 if (needsDetach) {
225 detachJNI();
226 }
227 }
228
229 // ----------------------------------------------------------------------------
230
231 extern "C" {
232
ImageReader_getContext(JNIEnv * env,jobject thiz)233 static JNIImageReaderContext* ImageReader_getContext(JNIEnv* env, jobject thiz)
234 {
235 JNIImageReaderContext *ctx;
236 ctx = reinterpret_cast<JNIImageReaderContext *>
237 (env->GetLongField(thiz, gImageReaderClassInfo.mNativeContext));
238 return ctx;
239 }
240
ImageReader_getProducer(JNIEnv * env,jobject thiz)241 static IGraphicBufferProducer* ImageReader_getProducer(JNIEnv* env, jobject thiz)
242 {
243 ALOGV("%s:", __FUNCTION__);
244 JNIImageReaderContext* const ctx = ImageReader_getContext(env, thiz);
245 if (ctx == NULL) {
246 jniThrowRuntimeException(env, "ImageReaderContext is not initialized");
247 return NULL;
248 }
249
250 return ctx->getProducer();
251 }
252
ImageReader_setNativeContext(JNIEnv * env,jobject thiz,sp<JNIImageReaderContext> ctx)253 static void ImageReader_setNativeContext(JNIEnv* env,
254 jobject thiz, sp<JNIImageReaderContext> ctx)
255 {
256 ALOGV("%s:", __FUNCTION__);
257 JNIImageReaderContext* const p = ImageReader_getContext(env, thiz);
258 if (ctx != 0) {
259 ctx->incStrong((void*)ImageReader_setNativeContext);
260 }
261 if (p) {
262 p->decStrong((void*)ImageReader_setNativeContext);
263 }
264 env->SetLongField(thiz, gImageReaderClassInfo.mNativeContext,
265 reinterpret_cast<jlong>(ctx.get()));
266 }
267
ImageReader_getBufferConsumer(JNIEnv * env,jobject thiz)268 static BufferItemConsumer* ImageReader_getBufferConsumer(JNIEnv* env, jobject thiz)
269 {
270 ALOGV("%s:", __FUNCTION__);
271 JNIImageReaderContext* const ctx = ImageReader_getContext(env, thiz);
272 if (ctx == NULL) {
273 jniThrowRuntimeException(env, "ImageReaderContext is not initialized");
274 return NULL;
275 }
276
277 return ctx->getBufferConsumer();
278 }
279
Image_setBufferItem(JNIEnv * env,jobject thiz,const BufferItem * buffer)280 static void Image_setBufferItem(JNIEnv* env, jobject thiz,
281 const BufferItem* buffer)
282 {
283 env->SetLongField(thiz, gSurfaceImageClassInfo.mNativeBuffer, reinterpret_cast<jlong>(buffer));
284 }
285
Image_getBufferItem(JNIEnv * env,jobject image)286 static BufferItem* Image_getBufferItem(JNIEnv* env, jobject image)
287 {
288 return reinterpret_cast<BufferItem*>(
289 env->GetLongField(image, gSurfaceImageClassInfo.mNativeBuffer));
290 }
291
292
293 // ----------------------------------------------------------------------------
294
ImageReader_classInit(JNIEnv * env,jclass clazz)295 static void ImageReader_classInit(JNIEnv* env, jclass clazz)
296 {
297 ALOGV("%s:", __FUNCTION__);
298
299 jclass imageClazz = env->FindClass("android/media/ImageReader$SurfaceImage");
300 LOG_ALWAYS_FATAL_IF(imageClazz == NULL,
301 "can't find android/graphics/ImageReader$SurfaceImage");
302 gSurfaceImageClassInfo.mNativeBuffer = env->GetFieldID(
303 imageClazz, ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID, "J");
304 LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mNativeBuffer == NULL,
305 "can't find android/graphics/ImageReader.%s",
306 ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID);
307
308 gSurfaceImageClassInfo.mTimestamp = env->GetFieldID(
309 imageClazz, ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID, "J");
310 LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mTimestamp == NULL,
311 "can't find android/graphics/ImageReader.%s",
312 ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID);
313
314 gSurfaceImageClassInfo.mTransform = env->GetFieldID(
315 imageClazz, ANDROID_MEDIA_SURFACEIMAGE_TF_JNI_ID, "I");
316 LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mTransform == NULL,
317 "can't find android/graphics/ImageReader.%s",
318 ANDROID_MEDIA_SURFACEIMAGE_TF_JNI_ID);
319
320 gSurfaceImageClassInfo.mScalingMode = env->GetFieldID(
321 imageClazz, ANDROID_MEDIA_SURFACEIMAGE_SM_JNI_ID, "I");
322 LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mScalingMode == NULL,
323 "can't find android/graphics/ImageReader.%s",
324 ANDROID_MEDIA_SURFACEIMAGE_SM_JNI_ID);
325
326 gSurfaceImageClassInfo.mPlanes = env->GetFieldID(
327 imageClazz, "mPlanes", "[Landroid/media/ImageReader$SurfaceImage$SurfacePlane;");
328 LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mPlanes == NULL,
329 "can't find android/media/ImageReader$ReaderSurfaceImage.mPlanes");
330
331 gImageReaderClassInfo.mNativeContext = env->GetFieldID(
332 clazz, ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID, "J");
333 LOG_ALWAYS_FATAL_IF(gImageReaderClassInfo.mNativeContext == NULL,
334 "can't find android/graphics/ImageReader.%s",
335 ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID);
336
337 gImageReaderClassInfo.postEventFromNative = env->GetStaticMethodID(
338 clazz, "postEventFromNative", "(Ljava/lang/Object;)V");
339 LOG_ALWAYS_FATAL_IF(gImageReaderClassInfo.postEventFromNative == NULL,
340 "can't find android/graphics/ImageReader.postEventFromNative");
341
342 jclass planeClazz = env->FindClass("android/media/ImageReader$SurfaceImage$SurfacePlane");
343 LOG_ALWAYS_FATAL_IF(planeClazz == NULL, "Can not find SurfacePlane class");
344 // FindClass only gives a local reference of jclass object.
345 gSurfacePlaneClassInfo.clazz = (jclass) env->NewGlobalRef(planeClazz);
346 gSurfacePlaneClassInfo.ctor = env->GetMethodID(gSurfacePlaneClassInfo.clazz, "<init>",
347 "(Landroid/media/ImageReader$SurfaceImage;IILjava/nio/ByteBuffer;)V");
348 LOG_ALWAYS_FATAL_IF(gSurfacePlaneClassInfo.ctor == NULL,
349 "Can not find SurfacePlane constructor");
350 }
351
ImageReader_init(JNIEnv * env,jobject thiz,jobject weakThiz,jint width,jint height,jint format,jint maxImages,jlong ndkUsage)352 static void ImageReader_init(JNIEnv* env, jobject thiz, jobject weakThiz, jint width, jint height,
353 jint format, jint maxImages, jlong ndkUsage)
354 {
355 status_t res;
356 int nativeFormat;
357 android_dataspace nativeDataspace;
358
359 ALOGV("%s: width:%d, height: %d, format: 0x%x, maxImages:%d",
360 __FUNCTION__, width, height, format, maxImages);
361
362 PublicFormat publicFormat = static_cast<PublicFormat>(format);
363 nativeFormat = android_view_Surface_mapPublicFormatToHalFormat(
364 publicFormat);
365 nativeDataspace = android_view_Surface_mapPublicFormatToHalDataspace(
366 publicFormat);
367
368 jclass clazz = env->GetObjectClass(thiz);
369 if (clazz == NULL) {
370 jniThrowRuntimeException(env, "Can't find android/graphics/ImageReader");
371 return;
372 }
373 sp<JNIImageReaderContext> ctx(new JNIImageReaderContext(env, weakThiz, clazz, maxImages));
374
375 sp<IGraphicBufferProducer> gbProducer;
376 sp<IGraphicBufferConsumer> gbConsumer;
377 BufferQueue::createBufferQueue(&gbProducer, &gbConsumer);
378 sp<BufferItemConsumer> bufferConsumer;
379 String8 consumerName = String8::format("ImageReader-%dx%df%xm%d-%d-%d",
380 width, height, format, maxImages, getpid(),
381 createProcessUniqueId());
382 uint32_t consumerUsage = GRALLOC_USAGE_SW_READ_OFTEN;
383 bool needUsageOverride = ndkUsage != CONSUMER_BUFFER_USAGE_UNKNOWN;
384 uint64_t outProducerUsage = 0;
385 uint64_t outConsumerUsage = 0;
386 android_hardware_HardwareBuffer_convertToGrallocUsageBits(&outProducerUsage, &outConsumerUsage,
387 ndkUsage, 0);
388
389 if (isFormatOpaque(nativeFormat)) {
390 // Use the SW_READ_NEVER usage to tell producer that this format is not for preview or video
391 // encoding. The only possibility will be ZSL output.
392 consumerUsage = GRALLOC_USAGE_SW_READ_NEVER;
393 if (needUsageOverride) {
394 consumerUsage = android_convertGralloc1To0Usage(0, outConsumerUsage);
395 }
396 } else if (needUsageOverride) {
397 ALOGW("Consumer usage override for non-opaque format is not implemented yet, "
398 "ignore the provided usage from the application");
399 }
400 bufferConsumer = new BufferItemConsumer(gbConsumer, consumerUsage, maxImages,
401 /*controlledByApp*/true);
402 if (bufferConsumer == nullptr) {
403 jniThrowExceptionFmt(env, "java/lang/RuntimeException",
404 "Failed to allocate native buffer consumer for format 0x%x and usage 0x%x",
405 nativeFormat, consumerUsage);
406 return;
407 }
408 ctx->setBufferConsumer(bufferConsumer);
409 bufferConsumer->setName(consumerName);
410
411 ctx->setProducer(gbProducer);
412 bufferConsumer->setFrameAvailableListener(ctx);
413 ImageReader_setNativeContext(env, thiz, ctx);
414 ctx->setBufferFormat(nativeFormat);
415 ctx->setBufferDataspace(nativeDataspace);
416 ctx->setBufferWidth(width);
417 ctx->setBufferHeight(height);
418
419 // Set the width/height/format/dataspace to the bufferConsumer.
420 res = bufferConsumer->setDefaultBufferSize(width, height);
421 if (res != OK) {
422 jniThrowExceptionFmt(env, "java/lang/IllegalStateException",
423 "Failed to set buffer consumer default size (%dx%d) for format 0x%x",
424 width, height, nativeFormat);
425 return;
426 }
427 res = bufferConsumer->setDefaultBufferFormat(nativeFormat);
428 if (res != OK) {
429 jniThrowExceptionFmt(env, "java/lang/IllegalStateException",
430 "Failed to set buffer consumer default format 0x%x", nativeFormat);
431 }
432 res = bufferConsumer->setDefaultBufferDataSpace(nativeDataspace);
433 if (res != OK) {
434 jniThrowExceptionFmt(env, "java/lang/IllegalStateException",
435 "Failed to set buffer consumer default dataSpace 0x%x", nativeDataspace);
436 }
437 }
438
ImageReader_close(JNIEnv * env,jobject thiz)439 static void ImageReader_close(JNIEnv* env, jobject thiz)
440 {
441 ALOGV("%s:", __FUNCTION__);
442
443 JNIImageReaderContext* const ctx = ImageReader_getContext(env, thiz);
444 if (ctx == NULL) {
445 // ImageReader is already closed.
446 return;
447 }
448
449 BufferItemConsumer* consumer = NULL;
450 consumer = ImageReader_getBufferConsumer(env, thiz);
451
452 if (consumer != NULL) {
453 consumer->abandon();
454 consumer->setFrameAvailableListener(NULL);
455 }
456 ImageReader_setNativeContext(env, thiz, NULL);
457 }
458
Image_unlockIfLocked(JNIEnv * env,jobject image)459 static sp<Fence> Image_unlockIfLocked(JNIEnv* env, jobject image) {
460 ALOGV("%s", __FUNCTION__);
461 BufferItem* buffer = Image_getBufferItem(env, image);
462 if (buffer == NULL) {
463 jniThrowException(env, "java/lang/IllegalStateException",
464 "Image is not initialized");
465 return Fence::NO_FENCE;
466 }
467
468 // Is locked?
469 bool wasBufferLocked = false;
470 jobject planes = NULL;
471 if (!isFormatOpaque(buffer->mGraphicBuffer->getPixelFormat())) {
472 planes = env->GetObjectField(image, gSurfaceImageClassInfo.mPlanes);
473 }
474 wasBufferLocked = (planes != NULL);
475 if (wasBufferLocked) {
476 status_t res = OK;
477 int fenceFd = -1;
478 if (wasBufferLocked) {
479 res = buffer->mGraphicBuffer->unlockAsync(&fenceFd);
480 if (res != OK) {
481 jniThrowRuntimeException(env, "unlock buffer failed");
482 return Fence::NO_FENCE;
483 }
484 }
485 sp<Fence> releaseFence = new Fence(fenceFd);
486 return releaseFence;
487 ALOGV("Successfully unlocked the image");
488 }
489 return Fence::NO_FENCE;
490 }
491
ImageReader_imageRelease(JNIEnv * env,jobject thiz,jobject image)492 static void ImageReader_imageRelease(JNIEnv* env, jobject thiz, jobject image)
493 {
494 ALOGV("%s:", __FUNCTION__);
495 JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz);
496 if (ctx == NULL) {
497 ALOGW("ImageReader#close called before Image#close, consider calling Image#close first");
498 return;
499 }
500
501 BufferItemConsumer* bufferConsumer = ctx->getBufferConsumer();
502 BufferItem* buffer = Image_getBufferItem(env, image);
503 if (buffer == nullptr) {
504 // Release an already closed image is harmless.
505 return;
506 }
507
508 sp<Fence> releaseFence = Image_unlockIfLocked(env, image);
509 bufferConsumer->releaseBuffer(*buffer, releaseFence);
510 Image_setBufferItem(env, image, NULL);
511 ctx->returnBufferItem(buffer);
512 ALOGV("%s: Image (format: 0x%x) has been released", __FUNCTION__, ctx->getBufferFormat());
513 }
514
ImageReader_imageSetup(JNIEnv * env,jobject thiz,jobject image)515 static jint ImageReader_imageSetup(JNIEnv* env, jobject thiz, jobject image) {
516 ALOGV("%s:", __FUNCTION__);
517 JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz);
518 if (ctx == NULL) {
519 jniThrowException(env, "java/lang/IllegalStateException",
520 "ImageReader is not initialized or was already closed");
521 return -1;
522 }
523
524 BufferItemConsumer* bufferConsumer = ctx->getBufferConsumer();
525 BufferItem* buffer = ctx->getBufferItem();
526 if (buffer == NULL) {
527 ALOGW("Unable to acquire a buffer item, very likely client tried to acquire more than"
528 " maxImages buffers");
529 return ACQUIRE_MAX_IMAGES;
530 }
531
532 status_t res = bufferConsumer->acquireBuffer(buffer, 0);
533 if (res != OK) {
534 ctx->returnBufferItem(buffer);
535 if (res != BufferQueue::NO_BUFFER_AVAILABLE) {
536 if (res == INVALID_OPERATION) {
537 // Max number of images were already acquired.
538 ALOGE("%s: Max number of buffers allowed are already acquired : %s (%d)",
539 __FUNCTION__, strerror(-res), res);
540 return ACQUIRE_MAX_IMAGES;
541 } else {
542 ALOGE("%s: Acquire image failed with some unknown error: %s (%d)",
543 __FUNCTION__, strerror(-res), res);
544 jniThrowExceptionFmt(env, "java/lang/IllegalStateException",
545 "Unknown error (%d) when we tried to acquire an image.",
546 res);
547 return ACQUIRE_NO_BUFFERS;
548 }
549 }
550 // This isn't really an error case, as the application may acquire buffer at any time.
551 return ACQUIRE_NO_BUFFERS;
552 }
553
554 // Add some extra checks for non-opaque formats.
555 if (!isFormatOpaque(ctx->getBufferFormat())) {
556 // Check if the left-top corner of the crop rect is origin, we currently assume this point is
557 // zero, will revisit this once this assumption turns out problematic.
558 Point lt = buffer->mCrop.leftTop();
559 if (lt.x != 0 || lt.y != 0) {
560 jniThrowExceptionFmt(env, "java/lang/UnsupportedOperationException",
561 "crop left top corner [%d, %d] need to be at origin", lt.x, lt.y);
562 return -1;
563 }
564
565 // Check if the producer buffer configurations match what ImageReader configured.
566 int outputWidth = getBufferWidth(buffer);
567 int outputHeight = getBufferHeight(buffer);
568
569 int imgReaderFmt = ctx->getBufferFormat();
570 int imageReaderWidth = ctx->getBufferWidth();
571 int imageReaderHeight = ctx->getBufferHeight();
572 int bufferFormat = buffer->mGraphicBuffer->getPixelFormat();
573 if ((bufferFormat != HAL_PIXEL_FORMAT_BLOB) && (imgReaderFmt != HAL_PIXEL_FORMAT_BLOB) &&
574 (imageReaderWidth != outputWidth || imageReaderHeight != outputHeight)) {
575 ALOGV("%s: Producer buffer size: %dx%d, doesn't match ImageReader configured size: %dx%d",
576 __FUNCTION__, outputWidth, outputHeight, imageReaderWidth, imageReaderHeight);
577 }
578 if (imgReaderFmt != bufferFormat) {
579 if (imgReaderFmt == HAL_PIXEL_FORMAT_YCbCr_420_888 &&
580 isPossiblyYUV(bufferFormat)) {
581 // Treat formats that are compatible with flexible YUV
582 // (HAL_PIXEL_FORMAT_YCbCr_420_888) as HAL_PIXEL_FORMAT_YCbCr_420_888.
583 ALOGV("%s: Treat buffer format to 0x%x as HAL_PIXEL_FORMAT_YCbCr_420_888",
584 __FUNCTION__, bufferFormat);
585 } else if (imgReaderFmt == HAL_PIXEL_FORMAT_BLOB &&
586 bufferFormat == HAL_PIXEL_FORMAT_RGBA_8888) {
587 // Using HAL_PIXEL_FORMAT_RGBA_8888 Gralloc buffers containing JPEGs to get around
588 // SW write limitations for (b/17379185).
589 ALOGV("%s: Receiving JPEG in HAL_PIXEL_FORMAT_RGBA_8888 buffer.", __FUNCTION__);
590 } else {
591 // Return the buffer to the queue. No need to provide fence, as this buffer wasn't
592 // used anywhere yet.
593 bufferConsumer->releaseBuffer(*buffer);
594 ctx->returnBufferItem(buffer);
595
596 // Throw exception
597 ALOGE("Producer output buffer format: 0x%x, ImageReader configured format: 0x%x",
598 bufferFormat, ctx->getBufferFormat());
599 String8 msg;
600 msg.appendFormat("The producer output buffer format 0x%x doesn't "
601 "match the ImageReader's configured buffer format 0x%x.",
602 bufferFormat, ctx->getBufferFormat());
603 jniThrowException(env, "java/lang/UnsupportedOperationException",
604 msg.string());
605 return -1;
606 }
607 }
608
609 }
610
611 // Set SurfaceImage instance member variables
612 Image_setBufferItem(env, image, buffer);
613 env->SetLongField(image, gSurfaceImageClassInfo.mTimestamp,
614 static_cast<jlong>(buffer->mTimestamp));
615 env->SetIntField(image, gSurfaceImageClassInfo.mTransform,
616 static_cast<jint>(buffer->mTransform));
617 env->SetIntField(image, gSurfaceImageClassInfo.mScalingMode,
618 static_cast<jint>(buffer->mScalingMode));
619
620 return ACQUIRE_SUCCESS;
621 }
622
ImageReader_detachImage(JNIEnv * env,jobject thiz,jobject image)623 static jint ImageReader_detachImage(JNIEnv* env, jobject thiz, jobject image) {
624 ALOGV("%s:", __FUNCTION__);
625 JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz);
626 if (ctx == NULL) {
627 jniThrowException(env, "java/lang/IllegalStateException", "ImageReader was already closed");
628 return -1;
629 }
630
631 BufferItemConsumer* bufferConsumer = ctx->getBufferConsumer();
632 BufferItem* buffer = Image_getBufferItem(env, image);
633 if (!buffer) {
634 ALOGE(
635 "Image already released and can not be detached from ImageReader!!!");
636 jniThrowException(env, "java/lang/IllegalStateException",
637 "Image detach from ImageReader failed: buffer was already released");
638 return -1;
639 }
640
641 status_t res = OK;
642 Image_unlockIfLocked(env, image);
643 res = bufferConsumer->detachBuffer(buffer->mSlot);
644 if (res != OK) {
645 ALOGE("Image detach failed: %s (%d)!!!", strerror(-res), res);
646 jniThrowRuntimeException(env,
647 "nativeDetachImage failed for image!!!");
648 return res;
649 }
650 return OK;
651 }
652
ImageReader_discardFreeBuffers(JNIEnv * env,jobject thiz)653 static void ImageReader_discardFreeBuffers(JNIEnv* env, jobject thiz) {
654 ALOGV("%s:", __FUNCTION__);
655 JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz);
656 if (ctx == NULL) {
657 jniThrowException(env, "java/lang/IllegalStateException", "ImageReader was already closed");
658 return;
659 }
660
661 BufferItemConsumer* bufferConsumer = ctx->getBufferConsumer();
662 status_t res = bufferConsumer->discardFreeBuffers();
663 if (res != OK) {
664 ALOGE("Buffer discard failed: %s (%d)", strerror(-res), res);
665 jniThrowRuntimeException(env,
666 "nativeDicardFreebuffers failed");
667 }
668 }
669
ImageReader_getSurface(JNIEnv * env,jobject thiz)670 static jobject ImageReader_getSurface(JNIEnv* env, jobject thiz)
671 {
672 ALOGV("%s: ", __FUNCTION__);
673
674 IGraphicBufferProducer* gbp = ImageReader_getProducer(env, thiz);
675 if (gbp == NULL) {
676 jniThrowRuntimeException(env, "Buffer consumer is uninitialized");
677 return NULL;
678 }
679
680 // Wrap the IGBP in a Java-language Surface.
681 return android_view_Surface_createFromIGraphicBufferProducer(env, gbp);
682 }
683
Image_getLockedImage(JNIEnv * env,jobject thiz,LockedImage * image)684 static void Image_getLockedImage(JNIEnv* env, jobject thiz, LockedImage *image) {
685 ALOGV("%s", __FUNCTION__);
686 BufferItem* buffer = Image_getBufferItem(env, thiz);
687 if (buffer == NULL) {
688 jniThrowException(env, "java/lang/IllegalStateException",
689 "Image is not initialized");
690 return;
691 }
692
693 status_t res = lockImageFromBuffer(buffer,
694 GRALLOC_USAGE_SW_READ_OFTEN, buffer->mFence->dup(), image);
695 if (res != OK) {
696 jniThrowExceptionFmt(env, "java/lang/RuntimeException",
697 "lock buffer failed for format 0x%x",
698 buffer->mGraphicBuffer->getPixelFormat());
699 return;
700 }
701
702 // Carry over some fields from BufferItem.
703 image->crop = buffer->mCrop;
704 image->transform = buffer->mTransform;
705 image->scalingMode = buffer->mScalingMode;
706 image->timestamp = buffer->mTimestamp;
707 image->dataSpace = buffer->mDataSpace;
708 image->frameNumber = buffer->mFrameNumber;
709
710 ALOGV("%s: Successfully locked the image", __FUNCTION__);
711 // crop, transform, scalingMode, timestamp, and frameNumber should be set by producer,
712 // and we don't set them here.
713 }
714
Image_getLockedImageInfo(JNIEnv * env,LockedImage * buffer,int idx,int32_t writerFormat,uint8_t ** base,uint32_t * size,int * pixelStride,int * rowStride)715 static void Image_getLockedImageInfo(JNIEnv* env, LockedImage* buffer, int idx,
716 int32_t writerFormat, uint8_t **base, uint32_t *size, int *pixelStride, int *rowStride) {
717 ALOGV("%s", __FUNCTION__);
718
719 status_t res = getLockedImageInfo(buffer, idx, writerFormat, base, size,
720 pixelStride, rowStride);
721 if (res != OK) {
722 jniThrowExceptionFmt(env, "java/lang/UnsupportedOperationException",
723 "Pixel format: 0x%x is unsupported", buffer->flexFormat);
724 }
725 }
726
Image_createSurfacePlanes(JNIEnv * env,jobject thiz,int numPlanes,int readerFormat)727 static jobjectArray Image_createSurfacePlanes(JNIEnv* env, jobject thiz,
728 int numPlanes, int readerFormat)
729 {
730 ALOGV("%s: create SurfacePlane array with size %d", __FUNCTION__, numPlanes);
731 int rowStride = 0;
732 int pixelStride = 0;
733 uint8_t *pData = NULL;
734 uint32_t dataSize = 0;
735 jobject byteBuffer = NULL;
736
737 PublicFormat publicReaderFormat = static_cast<PublicFormat>(readerFormat);
738 int halReaderFormat = android_view_Surface_mapPublicFormatToHalFormat(
739 publicReaderFormat);
740
741 if (isFormatOpaque(halReaderFormat) && numPlanes > 0) {
742 String8 msg;
743 msg.appendFormat("Format 0x%x is opaque, thus not writable, the number of planes (%d)"
744 " must be 0", halReaderFormat, numPlanes);
745 jniThrowException(env, "java/lang/IllegalArgumentException", msg.string());
746 return NULL;
747 }
748
749 jobjectArray surfacePlanes = env->NewObjectArray(numPlanes, gSurfacePlaneClassInfo.clazz,
750 /*initial_element*/NULL);
751 if (surfacePlanes == NULL) {
752 jniThrowRuntimeException(env, "Failed to create SurfacePlane arrays,"
753 " probably out of memory");
754 return NULL;
755 }
756 if (isFormatOpaque(halReaderFormat)) {
757 // Return 0 element surface array.
758 return surfacePlanes;
759 }
760
761 LockedImage lockedImg = LockedImage();
762 Image_getLockedImage(env, thiz, &lockedImg);
763 if (env->ExceptionCheck()) {
764 return NULL;
765 }
766 // Create all SurfacePlanes
767 for (int i = 0; i < numPlanes; i++) {
768 Image_getLockedImageInfo(env, &lockedImg, i, halReaderFormat,
769 &pData, &dataSize, &pixelStride, &rowStride);
770 byteBuffer = env->NewDirectByteBuffer(pData, dataSize);
771 if ((byteBuffer == NULL) && (env->ExceptionCheck() == false)) {
772 jniThrowException(env, "java/lang/IllegalStateException",
773 "Failed to allocate ByteBuffer");
774 return NULL;
775 }
776
777 // Finally, create this SurfacePlane.
778 jobject surfacePlane = env->NewObject(gSurfacePlaneClassInfo.clazz,
779 gSurfacePlaneClassInfo.ctor, thiz, rowStride, pixelStride, byteBuffer);
780 env->SetObjectArrayElement(surfacePlanes, i, surfacePlane);
781 }
782
783 return surfacePlanes;
784 }
785
Image_getWidth(JNIEnv * env,jobject thiz)786 static jint Image_getWidth(JNIEnv* env, jobject thiz)
787 {
788 BufferItem* buffer = Image_getBufferItem(env, thiz);
789 return getBufferWidth(buffer);
790 }
791
Image_getHeight(JNIEnv * env,jobject thiz)792 static jint Image_getHeight(JNIEnv* env, jobject thiz)
793 {
794 BufferItem* buffer = Image_getBufferItem(env, thiz);
795 return getBufferHeight(buffer);
796 }
797
Image_getFormat(JNIEnv * env,jobject thiz,jint readerFormat)798 static jint Image_getFormat(JNIEnv* env, jobject thiz, jint readerFormat)
799 {
800 if (isFormatOpaque(readerFormat)) {
801 // Assuming opaque reader produce opaque images.
802 return static_cast<jint>(PublicFormat::PRIVATE);
803 } else {
804 BufferItem* buffer = Image_getBufferItem(env, thiz);
805 int readerHalFormat = android_view_Surface_mapPublicFormatToHalFormat(
806 static_cast<PublicFormat>(readerFormat));
807 int32_t fmt = applyFormatOverrides(
808 buffer->mGraphicBuffer->getPixelFormat(), readerHalFormat);
809 // Override the image format to HAL_PIXEL_FORMAT_YCbCr_420_888 if the actual format is
810 // NV21 or YV12. This could only happen when the Gralloc HAL version is v0.1 thus doesn't
811 // support lockycbcr(), the CpuConsumer need to use the lock() method in the
812 // lockNextBuffer() call. For Gralloc HAL v0.2 or newer, this format should already be
813 // overridden to HAL_PIXEL_FORMAT_YCbCr_420_888 for the flexible YUV compatible formats.
814 if (isPossiblyYUV(fmt)) {
815 fmt = HAL_PIXEL_FORMAT_YCbCr_420_888;
816 }
817 PublicFormat publicFmt = android_view_Surface_mapHalFormatDataspaceToPublicFormat(
818 fmt, buffer->mDataSpace);
819 return static_cast<jint>(publicFmt);
820 }
821 }
822
Image_getHardwareBuffer(JNIEnv * env,jobject thiz)823 static jobject Image_getHardwareBuffer(JNIEnv* env, jobject thiz) {
824 BufferItem* buffer = Image_getBufferItem(env, thiz);
825 AHardwareBuffer* b = AHardwareBuffer_from_GraphicBuffer(buffer->mGraphicBuffer.get());
826 // don't user the public AHardwareBuffer_toHardwareBuffer() because this would force us
827 // to link against libandroid.so
828 return android_hardware_HardwareBuffer_createFromAHardwareBuffer(env, b);
829 }
830
831 } // extern "C"
832
833 // ----------------------------------------------------------------------------
834
835 static const JNINativeMethod gImageReaderMethods[] = {
836 {"nativeClassInit", "()V", (void*)ImageReader_classInit },
837 {"nativeInit", "(Ljava/lang/Object;IIIIJ)V", (void*)ImageReader_init },
838 {"nativeClose", "()V", (void*)ImageReader_close },
839 {"nativeReleaseImage", "(Landroid/media/Image;)V", (void*)ImageReader_imageRelease },
840 {"nativeImageSetup", "(Landroid/media/Image;)I", (void*)ImageReader_imageSetup },
841 {"nativeGetSurface", "()Landroid/view/Surface;", (void*)ImageReader_getSurface },
842 {"nativeDetachImage", "(Landroid/media/Image;)I", (void*)ImageReader_detachImage },
843 {"nativeDiscardFreeBuffers", "()V", (void*)ImageReader_discardFreeBuffers }
844 };
845
846 static const JNINativeMethod gImageMethods[] = {
847 {"nativeCreatePlanes", "(II)[Landroid/media/ImageReader$SurfaceImage$SurfacePlane;",
848 (void*)Image_createSurfacePlanes },
849 {"nativeGetWidth", "()I", (void*)Image_getWidth },
850 {"nativeGetHeight", "()I", (void*)Image_getHeight },
851 {"nativeGetFormat", "(I)I", (void*)Image_getFormat },
852 {"nativeGetHardwareBuffer", "()Landroid/hardware/HardwareBuffer;",
853 (void*)Image_getHardwareBuffer },
854 };
855
register_android_media_ImageReader(JNIEnv * env)856 int register_android_media_ImageReader(JNIEnv *env) {
857
858 int ret1 = AndroidRuntime::registerNativeMethods(env,
859 "android/media/ImageReader", gImageReaderMethods, NELEM(gImageReaderMethods));
860
861 int ret2 = AndroidRuntime::registerNativeMethods(env,
862 "android/media/ImageReader$SurfaceImage", gImageMethods, NELEM(gImageMethods));
863
864 return (ret1 || ret2);
865 }
866