1 /*
2 * Copyright 2013 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 //#define LOG_NDEBUG 0
18 #define LOG_TAG "ImageReader_JNI"
19 #include "android_media_Utils.h"
20 #include <cutils/atomic.h>
21 #include <utils/Log.h>
22 #include <utils/misc.h>
23 #include <utils/List.h>
24 #include <utils/String8.h>
25
26 #include <cstdio>
27
28 #include <gui/BufferItemConsumer.h>
29 #include <gui/Surface.h>
30
31 #include <android_runtime/AndroidRuntime.h>
32 #include <android_runtime/android_view_Surface.h>
33 #include <android_runtime/android_graphics_GraphicBuffer.h>
34 #include <android_runtime/android_hardware_HardwareBuffer.h>
35 #include <grallocusage/GrallocUsageConversion.h>
36
37 #include <private/android/AHardwareBufferHelpers.h>
38
39 #include <jni.h>
40 #include <nativehelper/JNIHelp.h>
41
42 #include <stdint.h>
43 #include <inttypes.h>
44 #include <android/hardware_buffer_jni.h>
45
46 #include <ui/Rect.h>
47
48 #define ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID "mNativeContext"
49 #define ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID "mNativeBuffer"
50 #define ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID "mTimestamp"
51 #define ANDROID_MEDIA_SURFACEIMAGE_DS_JNI_ID "mDataSpace"
52 #define ANDROID_MEDIA_SURFACEIMAGE_TF_JNI_ID "mTransform"
53 #define ANDROID_MEDIA_SURFACEIMAGE_SM_JNI_ID "mScalingMode"
54
55 #define CONSUMER_BUFFER_USAGE_UNKNOWN 0;
56 // ----------------------------------------------------------------------------
57
58 using namespace android;
59
60
61 enum {
62 ACQUIRE_SUCCESS = 0,
63 ACQUIRE_NO_BUFFERS = 1,
64 ACQUIRE_MAX_IMAGES = 2,
65 };
66
67 static struct {
68 jfieldID mNativeContext;
69 jmethodID postEventFromNative;
70 } gImageReaderClassInfo;
71
72 static struct {
73 jfieldID mNativeBuffer;
74 jfieldID mTimestamp;
75 jfieldID mDataSpace;
76 jfieldID mTransform;
77 jfieldID mScalingMode;
78 jfieldID mPlanes;
79 } gSurfaceImageClassInfo;
80
81 static struct {
82 jclass clazz;
83 jmethodID ctor;
84 } gSurfacePlaneClassInfo;
85
86 static struct {
87 jclass clazz;
88 jmethodID ctor;
89 } gImagePlaneClassInfo;
90
91 // Get an ID that's unique within this process.
createProcessUniqueId()92 static int32_t createProcessUniqueId() {
93 static volatile int32_t globalCounter = 0;
94 return android_atomic_inc(&globalCounter);
95 }
96
97 // ----------------------------------------------------------------------------
98
99 class JNIImageReaderContext : public ConsumerBase::FrameAvailableListener
100 {
101 public:
102 JNIImageReaderContext(JNIEnv* env, jobject weakThiz, jclass clazz, int maxImages);
103
104 virtual ~JNIImageReaderContext();
105
106 virtual void onFrameAvailable(const BufferItem& item);
107
108 BufferItem* getBufferItem();
109 void returnBufferItem(BufferItem* buffer);
110
111
setBufferConsumer(const sp<BufferItemConsumer> & consumer)112 void setBufferConsumer(const sp<BufferItemConsumer>& consumer) { mConsumer = consumer; }
getBufferConsumer()113 BufferItemConsumer* getBufferConsumer() { return mConsumer.get(); }
114
setProducer(const sp<IGraphicBufferProducer> & producer)115 void setProducer(const sp<IGraphicBufferProducer>& producer) { mProducer = producer; }
getProducer()116 IGraphicBufferProducer* getProducer() { return mProducer.get(); }
117
setBufferFormat(int format)118 void setBufferFormat(int format) { mFormat = format; }
getBufferFormat()119 int getBufferFormat() { return mFormat; }
120
setBufferDataspace(android_dataspace dataSpace)121 void setBufferDataspace(android_dataspace dataSpace) { mDataSpace = dataSpace; }
getBufferDataspace()122 android_dataspace getBufferDataspace() { return mDataSpace; }
123
setBufferWidth(int width)124 void setBufferWidth(int width) { mWidth = width; }
getBufferWidth()125 int getBufferWidth() { return mWidth; }
126
setBufferHeight(int height)127 void setBufferHeight(int height) { mHeight = height; }
getBufferHeight()128 int getBufferHeight() { return mHeight; }
129
130 private:
131 static JNIEnv* getJNIEnv(bool* needsDetach);
132 static void detachJNI();
133
134 List<BufferItem*> mBuffers;
135 sp<BufferItemConsumer> mConsumer;
136 sp<IGraphicBufferProducer> mProducer;
137 jobject mWeakThiz;
138 jclass mClazz;
139 int mFormat;
140 android_dataspace mDataSpace;
141 int mWidth;
142 int mHeight;
143 };
144
JNIImageReaderContext(JNIEnv * env,jobject weakThiz,jclass clazz,int maxImages)145 JNIImageReaderContext::JNIImageReaderContext(JNIEnv* env,
146 jobject weakThiz, jclass clazz, int maxImages) :
147 mWeakThiz(env->NewGlobalRef(weakThiz)),
148 mClazz((jclass)env->NewGlobalRef(clazz)),
149 mFormat(0),
150 mDataSpace(HAL_DATASPACE_UNKNOWN),
151 mWidth(-1),
152 mHeight(-1) {
153 for (int i = 0; i < maxImages; i++) {
154 BufferItem* buffer = new BufferItem;
155 mBuffers.push_back(buffer);
156 }
157 }
158
getJNIEnv(bool * needsDetach)159 JNIEnv* JNIImageReaderContext::getJNIEnv(bool* needsDetach) {
160 LOG_ALWAYS_FATAL_IF(needsDetach == NULL, "needsDetach is null!!!");
161 *needsDetach = false;
162 JNIEnv* env = AndroidRuntime::getJNIEnv();
163 if (env == NULL) {
164 JavaVMAttachArgs args = {JNI_VERSION_1_4, NULL, NULL};
165 JavaVM* vm = AndroidRuntime::getJavaVM();
166 int result = vm->AttachCurrentThread(&env, (void*) &args);
167 if (result != JNI_OK) {
168 ALOGE("thread attach failed: %#x", result);
169 return NULL;
170 }
171 *needsDetach = true;
172 }
173 return env;
174 }
175
detachJNI()176 void JNIImageReaderContext::detachJNI() {
177 JavaVM* vm = AndroidRuntime::getJavaVM();
178 int result = vm->DetachCurrentThread();
179 if (result != JNI_OK) {
180 ALOGE("thread detach failed: %#x", result);
181 }
182 }
183
getBufferItem()184 BufferItem* JNIImageReaderContext::getBufferItem() {
185 if (mBuffers.empty()) {
186 return NULL;
187 }
188 // Return a BufferItem pointer and remove it from the list
189 List<BufferItem*>::iterator it = mBuffers.begin();
190 BufferItem* buffer = *it;
191 mBuffers.erase(it);
192 return buffer;
193 }
194
returnBufferItem(BufferItem * buffer)195 void JNIImageReaderContext::returnBufferItem(BufferItem* buffer) {
196 buffer->mGraphicBuffer = nullptr;
197 mBuffers.push_back(buffer);
198 }
199
~JNIImageReaderContext()200 JNIImageReaderContext::~JNIImageReaderContext() {
201 bool needsDetach = false;
202 JNIEnv* env = getJNIEnv(&needsDetach);
203 if (env != NULL) {
204 env->DeleteGlobalRef(mWeakThiz);
205 env->DeleteGlobalRef(mClazz);
206 } else {
207 ALOGW("leaking JNI object references");
208 }
209 if (needsDetach) {
210 detachJNI();
211 }
212
213 // Delete buffer items.
214 for (List<BufferItem *>::iterator it = mBuffers.begin();
215 it != mBuffers.end(); it++) {
216 delete *it;
217 }
218
219 if (mConsumer != 0) {
220 mConsumer.clear();
221 }
222 }
223
onFrameAvailable(const BufferItem &)224 void JNIImageReaderContext::onFrameAvailable(const BufferItem& /*item*/)
225 {
226 ALOGV("%s: frame available", __FUNCTION__);
227 bool needsDetach = false;
228 JNIEnv* env = getJNIEnv(&needsDetach);
229 if (env != NULL) {
230 env->CallStaticVoidMethod(mClazz, gImageReaderClassInfo.postEventFromNative, mWeakThiz);
231 } else {
232 ALOGW("onFrameAvailable event will not posted");
233 }
234 if (needsDetach) {
235 detachJNI();
236 }
237 }
238
239 // ----------------------------------------------------------------------------
240
241 extern "C" {
242
ImageReader_getContext(JNIEnv * env,jobject thiz)243 static JNIImageReaderContext* ImageReader_getContext(JNIEnv* env, jobject thiz)
244 {
245 JNIImageReaderContext *ctx;
246 ctx = reinterpret_cast<JNIImageReaderContext *>
247 (env->GetLongField(thiz, gImageReaderClassInfo.mNativeContext));
248 return ctx;
249 }
250
ImageReader_getProducer(JNIEnv * env,jobject thiz)251 static IGraphicBufferProducer* ImageReader_getProducer(JNIEnv* env, jobject thiz)
252 {
253 ALOGV("%s:", __FUNCTION__);
254 JNIImageReaderContext* const ctx = ImageReader_getContext(env, thiz);
255 if (ctx == NULL) {
256 jniThrowRuntimeException(env, "ImageReaderContext is not initialized");
257 return NULL;
258 }
259
260 return ctx->getProducer();
261 }
262
ImageReader_setNativeContext(JNIEnv * env,jobject thiz,sp<JNIImageReaderContext> ctx)263 static void ImageReader_setNativeContext(JNIEnv* env,
264 jobject thiz, sp<JNIImageReaderContext> ctx)
265 {
266 ALOGV("%s:", __FUNCTION__);
267 JNIImageReaderContext* const p = ImageReader_getContext(env, thiz);
268 if (ctx != 0) {
269 ctx->incStrong((void*)ImageReader_setNativeContext);
270 }
271 if (p) {
272 p->decStrong((void*)ImageReader_setNativeContext);
273 }
274 env->SetLongField(thiz, gImageReaderClassInfo.mNativeContext,
275 reinterpret_cast<jlong>(ctx.get()));
276 }
277
ImageReader_getBufferConsumer(JNIEnv * env,jobject thiz)278 static BufferItemConsumer* ImageReader_getBufferConsumer(JNIEnv* env, jobject thiz)
279 {
280 ALOGV("%s:", __FUNCTION__);
281 JNIImageReaderContext* const ctx = ImageReader_getContext(env, thiz);
282 if (ctx == NULL) {
283 jniThrowRuntimeException(env, "ImageReaderContext is not initialized");
284 return NULL;
285 }
286
287 return ctx->getBufferConsumer();
288 }
289
Image_setBufferItem(JNIEnv * env,jobject thiz,const BufferItem * buffer)290 static void Image_setBufferItem(JNIEnv* env, jobject thiz,
291 const BufferItem* buffer)
292 {
293 env->SetLongField(thiz, gSurfaceImageClassInfo.mNativeBuffer, reinterpret_cast<jlong>(buffer));
294 }
295
Image_getBufferItem(JNIEnv * env,jobject image)296 static BufferItem* Image_getBufferItem(JNIEnv* env, jobject image)
297 {
298 return reinterpret_cast<BufferItem*>(
299 env->GetLongField(image, gSurfaceImageClassInfo.mNativeBuffer));
300 }
301
302
303 // ----------------------------------------------------------------------------
304
ImageReader_classInit(JNIEnv * env,jclass clazz)305 static void ImageReader_classInit(JNIEnv* env, jclass clazz)
306 {
307 ALOGV("%s:", __FUNCTION__);
308
309 jclass imageClazz = env->FindClass("android/media/ImageReader$SurfaceImage");
310 LOG_ALWAYS_FATAL_IF(imageClazz == NULL,
311 "can't find android/graphics/ImageReader$SurfaceImage");
312 gSurfaceImageClassInfo.mNativeBuffer = env->GetFieldID(
313 imageClazz, ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID, "J");
314 LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mNativeBuffer == NULL,
315 "can't find android/graphics/ImageReader.%s",
316 ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID);
317
318 gSurfaceImageClassInfo.mTimestamp = env->GetFieldID(
319 imageClazz, ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID, "J");
320 LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mTimestamp == NULL,
321 "can't find android/graphics/ImageReader.%s",
322 ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID);
323
324 gSurfaceImageClassInfo.mDataSpace = env->GetFieldID(
325 imageClazz, ANDROID_MEDIA_SURFACEIMAGE_DS_JNI_ID, "I");
326 LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mDataSpace == NULL,
327 "can't find android/graphics/ImageReader.%s",
328 ANDROID_MEDIA_SURFACEIMAGE_DS_JNI_ID);
329
330 gSurfaceImageClassInfo.mTransform = env->GetFieldID(
331 imageClazz, ANDROID_MEDIA_SURFACEIMAGE_TF_JNI_ID, "I");
332 LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mTransform == NULL,
333 "can't find android/graphics/ImageReader.%s",
334 ANDROID_MEDIA_SURFACEIMAGE_TF_JNI_ID);
335
336 gSurfaceImageClassInfo.mScalingMode = env->GetFieldID(
337 imageClazz, ANDROID_MEDIA_SURFACEIMAGE_SM_JNI_ID, "I");
338 LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mScalingMode == NULL,
339 "can't find android/graphics/ImageReader.%s",
340 ANDROID_MEDIA_SURFACEIMAGE_SM_JNI_ID);
341
342 gSurfaceImageClassInfo.mPlanes = env->GetFieldID(
343 imageClazz, "mPlanes", "[Landroid/media/ImageReader$SurfaceImage$SurfacePlane;");
344 LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mPlanes == NULL,
345 "can't find android/media/ImageReader$ReaderSurfaceImage.mPlanes");
346
347 gImageReaderClassInfo.mNativeContext = env->GetFieldID(
348 clazz, ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID, "J");
349 LOG_ALWAYS_FATAL_IF(gImageReaderClassInfo.mNativeContext == NULL,
350 "can't find android/graphics/ImageReader.%s",
351 ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID);
352
353 gImageReaderClassInfo.postEventFromNative = env->GetStaticMethodID(
354 clazz, "postEventFromNative", "(Ljava/lang/Object;)V");
355 LOG_ALWAYS_FATAL_IF(gImageReaderClassInfo.postEventFromNative == NULL,
356 "can't find android/graphics/ImageReader.postEventFromNative");
357
358 jclass planeClazz = env->FindClass("android/media/ImageReader$SurfaceImage$SurfacePlane");
359 LOG_ALWAYS_FATAL_IF(planeClazz == NULL, "Can not find SurfacePlane class");
360 // FindClass only gives a local reference of jclass object.
361 gSurfacePlaneClassInfo.clazz = (jclass) env->NewGlobalRef(planeClazz);
362 gSurfacePlaneClassInfo.ctor = env->GetMethodID(gSurfacePlaneClassInfo.clazz, "<init>",
363 "(Landroid/media/ImageReader$SurfaceImage;IILjava/nio/ByteBuffer;)V");
364 LOG_ALWAYS_FATAL_IF(gSurfacePlaneClassInfo.ctor == NULL,
365 "Can not find SurfacePlane constructor");
366
367 planeClazz = env->FindClass("android/media/ImageReader$ImagePlane");
368 LOG_ALWAYS_FATAL_IF(planeClazz == NULL, "Can not find ImagePlane class");
369 // FindClass only gives a local reference of jclass object.
370 gImagePlaneClassInfo.clazz = (jclass) env->NewGlobalRef(planeClazz);
371 gImagePlaneClassInfo.ctor = env->GetMethodID(gImagePlaneClassInfo.clazz, "<init>",
372 "(IILjava/nio/ByteBuffer;)V");
373 LOG_ALWAYS_FATAL_IF(gImagePlaneClassInfo.ctor == NULL,
374 "Can not find ImagePlane constructor");
375 }
376
ImageReader_init(JNIEnv * env,jobject thiz,jobject weakThiz,jint width,jint height,jint maxImages,jlong ndkUsage,jint nativeFormat,jint dataSpace)377 static void ImageReader_init(JNIEnv* env, jobject thiz, jobject weakThiz, jint width, jint height,
378 jint maxImages, jlong ndkUsage, jint nativeFormat, jint dataSpace) {
379 status_t res;
380
381 ALOGV("%s: width:%d, height: %d, nativeFormat: %d, maxImages:%d",
382 __FUNCTION__, width, height, nativeFormat, maxImages);
383
384 android_dataspace nativeDataspace = static_cast<android_dataspace>(dataSpace);
385
386 jclass clazz = env->GetObjectClass(thiz);
387 if (clazz == NULL) {
388 jniThrowRuntimeException(env, "Can't find android/graphics/ImageReader");
389 return;
390 }
391 sp<JNIImageReaderContext> ctx(new JNIImageReaderContext(env, weakThiz, clazz, maxImages));
392
393 sp<IGraphicBufferProducer> gbProducer;
394 sp<IGraphicBufferConsumer> gbConsumer;
395 BufferQueue::createBufferQueue(&gbProducer, &gbConsumer);
396 sp<BufferItemConsumer> bufferConsumer;
397 String8 consumerName = String8::format("ImageReader-%dx%df%xm%d-%d-%d",
398 width, height, nativeFormat, maxImages, getpid(),
399 createProcessUniqueId());
400 uint64_t consumerUsage =
401 android_hardware_HardwareBuffer_convertToGrallocUsageBits(ndkUsage);
402
403 bufferConsumer = new BufferItemConsumer(gbConsumer, consumerUsage, maxImages,
404 /*controlledByApp*/true);
405 if (bufferConsumer == nullptr) {
406 jniThrowExceptionFmt(env, "java/lang/RuntimeException",
407 "Failed to allocate native buffer consumer for format 0x%x and usage 0x%x",
408 nativeFormat, consumerUsage);
409 return;
410 }
411
412 if (consumerUsage & GRALLOC_USAGE_PROTECTED) {
413 gbConsumer->setConsumerIsProtected(true);
414 }
415
416 ctx->setBufferConsumer(bufferConsumer);
417 bufferConsumer->setName(consumerName);
418
419 ctx->setProducer(gbProducer);
420 bufferConsumer->setFrameAvailableListener(ctx);
421 ImageReader_setNativeContext(env, thiz, ctx);
422 ctx->setBufferFormat(nativeFormat);
423 ctx->setBufferDataspace(nativeDataspace);
424 ctx->setBufferWidth(width);
425 ctx->setBufferHeight(height);
426
427 // Set the width/height/format/dataspace to the bufferConsumer.
428 res = bufferConsumer->setDefaultBufferSize(width, height);
429 if (res != OK) {
430 jniThrowExceptionFmt(env, "java/lang/IllegalStateException",
431 "Failed to set buffer consumer default size (%dx%d) for format 0x%x",
432 width, height, nativeFormat);
433 return;
434 }
435 res = bufferConsumer->setDefaultBufferFormat(nativeFormat);
436 if (res != OK) {
437 jniThrowExceptionFmt(env, "java/lang/IllegalStateException",
438 "Failed to set buffer consumer default format 0x%x", nativeFormat);
439 return;
440 }
441 res = bufferConsumer->setDefaultBufferDataSpace(nativeDataspace);
442 if (res != OK) {
443 jniThrowExceptionFmt(env, "java/lang/IllegalStateException",
444 "Failed to set buffer consumer default dataSpace 0x%x", nativeDataspace);
445 return;
446 }
447 }
448
ImageReader_close(JNIEnv * env,jobject thiz)449 static void ImageReader_close(JNIEnv* env, jobject thiz)
450 {
451 ALOGV("%s:", __FUNCTION__);
452
453 JNIImageReaderContext* const ctx = ImageReader_getContext(env, thiz);
454 if (ctx == NULL) {
455 // ImageReader is already closed.
456 return;
457 }
458
459 BufferItemConsumer* consumer = NULL;
460 consumer = ImageReader_getBufferConsumer(env, thiz);
461
462 if (consumer != NULL) {
463 consumer->abandon();
464 consumer->setFrameAvailableListener(NULL);
465 }
466 ImageReader_setNativeContext(env, thiz, NULL);
467 }
468
Image_unlockIfLocked(JNIEnv * env,jobject image)469 static sp<Fence> Image_unlockIfLocked(JNIEnv* env, jobject image) {
470 ALOGV("%s", __FUNCTION__);
471 BufferItem* buffer = Image_getBufferItem(env, image);
472 if (buffer == NULL) {
473 jniThrowException(env, "java/lang/IllegalStateException",
474 "Image is not initialized");
475 return Fence::NO_FENCE;
476 }
477
478 // Is locked?
479 bool wasBufferLocked = false;
480 jobject planes = NULL;
481 if (!isFormatOpaque(buffer->mGraphicBuffer->getPixelFormat())) {
482 planes = env->GetObjectField(image, gSurfaceImageClassInfo.mPlanes);
483 }
484 wasBufferLocked = (planes != NULL);
485 if (wasBufferLocked) {
486 status_t res = OK;
487 int fenceFd = -1;
488 if (wasBufferLocked) {
489 res = buffer->mGraphicBuffer->unlockAsync(&fenceFd);
490 if (res != OK) {
491 jniThrowRuntimeException(env, "unlock buffer failed");
492 return Fence::NO_FENCE;
493 }
494 }
495 sp<Fence> releaseFence = new Fence(fenceFd);
496 return releaseFence;
497 ALOGV("Successfully unlocked the image");
498 }
499 return Fence::NO_FENCE;
500 }
501
ImageReader_imageRelease(JNIEnv * env,jobject thiz,jobject image)502 static void ImageReader_imageRelease(JNIEnv* env, jobject thiz, jobject image)
503 {
504 ALOGV("%s:", __FUNCTION__);
505 JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz);
506 if (ctx == NULL) {
507 ALOGW("ImageReader#close called before Image#close, consider calling Image#close first");
508 return;
509 }
510
511 BufferItemConsumer* bufferConsumer = ctx->getBufferConsumer();
512 BufferItem* buffer = Image_getBufferItem(env, image);
513 if (buffer == nullptr) {
514 // Release an already closed image is harmless.
515 return;
516 }
517
518 sp<Fence> releaseFence = Image_unlockIfLocked(env, image);
519 bufferConsumer->releaseBuffer(*buffer, releaseFence);
520 Image_setBufferItem(env, image, NULL);
521 ctx->returnBufferItem(buffer);
522 ALOGV("%s: Image (format: 0x%x) has been released", __FUNCTION__, ctx->getBufferFormat());
523 }
524
ImageReader_imageSetup(JNIEnv * env,jobject thiz,jobject image,jboolean legacyValidateImageFormat)525 static jint ImageReader_imageSetup(JNIEnv* env, jobject thiz, jobject image,
526 jboolean legacyValidateImageFormat) {
527 ALOGV("%s:", __FUNCTION__);
528 JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz);
529 if (ctx == NULL) {
530 jniThrowException(env, "java/lang/IllegalStateException",
531 "ImageReader is not initialized or was already closed");
532 return -1;
533 }
534
535 BufferItemConsumer* bufferConsumer = ctx->getBufferConsumer();
536 BufferItem* buffer = ctx->getBufferItem();
537 if (buffer == NULL) {
538 ALOGW("Unable to acquire a buffer item, very likely client tried to acquire more than"
539 " maxImages buffers");
540 return ACQUIRE_MAX_IMAGES;
541 }
542
543 status_t res = bufferConsumer->acquireBuffer(buffer, 0);
544 if (res != OK) {
545 ctx->returnBufferItem(buffer);
546 if (res != BufferQueue::NO_BUFFER_AVAILABLE) {
547 if (res == INVALID_OPERATION) {
548 // Max number of images were already acquired.
549 ALOGE("%s: Max number of buffers allowed are already acquired : %s (%d)",
550 __FUNCTION__, strerror(-res), res);
551 return ACQUIRE_MAX_IMAGES;
552 } else {
553 ALOGE("%s: Acquire image failed with some unknown error: %s (%d)",
554 __FUNCTION__, strerror(-res), res);
555 jniThrowExceptionFmt(env, "java/lang/IllegalStateException",
556 "Unknown error (%d) when we tried to acquire an image.",
557 res);
558 return ACQUIRE_NO_BUFFERS;
559 }
560 }
561 // This isn't really an error case, as the application may acquire buffer at any time.
562 return ACQUIRE_NO_BUFFERS;
563 }
564
565 // Add some extra checks for non-opaque formats.
566 if (!isFormatOpaque(ctx->getBufferFormat())) {
567 // Check if the left-top corner of the crop rect is origin, we currently assume this point is
568 // zero, will revisit this once this assumption turns out problematic.
569 Point lt = buffer->mCrop.leftTop();
570 if (lt.x != 0 || lt.y != 0) {
571 jniThrowExceptionFmt(env, "java/lang/UnsupportedOperationException",
572 "crop left top corner [%d, %d] need to be at origin", lt.x, lt.y);
573 return -1;
574 }
575
576 // Check if the producer buffer configurations match what ImageReader configured.
577 int outputWidth = getBufferWidth(buffer);
578 int outputHeight = getBufferHeight(buffer);
579
580 int imgReaderFmt = ctx->getBufferFormat();
581 int imageReaderWidth = ctx->getBufferWidth();
582 int imageReaderHeight = ctx->getBufferHeight();
583 int bufferFormat = buffer->mGraphicBuffer->getPixelFormat();
584 if ((bufferFormat != HAL_PIXEL_FORMAT_BLOB) && (imgReaderFmt != HAL_PIXEL_FORMAT_BLOB) &&
585 (imageReaderWidth != outputWidth || imageReaderHeight != outputHeight)) {
586 ALOGV("%s: Producer buffer size: %dx%d, doesn't match ImageReader configured size: %dx%d",
587 __FUNCTION__, outputWidth, outputHeight, imageReaderWidth, imageReaderHeight);
588 }
589 if (legacyValidateImageFormat && imgReaderFmt != bufferFormat) {
590 if (imgReaderFmt == HAL_PIXEL_FORMAT_YCbCr_420_888 &&
591 isPossiblyYUV(bufferFormat)) {
592 // Treat formats that are compatible with flexible YUV
593 // (HAL_PIXEL_FORMAT_YCbCr_420_888) as HAL_PIXEL_FORMAT_YCbCr_420_888.
594 ALOGV("%s: Treat buffer format to 0x%x as HAL_PIXEL_FORMAT_YCbCr_420_888",
595 __FUNCTION__, bufferFormat);
596 } else if (imgReaderFmt == HAL_PIXEL_FORMAT_YCBCR_P010 &&
597 isPossibly10BitYUV(bufferFormat)) {
598 // Treat formats that are compatible with flexible 10-bit YUV
599 // (HAL_PIXEL_FORMAT_YCBCR_P010) as HAL_PIXEL_FORMAT_YCBCR_P010.
600 ALOGV("%s: Treat buffer format to 0x%x as HAL_PIXEL_FORMAT_YCBCR_P010",
601 __FUNCTION__, bufferFormat);
602 } else if (imgReaderFmt == HAL_PIXEL_FORMAT_BLOB &&
603 bufferFormat == HAL_PIXEL_FORMAT_RGBA_8888) {
604 // Using HAL_PIXEL_FORMAT_RGBA_8888 Gralloc buffers containing JPEGs to get around
605 // SW write limitations for (b/17379185).
606 ALOGV("%s: Receiving JPEG in HAL_PIXEL_FORMAT_RGBA_8888 buffer.", __FUNCTION__);
607 } else {
608 // Return the buffer to the queue. No need to provide fence, as this buffer wasn't
609 // used anywhere yet.
610 bufferConsumer->releaseBuffer(*buffer);
611 ctx->returnBufferItem(buffer);
612
613 // Throw exception
614 ALOGE("Producer output buffer format: 0x%x, ImageReader configured format: 0x%x",
615 bufferFormat, ctx->getBufferFormat());
616 String8 msg;
617 msg.appendFormat("The producer output buffer format 0x%x doesn't "
618 "match the ImageReader's configured buffer format 0x%x.",
619 bufferFormat, ctx->getBufferFormat());
620 jniThrowException(env, "java/lang/UnsupportedOperationException",
621 msg.string());
622 return -1;
623 }
624 }
625
626 }
627
628 // Set SurfaceImage instance member variables
629 Image_setBufferItem(env, image, buffer);
630 env->SetLongField(image, gSurfaceImageClassInfo.mTimestamp,
631 static_cast<jlong>(buffer->mTimestamp));
632 env->SetIntField(image, gSurfaceImageClassInfo.mDataSpace,
633 static_cast<jint>(buffer->mDataSpace));
634 auto transform = buffer->mTransform;
635 if (buffer->mTransformToDisplayInverse) {
636 transform |= NATIVE_WINDOW_TRANSFORM_INVERSE_DISPLAY;
637 }
638 env->SetIntField(image, gSurfaceImageClassInfo.mTransform,
639 static_cast<jint>(transform));
640 env->SetIntField(image, gSurfaceImageClassInfo.mScalingMode,
641 static_cast<jint>(buffer->mScalingMode));
642
643 return ACQUIRE_SUCCESS;
644 }
645
ImageReader_detachImage(JNIEnv * env,jobject thiz,jobject image)646 static jint ImageReader_detachImage(JNIEnv* env, jobject thiz, jobject image) {
647 ALOGV("%s:", __FUNCTION__);
648 JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz);
649 if (ctx == NULL) {
650 jniThrowException(env, "java/lang/IllegalStateException", "ImageReader was already closed");
651 return -1;
652 }
653
654 BufferItemConsumer* bufferConsumer = ctx->getBufferConsumer();
655 BufferItem* buffer = Image_getBufferItem(env, image);
656 if (!buffer) {
657 ALOGE(
658 "Image already released and can not be detached from ImageReader!!!");
659 jniThrowException(env, "java/lang/IllegalStateException",
660 "Image detach from ImageReader failed: buffer was already released");
661 return -1;
662 }
663
664 status_t res = OK;
665 Image_unlockIfLocked(env, image);
666 res = bufferConsumer->detachBuffer(buffer->mSlot);
667 if (res != OK) {
668 ALOGE("Image detach failed: %s (%d)!!!", strerror(-res), res);
669 jniThrowRuntimeException(env,
670 "nativeDetachImage failed for image!!!");
671 return res;
672 }
673 return OK;
674 }
675
ImageReader_discardFreeBuffers(JNIEnv * env,jobject thiz)676 static void ImageReader_discardFreeBuffers(JNIEnv* env, jobject thiz) {
677 ALOGV("%s:", __FUNCTION__);
678 JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz);
679 if (ctx == NULL) {
680 jniThrowException(env, "java/lang/IllegalStateException", "ImageReader was already closed");
681 return;
682 }
683
684 BufferItemConsumer* bufferConsumer = ctx->getBufferConsumer();
685 status_t res = bufferConsumer->discardFreeBuffers();
686 if (res != OK) {
687 ALOGE("Buffer discard failed: %s (%d)", strerror(-res), res);
688 jniThrowRuntimeException(env,
689 "nativeDicardFreebuffers failed");
690 }
691 }
692
ImageReader_getSurface(JNIEnv * env,jobject thiz)693 static jobject ImageReader_getSurface(JNIEnv* env, jobject thiz)
694 {
695 ALOGV("%s: ", __FUNCTION__);
696
697 IGraphicBufferProducer* gbp = ImageReader_getProducer(env, thiz);
698 if (gbp == NULL) {
699 jniThrowRuntimeException(env, "Buffer consumer is uninitialized");
700 return NULL;
701 }
702
703 // Wrap the IGBP in a Java-language Surface.
704 return android_view_Surface_createFromIGraphicBufferProducer(env, gbp);
705 }
706
Image_getLockedImage(JNIEnv * env,jobject thiz,LockedImage * image,uint64_t ndkReaderUsage)707 static void Image_getLockedImage(JNIEnv* env, jobject thiz, LockedImage *image,
708 uint64_t ndkReaderUsage) {
709 ALOGV("%s", __FUNCTION__);
710 BufferItem* buffer = Image_getBufferItem(env, thiz);
711 if (buffer == NULL) {
712 jniThrowException(env, "java/lang/IllegalStateException",
713 "Image is not initialized");
714 return;
715 }
716
717 uint32_t lockUsage;
718 if ((ndkReaderUsage & (AHARDWAREBUFFER_USAGE_CPU_WRITE_RARELY
719 | AHARDWAREBUFFER_USAGE_CPU_WRITE_OFTEN)) != 0) {
720 lockUsage = GRALLOC_USAGE_SW_READ_OFTEN | GRALLOC_USAGE_SW_WRITE_OFTEN;
721 } else {
722 lockUsage = GRALLOC_USAGE_SW_READ_OFTEN;
723 }
724
725 status_t res = lockImageFromBuffer(buffer, lockUsage, buffer->mFence->dup(), image);
726
727 if (res != OK) {
728 jniThrowExceptionFmt(env, "java/lang/RuntimeException",
729 "lock buffer failed for format 0x%x",
730 buffer->mGraphicBuffer->getPixelFormat());
731 return;
732 }
733
734 // Carry over some fields from BufferItem.
735 image->crop = buffer->mCrop;
736 image->transform = buffer->mTransform;
737 image->scalingMode = buffer->mScalingMode;
738 image->timestamp = buffer->mTimestamp;
739 image->dataSpace = buffer->mDataSpace;
740 image->frameNumber = buffer->mFrameNumber;
741
742 ALOGV("%s: Successfully locked the image", __FUNCTION__);
743 // crop, transform, scalingMode, timestamp, and frameNumber should be set by producer,
744 // and we don't set them here.
745 }
746
Image_getLockedImageInfo(JNIEnv * env,LockedImage * buffer,int idx,int32_t writerFormat,uint8_t ** base,uint32_t * size,int * pixelStride,int * rowStride)747 static bool Image_getLockedImageInfo(JNIEnv* env, LockedImage* buffer, int idx,
748 int32_t writerFormat, uint8_t **base, uint32_t *size, int *pixelStride, int *rowStride) {
749 ALOGV("%s", __FUNCTION__);
750
751 status_t res = getLockedImageInfo(buffer, idx, writerFormat, base, size,
752 pixelStride, rowStride);
753 if (res != OK) {
754 jniThrowExceptionFmt(env, "java/lang/UnsupportedOperationException",
755 "Pixel format: 0x%x is unsupported", buffer->flexFormat);
756 return false;
757 }
758 return true;
759 }
760
ImageReader_unlockGraphicBuffer(JNIEnv * env,jobject,jobject buffer)761 static void ImageReader_unlockGraphicBuffer(JNIEnv* env, jobject /*thiz*/,
762 jobject buffer) {
763 sp<GraphicBuffer> graphicBuffer =
764 android_graphics_GraphicBuffer_getNativeGraphicsBuffer(env, buffer);
765 if (graphicBuffer.get() == NULL) {
766 jniThrowRuntimeException(env, "Invalid graphic buffer!");
767 }
768
769 status_t res = graphicBuffer->unlock();
770 if (res != OK) {
771 jniThrowRuntimeException(env, "unlock buffer failed");
772 }
773 }
774
ImageReader_createImagePlanes(JNIEnv * env,jobject,int numPlanes,jobject buffer,int fenceFd,int format,int cropLeft,int cropTop,int cropRight,int cropBottom)775 static jobjectArray ImageReader_createImagePlanes(JNIEnv* env, jobject /*thiz*/,
776 int numPlanes, jobject buffer, int fenceFd, int format, int cropLeft, int cropTop,
777 int cropRight, int cropBottom)
778 {
779 ALOGV("%s: create ImagePlane array with size %d", __FUNCTION__, numPlanes);
780 int rowStride = 0;
781 int pixelStride = 0;
782 uint8_t *pData = NULL;
783 uint32_t dataSize = 0;
784 jobject byteBuffer = NULL;
785
786 PublicFormat publicReaderFormat = static_cast<PublicFormat>(format);
787 int halReaderFormat = mapPublicFormatToHalFormat(publicReaderFormat);
788
789 if (isFormatOpaque(halReaderFormat) && numPlanes > 0) {
790 String8 msg;
791 msg.appendFormat("Format 0x%x is opaque, thus not writable, the number of planes (%d)"
792 " must be 0", halReaderFormat, numPlanes);
793 jniThrowException(env, "java/lang/IllegalArgumentException", msg.string());
794 return NULL;
795 }
796
797 jobjectArray imagePlanes = env->NewObjectArray(numPlanes, gImagePlaneClassInfo.clazz,
798 /*initial_element*/NULL);
799 if (imagePlanes == NULL) {
800 jniThrowRuntimeException(env, "Failed to create ImagePlane arrays,"
801 " probably out of memory");
802 return NULL;
803 }
804 if (isFormatOpaque(halReaderFormat)) {
805 // Return 0 element surface array.
806 return imagePlanes;
807 }
808
809 LockedImage lockedImg = LockedImage();
810 uint32_t lockUsage = GRALLOC_USAGE_SW_READ_OFTEN;
811
812 Rect cropRect(cropLeft, cropTop, cropRight, cropBottom);
813 status_t res = lockImageFromBuffer(
814 android_graphics_GraphicBuffer_getNativeGraphicsBuffer(env, buffer), lockUsage,
815 cropRect, fenceFd, &lockedImg);
816 if (res != OK) {
817 jniThrowExceptionFmt(env, "java/lang/RuntimeException",
818 "lock buffer failed for format 0x%x", format);
819 return nullptr;
820 }
821
822 // Create all ImagePlanes
823 for (int i = 0; i < numPlanes; i++) {
824 if (!Image_getLockedImageInfo(env, &lockedImg, i, halReaderFormat,
825 &pData, &dataSize, &pixelStride, &rowStride)) {
826 return NULL;
827 }
828 byteBuffer = env->NewDirectByteBuffer(pData, dataSize);
829 if ((byteBuffer == NULL) && (env->ExceptionCheck() == false)) {
830 jniThrowException(env, "java/lang/IllegalStateException",
831 "Failed to allocate ByteBuffer");
832 return NULL;
833 }
834
835 // Finally, create this ImagePlane.
836 jobject imagePlane = env->NewObject(gImagePlaneClassInfo.clazz,
837 gImagePlaneClassInfo.ctor, rowStride, pixelStride, byteBuffer);
838 env->SetObjectArrayElement(imagePlanes, i, imagePlane);
839 }
840
841 return imagePlanes;
842 }
843
Image_createSurfacePlanes(JNIEnv * env,jobject thiz,int numPlanes,int readerFormat,uint64_t ndkReaderUsage)844 static jobjectArray Image_createSurfacePlanes(JNIEnv* env, jobject thiz,
845 int numPlanes, int readerFormat, uint64_t ndkReaderUsage)
846 {
847 ALOGV("%s: create SurfacePlane array with size %d", __FUNCTION__, numPlanes);
848 int rowStride = 0;
849 int pixelStride = 0;
850 uint8_t *pData = NULL;
851 uint32_t dataSize = 0;
852 jobject byteBuffer = NULL;
853
854 PublicFormat publicReaderFormat = static_cast<PublicFormat>(readerFormat);
855 int halReaderFormat = mapPublicFormatToHalFormat(publicReaderFormat);
856
857 if (isFormatOpaque(halReaderFormat) && numPlanes > 0) {
858 String8 msg;
859 msg.appendFormat("Format 0x%x is opaque, thus not writable, the number of planes (%d)"
860 " must be 0", halReaderFormat, numPlanes);
861 jniThrowException(env, "java/lang/IllegalArgumentException", msg.string());
862 return NULL;
863 }
864
865 jobjectArray surfacePlanes = env->NewObjectArray(numPlanes, gSurfacePlaneClassInfo.clazz,
866 /*initial_element*/NULL);
867 if (surfacePlanes == NULL) {
868 jniThrowRuntimeException(env, "Failed to create SurfacePlane arrays,"
869 " probably out of memory");
870 return NULL;
871 }
872 if (isFormatOpaque(halReaderFormat)) {
873 // Return 0 element surface array.
874 return surfacePlanes;
875 }
876
877 LockedImage lockedImg = LockedImage();
878 Image_getLockedImage(env, thiz, &lockedImg, ndkReaderUsage);
879 if (env->ExceptionCheck()) {
880 return NULL;
881 }
882 // Create all SurfacePlanes
883 for (int i = 0; i < numPlanes; i++) {
884 if (!Image_getLockedImageInfo(env, &lockedImg, i, halReaderFormat,
885 &pData, &dataSize, &pixelStride, &rowStride)) {
886 return NULL;
887 }
888 byteBuffer = env->NewDirectByteBuffer(pData, dataSize);
889 if ((byteBuffer == NULL) && (env->ExceptionCheck() == false)) {
890 jniThrowException(env, "java/lang/IllegalStateException",
891 "Failed to allocate ByteBuffer");
892 return NULL;
893 }
894
895 // Finally, create this SurfacePlane.
896 jobject surfacePlane = env->NewObject(gSurfacePlaneClassInfo.clazz,
897 gSurfacePlaneClassInfo.ctor, thiz, rowStride, pixelStride, byteBuffer);
898 env->SetObjectArrayElement(surfacePlanes, i, surfacePlane);
899 }
900
901 return surfacePlanes;
902 }
903
Image_getWidth(JNIEnv * env,jobject thiz)904 static jint Image_getWidth(JNIEnv* env, jobject thiz)
905 {
906 BufferItem* buffer = Image_getBufferItem(env, thiz);
907 return getBufferWidth(buffer);
908 }
909
Image_getHeight(JNIEnv * env,jobject thiz)910 static jint Image_getHeight(JNIEnv* env, jobject thiz)
911 {
912 BufferItem* buffer = Image_getBufferItem(env, thiz);
913 return getBufferHeight(buffer);
914 }
915
Image_getFenceFd(JNIEnv * env,jobject thiz)916 static jint Image_getFenceFd(JNIEnv* env, jobject thiz)
917 {
918 BufferItem* buffer = Image_getBufferItem(env, thiz);
919 if ((buffer != NULL) && (buffer->mFence.get() != NULL)){
920 return buffer->mFence->get();
921 }
922
923 return -1;
924 }
925
Image_getFormat(JNIEnv * env,jobject thiz,jint readerFormat)926 static jint Image_getFormat(JNIEnv* env, jobject thiz, jint readerFormat)
927 {
928 if (isFormatOpaque(readerFormat)) {
929 // Assuming opaque reader produce opaque images.
930 return static_cast<jint>(PublicFormat::PRIVATE);
931 } else {
932 BufferItem* buffer = Image_getBufferItem(env, thiz);
933 int readerHalFormat = mapPublicFormatToHalFormat(static_cast<PublicFormat>(readerFormat));
934 int32_t fmt = applyFormatOverrides(
935 buffer->mGraphicBuffer->getPixelFormat(), readerHalFormat);
936 // Override the image format to HAL_PIXEL_FORMAT_YCbCr_420_888 if the actual format is
937 // NV21 or YV12. This could only happen when the Gralloc HAL version is v0.1 thus doesn't
938 // support lockycbcr(), the CpuConsumer need to use the lock() method in the
939 // lockNextBuffer() call. For Gralloc HAL v0.2 or newer, this format should already be
940 // overridden to HAL_PIXEL_FORMAT_YCbCr_420_888 for the flexible YUV compatible formats.
941 if (isPossiblyYUV(fmt)) {
942 fmt = HAL_PIXEL_FORMAT_YCbCr_420_888;
943 }
944 PublicFormat publicFmt = mapHalFormatDataspaceToPublicFormat(fmt, buffer->mDataSpace);
945 return static_cast<jint>(publicFmt);
946 }
947 }
948
Image_getHardwareBuffer(JNIEnv * env,jobject thiz)949 static jobject Image_getHardwareBuffer(JNIEnv* env, jobject thiz) {
950 BufferItem* buffer = Image_getBufferItem(env, thiz);
951 AHardwareBuffer* b = AHardwareBuffer_from_GraphicBuffer(buffer->mGraphicBuffer.get());
952 // don't user the public AHardwareBuffer_toHardwareBuffer() because this would force us
953 // to link against libandroid.so
954 return android_hardware_HardwareBuffer_createFromAHardwareBuffer(env, b);
955 }
956
957 } // extern "C"
958
959 // ----------------------------------------------------------------------------
960
961 static const JNINativeMethod gImageReaderMethods[] = {
962 {"nativeClassInit", "()V", (void*)ImageReader_classInit },
963 {"nativeInit", "(Ljava/lang/Object;IIIJII)V", (void*)ImageReader_init },
964 {"nativeClose", "()V", (void*)ImageReader_close },
965 {"nativeReleaseImage", "(Landroid/media/Image;)V", (void*)ImageReader_imageRelease },
966 {"nativeImageSetup", "(Landroid/media/Image;Z)I", (void*)ImageReader_imageSetup },
967 {"nativeGetSurface", "()Landroid/view/Surface;", (void*)ImageReader_getSurface },
968 {"nativeDetachImage", "(Landroid/media/Image;)I", (void*)ImageReader_detachImage },
969 {"nativeCreateImagePlanes",
970 "(ILandroid/graphics/GraphicBuffer;IIIIII)[Landroid/media/ImageReader$ImagePlane;",
971 (void*)ImageReader_createImagePlanes },
972 {"nativeUnlockGraphicBuffer",
973 "(Landroid/graphics/GraphicBuffer;)V", (void*)ImageReader_unlockGraphicBuffer },
974 {"nativeDiscardFreeBuffers", "()V", (void*)ImageReader_discardFreeBuffers }
975 };
976
977 static const JNINativeMethod gImageMethods[] = {
978 {"nativeCreatePlanes", "(IIJ)[Landroid/media/ImageReader$SurfaceImage$SurfacePlane;",
979 (void*)Image_createSurfacePlanes },
980 {"nativeGetWidth", "()I", (void*)Image_getWidth },
981 {"nativeGetHeight", "()I", (void*)Image_getHeight },
982 {"nativeGetFormat", "(I)I", (void*)Image_getFormat },
983 {"nativeGetFenceFd", "()I", (void*)Image_getFenceFd },
984 {"nativeGetHardwareBuffer", "()Landroid/hardware/HardwareBuffer;",
985 (void*)Image_getHardwareBuffer },
986 };
987
register_android_media_ImageReader(JNIEnv * env)988 int register_android_media_ImageReader(JNIEnv *env) {
989
990 int ret1 = AndroidRuntime::registerNativeMethods(env,
991 "android/media/ImageReader", gImageReaderMethods, NELEM(gImageReaderMethods));
992
993 int ret2 = AndroidRuntime::registerNativeMethods(env,
994 "android/media/ImageReader$SurfaceImage", gImageMethods, NELEM(gImageMethods));
995
996 return (ret1 || ret2);
997 }
998