1 /*
2 * Copyright 2015 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 //#define LOG_NDEBUG 0
18 #define LOG_TAG "ImageWriter_JNI"
19 #include "android_media_Utils.h"
20
21 #include <utils/Condition.h>
22 #include <utils/Log.h>
23 #include <utils/Mutex.h>
24 #include <utils/String8.h>
25 #include <utils/Thread.h>
26
27 #include <gui/IProducerListener.h>
28 #include <gui/Surface.h>
29 #include <ui/PublicFormat.h>
30 #include <android_runtime/AndroidRuntime.h>
31 #include <android_runtime/android_view_Surface.h>
32 #include <android_runtime/android_graphics_GraphicBuffer.h>
33 #include <android_runtime/android_hardware_HardwareBuffer.h>
34 #include <private/android/AHardwareBufferHelpers.h>
35 #include <jni.h>
36 #include <nativehelper/JNIHelp.h>
37
38 #include <stdint.h>
39 #include <inttypes.h>
40 #include <android/hardware_buffer_jni.h>
41
42 #include <deque>
43
44 #define IMAGE_BUFFER_JNI_ID "mNativeBuffer"
45 #define IMAGE_FORMAT_UNKNOWN 0 // This is the same value as ImageFormat#UNKNOWN.
46 // ----------------------------------------------------------------------------
47
48 using namespace android;
49
50 static struct {
51 jmethodID postEventFromNative;
52 jfieldID mWriterFormat;
53 } gImageWriterClassInfo;
54
55 static struct {
56 jfieldID mNativeBuffer;
57 jfieldID mNativeFenceFd;
58 jfieldID mPlanes;
59 } gSurfaceImageClassInfo;
60
61 static struct {
62 jclass clazz;
63 jmethodID ctor;
64 } gSurfacePlaneClassInfo;
65
66 // ----------------------------------------------------------------------------
67
68 class JNIImageWriterContext : public BnProducerListener {
69 public:
70 JNIImageWriterContext(JNIEnv* env, jobject weakThiz, jclass clazz);
71
72 virtual ~JNIImageWriterContext();
73
74 // Implementation of IProducerListener, used to notify the ImageWriter that the consumer
75 // has returned a buffer and it is ready for ImageWriter to dequeue.
76 virtual void onBufferReleased();
77
setProducer(const sp<Surface> & producer)78 void setProducer(const sp<Surface>& producer) { mProducer = producer; }
getProducer()79 Surface* getProducer() { return mProducer.get(); }
80
setBufferFormat(int format)81 void setBufferFormat(int format) { mFormat = format; }
getBufferFormat()82 int getBufferFormat() { return mFormat; }
83
setBufferWidth(int width)84 void setBufferWidth(int width) { mWidth = width; }
getBufferWidth()85 int getBufferWidth() { return mWidth; }
86
setBufferHeight(int height)87 void setBufferHeight(int height) { mHeight = height; }
getBufferHeight()88 int getBufferHeight() { return mHeight; }
89
queueAttachedFlag(bool isAttached)90 void queueAttachedFlag(bool isAttached) {
91 Mutex::Autolock l(mAttachedFlagQueueLock);
92 mAttachedFlagQueue.push_back(isAttached);
93 }
dequeueAttachedFlag()94 void dequeueAttachedFlag() {
95 Mutex::Autolock l(mAttachedFlagQueueLock);
96 mAttachedFlagQueue.pop_back();
97 }
98 private:
99 static JNIEnv* getJNIEnv(bool* needsDetach);
100 static void detachJNI();
101
102 sp<Surface> mProducer;
103 jobject mWeakThiz;
104 jclass mClazz;
105 int mFormat;
106 int mWidth;
107 int mHeight;
108
109 // Class for a shared thread used to detach buffers from buffer queues
110 // to discard buffers after consumers are done using them.
111 // This is needed because detaching buffers in onBufferReleased callback
112 // can lead to deadlock when consumer/producer are on the same process.
113 class BufferDetacher {
114 public:
115 // Called by JNIImageWriterContext ctor. Will start the thread for first ref.
116 void addRef();
117 // Called by JNIImageWriterContext dtor. Will stop the thread after ref goes to 0.
118 void removeRef();
119 // Called by onBufferReleased to signal this thread to detach a buffer
120 void detach(wp<Surface>);
121
122 private:
123
124 class DetachThread : public Thread {
125 public:
DetachThread()126 DetachThread() : Thread(/*canCallJava*/false) {};
127
128 void detach(wp<Surface>);
129
130 virtual void requestExit() override;
131
132 private:
133 virtual bool threadLoop() override;
134
135 Mutex mLock;
136 Condition mCondition;
137 std::deque<wp<Surface>> mQueue;
138
139 static const nsecs_t kWaitDuration = 500000000; // 500 ms
140 };
141 sp<DetachThread> mThread;
142
143 Mutex mLock;
144 int mRefCount = 0;
145 };
146
147 static BufferDetacher sBufferDetacher;
148
149 // Buffer queue guarantees both producer and consumer side buffer flows are
150 // in order. See b/19977520. As a result, we can use a queue here.
151 Mutex mAttachedFlagQueueLock;
152 std::deque<bool> mAttachedFlagQueue;
153 };
154
155 JNIImageWriterContext::BufferDetacher JNIImageWriterContext::sBufferDetacher;
156
addRef()157 void JNIImageWriterContext::BufferDetacher::addRef() {
158 Mutex::Autolock l(mLock);
159 mRefCount++;
160 if (mRefCount == 1) {
161 mThread = new DetachThread();
162 mThread->run("BufDtchThrd");
163 }
164 }
165
removeRef()166 void JNIImageWriterContext::BufferDetacher::removeRef() {
167 Mutex::Autolock l(mLock);
168 mRefCount--;
169 if (mRefCount == 0) {
170 mThread->requestExit();
171 mThread->join();
172 mThread.clear();
173 }
174 }
175
detach(wp<Surface> bq)176 void JNIImageWriterContext::BufferDetacher::detach(wp<Surface> bq) {
177 Mutex::Autolock l(mLock);
178 if (mThread == nullptr) {
179 ALOGE("%s: buffer detach thread is gone!", __FUNCTION__);
180 return;
181 }
182 mThread->detach(bq);
183 }
184
detach(wp<Surface> bq)185 void JNIImageWriterContext::BufferDetacher::DetachThread::detach(wp<Surface> bq) {
186 Mutex::Autolock l(mLock);
187 mQueue.push_back(bq);
188 mCondition.signal();
189 }
190
requestExit()191 void JNIImageWriterContext::BufferDetacher::DetachThread::requestExit() {
192 Thread::requestExit();
193 {
194 Mutex::Autolock l(mLock);
195 mQueue.clear();
196 }
197 mCondition.signal();
198 }
199
threadLoop()200 bool JNIImageWriterContext::BufferDetacher::DetachThread::threadLoop() {
201 Mutex::Autolock l(mLock);
202 mCondition.waitRelative(mLock, kWaitDuration);
203
204 while (!mQueue.empty()) {
205 if (exitPending()) {
206 return false;
207 }
208
209 wp<Surface> wbq = mQueue.front();
210 mQueue.pop_front();
211 sp<Surface> bq = wbq.promote();
212 if (bq != nullptr) {
213 sp<Fence> fence;
214 sp<GraphicBuffer> buffer;
215 ALOGV("%s: One buffer is detached", __FUNCTION__);
216 mLock.unlock();
217 bq->detachNextBuffer(&buffer, &fence);
218 mLock.lock();
219 }
220 }
221 return !exitPending();
222 }
223
JNIImageWriterContext(JNIEnv * env,jobject weakThiz,jclass clazz)224 JNIImageWriterContext::JNIImageWriterContext(JNIEnv* env, jobject weakThiz, jclass clazz) :
225 mWeakThiz(env->NewGlobalRef(weakThiz)),
226 mClazz((jclass)env->NewGlobalRef(clazz)),
227 mFormat(0),
228 mWidth(-1),
229 mHeight(-1) {
230 sBufferDetacher.addRef();
231 }
232
~JNIImageWriterContext()233 JNIImageWriterContext::~JNIImageWriterContext() {
234 ALOGV("%s", __FUNCTION__);
235 bool needsDetach = false;
236 JNIEnv* env = getJNIEnv(&needsDetach);
237 if (env != NULL) {
238 env->DeleteGlobalRef(mWeakThiz);
239 env->DeleteGlobalRef(mClazz);
240 } else {
241 ALOGW("leaking JNI object references");
242 }
243 if (needsDetach) {
244 detachJNI();
245 }
246
247 mProducer.clear();
248 sBufferDetacher.removeRef();
249 }
250
getJNIEnv(bool * needsDetach)251 JNIEnv* JNIImageWriterContext::getJNIEnv(bool* needsDetach) {
252 ALOGV("%s", __FUNCTION__);
253 LOG_ALWAYS_FATAL_IF(needsDetach == NULL, "needsDetach is null!!!");
254 *needsDetach = false;
255 JNIEnv* env = AndroidRuntime::getJNIEnv();
256 if (env == NULL) {
257 JavaVMAttachArgs args = {JNI_VERSION_1_4, NULL, NULL};
258 JavaVM* vm = AndroidRuntime::getJavaVM();
259 int result = vm->AttachCurrentThread(&env, (void*) &args);
260 if (result != JNI_OK) {
261 ALOGE("thread attach failed: %#x", result);
262 return NULL;
263 }
264 *needsDetach = true;
265 }
266 return env;
267 }
268
detachJNI()269 void JNIImageWriterContext::detachJNI() {
270 ALOGV("%s", __FUNCTION__);
271 JavaVM* vm = AndroidRuntime::getJavaVM();
272 int result = vm->DetachCurrentThread();
273 if (result != JNI_OK) {
274 ALOGE("thread detach failed: %#x", result);
275 }
276 }
277
onBufferReleased()278 void JNIImageWriterContext::onBufferReleased() {
279 ALOGV("%s: buffer released", __FUNCTION__);
280 bool needsDetach = false;
281 JNIEnv* env = getJNIEnv(&needsDetach);
282
283 bool bufferIsAttached = false;
284 {
285 Mutex::Autolock l(mAttachedFlagQueueLock);
286 if (!mAttachedFlagQueue.empty()) {
287 bufferIsAttached = mAttachedFlagQueue.front();
288 mAttachedFlagQueue.pop_front();
289 } else {
290 ALOGW("onBufferReleased called with no attached flag queued");
291 }
292 }
293
294 if (env != NULL) {
295 // Detach the buffer every time when a buffer consumption is done,
296 // need let this callback give a BufferItem, then only detach if it was attached to this
297 // Writer. see b/19977520
298 if (mFormat == HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED || bufferIsAttached) {
299 sBufferDetacher.detach(mProducer);
300 }
301
302 env->CallStaticVoidMethod(mClazz, gImageWriterClassInfo.postEventFromNative, mWeakThiz);
303 } else {
304 ALOGW("onBufferReleased event will not posted");
305 }
306
307 if (needsDetach) {
308 detachJNI();
309 }
310 }
311
312 // ----------------------------------------------------------------------------
313
314 extern "C" {
315
316 // -------------------------------Private method declarations--------------
317
318 static void Image_setNativeContext(JNIEnv* env, jobject thiz,
319 sp<GraphicBuffer> buffer, int fenceFd);
320 static void Image_getNativeContext(JNIEnv* env, jobject thiz,
321 GraphicBuffer** buffer, int* fenceFd);
322 static void Image_unlockIfLocked(JNIEnv* env, jobject thiz);
323
324 // --------------------------ImageWriter methods---------------------------------------
325
ImageWriter_classInit(JNIEnv * env,jclass clazz)326 static void ImageWriter_classInit(JNIEnv* env, jclass clazz) {
327 ALOGV("%s:", __FUNCTION__);
328 jclass imageClazz = env->FindClass("android/media/ImageWriter$WriterSurfaceImage");
329 LOG_ALWAYS_FATAL_IF(imageClazz == NULL,
330 "can't find android/media/ImageWriter$WriterSurfaceImage");
331 gSurfaceImageClassInfo.mNativeBuffer = env->GetFieldID(
332 imageClazz, IMAGE_BUFFER_JNI_ID, "J");
333 LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mNativeBuffer == NULL,
334 "can't find android/media/ImageWriter$WriterSurfaceImage.%s", IMAGE_BUFFER_JNI_ID);
335
336 gSurfaceImageClassInfo.mNativeFenceFd = env->GetFieldID(
337 imageClazz, "mNativeFenceFd", "I");
338 LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mNativeFenceFd == NULL,
339 "can't find android/media/ImageWriter$WriterSurfaceImage.mNativeFenceFd");
340
341 gSurfaceImageClassInfo.mPlanes = env->GetFieldID(
342 imageClazz, "mPlanes", "[Landroid/media/ImageWriter$WriterSurfaceImage$SurfacePlane;");
343 LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mPlanes == NULL,
344 "can't find android/media/ImageWriter$WriterSurfaceImage.mPlanes");
345
346 gImageWriterClassInfo.postEventFromNative = env->GetStaticMethodID(
347 clazz, "postEventFromNative", "(Ljava/lang/Object;)V");
348 LOG_ALWAYS_FATAL_IF(gImageWriterClassInfo.postEventFromNative == NULL,
349 "can't find android/media/ImageWriter.postEventFromNative");
350
351 gImageWriterClassInfo.mWriterFormat = env->GetFieldID(
352 clazz, "mWriterFormat", "I");
353 LOG_ALWAYS_FATAL_IF(gImageWriterClassInfo.mWriterFormat == NULL,
354 "can't find android/media/ImageWriter.mWriterFormat");
355
356 jclass planeClazz = env->FindClass("android/media/ImageWriter$WriterSurfaceImage$SurfacePlane");
357 LOG_ALWAYS_FATAL_IF(planeClazz == NULL, "Can not find SurfacePlane class");
358 // FindClass only gives a local reference of jclass object.
359 gSurfacePlaneClassInfo.clazz = (jclass) env->NewGlobalRef(planeClazz);
360 gSurfacePlaneClassInfo.ctor = env->GetMethodID(gSurfacePlaneClassInfo.clazz, "<init>",
361 "(Landroid/media/ImageWriter$WriterSurfaceImage;IILjava/nio/ByteBuffer;)V");
362 LOG_ALWAYS_FATAL_IF(gSurfacePlaneClassInfo.ctor == NULL,
363 "Can not find SurfacePlane constructor");
364 }
365
ImageWriter_init(JNIEnv * env,jobject thiz,jobject weakThiz,jobject jsurface,jint maxImages,jint userFormat,jint userWidth,jint userHeight)366 static jlong ImageWriter_init(JNIEnv* env, jobject thiz, jobject weakThiz, jobject jsurface,
367 jint maxImages, jint userFormat, jint userWidth, jint userHeight) {
368 status_t res;
369
370 ALOGV("%s: maxImages:%d", __FUNCTION__, maxImages);
371
372 sp<Surface> surface(android_view_Surface_getSurface(env, jsurface));
373 if (surface == NULL) {
374 jniThrowException(env,
375 "java/lang/IllegalArgumentException",
376 "The surface has been released");
377 return 0;
378 }
379 sp<IGraphicBufferProducer> bufferProducer = surface->getIGraphicBufferProducer();
380
381 jclass clazz = env->GetObjectClass(thiz);
382 if (clazz == NULL) {
383 jniThrowRuntimeException(env, "Can't find android/graphics/ImageWriter");
384 return 0;
385 }
386 sp<JNIImageWriterContext> ctx(new JNIImageWriterContext(env, weakThiz, clazz));
387
388 sp<Surface> producer = new Surface(bufferProducer, /*controlledByApp*/false);
389 ctx->setProducer(producer);
390 /**
391 * NATIVE_WINDOW_API_CPU isn't a good choice here, as it makes the bufferQueue not connectable
392 * after disconnect. MEDIA or CAMERA are treated the same internally. The producer listener
393 * will be cleared after disconnect call.
394 */
395 res = producer->connect(/*api*/NATIVE_WINDOW_API_CAMERA, /*listener*/ctx);
396 if (res != OK) {
397 ALOGE("%s: Connecting to surface producer interface failed: %s (%d)",
398 __FUNCTION__, strerror(-res), res);
399 jniThrowRuntimeException(env, "Failed to connect to native window");
400 return 0;
401 }
402
403 jlong nativeCtx = reinterpret_cast<jlong>(ctx.get());
404
405 // Get the dimension and format of the producer.
406 sp<ANativeWindow> anw = producer;
407 int32_t width, height, surfaceFormat;
408 if (userWidth < 0) {
409 if ((res = anw->query(anw.get(), NATIVE_WINDOW_WIDTH, &width)) != OK) {
410 ALOGE("%s: Query Surface width failed: %s (%d)", __FUNCTION__, strerror(-res), res);
411 jniThrowRuntimeException(env, "Failed to query Surface width");
412 return 0;
413 }
414 } else {
415 width = userWidth;
416 }
417
418 ctx->setBufferWidth(width);
419
420 if (userHeight < 0) {
421 if ((res = anw->query(anw.get(), NATIVE_WINDOW_HEIGHT, &height)) != OK) {
422 ALOGE("%s: Query Surface height failed: %s (%d)", __FUNCTION__, strerror(-res), res);
423 jniThrowRuntimeException(env, "Failed to query Surface height");
424 return 0;
425 }
426 } else {
427 height = userHeight;
428 }
429 ctx->setBufferHeight(height);
430
431 if ((userWidth > 0) && (userHeight > 0)) {
432 res = native_window_set_buffers_user_dimensions(anw.get(), userWidth, userHeight);
433 if (res != OK) {
434 ALOGE("%s: Set buffer dimensions failed: %s (%d)", __FUNCTION__, strerror(-res), res);
435 jniThrowRuntimeException(env, "Set buffer dimensions failed");
436 return 0;
437 }
438 }
439
440 // Query surface format if no valid user format is specified, otherwise, override surface format
441 // with user format.
442 if (userFormat == IMAGE_FORMAT_UNKNOWN) {
443 if ((res = anw->query(anw.get(), NATIVE_WINDOW_FORMAT, &surfaceFormat)) != OK) {
444 ALOGE("%s: Query Surface format failed: %s (%d)", __FUNCTION__, strerror(-res), res);
445 jniThrowRuntimeException(env, "Failed to query Surface format");
446 return 0;
447 }
448 } else {
449 // Set consumer buffer format to user specified format
450 PublicFormat publicFormat = static_cast<PublicFormat>(userFormat);
451 int nativeFormat = mapPublicFormatToHalFormat(publicFormat);
452 android_dataspace nativeDataspace = mapPublicFormatToHalDataspace(publicFormat);
453 res = native_window_set_buffers_format(anw.get(), nativeFormat);
454 if (res != OK) {
455 ALOGE("%s: Unable to configure consumer native buffer format to %#x",
456 __FUNCTION__, nativeFormat);
457 jniThrowRuntimeException(env, "Failed to set Surface format");
458 return 0;
459 }
460
461 res = native_window_set_buffers_data_space(anw.get(), nativeDataspace);
462 if (res != OK) {
463 ALOGE("%s: Unable to configure consumer dataspace %#x",
464 __FUNCTION__, nativeDataspace);
465 jniThrowRuntimeException(env, "Failed to set Surface dataspace");
466 return 0;
467 }
468 surfaceFormat = userFormat;
469 }
470
471 ctx->setBufferFormat(surfaceFormat);
472 env->SetIntField(thiz,
473 gImageWriterClassInfo.mWriterFormat, reinterpret_cast<jint>(surfaceFormat));
474
475 if (!isFormatOpaque(surfaceFormat)) {
476 res = native_window_set_usage(anw.get(), GRALLOC_USAGE_SW_WRITE_OFTEN);
477 if (res != OK) {
478 ALOGE("%s: Configure usage %08x for format %08x failed: %s (%d)",
479 __FUNCTION__, static_cast<unsigned int>(GRALLOC_USAGE_SW_WRITE_OFTEN),
480 surfaceFormat, strerror(-res), res);
481 jniThrowRuntimeException(env, "Failed to SW_WRITE_OFTEN configure usage");
482 return 0;
483 }
484 }
485
486 int minUndequeuedBufferCount = 0;
487 res = anw->query(anw.get(),
488 NATIVE_WINDOW_MIN_UNDEQUEUED_BUFFERS, &minUndequeuedBufferCount);
489 if (res != OK) {
490 ALOGE("%s: Query producer undequeued buffer count failed: %s (%d)",
491 __FUNCTION__, strerror(-res), res);
492 jniThrowRuntimeException(env, "Query producer undequeued buffer count failed");
493 return 0;
494 }
495
496 size_t totalBufferCount = maxImages + minUndequeuedBufferCount;
497 res = native_window_set_buffer_count(anw.get(), totalBufferCount);
498 if (res != OK) {
499 ALOGE("%s: Set buffer count failed: %s (%d)", __FUNCTION__, strerror(-res), res);
500 jniThrowRuntimeException(env, "Set buffer count failed");
501 return 0;
502 }
503
504 if (ctx != 0) {
505 ctx->incStrong((void*)ImageWriter_init);
506 }
507 return nativeCtx;
508 }
509
ImageWriter_dequeueImage(JNIEnv * env,jobject thiz,jlong nativeCtx,jobject image)510 static void ImageWriter_dequeueImage(JNIEnv* env, jobject thiz, jlong nativeCtx, jobject image) {
511 ALOGV("%s", __FUNCTION__);
512 JNIImageWriterContext* const ctx = reinterpret_cast<JNIImageWriterContext *>(nativeCtx);
513 if (ctx == NULL || thiz == NULL) {
514 jniThrowException(env, "java/lang/IllegalStateException",
515 "ImageWriterContext is not initialized");
516 return;
517 }
518
519 sp<ANativeWindow> anw = ctx->getProducer();
520 android_native_buffer_t *anb = NULL;
521 int fenceFd = -1;
522 status_t res = anw->dequeueBuffer(anw.get(), &anb, &fenceFd);
523 if (res != OK) {
524 ALOGE("%s: Dequeue buffer failed: %s (%d)", __FUNCTION__, strerror(-res), res);
525 switch (res) {
526 case NO_INIT:
527 jniThrowException(env, "java/lang/IllegalStateException",
528 "Surface has been abandoned");
529 break;
530 default:
531 // TODO: handle other error cases here.
532 jniThrowRuntimeException(env, "dequeue buffer failed");
533 }
534 return;
535 }
536 // New GraphicBuffer object doesn't own the handle, thus the native buffer
537 // won't be freed when this object is destroyed.
538 sp<GraphicBuffer> buffer(GraphicBuffer::from(anb));
539
540 // Note that:
541 // 1. No need to lock buffer now, will only lock it when the first getPlanes() is called.
542 // 2. Fence will be saved to mNativeFenceFd, and will consumed by lock/queue/cancel buffer
543 // later.
544 // 3. need use lockAsync here, as it will handle the dequeued fence for us automatically.
545
546 // Finally, set the native info into image object.
547 Image_setNativeContext(env, image, buffer, fenceFd);
548 }
549
ImageWriter_close(JNIEnv * env,jobject thiz,jlong nativeCtx)550 static void ImageWriter_close(JNIEnv* env, jobject thiz, jlong nativeCtx) {
551 ALOGV("%s:", __FUNCTION__);
552 JNIImageWriterContext* const ctx = reinterpret_cast<JNIImageWriterContext *>(nativeCtx);
553 if (ctx == NULL || thiz == NULL) {
554 // ImageWriter is already closed.
555 return;
556 }
557
558 ANativeWindow* producer = ctx->getProducer();
559 if (producer != NULL) {
560 /**
561 * NATIVE_WINDOW_API_CPU isn't a good choice here, as it makes the bufferQueue not
562 * connectable after disconnect. MEDIA or CAMERA are treated the same internally.
563 * The producer listener will be cleared after disconnect call.
564 */
565 status_t res = native_window_api_disconnect(producer, /*api*/NATIVE_WINDOW_API_CAMERA);
566 /**
567 * This is not an error. if client calling process dies, the window will
568 * also die and all calls to it will return DEAD_OBJECT, thus it's already
569 * "disconnected"
570 */
571 if (res == DEAD_OBJECT) {
572 ALOGW("%s: While disconnecting ImageWriter from native window, the"
573 " native window died already", __FUNCTION__);
574 } else if (res != OK) {
575 ALOGE("%s: native window disconnect failed: %s (%d)",
576 __FUNCTION__, strerror(-res), res);
577 jniThrowRuntimeException(env, "Native window disconnect failed");
578 return;
579 }
580 }
581
582 ctx->decStrong((void*)ImageWriter_init);
583 }
584
ImageWriter_cancelImage(JNIEnv * env,jobject thiz,jlong nativeCtx,jobject image)585 static void ImageWriter_cancelImage(JNIEnv* env, jobject thiz, jlong nativeCtx, jobject image) {
586 ALOGV("%s", __FUNCTION__);
587 JNIImageWriterContext* const ctx = reinterpret_cast<JNIImageWriterContext *>(nativeCtx);
588 if (ctx == NULL || thiz == NULL) {
589 ALOGW("ImageWriter#close called before Image#close, consider calling Image#close first");
590 return;
591 }
592
593 sp<ANativeWindow> anw = ctx->getProducer();
594
595 GraphicBuffer *buffer = NULL;
596 int fenceFd = -1;
597 Image_getNativeContext(env, image, &buffer, &fenceFd);
598 if (buffer == NULL) {
599 // Cancel an already cancelled image is harmless.
600 return;
601 }
602
603 // Unlock the image if it was locked
604 Image_unlockIfLocked(env, image);
605
606 anw->cancelBuffer(anw.get(), buffer, fenceFd);
607
608 Image_setNativeContext(env, image, NULL, -1);
609 }
610
ImageWriter_queueImage(JNIEnv * env,jobject thiz,jlong nativeCtx,jobject image,jlong timestampNs,jint left,jint top,jint right,jint bottom,jint transform,jint scalingMode)611 static void ImageWriter_queueImage(JNIEnv* env, jobject thiz, jlong nativeCtx, jobject image,
612 jlong timestampNs, jint left, jint top, jint right, jint bottom, jint transform,
613 jint scalingMode) {
614 ALOGV("%s", __FUNCTION__);
615 JNIImageWriterContext* const ctx = reinterpret_cast<JNIImageWriterContext *>(nativeCtx);
616 if (ctx == NULL || thiz == NULL) {
617 jniThrowException(env, "java/lang/IllegalStateException",
618 "ImageWriterContext is not initialized");
619 return;
620 }
621
622 status_t res = OK;
623 sp<ANativeWindow> anw = ctx->getProducer();
624
625 GraphicBuffer *buffer = NULL;
626 int fenceFd = -1;
627 Image_getNativeContext(env, image, &buffer, &fenceFd);
628 if (buffer == NULL) {
629 jniThrowException(env, "java/lang/IllegalStateException",
630 "Image is not initialized");
631 return;
632 }
633
634 // Unlock image if it was locked.
635 Image_unlockIfLocked(env, image);
636
637 // Set timestamp
638 ALOGV("timestamp to be queued: %" PRId64, timestampNs);
639 res = native_window_set_buffers_timestamp(anw.get(), timestampNs);
640 if (res != OK) {
641 jniThrowRuntimeException(env, "Set timestamp failed");
642 return;
643 }
644
645 // Set crop
646 android_native_rect_t cropRect;
647 cropRect.left = left;
648 cropRect.top = top;
649 cropRect.right = right;
650 cropRect.bottom = bottom;
651 res = native_window_set_crop(anw.get(), &cropRect);
652 if (res != OK) {
653 jniThrowRuntimeException(env, "Set crop rect failed");
654 return;
655 }
656
657 res = native_window_set_buffers_transform(anw.get(), transform);
658 if (res != OK) {
659 jniThrowRuntimeException(env, "Set transform failed");
660 return;
661 }
662
663 res = native_window_set_scaling_mode(anw.get(), scalingMode);
664 if (res != OK) {
665 jniThrowRuntimeException(env, "Set scaling mode failed");
666 return;
667 }
668
669 // Finally, queue input buffer.
670 //
671 // Because onBufferReleased may be called before queueBuffer() returns,
672 // queue the "attached" flag before calling queueBuffer. In case
673 // queueBuffer() fails, remove it from the queue.
674 ctx->queueAttachedFlag(false);
675 res = anw->queueBuffer(anw.get(), buffer, fenceFd);
676 if (res != OK) {
677 ALOGE("%s: Queue buffer failed: %s (%d)", __FUNCTION__, strerror(-res), res);
678 ctx->dequeueAttachedFlag();
679 switch (res) {
680 case NO_INIT:
681 jniThrowException(env, "java/lang/IllegalStateException",
682 "Surface has been abandoned");
683 break;
684 default:
685 // TODO: handle other error cases here.
686 jniThrowRuntimeException(env, "Queue input buffer failed");
687 }
688 return;
689 }
690
691 // Clear the image native context: end of this image's lifecycle in public API.
692 Image_setNativeContext(env, image, NULL, -1);
693 }
694
attachAndQeueuGraphicBuffer(JNIEnv * env,JNIImageWriterContext * ctx,sp<Surface> surface,sp<GraphicBuffer> gb,jlong timestampNs,jint left,jint top,jint right,jint bottom,jint transform,jint scalingMode)695 static status_t attachAndQeueuGraphicBuffer(JNIEnv* env, JNIImageWriterContext *ctx,
696 sp<Surface> surface, sp<GraphicBuffer> gb, jlong timestampNs, jint left, jint top,
697 jint right, jint bottom, jint transform, jint scalingMode) {
698 status_t res = OK;
699 // Step 1. Attach Image
700 res = surface->attachBuffer(gb.get());
701 if (res != OK) {
702 ALOGE("Attach image failed: %s (%d)", strerror(-res), res);
703 switch (res) {
704 case NO_INIT:
705 jniThrowException(env, "java/lang/IllegalStateException",
706 "Surface has been abandoned");
707 break;
708 default:
709 // TODO: handle other error cases here.
710 jniThrowRuntimeException(env, "nativeAttachImage failed!!!");
711 }
712 return res;
713 }
714 sp < ANativeWindow > anw = surface;
715
716 // Step 2. Set timestamp, crop, transform and scaling mode. Note that we do not need unlock the
717 // image because it was not locked.
718 ALOGV("timestamp to be queued: %" PRId64, timestampNs);
719 res = native_window_set_buffers_timestamp(anw.get(), timestampNs);
720 if (res != OK) {
721 jniThrowRuntimeException(env, "Set timestamp failed");
722 return res;
723 }
724
725 android_native_rect_t cropRect;
726 cropRect.left = left;
727 cropRect.top = top;
728 cropRect.right = right;
729 cropRect.bottom = bottom;
730 res = native_window_set_crop(anw.get(), &cropRect);
731 if (res != OK) {
732 jniThrowRuntimeException(env, "Set crop rect failed");
733 return res;
734 }
735
736 res = native_window_set_buffers_transform(anw.get(), transform);
737 if (res != OK) {
738 jniThrowRuntimeException(env, "Set transform failed");
739 return res;
740 }
741
742 res = native_window_set_scaling_mode(anw.get(), scalingMode);
743 if (res != OK) {
744 jniThrowRuntimeException(env, "Set scaling mode failed");
745 return res;
746 }
747
748 // Step 3. Queue Image.
749 //
750 // Because onBufferReleased may be called before queueBuffer() returns,
751 // queue the "attached" flag before calling queueBuffer. In case
752 // queueBuffer() fails, remove it from the queue.
753 ctx->queueAttachedFlag(true);
754 res = anw->queueBuffer(anw.get(), gb.get(), /*fenceFd*/
755 -1);
756 if (res != OK) {
757 ALOGE("%s: Queue buffer failed: %s (%d)", __FUNCTION__, strerror(-res), res);
758 ctx->dequeueAttachedFlag();
759 switch (res) {
760 case NO_INIT:
761 jniThrowException(env, "java/lang/IllegalStateException",
762 "Surface has been abandoned");
763 break;
764 default:
765 // TODO: handle other error cases here.
766 jniThrowRuntimeException(env, "Queue input buffer failed");
767 }
768 return res;
769 }
770
771 // Do not set the image native context. Since it would overwrite the existing native context
772 // of the image that is from ImageReader, the subsequent image close will run into issues.
773
774 return res;
775 }
776
ImageWriter_attachAndQueueImage(JNIEnv * env,jobject thiz,jlong nativeCtx,jlong nativeBuffer,jint imageFormat,jlong timestampNs,jint left,jint top,jint right,jint bottom,jint transform,jint scalingMode)777 static jint ImageWriter_attachAndQueueImage(JNIEnv* env, jobject thiz, jlong nativeCtx,
778 jlong nativeBuffer, jint imageFormat, jlong timestampNs, jint left, jint top,
779 jint right, jint bottom, jint transform, jint scalingMode) {
780 ALOGV("%s", __FUNCTION__);
781 JNIImageWriterContext* const ctx = reinterpret_cast<JNIImageWriterContext *>(nativeCtx);
782 if (ctx == NULL || thiz == NULL) {
783 jniThrowException(env, "java/lang/IllegalStateException",
784 "ImageWriterContext is not initialized");
785 return -1;
786 }
787
788 sp<Surface> surface = ctx->getProducer();
789 if (isFormatOpaque(imageFormat) != isFormatOpaque(ctx->getBufferFormat())) {
790 jniThrowException(env, "java/lang/IllegalStateException",
791 "Trying to attach an opaque image into a non-opaque ImageWriter, or vice versa");
792 return -1;
793 }
794
795 // Image is guaranteed to be from ImageReader at this point, so it is safe to
796 // cast to BufferItem pointer.
797 BufferItem* buffer = reinterpret_cast<BufferItem*>(nativeBuffer);
798 if (buffer == NULL) {
799 jniThrowException(env, "java/lang/IllegalStateException",
800 "Image is not initialized or already closed");
801 return -1;
802 }
803
804 return attachAndQeueuGraphicBuffer(env, ctx, surface, buffer->mGraphicBuffer, timestampNs, left,
805 top, right, bottom, transform, scalingMode);
806 }
807
ImageWriter_attachAndQueueGraphicBuffer(JNIEnv * env,jobject thiz,jlong nativeCtx,jobject buffer,jint format,jlong timestampNs,jint left,jint top,jint right,jint bottom,jint transform,jint scalingMode)808 static jint ImageWriter_attachAndQueueGraphicBuffer(JNIEnv* env, jobject thiz, jlong nativeCtx,
809 jobject buffer, jint format, jlong timestampNs, jint left, jint top,
810 jint right, jint bottom, jint transform, jint scalingMode) {
811 ALOGV("%s", __FUNCTION__);
812 JNIImageWriterContext* const ctx = reinterpret_cast<JNIImageWriterContext *>(nativeCtx);
813 if (ctx == NULL || thiz == NULL) {
814 jniThrowException(env, "java/lang/IllegalStateException",
815 "ImageWriterContext is not initialized");
816 return -1;
817 }
818
819 sp<Surface> surface = ctx->getProducer();
820 if (isFormatOpaque(format) != isFormatOpaque(ctx->getBufferFormat())) {
821 jniThrowException(env, "java/lang/IllegalStateException",
822 "Trying to attach an opaque image into a non-opaque ImageWriter, or vice versa");
823 return -1;
824 }
825
826 sp<GraphicBuffer> graphicBuffer = android_graphics_GraphicBuffer_getNativeGraphicsBuffer(env,
827 buffer);
828 if (graphicBuffer.get() == NULL) {
829 jniThrowException(env, "java/lang/IllegalArgumentException",
830 "Trying to attach an invalid graphic buffer");
831 return -1;
832 }
833
834 return attachAndQeueuGraphicBuffer(env, ctx, surface, graphicBuffer, timestampNs, left,
835 top, right, bottom, transform, scalingMode);
836 }
837
838 // --------------------------Image methods---------------------------------------
839
Image_getNativeContext(JNIEnv * env,jobject thiz,GraphicBuffer ** buffer,int * fenceFd)840 static void Image_getNativeContext(JNIEnv* env, jobject thiz,
841 GraphicBuffer** buffer, int* fenceFd) {
842 ALOGV("%s", __FUNCTION__);
843 if (buffer != NULL) {
844 GraphicBuffer *gb = reinterpret_cast<GraphicBuffer *>
845 (env->GetLongField(thiz, gSurfaceImageClassInfo.mNativeBuffer));
846 *buffer = gb;
847 }
848
849 if (fenceFd != NULL) {
850 *fenceFd = reinterpret_cast<jint>(env->GetIntField(
851 thiz, gSurfaceImageClassInfo.mNativeFenceFd));
852 }
853 }
854
Image_setNativeContext(JNIEnv * env,jobject thiz,sp<GraphicBuffer> buffer,int fenceFd)855 static void Image_setNativeContext(JNIEnv* env, jobject thiz,
856 sp<GraphicBuffer> buffer, int fenceFd) {
857 ALOGV("%s:", __FUNCTION__);
858 GraphicBuffer* p = NULL;
859 Image_getNativeContext(env, thiz, &p, /*fenceFd*/NULL);
860 if (buffer != 0) {
861 buffer->incStrong((void*)Image_setNativeContext);
862 }
863 if (p) {
864 p->decStrong((void*)Image_setNativeContext);
865 }
866 env->SetLongField(thiz, gSurfaceImageClassInfo.mNativeBuffer,
867 reinterpret_cast<jlong>(buffer.get()));
868
869 env->SetIntField(thiz, gSurfaceImageClassInfo.mNativeFenceFd, reinterpret_cast<jint>(fenceFd));
870 }
871
Image_unlockIfLocked(JNIEnv * env,jobject thiz)872 static void Image_unlockIfLocked(JNIEnv* env, jobject thiz) {
873 ALOGV("%s", __FUNCTION__);
874 GraphicBuffer* buffer;
875 Image_getNativeContext(env, thiz, &buffer, NULL);
876 if (buffer == NULL) {
877 jniThrowException(env, "java/lang/IllegalStateException",
878 "Image is not initialized");
879 return;
880 }
881
882 // Is locked?
883 bool isLocked = false;
884 jobject planes = NULL;
885 if (!isFormatOpaque(buffer->getPixelFormat())) {
886 planes = env->GetObjectField(thiz, gSurfaceImageClassInfo.mPlanes);
887 }
888 isLocked = (planes != NULL);
889 if (isLocked) {
890 // no need to use fence here, as we it will be consumed by either cancel or queue buffer.
891 status_t res = buffer->unlock();
892 if (res != OK) {
893 jniThrowRuntimeException(env, "unlock buffer failed");
894 return;
895 }
896 ALOGV("Successfully unlocked the image");
897 }
898 }
899
Image_getWidth(JNIEnv * env,jobject thiz)900 static jint Image_getWidth(JNIEnv* env, jobject thiz) {
901 ALOGV("%s", __FUNCTION__);
902 GraphicBuffer* buffer;
903 Image_getNativeContext(env, thiz, &buffer, NULL);
904 if (buffer == NULL) {
905 jniThrowException(env, "java/lang/IllegalStateException",
906 "Image is not initialized");
907 return -1;
908 }
909
910 return buffer->getWidth();
911 }
912
Image_getHeight(JNIEnv * env,jobject thiz)913 static jint Image_getHeight(JNIEnv* env, jobject thiz) {
914 ALOGV("%s", __FUNCTION__);
915 GraphicBuffer* buffer;
916 Image_getNativeContext(env, thiz, &buffer, NULL);
917 if (buffer == NULL) {
918 jniThrowException(env, "java/lang/IllegalStateException",
919 "Image is not initialized");
920 return -1;
921 }
922
923 return buffer->getHeight();
924 }
925
Image_getFormat(JNIEnv * env,jobject thiz)926 static jint Image_getFormat(JNIEnv* env, jobject thiz) {
927 ALOGV("%s", __FUNCTION__);
928 GraphicBuffer* buffer;
929 Image_getNativeContext(env, thiz, &buffer, NULL);
930 if (buffer == NULL) {
931 jniThrowException(env, "java/lang/IllegalStateException",
932 "Image is not initialized");
933 return 0;
934 }
935
936 // ImageWriter doesn't support data space yet, assuming it is unknown.
937 PublicFormat publicFmt = mapHalFormatDataspaceToPublicFormat(buffer->getPixelFormat(),
938 HAL_DATASPACE_UNKNOWN);
939 return static_cast<jint>(publicFmt);
940 }
941
Image_getHardwareBuffer(JNIEnv * env,jobject thiz)942 static jobject Image_getHardwareBuffer(JNIEnv* env, jobject thiz) {
943 GraphicBuffer* buffer;
944 Image_getNativeContext(env, thiz, &buffer, NULL);
945 if (buffer == NULL) {
946 jniThrowException(env, "java/lang/IllegalStateException",
947 "Image is not initialized");
948 return NULL;
949 }
950 AHardwareBuffer* b = AHardwareBuffer_from_GraphicBuffer(buffer);
951 // don't user the public AHardwareBuffer_toHardwareBuffer() because this would force us
952 // to link against libandroid.so
953 return android_hardware_HardwareBuffer_createFromAHardwareBuffer(env, b);
954 }
955
Image_setFenceFd(JNIEnv * env,jobject thiz,int fenceFd)956 static void Image_setFenceFd(JNIEnv* env, jobject thiz, int fenceFd) {
957 ALOGV("%s:", __FUNCTION__);
958 env->SetIntField(thiz, gSurfaceImageClassInfo.mNativeFenceFd, reinterpret_cast<jint>(fenceFd));
959 }
960
Image_getLockedImage(JNIEnv * env,jobject thiz,LockedImage * image)961 static void Image_getLockedImage(JNIEnv* env, jobject thiz, LockedImage *image) {
962 ALOGV("%s", __FUNCTION__);
963 GraphicBuffer* buffer;
964 int fenceFd = -1;
965 Image_getNativeContext(env, thiz, &buffer, &fenceFd);
966 if (buffer == NULL) {
967 jniThrowException(env, "java/lang/IllegalStateException",
968 "Image is not initialized");
969 return;
970 }
971
972 // ImageWriter doesn't use crop by itself, app sets it, use the no crop version.
973 const Rect noCrop(buffer->width, buffer->height);
974 status_t res = lockImageFromBuffer(
975 buffer, GRALLOC_USAGE_SW_WRITE_OFTEN, noCrop, fenceFd, image);
976 // Clear the fenceFd as it is already consumed by lock call.
977 Image_setFenceFd(env, thiz, /*fenceFd*/-1);
978 if (res != OK) {
979 jniThrowExceptionFmt(env, "java/lang/RuntimeException",
980 "lock buffer failed for format 0x%x",
981 buffer->getPixelFormat());
982 return;
983 }
984
985 ALOGV("%s: Successfully locked the image", __FUNCTION__);
986 // crop, transform, scalingMode, timestamp, and frameNumber should be set by producer,
987 // and we don't set them here.
988 }
989
Image_getLockedImageInfo(JNIEnv * env,LockedImage * buffer,int idx,int32_t writerFormat,uint8_t ** base,uint32_t * size,int * pixelStride,int * rowStride)990 static bool Image_getLockedImageInfo(JNIEnv* env, LockedImage* buffer, int idx,
991 int32_t writerFormat, uint8_t **base, uint32_t *size, int *pixelStride, int *rowStride) {
992 ALOGV("%s", __FUNCTION__);
993
994 status_t res = getLockedImageInfo(buffer, idx, writerFormat, base, size,
995 pixelStride, rowStride);
996 if (res != OK) {
997 jniThrowExceptionFmt(env, "java/lang/UnsupportedOperationException",
998 "Pixel format: 0x%x is unsupported", writerFormat);
999 return false;
1000 }
1001 return true;
1002 }
1003
Image_createSurfacePlanes(JNIEnv * env,jobject thiz,int numPlanes,int writerFormat)1004 static jobjectArray Image_createSurfacePlanes(JNIEnv* env, jobject thiz,
1005 int numPlanes, int writerFormat) {
1006 ALOGV("%s: create SurfacePlane array with size %d", __FUNCTION__, numPlanes);
1007 int rowStride, pixelStride;
1008 uint8_t *pData;
1009 uint32_t dataSize;
1010 jobject byteBuffer;
1011
1012 int format = Image_getFormat(env, thiz);
1013 if (isFormatOpaque(format) && numPlanes > 0) {
1014 String8 msg;
1015 msg.appendFormat("Format 0x%x is opaque, thus not writable, the number of planes (%d)"
1016 " must be 0", format, numPlanes);
1017 jniThrowException(env, "java/lang/IllegalArgumentException", msg.string());
1018 return NULL;
1019 }
1020
1021 jobjectArray surfacePlanes = env->NewObjectArray(numPlanes, gSurfacePlaneClassInfo.clazz,
1022 /*initial_element*/NULL);
1023 if (surfacePlanes == NULL) {
1024 jniThrowRuntimeException(env, "Failed to create SurfacePlane arrays,"
1025 " probably out of memory");
1026 return NULL;
1027 }
1028 if (isFormatOpaque(format)) {
1029 return surfacePlanes;
1030 }
1031
1032 // Buildup buffer info: rowStride, pixelStride and byteBuffers.
1033 LockedImage lockedImg = LockedImage();
1034 Image_getLockedImage(env, thiz, &lockedImg);
1035
1036 // Create all SurfacePlanes
1037 PublicFormat publicWriterFormat = static_cast<PublicFormat>(writerFormat);
1038 writerFormat = mapPublicFormatToHalFormat(publicWriterFormat);
1039 for (int i = 0; i < numPlanes; i++) {
1040 if (!Image_getLockedImageInfo(env, &lockedImg, i, writerFormat,
1041 &pData, &dataSize, &pixelStride, &rowStride)) {
1042 return NULL;
1043 }
1044 byteBuffer = env->NewDirectByteBuffer(pData, dataSize);
1045 if ((byteBuffer == NULL) && (env->ExceptionCheck() == false)) {
1046 jniThrowException(env, "java/lang/IllegalStateException",
1047 "Failed to allocate ByteBuffer");
1048 return NULL;
1049 }
1050
1051 // Finally, create this SurfacePlane.
1052 jobject surfacePlane = env->NewObject(gSurfacePlaneClassInfo.clazz,
1053 gSurfacePlaneClassInfo.ctor, thiz, rowStride, pixelStride, byteBuffer);
1054 env->SetObjectArrayElement(surfacePlanes, i, surfacePlane);
1055 }
1056
1057 return surfacePlanes;
1058 }
1059
1060 } // extern "C"
1061
1062 // ----------------------------------------------------------------------------
1063
1064 static JNINativeMethod gImageWriterMethods[] = {
1065 {"nativeClassInit", "()V", (void*)ImageWriter_classInit },
1066 {"nativeInit", "(Ljava/lang/Object;Landroid/view/Surface;IIII)J",
1067 (void*)ImageWriter_init },
1068 {"nativeClose", "(J)V", (void*)ImageWriter_close },
1069 {"nativeAttachAndQueueImage", "(JJIJIIIIII)I", (void*)ImageWriter_attachAndQueueImage },
1070 {"nativeAttachAndQueueGraphicBuffer",
1071 "(JLandroid/graphics/GraphicBuffer;IJIIIIII)I",
1072 (void*)ImageWriter_attachAndQueueGraphicBuffer },
1073 {"nativeDequeueInputImage", "(JLandroid/media/Image;)V", (void*)ImageWriter_dequeueImage },
1074 {"nativeQueueInputImage", "(JLandroid/media/Image;JIIIIII)V", (void*)ImageWriter_queueImage },
1075 {"cancelImage", "(JLandroid/media/Image;)V", (void*)ImageWriter_cancelImage },
1076 };
1077
1078 static JNINativeMethod gImageMethods[] = {
1079 {"nativeCreatePlanes", "(II)[Landroid/media/ImageWriter$WriterSurfaceImage$SurfacePlane;",
1080 (void*)Image_createSurfacePlanes },
1081 {"nativeGetWidth", "()I", (void*)Image_getWidth },
1082 {"nativeGetHeight", "()I", (void*)Image_getHeight },
1083 {"nativeGetFormat", "()I", (void*)Image_getFormat },
1084 {"nativeGetHardwareBuffer", "()Landroid/hardware/HardwareBuffer;",
1085 (void*)Image_getHardwareBuffer },
1086 };
1087
register_android_media_ImageWriter(JNIEnv * env)1088 int register_android_media_ImageWriter(JNIEnv *env) {
1089
1090 int ret1 = AndroidRuntime::registerNativeMethods(env,
1091 "android/media/ImageWriter", gImageWriterMethods, NELEM(gImageWriterMethods));
1092
1093 int ret2 = AndroidRuntime::registerNativeMethods(env,
1094 "android/media/ImageWriter$WriterSurfaceImage", gImageMethods, NELEM(gImageMethods));
1095
1096 return (ret1 || ret2);
1097 }
1098