1 /*
2 * Copyright (C) 2016 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include <inttypes.h>
18
19 //#define LOG_NDEBUG 0
20 #define LOG_TAG "NdkImageReader"
21
22 #include "NdkImagePriv.h"
23 #include "NdkImageReaderPriv.h"
24
25 #include <utils/Log.h>
26 #include <android_runtime/android_view_Surface.h>
27
28 using namespace android;
29
30 namespace {
31 // Get an ID that's unique within this process.
createProcessUniqueId()32 static int32_t createProcessUniqueId() {
33 static volatile int32_t globalCounter = 0;
34 return android_atomic_inc(&globalCounter);
35 }
36 }
37
38 const char* AImageReader::kCallbackFpKey = "Callback";
39 const char* AImageReader::kContextKey = "Context";
40
41 bool
isSupportedFormat(int32_t format)42 AImageReader::isSupportedFormat(int32_t format) {
43 switch (format) {
44 case AIMAGE_FORMAT_YUV_420_888:
45 case AIMAGE_FORMAT_JPEG:
46 case AIMAGE_FORMAT_RAW16:
47 case AIMAGE_FORMAT_RAW_PRIVATE:
48 case AIMAGE_FORMAT_RAW10:
49 case AIMAGE_FORMAT_RAW12:
50 case AIMAGE_FORMAT_DEPTH16:
51 case AIMAGE_FORMAT_DEPTH_POINT_CLOUD:
52 return true;
53 default:
54 return false;
55 }
56 }
57
58 int
getNumPlanesForFormat(int32_t format)59 AImageReader::getNumPlanesForFormat(int32_t format) {
60 switch (format) {
61 case AIMAGE_FORMAT_YUV_420_888:
62 return 3;
63 case AIMAGE_FORMAT_JPEG:
64 case AIMAGE_FORMAT_RAW16:
65 case AIMAGE_FORMAT_RAW_PRIVATE:
66 case AIMAGE_FORMAT_RAW10:
67 case AIMAGE_FORMAT_RAW12:
68 case AIMAGE_FORMAT_DEPTH16:
69 case AIMAGE_FORMAT_DEPTH_POINT_CLOUD:
70 return 1;
71 default:
72 return -1;
73 }
74 }
75
76 void
onFrameAvailable(const BufferItem &)77 AImageReader::FrameListener::onFrameAvailable(const BufferItem& /*item*/) {
78 Mutex::Autolock _l(mLock);
79 sp<AImageReader> reader = mReader.promote();
80 if (reader == nullptr) {
81 ALOGW("A frame is available after AImageReader closed!");
82 return; // reader has been closed
83 }
84 if (mListener.onImageAvailable == nullptr) {
85 return; // No callback registered
86 }
87
88 sp<AMessage> msg = new AMessage(AImageReader::kWhatImageAvailable, reader->mHandler);
89 msg->setPointer(AImageReader::kCallbackFpKey, (void *) mListener.onImageAvailable);
90 msg->setPointer(AImageReader::kContextKey, mListener.context);
91 msg->post();
92 }
93
94 media_status_t
setImageListener(AImageReader_ImageListener * listener)95 AImageReader::FrameListener::setImageListener(AImageReader_ImageListener* listener) {
96 Mutex::Autolock _l(mLock);
97 if (listener == nullptr) {
98 mListener.context = nullptr;
99 mListener.onImageAvailable = nullptr;
100 } else {
101 mListener = *listener;
102 }
103 return AMEDIA_OK;
104 }
105
106 media_status_t
setImageListenerLocked(AImageReader_ImageListener * listener)107 AImageReader::setImageListenerLocked(AImageReader_ImageListener* listener) {
108 return mFrameListener->setImageListener(listener);
109 }
110
111 media_status_t
setImageListener(AImageReader_ImageListener * listener)112 AImageReader::setImageListener(AImageReader_ImageListener* listener) {
113 Mutex::Autolock _l(mLock);
114 return setImageListenerLocked(listener);
115 }
116
onMessageReceived(const sp<AMessage> & msg)117 void AImageReader::CallbackHandler::onMessageReceived(
118 const sp<AMessage> &msg) {
119 switch (msg->what()) {
120 case kWhatImageAvailable:
121 {
122 AImageReader_ImageCallback onImageAvailable;
123 void* context;
124 bool found = msg->findPointer(kCallbackFpKey, (void**) &onImageAvailable);
125 if (!found || onImageAvailable == nullptr) {
126 ALOGE("%s: Cannot find onImageAvailable callback fp!", __FUNCTION__);
127 return;
128 }
129 found = msg->findPointer(kContextKey, &context);
130 if (!found) {
131 ALOGE("%s: Cannot find callback context!", __FUNCTION__);
132 return;
133 }
134 (*onImageAvailable)(context, mReader);
135 break;
136 }
137 default:
138 ALOGE("%s: unknown message type %d", __FUNCTION__, msg->what());
139 break;
140 }
141 }
142
AImageReader(int32_t width,int32_t height,int32_t format,int32_t maxImages)143 AImageReader::AImageReader(int32_t width, int32_t height, int32_t format, int32_t maxImages) :
144 mWidth(width), mHeight(height), mFormat(format), mMaxImages(maxImages),
145 mNumPlanes(getNumPlanesForFormat(format)),
146 mFrameListener(new FrameListener(this)) {}
147
148 media_status_t
init()149 AImageReader::init() {
150 PublicFormat publicFormat = static_cast<PublicFormat>(mFormat);
151 mHalFormat = android_view_Surface_mapPublicFormatToHalFormat(publicFormat);
152 mHalDataSpace = android_view_Surface_mapPublicFormatToHalDataspace(publicFormat);
153
154 sp<IGraphicBufferProducer> gbProducer;
155 sp<IGraphicBufferConsumer> gbConsumer;
156 BufferQueue::createBufferQueue(&gbProducer, &gbConsumer);
157
158 sp<CpuConsumer> cpuConsumer;
159 String8 consumerName = String8::format("ImageReader-%dx%df%xm%d-%d-%d",
160 mWidth, mHeight, mFormat, mMaxImages, getpid(),
161 createProcessUniqueId());
162
163 cpuConsumer = new CpuConsumer(gbConsumer, mMaxImages, /*controlledByApp*/true);
164 if (cpuConsumer == nullptr) {
165 ALOGE("Failed to allocate CpuConsumer");
166 return AMEDIA_ERROR_UNKNOWN;
167 }
168
169 mCpuConsumer = cpuConsumer;
170 mCpuConsumer->setName(consumerName);
171 mProducer = gbProducer;
172
173 sp<ConsumerBase> consumer = cpuConsumer;
174 consumer->setFrameAvailableListener(mFrameListener);
175
176 status_t res;
177 res = cpuConsumer->setDefaultBufferSize(mWidth, mHeight);
178 if (res != OK) {
179 ALOGE("Failed to set CpuConsumer buffer size");
180 return AMEDIA_ERROR_UNKNOWN;
181 }
182 res = cpuConsumer->setDefaultBufferFormat(mHalFormat);
183 if (res != OK) {
184 ALOGE("Failed to set CpuConsumer buffer format");
185 return AMEDIA_ERROR_UNKNOWN;
186 }
187 res = cpuConsumer->setDefaultBufferDataSpace(mHalDataSpace);
188 if (res != OK) {
189 ALOGE("Failed to set CpuConsumer buffer dataSpace");
190 return AMEDIA_ERROR_UNKNOWN;
191 }
192
193 mSurface = new Surface(mProducer, /*controlledByApp*/true);
194 if (mSurface == nullptr) {
195 ALOGE("Failed to create surface");
196 return AMEDIA_ERROR_UNKNOWN;
197 }
198 mWindow = static_cast<ANativeWindow*>(mSurface.get());
199
200 for (int i = 0; i < mMaxImages; i++) {
201 CpuConsumer::LockedBuffer* buffer = new CpuConsumer::LockedBuffer;
202 mBuffers.push_back(buffer);
203 }
204
205 mCbLooper = new ALooper;
206 mCbLooper->setName(consumerName.string());
207 res = mCbLooper->start(
208 /*runOnCallingThread*/false,
209 /*canCallJava*/ true,
210 PRIORITY_DEFAULT);
211 if (res != OK) {
212 ALOGE("Failed to start the looper");
213 return AMEDIA_ERROR_UNKNOWN;
214 }
215 mHandler = new CallbackHandler(this);
216 mCbLooper->registerHandler(mHandler);
217
218 return AMEDIA_OK;
219 }
220
~AImageReader()221 AImageReader::~AImageReader() {
222 Mutex::Autolock _l(mLock);
223 AImageReader_ImageListener nullListener = {nullptr, nullptr};
224 setImageListenerLocked(&nullListener);
225
226 if (mCbLooper != nullptr) {
227 mCbLooper->unregisterHandler(mHandler->id());
228 mCbLooper->stop();
229 }
230 mCbLooper.clear();
231 mHandler.clear();
232
233 // Close all previously acquired images
234 for (auto it = mAcquiredImages.begin();
235 it != mAcquiredImages.end(); it++) {
236 AImage* image = *it;
237 image->close();
238 }
239
240 // Delete LockedBuffers
241 for (auto it = mBuffers.begin();
242 it != mBuffers.end(); it++) {
243 delete *it;
244 }
245
246 if (mCpuConsumer != nullptr) {
247 mCpuConsumer->abandon();
248 mCpuConsumer->setFrameAvailableListener(nullptr);
249 }
250 }
251
252 media_status_t
acquireCpuConsumerImageLocked(AImage ** image)253 AImageReader::acquireCpuConsumerImageLocked(/*out*/AImage** image) {
254 *image = nullptr;
255 CpuConsumer::LockedBuffer* buffer = getLockedBufferLocked();
256 if (buffer == nullptr) {
257 ALOGW("Unable to acquire a lockedBuffer, very likely client tries to lock more than"
258 " maxImages buffers");
259 return AMEDIA_IMGREADER_MAX_IMAGES_ACQUIRED;
260 }
261
262 status_t res = mCpuConsumer->lockNextBuffer(buffer);
263 if (res != NO_ERROR) {
264 returnLockedBufferLocked(buffer);
265 if (res != BAD_VALUE /*no buffers*/) {
266 if (res == NOT_ENOUGH_DATA) {
267 return AMEDIA_IMGREADER_MAX_IMAGES_ACQUIRED;
268 } else {
269 ALOGE("%s Fail to lockNextBuffer with error: %d ",
270 __FUNCTION__, res);
271 return AMEDIA_ERROR_UNKNOWN;
272 }
273 }
274 return AMEDIA_IMGREADER_NO_BUFFER_AVAILABLE;
275 }
276
277 if (buffer->flexFormat == HAL_PIXEL_FORMAT_YCrCb_420_SP) {
278 ALOGE("NV21 format is not supported by AImageReader");
279 return AMEDIA_ERROR_UNSUPPORTED;
280 }
281
282 // Check if the left-top corner of the crop rect is origin, we currently assume this point is
283 // zero, will revist this once this assumption turns out problematic.
284 Point lt = buffer->crop.leftTop();
285 if (lt.x != 0 || lt.y != 0) {
286 ALOGE("crop left top corner [%d, %d] need to be at origin", lt.x, lt.y);
287 return AMEDIA_ERROR_UNKNOWN;
288 }
289
290 // Check if the producer buffer configurations match what ImageReader configured.
291 int outputWidth = getBufferWidth(buffer);
292 int outputHeight = getBufferHeight(buffer);
293
294 int readerFmt = mHalFormat;
295 int readerWidth = mWidth;
296 int readerHeight = mHeight;
297
298 if ((buffer->format != HAL_PIXEL_FORMAT_BLOB) && (readerFmt != HAL_PIXEL_FORMAT_BLOB) &&
299 (readerWidth != outputWidth || readerHeight != outputHeight)) {
300 ALOGW("%s: Producer buffer size: %dx%d, doesn't match AImageReader configured size: %dx%d",
301 __FUNCTION__, outputWidth, outputHeight, readerWidth, readerHeight);
302 }
303
304 int bufFmt = buffer->format;
305 if (readerFmt == HAL_PIXEL_FORMAT_YCbCr_420_888) {
306 bufFmt = buffer->flexFormat;
307 }
308
309 if (readerFmt != bufFmt) {
310 if (readerFmt == HAL_PIXEL_FORMAT_YCbCr_420_888 && (bufFmt ==
311 HAL_PIXEL_FORMAT_YCrCb_420_SP || bufFmt == HAL_PIXEL_FORMAT_YV12)) {
312 // Special casing for when producer switches to a format compatible with flexible YUV
313 // (HAL_PIXEL_FORMAT_YCbCr_420_888).
314 mHalFormat = bufFmt;
315 ALOGD("%s: Overriding buffer format YUV_420_888 to %x.", __FUNCTION__, bufFmt);
316 } else {
317 // Return the buffer to the queue.
318 mCpuConsumer->unlockBuffer(*buffer);
319 returnLockedBufferLocked(buffer);
320
321 ALOGE("Producer output buffer format: 0x%x, ImageReader configured format: 0x%x",
322 buffer->format, readerFmt);
323
324 return AMEDIA_ERROR_UNKNOWN;
325 }
326 }
327
328 if (mHalFormat == HAL_PIXEL_FORMAT_BLOB) {
329 *image = new AImage(this, mFormat, buffer, buffer->timestamp,
330 readerWidth, readerHeight, mNumPlanes);
331 } else {
332 *image = new AImage(this, mFormat, buffer, buffer->timestamp,
333 outputWidth, outputHeight, mNumPlanes);
334 }
335 mAcquiredImages.push_back(*image);
336 return AMEDIA_OK;
337 }
338
339 CpuConsumer::LockedBuffer*
getLockedBufferLocked()340 AImageReader::getLockedBufferLocked() {
341 if (mBuffers.empty()) {
342 return nullptr;
343 }
344 // Return a LockedBuffer pointer and remove it from the list
345 auto it = mBuffers.begin();
346 CpuConsumer::LockedBuffer* buffer = *it;
347 mBuffers.erase(it);
348 return buffer;
349 }
350
351 void
returnLockedBufferLocked(CpuConsumer::LockedBuffer * buffer)352 AImageReader::returnLockedBufferLocked(CpuConsumer::LockedBuffer* buffer) {
353 mBuffers.push_back(buffer);
354 }
355
356 void
releaseImageLocked(AImage * image)357 AImageReader::releaseImageLocked(AImage* image) {
358 CpuConsumer::LockedBuffer* buffer = image->mBuffer;
359 if (buffer == nullptr) {
360 // This should not happen, but is not fatal
361 ALOGW("AImage %p has no buffer!", image);
362 return;
363 }
364
365 mCpuConsumer->unlockBuffer(*buffer);
366 returnLockedBufferLocked(buffer);
367 image->mBuffer = nullptr;
368
369 bool found = false;
370 // cleanup acquired image list
371 for (auto it = mAcquiredImages.begin();
372 it != mAcquiredImages.end(); it++) {
373 AImage* readerCopy = *it;
374 if (readerCopy == image) {
375 found = true;
376 mAcquiredImages.erase(it);
377 break;
378 }
379 }
380 if (!found) {
381 ALOGE("Error: AImage %p is not generated by AImageReader %p",
382 image, this);
383 }
384 }
385
386 int
getBufferWidth(CpuConsumer::LockedBuffer * buffer)387 AImageReader::getBufferWidth(CpuConsumer::LockedBuffer* buffer) {
388 if (buffer == nullptr) return -1;
389
390 if (!buffer->crop.isEmpty()) {
391 return buffer->crop.getWidth();
392 }
393 return buffer->width;
394 }
395
396 int
getBufferHeight(CpuConsumer::LockedBuffer * buffer)397 AImageReader::getBufferHeight(CpuConsumer::LockedBuffer* buffer) {
398 if (buffer == nullptr) return -1;
399
400 if (!buffer->crop.isEmpty()) {
401 return buffer->crop.getHeight();
402 }
403 return buffer->height;
404 }
405
406 media_status_t
acquireNextImage(AImage ** image)407 AImageReader::acquireNextImage(/*out*/AImage** image) {
408 Mutex::Autolock _l(mLock);
409 return acquireCpuConsumerImageLocked(image);
410 }
411
412 media_status_t
acquireLatestImage(AImage ** image)413 AImageReader::acquireLatestImage(/*out*/AImage** image) {
414 if (image == nullptr) {
415 return AMEDIA_ERROR_INVALID_PARAMETER;
416 }
417 Mutex::Autolock _l(mLock);
418 *image = nullptr;
419 AImage* prevImage = nullptr;
420 AImage* nextImage = nullptr;
421 media_status_t ret = acquireCpuConsumerImageLocked(&prevImage);
422 if (prevImage == nullptr) {
423 return ret;
424 }
425 for (;;) {
426 ret = acquireCpuConsumerImageLocked(&nextImage);
427 if (nextImage == nullptr) {
428 *image = prevImage;
429 return AMEDIA_OK;
430 }
431 prevImage->close();
432 prevImage->free();
433 prevImage = nextImage;
434 nextImage = nullptr;
435 }
436 }
437
438 EXPORT
AImageReader_new(int32_t width,int32_t height,int32_t format,int32_t maxImages,AImageReader ** reader)439 media_status_t AImageReader_new(
440 int32_t width, int32_t height, int32_t format, int32_t maxImages,
441 /*out*/AImageReader** reader) {
442 ALOGV("%s", __FUNCTION__);
443
444 if (width < 1 || height < 1) {
445 ALOGE("%s: image dimension must be positive: w:%d h:%d",
446 __FUNCTION__, width, height);
447 return AMEDIA_ERROR_INVALID_PARAMETER;
448 }
449
450 if (maxImages < 1) {
451 ALOGE("%s: max outstanding image count must be at least 1 (%d)",
452 __FUNCTION__, maxImages);
453 return AMEDIA_ERROR_INVALID_PARAMETER;
454 }
455
456 if (!AImageReader::isSupportedFormat(format)) {
457 ALOGE("%s: format %d is not supported by AImageReader",
458 __FUNCTION__, format);
459 return AMEDIA_ERROR_INVALID_PARAMETER;
460 }
461
462 if (reader == nullptr) {
463 ALOGE("%s: reader argument is null", __FUNCTION__);
464 return AMEDIA_ERROR_INVALID_PARAMETER;
465 }
466
467 //*reader = new AImageReader(width, height, format, maxImages);
468 AImageReader* tmpReader = new AImageReader(width, height, format, maxImages);
469 if (tmpReader == nullptr) {
470 ALOGE("%s: AImageReader allocation failed", __FUNCTION__);
471 return AMEDIA_ERROR_UNKNOWN;
472 }
473 media_status_t ret = tmpReader->init();
474 if (ret != AMEDIA_OK) {
475 ALOGE("%s: AImageReader initialization failed!", __FUNCTION__);
476 delete tmpReader;
477 return ret;
478 }
479 *reader = tmpReader;
480 (*reader)->incStrong((void*) AImageReader_new);
481 return AMEDIA_OK;
482 }
483
484 EXPORT
AImageReader_delete(AImageReader * reader)485 void AImageReader_delete(AImageReader* reader) {
486 ALOGV("%s", __FUNCTION__);
487 if (reader != nullptr) {
488 reader->decStrong((void*) AImageReader_delete);
489 }
490 return;
491 }
492
493 EXPORT
AImageReader_getWindow(AImageReader * reader,ANativeWindow ** window)494 media_status_t AImageReader_getWindow(AImageReader* reader, /*out*/ANativeWindow** window) {
495 ALOGE("%s", __FUNCTION__);
496 if (reader == nullptr || window == nullptr) {
497 ALOGE("%s: invalid argument. reader %p, window %p",
498 __FUNCTION__, reader, window);
499 return AMEDIA_ERROR_INVALID_PARAMETER;
500 }
501 *window = reader->getWindow();
502 return AMEDIA_OK;
503 }
504
505 EXPORT
AImageReader_getWidth(const AImageReader * reader,int32_t * width)506 media_status_t AImageReader_getWidth(const AImageReader* reader, /*out*/int32_t* width) {
507 ALOGV("%s", __FUNCTION__);
508 if (reader == nullptr || width == nullptr) {
509 ALOGE("%s: invalid argument. reader %p, width %p",
510 __FUNCTION__, reader, width);
511 return AMEDIA_ERROR_INVALID_PARAMETER;
512 }
513 *width = reader->getWidth();
514 return AMEDIA_OK;
515 }
516
517 EXPORT
AImageReader_getHeight(const AImageReader * reader,int32_t * height)518 media_status_t AImageReader_getHeight(const AImageReader* reader, /*out*/int32_t* height) {
519 ALOGV("%s", __FUNCTION__);
520 if (reader == nullptr || height == nullptr) {
521 ALOGE("%s: invalid argument. reader %p, height %p",
522 __FUNCTION__, reader, height);
523 return AMEDIA_ERROR_INVALID_PARAMETER;
524 }
525 *height = reader->getHeight();
526 return AMEDIA_OK;
527 }
528
529 EXPORT
AImageReader_getFormat(const AImageReader * reader,int32_t * format)530 media_status_t AImageReader_getFormat(const AImageReader* reader, /*out*/int32_t* format) {
531 ALOGV("%s", __FUNCTION__);
532 if (reader == nullptr || format == nullptr) {
533 ALOGE("%s: invalid argument. reader %p, format %p",
534 __FUNCTION__, reader, format);
535 return AMEDIA_ERROR_INVALID_PARAMETER;
536 }
537 *format = reader->getFormat();
538 return AMEDIA_OK;
539 }
540
541 EXPORT
AImageReader_getMaxImages(const AImageReader * reader,int32_t * maxImages)542 media_status_t AImageReader_getMaxImages(const AImageReader* reader, /*out*/int32_t* maxImages) {
543 ALOGV("%s", __FUNCTION__);
544 if (reader == nullptr || maxImages == nullptr) {
545 ALOGE("%s: invalid argument. reader %p, maxImages %p",
546 __FUNCTION__, reader, maxImages);
547 return AMEDIA_ERROR_INVALID_PARAMETER;
548 }
549 *maxImages = reader->getMaxImages();
550 return AMEDIA_OK;
551 }
552
553 EXPORT
AImageReader_acquireNextImage(AImageReader * reader,AImage ** image)554 media_status_t AImageReader_acquireNextImage(AImageReader* reader, /*out*/AImage** image) {
555 ALOGV("%s", __FUNCTION__);
556 if (reader == nullptr || image == nullptr) {
557 ALOGE("%s: invalid argument. reader %p, maxImages %p",
558 __FUNCTION__, reader, image);
559 return AMEDIA_ERROR_INVALID_PARAMETER;
560 }
561 return reader->acquireNextImage(image);
562 }
563
564 EXPORT
AImageReader_acquireLatestImage(AImageReader * reader,AImage ** image)565 media_status_t AImageReader_acquireLatestImage(AImageReader* reader, /*out*/AImage** image) {
566 ALOGV("%s", __FUNCTION__);
567 if (reader == nullptr || image == nullptr) {
568 ALOGE("%s: invalid argument. reader %p, maxImages %p",
569 __FUNCTION__, reader, image);
570 return AMEDIA_ERROR_INVALID_PARAMETER;
571 }
572 return reader->acquireLatestImage(image);
573 }
574
575 EXPORT
AImageReader_setImageListener(AImageReader * reader,AImageReader_ImageListener * listener)576 media_status_t AImageReader_setImageListener(
577 AImageReader* reader, AImageReader_ImageListener* listener) {
578 ALOGV("%s", __FUNCTION__);
579 if (reader == nullptr) {
580 ALOGE("%s: invalid argument! reader %p", __FUNCTION__, reader);
581 return AMEDIA_ERROR_INVALID_PARAMETER;
582 }
583
584 reader->setImageListener(listener);
585 return AMEDIA_OK;
586 }
587