1 /*
2 * Copyright (C) 2012-2018 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #define LOG_TAG "Camera2-CallbackProcessor"
18 #define ATRACE_TAG ATRACE_TAG_CAMERA
19 //#define LOG_NDEBUG 0
20
21 #include <utils/Log.h>
22 #include <utils/Trace.h>
23 #include <gui/Surface.h>
24
25 #include "common/CameraDeviceBase.h"
26 #include "api1/Camera2Client.h"
27 #include "api1/client2/CallbackProcessor.h"
28
29 #define ALIGN(x, mask) ( ((x) + (mask) - 1) & ~((mask) - 1) )
30
31 namespace android {
32 namespace camera2 {
33
34 using android::camera3::CAMERA_STREAM_ROTATION_0;
35
CallbackProcessor(sp<Camera2Client> client)36 CallbackProcessor::CallbackProcessor(sp<Camera2Client> client):
37 Thread(false),
38 mClient(client),
39 mDevice(client->getCameraDevice()),
40 mId(client->getCameraId()),
41 mCallbackAvailable(false),
42 mCallbackPaused(true),
43 mCallbackToApp(false),
44 mCallbackStreamId(NO_STREAM) {
45 }
46
~CallbackProcessor()47 CallbackProcessor::~CallbackProcessor() {
48 ALOGV("%s: Exit", __FUNCTION__);
49 deleteStream();
50 }
51
onFrameAvailable(const BufferItem &)52 void CallbackProcessor::onFrameAvailable(const BufferItem& /*item*/) {
53 Mutex::Autolock l(mInputMutex);
54 if (!mCallbackAvailable) {
55 mCallbackAvailable = true;
56 mCallbackAvailableSignal.signal();
57 }
58 }
59
setCallbackWindow(const sp<Surface> & callbackWindow)60 status_t CallbackProcessor::setCallbackWindow(
61 const sp<Surface>& callbackWindow) {
62 ATRACE_CALL();
63 status_t res;
64
65 Mutex::Autolock l(mInputMutex);
66
67 sp<Camera2Client> client = mClient.promote();
68 if (client == 0) return OK;
69 sp<CameraDeviceBase> device = client->getCameraDevice();
70
71 // If the window is changing, clear out stream if it already exists
72 if (mCallbackWindow != callbackWindow && mCallbackStreamId != NO_STREAM) {
73 res = device->deleteStream(mCallbackStreamId);
74 if (res != OK) {
75 ALOGE("%s: Camera %d: Unable to delete old stream "
76 "for callbacks: %s (%d)", __FUNCTION__,
77 client->getCameraId(), strerror(-res), res);
78 return res;
79 }
80 mCallbackStreamId = NO_STREAM;
81 mCallbackConsumer.clear();
82 }
83 mCallbackWindow = callbackWindow;
84 mCallbackToApp = (mCallbackWindow != NULL);
85
86 return OK;
87 }
88
updateStream(const Parameters & params)89 status_t CallbackProcessor::updateStream(const Parameters ¶ms) {
90 ATRACE_CALL();
91 status_t res;
92
93 Mutex::Autolock l(mInputMutex);
94
95 sp<CameraDeviceBase> device = mDevice.promote();
96 if (device == 0) {
97 ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId);
98 return INVALID_OPERATION;
99 }
100
101 // If possible, use the flexible YUV format
102 int32_t callbackFormat = params.previewFormat;
103 if (mCallbackToApp) {
104 // TODO: etalvala: This should use the flexible YUV format as well, but
105 // need to reconcile HAL2/HAL3 requirements.
106 callbackFormat = HAL_PIXEL_FORMAT_YV12;
107 } else if(params.fastInfo.useFlexibleYuv &&
108 (params.previewFormat == HAL_PIXEL_FORMAT_YCrCb_420_SP ||
109 params.previewFormat == HAL_PIXEL_FORMAT_YV12) ) {
110 callbackFormat = HAL_PIXEL_FORMAT_YCbCr_420_888;
111 }
112
113 if (!mCallbackToApp && mCallbackConsumer == 0) {
114 // Create CPU buffer queue endpoint, since app hasn't given us one
115 // Make it async to avoid disconnect deadlocks
116 sp<IGraphicBufferProducer> producer;
117 sp<IGraphicBufferConsumer> consumer;
118 BufferQueue::createBufferQueue(&producer, &consumer);
119 mCallbackConsumer = new CpuConsumer(consumer, kCallbackHeapCount);
120 mCallbackConsumer->setFrameAvailableListener(this);
121 mCallbackConsumer->setName(String8("Camera2-CallbackConsumer"));
122 mCallbackWindow = new Surface(producer);
123 }
124
125 if (mCallbackStreamId != NO_STREAM) {
126 // Check if stream parameters have to change
127 CameraDeviceBase::StreamInfo streamInfo;
128 res = device->getStreamInfo(mCallbackStreamId, &streamInfo);
129 if (res != OK) {
130 ALOGE("%s: Camera %d: Error querying callback output stream info: "
131 "%s (%d)", __FUNCTION__, mId,
132 strerror(-res), res);
133 return res;
134 }
135 if (streamInfo.width != (uint32_t)params.previewWidth ||
136 streamInfo.height != (uint32_t)params.previewHeight ||
137 !streamInfo.matchFormat((uint32_t)callbackFormat)) {
138 // Since size should only change while preview is not running,
139 // assuming that all existing use of old callback stream is
140 // completed.
141 ALOGV("%s: Camera %d: Deleting stream %d since the buffer "
142 "parameters changed", __FUNCTION__, mId, mCallbackStreamId);
143 res = device->deleteStream(mCallbackStreamId);
144 if (res != OK) {
145 ALOGE("%s: Camera %d: Unable to delete old output stream "
146 "for callbacks: %s (%d)", __FUNCTION__,
147 mId, strerror(-res), res);
148 return res;
149 }
150 mCallbackStreamId = NO_STREAM;
151 }
152 }
153
154 if (mCallbackStreamId == NO_STREAM) {
155 ALOGV("Creating callback stream: %d x %d, format 0x%x, API format 0x%x",
156 params.previewWidth, params.previewHeight,
157 callbackFormat, params.previewFormat);
158 res = device->createStream(mCallbackWindow,
159 params.previewWidth, params.previewHeight, callbackFormat,
160 HAL_DATASPACE_V0_JFIF, CAMERA_STREAM_ROTATION_0, &mCallbackStreamId,
161 String8(), std::unordered_set<int32_t>{ANDROID_SENSOR_PIXEL_MODE_DEFAULT});
162 if (res != OK) {
163 ALOGE("%s: Camera %d: Can't create output stream for callbacks: "
164 "%s (%d)", __FUNCTION__, mId,
165 strerror(-res), res);
166 return res;
167 }
168 }
169
170 return OK;
171 }
172
deleteStream()173 status_t CallbackProcessor::deleteStream() {
174 ATRACE_CALL();
175 sp<CameraDeviceBase> device;
176 status_t res;
177 {
178 Mutex::Autolock l(mInputMutex);
179
180 if (mCallbackStreamId == NO_STREAM) {
181 return OK;
182 }
183 device = mDevice.promote();
184 if (device == 0) {
185 ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId);
186 return INVALID_OPERATION;
187 }
188 }
189 res = device->waitUntilDrained();
190 if (res != OK) {
191 ALOGE("%s: Error waiting for HAL to drain: %s (%d)",
192 __FUNCTION__, strerror(-res), res);
193 return res;
194 }
195
196 res = device->deleteStream(mCallbackStreamId);
197 if (res != OK) {
198 ALOGE("%s: Unable to delete callback stream: %s (%d)",
199 __FUNCTION__, strerror(-res), res);
200 return res;
201 }
202
203 {
204 Mutex::Autolock l(mInputMutex);
205
206 mCallbackHeap.clear();
207 mCallbackWindow.clear();
208 mCallbackConsumer.clear();
209
210 mCallbackStreamId = NO_STREAM;
211 }
212 return OK;
213 }
214
getStreamId() const215 int CallbackProcessor::getStreamId() const {
216 Mutex::Autolock l(mInputMutex);
217 return mCallbackStreamId;
218 }
219
unpauseCallback()220 void CallbackProcessor::unpauseCallback() {
221 mCallbackPaused = false;
222 }
223
pauseCallback()224 void CallbackProcessor::pauseCallback() {
225 mCallbackPaused = true;
226 }
227
dump(int,const Vector<String16> &) const228 void CallbackProcessor::dump(int /*fd*/, const Vector<String16>& /*args*/) const {
229 }
230
threadLoop()231 bool CallbackProcessor::threadLoop() {
232 status_t res;
233
234 {
235 Mutex::Autolock l(mInputMutex);
236 while (!mCallbackAvailable) {
237 res = mCallbackAvailableSignal.waitRelative(mInputMutex,
238 kWaitDuration);
239 if (res == TIMED_OUT) return true;
240 }
241 mCallbackAvailable = false;
242 }
243
244 do {
245 sp<Camera2Client> client = mClient.promote();
246 if (client == 0 || mCallbackPaused) {
247 res = discardNewCallback();
248 } else {
249 res = processNewCallback(client);
250 }
251 } while (res == OK);
252
253 return true;
254 }
255
discardNewCallback()256 status_t CallbackProcessor::discardNewCallback() {
257 ATRACE_CALL();
258 status_t res;
259 CpuConsumer::LockedBuffer imgBuffer;
260 res = mCallbackConsumer->lockNextBuffer(&imgBuffer);
261 if (res != OK) {
262 if (res != BAD_VALUE) {
263 ALOGE("%s: Camera %d: Error receiving next callback buffer: "
264 "%s (%d)", __FUNCTION__, mId, strerror(-res), res);
265 }
266 return res;
267 }
268 mCallbackConsumer->unlockBuffer(imgBuffer);
269 return OK;
270 }
271
processNewCallback(sp<Camera2Client> & client)272 status_t CallbackProcessor::processNewCallback(sp<Camera2Client> &client) {
273 ATRACE_CALL();
274 status_t res;
275
276 sp<Camera2Heap> callbackHeap;
277 bool useFlexibleYuv = false;
278 int32_t previewFormat = 0;
279 size_t heapIdx;
280
281 {
282 /* acquire SharedParameters before mMutex so we don't dead lock
283 with Camera2Client code calling into StreamingProcessor */
284 SharedParameters::Lock l(client->getParameters());
285 Mutex::Autolock m(mInputMutex);
286 CpuConsumer::LockedBuffer imgBuffer;
287 if (mCallbackStreamId == NO_STREAM) {
288 ALOGV("%s: Camera %d:No stream is available"
289 , __FUNCTION__, mId);
290 return INVALID_OPERATION;
291 }
292
293 ALOGV("%s: Getting buffer", __FUNCTION__);
294 res = mCallbackConsumer->lockNextBuffer(&imgBuffer);
295 if (res != OK) {
296 if (res != BAD_VALUE) {
297 ALOGE("%s: Camera %d: Error receiving next callback buffer: "
298 "%s (%d)", __FUNCTION__, mId, strerror(-res), res);
299 }
300 return res;
301 }
302 ALOGV("%s: Camera %d: Preview callback available", __FUNCTION__,
303 mId);
304
305 if ( l.mParameters.state != Parameters::PREVIEW
306 && l.mParameters.state != Parameters::RECORD
307 && l.mParameters.state != Parameters::VIDEO_SNAPSHOT) {
308 ALOGV("%s: Camera %d: No longer streaming",
309 __FUNCTION__, mId);
310 mCallbackConsumer->unlockBuffer(imgBuffer);
311 return OK;
312 }
313
314 if (! (l.mParameters.previewCallbackFlags &
315 CAMERA_FRAME_CALLBACK_FLAG_ENABLE_MASK) ) {
316 ALOGV("%s: No longer enabled, dropping", __FUNCTION__);
317 mCallbackConsumer->unlockBuffer(imgBuffer);
318 return OK;
319 }
320 if ((l.mParameters.previewCallbackFlags &
321 CAMERA_FRAME_CALLBACK_FLAG_ONE_SHOT_MASK) &&
322 !l.mParameters.previewCallbackOneShot) {
323 ALOGV("%s: One shot mode, already sent, dropping", __FUNCTION__);
324 mCallbackConsumer->unlockBuffer(imgBuffer);
325 return OK;
326 }
327
328 if (imgBuffer.width != static_cast<uint32_t>(l.mParameters.previewWidth) ||
329 imgBuffer.height != static_cast<uint32_t>(l.mParameters.previewHeight)) {
330 ALOGW("%s: The preview size has changed to %d x %d from %d x %d, this buffer is"
331 " no longer valid, dropping",__FUNCTION__,
332 l.mParameters.previewWidth, l.mParameters.previewHeight,
333 imgBuffer.width, imgBuffer.height);
334 mCallbackConsumer->unlockBuffer(imgBuffer);
335 return OK;
336 }
337
338 previewFormat = l.mParameters.previewFormat;
339 useFlexibleYuv = l.mParameters.fastInfo.useFlexibleYuv &&
340 (previewFormat == HAL_PIXEL_FORMAT_YCrCb_420_SP ||
341 previewFormat == HAL_PIXEL_FORMAT_YV12);
342
343 int32_t expectedFormat = useFlexibleYuv ?
344 HAL_PIXEL_FORMAT_YCbCr_420_888 : previewFormat;
345
346 if (imgBuffer.format != expectedFormat) {
347 ALOGE("%s: Camera %d: Unexpected format for callback: "
348 "0x%x, expected 0x%x", __FUNCTION__, mId,
349 imgBuffer.format, expectedFormat);
350 mCallbackConsumer->unlockBuffer(imgBuffer);
351 return INVALID_OPERATION;
352 }
353
354 // In one-shot mode, stop sending callbacks after the first one
355 if (l.mParameters.previewCallbackFlags &
356 CAMERA_FRAME_CALLBACK_FLAG_ONE_SHOT_MASK) {
357 ALOGV("%s: clearing oneshot", __FUNCTION__);
358 l.mParameters.previewCallbackOneShot = false;
359 }
360
361 uint32_t destYStride = 0;
362 uint32_t destCStride = 0;
363 if (useFlexibleYuv) {
364 if (previewFormat == HAL_PIXEL_FORMAT_YV12) {
365 // Strides must align to 16 for YV12
366 destYStride = ALIGN(imgBuffer.width, 16);
367 destCStride = ALIGN(destYStride / 2, 16);
368 } else {
369 // No padding for NV21
370 ALOG_ASSERT(previewFormat == HAL_PIXEL_FORMAT_YCrCb_420_SP,
371 "Unexpected preview format 0x%x", previewFormat);
372 destYStride = imgBuffer.width;
373 destCStride = destYStride / 2;
374 }
375 } else {
376 destYStride = imgBuffer.stride;
377 // don't care about cStride
378 }
379
380 size_t bufferSize = Camera2Client::calculateBufferSize(
381 imgBuffer.width, imgBuffer.height,
382 previewFormat, destYStride);
383 size_t currentBufferSize = (mCallbackHeap == 0) ?
384 0 : (mCallbackHeap->mHeap->getSize() / kCallbackHeapCount);
385 if (bufferSize != currentBufferSize) {
386 mCallbackHeap.clear();
387 mCallbackHeap = new Camera2Heap(bufferSize, kCallbackHeapCount,
388 "Camera2Client::CallbackHeap");
389 if (mCallbackHeap->mHeap->getSize() == 0) {
390 ALOGE("%s: Camera %d: Unable to allocate memory for callbacks",
391 __FUNCTION__, mId);
392 mCallbackConsumer->unlockBuffer(imgBuffer);
393 return INVALID_OPERATION;
394 }
395
396 mCallbackHeapHead = 0;
397 mCallbackHeapFree = kCallbackHeapCount;
398 }
399
400 if (mCallbackHeapFree == 0) {
401 ALOGE("%s: Camera %d: No free callback buffers, dropping frame",
402 __FUNCTION__, mId);
403 mCallbackConsumer->unlockBuffer(imgBuffer);
404 return OK;
405 }
406
407 heapIdx = mCallbackHeapHead;
408
409 mCallbackHeapHead = (mCallbackHeapHead + 1) % kCallbackHeapCount;
410 mCallbackHeapFree--;
411
412 // TODO: Get rid of this copy by passing the gralloc queue all the way
413 // to app
414
415 ssize_t offset;
416 size_t size;
417 sp<IMemoryHeap> heap =
418 mCallbackHeap->mBuffers[heapIdx]->getMemory(&offset,
419 &size);
420 uint8_t *data = (uint8_t*)heap->getBase() + offset;
421
422 if (!useFlexibleYuv) {
423 // Can just memcpy when HAL format matches API format
424 memcpy(data, imgBuffer.data, bufferSize);
425 } else {
426 res = convertFromFlexibleYuv(previewFormat, data, imgBuffer,
427 destYStride, destCStride);
428 if (res != OK) {
429 ALOGE("%s: Camera %d: Can't convert between 0x%x and 0x%x formats!",
430 __FUNCTION__, mId, imgBuffer.format, previewFormat);
431 mCallbackConsumer->unlockBuffer(imgBuffer);
432 return BAD_VALUE;
433 }
434 }
435
436 ALOGV("%s: Freeing buffer", __FUNCTION__);
437 mCallbackConsumer->unlockBuffer(imgBuffer);
438
439 // mCallbackHeap may get freed up once input mutex is released
440 callbackHeap = mCallbackHeap;
441 }
442
443 // Call outside parameter lock to allow re-entrancy from notification
444 {
445 Camera2Client::SharedCameraCallbacks::Lock
446 l(client->mSharedCameraCallbacks);
447 if (l.mRemoteCallback != 0) {
448 ALOGV("%s: Camera %d: Invoking client data callback",
449 __FUNCTION__, mId);
450 l.mRemoteCallback->dataCallback(CAMERA_MSG_PREVIEW_FRAME,
451 callbackHeap->mBuffers[heapIdx], NULL);
452 }
453 }
454
455 // Only increment free if we're still using the same heap
456 mCallbackHeapFree++;
457
458 ALOGV("%s: exit", __FUNCTION__);
459
460 return OK;
461 }
462
convertFromFlexibleYuv(int32_t previewFormat,uint8_t * dst,const CpuConsumer::LockedBuffer & src,uint32_t dstYStride,uint32_t dstCStride) const463 status_t CallbackProcessor::convertFromFlexibleYuv(int32_t previewFormat,
464 uint8_t *dst,
465 const CpuConsumer::LockedBuffer &src,
466 uint32_t dstYStride,
467 uint32_t dstCStride) const {
468
469 if (previewFormat != HAL_PIXEL_FORMAT_YCrCb_420_SP &&
470 previewFormat != HAL_PIXEL_FORMAT_YV12) {
471 ALOGE("%s: Camera %d: Unexpected preview format when using "
472 "flexible YUV: 0x%x", __FUNCTION__, mId, previewFormat);
473 return INVALID_OPERATION;
474 }
475
476 // Copy Y plane, adjusting for stride
477 const uint8_t *ySrc = src.data;
478 uint8_t *yDst = dst;
479 for (size_t row = 0; row < src.height; row++) {
480 memcpy(yDst, ySrc, src.width);
481 ySrc += src.stride;
482 yDst += dstYStride;
483 }
484
485 // Copy/swizzle chroma planes, 4:2:0 subsampling
486 const uint8_t *cbSrc = src.dataCb;
487 const uint8_t *crSrc = src.dataCr;
488 size_t chromaHeight = src.height / 2;
489 size_t chromaWidth = src.width / 2;
490 ssize_t chromaGap = src.chromaStride -
491 (chromaWidth * src.chromaStep);
492 size_t dstChromaGap = dstCStride - chromaWidth;
493
494 if (previewFormat == HAL_PIXEL_FORMAT_YCrCb_420_SP) {
495 // Flexible YUV chroma to NV21 chroma
496 uint8_t *crcbDst = yDst;
497 // Check for shortcuts
498 if (cbSrc == crSrc + 1 && src.chromaStep == 2) {
499 ALOGV("%s: Fast NV21->NV21", __FUNCTION__);
500 // Source has semiplanar CrCb chroma layout, can copy by rows
501 for (size_t row = 0; row < chromaHeight; row++) {
502 memcpy(crcbDst, crSrc, src.width);
503 crcbDst += src.width;
504 crSrc += src.chromaStride;
505 }
506 } else {
507 ALOGV("%s: Generic->NV21", __FUNCTION__);
508 // Generic copy, always works but not very efficient
509 for (size_t row = 0; row < chromaHeight; row++) {
510 for (size_t col = 0; col < chromaWidth; col++) {
511 *(crcbDst++) = *crSrc;
512 *(crcbDst++) = *cbSrc;
513 crSrc += src.chromaStep;
514 cbSrc += src.chromaStep;
515 }
516 crSrc += chromaGap;
517 cbSrc += chromaGap;
518 }
519 }
520 } else {
521 // flexible YUV chroma to YV12 chroma
522 ALOG_ASSERT(previewFormat == HAL_PIXEL_FORMAT_YV12,
523 "Unexpected preview format 0x%x", previewFormat);
524 uint8_t *crDst = yDst;
525 uint8_t *cbDst = yDst + chromaHeight * dstCStride;
526 if (src.chromaStep == 1) {
527 ALOGV("%s: Fast YV12->YV12", __FUNCTION__);
528 // Source has planar chroma layout, can copy by row
529 for (size_t row = 0; row < chromaHeight; row++) {
530 memcpy(crDst, crSrc, chromaWidth);
531 crDst += dstCStride;
532 crSrc += src.chromaStride;
533 }
534 for (size_t row = 0; row < chromaHeight; row++) {
535 memcpy(cbDst, cbSrc, chromaWidth);
536 cbDst += dstCStride;
537 cbSrc += src.chromaStride;
538 }
539 } else {
540 ALOGV("%s: Generic->YV12", __FUNCTION__);
541 // Generic copy, always works but not very efficient
542 for (size_t row = 0; row < chromaHeight; row++) {
543 for (size_t col = 0; col < chromaWidth; col++) {
544 *(crDst++) = *crSrc;
545 *(cbDst++) = *cbSrc;
546 crSrc += src.chromaStep;
547 cbSrc += src.chromaStep;
548 }
549 crSrc += chromaGap;
550 cbSrc += chromaGap;
551 crDst += dstChromaGap;
552 cbDst += dstChromaGap;
553 }
554 }
555 }
556
557 return OK;
558 }
559
560 }; // namespace camera2
561 }; // namespace android
562