1 /*
2 * Copyright (C) 2018 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 #define LOG_TAG "ExtCamDevSsn@3.4"
17 //#define LOG_NDEBUG 0
18 #define ATRACE_TAG ATRACE_TAG_CAMERA
19 #include <log/log.h>
20
21 #include <inttypes.h>
22 #include "ExternalCameraDeviceSession.h"
23
24 #include "android-base/macros.h"
25 #include <utils/Timers.h>
26 #include <utils/Trace.h>
27 #include <linux/videodev2.h>
28 #include <sync/sync.h>
29
30 #define HAVE_JPEG // required for libyuv.h to export MJPEG decode APIs
31 #include <libyuv.h>
32
33 #include <jpeglib.h>
34
35
36 namespace android {
37 namespace hardware {
38 namespace camera {
39 namespace device {
40 namespace V3_4 {
41 namespace implementation {
42
43 namespace {
44 // Size of request/result metadata fast message queue. Change to 0 to always use hwbinder buffer.
45 static constexpr size_t kMetadataMsgQueueSize = 1 << 18 /* 256kB */;
46
47 const int kBadFramesAfterStreamOn = 1; // drop x frames after streamOn to get rid of some initial
48 // bad frames. TODO: develop a better bad frame detection
49 // method
50 constexpr int MAX_RETRY = 15; // Allow retry some ioctl failures a few times to account for some
51 // webcam showing temporarily ioctl failures.
52 constexpr int IOCTL_RETRY_SLEEP_US = 33000; // 33ms * MAX_RETRY = 0.5 seconds
53
54 // Constants for tryLock during dumpstate
55 static constexpr int kDumpLockRetries = 50;
56 static constexpr int kDumpLockSleep = 60000;
57
tryLock(Mutex & mutex)58 bool tryLock(Mutex& mutex)
59 {
60 bool locked = false;
61 for (int i = 0; i < kDumpLockRetries; ++i) {
62 if (mutex.tryLock() == NO_ERROR) {
63 locked = true;
64 break;
65 }
66 usleep(kDumpLockSleep);
67 }
68 return locked;
69 }
70
tryLock(std::mutex & mutex)71 bool tryLock(std::mutex& mutex)
72 {
73 bool locked = false;
74 for (int i = 0; i < kDumpLockRetries; ++i) {
75 if (mutex.try_lock()) {
76 locked = true;
77 break;
78 }
79 usleep(kDumpLockSleep);
80 }
81 return locked;
82 }
83
84 } // Anonymous namespace
85
86 // Static instances
87 const int ExternalCameraDeviceSession::kMaxProcessedStream;
88 const int ExternalCameraDeviceSession::kMaxStallStream;
89 HandleImporter ExternalCameraDeviceSession::sHandleImporter;
90
ExternalCameraDeviceSession(const sp<ICameraDeviceCallback> & callback,const ExternalCameraConfig & cfg,const std::vector<SupportedV4L2Format> & sortedFormats,const CroppingType & croppingType,const common::V1_0::helper::CameraMetadata & chars,const std::string & cameraId,unique_fd v4l2Fd)91 ExternalCameraDeviceSession::ExternalCameraDeviceSession(
92 const sp<ICameraDeviceCallback>& callback,
93 const ExternalCameraConfig& cfg,
94 const std::vector<SupportedV4L2Format>& sortedFormats,
95 const CroppingType& croppingType,
96 const common::V1_0::helper::CameraMetadata& chars,
97 const std::string& cameraId,
98 unique_fd v4l2Fd) :
99 mCallback(callback),
100 mCfg(cfg),
101 mCameraCharacteristics(chars),
102 mSupportedFormats(sortedFormats),
103 mCroppingType(croppingType),
104 mCameraId(cameraId),
105 mV4l2Fd(std::move(v4l2Fd)),
106 mMaxThumbResolution(getMaxThumbResolution()),
107 mMaxJpegResolution(getMaxJpegResolution()) {}
108
initialize()109 bool ExternalCameraDeviceSession::initialize() {
110 if (mV4l2Fd.get() < 0) {
111 ALOGE("%s: invalid v4l2 device fd %d!", __FUNCTION__, mV4l2Fd.get());
112 return true;
113 }
114
115 struct v4l2_capability capability;
116 int ret = ioctl(mV4l2Fd.get(), VIDIOC_QUERYCAP, &capability);
117 std::string make, model;
118 if (ret < 0) {
119 ALOGW("%s v4l2 QUERYCAP failed", __FUNCTION__);
120 mExifMake = "Generic UVC webcam";
121 mExifModel = "Generic UVC webcam";
122 } else {
123 // capability.card is UTF-8 encoded
124 char card[32];
125 int j = 0;
126 for (int i = 0; i < 32; i++) {
127 if (capability.card[i] < 128) {
128 card[j++] = capability.card[i];
129 }
130 if (capability.card[i] == '\0') {
131 break;
132 }
133 }
134 if (j == 0 || card[j - 1] != '\0') {
135 mExifMake = "Generic UVC webcam";
136 mExifModel = "Generic UVC webcam";
137 } else {
138 mExifMake = card;
139 mExifModel = card;
140 }
141 }
142
143 initOutputThread();
144 if (mOutputThread == nullptr) {
145 ALOGE("%s: init OutputThread failed!", __FUNCTION__);
146 return true;
147 }
148 mOutputThread->setExifMakeModel(mExifMake, mExifModel);
149
150 status_t status = initDefaultRequests();
151 if (status != OK) {
152 ALOGE("%s: init default requests failed!", __FUNCTION__);
153 return true;
154 }
155
156 mRequestMetadataQueue = std::make_unique<RequestMetadataQueue>(
157 kMetadataMsgQueueSize, false /* non blocking */);
158 if (!mRequestMetadataQueue->isValid()) {
159 ALOGE("%s: invalid request fmq", __FUNCTION__);
160 return true;
161 }
162 mResultMetadataQueue = std::make_shared<ResultMetadataQueue>(
163 kMetadataMsgQueueSize, false /* non blocking */);
164 if (!mResultMetadataQueue->isValid()) {
165 ALOGE("%s: invalid result fmq", __FUNCTION__);
166 return true;
167 }
168
169 // TODO: check is PRIORITY_DISPLAY enough?
170 mOutputThread->run("ExtCamOut", PRIORITY_DISPLAY);
171 return false;
172 }
173
isInitFailed()174 bool ExternalCameraDeviceSession::isInitFailed() {
175 Mutex::Autolock _l(mLock);
176 if (!mInitialized) {
177 mInitFail = initialize();
178 mInitialized = true;
179 }
180 return mInitFail;
181 }
182
initOutputThread()183 void ExternalCameraDeviceSession::initOutputThread() {
184 mOutputThread = new OutputThread(this, mCroppingType, mCameraCharacteristics);
185 }
186
closeOutputThread()187 void ExternalCameraDeviceSession::closeOutputThread() {
188 closeOutputThreadImpl();
189 }
190
closeOutputThreadImpl()191 void ExternalCameraDeviceSession::closeOutputThreadImpl() {
192 if (mOutputThread) {
193 mOutputThread->flush();
194 mOutputThread->requestExit();
195 mOutputThread->join();
196 mOutputThread.clear();
197 }
198 }
199
initStatus() const200 Status ExternalCameraDeviceSession::initStatus() const {
201 Mutex::Autolock _l(mLock);
202 Status status = Status::OK;
203 if (mInitFail || mClosed) {
204 ALOGI("%s: sesssion initFailed %d closed %d", __FUNCTION__, mInitFail, mClosed);
205 status = Status::INTERNAL_ERROR;
206 }
207 return status;
208 }
209
~ExternalCameraDeviceSession()210 ExternalCameraDeviceSession::~ExternalCameraDeviceSession() {
211 if (!isClosed()) {
212 ALOGE("ExternalCameraDeviceSession deleted before close!");
213 close(/*callerIsDtor*/true);
214 }
215 }
216
217
dumpState(const native_handle_t * handle)218 void ExternalCameraDeviceSession::dumpState(const native_handle_t* handle) {
219 if (handle->numFds != 1 || handle->numInts != 0) {
220 ALOGE("%s: handle must contain 1 FD and 0 integers! Got %d FDs and %d ints",
221 __FUNCTION__, handle->numFds, handle->numInts);
222 return;
223 }
224 int fd = handle->data[0];
225
226 bool intfLocked = tryLock(mInterfaceLock);
227 if (!intfLocked) {
228 dprintf(fd, "!! ExternalCameraDeviceSession interface may be deadlocked !!\n");
229 }
230
231 if (isClosed()) {
232 dprintf(fd, "External camera %s is closed\n", mCameraId.c_str());
233 return;
234 }
235
236 bool streaming = false;
237 size_t v4L2BufferCount = 0;
238 SupportedV4L2Format streamingFmt;
239 {
240 bool sessionLocked = tryLock(mLock);
241 if (!sessionLocked) {
242 dprintf(fd, "!! ExternalCameraDeviceSession mLock may be deadlocked !!\n");
243 }
244 streaming = mV4l2Streaming;
245 streamingFmt = mV4l2StreamingFmt;
246 v4L2BufferCount = mV4L2BufferCount;
247
248 if (sessionLocked) {
249 mLock.unlock();
250 }
251 }
252
253 std::unordered_set<uint32_t> inflightFrames;
254 {
255 bool iffLocked = tryLock(mInflightFramesLock);
256 if (!iffLocked) {
257 dprintf(fd,
258 "!! ExternalCameraDeviceSession mInflightFramesLock may be deadlocked !!\n");
259 }
260 inflightFrames = mInflightFrames;
261 if (iffLocked) {
262 mInflightFramesLock.unlock();
263 }
264 }
265
266 dprintf(fd, "External camera %s V4L2 FD %d, cropping type %s, %s\n",
267 mCameraId.c_str(), mV4l2Fd.get(),
268 (mCroppingType == VERTICAL) ? "vertical" : "horizontal",
269 streaming ? "streaming" : "not streaming");
270 if (streaming) {
271 // TODO: dump fps later
272 dprintf(fd, "Current V4L2 format %c%c%c%c %dx%d @ %ffps\n",
273 streamingFmt.fourcc & 0xFF,
274 (streamingFmt.fourcc >> 8) & 0xFF,
275 (streamingFmt.fourcc >> 16) & 0xFF,
276 (streamingFmt.fourcc >> 24) & 0xFF,
277 streamingFmt.width, streamingFmt.height,
278 mV4l2StreamingFps);
279
280 size_t numDequeuedV4l2Buffers = 0;
281 {
282 std::lock_guard<std::mutex> lk(mV4l2BufferLock);
283 numDequeuedV4l2Buffers = mNumDequeuedV4l2Buffers;
284 }
285 dprintf(fd, "V4L2 buffer queue size %zu, dequeued %zu\n",
286 v4L2BufferCount, numDequeuedV4l2Buffers);
287 }
288
289 dprintf(fd, "In-flight frames (not sorted):");
290 for (const auto& frameNumber : inflightFrames) {
291 dprintf(fd, "%d, ", frameNumber);
292 }
293 dprintf(fd, "\n");
294 mOutputThread->dump(fd);
295 dprintf(fd, "\n");
296
297 if (intfLocked) {
298 mInterfaceLock.unlock();
299 }
300
301 return;
302 }
303
constructDefaultRequestSettings(V3_2::RequestTemplate type,V3_2::ICameraDeviceSession::constructDefaultRequestSettings_cb _hidl_cb)304 Return<void> ExternalCameraDeviceSession::constructDefaultRequestSettings(
305 V3_2::RequestTemplate type,
306 V3_2::ICameraDeviceSession::constructDefaultRequestSettings_cb _hidl_cb) {
307 V3_2::CameraMetadata outMetadata;
308 Status status = constructDefaultRequestSettingsRaw(
309 static_cast<RequestTemplate>(type), &outMetadata);
310 _hidl_cb(status, outMetadata);
311 return Void();
312 }
313
constructDefaultRequestSettingsRaw(RequestTemplate type,V3_2::CameraMetadata * outMetadata)314 Status ExternalCameraDeviceSession::constructDefaultRequestSettingsRaw(RequestTemplate type,
315 V3_2::CameraMetadata *outMetadata) {
316 CameraMetadata emptyMd;
317 Status status = initStatus();
318 if (status != Status::OK) {
319 return status;
320 }
321
322 switch (type) {
323 case RequestTemplate::PREVIEW:
324 case RequestTemplate::STILL_CAPTURE:
325 case RequestTemplate::VIDEO_RECORD:
326 case RequestTemplate::VIDEO_SNAPSHOT: {
327 *outMetadata = mDefaultRequests[type];
328 break;
329 }
330 case RequestTemplate::MANUAL:
331 case RequestTemplate::ZERO_SHUTTER_LAG:
332 // Don't support MANUAL, ZSL templates
333 status = Status::ILLEGAL_ARGUMENT;
334 break;
335 default:
336 ALOGE("%s: unknown request template type %d", __FUNCTION__, static_cast<int>(type));
337 status = Status::ILLEGAL_ARGUMENT;
338 break;
339 }
340 return status;
341 }
342
configureStreams(const V3_2::StreamConfiguration & streams,ICameraDeviceSession::configureStreams_cb _hidl_cb)343 Return<void> ExternalCameraDeviceSession::configureStreams(
344 const V3_2::StreamConfiguration& streams,
345 ICameraDeviceSession::configureStreams_cb _hidl_cb) {
346 V3_2::HalStreamConfiguration outStreams;
347 V3_3::HalStreamConfiguration outStreams_v33;
348 Mutex::Autolock _il(mInterfaceLock);
349
350 Status status = configureStreams(streams, &outStreams_v33);
351 size_t size = outStreams_v33.streams.size();
352 outStreams.streams.resize(size);
353 for (size_t i = 0; i < size; i++) {
354 outStreams.streams[i] = outStreams_v33.streams[i].v3_2;
355 }
356 _hidl_cb(status, outStreams);
357 return Void();
358 }
359
configureStreams_3_3(const V3_2::StreamConfiguration & streams,ICameraDeviceSession::configureStreams_3_3_cb _hidl_cb)360 Return<void> ExternalCameraDeviceSession::configureStreams_3_3(
361 const V3_2::StreamConfiguration& streams,
362 ICameraDeviceSession::configureStreams_3_3_cb _hidl_cb) {
363 V3_3::HalStreamConfiguration outStreams;
364 Mutex::Autolock _il(mInterfaceLock);
365
366 Status status = configureStreams(streams, &outStreams);
367 _hidl_cb(status, outStreams);
368 return Void();
369 }
370
configureStreams_3_4(const V3_4::StreamConfiguration & requestedConfiguration,ICameraDeviceSession::configureStreams_3_4_cb _hidl_cb)371 Return<void> ExternalCameraDeviceSession::configureStreams_3_4(
372 const V3_4::StreamConfiguration& requestedConfiguration,
373 ICameraDeviceSession::configureStreams_3_4_cb _hidl_cb) {
374 V3_2::StreamConfiguration config_v32;
375 V3_3::HalStreamConfiguration outStreams_v33;
376 V3_4::HalStreamConfiguration outStreams;
377 Mutex::Autolock _il(mInterfaceLock);
378
379 config_v32.operationMode = requestedConfiguration.operationMode;
380 config_v32.streams.resize(requestedConfiguration.streams.size());
381 uint32_t blobBufferSize = 0;
382 int numStallStream = 0;
383 for (size_t i = 0; i < config_v32.streams.size(); i++) {
384 config_v32.streams[i] = requestedConfiguration.streams[i].v3_2;
385 if (config_v32.streams[i].format == PixelFormat::BLOB) {
386 blobBufferSize = requestedConfiguration.streams[i].bufferSize;
387 numStallStream++;
388 }
389 }
390
391 // Fail early if there are multiple BLOB streams
392 if (numStallStream > kMaxStallStream) {
393 ALOGE("%s: too many stall streams (expect <= %d, got %d)", __FUNCTION__,
394 kMaxStallStream, numStallStream);
395 _hidl_cb(Status::ILLEGAL_ARGUMENT, outStreams);
396 return Void();
397 }
398
399 Status status = configureStreams(config_v32, &outStreams_v33, blobBufferSize);
400
401 outStreams.streams.resize(outStreams_v33.streams.size());
402 for (size_t i = 0; i < outStreams.streams.size(); i++) {
403 outStreams.streams[i].v3_3 = outStreams_v33.streams[i];
404 }
405 _hidl_cb(status, outStreams);
406 return Void();
407 }
408
getCaptureRequestMetadataQueue(ICameraDeviceSession::getCaptureRequestMetadataQueue_cb _hidl_cb)409 Return<void> ExternalCameraDeviceSession::getCaptureRequestMetadataQueue(
410 ICameraDeviceSession::getCaptureRequestMetadataQueue_cb _hidl_cb) {
411 Mutex::Autolock _il(mInterfaceLock);
412 _hidl_cb(*mRequestMetadataQueue->getDesc());
413 return Void();
414 }
415
getCaptureResultMetadataQueue(ICameraDeviceSession::getCaptureResultMetadataQueue_cb _hidl_cb)416 Return<void> ExternalCameraDeviceSession::getCaptureResultMetadataQueue(
417 ICameraDeviceSession::getCaptureResultMetadataQueue_cb _hidl_cb) {
418 Mutex::Autolock _il(mInterfaceLock);
419 _hidl_cb(*mResultMetadataQueue->getDesc());
420 return Void();
421 }
422
processCaptureRequest(const hidl_vec<CaptureRequest> & requests,const hidl_vec<BufferCache> & cachesToRemove,ICameraDeviceSession::processCaptureRequest_cb _hidl_cb)423 Return<void> ExternalCameraDeviceSession::processCaptureRequest(
424 const hidl_vec<CaptureRequest>& requests,
425 const hidl_vec<BufferCache>& cachesToRemove,
426 ICameraDeviceSession::processCaptureRequest_cb _hidl_cb) {
427 Mutex::Autolock _il(mInterfaceLock);
428 updateBufferCaches(cachesToRemove);
429
430 uint32_t numRequestProcessed = 0;
431 Status s = Status::OK;
432 for (size_t i = 0; i < requests.size(); i++, numRequestProcessed++) {
433 s = processOneCaptureRequest(requests[i]);
434 if (s != Status::OK) {
435 break;
436 }
437 }
438
439 _hidl_cb(s, numRequestProcessed);
440 return Void();
441 }
442
processCaptureRequest_3_4(const hidl_vec<V3_4::CaptureRequest> & requests,const hidl_vec<V3_2::BufferCache> & cachesToRemove,ICameraDeviceSession::processCaptureRequest_3_4_cb _hidl_cb)443 Return<void> ExternalCameraDeviceSession::processCaptureRequest_3_4(
444 const hidl_vec<V3_4::CaptureRequest>& requests,
445 const hidl_vec<V3_2::BufferCache>& cachesToRemove,
446 ICameraDeviceSession::processCaptureRequest_3_4_cb _hidl_cb) {
447 Mutex::Autolock _il(mInterfaceLock);
448 updateBufferCaches(cachesToRemove);
449
450 uint32_t numRequestProcessed = 0;
451 Status s = Status::OK;
452 for (size_t i = 0; i < requests.size(); i++, numRequestProcessed++) {
453 s = processOneCaptureRequest(requests[i].v3_2);
454 if (s != Status::OK) {
455 break;
456 }
457 }
458
459 _hidl_cb(s, numRequestProcessed);
460 return Void();
461 }
462
flush()463 Return<Status> ExternalCameraDeviceSession::flush() {
464 ATRACE_CALL();
465 Mutex::Autolock _il(mInterfaceLock);
466 Status status = initStatus();
467 if (status != Status::OK) {
468 return status;
469 }
470 mOutputThread->flush();
471 return Status::OK;
472 }
473
close(bool callerIsDtor)474 Return<void> ExternalCameraDeviceSession::close(bool callerIsDtor) {
475 Mutex::Autolock _il(mInterfaceLock);
476 bool closed = isClosed();
477 if (!closed) {
478 if (callerIsDtor) {
479 closeOutputThreadImpl();
480 } else {
481 closeOutputThread();
482 }
483
484 Mutex::Autolock _l(mLock);
485 // free all buffers
486 {
487 Mutex::Autolock _l(mCbsLock);
488 for(auto pair : mStreamMap) {
489 cleanupBuffersLocked(/*Stream ID*/pair.first);
490 }
491 }
492 v4l2StreamOffLocked();
493 ALOGV("%s: closing V4L2 camera FD %d", __FUNCTION__, mV4l2Fd.get());
494 mV4l2Fd.reset();
495 mClosed = true;
496 }
497 return Void();
498 }
499
importRequestLocked(const CaptureRequest & request,hidl_vec<buffer_handle_t * > & allBufPtrs,hidl_vec<int> & allFences)500 Status ExternalCameraDeviceSession::importRequestLocked(
501 const CaptureRequest& request,
502 hidl_vec<buffer_handle_t*>& allBufPtrs,
503 hidl_vec<int>& allFences) {
504 return importRequestLockedImpl(request, allBufPtrs, allFences);
505 }
506
importBuffer(int32_t streamId,uint64_t bufId,buffer_handle_t buf,buffer_handle_t ** outBufPtr,bool allowEmptyBuf)507 Status ExternalCameraDeviceSession::importBuffer(int32_t streamId,
508 uint64_t bufId, buffer_handle_t buf,
509 /*out*/buffer_handle_t** outBufPtr,
510 bool allowEmptyBuf) {
511 Mutex::Autolock _l(mCbsLock);
512 return importBufferLocked(streamId, bufId, buf, outBufPtr, allowEmptyBuf);
513 }
514
importBufferLocked(int32_t streamId,uint64_t bufId,buffer_handle_t buf,buffer_handle_t ** outBufPtr,bool allowEmptyBuf)515 Status ExternalCameraDeviceSession::importBufferLocked(int32_t streamId,
516 uint64_t bufId, buffer_handle_t buf,
517 /*out*/buffer_handle_t** outBufPtr,
518 bool allowEmptyBuf) {
519 return importBufferImpl(
520 mCirculatingBuffers, sHandleImporter, streamId,
521 bufId, buf, outBufPtr, allowEmptyBuf);
522 }
523
importRequestLockedImpl(const CaptureRequest & request,hidl_vec<buffer_handle_t * > & allBufPtrs,hidl_vec<int> & allFences,bool allowEmptyBuf)524 Status ExternalCameraDeviceSession::importRequestLockedImpl(
525 const CaptureRequest& request,
526 hidl_vec<buffer_handle_t*>& allBufPtrs,
527 hidl_vec<int>& allFences,
528 bool allowEmptyBuf) {
529 size_t numOutputBufs = request.outputBuffers.size();
530 size_t numBufs = numOutputBufs;
531 // Validate all I/O buffers
532 hidl_vec<buffer_handle_t> allBufs;
533 hidl_vec<uint64_t> allBufIds;
534 allBufs.resize(numBufs);
535 allBufIds.resize(numBufs);
536 allBufPtrs.resize(numBufs);
537 allFences.resize(numBufs);
538 std::vector<int32_t> streamIds(numBufs);
539
540 for (size_t i = 0; i < numOutputBufs; i++) {
541 allBufs[i] = request.outputBuffers[i].buffer.getNativeHandle();
542 allBufIds[i] = request.outputBuffers[i].bufferId;
543 allBufPtrs[i] = &allBufs[i];
544 streamIds[i] = request.outputBuffers[i].streamId;
545 }
546
547 {
548 Mutex::Autolock _l(mCbsLock);
549 for (size_t i = 0; i < numBufs; i++) {
550 Status st = importBufferLocked(
551 streamIds[i], allBufIds[i], allBufs[i], &allBufPtrs[i],
552 allowEmptyBuf);
553 if (st != Status::OK) {
554 // Detailed error logs printed in importBuffer
555 return st;
556 }
557 }
558 }
559
560 // All buffers are imported. Now validate output buffer acquire fences
561 for (size_t i = 0; i < numOutputBufs; i++) {
562 if (!sHandleImporter.importFence(
563 request.outputBuffers[i].acquireFence, allFences[i])) {
564 ALOGE("%s: output buffer %zu acquire fence is invalid", __FUNCTION__, i);
565 cleanupInflightFences(allFences, i);
566 return Status::INTERNAL_ERROR;
567 }
568 }
569 return Status::OK;
570 }
571
cleanupInflightFences(hidl_vec<int> & allFences,size_t numFences)572 void ExternalCameraDeviceSession::cleanupInflightFences(
573 hidl_vec<int>& allFences, size_t numFences) {
574 for (size_t j = 0; j < numFences; j++) {
575 sHandleImporter.closeFence(allFences[j]);
576 }
577 }
578
waitForV4L2BufferReturnLocked(std::unique_lock<std::mutex> & lk)579 int ExternalCameraDeviceSession::waitForV4L2BufferReturnLocked(std::unique_lock<std::mutex>& lk) {
580 ATRACE_CALL();
581 std::chrono::seconds timeout = std::chrono::seconds(kBufferWaitTimeoutSec);
582 mLock.unlock();
583 auto st = mV4L2BufferReturned.wait_for(lk, timeout);
584 // Here we introduce a order where mV4l2BufferLock is acquired before mLock, while
585 // the normal lock acquisition order is reversed. This is fine because in most of
586 // cases we are protected by mInterfaceLock. The only thread that can cause deadlock
587 // is the OutputThread, where we do need to make sure we don't acquire mLock then
588 // mV4l2BufferLock
589 mLock.lock();
590 if (st == std::cv_status::timeout) {
591 ALOGE("%s: wait for V4L2 buffer return timeout!", __FUNCTION__);
592 return -1;
593 }
594 return 0;
595 }
596
processOneCaptureRequest(const CaptureRequest & request)597 Status ExternalCameraDeviceSession::processOneCaptureRequest(const CaptureRequest& request) {
598 ATRACE_CALL();
599 Status status = initStatus();
600 if (status != Status::OK) {
601 return status;
602 }
603
604 if (request.inputBuffer.streamId != -1) {
605 ALOGE("%s: external camera does not support reprocessing!", __FUNCTION__);
606 return Status::ILLEGAL_ARGUMENT;
607 }
608
609 Mutex::Autolock _l(mLock);
610 if (!mV4l2Streaming) {
611 ALOGE("%s: cannot process request in streamOff state!", __FUNCTION__);
612 return Status::INTERNAL_ERROR;
613 }
614
615 const camera_metadata_t *rawSettings = nullptr;
616 bool converted = true;
617 CameraMetadata settingsFmq; // settings from FMQ
618 if (request.fmqSettingsSize > 0) {
619 // non-blocking read; client must write metadata before calling
620 // processOneCaptureRequest
621 settingsFmq.resize(request.fmqSettingsSize);
622 bool read = mRequestMetadataQueue->read(settingsFmq.data(), request.fmqSettingsSize);
623 if (read) {
624 converted = V3_2::implementation::convertFromHidl(settingsFmq, &rawSettings);
625 } else {
626 ALOGE("%s: capture request settings metadata couldn't be read from fmq!", __FUNCTION__);
627 converted = false;
628 }
629 } else {
630 converted = V3_2::implementation::convertFromHidl(request.settings, &rawSettings);
631 }
632
633 if (converted && rawSettings != nullptr) {
634 mLatestReqSetting = rawSettings;
635 }
636
637 if (!converted) {
638 ALOGE("%s: capture request settings metadata is corrupt!", __FUNCTION__);
639 return Status::ILLEGAL_ARGUMENT;
640 }
641
642 if (mFirstRequest && rawSettings == nullptr) {
643 ALOGE("%s: capture request settings must not be null for first request!",
644 __FUNCTION__);
645 return Status::ILLEGAL_ARGUMENT;
646 }
647
648 hidl_vec<buffer_handle_t*> allBufPtrs;
649 hidl_vec<int> allFences;
650 size_t numOutputBufs = request.outputBuffers.size();
651
652 if (numOutputBufs == 0) {
653 ALOGE("%s: capture request must have at least one output buffer!", __FUNCTION__);
654 return Status::ILLEGAL_ARGUMENT;
655 }
656
657 camera_metadata_entry fpsRange = mLatestReqSetting.find(ANDROID_CONTROL_AE_TARGET_FPS_RANGE);
658 if (fpsRange.count == 2) {
659 double requestFpsMax = fpsRange.data.i32[1];
660 double closestFps = 0.0;
661 double fpsError = 1000.0;
662 bool fpsSupported = false;
663 for (const auto& fr : mV4l2StreamingFmt.frameRates) {
664 double f = fr.getDouble();
665 if (std::fabs(requestFpsMax - f) < 1.0) {
666 fpsSupported = true;
667 break;
668 }
669 if (std::fabs(requestFpsMax - f) < fpsError) {
670 fpsError = std::fabs(requestFpsMax - f);
671 closestFps = f;
672 }
673 }
674 if (!fpsSupported) {
675 /* This can happen in a few scenarios:
676 * 1. The application is sending a FPS range not supported by the configured outputs.
677 * 2. The application is sending a valid FPS range for all cofigured outputs, but
678 * the selected V4L2 size can only run at slower speed. This should be very rare
679 * though: for this to happen a sensor needs to support at least 3 different aspect
680 * ratio outputs, and when (at least) two outputs are both not the main aspect ratio
681 * of the webcam, a third size that's larger might be picked and runs into this
682 * issue.
683 */
684 ALOGW("%s: cannot reach fps %d! Will do %f instead",
685 __FUNCTION__, fpsRange.data.i32[1], closestFps);
686 requestFpsMax = closestFps;
687 }
688
689 if (requestFpsMax != mV4l2StreamingFps) {
690 {
691 std::unique_lock<std::mutex> lk(mV4l2BufferLock);
692 while (mNumDequeuedV4l2Buffers != 0) {
693 // Wait until pipeline is idle before reconfigure stream
694 int waitRet = waitForV4L2BufferReturnLocked(lk);
695 if (waitRet != 0) {
696 ALOGE("%s: wait for pipeline idle failed!", __FUNCTION__);
697 return Status::INTERNAL_ERROR;
698 }
699 }
700 }
701 configureV4l2StreamLocked(mV4l2StreamingFmt, requestFpsMax);
702 }
703 }
704
705 status = importRequestLocked(request, allBufPtrs, allFences);
706 if (status != Status::OK) {
707 return status;
708 }
709
710 nsecs_t shutterTs = 0;
711 sp<V4L2Frame> frameIn = dequeueV4l2FrameLocked(&shutterTs);
712 if ( frameIn == nullptr) {
713 ALOGE("%s: V4L2 deque frame failed!", __FUNCTION__);
714 return Status::INTERNAL_ERROR;
715 }
716
717 std::shared_ptr<HalRequest> halReq = std::make_shared<HalRequest>();
718 halReq->frameNumber = request.frameNumber;
719 halReq->setting = mLatestReqSetting;
720 halReq->frameIn = frameIn;
721 halReq->shutterTs = shutterTs;
722 halReq->buffers.resize(numOutputBufs);
723 for (size_t i = 0; i < numOutputBufs; i++) {
724 HalStreamBuffer& halBuf = halReq->buffers[i];
725 int streamId = halBuf.streamId = request.outputBuffers[i].streamId;
726 halBuf.bufferId = request.outputBuffers[i].bufferId;
727 const Stream& stream = mStreamMap[streamId];
728 halBuf.width = stream.width;
729 halBuf.height = stream.height;
730 halBuf.format = stream.format;
731 halBuf.usage = stream.usage;
732 halBuf.bufPtr = allBufPtrs[i];
733 halBuf.acquireFence = allFences[i];
734 halBuf.fenceTimeout = false;
735 }
736 {
737 std::lock_guard<std::mutex> lk(mInflightFramesLock);
738 mInflightFrames.insert(halReq->frameNumber);
739 }
740 // Send request to OutputThread for the rest of processing
741 mOutputThread->submitRequest(halReq);
742 mFirstRequest = false;
743 return Status::OK;
744 }
745
notifyShutter(uint32_t frameNumber,nsecs_t shutterTs)746 void ExternalCameraDeviceSession::notifyShutter(uint32_t frameNumber, nsecs_t shutterTs) {
747 NotifyMsg msg;
748 msg.type = MsgType::SHUTTER;
749 msg.msg.shutter.frameNumber = frameNumber;
750 msg.msg.shutter.timestamp = shutterTs;
751 mCallback->notify({msg});
752 }
753
notifyError(uint32_t frameNumber,int32_t streamId,ErrorCode ec)754 void ExternalCameraDeviceSession::notifyError(
755 uint32_t frameNumber, int32_t streamId, ErrorCode ec) {
756 NotifyMsg msg;
757 msg.type = MsgType::ERROR;
758 msg.msg.error.frameNumber = frameNumber;
759 msg.msg.error.errorStreamId = streamId;
760 msg.msg.error.errorCode = ec;
761 mCallback->notify({msg});
762 }
763
764 //TODO: refactor with processCaptureResult
processCaptureRequestError(const std::shared_ptr<HalRequest> & req,std::vector<NotifyMsg> * outMsgs,std::vector<CaptureResult> * outResults)765 Status ExternalCameraDeviceSession::processCaptureRequestError(
766 const std::shared_ptr<HalRequest>& req,
767 /*out*/std::vector<NotifyMsg>* outMsgs,
768 /*out*/std::vector<CaptureResult>* outResults) {
769 ATRACE_CALL();
770 // Return V4L2 buffer to V4L2 buffer queue
771 sp<V3_4::implementation::V4L2Frame> v4l2Frame =
772 static_cast<V3_4::implementation::V4L2Frame*>(req->frameIn.get());
773 enqueueV4l2Frame(v4l2Frame);
774
775 if (outMsgs == nullptr) {
776 notifyShutter(req->frameNumber, req->shutterTs);
777 notifyError(/*frameNum*/req->frameNumber, /*stream*/-1, ErrorCode::ERROR_REQUEST);
778 } else {
779 NotifyMsg shutter;
780 shutter.type = MsgType::SHUTTER;
781 shutter.msg.shutter.frameNumber = req->frameNumber;
782 shutter.msg.shutter.timestamp = req->shutterTs;
783
784 NotifyMsg error;
785 error.type = MsgType::ERROR;
786 error.msg.error.frameNumber = req->frameNumber;
787 error.msg.error.errorStreamId = -1;
788 error.msg.error.errorCode = ErrorCode::ERROR_REQUEST;
789 outMsgs->push_back(shutter);
790 outMsgs->push_back(error);
791 }
792
793 // Fill output buffers
794 hidl_vec<CaptureResult> results;
795 results.resize(1);
796 CaptureResult& result = results[0];
797 result.frameNumber = req->frameNumber;
798 result.partialResult = 1;
799 result.inputBuffer.streamId = -1;
800 result.outputBuffers.resize(req->buffers.size());
801 for (size_t i = 0; i < req->buffers.size(); i++) {
802 result.outputBuffers[i].streamId = req->buffers[i].streamId;
803 result.outputBuffers[i].bufferId = req->buffers[i].bufferId;
804 result.outputBuffers[i].status = BufferStatus::ERROR;
805 if (req->buffers[i].acquireFence >= 0) {
806 native_handle_t* handle = native_handle_create(/*numFds*/1, /*numInts*/0);
807 handle->data[0] = req->buffers[i].acquireFence;
808 result.outputBuffers[i].releaseFence.setTo(handle, /*shouldOwn*/false);
809 }
810 }
811
812 // update inflight records
813 {
814 std::lock_guard<std::mutex> lk(mInflightFramesLock);
815 mInflightFrames.erase(req->frameNumber);
816 }
817
818 if (outResults == nullptr) {
819 // Callback into framework
820 invokeProcessCaptureResultCallback(results, /* tryWriteFmq */true);
821 freeReleaseFences(results);
822 } else {
823 outResults->push_back(result);
824 }
825 return Status::OK;
826 }
827
processCaptureResult(std::shared_ptr<HalRequest> & req)828 Status ExternalCameraDeviceSession::processCaptureResult(std::shared_ptr<HalRequest>& req) {
829 ATRACE_CALL();
830 // Return V4L2 buffer to V4L2 buffer queue
831 sp<V3_4::implementation::V4L2Frame> v4l2Frame =
832 static_cast<V3_4::implementation::V4L2Frame*>(req->frameIn.get());
833 enqueueV4l2Frame(v4l2Frame);
834
835 // NotifyShutter
836 notifyShutter(req->frameNumber, req->shutterTs);
837
838 // Fill output buffers
839 hidl_vec<CaptureResult> results;
840 results.resize(1);
841 CaptureResult& result = results[0];
842 result.frameNumber = req->frameNumber;
843 result.partialResult = 1;
844 result.inputBuffer.streamId = -1;
845 result.outputBuffers.resize(req->buffers.size());
846 for (size_t i = 0; i < req->buffers.size(); i++) {
847 result.outputBuffers[i].streamId = req->buffers[i].streamId;
848 result.outputBuffers[i].bufferId = req->buffers[i].bufferId;
849 if (req->buffers[i].fenceTimeout) {
850 result.outputBuffers[i].status = BufferStatus::ERROR;
851 if (req->buffers[i].acquireFence >= 0) {
852 native_handle_t* handle = native_handle_create(/*numFds*/1, /*numInts*/0);
853 handle->data[0] = req->buffers[i].acquireFence;
854 result.outputBuffers[i].releaseFence.setTo(handle, /*shouldOwn*/false);
855 }
856 notifyError(req->frameNumber, req->buffers[i].streamId, ErrorCode::ERROR_BUFFER);
857 } else {
858 result.outputBuffers[i].status = BufferStatus::OK;
859 // TODO: refactor
860 if (req->buffers[i].acquireFence >= 0) {
861 native_handle_t* handle = native_handle_create(/*numFds*/1, /*numInts*/0);
862 handle->data[0] = req->buffers[i].acquireFence;
863 result.outputBuffers[i].releaseFence.setTo(handle, /*shouldOwn*/false);
864 }
865 }
866 }
867
868 // Fill capture result metadata
869 fillCaptureResult(req->setting, req->shutterTs);
870 const camera_metadata_t *rawResult = req->setting.getAndLock();
871 V3_2::implementation::convertToHidl(rawResult, &result.result);
872 req->setting.unlock(rawResult);
873
874 // update inflight records
875 {
876 std::lock_guard<std::mutex> lk(mInflightFramesLock);
877 mInflightFrames.erase(req->frameNumber);
878 }
879
880 // Callback into framework
881 invokeProcessCaptureResultCallback(results, /* tryWriteFmq */true);
882 freeReleaseFences(results);
883 return Status::OK;
884 }
885
invokeProcessCaptureResultCallback(hidl_vec<CaptureResult> & results,bool tryWriteFmq)886 void ExternalCameraDeviceSession::invokeProcessCaptureResultCallback(
887 hidl_vec<CaptureResult> &results, bool tryWriteFmq) {
888 if (mProcessCaptureResultLock.tryLock() != OK) {
889 const nsecs_t NS_TO_SECOND = 1000000000;
890 ALOGV("%s: previous call is not finished! waiting 1s...", __FUNCTION__);
891 if (mProcessCaptureResultLock.timedLock(/* 1s */NS_TO_SECOND) != OK) {
892 ALOGE("%s: cannot acquire lock in 1s, cannot proceed",
893 __FUNCTION__);
894 return;
895 }
896 }
897 if (tryWriteFmq && mResultMetadataQueue->availableToWrite() > 0) {
898 for (CaptureResult &result : results) {
899 if (result.result.size() > 0) {
900 if (mResultMetadataQueue->write(result.result.data(), result.result.size())) {
901 result.fmqResultSize = result.result.size();
902 result.result.resize(0);
903 } else {
904 ALOGW("%s: couldn't utilize fmq, fall back to hwbinder", __FUNCTION__);
905 result.fmqResultSize = 0;
906 }
907 } else {
908 result.fmqResultSize = 0;
909 }
910 }
911 }
912 auto status = mCallback->processCaptureResult(results);
913 if (!status.isOk()) {
914 ALOGE("%s: processCaptureResult ERROR : %s", __FUNCTION__,
915 status.description().c_str());
916 }
917
918 mProcessCaptureResultLock.unlock();
919 }
920
OutputThread(wp<OutputThreadInterface> parent,CroppingType ct,const common::V1_0::helper::CameraMetadata & chars)921 ExternalCameraDeviceSession::OutputThread::OutputThread(
922 wp<OutputThreadInterface> parent, CroppingType ct,
923 const common::V1_0::helper::CameraMetadata& chars) :
924 mParent(parent), mCroppingType(ct), mCameraCharacteristics(chars) {}
925
~OutputThread()926 ExternalCameraDeviceSession::OutputThread::~OutputThread() {}
927
setExifMakeModel(const std::string & make,const std::string & model)928 void ExternalCameraDeviceSession::OutputThread::setExifMakeModel(
929 const std::string& make, const std::string& model) {
930 mExifMake = make;
931 mExifModel = model;
932 }
933
cropAndScaleLocked(sp<AllocatedFrame> & in,const Size & outSz,YCbCrLayout * out)934 int ExternalCameraDeviceSession::OutputThread::cropAndScaleLocked(
935 sp<AllocatedFrame>& in, const Size& outSz, YCbCrLayout* out) {
936 Size inSz = {in->mWidth, in->mHeight};
937
938 int ret;
939 if (inSz == outSz) {
940 ret = in->getLayout(out);
941 if (ret != 0) {
942 ALOGE("%s: failed to get input image layout", __FUNCTION__);
943 return ret;
944 }
945 return ret;
946 }
947
948 // Cropping to output aspect ratio
949 IMapper::Rect inputCrop;
950 ret = getCropRect(mCroppingType, inSz, outSz, &inputCrop);
951 if (ret != 0) {
952 ALOGE("%s: failed to compute crop rect for output size %dx%d",
953 __FUNCTION__, outSz.width, outSz.height);
954 return ret;
955 }
956
957 YCbCrLayout croppedLayout;
958 ret = in->getCroppedLayout(inputCrop, &croppedLayout);
959 if (ret != 0) {
960 ALOGE("%s: failed to crop input image %dx%d to output size %dx%d",
961 __FUNCTION__, inSz.width, inSz.height, outSz.width, outSz.height);
962 return ret;
963 }
964
965 if ((mCroppingType == VERTICAL && inSz.width == outSz.width) ||
966 (mCroppingType == HORIZONTAL && inSz.height == outSz.height)) {
967 // No scale is needed
968 *out = croppedLayout;
969 return 0;
970 }
971
972 auto it = mScaledYu12Frames.find(outSz);
973 sp<AllocatedFrame> scaledYu12Buf;
974 if (it != mScaledYu12Frames.end()) {
975 scaledYu12Buf = it->second;
976 } else {
977 it = mIntermediateBuffers.find(outSz);
978 if (it == mIntermediateBuffers.end()) {
979 ALOGE("%s: failed to find intermediate buffer size %dx%d",
980 __FUNCTION__, outSz.width, outSz.height);
981 return -1;
982 }
983 scaledYu12Buf = it->second;
984 }
985 // Scale
986 YCbCrLayout outLayout;
987 ret = scaledYu12Buf->getLayout(&outLayout);
988 if (ret != 0) {
989 ALOGE("%s: failed to get output buffer layout", __FUNCTION__);
990 return ret;
991 }
992
993 ret = libyuv::I420Scale(
994 static_cast<uint8_t*>(croppedLayout.y),
995 croppedLayout.yStride,
996 static_cast<uint8_t*>(croppedLayout.cb),
997 croppedLayout.cStride,
998 static_cast<uint8_t*>(croppedLayout.cr),
999 croppedLayout.cStride,
1000 inputCrop.width,
1001 inputCrop.height,
1002 static_cast<uint8_t*>(outLayout.y),
1003 outLayout.yStride,
1004 static_cast<uint8_t*>(outLayout.cb),
1005 outLayout.cStride,
1006 static_cast<uint8_t*>(outLayout.cr),
1007 outLayout.cStride,
1008 outSz.width,
1009 outSz.height,
1010 // TODO: b/72261744 see if we can use better filter without losing too much perf
1011 libyuv::FilterMode::kFilterNone);
1012
1013 if (ret != 0) {
1014 ALOGE("%s: failed to scale buffer from %dx%d to %dx%d. Ret %d",
1015 __FUNCTION__, inputCrop.width, inputCrop.height,
1016 outSz.width, outSz.height, ret);
1017 return ret;
1018 }
1019
1020 *out = outLayout;
1021 mScaledYu12Frames.insert({outSz, scaledYu12Buf});
1022 return 0;
1023 }
1024
1025
cropAndScaleThumbLocked(sp<AllocatedFrame> & in,const Size & outSz,YCbCrLayout * out)1026 int ExternalCameraDeviceSession::OutputThread::cropAndScaleThumbLocked(
1027 sp<AllocatedFrame>& in, const Size &outSz, YCbCrLayout* out) {
1028 Size inSz {in->mWidth, in->mHeight};
1029
1030 if ((outSz.width * outSz.height) >
1031 (mYu12ThumbFrame->mWidth * mYu12ThumbFrame->mHeight)) {
1032 ALOGE("%s: Requested thumbnail size too big (%d,%d) > (%d,%d)",
1033 __FUNCTION__, outSz.width, outSz.height,
1034 mYu12ThumbFrame->mWidth, mYu12ThumbFrame->mHeight);
1035 return -1;
1036 }
1037
1038 int ret;
1039
1040 /* This will crop-and-zoom the input YUV frame to the thumbnail size
1041 * Based on the following logic:
1042 * 1) Square pixels come in, square pixels come out, therefore single
1043 * scale factor is computed to either make input bigger or smaller
1044 * depending on if we are upscaling or downscaling
1045 * 2) That single scale factor would either make height too tall or width
1046 * too wide so we need to crop the input either horizontally or vertically
1047 * but not both
1048 */
1049
1050 /* Convert the input and output dimensions into floats for ease of math */
1051 float fWin = static_cast<float>(inSz.width);
1052 float fHin = static_cast<float>(inSz.height);
1053 float fWout = static_cast<float>(outSz.width);
1054 float fHout = static_cast<float>(outSz.height);
1055
1056 /* Compute the one scale factor from (1) above, it will be the smaller of
1057 * the two possibilities. */
1058 float scaleFactor = std::min( fHin / fHout, fWin / fWout );
1059
1060 /* Since we are crop-and-zooming (as opposed to letter/pillar boxing) we can
1061 * simply multiply the output by our scaleFactor to get the cropped input
1062 * size. Note that at least one of {fWcrop, fHcrop} is going to wind up
1063 * being {fWin, fHin} respectively because fHout or fWout cancels out the
1064 * scaleFactor calculation above.
1065 *
1066 * Specifically:
1067 * if ( fHin / fHout ) < ( fWin / fWout ) we crop the sides off
1068 * input, in which case
1069 * scaleFactor = fHin / fHout
1070 * fWcrop = fHin / fHout * fWout
1071 * fHcrop = fHin
1072 *
1073 * Note that fWcrop <= fWin ( because ( fHin / fHout ) * fWout < fWin, which
1074 * is just the inequality above with both sides multiplied by fWout
1075 *
1076 * on the other hand if ( fWin / fWout ) < ( fHin / fHout) we crop the top
1077 * and the bottom off of input, and
1078 * scaleFactor = fWin / fWout
1079 * fWcrop = fWin
1080 * fHCrop = fWin / fWout * fHout
1081 */
1082 float fWcrop = scaleFactor * fWout;
1083 float fHcrop = scaleFactor * fHout;
1084
1085 /* Convert to integer and truncate to an even number */
1086 Size cropSz = { 2*static_cast<uint32_t>(fWcrop/2.0f),
1087 2*static_cast<uint32_t>(fHcrop/2.0f) };
1088
1089 /* Convert to a centered rectange with even top/left */
1090 IMapper::Rect inputCrop {
1091 2*static_cast<int32_t>((inSz.width - cropSz.width)/4),
1092 2*static_cast<int32_t>((inSz.height - cropSz.height)/4),
1093 static_cast<int32_t>(cropSz.width),
1094 static_cast<int32_t>(cropSz.height) };
1095
1096 if ((inputCrop.top < 0) ||
1097 (inputCrop.top >= static_cast<int32_t>(inSz.height)) ||
1098 (inputCrop.left < 0) ||
1099 (inputCrop.left >= static_cast<int32_t>(inSz.width)) ||
1100 (inputCrop.width <= 0) ||
1101 (inputCrop.width + inputCrop.left > static_cast<int32_t>(inSz.width)) ||
1102 (inputCrop.height <= 0) ||
1103 (inputCrop.height + inputCrop.top > static_cast<int32_t>(inSz.height)))
1104 {
1105 ALOGE("%s: came up with really wrong crop rectangle",__FUNCTION__);
1106 ALOGE("%s: input layout %dx%d to for output size %dx%d",
1107 __FUNCTION__, inSz.width, inSz.height, outSz.width, outSz.height);
1108 ALOGE("%s: computed input crop +%d,+%d %dx%d",
1109 __FUNCTION__, inputCrop.left, inputCrop.top,
1110 inputCrop.width, inputCrop.height);
1111 return -1;
1112 }
1113
1114 YCbCrLayout inputLayout;
1115 ret = in->getCroppedLayout(inputCrop, &inputLayout);
1116 if (ret != 0) {
1117 ALOGE("%s: failed to crop input layout %dx%d to for output size %dx%d",
1118 __FUNCTION__, inSz.width, inSz.height, outSz.width, outSz.height);
1119 ALOGE("%s: computed input crop +%d,+%d %dx%d",
1120 __FUNCTION__, inputCrop.left, inputCrop.top,
1121 inputCrop.width, inputCrop.height);
1122 return ret;
1123 }
1124 ALOGV("%s: crop input layout %dx%d to for output size %dx%d",
1125 __FUNCTION__, inSz.width, inSz.height, outSz.width, outSz.height);
1126 ALOGV("%s: computed input crop +%d,+%d %dx%d",
1127 __FUNCTION__, inputCrop.left, inputCrop.top,
1128 inputCrop.width, inputCrop.height);
1129
1130
1131 // Scale
1132 YCbCrLayout outFullLayout;
1133
1134 ret = mYu12ThumbFrame->getLayout(&outFullLayout);
1135 if (ret != 0) {
1136 ALOGE("%s: failed to get output buffer layout", __FUNCTION__);
1137 return ret;
1138 }
1139
1140
1141 ret = libyuv::I420Scale(
1142 static_cast<uint8_t*>(inputLayout.y),
1143 inputLayout.yStride,
1144 static_cast<uint8_t*>(inputLayout.cb),
1145 inputLayout.cStride,
1146 static_cast<uint8_t*>(inputLayout.cr),
1147 inputLayout.cStride,
1148 inputCrop.width,
1149 inputCrop.height,
1150 static_cast<uint8_t*>(outFullLayout.y),
1151 outFullLayout.yStride,
1152 static_cast<uint8_t*>(outFullLayout.cb),
1153 outFullLayout.cStride,
1154 static_cast<uint8_t*>(outFullLayout.cr),
1155 outFullLayout.cStride,
1156 outSz.width,
1157 outSz.height,
1158 libyuv::FilterMode::kFilterNone);
1159
1160 if (ret != 0) {
1161 ALOGE("%s: failed to scale buffer from %dx%d to %dx%d. Ret %d",
1162 __FUNCTION__, inputCrop.width, inputCrop.height,
1163 outSz.width, outSz.height, ret);
1164 return ret;
1165 }
1166
1167 *out = outFullLayout;
1168 return 0;
1169 }
1170
1171 /*
1172 * TODO: There needs to be a mechanism to discover allocated buffer size
1173 * in the HAL.
1174 *
1175 * This is very fragile because it is duplicated computation from:
1176 * frameworks/av/services/camera/libcameraservice/device3/Camera3Device.cpp
1177 *
1178 */
1179
1180 /* This assumes mSupportedFormats have all been declared as supporting
1181 * HAL_PIXEL_FORMAT_BLOB to the framework */
getMaxJpegResolution() const1182 Size ExternalCameraDeviceSession::getMaxJpegResolution() const {
1183 Size ret { 0, 0 };
1184 for(auto & fmt : mSupportedFormats) {
1185 if(fmt.width * fmt.height > ret.width * ret.height) {
1186 ret = Size { fmt.width, fmt.height };
1187 }
1188 }
1189 return ret;
1190 }
1191
getMaxThumbResolution() const1192 Size ExternalCameraDeviceSession::getMaxThumbResolution() const {
1193 return getMaxThumbnailResolution(mCameraCharacteristics);
1194 }
1195
getJpegBufferSize(uint32_t width,uint32_t height) const1196 ssize_t ExternalCameraDeviceSession::getJpegBufferSize(
1197 uint32_t width, uint32_t height) const {
1198 // Constant from camera3.h
1199 const ssize_t kMinJpegBufferSize = 256 * 1024 + sizeof(CameraBlob);
1200 // Get max jpeg size (area-wise).
1201 if (mMaxJpegResolution.width == 0) {
1202 ALOGE("%s: Do not have a single supported JPEG stream",
1203 __FUNCTION__);
1204 return BAD_VALUE;
1205 }
1206
1207 // Get max jpeg buffer size
1208 ssize_t maxJpegBufferSize = 0;
1209 camera_metadata_ro_entry jpegBufMaxSize =
1210 mCameraCharacteristics.find(ANDROID_JPEG_MAX_SIZE);
1211 if (jpegBufMaxSize.count == 0) {
1212 ALOGE("%s: Can't find maximum JPEG size in static metadata!",
1213 __FUNCTION__);
1214 return BAD_VALUE;
1215 }
1216 maxJpegBufferSize = jpegBufMaxSize.data.i32[0];
1217
1218 if (maxJpegBufferSize <= kMinJpegBufferSize) {
1219 ALOGE("%s: ANDROID_JPEG_MAX_SIZE (%zd) <= kMinJpegBufferSize (%zd)",
1220 __FUNCTION__, maxJpegBufferSize, kMinJpegBufferSize);
1221 return BAD_VALUE;
1222 }
1223
1224 // Calculate final jpeg buffer size for the given resolution.
1225 float scaleFactor = ((float) (width * height)) /
1226 (mMaxJpegResolution.width * mMaxJpegResolution.height);
1227 ssize_t jpegBufferSize = scaleFactor * (maxJpegBufferSize - kMinJpegBufferSize) +
1228 kMinJpegBufferSize;
1229 if (jpegBufferSize > maxJpegBufferSize) {
1230 jpegBufferSize = maxJpegBufferSize;
1231 }
1232
1233 return jpegBufferSize;
1234 }
1235
createJpegLocked(HalStreamBuffer & halBuf,const common::V1_0::helper::CameraMetadata & setting)1236 int ExternalCameraDeviceSession::OutputThread::createJpegLocked(
1237 HalStreamBuffer &halBuf,
1238 const common::V1_0::helper::CameraMetadata& setting)
1239 {
1240 ATRACE_CALL();
1241 int ret;
1242 auto lfail = [&](auto... args) {
1243 ALOGE(args...);
1244
1245 return 1;
1246 };
1247 auto parent = mParent.promote();
1248 if (parent == nullptr) {
1249 ALOGE("%s: session has been disconnected!", __FUNCTION__);
1250 return 1;
1251 }
1252
1253 ALOGV("%s: HAL buffer sid: %d bid: %" PRIu64 " w: %u h: %u",
1254 __FUNCTION__, halBuf.streamId, static_cast<uint64_t>(halBuf.bufferId),
1255 halBuf.width, halBuf.height);
1256 ALOGV("%s: HAL buffer fmt: %x usage: %" PRIx64 " ptr: %p",
1257 __FUNCTION__, halBuf.format, static_cast<uint64_t>(halBuf.usage),
1258 halBuf.bufPtr);
1259 ALOGV("%s: YV12 buffer %d x %d",
1260 __FUNCTION__,
1261 mYu12Frame->mWidth, mYu12Frame->mHeight);
1262
1263 int jpegQuality, thumbQuality;
1264 Size thumbSize;
1265 bool outputThumbnail = true;
1266
1267 if (setting.exists(ANDROID_JPEG_QUALITY)) {
1268 camera_metadata_ro_entry entry =
1269 setting.find(ANDROID_JPEG_QUALITY);
1270 jpegQuality = entry.data.u8[0];
1271 } else {
1272 return lfail("%s: ANDROID_JPEG_QUALITY not set",__FUNCTION__);
1273 }
1274
1275 if (setting.exists(ANDROID_JPEG_THUMBNAIL_QUALITY)) {
1276 camera_metadata_ro_entry entry =
1277 setting.find(ANDROID_JPEG_THUMBNAIL_QUALITY);
1278 thumbQuality = entry.data.u8[0];
1279 } else {
1280 return lfail(
1281 "%s: ANDROID_JPEG_THUMBNAIL_QUALITY not set",
1282 __FUNCTION__);
1283 }
1284
1285 if (setting.exists(ANDROID_JPEG_THUMBNAIL_SIZE)) {
1286 camera_metadata_ro_entry entry =
1287 setting.find(ANDROID_JPEG_THUMBNAIL_SIZE);
1288 thumbSize = Size { static_cast<uint32_t>(entry.data.i32[0]),
1289 static_cast<uint32_t>(entry.data.i32[1])
1290 };
1291 if (thumbSize.width == 0 && thumbSize.height == 0) {
1292 outputThumbnail = false;
1293 }
1294 } else {
1295 return lfail(
1296 "%s: ANDROID_JPEG_THUMBNAIL_SIZE not set", __FUNCTION__);
1297 }
1298
1299 /* Cropped and scaled YU12 buffer for main and thumbnail */
1300 YCbCrLayout yu12Main;
1301 Size jpegSize { halBuf.width, halBuf.height };
1302
1303 /* Compute temporary buffer sizes accounting for the following:
1304 * thumbnail can't exceed APP1 size of 64K
1305 * main image needs to hold APP1, headers, and at most a poorly
1306 * compressed image */
1307 const ssize_t maxThumbCodeSize = 64 * 1024;
1308 const ssize_t maxJpegCodeSize = mBlobBufferSize == 0 ?
1309 parent->getJpegBufferSize(jpegSize.width, jpegSize.height) :
1310 mBlobBufferSize;
1311
1312 /* Check that getJpegBufferSize did not return an error */
1313 if (maxJpegCodeSize < 0) {
1314 return lfail(
1315 "%s: getJpegBufferSize returned %zd",__FUNCTION__,maxJpegCodeSize);
1316 }
1317
1318
1319 /* Hold actual thumbnail and main image code sizes */
1320 size_t thumbCodeSize = 0, jpegCodeSize = 0;
1321 /* Temporary thumbnail code buffer */
1322 std::vector<uint8_t> thumbCode(outputThumbnail ? maxThumbCodeSize : 0);
1323
1324 YCbCrLayout yu12Thumb;
1325 if (outputThumbnail) {
1326 ret = cropAndScaleThumbLocked(mYu12Frame, thumbSize, &yu12Thumb);
1327
1328 if (ret != 0) {
1329 return lfail(
1330 "%s: crop and scale thumbnail failed!", __FUNCTION__);
1331 }
1332 }
1333
1334 /* Scale and crop main jpeg */
1335 ret = cropAndScaleLocked(mYu12Frame, jpegSize, &yu12Main);
1336
1337 if (ret != 0) {
1338 return lfail("%s: crop and scale main failed!", __FUNCTION__);
1339 }
1340
1341 /* Encode the thumbnail image */
1342 if (outputThumbnail) {
1343 ret = encodeJpegYU12(thumbSize, yu12Thumb,
1344 thumbQuality, 0, 0,
1345 &thumbCode[0], maxThumbCodeSize, thumbCodeSize);
1346
1347 if (ret != 0) {
1348 return lfail("%s: thumbnail encodeJpegYU12 failed with %d",__FUNCTION__, ret);
1349 }
1350 }
1351
1352 /* Combine camera characteristics with request settings to form EXIF
1353 * metadata */
1354 common::V1_0::helper::CameraMetadata meta(mCameraCharacteristics);
1355 meta.append(setting);
1356
1357 /* Generate EXIF object */
1358 std::unique_ptr<ExifUtils> utils(ExifUtils::create());
1359 /* Make sure it's initialized */
1360 utils->initialize();
1361
1362 utils->setFromMetadata(meta, jpegSize.width, jpegSize.height);
1363 utils->setMake(mExifMake);
1364 utils->setModel(mExifModel);
1365
1366 ret = utils->generateApp1(outputThumbnail ? &thumbCode[0] : 0, thumbCodeSize);
1367
1368 if (!ret) {
1369 return lfail("%s: generating APP1 failed", __FUNCTION__);
1370 }
1371
1372 /* Get internal buffer */
1373 size_t exifDataSize = utils->getApp1Length();
1374 const uint8_t* exifData = utils->getApp1Buffer();
1375
1376 /* Lock the HAL jpeg code buffer */
1377 void *bufPtr = sHandleImporter.lock(
1378 *(halBuf.bufPtr), halBuf.usage, maxJpegCodeSize);
1379
1380 if (!bufPtr) {
1381 return lfail("%s: could not lock %zu bytes", __FUNCTION__, maxJpegCodeSize);
1382 }
1383
1384 /* Encode the main jpeg image */
1385 ret = encodeJpegYU12(jpegSize, yu12Main,
1386 jpegQuality, exifData, exifDataSize,
1387 bufPtr, maxJpegCodeSize, jpegCodeSize);
1388
1389 /* TODO: Not sure this belongs here, maybe better to pass jpegCodeSize out
1390 * and do this when returning buffer to parent */
1391 CameraBlob blob { CameraBlobId::JPEG, static_cast<uint32_t>(jpegCodeSize) };
1392 void *blobDst =
1393 reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(bufPtr) +
1394 maxJpegCodeSize -
1395 sizeof(CameraBlob));
1396 memcpy(blobDst, &blob, sizeof(CameraBlob));
1397
1398 /* Unlock the HAL jpeg code buffer */
1399 int relFence = sHandleImporter.unlock(*(halBuf.bufPtr));
1400 if (relFence >= 0) {
1401 halBuf.acquireFence = relFence;
1402 }
1403
1404 /* Check if our JPEG actually succeeded */
1405 if (ret != 0) {
1406 return lfail(
1407 "%s: encodeJpegYU12 failed with %d",__FUNCTION__, ret);
1408 }
1409
1410 ALOGV("%s: encoded JPEG (ret:%d) with Q:%d max size: %zu",
1411 __FUNCTION__, ret, jpegQuality, maxJpegCodeSize);
1412
1413 return 0;
1414 }
1415
threadLoop()1416 bool ExternalCameraDeviceSession::OutputThread::threadLoop() {
1417 std::shared_ptr<HalRequest> req;
1418 auto parent = mParent.promote();
1419 if (parent == nullptr) {
1420 ALOGE("%s: session has been disconnected!", __FUNCTION__);
1421 return false;
1422 }
1423
1424 // TODO: maybe we need to setup a sensor thread to dq/enq v4l frames
1425 // regularly to prevent v4l buffer queue filled with stale buffers
1426 // when app doesn't program a preveiw request
1427 waitForNextRequest(&req);
1428 if (req == nullptr) {
1429 // No new request, wait again
1430 return true;
1431 }
1432
1433 auto onDeviceError = [&](auto... args) {
1434 ALOGE(args...);
1435 parent->notifyError(
1436 req->frameNumber, /*stream*/-1, ErrorCode::ERROR_DEVICE);
1437 signalRequestDone();
1438 return false;
1439 };
1440
1441 if (req->frameIn->mFourcc != V4L2_PIX_FMT_MJPEG && req->frameIn->mFourcc != V4L2_PIX_FMT_Z16) {
1442 return onDeviceError("%s: do not support V4L2 format %c%c%c%c", __FUNCTION__,
1443 req->frameIn->mFourcc & 0xFF,
1444 (req->frameIn->mFourcc >> 8) & 0xFF,
1445 (req->frameIn->mFourcc >> 16) & 0xFF,
1446 (req->frameIn->mFourcc >> 24) & 0xFF);
1447 }
1448
1449 int res = requestBufferStart(req->buffers);
1450 if (res != 0) {
1451 ALOGE("%s: send BufferRequest failed! res %d", __FUNCTION__, res);
1452 return onDeviceError("%s: failed to send buffer request!", __FUNCTION__);
1453 }
1454
1455 std::unique_lock<std::mutex> lk(mBufferLock);
1456 // Convert input V4L2 frame to YU12 of the same size
1457 // TODO: see if we can save some computation by converting to YV12 here
1458 uint8_t* inData;
1459 size_t inDataSize;
1460 if (req->frameIn->getData(&inData, &inDataSize) != 0) {
1461 lk.unlock();
1462 return onDeviceError("%s: V4L2 buffer map failed", __FUNCTION__);
1463 }
1464
1465 // Process camera mute state
1466 auto testPatternMode = req->setting.find(ANDROID_SENSOR_TEST_PATTERN_MODE);
1467 if (testPatternMode.count == 1) {
1468 if (mCameraMuted != (testPatternMode.data.u8[0] != ANDROID_SENSOR_TEST_PATTERN_MODE_OFF)) {
1469 mCameraMuted = !mCameraMuted;
1470 // Get solid color for test pattern, if any was set
1471 if (testPatternMode.data.u8[0] == ANDROID_SENSOR_TEST_PATTERN_MODE_SOLID_COLOR) {
1472 auto entry = req->setting.find(ANDROID_SENSOR_TEST_PATTERN_DATA);
1473 if (entry.count == 4) {
1474 // Update the mute frame if the pattern color has changed
1475 if (memcmp(entry.data.i32, mTestPatternData, sizeof(mTestPatternData)) != 0) {
1476 memcpy(mTestPatternData, entry.data.i32, sizeof(mTestPatternData));
1477 // Fill the mute frame with the solid color, use only 8 MSB of RGGB as RGB
1478 for (int i = 0; i < mMuteTestPatternFrame.size(); i += 3) {
1479 mMuteTestPatternFrame[i] = entry.data.i32[0] >> 24;
1480 mMuteTestPatternFrame[i + 1] = entry.data.i32[1] >> 24;
1481 mMuteTestPatternFrame[i + 2] = entry.data.i32[3] >> 24;
1482 }
1483 }
1484 }
1485 }
1486 }
1487 }
1488
1489 // TODO: in some special case maybe we can decode jpg directly to gralloc output?
1490 if (req->frameIn->mFourcc == V4L2_PIX_FMT_MJPEG) {
1491 ATRACE_BEGIN("MJPGtoI420");
1492 int res = 0;
1493 if (mCameraMuted) {
1494 res = libyuv::ConvertToI420(
1495 mMuteTestPatternFrame.data(), mMuteTestPatternFrame.size(),
1496 static_cast<uint8_t*>(mYu12FrameLayout.y), mYu12FrameLayout.yStride,
1497 static_cast<uint8_t*>(mYu12FrameLayout.cb), mYu12FrameLayout.cStride,
1498 static_cast<uint8_t*>(mYu12FrameLayout.cr), mYu12FrameLayout.cStride, 0, 0,
1499 mYu12Frame->mWidth, mYu12Frame->mHeight, mYu12Frame->mWidth,
1500 mYu12Frame->mHeight, libyuv::kRotate0, libyuv::FOURCC_RAW);
1501 } else {
1502 res = libyuv::MJPGToI420(
1503 inData, inDataSize, static_cast<uint8_t*>(mYu12FrameLayout.y),
1504 mYu12FrameLayout.yStride, static_cast<uint8_t*>(mYu12FrameLayout.cb),
1505 mYu12FrameLayout.cStride, static_cast<uint8_t*>(mYu12FrameLayout.cr),
1506 mYu12FrameLayout.cStride, mYu12Frame->mWidth, mYu12Frame->mHeight,
1507 mYu12Frame->mWidth, mYu12Frame->mHeight);
1508 }
1509 ATRACE_END();
1510
1511 if (res != 0) {
1512 // For some webcam, the first few V4L2 frames might be malformed...
1513 ALOGE("%s: Convert V4L2 frame to YU12 failed! res %d", __FUNCTION__, res);
1514 lk.unlock();
1515 Status st = parent->processCaptureRequestError(req);
1516 if (st != Status::OK) {
1517 return onDeviceError("%s: failed to process capture request error!", __FUNCTION__);
1518 }
1519 signalRequestDone();
1520 return true;
1521 }
1522 }
1523
1524 ATRACE_BEGIN("Wait for BufferRequest done");
1525 res = waitForBufferRequestDone(&req->buffers);
1526 ATRACE_END();
1527
1528 if (res != 0) {
1529 ALOGE("%s: wait for BufferRequest done failed! res %d", __FUNCTION__, res);
1530 lk.unlock();
1531 return onDeviceError("%s: failed to process buffer request error!", __FUNCTION__);
1532 }
1533
1534 ALOGV("%s processing new request", __FUNCTION__);
1535 const int kSyncWaitTimeoutMs = 500;
1536 for (auto& halBuf : req->buffers) {
1537 if (*(halBuf.bufPtr) == nullptr) {
1538 ALOGW("%s: buffer for stream %d missing", __FUNCTION__, halBuf.streamId);
1539 halBuf.fenceTimeout = true;
1540 } else if (halBuf.acquireFence >= 0) {
1541 int ret = sync_wait(halBuf.acquireFence, kSyncWaitTimeoutMs);
1542 if (ret) {
1543 halBuf.fenceTimeout = true;
1544 } else {
1545 ::close(halBuf.acquireFence);
1546 halBuf.acquireFence = -1;
1547 }
1548 }
1549
1550 if (halBuf.fenceTimeout) {
1551 continue;
1552 }
1553
1554 // Gralloc lockYCbCr the buffer
1555 switch (halBuf.format) {
1556 case PixelFormat::BLOB: {
1557 int ret = createJpegLocked(halBuf, req->setting);
1558
1559 if(ret != 0) {
1560 lk.unlock();
1561 return onDeviceError("%s: createJpegLocked failed with %d",
1562 __FUNCTION__, ret);
1563 }
1564 } break;
1565 case PixelFormat::Y16: {
1566 void* outLayout = sHandleImporter.lock(*(halBuf.bufPtr), halBuf.usage, inDataSize);
1567
1568 std::memcpy(outLayout, inData, inDataSize);
1569
1570 int relFence = sHandleImporter.unlock(*(halBuf.bufPtr));
1571 if (relFence >= 0) {
1572 halBuf.acquireFence = relFence;
1573 }
1574 } break;
1575 case PixelFormat::YCBCR_420_888:
1576 case PixelFormat::YV12: {
1577 IMapper::Rect outRect {0, 0,
1578 static_cast<int32_t>(halBuf.width),
1579 static_cast<int32_t>(halBuf.height)};
1580 YCbCrLayout outLayout = sHandleImporter.lockYCbCr(
1581 *(halBuf.bufPtr), halBuf.usage, outRect);
1582 ALOGV("%s: outLayout y %p cb %p cr %p y_str %d c_str %d c_step %d",
1583 __FUNCTION__, outLayout.y, outLayout.cb, outLayout.cr,
1584 outLayout.yStride, outLayout.cStride, outLayout.chromaStep);
1585
1586 // Convert to output buffer size/format
1587 uint32_t outputFourcc = getFourCcFromLayout(outLayout);
1588 ALOGV("%s: converting to format %c%c%c%c", __FUNCTION__,
1589 outputFourcc & 0xFF,
1590 (outputFourcc >> 8) & 0xFF,
1591 (outputFourcc >> 16) & 0xFF,
1592 (outputFourcc >> 24) & 0xFF);
1593
1594 YCbCrLayout cropAndScaled;
1595 ATRACE_BEGIN("cropAndScaleLocked");
1596 int ret = cropAndScaleLocked(
1597 mYu12Frame,
1598 Size { halBuf.width, halBuf.height },
1599 &cropAndScaled);
1600 ATRACE_END();
1601 if (ret != 0) {
1602 lk.unlock();
1603 return onDeviceError("%s: crop and scale failed!", __FUNCTION__);
1604 }
1605
1606 Size sz {halBuf.width, halBuf.height};
1607 ATRACE_BEGIN("formatConvert");
1608 ret = formatConvert(cropAndScaled, outLayout, sz, outputFourcc);
1609 ATRACE_END();
1610 if (ret != 0) {
1611 lk.unlock();
1612 return onDeviceError("%s: format coversion failed!", __FUNCTION__);
1613 }
1614 int relFence = sHandleImporter.unlock(*(halBuf.bufPtr));
1615 if (relFence >= 0) {
1616 halBuf.acquireFence = relFence;
1617 }
1618 } break;
1619 default:
1620 lk.unlock();
1621 return onDeviceError("%s: unknown output format %x", __FUNCTION__, halBuf.format);
1622 }
1623 } // for each buffer
1624 mScaledYu12Frames.clear();
1625
1626 // Don't hold the lock while calling back to parent
1627 lk.unlock();
1628 Status st = parent->processCaptureResult(req);
1629 if (st != Status::OK) {
1630 return onDeviceError("%s: failed to process capture result!", __FUNCTION__);
1631 }
1632 signalRequestDone();
1633 return true;
1634 }
1635
allocateIntermediateBuffers(const Size & v4lSize,const Size & thumbSize,const hidl_vec<Stream> & streams,uint32_t blobBufferSize)1636 Status ExternalCameraDeviceSession::OutputThread::allocateIntermediateBuffers(
1637 const Size& v4lSize, const Size& thumbSize,
1638 const hidl_vec<Stream>& streams,
1639 uint32_t blobBufferSize) {
1640 std::lock_guard<std::mutex> lk(mBufferLock);
1641 if (mScaledYu12Frames.size() != 0) {
1642 ALOGE("%s: intermediate buffer pool has %zu inflight buffers! (expect 0)",
1643 __FUNCTION__, mScaledYu12Frames.size());
1644 return Status::INTERNAL_ERROR;
1645 }
1646
1647 // Allocating intermediate YU12 frame
1648 if (mYu12Frame == nullptr || mYu12Frame->mWidth != v4lSize.width ||
1649 mYu12Frame->mHeight != v4lSize.height) {
1650 mYu12Frame.clear();
1651 mYu12Frame = new AllocatedFrame(v4lSize.width, v4lSize.height);
1652 int ret = mYu12Frame->allocate(&mYu12FrameLayout);
1653 if (ret != 0) {
1654 ALOGE("%s: allocating YU12 frame failed!", __FUNCTION__);
1655 return Status::INTERNAL_ERROR;
1656 }
1657 }
1658
1659 // Allocating intermediate YU12 thumbnail frame
1660 if (mYu12ThumbFrame == nullptr ||
1661 mYu12ThumbFrame->mWidth != thumbSize.width ||
1662 mYu12ThumbFrame->mHeight != thumbSize.height) {
1663 mYu12ThumbFrame.clear();
1664 mYu12ThumbFrame = new AllocatedFrame(thumbSize.width, thumbSize.height);
1665 int ret = mYu12ThumbFrame->allocate(&mYu12ThumbFrameLayout);
1666 if (ret != 0) {
1667 ALOGE("%s: allocating YU12 thumb frame failed!", __FUNCTION__);
1668 return Status::INTERNAL_ERROR;
1669 }
1670 }
1671
1672 // Allocating scaled buffers
1673 for (const auto& stream : streams) {
1674 Size sz = {stream.width, stream.height};
1675 if (sz == v4lSize) {
1676 continue; // Don't need an intermediate buffer same size as v4lBuffer
1677 }
1678 if (mIntermediateBuffers.count(sz) == 0) {
1679 // Create new intermediate buffer
1680 sp<AllocatedFrame> buf = new AllocatedFrame(stream.width, stream.height);
1681 int ret = buf->allocate();
1682 if (ret != 0) {
1683 ALOGE("%s: allocating intermediate YU12 frame %dx%d failed!",
1684 __FUNCTION__, stream.width, stream.height);
1685 return Status::INTERNAL_ERROR;
1686 }
1687 mIntermediateBuffers[sz] = buf;
1688 }
1689 }
1690
1691 // Remove unconfigured buffers
1692 auto it = mIntermediateBuffers.begin();
1693 while (it != mIntermediateBuffers.end()) {
1694 bool configured = false;
1695 auto sz = it->first;
1696 for (const auto& stream : streams) {
1697 if (stream.width == sz.width && stream.height == sz.height) {
1698 configured = true;
1699 break;
1700 }
1701 }
1702 if (configured) {
1703 it++;
1704 } else {
1705 it = mIntermediateBuffers.erase(it);
1706 }
1707 }
1708
1709 // Allocate mute test pattern frame
1710 mMuteTestPatternFrame.resize(mYu12Frame->mWidth * mYu12Frame->mHeight * 3);
1711
1712 mBlobBufferSize = blobBufferSize;
1713 return Status::OK;
1714 }
1715
clearIntermediateBuffers()1716 void ExternalCameraDeviceSession::OutputThread::clearIntermediateBuffers() {
1717 std::lock_guard<std::mutex> lk(mBufferLock);
1718 mYu12Frame.clear();
1719 mYu12ThumbFrame.clear();
1720 mIntermediateBuffers.clear();
1721 mMuteTestPatternFrame.clear();
1722 mBlobBufferSize = 0;
1723 }
1724
submitRequest(const std::shared_ptr<HalRequest> & req)1725 Status ExternalCameraDeviceSession::OutputThread::submitRequest(
1726 const std::shared_ptr<HalRequest>& req) {
1727 std::unique_lock<std::mutex> lk(mRequestListLock);
1728 mRequestList.push_back(req);
1729 lk.unlock();
1730 mRequestCond.notify_one();
1731 return Status::OK;
1732 }
1733
flush()1734 void ExternalCameraDeviceSession::OutputThread::flush() {
1735 ATRACE_CALL();
1736 auto parent = mParent.promote();
1737 if (parent == nullptr) {
1738 ALOGE("%s: session has been disconnected!", __FUNCTION__);
1739 return;
1740 }
1741
1742 std::unique_lock<std::mutex> lk(mRequestListLock);
1743 std::list<std::shared_ptr<HalRequest>> reqs = std::move(mRequestList);
1744 mRequestList.clear();
1745 if (mProcessingRequest) {
1746 std::chrono::seconds timeout = std::chrono::seconds(kFlushWaitTimeoutSec);
1747 auto st = mRequestDoneCond.wait_for(lk, timeout);
1748 if (st == std::cv_status::timeout) {
1749 ALOGE("%s: wait for inflight request finish timeout!", __FUNCTION__);
1750 }
1751 }
1752
1753 ALOGV("%s: flusing inflight requests", __FUNCTION__);
1754 lk.unlock();
1755 for (const auto& req : reqs) {
1756 parent->processCaptureRequestError(req);
1757 }
1758 }
1759
1760 std::list<std::shared_ptr<HalRequest>>
switchToOffline()1761 ExternalCameraDeviceSession::OutputThread::switchToOffline() {
1762 ATRACE_CALL();
1763 std::list<std::shared_ptr<HalRequest>> emptyList;
1764 auto parent = mParent.promote();
1765 if (parent == nullptr) {
1766 ALOGE("%s: session has been disconnected!", __FUNCTION__);
1767 return emptyList;
1768 }
1769
1770 std::unique_lock<std::mutex> lk(mRequestListLock);
1771 std::list<std::shared_ptr<HalRequest>> reqs = std::move(mRequestList);
1772 mRequestList.clear();
1773 if (mProcessingRequest) {
1774 std::chrono::seconds timeout = std::chrono::seconds(kFlushWaitTimeoutSec);
1775 auto st = mRequestDoneCond.wait_for(lk, timeout);
1776 if (st == std::cv_status::timeout) {
1777 ALOGE("%s: wait for inflight request finish timeout!", __FUNCTION__);
1778 }
1779 }
1780 lk.unlock();
1781 clearIntermediateBuffers();
1782 ALOGV("%s: returning %zu request for offline processing", __FUNCTION__, reqs.size());
1783 return reqs;
1784 }
1785
waitForNextRequest(std::shared_ptr<HalRequest> * out)1786 void ExternalCameraDeviceSession::OutputThread::waitForNextRequest(
1787 std::shared_ptr<HalRequest>* out) {
1788 ATRACE_CALL();
1789 if (out == nullptr) {
1790 ALOGE("%s: out is null", __FUNCTION__);
1791 return;
1792 }
1793
1794 std::unique_lock<std::mutex> lk(mRequestListLock);
1795 int waitTimes = 0;
1796 while (mRequestList.empty()) {
1797 if (exitPending()) {
1798 return;
1799 }
1800 std::chrono::milliseconds timeout = std::chrono::milliseconds(kReqWaitTimeoutMs);
1801 auto st = mRequestCond.wait_for(lk, timeout);
1802 if (st == std::cv_status::timeout) {
1803 waitTimes++;
1804 if (waitTimes == kReqWaitTimesMax) {
1805 // no new request, return
1806 return;
1807 }
1808 }
1809 }
1810 *out = mRequestList.front();
1811 mRequestList.pop_front();
1812 mProcessingRequest = true;
1813 mProcessingFrameNumer = (*out)->frameNumber;
1814 }
1815
signalRequestDone()1816 void ExternalCameraDeviceSession::OutputThread::signalRequestDone() {
1817 std::unique_lock<std::mutex> lk(mRequestListLock);
1818 mProcessingRequest = false;
1819 mProcessingFrameNumer = 0;
1820 lk.unlock();
1821 mRequestDoneCond.notify_one();
1822 }
1823
dump(int fd)1824 void ExternalCameraDeviceSession::OutputThread::dump(int fd) {
1825 std::lock_guard<std::mutex> lk(mRequestListLock);
1826 if (mProcessingRequest) {
1827 dprintf(fd, "OutputThread processing frame %d\n", mProcessingFrameNumer);
1828 } else {
1829 dprintf(fd, "OutputThread not processing any frames\n");
1830 }
1831 dprintf(fd, "OutputThread request list contains frame: ");
1832 for (const auto& req : mRequestList) {
1833 dprintf(fd, "%d, ", req->frameNumber);
1834 }
1835 dprintf(fd, "\n");
1836 }
1837
cleanupBuffersLocked(int id)1838 void ExternalCameraDeviceSession::cleanupBuffersLocked(int id) {
1839 for (auto& pair : mCirculatingBuffers.at(id)) {
1840 sHandleImporter.freeBuffer(pair.second);
1841 }
1842 mCirculatingBuffers[id].clear();
1843 mCirculatingBuffers.erase(id);
1844 }
1845
updateBufferCaches(const hidl_vec<BufferCache> & cachesToRemove)1846 void ExternalCameraDeviceSession::updateBufferCaches(const hidl_vec<BufferCache>& cachesToRemove) {
1847 Mutex::Autolock _l(mCbsLock);
1848 for (auto& cache : cachesToRemove) {
1849 auto cbsIt = mCirculatingBuffers.find(cache.streamId);
1850 if (cbsIt == mCirculatingBuffers.end()) {
1851 // The stream could have been removed
1852 continue;
1853 }
1854 CirculatingBuffers& cbs = cbsIt->second;
1855 auto it = cbs.find(cache.bufferId);
1856 if (it != cbs.end()) {
1857 sHandleImporter.freeBuffer(it->second);
1858 cbs.erase(it);
1859 } else {
1860 ALOGE("%s: stream %d buffer %" PRIu64 " is not cached",
1861 __FUNCTION__, cache.streamId, cache.bufferId);
1862 }
1863 }
1864 }
1865
isSupported(const Stream & stream,const std::vector<SupportedV4L2Format> & supportedFormats,const ExternalCameraConfig & devCfg)1866 bool ExternalCameraDeviceSession::isSupported(const Stream& stream,
1867 const std::vector<SupportedV4L2Format>& supportedFormats,
1868 const ExternalCameraConfig& devCfg) {
1869 int32_t ds = static_cast<int32_t>(stream.dataSpace);
1870 PixelFormat fmt = stream.format;
1871 uint32_t width = stream.width;
1872 uint32_t height = stream.height;
1873 // TODO: check usage flags
1874
1875 if (stream.streamType != StreamType::OUTPUT) {
1876 ALOGE("%s: does not support non-output stream type", __FUNCTION__);
1877 return false;
1878 }
1879
1880 if (stream.rotation != StreamRotation::ROTATION_0) {
1881 ALOGE("%s: does not support stream rotation", __FUNCTION__);
1882 return false;
1883 }
1884
1885 switch (fmt) {
1886 case PixelFormat::BLOB:
1887 if (ds != static_cast<int32_t>(Dataspace::V0_JFIF)) {
1888 ALOGI("%s: BLOB format does not support dataSpace %x", __FUNCTION__, ds);
1889 return false;
1890 }
1891 break;
1892 case PixelFormat::IMPLEMENTATION_DEFINED:
1893 case PixelFormat::YCBCR_420_888:
1894 case PixelFormat::YV12:
1895 // TODO: check what dataspace we can support here.
1896 // intentional no-ops.
1897 break;
1898 case PixelFormat::Y16:
1899 if (!devCfg.depthEnabled) {
1900 ALOGI("%s: Depth is not Enabled", __FUNCTION__);
1901 return false;
1902 }
1903 if (!(ds & Dataspace::DEPTH)) {
1904 ALOGI("%s: Y16 supports only dataSpace DEPTH", __FUNCTION__);
1905 return false;
1906 }
1907 break;
1908 default:
1909 ALOGI("%s: does not support format %x", __FUNCTION__, fmt);
1910 return false;
1911 }
1912
1913 // Assume we can convert any V4L2 format to any of supported output format for now, i.e,
1914 // ignoring v4l2Fmt.fourcc for now. Might need more subtle check if we support more v4l format
1915 // in the futrue.
1916 for (const auto& v4l2Fmt : supportedFormats) {
1917 if (width == v4l2Fmt.width && height == v4l2Fmt.height) {
1918 return true;
1919 }
1920 }
1921 ALOGI("%s: resolution %dx%d is not supported", __FUNCTION__, width, height);
1922 return false;
1923 }
1924
v4l2StreamOffLocked()1925 int ExternalCameraDeviceSession::v4l2StreamOffLocked() {
1926 if (!mV4l2Streaming) {
1927 return OK;
1928 }
1929
1930 {
1931 std::lock_guard<std::mutex> lk(mV4l2BufferLock);
1932 if (mNumDequeuedV4l2Buffers != 0) {
1933 ALOGE("%s: there are %zu inflight V4L buffers",
1934 __FUNCTION__, mNumDequeuedV4l2Buffers);
1935 return -1;
1936 }
1937 }
1938 mV4L2BufferCount = 0;
1939
1940 // VIDIOC_STREAMOFF
1941 v4l2_buf_type capture_type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
1942 if (TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_STREAMOFF, &capture_type)) < 0) {
1943 ALOGE("%s: STREAMOFF failed: %s", __FUNCTION__, strerror(errno));
1944 return -errno;
1945 }
1946
1947 // VIDIOC_REQBUFS: clear buffers
1948 v4l2_requestbuffers req_buffers{};
1949 req_buffers.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
1950 req_buffers.memory = V4L2_MEMORY_MMAP;
1951 req_buffers.count = 0;
1952 if (TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_REQBUFS, &req_buffers)) < 0) {
1953 ALOGE("%s: REQBUFS failed: %s", __FUNCTION__, strerror(errno));
1954 return -errno;
1955 }
1956
1957 mV4l2Streaming = false;
1958 return OK;
1959 }
1960
setV4l2FpsLocked(double fps)1961 int ExternalCameraDeviceSession::setV4l2FpsLocked(double fps) {
1962 // VIDIOC_G_PARM/VIDIOC_S_PARM: set fps
1963 v4l2_streamparm streamparm = { .type = V4L2_BUF_TYPE_VIDEO_CAPTURE };
1964 // The following line checks that the driver knows about framerate get/set.
1965 int ret = TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_G_PARM, &streamparm));
1966 if (ret != 0) {
1967 if (errno == -EINVAL) {
1968 ALOGW("%s: device does not support VIDIOC_G_PARM", __FUNCTION__);
1969 }
1970 return -errno;
1971 }
1972 // Now check if the device is able to accept a capture framerate set.
1973 if (!(streamparm.parm.capture.capability & V4L2_CAP_TIMEPERFRAME)) {
1974 ALOGW("%s: device does not support V4L2_CAP_TIMEPERFRAME", __FUNCTION__);
1975 return -EINVAL;
1976 }
1977
1978 // fps is float, approximate by a fraction.
1979 const int kFrameRatePrecision = 10000;
1980 streamparm.parm.capture.timeperframe.numerator = kFrameRatePrecision;
1981 streamparm.parm.capture.timeperframe.denominator =
1982 (fps * kFrameRatePrecision);
1983
1984 if (TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_S_PARM, &streamparm)) < 0) {
1985 ALOGE("%s: failed to set framerate to %f: %s", __FUNCTION__, fps, strerror(errno));
1986 return -1;
1987 }
1988
1989 double retFps = streamparm.parm.capture.timeperframe.denominator /
1990 static_cast<double>(streamparm.parm.capture.timeperframe.numerator);
1991 if (std::fabs(fps - retFps) > 1.0) {
1992 ALOGE("%s: expect fps %f, got %f instead", __FUNCTION__, fps, retFps);
1993 return -1;
1994 }
1995 mV4l2StreamingFps = fps;
1996 return 0;
1997 }
1998
configureV4l2StreamLocked(const SupportedV4L2Format & v4l2Fmt,double requestFps)1999 int ExternalCameraDeviceSession::configureV4l2StreamLocked(
2000 const SupportedV4L2Format& v4l2Fmt, double requestFps) {
2001 ATRACE_CALL();
2002 int ret = v4l2StreamOffLocked();
2003 if (ret != OK) {
2004 ALOGE("%s: stop v4l2 streaming failed: ret %d", __FUNCTION__, ret);
2005 return ret;
2006 }
2007
2008 // VIDIOC_S_FMT w/h/fmt
2009 v4l2_format fmt;
2010 fmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
2011 fmt.fmt.pix.width = v4l2Fmt.width;
2012 fmt.fmt.pix.height = v4l2Fmt.height;
2013 fmt.fmt.pix.pixelformat = v4l2Fmt.fourcc;
2014 ret = TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_S_FMT, &fmt));
2015 if (ret < 0) {
2016 int numAttempt = 0;
2017 while (ret < 0) {
2018 ALOGW("%s: VIDIOC_S_FMT failed, wait 33ms and try again", __FUNCTION__);
2019 usleep(IOCTL_RETRY_SLEEP_US); // sleep and try again
2020 ret = TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_S_FMT, &fmt));
2021 if (numAttempt == MAX_RETRY) {
2022 break;
2023 }
2024 numAttempt++;
2025 }
2026 if (ret < 0) {
2027 ALOGE("%s: S_FMT ioctl failed: %s", __FUNCTION__, strerror(errno));
2028 return -errno;
2029 }
2030 }
2031
2032 if (v4l2Fmt.width != fmt.fmt.pix.width || v4l2Fmt.height != fmt.fmt.pix.height ||
2033 v4l2Fmt.fourcc != fmt.fmt.pix.pixelformat) {
2034 ALOGE("%s: S_FMT expect %c%c%c%c %dx%d, got %c%c%c%c %dx%d instead!", __FUNCTION__,
2035 v4l2Fmt.fourcc & 0xFF,
2036 (v4l2Fmt.fourcc >> 8) & 0xFF,
2037 (v4l2Fmt.fourcc >> 16) & 0xFF,
2038 (v4l2Fmt.fourcc >> 24) & 0xFF,
2039 v4l2Fmt.width, v4l2Fmt.height,
2040 fmt.fmt.pix.pixelformat & 0xFF,
2041 (fmt.fmt.pix.pixelformat >> 8) & 0xFF,
2042 (fmt.fmt.pix.pixelformat >> 16) & 0xFF,
2043 (fmt.fmt.pix.pixelformat >> 24) & 0xFF,
2044 fmt.fmt.pix.width, fmt.fmt.pix.height);
2045 return -EINVAL;
2046 }
2047 uint32_t bufferSize = fmt.fmt.pix.sizeimage;
2048 ALOGI("%s: V4L2 buffer size is %d", __FUNCTION__, bufferSize);
2049 uint32_t expectedMaxBufferSize = kMaxBytesPerPixel * fmt.fmt.pix.width * fmt.fmt.pix.height;
2050 if ((bufferSize == 0) || (bufferSize > expectedMaxBufferSize)) {
2051 ALOGE("%s: V4L2 buffer size: %u looks invalid. Expected maximum size: %u", __FUNCTION__,
2052 bufferSize, expectedMaxBufferSize);
2053 return -EINVAL;
2054 }
2055 mMaxV4L2BufferSize = bufferSize;
2056
2057 const double kDefaultFps = 30.0;
2058 double fps = 1000.0;
2059 if (requestFps != 0.0) {
2060 fps = requestFps;
2061 } else {
2062 double maxFps = -1.0;
2063 // Try to pick the slowest fps that is at least 30
2064 for (const auto& fr : v4l2Fmt.frameRates) {
2065 double f = fr.getDouble();
2066 if (maxFps < f) {
2067 maxFps = f;
2068 }
2069 if (f >= kDefaultFps && f < fps) {
2070 fps = f;
2071 }
2072 }
2073 if (fps == 1000.0) {
2074 fps = maxFps;
2075 }
2076 }
2077
2078 int fpsRet = setV4l2FpsLocked(fps);
2079 if (fpsRet != 0 && fpsRet != -EINVAL) {
2080 ALOGE("%s: set fps failed: %s", __FUNCTION__, strerror(fpsRet));
2081 return fpsRet;
2082 }
2083
2084 uint32_t v4lBufferCount = (fps >= kDefaultFps) ?
2085 mCfg.numVideoBuffers : mCfg.numStillBuffers;
2086 // VIDIOC_REQBUFS: create buffers
2087 v4l2_requestbuffers req_buffers{};
2088 req_buffers.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
2089 req_buffers.memory = V4L2_MEMORY_MMAP;
2090 req_buffers.count = v4lBufferCount;
2091 if (TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_REQBUFS, &req_buffers)) < 0) {
2092 ALOGE("%s: VIDIOC_REQBUFS failed: %s", __FUNCTION__, strerror(errno));
2093 return -errno;
2094 }
2095
2096 // Driver can indeed return more buffer if it needs more to operate
2097 if (req_buffers.count < v4lBufferCount) {
2098 ALOGE("%s: VIDIOC_REQBUFS expected %d buffers, got %d instead",
2099 __FUNCTION__, v4lBufferCount, req_buffers.count);
2100 return NO_MEMORY;
2101 }
2102
2103 // VIDIOC_QUERYBUF: get buffer offset in the V4L2 fd
2104 // VIDIOC_QBUF: send buffer to driver
2105 mV4L2BufferCount = req_buffers.count;
2106 for (uint32_t i = 0; i < req_buffers.count; i++) {
2107 v4l2_buffer buffer = {
2108 .index = i, .type = V4L2_BUF_TYPE_VIDEO_CAPTURE, .memory = V4L2_MEMORY_MMAP};
2109
2110 if (TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_QUERYBUF, &buffer)) < 0) {
2111 ALOGE("%s: QUERYBUF %d failed: %s", __FUNCTION__, i, strerror(errno));
2112 return -errno;
2113 }
2114
2115 if (TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_QBUF, &buffer)) < 0) {
2116 ALOGE("%s: QBUF %d failed: %s", __FUNCTION__, i, strerror(errno));
2117 return -errno;
2118 }
2119 }
2120
2121 // VIDIOC_STREAMON: start streaming
2122 v4l2_buf_type capture_type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
2123 ret = TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_STREAMON, &capture_type));
2124 if (ret < 0) {
2125 int numAttempt = 0;
2126 while (ret < 0) {
2127 ALOGW("%s: VIDIOC_STREAMON failed, wait 33ms and try again", __FUNCTION__);
2128 usleep(IOCTL_RETRY_SLEEP_US); // sleep 100 ms and try again
2129 ret = TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_STREAMON, &capture_type));
2130 if (numAttempt == MAX_RETRY) {
2131 break;
2132 }
2133 numAttempt++;
2134 }
2135 if (ret < 0) {
2136 ALOGE("%s: VIDIOC_STREAMON ioctl failed: %s", __FUNCTION__, strerror(errno));
2137 return -errno;
2138 }
2139 }
2140
2141 // Swallow first few frames after streamOn to account for bad frames from some devices
2142 for (int i = 0; i < kBadFramesAfterStreamOn; i++) {
2143 v4l2_buffer buffer{};
2144 buffer.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
2145 buffer.memory = V4L2_MEMORY_MMAP;
2146 if (TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_DQBUF, &buffer)) < 0) {
2147 ALOGE("%s: DQBUF fails: %s", __FUNCTION__, strerror(errno));
2148 return -errno;
2149 }
2150
2151 if (TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_QBUF, &buffer)) < 0) {
2152 ALOGE("%s: QBUF index %d fails: %s", __FUNCTION__, buffer.index, strerror(errno));
2153 return -errno;
2154 }
2155 }
2156
2157 ALOGI("%s: start V4L2 streaming %dx%d@%ffps",
2158 __FUNCTION__, v4l2Fmt.width, v4l2Fmt.height, fps);
2159 mV4l2StreamingFmt = v4l2Fmt;
2160 mV4l2Streaming = true;
2161 return OK;
2162 }
2163
dequeueV4l2FrameLocked(nsecs_t * shutterTs)2164 sp<V4L2Frame> ExternalCameraDeviceSession::dequeueV4l2FrameLocked(/*out*/nsecs_t* shutterTs) {
2165 ATRACE_CALL();
2166 sp<V4L2Frame> ret = nullptr;
2167
2168 if (shutterTs == nullptr) {
2169 ALOGE("%s: shutterTs must not be null!", __FUNCTION__);
2170 return ret;
2171 }
2172
2173 {
2174 std::unique_lock<std::mutex> lk(mV4l2BufferLock);
2175 if (mNumDequeuedV4l2Buffers == mV4L2BufferCount) {
2176 int waitRet = waitForV4L2BufferReturnLocked(lk);
2177 if (waitRet != 0) {
2178 return ret;
2179 }
2180 }
2181 }
2182
2183 ATRACE_BEGIN("VIDIOC_DQBUF");
2184 v4l2_buffer buffer{};
2185 buffer.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
2186 buffer.memory = V4L2_MEMORY_MMAP;
2187 if (TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_DQBUF, &buffer)) < 0) {
2188 ALOGE("%s: DQBUF fails: %s", __FUNCTION__, strerror(errno));
2189 return ret;
2190 }
2191 ATRACE_END();
2192
2193 if (buffer.index >= mV4L2BufferCount) {
2194 ALOGE("%s: Invalid buffer id: %d", __FUNCTION__, buffer.index);
2195 return ret;
2196 }
2197
2198 if (buffer.flags & V4L2_BUF_FLAG_ERROR) {
2199 ALOGE("%s: v4l2 buf error! buf flag 0x%x", __FUNCTION__, buffer.flags);
2200 // TODO: try to dequeue again
2201 }
2202
2203 if (buffer.bytesused > mMaxV4L2BufferSize) {
2204 ALOGE("%s: v4l2 buffer bytes used: %u maximum %u", __FUNCTION__, buffer.bytesused,
2205 mMaxV4L2BufferSize);
2206 return ret;
2207 }
2208
2209 if (buffer.flags & V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC) {
2210 // Ideally we should also check for V4L2_BUF_FLAG_TSTAMP_SRC_SOE, but
2211 // even V4L2_BUF_FLAG_TSTAMP_SRC_EOF is better than capture a timestamp now
2212 *shutterTs = static_cast<nsecs_t>(buffer.timestamp.tv_sec)*1000000000LL +
2213 buffer.timestamp.tv_usec * 1000LL;
2214 } else {
2215 *shutterTs = systemTime(SYSTEM_TIME_MONOTONIC);
2216 }
2217
2218 {
2219 std::lock_guard<std::mutex> lk(mV4l2BufferLock);
2220 mNumDequeuedV4l2Buffers++;
2221 }
2222 return new V4L2Frame(
2223 mV4l2StreamingFmt.width, mV4l2StreamingFmt.height, mV4l2StreamingFmt.fourcc,
2224 buffer.index, mV4l2Fd.get(), buffer.bytesused, buffer.m.offset);
2225 }
2226
enqueueV4l2Frame(const sp<V4L2Frame> & frame)2227 void ExternalCameraDeviceSession::enqueueV4l2Frame(const sp<V4L2Frame>& frame) {
2228 ATRACE_CALL();
2229 frame->unmap();
2230 ATRACE_BEGIN("VIDIOC_QBUF");
2231 v4l2_buffer buffer{};
2232 buffer.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
2233 buffer.memory = V4L2_MEMORY_MMAP;
2234 buffer.index = frame->mBufferIndex;
2235 if (TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_QBUF, &buffer)) < 0) {
2236 ALOGE("%s: QBUF index %d fails: %s", __FUNCTION__,
2237 frame->mBufferIndex, strerror(errno));
2238 return;
2239 }
2240 ATRACE_END();
2241
2242 {
2243 std::lock_guard<std::mutex> lk(mV4l2BufferLock);
2244 mNumDequeuedV4l2Buffers--;
2245 }
2246 mV4L2BufferReturned.notify_one();
2247 }
2248
isStreamCombinationSupported(const V3_2::StreamConfiguration & config,const std::vector<SupportedV4L2Format> & supportedFormats,const ExternalCameraConfig & devCfg)2249 Status ExternalCameraDeviceSession::isStreamCombinationSupported(
2250 const V3_2::StreamConfiguration& config,
2251 const std::vector<SupportedV4L2Format>& supportedFormats,
2252 const ExternalCameraConfig& devCfg) {
2253 if (config.operationMode != StreamConfigurationMode::NORMAL_MODE) {
2254 ALOGE("%s: unsupported operation mode: %d", __FUNCTION__, config.operationMode);
2255 return Status::ILLEGAL_ARGUMENT;
2256 }
2257
2258 if (config.streams.size() == 0) {
2259 ALOGE("%s: cannot configure zero stream", __FUNCTION__);
2260 return Status::ILLEGAL_ARGUMENT;
2261 }
2262
2263 int numProcessedStream = 0;
2264 int numStallStream = 0;
2265 for (const auto& stream : config.streams) {
2266 // Check if the format/width/height combo is supported
2267 if (!isSupported(stream, supportedFormats, devCfg)) {
2268 return Status::ILLEGAL_ARGUMENT;
2269 }
2270 if (stream.format == PixelFormat::BLOB) {
2271 numStallStream++;
2272 } else {
2273 numProcessedStream++;
2274 }
2275 }
2276
2277 if (numProcessedStream > kMaxProcessedStream) {
2278 ALOGE("%s: too many processed streams (expect <= %d, got %d)", __FUNCTION__,
2279 kMaxProcessedStream, numProcessedStream);
2280 return Status::ILLEGAL_ARGUMENT;
2281 }
2282
2283 if (numStallStream > kMaxStallStream) {
2284 ALOGE("%s: too many stall streams (expect <= %d, got %d)", __FUNCTION__,
2285 kMaxStallStream, numStallStream);
2286 return Status::ILLEGAL_ARGUMENT;
2287 }
2288
2289 return Status::OK;
2290 }
2291
configureStreams(const V3_2::StreamConfiguration & config,V3_3::HalStreamConfiguration * out,uint32_t blobBufferSize)2292 Status ExternalCameraDeviceSession::configureStreams(
2293 const V3_2::StreamConfiguration& config,
2294 V3_3::HalStreamConfiguration* out,
2295 uint32_t blobBufferSize) {
2296 ATRACE_CALL();
2297
2298 Status status = isStreamCombinationSupported(config, mSupportedFormats, mCfg);
2299 if (status != Status::OK) {
2300 return status;
2301 }
2302
2303 status = initStatus();
2304 if (status != Status::OK) {
2305 return status;
2306 }
2307
2308
2309 {
2310 std::lock_guard<std::mutex> lk(mInflightFramesLock);
2311 if (!mInflightFrames.empty()) {
2312 ALOGE("%s: trying to configureStreams while there are still %zu inflight frames!",
2313 __FUNCTION__, mInflightFrames.size());
2314 return Status::INTERNAL_ERROR;
2315 }
2316 }
2317
2318 Mutex::Autolock _l(mLock);
2319 {
2320 Mutex::Autolock _l(mCbsLock);
2321 // Add new streams
2322 for (const auto& stream : config.streams) {
2323 if (mStreamMap.count(stream.id) == 0) {
2324 mStreamMap[stream.id] = stream;
2325 mCirculatingBuffers.emplace(stream.id, CirculatingBuffers{});
2326 }
2327 }
2328
2329 // Cleanup removed streams
2330 for(auto it = mStreamMap.begin(); it != mStreamMap.end();) {
2331 int id = it->first;
2332 bool found = false;
2333 for (const auto& stream : config.streams) {
2334 if (id == stream.id) {
2335 found = true;
2336 break;
2337 }
2338 }
2339 if (!found) {
2340 // Unmap all buffers of deleted stream
2341 cleanupBuffersLocked(id);
2342 it = mStreamMap.erase(it);
2343 } else {
2344 ++it;
2345 }
2346 }
2347 }
2348
2349 // Now select a V4L2 format to produce all output streams
2350 float desiredAr = (mCroppingType == VERTICAL) ? kMaxAspectRatio : kMinAspectRatio;
2351 uint32_t maxDim = 0;
2352 for (const auto& stream : config.streams) {
2353 float aspectRatio = ASPECT_RATIO(stream);
2354 ALOGI("%s: request stream %dx%d", __FUNCTION__, stream.width, stream.height);
2355 if ((mCroppingType == VERTICAL && aspectRatio < desiredAr) ||
2356 (mCroppingType == HORIZONTAL && aspectRatio > desiredAr)) {
2357 desiredAr = aspectRatio;
2358 }
2359
2360 // The dimension that's not cropped
2361 uint32_t dim = (mCroppingType == VERTICAL) ? stream.width : stream.height;
2362 if (dim > maxDim) {
2363 maxDim = dim;
2364 }
2365 }
2366 // Find the smallest format that matches the desired aspect ratio and is wide/high enough
2367 SupportedV4L2Format v4l2Fmt {.width = 0, .height = 0};
2368 for (const auto& fmt : mSupportedFormats) {
2369 uint32_t dim = (mCroppingType == VERTICAL) ? fmt.width : fmt.height;
2370 if (dim >= maxDim) {
2371 float aspectRatio = ASPECT_RATIO(fmt);
2372 if (isAspectRatioClose(aspectRatio, desiredAr)) {
2373 v4l2Fmt = fmt;
2374 // since mSupportedFormats is sorted by width then height, the first matching fmt
2375 // will be the smallest one with matching aspect ratio
2376 break;
2377 }
2378 }
2379 }
2380 if (v4l2Fmt.width == 0) {
2381 // Cannot find exact good aspect ratio candidate, try to find a close one
2382 for (const auto& fmt : mSupportedFormats) {
2383 uint32_t dim = (mCroppingType == VERTICAL) ? fmt.width : fmt.height;
2384 if (dim >= maxDim) {
2385 float aspectRatio = ASPECT_RATIO(fmt);
2386 if ((mCroppingType == VERTICAL && aspectRatio < desiredAr) ||
2387 (mCroppingType == HORIZONTAL && aspectRatio > desiredAr)) {
2388 v4l2Fmt = fmt;
2389 break;
2390 }
2391 }
2392 }
2393 }
2394
2395 if (v4l2Fmt.width == 0) {
2396 ALOGE("%s: unable to find a resolution matching (%s at least %d, aspect ratio %f)"
2397 , __FUNCTION__, (mCroppingType == VERTICAL) ? "width" : "height",
2398 maxDim, desiredAr);
2399 return Status::ILLEGAL_ARGUMENT;
2400 }
2401
2402 if (configureV4l2StreamLocked(v4l2Fmt) != 0) {
2403 ALOGE("V4L configuration failed!, format:%c%c%c%c, w %d, h %d",
2404 v4l2Fmt.fourcc & 0xFF,
2405 (v4l2Fmt.fourcc >> 8) & 0xFF,
2406 (v4l2Fmt.fourcc >> 16) & 0xFF,
2407 (v4l2Fmt.fourcc >> 24) & 0xFF,
2408 v4l2Fmt.width, v4l2Fmt.height);
2409 return Status::INTERNAL_ERROR;
2410 }
2411
2412 Size v4lSize = {v4l2Fmt.width, v4l2Fmt.height};
2413 Size thumbSize { 0, 0 };
2414 camera_metadata_ro_entry entry =
2415 mCameraCharacteristics.find(ANDROID_JPEG_AVAILABLE_THUMBNAIL_SIZES);
2416 for(uint32_t i = 0; i < entry.count; i += 2) {
2417 Size sz { static_cast<uint32_t>(entry.data.i32[i]),
2418 static_cast<uint32_t>(entry.data.i32[i+1]) };
2419 if(sz.width * sz.height > thumbSize.width * thumbSize.height) {
2420 thumbSize = sz;
2421 }
2422 }
2423
2424 if (thumbSize.width * thumbSize.height == 0) {
2425 ALOGE("%s: non-zero thumbnail size not available", __FUNCTION__);
2426 return Status::INTERNAL_ERROR;
2427 }
2428
2429 mBlobBufferSize = blobBufferSize;
2430 status = mOutputThread->allocateIntermediateBuffers(v4lSize,
2431 mMaxThumbResolution, config.streams, blobBufferSize);
2432 if (status != Status::OK) {
2433 ALOGE("%s: allocating intermediate buffers failed!", __FUNCTION__);
2434 return status;
2435 }
2436
2437 out->streams.resize(config.streams.size());
2438 for (size_t i = 0; i < config.streams.size(); i++) {
2439 out->streams[i].overrideDataSpace = config.streams[i].dataSpace;
2440 out->streams[i].v3_2.id = config.streams[i].id;
2441 // TODO: double check should we add those CAMERA flags
2442 mStreamMap[config.streams[i].id].usage =
2443 out->streams[i].v3_2.producerUsage = config.streams[i].usage |
2444 BufferUsage::CPU_WRITE_OFTEN |
2445 BufferUsage::CAMERA_OUTPUT;
2446 out->streams[i].v3_2.consumerUsage = 0;
2447 out->streams[i].v3_2.maxBuffers = mV4L2BufferCount;
2448
2449 switch (config.streams[i].format) {
2450 case PixelFormat::BLOB:
2451 case PixelFormat::YCBCR_420_888:
2452 case PixelFormat::YV12: // Used by SurfaceTexture
2453 case PixelFormat::Y16:
2454 // No override
2455 out->streams[i].v3_2.overrideFormat = config.streams[i].format;
2456 break;
2457 case PixelFormat::IMPLEMENTATION_DEFINED:
2458 // Override based on VIDEO or not
2459 out->streams[i].v3_2.overrideFormat =
2460 (config.streams[i].usage & BufferUsage::VIDEO_ENCODER) ?
2461 PixelFormat::YCBCR_420_888 : PixelFormat::YV12;
2462 // Save overridden formt in mStreamMap
2463 mStreamMap[config.streams[i].id].format = out->streams[i].v3_2.overrideFormat;
2464 break;
2465 default:
2466 ALOGE("%s: unsupported format 0x%x", __FUNCTION__, config.streams[i].format);
2467 return Status::ILLEGAL_ARGUMENT;
2468 }
2469 }
2470
2471 mFirstRequest = true;
2472 return Status::OK;
2473 }
2474
isClosed()2475 bool ExternalCameraDeviceSession::isClosed() {
2476 Mutex::Autolock _l(mLock);
2477 return mClosed;
2478 }
2479
2480 #define ARRAY_SIZE(a) (sizeof(a) / sizeof(a[0]))
2481 #define UPDATE(md, tag, data, size) \
2482 do { \
2483 if ((md).update((tag), (data), (size))) { \
2484 ALOGE("Update " #tag " failed!"); \
2485 return BAD_VALUE; \
2486 } \
2487 } while (0)
2488
initDefaultRequests()2489 status_t ExternalCameraDeviceSession::initDefaultRequests() {
2490 ::android::hardware::camera::common::V1_0::helper::CameraMetadata md;
2491
2492 const uint8_t aberrationMode = ANDROID_COLOR_CORRECTION_ABERRATION_MODE_OFF;
2493 UPDATE(md, ANDROID_COLOR_CORRECTION_ABERRATION_MODE, &aberrationMode, 1);
2494
2495 const int32_t exposureCompensation = 0;
2496 UPDATE(md, ANDROID_CONTROL_AE_EXPOSURE_COMPENSATION, &exposureCompensation, 1);
2497
2498 const uint8_t videoStabilizationMode = ANDROID_CONTROL_VIDEO_STABILIZATION_MODE_OFF;
2499 UPDATE(md, ANDROID_CONTROL_VIDEO_STABILIZATION_MODE, &videoStabilizationMode, 1);
2500
2501 const uint8_t awbMode = ANDROID_CONTROL_AWB_MODE_AUTO;
2502 UPDATE(md, ANDROID_CONTROL_AWB_MODE, &awbMode, 1);
2503
2504 const uint8_t aeMode = ANDROID_CONTROL_AE_MODE_ON;
2505 UPDATE(md, ANDROID_CONTROL_AE_MODE, &aeMode, 1);
2506
2507 const uint8_t aePrecaptureTrigger = ANDROID_CONTROL_AE_PRECAPTURE_TRIGGER_IDLE;
2508 UPDATE(md, ANDROID_CONTROL_AE_PRECAPTURE_TRIGGER, &aePrecaptureTrigger, 1);
2509
2510 const uint8_t afMode = ANDROID_CONTROL_AF_MODE_AUTO;
2511 UPDATE(md, ANDROID_CONTROL_AF_MODE, &afMode, 1);
2512
2513 const uint8_t afTrigger = ANDROID_CONTROL_AF_TRIGGER_IDLE;
2514 UPDATE(md, ANDROID_CONTROL_AF_TRIGGER, &afTrigger, 1);
2515
2516 const uint8_t sceneMode = ANDROID_CONTROL_SCENE_MODE_DISABLED;
2517 UPDATE(md, ANDROID_CONTROL_SCENE_MODE, &sceneMode, 1);
2518
2519 const uint8_t effectMode = ANDROID_CONTROL_EFFECT_MODE_OFF;
2520 UPDATE(md, ANDROID_CONTROL_EFFECT_MODE, &effectMode, 1);
2521
2522 const uint8_t flashMode = ANDROID_FLASH_MODE_OFF;
2523 UPDATE(md, ANDROID_FLASH_MODE, &flashMode, 1);
2524
2525 const int32_t thumbnailSize[] = {240, 180};
2526 UPDATE(md, ANDROID_JPEG_THUMBNAIL_SIZE, thumbnailSize, 2);
2527
2528 const uint8_t jpegQuality = 90;
2529 UPDATE(md, ANDROID_JPEG_QUALITY, &jpegQuality, 1);
2530 UPDATE(md, ANDROID_JPEG_THUMBNAIL_QUALITY, &jpegQuality, 1);
2531
2532 const int32_t jpegOrientation = 0;
2533 UPDATE(md, ANDROID_JPEG_ORIENTATION, &jpegOrientation, 1);
2534
2535 const uint8_t oisMode = ANDROID_LENS_OPTICAL_STABILIZATION_MODE_OFF;
2536 UPDATE(md, ANDROID_LENS_OPTICAL_STABILIZATION_MODE, &oisMode, 1);
2537
2538 const uint8_t nrMode = ANDROID_NOISE_REDUCTION_MODE_OFF;
2539 UPDATE(md, ANDROID_NOISE_REDUCTION_MODE, &nrMode, 1);
2540
2541 const int32_t testPatternModes = ANDROID_SENSOR_TEST_PATTERN_MODE_OFF;
2542 UPDATE(md, ANDROID_SENSOR_TEST_PATTERN_MODE, &testPatternModes, 1);
2543
2544 const uint8_t fdMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF;
2545 UPDATE(md, ANDROID_STATISTICS_FACE_DETECT_MODE, &fdMode, 1);
2546
2547 const uint8_t hotpixelMode = ANDROID_STATISTICS_HOT_PIXEL_MAP_MODE_OFF;
2548 UPDATE(md, ANDROID_STATISTICS_HOT_PIXEL_MAP_MODE, &hotpixelMode, 1);
2549
2550 bool support30Fps = false;
2551 int32_t maxFps = std::numeric_limits<int32_t>::min();
2552 for (const auto& supportedFormat : mSupportedFormats) {
2553 for (const auto& fr : supportedFormat.frameRates) {
2554 int32_t framerateInt = static_cast<int32_t>(fr.getDouble());
2555 if (maxFps < framerateInt) {
2556 maxFps = framerateInt;
2557 }
2558 if (framerateInt == 30) {
2559 support30Fps = true;
2560 break;
2561 }
2562 }
2563 if (support30Fps) {
2564 break;
2565 }
2566 }
2567 int32_t defaultFramerate = support30Fps ? 30 : maxFps;
2568 int32_t defaultFpsRange[] = {defaultFramerate / 2, defaultFramerate};
2569 UPDATE(md, ANDROID_CONTROL_AE_TARGET_FPS_RANGE, defaultFpsRange, ARRAY_SIZE(defaultFpsRange));
2570
2571 uint8_t antibandingMode = ANDROID_CONTROL_AE_ANTIBANDING_MODE_AUTO;
2572 UPDATE(md, ANDROID_CONTROL_AE_ANTIBANDING_MODE, &antibandingMode, 1);
2573
2574 const uint8_t controlMode = ANDROID_CONTROL_MODE_AUTO;
2575 UPDATE(md, ANDROID_CONTROL_MODE, &controlMode, 1);
2576
2577 auto requestTemplates = hidl_enum_range<RequestTemplate>();
2578 for (RequestTemplate type : requestTemplates) {
2579 ::android::hardware::camera::common::V1_0::helper::CameraMetadata mdCopy = md;
2580 uint8_t intent = ANDROID_CONTROL_CAPTURE_INTENT_PREVIEW;
2581 switch (type) {
2582 case RequestTemplate::PREVIEW:
2583 intent = ANDROID_CONTROL_CAPTURE_INTENT_PREVIEW;
2584 break;
2585 case RequestTemplate::STILL_CAPTURE:
2586 intent = ANDROID_CONTROL_CAPTURE_INTENT_STILL_CAPTURE;
2587 break;
2588 case RequestTemplate::VIDEO_RECORD:
2589 intent = ANDROID_CONTROL_CAPTURE_INTENT_VIDEO_RECORD;
2590 break;
2591 case RequestTemplate::VIDEO_SNAPSHOT:
2592 intent = ANDROID_CONTROL_CAPTURE_INTENT_VIDEO_SNAPSHOT;
2593 break;
2594 default:
2595 ALOGV("%s: unsupported RequestTemplate type %d", __FUNCTION__, type);
2596 continue;
2597 }
2598 UPDATE(mdCopy, ANDROID_CONTROL_CAPTURE_INTENT, &intent, 1);
2599
2600 camera_metadata_t* rawMd = mdCopy.release();
2601 CameraMetadata hidlMd;
2602 hidlMd.setToExternal(
2603 (uint8_t*) rawMd, get_camera_metadata_size(rawMd));
2604 mDefaultRequests[type] = hidlMd;
2605 free_camera_metadata(rawMd);
2606 }
2607
2608 return OK;
2609 }
2610
fillCaptureResult(common::V1_0::helper::CameraMetadata & md,nsecs_t timestamp)2611 status_t ExternalCameraDeviceSession::fillCaptureResult(
2612 common::V1_0::helper::CameraMetadata &md, nsecs_t timestamp) {
2613 bool afTrigger = false;
2614 {
2615 std::lock_guard<std::mutex> lk(mAfTriggerLock);
2616 afTrigger = mAfTrigger;
2617 if (md.exists(ANDROID_CONTROL_AF_TRIGGER)) {
2618 camera_metadata_entry entry = md.find(ANDROID_CONTROL_AF_TRIGGER);
2619 if (entry.data.u8[0] == ANDROID_CONTROL_AF_TRIGGER_START) {
2620 mAfTrigger = afTrigger = true;
2621 } else if (entry.data.u8[0] == ANDROID_CONTROL_AF_TRIGGER_CANCEL) {
2622 mAfTrigger = afTrigger = false;
2623 }
2624 }
2625 }
2626
2627 // For USB camera, the USB camera handles everything and we don't have control
2628 // over AF. We only simply fake the AF metadata based on the request
2629 // received here.
2630 uint8_t afState;
2631 if (afTrigger) {
2632 afState = ANDROID_CONTROL_AF_STATE_FOCUSED_LOCKED;
2633 } else {
2634 afState = ANDROID_CONTROL_AF_STATE_INACTIVE;
2635 }
2636 UPDATE(md, ANDROID_CONTROL_AF_STATE, &afState, 1);
2637
2638 camera_metadata_ro_entry activeArraySize =
2639 mCameraCharacteristics.find(ANDROID_SENSOR_INFO_ACTIVE_ARRAY_SIZE);
2640
2641 return fillCaptureResultCommon(md, timestamp, activeArraySize);
2642 }
2643
2644 #undef ARRAY_SIZE
2645 #undef UPDATE
2646
2647 } // namespace implementation
2648 } // namespace V3_4
2649 } // namespace device
2650 } // namespace camera
2651 } // namespace hardware
2652 } // namespace android
2653