1 /*
2 * Copyright (C) 2022 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #define LOG_TAG "ExtCamDevSsn"
18 // #define LOG_NDEBUG 0
19 #include <log/log.h>
20
21 #include "ExternalCameraDeviceSession.h"
22
23 #include <Exif.h>
24 #include <ExternalCameraOfflineSession.h>
25 #include <aidl/android/hardware/camera/device/CameraBlob.h>
26 #include <aidl/android/hardware/camera/device/CameraBlobId.h>
27 #include <aidl/android/hardware/camera/device/ErrorMsg.h>
28 #include <aidl/android/hardware/camera/device/ShutterMsg.h>
29 #include <aidl/android/hardware/camera/device/StreamBufferRet.h>
30 #include <aidl/android/hardware/camera/device/StreamBuffersVal.h>
31 #include <aidl/android/hardware/camera/device/StreamConfigurationMode.h>
32 #include <aidl/android/hardware/camera/device/StreamRotation.h>
33 #include <aidl/android/hardware/camera/device/StreamType.h>
34 #include <aidl/android/hardware/graphics/common/Dataspace.h>
35 #include <aidlcommonsupport/NativeHandle.h>
36 #include <convert.h>
37 #include <linux/videodev2.h>
38 #include <sync/sync.h>
39 #include <utils/Trace.h>
40 #include <deque>
41
42 #define HAVE_JPEG // required for libyuv.h to export MJPEG decode APIs
43 #include <libyuv.h>
44 #include <libyuv/convert.h>
45
46 namespace android {
47 namespace hardware {
48 namespace camera {
49 namespace device {
50 namespace implementation {
51
52 namespace {
53
54 // Size of request/result metadata fast message queue. Change to 0 to always use hwbinder buffer.
55 static constexpr size_t kMetadataMsgQueueSize = 1 << 18 /* 256kB */;
56
57 const int kBadFramesAfterStreamOn = 1; // drop x frames after streamOn to get rid of some initial
58 // bad frames. TODO: develop a better bad frame detection
59 // method
60 constexpr int MAX_RETRY = 15; // Allow retry some ioctl failures a few times to account for some
61 // webcam showing temporarily ioctl failures.
62 constexpr int IOCTL_RETRY_SLEEP_US = 33000; // 33ms * MAX_RETRY = 0.5 seconds
63
64 // Constants for tryLock during dumpstate
65 static constexpr int kDumpLockRetries = 50;
66 static constexpr int kDumpLockSleep = 60000;
67
tryLock(Mutex & mutex)68 bool tryLock(Mutex& mutex) {
69 bool locked = false;
70 for (int i = 0; i < kDumpLockRetries; ++i) {
71 if (mutex.tryLock() == NO_ERROR) {
72 locked = true;
73 break;
74 }
75 usleep(kDumpLockSleep);
76 }
77 return locked;
78 }
79
tryLock(std::mutex & mutex)80 bool tryLock(std::mutex& mutex) {
81 bool locked = false;
82 for (int i = 0; i < kDumpLockRetries; ++i) {
83 if (mutex.try_lock()) {
84 locked = true;
85 break;
86 }
87 usleep(kDumpLockSleep);
88 }
89 return locked;
90 }
91
92 } // anonymous namespace
93
94 using ::aidl::android::hardware::camera::device::BufferRequestStatus;
95 using ::aidl::android::hardware::camera::device::CameraBlob;
96 using ::aidl::android::hardware::camera::device::CameraBlobId;
97 using ::aidl::android::hardware::camera::device::ErrorMsg;
98 using ::aidl::android::hardware::camera::device::ShutterMsg;
99 using ::aidl::android::hardware::camera::device::StreamBuffer;
100 using ::aidl::android::hardware::camera::device::StreamBufferRet;
101 using ::aidl::android::hardware::camera::device::StreamBuffersVal;
102 using ::aidl::android::hardware::camera::device::StreamConfigurationMode;
103 using ::aidl::android::hardware::camera::device::StreamRotation;
104 using ::aidl::android::hardware::camera::device::StreamType;
105 using ::aidl::android::hardware::graphics::common::Dataspace;
106 using ::android::hardware::camera::common::V1_0::helper::ExifUtils;
107
108 // Static instances
109 const int ExternalCameraDeviceSession::kMaxProcessedStream;
110 const int ExternalCameraDeviceSession::kMaxStallStream;
111 HandleImporter ExternalCameraDeviceSession::sHandleImporter;
112
ExternalCameraDeviceSession(const std::shared_ptr<ICameraDeviceCallback> & callback,const ExternalCameraConfig & cfg,const std::vector<SupportedV4L2Format> & sortedFormats,const CroppingType & croppingType,const common::V1_0::helper::CameraMetadata & chars,const std::string & cameraId,unique_fd v4l2Fd)113 ExternalCameraDeviceSession::ExternalCameraDeviceSession(
114 const std::shared_ptr<ICameraDeviceCallback>& callback, const ExternalCameraConfig& cfg,
115 const std::vector<SupportedV4L2Format>& sortedFormats, const CroppingType& croppingType,
116 const common::V1_0::helper::CameraMetadata& chars, const std::string& cameraId,
117 unique_fd v4l2Fd)
118 : mCallback(callback),
119 mCfg(cfg),
120 mCameraCharacteristics(chars),
121 mSupportedFormats(sortedFormats),
122 mCroppingType(croppingType),
123 mCameraId(cameraId),
124 mV4l2Fd(std::move(v4l2Fd)),
125 mMaxThumbResolution(getMaxThumbResolution()),
126 mMaxJpegResolution(getMaxJpegResolution()) {}
127
getMaxThumbResolution() const128 Size ExternalCameraDeviceSession::getMaxThumbResolution() const {
129 return getMaxThumbnailResolution(mCameraCharacteristics);
130 }
131
getMaxJpegResolution() const132 Size ExternalCameraDeviceSession::getMaxJpegResolution() const {
133 Size ret{0, 0};
134 for (auto& fmt : mSupportedFormats) {
135 if (fmt.width * fmt.height > ret.width * ret.height) {
136 ret = Size{fmt.width, fmt.height};
137 }
138 }
139 return ret;
140 }
141
initialize()142 bool ExternalCameraDeviceSession::initialize() {
143 if (mV4l2Fd.get() < 0) {
144 ALOGE("%s: invalid v4l2 device fd %d!", __FUNCTION__, mV4l2Fd.get());
145 return true;
146 }
147
148 struct v4l2_capability capability;
149 int ret = ioctl(mV4l2Fd.get(), VIDIOC_QUERYCAP, &capability);
150 std::string make, model;
151 if (ret < 0) {
152 ALOGW("%s v4l2 QUERYCAP failed", __FUNCTION__);
153 mExifMake = "Generic UVC webcam";
154 mExifModel = "Generic UVC webcam";
155 } else {
156 // capability.card is UTF-8 encoded
157 char card[32];
158 int j = 0;
159 for (int i = 0; i < 32; i++) {
160 if (capability.card[i] < 128) {
161 card[j++] = capability.card[i];
162 }
163 if (capability.card[i] == '\0') {
164 break;
165 }
166 }
167 if (j == 0 || card[j - 1] != '\0') {
168 mExifMake = "Generic UVC webcam";
169 mExifModel = "Generic UVC webcam";
170 } else {
171 mExifMake = card;
172 mExifModel = card;
173 }
174 }
175
176 initOutputThread();
177 if (mOutputThread == nullptr) {
178 ALOGE("%s: init OutputThread failed!", __FUNCTION__);
179 return true;
180 }
181 mOutputThread->setExifMakeModel(mExifMake, mExifModel);
182
183 status_t status = initDefaultRequests();
184 if (status != OK) {
185 ALOGE("%s: init default requests failed!", __FUNCTION__);
186 return true;
187 }
188
189 mRequestMetadataQueue =
190 std::make_unique<RequestMetadataQueue>(kMetadataMsgQueueSize, false /* non blocking */);
191 if (!mRequestMetadataQueue->isValid()) {
192 ALOGE("%s: invalid request fmq", __FUNCTION__);
193 return true;
194 }
195
196 mResultMetadataQueue =
197 std::make_shared<ResultMetadataQueue>(kMetadataMsgQueueSize, false /* non blocking */);
198 if (!mResultMetadataQueue->isValid()) {
199 ALOGE("%s: invalid result fmq", __FUNCTION__);
200 return true;
201 }
202
203 mOutputThread->run();
204 return false;
205 }
206
isInitFailed()207 bool ExternalCameraDeviceSession::isInitFailed() {
208 Mutex::Autolock _l(mLock);
209 if (!mInitialized) {
210 mInitFail = initialize();
211 mInitialized = true;
212 }
213 return mInitFail;
214 }
215
initOutputThread()216 void ExternalCameraDeviceSession::initOutputThread() {
217 // Grab a shared_ptr to 'this' from ndk::SharedRefBase::ref()
218 std::shared_ptr<ExternalCameraDeviceSession> thiz = ref<ExternalCameraDeviceSession>();
219
220 mBufferRequestThread = std::make_shared<BufferRequestThread>(/*parent=*/thiz, mCallback);
221 mBufferRequestThread->run();
222 mOutputThread = std::make_shared<OutputThread>(/*parent=*/thiz, mCroppingType,
223 mCameraCharacteristics, mBufferRequestThread);
224 }
225
closeOutputThread()226 void ExternalCameraDeviceSession::closeOutputThread() {
227 if (mOutputThread != nullptr) {
228 mOutputThread->flush();
229 mOutputThread->requestExitAndWait();
230 mOutputThread.reset();
231 }
232 }
233
closeBufferRequestThread()234 void ExternalCameraDeviceSession::closeBufferRequestThread() {
235 if (mBufferRequestThread != nullptr) {
236 mBufferRequestThread->requestExitAndWait();
237 mBufferRequestThread.reset();
238 }
239 }
240
initStatus() const241 Status ExternalCameraDeviceSession::initStatus() const {
242 Mutex::Autolock _l(mLock);
243 Status status = Status::OK;
244 if (mInitFail || mClosed) {
245 ALOGI("%s: session initFailed %d closed %d", __FUNCTION__, mInitFail, mClosed);
246 status = Status::INTERNAL_ERROR;
247 }
248 return status;
249 }
250
~ExternalCameraDeviceSession()251 ExternalCameraDeviceSession::~ExternalCameraDeviceSession() {
252 if (!isClosed()) {
253 ALOGE("ExternalCameraDeviceSession deleted before close!");
254 closeImpl();
255 }
256 }
257
constructDefaultRequestSettings(RequestTemplate in_type,CameraMetadata * _aidl_return)258 ScopedAStatus ExternalCameraDeviceSession::constructDefaultRequestSettings(
259 RequestTemplate in_type, CameraMetadata* _aidl_return) {
260 CameraMetadata emptyMetadata;
261 Status status = initStatus();
262 if (status != Status::OK) {
263 return fromStatus(status);
264 }
265 switch (in_type) {
266 case RequestTemplate::PREVIEW:
267 case RequestTemplate::STILL_CAPTURE:
268 case RequestTemplate::VIDEO_RECORD:
269 case RequestTemplate::VIDEO_SNAPSHOT: {
270 *_aidl_return = mDefaultRequests[in_type];
271 break;
272 }
273 case RequestTemplate::MANUAL:
274 case RequestTemplate::ZERO_SHUTTER_LAG:
275 // Don't support MANUAL, ZSL templates
276 status = Status::ILLEGAL_ARGUMENT;
277 break;
278 default:
279 ALOGE("%s: unknown request template type %d", __FUNCTION__, static_cast<int>(in_type));
280 status = Status::ILLEGAL_ARGUMENT;
281 break;
282 }
283 return fromStatus(status);
284 }
285
configureStreams(const StreamConfiguration & in_requestedConfiguration,std::vector<HalStream> * _aidl_return)286 ScopedAStatus ExternalCameraDeviceSession::configureStreams(
287 const StreamConfiguration& in_requestedConfiguration,
288 std::vector<HalStream>* _aidl_return) {
289 uint32_t blobBufferSize = 0;
290 _aidl_return->clear();
291 Mutex::Autolock _il(mInterfaceLock);
292
293 Status status =
294 isStreamCombinationSupported(in_requestedConfiguration, mSupportedFormats, mCfg);
295 if (status != Status::OK) {
296 return fromStatus(status);
297 }
298
299 status = initStatus();
300 if (status != Status::OK) {
301 return fromStatus(status);
302 }
303
304 {
305 std::lock_guard<std::mutex> lk(mInflightFramesLock);
306 if (!mInflightFrames.empty()) {
307 ALOGE("%s: trying to configureStreams while there are still %zu inflight frames!",
308 __FUNCTION__, mInflightFrames.size());
309 return fromStatus(Status::INTERNAL_ERROR);
310 }
311 }
312
313 Mutex::Autolock _l(mLock);
314 {
315 Mutex::Autolock _cl(mCbsLock);
316 // Add new streams
317 for (const auto& stream : in_requestedConfiguration.streams) {
318 if (mStreamMap.count(stream.id) == 0) {
319 mStreamMap[stream.id] = stream;
320 mCirculatingBuffers.emplace(stream.id, CirculatingBuffers{});
321 }
322 }
323
324 // Cleanup removed streams
325 for (auto it = mStreamMap.begin(); it != mStreamMap.end();) {
326 int id = it->first;
327 bool found = false;
328 for (const auto& stream : in_requestedConfiguration.streams) {
329 if (id == stream.id) {
330 found = true;
331 break;
332 }
333 }
334 if (!found) {
335 // Unmap all buffers of deleted stream
336 cleanupBuffersLocked(id);
337 it = mStreamMap.erase(it);
338 } else {
339 ++it;
340 }
341 }
342 }
343
344 // Now select a V4L2 format to produce all output streams
345 float desiredAr = (mCroppingType == VERTICAL) ? kMaxAspectRatio : kMinAspectRatio;
346 uint32_t maxDim = 0;
347 for (const auto& stream : in_requestedConfiguration.streams) {
348 float aspectRatio = ASPECT_RATIO(stream);
349 ALOGI("%s: request stream %dx%d", __FUNCTION__, stream.width, stream.height);
350 if ((mCroppingType == VERTICAL && aspectRatio < desiredAr) ||
351 (mCroppingType == HORIZONTAL && aspectRatio > desiredAr)) {
352 desiredAr = aspectRatio;
353 }
354
355 // The dimension that's not cropped
356 uint32_t dim = (mCroppingType == VERTICAL) ? stream.width : stream.height;
357 if (dim > maxDim) {
358 maxDim = dim;
359 }
360 }
361
362 // Find the smallest format that matches the desired aspect ratio and is wide/high enough
363 SupportedV4L2Format v4l2Fmt{.width = 0, .height = 0};
364 for (const auto& fmt : mSupportedFormats) {
365 uint32_t dim = (mCroppingType == VERTICAL) ? fmt.width : fmt.height;
366 if (dim >= maxDim) {
367 float aspectRatio = ASPECT_RATIO(fmt);
368 if (isAspectRatioClose(aspectRatio, desiredAr)) {
369 v4l2Fmt = fmt;
370 // since mSupportedFormats is sorted by width then height, the first matching fmt
371 // will be the smallest one with matching aspect ratio
372 break;
373 }
374 }
375 }
376
377 if (v4l2Fmt.width == 0) {
378 // Cannot find exact good aspect ratio candidate, try to find a close one
379 for (const auto& fmt : mSupportedFormats) {
380 uint32_t dim = (mCroppingType == VERTICAL) ? fmt.width : fmt.height;
381 if (dim >= maxDim) {
382 float aspectRatio = ASPECT_RATIO(fmt);
383 if ((mCroppingType == VERTICAL && aspectRatio < desiredAr) ||
384 (mCroppingType == HORIZONTAL && aspectRatio > desiredAr)) {
385 v4l2Fmt = fmt;
386 break;
387 }
388 }
389 }
390 }
391
392 if (v4l2Fmt.width == 0) {
393 ALOGE("%s: unable to find a resolution matching (%s at least %d, aspect ratio %f)",
394 __FUNCTION__, (mCroppingType == VERTICAL) ? "width" : "height", maxDim, desiredAr);
395 return fromStatus(Status::ILLEGAL_ARGUMENT);
396 }
397
398 if (configureV4l2StreamLocked(v4l2Fmt) != 0) {
399 ALOGE("V4L configuration failed!, format:%c%c%c%c, w %d, h %d", v4l2Fmt.fourcc & 0xFF,
400 (v4l2Fmt.fourcc >> 8) & 0xFF, (v4l2Fmt.fourcc >> 16) & 0xFF,
401 (v4l2Fmt.fourcc >> 24) & 0xFF, v4l2Fmt.width, v4l2Fmt.height);
402 return fromStatus(Status::INTERNAL_ERROR);
403 }
404
405 Size v4lSize = {v4l2Fmt.width, v4l2Fmt.height};
406 Size thumbSize{0, 0};
407 camera_metadata_ro_entry entry =
408 mCameraCharacteristics.find(ANDROID_JPEG_AVAILABLE_THUMBNAIL_SIZES);
409 for (uint32_t i = 0; i < entry.count; i += 2) {
410 Size sz{entry.data.i32[i], entry.data.i32[i + 1]};
411 if (sz.width * sz.height > thumbSize.width * thumbSize.height) {
412 thumbSize = sz;
413 }
414 }
415
416 if (thumbSize.width * thumbSize.height == 0) {
417 ALOGE("%s: non-zero thumbnail size not available", __FUNCTION__);
418 return fromStatus(Status::INTERNAL_ERROR);
419 }
420
421 mBlobBufferSize = blobBufferSize;
422 status = mOutputThread->allocateIntermediateBuffers(
423 v4lSize, mMaxThumbResolution, in_requestedConfiguration.streams, blobBufferSize);
424 if (status != Status::OK) {
425 ALOGE("%s: allocating intermediate buffers failed!", __FUNCTION__);
426 return fromStatus(status);
427 }
428
429 std::vector<HalStream>& out = *_aidl_return;
430 out.resize(in_requestedConfiguration.streams.size());
431 for (size_t i = 0; i < in_requestedConfiguration.streams.size(); i++) {
432 out[i].overrideDataSpace = in_requestedConfiguration.streams[i].dataSpace;
433 out[i].id = in_requestedConfiguration.streams[i].id;
434 // TODO: double check should we add those CAMERA flags
435 mStreamMap[in_requestedConfiguration.streams[i].id].usage = out[i].producerUsage =
436 static_cast<BufferUsage>(((int64_t)in_requestedConfiguration.streams[i].usage) |
437 ((int64_t)BufferUsage::CPU_WRITE_OFTEN) |
438 ((int64_t)BufferUsage::CAMERA_OUTPUT));
439 out[i].consumerUsage = static_cast<BufferUsage>(0);
440 out[i].maxBuffers = static_cast<int32_t>(mV4L2BufferCount);
441
442 switch (in_requestedConfiguration.streams[i].format) {
443 case PixelFormat::BLOB:
444 case PixelFormat::YCBCR_420_888:
445 case PixelFormat::YV12: // Used by SurfaceTexture
446 case PixelFormat::Y16:
447 // No override
448 out[i].overrideFormat = in_requestedConfiguration.streams[i].format;
449 break;
450 case PixelFormat::IMPLEMENTATION_DEFINED:
451 // Implementation Defined
452 // This should look at the Stream's dataspace flag to determine the format or leave
453 // it as is if the rest of the system knows how to handle a private format. To keep
454 // this HAL generic, this is being overridden to YUV420
455 out[i].overrideFormat = PixelFormat::YCBCR_420_888;
456 // Save overridden format in mStreamMap
457 mStreamMap[in_requestedConfiguration.streams[i].id].format = out[i].overrideFormat;
458 break;
459 default:
460 ALOGE("%s: unsupported format 0x%x", __FUNCTION__,
461 in_requestedConfiguration.streams[i].format);
462 return fromStatus(Status::ILLEGAL_ARGUMENT);
463 }
464 }
465
466 mFirstRequest = true;
467 mLastStreamConfigCounter = in_requestedConfiguration.streamConfigCounter;
468 return fromStatus(Status::OK);
469 }
470
flush()471 ScopedAStatus ExternalCameraDeviceSession::flush() {
472 ATRACE_CALL();
473 Mutex::Autolock _il(mInterfaceLock);
474 Status status = initStatus();
475 if (status != Status::OK) {
476 return fromStatus(status);
477 }
478 mOutputThread->flush();
479 return fromStatus(Status::OK);
480 }
481
getCaptureRequestMetadataQueue(MQDescriptor<int8_t,SynchronizedReadWrite> * _aidl_return)482 ScopedAStatus ExternalCameraDeviceSession::getCaptureRequestMetadataQueue(
483 MQDescriptor<int8_t, SynchronizedReadWrite>* _aidl_return) {
484 Mutex::Autolock _il(mInterfaceLock);
485 *_aidl_return = mRequestMetadataQueue->dupeDesc();
486 return fromStatus(Status::OK);
487 }
488
getCaptureResultMetadataQueue(MQDescriptor<int8_t,SynchronizedReadWrite> * _aidl_return)489 ScopedAStatus ExternalCameraDeviceSession::getCaptureResultMetadataQueue(
490 MQDescriptor<int8_t, SynchronizedReadWrite>* _aidl_return) {
491 Mutex::Autolock _il(mInterfaceLock);
492 *_aidl_return = mResultMetadataQueue->dupeDesc();
493 return fromStatus(Status::OK);
494 }
495
isReconfigurationRequired(const CameraMetadata & in_oldSessionParams,const CameraMetadata & in_newSessionParams,bool * _aidl_return)496 ScopedAStatus ExternalCameraDeviceSession::isReconfigurationRequired(
497 const CameraMetadata& in_oldSessionParams, const CameraMetadata& in_newSessionParams,
498 bool* _aidl_return) {
499 // reconfiguration required if there is any change in the session params
500 *_aidl_return = in_oldSessionParams != in_newSessionParams;
501 return fromStatus(Status::OK);
502 }
503
processCaptureRequest(const std::vector<CaptureRequest> & in_requests,const std::vector<BufferCache> & in_cachesToRemove,int32_t * _aidl_return)504 ScopedAStatus ExternalCameraDeviceSession::processCaptureRequest(
505 const std::vector<CaptureRequest>& in_requests,
506 const std::vector<BufferCache>& in_cachesToRemove, int32_t* _aidl_return) {
507 Mutex::Autolock _il(mInterfaceLock);
508 updateBufferCaches(in_cachesToRemove);
509
510 int32_t& numRequestProcessed = *_aidl_return;
511 numRequestProcessed = 0;
512 Status s = Status::OK;
513 for (size_t i = 0; i < in_requests.size(); i++, numRequestProcessed++) {
514 s = processOneCaptureRequest(in_requests[i]);
515 if (s != Status::OK) {
516 break;
517 }
518 }
519
520 return fromStatus(s);
521 }
522
processOneCaptureRequest(const CaptureRequest & request)523 Status ExternalCameraDeviceSession::processOneCaptureRequest(const CaptureRequest& request) {
524 ATRACE_CALL();
525 Status status = initStatus();
526 if (status != Status::OK) {
527 return status;
528 }
529
530 if (request.inputBuffer.streamId != -1) {
531 ALOGE("%s: external camera does not support reprocessing!", __FUNCTION__);
532 return Status::ILLEGAL_ARGUMENT;
533 }
534
535 Mutex::Autolock _l(mLock);
536 if (!mV4l2Streaming) {
537 ALOGE("%s: cannot process request in streamOff state!", __FUNCTION__);
538 return Status::INTERNAL_ERROR;
539 }
540
541 if (request.outputBuffers.empty()) {
542 ALOGE("%s: No output buffers provided.", __FUNCTION__);
543 return Status::ILLEGAL_ARGUMENT;
544 }
545
546 for (auto& outputBuf : request.outputBuffers) {
547 if (outputBuf.streamId == -1 || mStreamMap.find(outputBuf.streamId) == mStreamMap.end()) {
548 ALOGE("%s: Invalid streamId in CaptureRequest.outputBuffers: %d", __FUNCTION__,
549 outputBuf.streamId);
550 return Status::ILLEGAL_ARGUMENT;
551 }
552 }
553
554 const camera_metadata_t* rawSettings = nullptr;
555 bool converted;
556 CameraMetadata settingsFmq; // settings from FMQ
557
558 if (request.fmqSettingsSize > 0) {
559 // non-blocking read; client must write metadata before calling
560 // processOneCaptureRequest
561 settingsFmq.metadata.resize(request.fmqSettingsSize);
562 bool read = mRequestMetadataQueue->read(
563 reinterpret_cast<int8_t*>(settingsFmq.metadata.data()), request.fmqSettingsSize);
564 if (read) {
565 converted = convertFromAidl(settingsFmq, &rawSettings);
566 } else {
567 ALOGE("%s: capture request settings metadata couldn't be read from fmq!", __FUNCTION__);
568 converted = false;
569 }
570 } else {
571 converted = convertFromAidl(request.settings, &rawSettings);
572 }
573
574 if (converted && rawSettings != nullptr) {
575 mLatestReqSetting = rawSettings;
576 }
577
578 if (!converted) {
579 ALOGE("%s: capture request settings metadata is corrupt!", __FUNCTION__);
580 return Status::ILLEGAL_ARGUMENT;
581 }
582
583 if (mFirstRequest && rawSettings == nullptr) {
584 ALOGE("%s: capture request settings must not be null for first request!", __FUNCTION__);
585 return Status::ILLEGAL_ARGUMENT;
586 }
587
588 size_t numOutputBufs = request.outputBuffers.size();
589
590 if (numOutputBufs == 0) {
591 ALOGE("%s: capture request must have at least one output buffer!", __FUNCTION__);
592 return Status::ILLEGAL_ARGUMENT;
593 }
594
595 camera_metadata_entry fpsRange = mLatestReqSetting.find(ANDROID_CONTROL_AE_TARGET_FPS_RANGE);
596 if (fpsRange.count == 2) {
597 double requestFpsMax = fpsRange.data.i32[1];
598 double closestFps = 0.0;
599 double fpsError = 1000.0;
600 bool fpsSupported = false;
601 for (const auto& fr : mV4l2StreamingFmt.frameRates) {
602 double f = fr.getFramesPerSecond();
603 if (std::fabs(requestFpsMax - f) < 1.0) {
604 fpsSupported = true;
605 break;
606 }
607 if (std::fabs(requestFpsMax - f) < fpsError) {
608 fpsError = std::fabs(requestFpsMax - f);
609 closestFps = f;
610 }
611 }
612 if (!fpsSupported) {
613 /* This can happen in a few scenarios:
614 * 1. The application is sending an FPS range not supported by the configured outputs.
615 * 2. The application is sending a valid FPS range for all configured outputs, but
616 * the selected V4L2 size can only run at slower speed. This should be very rare
617 * though: for this to happen a sensor needs to support at least 3 different aspect
618 * ratio outputs, and when (at least) two outputs are both not the main aspect ratio
619 * of the webcam, a third size that's larger might be picked and runs into this
620 * issue.
621 */
622 ALOGW("%s: cannot reach fps %d! Will do %f instead", __FUNCTION__, fpsRange.data.i32[1],
623 closestFps);
624 requestFpsMax = closestFps;
625 }
626
627 if (requestFpsMax != mV4l2StreamingFps) {
628 {
629 std::unique_lock<std::mutex> lk(mV4l2BufferLock);
630 while (mNumDequeuedV4l2Buffers != 0) {
631 // Wait until pipeline is idle before reconfigure stream
632 int waitRet = waitForV4L2BufferReturnLocked(lk);
633 if (waitRet != 0) {
634 ALOGE("%s: wait for pipeline idle failed!", __FUNCTION__);
635 return Status::INTERNAL_ERROR;
636 }
637 }
638 }
639 configureV4l2StreamLocked(mV4l2StreamingFmt, requestFpsMax);
640 }
641 }
642
643 nsecs_t shutterTs = 0;
644 std::unique_ptr<V4L2Frame> frameIn = dequeueV4l2FrameLocked(&shutterTs);
645 if (frameIn == nullptr) {
646 ALOGE("%s: V4L2 deque frame failed!", __FUNCTION__);
647 return Status::INTERNAL_ERROR;
648 }
649
650 std::shared_ptr<HalRequest> halReq = std::make_shared<HalRequest>();
651 halReq->frameNumber = request.frameNumber;
652 halReq->setting = mLatestReqSetting;
653 halReq->frameIn = std::move(frameIn);
654 halReq->shutterTs = shutterTs;
655 halReq->buffers.resize(numOutputBufs);
656 for (size_t i = 0; i < numOutputBufs; i++) {
657 HalStreamBuffer& halBuf = halReq->buffers[i];
658 int streamId = halBuf.streamId = request.outputBuffers[i].streamId;
659 halBuf.bufferId = request.outputBuffers[i].bufferId;
660 const Stream& stream = mStreamMap[streamId];
661 halBuf.width = stream.width;
662 halBuf.height = stream.height;
663 halBuf.format = stream.format;
664 halBuf.usage = stream.usage;
665 halBuf.bufPtr = nullptr; // threadloop will request buffer from cameraservice
666 halBuf.acquireFence = 0; // threadloop will request fence from cameraservice
667 halBuf.fenceTimeout = false;
668 }
669 {
670 std::lock_guard<std::mutex> lk(mInflightFramesLock);
671 mInflightFrames.insert(halReq->frameNumber);
672 }
673 // Send request to OutputThread for the rest of processing
674 mOutputThread->submitRequest(halReq);
675 mFirstRequest = false;
676 return Status::OK;
677 }
678
signalStreamFlush(const std::vector<int32_t> &,int32_t in_streamConfigCounter)679 ScopedAStatus ExternalCameraDeviceSession::signalStreamFlush(
680 const std::vector<int32_t>& /*in_streamIds*/, int32_t in_streamConfigCounter) {
681 {
682 Mutex::Autolock _l(mLock);
683 if (in_streamConfigCounter < mLastStreamConfigCounter) {
684 // stale call. new streams have been configured since this call was issued.
685 // Do nothing.
686 return fromStatus(Status::OK);
687 }
688 }
689
690 // TODO: implement if needed.
691 return fromStatus(Status::OK);
692 }
693
switchToOffline(const std::vector<int32_t> & in_streamsToKeep,CameraOfflineSessionInfo * out_offlineSessionInfo,std::shared_ptr<ICameraOfflineSession> * _aidl_return)694 ScopedAStatus ExternalCameraDeviceSession::switchToOffline(
695 const std::vector<int32_t>& in_streamsToKeep,
696 CameraOfflineSessionInfo* out_offlineSessionInfo,
697 std::shared_ptr<ICameraOfflineSession>* _aidl_return) {
698 std::vector<NotifyMsg> msgs;
699 std::vector<CaptureResult> results;
700 CameraOfflineSessionInfo info;
701 std::shared_ptr<ICameraOfflineSession> session;
702 Status st = switchToOffline(in_streamsToKeep, &msgs, &results, &info, &session);
703
704 mCallback->notify(msgs);
705 invokeProcessCaptureResultCallback(results, /* tryWriteFmq= */ true);
706 freeReleaseFences(results);
707
708 // setup return values
709 *out_offlineSessionInfo = info;
710 *_aidl_return = session;
711 return fromStatus(st);
712 }
713
switchToOffline(const std::vector<int32_t> & offlineStreams,std::vector<NotifyMsg> * msgs,std::vector<CaptureResult> * results,CameraOfflineSessionInfo * info,std::shared_ptr<ICameraOfflineSession> * session)714 Status ExternalCameraDeviceSession::switchToOffline(
715 const std::vector<int32_t>& offlineStreams, std::vector<NotifyMsg>* msgs,
716 std::vector<CaptureResult>* results, CameraOfflineSessionInfo* info,
717 std::shared_ptr<ICameraOfflineSession>* session) {
718 ATRACE_CALL();
719 if (offlineStreams.size() > 1) {
720 ALOGE("%s: more than one offline stream is not supported", __FUNCTION__);
721 return Status::ILLEGAL_ARGUMENT;
722 }
723
724 if (msgs == nullptr || results == nullptr || info == nullptr || session == nullptr) {
725 ALOGE("%s, output arguments (%p, %p, %p, %p) must not be null", __FUNCTION__, msgs, results,
726 info, session);
727 }
728
729 Mutex::Autolock _il(mInterfaceLock);
730 Status status = initStatus();
731 if (status != Status::OK) {
732 return status;
733 }
734
735 Mutex::Autolock _l(mLock);
736 for (auto streamId : offlineStreams) {
737 if (!supportOfflineLocked(streamId)) {
738 return Status::ILLEGAL_ARGUMENT;
739 }
740 }
741
742 // pause output thread and get all remaining inflight requests
743 auto remainingReqs = mOutputThread->switchToOffline();
744 std::vector<std::shared_ptr<HalRequest>> halReqs;
745
746 // Send out buffer/request error for remaining requests and filter requests
747 // to be handled in offline mode
748 for (auto& halReq : remainingReqs) {
749 bool dropReq = canDropRequest(offlineStreams, halReq);
750 if (dropReq) {
751 // Request is dropped completely. Just send request error and
752 // there is no need to send the request to offline session
753 processCaptureRequestError(halReq, msgs, results);
754 continue;
755 }
756
757 // All requests reach here must have at least one offline stream output
758 NotifyMsg shutter;
759 aidl::android::hardware::camera::device::ShutterMsg shutterMsg = {
760 .frameNumber = static_cast<int32_t>(halReq->frameNumber),
761 .timestamp = halReq->shutterTs};
762 shutter.set<NotifyMsg::Tag::shutter>(shutterMsg);
763 msgs->push_back(shutter);
764
765 std::vector<HalStreamBuffer> offlineBuffers;
766 for (const auto& buffer : halReq->buffers) {
767 bool dropBuffer = true;
768 for (auto offlineStreamId : offlineStreams) {
769 if (buffer.streamId == offlineStreamId) {
770 dropBuffer = false;
771 break;
772 }
773 }
774 if (dropBuffer) {
775 aidl::android::hardware::camera::device::ErrorMsg errorMsg = {
776 .frameNumber = static_cast<int32_t>(halReq->frameNumber),
777 .errorStreamId = buffer.streamId,
778 .errorCode = ErrorCode::ERROR_BUFFER};
779
780 NotifyMsg error;
781 error.set<NotifyMsg::Tag::error>(errorMsg);
782 msgs->push_back(error);
783
784 results->push_back({
785 .frameNumber = static_cast<int32_t>(halReq->frameNumber),
786 .outputBuffers = {},
787 .inputBuffer = {.streamId = -1},
788 .partialResult = 0, // buffer only result
789 });
790
791 CaptureResult& result = results->back();
792 result.outputBuffers.resize(1);
793 StreamBuffer& outputBuffer = result.outputBuffers[0];
794 outputBuffer.streamId = buffer.streamId;
795 outputBuffer.bufferId = buffer.bufferId;
796 outputBuffer.status = BufferStatus::ERROR;
797 if (buffer.acquireFence >= 0) {
798 native_handle_t* handle = native_handle_create(/*numFds*/ 1, /*numInts*/ 0);
799 handle->data[0] = buffer.acquireFence;
800 outputBuffer.releaseFence = android::dupToAidl(handle);
801 native_handle_delete(handle);
802 }
803 } else {
804 offlineBuffers.push_back(buffer);
805 }
806 }
807 halReq->buffers = offlineBuffers;
808 halReqs.push_back(halReq);
809 }
810
811 // convert hal requests to offline request
812 std::deque<std::shared_ptr<HalRequest>> offlineReqs(halReqs.size());
813 size_t i = 0;
814 for (auto& v4lReq : halReqs) {
815 offlineReqs[i] = std::make_shared<HalRequest>();
816 offlineReqs[i]->frameNumber = v4lReq->frameNumber;
817 offlineReqs[i]->setting = v4lReq->setting;
818 offlineReqs[i]->shutterTs = v4lReq->shutterTs;
819 offlineReqs[i]->buffers = v4lReq->buffers;
820 std::shared_ptr<V4L2Frame> v4l2Frame(static_cast<V4L2Frame*>(v4lReq->frameIn.get()));
821 offlineReqs[i]->frameIn = std::make_shared<AllocatedV4L2Frame>(v4l2Frame);
822 i++;
823 // enqueue V4L2 frame
824 enqueueV4l2Frame(v4l2Frame);
825 }
826
827 // Collect buffer caches/streams
828 std::vector<Stream> streamInfos(offlineStreams.size());
829 std::map<int, CirculatingBuffers> circulatingBuffers;
830 {
831 Mutex::Autolock _cbsl(mCbsLock);
832 for (auto streamId : offlineStreams) {
833 circulatingBuffers[streamId] = mCirculatingBuffers.at(streamId);
834 mCirculatingBuffers.erase(streamId);
835 streamInfos.push_back(mStreamMap.at(streamId));
836 mStreamMap.erase(streamId);
837 }
838 }
839
840 fillOfflineSessionInfo(offlineStreams, offlineReqs, circulatingBuffers, info);
841 // create the offline session object
842 bool afTrigger;
843 {
844 std::lock_guard<std::mutex> _lk(mAfTriggerLock);
845 afTrigger = mAfTrigger;
846 }
847
848 std::shared_ptr<ExternalCameraOfflineSession> sessionImpl =
849 ndk::SharedRefBase::make<ExternalCameraOfflineSession>(
850 mCroppingType, mCameraCharacteristics, mCameraId, mExifMake, mExifModel,
851 mBlobBufferSize, afTrigger, streamInfos, offlineReqs, circulatingBuffers);
852
853 bool initFailed = sessionImpl->initialize();
854 if (initFailed) {
855 ALOGE("%s: offline session initialize failed!", __FUNCTION__);
856 return Status::INTERNAL_ERROR;
857 }
858
859 // cleanup stream and buffer caches
860 {
861 Mutex::Autolock _cbsl(mCbsLock);
862 for (auto pair : mStreamMap) {
863 cleanupBuffersLocked(/*Stream ID*/ pair.first);
864 }
865 mCirculatingBuffers.clear();
866 }
867 mStreamMap.clear();
868
869 // update inflight records
870 {
871 std::lock_guard<std::mutex> _lk(mInflightFramesLock);
872 mInflightFrames.clear();
873 }
874
875 // stop v4l2 streaming
876 if (v4l2StreamOffLocked() != 0) {
877 ALOGE("%s: stop V4L2 streaming failed!", __FUNCTION__);
878 return Status::INTERNAL_ERROR;
879 }
880
881 // No need to return session if there is no offline requests left
882 if (!offlineReqs.empty()) {
883 *session = sessionImpl;
884 } else {
885 *session = nullptr;
886 }
887
888 return Status::OK;
889 }
890
891 #define ARRAY_SIZE(a) (sizeof(a) / sizeof(a[0]))
892 #define UPDATE(md, tag, data, size) \
893 do { \
894 if ((md).update((tag), (data), (size))) { \
895 ALOGE("Update " #tag " failed!"); \
896 return BAD_VALUE; \
897 } \
898 } while (0)
899
initDefaultRequests()900 status_t ExternalCameraDeviceSession::initDefaultRequests() {
901 common::V1_0::helper::CameraMetadata md;
902
903 const uint8_t aberrationMode = ANDROID_COLOR_CORRECTION_ABERRATION_MODE_OFF;
904 UPDATE(md, ANDROID_COLOR_CORRECTION_ABERRATION_MODE, &aberrationMode, 1);
905
906 const int32_t exposureCompensation = 0;
907 UPDATE(md, ANDROID_CONTROL_AE_EXPOSURE_COMPENSATION, &exposureCompensation, 1);
908
909 const uint8_t videoStabilizationMode = ANDROID_CONTROL_VIDEO_STABILIZATION_MODE_OFF;
910 UPDATE(md, ANDROID_CONTROL_VIDEO_STABILIZATION_MODE, &videoStabilizationMode, 1);
911
912 const uint8_t awbMode = ANDROID_CONTROL_AWB_MODE_AUTO;
913 UPDATE(md, ANDROID_CONTROL_AWB_MODE, &awbMode, 1);
914
915 const uint8_t aeMode = ANDROID_CONTROL_AE_MODE_ON;
916 UPDATE(md, ANDROID_CONTROL_AE_MODE, &aeMode, 1);
917
918 const uint8_t aePrecaptureTrigger = ANDROID_CONTROL_AE_PRECAPTURE_TRIGGER_IDLE;
919 UPDATE(md, ANDROID_CONTROL_AE_PRECAPTURE_TRIGGER, &aePrecaptureTrigger, 1);
920
921 const uint8_t afMode = ANDROID_CONTROL_AF_MODE_AUTO;
922 UPDATE(md, ANDROID_CONTROL_AF_MODE, &afMode, 1);
923
924 const uint8_t afTrigger = ANDROID_CONTROL_AF_TRIGGER_IDLE;
925 UPDATE(md, ANDROID_CONTROL_AF_TRIGGER, &afTrigger, 1);
926
927 const uint8_t sceneMode = ANDROID_CONTROL_SCENE_MODE_DISABLED;
928 UPDATE(md, ANDROID_CONTROL_SCENE_MODE, &sceneMode, 1);
929
930 const uint8_t effectMode = ANDROID_CONTROL_EFFECT_MODE_OFF;
931 UPDATE(md, ANDROID_CONTROL_EFFECT_MODE, &effectMode, 1);
932
933 const uint8_t flashMode = ANDROID_FLASH_MODE_OFF;
934 UPDATE(md, ANDROID_FLASH_MODE, &flashMode, 1);
935
936 const int32_t thumbnailSize[] = {240, 180};
937 UPDATE(md, ANDROID_JPEG_THUMBNAIL_SIZE, thumbnailSize, 2);
938
939 const uint8_t jpegQuality = 90;
940 UPDATE(md, ANDROID_JPEG_QUALITY, &jpegQuality, 1);
941 UPDATE(md, ANDROID_JPEG_THUMBNAIL_QUALITY, &jpegQuality, 1);
942
943 const int32_t jpegOrientation = 0;
944 UPDATE(md, ANDROID_JPEG_ORIENTATION, &jpegOrientation, 1);
945
946 const uint8_t oisMode = ANDROID_LENS_OPTICAL_STABILIZATION_MODE_OFF;
947 UPDATE(md, ANDROID_LENS_OPTICAL_STABILIZATION_MODE, &oisMode, 1);
948
949 const uint8_t nrMode = ANDROID_NOISE_REDUCTION_MODE_OFF;
950 UPDATE(md, ANDROID_NOISE_REDUCTION_MODE, &nrMode, 1);
951
952 const int32_t testPatternModes = ANDROID_SENSOR_TEST_PATTERN_MODE_OFF;
953 UPDATE(md, ANDROID_SENSOR_TEST_PATTERN_MODE, &testPatternModes, 1);
954
955 const uint8_t fdMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF;
956 UPDATE(md, ANDROID_STATISTICS_FACE_DETECT_MODE, &fdMode, 1);
957
958 const uint8_t hotpixelMode = ANDROID_STATISTICS_HOT_PIXEL_MAP_MODE_OFF;
959 UPDATE(md, ANDROID_STATISTICS_HOT_PIXEL_MAP_MODE, &hotpixelMode, 1);
960
961 bool support30Fps = false;
962 int32_t maxFps = std::numeric_limits<int32_t>::min();
963 for (const auto& supportedFormat : mSupportedFormats) {
964 for (const auto& fr : supportedFormat.frameRates) {
965 int32_t framerateInt = static_cast<int32_t>(fr.getFramesPerSecond());
966 if (maxFps < framerateInt) {
967 maxFps = framerateInt;
968 }
969 if (framerateInt == 30) {
970 support30Fps = true;
971 break;
972 }
973 }
974 if (support30Fps) {
975 break;
976 }
977 }
978
979 int32_t defaultFramerate = support30Fps ? 30 : maxFps;
980 int32_t defaultFpsRange[] = {defaultFramerate / 2, defaultFramerate};
981 UPDATE(md, ANDROID_CONTROL_AE_TARGET_FPS_RANGE, defaultFpsRange, ARRAY_SIZE(defaultFpsRange));
982
983 uint8_t antibandingMode = ANDROID_CONTROL_AE_ANTIBANDING_MODE_AUTO;
984 UPDATE(md, ANDROID_CONTROL_AE_ANTIBANDING_MODE, &antibandingMode, 1);
985
986 const uint8_t controlMode = ANDROID_CONTROL_MODE_AUTO;
987 UPDATE(md, ANDROID_CONTROL_MODE, &controlMode, 1);
988
989 for (const auto& type : ndk::enum_range<RequestTemplate>()) {
990 common::V1_0::helper::CameraMetadata mdCopy = md;
991 uint8_t intent = ANDROID_CONTROL_CAPTURE_INTENT_PREVIEW;
992 switch (type) {
993 case RequestTemplate::PREVIEW:
994 intent = ANDROID_CONTROL_CAPTURE_INTENT_PREVIEW;
995 break;
996 case RequestTemplate::STILL_CAPTURE:
997 intent = ANDROID_CONTROL_CAPTURE_INTENT_STILL_CAPTURE;
998 break;
999 case RequestTemplate::VIDEO_RECORD:
1000 intent = ANDROID_CONTROL_CAPTURE_INTENT_VIDEO_RECORD;
1001 break;
1002 case RequestTemplate::VIDEO_SNAPSHOT:
1003 intent = ANDROID_CONTROL_CAPTURE_INTENT_VIDEO_SNAPSHOT;
1004 break;
1005 default:
1006 ALOGV("%s: unsupported RequestTemplate type %d", __FUNCTION__, type);
1007 continue;
1008 }
1009 UPDATE(mdCopy, ANDROID_CONTROL_CAPTURE_INTENT, &intent, 1);
1010 camera_metadata_t* mdPtr = mdCopy.release();
1011 uint8_t* rawMd = reinterpret_cast<uint8_t*>(mdPtr);
1012 CameraMetadata aidlMd;
1013 aidlMd.metadata.assign(rawMd, rawMd + get_camera_metadata_size(mdPtr));
1014 mDefaultRequests[type] = aidlMd;
1015 free_camera_metadata(mdPtr);
1016 }
1017 return OK;
1018 }
1019
fillCaptureResult(common::V1_0::helper::CameraMetadata & md,nsecs_t timestamp)1020 status_t ExternalCameraDeviceSession::fillCaptureResult(common::V1_0::helper::CameraMetadata& md,
1021 nsecs_t timestamp) {
1022 bool afTrigger = false;
1023 {
1024 std::lock_guard<std::mutex> lk(mAfTriggerLock);
1025 afTrigger = mAfTrigger;
1026 if (md.exists(ANDROID_CONTROL_AF_TRIGGER)) {
1027 camera_metadata_entry entry = md.find(ANDROID_CONTROL_AF_TRIGGER);
1028 if (entry.data.u8[0] == ANDROID_CONTROL_AF_TRIGGER_START) {
1029 mAfTrigger = afTrigger = true;
1030 } else if (entry.data.u8[0] == ANDROID_CONTROL_AF_TRIGGER_CANCEL) {
1031 mAfTrigger = afTrigger = false;
1032 }
1033 }
1034 }
1035
1036 // For USB camera, the USB camera handles everything and we don't have control
1037 // over AF. We only simply fake the AF metadata based on the request
1038 // received here.
1039 uint8_t afState;
1040 if (afTrigger) {
1041 afState = ANDROID_CONTROL_AF_STATE_FOCUSED_LOCKED;
1042 } else {
1043 afState = ANDROID_CONTROL_AF_STATE_INACTIVE;
1044 }
1045 UPDATE(md, ANDROID_CONTROL_AF_STATE, &afState, 1);
1046
1047 camera_metadata_ro_entry activeArraySize =
1048 mCameraCharacteristics.find(ANDROID_SENSOR_INFO_ACTIVE_ARRAY_SIZE);
1049
1050 return fillCaptureResultCommon(md, timestamp, activeArraySize);
1051 }
1052
configureV4l2StreamLocked(const SupportedV4L2Format & v4l2Fmt,double requestFps)1053 int ExternalCameraDeviceSession::configureV4l2StreamLocked(const SupportedV4L2Format& v4l2Fmt,
1054 double requestFps) {
1055 ATRACE_CALL();
1056 int ret = v4l2StreamOffLocked();
1057 if (ret != OK) {
1058 ALOGE("%s: stop v4l2 streaming failed: ret %d", __FUNCTION__, ret);
1059 return ret;
1060 }
1061
1062 // VIDIOC_S_FMT w/h/fmt
1063 v4l2_format fmt;
1064 fmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
1065 fmt.fmt.pix.width = v4l2Fmt.width;
1066 fmt.fmt.pix.height = v4l2Fmt.height;
1067 fmt.fmt.pix.pixelformat = v4l2Fmt.fourcc;
1068
1069 {
1070 int numAttempt = 0;
1071 do {
1072 ret = TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_S_FMT, &fmt));
1073 if (numAttempt == MAX_RETRY) {
1074 break;
1075 }
1076 numAttempt++;
1077 if (ret < 0) {
1078 ALOGW("%s: VIDIOC_S_FMT failed, wait 33ms and try again", __FUNCTION__);
1079 usleep(IOCTL_RETRY_SLEEP_US); // sleep and try again
1080 }
1081 } while (ret < 0);
1082 if (ret < 0) {
1083 ALOGE("%s: S_FMT ioctl failed: %s", __FUNCTION__, strerror(errno));
1084 return -errno;
1085 }
1086 }
1087
1088 if (v4l2Fmt.width != fmt.fmt.pix.width || v4l2Fmt.height != fmt.fmt.pix.height ||
1089 v4l2Fmt.fourcc != fmt.fmt.pix.pixelformat) {
1090 ALOGE("%s: S_FMT expect %c%c%c%c %dx%d, got %c%c%c%c %dx%d instead!", __FUNCTION__,
1091 v4l2Fmt.fourcc & 0xFF, (v4l2Fmt.fourcc >> 8) & 0xFF, (v4l2Fmt.fourcc >> 16) & 0xFF,
1092 (v4l2Fmt.fourcc >> 24) & 0xFF, v4l2Fmt.width, v4l2Fmt.height,
1093 fmt.fmt.pix.pixelformat & 0xFF, (fmt.fmt.pix.pixelformat >> 8) & 0xFF,
1094 (fmt.fmt.pix.pixelformat >> 16) & 0xFF, (fmt.fmt.pix.pixelformat >> 24) & 0xFF,
1095 fmt.fmt.pix.width, fmt.fmt.pix.height);
1096 return -EINVAL;
1097 }
1098
1099 uint32_t bufferSize = fmt.fmt.pix.sizeimage;
1100 ALOGI("%s: V4L2 buffer size is %d", __FUNCTION__, bufferSize);
1101 uint32_t expectedMaxBufferSize = kMaxBytesPerPixel * fmt.fmt.pix.width * fmt.fmt.pix.height;
1102 if ((bufferSize == 0) || (bufferSize > expectedMaxBufferSize)) {
1103 ALOGE("%s: V4L2 buffer size: %u looks invalid. Expected maximum size: %u", __FUNCTION__,
1104 bufferSize, expectedMaxBufferSize);
1105 return -EINVAL;
1106 }
1107 mMaxV4L2BufferSize = bufferSize;
1108
1109 const double kDefaultFps = 30.0;
1110 double fps = std::numeric_limits<double>::max();
1111 if (requestFps != 0.0) {
1112 fps = requestFps;
1113 } else {
1114 double maxFps = -1.0;
1115 // Try to pick the slowest fps that is at least 30
1116 for (const auto& fr : v4l2Fmt.frameRates) {
1117 double f = fr.getFramesPerSecond();
1118 if (maxFps < f) {
1119 maxFps = f;
1120 }
1121 if (f >= kDefaultFps && f < fps) {
1122 fps = f;
1123 }
1124 }
1125 // No fps > 30 found, use the highest fps available within supported formats.
1126 if (fps == std::numeric_limits<double>::max()) {
1127 fps = maxFps;
1128 }
1129 }
1130
1131 int fpsRet = setV4l2FpsLocked(fps);
1132 if (fpsRet != 0 && fpsRet != -EINVAL) {
1133 ALOGE("%s: set fps failed: %s", __FUNCTION__, strerror(fpsRet));
1134 return fpsRet;
1135 }
1136
1137 uint32_t v4lBufferCount = (fps >= kDefaultFps) ? mCfg.numVideoBuffers : mCfg.numStillBuffers;
1138
1139 // VIDIOC_REQBUFS: create buffers
1140 v4l2_requestbuffers req_buffers{};
1141 req_buffers.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
1142 req_buffers.memory = V4L2_MEMORY_MMAP;
1143 req_buffers.count = v4lBufferCount;
1144 if (TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_REQBUFS, &req_buffers)) < 0) {
1145 ALOGE("%s: VIDIOC_REQBUFS failed: %s", __FUNCTION__, strerror(errno));
1146 return -errno;
1147 }
1148
1149 // Driver can indeed return more buffer if it needs more to operate
1150 if (req_buffers.count < v4lBufferCount) {
1151 ALOGE("%s: VIDIOC_REQBUFS expected %d buffers, got %d instead", __FUNCTION__,
1152 v4lBufferCount, req_buffers.count);
1153 return NO_MEMORY;
1154 }
1155
1156 // VIDIOC_QUERYBUF: get buffer offset in the V4L2 fd
1157 // VIDIOC_QBUF: send buffer to driver
1158 mV4L2BufferCount = req_buffers.count;
1159 for (uint32_t i = 0; i < req_buffers.count; i++) {
1160 v4l2_buffer buffer = {
1161 .index = i, .type = V4L2_BUF_TYPE_VIDEO_CAPTURE, .memory = V4L2_MEMORY_MMAP};
1162
1163 if (TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_QUERYBUF, &buffer)) < 0) {
1164 ALOGE("%s: QUERYBUF %d failed: %s", __FUNCTION__, i, strerror(errno));
1165 return -errno;
1166 }
1167
1168 if (TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_QBUF, &buffer)) < 0) {
1169 ALOGE("%s: QBUF %d failed: %s", __FUNCTION__, i, strerror(errno));
1170 return -errno;
1171 }
1172 }
1173
1174 {
1175 // VIDIOC_STREAMON: start streaming
1176 v4l2_buf_type capture_type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
1177 int numAttempt = 0;
1178 do {
1179 ret = TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_STREAMON, &capture_type));
1180 if (numAttempt == MAX_RETRY) {
1181 break;
1182 }
1183 if (ret < 0) {
1184 ALOGW("%s: VIDIOC_STREAMON failed, wait 33ms and try again", __FUNCTION__);
1185 usleep(IOCTL_RETRY_SLEEP_US); // sleep 100 ms and try again
1186 }
1187 } while (ret < 0);
1188
1189 if (ret < 0) {
1190 ALOGE("%s: VIDIOC_STREAMON ioctl failed: %s", __FUNCTION__, strerror(errno));
1191 return -errno;
1192 }
1193 }
1194
1195 // Swallow first few frames after streamOn to account for bad frames from some devices
1196 for (int i = 0; i < kBadFramesAfterStreamOn; i++) {
1197 v4l2_buffer buffer{};
1198 buffer.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
1199 buffer.memory = V4L2_MEMORY_MMAP;
1200 if (TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_DQBUF, &buffer)) < 0) {
1201 ALOGE("%s: DQBUF fails: %s", __FUNCTION__, strerror(errno));
1202 return -errno;
1203 }
1204
1205 if (TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_QBUF, &buffer)) < 0) {
1206 ALOGE("%s: QBUF index %d fails: %s", __FUNCTION__, buffer.index, strerror(errno));
1207 return -errno;
1208 }
1209 }
1210
1211 ALOGI("%s: start V4L2 streaming %dx%d@%ffps", __FUNCTION__, v4l2Fmt.width, v4l2Fmt.height, fps);
1212 mV4l2StreamingFmt = v4l2Fmt;
1213 mV4l2Streaming = true;
1214 return OK;
1215 }
1216
dequeueV4l2FrameLocked(nsecs_t * shutterTs)1217 std::unique_ptr<V4L2Frame> ExternalCameraDeviceSession::dequeueV4l2FrameLocked(nsecs_t* shutterTs) {
1218 ATRACE_CALL();
1219 std::unique_ptr<V4L2Frame> ret = nullptr;
1220 if (shutterTs == nullptr) {
1221 ALOGE("%s: shutterTs must not be null!", __FUNCTION__);
1222 return ret;
1223 }
1224
1225 {
1226 std::unique_lock<std::mutex> lk(mV4l2BufferLock);
1227 if (mNumDequeuedV4l2Buffers == mV4L2BufferCount) {
1228 int waitRet = waitForV4L2BufferReturnLocked(lk);
1229 if (waitRet != 0) {
1230 return ret;
1231 }
1232 }
1233 }
1234
1235 ATRACE_BEGIN("VIDIOC_DQBUF");
1236 v4l2_buffer buffer{};
1237 buffer.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
1238 buffer.memory = V4L2_MEMORY_MMAP;
1239 if (TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_DQBUF, &buffer)) < 0) {
1240 ALOGE("%s: DQBUF fails: %s", __FUNCTION__, strerror(errno));
1241 return ret;
1242 }
1243 ATRACE_END();
1244
1245 if (buffer.index >= mV4L2BufferCount) {
1246 ALOGE("%s: Invalid buffer id: %d", __FUNCTION__, buffer.index);
1247 return ret;
1248 }
1249
1250 if (buffer.flags & V4L2_BUF_FLAG_ERROR) {
1251 ALOGE("%s: v4l2 buf error! buf flag 0x%x", __FUNCTION__, buffer.flags);
1252 // TODO: try to dequeue again
1253 }
1254
1255 if (buffer.bytesused > mMaxV4L2BufferSize) {
1256 ALOGE("%s: v4l2 buffer bytes used: %u maximum %u", __FUNCTION__, buffer.bytesused,
1257 mMaxV4L2BufferSize);
1258 return ret;
1259 }
1260
1261 if (buffer.flags & V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC) {
1262 // Ideally we should also check for V4L2_BUF_FLAG_TSTAMP_SRC_SOE, but
1263 // even V4L2_BUF_FLAG_TSTAMP_SRC_EOF is better than capture a timestamp now
1264 *shutterTs = static_cast<nsecs_t>(buffer.timestamp.tv_sec) * 1000000000LL +
1265 buffer.timestamp.tv_usec * 1000LL;
1266 } else {
1267 *shutterTs = systemTime(SYSTEM_TIME_MONOTONIC);
1268 }
1269
1270 {
1271 std::lock_guard<std::mutex> lk(mV4l2BufferLock);
1272 mNumDequeuedV4l2Buffers++;
1273 }
1274
1275 return std::make_unique<V4L2Frame>(mV4l2StreamingFmt.width, mV4l2StreamingFmt.height,
1276 mV4l2StreamingFmt.fourcc, buffer.index, mV4l2Fd.get(),
1277 buffer.bytesused, buffer.m.offset);
1278 }
1279
enqueueV4l2Frame(const std::shared_ptr<V4L2Frame> & frame)1280 void ExternalCameraDeviceSession::enqueueV4l2Frame(const std::shared_ptr<V4L2Frame>& frame) {
1281 ATRACE_CALL();
1282 frame->unmap();
1283 ATRACE_BEGIN("VIDIOC_QBUF");
1284 v4l2_buffer buffer{};
1285 buffer.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
1286 buffer.memory = V4L2_MEMORY_MMAP;
1287 buffer.index = frame->mBufferIndex;
1288 if (TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_QBUF, &buffer)) < 0) {
1289 ALOGE("%s: QBUF index %d fails: %s", __FUNCTION__, frame->mBufferIndex, strerror(errno));
1290 return;
1291 }
1292 ATRACE_END();
1293
1294 {
1295 std::lock_guard<std::mutex> lk(mV4l2BufferLock);
1296 mNumDequeuedV4l2Buffers--;
1297 }
1298 mV4L2BufferReturned.notify_one();
1299 }
1300
isSupported(const Stream & stream,const std::vector<SupportedV4L2Format> & supportedFormats,const ExternalCameraConfig & devCfg)1301 bool ExternalCameraDeviceSession::isSupported(
1302 const Stream& stream, const std::vector<SupportedV4L2Format>& supportedFormats,
1303 const ExternalCameraConfig& devCfg) {
1304 Dataspace ds = stream.dataSpace;
1305 PixelFormat fmt = stream.format;
1306 uint32_t width = stream.width;
1307 uint32_t height = stream.height;
1308 // TODO: check usage flags
1309
1310 if (stream.streamType != StreamType::OUTPUT) {
1311 ALOGE("%s: does not support non-output stream type", __FUNCTION__);
1312 return false;
1313 }
1314
1315 if (stream.rotation != StreamRotation::ROTATION_0) {
1316 ALOGE("%s: does not support stream rotation", __FUNCTION__);
1317 return false;
1318 }
1319
1320 switch (fmt) {
1321 case PixelFormat::BLOB:
1322 if (ds != Dataspace::JFIF) {
1323 ALOGI("%s: BLOB format does not support dataSpace %x", __FUNCTION__, ds);
1324 return false;
1325 }
1326 break;
1327 case PixelFormat::IMPLEMENTATION_DEFINED:
1328 case PixelFormat::YCBCR_420_888:
1329 case PixelFormat::YV12:
1330 // TODO: check what dataspace we can support here.
1331 // intentional no-ops.
1332 break;
1333 case PixelFormat::Y16:
1334 if (!devCfg.depthEnabled) {
1335 ALOGI("%s: Depth is not Enabled", __FUNCTION__);
1336 return false;
1337 }
1338 if (!(static_cast<int32_t>(ds) & static_cast<int32_t>(Dataspace::DEPTH))) {
1339 ALOGI("%s: Y16 supports only dataSpace DEPTH", __FUNCTION__);
1340 return false;
1341 }
1342 break;
1343 default:
1344 ALOGI("%s: does not support format %x", __FUNCTION__, fmt);
1345 return false;
1346 }
1347
1348 // Assume we can convert any V4L2 format to any of supported output format for now, i.e.
1349 // ignoring v4l2Fmt.fourcc for now. Might need more subtle check if we support more v4l format
1350 // in the futrue.
1351 for (const auto& v4l2Fmt : supportedFormats) {
1352 if (width == v4l2Fmt.width && height == v4l2Fmt.height) {
1353 return true;
1354 }
1355 }
1356 ALOGI("%s: resolution %dx%d is not supported", __FUNCTION__, width, height);
1357 return false;
1358 }
1359
importBuffer(int32_t streamId,uint64_t bufId,buffer_handle_t buf,buffer_handle_t ** outBufPtr)1360 Status ExternalCameraDeviceSession::importBuffer(int32_t streamId, uint64_t bufId,
1361 buffer_handle_t buf,
1362 /*out*/ buffer_handle_t** outBufPtr) {
1363 Mutex::Autolock _l(mCbsLock);
1364 return importBufferLocked(streamId, bufId, buf, outBufPtr);
1365 }
1366
importBufferLocked(int32_t streamId,uint64_t bufId,buffer_handle_t buf,buffer_handle_t ** outBufPtr)1367 Status ExternalCameraDeviceSession::importBufferLocked(int32_t streamId, uint64_t bufId,
1368 buffer_handle_t buf,
1369 buffer_handle_t** outBufPtr) {
1370 return importBufferImpl(mCirculatingBuffers, sHandleImporter, streamId, bufId, buf, outBufPtr);
1371 }
1372
close()1373 ScopedAStatus ExternalCameraDeviceSession::close() {
1374 closeImpl();
1375 return fromStatus(Status::OK);
1376 }
1377
closeImpl()1378 void ExternalCameraDeviceSession::closeImpl() {
1379 Mutex::Autolock _il(mInterfaceLock);
1380 bool closed = isClosed();
1381 if (!closed) {
1382 closeOutputThread();
1383 closeBufferRequestThread();
1384
1385 Mutex::Autolock _l(mLock);
1386 // free all buffers
1387 {
1388 Mutex::Autolock _cbsl(mCbsLock);
1389 for (auto pair : mStreamMap) {
1390 cleanupBuffersLocked(/*Stream ID*/ pair.first);
1391 }
1392 }
1393 v4l2StreamOffLocked();
1394 ALOGV("%s: closing V4L2 camera FD %d", __FUNCTION__, mV4l2Fd.get());
1395 mV4l2Fd.reset();
1396 mClosed = true;
1397 }
1398 }
1399
isClosed()1400 bool ExternalCameraDeviceSession::isClosed() {
1401 Mutex::Autolock _l(mLock);
1402 return mClosed;
1403 }
1404
repeatingRequestEnd(int32_t,const std::vector<int32_t> &)1405 ScopedAStatus ExternalCameraDeviceSession::repeatingRequestEnd(
1406 int32_t /*in_frameNumber*/, const std::vector<int32_t>& /*in_streamIds*/) {
1407 // TODO: Figure this one out.
1408 return fromStatus(Status::OK);
1409 }
1410
v4l2StreamOffLocked()1411 int ExternalCameraDeviceSession::v4l2StreamOffLocked() {
1412 if (!mV4l2Streaming) {
1413 return OK;
1414 }
1415
1416 {
1417 std::lock_guard<std::mutex> lk(mV4l2BufferLock);
1418 if (mNumDequeuedV4l2Buffers != 0) {
1419 ALOGE("%s: there are %zu inflight V4L buffers", __FUNCTION__, mNumDequeuedV4l2Buffers);
1420 return -1;
1421 }
1422 }
1423 mV4L2BufferCount = 0;
1424
1425 // VIDIOC_STREAMOFF
1426 v4l2_buf_type capture_type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
1427 if (TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_STREAMOFF, &capture_type)) < 0) {
1428 ALOGE("%s: STREAMOFF failed: %s", __FUNCTION__, strerror(errno));
1429 return -errno;
1430 }
1431
1432 // VIDIOC_REQBUFS: clear buffers
1433 v4l2_requestbuffers req_buffers{};
1434 req_buffers.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
1435 req_buffers.memory = V4L2_MEMORY_MMAP;
1436 req_buffers.count = 0;
1437 if (TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_REQBUFS, &req_buffers)) < 0) {
1438 ALOGE("%s: REQBUFS failed: %s", __FUNCTION__, strerror(errno));
1439 return -errno;
1440 }
1441
1442 mV4l2Streaming = false;
1443 return OK;
1444 }
1445
setV4l2FpsLocked(double fps)1446 int ExternalCameraDeviceSession::setV4l2FpsLocked(double fps) {
1447 // VIDIOC_G_PARM/VIDIOC_S_PARM: set fps
1448 v4l2_streamparm streamparm = {.type = V4L2_BUF_TYPE_VIDEO_CAPTURE};
1449 // The following line checks that the driver knows about framerate get/set.
1450 int ret = TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_G_PARM, &streamparm));
1451 if (ret != 0) {
1452 if (errno == -EINVAL) {
1453 ALOGW("%s: device does not support VIDIOC_G_PARM", __FUNCTION__);
1454 }
1455 return -errno;
1456 }
1457 // Now check if the device is able to accept a capture framerate set.
1458 if (!(streamparm.parm.capture.capability & V4L2_CAP_TIMEPERFRAME)) {
1459 ALOGW("%s: device does not support V4L2_CAP_TIMEPERFRAME", __FUNCTION__);
1460 return -EINVAL;
1461 }
1462
1463 // fps is float, approximate by a fraction.
1464 const int kFrameRatePrecision = 10000;
1465 streamparm.parm.capture.timeperframe.numerator = kFrameRatePrecision;
1466 streamparm.parm.capture.timeperframe.denominator = (fps * kFrameRatePrecision);
1467
1468 if (TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_S_PARM, &streamparm)) < 0) {
1469 ALOGE("%s: failed to set framerate to %f: %s", __FUNCTION__, fps, strerror(errno));
1470 return -1;
1471 }
1472
1473 double retFps = streamparm.parm.capture.timeperframe.denominator /
1474 static_cast<double>(streamparm.parm.capture.timeperframe.numerator);
1475 if (std::fabs(fps - retFps) > 1.0) {
1476 ALOGE("%s: expect fps %f, got %f instead", __FUNCTION__, fps, retFps);
1477 return -1;
1478 }
1479 mV4l2StreamingFps = fps;
1480 return 0;
1481 }
1482
cleanupInflightFences(std::vector<int> & allFences,size_t numFences)1483 void ExternalCameraDeviceSession::cleanupInflightFences(std::vector<int>& allFences,
1484 size_t numFences) {
1485 for (size_t j = 0; j < numFences; j++) {
1486 sHandleImporter.closeFence(allFences[j]);
1487 }
1488 }
1489
cleanupBuffersLocked(int id)1490 void ExternalCameraDeviceSession::cleanupBuffersLocked(int id) {
1491 for (auto& pair : mCirculatingBuffers.at(id)) {
1492 sHandleImporter.freeBuffer(pair.second);
1493 }
1494 mCirculatingBuffers[id].clear();
1495 mCirculatingBuffers.erase(id);
1496 }
1497
notifyShutter(int32_t frameNumber,nsecs_t shutterTs)1498 void ExternalCameraDeviceSession::notifyShutter(int32_t frameNumber, nsecs_t shutterTs) {
1499 NotifyMsg msg;
1500 msg.set<NotifyMsg::Tag::shutter>(ShutterMsg{
1501 .frameNumber = frameNumber,
1502 .timestamp = shutterTs,
1503 });
1504 mCallback->notify({msg});
1505 }
notifyError(int32_t frameNumber,int32_t streamId,ErrorCode ec)1506 void ExternalCameraDeviceSession::notifyError(int32_t frameNumber, int32_t streamId, ErrorCode ec) {
1507 NotifyMsg msg;
1508 msg.set<NotifyMsg::Tag::error>(ErrorMsg{
1509 .frameNumber = frameNumber,
1510 .errorStreamId = streamId,
1511 .errorCode = ec,
1512 });
1513 mCallback->notify({msg});
1514 }
1515
invokeProcessCaptureResultCallback(std::vector<CaptureResult> & results,bool tryWriteFmq)1516 void ExternalCameraDeviceSession::invokeProcessCaptureResultCallback(
1517 std::vector<CaptureResult>& results, bool tryWriteFmq) {
1518 if (mProcessCaptureResultLock.tryLock() != OK) {
1519 const nsecs_t NS_TO_SECOND = 1000000000;
1520 ALOGV("%s: previous call is not finished! waiting 1s...", __FUNCTION__);
1521 if (mProcessCaptureResultLock.timedLock(/* 1s */ NS_TO_SECOND) != OK) {
1522 ALOGE("%s: cannot acquire lock in 1s, cannot proceed", __FUNCTION__);
1523 return;
1524 }
1525 }
1526 if (tryWriteFmq && mResultMetadataQueue->availableToWrite() > 0) {
1527 for (CaptureResult& result : results) {
1528 CameraMetadata& md = result.result;
1529 if (!md.metadata.empty()) {
1530 if (mResultMetadataQueue->write(reinterpret_cast<int8_t*>(md.metadata.data()),
1531 md.metadata.size())) {
1532 result.fmqResultSize = md.metadata.size();
1533 md.metadata.resize(0);
1534 } else {
1535 ALOGW("%s: couldn't utilize fmq, fall back to hwbinder", __FUNCTION__);
1536 result.fmqResultSize = 0;
1537 }
1538 } else {
1539 result.fmqResultSize = 0;
1540 }
1541 }
1542 }
1543 auto status = mCallback->processCaptureResult(results);
1544 if (!status.isOk()) {
1545 ALOGE("%s: processCaptureResult ERROR : %d:%d", __FUNCTION__, status.getExceptionCode(),
1546 status.getServiceSpecificError());
1547 }
1548
1549 mProcessCaptureResultLock.unlock();
1550 }
1551
waitForV4L2BufferReturnLocked(std::unique_lock<std::mutex> & lk)1552 int ExternalCameraDeviceSession::waitForV4L2BufferReturnLocked(std::unique_lock<std::mutex>& lk) {
1553 ATRACE_CALL();
1554 auto timeout = std::chrono::seconds(kBufferWaitTimeoutSec);
1555 mLock.unlock();
1556 auto st = mV4L2BufferReturned.wait_for(lk, timeout);
1557 // Here we introduce an order where mV4l2BufferLock is acquired before mLock, while
1558 // the normal lock acquisition order is reversed. This is fine because in most of
1559 // cases we are protected by mInterfaceLock. The only thread that can cause deadlock
1560 // is the OutputThread, where we do need to make sure we don't acquire mLock then
1561 // mV4l2BufferLock
1562 mLock.lock();
1563 if (st == std::cv_status::timeout) {
1564 ALOGE("%s: wait for V4L2 buffer return timeout!", __FUNCTION__);
1565 return -1;
1566 }
1567 return 0;
1568 }
1569
supportOfflineLocked(int32_t streamId)1570 bool ExternalCameraDeviceSession::supportOfflineLocked(int32_t streamId) {
1571 const Stream& stream = mStreamMap[streamId];
1572 if (stream.format == PixelFormat::BLOB &&
1573 static_cast<int32_t>(stream.dataSpace) == static_cast<int32_t>(Dataspace::JFIF)) {
1574 return true;
1575 }
1576 // TODO: support YUV output stream?
1577 return false;
1578 }
1579
canDropRequest(const std::vector<int32_t> & offlineStreams,std::shared_ptr<HalRequest> halReq)1580 bool ExternalCameraDeviceSession::canDropRequest(const std::vector<int32_t>& offlineStreams,
1581 std::shared_ptr<HalRequest> halReq) {
1582 for (const auto& buffer : halReq->buffers) {
1583 for (auto offlineStreamId : offlineStreams) {
1584 if (buffer.streamId == offlineStreamId) {
1585 return false;
1586 }
1587 }
1588 }
1589 // Only drop a request completely if it has no offline output
1590 return true;
1591 }
1592
fillOfflineSessionInfo(const std::vector<int32_t> & offlineStreams,std::deque<std::shared_ptr<HalRequest>> & offlineReqs,const std::map<int,CirculatingBuffers> & circulatingBuffers,CameraOfflineSessionInfo * info)1593 void ExternalCameraDeviceSession::fillOfflineSessionInfo(
1594 const std::vector<int32_t>& offlineStreams,
1595 std::deque<std::shared_ptr<HalRequest>>& offlineReqs,
1596 const std::map<int, CirculatingBuffers>& circulatingBuffers,
1597 CameraOfflineSessionInfo* info) {
1598 if (info == nullptr) {
1599 ALOGE("%s: output info must not be null!", __FUNCTION__);
1600 return;
1601 }
1602
1603 info->offlineStreams.resize(offlineStreams.size());
1604 info->offlineRequests.resize(offlineReqs.size());
1605
1606 // Fill in offline reqs and count outstanding buffers
1607 for (size_t i = 0; i < offlineReqs.size(); i++) {
1608 info->offlineRequests[i].frameNumber = offlineReqs[i]->frameNumber;
1609 info->offlineRequests[i].pendingStreams.resize(offlineReqs[i]->buffers.size());
1610 for (size_t bIdx = 0; bIdx < offlineReqs[i]->buffers.size(); bIdx++) {
1611 int32_t streamId = offlineReqs[i]->buffers[bIdx].streamId;
1612 info->offlineRequests[i].pendingStreams[bIdx] = streamId;
1613 }
1614 }
1615
1616 for (size_t i = 0; i < offlineStreams.size(); i++) {
1617 int32_t streamId = offlineStreams[i];
1618 info->offlineStreams[i].id = streamId;
1619 // outstanding buffers are 0 since we are doing hal buffer management and
1620 // offline session will ask for those buffers later
1621 info->offlineStreams[i].numOutstandingBuffers = 0;
1622 const CirculatingBuffers& bufIdMap = circulatingBuffers.at(streamId);
1623 info->offlineStreams[i].circulatingBufferIds.resize(bufIdMap.size());
1624 size_t bIdx = 0;
1625 for (const auto& pair : bufIdMap) {
1626 // Fill in bufferId
1627 info->offlineStreams[i].circulatingBufferIds[bIdx++] = pair.first;
1628 }
1629 }
1630 }
1631
isStreamCombinationSupported(const StreamConfiguration & config,const std::vector<SupportedV4L2Format> & supportedFormats,const ExternalCameraConfig & devCfg)1632 Status ExternalCameraDeviceSession::isStreamCombinationSupported(
1633 const StreamConfiguration& config, const std::vector<SupportedV4L2Format>& supportedFormats,
1634 const ExternalCameraConfig& devCfg) {
1635 if (config.operationMode != StreamConfigurationMode::NORMAL_MODE) {
1636 ALOGE("%s: unsupported operation mode: %d", __FUNCTION__, config.operationMode);
1637 return Status::ILLEGAL_ARGUMENT;
1638 }
1639
1640 if (config.streams.size() == 0) {
1641 ALOGE("%s: cannot configure zero stream", __FUNCTION__);
1642 return Status::ILLEGAL_ARGUMENT;
1643 }
1644
1645 int numProcessedStream = 0;
1646 int numStallStream = 0;
1647 for (const auto& stream : config.streams) {
1648 // Check if the format/width/height combo is supported
1649 if (!isSupported(stream, supportedFormats, devCfg)) {
1650 return Status::ILLEGAL_ARGUMENT;
1651 }
1652 if (stream.format == PixelFormat::BLOB) {
1653 numStallStream++;
1654 } else {
1655 numProcessedStream++;
1656 }
1657 }
1658
1659 if (numProcessedStream > kMaxProcessedStream) {
1660 ALOGE("%s: too many processed streams (expect <= %d, got %d)", __FUNCTION__,
1661 kMaxProcessedStream, numProcessedStream);
1662 return Status::ILLEGAL_ARGUMENT;
1663 }
1664
1665 if (numStallStream > kMaxStallStream) {
1666 ALOGE("%s: too many stall streams (expect <= %d, got %d)", __FUNCTION__, kMaxStallStream,
1667 numStallStream);
1668 return Status::ILLEGAL_ARGUMENT;
1669 }
1670
1671 return Status::OK;
1672 }
updateBufferCaches(const std::vector<BufferCache> & cachesToRemove)1673 void ExternalCameraDeviceSession::updateBufferCaches(
1674 const std::vector<BufferCache>& cachesToRemove) {
1675 Mutex::Autolock _l(mCbsLock);
1676 for (auto& cache : cachesToRemove) {
1677 auto cbsIt = mCirculatingBuffers.find(cache.streamId);
1678 if (cbsIt == mCirculatingBuffers.end()) {
1679 // The stream could have been removed
1680 continue;
1681 }
1682 CirculatingBuffers& cbs = cbsIt->second;
1683 auto it = cbs.find(cache.bufferId);
1684 if (it != cbs.end()) {
1685 sHandleImporter.freeBuffer(it->second);
1686 cbs.erase(it);
1687 } else {
1688 ALOGE("%s: stream %d buffer %" PRIu64 " is not cached", __FUNCTION__, cache.streamId,
1689 cache.bufferId);
1690 }
1691 }
1692 }
1693
processCaptureRequestError(const std::shared_ptr<HalRequest> & req,std::vector<NotifyMsg> * outMsgs,std::vector<CaptureResult> * outResults)1694 Status ExternalCameraDeviceSession::processCaptureRequestError(
1695 const std::shared_ptr<HalRequest>& req, std::vector<NotifyMsg>* outMsgs,
1696 std::vector<CaptureResult>* outResults) {
1697 ATRACE_CALL();
1698 // Return V4L2 buffer to V4L2 buffer queue
1699 std::shared_ptr<V4L2Frame> v4l2Frame = std::static_pointer_cast<V4L2Frame>(req->frameIn);
1700 enqueueV4l2Frame(v4l2Frame);
1701
1702 if (outMsgs == nullptr) {
1703 notifyShutter(req->frameNumber, req->shutterTs);
1704 notifyError(/*frameNum*/ req->frameNumber, /*stream*/ -1, ErrorCode::ERROR_REQUEST);
1705 } else {
1706 NotifyMsg shutter;
1707 shutter.set<NotifyMsg::Tag::shutter>(
1708 ShutterMsg{.frameNumber = req->frameNumber, .timestamp = req->shutterTs});
1709
1710 NotifyMsg error;
1711 error.set<NotifyMsg::Tag::error>(ErrorMsg{.frameNumber = req->frameNumber,
1712 .errorStreamId = -1,
1713 .errorCode = ErrorCode::ERROR_REQUEST});
1714 outMsgs->push_back(shutter);
1715 outMsgs->push_back(error);
1716 }
1717
1718 // Fill output buffers
1719 CaptureResult result;
1720 result.frameNumber = req->frameNumber;
1721 result.partialResult = 1;
1722 result.inputBuffer.streamId = -1;
1723 result.outputBuffers.resize(req->buffers.size());
1724 for (size_t i = 0; i < req->buffers.size(); i++) {
1725 result.outputBuffers[i].streamId = req->buffers[i].streamId;
1726 result.outputBuffers[i].bufferId = req->buffers[i].bufferId;
1727 result.outputBuffers[i].status = BufferStatus::ERROR;
1728 if (req->buffers[i].acquireFence >= 0) {
1729 // numFds = 0 for error
1730 native_handle_t* handle = native_handle_create(/*numFds*/ 0, /*numInts*/ 0);
1731 result.outputBuffers[i].releaseFence = android::dupToAidl(handle);
1732 native_handle_delete(handle);
1733 }
1734 }
1735
1736 // update inflight records
1737 {
1738 std::lock_guard<std::mutex> lk(mInflightFramesLock);
1739 mInflightFrames.erase(req->frameNumber);
1740 }
1741
1742 if (outResults == nullptr) {
1743 // Callback into framework
1744 std::vector<CaptureResult> results(1);
1745 results[0] = std::move(result);
1746 invokeProcessCaptureResultCallback(results, /* tryWriteFmq */ true);
1747 freeReleaseFences(results);
1748 } else {
1749 outResults->push_back(std::move(result));
1750 }
1751 return Status::OK;
1752 }
1753
processCaptureResult(std::shared_ptr<HalRequest> & req)1754 Status ExternalCameraDeviceSession::processCaptureResult(std::shared_ptr<HalRequest>& req) {
1755 ATRACE_CALL();
1756 // Return V4L2 buffer to V4L2 buffer queue
1757 std::shared_ptr<V4L2Frame> v4l2Frame = std::static_pointer_cast<V4L2Frame>(req->frameIn);
1758 enqueueV4l2Frame(v4l2Frame);
1759
1760 // NotifyShutter
1761 notifyShutter(req->frameNumber, req->shutterTs);
1762
1763 // Fill output buffers;
1764 std::vector<CaptureResult> results(1);
1765 CaptureResult& result = results[0];
1766 result.frameNumber = req->frameNumber;
1767 result.partialResult = 1;
1768 result.inputBuffer.streamId = -1;
1769 result.outputBuffers.resize(req->buffers.size());
1770 for (size_t i = 0; i < req->buffers.size(); i++) {
1771 result.outputBuffers[i].streamId = req->buffers[i].streamId;
1772 result.outputBuffers[i].bufferId = req->buffers[i].bufferId;
1773 if (req->buffers[i].fenceTimeout) {
1774 result.outputBuffers[i].status = BufferStatus::ERROR;
1775 if (req->buffers[i].acquireFence >= 0) {
1776 native_handle_t* handle = native_handle_create(/*numFds*/ 1, /*numInts*/ 0);
1777 handle->data[0] = req->buffers[i].acquireFence;
1778 result.outputBuffers[i].releaseFence = android::dupToAidl(handle);
1779 native_handle_delete(handle);
1780 }
1781 notifyError(req->frameNumber, req->buffers[i].streamId, ErrorCode::ERROR_BUFFER);
1782 } else {
1783 result.outputBuffers[i].status = BufferStatus::OK;
1784 // TODO: refactor
1785 if (req->buffers[i].acquireFence >= 0) {
1786 native_handle_t* handle = native_handle_create(/*numFds*/ 1, /*numInts*/ 0);
1787 handle->data[0] = req->buffers[i].acquireFence;
1788 result.outputBuffers[i].releaseFence = android::dupToAidl(handle);
1789 native_handle_delete(handle);
1790 }
1791 }
1792 }
1793
1794 // Fill capture result metadata
1795 fillCaptureResult(req->setting, req->shutterTs);
1796 const camera_metadata_t* rawResult = req->setting.getAndLock();
1797 convertToAidl(rawResult, &result.result);
1798 req->setting.unlock(rawResult);
1799
1800 // update inflight records
1801 {
1802 std::lock_guard<std::mutex> lk(mInflightFramesLock);
1803 mInflightFrames.erase(req->frameNumber);
1804 }
1805
1806 // Callback into framework
1807 invokeProcessCaptureResultCallback(results, /* tryWriteFmq */ true);
1808 freeReleaseFences(results);
1809 return Status::OK;
1810 }
1811
getJpegBufferSize(int32_t width,int32_t height) const1812 ssize_t ExternalCameraDeviceSession::getJpegBufferSize(int32_t width, int32_t height) const {
1813 // Constant from camera3.h
1814 const ssize_t kMinJpegBufferSize = 256 * 1024 + sizeof(CameraBlob);
1815 // Get max jpeg size (area-wise).
1816 if (mMaxJpegResolution.width == 0) {
1817 ALOGE("%s: No supported JPEG stream", __FUNCTION__);
1818 return BAD_VALUE;
1819 }
1820
1821 // Get max jpeg buffer size
1822 ssize_t maxJpegBufferSize = 0;
1823 camera_metadata_ro_entry jpegBufMaxSize = mCameraCharacteristics.find(ANDROID_JPEG_MAX_SIZE);
1824 if (jpegBufMaxSize.count == 0) {
1825 ALOGE("%s: Can't find maximum JPEG size in static metadata!", __FUNCTION__);
1826 return BAD_VALUE;
1827 }
1828 maxJpegBufferSize = jpegBufMaxSize.data.i32[0];
1829
1830 if (maxJpegBufferSize <= kMinJpegBufferSize) {
1831 ALOGE("%s: ANDROID_JPEG_MAX_SIZE (%zd) <= kMinJpegBufferSize (%zd)", __FUNCTION__,
1832 maxJpegBufferSize, kMinJpegBufferSize);
1833 return BAD_VALUE;
1834 }
1835
1836 // Calculate final jpeg buffer size for the given resolution.
1837 float scaleFactor =
1838 ((float)(width * height)) / (mMaxJpegResolution.width * mMaxJpegResolution.height);
1839 ssize_t jpegBufferSize =
1840 scaleFactor * (maxJpegBufferSize - kMinJpegBufferSize) + kMinJpegBufferSize;
1841 if (jpegBufferSize > maxJpegBufferSize) {
1842 jpegBufferSize = maxJpegBufferSize;
1843 }
1844
1845 return jpegBufferSize;
1846 }
dump(int fd,const char **,uint32_t)1847 binder_status_t ExternalCameraDeviceSession::dump(int fd, const char** /*args*/,
1848 uint32_t /*numArgs*/) {
1849 bool intfLocked = tryLock(mInterfaceLock);
1850 if (!intfLocked) {
1851 dprintf(fd, "!! ExternalCameraDeviceSession interface may be deadlocked !!\n");
1852 }
1853
1854 if (isClosed()) {
1855 dprintf(fd, "External camera %s is closed\n", mCameraId.c_str());
1856 return STATUS_OK;
1857 }
1858
1859 bool streaming = false;
1860 size_t v4L2BufferCount = 0;
1861 SupportedV4L2Format streamingFmt;
1862 {
1863 bool sessionLocked = tryLock(mLock);
1864 if (!sessionLocked) {
1865 dprintf(fd, "!! ExternalCameraDeviceSession mLock may be deadlocked !!\n");
1866 }
1867 streaming = mV4l2Streaming;
1868 streamingFmt = mV4l2StreamingFmt;
1869 v4L2BufferCount = mV4L2BufferCount;
1870
1871 if (sessionLocked) {
1872 mLock.unlock();
1873 }
1874 }
1875
1876 std::unordered_set<uint32_t> inflightFrames;
1877 {
1878 bool iffLocked = tryLock(mInflightFramesLock);
1879 if (!iffLocked) {
1880 dprintf(fd,
1881 "!! ExternalCameraDeviceSession mInflightFramesLock may be deadlocked !!\n");
1882 }
1883 inflightFrames = mInflightFrames;
1884 if (iffLocked) {
1885 mInflightFramesLock.unlock();
1886 }
1887 }
1888
1889 dprintf(fd, "External camera %s V4L2 FD %d, cropping type %s, %s\n", mCameraId.c_str(),
1890 mV4l2Fd.get(), (mCroppingType == VERTICAL) ? "vertical" : "horizontal",
1891 streaming ? "streaming" : "not streaming");
1892
1893 if (streaming) {
1894 // TODO: dump fps later
1895 dprintf(fd, "Current V4L2 format %c%c%c%c %dx%d @ %ffps\n", streamingFmt.fourcc & 0xFF,
1896 (streamingFmt.fourcc >> 8) & 0xFF, (streamingFmt.fourcc >> 16) & 0xFF,
1897 (streamingFmt.fourcc >> 24) & 0xFF, streamingFmt.width, streamingFmt.height,
1898 mV4l2StreamingFps);
1899
1900 size_t numDequeuedV4l2Buffers = 0;
1901 {
1902 std::lock_guard<std::mutex> lk(mV4l2BufferLock);
1903 numDequeuedV4l2Buffers = mNumDequeuedV4l2Buffers;
1904 }
1905 dprintf(fd, "V4L2 buffer queue size %zu, dequeued %zu\n", v4L2BufferCount,
1906 numDequeuedV4l2Buffers);
1907 }
1908
1909 dprintf(fd, "In-flight frames (not sorted):");
1910 for (const auto& frameNumber : inflightFrames) {
1911 dprintf(fd, "%d, ", frameNumber);
1912 }
1913 dprintf(fd, "\n");
1914 mOutputThread->dump(fd);
1915 dprintf(fd, "\n");
1916
1917 if (intfLocked) {
1918 mInterfaceLock.unlock();
1919 }
1920
1921 return STATUS_OK;
1922 }
1923
1924 // Start ExternalCameraDeviceSession::BufferRequestThread functions
BufferRequestThread(std::weak_ptr<OutputThreadInterface> parent,std::shared_ptr<ICameraDeviceCallback> callbacks)1925 ExternalCameraDeviceSession::BufferRequestThread::BufferRequestThread(
1926 std::weak_ptr<OutputThreadInterface> parent,
1927 std::shared_ptr<ICameraDeviceCallback> callbacks)
1928 : mParent(parent), mCallbacks(callbacks) {}
1929
requestBufferStart(const std::vector<HalStreamBuffer> & bufReqs)1930 int ExternalCameraDeviceSession::BufferRequestThread::requestBufferStart(
1931 const std::vector<HalStreamBuffer>& bufReqs) {
1932 if (bufReqs.empty()) {
1933 ALOGE("%s: bufReqs is empty!", __FUNCTION__);
1934 return -1;
1935 }
1936
1937 {
1938 std::lock_guard<std::mutex> lk(mLock);
1939 if (mRequestingBuffer) {
1940 ALOGE("%s: BufferRequestThread does not support more than one concurrent request!",
1941 __FUNCTION__);
1942 return -1;
1943 }
1944
1945 mBufferReqs = bufReqs;
1946 mRequestingBuffer = true;
1947 }
1948 mRequestCond.notify_one();
1949 return 0;
1950 }
1951
waitForBufferRequestDone(std::vector<HalStreamBuffer> * outBufReqs)1952 int ExternalCameraDeviceSession::BufferRequestThread::waitForBufferRequestDone(
1953 std::vector<HalStreamBuffer>* outBufReqs) {
1954 std::unique_lock<std::mutex> lk(mLock);
1955 if (!mRequestingBuffer) {
1956 ALOGE("%s: no pending buffer request!", __FUNCTION__);
1957 return -1;
1958 }
1959
1960 if (mPendingReturnBufferReqs.empty()) {
1961 std::chrono::milliseconds timeout = std::chrono::milliseconds(kReqProcTimeoutMs);
1962 auto st = mRequestDoneCond.wait_for(lk, timeout);
1963 if (st == std::cv_status::timeout) {
1964 mRequestingBuffer = false;
1965 ALOGE("%s: wait for buffer request finish timeout!", __FUNCTION__);
1966 return -1;
1967 }
1968
1969 if (mPendingReturnBufferReqs.empty()) {
1970 mRequestingBuffer = false;
1971 ALOGE("%s: cameraservice did not return any buffers!", __FUNCTION__);
1972 return -1;
1973 }
1974 }
1975 mRequestingBuffer = false;
1976 *outBufReqs = std::move(mPendingReturnBufferReqs);
1977 mPendingReturnBufferReqs.clear();
1978 return 0;
1979 }
1980
waitForNextRequest()1981 void ExternalCameraDeviceSession::BufferRequestThread::waitForNextRequest() {
1982 ATRACE_CALL();
1983 std::unique_lock<std::mutex> lk(mLock);
1984 int waitTimes = 0;
1985 while (mBufferReqs.empty()) {
1986 if (exitPending()) {
1987 return;
1988 }
1989 auto timeout = std::chrono::milliseconds(kReqWaitTimeoutMs);
1990 auto st = mRequestCond.wait_for(lk, timeout);
1991 if (st == std::cv_status::timeout) {
1992 waitTimes++;
1993 if (waitTimes == kReqWaitTimesWarn) {
1994 // BufferRequestThread just wait forever for new buffer request
1995 // But it will print some periodic warning indicating it's waiting
1996 ALOGV("%s: still waiting for new buffer request", __FUNCTION__);
1997 waitTimes = 0;
1998 }
1999 }
2000 }
2001
2002 // Fill in BufferRequest
2003 mHalBufferReqs.resize(mBufferReqs.size());
2004 for (size_t i = 0; i < mHalBufferReqs.size(); i++) {
2005 mHalBufferReqs[i].streamId = mBufferReqs[i].streamId;
2006 mHalBufferReqs[i].numBuffersRequested = 1;
2007 }
2008 }
2009
threadLoop()2010 bool ExternalCameraDeviceSession::BufferRequestThread::threadLoop() {
2011 waitForNextRequest();
2012 if (exitPending()) {
2013 return false;
2014 }
2015
2016 ATRACE_BEGIN("AIDL requestStreamBuffers");
2017 BufferRequestStatus status;
2018 std::vector<StreamBufferRet> bufRets;
2019 ScopedAStatus ret = mCallbacks->requestStreamBuffers(mHalBufferReqs, &bufRets, &status);
2020 if (!ret.isOk()) {
2021 ALOGE("%s: Transaction error: %d:%d", __FUNCTION__, ret.getExceptionCode(),
2022 ret.getServiceSpecificError());
2023 mBufferReqs.clear();
2024 mRequestDoneCond.notify_one();
2025 return false;
2026 }
2027
2028 std::unique_lock<std::mutex> lk(mLock);
2029 if (status == BufferRequestStatus::OK || status == BufferRequestStatus::FAILED_PARTIAL) {
2030 if (bufRets.size() != mHalBufferReqs.size()) {
2031 ALOGE("%s: expect %zu buffer requests returned, only got %zu", __FUNCTION__,
2032 mHalBufferReqs.size(), bufRets.size());
2033 mBufferReqs.clear();
2034 lk.unlock();
2035 mRequestDoneCond.notify_one();
2036 return false;
2037 }
2038
2039 auto parent = mParent.lock();
2040 if (parent == nullptr) {
2041 ALOGE("%s: session has been disconnected!", __FUNCTION__);
2042 mBufferReqs.clear();
2043 lk.unlock();
2044 mRequestDoneCond.notify_one();
2045 return false;
2046 }
2047
2048 std::vector<int> importedFences;
2049 importedFences.resize(bufRets.size());
2050 bool hasError = false;
2051 for (size_t i = 0; i < bufRets.size(); i++) {
2052 int streamId = bufRets[i].streamId;
2053 switch (bufRets[i].val.getTag()) {
2054 case StreamBuffersVal::Tag::error:
2055 continue;
2056 case StreamBuffersVal::Tag::buffers: {
2057 const std::vector<StreamBuffer>& hBufs =
2058 bufRets[i].val.get<StreamBuffersVal::Tag::buffers>();
2059 if (hBufs.size() != 1) {
2060 ALOGE("%s: expect 1 buffer returned, got %zu!", __FUNCTION__, hBufs.size());
2061 hasError = true;
2062 break;
2063 }
2064 const StreamBuffer& hBuf = hBufs[0];
2065
2066 mBufferReqs[i].bufferId = hBuf.bufferId;
2067 // TODO: create a batch import API so we don't need to lock/unlock mCbsLock
2068 // repeatedly?
2069 lk.unlock();
2070 native_handle_t* h = makeFromAidl(hBuf.buffer);
2071 Status s = parent->importBuffer(streamId, hBuf.bufferId, h,
2072 /*out*/ &mBufferReqs[i].bufPtr);
2073 native_handle_delete(h);
2074 lk.lock();
2075
2076 if (s != Status::OK) {
2077 ALOGE("%s: stream %d import buffer failed!", __FUNCTION__, streamId);
2078 cleanupInflightFences(importedFences, i - 1);
2079 hasError = true;
2080 break;
2081 }
2082 h = makeFromAidl(hBuf.acquireFence);
2083 if (!sHandleImporter.importFence(h, mBufferReqs[i].acquireFence)) {
2084 ALOGE("%s: stream %d import fence failed!", __FUNCTION__, streamId);
2085 cleanupInflightFences(importedFences, i - 1);
2086 native_handle_delete(h);
2087 hasError = true;
2088 break;
2089 }
2090 native_handle_delete(h);
2091 importedFences[i] = mBufferReqs[i].acquireFence;
2092 } break;
2093 default:
2094 ALOGE("%s: Unknown StreamBuffersVal!", __FUNCTION__);
2095 hasError = true;
2096 break;
2097 }
2098 if (hasError) {
2099 mBufferReqs.clear();
2100 lk.unlock();
2101 mRequestDoneCond.notify_one();
2102 return true;
2103 }
2104 }
2105 } else {
2106 ALOGE("%s: requestStreamBuffers call failed!", __FUNCTION__);
2107 mBufferReqs.clear();
2108 lk.unlock();
2109 mRequestDoneCond.notify_one();
2110 return true;
2111 }
2112
2113 mPendingReturnBufferReqs = std::move(mBufferReqs);
2114 mBufferReqs.clear();
2115
2116 lk.unlock();
2117 mRequestDoneCond.notify_one();
2118 return true;
2119 }
2120
2121 // End ExternalCameraDeviceSession::BufferRequestThread functions
2122
2123 // Start ExternalCameraDeviceSession::OutputThread functions
2124
OutputThread(std::weak_ptr<OutputThreadInterface> parent,CroppingType ct,const common::V1_0::helper::CameraMetadata & chars,std::shared_ptr<BufferRequestThread> bufReqThread)2125 ExternalCameraDeviceSession::OutputThread::OutputThread(
2126 std::weak_ptr<OutputThreadInterface> parent, CroppingType ct,
2127 const common::V1_0::helper::CameraMetadata& chars,
2128 std::shared_ptr<BufferRequestThread> bufReqThread)
2129 : mParent(parent),
2130 mCroppingType(ct),
2131 mCameraCharacteristics(chars),
2132 mBufferRequestThread(bufReqThread) {}
2133
~OutputThread()2134 ExternalCameraDeviceSession::OutputThread::~OutputThread() {}
2135
allocateIntermediateBuffers(const Size & v4lSize,const Size & thumbSize,const std::vector<Stream> & streams,uint32_t blobBufferSize)2136 Status ExternalCameraDeviceSession::OutputThread::allocateIntermediateBuffers(
2137 const Size& v4lSize, const Size& thumbSize, const std::vector<Stream>& streams,
2138 uint32_t blobBufferSize) {
2139 std::lock_guard<std::mutex> lk(mBufferLock);
2140 if (!mScaledYu12Frames.empty()) {
2141 ALOGE("%s: intermediate buffer pool has %zu inflight buffers! (expect 0)", __FUNCTION__,
2142 mScaledYu12Frames.size());
2143 return Status::INTERNAL_ERROR;
2144 }
2145
2146 // Allocating intermediate YU12 frame
2147 if (mYu12Frame == nullptr || mYu12Frame->mWidth != v4lSize.width ||
2148 mYu12Frame->mHeight != v4lSize.height) {
2149 mYu12Frame.reset();
2150 mYu12Frame = std::make_shared<AllocatedFrame>(v4lSize.width, v4lSize.height);
2151 int ret = mYu12Frame->allocate(&mYu12FrameLayout);
2152 if (ret != 0) {
2153 ALOGE("%s: allocating YU12 frame failed!", __FUNCTION__);
2154 return Status::INTERNAL_ERROR;
2155 }
2156 }
2157
2158 // Allocating intermediate YU12 thumbnail frame
2159 if (mYu12ThumbFrame == nullptr || mYu12ThumbFrame->mWidth != thumbSize.width ||
2160 mYu12ThumbFrame->mHeight != thumbSize.height) {
2161 mYu12ThumbFrame.reset();
2162 mYu12ThumbFrame = std::make_shared<AllocatedFrame>(thumbSize.width, thumbSize.height);
2163 int ret = mYu12ThumbFrame->allocate(&mYu12ThumbFrameLayout);
2164 if (ret != 0) {
2165 ALOGE("%s: allocating YU12 thumb frame failed!", __FUNCTION__);
2166 return Status::INTERNAL_ERROR;
2167 }
2168 }
2169
2170 // Allocating scaled buffers
2171 for (const auto& stream : streams) {
2172 Size sz = {stream.width, stream.height};
2173 if (sz == v4lSize) {
2174 continue; // Don't need an intermediate buffer same size as v4lBuffer
2175 }
2176 if (mIntermediateBuffers.count(sz) == 0) {
2177 // Create new intermediate buffer
2178 std::shared_ptr<AllocatedFrame> buf =
2179 std::make_shared<AllocatedFrame>(stream.width, stream.height);
2180 int ret = buf->allocate();
2181 if (ret != 0) {
2182 ALOGE("%s: allocating intermediate YU12 frame %dx%d failed!", __FUNCTION__,
2183 stream.width, stream.height);
2184 return Status::INTERNAL_ERROR;
2185 }
2186 mIntermediateBuffers[sz] = buf;
2187 }
2188 }
2189
2190 // Remove unconfigured buffers
2191 auto it = mIntermediateBuffers.begin();
2192 while (it != mIntermediateBuffers.end()) {
2193 bool configured = false;
2194 auto sz = it->first;
2195 for (const auto& stream : streams) {
2196 if (stream.width == sz.width && stream.height == sz.height) {
2197 configured = true;
2198 break;
2199 }
2200 }
2201 if (configured) {
2202 it++;
2203 } else {
2204 it = mIntermediateBuffers.erase(it);
2205 }
2206 }
2207
2208 // Allocate mute test pattern frame
2209 mMuteTestPatternFrame.resize(mYu12Frame->mWidth * mYu12Frame->mHeight * 3);
2210
2211 mBlobBufferSize = blobBufferSize;
2212 return Status::OK;
2213 }
2214
submitRequest(const std::shared_ptr<HalRequest> & req)2215 Status ExternalCameraDeviceSession::OutputThread::submitRequest(
2216 const std::shared_ptr<HalRequest>& req) {
2217 std::unique_lock<std::mutex> lk(mRequestListLock);
2218 mRequestList.push_back(req);
2219 lk.unlock();
2220 mRequestCond.notify_one();
2221 return Status::OK;
2222 }
2223
flush()2224 void ExternalCameraDeviceSession::OutputThread::flush() {
2225 ATRACE_CALL();
2226 auto parent = mParent.lock();
2227 if (parent == nullptr) {
2228 ALOGE("%s: session has been disconnected!", __FUNCTION__);
2229 return;
2230 }
2231
2232 std::unique_lock<std::mutex> lk(mRequestListLock);
2233 std::list<std::shared_ptr<HalRequest>> reqs = std::move(mRequestList);
2234 mRequestList.clear();
2235 if (mProcessingRequest) {
2236 auto timeout = std::chrono::seconds(kFlushWaitTimeoutSec);
2237 auto st = mRequestDoneCond.wait_for(lk, timeout);
2238 if (st == std::cv_status::timeout) {
2239 ALOGE("%s: wait for inflight request finish timeout!", __FUNCTION__);
2240 }
2241 }
2242
2243 ALOGV("%s: flushing inflight requests", __FUNCTION__);
2244 lk.unlock();
2245 for (const auto& req : reqs) {
2246 parent->processCaptureRequestError(req);
2247 }
2248 }
2249
dump(int fd)2250 void ExternalCameraDeviceSession::OutputThread::dump(int fd) {
2251 std::lock_guard<std::mutex> lk(mRequestListLock);
2252 if (mProcessingRequest) {
2253 dprintf(fd, "OutputThread processing frame %d\n", mProcessingFrameNumber);
2254 } else {
2255 dprintf(fd, "OutputThread not processing any frames\n");
2256 }
2257 dprintf(fd, "OutputThread request list contains frame: ");
2258 for (const auto& req : mRequestList) {
2259 dprintf(fd, "%d, ", req->frameNumber);
2260 }
2261 dprintf(fd, "\n");
2262 }
2263
setExifMakeModel(const std::string & make,const std::string & model)2264 void ExternalCameraDeviceSession::OutputThread::setExifMakeModel(const std::string& make,
2265 const std::string& model) {
2266 mExifMake = make;
2267 mExifModel = model;
2268 }
2269
2270 std::list<std::shared_ptr<HalRequest>>
switchToOffline()2271 ExternalCameraDeviceSession::OutputThread::switchToOffline() {
2272 ATRACE_CALL();
2273 auto parent = mParent.lock();
2274 if (parent == nullptr) {
2275 ALOGE("%s: session has been disconnected!", __FUNCTION__);
2276 return {};
2277 }
2278
2279 std::unique_lock<std::mutex> lk(mRequestListLock);
2280 std::list<std::shared_ptr<HalRequest>> reqs = std::move(mRequestList);
2281 mRequestList.clear();
2282 if (mProcessingRequest) {
2283 auto timeout = std::chrono::seconds(kFlushWaitTimeoutSec);
2284 auto st = mRequestDoneCond.wait_for(lk, timeout);
2285 if (st == std::cv_status::timeout) {
2286 ALOGE("%s: wait for inflight request finish timeout!", __FUNCTION__);
2287 }
2288 }
2289 lk.unlock();
2290 clearIntermediateBuffers();
2291 ALOGV("%s: returning %zu request for offline processing", __FUNCTION__, reqs.size());
2292 return reqs;
2293 }
2294
requestBufferStart(const std::vector<HalStreamBuffer> & bufs)2295 int ExternalCameraDeviceSession::OutputThread::requestBufferStart(
2296 const std::vector<HalStreamBuffer>& bufs) {
2297 if (mBufferRequestThread == nullptr) {
2298 return 0;
2299 }
2300 return mBufferRequestThread->requestBufferStart(bufs);
2301 }
2302
waitForBufferRequestDone(std::vector<HalStreamBuffer> * outBufs)2303 int ExternalCameraDeviceSession::OutputThread::waitForBufferRequestDone(
2304 std::vector<HalStreamBuffer>* outBufs) {
2305 if (mBufferRequestThread == nullptr) {
2306 return 0;
2307 }
2308 return mBufferRequestThread->waitForBufferRequestDone(outBufs);
2309 }
2310
waitForNextRequest(std::shared_ptr<HalRequest> * out)2311 void ExternalCameraDeviceSession::OutputThread::waitForNextRequest(
2312 std::shared_ptr<HalRequest>* out) {
2313 ATRACE_CALL();
2314 if (out == nullptr) {
2315 ALOGE("%s: out is null", __FUNCTION__);
2316 return;
2317 }
2318
2319 std::unique_lock<std::mutex> lk(mRequestListLock);
2320 int waitTimes = 0;
2321 while (mRequestList.empty()) {
2322 if (exitPending()) {
2323 return;
2324 }
2325 auto timeout = std::chrono::milliseconds(kReqWaitTimeoutMs);
2326 auto st = mRequestCond.wait_for(lk, timeout);
2327 if (st == std::cv_status::timeout) {
2328 waitTimes++;
2329 if (waitTimes == kReqWaitTimesMax) {
2330 // no new request, return
2331 return;
2332 }
2333 }
2334 }
2335 *out = mRequestList.front();
2336 mRequestList.pop_front();
2337 mProcessingRequest = true;
2338 mProcessingFrameNumber = (*out)->frameNumber;
2339 }
2340
signalRequestDone()2341 void ExternalCameraDeviceSession::OutputThread::signalRequestDone() {
2342 std::unique_lock<std::mutex> lk(mRequestListLock);
2343 mProcessingRequest = false;
2344 mProcessingFrameNumber = 0;
2345 lk.unlock();
2346 mRequestDoneCond.notify_one();
2347 }
2348
cropAndScaleLocked(std::shared_ptr<AllocatedFrame> & in,const Size & outSz,YCbCrLayout * out)2349 int ExternalCameraDeviceSession::OutputThread::cropAndScaleLocked(
2350 std::shared_ptr<AllocatedFrame>& in, const Size& outSz, YCbCrLayout* out) {
2351 Size inSz = {in->mWidth, in->mHeight};
2352
2353 int ret;
2354 if (inSz == outSz) {
2355 ret = in->getLayout(out);
2356 if (ret != 0) {
2357 ALOGE("%s: failed to get input image layout", __FUNCTION__);
2358 return ret;
2359 }
2360 return ret;
2361 }
2362
2363 // Cropping to output aspect ratio
2364 IMapper::Rect inputCrop;
2365 ret = getCropRect(mCroppingType, inSz, outSz, &inputCrop);
2366 if (ret != 0) {
2367 ALOGE("%s: failed to compute crop rect for output size %dx%d", __FUNCTION__, outSz.width,
2368 outSz.height);
2369 return ret;
2370 }
2371
2372 YCbCrLayout croppedLayout;
2373 ret = in->getCroppedLayout(inputCrop, &croppedLayout);
2374 if (ret != 0) {
2375 ALOGE("%s: failed to crop input image %dx%d to output size %dx%d", __FUNCTION__, inSz.width,
2376 inSz.height, outSz.width, outSz.height);
2377 return ret;
2378 }
2379
2380 if ((mCroppingType == VERTICAL && inSz.width == outSz.width) ||
2381 (mCroppingType == HORIZONTAL && inSz.height == outSz.height)) {
2382 // No scale is needed
2383 *out = croppedLayout;
2384 return 0;
2385 }
2386
2387 auto it = mScaledYu12Frames.find(outSz);
2388 std::shared_ptr<AllocatedFrame> scaledYu12Buf;
2389 if (it != mScaledYu12Frames.end()) {
2390 scaledYu12Buf = it->second;
2391 } else {
2392 it = mIntermediateBuffers.find(outSz);
2393 if (it == mIntermediateBuffers.end()) {
2394 ALOGE("%s: failed to find intermediate buffer size %dx%d", __FUNCTION__, outSz.width,
2395 outSz.height);
2396 return -1;
2397 }
2398 scaledYu12Buf = it->second;
2399 }
2400 // Scale
2401 YCbCrLayout outLayout;
2402 ret = scaledYu12Buf->getLayout(&outLayout);
2403 if (ret != 0) {
2404 ALOGE("%s: failed to get output buffer layout", __FUNCTION__);
2405 return ret;
2406 }
2407
2408 ret = libyuv::I420Scale(
2409 static_cast<uint8_t*>(croppedLayout.y), croppedLayout.yStride,
2410 static_cast<uint8_t*>(croppedLayout.cb), croppedLayout.cStride,
2411 static_cast<uint8_t*>(croppedLayout.cr), croppedLayout.cStride, inputCrop.width,
2412 inputCrop.height, static_cast<uint8_t*>(outLayout.y), outLayout.yStride,
2413 static_cast<uint8_t*>(outLayout.cb), outLayout.cStride,
2414 static_cast<uint8_t*>(outLayout.cr), outLayout.cStride, outSz.width, outSz.height,
2415 // TODO: b/72261744 see if we can use better filter without losing too much perf
2416 libyuv::FilterMode::kFilterNone);
2417
2418 if (ret != 0) {
2419 ALOGE("%s: failed to scale buffer from %dx%d to %dx%d. Ret %d", __FUNCTION__,
2420 inputCrop.width, inputCrop.height, outSz.width, outSz.height, ret);
2421 return ret;
2422 }
2423
2424 *out = outLayout;
2425 mScaledYu12Frames.insert({outSz, scaledYu12Buf});
2426 return 0;
2427 }
2428
cropAndScaleThumbLocked(std::shared_ptr<AllocatedFrame> & in,const Size & outSz,YCbCrLayout * out)2429 int ExternalCameraDeviceSession::OutputThread::cropAndScaleThumbLocked(
2430 std::shared_ptr<AllocatedFrame>& in, const Size& outSz, YCbCrLayout* out) {
2431 Size inSz{in->mWidth, in->mHeight};
2432
2433 if ((outSz.width * outSz.height) > (mYu12ThumbFrame->mWidth * mYu12ThumbFrame->mHeight)) {
2434 ALOGE("%s: Requested thumbnail size too big (%d,%d) > (%d,%d)", __FUNCTION__, outSz.width,
2435 outSz.height, mYu12ThumbFrame->mWidth, mYu12ThumbFrame->mHeight);
2436 return -1;
2437 }
2438
2439 int ret;
2440
2441 /* This will crop-and-zoom the input YUV frame to the thumbnail size
2442 * Based on the following logic:
2443 * 1) Square pixels come in, square pixels come out, therefore single
2444 * scale factor is computed to either make input bigger or smaller
2445 * depending on if we are upscaling or downscaling
2446 * 2) That single scale factor would either make height too tall or width
2447 * too wide so we need to crop the input either horizontally or vertically
2448 * but not both
2449 */
2450
2451 /* Convert the input and output dimensions into floats for ease of math */
2452 float fWin = static_cast<float>(inSz.width);
2453 float fHin = static_cast<float>(inSz.height);
2454 float fWout = static_cast<float>(outSz.width);
2455 float fHout = static_cast<float>(outSz.height);
2456
2457 /* Compute the one scale factor from (1) above, it will be the smaller of
2458 * the two possibilities. */
2459 float scaleFactor = std::min(fHin / fHout, fWin / fWout);
2460
2461 /* Since we are crop-and-zooming (as opposed to letter/pillar boxing) we can
2462 * simply multiply the output by our scaleFactor to get the cropped input
2463 * size. Note that at least one of {fWcrop, fHcrop} is going to wind up
2464 * being {fWin, fHin} respectively because fHout or fWout cancels out the
2465 * scaleFactor calculation above.
2466 *
2467 * Specifically:
2468 * if ( fHin / fHout ) < ( fWin / fWout ) we crop the sides off
2469 * input, in which case
2470 * scaleFactor = fHin / fHout
2471 * fWcrop = fHin / fHout * fWout
2472 * fHcrop = fHin
2473 *
2474 * Note that fWcrop <= fWin ( because ( fHin / fHout ) * fWout < fWin, which
2475 * is just the inequality above with both sides multiplied by fWout
2476 *
2477 * on the other hand if ( fWin / fWout ) < ( fHin / fHout) we crop the top
2478 * and the bottom off of input, and
2479 * scaleFactor = fWin / fWout
2480 * fWcrop = fWin
2481 * fHCrop = fWin / fWout * fHout
2482 */
2483 float fWcrop = scaleFactor * fWout;
2484 float fHcrop = scaleFactor * fHout;
2485
2486 /* Convert to integer and truncate to an even number */
2487 Size cropSz = {.width = 2 * static_cast<int32_t>(fWcrop / 2.0f),
2488 .height = 2 * static_cast<int32_t>(fHcrop / 2.0f)};
2489
2490 /* Convert to a centered rectange with even top/left */
2491 IMapper::Rect inputCrop{.left = 2 * static_cast<int32_t>((inSz.width - cropSz.width) / 4),
2492 .top = 2 * static_cast<int32_t>((inSz.height - cropSz.height) / 4),
2493 .width = static_cast<int32_t>(cropSz.width),
2494 .height = static_cast<int32_t>(cropSz.height)};
2495
2496 if ((inputCrop.top < 0) || (inputCrop.top >= static_cast<int32_t>(inSz.height)) ||
2497 (inputCrop.left < 0) || (inputCrop.left >= static_cast<int32_t>(inSz.width)) ||
2498 (inputCrop.width <= 0) ||
2499 (inputCrop.width + inputCrop.left > static_cast<int32_t>(inSz.width)) ||
2500 (inputCrop.height <= 0) ||
2501 (inputCrop.height + inputCrop.top > static_cast<int32_t>(inSz.height))) {
2502 ALOGE("%s: came up with really wrong crop rectangle", __FUNCTION__);
2503 ALOGE("%s: input layout %dx%d to for output size %dx%d", __FUNCTION__, inSz.width,
2504 inSz.height, outSz.width, outSz.height);
2505 ALOGE("%s: computed input crop +%d,+%d %dx%d", __FUNCTION__, inputCrop.left, inputCrop.top,
2506 inputCrop.width, inputCrop.height);
2507 return -1;
2508 }
2509
2510 YCbCrLayout inputLayout;
2511 ret = in->getCroppedLayout(inputCrop, &inputLayout);
2512 if (ret != 0) {
2513 ALOGE("%s: failed to crop input layout %dx%d to for output size %dx%d", __FUNCTION__,
2514 inSz.width, inSz.height, outSz.width, outSz.height);
2515 ALOGE("%s: computed input crop +%d,+%d %dx%d", __FUNCTION__, inputCrop.left, inputCrop.top,
2516 inputCrop.width, inputCrop.height);
2517 return ret;
2518 }
2519 ALOGV("%s: crop input layout %dx%d to for output size %dx%d", __FUNCTION__, inSz.width,
2520 inSz.height, outSz.width, outSz.height);
2521 ALOGV("%s: computed input crop +%d,+%d %dx%d", __FUNCTION__, inputCrop.left, inputCrop.top,
2522 inputCrop.width, inputCrop.height);
2523
2524 // Scale
2525 YCbCrLayout outFullLayout;
2526
2527 ret = mYu12ThumbFrame->getLayout(&outFullLayout);
2528 if (ret != 0) {
2529 ALOGE("%s: failed to get output buffer layout", __FUNCTION__);
2530 return ret;
2531 }
2532
2533 ret = libyuv::I420Scale(static_cast<uint8_t*>(inputLayout.y), inputLayout.yStride,
2534 static_cast<uint8_t*>(inputLayout.cb), inputLayout.cStride,
2535 static_cast<uint8_t*>(inputLayout.cr), inputLayout.cStride,
2536 inputCrop.width, inputCrop.height,
2537 static_cast<uint8_t*>(outFullLayout.y), outFullLayout.yStride,
2538 static_cast<uint8_t*>(outFullLayout.cb), outFullLayout.cStride,
2539 static_cast<uint8_t*>(outFullLayout.cr), outFullLayout.cStride,
2540 outSz.width, outSz.height, libyuv::FilterMode::kFilterNone);
2541
2542 if (ret != 0) {
2543 ALOGE("%s: failed to scale buffer from %dx%d to %dx%d. Ret %d", __FUNCTION__,
2544 inputCrop.width, inputCrop.height, outSz.width, outSz.height, ret);
2545 return ret;
2546 }
2547
2548 *out = outFullLayout;
2549 return 0;
2550 }
2551
createJpegLocked(HalStreamBuffer & halBuf,const common::V1_0::helper::CameraMetadata & setting)2552 int ExternalCameraDeviceSession::OutputThread::createJpegLocked(
2553 HalStreamBuffer& halBuf, const common::V1_0::helper::CameraMetadata& setting) {
2554 ATRACE_CALL();
2555 int ret;
2556 auto lfail = [&](auto... args) {
2557 ALOGE(args...);
2558
2559 return 1;
2560 };
2561 auto parent = mParent.lock();
2562 if (parent == nullptr) {
2563 ALOGE("%s: session has been disconnected!", __FUNCTION__);
2564 return 1;
2565 }
2566
2567 ALOGV("%s: HAL buffer sid: %d bid: %" PRIu64 " w: %u h: %u", __FUNCTION__, halBuf.streamId,
2568 static_cast<uint64_t>(halBuf.bufferId), halBuf.width, halBuf.height);
2569 ALOGV("%s: HAL buffer fmt: %x usage: %" PRIx64 " ptr: %p", __FUNCTION__, halBuf.format,
2570 static_cast<uint64_t>(halBuf.usage), halBuf.bufPtr);
2571 ALOGV("%s: YV12 buffer %d x %d", __FUNCTION__, mYu12Frame->mWidth, mYu12Frame->mHeight);
2572
2573 int jpegQuality, thumbQuality;
2574 Size thumbSize;
2575 bool outputThumbnail = true;
2576
2577 if (setting.exists(ANDROID_JPEG_QUALITY)) {
2578 camera_metadata_ro_entry entry = setting.find(ANDROID_JPEG_QUALITY);
2579 jpegQuality = entry.data.u8[0];
2580 } else {
2581 return lfail("%s: ANDROID_JPEG_QUALITY not set", __FUNCTION__);
2582 }
2583
2584 if (setting.exists(ANDROID_JPEG_THUMBNAIL_QUALITY)) {
2585 camera_metadata_ro_entry entry = setting.find(ANDROID_JPEG_THUMBNAIL_QUALITY);
2586 thumbQuality = entry.data.u8[0];
2587 } else {
2588 return lfail("%s: ANDROID_JPEG_THUMBNAIL_QUALITY not set", __FUNCTION__);
2589 }
2590
2591 if (setting.exists(ANDROID_JPEG_THUMBNAIL_SIZE)) {
2592 camera_metadata_ro_entry entry = setting.find(ANDROID_JPEG_THUMBNAIL_SIZE);
2593 thumbSize = Size{.width = entry.data.i32[0], .height = entry.data.i32[1]};
2594 if (thumbSize.width == 0 && thumbSize.height == 0) {
2595 outputThumbnail = false;
2596 }
2597 } else {
2598 return lfail("%s: ANDROID_JPEG_THUMBNAIL_SIZE not set", __FUNCTION__);
2599 }
2600
2601 /* Cropped and scaled YU12 buffer for main and thumbnail */
2602 YCbCrLayout yu12Main;
2603 Size jpegSize{halBuf.width, halBuf.height};
2604
2605 /* Compute temporary buffer sizes accounting for the following:
2606 * thumbnail can't exceed APP1 size of 64K
2607 * main image needs to hold APP1, headers, and at most a poorly
2608 * compressed image */
2609 const ssize_t maxThumbCodeSize = 64 * 1024;
2610 const ssize_t maxJpegCodeSize =
2611 mBlobBufferSize == 0 ? parent->getJpegBufferSize(jpegSize.width, jpegSize.height)
2612 : mBlobBufferSize;
2613
2614 /* Check that getJpegBufferSize did not return an error */
2615 if (maxJpegCodeSize < 0) {
2616 return lfail("%s: getJpegBufferSize returned %zd", __FUNCTION__, maxJpegCodeSize);
2617 }
2618
2619 /* Hold actual thumbnail and main image code sizes */
2620 size_t thumbCodeSize = 0, jpegCodeSize = 0;
2621 /* Temporary thumbnail code buffer */
2622 std::vector<uint8_t> thumbCode(outputThumbnail ? maxThumbCodeSize : 0);
2623
2624 YCbCrLayout yu12Thumb;
2625 if (outputThumbnail) {
2626 ret = cropAndScaleThumbLocked(mYu12Frame, thumbSize, &yu12Thumb);
2627
2628 if (ret != 0) {
2629 return lfail("%s: crop and scale thumbnail failed!", __FUNCTION__);
2630 }
2631 }
2632
2633 /* Scale and crop main jpeg */
2634 ret = cropAndScaleLocked(mYu12Frame, jpegSize, &yu12Main);
2635
2636 if (ret != 0) {
2637 return lfail("%s: crop and scale main failed!", __FUNCTION__);
2638 }
2639
2640 /* Encode the thumbnail image */
2641 if (outputThumbnail) {
2642 ret = encodeJpegYU12(thumbSize, yu12Thumb, thumbQuality, 0, 0, &thumbCode[0],
2643 maxThumbCodeSize, thumbCodeSize);
2644
2645 if (ret != 0) {
2646 return lfail("%s: thumbnail encodeJpegYU12 failed with %d", __FUNCTION__, ret);
2647 }
2648 }
2649
2650 /* Combine camera characteristics with request settings to form EXIF
2651 * metadata */
2652 common::V1_0::helper::CameraMetadata meta(mCameraCharacteristics);
2653 meta.append(setting);
2654
2655 /* Generate EXIF object */
2656 std::unique_ptr<ExifUtils> utils(ExifUtils::create());
2657 /* Make sure it's initialized */
2658 utils->initialize();
2659
2660 utils->setFromMetadata(meta, jpegSize.width, jpegSize.height);
2661 utils->setMake(mExifMake);
2662 utils->setModel(mExifModel);
2663
2664 ret = utils->generateApp1(outputThumbnail ? &thumbCode[0] : nullptr, thumbCodeSize);
2665
2666 if (!ret) {
2667 return lfail("%s: generating APP1 failed", __FUNCTION__);
2668 }
2669
2670 /* Get internal buffer */
2671 size_t exifDataSize = utils->getApp1Length();
2672 const uint8_t* exifData = utils->getApp1Buffer();
2673
2674 /* Lock the HAL jpeg code buffer */
2675 void* bufPtr = sHandleImporter.lock(*(halBuf.bufPtr), static_cast<uint64_t>(halBuf.usage),
2676 maxJpegCodeSize);
2677
2678 if (!bufPtr) {
2679 return lfail("%s: could not lock %zu bytes", __FUNCTION__, maxJpegCodeSize);
2680 }
2681
2682 /* Encode the main jpeg image */
2683 ret = encodeJpegYU12(jpegSize, yu12Main, jpegQuality, exifData, exifDataSize, bufPtr,
2684 maxJpegCodeSize, jpegCodeSize);
2685
2686 /* TODO: Not sure this belongs here, maybe better to pass jpegCodeSize out
2687 * and do this when returning buffer to parent */
2688 CameraBlob blob{CameraBlobId::JPEG, static_cast<int32_t>(jpegCodeSize)};
2689 void* blobDst = reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(bufPtr) + maxJpegCodeSize -
2690 sizeof(CameraBlob));
2691 memcpy(blobDst, &blob, sizeof(CameraBlob));
2692
2693 /* Unlock the HAL jpeg code buffer */
2694 int relFence = sHandleImporter.unlock(*(halBuf.bufPtr));
2695 if (relFence >= 0) {
2696 halBuf.acquireFence = relFence;
2697 }
2698
2699 /* Check if our JPEG actually succeeded */
2700 if (ret != 0) {
2701 return lfail("%s: encodeJpegYU12 failed with %d", __FUNCTION__, ret);
2702 }
2703
2704 ALOGV("%s: encoded JPEG (ret:%d) with Q:%d max size: %zu", __FUNCTION__, ret, jpegQuality,
2705 maxJpegCodeSize);
2706
2707 return 0;
2708 }
2709
clearIntermediateBuffers()2710 void ExternalCameraDeviceSession::OutputThread::clearIntermediateBuffers() {
2711 std::lock_guard<std::mutex> lk(mBufferLock);
2712 mYu12Frame.reset();
2713 mYu12ThumbFrame.reset();
2714 mIntermediateBuffers.clear();
2715 mMuteTestPatternFrame.clear();
2716 mBlobBufferSize = 0;
2717 }
2718
threadLoop()2719 bool ExternalCameraDeviceSession::OutputThread::threadLoop() {
2720 std::shared_ptr<HalRequest> req;
2721 auto parent = mParent.lock();
2722 if (parent == nullptr) {
2723 ALOGE("%s: session has been disconnected!", __FUNCTION__);
2724 return false;
2725 }
2726
2727 // TODO: maybe we need to setup a sensor thread to dq/enq v4l frames
2728 // regularly to prevent v4l buffer queue filled with stale buffers
2729 // when app doesn't program a preview request
2730 waitForNextRequest(&req);
2731 if (req == nullptr) {
2732 // No new request, wait again
2733 return true;
2734 }
2735
2736 auto onDeviceError = [&](auto... args) {
2737 ALOGE(args...);
2738 parent->notifyError(req->frameNumber, /*stream*/ -1, ErrorCode::ERROR_DEVICE);
2739 signalRequestDone();
2740 return false;
2741 };
2742
2743 if (req->frameIn->mFourcc != V4L2_PIX_FMT_MJPEG && req->frameIn->mFourcc != V4L2_PIX_FMT_Z16) {
2744 return onDeviceError("%s: do not support V4L2 format %c%c%c%c", __FUNCTION__,
2745 req->frameIn->mFourcc & 0xFF, (req->frameIn->mFourcc >> 8) & 0xFF,
2746 (req->frameIn->mFourcc >> 16) & 0xFF,
2747 (req->frameIn->mFourcc >> 24) & 0xFF);
2748 }
2749
2750 int res = requestBufferStart(req->buffers);
2751 if (res != 0) {
2752 ALOGE("%s: send BufferRequest failed! res %d", __FUNCTION__, res);
2753 return onDeviceError("%s: failed to send buffer request!", __FUNCTION__);
2754 }
2755
2756 std::unique_lock<std::mutex> lk(mBufferLock);
2757 // Convert input V4L2 frame to YU12 of the same size
2758 // TODO: see if we can save some computation by converting to YV12 here
2759 uint8_t* inData;
2760 size_t inDataSize;
2761 if (req->frameIn->getData(&inData, &inDataSize) != 0) {
2762 lk.unlock();
2763 return onDeviceError("%s: V4L2 buffer map failed", __FUNCTION__);
2764 }
2765
2766 // Process camera mute state
2767 auto testPatternMode = req->setting.find(ANDROID_SENSOR_TEST_PATTERN_MODE);
2768 if (testPatternMode.count == 1) {
2769 if (mCameraMuted != (testPatternMode.data.u8[0] != ANDROID_SENSOR_TEST_PATTERN_MODE_OFF)) {
2770 mCameraMuted = !mCameraMuted;
2771 // Get solid color for test pattern, if any was set
2772 if (testPatternMode.data.u8[0] == ANDROID_SENSOR_TEST_PATTERN_MODE_SOLID_COLOR) {
2773 auto entry = req->setting.find(ANDROID_SENSOR_TEST_PATTERN_DATA);
2774 if (entry.count == 4) {
2775 // Update the mute frame if the pattern color has changed
2776 if (memcmp(entry.data.i32, mTestPatternData, sizeof(mTestPatternData)) != 0) {
2777 memcpy(mTestPatternData, entry.data.i32, sizeof(mTestPatternData));
2778 // Fill the mute frame with the solid color, use only 8 MSB of RGGB as RGB
2779 for (int i = 0; i < mMuteTestPatternFrame.size(); i += 3) {
2780 mMuteTestPatternFrame[i] = entry.data.i32[0] >> 24;
2781 mMuteTestPatternFrame[i + 1] = entry.data.i32[1] >> 24;
2782 mMuteTestPatternFrame[i + 2] = entry.data.i32[3] >> 24;
2783 }
2784 }
2785 }
2786 }
2787 }
2788 }
2789
2790 // TODO: in some special case maybe we can decode jpg directly to gralloc output?
2791 if (req->frameIn->mFourcc == V4L2_PIX_FMT_MJPEG) {
2792 ATRACE_BEGIN("MJPGtoI420");
2793 res = 0;
2794 if (mCameraMuted) {
2795 res = libyuv::ConvertToI420(
2796 mMuteTestPatternFrame.data(), mMuteTestPatternFrame.size(),
2797 static_cast<uint8_t*>(mYu12FrameLayout.y), mYu12FrameLayout.yStride,
2798 static_cast<uint8_t*>(mYu12FrameLayout.cb), mYu12FrameLayout.cStride,
2799 static_cast<uint8_t*>(mYu12FrameLayout.cr), mYu12FrameLayout.cStride, 0, 0,
2800 mYu12Frame->mWidth, mYu12Frame->mHeight, mYu12Frame->mWidth,
2801 mYu12Frame->mHeight, libyuv::kRotate0, libyuv::FOURCC_RAW);
2802 } else {
2803 res = libyuv::MJPGToI420(
2804 inData, inDataSize, static_cast<uint8_t*>(mYu12FrameLayout.y),
2805 mYu12FrameLayout.yStride, static_cast<uint8_t*>(mYu12FrameLayout.cb),
2806 mYu12FrameLayout.cStride, static_cast<uint8_t*>(mYu12FrameLayout.cr),
2807 mYu12FrameLayout.cStride, mYu12Frame->mWidth, mYu12Frame->mHeight,
2808 mYu12Frame->mWidth, mYu12Frame->mHeight);
2809 }
2810 ATRACE_END();
2811
2812 if (res != 0) {
2813 // For some webcam, the first few V4L2 frames might be malformed...
2814 ALOGE("%s: Convert V4L2 frame to YU12 failed! res %d", __FUNCTION__, res);
2815
2816 ATRACE_BEGIN("Wait for BufferRequest done");
2817 res = waitForBufferRequestDone(&req->buffers);
2818 ATRACE_END();
2819
2820 lk.unlock();
2821 Status st = parent->processCaptureRequestError(req);
2822 if (st != Status::OK) {
2823 return onDeviceError("%s: failed to process capture request error!", __FUNCTION__);
2824 }
2825 signalRequestDone();
2826 return true;
2827 }
2828 }
2829
2830 ATRACE_BEGIN("Wait for BufferRequest done");
2831 res = waitForBufferRequestDone(&req->buffers);
2832 ATRACE_END();
2833
2834 if (res != 0) {
2835 // HAL buffer management buffer request can fail
2836 ALOGE("%s: wait for BufferRequest done failed! res %d", __FUNCTION__, res);
2837 lk.unlock();
2838 Status st = parent->processCaptureRequestError(req);
2839 if (st != Status::OK) {
2840 return onDeviceError("%s: failed to process capture request error!", __FUNCTION__);
2841 }
2842 signalRequestDone();
2843 return true;
2844 }
2845
2846 ALOGV("%s processing new request", __FUNCTION__);
2847 const int kSyncWaitTimeoutMs = 500;
2848 for (auto& halBuf : req->buffers) {
2849 if (*(halBuf.bufPtr) == nullptr) {
2850 ALOGW("%s: buffer for stream %d missing", __FUNCTION__, halBuf.streamId);
2851 halBuf.fenceTimeout = true;
2852 } else if (halBuf.acquireFence >= 0) {
2853 int ret = sync_wait(halBuf.acquireFence, kSyncWaitTimeoutMs);
2854 if (ret) {
2855 halBuf.fenceTimeout = true;
2856 } else {
2857 ::close(halBuf.acquireFence);
2858 halBuf.acquireFence = -1;
2859 }
2860 }
2861
2862 if (halBuf.fenceTimeout) {
2863 continue;
2864 }
2865
2866 // Gralloc lockYCbCr the buffer
2867 switch (halBuf.format) {
2868 case PixelFormat::BLOB: {
2869 int ret = createJpegLocked(halBuf, req->setting);
2870
2871 if (ret != 0) {
2872 lk.unlock();
2873 return onDeviceError("%s: createJpegLocked failed with %d", __FUNCTION__, ret);
2874 }
2875 } break;
2876 case PixelFormat::Y16: {
2877 void* outLayout = sHandleImporter.lock(
2878 *(halBuf.bufPtr), static_cast<uint64_t>(halBuf.usage), inDataSize);
2879
2880 std::memcpy(outLayout, inData, inDataSize);
2881
2882 int relFence = sHandleImporter.unlock(*(halBuf.bufPtr));
2883 if (relFence >= 0) {
2884 halBuf.acquireFence = relFence;
2885 }
2886 } break;
2887 case PixelFormat::YCBCR_420_888:
2888 case PixelFormat::YV12: {
2889 android::Rect outRect{0, 0, static_cast<int32_t>(halBuf.width),
2890 static_cast<int32_t>(halBuf.height)};
2891 android_ycbcr result = sHandleImporter.lockYCbCr(
2892 *(halBuf.bufPtr), static_cast<uint64_t>(halBuf.usage), outRect);
2893 ALOGV("%s: outLayout y %p cb %p cr %p y_str %zu c_str %zu c_step %zu", __FUNCTION__,
2894 result.y, result.cb, result.cr, result.ystride, result.cstride,
2895 result.chroma_step);
2896 if (result.ystride > UINT32_MAX || result.cstride > UINT32_MAX ||
2897 result.chroma_step > UINT32_MAX) {
2898 return onDeviceError("%s: lockYCbCr failed. Unexpected values!", __FUNCTION__);
2899 }
2900 YCbCrLayout outLayout = {.y = result.y,
2901 .cb = result.cb,
2902 .cr = result.cr,
2903 .yStride = static_cast<uint32_t>(result.ystride),
2904 .cStride = static_cast<uint32_t>(result.cstride),
2905 .chromaStep = static_cast<uint32_t>(result.chroma_step)};
2906
2907 // Convert to output buffer size/format
2908 uint32_t outputFourcc = getFourCcFromLayout(outLayout);
2909 ALOGV("%s: converting to format %c%c%c%c", __FUNCTION__, outputFourcc & 0xFF,
2910 (outputFourcc >> 8) & 0xFF, (outputFourcc >> 16) & 0xFF,
2911 (outputFourcc >> 24) & 0xFF);
2912
2913 YCbCrLayout cropAndScaled;
2914 ATRACE_BEGIN("cropAndScaleLocked");
2915 int ret = cropAndScaleLocked(mYu12Frame, Size{halBuf.width, halBuf.height},
2916 &cropAndScaled);
2917 ATRACE_END();
2918 if (ret != 0) {
2919 lk.unlock();
2920 return onDeviceError("%s: crop and scale failed!", __FUNCTION__);
2921 }
2922
2923 Size sz{halBuf.width, halBuf.height};
2924 ATRACE_BEGIN("formatConvert");
2925 ret = formatConvert(cropAndScaled, outLayout, sz, outputFourcc);
2926 ATRACE_END();
2927 if (ret != 0) {
2928 lk.unlock();
2929 return onDeviceError("%s: format conversion failed!", __FUNCTION__);
2930 }
2931 int relFence = sHandleImporter.unlock(*(halBuf.bufPtr));
2932 if (relFence >= 0) {
2933 halBuf.acquireFence = relFence;
2934 }
2935 } break;
2936 default:
2937 lk.unlock();
2938 return onDeviceError("%s: unknown output format %x", __FUNCTION__, halBuf.format);
2939 }
2940 } // for each buffer
2941 mScaledYu12Frames.clear();
2942
2943 // Don't hold the lock while calling back to parent
2944 lk.unlock();
2945 Status st = parent->processCaptureResult(req);
2946 if (st != Status::OK) {
2947 return onDeviceError("%s: failed to process capture result!", __FUNCTION__);
2948 }
2949 signalRequestDone();
2950 return true;
2951 }
2952
2953 // End ExternalCameraDeviceSession::OutputThread functions
2954
2955 } // namespace implementation
2956 } // namespace device
2957 } // namespace camera
2958 } // namespace hardware
2959 } // namespace android
2960