• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (C) 2022 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #define LOG_TAG "ExtCamDevSsn"
18 // #define LOG_NDEBUG 0
19 #include <log/log.h>
20 
21 #include "ExternalCameraDeviceSession.h"
22 
23 #include <Exif.h>
24 #include <ExternalCameraOfflineSession.h>
25 #include <aidl/android/hardware/camera/device/CameraBlob.h>
26 #include <aidl/android/hardware/camera/device/CameraBlobId.h>
27 #include <aidl/android/hardware/camera/device/ErrorMsg.h>
28 #include <aidl/android/hardware/camera/device/ShutterMsg.h>
29 #include <aidl/android/hardware/camera/device/StreamBufferRet.h>
30 #include <aidl/android/hardware/camera/device/StreamBuffersVal.h>
31 #include <aidl/android/hardware/camera/device/StreamConfigurationMode.h>
32 #include <aidl/android/hardware/camera/device/StreamRotation.h>
33 #include <aidl/android/hardware/camera/device/StreamType.h>
34 #include <aidl/android/hardware/graphics/common/Dataspace.h>
35 #include <aidlcommonsupport/NativeHandle.h>
36 #include <convert.h>
37 #include <linux/videodev2.h>
38 #include <sync/sync.h>
39 #include <utils/Trace.h>
40 #include <deque>
41 
42 #define HAVE_JPEG  // required for libyuv.h to export MJPEG decode APIs
43 #include <libyuv.h>
44 #include <libyuv/convert.h>
45 
46 namespace android {
47 namespace hardware {
48 namespace camera {
49 namespace device {
50 namespace implementation {
51 
52 namespace {
53 
54 // Size of request/result metadata fast message queue. Change to 0 to always use hwbinder buffer.
55 static constexpr size_t kMetadataMsgQueueSize = 1 << 18 /* 256kB */;
56 
57 const int kBadFramesAfterStreamOn = 1;  // drop x frames after streamOn to get rid of some initial
58                                         // bad frames. TODO: develop a better bad frame detection
59                                         // method
60 constexpr int MAX_RETRY = 15;  // Allow retry some ioctl failures a few times to account for some
61                                // webcam showing temporarily ioctl failures.
62 constexpr int IOCTL_RETRY_SLEEP_US = 33000;  // 33ms * MAX_RETRY = 0.5 seconds
63 
64 // Constants for tryLock during dumpstate
65 static constexpr int kDumpLockRetries = 50;
66 static constexpr int kDumpLockSleep = 60000;
67 
tryLock(Mutex & mutex)68 bool tryLock(Mutex& mutex) {
69     bool locked = false;
70     for (int i = 0; i < kDumpLockRetries; ++i) {
71         if (mutex.tryLock() == NO_ERROR) {
72             locked = true;
73             break;
74         }
75         usleep(kDumpLockSleep);
76     }
77     return locked;
78 }
79 
tryLock(std::mutex & mutex)80 bool tryLock(std::mutex& mutex) {
81     bool locked = false;
82     for (int i = 0; i < kDumpLockRetries; ++i) {
83         if (mutex.try_lock()) {
84             locked = true;
85             break;
86         }
87         usleep(kDumpLockSleep);
88     }
89     return locked;
90 }
91 
92 }  // anonymous namespace
93 
94 using ::aidl::android::hardware::camera::device::BufferRequestStatus;
95 using ::aidl::android::hardware::camera::device::CameraBlob;
96 using ::aidl::android::hardware::camera::device::CameraBlobId;
97 using ::aidl::android::hardware::camera::device::ErrorMsg;
98 using ::aidl::android::hardware::camera::device::ShutterMsg;
99 using ::aidl::android::hardware::camera::device::StreamBuffer;
100 using ::aidl::android::hardware::camera::device::StreamBufferRet;
101 using ::aidl::android::hardware::camera::device::StreamBuffersVal;
102 using ::aidl::android::hardware::camera::device::StreamConfigurationMode;
103 using ::aidl::android::hardware::camera::device::StreamRotation;
104 using ::aidl::android::hardware::camera::device::StreamType;
105 using ::aidl::android::hardware::graphics::common::Dataspace;
106 using ::android::hardware::camera::common::V1_0::helper::ExifUtils;
107 
108 // Static instances
109 const int ExternalCameraDeviceSession::kMaxProcessedStream;
110 const int ExternalCameraDeviceSession::kMaxStallStream;
111 HandleImporter ExternalCameraDeviceSession::sHandleImporter;
112 
ExternalCameraDeviceSession(const std::shared_ptr<ICameraDeviceCallback> & callback,const ExternalCameraConfig & cfg,const std::vector<SupportedV4L2Format> & sortedFormats,const CroppingType & croppingType,const common::V1_0::helper::CameraMetadata & chars,const std::string & cameraId,unique_fd v4l2Fd)113 ExternalCameraDeviceSession::ExternalCameraDeviceSession(
114         const std::shared_ptr<ICameraDeviceCallback>& callback, const ExternalCameraConfig& cfg,
115         const std::vector<SupportedV4L2Format>& sortedFormats, const CroppingType& croppingType,
116         const common::V1_0::helper::CameraMetadata& chars, const std::string& cameraId,
117         unique_fd v4l2Fd)
118     : mCallback(callback),
119       mCfg(cfg),
120       mCameraCharacteristics(chars),
121       mSupportedFormats(sortedFormats),
122       mCroppingType(croppingType),
123       mCameraId(cameraId),
124       mV4l2Fd(std::move(v4l2Fd)),
125       mMaxThumbResolution(getMaxThumbResolution()),
126       mMaxJpegResolution(getMaxJpegResolution()) {}
127 
getMaxThumbResolution() const128 Size ExternalCameraDeviceSession::getMaxThumbResolution() const {
129     return getMaxThumbnailResolution(mCameraCharacteristics);
130 }
131 
getMaxJpegResolution() const132 Size ExternalCameraDeviceSession::getMaxJpegResolution() const {
133     Size ret{0, 0};
134     for (auto& fmt : mSupportedFormats) {
135         if (fmt.width * fmt.height > ret.width * ret.height) {
136             ret = Size{fmt.width, fmt.height};
137         }
138     }
139     return ret;
140 }
141 
initialize()142 bool ExternalCameraDeviceSession::initialize() {
143     if (mV4l2Fd.get() < 0) {
144         ALOGE("%s: invalid v4l2 device fd %d!", __FUNCTION__, mV4l2Fd.get());
145         return true;
146     }
147 
148     struct v4l2_capability capability;
149     int ret = ioctl(mV4l2Fd.get(), VIDIOC_QUERYCAP, &capability);
150     std::string make, model;
151     if (ret < 0) {
152         ALOGW("%s v4l2 QUERYCAP failed", __FUNCTION__);
153         mExifMake = "Generic UVC webcam";
154         mExifModel = "Generic UVC webcam";
155     } else {
156         // capability.card is UTF-8 encoded
157         char card[32];
158         int j = 0;
159         for (int i = 0; i < 32; i++) {
160             if (capability.card[i] < 128) {
161                 card[j++] = capability.card[i];
162             }
163             if (capability.card[i] == '\0') {
164                 break;
165             }
166         }
167         if (j == 0 || card[j - 1] != '\0') {
168             mExifMake = "Generic UVC webcam";
169             mExifModel = "Generic UVC webcam";
170         } else {
171             mExifMake = card;
172             mExifModel = card;
173         }
174     }
175 
176     initOutputThread();
177     if (mOutputThread == nullptr) {
178         ALOGE("%s: init OutputThread failed!", __FUNCTION__);
179         return true;
180     }
181     mOutputThread->setExifMakeModel(mExifMake, mExifModel);
182 
183     status_t status = initDefaultRequests();
184     if (status != OK) {
185         ALOGE("%s: init default requests failed!", __FUNCTION__);
186         return true;
187     }
188 
189     mRequestMetadataQueue =
190             std::make_unique<RequestMetadataQueue>(kMetadataMsgQueueSize, false /* non blocking */);
191     if (!mRequestMetadataQueue->isValid()) {
192         ALOGE("%s: invalid request fmq", __FUNCTION__);
193         return true;
194     }
195 
196     mResultMetadataQueue =
197             std::make_shared<ResultMetadataQueue>(kMetadataMsgQueueSize, false /* non blocking */);
198     if (!mResultMetadataQueue->isValid()) {
199         ALOGE("%s: invalid result fmq", __FUNCTION__);
200         return true;
201     }
202 
203     mOutputThread->run();
204     return false;
205 }
206 
isInitFailed()207 bool ExternalCameraDeviceSession::isInitFailed() {
208     Mutex::Autolock _l(mLock);
209     if (!mInitialized) {
210         mInitFail = initialize();
211         mInitialized = true;
212     }
213     return mInitFail;
214 }
215 
initOutputThread()216 void ExternalCameraDeviceSession::initOutputThread() {
217     // Grab a shared_ptr to 'this' from ndk::SharedRefBase::ref()
218     std::shared_ptr<ExternalCameraDeviceSession> thiz = ref<ExternalCameraDeviceSession>();
219 
220     mBufferRequestThread = std::make_shared<BufferRequestThread>(/*parent=*/thiz, mCallback);
221     mBufferRequestThread->run();
222     mOutputThread = std::make_shared<OutputThread>(/*parent=*/thiz, mCroppingType,
223                                                    mCameraCharacteristics, mBufferRequestThread);
224 }
225 
closeOutputThread()226 void ExternalCameraDeviceSession::closeOutputThread() {
227     if (mOutputThread != nullptr) {
228         mOutputThread->flush();
229         mOutputThread->requestExitAndWait();
230         mOutputThread.reset();
231     }
232 }
233 
closeBufferRequestThread()234 void ExternalCameraDeviceSession::closeBufferRequestThread() {
235     if (mBufferRequestThread != nullptr) {
236         mBufferRequestThread->requestExitAndWait();
237         mBufferRequestThread.reset();
238     }
239 }
240 
initStatus() const241 Status ExternalCameraDeviceSession::initStatus() const {
242     Mutex::Autolock _l(mLock);
243     Status status = Status::OK;
244     if (mInitFail || mClosed) {
245         ALOGI("%s: session initFailed %d closed %d", __FUNCTION__, mInitFail, mClosed);
246         status = Status::INTERNAL_ERROR;
247     }
248     return status;
249 }
250 
~ExternalCameraDeviceSession()251 ExternalCameraDeviceSession::~ExternalCameraDeviceSession() {
252     if (!isClosed()) {
253         ALOGE("ExternalCameraDeviceSession deleted before close!");
254         closeImpl();
255     }
256 }
257 
constructDefaultRequestSettings(RequestTemplate in_type,CameraMetadata * _aidl_return)258 ScopedAStatus ExternalCameraDeviceSession::constructDefaultRequestSettings(
259         RequestTemplate in_type, CameraMetadata* _aidl_return) {
260     CameraMetadata emptyMetadata;
261     Status status = initStatus();
262     if (status != Status::OK) {
263         return fromStatus(status);
264     }
265     switch (in_type) {
266         case RequestTemplate::PREVIEW:
267         case RequestTemplate::STILL_CAPTURE:
268         case RequestTemplate::VIDEO_RECORD:
269         case RequestTemplate::VIDEO_SNAPSHOT: {
270             *_aidl_return = mDefaultRequests[in_type];
271             break;
272         }
273         case RequestTemplate::MANUAL:
274         case RequestTemplate::ZERO_SHUTTER_LAG:
275             // Don't support MANUAL, ZSL templates
276             status = Status::ILLEGAL_ARGUMENT;
277             break;
278         default:
279             ALOGE("%s: unknown request template type %d", __FUNCTION__, static_cast<int>(in_type));
280             status = Status::ILLEGAL_ARGUMENT;
281             break;
282     }
283     return fromStatus(status);
284 }
285 
configureStreams(const StreamConfiguration & in_requestedConfiguration,std::vector<HalStream> * _aidl_return)286 ScopedAStatus ExternalCameraDeviceSession::configureStreams(
287         const StreamConfiguration& in_requestedConfiguration,
288         std::vector<HalStream>* _aidl_return) {
289     uint32_t blobBufferSize = 0;
290     _aidl_return->clear();
291     Mutex::Autolock _il(mInterfaceLock);
292 
293     Status status =
294             isStreamCombinationSupported(in_requestedConfiguration, mSupportedFormats, mCfg);
295     if (status != Status::OK) {
296         return fromStatus(status);
297     }
298 
299     status = initStatus();
300     if (status != Status::OK) {
301         return fromStatus(status);
302     }
303 
304     {
305         std::lock_guard<std::mutex> lk(mInflightFramesLock);
306         if (!mInflightFrames.empty()) {
307             ALOGE("%s: trying to configureStreams while there are still %zu inflight frames!",
308                   __FUNCTION__, mInflightFrames.size());
309             return fromStatus(Status::INTERNAL_ERROR);
310         }
311     }
312 
313     Mutex::Autolock _l(mLock);
314     {
315         Mutex::Autolock _cl(mCbsLock);
316         // Add new streams
317         for (const auto& stream : in_requestedConfiguration.streams) {
318             if (mStreamMap.count(stream.id) == 0) {
319                 mStreamMap[stream.id] = stream;
320                 mCirculatingBuffers.emplace(stream.id, CirculatingBuffers{});
321             }
322         }
323 
324         // Cleanup removed streams
325         for (auto it = mStreamMap.begin(); it != mStreamMap.end();) {
326             int id = it->first;
327             bool found = false;
328             for (const auto& stream : in_requestedConfiguration.streams) {
329                 if (id == stream.id) {
330                     found = true;
331                     break;
332                 }
333             }
334             if (!found) {
335                 // Unmap all buffers of deleted stream
336                 cleanupBuffersLocked(id);
337                 it = mStreamMap.erase(it);
338             } else {
339                 ++it;
340             }
341         }
342     }
343 
344     // Now select a V4L2 format to produce all output streams
345     float desiredAr = (mCroppingType == VERTICAL) ? kMaxAspectRatio : kMinAspectRatio;
346     uint32_t maxDim = 0;
347     for (const auto& stream : in_requestedConfiguration.streams) {
348         float aspectRatio = ASPECT_RATIO(stream);
349         ALOGI("%s: request stream %dx%d", __FUNCTION__, stream.width, stream.height);
350         if ((mCroppingType == VERTICAL && aspectRatio < desiredAr) ||
351             (mCroppingType == HORIZONTAL && aspectRatio > desiredAr)) {
352             desiredAr = aspectRatio;
353         }
354 
355         // The dimension that's not cropped
356         uint32_t dim = (mCroppingType == VERTICAL) ? stream.width : stream.height;
357         if (dim > maxDim) {
358             maxDim = dim;
359         }
360     }
361 
362     // Find the smallest format that matches the desired aspect ratio and is wide/high enough
363     SupportedV4L2Format v4l2Fmt{.width = 0, .height = 0};
364     for (const auto& fmt : mSupportedFormats) {
365         uint32_t dim = (mCroppingType == VERTICAL) ? fmt.width : fmt.height;
366         if (dim >= maxDim) {
367             float aspectRatio = ASPECT_RATIO(fmt);
368             if (isAspectRatioClose(aspectRatio, desiredAr)) {
369                 v4l2Fmt = fmt;
370                 // since mSupportedFormats is sorted by width then height, the first matching fmt
371                 // will be the smallest one with matching aspect ratio
372                 break;
373             }
374         }
375     }
376 
377     if (v4l2Fmt.width == 0) {
378         // Cannot find exact good aspect ratio candidate, try to find a close one
379         for (const auto& fmt : mSupportedFormats) {
380             uint32_t dim = (mCroppingType == VERTICAL) ? fmt.width : fmt.height;
381             if (dim >= maxDim) {
382                 float aspectRatio = ASPECT_RATIO(fmt);
383                 if ((mCroppingType == VERTICAL && aspectRatio < desiredAr) ||
384                     (mCroppingType == HORIZONTAL && aspectRatio > desiredAr)) {
385                     v4l2Fmt = fmt;
386                     break;
387                 }
388             }
389         }
390     }
391 
392     if (v4l2Fmt.width == 0) {
393         ALOGE("%s: unable to find a resolution matching (%s at least %d, aspect ratio %f)",
394               __FUNCTION__, (mCroppingType == VERTICAL) ? "width" : "height", maxDim, desiredAr);
395         return fromStatus(Status::ILLEGAL_ARGUMENT);
396     }
397 
398     if (configureV4l2StreamLocked(v4l2Fmt) != 0) {
399         ALOGE("V4L configuration failed!, format:%c%c%c%c, w %d, h %d", v4l2Fmt.fourcc & 0xFF,
400               (v4l2Fmt.fourcc >> 8) & 0xFF, (v4l2Fmt.fourcc >> 16) & 0xFF,
401               (v4l2Fmt.fourcc >> 24) & 0xFF, v4l2Fmt.width, v4l2Fmt.height);
402         return fromStatus(Status::INTERNAL_ERROR);
403     }
404 
405     Size v4lSize = {v4l2Fmt.width, v4l2Fmt.height};
406     Size thumbSize{0, 0};
407     camera_metadata_ro_entry entry =
408             mCameraCharacteristics.find(ANDROID_JPEG_AVAILABLE_THUMBNAIL_SIZES);
409     for (uint32_t i = 0; i < entry.count; i += 2) {
410         Size sz{entry.data.i32[i], entry.data.i32[i + 1]};
411         if (sz.width * sz.height > thumbSize.width * thumbSize.height) {
412             thumbSize = sz;
413         }
414     }
415 
416     if (thumbSize.width * thumbSize.height == 0) {
417         ALOGE("%s: non-zero thumbnail size not available", __FUNCTION__);
418         return fromStatus(Status::INTERNAL_ERROR);
419     }
420 
421     mBlobBufferSize = blobBufferSize;
422     status = mOutputThread->allocateIntermediateBuffers(
423             v4lSize, mMaxThumbResolution, in_requestedConfiguration.streams, blobBufferSize);
424     if (status != Status::OK) {
425         ALOGE("%s: allocating intermediate buffers failed!", __FUNCTION__);
426         return fromStatus(status);
427     }
428 
429     std::vector<HalStream>& out = *_aidl_return;
430     out.resize(in_requestedConfiguration.streams.size());
431     for (size_t i = 0; i < in_requestedConfiguration.streams.size(); i++) {
432         out[i].overrideDataSpace = in_requestedConfiguration.streams[i].dataSpace;
433         out[i].id = in_requestedConfiguration.streams[i].id;
434         // TODO: double check should we add those CAMERA flags
435         mStreamMap[in_requestedConfiguration.streams[i].id].usage = out[i].producerUsage =
436                 static_cast<BufferUsage>(((int64_t)in_requestedConfiguration.streams[i].usage) |
437                                          ((int64_t)BufferUsage::CPU_WRITE_OFTEN) |
438                                          ((int64_t)BufferUsage::CAMERA_OUTPUT));
439         out[i].consumerUsage = static_cast<BufferUsage>(0);
440         out[i].maxBuffers = static_cast<int32_t>(mV4L2BufferCount);
441 
442         switch (in_requestedConfiguration.streams[i].format) {
443             case PixelFormat::BLOB:
444             case PixelFormat::YCBCR_420_888:
445             case PixelFormat::YV12:  // Used by SurfaceTexture
446             case PixelFormat::Y16:
447                 // No override
448                 out[i].overrideFormat = in_requestedConfiguration.streams[i].format;
449                 break;
450             case PixelFormat::IMPLEMENTATION_DEFINED:
451                 // Implementation Defined
452                 // This should look at the Stream's dataspace flag to determine the format or leave
453                 // it as is if the rest of the system knows how to handle a private format. To keep
454                 // this HAL generic, this is being overridden to YUV420
455                 out[i].overrideFormat = PixelFormat::YCBCR_420_888;
456                 // Save overridden format in mStreamMap
457                 mStreamMap[in_requestedConfiguration.streams[i].id].format = out[i].overrideFormat;
458                 break;
459             default:
460                 ALOGE("%s: unsupported format 0x%x", __FUNCTION__,
461                       in_requestedConfiguration.streams[i].format);
462                 return fromStatus(Status::ILLEGAL_ARGUMENT);
463         }
464     }
465 
466     mFirstRequest = true;
467     mLastStreamConfigCounter = in_requestedConfiguration.streamConfigCounter;
468     return fromStatus(Status::OK);
469 }
470 
flush()471 ScopedAStatus ExternalCameraDeviceSession::flush() {
472     ATRACE_CALL();
473     Mutex::Autolock _il(mInterfaceLock);
474     Status status = initStatus();
475     if (status != Status::OK) {
476         return fromStatus(status);
477     }
478     mOutputThread->flush();
479     return fromStatus(Status::OK);
480 }
481 
getCaptureRequestMetadataQueue(MQDescriptor<int8_t,SynchronizedReadWrite> * _aidl_return)482 ScopedAStatus ExternalCameraDeviceSession::getCaptureRequestMetadataQueue(
483         MQDescriptor<int8_t, SynchronizedReadWrite>* _aidl_return) {
484     Mutex::Autolock _il(mInterfaceLock);
485     *_aidl_return = mRequestMetadataQueue->dupeDesc();
486     return fromStatus(Status::OK);
487 }
488 
getCaptureResultMetadataQueue(MQDescriptor<int8_t,SynchronizedReadWrite> * _aidl_return)489 ScopedAStatus ExternalCameraDeviceSession::getCaptureResultMetadataQueue(
490         MQDescriptor<int8_t, SynchronizedReadWrite>* _aidl_return) {
491     Mutex::Autolock _il(mInterfaceLock);
492     *_aidl_return = mResultMetadataQueue->dupeDesc();
493     return fromStatus(Status::OK);
494 }
495 
isReconfigurationRequired(const CameraMetadata & in_oldSessionParams,const CameraMetadata & in_newSessionParams,bool * _aidl_return)496 ScopedAStatus ExternalCameraDeviceSession::isReconfigurationRequired(
497         const CameraMetadata& in_oldSessionParams, const CameraMetadata& in_newSessionParams,
498         bool* _aidl_return) {
499     // reconfiguration required if there is any change in the session params
500     *_aidl_return = in_oldSessionParams != in_newSessionParams;
501     return fromStatus(Status::OK);
502 }
503 
processCaptureRequest(const std::vector<CaptureRequest> & in_requests,const std::vector<BufferCache> & in_cachesToRemove,int32_t * _aidl_return)504 ScopedAStatus ExternalCameraDeviceSession::processCaptureRequest(
505         const std::vector<CaptureRequest>& in_requests,
506         const std::vector<BufferCache>& in_cachesToRemove, int32_t* _aidl_return) {
507     Mutex::Autolock _il(mInterfaceLock);
508     updateBufferCaches(in_cachesToRemove);
509 
510     int32_t& numRequestProcessed = *_aidl_return;
511     numRequestProcessed = 0;
512     Status s = Status::OK;
513     for (size_t i = 0; i < in_requests.size(); i++, numRequestProcessed++) {
514         s = processOneCaptureRequest(in_requests[i]);
515         if (s != Status::OK) {
516             break;
517         }
518     }
519 
520     return fromStatus(s);
521 }
522 
processOneCaptureRequest(const CaptureRequest & request)523 Status ExternalCameraDeviceSession::processOneCaptureRequest(const CaptureRequest& request) {
524     ATRACE_CALL();
525     Status status = initStatus();
526     if (status != Status::OK) {
527         return status;
528     }
529 
530     if (request.inputBuffer.streamId != -1) {
531         ALOGE("%s: external camera does not support reprocessing!", __FUNCTION__);
532         return Status::ILLEGAL_ARGUMENT;
533     }
534 
535     Mutex::Autolock _l(mLock);
536     if (!mV4l2Streaming) {
537         ALOGE("%s: cannot process request in streamOff state!", __FUNCTION__);
538         return Status::INTERNAL_ERROR;
539     }
540 
541     if (request.outputBuffers.empty()) {
542         ALOGE("%s: No output buffers provided.", __FUNCTION__);
543         return Status::ILLEGAL_ARGUMENT;
544     }
545 
546     for (auto& outputBuf : request.outputBuffers) {
547         if (outputBuf.streamId == -1 || mStreamMap.find(outputBuf.streamId) == mStreamMap.end()) {
548             ALOGE("%s: Invalid streamId in CaptureRequest.outputBuffers: %d", __FUNCTION__,
549                   outputBuf.streamId);
550             return Status::ILLEGAL_ARGUMENT;
551         }
552     }
553 
554     const camera_metadata_t* rawSettings = nullptr;
555     bool converted;
556     CameraMetadata settingsFmq;  // settings from FMQ
557 
558     if (request.fmqSettingsSize > 0) {
559         // non-blocking read; client must write metadata before calling
560         // processOneCaptureRequest
561         settingsFmq.metadata.resize(request.fmqSettingsSize);
562         bool read = mRequestMetadataQueue->read(
563                 reinterpret_cast<int8_t*>(settingsFmq.metadata.data()), request.fmqSettingsSize);
564         if (read) {
565             converted = convertFromAidl(settingsFmq, &rawSettings);
566         } else {
567             ALOGE("%s: capture request settings metadata couldn't be read from fmq!", __FUNCTION__);
568             converted = false;
569         }
570     } else {
571         converted = convertFromAidl(request.settings, &rawSettings);
572     }
573 
574     if (converted && rawSettings != nullptr) {
575         mLatestReqSetting = rawSettings;
576     }
577 
578     if (!converted) {
579         ALOGE("%s: capture request settings metadata is corrupt!", __FUNCTION__);
580         return Status::ILLEGAL_ARGUMENT;
581     }
582 
583     if (mFirstRequest && rawSettings == nullptr) {
584         ALOGE("%s: capture request settings must not be null for first request!", __FUNCTION__);
585         return Status::ILLEGAL_ARGUMENT;
586     }
587 
588     size_t numOutputBufs = request.outputBuffers.size();
589 
590     if (numOutputBufs == 0) {
591         ALOGE("%s: capture request must have at least one output buffer!", __FUNCTION__);
592         return Status::ILLEGAL_ARGUMENT;
593     }
594 
595     camera_metadata_entry fpsRange = mLatestReqSetting.find(ANDROID_CONTROL_AE_TARGET_FPS_RANGE);
596     if (fpsRange.count == 2) {
597         double requestFpsMax = fpsRange.data.i32[1];
598         double closestFps = 0.0;
599         double fpsError = 1000.0;
600         bool fpsSupported = false;
601         for (const auto& fr : mV4l2StreamingFmt.frameRates) {
602             double f = fr.getFramesPerSecond();
603             if (std::fabs(requestFpsMax - f) < 1.0) {
604                 fpsSupported = true;
605                 break;
606             }
607             if (std::fabs(requestFpsMax - f) < fpsError) {
608                 fpsError = std::fabs(requestFpsMax - f);
609                 closestFps = f;
610             }
611         }
612         if (!fpsSupported) {
613             /* This can happen in a few scenarios:
614              * 1. The application is sending an FPS range not supported by the configured outputs.
615              * 2. The application is sending a valid FPS range for all configured outputs, but
616              *    the selected V4L2 size can only run at slower speed. This should be very rare
617              *    though: for this to happen a sensor needs to support at least 3 different aspect
618              *    ratio outputs, and when (at least) two outputs are both not the main aspect ratio
619              *    of the webcam, a third size that's larger might be picked and runs into this
620              *    issue.
621              */
622             ALOGW("%s: cannot reach fps %d! Will do %f instead", __FUNCTION__, fpsRange.data.i32[1],
623                   closestFps);
624             requestFpsMax = closestFps;
625         }
626 
627         if (requestFpsMax != mV4l2StreamingFps) {
628             {
629                 std::unique_lock<std::mutex> lk(mV4l2BufferLock);
630                 while (mNumDequeuedV4l2Buffers != 0) {
631                     // Wait until pipeline is idle before reconfigure stream
632                     int waitRet = waitForV4L2BufferReturnLocked(lk);
633                     if (waitRet != 0) {
634                         ALOGE("%s: wait for pipeline idle failed!", __FUNCTION__);
635                         return Status::INTERNAL_ERROR;
636                     }
637                 }
638             }
639             configureV4l2StreamLocked(mV4l2StreamingFmt, requestFpsMax);
640         }
641     }
642 
643     nsecs_t shutterTs = 0;
644     std::unique_ptr<V4L2Frame> frameIn = dequeueV4l2FrameLocked(&shutterTs);
645     if (frameIn == nullptr) {
646         ALOGE("%s: V4L2 deque frame failed!", __FUNCTION__);
647         return Status::INTERNAL_ERROR;
648     }
649 
650     std::shared_ptr<HalRequest> halReq = std::make_shared<HalRequest>();
651     halReq->frameNumber = request.frameNumber;
652     halReq->setting = mLatestReqSetting;
653     halReq->frameIn = std::move(frameIn);
654     halReq->shutterTs = shutterTs;
655     halReq->buffers.resize(numOutputBufs);
656     for (size_t i = 0; i < numOutputBufs; i++) {
657         HalStreamBuffer& halBuf = halReq->buffers[i];
658         int streamId = halBuf.streamId = request.outputBuffers[i].streamId;
659         halBuf.bufferId = request.outputBuffers[i].bufferId;
660         const Stream& stream = mStreamMap[streamId];
661         halBuf.width = stream.width;
662         halBuf.height = stream.height;
663         halBuf.format = stream.format;
664         halBuf.usage = stream.usage;
665         halBuf.bufPtr = nullptr;  // threadloop will request buffer from cameraservice
666         halBuf.acquireFence = 0;  // threadloop will request fence from cameraservice
667         halBuf.fenceTimeout = false;
668     }
669     {
670         std::lock_guard<std::mutex> lk(mInflightFramesLock);
671         mInflightFrames.insert(halReq->frameNumber);
672     }
673     // Send request to OutputThread for the rest of processing
674     mOutputThread->submitRequest(halReq);
675     mFirstRequest = false;
676     return Status::OK;
677 }
678 
signalStreamFlush(const std::vector<int32_t> &,int32_t in_streamConfigCounter)679 ScopedAStatus ExternalCameraDeviceSession::signalStreamFlush(
680         const std::vector<int32_t>& /*in_streamIds*/, int32_t in_streamConfigCounter) {
681     {
682         Mutex::Autolock _l(mLock);
683         if (in_streamConfigCounter < mLastStreamConfigCounter) {
684             // stale call. new streams have been configured since this call was issued.
685             // Do nothing.
686             return fromStatus(Status::OK);
687         }
688     }
689 
690     // TODO: implement if needed.
691     return fromStatus(Status::OK);
692 }
693 
switchToOffline(const std::vector<int32_t> & in_streamsToKeep,CameraOfflineSessionInfo * out_offlineSessionInfo,std::shared_ptr<ICameraOfflineSession> * _aidl_return)694 ScopedAStatus ExternalCameraDeviceSession::switchToOffline(
695         const std::vector<int32_t>& in_streamsToKeep,
696         CameraOfflineSessionInfo* out_offlineSessionInfo,
697         std::shared_ptr<ICameraOfflineSession>* _aidl_return) {
698     std::vector<NotifyMsg> msgs;
699     std::vector<CaptureResult> results;
700     CameraOfflineSessionInfo info;
701     std::shared_ptr<ICameraOfflineSession> session;
702     Status st = switchToOffline(in_streamsToKeep, &msgs, &results, &info, &session);
703 
704     mCallback->notify(msgs);
705     invokeProcessCaptureResultCallback(results, /* tryWriteFmq= */ true);
706     freeReleaseFences(results);
707 
708     // setup return values
709     *out_offlineSessionInfo = info;
710     *_aidl_return = session;
711     return fromStatus(st);
712 }
713 
switchToOffline(const std::vector<int32_t> & offlineStreams,std::vector<NotifyMsg> * msgs,std::vector<CaptureResult> * results,CameraOfflineSessionInfo * info,std::shared_ptr<ICameraOfflineSession> * session)714 Status ExternalCameraDeviceSession::switchToOffline(
715         const std::vector<int32_t>& offlineStreams, std::vector<NotifyMsg>* msgs,
716         std::vector<CaptureResult>* results, CameraOfflineSessionInfo* info,
717         std::shared_ptr<ICameraOfflineSession>* session) {
718     ATRACE_CALL();
719     if (offlineStreams.size() > 1) {
720         ALOGE("%s: more than one offline stream is not supported", __FUNCTION__);
721         return Status::ILLEGAL_ARGUMENT;
722     }
723 
724     if (msgs == nullptr || results == nullptr || info == nullptr || session == nullptr) {
725         ALOGE("%s, output arguments (%p, %p, %p, %p) must not be null", __FUNCTION__, msgs, results,
726               info, session);
727     }
728 
729     Mutex::Autolock _il(mInterfaceLock);
730     Status status = initStatus();
731     if (status != Status::OK) {
732         return status;
733     }
734 
735     Mutex::Autolock _l(mLock);
736     for (auto streamId : offlineStreams) {
737         if (!supportOfflineLocked(streamId)) {
738             return Status::ILLEGAL_ARGUMENT;
739         }
740     }
741 
742     // pause output thread and get all remaining inflight requests
743     auto remainingReqs = mOutputThread->switchToOffline();
744     std::vector<std::shared_ptr<HalRequest>> halReqs;
745 
746     // Send out buffer/request error for remaining requests and filter requests
747     // to be handled in offline mode
748     for (auto& halReq : remainingReqs) {
749         bool dropReq = canDropRequest(offlineStreams, halReq);
750         if (dropReq) {
751             // Request is dropped completely. Just send request error and
752             // there is no need to send the request to offline session
753             processCaptureRequestError(halReq, msgs, results);
754             continue;
755         }
756 
757         // All requests reach here must have at least one offline stream output
758         NotifyMsg shutter;
759         aidl::android::hardware::camera::device::ShutterMsg shutterMsg = {
760                 .frameNumber = static_cast<int32_t>(halReq->frameNumber),
761                 .timestamp = halReq->shutterTs};
762         shutter.set<NotifyMsg::Tag::shutter>(shutterMsg);
763         msgs->push_back(shutter);
764 
765         std::vector<HalStreamBuffer> offlineBuffers;
766         for (const auto& buffer : halReq->buffers) {
767             bool dropBuffer = true;
768             for (auto offlineStreamId : offlineStreams) {
769                 if (buffer.streamId == offlineStreamId) {
770                     dropBuffer = false;
771                     break;
772                 }
773             }
774             if (dropBuffer) {
775                 aidl::android::hardware::camera::device::ErrorMsg errorMsg = {
776                         .frameNumber = static_cast<int32_t>(halReq->frameNumber),
777                         .errorStreamId = buffer.streamId,
778                         .errorCode = ErrorCode::ERROR_BUFFER};
779 
780                 NotifyMsg error;
781                 error.set<NotifyMsg::Tag::error>(errorMsg);
782                 msgs->push_back(error);
783 
784                 results->push_back({
785                         .frameNumber = static_cast<int32_t>(halReq->frameNumber),
786                         .outputBuffers = {},
787                         .inputBuffer = {.streamId = -1},
788                         .partialResult = 0,  // buffer only result
789                 });
790 
791                 CaptureResult& result = results->back();
792                 result.outputBuffers.resize(1);
793                 StreamBuffer& outputBuffer = result.outputBuffers[0];
794                 outputBuffer.streamId = buffer.streamId;
795                 outputBuffer.bufferId = buffer.bufferId;
796                 outputBuffer.status = BufferStatus::ERROR;
797                 if (buffer.acquireFence >= 0) {
798                     native_handle_t* handle = native_handle_create(/*numFds*/ 1, /*numInts*/ 0);
799                     handle->data[0] = buffer.acquireFence;
800                     outputBuffer.releaseFence = android::dupToAidl(handle);
801                     native_handle_delete(handle);
802                 }
803             } else {
804                 offlineBuffers.push_back(buffer);
805             }
806         }
807         halReq->buffers = offlineBuffers;
808         halReqs.push_back(halReq);
809     }
810 
811     // convert hal requests to offline request
812     std::deque<std::shared_ptr<HalRequest>> offlineReqs(halReqs.size());
813     size_t i = 0;
814     for (auto& v4lReq : halReqs) {
815         offlineReqs[i] = std::make_shared<HalRequest>();
816         offlineReqs[i]->frameNumber = v4lReq->frameNumber;
817         offlineReqs[i]->setting = v4lReq->setting;
818         offlineReqs[i]->shutterTs = v4lReq->shutterTs;
819         offlineReqs[i]->buffers = v4lReq->buffers;
820         std::shared_ptr<V4L2Frame> v4l2Frame(static_cast<V4L2Frame*>(v4lReq->frameIn.get()));
821         offlineReqs[i]->frameIn = std::make_shared<AllocatedV4L2Frame>(v4l2Frame);
822         i++;
823         // enqueue V4L2 frame
824         enqueueV4l2Frame(v4l2Frame);
825     }
826 
827     // Collect buffer caches/streams
828     std::vector<Stream> streamInfos(offlineStreams.size());
829     std::map<int, CirculatingBuffers> circulatingBuffers;
830     {
831         Mutex::Autolock _cbsl(mCbsLock);
832         for (auto streamId : offlineStreams) {
833             circulatingBuffers[streamId] = mCirculatingBuffers.at(streamId);
834             mCirculatingBuffers.erase(streamId);
835             streamInfos.push_back(mStreamMap.at(streamId));
836             mStreamMap.erase(streamId);
837         }
838     }
839 
840     fillOfflineSessionInfo(offlineStreams, offlineReqs, circulatingBuffers, info);
841     // create the offline session object
842     bool afTrigger;
843     {
844         std::lock_guard<std::mutex> _lk(mAfTriggerLock);
845         afTrigger = mAfTrigger;
846     }
847 
848     std::shared_ptr<ExternalCameraOfflineSession> sessionImpl =
849             ndk::SharedRefBase::make<ExternalCameraOfflineSession>(
850                     mCroppingType, mCameraCharacteristics, mCameraId, mExifMake, mExifModel,
851                     mBlobBufferSize, afTrigger, streamInfos, offlineReqs, circulatingBuffers);
852 
853     bool initFailed = sessionImpl->initialize();
854     if (initFailed) {
855         ALOGE("%s: offline session initialize failed!", __FUNCTION__);
856         return Status::INTERNAL_ERROR;
857     }
858 
859     // cleanup stream and buffer caches
860     {
861         Mutex::Autolock _cbsl(mCbsLock);
862         for (auto pair : mStreamMap) {
863             cleanupBuffersLocked(/*Stream ID*/ pair.first);
864         }
865         mCirculatingBuffers.clear();
866     }
867     mStreamMap.clear();
868 
869     // update inflight records
870     {
871         std::lock_guard<std::mutex> _lk(mInflightFramesLock);
872         mInflightFrames.clear();
873     }
874 
875     // stop v4l2 streaming
876     if (v4l2StreamOffLocked() != 0) {
877         ALOGE("%s: stop V4L2 streaming failed!", __FUNCTION__);
878         return Status::INTERNAL_ERROR;
879     }
880 
881     // No need to return session if there is no offline requests left
882     if (!offlineReqs.empty()) {
883         *session = sessionImpl;
884     } else {
885         *session = nullptr;
886     }
887 
888     return Status::OK;
889 }
890 
891 #define ARRAY_SIZE(a) (sizeof(a) / sizeof(a[0]))
892 #define UPDATE(md, tag, data, size)               \
893     do {                                          \
894         if ((md).update((tag), (data), (size))) { \
895             ALOGE("Update " #tag " failed!");     \
896             return BAD_VALUE;                     \
897         }                                         \
898     } while (0)
899 
initDefaultRequests()900 status_t ExternalCameraDeviceSession::initDefaultRequests() {
901     common::V1_0::helper::CameraMetadata md;
902 
903     const uint8_t aberrationMode = ANDROID_COLOR_CORRECTION_ABERRATION_MODE_OFF;
904     UPDATE(md, ANDROID_COLOR_CORRECTION_ABERRATION_MODE, &aberrationMode, 1);
905 
906     const int32_t exposureCompensation = 0;
907     UPDATE(md, ANDROID_CONTROL_AE_EXPOSURE_COMPENSATION, &exposureCompensation, 1);
908 
909     const uint8_t videoStabilizationMode = ANDROID_CONTROL_VIDEO_STABILIZATION_MODE_OFF;
910     UPDATE(md, ANDROID_CONTROL_VIDEO_STABILIZATION_MODE, &videoStabilizationMode, 1);
911 
912     const uint8_t awbMode = ANDROID_CONTROL_AWB_MODE_AUTO;
913     UPDATE(md, ANDROID_CONTROL_AWB_MODE, &awbMode, 1);
914 
915     const uint8_t aeMode = ANDROID_CONTROL_AE_MODE_ON;
916     UPDATE(md, ANDROID_CONTROL_AE_MODE, &aeMode, 1);
917 
918     const uint8_t aePrecaptureTrigger = ANDROID_CONTROL_AE_PRECAPTURE_TRIGGER_IDLE;
919     UPDATE(md, ANDROID_CONTROL_AE_PRECAPTURE_TRIGGER, &aePrecaptureTrigger, 1);
920 
921     const uint8_t afMode = ANDROID_CONTROL_AF_MODE_AUTO;
922     UPDATE(md, ANDROID_CONTROL_AF_MODE, &afMode, 1);
923 
924     const uint8_t afTrigger = ANDROID_CONTROL_AF_TRIGGER_IDLE;
925     UPDATE(md, ANDROID_CONTROL_AF_TRIGGER, &afTrigger, 1);
926 
927     const uint8_t sceneMode = ANDROID_CONTROL_SCENE_MODE_DISABLED;
928     UPDATE(md, ANDROID_CONTROL_SCENE_MODE, &sceneMode, 1);
929 
930     const uint8_t effectMode = ANDROID_CONTROL_EFFECT_MODE_OFF;
931     UPDATE(md, ANDROID_CONTROL_EFFECT_MODE, &effectMode, 1);
932 
933     const uint8_t flashMode = ANDROID_FLASH_MODE_OFF;
934     UPDATE(md, ANDROID_FLASH_MODE, &flashMode, 1);
935 
936     const int32_t thumbnailSize[] = {240, 180};
937     UPDATE(md, ANDROID_JPEG_THUMBNAIL_SIZE, thumbnailSize, 2);
938 
939     const uint8_t jpegQuality = 90;
940     UPDATE(md, ANDROID_JPEG_QUALITY, &jpegQuality, 1);
941     UPDATE(md, ANDROID_JPEG_THUMBNAIL_QUALITY, &jpegQuality, 1);
942 
943     const int32_t jpegOrientation = 0;
944     UPDATE(md, ANDROID_JPEG_ORIENTATION, &jpegOrientation, 1);
945 
946     const uint8_t oisMode = ANDROID_LENS_OPTICAL_STABILIZATION_MODE_OFF;
947     UPDATE(md, ANDROID_LENS_OPTICAL_STABILIZATION_MODE, &oisMode, 1);
948 
949     const uint8_t nrMode = ANDROID_NOISE_REDUCTION_MODE_OFF;
950     UPDATE(md, ANDROID_NOISE_REDUCTION_MODE, &nrMode, 1);
951 
952     const int32_t testPatternModes = ANDROID_SENSOR_TEST_PATTERN_MODE_OFF;
953     UPDATE(md, ANDROID_SENSOR_TEST_PATTERN_MODE, &testPatternModes, 1);
954 
955     const uint8_t fdMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF;
956     UPDATE(md, ANDROID_STATISTICS_FACE_DETECT_MODE, &fdMode, 1);
957 
958     const uint8_t hotpixelMode = ANDROID_STATISTICS_HOT_PIXEL_MAP_MODE_OFF;
959     UPDATE(md, ANDROID_STATISTICS_HOT_PIXEL_MAP_MODE, &hotpixelMode, 1);
960 
961     bool support30Fps = false;
962     int32_t maxFps = std::numeric_limits<int32_t>::min();
963     for (const auto& supportedFormat : mSupportedFormats) {
964         for (const auto& fr : supportedFormat.frameRates) {
965             int32_t framerateInt = static_cast<int32_t>(fr.getFramesPerSecond());
966             if (maxFps < framerateInt) {
967                 maxFps = framerateInt;
968             }
969             if (framerateInt == 30) {
970                 support30Fps = true;
971                 break;
972             }
973         }
974         if (support30Fps) {
975             break;
976         }
977     }
978 
979     int32_t defaultFramerate = support30Fps ? 30 : maxFps;
980     int32_t defaultFpsRange[] = {defaultFramerate / 2, defaultFramerate};
981     UPDATE(md, ANDROID_CONTROL_AE_TARGET_FPS_RANGE, defaultFpsRange, ARRAY_SIZE(defaultFpsRange));
982 
983     uint8_t antibandingMode = ANDROID_CONTROL_AE_ANTIBANDING_MODE_AUTO;
984     UPDATE(md, ANDROID_CONTROL_AE_ANTIBANDING_MODE, &antibandingMode, 1);
985 
986     const uint8_t controlMode = ANDROID_CONTROL_MODE_AUTO;
987     UPDATE(md, ANDROID_CONTROL_MODE, &controlMode, 1);
988 
989     for (const auto& type : ndk::enum_range<RequestTemplate>()) {
990         common::V1_0::helper::CameraMetadata mdCopy = md;
991         uint8_t intent = ANDROID_CONTROL_CAPTURE_INTENT_PREVIEW;
992         switch (type) {
993             case RequestTemplate::PREVIEW:
994                 intent = ANDROID_CONTROL_CAPTURE_INTENT_PREVIEW;
995                 break;
996             case RequestTemplate::STILL_CAPTURE:
997                 intent = ANDROID_CONTROL_CAPTURE_INTENT_STILL_CAPTURE;
998                 break;
999             case RequestTemplate::VIDEO_RECORD:
1000                 intent = ANDROID_CONTROL_CAPTURE_INTENT_VIDEO_RECORD;
1001                 break;
1002             case RequestTemplate::VIDEO_SNAPSHOT:
1003                 intent = ANDROID_CONTROL_CAPTURE_INTENT_VIDEO_SNAPSHOT;
1004                 break;
1005             default:
1006                 ALOGV("%s: unsupported RequestTemplate type %d", __FUNCTION__, type);
1007                 continue;
1008         }
1009         UPDATE(mdCopy, ANDROID_CONTROL_CAPTURE_INTENT, &intent, 1);
1010         camera_metadata_t* mdPtr = mdCopy.release();
1011         uint8_t* rawMd = reinterpret_cast<uint8_t*>(mdPtr);
1012         CameraMetadata aidlMd;
1013         aidlMd.metadata.assign(rawMd, rawMd + get_camera_metadata_size(mdPtr));
1014         mDefaultRequests[type] = aidlMd;
1015         free_camera_metadata(mdPtr);
1016     }
1017     return OK;
1018 }
1019 
fillCaptureResult(common::V1_0::helper::CameraMetadata & md,nsecs_t timestamp)1020 status_t ExternalCameraDeviceSession::fillCaptureResult(common::V1_0::helper::CameraMetadata& md,
1021                                                         nsecs_t timestamp) {
1022     bool afTrigger = false;
1023     {
1024         std::lock_guard<std::mutex> lk(mAfTriggerLock);
1025         afTrigger = mAfTrigger;
1026         if (md.exists(ANDROID_CONTROL_AF_TRIGGER)) {
1027             camera_metadata_entry entry = md.find(ANDROID_CONTROL_AF_TRIGGER);
1028             if (entry.data.u8[0] == ANDROID_CONTROL_AF_TRIGGER_START) {
1029                 mAfTrigger = afTrigger = true;
1030             } else if (entry.data.u8[0] == ANDROID_CONTROL_AF_TRIGGER_CANCEL) {
1031                 mAfTrigger = afTrigger = false;
1032             }
1033         }
1034     }
1035 
1036     // For USB camera, the USB camera handles everything and we don't have control
1037     // over AF. We only simply fake the AF metadata based on the request
1038     // received here.
1039     uint8_t afState;
1040     if (afTrigger) {
1041         afState = ANDROID_CONTROL_AF_STATE_FOCUSED_LOCKED;
1042     } else {
1043         afState = ANDROID_CONTROL_AF_STATE_INACTIVE;
1044     }
1045     UPDATE(md, ANDROID_CONTROL_AF_STATE, &afState, 1);
1046 
1047     camera_metadata_ro_entry activeArraySize =
1048             mCameraCharacteristics.find(ANDROID_SENSOR_INFO_ACTIVE_ARRAY_SIZE);
1049 
1050     return fillCaptureResultCommon(md, timestamp, activeArraySize);
1051 }
1052 
configureV4l2StreamLocked(const SupportedV4L2Format & v4l2Fmt,double requestFps)1053 int ExternalCameraDeviceSession::configureV4l2StreamLocked(const SupportedV4L2Format& v4l2Fmt,
1054                                                            double requestFps) {
1055     ATRACE_CALL();
1056     int ret = v4l2StreamOffLocked();
1057     if (ret != OK) {
1058         ALOGE("%s: stop v4l2 streaming failed: ret %d", __FUNCTION__, ret);
1059         return ret;
1060     }
1061 
1062     // VIDIOC_S_FMT w/h/fmt
1063     v4l2_format fmt;
1064     fmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
1065     fmt.fmt.pix.width = v4l2Fmt.width;
1066     fmt.fmt.pix.height = v4l2Fmt.height;
1067     fmt.fmt.pix.pixelformat = v4l2Fmt.fourcc;
1068 
1069     {
1070         int numAttempt = 0;
1071         do {
1072             ret = TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_S_FMT, &fmt));
1073             if (numAttempt == MAX_RETRY) {
1074                 break;
1075             }
1076             numAttempt++;
1077             if (ret < 0) {
1078                 ALOGW("%s: VIDIOC_S_FMT failed, wait 33ms and try again", __FUNCTION__);
1079                 usleep(IOCTL_RETRY_SLEEP_US);  // sleep and try again
1080             }
1081         } while (ret < 0);
1082         if (ret < 0) {
1083             ALOGE("%s: S_FMT ioctl failed: %s", __FUNCTION__, strerror(errno));
1084             return -errno;
1085         }
1086     }
1087 
1088     if (v4l2Fmt.width != fmt.fmt.pix.width || v4l2Fmt.height != fmt.fmt.pix.height ||
1089         v4l2Fmt.fourcc != fmt.fmt.pix.pixelformat) {
1090         ALOGE("%s: S_FMT expect %c%c%c%c %dx%d, got %c%c%c%c %dx%d instead!", __FUNCTION__,
1091               v4l2Fmt.fourcc & 0xFF, (v4l2Fmt.fourcc >> 8) & 0xFF, (v4l2Fmt.fourcc >> 16) & 0xFF,
1092               (v4l2Fmt.fourcc >> 24) & 0xFF, v4l2Fmt.width, v4l2Fmt.height,
1093               fmt.fmt.pix.pixelformat & 0xFF, (fmt.fmt.pix.pixelformat >> 8) & 0xFF,
1094               (fmt.fmt.pix.pixelformat >> 16) & 0xFF, (fmt.fmt.pix.pixelformat >> 24) & 0xFF,
1095               fmt.fmt.pix.width, fmt.fmt.pix.height);
1096         return -EINVAL;
1097     }
1098 
1099     uint32_t bufferSize = fmt.fmt.pix.sizeimage;
1100     ALOGI("%s: V4L2 buffer size is %d", __FUNCTION__, bufferSize);
1101     uint32_t expectedMaxBufferSize = kMaxBytesPerPixel * fmt.fmt.pix.width * fmt.fmt.pix.height;
1102     if ((bufferSize == 0) || (bufferSize > expectedMaxBufferSize)) {
1103         ALOGE("%s: V4L2 buffer size: %u looks invalid. Expected maximum size: %u", __FUNCTION__,
1104               bufferSize, expectedMaxBufferSize);
1105         return -EINVAL;
1106     }
1107     mMaxV4L2BufferSize = bufferSize;
1108 
1109     const double kDefaultFps = 30.0;
1110     double fps = std::numeric_limits<double>::max();
1111     if (requestFps != 0.0) {
1112         fps = requestFps;
1113     } else {
1114         double maxFps = -1.0;
1115         // Try to pick the slowest fps that is at least 30
1116         for (const auto& fr : v4l2Fmt.frameRates) {
1117             double f = fr.getFramesPerSecond();
1118             if (maxFps < f) {
1119                 maxFps = f;
1120             }
1121             if (f >= kDefaultFps && f < fps) {
1122                 fps = f;
1123             }
1124         }
1125         // No fps > 30 found, use the highest fps available within supported formats.
1126         if (fps == std::numeric_limits<double>::max()) {
1127             fps = maxFps;
1128         }
1129     }
1130 
1131     int fpsRet = setV4l2FpsLocked(fps);
1132     if (fpsRet != 0 && fpsRet != -EINVAL) {
1133         ALOGE("%s: set fps failed: %s", __FUNCTION__, strerror(fpsRet));
1134         return fpsRet;
1135     }
1136 
1137     uint32_t v4lBufferCount = (fps >= kDefaultFps) ? mCfg.numVideoBuffers : mCfg.numStillBuffers;
1138 
1139     // VIDIOC_REQBUFS: create buffers
1140     v4l2_requestbuffers req_buffers{};
1141     req_buffers.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
1142     req_buffers.memory = V4L2_MEMORY_MMAP;
1143     req_buffers.count = v4lBufferCount;
1144     if (TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_REQBUFS, &req_buffers)) < 0) {
1145         ALOGE("%s: VIDIOC_REQBUFS failed: %s", __FUNCTION__, strerror(errno));
1146         return -errno;
1147     }
1148 
1149     // Driver can indeed return more buffer if it needs more to operate
1150     if (req_buffers.count < v4lBufferCount) {
1151         ALOGE("%s: VIDIOC_REQBUFS expected %d buffers, got %d instead", __FUNCTION__,
1152               v4lBufferCount, req_buffers.count);
1153         return NO_MEMORY;
1154     }
1155 
1156     // VIDIOC_QUERYBUF:  get buffer offset in the V4L2 fd
1157     // VIDIOC_QBUF: send buffer to driver
1158     mV4L2BufferCount = req_buffers.count;
1159     for (uint32_t i = 0; i < req_buffers.count; i++) {
1160         v4l2_buffer buffer = {
1161                 .index = i, .type = V4L2_BUF_TYPE_VIDEO_CAPTURE, .memory = V4L2_MEMORY_MMAP};
1162 
1163         if (TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_QUERYBUF, &buffer)) < 0) {
1164             ALOGE("%s: QUERYBUF %d failed: %s", __FUNCTION__, i, strerror(errno));
1165             return -errno;
1166         }
1167 
1168         if (TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_QBUF, &buffer)) < 0) {
1169             ALOGE("%s: QBUF %d failed: %s", __FUNCTION__, i, strerror(errno));
1170             return -errno;
1171         }
1172     }
1173 
1174     {
1175         // VIDIOC_STREAMON: start streaming
1176         v4l2_buf_type capture_type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
1177         int numAttempt = 0;
1178         do {
1179             ret = TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_STREAMON, &capture_type));
1180             if (numAttempt == MAX_RETRY) {
1181                 break;
1182             }
1183             if (ret < 0) {
1184                 ALOGW("%s: VIDIOC_STREAMON failed, wait 33ms and try again", __FUNCTION__);
1185                 usleep(IOCTL_RETRY_SLEEP_US);  // sleep 100 ms and try again
1186             }
1187         } while (ret < 0);
1188 
1189         if (ret < 0) {
1190             ALOGE("%s: VIDIOC_STREAMON ioctl failed: %s", __FUNCTION__, strerror(errno));
1191             return -errno;
1192         }
1193     }
1194 
1195     // Swallow first few frames after streamOn to account for bad frames from some devices
1196     for (int i = 0; i < kBadFramesAfterStreamOn; i++) {
1197         v4l2_buffer buffer{};
1198         buffer.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
1199         buffer.memory = V4L2_MEMORY_MMAP;
1200         if (TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_DQBUF, &buffer)) < 0) {
1201             ALOGE("%s: DQBUF fails: %s", __FUNCTION__, strerror(errno));
1202             return -errno;
1203         }
1204 
1205         if (TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_QBUF, &buffer)) < 0) {
1206             ALOGE("%s: QBUF index %d fails: %s", __FUNCTION__, buffer.index, strerror(errno));
1207             return -errno;
1208         }
1209     }
1210 
1211     ALOGI("%s: start V4L2 streaming %dx%d@%ffps", __FUNCTION__, v4l2Fmt.width, v4l2Fmt.height, fps);
1212     mV4l2StreamingFmt = v4l2Fmt;
1213     mV4l2Streaming = true;
1214     return OK;
1215 }
1216 
dequeueV4l2FrameLocked(nsecs_t * shutterTs)1217 std::unique_ptr<V4L2Frame> ExternalCameraDeviceSession::dequeueV4l2FrameLocked(nsecs_t* shutterTs) {
1218     ATRACE_CALL();
1219     std::unique_ptr<V4L2Frame> ret = nullptr;
1220     if (shutterTs == nullptr) {
1221         ALOGE("%s: shutterTs must not be null!", __FUNCTION__);
1222         return ret;
1223     }
1224 
1225     {
1226         std::unique_lock<std::mutex> lk(mV4l2BufferLock);
1227         if (mNumDequeuedV4l2Buffers == mV4L2BufferCount) {
1228             int waitRet = waitForV4L2BufferReturnLocked(lk);
1229             if (waitRet != 0) {
1230                 return ret;
1231             }
1232         }
1233     }
1234 
1235     ATRACE_BEGIN("VIDIOC_DQBUF");
1236     v4l2_buffer buffer{};
1237     buffer.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
1238     buffer.memory = V4L2_MEMORY_MMAP;
1239     if (TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_DQBUF, &buffer)) < 0) {
1240         ALOGE("%s: DQBUF fails: %s", __FUNCTION__, strerror(errno));
1241         return ret;
1242     }
1243     ATRACE_END();
1244 
1245     if (buffer.index >= mV4L2BufferCount) {
1246         ALOGE("%s: Invalid buffer id: %d", __FUNCTION__, buffer.index);
1247         return ret;
1248     }
1249 
1250     if (buffer.flags & V4L2_BUF_FLAG_ERROR) {
1251         ALOGE("%s: v4l2 buf error! buf flag 0x%x", __FUNCTION__, buffer.flags);
1252         // TODO: try to dequeue again
1253     }
1254 
1255     if (buffer.bytesused > mMaxV4L2BufferSize) {
1256         ALOGE("%s: v4l2 buffer bytes used: %u maximum %u", __FUNCTION__, buffer.bytesused,
1257               mMaxV4L2BufferSize);
1258         return ret;
1259     }
1260 
1261     if (buffer.flags & V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC) {
1262         // Ideally we should also check for V4L2_BUF_FLAG_TSTAMP_SRC_SOE, but
1263         // even V4L2_BUF_FLAG_TSTAMP_SRC_EOF is better than capture a timestamp now
1264         *shutterTs = static_cast<nsecs_t>(buffer.timestamp.tv_sec) * 1000000000LL +
1265                      buffer.timestamp.tv_usec * 1000LL;
1266     } else {
1267         *shutterTs = systemTime(SYSTEM_TIME_MONOTONIC);
1268     }
1269 
1270     {
1271         std::lock_guard<std::mutex> lk(mV4l2BufferLock);
1272         mNumDequeuedV4l2Buffers++;
1273     }
1274 
1275     return std::make_unique<V4L2Frame>(mV4l2StreamingFmt.width, mV4l2StreamingFmt.height,
1276                                        mV4l2StreamingFmt.fourcc, buffer.index, mV4l2Fd.get(),
1277                                        buffer.bytesused, buffer.m.offset);
1278 }
1279 
enqueueV4l2Frame(const std::shared_ptr<V4L2Frame> & frame)1280 void ExternalCameraDeviceSession::enqueueV4l2Frame(const std::shared_ptr<V4L2Frame>& frame) {
1281     ATRACE_CALL();
1282     frame->unmap();
1283     ATRACE_BEGIN("VIDIOC_QBUF");
1284     v4l2_buffer buffer{};
1285     buffer.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
1286     buffer.memory = V4L2_MEMORY_MMAP;
1287     buffer.index = frame->mBufferIndex;
1288     if (TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_QBUF, &buffer)) < 0) {
1289         ALOGE("%s: QBUF index %d fails: %s", __FUNCTION__, frame->mBufferIndex, strerror(errno));
1290         return;
1291     }
1292     ATRACE_END();
1293 
1294     {
1295         std::lock_guard<std::mutex> lk(mV4l2BufferLock);
1296         mNumDequeuedV4l2Buffers--;
1297     }
1298     mV4L2BufferReturned.notify_one();
1299 }
1300 
isSupported(const Stream & stream,const std::vector<SupportedV4L2Format> & supportedFormats,const ExternalCameraConfig & devCfg)1301 bool ExternalCameraDeviceSession::isSupported(
1302         const Stream& stream, const std::vector<SupportedV4L2Format>& supportedFormats,
1303         const ExternalCameraConfig& devCfg) {
1304     Dataspace ds = stream.dataSpace;
1305     PixelFormat fmt = stream.format;
1306     uint32_t width = stream.width;
1307     uint32_t height = stream.height;
1308     // TODO: check usage flags
1309 
1310     if (stream.streamType != StreamType::OUTPUT) {
1311         ALOGE("%s: does not support non-output stream type", __FUNCTION__);
1312         return false;
1313     }
1314 
1315     if (stream.rotation != StreamRotation::ROTATION_0) {
1316         ALOGE("%s: does not support stream rotation", __FUNCTION__);
1317         return false;
1318     }
1319 
1320     switch (fmt) {
1321         case PixelFormat::BLOB:
1322             if (ds != Dataspace::JFIF) {
1323                 ALOGI("%s: BLOB format does not support dataSpace %x", __FUNCTION__, ds);
1324                 return false;
1325             }
1326             break;
1327         case PixelFormat::IMPLEMENTATION_DEFINED:
1328         case PixelFormat::YCBCR_420_888:
1329         case PixelFormat::YV12:
1330             // TODO: check what dataspace we can support here.
1331             // intentional no-ops.
1332             break;
1333         case PixelFormat::Y16:
1334             if (!devCfg.depthEnabled) {
1335                 ALOGI("%s: Depth is not Enabled", __FUNCTION__);
1336                 return false;
1337             }
1338             if (!(static_cast<int32_t>(ds) & static_cast<int32_t>(Dataspace::DEPTH))) {
1339                 ALOGI("%s: Y16 supports only dataSpace DEPTH", __FUNCTION__);
1340                 return false;
1341             }
1342             break;
1343         default:
1344             ALOGI("%s: does not support format %x", __FUNCTION__, fmt);
1345             return false;
1346     }
1347 
1348     // Assume we can convert any V4L2 format to any of supported output format for now, i.e.
1349     // ignoring v4l2Fmt.fourcc for now. Might need more subtle check if we support more v4l format
1350     // in the futrue.
1351     for (const auto& v4l2Fmt : supportedFormats) {
1352         if (width == v4l2Fmt.width && height == v4l2Fmt.height) {
1353             return true;
1354         }
1355     }
1356     ALOGI("%s: resolution %dx%d is not supported", __FUNCTION__, width, height);
1357     return false;
1358 }
1359 
importBuffer(int32_t streamId,uint64_t bufId,buffer_handle_t buf,buffer_handle_t ** outBufPtr)1360 Status ExternalCameraDeviceSession::importBuffer(int32_t streamId, uint64_t bufId,
1361                                                  buffer_handle_t buf,
1362                                                  /*out*/ buffer_handle_t** outBufPtr) {
1363     Mutex::Autolock _l(mCbsLock);
1364     return importBufferLocked(streamId, bufId, buf, outBufPtr);
1365 }
1366 
importBufferLocked(int32_t streamId,uint64_t bufId,buffer_handle_t buf,buffer_handle_t ** outBufPtr)1367 Status ExternalCameraDeviceSession::importBufferLocked(int32_t streamId, uint64_t bufId,
1368                                                        buffer_handle_t buf,
1369                                                        buffer_handle_t** outBufPtr) {
1370     return importBufferImpl(mCirculatingBuffers, sHandleImporter, streamId, bufId, buf, outBufPtr);
1371 }
1372 
close()1373 ScopedAStatus ExternalCameraDeviceSession::close() {
1374     closeImpl();
1375     return fromStatus(Status::OK);
1376 }
1377 
closeImpl()1378 void ExternalCameraDeviceSession::closeImpl() {
1379     Mutex::Autolock _il(mInterfaceLock);
1380     bool closed = isClosed();
1381     if (!closed) {
1382         closeOutputThread();
1383         closeBufferRequestThread();
1384 
1385         Mutex::Autolock _l(mLock);
1386         // free all buffers
1387         {
1388             Mutex::Autolock _cbsl(mCbsLock);
1389             for (auto pair : mStreamMap) {
1390                 cleanupBuffersLocked(/*Stream ID*/ pair.first);
1391             }
1392         }
1393         v4l2StreamOffLocked();
1394         ALOGV("%s: closing V4L2 camera FD %d", __FUNCTION__, mV4l2Fd.get());
1395         mV4l2Fd.reset();
1396         mClosed = true;
1397     }
1398 }
1399 
isClosed()1400 bool ExternalCameraDeviceSession::isClosed() {
1401     Mutex::Autolock _l(mLock);
1402     return mClosed;
1403 }
1404 
repeatingRequestEnd(int32_t,const std::vector<int32_t> &)1405 ScopedAStatus ExternalCameraDeviceSession::repeatingRequestEnd(
1406         int32_t /*in_frameNumber*/, const std::vector<int32_t>& /*in_streamIds*/) {
1407     // TODO: Figure this one out.
1408     return fromStatus(Status::OK);
1409 }
1410 
v4l2StreamOffLocked()1411 int ExternalCameraDeviceSession::v4l2StreamOffLocked() {
1412     if (!mV4l2Streaming) {
1413         return OK;
1414     }
1415 
1416     {
1417         std::lock_guard<std::mutex> lk(mV4l2BufferLock);
1418         if (mNumDequeuedV4l2Buffers != 0) {
1419             ALOGE("%s: there are %zu inflight V4L buffers", __FUNCTION__, mNumDequeuedV4l2Buffers);
1420             return -1;
1421         }
1422     }
1423     mV4L2BufferCount = 0;
1424 
1425     // VIDIOC_STREAMOFF
1426     v4l2_buf_type capture_type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
1427     if (TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_STREAMOFF, &capture_type)) < 0) {
1428         ALOGE("%s: STREAMOFF failed: %s", __FUNCTION__, strerror(errno));
1429         return -errno;
1430     }
1431 
1432     // VIDIOC_REQBUFS: clear buffers
1433     v4l2_requestbuffers req_buffers{};
1434     req_buffers.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
1435     req_buffers.memory = V4L2_MEMORY_MMAP;
1436     req_buffers.count = 0;
1437     if (TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_REQBUFS, &req_buffers)) < 0) {
1438         ALOGE("%s: REQBUFS failed: %s", __FUNCTION__, strerror(errno));
1439         return -errno;
1440     }
1441 
1442     mV4l2Streaming = false;
1443     return OK;
1444 }
1445 
setV4l2FpsLocked(double fps)1446 int ExternalCameraDeviceSession::setV4l2FpsLocked(double fps) {
1447     // VIDIOC_G_PARM/VIDIOC_S_PARM: set fps
1448     v4l2_streamparm streamparm = {.type = V4L2_BUF_TYPE_VIDEO_CAPTURE};
1449     // The following line checks that the driver knows about framerate get/set.
1450     int ret = TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_G_PARM, &streamparm));
1451     if (ret != 0) {
1452         if (errno == -EINVAL) {
1453             ALOGW("%s: device does not support VIDIOC_G_PARM", __FUNCTION__);
1454         }
1455         return -errno;
1456     }
1457     // Now check if the device is able to accept a capture framerate set.
1458     if (!(streamparm.parm.capture.capability & V4L2_CAP_TIMEPERFRAME)) {
1459         ALOGW("%s: device does not support V4L2_CAP_TIMEPERFRAME", __FUNCTION__);
1460         return -EINVAL;
1461     }
1462 
1463     // fps is float, approximate by a fraction.
1464     const int kFrameRatePrecision = 10000;
1465     streamparm.parm.capture.timeperframe.numerator = kFrameRatePrecision;
1466     streamparm.parm.capture.timeperframe.denominator = (fps * kFrameRatePrecision);
1467 
1468     if (TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_S_PARM, &streamparm)) < 0) {
1469         ALOGE("%s: failed to set framerate to %f: %s", __FUNCTION__, fps, strerror(errno));
1470         return -1;
1471     }
1472 
1473     double retFps = streamparm.parm.capture.timeperframe.denominator /
1474                     static_cast<double>(streamparm.parm.capture.timeperframe.numerator);
1475     if (std::fabs(fps - retFps) > 1.0) {
1476         ALOGE("%s: expect fps %f, got %f instead", __FUNCTION__, fps, retFps);
1477         return -1;
1478     }
1479     mV4l2StreamingFps = fps;
1480     return 0;
1481 }
1482 
cleanupInflightFences(std::vector<int> & allFences,size_t numFences)1483 void ExternalCameraDeviceSession::cleanupInflightFences(std::vector<int>& allFences,
1484                                                         size_t numFences) {
1485     for (size_t j = 0; j < numFences; j++) {
1486         sHandleImporter.closeFence(allFences[j]);
1487     }
1488 }
1489 
cleanupBuffersLocked(int id)1490 void ExternalCameraDeviceSession::cleanupBuffersLocked(int id) {
1491     for (auto& pair : mCirculatingBuffers.at(id)) {
1492         sHandleImporter.freeBuffer(pair.second);
1493     }
1494     mCirculatingBuffers[id].clear();
1495     mCirculatingBuffers.erase(id);
1496 }
1497 
notifyShutter(int32_t frameNumber,nsecs_t shutterTs)1498 void ExternalCameraDeviceSession::notifyShutter(int32_t frameNumber, nsecs_t shutterTs) {
1499     NotifyMsg msg;
1500     msg.set<NotifyMsg::Tag::shutter>(ShutterMsg{
1501             .frameNumber = frameNumber,
1502             .timestamp = shutterTs,
1503     });
1504     mCallback->notify({msg});
1505 }
notifyError(int32_t frameNumber,int32_t streamId,ErrorCode ec)1506 void ExternalCameraDeviceSession::notifyError(int32_t frameNumber, int32_t streamId, ErrorCode ec) {
1507     NotifyMsg msg;
1508     msg.set<NotifyMsg::Tag::error>(ErrorMsg{
1509             .frameNumber = frameNumber,
1510             .errorStreamId = streamId,
1511             .errorCode = ec,
1512     });
1513     mCallback->notify({msg});
1514 }
1515 
invokeProcessCaptureResultCallback(std::vector<CaptureResult> & results,bool tryWriteFmq)1516 void ExternalCameraDeviceSession::invokeProcessCaptureResultCallback(
1517         std::vector<CaptureResult>& results, bool tryWriteFmq) {
1518     if (mProcessCaptureResultLock.tryLock() != OK) {
1519         const nsecs_t NS_TO_SECOND = 1000000000;
1520         ALOGV("%s: previous call is not finished! waiting 1s...", __FUNCTION__);
1521         if (mProcessCaptureResultLock.timedLock(/* 1s */ NS_TO_SECOND) != OK) {
1522             ALOGE("%s: cannot acquire lock in 1s, cannot proceed", __FUNCTION__);
1523             return;
1524         }
1525     }
1526     if (tryWriteFmq && mResultMetadataQueue->availableToWrite() > 0) {
1527         for (CaptureResult& result : results) {
1528             CameraMetadata& md = result.result;
1529             if (!md.metadata.empty()) {
1530                 if (mResultMetadataQueue->write(reinterpret_cast<int8_t*>(md.metadata.data()),
1531                                                 md.metadata.size())) {
1532                     result.fmqResultSize = md.metadata.size();
1533                     md.metadata.resize(0);
1534                 } else {
1535                     ALOGW("%s: couldn't utilize fmq, fall back to hwbinder", __FUNCTION__);
1536                     result.fmqResultSize = 0;
1537                 }
1538             } else {
1539                 result.fmqResultSize = 0;
1540             }
1541         }
1542     }
1543     auto status = mCallback->processCaptureResult(results);
1544     if (!status.isOk()) {
1545         ALOGE("%s: processCaptureResult ERROR : %d:%d", __FUNCTION__, status.getExceptionCode(),
1546               status.getServiceSpecificError());
1547     }
1548 
1549     mProcessCaptureResultLock.unlock();
1550 }
1551 
waitForV4L2BufferReturnLocked(std::unique_lock<std::mutex> & lk)1552 int ExternalCameraDeviceSession::waitForV4L2BufferReturnLocked(std::unique_lock<std::mutex>& lk) {
1553     ATRACE_CALL();
1554     auto timeout = std::chrono::seconds(kBufferWaitTimeoutSec);
1555     mLock.unlock();
1556     auto st = mV4L2BufferReturned.wait_for(lk, timeout);
1557     // Here we introduce an order where mV4l2BufferLock is acquired before mLock, while
1558     // the normal lock acquisition order is reversed. This is fine because in most of
1559     // cases we are protected by mInterfaceLock. The only thread that can cause deadlock
1560     // is the OutputThread, where we do need to make sure we don't acquire mLock then
1561     // mV4l2BufferLock
1562     mLock.lock();
1563     if (st == std::cv_status::timeout) {
1564         ALOGE("%s: wait for V4L2 buffer return timeout!", __FUNCTION__);
1565         return -1;
1566     }
1567     return 0;
1568 }
1569 
supportOfflineLocked(int32_t streamId)1570 bool ExternalCameraDeviceSession::supportOfflineLocked(int32_t streamId) {
1571     const Stream& stream = mStreamMap[streamId];
1572     if (stream.format == PixelFormat::BLOB &&
1573         static_cast<int32_t>(stream.dataSpace) == static_cast<int32_t>(Dataspace::JFIF)) {
1574         return true;
1575     }
1576     // TODO: support YUV output stream?
1577     return false;
1578 }
1579 
canDropRequest(const std::vector<int32_t> & offlineStreams,std::shared_ptr<HalRequest> halReq)1580 bool ExternalCameraDeviceSession::canDropRequest(const std::vector<int32_t>& offlineStreams,
1581                                                  std::shared_ptr<HalRequest> halReq) {
1582     for (const auto& buffer : halReq->buffers) {
1583         for (auto offlineStreamId : offlineStreams) {
1584             if (buffer.streamId == offlineStreamId) {
1585                 return false;
1586             }
1587         }
1588     }
1589     // Only drop a request completely if it has no offline output
1590     return true;
1591 }
1592 
fillOfflineSessionInfo(const std::vector<int32_t> & offlineStreams,std::deque<std::shared_ptr<HalRequest>> & offlineReqs,const std::map<int,CirculatingBuffers> & circulatingBuffers,CameraOfflineSessionInfo * info)1593 void ExternalCameraDeviceSession::fillOfflineSessionInfo(
1594         const std::vector<int32_t>& offlineStreams,
1595         std::deque<std::shared_ptr<HalRequest>>& offlineReqs,
1596         const std::map<int, CirculatingBuffers>& circulatingBuffers,
1597         CameraOfflineSessionInfo* info) {
1598     if (info == nullptr) {
1599         ALOGE("%s: output info must not be null!", __FUNCTION__);
1600         return;
1601     }
1602 
1603     info->offlineStreams.resize(offlineStreams.size());
1604     info->offlineRequests.resize(offlineReqs.size());
1605 
1606     // Fill in offline reqs and count outstanding buffers
1607     for (size_t i = 0; i < offlineReqs.size(); i++) {
1608         info->offlineRequests[i].frameNumber = offlineReqs[i]->frameNumber;
1609         info->offlineRequests[i].pendingStreams.resize(offlineReqs[i]->buffers.size());
1610         for (size_t bIdx = 0; bIdx < offlineReqs[i]->buffers.size(); bIdx++) {
1611             int32_t streamId = offlineReqs[i]->buffers[bIdx].streamId;
1612             info->offlineRequests[i].pendingStreams[bIdx] = streamId;
1613         }
1614     }
1615 
1616     for (size_t i = 0; i < offlineStreams.size(); i++) {
1617         int32_t streamId = offlineStreams[i];
1618         info->offlineStreams[i].id = streamId;
1619         // outstanding buffers are 0 since we are doing hal buffer management and
1620         // offline session will ask for those buffers later
1621         info->offlineStreams[i].numOutstandingBuffers = 0;
1622         const CirculatingBuffers& bufIdMap = circulatingBuffers.at(streamId);
1623         info->offlineStreams[i].circulatingBufferIds.resize(bufIdMap.size());
1624         size_t bIdx = 0;
1625         for (const auto& pair : bufIdMap) {
1626             // Fill in bufferId
1627             info->offlineStreams[i].circulatingBufferIds[bIdx++] = pair.first;
1628         }
1629     }
1630 }
1631 
isStreamCombinationSupported(const StreamConfiguration & config,const std::vector<SupportedV4L2Format> & supportedFormats,const ExternalCameraConfig & devCfg)1632 Status ExternalCameraDeviceSession::isStreamCombinationSupported(
1633         const StreamConfiguration& config, const std::vector<SupportedV4L2Format>& supportedFormats,
1634         const ExternalCameraConfig& devCfg) {
1635     if (config.operationMode != StreamConfigurationMode::NORMAL_MODE) {
1636         ALOGE("%s: unsupported operation mode: %d", __FUNCTION__, config.operationMode);
1637         return Status::ILLEGAL_ARGUMENT;
1638     }
1639 
1640     if (config.streams.size() == 0) {
1641         ALOGE("%s: cannot configure zero stream", __FUNCTION__);
1642         return Status::ILLEGAL_ARGUMENT;
1643     }
1644 
1645     int numProcessedStream = 0;
1646     int numStallStream = 0;
1647     for (const auto& stream : config.streams) {
1648         // Check if the format/width/height combo is supported
1649         if (!isSupported(stream, supportedFormats, devCfg)) {
1650             return Status::ILLEGAL_ARGUMENT;
1651         }
1652         if (stream.format == PixelFormat::BLOB) {
1653             numStallStream++;
1654         } else {
1655             numProcessedStream++;
1656         }
1657     }
1658 
1659     if (numProcessedStream > kMaxProcessedStream) {
1660         ALOGE("%s: too many processed streams (expect <= %d, got %d)", __FUNCTION__,
1661               kMaxProcessedStream, numProcessedStream);
1662         return Status::ILLEGAL_ARGUMENT;
1663     }
1664 
1665     if (numStallStream > kMaxStallStream) {
1666         ALOGE("%s: too many stall streams (expect <= %d, got %d)", __FUNCTION__, kMaxStallStream,
1667               numStallStream);
1668         return Status::ILLEGAL_ARGUMENT;
1669     }
1670 
1671     return Status::OK;
1672 }
updateBufferCaches(const std::vector<BufferCache> & cachesToRemove)1673 void ExternalCameraDeviceSession::updateBufferCaches(
1674         const std::vector<BufferCache>& cachesToRemove) {
1675     Mutex::Autolock _l(mCbsLock);
1676     for (auto& cache : cachesToRemove) {
1677         auto cbsIt = mCirculatingBuffers.find(cache.streamId);
1678         if (cbsIt == mCirculatingBuffers.end()) {
1679             // The stream could have been removed
1680             continue;
1681         }
1682         CirculatingBuffers& cbs = cbsIt->second;
1683         auto it = cbs.find(cache.bufferId);
1684         if (it != cbs.end()) {
1685             sHandleImporter.freeBuffer(it->second);
1686             cbs.erase(it);
1687         } else {
1688             ALOGE("%s: stream %d buffer %" PRIu64 " is not cached", __FUNCTION__, cache.streamId,
1689                   cache.bufferId);
1690         }
1691     }
1692 }
1693 
processCaptureRequestError(const std::shared_ptr<HalRequest> & req,std::vector<NotifyMsg> * outMsgs,std::vector<CaptureResult> * outResults)1694 Status ExternalCameraDeviceSession::processCaptureRequestError(
1695         const std::shared_ptr<HalRequest>& req, std::vector<NotifyMsg>* outMsgs,
1696         std::vector<CaptureResult>* outResults) {
1697     ATRACE_CALL();
1698     // Return V4L2 buffer to V4L2 buffer queue
1699     std::shared_ptr<V4L2Frame> v4l2Frame = std::static_pointer_cast<V4L2Frame>(req->frameIn);
1700     enqueueV4l2Frame(v4l2Frame);
1701 
1702     if (outMsgs == nullptr) {
1703         notifyShutter(req->frameNumber, req->shutterTs);
1704         notifyError(/*frameNum*/ req->frameNumber, /*stream*/ -1, ErrorCode::ERROR_REQUEST);
1705     } else {
1706         NotifyMsg shutter;
1707         shutter.set<NotifyMsg::Tag::shutter>(
1708                 ShutterMsg{.frameNumber = req->frameNumber, .timestamp = req->shutterTs});
1709 
1710         NotifyMsg error;
1711         error.set<NotifyMsg::Tag::error>(ErrorMsg{.frameNumber = req->frameNumber,
1712                                                   .errorStreamId = -1,
1713                                                   .errorCode = ErrorCode::ERROR_REQUEST});
1714         outMsgs->push_back(shutter);
1715         outMsgs->push_back(error);
1716     }
1717 
1718     // Fill output buffers
1719     CaptureResult result;
1720     result.frameNumber = req->frameNumber;
1721     result.partialResult = 1;
1722     result.inputBuffer.streamId = -1;
1723     result.outputBuffers.resize(req->buffers.size());
1724     for (size_t i = 0; i < req->buffers.size(); i++) {
1725         result.outputBuffers[i].streamId = req->buffers[i].streamId;
1726         result.outputBuffers[i].bufferId = req->buffers[i].bufferId;
1727         result.outputBuffers[i].status = BufferStatus::ERROR;
1728         if (req->buffers[i].acquireFence >= 0) {
1729             // numFds = 0 for error
1730             native_handle_t* handle = native_handle_create(/*numFds*/ 0, /*numInts*/ 0);
1731             result.outputBuffers[i].releaseFence = android::dupToAidl(handle);
1732             native_handle_delete(handle);
1733         }
1734     }
1735 
1736     // update inflight records
1737     {
1738         std::lock_guard<std::mutex> lk(mInflightFramesLock);
1739         mInflightFrames.erase(req->frameNumber);
1740     }
1741 
1742     if (outResults == nullptr) {
1743         // Callback into framework
1744         std::vector<CaptureResult> results(1);
1745         results[0] = std::move(result);
1746         invokeProcessCaptureResultCallback(results, /* tryWriteFmq */ true);
1747         freeReleaseFences(results);
1748     } else {
1749         outResults->push_back(std::move(result));
1750     }
1751     return Status::OK;
1752 }
1753 
processCaptureResult(std::shared_ptr<HalRequest> & req)1754 Status ExternalCameraDeviceSession::processCaptureResult(std::shared_ptr<HalRequest>& req) {
1755     ATRACE_CALL();
1756     // Return V4L2 buffer to V4L2 buffer queue
1757     std::shared_ptr<V4L2Frame> v4l2Frame = std::static_pointer_cast<V4L2Frame>(req->frameIn);
1758     enqueueV4l2Frame(v4l2Frame);
1759 
1760     // NotifyShutter
1761     notifyShutter(req->frameNumber, req->shutterTs);
1762 
1763     // Fill output buffers;
1764     std::vector<CaptureResult> results(1);
1765     CaptureResult& result = results[0];
1766     result.frameNumber = req->frameNumber;
1767     result.partialResult = 1;
1768     result.inputBuffer.streamId = -1;
1769     result.outputBuffers.resize(req->buffers.size());
1770     for (size_t i = 0; i < req->buffers.size(); i++) {
1771         result.outputBuffers[i].streamId = req->buffers[i].streamId;
1772         result.outputBuffers[i].bufferId = req->buffers[i].bufferId;
1773         if (req->buffers[i].fenceTimeout) {
1774             result.outputBuffers[i].status = BufferStatus::ERROR;
1775             if (req->buffers[i].acquireFence >= 0) {
1776                 native_handle_t* handle = native_handle_create(/*numFds*/ 1, /*numInts*/ 0);
1777                 handle->data[0] = req->buffers[i].acquireFence;
1778                 result.outputBuffers[i].releaseFence = android::dupToAidl(handle);
1779                 native_handle_delete(handle);
1780             }
1781             notifyError(req->frameNumber, req->buffers[i].streamId, ErrorCode::ERROR_BUFFER);
1782         } else {
1783             result.outputBuffers[i].status = BufferStatus::OK;
1784             // TODO: refactor
1785             if (req->buffers[i].acquireFence >= 0) {
1786                 native_handle_t* handle = native_handle_create(/*numFds*/ 1, /*numInts*/ 0);
1787                 handle->data[0] = req->buffers[i].acquireFence;
1788                 result.outputBuffers[i].releaseFence = android::dupToAidl(handle);
1789                 native_handle_delete(handle);
1790             }
1791         }
1792     }
1793 
1794     // Fill capture result metadata
1795     fillCaptureResult(req->setting, req->shutterTs);
1796     const camera_metadata_t* rawResult = req->setting.getAndLock();
1797     convertToAidl(rawResult, &result.result);
1798     req->setting.unlock(rawResult);
1799 
1800     // update inflight records
1801     {
1802         std::lock_guard<std::mutex> lk(mInflightFramesLock);
1803         mInflightFrames.erase(req->frameNumber);
1804     }
1805 
1806     // Callback into framework
1807     invokeProcessCaptureResultCallback(results, /* tryWriteFmq */ true);
1808     freeReleaseFences(results);
1809     return Status::OK;
1810 }
1811 
getJpegBufferSize(int32_t width,int32_t height) const1812 ssize_t ExternalCameraDeviceSession::getJpegBufferSize(int32_t width, int32_t height) const {
1813     // Constant from camera3.h
1814     const ssize_t kMinJpegBufferSize = 256 * 1024 + sizeof(CameraBlob);
1815     // Get max jpeg size (area-wise).
1816     if (mMaxJpegResolution.width == 0) {
1817         ALOGE("%s: No supported JPEG stream", __FUNCTION__);
1818         return BAD_VALUE;
1819     }
1820 
1821     // Get max jpeg buffer size
1822     ssize_t maxJpegBufferSize = 0;
1823     camera_metadata_ro_entry jpegBufMaxSize = mCameraCharacteristics.find(ANDROID_JPEG_MAX_SIZE);
1824     if (jpegBufMaxSize.count == 0) {
1825         ALOGE("%s: Can't find maximum JPEG size in static metadata!", __FUNCTION__);
1826         return BAD_VALUE;
1827     }
1828     maxJpegBufferSize = jpegBufMaxSize.data.i32[0];
1829 
1830     if (maxJpegBufferSize <= kMinJpegBufferSize) {
1831         ALOGE("%s: ANDROID_JPEG_MAX_SIZE (%zd) <= kMinJpegBufferSize (%zd)", __FUNCTION__,
1832               maxJpegBufferSize, kMinJpegBufferSize);
1833         return BAD_VALUE;
1834     }
1835 
1836     // Calculate final jpeg buffer size for the given resolution.
1837     float scaleFactor =
1838             ((float)(width * height)) / (mMaxJpegResolution.width * mMaxJpegResolution.height);
1839     ssize_t jpegBufferSize =
1840             scaleFactor * (maxJpegBufferSize - kMinJpegBufferSize) + kMinJpegBufferSize;
1841     if (jpegBufferSize > maxJpegBufferSize) {
1842         jpegBufferSize = maxJpegBufferSize;
1843     }
1844 
1845     return jpegBufferSize;
1846 }
dump(int fd,const char **,uint32_t)1847 binder_status_t ExternalCameraDeviceSession::dump(int fd, const char** /*args*/,
1848                                                   uint32_t /*numArgs*/) {
1849     bool intfLocked = tryLock(mInterfaceLock);
1850     if (!intfLocked) {
1851         dprintf(fd, "!! ExternalCameraDeviceSession interface may be deadlocked !!\n");
1852     }
1853 
1854     if (isClosed()) {
1855         dprintf(fd, "External camera %s is closed\n", mCameraId.c_str());
1856         return STATUS_OK;
1857     }
1858 
1859     bool streaming = false;
1860     size_t v4L2BufferCount = 0;
1861     SupportedV4L2Format streamingFmt;
1862     {
1863         bool sessionLocked = tryLock(mLock);
1864         if (!sessionLocked) {
1865             dprintf(fd, "!! ExternalCameraDeviceSession mLock may be deadlocked !!\n");
1866         }
1867         streaming = mV4l2Streaming;
1868         streamingFmt = mV4l2StreamingFmt;
1869         v4L2BufferCount = mV4L2BufferCount;
1870 
1871         if (sessionLocked) {
1872             mLock.unlock();
1873         }
1874     }
1875 
1876     std::unordered_set<uint32_t> inflightFrames;
1877     {
1878         bool iffLocked = tryLock(mInflightFramesLock);
1879         if (!iffLocked) {
1880             dprintf(fd,
1881                     "!! ExternalCameraDeviceSession mInflightFramesLock may be deadlocked !!\n");
1882         }
1883         inflightFrames = mInflightFrames;
1884         if (iffLocked) {
1885             mInflightFramesLock.unlock();
1886         }
1887     }
1888 
1889     dprintf(fd, "External camera %s V4L2 FD %d, cropping type %s, %s\n", mCameraId.c_str(),
1890             mV4l2Fd.get(), (mCroppingType == VERTICAL) ? "vertical" : "horizontal",
1891             streaming ? "streaming" : "not streaming");
1892 
1893     if (streaming) {
1894         // TODO: dump fps later
1895         dprintf(fd, "Current V4L2 format %c%c%c%c %dx%d @ %ffps\n", streamingFmt.fourcc & 0xFF,
1896                 (streamingFmt.fourcc >> 8) & 0xFF, (streamingFmt.fourcc >> 16) & 0xFF,
1897                 (streamingFmt.fourcc >> 24) & 0xFF, streamingFmt.width, streamingFmt.height,
1898                 mV4l2StreamingFps);
1899 
1900         size_t numDequeuedV4l2Buffers = 0;
1901         {
1902             std::lock_guard<std::mutex> lk(mV4l2BufferLock);
1903             numDequeuedV4l2Buffers = mNumDequeuedV4l2Buffers;
1904         }
1905         dprintf(fd, "V4L2 buffer queue size %zu, dequeued %zu\n", v4L2BufferCount,
1906                 numDequeuedV4l2Buffers);
1907     }
1908 
1909     dprintf(fd, "In-flight frames (not sorted):");
1910     for (const auto& frameNumber : inflightFrames) {
1911         dprintf(fd, "%d, ", frameNumber);
1912     }
1913     dprintf(fd, "\n");
1914     mOutputThread->dump(fd);
1915     dprintf(fd, "\n");
1916 
1917     if (intfLocked) {
1918         mInterfaceLock.unlock();
1919     }
1920 
1921     return STATUS_OK;
1922 }
1923 
1924 // Start ExternalCameraDeviceSession::BufferRequestThread functions
BufferRequestThread(std::weak_ptr<OutputThreadInterface> parent,std::shared_ptr<ICameraDeviceCallback> callbacks)1925 ExternalCameraDeviceSession::BufferRequestThread::BufferRequestThread(
1926         std::weak_ptr<OutputThreadInterface> parent,
1927         std::shared_ptr<ICameraDeviceCallback> callbacks)
1928     : mParent(parent), mCallbacks(callbacks) {}
1929 
requestBufferStart(const std::vector<HalStreamBuffer> & bufReqs)1930 int ExternalCameraDeviceSession::BufferRequestThread::requestBufferStart(
1931         const std::vector<HalStreamBuffer>& bufReqs) {
1932     if (bufReqs.empty()) {
1933         ALOGE("%s: bufReqs is empty!", __FUNCTION__);
1934         return -1;
1935     }
1936 
1937     {
1938         std::lock_guard<std::mutex> lk(mLock);
1939         if (mRequestingBuffer) {
1940             ALOGE("%s: BufferRequestThread does not support more than one concurrent request!",
1941                   __FUNCTION__);
1942             return -1;
1943         }
1944 
1945         mBufferReqs = bufReqs;
1946         mRequestingBuffer = true;
1947     }
1948     mRequestCond.notify_one();
1949     return 0;
1950 }
1951 
waitForBufferRequestDone(std::vector<HalStreamBuffer> * outBufReqs)1952 int ExternalCameraDeviceSession::BufferRequestThread::waitForBufferRequestDone(
1953         std::vector<HalStreamBuffer>* outBufReqs) {
1954     std::unique_lock<std::mutex> lk(mLock);
1955     if (!mRequestingBuffer) {
1956         ALOGE("%s: no pending buffer request!", __FUNCTION__);
1957         return -1;
1958     }
1959 
1960     if (mPendingReturnBufferReqs.empty()) {
1961         std::chrono::milliseconds timeout = std::chrono::milliseconds(kReqProcTimeoutMs);
1962         auto st = mRequestDoneCond.wait_for(lk, timeout);
1963         if (st == std::cv_status::timeout) {
1964             mRequestingBuffer = false;
1965             ALOGE("%s: wait for buffer request finish timeout!", __FUNCTION__);
1966             return -1;
1967         }
1968 
1969         if (mPendingReturnBufferReqs.empty()) {
1970             mRequestingBuffer = false;
1971             ALOGE("%s: cameraservice did not return any buffers!", __FUNCTION__);
1972             return -1;
1973         }
1974     }
1975     mRequestingBuffer = false;
1976     *outBufReqs = std::move(mPendingReturnBufferReqs);
1977     mPendingReturnBufferReqs.clear();
1978     return 0;
1979 }
1980 
waitForNextRequest()1981 void ExternalCameraDeviceSession::BufferRequestThread::waitForNextRequest() {
1982     ATRACE_CALL();
1983     std::unique_lock<std::mutex> lk(mLock);
1984     int waitTimes = 0;
1985     while (mBufferReqs.empty()) {
1986         if (exitPending()) {
1987             return;
1988         }
1989         auto timeout = std::chrono::milliseconds(kReqWaitTimeoutMs);
1990         auto st = mRequestCond.wait_for(lk, timeout);
1991         if (st == std::cv_status::timeout) {
1992             waitTimes++;
1993             if (waitTimes == kReqWaitTimesWarn) {
1994                 // BufferRequestThread just wait forever for new buffer request
1995                 // But it will print some periodic warning indicating it's waiting
1996                 ALOGV("%s: still waiting for new buffer request", __FUNCTION__);
1997                 waitTimes = 0;
1998             }
1999         }
2000     }
2001 
2002     // Fill in BufferRequest
2003     mHalBufferReqs.resize(mBufferReqs.size());
2004     for (size_t i = 0; i < mHalBufferReqs.size(); i++) {
2005         mHalBufferReqs[i].streamId = mBufferReqs[i].streamId;
2006         mHalBufferReqs[i].numBuffersRequested = 1;
2007     }
2008 }
2009 
threadLoop()2010 bool ExternalCameraDeviceSession::BufferRequestThread::threadLoop() {
2011     waitForNextRequest();
2012     if (exitPending()) {
2013         return false;
2014     }
2015 
2016     ATRACE_BEGIN("AIDL requestStreamBuffers");
2017     BufferRequestStatus status;
2018     std::vector<StreamBufferRet> bufRets;
2019     ScopedAStatus ret = mCallbacks->requestStreamBuffers(mHalBufferReqs, &bufRets, &status);
2020     if (!ret.isOk()) {
2021         ALOGE("%s: Transaction error: %d:%d", __FUNCTION__, ret.getExceptionCode(),
2022               ret.getServiceSpecificError());
2023         mBufferReqs.clear();
2024         mRequestDoneCond.notify_one();
2025         return false;
2026     }
2027 
2028     std::unique_lock<std::mutex> lk(mLock);
2029     if (status == BufferRequestStatus::OK || status == BufferRequestStatus::FAILED_PARTIAL) {
2030         if (bufRets.size() != mHalBufferReqs.size()) {
2031             ALOGE("%s: expect %zu buffer requests returned, only got %zu", __FUNCTION__,
2032                   mHalBufferReqs.size(), bufRets.size());
2033             mBufferReqs.clear();
2034             lk.unlock();
2035             mRequestDoneCond.notify_one();
2036             return false;
2037         }
2038 
2039         auto parent = mParent.lock();
2040         if (parent == nullptr) {
2041             ALOGE("%s: session has been disconnected!", __FUNCTION__);
2042             mBufferReqs.clear();
2043             lk.unlock();
2044             mRequestDoneCond.notify_one();
2045             return false;
2046         }
2047 
2048         std::vector<int> importedFences;
2049         importedFences.resize(bufRets.size());
2050         bool hasError = false;
2051         for (size_t i = 0; i < bufRets.size(); i++) {
2052             int streamId = bufRets[i].streamId;
2053             switch (bufRets[i].val.getTag()) {
2054                 case StreamBuffersVal::Tag::error:
2055                     continue;
2056                 case StreamBuffersVal::Tag::buffers: {
2057                     const std::vector<StreamBuffer>& hBufs =
2058                             bufRets[i].val.get<StreamBuffersVal::Tag::buffers>();
2059                     if (hBufs.size() != 1) {
2060                         ALOGE("%s: expect 1 buffer returned, got %zu!", __FUNCTION__, hBufs.size());
2061                         hasError = true;
2062                         break;
2063                     }
2064                     const StreamBuffer& hBuf = hBufs[0];
2065 
2066                     mBufferReqs[i].bufferId = hBuf.bufferId;
2067                     // TODO: create a batch import API so we don't need to lock/unlock mCbsLock
2068                     // repeatedly?
2069                     lk.unlock();
2070                     native_handle_t* h = makeFromAidl(hBuf.buffer);
2071                     Status s = parent->importBuffer(streamId, hBuf.bufferId, h,
2072                                                     /*out*/ &mBufferReqs[i].bufPtr);
2073                     native_handle_delete(h);
2074                     lk.lock();
2075 
2076                     if (s != Status::OK) {
2077                         ALOGE("%s: stream %d import buffer failed!", __FUNCTION__, streamId);
2078                         cleanupInflightFences(importedFences, i - 1);
2079                         hasError = true;
2080                         break;
2081                     }
2082                     h = makeFromAidl(hBuf.acquireFence);
2083                     if (!sHandleImporter.importFence(h, mBufferReqs[i].acquireFence)) {
2084                         ALOGE("%s: stream %d import fence failed!", __FUNCTION__, streamId);
2085                         cleanupInflightFences(importedFences, i - 1);
2086                         native_handle_delete(h);
2087                         hasError = true;
2088                         break;
2089                     }
2090                     native_handle_delete(h);
2091                     importedFences[i] = mBufferReqs[i].acquireFence;
2092                 } break;
2093                 default:
2094                     ALOGE("%s: Unknown StreamBuffersVal!", __FUNCTION__);
2095                     hasError = true;
2096                     break;
2097             }
2098             if (hasError) {
2099                 mBufferReqs.clear();
2100                 lk.unlock();
2101                 mRequestDoneCond.notify_one();
2102                 return true;
2103             }
2104         }
2105     } else {
2106         ALOGE("%s: requestStreamBuffers call failed!", __FUNCTION__);
2107         mBufferReqs.clear();
2108         lk.unlock();
2109         mRequestDoneCond.notify_one();
2110         return true;
2111     }
2112 
2113     mPendingReturnBufferReqs = std::move(mBufferReqs);
2114     mBufferReqs.clear();
2115 
2116     lk.unlock();
2117     mRequestDoneCond.notify_one();
2118     return true;
2119 }
2120 
2121 // End ExternalCameraDeviceSession::BufferRequestThread functions
2122 
2123 // Start ExternalCameraDeviceSession::OutputThread functions
2124 
OutputThread(std::weak_ptr<OutputThreadInterface> parent,CroppingType ct,const common::V1_0::helper::CameraMetadata & chars,std::shared_ptr<BufferRequestThread> bufReqThread)2125 ExternalCameraDeviceSession::OutputThread::OutputThread(
2126         std::weak_ptr<OutputThreadInterface> parent, CroppingType ct,
2127         const common::V1_0::helper::CameraMetadata& chars,
2128         std::shared_ptr<BufferRequestThread> bufReqThread)
2129     : mParent(parent),
2130       mCroppingType(ct),
2131       mCameraCharacteristics(chars),
2132       mBufferRequestThread(bufReqThread) {}
2133 
~OutputThread()2134 ExternalCameraDeviceSession::OutputThread::~OutputThread() {}
2135 
allocateIntermediateBuffers(const Size & v4lSize,const Size & thumbSize,const std::vector<Stream> & streams,uint32_t blobBufferSize)2136 Status ExternalCameraDeviceSession::OutputThread::allocateIntermediateBuffers(
2137         const Size& v4lSize, const Size& thumbSize, const std::vector<Stream>& streams,
2138         uint32_t blobBufferSize) {
2139     std::lock_guard<std::mutex> lk(mBufferLock);
2140     if (!mScaledYu12Frames.empty()) {
2141         ALOGE("%s: intermediate buffer pool has %zu inflight buffers! (expect 0)", __FUNCTION__,
2142               mScaledYu12Frames.size());
2143         return Status::INTERNAL_ERROR;
2144     }
2145 
2146     // Allocating intermediate YU12 frame
2147     if (mYu12Frame == nullptr || mYu12Frame->mWidth != v4lSize.width ||
2148         mYu12Frame->mHeight != v4lSize.height) {
2149         mYu12Frame.reset();
2150         mYu12Frame = std::make_shared<AllocatedFrame>(v4lSize.width, v4lSize.height);
2151         int ret = mYu12Frame->allocate(&mYu12FrameLayout);
2152         if (ret != 0) {
2153             ALOGE("%s: allocating YU12 frame failed!", __FUNCTION__);
2154             return Status::INTERNAL_ERROR;
2155         }
2156     }
2157 
2158     // Allocating intermediate YU12 thumbnail frame
2159     if (mYu12ThumbFrame == nullptr || mYu12ThumbFrame->mWidth != thumbSize.width ||
2160         mYu12ThumbFrame->mHeight != thumbSize.height) {
2161         mYu12ThumbFrame.reset();
2162         mYu12ThumbFrame = std::make_shared<AllocatedFrame>(thumbSize.width, thumbSize.height);
2163         int ret = mYu12ThumbFrame->allocate(&mYu12ThumbFrameLayout);
2164         if (ret != 0) {
2165             ALOGE("%s: allocating YU12 thumb frame failed!", __FUNCTION__);
2166             return Status::INTERNAL_ERROR;
2167         }
2168     }
2169 
2170     // Allocating scaled buffers
2171     for (const auto& stream : streams) {
2172         Size sz = {stream.width, stream.height};
2173         if (sz == v4lSize) {
2174             continue;  // Don't need an intermediate buffer same size as v4lBuffer
2175         }
2176         if (mIntermediateBuffers.count(sz) == 0) {
2177             // Create new intermediate buffer
2178             std::shared_ptr<AllocatedFrame> buf =
2179                     std::make_shared<AllocatedFrame>(stream.width, stream.height);
2180             int ret = buf->allocate();
2181             if (ret != 0) {
2182                 ALOGE("%s: allocating intermediate YU12 frame %dx%d failed!", __FUNCTION__,
2183                       stream.width, stream.height);
2184                 return Status::INTERNAL_ERROR;
2185             }
2186             mIntermediateBuffers[sz] = buf;
2187         }
2188     }
2189 
2190     // Remove unconfigured buffers
2191     auto it = mIntermediateBuffers.begin();
2192     while (it != mIntermediateBuffers.end()) {
2193         bool configured = false;
2194         auto sz = it->first;
2195         for (const auto& stream : streams) {
2196             if (stream.width == sz.width && stream.height == sz.height) {
2197                 configured = true;
2198                 break;
2199             }
2200         }
2201         if (configured) {
2202             it++;
2203         } else {
2204             it = mIntermediateBuffers.erase(it);
2205         }
2206     }
2207 
2208     // Allocate mute test pattern frame
2209     mMuteTestPatternFrame.resize(mYu12Frame->mWidth * mYu12Frame->mHeight * 3);
2210 
2211     mBlobBufferSize = blobBufferSize;
2212     return Status::OK;
2213 }
2214 
submitRequest(const std::shared_ptr<HalRequest> & req)2215 Status ExternalCameraDeviceSession::OutputThread::submitRequest(
2216         const std::shared_ptr<HalRequest>& req) {
2217     std::unique_lock<std::mutex> lk(mRequestListLock);
2218     mRequestList.push_back(req);
2219     lk.unlock();
2220     mRequestCond.notify_one();
2221     return Status::OK;
2222 }
2223 
flush()2224 void ExternalCameraDeviceSession::OutputThread::flush() {
2225     ATRACE_CALL();
2226     auto parent = mParent.lock();
2227     if (parent == nullptr) {
2228         ALOGE("%s: session has been disconnected!", __FUNCTION__);
2229         return;
2230     }
2231 
2232     std::unique_lock<std::mutex> lk(mRequestListLock);
2233     std::list<std::shared_ptr<HalRequest>> reqs = std::move(mRequestList);
2234     mRequestList.clear();
2235     if (mProcessingRequest) {
2236         auto timeout = std::chrono::seconds(kFlushWaitTimeoutSec);
2237         auto st = mRequestDoneCond.wait_for(lk, timeout);
2238         if (st == std::cv_status::timeout) {
2239             ALOGE("%s: wait for inflight request finish timeout!", __FUNCTION__);
2240         }
2241     }
2242 
2243     ALOGV("%s: flushing inflight requests", __FUNCTION__);
2244     lk.unlock();
2245     for (const auto& req : reqs) {
2246         parent->processCaptureRequestError(req);
2247     }
2248 }
2249 
dump(int fd)2250 void ExternalCameraDeviceSession::OutputThread::dump(int fd) {
2251     std::lock_guard<std::mutex> lk(mRequestListLock);
2252     if (mProcessingRequest) {
2253         dprintf(fd, "OutputThread processing frame %d\n", mProcessingFrameNumber);
2254     } else {
2255         dprintf(fd, "OutputThread not processing any frames\n");
2256     }
2257     dprintf(fd, "OutputThread request list contains frame: ");
2258     for (const auto& req : mRequestList) {
2259         dprintf(fd, "%d, ", req->frameNumber);
2260     }
2261     dprintf(fd, "\n");
2262 }
2263 
setExifMakeModel(const std::string & make,const std::string & model)2264 void ExternalCameraDeviceSession::OutputThread::setExifMakeModel(const std::string& make,
2265                                                                  const std::string& model) {
2266     mExifMake = make;
2267     mExifModel = model;
2268 }
2269 
2270 std::list<std::shared_ptr<HalRequest>>
switchToOffline()2271 ExternalCameraDeviceSession::OutputThread::switchToOffline() {
2272     ATRACE_CALL();
2273     auto parent = mParent.lock();
2274     if (parent == nullptr) {
2275         ALOGE("%s: session has been disconnected!", __FUNCTION__);
2276         return {};
2277     }
2278 
2279     std::unique_lock<std::mutex> lk(mRequestListLock);
2280     std::list<std::shared_ptr<HalRequest>> reqs = std::move(mRequestList);
2281     mRequestList.clear();
2282     if (mProcessingRequest) {
2283         auto timeout = std::chrono::seconds(kFlushWaitTimeoutSec);
2284         auto st = mRequestDoneCond.wait_for(lk, timeout);
2285         if (st == std::cv_status::timeout) {
2286             ALOGE("%s: wait for inflight request finish timeout!", __FUNCTION__);
2287         }
2288     }
2289     lk.unlock();
2290     clearIntermediateBuffers();
2291     ALOGV("%s: returning %zu request for offline processing", __FUNCTION__, reqs.size());
2292     return reqs;
2293 }
2294 
requestBufferStart(const std::vector<HalStreamBuffer> & bufs)2295 int ExternalCameraDeviceSession::OutputThread::requestBufferStart(
2296         const std::vector<HalStreamBuffer>& bufs) {
2297     if (mBufferRequestThread == nullptr) {
2298         return 0;
2299     }
2300     return mBufferRequestThread->requestBufferStart(bufs);
2301 }
2302 
waitForBufferRequestDone(std::vector<HalStreamBuffer> * outBufs)2303 int ExternalCameraDeviceSession::OutputThread::waitForBufferRequestDone(
2304         std::vector<HalStreamBuffer>* outBufs) {
2305     if (mBufferRequestThread == nullptr) {
2306         return 0;
2307     }
2308     return mBufferRequestThread->waitForBufferRequestDone(outBufs);
2309 }
2310 
waitForNextRequest(std::shared_ptr<HalRequest> * out)2311 void ExternalCameraDeviceSession::OutputThread::waitForNextRequest(
2312         std::shared_ptr<HalRequest>* out) {
2313     ATRACE_CALL();
2314     if (out == nullptr) {
2315         ALOGE("%s: out is null", __FUNCTION__);
2316         return;
2317     }
2318 
2319     std::unique_lock<std::mutex> lk(mRequestListLock);
2320     int waitTimes = 0;
2321     while (mRequestList.empty()) {
2322         if (exitPending()) {
2323             return;
2324         }
2325         auto timeout = std::chrono::milliseconds(kReqWaitTimeoutMs);
2326         auto st = mRequestCond.wait_for(lk, timeout);
2327         if (st == std::cv_status::timeout) {
2328             waitTimes++;
2329             if (waitTimes == kReqWaitTimesMax) {
2330                 // no new request, return
2331                 return;
2332             }
2333         }
2334     }
2335     *out = mRequestList.front();
2336     mRequestList.pop_front();
2337     mProcessingRequest = true;
2338     mProcessingFrameNumber = (*out)->frameNumber;
2339 }
2340 
signalRequestDone()2341 void ExternalCameraDeviceSession::OutputThread::signalRequestDone() {
2342     std::unique_lock<std::mutex> lk(mRequestListLock);
2343     mProcessingRequest = false;
2344     mProcessingFrameNumber = 0;
2345     lk.unlock();
2346     mRequestDoneCond.notify_one();
2347 }
2348 
cropAndScaleLocked(std::shared_ptr<AllocatedFrame> & in,const Size & outSz,YCbCrLayout * out)2349 int ExternalCameraDeviceSession::OutputThread::cropAndScaleLocked(
2350         std::shared_ptr<AllocatedFrame>& in, const Size& outSz, YCbCrLayout* out) {
2351     Size inSz = {in->mWidth, in->mHeight};
2352 
2353     int ret;
2354     if (inSz == outSz) {
2355         ret = in->getLayout(out);
2356         if (ret != 0) {
2357             ALOGE("%s: failed to get input image layout", __FUNCTION__);
2358             return ret;
2359         }
2360         return ret;
2361     }
2362 
2363     // Cropping to output aspect ratio
2364     IMapper::Rect inputCrop;
2365     ret = getCropRect(mCroppingType, inSz, outSz, &inputCrop);
2366     if (ret != 0) {
2367         ALOGE("%s: failed to compute crop rect for output size %dx%d", __FUNCTION__, outSz.width,
2368               outSz.height);
2369         return ret;
2370     }
2371 
2372     YCbCrLayout croppedLayout;
2373     ret = in->getCroppedLayout(inputCrop, &croppedLayout);
2374     if (ret != 0) {
2375         ALOGE("%s: failed to crop input image %dx%d to output size %dx%d", __FUNCTION__, inSz.width,
2376               inSz.height, outSz.width, outSz.height);
2377         return ret;
2378     }
2379 
2380     if ((mCroppingType == VERTICAL && inSz.width == outSz.width) ||
2381         (mCroppingType == HORIZONTAL && inSz.height == outSz.height)) {
2382         // No scale is needed
2383         *out = croppedLayout;
2384         return 0;
2385     }
2386 
2387     auto it = mScaledYu12Frames.find(outSz);
2388     std::shared_ptr<AllocatedFrame> scaledYu12Buf;
2389     if (it != mScaledYu12Frames.end()) {
2390         scaledYu12Buf = it->second;
2391     } else {
2392         it = mIntermediateBuffers.find(outSz);
2393         if (it == mIntermediateBuffers.end()) {
2394             ALOGE("%s: failed to find intermediate buffer size %dx%d", __FUNCTION__, outSz.width,
2395                   outSz.height);
2396             return -1;
2397         }
2398         scaledYu12Buf = it->second;
2399     }
2400     // Scale
2401     YCbCrLayout outLayout;
2402     ret = scaledYu12Buf->getLayout(&outLayout);
2403     if (ret != 0) {
2404         ALOGE("%s: failed to get output buffer layout", __FUNCTION__);
2405         return ret;
2406     }
2407 
2408     ret = libyuv::I420Scale(
2409             static_cast<uint8_t*>(croppedLayout.y), croppedLayout.yStride,
2410             static_cast<uint8_t*>(croppedLayout.cb), croppedLayout.cStride,
2411             static_cast<uint8_t*>(croppedLayout.cr), croppedLayout.cStride, inputCrop.width,
2412             inputCrop.height, static_cast<uint8_t*>(outLayout.y), outLayout.yStride,
2413             static_cast<uint8_t*>(outLayout.cb), outLayout.cStride,
2414             static_cast<uint8_t*>(outLayout.cr), outLayout.cStride, outSz.width, outSz.height,
2415             // TODO: b/72261744 see if we can use better filter without losing too much perf
2416             libyuv::FilterMode::kFilterNone);
2417 
2418     if (ret != 0) {
2419         ALOGE("%s: failed to scale buffer from %dx%d to %dx%d. Ret %d", __FUNCTION__,
2420               inputCrop.width, inputCrop.height, outSz.width, outSz.height, ret);
2421         return ret;
2422     }
2423 
2424     *out = outLayout;
2425     mScaledYu12Frames.insert({outSz, scaledYu12Buf});
2426     return 0;
2427 }
2428 
cropAndScaleThumbLocked(std::shared_ptr<AllocatedFrame> & in,const Size & outSz,YCbCrLayout * out)2429 int ExternalCameraDeviceSession::OutputThread::cropAndScaleThumbLocked(
2430         std::shared_ptr<AllocatedFrame>& in, const Size& outSz, YCbCrLayout* out) {
2431     Size inSz{in->mWidth, in->mHeight};
2432 
2433     if ((outSz.width * outSz.height) > (mYu12ThumbFrame->mWidth * mYu12ThumbFrame->mHeight)) {
2434         ALOGE("%s: Requested thumbnail size too big (%d,%d) > (%d,%d)", __FUNCTION__, outSz.width,
2435               outSz.height, mYu12ThumbFrame->mWidth, mYu12ThumbFrame->mHeight);
2436         return -1;
2437     }
2438 
2439     int ret;
2440 
2441     /* This will crop-and-zoom the input YUV frame to the thumbnail size
2442      * Based on the following logic:
2443      *  1) Square pixels come in, square pixels come out, therefore single
2444      *  scale factor is computed to either make input bigger or smaller
2445      *  depending on if we are upscaling or downscaling
2446      *  2) That single scale factor would either make height too tall or width
2447      *  too wide so we need to crop the input either horizontally or vertically
2448      *  but not both
2449      */
2450 
2451     /* Convert the input and output dimensions into floats for ease of math */
2452     float fWin = static_cast<float>(inSz.width);
2453     float fHin = static_cast<float>(inSz.height);
2454     float fWout = static_cast<float>(outSz.width);
2455     float fHout = static_cast<float>(outSz.height);
2456 
2457     /* Compute the one scale factor from (1) above, it will be the smaller of
2458      * the two possibilities. */
2459     float scaleFactor = std::min(fHin / fHout, fWin / fWout);
2460 
2461     /* Since we are crop-and-zooming (as opposed to letter/pillar boxing) we can
2462      * simply multiply the output by our scaleFactor to get the cropped input
2463      * size. Note that at least one of {fWcrop, fHcrop} is going to wind up
2464      * being {fWin, fHin} respectively because fHout or fWout cancels out the
2465      * scaleFactor calculation above.
2466      *
2467      * Specifically:
2468      *  if ( fHin / fHout ) < ( fWin / fWout ) we crop the sides off
2469      * input, in which case
2470      *    scaleFactor = fHin / fHout
2471      *    fWcrop = fHin / fHout * fWout
2472      *    fHcrop = fHin
2473      *
2474      * Note that fWcrop <= fWin ( because ( fHin / fHout ) * fWout < fWin, which
2475      * is just the inequality above with both sides multiplied by fWout
2476      *
2477      * on the other hand if ( fWin / fWout ) < ( fHin / fHout) we crop the top
2478      * and the bottom off of input, and
2479      *    scaleFactor = fWin / fWout
2480      *    fWcrop = fWin
2481      *    fHCrop = fWin / fWout * fHout
2482      */
2483     float fWcrop = scaleFactor * fWout;
2484     float fHcrop = scaleFactor * fHout;
2485 
2486     /* Convert to integer and truncate to an even number */
2487     Size cropSz = {.width = 2 * static_cast<int32_t>(fWcrop / 2.0f),
2488                    .height = 2 * static_cast<int32_t>(fHcrop / 2.0f)};
2489 
2490     /* Convert to a centered rectange with even top/left */
2491     IMapper::Rect inputCrop{.left = 2 * static_cast<int32_t>((inSz.width - cropSz.width) / 4),
2492                             .top = 2 * static_cast<int32_t>((inSz.height - cropSz.height) / 4),
2493                             .width = static_cast<int32_t>(cropSz.width),
2494                             .height = static_cast<int32_t>(cropSz.height)};
2495 
2496     if ((inputCrop.top < 0) || (inputCrop.top >= static_cast<int32_t>(inSz.height)) ||
2497         (inputCrop.left < 0) || (inputCrop.left >= static_cast<int32_t>(inSz.width)) ||
2498         (inputCrop.width <= 0) ||
2499         (inputCrop.width + inputCrop.left > static_cast<int32_t>(inSz.width)) ||
2500         (inputCrop.height <= 0) ||
2501         (inputCrop.height + inputCrop.top > static_cast<int32_t>(inSz.height))) {
2502         ALOGE("%s: came up with really wrong crop rectangle", __FUNCTION__);
2503         ALOGE("%s: input layout %dx%d to for output size %dx%d", __FUNCTION__, inSz.width,
2504               inSz.height, outSz.width, outSz.height);
2505         ALOGE("%s: computed input crop +%d,+%d %dx%d", __FUNCTION__, inputCrop.left, inputCrop.top,
2506               inputCrop.width, inputCrop.height);
2507         return -1;
2508     }
2509 
2510     YCbCrLayout inputLayout;
2511     ret = in->getCroppedLayout(inputCrop, &inputLayout);
2512     if (ret != 0) {
2513         ALOGE("%s: failed to crop input layout %dx%d to for output size %dx%d", __FUNCTION__,
2514               inSz.width, inSz.height, outSz.width, outSz.height);
2515         ALOGE("%s: computed input crop +%d,+%d %dx%d", __FUNCTION__, inputCrop.left, inputCrop.top,
2516               inputCrop.width, inputCrop.height);
2517         return ret;
2518     }
2519     ALOGV("%s: crop input layout %dx%d to for output size %dx%d", __FUNCTION__, inSz.width,
2520           inSz.height, outSz.width, outSz.height);
2521     ALOGV("%s: computed input crop +%d,+%d %dx%d", __FUNCTION__, inputCrop.left, inputCrop.top,
2522           inputCrop.width, inputCrop.height);
2523 
2524     // Scale
2525     YCbCrLayout outFullLayout;
2526 
2527     ret = mYu12ThumbFrame->getLayout(&outFullLayout);
2528     if (ret != 0) {
2529         ALOGE("%s: failed to get output buffer layout", __FUNCTION__);
2530         return ret;
2531     }
2532 
2533     ret = libyuv::I420Scale(static_cast<uint8_t*>(inputLayout.y), inputLayout.yStride,
2534                             static_cast<uint8_t*>(inputLayout.cb), inputLayout.cStride,
2535                             static_cast<uint8_t*>(inputLayout.cr), inputLayout.cStride,
2536                             inputCrop.width, inputCrop.height,
2537                             static_cast<uint8_t*>(outFullLayout.y), outFullLayout.yStride,
2538                             static_cast<uint8_t*>(outFullLayout.cb), outFullLayout.cStride,
2539                             static_cast<uint8_t*>(outFullLayout.cr), outFullLayout.cStride,
2540                             outSz.width, outSz.height, libyuv::FilterMode::kFilterNone);
2541 
2542     if (ret != 0) {
2543         ALOGE("%s: failed to scale buffer from %dx%d to %dx%d. Ret %d", __FUNCTION__,
2544               inputCrop.width, inputCrop.height, outSz.width, outSz.height, ret);
2545         return ret;
2546     }
2547 
2548     *out = outFullLayout;
2549     return 0;
2550 }
2551 
createJpegLocked(HalStreamBuffer & halBuf,const common::V1_0::helper::CameraMetadata & setting)2552 int ExternalCameraDeviceSession::OutputThread::createJpegLocked(
2553         HalStreamBuffer& halBuf, const common::V1_0::helper::CameraMetadata& setting) {
2554     ATRACE_CALL();
2555     int ret;
2556     auto lfail = [&](auto... args) {
2557         ALOGE(args...);
2558 
2559         return 1;
2560     };
2561     auto parent = mParent.lock();
2562     if (parent == nullptr) {
2563         ALOGE("%s: session has been disconnected!", __FUNCTION__);
2564         return 1;
2565     }
2566 
2567     ALOGV("%s: HAL buffer sid: %d bid: %" PRIu64 " w: %u h: %u", __FUNCTION__, halBuf.streamId,
2568           static_cast<uint64_t>(halBuf.bufferId), halBuf.width, halBuf.height);
2569     ALOGV("%s: HAL buffer fmt: %x usage: %" PRIx64 " ptr: %p", __FUNCTION__, halBuf.format,
2570           static_cast<uint64_t>(halBuf.usage), halBuf.bufPtr);
2571     ALOGV("%s: YV12 buffer %d x %d", __FUNCTION__, mYu12Frame->mWidth, mYu12Frame->mHeight);
2572 
2573     int jpegQuality, thumbQuality;
2574     Size thumbSize;
2575     bool outputThumbnail = true;
2576 
2577     if (setting.exists(ANDROID_JPEG_QUALITY)) {
2578         camera_metadata_ro_entry entry = setting.find(ANDROID_JPEG_QUALITY);
2579         jpegQuality = entry.data.u8[0];
2580     } else {
2581         return lfail("%s: ANDROID_JPEG_QUALITY not set", __FUNCTION__);
2582     }
2583 
2584     if (setting.exists(ANDROID_JPEG_THUMBNAIL_QUALITY)) {
2585         camera_metadata_ro_entry entry = setting.find(ANDROID_JPEG_THUMBNAIL_QUALITY);
2586         thumbQuality = entry.data.u8[0];
2587     } else {
2588         return lfail("%s: ANDROID_JPEG_THUMBNAIL_QUALITY not set", __FUNCTION__);
2589     }
2590 
2591     if (setting.exists(ANDROID_JPEG_THUMBNAIL_SIZE)) {
2592         camera_metadata_ro_entry entry = setting.find(ANDROID_JPEG_THUMBNAIL_SIZE);
2593         thumbSize = Size{.width = entry.data.i32[0], .height = entry.data.i32[1]};
2594         if (thumbSize.width == 0 && thumbSize.height == 0) {
2595             outputThumbnail = false;
2596         }
2597     } else {
2598         return lfail("%s: ANDROID_JPEG_THUMBNAIL_SIZE not set", __FUNCTION__);
2599     }
2600 
2601     /* Cropped and scaled YU12 buffer for main and thumbnail */
2602     YCbCrLayout yu12Main;
2603     Size jpegSize{halBuf.width, halBuf.height};
2604 
2605     /* Compute temporary buffer sizes accounting for the following:
2606      * thumbnail can't exceed APP1 size of 64K
2607      * main image needs to hold APP1, headers, and at most a poorly
2608      * compressed image */
2609     const ssize_t maxThumbCodeSize = 64 * 1024;
2610     const ssize_t maxJpegCodeSize =
2611             mBlobBufferSize == 0 ? parent->getJpegBufferSize(jpegSize.width, jpegSize.height)
2612                                  : mBlobBufferSize;
2613 
2614     /* Check that getJpegBufferSize did not return an error */
2615     if (maxJpegCodeSize < 0) {
2616         return lfail("%s: getJpegBufferSize returned %zd", __FUNCTION__, maxJpegCodeSize);
2617     }
2618 
2619     /* Hold actual thumbnail and main image code sizes */
2620     size_t thumbCodeSize = 0, jpegCodeSize = 0;
2621     /* Temporary thumbnail code buffer */
2622     std::vector<uint8_t> thumbCode(outputThumbnail ? maxThumbCodeSize : 0);
2623 
2624     YCbCrLayout yu12Thumb;
2625     if (outputThumbnail) {
2626         ret = cropAndScaleThumbLocked(mYu12Frame, thumbSize, &yu12Thumb);
2627 
2628         if (ret != 0) {
2629             return lfail("%s: crop and scale thumbnail failed!", __FUNCTION__);
2630         }
2631     }
2632 
2633     /* Scale and crop main jpeg */
2634     ret = cropAndScaleLocked(mYu12Frame, jpegSize, &yu12Main);
2635 
2636     if (ret != 0) {
2637         return lfail("%s: crop and scale main failed!", __FUNCTION__);
2638     }
2639 
2640     /* Encode the thumbnail image */
2641     if (outputThumbnail) {
2642         ret = encodeJpegYU12(thumbSize, yu12Thumb, thumbQuality, 0, 0, &thumbCode[0],
2643                              maxThumbCodeSize, thumbCodeSize);
2644 
2645         if (ret != 0) {
2646             return lfail("%s: thumbnail encodeJpegYU12 failed with %d", __FUNCTION__, ret);
2647         }
2648     }
2649 
2650     /* Combine camera characteristics with request settings to form EXIF
2651      * metadata */
2652     common::V1_0::helper::CameraMetadata meta(mCameraCharacteristics);
2653     meta.append(setting);
2654 
2655     /* Generate EXIF object */
2656     std::unique_ptr<ExifUtils> utils(ExifUtils::create());
2657     /* Make sure it's initialized */
2658     utils->initialize();
2659 
2660     utils->setFromMetadata(meta, jpegSize.width, jpegSize.height);
2661     utils->setMake(mExifMake);
2662     utils->setModel(mExifModel);
2663 
2664     ret = utils->generateApp1(outputThumbnail ? &thumbCode[0] : nullptr, thumbCodeSize);
2665 
2666     if (!ret) {
2667         return lfail("%s: generating APP1 failed", __FUNCTION__);
2668     }
2669 
2670     /* Get internal buffer */
2671     size_t exifDataSize = utils->getApp1Length();
2672     const uint8_t* exifData = utils->getApp1Buffer();
2673 
2674     /* Lock the HAL jpeg code buffer */
2675     void* bufPtr = sHandleImporter.lock(*(halBuf.bufPtr), static_cast<uint64_t>(halBuf.usage),
2676                                         maxJpegCodeSize);
2677 
2678     if (!bufPtr) {
2679         return lfail("%s: could not lock %zu bytes", __FUNCTION__, maxJpegCodeSize);
2680     }
2681 
2682     /* Encode the main jpeg image */
2683     ret = encodeJpegYU12(jpegSize, yu12Main, jpegQuality, exifData, exifDataSize, bufPtr,
2684                          maxJpegCodeSize, jpegCodeSize);
2685 
2686     /* TODO: Not sure this belongs here, maybe better to pass jpegCodeSize out
2687      * and do this when returning buffer to parent */
2688     CameraBlob blob{CameraBlobId::JPEG, static_cast<int32_t>(jpegCodeSize)};
2689     void* blobDst = reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(bufPtr) + maxJpegCodeSize -
2690                                             sizeof(CameraBlob));
2691     memcpy(blobDst, &blob, sizeof(CameraBlob));
2692 
2693     /* Unlock the HAL jpeg code buffer */
2694     int relFence = sHandleImporter.unlock(*(halBuf.bufPtr));
2695     if (relFence >= 0) {
2696         halBuf.acquireFence = relFence;
2697     }
2698 
2699     /* Check if our JPEG actually succeeded */
2700     if (ret != 0) {
2701         return lfail("%s: encodeJpegYU12 failed with %d", __FUNCTION__, ret);
2702     }
2703 
2704     ALOGV("%s: encoded JPEG (ret:%d) with Q:%d max size: %zu", __FUNCTION__, ret, jpegQuality,
2705           maxJpegCodeSize);
2706 
2707     return 0;
2708 }
2709 
clearIntermediateBuffers()2710 void ExternalCameraDeviceSession::OutputThread::clearIntermediateBuffers() {
2711     std::lock_guard<std::mutex> lk(mBufferLock);
2712     mYu12Frame.reset();
2713     mYu12ThumbFrame.reset();
2714     mIntermediateBuffers.clear();
2715     mMuteTestPatternFrame.clear();
2716     mBlobBufferSize = 0;
2717 }
2718 
threadLoop()2719 bool ExternalCameraDeviceSession::OutputThread::threadLoop() {
2720     std::shared_ptr<HalRequest> req;
2721     auto parent = mParent.lock();
2722     if (parent == nullptr) {
2723         ALOGE("%s: session has been disconnected!", __FUNCTION__);
2724         return false;
2725     }
2726 
2727     // TODO: maybe we need to setup a sensor thread to dq/enq v4l frames
2728     //       regularly to prevent v4l buffer queue filled with stale buffers
2729     //       when app doesn't program a preview request
2730     waitForNextRequest(&req);
2731     if (req == nullptr) {
2732         // No new request, wait again
2733         return true;
2734     }
2735 
2736     auto onDeviceError = [&](auto... args) {
2737         ALOGE(args...);
2738         parent->notifyError(req->frameNumber, /*stream*/ -1, ErrorCode::ERROR_DEVICE);
2739         signalRequestDone();
2740         return false;
2741     };
2742 
2743     if (req->frameIn->mFourcc != V4L2_PIX_FMT_MJPEG && req->frameIn->mFourcc != V4L2_PIX_FMT_Z16) {
2744         return onDeviceError("%s: do not support V4L2 format %c%c%c%c", __FUNCTION__,
2745                              req->frameIn->mFourcc & 0xFF, (req->frameIn->mFourcc >> 8) & 0xFF,
2746                              (req->frameIn->mFourcc >> 16) & 0xFF,
2747                              (req->frameIn->mFourcc >> 24) & 0xFF);
2748     }
2749 
2750     int res = requestBufferStart(req->buffers);
2751     if (res != 0) {
2752         ALOGE("%s: send BufferRequest failed! res %d", __FUNCTION__, res);
2753         return onDeviceError("%s: failed to send buffer request!", __FUNCTION__);
2754     }
2755 
2756     std::unique_lock<std::mutex> lk(mBufferLock);
2757     // Convert input V4L2 frame to YU12 of the same size
2758     // TODO: see if we can save some computation by converting to YV12 here
2759     uint8_t* inData;
2760     size_t inDataSize;
2761     if (req->frameIn->getData(&inData, &inDataSize) != 0) {
2762         lk.unlock();
2763         return onDeviceError("%s: V4L2 buffer map failed", __FUNCTION__);
2764     }
2765 
2766     // Process camera mute state
2767     auto testPatternMode = req->setting.find(ANDROID_SENSOR_TEST_PATTERN_MODE);
2768     if (testPatternMode.count == 1) {
2769         if (mCameraMuted != (testPatternMode.data.u8[0] != ANDROID_SENSOR_TEST_PATTERN_MODE_OFF)) {
2770             mCameraMuted = !mCameraMuted;
2771             // Get solid color for test pattern, if any was set
2772             if (testPatternMode.data.u8[0] == ANDROID_SENSOR_TEST_PATTERN_MODE_SOLID_COLOR) {
2773                 auto entry = req->setting.find(ANDROID_SENSOR_TEST_PATTERN_DATA);
2774                 if (entry.count == 4) {
2775                     // Update the mute frame if the pattern color has changed
2776                     if (memcmp(entry.data.i32, mTestPatternData, sizeof(mTestPatternData)) != 0) {
2777                         memcpy(mTestPatternData, entry.data.i32, sizeof(mTestPatternData));
2778                         // Fill the mute frame with the solid color, use only 8 MSB of RGGB as RGB
2779                         for (int i = 0; i < mMuteTestPatternFrame.size(); i += 3) {
2780                             mMuteTestPatternFrame[i] = entry.data.i32[0] >> 24;
2781                             mMuteTestPatternFrame[i + 1] = entry.data.i32[1] >> 24;
2782                             mMuteTestPatternFrame[i + 2] = entry.data.i32[3] >> 24;
2783                         }
2784                     }
2785                 }
2786             }
2787         }
2788     }
2789 
2790     // TODO: in some special case maybe we can decode jpg directly to gralloc output?
2791     if (req->frameIn->mFourcc == V4L2_PIX_FMT_MJPEG) {
2792         ATRACE_BEGIN("MJPGtoI420");
2793         res = 0;
2794         if (mCameraMuted) {
2795             res = libyuv::ConvertToI420(
2796                     mMuteTestPatternFrame.data(), mMuteTestPatternFrame.size(),
2797                     static_cast<uint8_t*>(mYu12FrameLayout.y), mYu12FrameLayout.yStride,
2798                     static_cast<uint8_t*>(mYu12FrameLayout.cb), mYu12FrameLayout.cStride,
2799                     static_cast<uint8_t*>(mYu12FrameLayout.cr), mYu12FrameLayout.cStride, 0, 0,
2800                     mYu12Frame->mWidth, mYu12Frame->mHeight, mYu12Frame->mWidth,
2801                     mYu12Frame->mHeight, libyuv::kRotate0, libyuv::FOURCC_RAW);
2802         } else {
2803             res = libyuv::MJPGToI420(
2804                     inData, inDataSize, static_cast<uint8_t*>(mYu12FrameLayout.y),
2805                     mYu12FrameLayout.yStride, static_cast<uint8_t*>(mYu12FrameLayout.cb),
2806                     mYu12FrameLayout.cStride, static_cast<uint8_t*>(mYu12FrameLayout.cr),
2807                     mYu12FrameLayout.cStride, mYu12Frame->mWidth, mYu12Frame->mHeight,
2808                     mYu12Frame->mWidth, mYu12Frame->mHeight);
2809         }
2810         ATRACE_END();
2811 
2812         if (res != 0) {
2813             // For some webcam, the first few V4L2 frames might be malformed...
2814             ALOGE("%s: Convert V4L2 frame to YU12 failed! res %d", __FUNCTION__, res);
2815 
2816             ATRACE_BEGIN("Wait for BufferRequest done");
2817             res = waitForBufferRequestDone(&req->buffers);
2818             ATRACE_END();
2819 
2820             lk.unlock();
2821             Status st = parent->processCaptureRequestError(req);
2822             if (st != Status::OK) {
2823                 return onDeviceError("%s: failed to process capture request error!", __FUNCTION__);
2824             }
2825             signalRequestDone();
2826             return true;
2827         }
2828     }
2829 
2830     ATRACE_BEGIN("Wait for BufferRequest done");
2831     res = waitForBufferRequestDone(&req->buffers);
2832     ATRACE_END();
2833 
2834     if (res != 0) {
2835         // HAL buffer management buffer request can fail
2836         ALOGE("%s: wait for BufferRequest done failed! res %d", __FUNCTION__, res);
2837         lk.unlock();
2838         Status st = parent->processCaptureRequestError(req);
2839         if (st != Status::OK) {
2840             return onDeviceError("%s: failed to process capture request error!", __FUNCTION__);
2841         }
2842         signalRequestDone();
2843         return true;
2844     }
2845 
2846     ALOGV("%s processing new request", __FUNCTION__);
2847     const int kSyncWaitTimeoutMs = 500;
2848     for (auto& halBuf : req->buffers) {
2849         if (*(halBuf.bufPtr) == nullptr) {
2850             ALOGW("%s: buffer for stream %d missing", __FUNCTION__, halBuf.streamId);
2851             halBuf.fenceTimeout = true;
2852         } else if (halBuf.acquireFence >= 0) {
2853             int ret = sync_wait(halBuf.acquireFence, kSyncWaitTimeoutMs);
2854             if (ret) {
2855                 halBuf.fenceTimeout = true;
2856             } else {
2857                 ::close(halBuf.acquireFence);
2858                 halBuf.acquireFence = -1;
2859             }
2860         }
2861 
2862         if (halBuf.fenceTimeout) {
2863             continue;
2864         }
2865 
2866         // Gralloc lockYCbCr the buffer
2867         switch (halBuf.format) {
2868             case PixelFormat::BLOB: {
2869                 int ret = createJpegLocked(halBuf, req->setting);
2870 
2871                 if (ret != 0) {
2872                     lk.unlock();
2873                     return onDeviceError("%s: createJpegLocked failed with %d", __FUNCTION__, ret);
2874                 }
2875             } break;
2876             case PixelFormat::Y16: {
2877                 void* outLayout = sHandleImporter.lock(
2878                         *(halBuf.bufPtr), static_cast<uint64_t>(halBuf.usage), inDataSize);
2879 
2880                 std::memcpy(outLayout, inData, inDataSize);
2881 
2882                 int relFence = sHandleImporter.unlock(*(halBuf.bufPtr));
2883                 if (relFence >= 0) {
2884                     halBuf.acquireFence = relFence;
2885                 }
2886             } break;
2887             case PixelFormat::YCBCR_420_888:
2888             case PixelFormat::YV12: {
2889                 android::Rect outRect{0, 0, static_cast<int32_t>(halBuf.width),
2890                                       static_cast<int32_t>(halBuf.height)};
2891                 android_ycbcr result = sHandleImporter.lockYCbCr(
2892                         *(halBuf.bufPtr), static_cast<uint64_t>(halBuf.usage), outRect);
2893                 ALOGV("%s: outLayout y %p cb %p cr %p y_str %zu c_str %zu c_step %zu", __FUNCTION__,
2894                       result.y, result.cb, result.cr, result.ystride, result.cstride,
2895                       result.chroma_step);
2896                 if (result.ystride > UINT32_MAX || result.cstride > UINT32_MAX ||
2897                     result.chroma_step > UINT32_MAX) {
2898                     return onDeviceError("%s: lockYCbCr failed. Unexpected values!", __FUNCTION__);
2899                 }
2900                 YCbCrLayout outLayout = {.y = result.y,
2901                                          .cb = result.cb,
2902                                          .cr = result.cr,
2903                                          .yStride = static_cast<uint32_t>(result.ystride),
2904                                          .cStride = static_cast<uint32_t>(result.cstride),
2905                                          .chromaStep = static_cast<uint32_t>(result.chroma_step)};
2906 
2907                 // Convert to output buffer size/format
2908                 uint32_t outputFourcc = getFourCcFromLayout(outLayout);
2909                 ALOGV("%s: converting to format %c%c%c%c", __FUNCTION__, outputFourcc & 0xFF,
2910                       (outputFourcc >> 8) & 0xFF, (outputFourcc >> 16) & 0xFF,
2911                       (outputFourcc >> 24) & 0xFF);
2912 
2913                 YCbCrLayout cropAndScaled;
2914                 ATRACE_BEGIN("cropAndScaleLocked");
2915                 int ret = cropAndScaleLocked(mYu12Frame, Size{halBuf.width, halBuf.height},
2916                                              &cropAndScaled);
2917                 ATRACE_END();
2918                 if (ret != 0) {
2919                     lk.unlock();
2920                     return onDeviceError("%s: crop and scale failed!", __FUNCTION__);
2921                 }
2922 
2923                 Size sz{halBuf.width, halBuf.height};
2924                 ATRACE_BEGIN("formatConvert");
2925                 ret = formatConvert(cropAndScaled, outLayout, sz, outputFourcc);
2926                 ATRACE_END();
2927                 if (ret != 0) {
2928                     lk.unlock();
2929                     return onDeviceError("%s: format conversion failed!", __FUNCTION__);
2930                 }
2931                 int relFence = sHandleImporter.unlock(*(halBuf.bufPtr));
2932                 if (relFence >= 0) {
2933                     halBuf.acquireFence = relFence;
2934                 }
2935             } break;
2936             default:
2937                 lk.unlock();
2938                 return onDeviceError("%s: unknown output format %x", __FUNCTION__, halBuf.format);
2939         }
2940     }  // for each buffer
2941     mScaledYu12Frames.clear();
2942 
2943     // Don't hold the lock while calling back to parent
2944     lk.unlock();
2945     Status st = parent->processCaptureResult(req);
2946     if (st != Status::OK) {
2947         return onDeviceError("%s: failed to process capture result!", __FUNCTION__);
2948     }
2949     signalRequestDone();
2950     return true;
2951 }
2952 
2953 // End ExternalCameraDeviceSession::OutputThread functions
2954 
2955 }  // namespace implementation
2956 }  // namespace device
2957 }  // namespace camera
2958 }  // namespace hardware
2959 }  // namespace android
2960