1 /* 2 * Copyright (C) 2022 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef HARDWARE_INTERFACES_CAMERA_DEVICE_DEFAULT_EXTERNALCAMERADEVICESESSION_H_ 18 #define HARDWARE_INTERFACES_CAMERA_DEVICE_DEFAULT_EXTERNALCAMERADEVICESESSION_H_ 19 20 #include <ExternalCameraUtils.h> 21 #include <SimpleThread.h> 22 #include <aidl/android/hardware/camera/common/Status.h> 23 #include <aidl/android/hardware/camera/device/BnCameraDeviceSession.h> 24 #include <aidl/android/hardware/camera/device/BufferRequest.h> 25 #include <aidl/android/hardware/camera/device/Stream.h> 26 #include <android-base/unique_fd.h> 27 #include <android/hardware/graphics/mapper/2.0/IMapper.h> 28 #include <android/hardware/graphics/mapper/3.0/IMapper.h> 29 #include <android/hardware/graphics/mapper/4.0/IMapper.h> 30 #include <fmq/AidlMessageQueue.h> 31 #include <utils/Thread.h> 32 #include <deque> 33 #include <list> 34 35 namespace android { 36 namespace hardware { 37 namespace camera { 38 namespace device { 39 namespace implementation { 40 41 using ::aidl::android::hardware::camera::common::Status; 42 using ::aidl::android::hardware::camera::device::BnCameraDeviceSession; 43 using ::aidl::android::hardware::camera::device::BufferCache; 44 using ::aidl::android::hardware::camera::device::BufferRequest; 45 using ::aidl::android::hardware::camera::device::CameraMetadata; 46 using ::aidl::android::hardware::camera::device::CameraOfflineSessionInfo; 47 using ::aidl::android::hardware::camera::device::CaptureRequest; 48 using ::aidl::android::hardware::camera::device::HalStream; 49 using ::aidl::android::hardware::camera::device::ICameraDeviceCallback; 50 using ::aidl::android::hardware::camera::device::ICameraOfflineSession; 51 using ::aidl::android::hardware::camera::device::RequestTemplate; 52 using ::aidl::android::hardware::camera::device::Stream; 53 using ::aidl::android::hardware::camera::device::StreamConfiguration; 54 using ::aidl::android::hardware::common::fmq::MQDescriptor; 55 using ::aidl::android::hardware::common::fmq::SynchronizedReadWrite; 56 using ::android::AidlMessageQueue; 57 using ::android::base::unique_fd; 58 using ::android::hardware::camera::common::helper::SimpleThread; 59 using ::android::hardware::camera::external::common::ExternalCameraConfig; 60 using ::android::hardware::camera::external::common::SizeHasher; 61 using ::android::hardware::graphics::mapper::V2_0::YCbCrLayout; 62 using ::ndk::ScopedAStatus; 63 64 class ExternalCameraDeviceSession : public BnCameraDeviceSession, public OutputThreadInterface { 65 public: 66 ExternalCameraDeviceSession(const std::shared_ptr<ICameraDeviceCallback>&, 67 const ExternalCameraConfig& cfg, 68 const std::vector<SupportedV4L2Format>& sortedFormats, 69 const CroppingType& croppingType, 70 const common::V1_0::helper::CameraMetadata& chars, 71 const std::string& cameraId, unique_fd v4l2Fd); 72 ~ExternalCameraDeviceSession() override; 73 74 // Caller must use this method to check if CameraDeviceSession ctor failed 75 bool isInitFailed(); 76 bool isClosed(); 77 78 ScopedAStatus close() override; 79 80 ScopedAStatus configureStreams(const StreamConfiguration& in_requestedConfiguration, 81 std::vector<HalStream>* _aidl_return) override; 82 ScopedAStatus constructDefaultRequestSettings(RequestTemplate in_type, 83 CameraMetadata* _aidl_return) override; 84 ScopedAStatus flush() override; 85 ScopedAStatus getCaptureRequestMetadataQueue( 86 MQDescriptor<int8_t, SynchronizedReadWrite>* _aidl_return) override; 87 ScopedAStatus getCaptureResultMetadataQueue( 88 MQDescriptor<int8_t, SynchronizedReadWrite>* _aidl_return) override; 89 ScopedAStatus isReconfigurationRequired(const CameraMetadata& in_oldSessionParams, 90 const CameraMetadata& in_newSessionParams, 91 bool* _aidl_return) override; 92 ScopedAStatus processCaptureRequest(const std::vector<CaptureRequest>& in_requests, 93 const std::vector<BufferCache>& in_cachesToRemove, 94 int32_t* _aidl_return) override; 95 ScopedAStatus signalStreamFlush(const std::vector<int32_t>& in_streamIds, 96 int32_t in_streamConfigCounter) override; 97 ScopedAStatus switchToOffline(const std::vector<int32_t>& in_streamsToKeep, 98 CameraOfflineSessionInfo* out_offlineSessionInfo, 99 std::shared_ptr<ICameraOfflineSession>* _aidl_return) override; 100 ScopedAStatus repeatingRequestEnd(int32_t in_frameNumber, 101 const std::vector<int32_t>& in_streamIds) override; 102 103 Status importBuffer(int32_t streamId, uint64_t bufId, buffer_handle_t buf, 104 buffer_handle_t** outBufPtr) override; 105 106 void notifyError(int32_t frameNumber, int32_t streamId, ErrorCode ec) override; 107 108 Status processCaptureRequestError(const std::shared_ptr<HalRequest>& ptr, 109 std::vector<NotifyMsg>* msgs, 110 std::vector<CaptureResult>* results) override; 111 112 Status processCaptureResult(std::shared_ptr<HalRequest>& ptr) override; 113 ssize_t getJpegBufferSize(int32_t width, int32_t height) const override; 114 115 // Called by CameraDevice to dump active device states 116 binder_status_t dump(int fd, const char** args, uint32_t numArgs) override; 117 118 static Status isStreamCombinationSupported( 119 const StreamConfiguration& config, 120 const std::vector<SupportedV4L2Format>& supportedFormats, 121 const ExternalCameraConfig& devCfg); 122 123 static const int kMaxProcessedStream = 2; 124 static const int kMaxStallStream = 1; 125 static const uint32_t kMaxBytesPerPixel = 2; 126 127 class BufferRequestThread : public SimpleThread { 128 public: 129 BufferRequestThread(std::weak_ptr<OutputThreadInterface> parent, 130 std::shared_ptr<ICameraDeviceCallback> callbacks); 131 132 int requestBufferStart(const std::vector<HalStreamBuffer>&); 133 int waitForBufferRequestDone( 134 /*out*/ std::vector<HalStreamBuffer>*); 135 136 bool threadLoop() override; 137 138 private: 139 void waitForNextRequest(); 140 141 const std::weak_ptr<OutputThreadInterface> mParent; 142 const std::shared_ptr<ICameraDeviceCallback> mCallbacks; 143 144 std::mutex mLock; 145 bool mRequestingBuffer = false; 146 147 std::vector<HalStreamBuffer> mBufferReqs; 148 std::vector<HalStreamBuffer> mPendingReturnBufferReqs; 149 // mHalBufferReqs is not under mLock protection during the HIDL transaction 150 std::vector<BufferRequest> mHalBufferReqs; 151 152 // request buffers takes much less time in steady state, but can take much longer 153 // when requesting 1st buffer from a stream. 154 // TODO: consider a separate timeout for new vs. steady state? 155 // TODO: or make sure framework is warming up the pipeline during configure new stream? 156 static const int kReqProcTimeoutMs = 66; 157 158 static const int kReqWaitTimeoutMs = 33; 159 static const int kReqWaitTimesWarn = 90; // 33ms * 90 ~= 3 sec 160 std::condition_variable mRequestCond; // signaled when a new buffer request incoming 161 std::condition_variable mRequestDoneCond; // signaled when a request is done 162 }; 163 164 class OutputThread : public SimpleThread { 165 public: 166 OutputThread(std::weak_ptr<OutputThreadInterface> parent, CroppingType, 167 const common::V1_0::helper::CameraMetadata&, 168 std::shared_ptr<BufferRequestThread> bufReqThread); 169 ~OutputThread(); 170 171 Status allocateIntermediateBuffers(const Size& v4lSize, const Size& thumbSize, 172 const std::vector<Stream>& streams, 173 uint32_t blobBufferSize); 174 Status submitRequest(const std::shared_ptr<HalRequest>&); 175 void flush(); 176 void dump(int fd); 177 bool threadLoop() override; 178 179 void setExifMakeModel(const std::string& make, const std::string& model); 180 181 // The remaining request list is returned for offline processing 182 std::list<std::shared_ptr<HalRequest>> switchToOffline(); 183 184 protected: 185 static const int kFlushWaitTimeoutSec = 3; // 3 sec 186 static const int kReqWaitTimeoutMs = 33; // 33ms 187 static const int kReqWaitTimesMax = 90; // 33ms * 90 ~= 3 sec 188 189 // Methods to request output buffer in parallel 190 int requestBufferStart(const std::vector<HalStreamBuffer>&); 191 int waitForBufferRequestDone( 192 /*out*/ std::vector<HalStreamBuffer>*); 193 194 void waitForNextRequest(std::shared_ptr<HalRequest>* out); 195 void signalRequestDone(); 196 197 int cropAndScaleLocked(std::shared_ptr<AllocatedFrame>& in, const Size& outSize, 198 YCbCrLayout* out); 199 200 int cropAndScaleThumbLocked(std::shared_ptr<AllocatedFrame>& in, const Size& outSize, 201 YCbCrLayout* out); 202 203 int createJpegLocked(HalStreamBuffer& halBuf, 204 const common::V1_0::helper::CameraMetadata& settings); 205 206 void clearIntermediateBuffers(); 207 208 const std::weak_ptr<OutputThreadInterface> mParent; 209 const CroppingType mCroppingType; 210 const common::V1_0::helper::CameraMetadata mCameraCharacteristics; 211 212 mutable std::mutex mRequestListLock; // Protect access to mRequestList, 213 // mProcessingRequest and mProcessingFrameNumber 214 std::condition_variable mRequestCond; // signaled when a new request is submitted 215 std::condition_variable mRequestDoneCond; // signaled when a request is done processing 216 std::list<std::shared_ptr<HalRequest>> mRequestList; 217 bool mProcessingRequest = false; 218 uint32_t mProcessingFrameNumber = 0; 219 220 // V4L2 frameIn 221 // (MJPG decode)-> mYu12Frame 222 // (Scale)-> mScaledYu12Frames 223 // (Format convert) -> output gralloc frames 224 mutable std::mutex mBufferLock; // Protect access to intermediate buffers 225 std::shared_ptr<AllocatedFrame> mYu12Frame; 226 std::shared_ptr<AllocatedFrame> mYu12ThumbFrame; 227 std::unordered_map<Size, std::shared_ptr<AllocatedFrame>, SizeHasher> mIntermediateBuffers; 228 std::unordered_map<Size, std::shared_ptr<AllocatedFrame>, SizeHasher> mScaledYu12Frames; 229 YCbCrLayout mYu12FrameLayout; 230 YCbCrLayout mYu12ThumbFrameLayout; 231 std::vector<uint8_t> mMuteTestPatternFrame; 232 uint32_t mTestPatternData[4] = {0, 0, 0, 0}; 233 bool mCameraMuted = false; 234 uint32_t mBlobBufferSize = 0; // 0 -> HAL derive buffer size, else: use given size 235 236 std::string mExifMake; 237 std::string mExifModel; 238 239 const std::shared_ptr<BufferRequestThread> mBufferRequestThread; 240 }; 241 242 private: 243 bool initialize(); 244 // To init/close different version of output thread 245 void initOutputThread(); 246 void closeOutputThread(); 247 void closeBufferRequestThread(); 248 249 void closeImpl(); 250 Status initStatus() const; 251 status_t initDefaultRequests(); 252 253 status_t fillCaptureResult(common::V1_0::helper::CameraMetadata& md, nsecs_t timestamp); 254 int configureV4l2StreamLocked(const SupportedV4L2Format& fmt, double fps = 0.0); 255 int v4l2StreamOffLocked(); 256 257 int setV4l2FpsLocked(double fps); 258 259 std::unique_ptr<V4L2Frame> dequeueV4l2FrameLocked( 260 /*out*/ nsecs_t* shutterTs); // Called with mLock held 261 262 void enqueueV4l2Frame(const std::shared_ptr<V4L2Frame>&); 263 264 // Check if input Stream is one of supported stream setting on this device 265 static bool isSupported(const Stream& stream, 266 const std::vector<SupportedV4L2Format>& supportedFormats, 267 const ExternalCameraConfig& cfg); 268 269 Status importBufferLocked(int32_t streamId, uint64_t bufId, buffer_handle_t buf, 270 /*out*/ buffer_handle_t** outBufPtr); 271 static void cleanupInflightFences(std::vector<int>& allFences, size_t numFences); 272 void cleanupBuffersLocked(int id); 273 274 void updateBufferCaches(const std::vector<BufferCache>& cachesToRemove); 275 276 Status processOneCaptureRequest(const CaptureRequest& request); 277 void notifyShutter(int32_t frameNumber, nsecs_t shutterTs); 278 279 void invokeProcessCaptureResultCallback(std::vector<CaptureResult>& results, bool tryWriteFmq); 280 Size getMaxJpegResolution() const; 281 282 Size getMaxThumbResolution() const; 283 284 int waitForV4L2BufferReturnLocked(std::unique_lock<std::mutex>& lk); 285 286 // Main body of switchToOffline. This method does not invoke any callbacks 287 // but instead returns the necessary callbacks in output arguments so callers 288 // can callback later without holding any locks 289 Status switchToOffline(const std::vector<int32_t>& offlineStreams, 290 /*out*/ std::vector<NotifyMsg>* msgs, 291 /*out*/ std::vector<CaptureResult>* results, 292 /*out*/ CameraOfflineSessionInfo* info, 293 /*out*/ std::shared_ptr<ICameraOfflineSession>* session); 294 295 bool supportOfflineLocked(int32_t streamId); 296 297 // Whether a request can be completely dropped when switching to offline 298 bool canDropRequest(const std::vector<int32_t>& offlineStreams, 299 std::shared_ptr<HalRequest> halReq); 300 301 void fillOfflineSessionInfo(const std::vector<int32_t>& offlineStreams, 302 std::deque<std::shared_ptr<HalRequest>>& offlineReqs, 303 const std::map<int, CirculatingBuffers>& circulatingBuffers, 304 /*out*/ CameraOfflineSessionInfo* info); 305 306 // Protect (most of) HIDL interface methods from synchronized-entering 307 mutable Mutex mInterfaceLock; 308 309 mutable Mutex mLock; // Protect all private members except otherwise noted 310 const std::shared_ptr<ICameraDeviceCallback> mCallback; 311 const ExternalCameraConfig& mCfg; 312 const common::V1_0::helper::CameraMetadata mCameraCharacteristics; 313 const std::vector<SupportedV4L2Format> mSupportedFormats; 314 const CroppingType mCroppingType; 315 const std::string mCameraId; 316 317 // Not protected by mLock, this is almost a const. 318 // Setup in constructor, reset in close() after OutputThread is joined 319 unique_fd mV4l2Fd; 320 321 // device is closed either 322 // - closed by user 323 // - init failed 324 // - camera disconnected 325 bool mClosed = false; 326 bool mInitialized = false; 327 bool mInitFail = false; 328 bool mFirstRequest = false; 329 common::V1_0::helper::CameraMetadata mLatestReqSetting; 330 331 bool mV4l2Streaming = false; 332 SupportedV4L2Format mV4l2StreamingFmt; 333 double mV4l2StreamingFps = 0.0; 334 size_t mV4L2BufferCount = 0; 335 336 static const int kBufferWaitTimeoutSec = 3; // TODO: handle long exposure (or not allowing) 337 std::mutex mV4l2BufferLock; // protect the buffer count and condition below 338 std::condition_variable mV4L2BufferReturned; 339 size_t mNumDequeuedV4l2Buffers = 0; 340 uint32_t mMaxV4L2BufferSize = 0; 341 342 // Not protected by mLock (but might be used when mLock is locked) 343 std::shared_ptr<OutputThread> mOutputThread; 344 345 // Stream ID -> Stream cache 346 std::unordered_map<int, Stream> mStreamMap; 347 348 std::mutex mInflightFramesLock; // protect mInflightFrames 349 std::unordered_set<uint32_t> mInflightFrames; 350 351 // Stream ID -> circulating buffers map 352 std::map<int, CirculatingBuffers> mCirculatingBuffers; 353 // Protect mCirculatingBuffers, must not lock mLock after acquiring this lock 354 mutable Mutex mCbsLock; 355 356 std::mutex mAfTriggerLock; // protect mAfTrigger 357 bool mAfTrigger = false; 358 359 uint32_t mBlobBufferSize = 0; 360 361 static HandleImporter sHandleImporter; 362 363 std::shared_ptr<BufferRequestThread> mBufferRequestThread; 364 365 /* Beginning of members not changed after initialize() */ 366 using RequestMetadataQueue = AidlMessageQueue<int8_t, SynchronizedReadWrite>; 367 std::unique_ptr<RequestMetadataQueue> mRequestMetadataQueue; 368 using ResultMetadataQueue = AidlMessageQueue<int8_t, SynchronizedReadWrite>; 369 std::shared_ptr<ResultMetadataQueue> mResultMetadataQueue; 370 371 // Protect against invokeProcessCaptureResultCallback() 372 Mutex mProcessCaptureResultLock; 373 374 // tracks last seen stream config counter 375 int32_t mLastStreamConfigCounter = -1; 376 377 std::unordered_map<RequestTemplate, CameraMetadata> mDefaultRequests; 378 379 const Size mMaxThumbResolution; 380 const Size mMaxJpegResolution; 381 382 std::string mExifMake; 383 std::string mExifModel; 384 /* End of members not changed after initialize() */ 385 }; 386 387 } // namespace implementation 388 } // namespace device 389 } // namespace camera 390 } // namespace hardware 391 } // namespace android 392 393 #endif // HARDWARE_INTERFACES_CAMERA_DEVICE_DEFAULT_EXTERNALCAMERADEVICESESSION_H_ 394