1 /* 2 * Copyright (C) 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 #ifndef _ACAMERA_DEVICE_H 17 #define _ACAMERA_DEVICE_H 18 19 #include "ACameraMetadata.h" 20 #include "utils.h" 21 22 #include <aidl/android/frameworks/cameraservice/common/Status.h> 23 #include <aidl/android/frameworks/cameraservice/device/BnCameraDeviceCallback.h> 24 #include <aidl/android/frameworks/cameraservice/device/CaptureResultExtras.h> 25 #include <aidl/android/frameworks/cameraservice/device/ErrorCode.h> 26 #include <aidl/android/frameworks/cameraservice/device/CaptureMetadataInfo.h> 27 #include <aidl/android/frameworks/cameraservice/device/ICameraDeviceUser.h> 28 #include <aidl/android/frameworks/cameraservice/device/PhysicalCameraSettings.h> 29 #include <aidl/android/frameworks/cameraservice/device/PhysicalCaptureResultInfo.h> 30 #include <aidl/android/frameworks/cameraservice/device/StreamConfigurationMode.h> 31 #include <aidl/android/frameworks/cameraservice/device/SubmitInfo.h> 32 #include <aidl/android/frameworks/cameraservice/service/CameraStatusAndId.h> 33 #include <atomic> 34 #include <camera/NdkCameraCaptureSession.h> 35 #include <camera/NdkCameraManager.h> 36 #include <fmq/AidlMessageQueue.h> 37 #include <fmq/MessageQueue.h> 38 #include <map> 39 #include <media/stagefright/foundation/AHandler.h> 40 #include <media/stagefright/foundation/ALooper.h> 41 #include <media/stagefright/foundation/AMessage.h> 42 #include <memory> 43 #include <set> 44 #include <utility> 45 #include <utils/List.h> 46 #include <utils/Mutex.h> 47 #include <utils/StrongPointer.h> 48 #include <utils/Vector.h> 49 #include <vector> 50 51 namespace android { 52 namespace acam { 53 54 using ::aidl::android::frameworks::cameraservice::common::Status; 55 using ::aidl::android::frameworks::cameraservice::device::BnCameraDeviceCallback; 56 using ::aidl::android::frameworks::cameraservice::device::CaptureResultExtras; 57 using ::aidl::android::frameworks::cameraservice::device::ErrorCode; 58 using ::aidl::android::frameworks::cameraservice::device::CaptureMetadataInfo; 59 using ::aidl::android::frameworks::cameraservice::device::ICameraDeviceCallback; 60 using ::aidl::android::frameworks::cameraservice::device::ICameraDeviceUser; 61 using ::aidl::android::frameworks::cameraservice::device::OutputConfiguration; 62 using ::aidl::android::frameworks::cameraservice::device::PhysicalCameraSettings; 63 using ::aidl::android::frameworks::cameraservice::device::PhysicalCaptureResultInfo; 64 using ::aidl::android::frameworks::cameraservice::device::StreamConfigurationMode; 65 using ::aidl::android::frameworks::cameraservice::device::SubmitInfo; 66 using ::aidl::android::frameworks::cameraservice::service::CameraStatusAndId; 67 using ::aidl::android::hardware::common::fmq::SynchronizedReadWrite; 68 using ::android::AidlMessageQueue; 69 70 71 using ResultMetadataQueue = AidlMessageQueue<int8_t, SynchronizedReadWrite>; 72 using RequestMetadataQueue = AidlMessageQueue<int8_t, SynchronizedReadWrite>; 73 74 using utils::CaptureRequest; 75 76 // Wrap ACameraCaptureFailure so it can be ref-counted 77 struct CameraCaptureFailure : public RefBase, public ACameraCaptureFailure { }; 78 79 // Wrap PhysicalCaptureResultInfo so that it can be ref-counted 80 struct PhysicalCaptureResultInfoLocal { 81 std::string physicalCameraId; 82 CameraMetadata physicalMetadata; 83 }; 84 85 struct ACameraPhysicalCaptureResultInfo: public RefBase { ACameraPhysicalCaptureResultInfoACameraPhysicalCaptureResultInfo86 ACameraPhysicalCaptureResultInfo(const std::vector<PhysicalCaptureResultInfoLocal>& info, 87 int64_t frameNumber) : 88 mPhysicalResultInfo(info), mFrameNumber(frameNumber) {} 89 90 std::vector<PhysicalCaptureResultInfoLocal> mPhysicalResultInfo; 91 int64_t mFrameNumber; 92 }; 93 94 class CameraDevice final : public std::enable_shared_from_this<CameraDevice> { 95 public: 96 CameraDevice(const char* id, ACameraDevice_StateCallbacks* cb, 97 sp<ACameraMetadata> chars, 98 ACameraDevice* wrapper, bool sharedMode); 99 ~CameraDevice(); 100 101 // Called to initialize fields that require shared_ptr to `this` 102 void init(); 103 getId()104 inline const char* getId() const { return mCameraId.c_str(); } 105 106 camera_status_t createCaptureRequest( 107 ACameraDevice_request_template templateId, 108 const ACameraIdList* physicalCameraIdList, 109 ACaptureRequest** request) const; 110 111 camera_status_t createCaptureSession( 112 const ACaptureSessionOutputContainer* outputs, 113 const ACaptureRequest* sessionParameters, 114 const ACameraCaptureSession_stateCallbacks* callbacks, 115 /*out*/ACameraCaptureSession** session); 116 117 camera_status_t isSessionConfigurationSupported( 118 const ACaptureSessionOutputContainer* sessionOutputContainer) const; 119 120 // Callbacks from camera service 121 class ServiceCallback : public BnCameraDeviceCallback { 122 public: ServiceCallback(std::weak_ptr<CameraDevice> device)123 explicit ServiceCallback(std::weak_ptr<CameraDevice> device) : 124 mDevice(std::move(device)) {} 125 126 ndk::ScopedAStatus onDeviceError(ErrorCode in_errorCode, 127 const CaptureResultExtras& in_resultExtras) override; 128 ndk::ScopedAStatus onDeviceIdle() override; 129 130 ndk::ScopedAStatus onCaptureStarted(const CaptureResultExtras& in_resultExtras, 131 int64_t in_timestamp) override; 132 ndk::ScopedAStatus onPrepared(int32_t in_streamId) override; 133 ndk::ScopedAStatus onRepeatingRequestError(int64_t in_lastFrameNumber, 134 int32_t in_repeatingRequestId) override; 135 ndk::ScopedAStatus onResultReceived(const CaptureMetadataInfo& in_result, 136 const CaptureResultExtras& in_resultExtras, 137 const std::vector<PhysicalCaptureResultInfo>& 138 in_physicalCaptureResultInfos) override; 139 ndk::ScopedAStatus onClientSharedAccessPriorityChanged(bool isPrimaryClient) override; 140 141 private: 142 camera_status_t readOneResultMetadata(const CaptureMetadataInfo& captureMetadataInfo, 143 ResultMetadataQueue* metadataQueue, CameraMetadata* metadata); 144 const std::weak_ptr<CameraDevice> mDevice; 145 }; getServiceCallback()146 inline std::shared_ptr<BnCameraDeviceCallback> getServiceCallback() { 147 return mServiceCallback; 148 }; 149 150 // Camera device is only functional after remote being set 151 void setRemoteDevice(std::shared_ptr<ICameraDeviceUser> remote); 152 153 bool setDeviceMetadataQueues(); getWrapper()154 inline ACameraDevice* getWrapper() const { return mWrapper; }; 155 156 // Stop the looper thread and unregister the handler 157 void stopLooperAndDisconnect(); setPrimaryClient(bool isPrimary)158 void setPrimaryClient(bool isPrimary) {mIsPrimaryClient = isPrimary;}; isPrimaryClient()159 bool isPrimaryClient() {return mIsPrimaryClient;}; isSharedMode()160 bool isSharedMode() {return mSharedMode;}; 161 162 private: 163 friend ACameraCaptureSession; 164 friend ACameraDevice; 165 166 camera_status_t checkCameraClosedOrErrorLocked() const; 167 168 // device goes into fatal error state after this 169 void setCameraDeviceErrorLocked(camera_status_t error); 170 171 void disconnectLocked(sp<ACameraCaptureSession>& session); // disconnect from camera service 172 173 camera_status_t stopRepeatingLocked(); 174 175 camera_status_t flushLocked(ACameraCaptureSession*); 176 177 camera_status_t waitUntilIdleLocked(); 178 179 template<class T> 180 camera_status_t captureLocked(sp<ACameraCaptureSession> session, 181 /*optional*/T* cbs, 182 int numRequests, ACaptureRequest** requests, 183 /*optional*/int* captureSequenceId); 184 185 template<class T> 186 camera_status_t setRepeatingRequestsLocked(sp<ACameraCaptureSession> session, 187 /*optional*/T* cbs, 188 int numRequests, ACaptureRequest** requests, 189 /*optional*/int* captureSequenceId); 190 191 template<class T> 192 camera_status_t submitRequestsLocked( 193 sp<ACameraCaptureSession> session, 194 /*optional*/T* cbs, 195 int numRequests, ACaptureRequest** requests, 196 /*out*/int* captureSequenceId, 197 bool isRepeating); 198 199 camera_status_t stopStreamingLocked(); 200 201 template<class T> 202 camera_status_t startStreamingLocked(ACameraCaptureSession* session, 203 /*optional*/T* callbacks, 204 int numOutputWindows, ANativeWindow** windows, /*optional*/int* captureSequenceId); 205 206 void addRequestSettingsMetadata(ACaptureRequest *aCaptureRequest, sp<CaptureRequest> &req); 207 208 camera_status_t updateOutputConfigurationLocked(ACaptureSessionOutput *output); 209 210 camera_status_t prepareLocked(ANativeWindow *window); 211 212 // Since this writes to ICameraDeviceUser's fmq, clients must take care that: 213 // a) This function is called serially. 214 // b) This function is called in accordance with ICameraDeviceUser.submitRequestList, 215 // otherwise, the wrong capture request might have the wrong settings 216 // metadata associated with it. 217 camera_status_t allocateCaptureRequestLocked( 218 const ACaptureRequest* request, sp<CaptureRequest>& outReq); 219 void allocateOneCaptureRequestMetadata( 220 PhysicalCameraSettings& cameraSettings, 221 const std::string& id, const sp<ACameraMetadata>& metadata); 222 223 static ACaptureRequest* allocateACaptureRequest(sp<CaptureRequest>& req, const char* deviceId); 224 static void freeACaptureRequest(ACaptureRequest*); 225 226 // only For session to hold device lock 227 // Always grab device lock before grabbing session lock lockDeviceForSessionOps()228 void lockDeviceForSessionOps() const { mDeviceLock.lock(); }; unlockDevice()229 void unlockDevice() const { mDeviceLock.unlock(); }; 230 231 // For capture session to notify its end of life 232 void notifySessionEndOfLifeLocked(ACameraCaptureSession* session); 233 234 camera_status_t configureStreamsLocked(const ACaptureSessionOutputContainer* outputs, 235 const ACaptureRequest* sessionParameters, nsecs_t startTimeNs); 236 237 // Input message will be posted and cleared after this returns 238 void postSessionMsgAndCleanup(sp<AMessage>& msg); 239 240 mutable Mutex mDeviceLock; 241 const std::string mCameraId; // Camera ID 242 const ACameraDevice_StateCallbacks mAppCallbacks; // Callback to app 243 const sp<ACameraMetadata> mChars; // Camera characteristics 244 std::shared_ptr<ServiceCallback> mServiceCallback; 245 ACameraDevice* mWrapper; 246 bool mSharedMode; 247 bool mIsPrimaryClient; 248 ACaptureRequest* mPreviewRequest = nullptr; 249 std::vector<ACameraOutputTarget*> mPreviewRequestOutputs; 250 251 // stream id -> pair of (ACameraWindowType* from application, OutputConfiguration used for 252 // camera service) 253 std::map<int, std::pair<ANativeWindow *, OutputConfiguration>> mConfiguredOutputs; 254 255 // TODO: maybe a bool will suffice for synchronous implementation? 256 std::atomic_bool mClosing; isClosed()257 inline bool isClosed() { return mClosing; } 258 259 bool mInError = false; 260 camera_status_t mError = ACAMERA_OK; 261 void onCaptureErrorLocked( 262 ErrorCode errorCode, 263 const CaptureResultExtras& resultExtras); 264 265 bool mIdle = true; 266 // This will avoid a busy session being deleted before it's back to idle state 267 sp<ACameraCaptureSession> mBusySession; 268 269 std::shared_ptr<ICameraDeviceUser> mRemote; 270 271 // Looper thread to handle callback to app 272 sp<ALooper> mCbLooper; 273 // definition of handler and message 274 enum { 275 // Device state callbacks 276 kWhatOnDisconnected, // onDisconnected 277 kWhatOnError, // onError 278 // Session state callbacks 279 kWhatSessionStateCb, // onReady, onActive 280 // Capture callbacks 281 kWhatCaptureStart, // onCaptureStarted 282 kWhatCaptureStart2, // onCaptureStarted2 283 kWhatCaptureResult, // onCaptureProgressed, onCaptureCompleted 284 kWhatLogicalCaptureResult, // onLogicalCameraCaptureCompleted 285 kWhatCaptureFail, // onCaptureFailed 286 kWhatLogicalCaptureFail, // onLogicalCameraCaptureFailed 287 kWhatCaptureSeqEnd, // onCaptureSequenceCompleted 288 kWhatCaptureSeqAbort, // onCaptureSequenceAborted 289 kWhatCaptureBufferLost, // onCaptureBufferLost 290 kWhatPreparedCb, // onPrepared 291 // Internal cleanup 292 kWhatCleanUpSessions, // Cleanup cached sp<ACameraCaptureSession> 293 kWhatClientSharedAccessPriorityChanged 294 }; 295 static const char* kContextKey; 296 static const char* kDeviceKey; 297 static const char* kErrorCodeKey; 298 static const char* kCallbackFpKey; 299 static const char* kSessionSpKey; 300 static const char* kCaptureRequestKey; 301 static const char* kTimeStampKey; 302 static const char* kCaptureResultKey; 303 static const char* kPhysicalCaptureResultKey; 304 static const char* kCaptureFailureKey; 305 static const char* kSequenceIdKey; 306 static const char* kFrameNumberKey; 307 static const char* kAnwKey; 308 static const char* kFailingPhysicalCameraId; 309 310 class CallbackHandler : public AHandler { 311 public: 312 explicit CallbackHandler(const char *id); 313 void onMessageReceived(const sp<AMessage> &msg) override; 314 315 private: 316 std::string mId; 317 // This handler will cache all capture session sp until kWhatCleanUpSessions 318 // is processed. This is used to guarantee the last session reference is always 319 // being removed in callback thread without holding camera device lock 320 std::vector<sp<ACameraCaptureSession>> mCachedSessions; 321 }; 322 sp<CallbackHandler> mHandler; 323 324 /*********************************** 325 * Capture session related members * 326 ***********************************/ 327 // The current active session 328 wp<ACameraCaptureSession> mCurrentSession; 329 bool mFlushing = false; 330 331 int mNextSessionId = 0; 332 // TODO: might need another looper/handler to handle callbacks from service 333 334 static const int REQUEST_ID_NONE = -1; 335 int mRepeatingSequenceId = REQUEST_ID_NONE; 336 337 // sequence id -> last frame number map 338 std::map<int32_t, int64_t> mSequenceLastFrameNumberMap; 339 340 struct CallbackHolder { 341 CallbackHolder(sp<ACameraCaptureSession> session, 342 std::vector<sp<CaptureRequest>> requests, 343 bool isRepeating, 344 ACameraCaptureSession_captureCallbacks* cbs); 345 CallbackHolder(sp<ACameraCaptureSession> session, 346 std::vector<sp<CaptureRequest>> requests, 347 bool isRepeating, 348 ACameraCaptureSession_logicalCamera_captureCallbacks* lcbs); 349 CallbackHolder(sp<ACameraCaptureSession> session, 350 std::vector<sp<CaptureRequest> > requests, 351 bool isRepeating, 352 ACameraCaptureSession_captureCallbacksV2* cbs); 353 CallbackHolder(sp<ACameraCaptureSession> session, 354 std::vector<sp<CaptureRequest> > requests, 355 bool isRepeating, 356 ACameraCaptureSession_logicalCamera_captureCallbacksV2* lcbs); clearCallbacksCallbackHolder357 void clearCallbacks() { 358 mContext = nullptr; 359 mOnCaptureStarted = nullptr; 360 mOnCaptureStarted2 = nullptr; 361 mOnCaptureProgressed = nullptr; 362 mOnCaptureCompleted = nullptr; 363 mOnLogicalCameraCaptureCompleted = nullptr; 364 mOnLogicalCameraCaptureFailed = nullptr; 365 mOnCaptureFailed = nullptr; 366 mOnCaptureSequenceCompleted = nullptr; 367 mOnCaptureSequenceAborted = nullptr; 368 mOnCaptureBufferLost = nullptr; 369 } 370 371 template <class T> initCaptureCallbacksV2CallbackHolder372 void initCaptureCallbacksV2(T* cbs) { 373 clearCallbacks(); 374 if (cbs != nullptr) { 375 mContext = cbs->context; 376 mOnCaptureStarted2 = cbs->onCaptureStarted; 377 mOnCaptureProgressed = cbs->onCaptureProgressed; 378 mOnCaptureSequenceCompleted = cbs->onCaptureSequenceCompleted; 379 mOnCaptureSequenceAborted = cbs->onCaptureSequenceAborted; 380 mOnCaptureBufferLost = cbs->onCaptureBufferLost; 381 } 382 } 383 384 template <class T> initCaptureCallbacksCallbackHolder385 void initCaptureCallbacks(T* cbs) { 386 clearCallbacks(); 387 if (cbs != nullptr) { 388 mContext = cbs->context; 389 mOnCaptureStarted = cbs->onCaptureStarted; 390 mOnCaptureProgressed = cbs->onCaptureProgressed; 391 mOnCaptureSequenceCompleted = cbs->onCaptureSequenceCompleted; 392 mOnCaptureSequenceAborted = cbs->onCaptureSequenceAborted; 393 mOnCaptureBufferLost = cbs->onCaptureBufferLost; 394 } 395 } 396 397 sp<ACameraCaptureSession> mSession; 398 std::vector<sp<CaptureRequest>> mRequests; 399 const bool mIsRepeating; 400 const bool mIs2Callback; 401 const bool mIsLogicalCameraCallback; 402 403 void* mContext; 404 ACameraCaptureSession_captureCallback_start mOnCaptureStarted; 405 ACameraCaptureSession_captureCallback_startV2 mOnCaptureStarted2; 406 ACameraCaptureSession_captureCallback_result mOnCaptureProgressed; 407 ACameraCaptureSession_captureCallback_result mOnCaptureCompleted; 408 ACameraCaptureSession_logicalCamera_captureCallback_result mOnLogicalCameraCaptureCompleted; 409 ACameraCaptureSession_logicalCamera_captureCallback_failed mOnLogicalCameraCaptureFailed; 410 ACameraCaptureSession_captureCallback_failed mOnCaptureFailed; 411 ACameraCaptureSession_captureCallback_sequenceEnd mOnCaptureSequenceCompleted; 412 ACameraCaptureSession_captureCallback_sequenceAbort mOnCaptureSequenceAborted; 413 ACameraCaptureSession_captureCallback_bufferLost mOnCaptureBufferLost; 414 }; 415 // sequence id -> callbacks map 416 std::map<int, CallbackHolder> mSequenceCallbackMap; 417 418 static const int64_t NO_FRAMES_CAPTURED = -1; 419 class FrameNumberTracker { 420 public: 421 // TODO: Called in onResultReceived and onCaptureErrorLocked 422 void updateTracker(int64_t frameNumber, bool isError); getCompletedFrameNumber()423 inline int64_t getCompletedFrameNumber() { return mCompletedFrameNumber; } 424 private: 425 void update(); 426 void updateCompletedFrameNumber(int64_t frameNumber); 427 428 int64_t mCompletedFrameNumber = NO_FRAMES_CAPTURED; 429 List<int64_t> mSkippedFrameNumbers; 430 std::set<int64_t> mFutureErrorSet; 431 }; 432 FrameNumberTracker mFrameNumberTracker; 433 434 void checkRepeatingSequenceCompleteLocked(const int sequenceId, const int64_t lastFrameNumber); 435 void checkAndFireSequenceCompleteLocked(); 436 437 // Misc variables 438 int32_t mShadingMapSize[2]; // const after constructor 439 int32_t mPartialResultCount; // const after constructor 440 std::shared_ptr<RequestMetadataQueue> mCaptureRequestMetadataQueue = nullptr; 441 std::shared_ptr<ResultMetadataQueue> mCaptureResultMetadataQueue = nullptr; 442 }; 443 444 } // namespace acam; 445 } // namespace android; 446 447 /** 448 * ACameraDevice opaque struct definition 449 * Leave outside of android namespace because it's NDK struct 450 */ 451 struct ACameraDevice { ACameraDeviceACameraDevice452 ACameraDevice(const char* id, ACameraDevice_StateCallbacks* cb, 453 sp<ACameraMetadata> chars, bool sharedMode) : 454 mDevice(std::make_shared<android::acam::CameraDevice>(id, cb, 455 std::move(chars), this, sharedMode)) { 456 mDevice->init(); 457 } 458 459 ~ACameraDevice(); 460 /******************* 461 * NDK public APIs * 462 *******************/ getIdACameraDevice463 inline const char* getId() const { return mDevice->getId(); } 464 createCaptureRequestACameraDevice465 camera_status_t createCaptureRequest( 466 ACameraDevice_request_template templateId, 467 const ACameraIdList* physicalCameraIdList, 468 ACaptureRequest** request) const { 469 return mDevice->createCaptureRequest(templateId, physicalCameraIdList, request); 470 } 471 createCaptureSessionACameraDevice472 camera_status_t createCaptureSession( 473 const ACaptureSessionOutputContainer* outputs, 474 const ACaptureRequest* sessionParameters, 475 const ACameraCaptureSession_stateCallbacks* callbacks, 476 /*out*/ACameraCaptureSession** session) { 477 return mDevice->createCaptureSession(outputs, sessionParameters, callbacks, session); 478 } 479 isSessionConfigurationSupportedACameraDevice480 camera_status_t isSessionConfigurationSupported( 481 const ACaptureSessionOutputContainer* sessionOutputContainer) const { 482 return mDevice->isSessionConfigurationSupported(sessionOutputContainer); 483 } 484 485 /*********************** 486 * Device interal APIs * 487 ***********************/ getServiceCallbackACameraDevice488 inline std::shared_ptr<android::acam::BnCameraDeviceCallback> getServiceCallback() { 489 return mDevice->getServiceCallback(); 490 }; 491 492 // Camera device is only functional after remote being set setRemoteDeviceACameraDevice493 inline void setRemoteDevice(std::shared_ptr< 494 ::aidl::android::frameworks::cameraservice::device::ICameraDeviceUser> remote) { 495 mDevice->setRemoteDevice(remote); 496 } setDeviceMetadataQueuesACameraDevice497 inline bool setDeviceMetadataQueues() { 498 return mDevice->setDeviceMetadataQueues(); 499 } setPrimaryClientACameraDevice500 inline void setPrimaryClient(bool isPrimary) { 501 mDevice->setPrimaryClient(isPrimary); 502 } isPrimaryClientACameraDevice503 inline bool isPrimaryClient() const { 504 return mDevice->isPrimaryClient(); 505 } isSharedModeACameraDevice506 inline bool isSharedMode() const { 507 return mDevice->isSharedMode(); 508 } 509 510 private: 511 std::shared_ptr<android::acam::CameraDevice> mDevice; 512 }; 513 514 #endif // _ACAMERA_DEVICE_H 515