1 /* 2 * Copyright (C) 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 #ifndef _ACAMERA_DEVICE_H 17 #define _ACAMERA_DEVICE_H 18 19 #include "ACameraMetadata.h" 20 #include "utils.h" 21 22 #include <aidl/android/frameworks/cameraservice/common/Status.h> 23 #include <aidl/android/frameworks/cameraservice/device/BnCameraDeviceCallback.h> 24 #include <aidl/android/frameworks/cameraservice/device/CaptureResultExtras.h> 25 #include <aidl/android/frameworks/cameraservice/device/ErrorCode.h> 26 #include <aidl/android/frameworks/cameraservice/device/CaptureMetadataInfo.h> 27 #include <aidl/android/frameworks/cameraservice/device/ICameraDeviceUser.h> 28 #include <aidl/android/frameworks/cameraservice/device/PhysicalCameraSettings.h> 29 #include <aidl/android/frameworks/cameraservice/device/PhysicalCaptureResultInfo.h> 30 #include <aidl/android/frameworks/cameraservice/device/StreamConfigurationMode.h> 31 #include <aidl/android/frameworks/cameraservice/device/SubmitInfo.h> 32 #include <aidl/android/frameworks/cameraservice/service/CameraStatusAndId.h> 33 #include <atomic> 34 #include <camera/NdkCameraCaptureSession.h> 35 #include <camera/NdkCameraManager.h> 36 #include <fmq/AidlMessageQueue.h> 37 #include <fmq/MessageQueue.h> 38 #include <map> 39 #include <media/stagefright/foundation/AHandler.h> 40 #include <media/stagefright/foundation/ALooper.h> 41 #include <media/stagefright/foundation/AMessage.h> 42 #include <memory> 43 #include <set> 44 #include <utility> 45 #include <utils/List.h> 46 #include <utils/Mutex.h> 47 #include <utils/StrongPointer.h> 48 #include <utils/Vector.h> 49 #include <vector> 50 51 namespace android { 52 namespace acam { 53 54 using ::aidl::android::frameworks::cameraservice::common::Status; 55 using ::aidl::android::frameworks::cameraservice::device::BnCameraDeviceCallback; 56 using ::aidl::android::frameworks::cameraservice::device::CaptureResultExtras; 57 using ::aidl::android::frameworks::cameraservice::device::ErrorCode; 58 using ::aidl::android::frameworks::cameraservice::device::CaptureMetadataInfo; 59 using ::aidl::android::frameworks::cameraservice::device::ICameraDeviceCallback; 60 using ::aidl::android::frameworks::cameraservice::device::ICameraDeviceUser; 61 using ::aidl::android::frameworks::cameraservice::device::OutputConfiguration; 62 using ::aidl::android::frameworks::cameraservice::device::PhysicalCameraSettings; 63 using ::aidl::android::frameworks::cameraservice::device::PhysicalCaptureResultInfo; 64 using ::aidl::android::frameworks::cameraservice::device::StreamConfigurationMode; 65 using ::aidl::android::frameworks::cameraservice::device::SubmitInfo; 66 using ::aidl::android::frameworks::cameraservice::service::CameraStatusAndId; 67 using ::aidl::android::hardware::common::fmq::SynchronizedReadWrite; 68 using ::android::AidlMessageQueue; 69 using ::android::acam::utils::native_handle_ptr_wrapper; 70 71 72 using ResultMetadataQueue = AidlMessageQueue<int8_t, SynchronizedReadWrite>; 73 using RequestMetadataQueue = AidlMessageQueue<int8_t, SynchronizedReadWrite>; 74 75 using utils::CaptureRequest; 76 77 // Wrap ACameraCaptureFailure so it can be ref-counted 78 struct CameraCaptureFailure : public RefBase, public ACameraCaptureFailure { }; 79 80 // Wrap PhysicalCaptureResultInfo so that it can be ref-counted 81 struct PhysicalCaptureResultInfoLocal { 82 std::string physicalCameraId; 83 CameraMetadata physicalMetadata; 84 }; 85 86 struct ACameraPhysicalCaptureResultInfo: public RefBase { ACameraPhysicalCaptureResultInfoACameraPhysicalCaptureResultInfo87 ACameraPhysicalCaptureResultInfo(const std::vector<PhysicalCaptureResultInfoLocal>& info, 88 int64_t frameNumber) : 89 mPhysicalResultInfo(info), mFrameNumber(frameNumber) {} 90 91 std::vector<PhysicalCaptureResultInfoLocal> mPhysicalResultInfo; 92 int64_t mFrameNumber; 93 }; 94 95 class CameraDevice final : public std::enable_shared_from_this<CameraDevice> { 96 public: 97 CameraDevice(const char* id, ACameraDevice_StateCallbacks* cb, 98 sp<ACameraMetadata> chars, 99 ACameraDevice* wrapper); 100 ~CameraDevice(); 101 102 // Called to initialize fields that require shared_ptr to `this` 103 void init(); 104 getId()105 inline const char* getId() const { return mCameraId.c_str(); } 106 107 camera_status_t createCaptureRequest( 108 ACameraDevice_request_template templateId, 109 const ACameraIdList* physicalCameraIdList, 110 ACaptureRequest** request) const; 111 112 camera_status_t createCaptureSession( 113 const ACaptureSessionOutputContainer* outputs, 114 const ACaptureRequest* sessionParameters, 115 const ACameraCaptureSession_stateCallbacks* callbacks, 116 /*out*/ACameraCaptureSession** session); 117 118 camera_status_t isSessionConfigurationSupported( 119 const ACaptureSessionOutputContainer* sessionOutputContainer) const; 120 121 // Callbacks from camera service 122 class ServiceCallback : public BnCameraDeviceCallback { 123 public: ServiceCallback(std::weak_ptr<CameraDevice> device)124 explicit ServiceCallback(std::weak_ptr<CameraDevice> device) : 125 mDevice(std::move(device)) {} 126 127 ndk::ScopedAStatus onDeviceError(ErrorCode in_errorCode, 128 const CaptureResultExtras& in_resultExtras) override; 129 ndk::ScopedAStatus onDeviceIdle() override; 130 131 ndk::ScopedAStatus onCaptureStarted(const CaptureResultExtras& in_resultExtras, 132 int64_t in_timestamp) override; 133 ndk::ScopedAStatus onPrepared(int32_t in_streamId) override; 134 ndk::ScopedAStatus onRepeatingRequestError(int64_t in_lastFrameNumber, 135 int32_t in_repeatingRequestId) override; 136 ndk::ScopedAStatus onResultReceived(const CaptureMetadataInfo& in_result, 137 const CaptureResultExtras& in_resultExtras, 138 const std::vector<PhysicalCaptureResultInfo>& 139 in_physicalCaptureResultInfos) override; 140 141 private: 142 camera_status_t readOneResultMetadata(const CaptureMetadataInfo& captureMetadataInfo, 143 ResultMetadataQueue* metadataQueue, CameraMetadata* metadata); 144 const std::weak_ptr<CameraDevice> mDevice; 145 }; getServiceCallback()146 inline std::shared_ptr<BnCameraDeviceCallback> getServiceCallback() { 147 return mServiceCallback; 148 }; 149 150 // Camera device is only functional after remote being set 151 void setRemoteDevice(std::shared_ptr<ICameraDeviceUser> remote); 152 153 bool setDeviceMetadataQueues(); getWrapper()154 inline ACameraDevice* getWrapper() const { return mWrapper; }; 155 156 // Stop the looper thread and unregister the handler 157 void stopLooperAndDisconnect(); 158 159 private: 160 friend ACameraCaptureSession; 161 friend ACameraDevice; 162 163 camera_status_t checkCameraClosedOrErrorLocked() const; 164 165 // device goes into fatal error state after this 166 void setCameraDeviceErrorLocked(camera_status_t error); 167 168 void disconnectLocked(sp<ACameraCaptureSession>& session); // disconnect from camera service 169 170 camera_status_t stopRepeatingLocked(); 171 172 camera_status_t flushLocked(ACameraCaptureSession*); 173 174 camera_status_t waitUntilIdleLocked(); 175 176 template<class T> 177 camera_status_t captureLocked(sp<ACameraCaptureSession> session, 178 /*optional*/T* cbs, 179 int numRequests, ACaptureRequest** requests, 180 /*optional*/int* captureSequenceId); 181 182 template<class T> 183 camera_status_t setRepeatingRequestsLocked(sp<ACameraCaptureSession> session, 184 /*optional*/T* cbs, 185 int numRequests, ACaptureRequest** requests, 186 /*optional*/int* captureSequenceId); 187 188 template<class T> 189 camera_status_t submitRequestsLocked( 190 sp<ACameraCaptureSession> session, 191 /*optional*/T* cbs, 192 int numRequests, ACaptureRequest** requests, 193 /*out*/int* captureSequenceId, 194 bool isRepeating); 195 196 void addRequestSettingsMetadata(ACaptureRequest *aCaptureRequest, sp<CaptureRequest> &req); 197 198 camera_status_t updateOutputConfigurationLocked(ACaptureSessionOutput *output); 199 200 camera_status_t prepareLocked(ACameraWindowType *window); 201 202 // Since this writes to ICameraDeviceUser's fmq, clients must take care that: 203 // a) This function is called serially. 204 // b) This function is called in accordance with ICameraDeviceUser.submitRequestList, 205 // otherwise, the wrong capture request might have the wrong settings 206 // metadata associated with it. 207 camera_status_t allocateCaptureRequestLocked( 208 const ACaptureRequest* request, sp<CaptureRequest>& outReq); 209 void allocateOneCaptureRequestMetadata( 210 PhysicalCameraSettings& cameraSettings, 211 const std::string& id, const sp<ACameraMetadata>& metadata); 212 213 static ACaptureRequest* allocateACaptureRequest(sp<CaptureRequest>& req, const char* deviceId); 214 static void freeACaptureRequest(ACaptureRequest*); 215 216 // only For session to hold device lock 217 // Always grab device lock before grabbing session lock lockDeviceForSessionOps()218 void lockDeviceForSessionOps() const { mDeviceLock.lock(); }; unlockDevice()219 void unlockDevice() const { mDeviceLock.unlock(); }; 220 221 // For capture session to notify its end of life 222 void notifySessionEndOfLifeLocked(ACameraCaptureSession* session); 223 224 camera_status_t configureStreamsLocked(const ACaptureSessionOutputContainer* outputs, 225 const ACaptureRequest* sessionParameters, nsecs_t startTimeNs); 226 227 // Input message will be posted and cleared after this returns 228 void postSessionMsgAndCleanup(sp<AMessage>& msg); 229 230 mutable Mutex mDeviceLock; 231 const std::string mCameraId; // Camera ID 232 const ACameraDevice_StateCallbacks mAppCallbacks; // Callback to app 233 const sp<ACameraMetadata> mChars; // Camera characteristics 234 std::shared_ptr<ServiceCallback> mServiceCallback; 235 ACameraDevice* mWrapper; 236 237 // stream id -> pair of (ACameraWindowType* from application, OutputConfiguration used for 238 // camera service) 239 std::map<int, std::pair<native_handle_ptr_wrapper, OutputConfiguration>> mConfiguredOutputs; 240 241 // TODO: maybe a bool will suffice for synchronous implementation? 242 std::atomic_bool mClosing; isClosed()243 inline bool isClosed() { return mClosing; } 244 245 bool mInError = false; 246 camera_status_t mError = ACAMERA_OK; 247 void onCaptureErrorLocked( 248 ErrorCode errorCode, 249 const CaptureResultExtras& resultExtras); 250 251 bool mIdle = true; 252 // This will avoid a busy session being deleted before it's back to idle state 253 sp<ACameraCaptureSession> mBusySession; 254 255 std::shared_ptr<ICameraDeviceUser> mRemote; 256 257 // Looper thread to handle callback to app 258 sp<ALooper> mCbLooper; 259 // definition of handler and message 260 enum { 261 // Device state callbacks 262 kWhatOnDisconnected, // onDisconnected 263 kWhatOnError, // onError 264 // Session state callbacks 265 kWhatSessionStateCb, // onReady, onActive 266 // Capture callbacks 267 kWhatCaptureStart, // onCaptureStarted 268 kWhatCaptureStart2, // onCaptureStarted2 269 kWhatCaptureResult, // onCaptureProgressed, onCaptureCompleted 270 kWhatLogicalCaptureResult, // onLogicalCameraCaptureCompleted 271 kWhatCaptureFail, // onCaptureFailed 272 kWhatLogicalCaptureFail, // onLogicalCameraCaptureFailed 273 kWhatCaptureSeqEnd, // onCaptureSequenceCompleted 274 kWhatCaptureSeqAbort, // onCaptureSequenceAborted 275 kWhatCaptureBufferLost, // onCaptureBufferLost 276 kWhatPreparedCb, // onPrepared 277 // Internal cleanup 278 kWhatCleanUpSessions // Cleanup cached sp<ACameraCaptureSession> 279 }; 280 static const char* kContextKey; 281 static const char* kDeviceKey; 282 static const char* kErrorCodeKey; 283 static const char* kCallbackFpKey; 284 static const char* kSessionSpKey; 285 static const char* kCaptureRequestKey; 286 static const char* kTimeStampKey; 287 static const char* kCaptureResultKey; 288 static const char* kPhysicalCaptureResultKey; 289 static const char* kCaptureFailureKey; 290 static const char* kSequenceIdKey; 291 static const char* kFrameNumberKey; 292 static const char* kAnwKey; 293 static const char* kFailingPhysicalCameraId; 294 295 class CallbackHandler : public AHandler { 296 public: 297 explicit CallbackHandler(const char *id); 298 void onMessageReceived(const sp<AMessage> &msg) override; 299 300 private: 301 std::string mId; 302 // This handler will cache all capture session sp until kWhatCleanUpSessions 303 // is processed. This is used to guarantee the last session reference is always 304 // being removed in callback thread without holding camera device lock 305 std::vector<sp<ACameraCaptureSession>> mCachedSessions; 306 }; 307 sp<CallbackHandler> mHandler; 308 309 /*********************************** 310 * Capture session related members * 311 ***********************************/ 312 // The current active session 313 wp<ACameraCaptureSession> mCurrentSession; 314 bool mFlushing = false; 315 316 int mNextSessionId = 0; 317 // TODO: might need another looper/handler to handle callbacks from service 318 319 static const int REQUEST_ID_NONE = -1; 320 int mRepeatingSequenceId = REQUEST_ID_NONE; 321 322 // sequence id -> last frame number map 323 std::map<int32_t, int64_t> mSequenceLastFrameNumberMap; 324 325 struct CallbackHolder { 326 CallbackHolder(sp<ACameraCaptureSession> session, 327 std::vector<sp<CaptureRequest>> requests, 328 bool isRepeating, 329 ACameraCaptureSession_captureCallbacks* cbs); 330 CallbackHolder(sp<ACameraCaptureSession> session, 331 std::vector<sp<CaptureRequest>> requests, 332 bool isRepeating, 333 ACameraCaptureSession_logicalCamera_captureCallbacks* lcbs); 334 CallbackHolder(sp<ACameraCaptureSession> session, 335 std::vector<sp<CaptureRequest> > requests, 336 bool isRepeating, 337 ACameraCaptureSession_captureCallbacksV2* cbs); 338 CallbackHolder(sp<ACameraCaptureSession> session, 339 std::vector<sp<CaptureRequest> > requests, 340 bool isRepeating, 341 ACameraCaptureSession_logicalCamera_captureCallbacksV2* lcbs); clearCallbacksCallbackHolder342 void clearCallbacks() { 343 mContext = nullptr; 344 mOnCaptureStarted = nullptr; 345 mOnCaptureStarted2 = nullptr; 346 mOnCaptureProgressed = nullptr; 347 mOnCaptureCompleted = nullptr; 348 mOnLogicalCameraCaptureCompleted = nullptr; 349 mOnLogicalCameraCaptureFailed = nullptr; 350 mOnCaptureFailed = nullptr; 351 mOnCaptureSequenceCompleted = nullptr; 352 mOnCaptureSequenceAborted = nullptr; 353 mOnCaptureBufferLost = nullptr; 354 } 355 356 template <class T> initCaptureCallbacksV2CallbackHolder357 void initCaptureCallbacksV2(T* cbs) { 358 clearCallbacks(); 359 if (cbs != nullptr) { 360 mContext = cbs->context; 361 mOnCaptureStarted2 = cbs->onCaptureStarted; 362 mOnCaptureProgressed = cbs->onCaptureProgressed; 363 mOnCaptureSequenceCompleted = cbs->onCaptureSequenceCompleted; 364 mOnCaptureSequenceAborted = cbs->onCaptureSequenceAborted; 365 mOnCaptureBufferLost = cbs->onCaptureBufferLost; 366 } 367 } 368 369 template <class T> initCaptureCallbacksCallbackHolder370 void initCaptureCallbacks(T* cbs) { 371 clearCallbacks(); 372 if (cbs != nullptr) { 373 mContext = cbs->context; 374 mOnCaptureStarted = cbs->onCaptureStarted; 375 mOnCaptureProgressed = cbs->onCaptureProgressed; 376 mOnCaptureSequenceCompleted = cbs->onCaptureSequenceCompleted; 377 mOnCaptureSequenceAborted = cbs->onCaptureSequenceAborted; 378 mOnCaptureBufferLost = cbs->onCaptureBufferLost; 379 } 380 } 381 382 sp<ACameraCaptureSession> mSession; 383 std::vector<sp<CaptureRequest>> mRequests; 384 const bool mIsRepeating; 385 const bool mIs2Callback; 386 const bool mIsLogicalCameraCallback; 387 388 void* mContext; 389 ACameraCaptureSession_captureCallback_start mOnCaptureStarted; 390 ACameraCaptureSession_captureCallback_startV2 mOnCaptureStarted2; 391 ACameraCaptureSession_captureCallback_result mOnCaptureProgressed; 392 ACameraCaptureSession_captureCallback_result mOnCaptureCompleted; 393 ACameraCaptureSession_logicalCamera_captureCallback_result mOnLogicalCameraCaptureCompleted; 394 ACameraCaptureSession_logicalCamera_captureCallback_failed mOnLogicalCameraCaptureFailed; 395 ACameraCaptureSession_captureCallback_failed mOnCaptureFailed; 396 ACameraCaptureSession_captureCallback_sequenceEnd mOnCaptureSequenceCompleted; 397 ACameraCaptureSession_captureCallback_sequenceAbort mOnCaptureSequenceAborted; 398 ACameraCaptureSession_captureCallback_bufferLost mOnCaptureBufferLost; 399 }; 400 // sequence id -> callbacks map 401 std::map<int, CallbackHolder> mSequenceCallbackMap; 402 403 static const int64_t NO_FRAMES_CAPTURED = -1; 404 class FrameNumberTracker { 405 public: 406 // TODO: Called in onResultReceived and onCaptureErrorLocked 407 void updateTracker(int64_t frameNumber, bool isError); getCompletedFrameNumber()408 inline int64_t getCompletedFrameNumber() { return mCompletedFrameNumber; } 409 private: 410 void update(); 411 void updateCompletedFrameNumber(int64_t frameNumber); 412 413 int64_t mCompletedFrameNumber = NO_FRAMES_CAPTURED; 414 List<int64_t> mSkippedFrameNumbers; 415 std::set<int64_t> mFutureErrorSet; 416 }; 417 FrameNumberTracker mFrameNumberTracker; 418 419 void checkRepeatingSequenceCompleteLocked(const int sequenceId, const int64_t lastFrameNumber); 420 void checkAndFireSequenceCompleteLocked(); 421 422 // Misc variables 423 int32_t mShadingMapSize[2]; // const after constructor 424 int32_t mPartialResultCount; // const after constructor 425 std::shared_ptr<RequestMetadataQueue> mCaptureRequestMetadataQueue = nullptr; 426 std::shared_ptr<ResultMetadataQueue> mCaptureResultMetadataQueue = nullptr; 427 }; 428 429 } // namespace acam; 430 } // namespace android; 431 432 /** 433 * ACameraDevice opaque struct definition 434 * Leave outside of android namespace because it's NDK struct 435 */ 436 struct ACameraDevice { ACameraDeviceACameraDevice437 ACameraDevice(const char* id, ACameraDevice_StateCallbacks* cb, 438 sp<ACameraMetadata> chars) : 439 mDevice(std::make_shared<android::acam::CameraDevice>(id, cb, 440 std::move(chars), this)) { 441 mDevice->init(); 442 } 443 444 ~ACameraDevice(); 445 /******************* 446 * NDK public APIs * 447 *******************/ getIdACameraDevice448 inline const char* getId() const { return mDevice->getId(); } 449 createCaptureRequestACameraDevice450 camera_status_t createCaptureRequest( 451 ACameraDevice_request_template templateId, 452 const ACameraIdList* physicalCameraIdList, 453 ACaptureRequest** request) const { 454 return mDevice->createCaptureRequest(templateId, physicalCameraIdList, request); 455 } 456 createCaptureSessionACameraDevice457 camera_status_t createCaptureSession( 458 const ACaptureSessionOutputContainer* outputs, 459 const ACaptureRequest* sessionParameters, 460 const ACameraCaptureSession_stateCallbacks* callbacks, 461 /*out*/ACameraCaptureSession** session) { 462 return mDevice->createCaptureSession(outputs, sessionParameters, callbacks, session); 463 } 464 isSessionConfigurationSupportedACameraDevice465 camera_status_t isSessionConfigurationSupported( 466 const ACaptureSessionOutputContainer* sessionOutputContainer) const { 467 return mDevice->isSessionConfigurationSupported(sessionOutputContainer); 468 } 469 470 /*********************** 471 * Device interal APIs * 472 ***********************/ getServiceCallbackACameraDevice473 inline std::shared_ptr<android::acam::BnCameraDeviceCallback> getServiceCallback() { 474 return mDevice->getServiceCallback(); 475 }; 476 477 // Camera device is only functional after remote being set setRemoteDeviceACameraDevice478 inline void setRemoteDevice(std::shared_ptr< 479 ::aidl::android::frameworks::cameraservice::device::ICameraDeviceUser> remote) { 480 mDevice->setRemoteDevice(remote); 481 } setDeviceMetadataQueuesACameraDevice482 inline bool setDeviceMetadataQueues() { 483 return mDevice->setDeviceMetadataQueues(); 484 } 485 private: 486 std::shared_ptr<android::acam::CameraDevice> mDevice; 487 }; 488 489 #endif // _ACAMERA_DEVICE_H 490