1 /* 2 * Copyright (C) 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 #ifndef _ACAMERA_DEVICE_H 17 #define _ACAMERA_DEVICE_H 18 19 #include <memory> 20 #include <map> 21 #include <set> 22 #include <atomic> 23 #include <utility> 24 #include <vector> 25 #include <utils/StrongPointer.h> 26 #include <utils/Mutex.h> 27 #include <utils/List.h> 28 #include <utils/Vector.h> 29 #include <android/frameworks/cameraservice/device/2.1/ICameraDeviceUser.h> 30 #include <android/frameworks/cameraservice/device/2.0/ICameraDeviceCallback.h> 31 #include <android/frameworks/cameraservice/device/2.0/types.h> 32 #include <fmq/MessageQueue.h> 33 #include <media/stagefright/foundation/ALooper.h> 34 #include <media/stagefright/foundation/AHandler.h> 35 #include <media/stagefright/foundation/AMessage.h> 36 37 #include <camera/NdkCameraManager.h> 38 #include <camera/NdkCameraCaptureSession.h> 39 40 #include "ACameraMetadata.h" 41 #include "utils.h" 42 43 namespace android { 44 namespace acam { 45 46 using ICameraDeviceCallback = frameworks::cameraservice::device::V2_0::ICameraDeviceCallback; 47 using ICameraDeviceUser_2_0 = frameworks::cameraservice::device::V2_0::ICameraDeviceUser; 48 using ICameraDeviceUser = frameworks::cameraservice::device::V2_1::ICameraDeviceUser; 49 using CaptureResultExtras = frameworks::cameraservice::device::V2_0::CaptureResultExtras; 50 using PhysicalCaptureResultInfo = frameworks::cameraservice::device::V2_0::PhysicalCaptureResultInfo; 51 using PhysicalCameraSettings = frameworks::cameraservice::device::V2_0::PhysicalCameraSettings; 52 using SubmitInfo = frameworks::cameraservice::device::V2_0::SubmitInfo; 53 using CaptureResultExtras = frameworks::cameraservice::device::V2_0::CaptureResultExtras; 54 using ErrorCode = frameworks::cameraservice::device::V2_0::ErrorCode; 55 using FmqSizeOrMetadata = frameworks::cameraservice::device::V2_0::FmqSizeOrMetadata; 56 using StreamConfigurationMode = frameworks::cameraservice::device::V2_0::StreamConfigurationMode; 57 using Status = frameworks::cameraservice::common::V2_0::Status; 58 using ResultMetadataQueue = hardware::MessageQueue<uint8_t, hardware::kSynchronizedReadWrite>; 59 using RequestMetadataQueue = hardware::MessageQueue<uint8_t, hardware::kSynchronizedReadWrite>; 60 using CameraStatusAndId = frameworks::cameraservice::service::V2_0::CameraStatusAndId; 61 62 using hardware::hidl_vec; 63 using hardware::hidl_string; 64 using utils::native_handle_ptr_wrapper; 65 using utils::CaptureRequest; 66 using utils::OutputConfigurationWrapper; 67 68 // Wrap ACameraCaptureFailure so it can be ref-counted 69 struct CameraCaptureFailure : public RefBase, public ACameraCaptureFailure { }; 70 71 // Wrap PhysicalCaptureResultInfo so that it can be ref-counted 72 struct PhysicalCaptureResultInfoLocal { 73 std::string physicalCameraId; 74 CameraMetadata physicalMetadata; 75 }; 76 77 struct ACameraPhysicalCaptureResultInfo: public RefBase { ACameraPhysicalCaptureResultInfoACameraPhysicalCaptureResultInfo78 ACameraPhysicalCaptureResultInfo(const std::vector<PhysicalCaptureResultInfoLocal>& info, 79 int64_t frameNumber) : 80 mPhysicalResultInfo(info), mFrameNumber(frameNumber) {} 81 82 std::vector<PhysicalCaptureResultInfoLocal> mPhysicalResultInfo; 83 int64_t mFrameNumber; 84 }; 85 86 class CameraDevice final : public RefBase { 87 public: 88 CameraDevice(const char* id, ACameraDevice_StateCallbacks* cb, 89 sp<ACameraMetadata> chars, 90 ACameraDevice* wrapper); 91 ~CameraDevice(); 92 getId()93 inline const char* getId() const { return mCameraId.c_str(); } 94 95 camera_status_t createCaptureRequest( 96 ACameraDevice_request_template templateId, 97 const ACameraIdList* physicalCameraIdList, 98 ACaptureRequest** request) const; 99 100 camera_status_t createCaptureSession( 101 const ACaptureSessionOutputContainer* outputs, 102 const ACaptureRequest* sessionParameters, 103 const ACameraCaptureSession_stateCallbacks* callbacks, 104 /*out*/ACameraCaptureSession** session); 105 106 camera_status_t isSessionConfigurationSupported( 107 const ACaptureSessionOutputContainer* sessionOutputContainer) const; 108 109 // Callbacks from camera service 110 class ServiceCallback : public ICameraDeviceCallback { 111 public: ServiceCallback(CameraDevice * device)112 explicit ServiceCallback(CameraDevice* device) : mDevice(device) {} 113 android::hardware::Return<void> onDeviceError(ErrorCode errorCode, 114 const CaptureResultExtras& resultExtras) override; 115 android::hardware::Return<void> onDeviceIdle() override; 116 android::hardware::Return<void> onCaptureStarted(const CaptureResultExtras& resultExtras, 117 uint64_t timestamp) override; 118 android::hardware::Return<void> onResultReceived(const FmqSizeOrMetadata& result, 119 const CaptureResultExtras& resultExtras, 120 const hidl_vec<PhysicalCaptureResultInfo>& physicalResultInfos) override; 121 android::hardware::Return<void> onRepeatingRequestError(uint64_t lastFrameNumber, 122 int32_t stoppedSequenceId) override; 123 private: 124 camera_status_t readOneResultMetadata(const FmqSizeOrMetadata& fmqSizeOrMetadata, 125 ResultMetadataQueue* metadataQueue, CameraMetadata* metadata); 126 const wp<CameraDevice> mDevice; 127 }; getServiceCallback()128 inline sp<ICameraDeviceCallback> getServiceCallback() { 129 return mServiceCallback; 130 }; 131 132 // Camera device is only functional after remote being set 133 void setRemoteDevice(sp<ICameraDeviceUser> remote); 134 135 bool setDeviceMetadataQueues(); getWrapper()136 inline ACameraDevice* getWrapper() const { return mWrapper; }; 137 138 // Stop the looper thread and unregister the handler 139 void stopLooperAndDisconnect(); 140 141 private: 142 friend ACameraCaptureSession; 143 friend ACameraDevice; 144 145 camera_status_t checkCameraClosedOrErrorLocked() const; 146 147 // device goes into fatal error state after this 148 void setCameraDeviceErrorLocked(camera_status_t error); 149 150 void disconnectLocked(sp<ACameraCaptureSession>& session); // disconnect from camera service 151 152 camera_status_t stopRepeatingLocked(); 153 154 camera_status_t flushLocked(ACameraCaptureSession*); 155 156 camera_status_t waitUntilIdleLocked(); 157 158 template<class T> 159 camera_status_t captureLocked(sp<ACameraCaptureSession> session, 160 /*optional*/T* cbs, 161 int numRequests, ACaptureRequest** requests, 162 /*optional*/int* captureSequenceId); 163 164 template<class T> 165 camera_status_t setRepeatingRequestsLocked(sp<ACameraCaptureSession> session, 166 /*optional*/T* cbs, 167 int numRequests, ACaptureRequest** requests, 168 /*optional*/int* captureSequenceId); 169 170 template<class T> 171 camera_status_t submitRequestsLocked( 172 sp<ACameraCaptureSession> session, 173 /*optional*/T* cbs, 174 int numRequests, ACaptureRequest** requests, 175 /*out*/int* captureSequenceId, 176 bool isRepeating); 177 178 void addRequestSettingsMetadata(ACaptureRequest *aCaptureRequest, sp<CaptureRequest> &req); 179 180 camera_status_t updateOutputConfigurationLocked(ACaptureSessionOutput *output); 181 182 // Since this writes to ICameraDeviceUser's fmq, clients must take care that: 183 // a) This function is called serially. 184 // b) This function is called in accordance with ICameraDeviceUser.submitRequestList, 185 // otherwise, the wrong capture request might have the wrong settings 186 // metadata associated with it. 187 camera_status_t allocateCaptureRequestLocked( 188 const ACaptureRequest* request, sp<CaptureRequest>& outReq); 189 void allocateOneCaptureRequestMetadata( 190 PhysicalCameraSettings& cameraSettings, 191 const std::string& id, const sp<ACameraMetadata>& metadata); 192 193 static ACaptureRequest* allocateACaptureRequest(sp<CaptureRequest>& req, const char* deviceId); 194 static void freeACaptureRequest(ACaptureRequest*); 195 196 // only For session to hold device lock 197 // Always grab device lock before grabbing session lock lockDeviceForSessionOps()198 void lockDeviceForSessionOps() const { mDeviceLock.lock(); }; unlockDevice()199 void unlockDevice() const { mDeviceLock.unlock(); }; 200 201 // For capture session to notify its end of life 202 void notifySessionEndOfLifeLocked(ACameraCaptureSession* session); 203 204 camera_status_t configureStreamsLocked(const ACaptureSessionOutputContainer* outputs, 205 const ACaptureRequest* sessionParameters, nsecs_t startTimeNs); 206 207 // Input message will be posted and cleared after this returns 208 void postSessionMsgAndCleanup(sp<AMessage>& msg); 209 210 mutable Mutex mDeviceLock; 211 const hidl_string mCameraId; // Camera ID 212 const ACameraDevice_StateCallbacks mAppCallbacks; // Callback to app 213 const sp<ACameraMetadata> mChars; // Camera characteristics 214 const sp<ServiceCallback> mServiceCallback; 215 ACameraDevice* mWrapper; 216 217 // stream id -> pair of (ACameraWindowType* from application, OutputConfiguration used for 218 // camera service) 219 std::map<int, std::pair<native_handle_ptr_wrapper, OutputConfigurationWrapper>> mConfiguredOutputs; 220 221 // TODO: maybe a bool will suffice for synchronous implementation? 222 std::atomic_bool mClosing; isClosed()223 inline bool isClosed() { return mClosing; } 224 225 bool mInError = false; 226 camera_status_t mError = ACAMERA_OK; 227 void onCaptureErrorLocked( 228 ErrorCode errorCode, 229 const CaptureResultExtras& resultExtras); 230 231 bool mIdle = true; 232 // This will avoid a busy session being deleted before it's back to idle state 233 sp<ACameraCaptureSession> mBusySession; 234 235 sp<ICameraDeviceUser> mRemote; 236 237 // Looper thread to handle callback to app 238 sp<ALooper> mCbLooper; 239 // definition of handler and message 240 enum { 241 // Device state callbacks 242 kWhatOnDisconnected, // onDisconnected 243 kWhatOnError, // onError 244 // Session state callbacks 245 kWhatSessionStateCb, // onReady, onActive 246 // Capture callbacks 247 kWhatCaptureStart, // onCaptureStarted 248 kWhatCaptureStart2, // onCaptureStarted2 249 kWhatCaptureResult, // onCaptureProgressed, onCaptureCompleted 250 kWhatLogicalCaptureResult, // onLogicalCameraCaptureCompleted 251 kWhatCaptureFail, // onCaptureFailed 252 kWhatLogicalCaptureFail, // onLogicalCameraCaptureFailed 253 kWhatCaptureSeqEnd, // onCaptureSequenceCompleted 254 kWhatCaptureSeqAbort, // onCaptureSequenceAborted 255 kWhatCaptureBufferLost,// onCaptureBufferLost 256 // Internal cleanup 257 kWhatCleanUpSessions // Cleanup cached sp<ACameraCaptureSession> 258 }; 259 static const char* kContextKey; 260 static const char* kDeviceKey; 261 static const char* kErrorCodeKey; 262 static const char* kCallbackFpKey; 263 static const char* kSessionSpKey; 264 static const char* kCaptureRequestKey; 265 static const char* kTimeStampKey; 266 static const char* kCaptureResultKey; 267 static const char* kPhysicalCaptureResultKey; 268 static const char* kCaptureFailureKey; 269 static const char* kSequenceIdKey; 270 static const char* kFrameNumberKey; 271 static const char* kAnwKey; 272 static const char* kFailingPhysicalCameraId; 273 274 class CallbackHandler : public AHandler { 275 public: 276 explicit CallbackHandler(const char *id); 277 void onMessageReceived(const sp<AMessage> &msg) override; 278 279 private: 280 std::string mId; 281 // This handler will cache all capture session sp until kWhatCleanUpSessions 282 // is processed. This is used to guarantee the last session reference is always 283 // being removed in callback thread without holding camera device lock 284 Vector<sp<ACameraCaptureSession>> mCachedSessions; 285 }; 286 sp<CallbackHandler> mHandler; 287 288 /*********************************** 289 * Capture session related members * 290 ***********************************/ 291 // The current active session 292 wp<ACameraCaptureSession> mCurrentSession; 293 bool mFlushing = false; 294 295 int mNextSessionId = 0; 296 // TODO: might need another looper/handler to handle callbacks from service 297 298 static const int REQUEST_ID_NONE = -1; 299 int mRepeatingSequenceId = REQUEST_ID_NONE; 300 301 // sequence id -> last frame number map 302 std::map<int32_t, int64_t> mSequenceLastFrameNumberMap; 303 304 struct CallbackHolder { 305 CallbackHolder(sp<ACameraCaptureSession> session, 306 const Vector<sp<CaptureRequest>>& requests, 307 bool isRepeating, 308 ACameraCaptureSession_captureCallbacks* cbs); 309 CallbackHolder(sp<ACameraCaptureSession> session, 310 const Vector<sp<CaptureRequest>>& requests, 311 bool isRepeating, 312 ACameraCaptureSession_logicalCamera_captureCallbacks* lcbs); 313 CallbackHolder(sp<ACameraCaptureSession> session, 314 const Vector<sp<CaptureRequest> >& requests, 315 bool isRepeating, 316 ACameraCaptureSession_captureCallbacksV2* cbs); 317 CallbackHolder(sp<ACameraCaptureSession> session, 318 const Vector<sp<CaptureRequest> >& requests, 319 bool isRepeating, 320 ACameraCaptureSession_logicalCamera_captureCallbacksV2* lcbs); clearCallbacksCallbackHolder321 void clearCallbacks() { 322 mContext = nullptr; 323 mOnCaptureStarted = nullptr; 324 mOnCaptureStarted2 = nullptr; 325 mOnCaptureProgressed = nullptr; 326 mOnCaptureCompleted = nullptr; 327 mOnLogicalCameraCaptureCompleted = nullptr; 328 mOnLogicalCameraCaptureFailed = nullptr; 329 mOnCaptureFailed = nullptr; 330 mOnCaptureSequenceCompleted = nullptr; 331 mOnCaptureSequenceAborted = nullptr; 332 mOnCaptureBufferLost = nullptr; 333 } 334 335 template <class T> initCaptureCallbacksV2CallbackHolder336 void initCaptureCallbacksV2(T* cbs) { 337 clearCallbacks(); 338 if (cbs != nullptr) { 339 mContext = cbs->context; 340 mOnCaptureStarted2 = cbs->onCaptureStarted; 341 mOnCaptureProgressed = cbs->onCaptureProgressed; 342 mOnCaptureSequenceCompleted = cbs->onCaptureSequenceCompleted; 343 mOnCaptureSequenceAborted = cbs->onCaptureSequenceAborted; 344 mOnCaptureBufferLost = cbs->onCaptureBufferLost; 345 } 346 } 347 348 template <class T> initCaptureCallbacksCallbackHolder349 void initCaptureCallbacks(T* cbs) { 350 clearCallbacks(); 351 if (cbs != nullptr) { 352 mContext = cbs->context; 353 mOnCaptureStarted = cbs->onCaptureStarted; 354 mOnCaptureProgressed = cbs->onCaptureProgressed; 355 mOnCaptureSequenceCompleted = cbs->onCaptureSequenceCompleted; 356 mOnCaptureSequenceAborted = cbs->onCaptureSequenceAborted; 357 mOnCaptureBufferLost = cbs->onCaptureBufferLost; 358 } 359 } 360 361 sp<ACameraCaptureSession> mSession; 362 Vector<sp<CaptureRequest>> mRequests; 363 const bool mIsRepeating; 364 const bool mIs2Callback; 365 const bool mIsLogicalCameraCallback; 366 367 void* mContext; 368 ACameraCaptureSession_captureCallback_start mOnCaptureStarted; 369 ACameraCaptureSession_captureCallback_startV2 mOnCaptureStarted2; 370 ACameraCaptureSession_captureCallback_result mOnCaptureProgressed; 371 ACameraCaptureSession_captureCallback_result mOnCaptureCompleted; 372 ACameraCaptureSession_logicalCamera_captureCallback_result mOnLogicalCameraCaptureCompleted; 373 ACameraCaptureSession_logicalCamera_captureCallback_failed mOnLogicalCameraCaptureFailed; 374 ACameraCaptureSession_captureCallback_failed mOnCaptureFailed; 375 ACameraCaptureSession_captureCallback_sequenceEnd mOnCaptureSequenceCompleted; 376 ACameraCaptureSession_captureCallback_sequenceAbort mOnCaptureSequenceAborted; 377 ACameraCaptureSession_captureCallback_bufferLost mOnCaptureBufferLost; 378 }; 379 // sequence id -> callbacks map 380 std::map<int, CallbackHolder> mSequenceCallbackMap; 381 382 static const int64_t NO_FRAMES_CAPTURED = -1; 383 class FrameNumberTracker { 384 public: 385 // TODO: Called in onResultReceived and onCaptureErrorLocked 386 void updateTracker(int64_t frameNumber, bool isError); getCompletedFrameNumber()387 inline int64_t getCompletedFrameNumber() { return mCompletedFrameNumber; } 388 private: 389 void update(); 390 void updateCompletedFrameNumber(int64_t frameNumber); 391 392 int64_t mCompletedFrameNumber = NO_FRAMES_CAPTURED; 393 List<int64_t> mSkippedFrameNumbers; 394 std::set<int64_t> mFutureErrorSet; 395 }; 396 FrameNumberTracker mFrameNumberTracker; 397 398 void checkRepeatingSequenceCompleteLocked(const int sequenceId, const int64_t lastFrameNumber); 399 void checkAndFireSequenceCompleteLocked(); 400 401 // Misc variables 402 int32_t mShadingMapSize[2]; // const after constructor 403 int32_t mPartialResultCount; // const after constructor 404 std::shared_ptr<ResultMetadataQueue> mCaptureRequestMetadataQueue = nullptr; 405 std::shared_ptr<ResultMetadataQueue> mCaptureResultMetadataQueue = nullptr; 406 }; 407 408 } // namespace acam; 409 } // namespace android; 410 411 /** 412 * ACameraDevice opaque struct definition 413 * Leave outside of android namespace because it's NDK struct 414 */ 415 struct ACameraDevice { ACameraDeviceACameraDevice416 ACameraDevice(const char* id, ACameraDevice_StateCallbacks* cb, 417 sp<ACameraMetadata> chars) : 418 mDevice(new android::acam::CameraDevice(id, cb, std::move(chars), this)) {} 419 420 ~ACameraDevice(); 421 /******************* 422 * NDK public APIs * 423 *******************/ getIdACameraDevice424 inline const char* getId() const { return mDevice->getId(); } 425 createCaptureRequestACameraDevice426 camera_status_t createCaptureRequest( 427 ACameraDevice_request_template templateId, 428 const ACameraIdList* physicalCameraIdList, 429 ACaptureRequest** request) const { 430 return mDevice->createCaptureRequest(templateId, physicalCameraIdList, request); 431 } 432 createCaptureSessionACameraDevice433 camera_status_t createCaptureSession( 434 const ACaptureSessionOutputContainer* outputs, 435 const ACaptureRequest* sessionParameters, 436 const ACameraCaptureSession_stateCallbacks* callbacks, 437 /*out*/ACameraCaptureSession** session) { 438 return mDevice->createCaptureSession(outputs, sessionParameters, callbacks, session); 439 } 440 isSessionConfigurationSupportedACameraDevice441 camera_status_t isSessionConfigurationSupported( 442 const ACaptureSessionOutputContainer* sessionOutputContainer) const { 443 return mDevice->isSessionConfigurationSupported(sessionOutputContainer); 444 } 445 446 /*********************** 447 * Device interal APIs * 448 ***********************/ getServiceCallbackACameraDevice449 inline android::sp<android::acam::ICameraDeviceCallback> getServiceCallback() { 450 return mDevice->getServiceCallback(); 451 }; 452 453 // Camera device is only functional after remote being set setRemoteDeviceACameraDevice454 inline void setRemoteDevice(android::sp<android::acam::ICameraDeviceUser> remote) { 455 mDevice->setRemoteDevice(remote); 456 } setDeviceMetadataQueuesACameraDevice457 inline bool setDeviceMetadataQueues() { 458 return mDevice->setDeviceMetadataQueues(); 459 } 460 private: 461 android::sp<android::acam::CameraDevice> mDevice; 462 }; 463 464 #endif // _ACAMERA_DEVICE_H 465