1 /* 2 * Copyright (C) 2015 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 #ifndef _ACAMERA_DEVICE_H 17 #define _ACAMERA_DEVICE_H 18 19 #include <memory> 20 #include <map> 21 #include <set> 22 #include <atomic> 23 #include <utility> 24 #include <vector> 25 #include <utils/StrongPointer.h> 26 #include <utils/Mutex.h> 27 #include <utils/String8.h> 28 #include <utils/List.h> 29 #include <utils/Vector.h> 30 31 #include <android/hardware/camera2/BnCameraDeviceCallbacks.h> 32 #include <android/hardware/camera2/ICameraDeviceUser.h> 33 #include <media/stagefright/foundation/ALooper.h> 34 #include <media/stagefright/foundation/AHandler.h> 35 #include <media/stagefright/foundation/AMessage.h> 36 #include <camera/CaptureResult.h> 37 #include <camera/camera2/OutputConfiguration.h> 38 #include <camera/camera2/SessionConfiguration.h> 39 #include <camera/camera2/CaptureRequest.h> 40 #include <fmq/AidlMessageQueueCpp.h> 41 42 #include <camera/NdkCameraCaptureSession.h> 43 #include <camera/NdkCameraManager.h> 44 #include <gui/Flags.h> // remove with WB_LIBCAMERASERVICE_WITH_DEPENDENCIES 45 46 #include "ACameraMetadata.h" 47 48 namespace android { 49 namespace acam { 50 51 using android::hardware::common::fmq::SynchronizedReadWrite; 52 using ResultMetadataQueue = AidlMessageQueueCpp<int8_t, SynchronizedReadWrite>; 53 54 // Wrap ACameraCaptureFailure so it can be ref-counted 55 struct CameraCaptureFailure : public RefBase, public ACameraCaptureFailure {}; 56 57 // Wrap PhysicalCaptureResultInfo so that it can be ref-counted 58 struct ACameraPhysicalCaptureResultInfo: public RefBase { ACameraPhysicalCaptureResultInfoACameraPhysicalCaptureResultInfo59 ACameraPhysicalCaptureResultInfo(const std::vector<PhysicalCaptureResultInfo>& info, 60 int64_t frameNumber) : 61 mPhysicalResultInfo(info), mFrameNumber(frameNumber) {} 62 63 std::vector<PhysicalCaptureResultInfo> mPhysicalResultInfo; 64 int64_t mFrameNumber; 65 }; 66 67 class CameraDevice final : public RefBase { 68 public: 69 70 using CameraMetadataInfo = android::hardware::camera2::CameraMetadataInfo; 71 CameraDevice(const char* id, ACameraDevice_StateCallbacks* cb, 72 sp<ACameraMetadata> chars, 73 ACameraDevice* wrapper, bool sharedMode); 74 ~CameraDevice(); 75 getId()76 inline const char* getId() const { return mCameraId.c_str(); } 77 78 camera_status_t createCaptureRequest( 79 ACameraDevice_request_template templateId, 80 const ACameraIdList* physicalIdList, 81 ACaptureRequest** request) const; 82 83 camera_status_t createCaptureSession( 84 const ACaptureSessionOutputContainer* outputs, 85 const ACaptureRequest* sessionParameters, 86 const ACameraCaptureSession_stateCallbacks* callbacks, 87 /*out*/ACameraCaptureSession** session); 88 89 camera_status_t isSessionConfigurationSupported( 90 const ACaptureSessionOutputContainer* sessionOutputContainer) const; 91 92 // Callbacks from camera service 93 class ServiceCallback : public hardware::camera2::BnCameraDeviceCallbacks { 94 public: ServiceCallback(CameraDevice * device)95 explicit ServiceCallback(CameraDevice* device) : mDevice(device) {} 96 binder::Status onDeviceError(int32_t errorCode, 97 const CaptureResultExtras& resultExtras) override; 98 binder::Status onDeviceIdle() override; 99 binder::Status onCaptureStarted(const CaptureResultExtras& resultExtras, 100 int64_t timestamp) override; 101 binder::Status onResultReceived(const CameraMetadataInfo &resultInfo, 102 const CaptureResultExtras& resultExtras, 103 const std::vector<PhysicalCaptureResultInfo>& physicalResultInfos) override; 104 binder::Status onPrepared(int streamId) override; 105 binder::Status onRequestQueueEmpty() override; 106 binder::Status onRepeatingRequestError(int64_t lastFrameNumber, 107 int32_t stoppedSequenceId) override; 108 binder::Status onClientSharedAccessPriorityChanged(bool isPrimaryClient) override; 109 private: 110 camera_status_t readOneResultMetadata( 111 const CameraMetadataInfo& resultInfo, ResultMetadataQueue* metadataQueue, 112 CameraMetadata* metadata); 113 const wp<CameraDevice> mDevice; 114 }; getServiceCallback()115 inline sp<hardware::camera2::ICameraDeviceCallbacks> getServiceCallback() { 116 return mServiceCallback; 117 }; 118 119 // Camera device is only functional after remote being set 120 void setRemoteDevice(sp<hardware::camera2::ICameraDeviceUser> remote); 121 bool setDeviceMetadataQueues(); 122 getWrapper()123 inline ACameraDevice* getWrapper() const { return mWrapper; }; 124 125 // Stop the looper thread and unregister the handler 126 void stopLooperAndDisconnect(); 127 setPrimaryClient(bool isPrimary)128 void setPrimaryClient(bool isPrimary) {mIsPrimaryClient = isPrimary;}; isPrimaryClient()129 bool isPrimaryClient() {return mIsPrimaryClient;}; isSharedMode()130 bool isSharedMode() {return mSharedMode;}; 131 132 private: 133 friend ACameraCaptureSession; 134 camera_status_t checkCameraClosedOrErrorLocked() const; 135 136 // device goes into fatal error state after this 137 void setCameraDeviceErrorLocked(camera_status_t error); 138 139 void disconnectLocked(sp<ACameraCaptureSession>& session); // disconnect from camera service 140 141 camera_status_t stopRepeatingLocked(); 142 143 camera_status_t flushLocked(ACameraCaptureSession*); 144 145 camera_status_t waitUntilIdleLocked(); 146 147 camera_status_t stopStreamingLocked(); 148 149 template<class T> 150 camera_status_t startStreamingLocked(ACameraCaptureSession* session, 151 /*optional*/T* callbacks, 152 int numOutputWindows, ANativeWindow** windows, /*optional*/int* captureSequenceId); 153 154 ACaptureRequest* mPreviewRequest = nullptr; 155 std::vector<ACameraOutputTarget*> mPreviewRequestOutputs; 156 157 template<class T> 158 camera_status_t captureLocked(sp<ACameraCaptureSession> session, 159 /*optional*/T* cbs, 160 int numRequests, ACaptureRequest** requests, 161 /*optional*/int* captureSequenceId); 162 163 template<class T> 164 camera_status_t setRepeatingRequestsLocked(sp<ACameraCaptureSession> session, 165 /*optional*/T* cbs, 166 int numRequests, ACaptureRequest** requests, 167 /*optional*/int* captureSequenceId); 168 169 template<class T> 170 camera_status_t submitRequestsLocked( 171 sp<ACameraCaptureSession> session, 172 /*optional*/T* cbs, 173 int numRequests, ACaptureRequest** requests, 174 /*out*/int* captureSequenceId, 175 bool isRepeating); 176 177 camera_status_t updateOutputConfigurationLocked(ACaptureSessionOutput *output); 178 179 camera_status_t prepareLocked(ANativeWindow *window); 180 181 camera_status_t allocateCaptureRequest( 182 const ACaptureRequest* request, sp<CaptureRequest>& outReq); 183 184 static ACaptureRequest* allocateACaptureRequest(sp<CaptureRequest>& req, 185 const std::string& deviceId); 186 static void freeACaptureRequest(ACaptureRequest*); 187 188 // only For session to hold device lock 189 // Always grab device lock before grabbing session lock lockDeviceForSessionOps()190 void lockDeviceForSessionOps() const { mDeviceLock.lock(); }; unlockDevice()191 void unlockDevice() const { mDeviceLock.unlock(); }; 192 193 // For capture session to notify its end of life 194 void notifySessionEndOfLifeLocked(ACameraCaptureSession* session); 195 196 camera_status_t configureStreamsLocked(const ACaptureSessionOutputContainer* outputs, 197 const ACaptureRequest* sessionParameters, nsecs_t startTimeNs); 198 199 // Input message will be posted and cleared after this returns 200 void postSessionMsgAndCleanup(sp<AMessage>& msg); 201 202 // Only used when WB_LIBCAMERASERVICE_WITH_DEPENDENCIES is active 203 static ParcelableSurfaceType convertSurfaceTypeToView(sp<SurfaceType> surface); 204 205 static camera_status_t getSurfacefromAnw(ANativeWindow* anw, sp<SurfaceType>& out); 206 207 static camera_status_t getSurfaceFromANativeWindow( 208 ANativeWindow* anw, sp<Surface>& out); 209 210 mutable Mutex mDeviceLock; 211 const String8 mCameraId; // Camera ID 212 const ACameraDevice_StateCallbacks mAppCallbacks; // Callback to app 213 const sp<ACameraMetadata> mChars; // Camera characteristics 214 const sp<ServiceCallback> mServiceCallback; 215 ACameraDevice* mWrapper; 216 bool mSharedMode; 217 bool mIsPrimaryClient; 218 219 // stream id -> pair of (ANW* from application, OutputConfiguration used for camera service) 220 std::map<int, std::pair<ANativeWindow*, OutputConfiguration>> mConfiguredOutputs; 221 222 // TODO: maybe a bool will suffice for synchronous implementation? 223 std::atomic_bool mClosing; isClosed()224 inline bool isClosed() { return mClosing; } 225 226 bool mInError = false; 227 camera_status_t mError = ACAMERA_OK; 228 void onCaptureErrorLocked( 229 int32_t errorCode, 230 const CaptureResultExtras& resultExtras); 231 232 bool mIdle = true; 233 // This will avoid a busy session being deleted before it's back to idle state 234 sp<ACameraCaptureSession> mBusySession; 235 236 sp<hardware::camera2::ICameraDeviceUser> mRemote; 237 238 // Looper thread to handle callback to app 239 sp<ALooper> mCbLooper; 240 // definition of handler and message 241 enum { 242 // Device state callbacks 243 kWhatOnDisconnected, // onDisconnected 244 kWhatOnError, // onError 245 // Session state callbacks 246 kWhatSessionStateCb, // onReady, onActive 247 // Capture callbacks 248 kWhatCaptureStart, // onCaptureStarted 249 kWhatCaptureStart2, // onCaptureStarted 250 kWhatCaptureResult, // onCaptureProgressed, onCaptureCompleted 251 kWhatLogicalCaptureResult, // onLogicalCameraCaptureCompleted 252 kWhatCaptureFail, // onCaptureFailed 253 kWhatLogicalCaptureFail, // onLogicalCameraCaptureFailed 254 kWhatCaptureSeqEnd, // onCaptureSequenceCompleted 255 kWhatCaptureSeqAbort, // onCaptureSequenceAborted 256 kWhatCaptureBufferLost, // onCaptureBufferLost 257 kWhatPreparedCb, // onWindowPrepared 258 // Internal cleanup 259 kWhatCleanUpSessions, // Cleanup cached sp<ACameraCaptureSession> 260 kWhatClientSharedAccessPriorityChanged 261 }; 262 static const char* kContextKey; 263 static const char* kDeviceKey; 264 static const char* kErrorCodeKey; 265 static const char* kCallbackFpKey; 266 static const char* kSessionSpKey; 267 static const char* kCaptureRequestKey; 268 static const char* kTimeStampKey; 269 static const char* kCaptureResultKey; 270 static const char* kPhysicalCaptureResultKey; 271 static const char* kCaptureFailureKey; 272 static const char* kSequenceIdKey; 273 static const char* kFrameNumberKey; 274 static const char* kAnwKey; 275 static const char* kFailingPhysicalCameraId; 276 277 class CallbackHandler : public AHandler { 278 public: 279 explicit CallbackHandler(const char* id); 280 void onMessageReceived(const sp<AMessage> &msg) override; 281 282 private: 283 std::string mId; 284 // This handler will cache all capture session sp until kWhatCleanUpSessions 285 // is processed. This is used to guarantee the last session reference is always 286 // being removed in callback thread without holding camera device lock 287 Vector<sp<ACameraCaptureSession>> mCachedSessions; 288 }; 289 sp<CallbackHandler> mHandler; 290 291 /*********************************** 292 * Capture session related members * 293 ***********************************/ 294 // The current active session 295 wp<ACameraCaptureSession> mCurrentSession; 296 bool mFlushing = false; 297 298 int mNextSessionId = 0; 299 // TODO: might need another looper/handler to handle callbacks from service 300 301 static const int REQUEST_ID_NONE = -1; 302 int mRepeatingSequenceId = REQUEST_ID_NONE; 303 304 // sequence id -> last frame number holder map 305 struct RequestLastFrameNumberHolder { 306 int64_t lastFrameNumber; 307 // Whether the current sequence is completed (capture results are 308 // generated). May be set to true, but 309 // not removed from the map if not all inflight requests in the sequence 310 // have been completed. 311 bool isSequenceCompleted = false; 312 // Whether all inflight requests in the sequence are completed 313 // (capture results and buffers are generated). May be 314 // set to true, but not removed from the map yet if the capture results 315 // haven't been delivered to the app yet. 316 bool isInflightCompleted = false; RequestLastFrameNumberHolderRequestLastFrameNumberHolder317 RequestLastFrameNumberHolder(int64_t lastFN) : 318 lastFrameNumber(lastFN) {} 319 }; 320 std::map<int, RequestLastFrameNumberHolder> mSequenceLastFrameNumberMap; 321 322 struct CallbackHolder { 323 CallbackHolder(sp<ACameraCaptureSession> session, 324 const Vector<sp<CaptureRequest> >& requests, 325 bool isRepeating, 326 ACameraCaptureSession_captureCallbacks* cbs); 327 CallbackHolder(sp<ACameraCaptureSession> session, 328 const Vector<sp<CaptureRequest> >& requests, 329 bool isRepeating, 330 ACameraCaptureSession_logicalCamera_captureCallbacks* lcbs); 331 CallbackHolder(sp<ACameraCaptureSession> session, 332 const Vector<sp<CaptureRequest> >& requests, 333 bool isRepeating, 334 ACameraCaptureSession_captureCallbacksV2* cbs); 335 CallbackHolder(sp<ACameraCaptureSession> session, 336 const Vector<sp<CaptureRequest> >& requests, 337 bool isRepeating, 338 ACameraCaptureSession_logicalCamera_captureCallbacksV2* lcbs); clearCallbacksCallbackHolder339 void clearCallbacks() { 340 mContext = nullptr; 341 mOnCaptureStarted = nullptr; 342 mOnCaptureStarted2 = nullptr; 343 mOnCaptureProgressed = nullptr; 344 mOnCaptureCompleted = nullptr; 345 mOnLogicalCameraCaptureCompleted = nullptr; 346 mOnLogicalCameraCaptureFailed = nullptr; 347 mOnCaptureFailed = nullptr; 348 mOnCaptureSequenceCompleted = nullptr; 349 mOnCaptureSequenceAborted = nullptr; 350 mOnCaptureBufferLost = nullptr; 351 } 352 353 template <class T> initCaptureCallbacksV2CallbackHolder354 void initCaptureCallbacksV2(T* cbs) { 355 clearCallbacks(); 356 if (cbs != nullptr) { 357 mContext = cbs->context; 358 mOnCaptureStarted2 = cbs->onCaptureStarted; 359 mOnCaptureProgressed = cbs->onCaptureProgressed; 360 mOnCaptureSequenceCompleted = cbs->onCaptureSequenceCompleted; 361 mOnCaptureSequenceAborted = cbs->onCaptureSequenceAborted; 362 mOnCaptureBufferLost = cbs->onCaptureBufferLost; 363 } 364 } 365 366 template <class T> initCaptureCallbacksCallbackHolder367 void initCaptureCallbacks(T* cbs) { 368 clearCallbacks(); 369 if (cbs != nullptr) { 370 mContext = cbs->context; 371 mOnCaptureStarted = cbs->onCaptureStarted; 372 mOnCaptureProgressed = cbs->onCaptureProgressed; 373 mOnCaptureSequenceCompleted = cbs->onCaptureSequenceCompleted; 374 mOnCaptureSequenceAborted = cbs->onCaptureSequenceAborted; 375 mOnCaptureBufferLost = cbs->onCaptureBufferLost; 376 } 377 } 378 sp<ACameraCaptureSession> mSession; 379 Vector<sp<CaptureRequest> > mRequests; 380 const bool mIsRepeating; 381 const bool mIsLogicalCameraCallback; 382 const bool mIs2Callback; 383 384 void* mContext; 385 ACameraCaptureSession_captureCallback_start mOnCaptureStarted; 386 ACameraCaptureSession_captureCallback_startV2 mOnCaptureStarted2; 387 ACameraCaptureSession_captureCallback_result mOnCaptureProgressed; 388 ACameraCaptureSession_captureCallback_result mOnCaptureCompleted; 389 ACameraCaptureSession_logicalCamera_captureCallback_result mOnLogicalCameraCaptureCompleted; 390 ACameraCaptureSession_logicalCamera_captureCallback_failed mOnLogicalCameraCaptureFailed; 391 ACameraCaptureSession_captureCallback_failed mOnCaptureFailed; 392 ACameraCaptureSession_captureCallback_sequenceEnd mOnCaptureSequenceCompleted; 393 ACameraCaptureSession_captureCallback_sequenceAbort mOnCaptureSequenceAborted; 394 ACameraCaptureSession_captureCallback_bufferLost mOnCaptureBufferLost; 395 }; 396 // sequence id -> callbacks map 397 std::map<int, CallbackHolder> mSequenceCallbackMap; 398 399 static const int64_t NO_FRAMES_CAPTURED = -1; 400 class FrameNumberTracker { 401 public: 402 // TODO: Called in onResultReceived and onCaptureErrorLocked 403 void updateTracker(int64_t frameNumber, bool isError); getCompletedFrameNumber()404 inline int64_t getCompletedFrameNumber() { return mCompletedFrameNumber; } 405 private: 406 void update(); 407 void updateCompletedFrameNumber(int64_t frameNumber); 408 409 int64_t mCompletedFrameNumber = NO_FRAMES_CAPTURED; 410 List<int64_t> mSkippedFrameNumbers; 411 std::set<int64_t> mFutureErrorSet; 412 }; 413 FrameNumberTracker mFrameNumberTracker; 414 415 void checkRepeatingSequenceCompleteLocked(const int sequenceId, const int64_t lastFrameNumber); 416 void checkAndFireSequenceCompleteLocked(); 417 void removeCompletedCallbackHolderLocked(int64_t lastCompletedRegularFrameNumber); 418 void sendCaptureSequenceCompletedLocked(int sequenceId, int64_t lastFrameNumber); 419 420 // Misc variables 421 int32_t mShadingMapSize[2]; // const after constructor 422 int32_t mPartialResultCount; // const after constructor 423 std::vector<std::string> mPhysicalIds; // const after constructor 424 425 // Metadata queue to write the result metadata to. 426 std::unique_ptr<ResultMetadataQueue> mCaptureResultMetadataQueue; 427 428 }; 429 430 } // namespace acam; 431 } // namespace android; 432 433 /** 434 * ACameraDevice opaque struct definition 435 * Leave outside of android namespace because it's NDK struct 436 */ 437 struct ACameraDevice { ACameraDeviceACameraDevice438 ACameraDevice(const char* id, ACameraDevice_StateCallbacks* cb, 439 sp<ACameraMetadata> chars, bool sharedMode) : 440 mDevice(new android::acam::CameraDevice(id, cb, chars, this, sharedMode)) {} 441 442 ~ACameraDevice(); 443 444 /******************* 445 * NDK public APIs * 446 *******************/ getIdACameraDevice447 inline const char* getId() const { return mDevice->getId(); } 448 createCaptureRequestACameraDevice449 camera_status_t createCaptureRequest( 450 ACameraDevice_request_template templateId, 451 const ACameraIdList* physicalCameraIdList, 452 ACaptureRequest** request) const { 453 return mDevice->createCaptureRequest(templateId, physicalCameraIdList, request); 454 } 455 createCaptureSessionACameraDevice456 camera_status_t createCaptureSession( 457 const ACaptureSessionOutputContainer* outputs, 458 const ACaptureRequest* sessionParameters, 459 const ACameraCaptureSession_stateCallbacks* callbacks, 460 /*out*/ACameraCaptureSession** session) { 461 return mDevice->createCaptureSession(outputs, sessionParameters, callbacks, session); 462 } 463 isSessionConfigurationSupportedACameraDevice464 camera_status_t isSessionConfigurationSupported( 465 const ACaptureSessionOutputContainer* sessionOutputContainer) const { 466 return mDevice->isSessionConfigurationSupported(sessionOutputContainer); 467 } 468 469 /*********************** 470 * Device interal APIs * 471 ***********************/ getServiceCallbackACameraDevice472 inline android::sp<android::hardware::camera2::ICameraDeviceCallbacks> getServiceCallback() { 473 return mDevice->getServiceCallback(); 474 }; 475 476 // Camera device is only functional after remote being set setRemoteDeviceACameraDevice477 inline void setRemoteDevice(android::sp<android::hardware::camera2::ICameraDeviceUser> remote) { 478 mDevice->setRemoteDevice(remote); 479 } 480 setDeviceMetadataQueuesACameraDevice481 inline bool setDeviceMetadataQueues() { 482 return mDevice->setDeviceMetadataQueues(); 483 } 484 setPrimaryClientACameraDevice485 inline void setPrimaryClient(bool isPrimary) { 486 mDevice->setPrimaryClient(isPrimary); 487 } 488 isPrimaryClientACameraDevice489 inline bool isPrimaryClient() const { 490 return mDevice->isPrimaryClient(); 491 } 492 isSharedModeACameraDevice493 inline bool isSharedMode() const{ 494 return mDevice->isSharedMode(); 495 } 496 497 private: 498 android::sp<android::acam::CameraDevice> mDevice; 499 }; 500 501 #endif // _ACAMERA_DEVICE_H 502