1 /* 2 * Copyright (C) 2015 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 #ifndef _ACAMERA_DEVICE_H 17 #define _ACAMERA_DEVICE_H 18 19 #include <memory> 20 #include <map> 21 #include <set> 22 #include <atomic> 23 #include <utility> 24 #include <vector> 25 #include <utils/StrongPointer.h> 26 #include <utils/Mutex.h> 27 #include <utils/String8.h> 28 #include <utils/List.h> 29 #include <utils/Vector.h> 30 31 #include <android/hardware/camera2/BnCameraDeviceCallbacks.h> 32 #include <android/hardware/camera2/ICameraDeviceUser.h> 33 #include <media/stagefright/foundation/ALooper.h> 34 #include <media/stagefright/foundation/AHandler.h> 35 #include <media/stagefright/foundation/AMessage.h> 36 #include <camera/CaptureResult.h> 37 #include <camera/camera2/OutputConfiguration.h> 38 #include <camera/camera2/SessionConfiguration.h> 39 #include <camera/camera2/CaptureRequest.h> 40 41 #include <camera/NdkCameraManager.h> 42 #include <camera/NdkCameraCaptureSession.h> 43 44 #include "ACameraMetadata.h" 45 46 namespace android { 47 namespace acam { 48 49 // Wrap ACameraCaptureFailure so it can be ref-counted 50 struct CameraCaptureFailure : public RefBase, public ACameraCaptureFailure {}; 51 52 // Wrap PhysicalCaptureResultInfo so that it can be ref-counted 53 struct ACameraPhysicalCaptureResultInfo: public RefBase { ACameraPhysicalCaptureResultInfoACameraPhysicalCaptureResultInfo54 ACameraPhysicalCaptureResultInfo(const std::vector<PhysicalCaptureResultInfo>& info, 55 int64_t frameNumber) : 56 mPhysicalResultInfo(info), mFrameNumber(frameNumber) {} 57 58 std::vector<PhysicalCaptureResultInfo> mPhysicalResultInfo; 59 int64_t mFrameNumber; 60 }; 61 62 class CameraDevice final : public RefBase { 63 public: 64 CameraDevice(const char* id, ACameraDevice_StateCallbacks* cb, 65 sp<ACameraMetadata> chars, 66 ACameraDevice* wrapper); 67 ~CameraDevice(); 68 getId()69 inline const char* getId() const { return mCameraId.c_str(); } 70 71 camera_status_t createCaptureRequest( 72 ACameraDevice_request_template templateId, 73 const ACameraIdList* physicalIdList, 74 ACaptureRequest** request) const; 75 76 camera_status_t createCaptureSession( 77 const ACaptureSessionOutputContainer* outputs, 78 const ACaptureRequest* sessionParameters, 79 const ACameraCaptureSession_stateCallbacks* callbacks, 80 /*out*/ACameraCaptureSession** session); 81 82 camera_status_t isSessionConfigurationSupported( 83 const ACaptureSessionOutputContainer* sessionOutputContainer) const; 84 85 // Callbacks from camera service 86 class ServiceCallback : public hardware::camera2::BnCameraDeviceCallbacks { 87 public: ServiceCallback(CameraDevice * device)88 explicit ServiceCallback(CameraDevice* device) : mDevice(device) {} 89 binder::Status onDeviceError(int32_t errorCode, 90 const CaptureResultExtras& resultExtras) override; 91 binder::Status onDeviceIdle() override; 92 binder::Status onCaptureStarted(const CaptureResultExtras& resultExtras, 93 int64_t timestamp) override; 94 binder::Status onResultReceived(const CameraMetadata& metadata, 95 const CaptureResultExtras& resultExtras, 96 const std::vector<PhysicalCaptureResultInfo>& physicalResultInfos) override; 97 binder::Status onPrepared(int streamId) override; 98 binder::Status onRequestQueueEmpty() override; 99 binder::Status onRepeatingRequestError(int64_t lastFrameNumber, 100 int32_t stoppedSequenceId) override; 101 private: 102 const wp<CameraDevice> mDevice; 103 }; getServiceCallback()104 inline sp<hardware::camera2::ICameraDeviceCallbacks> getServiceCallback() { 105 return mServiceCallback; 106 }; 107 108 // Camera device is only functional after remote being set 109 void setRemoteDevice(sp<hardware::camera2::ICameraDeviceUser> remote); 110 getWrapper()111 inline ACameraDevice* getWrapper() const { return mWrapper; }; 112 113 // Stop the looper thread and unregister the handler 114 void stopLooperAndDisconnect(); 115 116 private: 117 friend ACameraCaptureSession; 118 camera_status_t checkCameraClosedOrErrorLocked() const; 119 120 // device goes into fatal error state after this 121 void setCameraDeviceErrorLocked(camera_status_t error); 122 123 void disconnectLocked(sp<ACameraCaptureSession>& session); // disconnect from camera service 124 125 camera_status_t stopRepeatingLocked(); 126 127 camera_status_t flushLocked(ACameraCaptureSession*); 128 129 camera_status_t waitUntilIdleLocked(); 130 131 132 template<class T> 133 camera_status_t captureLocked(sp<ACameraCaptureSession> session, 134 /*optional*/T* cbs, 135 int numRequests, ACaptureRequest** requests, 136 /*optional*/int* captureSequenceId); 137 138 template<class T> 139 camera_status_t setRepeatingRequestsLocked(sp<ACameraCaptureSession> session, 140 /*optional*/T* cbs, 141 int numRequests, ACaptureRequest** requests, 142 /*optional*/int* captureSequenceId); 143 144 template<class T> 145 camera_status_t submitRequestsLocked( 146 sp<ACameraCaptureSession> session, 147 /*optional*/T* cbs, 148 int numRequests, ACaptureRequest** requests, 149 /*out*/int* captureSequenceId, 150 bool isRepeating); 151 152 camera_status_t updateOutputConfigurationLocked(ACaptureSessionOutput *output); 153 154 camera_status_t prepareLocked(ANativeWindow *window); 155 156 camera_status_t allocateCaptureRequest( 157 const ACaptureRequest* request, sp<CaptureRequest>& outReq); 158 159 static ACaptureRequest* allocateACaptureRequest(sp<CaptureRequest>& req, 160 const std::string& deviceId); 161 static void freeACaptureRequest(ACaptureRequest*); 162 163 // only For session to hold device lock 164 // Always grab device lock before grabbing session lock lockDeviceForSessionOps()165 void lockDeviceForSessionOps() const { mDeviceLock.lock(); }; unlockDevice()166 void unlockDevice() const { mDeviceLock.unlock(); }; 167 168 // For capture session to notify its end of life 169 void notifySessionEndOfLifeLocked(ACameraCaptureSession* session); 170 171 camera_status_t configureStreamsLocked(const ACaptureSessionOutputContainer* outputs, 172 const ACaptureRequest* sessionParameters, nsecs_t startTimeNs); 173 174 // Input message will be posted and cleared after this returns 175 void postSessionMsgAndCleanup(sp<AMessage>& msg); 176 177 static camera_status_t getIGBPfromAnw( 178 ANativeWindow* anw, sp<IGraphicBufferProducer>& out); 179 180 static camera_status_t getSurfaceFromANativeWindow( 181 ANativeWindow* anw, sp<Surface>& out); 182 183 mutable Mutex mDeviceLock; 184 const String8 mCameraId; // Camera ID 185 const ACameraDevice_StateCallbacks mAppCallbacks; // Callback to app 186 const sp<ACameraMetadata> mChars; // Camera characteristics 187 const sp<ServiceCallback> mServiceCallback; 188 ACameraDevice* mWrapper; 189 190 // stream id -> pair of (ANW* from application, OutputConfiguration used for camera service) 191 std::map<int, std::pair<ANativeWindow*, OutputConfiguration>> mConfiguredOutputs; 192 193 // TODO: maybe a bool will suffice for synchronous implementation? 194 std::atomic_bool mClosing; isClosed()195 inline bool isClosed() { return mClosing; } 196 197 bool mInError = false; 198 camera_status_t mError = ACAMERA_OK; 199 void onCaptureErrorLocked( 200 int32_t errorCode, 201 const CaptureResultExtras& resultExtras); 202 203 bool mIdle = true; 204 // This will avoid a busy session being deleted before it's back to idle state 205 sp<ACameraCaptureSession> mBusySession; 206 207 sp<hardware::camera2::ICameraDeviceUser> mRemote; 208 209 // Looper thread to handle callback to app 210 sp<ALooper> mCbLooper; 211 // definition of handler and message 212 enum { 213 // Device state callbacks 214 kWhatOnDisconnected, // onDisconnected 215 kWhatOnError, // onError 216 // Session state callbacks 217 kWhatSessionStateCb, // onReady, onActive 218 // Capture callbacks 219 kWhatCaptureStart, // onCaptureStarted 220 kWhatCaptureStart2, // onCaptureStarted 221 kWhatCaptureResult, // onCaptureProgressed, onCaptureCompleted 222 kWhatLogicalCaptureResult, // onLogicalCameraCaptureCompleted 223 kWhatCaptureFail, // onCaptureFailed 224 kWhatLogicalCaptureFail, // onLogicalCameraCaptureFailed 225 kWhatCaptureSeqEnd, // onCaptureSequenceCompleted 226 kWhatCaptureSeqAbort, // onCaptureSequenceAborted 227 kWhatCaptureBufferLost, // onCaptureBufferLost 228 kWhatPreparedCb, // onWindowPrepared 229 // Internal cleanup 230 kWhatCleanUpSessions // Cleanup cached sp<ACameraCaptureSession> 231 }; 232 static const char* kContextKey; 233 static const char* kDeviceKey; 234 static const char* kErrorCodeKey; 235 static const char* kCallbackFpKey; 236 static const char* kSessionSpKey; 237 static const char* kCaptureRequestKey; 238 static const char* kTimeStampKey; 239 static const char* kCaptureResultKey; 240 static const char* kPhysicalCaptureResultKey; 241 static const char* kCaptureFailureKey; 242 static const char* kSequenceIdKey; 243 static const char* kFrameNumberKey; 244 static const char* kAnwKey; 245 static const char* kFailingPhysicalCameraId; 246 247 class CallbackHandler : public AHandler { 248 public: 249 explicit CallbackHandler(const char* id); 250 void onMessageReceived(const sp<AMessage> &msg) override; 251 252 private: 253 std::string mId; 254 // This handler will cache all capture session sp until kWhatCleanUpSessions 255 // is processed. This is used to guarantee the last session reference is always 256 // being removed in callback thread without holding camera device lock 257 Vector<sp<ACameraCaptureSession>> mCachedSessions; 258 }; 259 sp<CallbackHandler> mHandler; 260 261 /*********************************** 262 * Capture session related members * 263 ***********************************/ 264 // The current active session 265 wp<ACameraCaptureSession> mCurrentSession; 266 bool mFlushing = false; 267 268 int mNextSessionId = 0; 269 // TODO: might need another looper/handler to handle callbacks from service 270 271 static const int REQUEST_ID_NONE = -1; 272 int mRepeatingSequenceId = REQUEST_ID_NONE; 273 274 // sequence id -> last frame number holder map 275 struct RequestLastFrameNumberHolder { 276 int64_t lastFrameNumber; 277 // Whether the current sequence is completed (capture results are 278 // generated). May be set to true, but 279 // not removed from the map if not all inflight requests in the sequence 280 // have been completed. 281 bool isSequenceCompleted = false; 282 // Whether all inflight requests in the sequence are completed 283 // (capture results and buffers are generated). May be 284 // set to true, but not removed from the map yet if the capture results 285 // haven't been delivered to the app yet. 286 bool isInflightCompleted = false; RequestLastFrameNumberHolderRequestLastFrameNumberHolder287 RequestLastFrameNumberHolder(int64_t lastFN) : 288 lastFrameNumber(lastFN) {} 289 }; 290 std::map<int, RequestLastFrameNumberHolder> mSequenceLastFrameNumberMap; 291 292 struct CallbackHolder { 293 CallbackHolder(sp<ACameraCaptureSession> session, 294 const Vector<sp<CaptureRequest> >& requests, 295 bool isRepeating, 296 ACameraCaptureSession_captureCallbacks* cbs); 297 CallbackHolder(sp<ACameraCaptureSession> session, 298 const Vector<sp<CaptureRequest> >& requests, 299 bool isRepeating, 300 ACameraCaptureSession_logicalCamera_captureCallbacks* lcbs); 301 CallbackHolder(sp<ACameraCaptureSession> session, 302 const Vector<sp<CaptureRequest> >& requests, 303 bool isRepeating, 304 ACameraCaptureSession_captureCallbacksV2* cbs); 305 CallbackHolder(sp<ACameraCaptureSession> session, 306 const Vector<sp<CaptureRequest> >& requests, 307 bool isRepeating, 308 ACameraCaptureSession_logicalCamera_captureCallbacksV2* lcbs); clearCallbacksCallbackHolder309 void clearCallbacks() { 310 mContext = nullptr; 311 mOnCaptureStarted = nullptr; 312 mOnCaptureStarted2 = nullptr; 313 mOnCaptureProgressed = nullptr; 314 mOnCaptureCompleted = nullptr; 315 mOnLogicalCameraCaptureCompleted = nullptr; 316 mOnLogicalCameraCaptureFailed = nullptr; 317 mOnCaptureFailed = nullptr; 318 mOnCaptureSequenceCompleted = nullptr; 319 mOnCaptureSequenceAborted = nullptr; 320 mOnCaptureBufferLost = nullptr; 321 } 322 323 template <class T> initCaptureCallbacksV2CallbackHolder324 void initCaptureCallbacksV2(T* cbs) { 325 clearCallbacks(); 326 if (cbs != nullptr) { 327 mContext = cbs->context; 328 mOnCaptureStarted2 = cbs->onCaptureStarted; 329 mOnCaptureProgressed = cbs->onCaptureProgressed; 330 mOnCaptureSequenceCompleted = cbs->onCaptureSequenceCompleted; 331 mOnCaptureSequenceAborted = cbs->onCaptureSequenceAborted; 332 mOnCaptureBufferLost = cbs->onCaptureBufferLost; 333 } 334 } 335 336 template <class T> initCaptureCallbacksCallbackHolder337 void initCaptureCallbacks(T* cbs) { 338 clearCallbacks(); 339 if (cbs != nullptr) { 340 mContext = cbs->context; 341 mOnCaptureStarted = cbs->onCaptureStarted; 342 mOnCaptureProgressed = cbs->onCaptureProgressed; 343 mOnCaptureSequenceCompleted = cbs->onCaptureSequenceCompleted; 344 mOnCaptureSequenceAborted = cbs->onCaptureSequenceAborted; 345 mOnCaptureBufferLost = cbs->onCaptureBufferLost; 346 } 347 } 348 sp<ACameraCaptureSession> mSession; 349 Vector<sp<CaptureRequest> > mRequests; 350 const bool mIsRepeating; 351 const bool mIsLogicalCameraCallback; 352 const bool mIs2Callback; 353 354 void* mContext; 355 ACameraCaptureSession_captureCallback_start mOnCaptureStarted; 356 ACameraCaptureSession_captureCallback_startV2 mOnCaptureStarted2; 357 ACameraCaptureSession_captureCallback_result mOnCaptureProgressed; 358 ACameraCaptureSession_captureCallback_result mOnCaptureCompleted; 359 ACameraCaptureSession_logicalCamera_captureCallback_result mOnLogicalCameraCaptureCompleted; 360 ACameraCaptureSession_logicalCamera_captureCallback_failed mOnLogicalCameraCaptureFailed; 361 ACameraCaptureSession_captureCallback_failed mOnCaptureFailed; 362 ACameraCaptureSession_captureCallback_sequenceEnd mOnCaptureSequenceCompleted; 363 ACameraCaptureSession_captureCallback_sequenceAbort mOnCaptureSequenceAborted; 364 ACameraCaptureSession_captureCallback_bufferLost mOnCaptureBufferLost; 365 }; 366 // sequence id -> callbacks map 367 std::map<int, CallbackHolder> mSequenceCallbackMap; 368 369 static const int64_t NO_FRAMES_CAPTURED = -1; 370 class FrameNumberTracker { 371 public: 372 // TODO: Called in onResultReceived and onCaptureErrorLocked 373 void updateTracker(int64_t frameNumber, bool isError); getCompletedFrameNumber()374 inline int64_t getCompletedFrameNumber() { return mCompletedFrameNumber; } 375 private: 376 void update(); 377 void updateCompletedFrameNumber(int64_t frameNumber); 378 379 int64_t mCompletedFrameNumber = NO_FRAMES_CAPTURED; 380 List<int64_t> mSkippedFrameNumbers; 381 std::set<int64_t> mFutureErrorSet; 382 }; 383 FrameNumberTracker mFrameNumberTracker; 384 385 void checkRepeatingSequenceCompleteLocked(const int sequenceId, const int64_t lastFrameNumber); 386 void checkAndFireSequenceCompleteLocked(); 387 void removeCompletedCallbackHolderLocked(int64_t lastCompletedRegularFrameNumber); 388 void sendCaptureSequenceCompletedLocked(int sequenceId, int64_t lastFrameNumber); 389 390 // Misc variables 391 int32_t mShadingMapSize[2]; // const after constructor 392 int32_t mPartialResultCount; // const after constructor 393 std::vector<std::string> mPhysicalIds; // const after constructor 394 395 }; 396 397 } // namespace acam; 398 } // namespace android; 399 400 /** 401 * ACameraDevice opaque struct definition 402 * Leave outside of android namespace because it's NDK struct 403 */ 404 struct ACameraDevice { ACameraDeviceACameraDevice405 ACameraDevice(const char* id, ACameraDevice_StateCallbacks* cb, 406 sp<ACameraMetadata> chars) : 407 mDevice(new android::acam::CameraDevice(id, cb, chars, this)) {} 408 409 ~ACameraDevice(); 410 411 /******************* 412 * NDK public APIs * 413 *******************/ getIdACameraDevice414 inline const char* getId() const { return mDevice->getId(); } 415 createCaptureRequestACameraDevice416 camera_status_t createCaptureRequest( 417 ACameraDevice_request_template templateId, 418 const ACameraIdList* physicalCameraIdList, 419 ACaptureRequest** request) const { 420 return mDevice->createCaptureRequest(templateId, physicalCameraIdList, request); 421 } 422 createCaptureSessionACameraDevice423 camera_status_t createCaptureSession( 424 const ACaptureSessionOutputContainer* outputs, 425 const ACaptureRequest* sessionParameters, 426 const ACameraCaptureSession_stateCallbacks* callbacks, 427 /*out*/ACameraCaptureSession** session) { 428 return mDevice->createCaptureSession(outputs, sessionParameters, callbacks, session); 429 } 430 isSessionConfigurationSupportedACameraDevice431 camera_status_t isSessionConfigurationSupported( 432 const ACaptureSessionOutputContainer* sessionOutputContainer) const { 433 return mDevice->isSessionConfigurationSupported(sessionOutputContainer); 434 } 435 436 /*********************** 437 * Device interal APIs * 438 ***********************/ getServiceCallbackACameraDevice439 inline android::sp<android::hardware::camera2::ICameraDeviceCallbacks> getServiceCallback() { 440 return mDevice->getServiceCallback(); 441 }; 442 443 // Camera device is only functional after remote being set setRemoteDeviceACameraDevice444 inline void setRemoteDevice(android::sp<android::hardware::camera2::ICameraDeviceUser> remote) { 445 mDevice->setRemoteDevice(remote); 446 } 447 448 private: 449 android::sp<android::acam::CameraDevice> mDevice; 450 }; 451 452 #endif // _ACAMERA_DEVICE_H 453