1 /* 2 * Copyright (C) 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef ANDROID_SERVERS_CAMERA3DEVICE_H 18 #define ANDROID_SERVERS_CAMERA3DEVICE_H 19 20 #include <utility> 21 #include <unordered_map> 22 23 #include <utils/Condition.h> 24 #include <utils/Errors.h> 25 #include <utils/List.h> 26 #include <utils/Mutex.h> 27 #include <utils/Thread.h> 28 #include <utils/KeyedVector.h> 29 #include <utils/Timers.h> 30 31 #include <android/hardware/camera/device/3.2/ICameraDevice.h> 32 #include <android/hardware/camera/device/3.2/ICameraDeviceSession.h> 33 #include <android/hardware/camera/device/3.3/ICameraDeviceSession.h> 34 #include <android/hardware/camera/device/3.2/ICameraDeviceCallback.h> 35 #include <fmq/MessageQueue.h> 36 #include <hardware/camera3.h> 37 38 #include <camera/CaptureResult.h> 39 40 #include "common/CameraDeviceBase.h" 41 #include "device3/StatusTracker.h" 42 #include "device3/Camera3BufferManager.h" 43 #include "utils/TagMonitor.h" 44 #include "utils/LatencyHistogram.h" 45 #include <camera_metadata_hidden.h> 46 47 /** 48 * Function pointer types with C calling convention to 49 * use for HAL callback functions. 50 */ 51 extern "C" { 52 typedef void (callbacks_process_capture_result_t)( 53 const struct camera3_callback_ops *, 54 const camera3_capture_result_t *); 55 56 typedef void (callbacks_notify_t)( 57 const struct camera3_callback_ops *, 58 const camera3_notify_msg_t *); 59 } 60 61 namespace android { 62 63 namespace camera3 { 64 65 class Camera3Stream; 66 class Camera3ZslStream; 67 class Camera3OutputStreamInterface; 68 class Camera3StreamInterface; 69 70 } // namespace camera3 71 72 /** 73 * CameraDevice for HAL devices with version CAMERA_DEVICE_API_VERSION_3_0 or higher. 74 */ 75 class Camera3Device : 76 public CameraDeviceBase, 77 virtual public hardware::camera::device::V3_2::ICameraDeviceCallback, 78 private camera3_callback_ops { 79 public: 80 81 explicit Camera3Device(const String8& id); 82 83 virtual ~Camera3Device(); 84 85 /** 86 * CameraDeviceBase interface 87 */ 88 89 const String8& getId() const override; 90 91 // Transitions to idle state on success. 92 status_t initialize(sp<CameraProviderManager> manager) override; 93 status_t disconnect() override; 94 status_t dump(int fd, const Vector<String16> &args) override; 95 const CameraMetadata& info() const override; 96 97 // Capture and setStreamingRequest will configure streams if currently in 98 // idle state 99 status_t capture(CameraMetadata &request, int64_t *lastFrameNumber = NULL) override; 100 status_t captureList(const List<const CameraMetadata> &requests, 101 const std::list<const SurfaceMap> &surfaceMaps, 102 int64_t *lastFrameNumber = NULL) override; 103 status_t setStreamingRequest(const CameraMetadata &request, 104 int64_t *lastFrameNumber = NULL) override; 105 status_t setStreamingRequestList(const List<const CameraMetadata> &requests, 106 const std::list<const SurfaceMap> &surfaceMaps, 107 int64_t *lastFrameNumber = NULL) override; 108 status_t clearStreamingRequest(int64_t *lastFrameNumber = NULL) override; 109 110 status_t waitUntilRequestReceived(int32_t requestId, nsecs_t timeout) override; 111 112 // Actual stream creation/deletion is delayed until first request is submitted 113 // If adding streams while actively capturing, will pause device before adding 114 // stream, reconfiguring device, and unpausing. If the client create a stream 115 // with nullptr consumer surface, the client must then call setConsumers() 116 // and finish the stream configuration before starting output streaming. 117 status_t createStream(sp<Surface> consumer, 118 uint32_t width, uint32_t height, int format, 119 android_dataspace dataSpace, camera3_stream_rotation_t rotation, int *id, 120 int streamSetId = camera3::CAMERA3_STREAM_SET_ID_INVALID, 121 bool isShared = false, uint64_t consumerUsage = 0) override; 122 status_t createStream(const std::vector<sp<Surface>>& consumers, 123 bool hasDeferredConsumer, uint32_t width, uint32_t height, int format, 124 android_dataspace dataSpace, camera3_stream_rotation_t rotation, int *id, 125 int streamSetId = camera3::CAMERA3_STREAM_SET_ID_INVALID, 126 bool isShared = false, uint64_t consumerUsage = 0) override; 127 128 status_t createInputStream( 129 uint32_t width, uint32_t height, int format, 130 int *id) override; 131 132 status_t getStreamInfo(int id, StreamInfo *streamInfo) override; 133 status_t setStreamTransform(int id, int transform) override; 134 135 status_t deleteStream(int id) override; 136 137 status_t configureStreams(int operatingMode = 138 static_cast<int>(hardware::camera::device::V3_2::StreamConfigurationMode::NORMAL_MODE)) 139 override; 140 status_t getInputBufferProducer( 141 sp<IGraphicBufferProducer> *producer) override; 142 143 status_t createDefaultRequest(int templateId, CameraMetadata *request) override; 144 145 // Transitions to the idle state on success 146 status_t waitUntilDrained() override; 147 148 status_t setNotifyCallback(wp<NotificationListener> listener) override; 149 bool willNotify3A() override; 150 status_t waitForNextFrame(nsecs_t timeout) override; 151 status_t getNextResult(CaptureResult *frame) override; 152 153 status_t triggerAutofocus(uint32_t id) override; 154 status_t triggerCancelAutofocus(uint32_t id) override; 155 status_t triggerPrecaptureMetering(uint32_t id) override; 156 157 status_t flush(int64_t *lastFrameNumber = NULL) override; 158 159 status_t prepare(int streamId) override; 160 161 status_t tearDown(int streamId) override; 162 163 status_t addBufferListenerForStream(int streamId, 164 wp<camera3::Camera3StreamBufferListener> listener) override; 165 166 status_t prepare(int maxCount, int streamId) override; 167 168 ssize_t getJpegBufferSize(uint32_t width, uint32_t height) const override; 169 ssize_t getPointCloudBufferSize() const; 170 ssize_t getRawOpaqueBufferSize(int32_t width, int32_t height) const; 171 172 // Methods called by subclasses 173 void notifyStatus(bool idle); // updates from StatusTracker 174 175 /** 176 * Set the deferred consumer surfaces to the output stream and finish the deferred 177 * consumer configuration. 178 */ 179 status_t setConsumerSurfaces(int streamId, const std::vector<sp<Surface>>& consumers) override; 180 181 private: 182 183 // internal typedefs 184 using RequestMetadataQueue = hardware::MessageQueue<uint8_t, hardware::kSynchronizedReadWrite>; 185 using ResultMetadataQueue = hardware::MessageQueue<uint8_t, hardware::kSynchronizedReadWrite>; 186 187 static const size_t kDumpLockAttempts = 10; 188 static const size_t kDumpSleepDuration = 100000; // 0.10 sec 189 static const nsecs_t kActiveTimeout = 500000000; // 500 ms 190 static const size_t kInFlightWarnLimit = 30; 191 static const size_t kInFlightWarnLimitHighSpeed = 256; // batch size 32 * pipe depth 8 192 static const nsecs_t kDefaultExpectedDuration = 100000000; // 100 ms 193 static const nsecs_t kMinInflightDuration = 5000000000; // 5 s 194 // SCHED_FIFO priority for request submission thread in HFR mode 195 static const int kRequestThreadPriority = 1; 196 197 struct RequestTrigger; 198 // minimal jpeg buffer size: 256KB + blob header 199 static const ssize_t kMinJpegBufferSize = 256 * 1024 + sizeof(camera3_jpeg_blob); 200 // Constant to use for stream ID when one doesn't exist 201 static const int NO_STREAM = -1; 202 203 // A lock to enforce serialization on the input/configure side 204 // of the public interface. 205 // Only locked by public methods inherited from CameraDeviceBase. 206 // Not locked by methods guarded by mOutputLock, since they may act 207 // concurrently to the input/configure side of the interface. 208 // Must be locked before mLock if both will be locked by a method 209 Mutex mInterfaceLock; 210 211 // The main lock on internal state 212 Mutex mLock; 213 214 // Camera device ID 215 const String8 mId; 216 217 // Current stream configuration mode; 218 int mOperatingMode; 219 // Constant to use for no set operating mode 220 static const int NO_MODE = -1; 221 222 // Flag indicating is the current active stream configuration is constrained high speed. 223 bool mIsConstrainedHighSpeedConfiguration; 224 225 // FMQ to write result on. Must be guarded by mProcessCaptureResultLock. 226 std::unique_ptr<ResultMetadataQueue> mResultMetadataQueue; 227 228 /**** Scope for mLock ****/ 229 230 /** 231 * Adapter for legacy HAL / HIDL HAL interface calls; calls either into legacy HALv3 or the 232 * HIDL HALv3 interfaces. 233 */ 234 class HalInterface : public camera3::Camera3StreamBufferFreedListener { 235 public: 236 HalInterface(sp<hardware::camera::device::V3_2::ICameraDeviceSession> &session, 237 std::shared_ptr<RequestMetadataQueue> queue); 238 HalInterface(const HalInterface &other); 239 HalInterface(); 240 241 // Returns true if constructed with a valid device or session, and not yet cleared 242 bool valid(); 243 244 // Reset this HalInterface object (does not call close()) 245 void clear(); 246 247 // Check if HalInterface support sending requests in batch 248 bool supportBatchRequest(); 249 250 // Calls into the HAL interface 251 252 // Caller takes ownership of requestTemplate 253 status_t constructDefaultRequestSettings(camera3_request_template_t templateId, 254 /*out*/ camera_metadata_t **requestTemplate); 255 status_t configureStreams(/*inout*/ camera3_stream_configuration *config); 256 status_t processCaptureRequest(camera3_capture_request_t *request); 257 status_t processBatchCaptureRequests( 258 std::vector<camera3_capture_request_t*>& requests, 259 /*out*/uint32_t* numRequestProcessed); 260 status_t flush(); 261 status_t dump(int fd); 262 status_t close(); 263 264 // Find a buffer_handle_t based on frame number and stream ID 265 status_t popInflightBuffer(int32_t frameNumber, int32_t streamId, 266 /*out*/ buffer_handle_t **buffer); 267 268 // Get a vector of (frameNumber, streamId) pair of currently inflight 269 // buffers 270 void getInflightBufferKeys(std::vector<std::pair<int32_t, int32_t>>* out); 271 272 private: 273 sp<hardware::camera::device::V3_2::ICameraDeviceSession> mHidlSession; 274 std::shared_ptr<RequestMetadataQueue> mRequestMetadataQueue; 275 276 std::mutex mInflightLock; 277 278 // The output HIDL request still depends on input camera3_capture_request_t 279 // Do not free input camera3_capture_request_t before output HIDL request 280 void wrapAsHidlRequest(camera3_capture_request_t* in, 281 /*out*/hardware::camera::device::V3_2::CaptureRequest* out, 282 /*out*/std::vector<native_handle_t*>* handlesCreated); 283 284 status_t pushInflightBufferLocked(int32_t frameNumber, int32_t streamId, 285 buffer_handle_t *buffer, int acquireFence); 286 // Cache of buffer handles keyed off (frameNumber << 32 | streamId) 287 // value is a pair of (buffer_handle_t*, acquire_fence FD) 288 std::unordered_map<uint64_t, std::pair<buffer_handle_t*, int>> mInflightBufferMap; 289 290 struct BufferHasher { operatorBufferHasher291 size_t operator()(const buffer_handle_t& buf) const { 292 if (buf == nullptr) 293 return 0; 294 295 size_t result = 1; 296 result = 31 * result + buf->numFds; 297 for (int i = 0; i < buf->numFds; i++) { 298 result = 31 * result + buf->data[i]; 299 } 300 return result; 301 } 302 }; 303 304 struct BufferComparator { operatorBufferComparator305 bool operator()(const buffer_handle_t& buf1, const buffer_handle_t& buf2) const { 306 if (buf1->numFds == buf2->numFds) { 307 for (int i = 0; i < buf1->numFds; i++) { 308 if (buf1->data[i] != buf2->data[i]) { 309 return false; 310 } 311 } 312 return true; 313 } 314 return false; 315 } 316 }; 317 318 std::mutex mBufferIdMapLock; // protecting mBufferIdMaps and mNextBufferId 319 typedef std::unordered_map<const buffer_handle_t, uint64_t, 320 BufferHasher, BufferComparator> BufferIdMap; 321 // stream ID -> per stream buffer ID map 322 std::unordered_map<int, BufferIdMap> mBufferIdMaps; 323 uint64_t mNextBufferId = 1; // 0 means no buffer 324 static const uint64_t BUFFER_ID_NO_BUFFER = 0; 325 326 // method to extract buffer's unique ID 327 // TODO: we should switch to use gralloc mapper's getBackingStore API 328 // once we ran in binderized gralloc mode, but before that is ready, 329 // we need to rely on the conventional buffer queue behavior where 330 // buffer_handle_t's FD won't change. 331 // return pair of (newlySeenBuffer?, bufferId) 332 std::pair<bool, uint64_t> getBufferId(const buffer_handle_t& buf, int streamId); 333 334 virtual void onBufferFreed(int streamId, const native_handle_t* handle) override; 335 336 std::vector<std::pair<int, uint64_t>> mFreedBuffers; 337 }; 338 339 sp<HalInterface> mInterface; 340 341 CameraMetadata mDeviceInfo; 342 343 CameraMetadata mRequestTemplateCache[CAMERA3_TEMPLATE_COUNT]; 344 345 struct Size { 346 uint32_t width; 347 uint32_t height; widthSize348 explicit Size(uint32_t w = 0, uint32_t h = 0) : width(w), height(h){} 349 }; 350 // Map from format to size. 351 Vector<Size> mSupportedOpaqueInputSizes; 352 353 enum Status { 354 STATUS_ERROR, 355 STATUS_UNINITIALIZED, 356 STATUS_UNCONFIGURED, 357 STATUS_CONFIGURED, 358 STATUS_ACTIVE 359 } mStatus; 360 361 // Only clear mRecentStatusUpdates, mStatusWaiters from waitUntilStateThenRelock 362 Vector<Status> mRecentStatusUpdates; 363 int mStatusWaiters; 364 365 Condition mStatusChanged; 366 367 // Tracking cause of fatal errors when in STATUS_ERROR 368 String8 mErrorCause; 369 370 // Mapping of stream IDs to stream instances 371 typedef KeyedVector<int, sp<camera3::Camera3OutputStreamInterface> > 372 StreamSet; 373 374 StreamSet mOutputStreams; 375 sp<camera3::Camera3Stream> mInputStream; 376 int mNextStreamId; 377 bool mNeedConfig; 378 379 int mDummyStreamId; 380 381 // Whether to send state updates upstream 382 // Pause when doing transparent reconfiguration 383 bool mPauseStateNotify; 384 385 // Need to hold on to stream references until configure completes. 386 Vector<sp<camera3::Camera3StreamInterface> > mDeletedStreams; 387 388 // Whether the HAL will send partial result 389 bool mUsePartialResult; 390 391 // Number of partial results that will be delivered by the HAL. 392 uint32_t mNumPartialResults; 393 394 /**** End scope for mLock ****/ 395 396 // The offset converting from clock domain of other subsystem 397 // (video/hardware composer) to that of camera. Assumption is that this 398 // offset won't change during the life cycle of the camera device. In other 399 // words, camera device shouldn't be open during CPU suspend. 400 nsecs_t mTimestampOffset; 401 402 class CaptureRequest : public LightRefBase<CaptureRequest> { 403 public: 404 CameraMetadata mSettings; 405 sp<camera3::Camera3Stream> mInputStream; 406 camera3_stream_buffer_t mInputBuffer; 407 Vector<sp<camera3::Camera3OutputStreamInterface> > 408 mOutputStreams; 409 SurfaceMap mOutputSurfaces; 410 CaptureResultExtras mResultExtras; 411 // The number of requests that should be submitted to HAL at a time. 412 // For example, if batch size is 8, this request and the following 7 413 // requests will be submitted to HAL at a time. The batch size for 414 // the following 7 requests will be ignored by the request thread. 415 int mBatchSize; 416 // Whether this request is from a repeating or repeating burst. 417 bool mRepeating; 418 }; 419 typedef List<sp<CaptureRequest> > RequestList; 420 421 status_t checkStatusOkToCaptureLocked(); 422 423 status_t convertMetadataListToRequestListLocked( 424 const List<const CameraMetadata> &metadataList, 425 const std::list<const SurfaceMap> &surfaceMaps, 426 bool repeating, 427 /*out*/ 428 RequestList *requestList); 429 430 void convertToRequestList(List<const CameraMetadata>& requests, 431 std::list<const SurfaceMap>& surfaceMaps, 432 const CameraMetadata& request); 433 434 status_t submitRequestsHelper(const List<const CameraMetadata> &requests, 435 const std::list<const SurfaceMap> &surfaceMaps, 436 bool repeating, 437 int64_t *lastFrameNumber = NULL); 438 439 440 /** 441 * Implementation of android::hardware::camera::device::V3_2::ICameraDeviceCallback 442 */ 443 444 hardware::Return<void> processCaptureResult( 445 const hardware::hidl_vec< 446 hardware::camera::device::V3_2::CaptureResult>& results) override; 447 hardware::Return<void> notify( 448 const hardware::hidl_vec< 449 hardware::camera::device::V3_2::NotifyMsg>& msgs) override; 450 451 // Handle one capture result. Assume that mProcessCaptureResultLock is held. 452 void processOneCaptureResultLocked( 453 const hardware::camera::device::V3_2::CaptureResult& results); 454 // Handle one notify message 455 void notify(const hardware::camera::device::V3_2::NotifyMsg& msg); 456 457 // lock to ensure only one processCaptureResult is called at a time. 458 Mutex mProcessCaptureResultLock; 459 460 /** 461 * Common initialization code shared by both HAL paths 462 * 463 * Must be called with mLock and mInterfaceLock held. 464 */ 465 status_t initializeCommonLocked(); 466 467 /** 468 * Get the last request submitted to the hal by the request thread. 469 * 470 * Must be called with mLock held. 471 */ 472 virtual CameraMetadata getLatestRequestLocked(); 473 474 /** 475 * Update the current device status and wake all waiting threads. 476 * 477 * Must be called with mLock held. 478 */ 479 void internalUpdateStatusLocked(Status status); 480 481 /** 482 * Pause processing and flush everything, but don't tell the clients. 483 * This is for reconfiguring outputs transparently when according to the 484 * CameraDeviceBase interface we shouldn't need to. 485 * Must be called with mLock and mInterfaceLock both held. 486 */ 487 status_t internalPauseAndWaitLocked(nsecs_t maxExpectedDuration); 488 489 /** 490 * Resume work after internalPauseAndWaitLocked() 491 * Must be called with mLock and mInterfaceLock both held. 492 */ 493 status_t internalResumeLocked(); 494 495 /** 496 * Wait until status tracker tells us we've transitioned to the target state 497 * set, which is either ACTIVE when active==true or IDLE (which is any 498 * non-ACTIVE state) when active==false. 499 * 500 * Needs to be called with mLock and mInterfaceLock held. This means there 501 * can ever only be one waiter at most. 502 * 503 * During the wait mLock is released. 504 * 505 */ 506 status_t waitUntilStateThenRelock(bool active, nsecs_t timeout); 507 508 /** 509 * Implementation of waitUntilDrained. On success, will transition to IDLE state. 510 * 511 * Need to be called with mLock and mInterfaceLock held. 512 */ 513 status_t waitUntilDrainedLocked(nsecs_t maxExpectedDuration); 514 515 /** 516 * Do common work for setting up a streaming or single capture request. 517 * On success, will transition to ACTIVE if in IDLE. 518 */ 519 sp<CaptureRequest> setUpRequestLocked(const CameraMetadata &request, 520 const SurfaceMap &surfaceMap); 521 522 /** 523 * Build a CaptureRequest request from the CameraDeviceBase request 524 * settings. 525 */ 526 sp<CaptureRequest> createCaptureRequest(const CameraMetadata &request, 527 const SurfaceMap &surfaceMap); 528 529 /** 530 * Take the currently-defined set of streams and configure the HAL to use 531 * them. This is a long-running operation (may be several hundered ms). 532 */ 533 status_t configureStreamsLocked(int operatingMode); 534 535 /** 536 * Cancel stream configuration that did not finish successfully. 537 */ 538 void cancelStreamsConfigurationLocked(); 539 540 /** 541 * Add a dummy stream to the current stream set as a workaround for 542 * not allowing 0 streams in the camera HAL spec. 543 */ 544 status_t addDummyStreamLocked(); 545 546 /** 547 * Remove a dummy stream if the current config includes real streams. 548 */ 549 status_t tryRemoveDummyStreamLocked(); 550 551 /** 552 * Set device into an error state due to some fatal failure, and set an 553 * error message to indicate why. Only the first call's message will be 554 * used. The message is also sent to the log. 555 */ 556 void setErrorState(const char *fmt, ...); 557 void setErrorStateV(const char *fmt, va_list args); 558 void setErrorStateLocked(const char *fmt, ...); 559 void setErrorStateLockedV(const char *fmt, va_list args); 560 561 /** 562 * Debugging trylock/spin method 563 * Try to acquire a lock a few times with sleeps between before giving up. 564 */ 565 bool tryLockSpinRightRound(Mutex& lock); 566 567 /** 568 * Helper function to determine if an input size for implementation defined 569 * format is supported. 570 */ 571 bool isOpaqueInputSizeSupported(uint32_t width, uint32_t height); 572 573 /** 574 * Helper function to get the largest Jpeg resolution (in area) 575 * Return Size(0, 0) if static metatdata is invalid 576 */ 577 Size getMaxJpegResolution() const; 578 579 /** 580 * Helper function to get the offset between MONOTONIC and BOOTTIME 581 * timestamp. 582 */ 583 static nsecs_t getMonoToBoottimeOffset(); 584 585 /** 586 * Helper functions to map between framework and HIDL values 587 */ 588 static hardware::graphics::common::V1_0::PixelFormat mapToPixelFormat(int frameworkFormat); 589 static hardware::camera::device::V3_2::DataspaceFlags mapToHidlDataspace( 590 android_dataspace dataSpace); 591 static hardware::camera::device::V3_2::BufferUsageFlags mapToConsumerUsage(uint64_t usage); 592 static hardware::camera::device::V3_2::StreamRotation mapToStreamRotation( 593 camera3_stream_rotation_t rotation); 594 // Returns a negative error code if the passed-in operation mode is not valid. 595 static status_t mapToStreamConfigurationMode(camera3_stream_configuration_mode_t operationMode, 596 /*out*/ hardware::camera::device::V3_2::StreamConfigurationMode *mode); 597 static camera3_buffer_status_t mapHidlBufferStatus(hardware::camera::device::V3_2::BufferStatus status); 598 static int mapToFrameworkFormat(hardware::graphics::common::V1_0::PixelFormat pixelFormat); 599 static android_dataspace mapToFrameworkDataspace( 600 hardware::camera::device::V3_2::DataspaceFlags); 601 static uint64_t mapConsumerToFrameworkUsage( 602 hardware::camera::device::V3_2::BufferUsageFlags usage); 603 static uint64_t mapProducerToFrameworkUsage( 604 hardware::camera::device::V3_2::BufferUsageFlags usage); 605 606 struct RequestTrigger { 607 // Metadata tag number, e.g. android.control.aePrecaptureTrigger 608 uint32_t metadataTag; 609 // Metadata value, e.g. 'START' or the trigger ID 610 int32_t entryValue; 611 612 // The last part of the fully qualified path, e.g. afTrigger getTagNameRequestTrigger613 const char *getTagName() const { 614 return get_camera_metadata_tag_name(metadataTag) ?: "NULL"; 615 } 616 617 // e.g. TYPE_BYTE, TYPE_INT32, etc. getTagTypeRequestTrigger618 int getTagType() const { 619 return get_camera_metadata_tag_type(metadataTag); 620 } 621 }; 622 623 /** 624 * Thread for managing capture request submission to HAL device. 625 */ 626 class RequestThread : public Thread { 627 628 public: 629 630 RequestThread(wp<Camera3Device> parent, 631 sp<camera3::StatusTracker> statusTracker, 632 sp<HalInterface> interface); 633 ~RequestThread(); 634 635 void setNotificationListener(wp<NotificationListener> listener); 636 637 /** 638 * Call after stream (re)-configuration is completed. 639 */ 640 void configurationComplete(bool isConstrainedHighSpeed); 641 642 /** 643 * Set or clear the list of repeating requests. Does not block 644 * on either. Use waitUntilPaused to wait until request queue 645 * has emptied out. 646 */ 647 status_t setRepeatingRequests(const RequestList& requests, 648 /*out*/ 649 int64_t *lastFrameNumber = NULL); 650 status_t clearRepeatingRequests(/*out*/ 651 int64_t *lastFrameNumber = NULL); 652 653 status_t queueRequestList(List<sp<CaptureRequest> > &requests, 654 /*out*/ 655 int64_t *lastFrameNumber = NULL); 656 657 /** 658 * Remove all queued and repeating requests, and pending triggers 659 */ 660 status_t clear(/*out*/int64_t *lastFrameNumber = NULL); 661 662 /** 663 * Flush all pending requests in HAL. 664 */ 665 status_t flush(); 666 667 /** 668 * Queue a trigger to be dispatched with the next outgoing 669 * process_capture_request. The settings for that request only 670 * will be temporarily rewritten to add the trigger tag/value. 671 * Subsequent requests will not be rewritten (for this tag). 672 */ 673 status_t queueTrigger(RequestTrigger trigger[], size_t count); 674 675 /** 676 * Pause/unpause the capture thread. Doesn't block, so use 677 * waitUntilPaused to wait until the thread is paused. 678 */ 679 void setPaused(bool paused); 680 681 /** 682 * Wait until thread processes the capture request with settings' 683 * android.request.id == requestId. 684 * 685 * Returns TIMED_OUT in case the thread does not process the request 686 * within the timeout. 687 */ 688 status_t waitUntilRequestProcessed(int32_t requestId, nsecs_t timeout); 689 690 /** 691 * Shut down the thread. Shutdown is asynchronous, so thread may 692 * still be running once this method returns. 693 */ 694 virtual void requestExit(); 695 696 /** 697 * Get the latest request that was sent to the HAL 698 * with process_capture_request. 699 */ 700 CameraMetadata getLatestRequest() const; 701 702 /** 703 * Returns true if the stream is a target of any queued or repeating 704 * capture request 705 */ 706 bool isStreamPending(sp<camera3::Camera3StreamInterface>& stream); 707 708 // dump processCaptureRequest latency dumpCaptureRequestLatency(int fd,const char * name)709 void dumpCaptureRequestLatency(int fd, const char* name) { 710 mRequestLatency.dump(fd, name); 711 } 712 713 protected: 714 715 virtual bool threadLoop(); 716 717 private: 718 static const String8& getId(const wp<Camera3Device> &device); 719 720 status_t queueTriggerLocked(RequestTrigger trigger); 721 // Mix-in queued triggers into this request 722 int32_t insertTriggers(const sp<CaptureRequest> &request); 723 // Purge the queued triggers from this request, 724 // restoring the old field values for those tags. 725 status_t removeTriggers(const sp<CaptureRequest> &request); 726 727 // HAL workaround: Make sure a trigger ID always exists if 728 // a trigger does 729 status_t addDummyTriggerIds(const sp<CaptureRequest> &request); 730 731 static const nsecs_t kRequestTimeout = 50e6; // 50 ms 732 733 // Used to prepare a batch of requests. 734 struct NextRequest { 735 sp<CaptureRequest> captureRequest; 736 camera3_capture_request_t halRequest; 737 Vector<camera3_stream_buffer_t> outputBuffers; 738 bool submitted; 739 }; 740 741 // Wait for the next batch of requests and put them in mNextRequests. mNextRequests will 742 // be empty if it times out. 743 void waitForNextRequestBatch(); 744 745 // Waits for a request, or returns NULL if times out. Must be called with mRequestLock hold. 746 sp<CaptureRequest> waitForNextRequestLocked(); 747 748 // Prepare HAL requests and output buffers in mNextRequests. Return TIMED_OUT if getting any 749 // output buffer timed out. If an error is returned, the caller should clean up the pending 750 // request batch. 751 status_t prepareHalRequests(); 752 753 // Return buffers, etc, for requests in mNextRequests that couldn't be fully constructed and 754 // send request errors if sendRequestError is true. The buffers will be returned in the 755 // ERROR state to mark them as not having valid data. mNextRequests will be cleared. 756 void cleanUpFailedRequests(bool sendRequestError); 757 758 // Stop the repeating request if any of its output streams is abandoned. 759 void checkAndStopRepeatingRequest(); 760 761 // Pause handling 762 bool waitIfPaused(); 763 void unpauseForNewRequests(); 764 765 // Relay error to parent device object setErrorState 766 void setErrorState(const char *fmt, ...); 767 768 // If the input request is in mRepeatingRequests. Must be called with mRequestLock hold 769 bool isRepeatingRequestLocked(const sp<CaptureRequest>&); 770 771 // Clear repeating requests. Must be called with mRequestLock held. 772 status_t clearRepeatingRequestsLocked(/*out*/ int64_t *lastFrameNumber = NULL); 773 774 // send request in mNextRequests to HAL one by one. Return true = sucssess 775 bool sendRequestsOneByOne(); 776 777 // send request in mNextRequests to HAL in a batch. Return true = sucssess 778 bool sendRequestsBatch(); 779 780 // Calculate the expected maximum duration for a request 781 nsecs_t calculateMaxExpectedDuration(const camera_metadata_t *request); 782 783 wp<Camera3Device> mParent; 784 wp<camera3::StatusTracker> mStatusTracker; 785 sp<HalInterface> mInterface; 786 787 wp<NotificationListener> mListener; 788 789 const String8& mId; // The camera ID 790 int mStatusId; // The RequestThread's component ID for 791 // status tracking 792 793 Mutex mRequestLock; 794 Condition mRequestSignal; 795 RequestList mRequestQueue; 796 RequestList mRepeatingRequests; 797 // The next batch of requests being prepped for submission to the HAL, no longer 798 // on the request queue. Read-only even with mRequestLock held, outside 799 // of threadLoop 800 Vector<NextRequest> mNextRequests; 801 802 // To protect flush() and sending a request batch to HAL. 803 Mutex mFlushLock; 804 805 bool mReconfigured; 806 807 // Used by waitIfPaused, waitForNextRequest, and waitUntilPaused 808 Mutex mPauseLock; 809 bool mDoPause; 810 Condition mDoPauseSignal; 811 bool mPaused; 812 Condition mPausedSignal; 813 814 sp<CaptureRequest> mPrevRequest; 815 int32_t mPrevTriggers; 816 817 uint32_t mFrameNumber; 818 819 mutable Mutex mLatestRequestMutex; 820 Condition mLatestRequestSignal; 821 // android.request.id for latest process_capture_request 822 int32_t mLatestRequestId; 823 CameraMetadata mLatestRequest; 824 825 typedef KeyedVector<uint32_t/*tag*/, RequestTrigger> TriggerMap; 826 Mutex mTriggerMutex; 827 TriggerMap mTriggerMap; 828 TriggerMap mTriggerRemovedMap; 829 TriggerMap mTriggerReplacedMap; 830 uint32_t mCurrentAfTriggerId; 831 uint32_t mCurrentPreCaptureTriggerId; 832 833 int64_t mRepeatingLastFrameNumber; 834 835 // Flag indicating if we should prepare video stream for video requests. 836 bool mPrepareVideoStream; 837 838 static const int32_t kRequestLatencyBinSize = 40; // in ms 839 CameraLatencyHistogram mRequestLatency; 840 }; 841 sp<RequestThread> mRequestThread; 842 843 /** 844 * In-flight queue for tracking completion of capture requests. 845 */ 846 847 struct InFlightRequest { 848 // Set by notify() SHUTTER call. 849 nsecs_t shutterTimestamp; 850 // Set by process_capture_result(). 851 nsecs_t sensorTimestamp; 852 int requestStatus; 853 // Set by process_capture_result call with valid metadata 854 bool haveResultMetadata; 855 // Decremented by calls to process_capture_result with valid output 856 // and input buffers 857 int numBuffersLeft; 858 CaptureResultExtras resultExtras; 859 // If this request has any input buffer 860 bool hasInputBuffer; 861 862 // The last metadata that framework receives from HAL and 863 // not yet send out because the shutter event hasn't arrived. 864 // It's added by process_capture_result and sent when framework 865 // receives the shutter event. 866 CameraMetadata pendingMetadata; 867 868 // The metadata of the partial results that framework receives from HAL so far 869 // and has sent out. 870 CameraMetadata collectedPartialResult; 871 872 // Buffers are added by process_capture_result when output buffers 873 // return from HAL but framework has not yet received the shutter 874 // event. They will be returned to the streams when framework receives 875 // the shutter event. 876 Vector<camera3_stream_buffer_t> pendingOutputBuffers; 877 878 // Whether this inflight request's shutter and result callback are to be 879 // called. The policy is that if the request is the last one in the constrained 880 // high speed recording request list, this flag will be true. If the request list 881 // is not for constrained high speed recording, this flag will also be true. 882 bool hasCallback; 883 884 // Maximum expected frame duration for this request. 885 // For manual captures, equal to the max of requested exposure time and frame duration 886 // For auto-exposure modes, equal to 1/(lower end of target FPS range) 887 nsecs_t maxExpectedDuration; 888 889 // Whether the result metadata for this request is to be skipped. The 890 // result metadata should be skipped in the case of 891 // REQUEST/RESULT error. 892 bool skipResultMetadata; 893 894 // Default constructor needed by KeyedVector InFlightRequestInFlightRequest895 InFlightRequest() : 896 shutterTimestamp(0), 897 sensorTimestamp(0), 898 requestStatus(OK), 899 haveResultMetadata(false), 900 numBuffersLeft(0), 901 hasInputBuffer(false), 902 hasCallback(true), 903 maxExpectedDuration(kDefaultExpectedDuration), 904 skipResultMetadata(false) { 905 } 906 InFlightRequestInFlightRequest907 InFlightRequest(int numBuffers, CaptureResultExtras extras, bool hasInput, 908 bool hasAppCallback, nsecs_t maxDuration) : 909 shutterTimestamp(0), 910 sensorTimestamp(0), 911 requestStatus(OK), 912 haveResultMetadata(false), 913 numBuffersLeft(numBuffers), 914 resultExtras(extras), 915 hasInputBuffer(hasInput), 916 hasCallback(hasAppCallback), 917 maxExpectedDuration(maxDuration), 918 skipResultMetadata(false) { 919 } 920 }; 921 922 // Map from frame number to the in-flight request state 923 typedef KeyedVector<uint32_t, InFlightRequest> InFlightMap; 924 925 926 Mutex mInFlightLock; // Protects mInFlightMap and 927 // mExpectedInflightDuration 928 InFlightMap mInFlightMap; 929 nsecs_t mExpectedInflightDuration = 0; 930 int mInFlightStatusId; 931 932 933 status_t registerInFlight(uint32_t frameNumber, 934 int32_t numBuffers, CaptureResultExtras resultExtras, bool hasInput, 935 bool callback, nsecs_t maxExpectedDuration); 936 937 /** 938 * Returns the maximum expected time it'll take for all currently in-flight 939 * requests to complete, based on their settings 940 */ 941 nsecs_t getExpectedInFlightDuration(); 942 943 /** 944 * Tracking for idle detection 945 */ 946 sp<camera3::StatusTracker> mStatusTracker; 947 948 /** 949 * Graphic buffer manager for output streams. Each device has a buffer manager, which is used 950 * by the output streams to get and return buffers if these streams are registered to this 951 * buffer manager. 952 */ 953 sp<camera3::Camera3BufferManager> mBufferManager; 954 955 /** 956 * Thread for preparing streams 957 */ 958 class PreparerThread : private Thread, public virtual RefBase { 959 public: 960 PreparerThread(); 961 ~PreparerThread(); 962 963 void setNotificationListener(wp<NotificationListener> listener); 964 965 /** 966 * Queue up a stream to be prepared. Streams are processed by a background thread in FIFO 967 * order. Pre-allocate up to maxCount buffers for the stream, or the maximum number needed 968 * for the pipeline if maxCount is ALLOCATE_PIPELINE_MAX. 969 */ 970 status_t prepare(int maxCount, sp<camera3::Camera3StreamInterface>& stream); 971 972 /** 973 * Cancel all current and pending stream preparation 974 */ 975 status_t clear(); 976 977 private: 978 Mutex mLock; 979 980 virtual bool threadLoop(); 981 982 // Guarded by mLock 983 984 wp<NotificationListener> mListener; 985 List<sp<camera3::Camera3StreamInterface> > mPendingStreams; 986 bool mActive; 987 bool mCancelNow; 988 989 // Only accessed by threadLoop and the destructor 990 991 sp<camera3::Camera3StreamInterface> mCurrentStream; 992 }; 993 sp<PreparerThread> mPreparerThread; 994 995 /** 996 * Output result queue and current HAL device 3A state 997 */ 998 999 // Lock for output side of device 1000 Mutex mOutputLock; 1001 1002 /**** Scope for mOutputLock ****/ 1003 // the minimal frame number of the next non-reprocess result 1004 uint32_t mNextResultFrameNumber; 1005 // the minimal frame number of the next reprocess result 1006 uint32_t mNextReprocessResultFrameNumber; 1007 // the minimal frame number of the next non-reprocess shutter 1008 uint32_t mNextShutterFrameNumber; 1009 // the minimal frame number of the next reprocess shutter 1010 uint32_t mNextReprocessShutterFrameNumber; 1011 List<CaptureResult> mResultQueue; 1012 Condition mResultSignal; 1013 wp<NotificationListener> mListener; 1014 1015 /**** End scope for mOutputLock ****/ 1016 1017 /** 1018 * Callback functions from HAL device 1019 */ 1020 void processCaptureResult(const camera3_capture_result *result); 1021 1022 void notify(const camera3_notify_msg *msg); 1023 1024 // Specific notify handlers 1025 void notifyError(const camera3_error_msg_t &msg, 1026 sp<NotificationListener> listener); 1027 void notifyShutter(const camera3_shutter_msg_t &msg, 1028 sp<NotificationListener> listener); 1029 1030 // helper function to return the output buffers to the streams. 1031 void returnOutputBuffers(const camera3_stream_buffer_t *outputBuffers, 1032 size_t numBuffers, nsecs_t timestamp); 1033 1034 // Send a partial capture result. 1035 void sendPartialCaptureResult(const camera_metadata_t * partialResult, 1036 const CaptureResultExtras &resultExtras, uint32_t frameNumber); 1037 1038 // Send a total capture result given the pending metadata and result extras, 1039 // partial results, and the frame number to the result queue. 1040 void sendCaptureResult(CameraMetadata &pendingMetadata, 1041 CaptureResultExtras &resultExtras, 1042 CameraMetadata &collectedPartialResult, uint32_t frameNumber, 1043 bool reprocess); 1044 1045 // Insert the result to the result queue after updating frame number and overriding AE 1046 // trigger cancel. 1047 // mOutputLock must be held when calling this function. 1048 void insertResultLocked(CaptureResult *result, uint32_t frameNumber); 1049 1050 /**** Scope for mInFlightLock ****/ 1051 1052 // Remove the in-flight map entry of the given index from mInFlightMap. 1053 // It must only be called with mInFlightLock held. 1054 void removeInFlightMapEntryLocked(int idx); 1055 // Remove the in-flight request of the given index from mInFlightMap 1056 // if it's no longer needed. It must only be called with mInFlightLock held. 1057 void removeInFlightRequestIfReadyLocked(int idx); 1058 // Remove all in-flight requests and return all buffers. 1059 // This is used after HAL interface is closed to cleanup any request/buffers 1060 // not returned by HAL. 1061 void flushInflightRequests(); 1062 1063 /**** End scope for mInFlightLock ****/ 1064 1065 // Debug tracker for metadata tag value changes 1066 // - Enabled with the -m <taglist> option to dumpsys, such as 1067 // dumpsys -m android.control.aeState,android.control.aeMode 1068 // - Disabled with -m off 1069 // - dumpsys -m 3a is a shortcut for ae/af/awbMode, State, and Triggers 1070 TagMonitor mTagMonitor; 1071 1072 void monitorMetadata(TagMonitor::eventSource source, int64_t frameNumber, 1073 nsecs_t timestamp, const CameraMetadata& metadata); 1074 1075 metadata_vendor_id_t mVendorTagId; 1076 1077 /** 1078 * Static callback forwarding methods from HAL to instance 1079 */ 1080 static callbacks_process_capture_result_t sProcessCaptureResult; 1081 1082 static callbacks_notify_t sNotify; 1083 1084 }; // class Camera3Device 1085 1086 }; // namespace android 1087 1088 #endif 1089