1 /* 2 * Copyright (C) 2013-2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef ANDROID_SERVERS_CAMERA3_STREAM_H 18 #define ANDROID_SERVERS_CAMERA3_STREAM_H 19 20 #include <gui/Surface.h> 21 #include <utils/RefBase.h> 22 #include <utils/String8.h> 23 #include <utils/String16.h> 24 #include <utils/List.h> 25 26 #include "utils/LatencyHistogram.h" 27 #include "Camera3StreamBufferListener.h" 28 #include "Camera3StreamInterface.h" 29 30 namespace android { 31 32 namespace camera3 { 33 34 /** 35 * A class for managing a single stream of input or output data from the camera 36 * device. 37 * 38 * The stream has an internal state machine to track whether it's 39 * connected/configured/etc. 40 * 41 * States: 42 * 43 * STATE_ERROR: A serious error has occurred, stream is unusable. Outstanding 44 * buffers may still be returned. 45 * 46 * STATE_CONSTRUCTED: The stream is ready for configuration, but buffers cannot 47 * be gotten yet. Not connected to any endpoint, no buffers are registered 48 * with the HAL. 49 * 50 * STATE_IN_CONFIG: Configuration has started, but not yet concluded. During this 51 * time, the usage, max_buffers, and priv fields of camera_stream returned by 52 * startConfiguration() may be modified. 53 * 54 * STATE_IN_RE_CONFIG: Configuration has started, and the stream has been 55 * configured before. Need to track separately from IN_CONFIG to avoid 56 * re-registering buffers with HAL. 57 * 58 * STATE_CONFIGURED: Stream is configured, and has registered buffers with the 59 * HAL (if necessary). The stream's getBuffer/returnBuffer work. The priv 60 * pointer may still be modified. 61 * 62 * STATE_PREPARING: The stream's buffers are being pre-allocated for use. On 63 * older HALs, this is done as part of configuration, but in newer HALs 64 * buffers may be allocated at time of first use. But some use cases require 65 * buffer allocation upfront, to minmize disruption due to lengthy allocation 66 * duration. In this state, only prepareNextBuffer() and cancelPrepare() 67 * may be called. 68 * 69 * STATE_IN_IDLE: This is a temporary state only intended to be used for input 70 * streams and only for the case where we need to re-configure the camera device 71 * while the input stream has an outstanding buffer. All other streams should not 72 * be able to switch to this state. For them this is invalid and should be handled 73 * as an unknown state. 74 * 75 * Transition table: 76 * 77 * <none> => STATE_CONSTRUCTED: 78 * When constructed with valid arguments 79 * <none> => STATE_ERROR: 80 * When constructed with invalid arguments 81 * STATE_CONSTRUCTED => STATE_IN_CONFIG: 82 * When startConfiguration() is called 83 * STATE_IN_CONFIG => STATE_CONFIGURED: 84 * When finishConfiguration() is called 85 * STATE_IN_CONFIG => STATE_ERROR: 86 * When finishConfiguration() fails to allocate or register buffers. 87 * STATE_CONFIGURED => STATE_IN_RE_CONFIG: * 88 * When startConfiguration() is called again, after making sure stream is 89 * idle with waitUntilIdle(). 90 * STATE_IN_RE_CONFIG => STATE_CONFIGURED: 91 * When finishConfiguration() is called. 92 * STATE_IN_RE_CONFIG => STATE_ERROR: 93 * When finishConfiguration() fails to allocate or register buffers. 94 * STATE_CONFIGURED => STATE_CONSTRUCTED: 95 * When disconnect() is called after making sure stream is idle with 96 * waitUntilIdle(). 97 * STATE_CONFIGURED => STATE_PREPARING: 98 * When startPrepare is called before the stream has a buffer 99 * queued back into it for the first time. 100 * STATE_PREPARING => STATE_CONFIGURED: 101 * When sufficient prepareNextBuffer calls have been made to allocate 102 * all stream buffers, or cancelPrepare is called. 103 * STATE_CONFIGURED => STATE_ABANDONED: 104 * When the buffer queue of the stream is abandoned. 105 * STATE_CONFIGURED => STATE_IN_IDLE: 106 * Only for an input stream which has an outstanding buffer. 107 * STATE_IN_IDLE => STATE_CONFIGURED: 108 * After the internal re-configuration, the input should revert back to 109 * the configured state. 110 * 111 * Status Tracking: 112 * Each stream is tracked by StatusTracker as a separate component, 113 * depending on the handed out buffer count. The state must be STATE_CONFIGURED 114 * in order for the component to be marked. 115 * 116 * It's marked in one of two ways: 117 * 118 * - ACTIVE: One or more buffers have been handed out (with #getBuffer). 119 * - IDLE: All buffers have been returned (with #returnBuffer), and their 120 * respective release_fence(s) have been signaled. The only exception to this 121 * rule is an input stream that moves to "STATE_IN_IDLE" during internal 122 * re-configuration. 123 * 124 * A typical use case is output streams. When the HAL has any buffers 125 * dequeued, the stream is marked ACTIVE. When the HAL returns all buffers 126 * (e.g. if no capture requests are active), the stream is marked IDLE. 127 * In this use case, the app consumer does not affect the component status. 128 * 129 */ 130 class Camera3Stream : 131 protected camera_stream, 132 public virtual Camera3StreamInterface, 133 public virtual RefBase { 134 public: 135 136 virtual ~Camera3Stream(); 137 138 static Camera3Stream* cast(camera_stream *stream); 139 static const Camera3Stream* cast(const camera_stream *stream); 140 141 // Queue corresponding HDR metadata to given native window. 142 static void queueHDRMetadata(buffer_handle_t buffer, sp<ANativeWindow>& anw, 143 int64_t dynamicRangeProfile); 144 145 /** 146 * Get the stream's ID 147 */ 148 int getId() const; 149 150 /** 151 * Get the output stream set id. 152 */ 153 int getStreamSetId() const; 154 /** 155 * Is this stream part of a multi-resolution stream set 156 */ 157 bool isMultiResolution() const; 158 /** 159 * Get the HAL stream group id for a multi-resolution stream set 160 */ 161 int getHalStreamGroupId() const; 162 163 /** 164 * Get the stream's dimensions and format 165 */ 166 uint32_t getWidth() const; 167 uint32_t getHeight() const; 168 int getFormat() const; 169 android_dataspace getDataSpace() const; 170 uint64_t getUsage() const; 171 void setUsage(uint64_t usage); 172 void setFormatOverride(bool formatOverriden); 173 bool isFormatOverridden() const; 174 int getOriginalFormat() const; 175 int64_t getDynamicRangeProfile() const; 176 void setDataSpaceOverride(bool dataSpaceOverriden); 177 bool isDataSpaceOverridden() const; 178 android_dataspace getOriginalDataSpace() const; 179 int getMaxHalBuffers() const; 180 const String8& physicalCameraId() const; 181 int64_t getStreamUseCase() const; 182 int getTimestampBase() const; 183 bool isDeviceTimeBaseRealtime() const; 184 185 void setOfflineProcessingSupport(bool) override; 186 bool getOfflineProcessingSupport() const override; 187 asHalStream()188 camera_stream* asHalStream() override { 189 return this; 190 } 191 192 /** 193 * Start the stream configuration process. Returns a handle to the stream's 194 * information to be passed into the HAL device's configure_streams call. 195 * 196 * Until finishConfiguration() is called, no other methods on the stream may be 197 * called. The usage and max_buffers fields of camera_stream may be modified 198 * between start/finishConfiguration, but may not be changed after that. 199 * 200 * Returns NULL in case of error starting configuration. 201 */ 202 camera_stream* startConfiguration(); 203 204 /** 205 * Check if the stream is mid-configuration (start has been called, but not 206 * finish). Used for lazy completion of configuration. 207 */ 208 bool isConfiguring() const; 209 210 /** 211 * Completes the stream configuration process. The stream information 212 * structure returned by startConfiguration() may no longer be modified 213 * after this call, but can still be read until the destruction of the 214 * stream. 215 * 216 * streamReconfigured: set to true when a stream is being reconfigured. 217 * 218 * Returns: 219 * OK on a successful configuration 220 * NO_INIT in case of a serious error from the HAL device 221 * NO_MEMORY in case of an error registering buffers 222 * INVALID_OPERATION in case connecting to the consumer failed or consumer 223 * doesn't exist yet. 224 */ 225 status_t finishConfiguration(/*out*/bool* streamReconfigured = nullptr); 226 227 /** 228 * Cancels the stream configuration process. This returns the stream to the 229 * initial state, allowing it to be configured again later. 230 * This is done if the HAL rejects the proposed combined stream configuration 231 */ 232 status_t cancelConfiguration(); 233 234 /** 235 * Determine whether the stream has already become in-use (has received 236 * a valid filled buffer), which determines if a stream can still have 237 * prepareNextBuffer called on it. 238 */ 239 bool isUnpreparable(); 240 241 /** 242 * Mark the stream as unpreparable. 243 */ 244 void markUnpreparable() override; 245 246 /** 247 * Start stream preparation. May only be called in the CONFIGURED state, 248 * when no valid buffers have yet been returned to this stream. Prepares 249 * up to maxCount buffers, or the maximum number of buffers needed by the 250 * pipeline if maxCount is ALLOCATE_PIPELINE_MAX. 251 * 252 * If no prepartion is necessary, returns OK and does not transition to 253 * PREPARING state. Otherwise, returns NOT_ENOUGH_DATA and transitions 254 * to PREPARING. 255 * 256 * This call performs no allocation, so is quick to call. 257 * 258 * blockRequest specifies whether prepare will block upcoming capture 259 * request. This flag should only be set to false if the caller guarantees 260 * the whole buffer preparation process is done before capture request 261 * comes in. 262 * 263 * Returns: 264 * OK if no more buffers need to be preallocated 265 * NOT_ENOUGH_DATA if calls to prepareNextBuffer are needed to finish 266 * buffer pre-allocation, and transitions to the PREPARING state. 267 * NO_INIT in case of a serious error from the HAL device 268 * INVALID_OPERATION if called when not in CONFIGURED state, or a 269 * valid buffer has already been returned to this stream. 270 */ 271 status_t startPrepare(int maxCount, bool blockRequest); 272 273 /** 274 * Check if the request on a stream is blocked by prepare. 275 */ 276 bool isBlockedByPrepare() const; 277 278 /** 279 * Continue stream buffer preparation by allocating the next 280 * buffer for this stream. May only be called in the PREPARED state. 281 * 282 * Returns OK and transitions to the CONFIGURED state if all buffers 283 * are allocated after the call concludes. Otherwise returns NOT_ENOUGH_DATA. 284 * 285 * This call allocates one buffer, which may take several milliseconds for 286 * large buffers. 287 * 288 * Returns: 289 * OK if no more buffers need to be preallocated, and transitions 290 * to the CONFIGURED state. 291 * NOT_ENOUGH_DATA if more calls to prepareNextBuffer are needed to finish 292 * buffer pre-allocation. 293 * NO_INIT in case of a serious error from the HAL device 294 * INVALID_OPERATION if called when not in CONFIGURED state, or a 295 * valid buffer has already been returned to this stream. 296 */ 297 status_t prepareNextBuffer(); 298 299 /** 300 * Cancel stream preparation early. In case allocation needs to be 301 * stopped, this method transitions the stream back to the CONFIGURED state. 302 * Buffers that have been allocated with prepareNextBuffer remain that way, 303 * but a later use of prepareNextBuffer will require just as many 304 * calls as if the earlier prepare attempt had not existed. 305 * 306 * Returns: 307 * OK if cancellation succeeded, and transitions to the CONFIGURED state 308 * INVALID_OPERATION if not in the PREPARING state 309 * NO_INIT in case of a serious error from the HAL device 310 */ 311 status_t cancelPrepare(); 312 313 /** 314 * Tear down memory for this stream. This frees all unused gralloc buffers 315 * allocated for this stream, but leaves it ready for operation afterward. 316 * 317 * May only be called in the CONFIGURED state, and keeps the stream in 318 * the CONFIGURED state. 319 * 320 * Returns: 321 * OK if teardown succeeded. 322 * INVALID_OPERATION if not in the CONFIGURED state 323 * NO_INIT in case of a serious error from the HAL device 324 */ 325 status_t tearDown(); 326 327 /** 328 * Fill in the camera_stream_buffer with the next valid buffer for this 329 * stream, to hand over to the HAL. 330 * 331 * Multiple surfaces could share the same HAL stream, but a request may 332 * be only for a subset of surfaces. In this case, the 333 * Camera3StreamInterface object needs the surface ID information to acquire 334 * buffers for those surfaces. 335 * 336 * This method may only be called once finishConfiguration has been called. 337 * For bidirectional streams, this method applies to the output-side 338 * buffers. 339 * 340 */ 341 status_t getBuffer(camera_stream_buffer *buffer, 342 nsecs_t waitBufferTimeout, 343 const std::vector<size_t>& surface_ids = std::vector<size_t>()); 344 345 /** 346 * Similar to getBuffer() except this method fills multiple buffers. 347 */ 348 status_t getBuffers(std::vector<OutstandingBuffer>* buffers, 349 nsecs_t waitBufferTimeout); 350 351 /** 352 * Return a buffer to the stream after use by the HAL. 353 * 354 * Multiple surfaces could share the same HAL stream, but a request may 355 * be only for a subset of surfaces. In this case, the 356 * Camera3StreamInterface object needs the surface ID information to attach 357 * buffers for those surfaces. 358 * 359 * This method may only be called for buffers provided by getBuffer(). 360 * For bidirectional streams, this method applies to the output-side buffers 361 */ 362 status_t returnBuffer(const camera_stream_buffer &buffer, 363 nsecs_t timestamp, nsecs_t readoutTimestamp, bool timestampIncreasing, 364 const std::vector<size_t>& surface_ids = std::vector<size_t>(), 365 uint64_t frameNumber = 0, int32_t transform = -1); 366 367 /** 368 * Fill in the camera_stream_buffer with the next valid buffer for this 369 * stream, to hand over to the HAL. 370 * 371 * This method may only be called once finishConfiguration has been called. 372 * For bidirectional streams, this method applies to the input-side 373 * buffers. 374 * 375 * This method also returns the size of the returned input buffer. 376 * 377 * Normally this call will block until the handed out buffer count is less than the stream 378 * max buffer count; if respectHalLimit is set to false, this is ignored. 379 */ 380 status_t getInputBuffer(camera_stream_buffer *buffer, 381 Size* size, bool respectHalLimit = true); 382 383 /** 384 * Return a buffer to the stream after use by the HAL. 385 * 386 * This method may only be called for buffers provided by getBuffer(). 387 * For bidirectional streams, this method applies to the input-side buffers 388 */ 389 status_t returnInputBuffer(const camera_stream_buffer &buffer); 390 391 // get the buffer producer of the input buffer queue. 392 // only apply to input streams. 393 status_t getInputBufferProducer(sp<IGraphicBufferProducer> *producer); 394 395 /** 396 * Whether any of the stream's buffers are currently in use by the HAL, 397 * including buffers that have been returned but not yet had their 398 * release fence signaled. 399 */ 400 bool hasOutstandingBuffers() const; 401 402 /** 403 * Get number of buffers currently handed out to HAL 404 */ 405 size_t getOutstandingBuffersCount() const; 406 407 enum { 408 TIMEOUT_NEVER = -1 409 }; 410 411 /** 412 * Set the status tracker to notify about idle transitions 413 */ 414 virtual status_t setStatusTracker(sp<StatusTracker> statusTracker); 415 416 /** 417 * Disconnect stream from its non-HAL endpoint. After this, 418 * start/finishConfiguration must be called before the stream can be used 419 * again. This cannot be called if the stream has outstanding dequeued 420 * buffers. 421 */ 422 status_t disconnect(); 423 424 /** 425 * Debug dump of the stream's state. 426 */ 427 virtual void dump(int fd, const Vector<String16> &args) const; 428 429 /** 430 * Add a camera3 buffer listener. Adding the same listener twice has 431 * no effect. 432 */ 433 void addBufferListener( 434 wp<Camera3StreamBufferListener> listener); 435 436 /** 437 * Remove a camera3 buffer listener. Removing the same listener twice 438 * or the listener that was never added has no effect. 439 */ 440 void removeBufferListener( 441 const sp<Camera3StreamBufferListener>& listener); 442 443 444 // Setting listener will remove previous listener (if exists) 445 virtual void setBufferFreedListener( 446 wp<Camera3StreamBufferFreedListener> listener) override; 447 448 /** 449 * Return if the buffer queue of the stream is abandoned. 450 */ 451 bool isAbandoned() const; 452 453 /** 454 * Switch a configured stream with possibly outstanding buffers in idle 455 * state. Configuration for such streams will be skipped assuming there 456 * are no changes to the stream parameters. 457 */ 458 status_t forceToIdle(); 459 460 /** 461 * Restore a forced idle stream to configured state, marking it active 462 * in case it contains outstanding buffers. 463 */ 464 status_t restoreConfiguredState(); 465 466 /** 467 * Notify buffer stream listeners about incoming request with particular frame number. 468 */ 469 void fireBufferRequestForFrameNumber(uint64_t frameNumber, 470 const CameraMetadata& settings) override; 471 472 protected: 473 const int mId; 474 /** 475 * Stream set id, used to indicate which group of this stream belongs to for buffer sharing 476 * across multiple streams. 477 * 478 * The default value is set to CAMERA3_STREAM_SET_ID_INVALID, which indicates that this stream 479 * doesn't intend to share buffers with any other streams, and this stream will fall back to 480 * the existing BufferQueue mechanism to manage the buffer allocations and buffer circulation. 481 * When a valid stream set id is set, this stream intends to use the Camera3BufferManager to 482 * manage the buffer allocations; the BufferQueue will only handle the buffer transaction 483 * between the producer and consumer. For this case, upon successfully registration, the streams 484 * with the same stream set id will potentially share the buffers allocated by 485 * Camera3BufferManager. 486 */ 487 const int mSetId; 488 489 const String8 mName; 490 // Zero for formats with fixed buffer size for given dimensions. 491 const size_t mMaxSize; 492 493 enum StreamState { 494 STATE_ERROR, 495 STATE_CONSTRUCTED, 496 STATE_IN_CONFIG, 497 STATE_IN_RECONFIG, 498 STATE_CONFIGURED, 499 STATE_PREPARING, 500 STATE_ABANDONED, 501 STATE_IN_IDLE 502 } mState; 503 504 mutable Mutex mLock; 505 506 Camera3Stream(int id, camera_stream_type type, 507 uint32_t width, uint32_t height, size_t maxSize, int format, 508 android_dataspace dataSpace, camera_stream_rotation_t rotation, 509 const String8& physicalCameraId, 510 const std::unordered_set<int32_t> &sensorPixelModesUsed, 511 int setId, bool isMultiResolution, int64_t dynamicRangeProfile, 512 int64_t streamUseCase, bool deviceTimeBaseIsRealtime, int timestampBase); 513 514 wp<Camera3StreamBufferFreedListener> mBufferFreedListener; 515 516 /** 517 * Interface to be implemented by derived classes 518 */ 519 520 // getBuffer / returnBuffer implementations 521 522 // Since camera_stream_buffer includes a raw pointer to the stream, 523 // cast to camera_stream*, implementations must increment the 524 // refcount of the stream manually in getBufferLocked, and decrement it in 525 // returnBufferLocked. 526 virtual status_t getBufferLocked(camera_stream_buffer *buffer, 527 const std::vector<size_t>& surface_ids = std::vector<size_t>()); 528 virtual status_t returnBufferLocked(const camera_stream_buffer &buffer, 529 nsecs_t timestamp, nsecs_t readoutTimestamp, int32_t transform, 530 const std::vector<size_t>& surface_ids = std::vector<size_t>()); 531 532 virtual status_t getBuffersLocked(std::vector<OutstandingBuffer>*); 533 534 virtual status_t getInputBufferLocked(camera_stream_buffer *buffer, Size* size); 535 536 virtual status_t returnInputBufferLocked( 537 const camera_stream_buffer &buffer); 538 virtual bool hasOutstandingBuffersLocked() const = 0; 539 // Get the buffer producer of the input buffer queue. Only apply to input streams. 540 virtual status_t getInputBufferProducerLocked(sp<IGraphicBufferProducer> *producer); 541 542 // Can return -ENOTCONN when we are already disconnected (not an error) 543 virtual status_t disconnectLocked() = 0; 544 545 // Configure the buffer queue interface to the other end of the stream, 546 // after the HAL has provided usage and max_buffers values. After this call, 547 // the stream must be ready to produce all buffers for registration with 548 // HAL. 549 // Returns NO_INIT or DEAD_OBJECT if the queue has been abandoned. 550 virtual status_t configureQueueLocked() = 0; 551 552 // Get the total number of buffers in the queue 553 virtual size_t getBufferCountLocked() = 0; 554 555 // Get handout output buffer count. 556 virtual size_t getHandoutOutputBufferCountLocked() const = 0; 557 558 // Get handout input buffer count. 559 virtual size_t getHandoutInputBufferCountLocked() = 0; 560 561 // Get cached output buffer count. 562 virtual size_t getCachedOutputBufferCountLocked() const = 0; 563 virtual size_t getMaxCachedOutputBuffersLocked() const = 0; 564 565 // Get the usage flags for the other endpoint, or return 566 // INVALID_OPERATION if they cannot be obtained. 567 virtual status_t getEndpointUsage(uint64_t *usage) const = 0; 568 569 // Return whether the buffer is in the list of outstanding buffers. 570 bool isOutstandingBuffer(const camera_stream_buffer& buffer) const; 571 572 // Tracking for idle state 573 wp<StatusTracker> mStatusTracker; 574 // Status tracker component ID 575 int mStatusId; 576 577 // Tracking for stream prepare - whether this stream can still have 578 // prepareNextBuffer called on it. 579 bool mStreamUnpreparable; 580 581 uint64_t mUsage; 582 583 Condition mOutputBufferReturnedSignal; 584 585 private: 586 // Previously configured stream properties (post HAL override) 587 uint64_t mOldUsage; 588 uint32_t mOldMaxBuffers; 589 int mOldFormat; 590 android_dataspace mOldDataSpace; 591 592 Condition mInputBufferReturnedSignal; 593 static const nsecs_t kWaitForBufferDuration = 3000000000LL; // 3000 ms 594 595 void fireBufferListenersLocked(const camera_stream_buffer& buffer, 596 bool acquired, bool output, nsecs_t timestamp = 0, uint64_t frameNumber = 0); 597 List<wp<Camera3StreamBufferListener> > mBufferListenerList; 598 599 status_t cancelPrepareLocked(); 600 601 // Remove the buffer from the list of outstanding buffers. 602 void removeOutstandingBuffer(const camera_stream_buffer& buffer); 603 604 // Tracking for PREPARING state 605 606 // State of buffer preallocation. Only true if either prepareNextBuffer 607 // has been called sufficient number of times, or stream configuration 608 // had to register buffers with the HAL 609 bool mPrepared; 610 bool mPrepareBlockRequest; 611 612 Vector<camera_stream_buffer_t> mPreparedBuffers; 613 size_t mPreparedBufferIdx; 614 615 // Number of buffers allocated on last prepare call. 616 size_t mLastMaxCount; 617 618 mutable Mutex mOutstandingBuffersLock; 619 // Outstanding buffers dequeued from the stream's buffer queue. 620 List<buffer_handle_t> mOutstandingBuffers; 621 622 // Latency histogram of the wait time for handout buffer count to drop below 623 // max_buffers. 624 static const int32_t kBufferLimitLatencyBinSize = 33; //in ms 625 CameraLatencyHistogram mBufferLimitLatency; 626 627 //Keep track of original format when the stream is created in case it gets overridden 628 bool mFormatOverridden; 629 const int mOriginalFormat; 630 631 //Keep track of original dataSpace in case it gets overridden 632 bool mDataSpaceOverridden; 633 const android_dataspace mOriginalDataSpace; 634 635 String8 mPhysicalCameraId; 636 nsecs_t mLastTimestamp; 637 638 bool mIsMultiResolution = false; 639 bool mSupportOfflineProcessing = false; 640 641 bool mDeviceTimeBaseIsRealtime; 642 int mTimestampBase; 643 }; // class Camera3Stream 644 645 }; // namespace camera3 646 647 }; // namespace android 648 649 #endif 650