1 /* 2 * Copyright (C) 2013-2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef ANDROID_SERVERS_CAMERA3_STREAM_H 18 #define ANDROID_SERVERS_CAMERA3_STREAM_H 19 20 #include <gui/Surface.h> 21 #include <utils/RefBase.h> 22 #include <utils/String8.h> 23 #include <utils/String16.h> 24 #include <utils/List.h> 25 26 #include "utils/LatencyHistogram.h" 27 #include "Camera3StreamBufferListener.h" 28 #include "Camera3StreamInterface.h" 29 30 namespace android { 31 32 namespace camera3 { 33 34 /** 35 * A class for managing a single stream of input or output data from the camera 36 * device. 37 * 38 * The stream has an internal state machine to track whether it's 39 * connected/configured/etc. 40 * 41 * States: 42 * 43 * STATE_ERROR: A serious error has occurred, stream is unusable. Outstanding 44 * buffers may still be returned. 45 * 46 * STATE_CONSTRUCTED: The stream is ready for configuration, but buffers cannot 47 * be gotten yet. Not connected to any endpoint, no buffers are registered 48 * with the HAL. 49 * 50 * STATE_IN_CONFIG: Configuration has started, but not yet concluded. During this 51 * time, the usage, max_buffers, and priv fields of camera_stream returned by 52 * startConfiguration() may be modified. 53 * 54 * STATE_IN_RE_CONFIG: Configuration has started, and the stream has been 55 * configured before. Need to track separately from IN_CONFIG to avoid 56 * re-registering buffers with HAL. 57 * 58 * STATE_CONFIGURED: Stream is configured, and has registered buffers with the 59 * HAL (if necessary). The stream's getBuffer/returnBuffer work. The priv 60 * pointer may still be modified. 61 * 62 * STATE_PREPARING: The stream's buffers are being pre-allocated for use. On 63 * older HALs, this is done as part of configuration, but in newer HALs 64 * buffers may be allocated at time of first use. But some use cases require 65 * buffer allocation upfront, to minmize disruption due to lengthy allocation 66 * duration. In this state, only prepareNextBuffer() and cancelPrepare() 67 * may be called. 68 * 69 * STATE_IN_IDLE: This is a temporary state only intended to be used for input 70 * streams and only for the case where we need to re-configure the camera device 71 * while the input stream has an outstanding buffer. All other streams should not 72 * be able to switch to this state. For them this is invalid and should be handled 73 * as an unknown state. 74 * 75 * Transition table: 76 * 77 * <none> => STATE_CONSTRUCTED: 78 * When constructed with valid arguments 79 * <none> => STATE_ERROR: 80 * When constructed with invalid arguments 81 * STATE_CONSTRUCTED => STATE_IN_CONFIG: 82 * When startConfiguration() is called 83 * STATE_IN_CONFIG => STATE_CONFIGURED: 84 * When finishConfiguration() is called 85 * STATE_IN_CONFIG => STATE_ERROR: 86 * When finishConfiguration() fails to allocate or register buffers. 87 * STATE_CONFIGURED => STATE_IN_RE_CONFIG: * 88 * When startConfiguration() is called again, after making sure stream is 89 * idle with waitUntilIdle(). 90 * STATE_IN_RE_CONFIG => STATE_CONFIGURED: 91 * When finishConfiguration() is called. 92 * STATE_IN_RE_CONFIG => STATE_ERROR: 93 * When finishConfiguration() fails to allocate or register buffers. 94 * STATE_CONFIGURED => STATE_CONSTRUCTED: 95 * When disconnect() is called after making sure stream is idle with 96 * waitUntilIdle(). 97 * STATE_CONFIGURED => STATE_PREPARING: 98 * When startPrepare is called before the stream has a buffer 99 * queued back into it for the first time. 100 * STATE_PREPARING => STATE_CONFIGURED: 101 * When sufficient prepareNextBuffer calls have been made to allocate 102 * all stream buffers, or cancelPrepare is called. 103 * STATE_CONFIGURED => STATE_ABANDONED: 104 * When the buffer queue of the stream is abandoned. 105 * STATE_CONFIGURED => STATE_IN_IDLE: 106 * Only for an input stream which has an outstanding buffer. 107 * STATE_IN_IDLE => STATE_CONFIGURED: 108 * After the internal re-configuration, the input should revert back to 109 * the configured state. 110 * 111 * Status Tracking: 112 * Each stream is tracked by StatusTracker as a separate component, 113 * depending on the handed out buffer count. The state must be STATE_CONFIGURED 114 * in order for the component to be marked. 115 * 116 * It's marked in one of two ways: 117 * 118 * - ACTIVE: One or more buffers have been handed out (with #getBuffer). 119 * - IDLE: All buffers have been returned (with #returnBuffer), and their 120 * respective release_fence(s) have been signaled. The only exception to this 121 * rule is an input stream that moves to "STATE_IN_IDLE" during internal 122 * re-configuration. 123 * 124 * A typical use case is output streams. When the HAL has any buffers 125 * dequeued, the stream is marked ACTIVE. When the HAL returns all buffers 126 * (e.g. if no capture requests are active), the stream is marked IDLE. 127 * In this use case, the app consumer does not affect the component status. 128 * 129 */ 130 class Camera3Stream : 131 protected camera_stream, 132 public virtual Camera3StreamInterface, 133 public virtual RefBase { 134 public: 135 136 virtual ~Camera3Stream(); 137 138 static Camera3Stream* cast(camera_stream *stream); 139 static const Camera3Stream* cast(const camera_stream *stream); 140 141 /** 142 * Get the stream's ID 143 */ 144 int getId() const; 145 146 /** 147 * Get the output stream set id. 148 */ 149 int getStreamSetId() const; 150 /** 151 * Is this stream part of a multi-resolution stream set 152 */ 153 bool isMultiResolution() const; 154 /** 155 * Get the HAL stream group id for a multi-resolution stream set 156 */ 157 int getHalStreamGroupId() const; 158 159 /** 160 * Get the stream's dimensions and format 161 */ 162 uint32_t getWidth() const; 163 uint32_t getHeight() const; 164 int getFormat() const; 165 android_dataspace getDataSpace() const; 166 uint64_t getUsage() const; 167 void setUsage(uint64_t usage); 168 void setFormatOverride(bool formatOverriden); 169 bool isFormatOverridden() const; 170 int getOriginalFormat() const; 171 void setDataSpaceOverride(bool dataSpaceOverriden); 172 bool isDataSpaceOverridden() const; 173 android_dataspace getOriginalDataSpace() const; 174 int getMaxHalBuffers() const; 175 const String8& physicalCameraId() const; 176 177 void setOfflineProcessingSupport(bool) override; 178 bool getOfflineProcessingSupport() const override; 179 asHalStream()180 camera_stream* asHalStream() override { 181 return this; 182 } 183 184 /** 185 * Start the stream configuration process. Returns a handle to the stream's 186 * information to be passed into the HAL device's configure_streams call. 187 * 188 * Until finishConfiguration() is called, no other methods on the stream may be 189 * called. The usage and max_buffers fields of camera_stream may be modified 190 * between start/finishConfiguration, but may not be changed after that. 191 * 192 * Returns NULL in case of error starting configuration. 193 */ 194 camera_stream* startConfiguration(); 195 196 /** 197 * Check if the stream is mid-configuration (start has been called, but not 198 * finish). Used for lazy completion of configuration. 199 */ 200 bool isConfiguring() const; 201 202 /** 203 * Completes the stream configuration process. The stream information 204 * structure returned by startConfiguration() may no longer be modified 205 * after this call, but can still be read until the destruction of the 206 * stream. 207 * 208 * streamReconfigured: set to true when a stream is being reconfigured. 209 * 210 * Returns: 211 * OK on a successful configuration 212 * NO_INIT in case of a serious error from the HAL device 213 * NO_MEMORY in case of an error registering buffers 214 * INVALID_OPERATION in case connecting to the consumer failed or consumer 215 * doesn't exist yet. 216 */ 217 status_t finishConfiguration(/*out*/bool* streamReconfigured = nullptr); 218 219 /** 220 * Cancels the stream configuration process. This returns the stream to the 221 * initial state, allowing it to be configured again later. 222 * This is done if the HAL rejects the proposed combined stream configuration 223 */ 224 status_t cancelConfiguration(); 225 226 /** 227 * Determine whether the stream has already become in-use (has received 228 * a valid filled buffer), which determines if a stream can still have 229 * prepareNextBuffer called on it. 230 */ 231 bool isUnpreparable(); 232 233 /** 234 * Mark the stream as unpreparable. 235 */ 236 void markUnpreparable() override; 237 238 /** 239 * Start stream preparation. May only be called in the CONFIGURED state, 240 * when no valid buffers have yet been returned to this stream. Prepares 241 * up to maxCount buffers, or the maximum number of buffers needed by the 242 * pipeline if maxCount is ALLOCATE_PIPELINE_MAX. 243 * 244 * If no prepartion is necessary, returns OK and does not transition to 245 * PREPARING state. Otherwise, returns NOT_ENOUGH_DATA and transitions 246 * to PREPARING. 247 * 248 * This call performs no allocation, so is quick to call. 249 * 250 * blockRequest specifies whether prepare will block upcoming capture 251 * request. This flag should only be set to false if the caller guarantees 252 * the whole buffer preparation process is done before capture request 253 * comes in. 254 * 255 * Returns: 256 * OK if no more buffers need to be preallocated 257 * NOT_ENOUGH_DATA if calls to prepareNextBuffer are needed to finish 258 * buffer pre-allocation, and transitions to the PREPARING state. 259 * NO_INIT in case of a serious error from the HAL device 260 * INVALID_OPERATION if called when not in CONFIGURED state, or a 261 * valid buffer has already been returned to this stream. 262 */ 263 status_t startPrepare(int maxCount, bool blockRequest); 264 265 /** 266 * Check if the request on a stream is blocked by prepare. 267 */ 268 bool isBlockedByPrepare() const; 269 270 /** 271 * Continue stream buffer preparation by allocating the next 272 * buffer for this stream. May only be called in the PREPARED state. 273 * 274 * Returns OK and transitions to the CONFIGURED state if all buffers 275 * are allocated after the call concludes. Otherwise returns NOT_ENOUGH_DATA. 276 * 277 * This call allocates one buffer, which may take several milliseconds for 278 * large buffers. 279 * 280 * Returns: 281 * OK if no more buffers need to be preallocated, and transitions 282 * to the CONFIGURED state. 283 * NOT_ENOUGH_DATA if more calls to prepareNextBuffer are needed to finish 284 * buffer pre-allocation. 285 * NO_INIT in case of a serious error from the HAL device 286 * INVALID_OPERATION if called when not in CONFIGURED state, or a 287 * valid buffer has already been returned to this stream. 288 */ 289 status_t prepareNextBuffer(); 290 291 /** 292 * Cancel stream preparation early. In case allocation needs to be 293 * stopped, this method transitions the stream back to the CONFIGURED state. 294 * Buffers that have been allocated with prepareNextBuffer remain that way, 295 * but a later use of prepareNextBuffer will require just as many 296 * calls as if the earlier prepare attempt had not existed. 297 * 298 * Returns: 299 * OK if cancellation succeeded, and transitions to the CONFIGURED state 300 * INVALID_OPERATION if not in the PREPARING state 301 * NO_INIT in case of a serious error from the HAL device 302 */ 303 status_t cancelPrepare(); 304 305 /** 306 * Tear down memory for this stream. This frees all unused gralloc buffers 307 * allocated for this stream, but leaves it ready for operation afterward. 308 * 309 * May only be called in the CONFIGURED state, and keeps the stream in 310 * the CONFIGURED state. 311 * 312 * Returns: 313 * OK if teardown succeeded. 314 * INVALID_OPERATION if not in the CONFIGURED state 315 * NO_INIT in case of a serious error from the HAL device 316 */ 317 status_t tearDown(); 318 319 /** 320 * Fill in the camera_stream_buffer with the next valid buffer for this 321 * stream, to hand over to the HAL. 322 * 323 * Multiple surfaces could share the same HAL stream, but a request may 324 * be only for a subset of surfaces. In this case, the 325 * Camera3StreamInterface object needs the surface ID information to acquire 326 * buffers for those surfaces. 327 * 328 * This method may only be called once finishConfiguration has been called. 329 * For bidirectional streams, this method applies to the output-side 330 * buffers. 331 * 332 */ 333 status_t getBuffer(camera_stream_buffer *buffer, 334 nsecs_t waitBufferTimeout, 335 const std::vector<size_t>& surface_ids = std::vector<size_t>()); 336 337 /** 338 * Similar to getBuffer() except this method fills multiple buffers. 339 */ 340 status_t getBuffers(std::vector<OutstandingBuffer>* buffers, 341 nsecs_t waitBufferTimeout); 342 343 /** 344 * Return a buffer to the stream after use by the HAL. 345 * 346 * Multiple surfaces could share the same HAL stream, but a request may 347 * be only for a subset of surfaces. In this case, the 348 * Camera3StreamInterface object needs the surface ID information to attach 349 * buffers for those surfaces. 350 * 351 * This method may only be called for buffers provided by getBuffer(). 352 * For bidirectional streams, this method applies to the output-side buffers 353 */ 354 status_t returnBuffer(const camera_stream_buffer &buffer, 355 nsecs_t timestamp, bool timestampIncreasing, 356 const std::vector<size_t>& surface_ids = std::vector<size_t>(), 357 uint64_t frameNumber = 0); 358 359 /** 360 * Fill in the camera_stream_buffer with the next valid buffer for this 361 * stream, to hand over to the HAL. 362 * 363 * This method may only be called once finishConfiguration has been called. 364 * For bidirectional streams, this method applies to the input-side 365 * buffers. 366 * 367 * This method also returns the size of the returned input buffer. 368 * 369 * Normally this call will block until the handed out buffer count is less than the stream 370 * max buffer count; if respectHalLimit is set to false, this is ignored. 371 */ 372 status_t getInputBuffer(camera_stream_buffer *buffer, 373 Size* size, bool respectHalLimit = true); 374 375 /** 376 * Return a buffer to the stream after use by the HAL. 377 * 378 * This method may only be called for buffers provided by getBuffer(). 379 * For bidirectional streams, this method applies to the input-side buffers 380 */ 381 status_t returnInputBuffer(const camera_stream_buffer &buffer); 382 383 // get the buffer producer of the input buffer queue. 384 // only apply to input streams. 385 status_t getInputBufferProducer(sp<IGraphicBufferProducer> *producer); 386 387 /** 388 * Whether any of the stream's buffers are currently in use by the HAL, 389 * including buffers that have been returned but not yet had their 390 * release fence signaled. 391 */ 392 bool hasOutstandingBuffers() const; 393 394 /** 395 * Get number of buffers currently handed out to HAL 396 */ 397 size_t getOutstandingBuffersCount() const; 398 399 enum { 400 TIMEOUT_NEVER = -1 401 }; 402 403 /** 404 * Set the status tracker to notify about idle transitions 405 */ 406 virtual status_t setStatusTracker(sp<StatusTracker> statusTracker); 407 408 /** 409 * Disconnect stream from its non-HAL endpoint. After this, 410 * start/finishConfiguration must be called before the stream can be used 411 * again. This cannot be called if the stream has outstanding dequeued 412 * buffers. 413 */ 414 status_t disconnect(); 415 416 /** 417 * Debug dump of the stream's state. 418 */ 419 virtual void dump(int fd, const Vector<String16> &args) const; 420 421 /** 422 * Add a camera3 buffer listener. Adding the same listener twice has 423 * no effect. 424 */ 425 void addBufferListener( 426 wp<Camera3StreamBufferListener> listener); 427 428 /** 429 * Remove a camera3 buffer listener. Removing the same listener twice 430 * or the listener that was never added has no effect. 431 */ 432 void removeBufferListener( 433 const sp<Camera3StreamBufferListener>& listener); 434 435 436 // Setting listener will remove previous listener (if exists) 437 virtual void setBufferFreedListener( 438 wp<Camera3StreamBufferFreedListener> listener) override; 439 440 /** 441 * Return if the buffer queue of the stream is abandoned. 442 */ 443 bool isAbandoned() const; 444 445 /** 446 * Switch a configured stream with possibly outstanding buffers in idle 447 * state. Configuration for such streams will be skipped assuming there 448 * are no changes to the stream parameters. 449 */ 450 status_t forceToIdle(); 451 452 /** 453 * Restore a forced idle stream to configured state, marking it active 454 * in case it contains outstanding buffers. 455 */ 456 status_t restoreConfiguredState(); 457 458 /** 459 * Notify buffer stream listeners about incoming request with particular frame number. 460 */ 461 void fireBufferRequestForFrameNumber(uint64_t frameNumber, 462 const CameraMetadata& settings) override; 463 464 protected: 465 const int mId; 466 /** 467 * Stream set id, used to indicate which group of this stream belongs to for buffer sharing 468 * across multiple streams. 469 * 470 * The default value is set to CAMERA3_STREAM_SET_ID_INVALID, which indicates that this stream 471 * doesn't intend to share buffers with any other streams, and this stream will fall back to 472 * the existing BufferQueue mechanism to manage the buffer allocations and buffer circulation. 473 * When a valid stream set id is set, this stream intends to use the Camera3BufferManager to 474 * manage the buffer allocations; the BufferQueue will only handle the buffer transaction 475 * between the producer and consumer. For this case, upon successfully registration, the streams 476 * with the same stream set id will potentially share the buffers allocated by 477 * Camera3BufferManager. 478 */ 479 const int mSetId; 480 481 const String8 mName; 482 // Zero for formats with fixed buffer size for given dimensions. 483 const size_t mMaxSize; 484 485 enum StreamState { 486 STATE_ERROR, 487 STATE_CONSTRUCTED, 488 STATE_IN_CONFIG, 489 STATE_IN_RECONFIG, 490 STATE_CONFIGURED, 491 STATE_PREPARING, 492 STATE_ABANDONED, 493 STATE_IN_IDLE 494 } mState; 495 496 mutable Mutex mLock; 497 498 Camera3Stream(int id, camera_stream_type type, 499 uint32_t width, uint32_t height, size_t maxSize, int format, 500 android_dataspace dataSpace, camera_stream_rotation_t rotation, 501 const String8& physicalCameraId, 502 const std::unordered_set<int32_t> &sensorPixelModesUsed, 503 int setId, bool isMultiResolution); 504 505 wp<Camera3StreamBufferFreedListener> mBufferFreedListener; 506 507 /** 508 * Interface to be implemented by derived classes 509 */ 510 511 // getBuffer / returnBuffer implementations 512 513 // Since camera_stream_buffer includes a raw pointer to the stream, 514 // cast to camera_stream*, implementations must increment the 515 // refcount of the stream manually in getBufferLocked, and decrement it in 516 // returnBufferLocked. 517 virtual status_t getBufferLocked(camera_stream_buffer *buffer, 518 const std::vector<size_t>& surface_ids = std::vector<size_t>()); 519 virtual status_t returnBufferLocked(const camera_stream_buffer &buffer, 520 nsecs_t timestamp, 521 const std::vector<size_t>& surface_ids = std::vector<size_t>()); 522 523 virtual status_t getBuffersLocked(std::vector<OutstandingBuffer>*); 524 525 virtual status_t getInputBufferLocked(camera_stream_buffer *buffer, Size* size); 526 527 virtual status_t returnInputBufferLocked( 528 const camera_stream_buffer &buffer); 529 virtual bool hasOutstandingBuffersLocked() const = 0; 530 // Get the buffer producer of the input buffer queue. Only apply to input streams. 531 virtual status_t getInputBufferProducerLocked(sp<IGraphicBufferProducer> *producer); 532 533 // Can return -ENOTCONN when we are already disconnected (not an error) 534 virtual status_t disconnectLocked() = 0; 535 536 // Configure the buffer queue interface to the other end of the stream, 537 // after the HAL has provided usage and max_buffers values. After this call, 538 // the stream must be ready to produce all buffers for registration with 539 // HAL. 540 // Returns NO_INIT or DEAD_OBJECT if the queue has been abandoned. 541 virtual status_t configureQueueLocked() = 0; 542 543 // Get the total number of buffers in the queue 544 virtual size_t getBufferCountLocked() = 0; 545 546 // Get handout output buffer count. 547 virtual size_t getHandoutOutputBufferCountLocked() const = 0; 548 549 // Get handout input buffer count. 550 virtual size_t getHandoutInputBufferCountLocked() = 0; 551 552 // Get the usage flags for the other endpoint, or return 553 // INVALID_OPERATION if they cannot be obtained. 554 virtual status_t getEndpointUsage(uint64_t *usage) const = 0; 555 556 // Return whether the buffer is in the list of outstanding buffers. 557 bool isOutstandingBuffer(const camera_stream_buffer& buffer) const; 558 559 // Tracking for idle state 560 wp<StatusTracker> mStatusTracker; 561 // Status tracker component ID 562 int mStatusId; 563 564 // Tracking for stream prepare - whether this stream can still have 565 // prepareNextBuffer called on it. 566 bool mStreamUnpreparable; 567 568 uint64_t mUsage; 569 570 private: 571 // Previously configured stream properties (post HAL override) 572 uint64_t mOldUsage; 573 uint32_t mOldMaxBuffers; 574 int mOldFormat; 575 android_dataspace mOldDataSpace; 576 577 Condition mOutputBufferReturnedSignal; 578 Condition mInputBufferReturnedSignal; 579 static const nsecs_t kWaitForBufferDuration = 3000000000LL; // 3000 ms 580 581 void fireBufferListenersLocked(const camera_stream_buffer& buffer, 582 bool acquired, bool output, nsecs_t timestamp = 0, uint64_t frameNumber = 0); 583 List<wp<Camera3StreamBufferListener> > mBufferListenerList; 584 585 status_t cancelPrepareLocked(); 586 587 // Remove the buffer from the list of outstanding buffers. 588 void removeOutstandingBuffer(const camera_stream_buffer& buffer); 589 590 // Tracking for PREPARING state 591 592 // State of buffer preallocation. Only true if either prepareNextBuffer 593 // has been called sufficient number of times, or stream configuration 594 // had to register buffers with the HAL 595 bool mPrepared; 596 bool mPrepareBlockRequest; 597 598 Vector<camera_stream_buffer_t> mPreparedBuffers; 599 size_t mPreparedBufferIdx; 600 601 // Number of buffers allocated on last prepare call. 602 size_t mLastMaxCount; 603 604 mutable Mutex mOutstandingBuffersLock; 605 // Outstanding buffers dequeued from the stream's buffer queue. 606 List<buffer_handle_t> mOutstandingBuffers; 607 608 // Latency histogram of the wait time for handout buffer count to drop below 609 // max_buffers. 610 static const int32_t kBufferLimitLatencyBinSize = 33; //in ms 611 CameraLatencyHistogram mBufferLimitLatency; 612 613 //Keep track of original format when the stream is created in case it gets overridden 614 bool mFormatOverridden; 615 const int mOriginalFormat; 616 617 //Keep track of original dataSpace in case it gets overridden 618 bool mDataSpaceOverridden; 619 const android_dataspace mOriginalDataSpace; 620 621 String8 mPhysicalCameraId; 622 nsecs_t mLastTimestamp; 623 624 bool mIsMultiResolution = false; 625 bool mSupportOfflineProcessing = false; 626 }; // class Camera3Stream 627 628 }; // namespace camera3 629 630 }; // namespace android 631 632 #endif 633