1 /* 2 * Copyright (C) 2013-2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef ANDROID_SERVERS_CAMERA3_OUTPUT_STREAM_H 18 #define ANDROID_SERVERS_CAMERA3_OUTPUT_STREAM_H 19 20 #include <mutex> 21 #include <optional> 22 #include <utils/RefBase.h> 23 #include <gui/IProducerListener.h> 24 #include <gui/Surface.h> 25 #include <gui/DisplayEventReceiver.h> 26 27 #include "utils/IPCTransport.h" 28 #include "utils/LatencyHistogram.h" 29 #include "Camera3Stream.h" 30 #include "Camera3IOStreamBase.h" 31 #include "Camera3OutputStreamInterface.h" 32 #include "Camera3BufferManager.h" 33 #include "PreviewFrameSpacer.h" 34 35 namespace android { 36 37 namespace camera3 { 38 39 class Camera3BufferManager; 40 41 /** 42 * Stream info structure that holds the necessary stream info for buffer manager to use for 43 * buffer allocation and management. 44 */ 45 struct StreamInfo { 46 int streamId; 47 int streamSetId; 48 uint32_t width; 49 uint32_t height; 50 uint32_t format; 51 android_dataspace dataSpace; 52 uint64_t combinedUsage; 53 size_t totalBufferCount; 54 bool isConfigured; 55 bool isMultiRes; 56 explicit StreamInfo(int id = CAMERA3_STREAM_ID_INVALID, 57 int setId = CAMERA3_STREAM_SET_ID_INVALID, 58 uint32_t w = 0, 59 uint32_t h = 0, 60 uint32_t fmt = 0, 61 android_dataspace ds = HAL_DATASPACE_UNKNOWN, 62 uint64_t usage = 0, 63 size_t bufferCount = 0, 64 bool configured = false, 65 bool multiRes = false) : streamIdStreamInfo66 streamId(id), 67 streamSetId(setId), 68 width(w), 69 height(h), 70 format(fmt), 71 dataSpace(ds), 72 combinedUsage(usage), 73 totalBufferCount(bufferCount), 74 isConfigured(configured), 75 isMultiRes(multiRes) {} 76 }; 77 78 /** 79 * A class for managing a single stream of output data from the camera device. 80 */ 81 class Camera3OutputStream : 82 public Camera3IOStreamBase, 83 public Camera3OutputStreamInterface { 84 public: 85 /** 86 * Set up a stream for formats that have 2 dimensions, such as RAW and YUV. 87 * A valid stream set id needs to be set to support buffer sharing between multiple 88 * streams. 89 */ 90 Camera3OutputStream(int id, sp<Surface> consumer, 91 uint32_t width, uint32_t height, int format, 92 android_dataspace dataSpace, camera_stream_rotation_t rotation, 93 nsecs_t timestampOffset, const std::string& physicalCameraId, 94 const std::unordered_set<int32_t> &sensorPixelModesUsed, IPCTransport transport, 95 int setId = CAMERA3_STREAM_SET_ID_INVALID, bool isMultiResolution = false, 96 int64_t dynamicProfile = ANDROID_REQUEST_AVAILABLE_DYNAMIC_RANGE_PROFILES_MAP_STANDARD, 97 int64_t streamUseCase = ANDROID_SCALER_AVAILABLE_STREAM_USE_CASES_DEFAULT, 98 bool deviceTimeBaseIsRealtime = false, 99 int timestampBase = OutputConfiguration::TIMESTAMP_BASE_DEFAULT, 100 int mirrorMode = OutputConfiguration::MIRROR_MODE_AUTO, 101 int32_t colorSpace = ANDROID_REQUEST_AVAILABLE_COLOR_SPACE_PROFILES_MAP_UNSPECIFIED, 102 bool useReadoutTimestamp = false); 103 /** 104 * Set up a stream for formats that have a variable buffer size for the same 105 * dimensions, such as compressed JPEG. 106 * A valid stream set id needs to be set to support buffer sharing between multiple 107 * streams. 108 */ 109 Camera3OutputStream(int id, sp<Surface> consumer, 110 uint32_t width, uint32_t height, size_t maxSize, int format, 111 android_dataspace dataSpace, camera_stream_rotation_t rotation, 112 nsecs_t timestampOffset, const std::string& physicalCameraId, 113 const std::unordered_set<int32_t> &sensorPixelModesUsed, IPCTransport transport, 114 int setId = CAMERA3_STREAM_SET_ID_INVALID, bool isMultiResolution = false, 115 int64_t dynamicProfile = ANDROID_REQUEST_AVAILABLE_DYNAMIC_RANGE_PROFILES_MAP_STANDARD, 116 int64_t streamUseCase = ANDROID_SCALER_AVAILABLE_STREAM_USE_CASES_DEFAULT, 117 bool deviceTimeBaseIsRealtime = false, 118 int timestampBase = OutputConfiguration::TIMESTAMP_BASE_DEFAULT, 119 int mirrorMode = OutputConfiguration::MIRROR_MODE_AUTO, 120 int32_t colorSpace = ANDROID_REQUEST_AVAILABLE_COLOR_SPACE_PROFILES_MAP_UNSPECIFIED, 121 bool useReadoutTimestamp = false); 122 /** 123 * Set up a stream with deferred consumer for formats that have 2 dimensions, such as 124 * RAW and YUV. The consumer must be set before using this stream for output. A valid 125 * stream set id needs to be set to support buffer sharing between multiple streams. 126 */ 127 Camera3OutputStream(int id, uint32_t width, uint32_t height, int format, 128 uint64_t consumerUsage, android_dataspace dataSpace, 129 camera_stream_rotation_t rotation, nsecs_t timestampOffset, 130 const std::string& physicalCameraId, 131 const std::unordered_set<int32_t> &sensorPixelModesUsed, IPCTransport transport, 132 int setId = CAMERA3_STREAM_SET_ID_INVALID, bool isMultiResolution = false, 133 int64_t dynamicProfile = ANDROID_REQUEST_AVAILABLE_DYNAMIC_RANGE_PROFILES_MAP_STANDARD, 134 int64_t streamUseCase = ANDROID_SCALER_AVAILABLE_STREAM_USE_CASES_DEFAULT, 135 bool deviceTimeBaseIsRealtime = false, 136 int timestampBase = OutputConfiguration::TIMESTAMP_BASE_DEFAULT, 137 int mirrorMode = OutputConfiguration::MIRROR_MODE_AUTO, 138 int32_t colorSpace = ANDROID_REQUEST_AVAILABLE_COLOR_SPACE_PROFILES_MAP_UNSPECIFIED, 139 bool useReadoutTimestamp = false); 140 141 virtual ~Camera3OutputStream(); 142 143 /** 144 * Camera3Stream interface 145 */ 146 147 virtual void dump(int fd, const Vector<String16> &args); 148 149 /** 150 * Set the transform on the output stream; one of the 151 * HAL_TRANSFORM_* / NATIVE_WINDOW_TRANSFORM_* constants. 152 */ 153 status_t setTransform(int transform, bool mayChangeMirror); 154 155 /** 156 * Return if this output stream is for video encoding. 157 */ 158 bool isVideoStream(); 159 /** 160 * Return if this output stream is consumed by hardware composer. 161 */ 162 bool isConsumedByHWComposer(); 163 164 /** 165 * Return if this output stream is consumed by hardware texture. 166 */ 167 bool isConsumedByHWTexture(); 168 169 /** 170 * Return if this output stream is consumed by CPU. 171 */ 172 bool isConsumedByCPU(); 173 174 /** 175 * Return if the consumer configuration of this stream is deferred. 176 */ 177 virtual bool isConsumerConfigurationDeferred(size_t surface_id) const; 178 179 /** 180 * Set the consumer surfaces to the output stream. 181 */ 182 virtual status_t setConsumers(const std::vector<sp<Surface>>& consumers); 183 184 class BufferProducerListener : public SurfaceListener { 185 public: BufferProducerListener(wp<Camera3OutputStream> parent,bool needsReleaseNotify)186 BufferProducerListener(wp<Camera3OutputStream> parent, bool needsReleaseNotify) 187 : mParent(parent), mNeedsReleaseNotify(needsReleaseNotify) {} 188 189 /** 190 * Implementation of IProducerListener, used to notify this stream that the consumer 191 * has returned a buffer and it is ready to return to Camera3BufferManager for reuse. 192 */ 193 virtual void onBufferReleased(); needsReleaseNotify()194 virtual bool needsReleaseNotify() { return mNeedsReleaseNotify; } 195 virtual void onBuffersDiscarded(const std::vector<sp<GraphicBuffer>>& buffers); 196 197 private: 198 wp<Camera3OutputStream> mParent; 199 bool mNeedsReleaseNotify; 200 }; 201 202 virtual status_t detachBuffer(sp<GraphicBuffer>* buffer, int* fenceFd); 203 204 /** 205 * Notify that the buffer is being released to the buffer queue instead of 206 * being queued to the consumer. 207 */ 208 virtual status_t notifyBufferReleased(ANativeWindowBuffer *anwBuffer); 209 210 /** 211 * Drop buffers if dropping is true. If dropping is false, do not drop buffers. 212 */ 213 virtual status_t dropBuffers(bool dropping) override; 214 215 /** 216 * Query the physical camera id for the output stream. 217 */ 218 virtual const std::string& getPhysicalCameraId() const override; 219 220 /** 221 * Set the graphic buffer manager to get/return the stream buffers. 222 * 223 * It is only legal to call this method when stream is in STATE_CONSTRUCTED state. 224 */ 225 status_t setBufferManager(sp<Camera3BufferManager> bufferManager); 226 227 /** 228 * Query the ouput surface id. 229 */ getSurfaceId(const sp<Surface> &)230 virtual ssize_t getSurfaceId(const sp<Surface> &/*surface*/) { return 0; } 231 getUniqueSurfaceIds(const std::vector<size_t> &,std::vector<size_t> *)232 virtual status_t getUniqueSurfaceIds(const std::vector<size_t>&, 233 /*out*/std::vector<size_t>*) { return INVALID_OPERATION; }; 234 235 /** 236 * Update the stream output surfaces. 237 */ 238 virtual status_t updateStream(const std::vector<sp<Surface>> &outputSurfaces, 239 const std::vector<OutputStreamInfo> &outputInfo, 240 const std::vector<size_t> &removedSurfaceIds, 241 KeyedVector<sp<Surface>, size_t> *outputMap/*out*/); 242 243 /** 244 * Set the batch size for buffer operations. The output stream will request 245 * buffers from buffer queue on a batch basis. Currently only video streams 246 * are allowed to set the batch size. Also if the stream is managed by 247 * buffer manager (Surface group in Java API) then batching is also not 248 * supported. Changing batch size on the fly while there is already batched 249 * buffers in the stream is also not supported. 250 * If the batch size is larger than the max dequeue count set 251 * by the camera HAL, the batch size will be set to the max dequeue count 252 * instead. 253 */ 254 virtual status_t setBatchSize(size_t batchSize = 1) override; 255 256 /** 257 * Notify the stream on change of min frame durations or variable/fixed 258 * frame rate. 259 */ 260 virtual void onMinDurationChanged(nsecs_t duration, bool fixedFps) override; 261 262 /** 263 * Modify stream use case 264 */ 265 virtual void setStreamUseCase(int64_t streamUseCase) override; 266 267 /** 268 * Apply ZSL related consumer usage quirk. 269 */ 270 static void applyZSLUsageQuirk(int format, uint64_t *consumerUsage /*inout*/); 271 setImageDumpMask(int mask)272 void setImageDumpMask(int mask) { mImageDumpMask = mask; } 273 bool shouldLogError(status_t res); 274 void onCachedBufferQueued(); 275 276 protected: 277 Camera3OutputStream(int id, camera_stream_type_t type, 278 uint32_t width, uint32_t height, int format, 279 android_dataspace dataSpace, camera_stream_rotation_t rotation, 280 const std::string& physicalCameraId, 281 const std::unordered_set<int32_t> &sensorPixelModesUsed, IPCTransport transport, 282 uint64_t consumerUsage = 0, nsecs_t timestampOffset = 0, 283 int setId = CAMERA3_STREAM_SET_ID_INVALID, bool isMultiResolution = false, 284 int64_t dynamicProfile = ANDROID_REQUEST_AVAILABLE_DYNAMIC_RANGE_PROFILES_MAP_STANDARD, 285 int64_t streamUseCase = ANDROID_SCALER_AVAILABLE_STREAM_USE_CASES_DEFAULT, 286 bool deviceTimeBaseIsRealtime = false, 287 int timestampBase = OutputConfiguration::TIMESTAMP_BASE_DEFAULT, 288 int mirrorMode = OutputConfiguration::MIRROR_MODE_AUTO, 289 int32_t colorSpace = ANDROID_REQUEST_AVAILABLE_COLOR_SPACE_PROFILES_MAP_UNSPECIFIED, 290 bool useReadoutTimestamp = false); 291 292 /** 293 * Note that we release the lock briefly in this function 294 */ 295 virtual status_t returnBufferCheckedLocked( 296 const camera_stream_buffer &buffer, 297 nsecs_t timestamp, 298 nsecs_t readoutTimestamp, 299 bool output, 300 int32_t transform, 301 const std::vector<size_t>& surface_ids, 302 /*out*/ 303 sp<Fence> *releaseFenceOut); 304 305 virtual status_t disconnectLocked(); 306 status_t fixUpHidlJpegBlobHeader(ANativeWindowBuffer* anwBuffer, int fence); 307 308 status_t getEndpointUsageForSurface(uint64_t *usage, const sp<Surface>& surface); 309 status_t configureConsumerQueueLocked(bool allowPreviewRespace); 310 311 // Consumer as the output of camera HAL 312 sp<Surface> mConsumer; 313 getPresetConsumerUsage()314 uint64_t getPresetConsumerUsage() const { return mConsumerUsage; } 315 316 static const nsecs_t kDequeueBufferTimeout = 1000000000; // 1 sec 317 318 status_t getBufferLockedCommon(ANativeWindowBuffer** anb, int* fenceFd); 319 320 321 private: 322 323 int mTransform; 324 325 virtual status_t setTransformLocked(int transform); 326 327 bool mTraceFirstBuffer; 328 329 /** 330 * GraphicBuffer manager this stream is registered to. Used to replace the buffer 331 * allocation/deallocation role of BufferQueue. 332 */ 333 sp<Camera3BufferManager> mBufferManager; 334 335 /** 336 * Buffer producer listener, used to handle notification when a buffer is released 337 * from consumer side, or a set of buffers are discarded by the consumer. 338 */ 339 sp<BufferProducerListener> mBufferProducerListener; 340 341 /** 342 * Flag indicating if the buffer manager is used to allocate the stream buffers 343 */ 344 bool mUseBufferManager; 345 346 /** 347 * Offset used to override camera HAL produced timestamps 348 * 349 * The offset is first initialized to bootTime - monotonicTime in 350 * constructor, and may later be updated based on the client's timestampBase 351 * setting. 352 */ 353 nsecs_t mTimestampOffset; 354 355 /** 356 * If camera readout time is used rather than the start-of-exposure time. 357 */ 358 bool mUseReadoutTime; 359 360 /** 361 * Consumer end point usage flag set by the constructor for the deferred 362 * consumer case. 363 */ 364 uint64_t mConsumerUsage; 365 366 /** 367 * Consumer end point usage flag retrieved from the buffer queue. 368 */ 369 std::optional<uint64_t> mConsumerUsageCachedValue; 370 371 // Whether to drop valid buffers. 372 bool mDropBuffers; 373 374 375 376 // The batch size for buffer operation 377 std::atomic_size_t mBatchSize = 1; 378 379 // Protecting batch states below, must be acquired after mLock 380 std::mutex mBatchLock; 381 // Prefetched buffers (ready to be handed to client) 382 std::vector<Surface::BatchBuffer> mBatchedBuffers; 383 // ---- End of mBatchLock protected scope ---- 384 385 const int mMirrorMode; 386 387 /** 388 * Internal Camera3Stream interface 389 */ 390 virtual status_t getBufferLocked(camera_stream_buffer *buffer, 391 const std::vector<size_t>& surface_ids); 392 393 virtual status_t returnBufferLocked( 394 const camera_stream_buffer &buffer, 395 nsecs_t timestamp, nsecs_t readoutTimestamp, 396 int32_t transform, const std::vector<size_t>& surface_ids); 397 398 virtual status_t queueBufferToConsumer(sp<ANativeWindow>& consumer, 399 ANativeWindowBuffer* buffer, int anwReleaseFence, 400 const std::vector<size_t>& surface_ids); 401 402 virtual status_t configureQueueLocked(); 403 404 virtual status_t getEndpointUsage(uint64_t *usage); 405 406 /** 407 * Private methods 408 */ 409 void onBuffersRemovedLocked(const std::vector<sp<GraphicBuffer>>&); 410 status_t detachBufferLocked(sp<GraphicBuffer>* buffer, int* fenceFd); 411 // Call this after each dequeueBuffer/attachBuffer/detachNextBuffer call to get update on 412 // removed buffers. Set notifyBufferManager to false when the call is initiated by buffer 413 // manager so buffer manager doesn't need to be notified. 414 void checkRemovedBuffersLocked(bool notifyBufferManager = true); 415 416 // Check return status of IGBP calls and set abandoned state accordingly 417 void checkRetAndSetAbandonedLocked(status_t res); 418 419 // If the status indicates abandonded stream, only log when state hasn't been updated to 420 // STATE_ABANDONED 421 static bool shouldLogError(status_t res, StreamState state); 422 423 // Dump images to disk before returning to consumer 424 void dumpImageToDisk(nsecs_t timestamp, ANativeWindowBuffer* anwBuffer, int fence); 425 426 void returnPrefetchedBuffersLocked(); 427 428 429 static const int32_t kDequeueLatencyBinSize = 5; // in ms 430 CameraLatencyHistogram mDequeueBufferLatency; 431 IPCTransport mIPCTransport = IPCTransport::INVALID; 432 433 int mImageDumpMask = 0; 434 435 // Re-space frames by overriding timestamp to align with display Vsync. 436 // Default is on for SurfaceView bound streams. 437 bool mFixedFps = false; 438 nsecs_t mMinExpectedDuration = 0; 439 bool mSyncToDisplay = false; 440 DisplayEventReceiver mDisplayEventReceiver; 441 nsecs_t mLastCaptureTime = 0; 442 nsecs_t mLastPresentTime = 0; 443 nsecs_t mCaptureToPresentOffset = 0; 444 static constexpr size_t kDisplaySyncExtraBuffer = 2; 445 static constexpr nsecs_t kSpacingResetIntervalNs = 50000000LL; // 50 millisecond 446 static constexpr nsecs_t kTimelineThresholdNs = 1000000LL; // 1 millisecond 447 static constexpr float kMaxIntervalRatioDeviation = 0.05f; 448 static constexpr int kMaxTimelines = 2; 449 nsecs_t syncTimestampToDisplayLocked(nsecs_t t, sp<Fence> releaseFence); 450 451 // In case of fence being used 452 sp<Fence> mReferenceFrameFence; 453 nsecs_t mReferenceCaptureTime = 0; 454 nsecs_t mReferenceArrivalTime = 0; 455 nsecs_t mFenceSignalOffset = 0; 456 VsyncEventData mRefVsyncData; 457 458 // Re-space frames by delaying queueBuffer so that frame delivery has 459 // the same cadence as capture. Default is on for SurfaceTexture bound 460 // streams. 461 sp<PreviewFrameSpacer> mPreviewFrameSpacer; 462 }; // class Camera3OutputStream 463 464 } // namespace camera3 465 466 } // namespace android 467 468 #endif 469