1 /* 2 * Copyright (C) 2013-2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef ANDROID_SERVERS_CAMERA3_OUTPUT_STREAM_H 18 #define ANDROID_SERVERS_CAMERA3_OUTPUT_STREAM_H 19 20 #include <mutex> 21 #include <utils/RefBase.h> 22 #include <gui/IProducerListener.h> 23 #include <gui/Surface.h> 24 #include <gui/DisplayEventReceiver.h> 25 26 #include "utils/IPCTransport.h" 27 #include "utils/LatencyHistogram.h" 28 #include "Camera3Stream.h" 29 #include "Camera3IOStreamBase.h" 30 #include "Camera3OutputStreamInterface.h" 31 #include "Camera3BufferManager.h" 32 #include "PreviewFrameSpacer.h" 33 34 namespace android { 35 36 namespace camera3 { 37 38 class Camera3BufferManager; 39 40 /** 41 * Stream info structure that holds the necessary stream info for buffer manager to use for 42 * buffer allocation and management. 43 */ 44 struct StreamInfo { 45 int streamId; 46 int streamSetId; 47 uint32_t width; 48 uint32_t height; 49 uint32_t format; 50 android_dataspace dataSpace; 51 uint64_t combinedUsage; 52 size_t totalBufferCount; 53 bool isConfigured; 54 bool isMultiRes; 55 explicit StreamInfo(int id = CAMERA3_STREAM_ID_INVALID, 56 int setId = CAMERA3_STREAM_SET_ID_INVALID, 57 uint32_t w = 0, 58 uint32_t h = 0, 59 uint32_t fmt = 0, 60 android_dataspace ds = HAL_DATASPACE_UNKNOWN, 61 uint64_t usage = 0, 62 size_t bufferCount = 0, 63 bool configured = false, 64 bool multiRes = false) : streamIdStreamInfo65 streamId(id), 66 streamSetId(setId), 67 width(w), 68 height(h), 69 format(fmt), 70 dataSpace(ds), 71 combinedUsage(usage), 72 totalBufferCount(bufferCount), 73 isConfigured(configured), 74 isMultiRes(multiRes) {} 75 }; 76 77 /** 78 * A class for managing a single stream of output data from the camera device. 79 */ 80 class Camera3OutputStream : 81 public Camera3IOStreamBase, 82 public Camera3OutputStreamInterface { 83 public: 84 /** 85 * Set up a stream for formats that have 2 dimensions, such as RAW and YUV. 86 * A valid stream set id needs to be set to support buffer sharing between multiple 87 * streams. 88 */ 89 Camera3OutputStream(int id, sp<Surface> consumer, 90 uint32_t width, uint32_t height, int format, 91 android_dataspace dataSpace, camera_stream_rotation_t rotation, 92 nsecs_t timestampOffset, const String8& physicalCameraId, 93 const std::unordered_set<int32_t> &sensorPixelModesUsed, IPCTransport transport, 94 int setId = CAMERA3_STREAM_SET_ID_INVALID, bool isMultiResolution = false, 95 int64_t dynamicProfile = ANDROID_REQUEST_AVAILABLE_DYNAMIC_RANGE_PROFILES_MAP_STANDARD, 96 int64_t streamUseCase = ANDROID_SCALER_AVAILABLE_STREAM_USE_CASES_DEFAULT, 97 bool deviceTimeBaseIsRealtime = false, 98 int timestampBase = OutputConfiguration::TIMESTAMP_BASE_DEFAULT, 99 int mirrorMode = OutputConfiguration::MIRROR_MODE_AUTO); 100 /** 101 * Set up a stream for formats that have a variable buffer size for the same 102 * dimensions, such as compressed JPEG. 103 * A valid stream set id needs to be set to support buffer sharing between multiple 104 * streams. 105 */ 106 Camera3OutputStream(int id, sp<Surface> consumer, 107 uint32_t width, uint32_t height, size_t maxSize, int format, 108 android_dataspace dataSpace, camera_stream_rotation_t rotation, 109 nsecs_t timestampOffset, const String8& physicalCameraId, 110 const std::unordered_set<int32_t> &sensorPixelModesUsed, IPCTransport transport, 111 int setId = CAMERA3_STREAM_SET_ID_INVALID, bool isMultiResolution = false, 112 int64_t dynamicProfile = ANDROID_REQUEST_AVAILABLE_DYNAMIC_RANGE_PROFILES_MAP_STANDARD, 113 int64_t streamUseCase = ANDROID_SCALER_AVAILABLE_STREAM_USE_CASES_DEFAULT, 114 bool deviceTimeBaseIsRealtime = false, 115 int timestampBase = OutputConfiguration::TIMESTAMP_BASE_DEFAULT, 116 int mirrorMode = OutputConfiguration::MIRROR_MODE_AUTO); 117 /** 118 * Set up a stream with deferred consumer for formats that have 2 dimensions, such as 119 * RAW and YUV. The consumer must be set before using this stream for output. A valid 120 * stream set id needs to be set to support buffer sharing between multiple streams. 121 */ 122 Camera3OutputStream(int id, uint32_t width, uint32_t height, int format, 123 uint64_t consumerUsage, android_dataspace dataSpace, 124 camera_stream_rotation_t rotation, nsecs_t timestampOffset, 125 const String8& physicalCameraId, 126 const std::unordered_set<int32_t> &sensorPixelModesUsed, IPCTransport transport, 127 int setId = CAMERA3_STREAM_SET_ID_INVALID, bool isMultiResolution = false, 128 int64_t dynamicProfile = ANDROID_REQUEST_AVAILABLE_DYNAMIC_RANGE_PROFILES_MAP_STANDARD, 129 int64_t streamUseCase = ANDROID_SCALER_AVAILABLE_STREAM_USE_CASES_DEFAULT, 130 bool deviceTimeBaseIsRealtime = false, 131 int timestampBase = OutputConfiguration::TIMESTAMP_BASE_DEFAULT, 132 int mirrorMode = OutputConfiguration::MIRROR_MODE_AUTO); 133 134 virtual ~Camera3OutputStream(); 135 136 /** 137 * Camera3Stream interface 138 */ 139 140 virtual void dump(int fd, const Vector<String16> &args) const; 141 142 /** 143 * Set the transform on the output stream; one of the 144 * HAL_TRANSFORM_* / NATIVE_WINDOW_TRANSFORM_* constants. 145 */ 146 status_t setTransform(int transform, bool mayChangeMirror); 147 148 /** 149 * Return if this output stream is for video encoding. 150 */ 151 bool isVideoStream() const; 152 /** 153 * Return if this output stream is consumed by hardware composer. 154 */ 155 bool isConsumedByHWComposer() const; 156 157 /** 158 * Return if this output stream is consumed by hardware texture. 159 */ 160 bool isConsumedByHWTexture() const; 161 162 /** 163 * Return if this output stream is consumed by CPU. 164 */ 165 bool isConsumedByCPU() const; 166 167 /** 168 * Return if the consumer configuration of this stream is deferred. 169 */ 170 virtual bool isConsumerConfigurationDeferred(size_t surface_id) const; 171 172 /** 173 * Set the consumer surfaces to the output stream. 174 */ 175 virtual status_t setConsumers(const std::vector<sp<Surface>>& consumers); 176 177 class BufferProducerListener : public SurfaceListener { 178 public: BufferProducerListener(wp<Camera3OutputStream> parent,bool needsReleaseNotify)179 BufferProducerListener(wp<Camera3OutputStream> parent, bool needsReleaseNotify) 180 : mParent(parent), mNeedsReleaseNotify(needsReleaseNotify) {} 181 182 /** 183 * Implementation of IProducerListener, used to notify this stream that the consumer 184 * has returned a buffer and it is ready to return to Camera3BufferManager for reuse. 185 */ 186 virtual void onBufferReleased(); needsReleaseNotify()187 virtual bool needsReleaseNotify() { return mNeedsReleaseNotify; } 188 virtual void onBuffersDiscarded(const std::vector<sp<GraphicBuffer>>& buffers); 189 190 private: 191 wp<Camera3OutputStream> mParent; 192 bool mNeedsReleaseNotify; 193 }; 194 195 virtual status_t detachBuffer(sp<GraphicBuffer>* buffer, int* fenceFd); 196 197 /** 198 * Notify that the buffer is being released to the buffer queue instead of 199 * being queued to the consumer. 200 */ 201 virtual status_t notifyBufferReleased(ANativeWindowBuffer *anwBuffer); 202 203 /** 204 * Drop buffers if dropping is true. If dropping is false, do not drop buffers. 205 */ 206 virtual status_t dropBuffers(bool dropping) override; 207 208 /** 209 * Query the physical camera id for the output stream. 210 */ 211 virtual const String8& getPhysicalCameraId() const override; 212 213 /** 214 * Set the graphic buffer manager to get/return the stream buffers. 215 * 216 * It is only legal to call this method when stream is in STATE_CONSTRUCTED state. 217 */ 218 status_t setBufferManager(sp<Camera3BufferManager> bufferManager); 219 220 /** 221 * Query the ouput surface id. 222 */ getSurfaceId(const sp<Surface> &)223 virtual ssize_t getSurfaceId(const sp<Surface> &/*surface*/) { return 0; } 224 getUniqueSurfaceIds(const std::vector<size_t> &,std::vector<size_t> *)225 virtual status_t getUniqueSurfaceIds(const std::vector<size_t>&, 226 /*out*/std::vector<size_t>*) { return INVALID_OPERATION; }; 227 228 /** 229 * Update the stream output surfaces. 230 */ 231 virtual status_t updateStream(const std::vector<sp<Surface>> &outputSurfaces, 232 const std::vector<OutputStreamInfo> &outputInfo, 233 const std::vector<size_t> &removedSurfaceIds, 234 KeyedVector<sp<Surface>, size_t> *outputMap/*out*/); 235 236 /** 237 * Set the batch size for buffer operations. The output stream will request 238 * buffers from buffer queue on a batch basis. Currently only video streams 239 * are allowed to set the batch size. Also if the stream is managed by 240 * buffer manager (Surface group in Java API) then batching is also not 241 * supported. Changing batch size on the fly while there is already batched 242 * buffers in the stream is also not supported. 243 * If the batch size is larger than the max dequeue count set 244 * by the camera HAL, the batch size will be set to the max dequeue count 245 * instead. 246 */ 247 virtual status_t setBatchSize(size_t batchSize = 1) override; 248 249 /** 250 * Notify the stream on change of min frame durations or variable/fixed 251 * frame rate. 252 */ 253 virtual void onMinDurationChanged(nsecs_t duration, bool fixedFps) override; 254 255 /** 256 * Modify stream use case 257 */ 258 virtual void setStreamUseCase(int64_t streamUseCase) override; 259 260 /** 261 * Apply ZSL related consumer usage quirk. 262 */ 263 static void applyZSLUsageQuirk(int format, uint64_t *consumerUsage /*inout*/); 264 setImageDumpMask(int mask)265 void setImageDumpMask(int mask) { mImageDumpMask = mask; } 266 bool shouldLogError(status_t res); 267 void onCachedBufferQueued(); 268 269 protected: 270 Camera3OutputStream(int id, camera_stream_type_t type, 271 uint32_t width, uint32_t height, int format, 272 android_dataspace dataSpace, camera_stream_rotation_t rotation, 273 const String8& physicalCameraId, 274 const std::unordered_set<int32_t> &sensorPixelModesUsed, IPCTransport transport, 275 uint64_t consumerUsage = 0, nsecs_t timestampOffset = 0, 276 int setId = CAMERA3_STREAM_SET_ID_INVALID, bool isMultiResolution = false, 277 int64_t dynamicProfile = ANDROID_REQUEST_AVAILABLE_DYNAMIC_RANGE_PROFILES_MAP_STANDARD, 278 int64_t streamUseCase = ANDROID_SCALER_AVAILABLE_STREAM_USE_CASES_DEFAULT, 279 bool deviceTimeBaseIsRealtime = false, 280 int timestampBase = OutputConfiguration::TIMESTAMP_BASE_DEFAULT, 281 int mirrorMode = OutputConfiguration::MIRROR_MODE_AUTO); 282 283 /** 284 * Note that we release the lock briefly in this function 285 */ 286 virtual status_t returnBufferCheckedLocked( 287 const camera_stream_buffer &buffer, 288 nsecs_t timestamp, 289 nsecs_t readoutTimestamp, 290 bool output, 291 int32_t transform, 292 const std::vector<size_t>& surface_ids, 293 /*out*/ 294 sp<Fence> *releaseFenceOut); 295 296 virtual status_t disconnectLocked(); 297 status_t fixUpHidlJpegBlobHeader(ANativeWindowBuffer* anwBuffer, int fence); 298 299 status_t getEndpointUsageForSurface(uint64_t *usage, 300 const sp<Surface>& surface) const; 301 status_t configureConsumerQueueLocked(bool allowPreviewRespace); 302 303 // Consumer as the output of camera HAL 304 sp<Surface> mConsumer; 305 getPresetConsumerUsage()306 uint64_t getPresetConsumerUsage() const { return mConsumerUsage; } 307 308 static const nsecs_t kDequeueBufferTimeout = 1000000000; // 1 sec 309 310 status_t getBufferLockedCommon(ANativeWindowBuffer** anb, int* fenceFd); 311 312 313 private: 314 315 int mTransform; 316 317 virtual status_t setTransformLocked(int transform); 318 319 bool mTraceFirstBuffer; 320 321 // Name of Surface consumer 322 String8 mConsumerName; 323 324 /** 325 * GraphicBuffer manager this stream is registered to. Used to replace the buffer 326 * allocation/deallocation role of BufferQueue. 327 */ 328 sp<Camera3BufferManager> mBufferManager; 329 330 /** 331 * Buffer producer listener, used to handle notification when a buffer is released 332 * from consumer side, or a set of buffers are discarded by the consumer. 333 */ 334 sp<BufferProducerListener> mBufferProducerListener; 335 336 /** 337 * Flag indicating if the buffer manager is used to allocate the stream buffers 338 */ 339 bool mUseBufferManager; 340 341 /** 342 * Offset used to override camera HAL produced timestamps 343 * 344 * The offset is first initialized to bootTime - monotonicTime in 345 * constructor, and may later be updated based on the client's timestampBase 346 * setting. 347 */ 348 nsecs_t mTimestampOffset; 349 350 /** 351 * If camera readout time is used rather than the start-of-exposure time. 352 */ 353 bool mUseReadoutTime; 354 355 /** 356 * Consumer end point usage flag set by the constructor for the deferred 357 * consumer case. 358 */ 359 uint64_t mConsumerUsage; 360 361 // Whether to drop valid buffers. 362 bool mDropBuffers; 363 364 365 366 // The batch size for buffer operation 367 std::atomic_size_t mBatchSize = 1; 368 369 // Protecting batch states below, must be acquired after mLock 370 std::mutex mBatchLock; 371 // Prefetched buffers (ready to be handed to client) 372 std::vector<Surface::BatchBuffer> mBatchedBuffers; 373 // ---- End of mBatchLock protected scope ---- 374 375 const int mMirrorMode; 376 377 /** 378 * Internal Camera3Stream interface 379 */ 380 virtual status_t getBufferLocked(camera_stream_buffer *buffer, 381 const std::vector<size_t>& surface_ids); 382 383 virtual status_t getBuffersLocked(/*out*/std::vector<OutstandingBuffer>* buffers) override; 384 385 virtual status_t returnBufferLocked( 386 const camera_stream_buffer &buffer, 387 nsecs_t timestamp, nsecs_t readoutTimestamp, 388 int32_t transform, const std::vector<size_t>& surface_ids); 389 390 virtual status_t queueBufferToConsumer(sp<ANativeWindow>& consumer, 391 ANativeWindowBuffer* buffer, int anwReleaseFence, 392 const std::vector<size_t>& surface_ids); 393 394 virtual status_t configureQueueLocked(); 395 396 virtual status_t getEndpointUsage(uint64_t *usage) const; 397 398 /** 399 * Private methods 400 */ 401 void onBuffersRemovedLocked(const std::vector<sp<GraphicBuffer>>&); 402 status_t detachBufferLocked(sp<GraphicBuffer>* buffer, int* fenceFd); 403 // Call this after each dequeueBuffer/attachBuffer/detachNextBuffer call to get update on 404 // removed buffers. Set notifyBufferManager to false when the call is initiated by buffer 405 // manager so buffer manager doesn't need to be notified. 406 void checkRemovedBuffersLocked(bool notifyBufferManager = true); 407 408 // Check return status of IGBP calls and set abandoned state accordingly 409 void checkRetAndSetAbandonedLocked(status_t res); 410 411 // If the status indicates abandonded stream, only log when state hasn't been updated to 412 // STATE_ABANDONED 413 static bool shouldLogError(status_t res, StreamState state); 414 415 // Dump images to disk before returning to consumer 416 void dumpImageToDisk(nsecs_t timestamp, ANativeWindowBuffer* anwBuffer, int fence); 417 418 void returnPrefetchedBuffersLocked(); 419 420 421 static const int32_t kDequeueLatencyBinSize = 5; // in ms 422 CameraLatencyHistogram mDequeueBufferLatency; 423 IPCTransport mIPCTransport = IPCTransport::INVALID; 424 425 int mImageDumpMask = 0; 426 427 // Re-space frames by overriding timestamp to align with display Vsync. 428 // Default is on for SurfaceView bound streams. 429 bool mFixedFps = false; 430 nsecs_t mMinExpectedDuration = 0; 431 bool mSyncToDisplay = false; 432 DisplayEventReceiver mDisplayEventReceiver; 433 nsecs_t mLastCaptureTime = 0; 434 nsecs_t mLastPresentTime = 0; 435 nsecs_t mCaptureToPresentOffset = 0; 436 static constexpr size_t kDisplaySyncExtraBuffer = 2; 437 static constexpr nsecs_t kSpacingResetIntervalNs = 50000000LL; // 50 millisecond 438 static constexpr nsecs_t kTimelineThresholdNs = 1000000LL; // 1 millisecond 439 static constexpr float kMaxIntervalRatioDeviation = 0.05f; 440 static constexpr int kMaxTimelines = 2; 441 nsecs_t syncTimestampToDisplayLocked(nsecs_t t); 442 443 // Re-space frames by delaying queueBuffer so that frame delivery has 444 // the same cadence as capture. Default is on for SurfaceTexture bound 445 // streams. 446 sp<PreviewFrameSpacer> mPreviewFrameSpacer; 447 }; // class Camera3OutputStream 448 449 } // namespace camera3 450 451 } // namespace android 452 453 #endif 454