1 /* 2 * Copyright 2019 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #pragma once 18 19 #include <android/hardware_buffer.h> 20 #include <gui/BufferQueueDefs.h> 21 #include <gui/ConsumerBase.h> 22 23 #include <gui/IGraphicBufferProducer.h> 24 #include <sys/cdefs.h> 25 #include <system/graphics.h> 26 #include <ui/FenceTime.h> 27 #include <ui/GraphicBuffer.h> 28 #include <utils/Mutex.h> 29 #include <utils/String8.h> 30 31 #include "EGLConsumer.h" 32 #include "ImageConsumer.h" 33 34 namespace android { 35 36 /* 37 * SurfaceTexture consumes buffers of graphics data from a BufferQueue, 38 * and makes them available to HWUI render thread as a SkImage and to 39 * an application GL render thread as an OpenGL texture. 40 * 41 * When attached to an application GL render thread, a typical usage 42 * pattern is to set up the SurfaceTexture with the 43 * desired options, and call updateTexImage() when a new frame is desired. 44 * If a new frame is available, the texture will be updated. If not, 45 * the previous contents are retained. 46 * 47 * When attached to a HWUI render thread, the TextureView implementation 48 * calls dequeueBuffer, which either pulls a new buffer or returns the 49 * last cached buffer if BufferQueue is empty. 50 * When attached to HWUI render thread, SurfaceTexture is compatible to 51 * both Vulkan and GL drawing pipelines. 52 */ 53 class ANDROID_API SurfaceTexture : public ConsumerBase { 54 public: 55 /** 56 * Callback function needed by dequeueBuffer. It creates a fence, 57 * that is signalled, when the previous buffer is no longer in use by HWUI 58 * and can be used by written by the producer. 59 */ 60 typedef status_t (*SurfaceTexture_createReleaseFence)(bool useFenceSync, EGLSyncKHR* eglFence, 61 EGLDisplay* display, int* releaseFence, 62 void* passThroughHandle); 63 64 /** 65 * Callback function needed by dequeueBuffer. It waits for the new buffer 66 * fence to signal, before issuing any draw commands. 67 */ 68 typedef status_t (*SurfaceTexture_fenceWait)(int fence, void* passThroughHandle); 69 70 enum { TEXTURE_EXTERNAL = 0x8D65 }; // GL_TEXTURE_EXTERNAL_OES 71 typedef ConsumerBase::FrameAvailableListener FrameAvailableListener; 72 73 /** 74 * SurfaceTexture constructs a new SurfaceTexture object. If the constructor 75 * with the tex parameter is used, tex indicates the name of the OpenGL ES 76 * texture to which images are to be streamed. texTarget specifies the 77 * OpenGL ES texture target to which the texture will be bound in 78 * updateTexImage. useFenceSync specifies whether fences should be used to 79 * synchronize access to buffers if that behavior is enabled at 80 * compile-time. 81 * 82 * A SurfaceTexture may be detached from one OpenGL ES context and then 83 * attached to a different context using the detachFromContext and 84 * attachToContext methods, respectively. The intention of these methods is 85 * purely to allow a SurfaceTexture to be transferred from one consumer 86 * context to another. If such a transfer is not needed there is no 87 * requirement that either of these methods be called. 88 * 89 * If the constructor with the tex parameter is used, the SurfaceTexture is 90 * created in a state where it is considered attached to an OpenGL ES 91 * context for the purposes of the attachToContext and detachFromContext 92 * methods. However, despite being considered "attached" to a context, the 93 * specific OpenGL ES context doesn't get latched until the first call to 94 * updateTexImage. After that point, all calls to updateTexImage must be 95 * made with the same OpenGL ES context current. 96 * 97 * If the constructor without the tex parameter is used, the SurfaceTexture 98 * is created in a detached state, and attachToContext must be called before 99 * calls to updateTexImage. 100 */ 101 #if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) 102 SurfaceTexture(uint32_t tex, uint32_t textureTarget, bool useFenceSync, bool isControlledByApp); 103 104 SurfaceTexture(uint32_t textureTarget, bool useFenceSync, bool isControlledByApp); 105 106 SurfaceTexture(const sp<IGraphicBufferConsumer>& bq, uint32_t tex, uint32_t textureTarget, 107 bool useFenceSync, bool isControlledByApp) 108 __attribute((deprecated("Prefer ctors that create their own surface and consumer."))); 109 110 SurfaceTexture(const sp<IGraphicBufferConsumer>& bq, uint32_t textureTarget, bool useFenceSync, 111 bool isControlledByApp) 112 __attribute((deprecated("Prefer ctors that create their own surface and consumer."))); 113 #else 114 SurfaceTexture(const sp<IGraphicBufferConsumer>& bq, uint32_t tex, uint32_t textureTarget, 115 bool useFenceSync, bool isControlledByApp); 116 117 SurfaceTexture(const sp<IGraphicBufferConsumer>& bq, uint32_t textureTarget, bool useFenceSync, 118 bool isControlledByApp); 119 #endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) 120 121 /** 122 * updateTexImage acquires the most recently queued buffer, and sets the 123 * image contents of the target texture to it. 124 * 125 * This call may only be made while the OpenGL ES context to which the 126 * target texture belongs is bound to the calling thread. 127 * 128 * This calls doGLFenceWait to ensure proper synchronization. 129 */ 130 status_t updateTexImage(); 131 132 /** 133 * releaseTexImage releases the texture acquired in updateTexImage(). 134 * This is intended to be used in single buffer mode. 135 * 136 * This call may only be made while the OpenGL ES context to which the 137 * target texture belongs is bound to the calling thread. 138 */ 139 status_t releaseTexImage(); 140 141 /** 142 * getTransformMatrix retrieves the 4x4 texture coordinate transform matrix 143 * associated with the texture image set by the most recent call to 144 * updateTexImage. 145 * 146 * This transform matrix maps 2D homogeneous texture coordinates of the form 147 * (s, t, 0, 1) with s and t in the inclusive range [0, 1] to the texture 148 * coordinate that should be used to sample that location from the texture. 149 * Sampling the texture outside of the range of this transform is undefined. 150 * 151 * This transform is necessary to compensate for transforms that the stream 152 * content producer may implicitly apply to the content. By forcing users of 153 * a SurfaceTexture to apply this transform we avoid performing an extra 154 * copy of the data that would be needed to hide the transform from the 155 * user. 156 * 157 * The matrix is stored in column-major order so that it may be passed 158 * directly to OpenGL ES via the glLoadMatrixf or glUniformMatrix4fv 159 * functions. 160 */ 161 void getTransformMatrix(float mtx[16]); 162 163 /** 164 * Computes the transform matrix documented by getTransformMatrix 165 * from the BufferItem sub parts. 166 */ 167 static void computeTransformMatrix(float outTransform[16], const sp<GraphicBuffer>& buf, 168 const Rect& cropRect, uint32_t transform, bool filtering); 169 170 /** 171 * Scale the crop down horizontally or vertically such that it has the 172 * same aspect ratio as the buffer does. 173 */ 174 static Rect scaleDownCrop(const Rect& crop, uint32_t bufferWidth, uint32_t bufferHeight); 175 176 /** 177 * getTimestamp retrieves the timestamp associated with the texture image 178 * set by the most recent call to updateTexImage. 179 * 180 * The timestamp is in nanoseconds, and is monotonically increasing. Its 181 * other semantics (zero point, etc) are source-dependent and should be 182 * documented by the source. 183 */ 184 int64_t getTimestamp(); 185 186 /** 187 * getDataSpace retrieves the DataSpace associated with the texture image 188 * set by the most recent call to updateTexImage. 189 */ 190 android_dataspace getCurrentDataSpace(); 191 192 /** 193 * getFrameNumber retrieves the frame number associated with the texture 194 * image set by the most recent call to updateTexImage. 195 * 196 * The frame number is an incrementing counter set to 0 at the creation of 197 * the BufferQueue associated with this consumer. 198 */ 199 uint64_t getFrameNumber(); 200 201 /** 202 * setDefaultBufferSize is used to set the size of buffers returned by 203 * requestBuffers when a with and height of zero is requested. 204 * A call to setDefaultBufferSize() may trigger requestBuffers() to 205 * be called from the client. 206 * The width and height parameters must be no greater than the minimum of 207 * GL_MAX_VIEWPORT_DIMS and GL_MAX_TEXTURE_SIZE (see: glGetIntegerv). 208 * An error due to invalid dimensions might not be reported until 209 * updateTexImage() is called. 210 */ 211 status_t setDefaultBufferSize(uint32_t width, uint32_t height); 212 213 /** 214 * setFilteringEnabled sets whether the transform matrix should be computed 215 * for use with bilinear filtering. 216 */ 217 void setFilteringEnabled(bool enabled); 218 219 /** 220 * getCurrentTextureTarget returns the texture target of the current 221 * texture as returned by updateTexImage(). 222 */ 223 uint32_t getCurrentTextureTarget() const; 224 225 /** 226 * getCurrentCrop returns the cropping rectangle of the current buffer. 227 */ 228 Rect getCurrentCrop() const; 229 230 /** 231 * getCurrentTransform returns the transform of the current buffer. 232 */ 233 uint32_t getCurrentTransform() const; 234 235 /** 236 * getCurrentScalingMode returns the scaling mode of the current buffer. 237 */ 238 uint32_t getCurrentScalingMode() const; 239 240 /** 241 * getCurrentFence returns the fence indicating when the current buffer is 242 * ready to be read from. 243 */ 244 sp<Fence> getCurrentFence() const; 245 246 /** 247 * getCurrentFence returns the FenceTime indicating when the current 248 * buffer is ready to be read from. 249 */ 250 std::shared_ptr<FenceTime> getCurrentFenceTime() const; 251 252 /** 253 * setConsumerUsageBits overrides the ConsumerBase method to OR 254 * DEFAULT_USAGE_FLAGS to usage. 255 */ 256 status_t setConsumerUsageBits(uint64_t usage); 257 258 /** 259 * detachFromContext detaches the SurfaceTexture from the calling thread's 260 * current OpenGL ES context. This context must be the same as the context 261 * that was current for previous calls to updateTexImage. 262 * 263 * Detaching a SurfaceTexture from an OpenGL ES context will result in the 264 * deletion of the OpenGL ES texture object into which the images were being 265 * streamed. After a SurfaceTexture has been detached from the OpenGL ES 266 * context calls to updateTexImage will fail returning INVALID_OPERATION 267 * until the SurfaceTexture is attached to a new OpenGL ES context using the 268 * attachToContext method. 269 */ 270 status_t detachFromContext(); 271 272 /** 273 * attachToContext attaches a SurfaceTexture that is currently in the 274 * 'detached' state to the current OpenGL ES context. A SurfaceTexture is 275 * in the 'detached' state iff detachFromContext has successfully been 276 * called and no calls to attachToContext have succeeded since the last 277 * detachFromContext call. Calls to attachToContext made on a 278 * SurfaceTexture that is not in the 'detached' state will result in an 279 * INVALID_OPERATION error. 280 * 281 * The tex argument specifies the OpenGL ES texture object name in the 282 * new context into which the image contents will be streamed. A successful 283 * call to attachToContext will result in this texture object being bound to 284 * the texture target and populated with the image contents that were 285 * current at the time of the last call to detachFromContext. 286 */ 287 status_t attachToContext(uint32_t tex); 288 289 sp<GraphicBuffer> dequeueBuffer(int* outSlotid, android_dataspace* outDataspace, 290 HdrMetadata* outHdrMetadata, float* outTransformMatrix, 291 uint32_t* outTransform, bool* outQueueEmpty, 292 SurfaceTexture_createReleaseFence createFence, 293 SurfaceTexture_fenceWait fenceWait, 294 void* fencePassThroughHandle, ARect* currentCrop); 295 296 /** 297 * takeConsumerOwnership attaches a SurfaceTexture that is currently in the 298 * 'detached' state to a consumer context (usually HWUI RenderThread). 299 */ 300 void takeConsumerOwnership(); 301 302 /** 303 * releaseConsumerOwnership detaches a SurfaceTexture from a consumer 304 * context (usually HWUI RenderThread). 305 */ 306 void releaseConsumerOwnership(); 307 308 /** 309 * Interface for SurfaceTexture callback(s). 310 */ 311 struct SurfaceTextureListener : public RefBase { 312 virtual void onFrameAvailable(const BufferItem& item) = 0; 313 virtual void onSetFrameRate(float frameRate, int8_t compatibility, 314 int8_t changeFrameRateStrategy) = 0; 315 }; 316 317 /** 318 * setSurfaceTextureListener registers a SurfaceTextureListener. 319 */ 320 void setSurfaceTextureListener(const sp<SurfaceTextureListener>&); 321 322 protected: 323 /** 324 * abandonLocked overrides the ConsumerBase method to clear 325 * mCurrentTextureImage in addition to the ConsumerBase behavior. 326 */ 327 virtual void abandonLocked(); 328 329 /** 330 * dumpLocked overrides the ConsumerBase method to dump SurfaceTexture- 331 * specific info in addition to the ConsumerBase behavior. 332 */ 333 virtual void dumpLocked(String8& result, const char* prefix) const override; 334 335 /** 336 * acquireBufferLocked overrides the ConsumerBase method to update the 337 * mEglSlots array in addition to the ConsumerBase behavior. 338 */ 339 virtual status_t acquireBufferLocked(BufferItem* item, nsecs_t presentWhen, 340 uint64_t maxFrameNumber = 0) override; 341 342 /** 343 * releaseBufferLocked overrides the ConsumerBase method to update the 344 * mEglSlots array in addition to the ConsumerBase. 345 */ 346 #if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(BQ_GL_FENCE_CLEANUP) 347 virtual status_t releaseBufferLocked(int slot, const sp<GraphicBuffer> graphicBuffer) override; 348 #else 349 virtual status_t releaseBufferLocked(int slot, const sp<GraphicBuffer> graphicBuffer, 350 EGLDisplay display = EGL_NO_DISPLAY, 351 EGLSyncKHR eglFence = EGL_NO_SYNC_KHR) override; 352 #endif 353 354 /** 355 * freeBufferLocked frees up the given buffer slot. If the slot has been 356 * initialized this will release the reference to the GraphicBuffer in that 357 * slot and destroy the EGLImage in that slot. Otherwise it has no effect. 358 * 359 * This method must be called with mMutex locked. 360 */ 361 virtual void freeBufferLocked(int slotIndex); 362 363 /** 364 * computeCurrentTransformMatrixLocked computes the transform matrix for the 365 * current texture. It uses mCurrentTransform and the current GraphicBuffer 366 * to compute this matrix and stores it in mCurrentTransformMatrix. 367 * mCurrentTextureImage must not be NULL. 368 */ 369 void computeCurrentTransformMatrixLocked(); 370 371 /** 372 * onSetFrameRate Notifies the consumer of a setFrameRate call from the producer side. 373 */ 374 #if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(BQ_SETFRAMERATE) 375 void onSetFrameRate(float frameRate, int8_t compatibility, 376 int8_t changeFrameRateStrategy) override; 377 #endif 378 379 /** 380 * The default consumer usage flags that SurfaceTexture always sets on its 381 * BufferQueue instance; these will be OR:d with any additional flags passed 382 * from the SurfaceTexture user. In particular, SurfaceTexture will always 383 * consume buffers as hardware textures. 384 */ 385 static const uint64_t DEFAULT_USAGE_FLAGS = GraphicBuffer::USAGE_HW_TEXTURE; 386 387 /** 388 * mCurrentCrop is the crop rectangle that applies to the current texture. 389 * It gets set each time updateTexImage is called. 390 */ 391 Rect mCurrentCrop; 392 393 /** 394 * mCurrentTransform is the transform identifier for the current texture. It 395 * gets set each time updateTexImage is called. 396 */ 397 uint32_t mCurrentTransform; 398 399 /** 400 * mCurrentScalingMode is the scaling mode for the current texture. It gets 401 * set each time updateTexImage is called. 402 */ 403 uint32_t mCurrentScalingMode; 404 405 /** 406 * mCurrentFence is the fence received from BufferQueue in updateTexImage. 407 */ 408 sp<Fence> mCurrentFence; 409 410 /** 411 * The FenceTime wrapper around mCurrentFence. 412 */ 413 std::shared_ptr<FenceTime> mCurrentFenceTime{FenceTime::NO_FENCE}; 414 415 /** 416 * mCurrentTransformMatrix is the transform matrix for the current texture. 417 * It gets computed by computeTransformMatrix each time updateTexImage is 418 * called. 419 */ 420 float mCurrentTransformMatrix[16]; 421 422 /** 423 * mCurrentTimestamp is the timestamp for the current texture. It 424 * gets set each time updateTexImage is called. 425 */ 426 int64_t mCurrentTimestamp; 427 428 /** 429 * mCurrentDataSpace is the dataspace for the current texture. It 430 * gets set each time updateTexImage is called. 431 */ 432 android_dataspace mCurrentDataSpace; 433 434 /** 435 * mCurrentFrameNumber is the frame counter for the current texture. 436 * It gets set each time updateTexImage is called. 437 */ 438 uint64_t mCurrentFrameNumber; 439 440 uint32_t mDefaultWidth, mDefaultHeight; 441 442 /** 443 * mFilteringEnabled indicates whether the transform matrix is computed for 444 * use with bilinear filtering. It defaults to true and is changed by 445 * setFilteringEnabled(). 446 */ 447 bool mFilteringEnabled; 448 449 /** 450 * mTexName is the name of the OpenGL texture to which streamed images will 451 * be bound when updateTexImage is called. It is set at construction time 452 * and can be changed with a call to attachToContext. 453 */ 454 uint32_t mTexName; 455 456 /** 457 * mUseFenceSync indicates whether creation of the EGL_KHR_fence_sync 458 * extension should be used to prevent buffers from being dequeued before 459 * it's safe for them to be written. It gets set at construction time and 460 * never changes. 461 */ 462 const bool mUseFenceSync; 463 464 /** 465 * mTexTarget is the GL texture target with which the GL texture object is 466 * associated. It is set in the constructor and never changed. It is 467 * almost always GL_TEXTURE_EXTERNAL_OES except for one use case in Android 468 * Browser. In that case it is set to GL_TEXTURE_2D to allow 469 * glCopyTexSubImage to read from the texture. This is a hack to work 470 * around a GL driver limitation on the number of FBO attachments, which the 471 * browser's tile cache exceeds. 472 */ 473 const uint32_t mTexTarget; 474 475 /** 476 * mCurrentTexture is the buffer slot index of the buffer that is currently 477 * bound to the OpenGL texture. It is initialized to INVALID_BUFFER_SLOT, 478 * indicating that no buffer slot is currently bound to the texture. Note, 479 * however, that a value of INVALID_BUFFER_SLOT does not necessarily mean 480 * that no buffer is bound to the texture. A call to setBufferCount will 481 * reset mCurrentTexture to INVALID_BUFFER_SLOT. 482 */ 483 int mCurrentTexture; 484 485 enum class OpMode { detached, attachedToConsumer, attachedToGL }; 486 /** 487 * mOpMode indicates whether the SurfaceTexture is currently attached to 488 * an OpenGL ES context or the consumer context. For legacy reasons, this 489 * is initialized to, "attachedToGL" indicating that the SurfaceTexture is 490 * considered to be attached to whatever GL context is current at the time 491 * of the first updateTexImage call. 492 * It is set to "detached" by detachFromContext, and then set to 493 * "attachedToGL" again by attachToContext. 494 * takeConsumerOwnership/releaseConsumerOwnership are used to attach/detach 495 * from a consumer context - usually HWUI RenderThread. 496 */ 497 OpMode mOpMode; 498 499 /** 500 * mEGLConsumer has SurfaceTexture logic used when attached to GL context. 501 */ 502 EGLConsumer mEGLConsumer; 503 504 /** 505 * mImageConsumer has SurfaceTexture logic used when attached to a consumer 506 * context (usually HWUI RenderThread). 507 */ 508 ImageConsumer mImageConsumer; 509 510 /** 511 * mSurfaceTextureListener holds the registered SurfaceTextureListener. 512 * Note that SurfaceTexture holds the lister with an sp<>, which means that the listener 513 * must only hold a wp<> to SurfaceTexture and not an sp<>. 514 */ 515 sp<SurfaceTextureListener> mSurfaceTextureListener; 516 517 friend class ImageConsumer; 518 friend class EGLConsumer; 519 520 private: 521 void initialize(); 522 523 // Proxy listener to avoid having SurfaceTexture directly implement FrameAvailableListener as it 524 // is extending ConsumerBase which also implements FrameAvailableListener. 525 class FrameAvailableListenerProxy : public ConsumerBase::FrameAvailableListener { 526 public: FrameAvailableListenerProxy(const wp<SurfaceTextureListener> & listener)527 FrameAvailableListenerProxy(const wp<SurfaceTextureListener>& listener) 528 : mSurfaceTextureListener(listener) {} 529 530 private: 531 void onFrameAvailable(const BufferItem& item) override; 532 533 const wp<SurfaceTextureListener> mSurfaceTextureListener; 534 }; 535 sp<FrameAvailableListenerProxy> mFrameAvailableListenerProxy; 536 }; 537 538 // ---------------------------------------------------------------------------- 539 } // namespace android 540