1 /* 2 * Copyright 2019 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #pragma once 18 19 #include <android/hardware_buffer.h> 20 #include <gui/BufferQueueDefs.h> 21 #include <gui/ConsumerBase.h> 22 #include <gui/IGraphicBufferProducer.h> 23 #include <sys/cdefs.h> 24 #include <system/graphics.h> 25 #include <ui/FenceTime.h> 26 #include <ui/GraphicBuffer.h> 27 #include <utils/Mutex.h> 28 #include <utils/String8.h> 29 30 #include "EGLConsumer.h" 31 #include "ImageConsumer.h" 32 33 namespace android { 34 35 /* 36 * SurfaceTexture consumes buffers of graphics data from a BufferQueue, 37 * and makes them available to HWUI render thread as a SkImage and to 38 * an application GL render thread as an OpenGL texture. 39 * 40 * When attached to an application GL render thread, a typical usage 41 * pattern is to set up the SurfaceTexture with the 42 * desired options, and call updateTexImage() when a new frame is desired. 43 * If a new frame is available, the texture will be updated. If not, 44 * the previous contents are retained. 45 * 46 * When attached to a HWUI render thread, the TextureView implementation 47 * calls dequeueBuffer, which either pulls a new buffer or returns the 48 * last cached buffer if BufferQueue is empty. 49 * When attached to HWUI render thread, SurfaceTexture is compatible to 50 * both Vulkan and GL drawing pipelines. 51 */ 52 class ANDROID_API SurfaceTexture : public ConsumerBase { 53 public: 54 /** 55 * Callback function needed by dequeueBuffer. It creates a fence, 56 * that is signalled, when the previous buffer is no longer in use by HWUI 57 * and can be used by written by the producer. 58 */ 59 typedef status_t (*SurfaceTexture_createReleaseFence)(bool useFenceSync, EGLSyncKHR* eglFence, 60 EGLDisplay* display, int* releaseFence, 61 void* passThroughHandle); 62 63 /** 64 * Callback function needed by dequeueBuffer. It waits for the new buffer 65 * fence to signal, before issuing any draw commands. 66 */ 67 typedef status_t (*SurfaceTexture_fenceWait)(int fence, void* passThroughHandle); 68 69 enum { TEXTURE_EXTERNAL = 0x8D65 }; // GL_TEXTURE_EXTERNAL_OES 70 typedef ConsumerBase::FrameAvailableListener FrameAvailableListener; 71 72 /** 73 * SurfaceTexture constructs a new SurfaceTexture object. If the constructor 74 * with the tex parameter is used, tex indicates the name of the OpenGL ES 75 * texture to which images are to be streamed. texTarget specifies the 76 * OpenGL ES texture target to which the texture will be bound in 77 * updateTexImage. useFenceSync specifies whether fences should be used to 78 * synchronize access to buffers if that behavior is enabled at 79 * compile-time. 80 * 81 * A SurfaceTexture may be detached from one OpenGL ES context and then 82 * attached to a different context using the detachFromContext and 83 * attachToContext methods, respectively. The intention of these methods is 84 * purely to allow a SurfaceTexture to be transferred from one consumer 85 * context to another. If such a transfer is not needed there is no 86 * requirement that either of these methods be called. 87 * 88 * If the constructor with the tex parameter is used, the SurfaceTexture is 89 * created in a state where it is considered attached to an OpenGL ES 90 * context for the purposes of the attachToContext and detachFromContext 91 * methods. However, despite being considered "attached" to a context, the 92 * specific OpenGL ES context doesn't get latched until the first call to 93 * updateTexImage. After that point, all calls to updateTexImage must be 94 * made with the same OpenGL ES context current. 95 * 96 * If the constructor without the tex parameter is used, the SurfaceTexture 97 * is created in a detached state, and attachToContext must be called before 98 * calls to updateTexImage. 99 */ 100 SurfaceTexture(const sp<IGraphicBufferConsumer>& bq, uint32_t tex, uint32_t textureTarget, 101 bool useFenceSync, bool isControlledByApp); 102 103 SurfaceTexture(const sp<IGraphicBufferConsumer>& bq, uint32_t textureTarget, bool useFenceSync, 104 bool isControlledByApp); 105 106 /** 107 * updateTexImage acquires the most recently queued buffer, and sets the 108 * image contents of the target texture to it. 109 * 110 * This call may only be made while the OpenGL ES context to which the 111 * target texture belongs is bound to the calling thread. 112 * 113 * This calls doGLFenceWait to ensure proper synchronization. 114 */ 115 status_t updateTexImage(); 116 117 /** 118 * releaseTexImage releases the texture acquired in updateTexImage(). 119 * This is intended to be used in single buffer mode. 120 * 121 * This call may only be made while the OpenGL ES context to which the 122 * target texture belongs is bound to the calling thread. 123 */ 124 status_t releaseTexImage(); 125 126 /** 127 * getTransformMatrix retrieves the 4x4 texture coordinate transform matrix 128 * associated with the texture image set by the most recent call to 129 * updateTexImage. 130 * 131 * This transform matrix maps 2D homogeneous texture coordinates of the form 132 * (s, t, 0, 1) with s and t in the inclusive range [0, 1] to the texture 133 * coordinate that should be used to sample that location from the texture. 134 * Sampling the texture outside of the range of this transform is undefined. 135 * 136 * This transform is necessary to compensate for transforms that the stream 137 * content producer may implicitly apply to the content. By forcing users of 138 * a SurfaceTexture to apply this transform we avoid performing an extra 139 * copy of the data that would be needed to hide the transform from the 140 * user. 141 * 142 * The matrix is stored in column-major order so that it may be passed 143 * directly to OpenGL ES via the glLoadMatrixf or glUniformMatrix4fv 144 * functions. 145 */ 146 void getTransformMatrix(float mtx[16]); 147 148 /** 149 * Computes the transform matrix documented by getTransformMatrix 150 * from the BufferItem sub parts. 151 */ 152 static void computeTransformMatrix(float outTransform[16], const sp<GraphicBuffer>& buf, 153 const Rect& cropRect, uint32_t transform, bool filtering); 154 155 /** 156 * Scale the crop down horizontally or vertically such that it has the 157 * same aspect ratio as the buffer does. 158 */ 159 static Rect scaleDownCrop(const Rect& crop, uint32_t bufferWidth, uint32_t bufferHeight); 160 161 /** 162 * getTimestamp retrieves the timestamp associated with the texture image 163 * set by the most recent call to updateTexImage. 164 * 165 * The timestamp is in nanoseconds, and is monotonically increasing. Its 166 * other semantics (zero point, etc) are source-dependent and should be 167 * documented by the source. 168 */ 169 int64_t getTimestamp(); 170 171 /** 172 * getDataSpace retrieves the DataSpace associated with the texture image 173 * set by the most recent call to updateTexImage. 174 */ 175 android_dataspace getCurrentDataSpace(); 176 177 /** 178 * getFrameNumber retrieves the frame number associated with the texture 179 * image set by the most recent call to updateTexImage. 180 * 181 * The frame number is an incrementing counter set to 0 at the creation of 182 * the BufferQueue associated with this consumer. 183 */ 184 uint64_t getFrameNumber(); 185 186 /** 187 * setDefaultBufferSize is used to set the size of buffers returned by 188 * requestBuffers when a with and height of zero is requested. 189 * A call to setDefaultBufferSize() may trigger requestBuffers() to 190 * be called from the client. 191 * The width and height parameters must be no greater than the minimum of 192 * GL_MAX_VIEWPORT_DIMS and GL_MAX_TEXTURE_SIZE (see: glGetIntegerv). 193 * An error due to invalid dimensions might not be reported until 194 * updateTexImage() is called. 195 */ 196 status_t setDefaultBufferSize(uint32_t width, uint32_t height); 197 198 /** 199 * setFilteringEnabled sets whether the transform matrix should be computed 200 * for use with bilinear filtering. 201 */ 202 void setFilteringEnabled(bool enabled); 203 204 /** 205 * getCurrentTextureTarget returns the texture target of the current 206 * texture as returned by updateTexImage(). 207 */ 208 uint32_t getCurrentTextureTarget() const; 209 210 /** 211 * getCurrentCrop returns the cropping rectangle of the current buffer. 212 */ 213 Rect getCurrentCrop() const; 214 215 /** 216 * getCurrentTransform returns the transform of the current buffer. 217 */ 218 uint32_t getCurrentTransform() const; 219 220 /** 221 * getCurrentScalingMode returns the scaling mode of the current buffer. 222 */ 223 uint32_t getCurrentScalingMode() const; 224 225 /** 226 * getCurrentFence returns the fence indicating when the current buffer is 227 * ready to be read from. 228 */ 229 sp<Fence> getCurrentFence() const; 230 231 /** 232 * getCurrentFence returns the FenceTime indicating when the current 233 * buffer is ready to be read from. 234 */ 235 std::shared_ptr<FenceTime> getCurrentFenceTime() const; 236 237 /** 238 * setConsumerUsageBits overrides the ConsumerBase method to OR 239 * DEFAULT_USAGE_FLAGS to usage. 240 */ 241 status_t setConsumerUsageBits(uint64_t usage); 242 243 /** 244 * detachFromContext detaches the SurfaceTexture from the calling thread's 245 * current OpenGL ES context. This context must be the same as the context 246 * that was current for previous calls to updateTexImage. 247 * 248 * Detaching a SurfaceTexture from an OpenGL ES context will result in the 249 * deletion of the OpenGL ES texture object into which the images were being 250 * streamed. After a SurfaceTexture has been detached from the OpenGL ES 251 * context calls to updateTexImage will fail returning INVALID_OPERATION 252 * until the SurfaceTexture is attached to a new OpenGL ES context using the 253 * attachToContext method. 254 */ 255 status_t detachFromContext(); 256 257 /** 258 * attachToContext attaches a SurfaceTexture that is currently in the 259 * 'detached' state to the current OpenGL ES context. A SurfaceTexture is 260 * in the 'detached' state iff detachFromContext has successfully been 261 * called and no calls to attachToContext have succeeded since the last 262 * detachFromContext call. Calls to attachToContext made on a 263 * SurfaceTexture that is not in the 'detached' state will result in an 264 * INVALID_OPERATION error. 265 * 266 * The tex argument specifies the OpenGL ES texture object name in the 267 * new context into which the image contents will be streamed. A successful 268 * call to attachToContext will result in this texture object being bound to 269 * the texture target and populated with the image contents that were 270 * current at the time of the last call to detachFromContext. 271 */ 272 status_t attachToContext(uint32_t tex); 273 274 sp<GraphicBuffer> dequeueBuffer(int* outSlotid, android_dataspace* outDataspace, 275 HdrMetadata* outHdrMetadata, float* outTransformMatrix, 276 uint32_t* outTransform, bool* outQueueEmpty, 277 SurfaceTexture_createReleaseFence createFence, 278 SurfaceTexture_fenceWait fenceWait, 279 void* fencePassThroughHandle, ARect* currentCrop); 280 281 /** 282 * takeConsumerOwnership attaches a SurfaceTexture that is currently in the 283 * 'detached' state to a consumer context (usually HWUI RenderThread). 284 */ 285 void takeConsumerOwnership(); 286 287 /** 288 * releaseConsumerOwnership detaches a SurfaceTexture from a consumer 289 * context (usually HWUI RenderThread). 290 */ 291 void releaseConsumerOwnership(); 292 293 protected: 294 /** 295 * abandonLocked overrides the ConsumerBase method to clear 296 * mCurrentTextureImage in addition to the ConsumerBase behavior. 297 */ 298 virtual void abandonLocked(); 299 300 /** 301 * dumpLocked overrides the ConsumerBase method to dump SurfaceTexture- 302 * specific info in addition to the ConsumerBase behavior. 303 */ 304 virtual void dumpLocked(String8& result, const char* prefix) const override; 305 306 /** 307 * acquireBufferLocked overrides the ConsumerBase method to update the 308 * mEglSlots array in addition to the ConsumerBase behavior. 309 */ 310 virtual status_t acquireBufferLocked(BufferItem* item, nsecs_t presentWhen, 311 uint64_t maxFrameNumber = 0) override; 312 313 /** 314 * releaseBufferLocked overrides the ConsumerBase method to update the 315 * mEglSlots array in addition to the ConsumerBase. 316 */ 317 virtual status_t releaseBufferLocked(int slot, const sp<GraphicBuffer> graphicBuffer, 318 EGLDisplay display, EGLSyncKHR eglFence) override; 319 320 /** 321 * freeBufferLocked frees up the given buffer slot. If the slot has been 322 * initialized this will release the reference to the GraphicBuffer in that 323 * slot and destroy the EGLImage in that slot. Otherwise it has no effect. 324 * 325 * This method must be called with mMutex locked. 326 */ 327 virtual void freeBufferLocked(int slotIndex); 328 329 /** 330 * computeCurrentTransformMatrixLocked computes the transform matrix for the 331 * current texture. It uses mCurrentTransform and the current GraphicBuffer 332 * to compute this matrix and stores it in mCurrentTransformMatrix. 333 * mCurrentTextureImage must not be NULL. 334 */ 335 void computeCurrentTransformMatrixLocked(); 336 337 /** 338 * The default consumer usage flags that SurfaceTexture always sets on its 339 * BufferQueue instance; these will be OR:d with any additional flags passed 340 * from the SurfaceTexture user. In particular, SurfaceTexture will always 341 * consume buffers as hardware textures. 342 */ 343 static const uint64_t DEFAULT_USAGE_FLAGS = GraphicBuffer::USAGE_HW_TEXTURE; 344 345 /** 346 * mCurrentCrop is the crop rectangle that applies to the current texture. 347 * It gets set each time updateTexImage is called. 348 */ 349 Rect mCurrentCrop; 350 351 /** 352 * mCurrentTransform is the transform identifier for the current texture. It 353 * gets set each time updateTexImage is called. 354 */ 355 uint32_t mCurrentTransform; 356 357 /** 358 * mCurrentScalingMode is the scaling mode for the current texture. It gets 359 * set each time updateTexImage is called. 360 */ 361 uint32_t mCurrentScalingMode; 362 363 /** 364 * mCurrentFence is the fence received from BufferQueue in updateTexImage. 365 */ 366 sp<Fence> mCurrentFence; 367 368 /** 369 * The FenceTime wrapper around mCurrentFence. 370 */ 371 std::shared_ptr<FenceTime> mCurrentFenceTime{FenceTime::NO_FENCE}; 372 373 /** 374 * mCurrentTransformMatrix is the transform matrix for the current texture. 375 * It gets computed by computeTransformMatrix each time updateTexImage is 376 * called. 377 */ 378 float mCurrentTransformMatrix[16]; 379 380 /** 381 * mCurrentTimestamp is the timestamp for the current texture. It 382 * gets set each time updateTexImage is called. 383 */ 384 int64_t mCurrentTimestamp; 385 386 /** 387 * mCurrentDataSpace is the dataspace for the current texture. It 388 * gets set each time updateTexImage is called. 389 */ 390 android_dataspace mCurrentDataSpace; 391 392 /** 393 * mCurrentFrameNumber is the frame counter for the current texture. 394 * It gets set each time updateTexImage is called. 395 */ 396 uint64_t mCurrentFrameNumber; 397 398 uint32_t mDefaultWidth, mDefaultHeight; 399 400 /** 401 * mFilteringEnabled indicates whether the transform matrix is computed for 402 * use with bilinear filtering. It defaults to true and is changed by 403 * setFilteringEnabled(). 404 */ 405 bool mFilteringEnabled; 406 407 /** 408 * mTexName is the name of the OpenGL texture to which streamed images will 409 * be bound when updateTexImage is called. It is set at construction time 410 * and can be changed with a call to attachToContext. 411 */ 412 uint32_t mTexName; 413 414 /** 415 * mUseFenceSync indicates whether creation of the EGL_KHR_fence_sync 416 * extension should be used to prevent buffers from being dequeued before 417 * it's safe for them to be written. It gets set at construction time and 418 * never changes. 419 */ 420 const bool mUseFenceSync; 421 422 /** 423 * mTexTarget is the GL texture target with which the GL texture object is 424 * associated. It is set in the constructor and never changed. It is 425 * almost always GL_TEXTURE_EXTERNAL_OES except for one use case in Android 426 * Browser. In that case it is set to GL_TEXTURE_2D to allow 427 * glCopyTexSubImage to read from the texture. This is a hack to work 428 * around a GL driver limitation on the number of FBO attachments, which the 429 * browser's tile cache exceeds. 430 */ 431 const uint32_t mTexTarget; 432 433 /** 434 * mCurrentTexture is the buffer slot index of the buffer that is currently 435 * bound to the OpenGL texture. It is initialized to INVALID_BUFFER_SLOT, 436 * indicating that no buffer slot is currently bound to the texture. Note, 437 * however, that a value of INVALID_BUFFER_SLOT does not necessarily mean 438 * that no buffer is bound to the texture. A call to setBufferCount will 439 * reset mCurrentTexture to INVALID_BUFFER_SLOT. 440 */ 441 int mCurrentTexture; 442 443 enum class OpMode { detached, attachedToConsumer, attachedToGL }; 444 /** 445 * mOpMode indicates whether the SurfaceTexture is currently attached to 446 * an OpenGL ES context or the consumer context. For legacy reasons, this 447 * is initialized to, "attachedToGL" indicating that the SurfaceTexture is 448 * considered to be attached to whatever GL context is current at the time 449 * of the first updateTexImage call. 450 * It is set to "detached" by detachFromContext, and then set to 451 * "attachedToGL" again by attachToContext. 452 * takeConsumerOwnership/releaseConsumerOwnership are used to attach/detach 453 * from a consumer context - usually HWUI RenderThread. 454 */ 455 OpMode mOpMode; 456 457 /** 458 * mEGLConsumer has SurfaceTexture logic used when attached to GL context. 459 */ 460 EGLConsumer mEGLConsumer; 461 462 /** 463 * mImageConsumer has SurfaceTexture logic used when attached to a consumer 464 * context (usually HWUI RenderThread). 465 */ 466 ImageConsumer mImageConsumer; 467 468 friend class ImageConsumer; 469 friend class EGLConsumer; 470 }; 471 472 // ---------------------------------------------------------------------------- 473 } // namespace android 474