1 /* 2 * Copyright 2019 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #pragma once 18 19 #include <android/hardware_buffer.h> 20 #include <gui/BufferQueueDefs.h> 21 #include <gui/ConsumerBase.h> 22 #include <gui/IGraphicBufferProducer.h> 23 #include <sys/cdefs.h> 24 #include <system/graphics.h> 25 #include <ui/FenceTime.h> 26 #include <ui/GraphicBuffer.h> 27 #include <utils/Mutex.h> 28 #include <utils/String8.h> 29 30 #include "EGLConsumer.h" 31 #include "ImageConsumer.h" 32 33 namespace android { 34 35 /* 36 * SurfaceTexture consumes buffers of graphics data from a BufferQueue, 37 * and makes them available to HWUI render thread as a SkImage and to 38 * an application GL render thread as an OpenGL texture. 39 * 40 * When attached to an application GL render thread, a typical usage 41 * pattern is to set up the SurfaceTexture with the 42 * desired options, and call updateTexImage() when a new frame is desired. 43 * If a new frame is available, the texture will be updated. If not, 44 * the previous contents are retained. 45 * 46 * When attached to a HWUI render thread, the TextureView implementation 47 * calls dequeueBuffer, which either pulls a new buffer or returns the 48 * last cached buffer if BufferQueue is empty. 49 * When attached to HWUI render thread, SurfaceTexture is compatible to 50 * both Vulkan and GL drawing pipelines. 51 */ 52 class ANDROID_API SurfaceTexture : public ConsumerBase { 53 public: 54 /** 55 * Callback function needed by dequeueBuffer. It creates a fence, 56 * that is signalled, when the previous buffer is no longer in use by HWUI 57 * and can be used by written by the producer. 58 */ 59 typedef status_t (*SurfaceTexture_createReleaseFence)(bool useFenceSync, EGLSyncKHR* eglFence, 60 EGLDisplay* display, int* releaseFence, 61 void* passThroughHandle); 62 63 /** 64 * Callback function needed by dequeueBuffer. It waits for the new buffer 65 * fence to signal, before issuing any draw commands. 66 */ 67 typedef status_t (*SurfaceTexture_fenceWait)(int fence, void* passThroughHandle); 68 69 enum { TEXTURE_EXTERNAL = 0x8D65 }; // GL_TEXTURE_EXTERNAL_OES 70 typedef ConsumerBase::FrameAvailableListener FrameAvailableListener; 71 72 /** 73 * SurfaceTexture constructs a new SurfaceTexture object. If the constructor 74 * with the tex parameter is used, tex indicates the name of the OpenGL ES 75 * texture to which images are to be streamed. texTarget specifies the 76 * OpenGL ES texture target to which the texture will be bound in 77 * updateTexImage. useFenceSync specifies whether fences should be used to 78 * synchronize access to buffers if that behavior is enabled at 79 * compile-time. 80 * 81 * A SurfaceTexture may be detached from one OpenGL ES context and then 82 * attached to a different context using the detachFromContext and 83 * attachToContext methods, respectively. The intention of these methods is 84 * purely to allow a SurfaceTexture to be transferred from one consumer 85 * context to another. If such a transfer is not needed there is no 86 * requirement that either of these methods be called. 87 * 88 * If the constructor with the tex parameter is used, the SurfaceTexture is 89 * created in a state where it is considered attached to an OpenGL ES 90 * context for the purposes of the attachToContext and detachFromContext 91 * methods. However, despite being considered "attached" to a context, the 92 * specific OpenGL ES context doesn't get latched until the first call to 93 * updateTexImage. After that point, all calls to updateTexImage must be 94 * made with the same OpenGL ES context current. 95 * 96 * If the constructor without the tex parameter is used, the SurfaceTexture 97 * is created in a detached state, and attachToContext must be called before 98 * calls to updateTexImage. 99 */ 100 SurfaceTexture(const sp<IGraphicBufferConsumer>& bq, uint32_t tex, uint32_t textureTarget, 101 bool useFenceSync, bool isControlledByApp); 102 103 SurfaceTexture(const sp<IGraphicBufferConsumer>& bq, uint32_t textureTarget, bool useFenceSync, 104 bool isControlledByApp); 105 106 /** 107 * updateTexImage acquires the most recently queued buffer, and sets the 108 * image contents of the target texture to it. 109 * 110 * This call may only be made while the OpenGL ES context to which the 111 * target texture belongs is bound to the calling thread. 112 * 113 * This calls doGLFenceWait to ensure proper synchronization. 114 */ 115 status_t updateTexImage(); 116 117 /** 118 * releaseTexImage releases the texture acquired in updateTexImage(). 119 * This is intended to be used in single buffer mode. 120 * 121 * This call may only be made while the OpenGL ES context to which the 122 * target texture belongs is bound to the calling thread. 123 */ 124 status_t releaseTexImage(); 125 126 /** 127 * getTransformMatrix retrieves the 4x4 texture coordinate transform matrix 128 * associated with the texture image set by the most recent call to 129 * updateTexImage. 130 * 131 * This transform matrix maps 2D homogeneous texture coordinates of the form 132 * (s, t, 0, 1) with s and t in the inclusive range [0, 1] to the texture 133 * coordinate that should be used to sample that location from the texture. 134 * Sampling the texture outside of the range of this transform is undefined. 135 * 136 * This transform is necessary to compensate for transforms that the stream 137 * content producer may implicitly apply to the content. By forcing users of 138 * a SurfaceTexture to apply this transform we avoid performing an extra 139 * copy of the data that would be needed to hide the transform from the 140 * user. 141 * 142 * The matrix is stored in column-major order so that it may be passed 143 * directly to OpenGL ES via the glLoadMatrixf or glUniformMatrix4fv 144 * functions. 145 */ 146 void getTransformMatrix(float mtx[16]); 147 148 /** 149 * Computes the transform matrix documented by getTransformMatrix 150 * from the BufferItem sub parts. 151 */ 152 static void computeTransformMatrix(float outTransform[16], const sp<GraphicBuffer>& buf, 153 const Rect& cropRect, uint32_t transform, bool filtering); 154 155 /** 156 * Scale the crop down horizontally or vertically such that it has the 157 * same aspect ratio as the buffer does. 158 */ 159 static Rect scaleDownCrop(const Rect& crop, uint32_t bufferWidth, uint32_t bufferHeight); 160 161 /** 162 * getTimestamp retrieves the timestamp associated with the texture image 163 * set by the most recent call to updateTexImage. 164 * 165 * The timestamp is in nanoseconds, and is monotonically increasing. Its 166 * other semantics (zero point, etc) are source-dependent and should be 167 * documented by the source. 168 */ 169 int64_t getTimestamp(); 170 171 /** 172 * getDataSpace retrieves the DataSpace associated with the texture image 173 * set by the most recent call to updateTexImage. 174 */ 175 android_dataspace getCurrentDataSpace(); 176 177 /** 178 * getFrameNumber retrieves the frame number associated with the texture 179 * image set by the most recent call to updateTexImage. 180 * 181 * The frame number is an incrementing counter set to 0 at the creation of 182 * the BufferQueue associated with this consumer. 183 */ 184 uint64_t getFrameNumber(); 185 186 /** 187 * setDefaultBufferSize is used to set the size of buffers returned by 188 * requestBuffers when a with and height of zero is requested. 189 * A call to setDefaultBufferSize() may trigger requestBuffers() to 190 * be called from the client. 191 * The width and height parameters must be no greater than the minimum of 192 * GL_MAX_VIEWPORT_DIMS and GL_MAX_TEXTURE_SIZE (see: glGetIntegerv). 193 * An error due to invalid dimensions might not be reported until 194 * updateTexImage() is called. 195 */ 196 status_t setDefaultBufferSize(uint32_t width, uint32_t height); 197 198 /** 199 * setFilteringEnabled sets whether the transform matrix should be computed 200 * for use with bilinear filtering. 201 */ 202 void setFilteringEnabled(bool enabled); 203 204 /** 205 * getCurrentTextureTarget returns the texture target of the current 206 * texture as returned by updateTexImage(). 207 */ 208 uint32_t getCurrentTextureTarget() const; 209 210 /** 211 * getCurrentCrop returns the cropping rectangle of the current buffer. 212 */ 213 Rect getCurrentCrop() const; 214 215 /** 216 * getCurrentTransform returns the transform of the current buffer. 217 */ 218 uint32_t getCurrentTransform() const; 219 220 /** 221 * getCurrentScalingMode returns the scaling mode of the current buffer. 222 */ 223 uint32_t getCurrentScalingMode() const; 224 225 /** 226 * getCurrentFence returns the fence indicating when the current buffer is 227 * ready to be read from. 228 */ 229 sp<Fence> getCurrentFence() const; 230 231 /** 232 * getCurrentFence returns the FenceTime indicating when the current 233 * buffer is ready to be read from. 234 */ 235 std::shared_ptr<FenceTime> getCurrentFenceTime() const; 236 237 /** 238 * setConsumerUsageBits overrides the ConsumerBase method to OR 239 * DEFAULT_USAGE_FLAGS to usage. 240 */ 241 status_t setConsumerUsageBits(uint64_t usage); 242 243 /** 244 * detachFromContext detaches the SurfaceTexture from the calling thread's 245 * current OpenGL ES context. This context must be the same as the context 246 * that was current for previous calls to updateTexImage. 247 * 248 * Detaching a SurfaceTexture from an OpenGL ES context will result in the 249 * deletion of the OpenGL ES texture object into which the images were being 250 * streamed. After a SurfaceTexture has been detached from the OpenGL ES 251 * context calls to updateTexImage will fail returning INVALID_OPERATION 252 * until the SurfaceTexture is attached to a new OpenGL ES context using the 253 * attachToContext method. 254 */ 255 status_t detachFromContext(); 256 257 /** 258 * attachToContext attaches a SurfaceTexture that is currently in the 259 * 'detached' state to the current OpenGL ES context. A SurfaceTexture is 260 * in the 'detached' state iff detachFromContext has successfully been 261 * called and no calls to attachToContext have succeeded since the last 262 * detachFromContext call. Calls to attachToContext made on a 263 * SurfaceTexture that is not in the 'detached' state will result in an 264 * INVALID_OPERATION error. 265 * 266 * The tex argument specifies the OpenGL ES texture object name in the 267 * new context into which the image contents will be streamed. A successful 268 * call to attachToContext will result in this texture object being bound to 269 * the texture target and populated with the image contents that were 270 * current at the time of the last call to detachFromContext. 271 */ 272 status_t attachToContext(uint32_t tex); 273 274 sp<GraphicBuffer> dequeueBuffer(int* outSlotid, android_dataspace* outDataspace, 275 float* outTransformMatrix, bool* outQueueEmpty, 276 SurfaceTexture_createReleaseFence createFence, 277 SurfaceTexture_fenceWait fenceWait, 278 void* fencePassThroughHandle); 279 280 /** 281 * takeConsumerOwnership attaches a SurfaceTexture that is currently in the 282 * 'detached' state to a consumer context (usually HWUI RenderThread). 283 */ 284 void takeConsumerOwnership(); 285 286 /** 287 * releaseConsumerOwnership detaches a SurfaceTexture from a consumer 288 * context (usually HWUI RenderThread). 289 */ 290 void releaseConsumerOwnership(); 291 292 protected: 293 /** 294 * abandonLocked overrides the ConsumerBase method to clear 295 * mCurrentTextureImage in addition to the ConsumerBase behavior. 296 */ 297 virtual void abandonLocked(); 298 299 /** 300 * dumpLocked overrides the ConsumerBase method to dump SurfaceTexture- 301 * specific info in addition to the ConsumerBase behavior. 302 */ 303 virtual void dumpLocked(String8& result, const char* prefix) const override; 304 305 /** 306 * acquireBufferLocked overrides the ConsumerBase method to update the 307 * mEglSlots array in addition to the ConsumerBase behavior. 308 */ 309 virtual status_t acquireBufferLocked(BufferItem* item, nsecs_t presentWhen, 310 uint64_t maxFrameNumber = 0) override; 311 312 /** 313 * releaseBufferLocked overrides the ConsumerBase method to update the 314 * mEglSlots array in addition to the ConsumerBase. 315 */ 316 virtual status_t releaseBufferLocked(int slot, const sp<GraphicBuffer> graphicBuffer, 317 EGLDisplay display, EGLSyncKHR eglFence) override; 318 319 /** 320 * freeBufferLocked frees up the given buffer slot. If the slot has been 321 * initialized this will release the reference to the GraphicBuffer in that 322 * slot and destroy the EGLImage in that slot. Otherwise it has no effect. 323 * 324 * This method must be called with mMutex locked. 325 */ 326 virtual void freeBufferLocked(int slotIndex); 327 328 /** 329 * computeCurrentTransformMatrixLocked computes the transform matrix for the 330 * current texture. It uses mCurrentTransform and the current GraphicBuffer 331 * to compute this matrix and stores it in mCurrentTransformMatrix. 332 * mCurrentTextureImage must not be NULL. 333 */ 334 void computeCurrentTransformMatrixLocked(); 335 336 /** 337 * The default consumer usage flags that SurfaceTexture always sets on its 338 * BufferQueue instance; these will be OR:d with any additional flags passed 339 * from the SurfaceTexture user. In particular, SurfaceTexture will always 340 * consume buffers as hardware textures. 341 */ 342 static const uint64_t DEFAULT_USAGE_FLAGS = GraphicBuffer::USAGE_HW_TEXTURE; 343 344 /** 345 * mCurrentCrop is the crop rectangle that applies to the current texture. 346 * It gets set each time updateTexImage is called. 347 */ 348 Rect mCurrentCrop; 349 350 /** 351 * mCurrentTransform is the transform identifier for the current texture. It 352 * gets set each time updateTexImage is called. 353 */ 354 uint32_t mCurrentTransform; 355 356 /** 357 * mCurrentScalingMode is the scaling mode for the current texture. It gets 358 * set each time updateTexImage is called. 359 */ 360 uint32_t mCurrentScalingMode; 361 362 /** 363 * mCurrentFence is the fence received from BufferQueue in updateTexImage. 364 */ 365 sp<Fence> mCurrentFence; 366 367 /** 368 * The FenceTime wrapper around mCurrentFence. 369 */ 370 std::shared_ptr<FenceTime> mCurrentFenceTime{FenceTime::NO_FENCE}; 371 372 /** 373 * mCurrentTransformMatrix is the transform matrix for the current texture. 374 * It gets computed by computeTransformMatrix each time updateTexImage is 375 * called. 376 */ 377 float mCurrentTransformMatrix[16]; 378 379 /** 380 * mCurrentTimestamp is the timestamp for the current texture. It 381 * gets set each time updateTexImage is called. 382 */ 383 int64_t mCurrentTimestamp; 384 385 /** 386 * mCurrentDataSpace is the dataspace for the current texture. It 387 * gets set each time updateTexImage is called. 388 */ 389 android_dataspace mCurrentDataSpace; 390 391 /** 392 * mCurrentFrameNumber is the frame counter for the current texture. 393 * It gets set each time updateTexImage is called. 394 */ 395 uint64_t mCurrentFrameNumber; 396 397 uint32_t mDefaultWidth, mDefaultHeight; 398 399 /** 400 * mFilteringEnabled indicates whether the transform matrix is computed for 401 * use with bilinear filtering. It defaults to true and is changed by 402 * setFilteringEnabled(). 403 */ 404 bool mFilteringEnabled; 405 406 /** 407 * mTexName is the name of the OpenGL texture to which streamed images will 408 * be bound when updateTexImage is called. It is set at construction time 409 * and can be changed with a call to attachToContext. 410 */ 411 uint32_t mTexName; 412 413 /** 414 * mUseFenceSync indicates whether creation of the EGL_KHR_fence_sync 415 * extension should be used to prevent buffers from being dequeued before 416 * it's safe for them to be written. It gets set at construction time and 417 * never changes. 418 */ 419 const bool mUseFenceSync; 420 421 /** 422 * mTexTarget is the GL texture target with which the GL texture object is 423 * associated. It is set in the constructor and never changed. It is 424 * almost always GL_TEXTURE_EXTERNAL_OES except for one use case in Android 425 * Browser. In that case it is set to GL_TEXTURE_2D to allow 426 * glCopyTexSubImage to read from the texture. This is a hack to work 427 * around a GL driver limitation on the number of FBO attachments, which the 428 * browser's tile cache exceeds. 429 */ 430 const uint32_t mTexTarget; 431 432 /** 433 * mCurrentTexture is the buffer slot index of the buffer that is currently 434 * bound to the OpenGL texture. It is initialized to INVALID_BUFFER_SLOT, 435 * indicating that no buffer slot is currently bound to the texture. Note, 436 * however, that a value of INVALID_BUFFER_SLOT does not necessarily mean 437 * that no buffer is bound to the texture. A call to setBufferCount will 438 * reset mCurrentTexture to INVALID_BUFFER_SLOT. 439 */ 440 int mCurrentTexture; 441 442 enum class OpMode { detached, attachedToConsumer, attachedToGL }; 443 /** 444 * mOpMode indicates whether the SurfaceTexture is currently attached to 445 * an OpenGL ES context or the consumer context. For legacy reasons, this 446 * is initialized to, "attachedToGL" indicating that the SurfaceTexture is 447 * considered to be attached to whatever GL context is current at the time 448 * of the first updateTexImage call. 449 * It is set to "detached" by detachFromContext, and then set to 450 * "attachedToGL" again by attachToContext. 451 * takeConsumerOwnership/releaseConsumerOwnership are used to attach/detach 452 * from a consumer context - usually HWUI RenderThread. 453 */ 454 OpMode mOpMode; 455 456 /** 457 * mEGLConsumer has SurfaceTexture logic used when attached to GL context. 458 */ 459 EGLConsumer mEGLConsumer; 460 461 /** 462 * mImageConsumer has SurfaceTexture logic used when attached to a consumer 463 * context (usually HWUI RenderThread). 464 */ 465 ImageConsumer mImageConsumer; 466 467 friend class ImageConsumer; 468 friend class EGLConsumer; 469 }; 470 471 // ---------------------------------------------------------------------------- 472 } // namespace android 473