1 /* 2 * Copyright (C) 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #pragma once 18 19 #include <gui/BufferQueueDefs.h> 20 #include <gui/ConsumerBase.h> 21 22 #include <ui/FenceTime.h> 23 #include <ui/GraphicBuffer.h> 24 25 #include <utils/Mutex.h> 26 #include <utils/String8.h> 27 28 #include "EGLConsumer.h" 29 #include "ImageConsumer.h" 30 31 namespace android { 32 33 namespace uirenderer { 34 class RenderState; 35 } 36 37 /* 38 * SurfaceTexture consumes buffers of graphics data from a BufferQueue, 39 * and makes them available to HWUI render thread as a SkImage and to 40 * an application GL render thread as an OpenGL texture. 41 * 42 * When attached to an application GL render thread, a typical usage 43 * pattern is to set up the SurfaceTexture with the 44 * desired options, and call updateTexImage() when a new frame is desired. 45 * If a new frame is available, the texture will be updated. If not, 46 * the previous contents are retained. 47 * 48 * When attached to a HWUI render thread, the TextureView implementation 49 * calls dequeueImage, which either pulls a new SkImage or returns the 50 * last cached SkImage if BufferQueue is empty. 51 * When attached to HWUI render thread, SurfaceTexture is compatible to 52 * both Vulkan and GL drawing pipelines. 53 */ 54 class ANDROID_API SurfaceTexture : public ConsumerBase { 55 public: 56 enum { TEXTURE_EXTERNAL = 0x8D65 }; // GL_TEXTURE_EXTERNAL_OES 57 typedef ConsumerBase::FrameAvailableListener FrameAvailableListener; 58 59 /** 60 * SurfaceTexture constructs a new SurfaceTexture object. If the constructor with 61 * the tex parameter is used, tex indicates the name of the OpenGL ES 62 * texture to which images are to be streamed. texTarget specifies the 63 * OpenGL ES texture target to which the texture will be bound in 64 * updateTexImage. useFenceSync specifies whether fences should be used to 65 * synchronize access to buffers if that behavior is enabled at 66 * compile-time. 67 * 68 * A SurfaceTexture may be detached from one OpenGL ES context and then 69 * attached to a different context using the detachFromContext and 70 * attachToContext methods, respectively. The intention of these methods is 71 * purely to allow a SurfaceTexture to be transferred from one consumer 72 * context to another. If such a transfer is not needed there is no 73 * requirement that either of these methods be called. 74 * 75 * If the constructor with the tex parameter is used, the SurfaceTexture is 76 * created in a state where it is considered attached to an OpenGL ES 77 * context for the purposes of the attachToContext and detachFromContext 78 * methods. However, despite being considered "attached" to a context, the 79 * specific OpenGL ES context doesn't get latched until the first call to 80 * updateTexImage. After that point, all calls to updateTexImage must be 81 * made with the same OpenGL ES context current. 82 * 83 * If the constructor without the tex parameter is used, the SurfaceTexture is 84 * created in a detached state, and attachToContext must be called before 85 * calls to updateTexImage. 86 */ 87 SurfaceTexture(const sp<IGraphicBufferConsumer>& bq, uint32_t tex, uint32_t texureTarget, 88 bool useFenceSync, bool isControlledByApp); 89 90 SurfaceTexture(const sp<IGraphicBufferConsumer>& bq, uint32_t texureTarget, bool useFenceSync, 91 bool isControlledByApp); 92 93 /** 94 * updateTexImage acquires the most recently queued buffer, and sets the 95 * image contents of the target texture to it. 96 * 97 * This call may only be made while the OpenGL ES context to which the 98 * target texture belongs is bound to the calling thread. 99 * 100 * This calls doGLFenceWait to ensure proper synchronization. 101 */ 102 status_t updateTexImage(); 103 104 /** 105 * releaseTexImage releases the texture acquired in updateTexImage(). 106 * This is intended to be used in single buffer mode. 107 * 108 * This call may only be made while the OpenGL ES context to which the 109 * target texture belongs is bound to the calling thread. 110 */ 111 status_t releaseTexImage(); 112 113 /** 114 * getTransformMatrix retrieves the 4x4 texture coordinate transform matrix 115 * associated with the texture image set by the most recent call to 116 * updateTexImage. 117 * 118 * This transform matrix maps 2D homogeneous texture coordinates of the form 119 * (s, t, 0, 1) with s and t in the inclusive range [0, 1] to the texture 120 * coordinate that should be used to sample that location from the texture. 121 * Sampling the texture outside of the range of this transform is undefined. 122 * 123 * This transform is necessary to compensate for transforms that the stream 124 * content producer may implicitly apply to the content. By forcing users of 125 * a SurfaceTexture to apply this transform we avoid performing an extra 126 * copy of the data that would be needed to hide the transform from the 127 * user. 128 * 129 * The matrix is stored in column-major order so that it may be passed 130 * directly to OpenGL ES via the glLoadMatrixf or glUniformMatrix4fv 131 * functions. 132 */ 133 void getTransformMatrix(float mtx[16]); 134 135 /** 136 * Computes the transform matrix documented by getTransformMatrix 137 * from the BufferItem sub parts. 138 */ 139 static void computeTransformMatrix(float outTransform[16], const sp<GraphicBuffer>& buf, 140 const Rect& cropRect, uint32_t transform, bool filtering); 141 142 /** 143 * Scale the crop down horizontally or vertically such that it has the 144 * same aspect ratio as the buffer does. 145 */ 146 static Rect scaleDownCrop(const Rect& crop, uint32_t bufferWidth, uint32_t bufferHeight); 147 148 /** 149 * getTimestamp retrieves the timestamp associated with the texture image 150 * set by the most recent call to updateTexImage. 151 * 152 * The timestamp is in nanoseconds, and is monotonically increasing. Its 153 * other semantics (zero point, etc) are source-dependent and should be 154 * documented by the source. 155 */ 156 int64_t getTimestamp(); 157 158 /** 159 * getDataSpace retrieves the DataSpace associated with the texture image 160 * set by the most recent call to updateTexImage. 161 */ 162 android_dataspace getCurrentDataSpace(); 163 164 /** 165 * getFrameNumber retrieves the frame number associated with the texture 166 * image set by the most recent call to updateTexImage. 167 * 168 * The frame number is an incrementing counter set to 0 at the creation of 169 * the BufferQueue associated with this consumer. 170 */ 171 uint64_t getFrameNumber(); 172 173 /** 174 * setDefaultBufferSize is used to set the size of buffers returned by 175 * requestBuffers when a with and height of zero is requested. 176 * A call to setDefaultBufferSize() may trigger requestBuffers() to 177 * be called from the client. 178 * The width and height parameters must be no greater than the minimum of 179 * GL_MAX_VIEWPORT_DIMS and GL_MAX_TEXTURE_SIZE (see: glGetIntegerv). 180 * An error due to invalid dimensions might not be reported until 181 * updateTexImage() is called. 182 */ 183 status_t setDefaultBufferSize(uint32_t width, uint32_t height); 184 185 /** 186 * setFilteringEnabled sets whether the transform matrix should be computed 187 * for use with bilinear filtering. 188 */ 189 void setFilteringEnabled(bool enabled); 190 191 /** 192 * getCurrentTextureTarget returns the texture target of the current 193 * texture as returned by updateTexImage(). 194 */ 195 uint32_t getCurrentTextureTarget() const; 196 197 /** 198 * getCurrentCrop returns the cropping rectangle of the current buffer. 199 */ 200 Rect getCurrentCrop() const; 201 202 /** 203 * getCurrentTransform returns the transform of the current buffer. 204 */ 205 uint32_t getCurrentTransform() const; 206 207 /** 208 * getCurrentScalingMode returns the scaling mode of the current buffer. 209 */ 210 uint32_t getCurrentScalingMode() const; 211 212 /** 213 * getCurrentFence returns the fence indicating when the current buffer is 214 * ready to be read from. 215 */ 216 sp<Fence> getCurrentFence() const; 217 218 /** 219 * getCurrentFence returns the FenceTime indicating when the current 220 * buffer is ready to be read from. 221 */ 222 std::shared_ptr<FenceTime> getCurrentFenceTime() const; 223 224 /** 225 * setConsumerUsageBits overrides the ConsumerBase method to OR 226 * DEFAULT_USAGE_FLAGS to usage. 227 */ 228 status_t setConsumerUsageBits(uint64_t usage); 229 230 /** 231 * detachFromContext detaches the SurfaceTexture from the calling thread's 232 * current OpenGL ES context. This context must be the same as the context 233 * that was current for previous calls to updateTexImage. 234 * 235 * Detaching a SurfaceTexture from an OpenGL ES context will result in the 236 * deletion of the OpenGL ES texture object into which the images were being 237 * streamed. After a SurfaceTexture has been detached from the OpenGL ES 238 * context calls to updateTexImage will fail returning INVALID_OPERATION 239 * until the SurfaceTexture is attached to a new OpenGL ES context using the 240 * attachToContext method. 241 */ 242 status_t detachFromContext(); 243 244 /** 245 * attachToContext attaches a SurfaceTexture that is currently in the 246 * 'detached' state to the current OpenGL ES context. A SurfaceTexture is 247 * in the 'detached' state iff detachFromContext has successfully been 248 * called and no calls to attachToContext have succeeded since the last 249 * detachFromContext call. Calls to attachToContext made on a 250 * SurfaceTexture that is not in the 'detached' state will result in an 251 * INVALID_OPERATION error. 252 * 253 * The tex argument specifies the OpenGL ES texture object name in the 254 * new context into which the image contents will be streamed. A successful 255 * call to attachToContext will result in this texture object being bound to 256 * the texture target and populated with the image contents that were 257 * current at the time of the last call to detachFromContext. 258 */ 259 status_t attachToContext(uint32_t tex); 260 261 sk_sp<SkImage> dequeueImage(SkMatrix& transformMatrix, bool* queueEmpty, 262 uirenderer::RenderState& renderState); 263 264 /** 265 * attachToView attaches a SurfaceTexture that is currently in the 266 * 'detached' state to HWUI View system. 267 */ 268 void attachToView(); 269 270 /** 271 * detachFromView detaches a SurfaceTexture from HWUI View system. 272 */ 273 void detachFromView(); 274 275 protected: 276 /** 277 * abandonLocked overrides the ConsumerBase method to clear 278 * mCurrentTextureImage in addition to the ConsumerBase behavior. 279 */ 280 virtual void abandonLocked(); 281 282 /** 283 * dumpLocked overrides the ConsumerBase method to dump SurfaceTexture- 284 * specific info in addition to the ConsumerBase behavior. 285 */ 286 virtual void dumpLocked(String8& result, const char* prefix) const override; 287 288 /** 289 * acquireBufferLocked overrides the ConsumerBase method to update the 290 * mEglSlots array in addition to the ConsumerBase behavior. 291 */ 292 virtual status_t acquireBufferLocked(BufferItem* item, nsecs_t presentWhen, 293 uint64_t maxFrameNumber = 0) override; 294 295 /** 296 * releaseBufferLocked overrides the ConsumerBase method to update the 297 * mEglSlots array in addition to the ConsumerBase. 298 */ 299 virtual status_t releaseBufferLocked(int slot, const sp<GraphicBuffer> graphicBuffer, 300 EGLDisplay display, EGLSyncKHR eglFence) override; 301 302 /** 303 * freeBufferLocked frees up the given buffer slot. If the slot has been 304 * initialized this will release the reference to the GraphicBuffer in that 305 * slot and destroy the EGLImage in that slot. Otherwise it has no effect. 306 * 307 * This method must be called with mMutex locked. 308 */ 309 virtual void freeBufferLocked(int slotIndex); 310 311 /** 312 * computeCurrentTransformMatrixLocked computes the transform matrix for the 313 * current texture. It uses mCurrentTransform and the current GraphicBuffer 314 * to compute this matrix and stores it in mCurrentTransformMatrix. 315 * mCurrentTextureImage must not be NULL. 316 */ 317 void computeCurrentTransformMatrixLocked(); 318 319 /** 320 * The default consumer usage flags that SurfaceTexture always sets on its 321 * BufferQueue instance; these will be OR:d with any additional flags passed 322 * from the SurfaceTexture user. In particular, SurfaceTexture will always 323 * consume buffers as hardware textures. 324 */ 325 static const uint64_t DEFAULT_USAGE_FLAGS = GraphicBuffer::USAGE_HW_TEXTURE; 326 327 /** 328 * mCurrentCrop is the crop rectangle that applies to the current texture. 329 * It gets set each time updateTexImage is called. 330 */ 331 Rect mCurrentCrop; 332 333 /** 334 * mCurrentTransform is the transform identifier for the current texture. It 335 * gets set each time updateTexImage is called. 336 */ 337 uint32_t mCurrentTransform; 338 339 /** 340 * mCurrentScalingMode is the scaling mode for the current texture. It gets 341 * set each time updateTexImage is called. 342 */ 343 uint32_t mCurrentScalingMode; 344 345 /** 346 * mCurrentFence is the fence received from BufferQueue in updateTexImage. 347 */ 348 sp<Fence> mCurrentFence; 349 350 /** 351 * The FenceTime wrapper around mCurrentFence. 352 */ 353 std::shared_ptr<FenceTime> mCurrentFenceTime{FenceTime::NO_FENCE}; 354 355 /** 356 * mCurrentTransformMatrix is the transform matrix for the current texture. 357 * It gets computed by computeTransformMatrix each time updateTexImage is 358 * called. 359 */ 360 float mCurrentTransformMatrix[16]; 361 362 /** 363 * mCurrentTimestamp is the timestamp for the current texture. It 364 * gets set each time updateTexImage is called. 365 */ 366 int64_t mCurrentTimestamp; 367 368 /** 369 * mCurrentDataSpace is the dataspace for the current texture. It 370 * gets set each time updateTexImage is called. 371 */ 372 android_dataspace mCurrentDataSpace; 373 374 /** 375 * mCurrentFrameNumber is the frame counter for the current texture. 376 * It gets set each time updateTexImage is called. 377 */ 378 uint64_t mCurrentFrameNumber; 379 380 uint32_t mDefaultWidth, mDefaultHeight; 381 382 /** 383 * mFilteringEnabled indicates whether the transform matrix is computed for 384 * use with bilinear filtering. It defaults to true and is changed by 385 * setFilteringEnabled(). 386 */ 387 bool mFilteringEnabled; 388 389 /** 390 * mTexName is the name of the OpenGL texture to which streamed images will 391 * be bound when updateTexImage is called. It is set at construction time 392 * and can be changed with a call to attachToContext. 393 */ 394 uint32_t mTexName; 395 396 /** 397 * mUseFenceSync indicates whether creation of the EGL_KHR_fence_sync 398 * extension should be used to prevent buffers from being dequeued before 399 * it's safe for them to be written. It gets set at construction time and 400 * never changes. 401 */ 402 const bool mUseFenceSync; 403 404 /** 405 * mTexTarget is the GL texture target with which the GL texture object is 406 * associated. It is set in the constructor and never changed. It is 407 * almost always GL_TEXTURE_EXTERNAL_OES except for one use case in Android 408 * Browser. In that case it is set to GL_TEXTURE_2D to allow 409 * glCopyTexSubImage to read from the texture. This is a hack to work 410 * around a GL driver limitation on the number of FBO attachments, which the 411 * browser's tile cache exceeds. 412 */ 413 const uint32_t mTexTarget; 414 415 /** 416 * mCurrentTexture is the buffer slot index of the buffer that is currently 417 * bound to the OpenGL texture. It is initialized to INVALID_BUFFER_SLOT, 418 * indicating that no buffer slot is currently bound to the texture. Note, 419 * however, that a value of INVALID_BUFFER_SLOT does not necessarily mean 420 * that no buffer is bound to the texture. A call to setBufferCount will 421 * reset mCurrentTexture to INVALID_BUFFER_SLOT. 422 */ 423 int mCurrentTexture; 424 425 enum class OpMode { detached, attachedToView, attachedToGL }; 426 /** 427 * mOpMode indicates whether the SurfaceTexture is currently attached to 428 * an OpenGL ES context or the HWUI view system. For legacy reasons, this is initialized to, 429 * "attachedToGL" indicating that the SurfaceTexture is considered to be attached to 430 * whatever GL context is current at the time of the first updateTexImage call. 431 * It is set to "detached" by detachFromContext, and then set to "attachedToGL" again by 432 * attachToContext. 433 * attachToView/detachFromView are used to attach/detach from HWUI view system. 434 */ 435 OpMode mOpMode; 436 437 /** 438 * mEGLConsumer has SurfaceTexture logic used when attached to GL context. 439 */ 440 EGLConsumer mEGLConsumer; 441 442 /** 443 * mImageConsumer has SurfaceTexture logic used when attached to HWUI view system. 444 */ 445 ImageConsumer mImageConsumer; 446 447 friend class ImageConsumer; 448 friend class EGLConsumer; 449 }; 450 451 // ---------------------------------------------------------------------------- 452 } // namespace android 453