1 /* 2 * Copyright (C) 2010 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.graphics; 18 19 import android.annotation.Nullable; 20 import android.compat.annotation.UnsupportedAppUsage; 21 import android.os.Build; 22 import android.os.Handler; 23 import android.os.Looper; 24 import android.os.Message; 25 import android.view.Surface; 26 27 import java.lang.ref.WeakReference; 28 29 /** 30 * Captures frames from an image stream as an OpenGL ES texture. 31 * 32 * <p>The image stream may come from either camera preview or video decode. A 33 * {@link android.view.Surface} created from a SurfaceTexture can be used as an output 34 * destination for the {@link android.hardware.camera2}, {@link android.media.MediaCodec}, 35 * {@link android.media.MediaPlayer}, and {@link android.renderscript.Allocation} APIs. 36 * When {@link #updateTexImage} is called, the contents of the texture object specified 37 * when the SurfaceTexture was created are updated to contain the most recent image from the image 38 * stream. This may cause some frames of the stream to be skipped. 39 * 40 * <p>A SurfaceTexture may also be used in place of a SurfaceHolder when specifying the output 41 * destination of the older {@link android.hardware.Camera} API. Doing so will cause all the 42 * frames from the image stream to be sent to the SurfaceTexture object rather than to the device's 43 * display. 44 * 45 * <p>When sampling from the texture one should first transform the texture coordinates using the 46 * matrix queried via {@link #getTransformMatrix(float[])}. The transform matrix may change each 47 * time {@link #updateTexImage} is called, so it should be re-queried each time the texture image 48 * is updated. 49 * This matrix transforms traditional 2D OpenGL ES texture coordinate column vectors of the form (s, 50 * t, 0, 1) where s and t are on the inclusive interval [0, 1] to the proper sampling location in 51 * the streamed texture. This transform compensates for any properties of the image stream source 52 * that cause it to appear different from a traditional OpenGL ES texture. For example, sampling 53 * from the bottom left corner of the image can be accomplished by transforming the column vector 54 * (0, 0, 0, 1) using the queried matrix, while sampling from the top right corner of the image can 55 * be done by transforming (1, 1, 0, 1). 56 * 57 * <p>The texture object uses the GL_TEXTURE_EXTERNAL_OES texture target, which is defined by the 58 * <a href="http://www.khronos.org/registry/gles/extensions/OES/OES_EGL_image_external.txt"> 59 * GL_OES_EGL_image_external</a> OpenGL ES extension. This limits how the texture may be used. 60 * Each time the texture is bound it must be bound to the GL_TEXTURE_EXTERNAL_OES target rather than 61 * the GL_TEXTURE_2D target. Additionally, any OpenGL ES 2.0 shader that samples from the texture 62 * must declare its use of this extension using, for example, an "#extension 63 * GL_OES_EGL_image_external : require" directive. Such shaders must also access the texture using 64 * the samplerExternalOES GLSL sampler type. 65 * 66 * <p>SurfaceTexture objects may be created on any thread. {@link #updateTexImage} may only be 67 * called on the thread with the OpenGL ES context that contains the texture object. The 68 * frame-available callback is called on an arbitrary thread, so unless special care is taken {@link 69 * #updateTexImage} should not be called directly from the callback. 70 */ 71 public class SurfaceTexture { 72 private final Looper mCreatorLooper; 73 @UnsupportedAppUsage(maxTargetSdk = Build.VERSION_CODES.R, trackingBug = 170729553) 74 private Handler mOnFrameAvailableHandler; 75 76 /** 77 * These fields are used by native code, do not access or modify. 78 */ 79 @UnsupportedAppUsage(trackingBug = 176388660) 80 private long mSurfaceTexture; 81 @UnsupportedAppUsage(trackingBug = 176388660) 82 private long mProducer; 83 @UnsupportedAppUsage(maxTargetSdk = Build.VERSION_CODES.R, trackingBug = 170729553) 84 private long mFrameAvailableListener; 85 86 private boolean mIsSingleBuffered; 87 88 /** 89 * Callback interface for being notified that a new stream frame is available. 90 */ 91 public interface OnFrameAvailableListener { onFrameAvailable(SurfaceTexture surfaceTexture)92 void onFrameAvailable(SurfaceTexture surfaceTexture); 93 } 94 95 /** 96 * Exception thrown when a SurfaceTexture couldn't be created or resized. 97 * 98 * @deprecated No longer thrown. {@link android.view.Surface.OutOfResourcesException} 99 * is used instead. 100 */ 101 @SuppressWarnings("serial") 102 @Deprecated 103 public static class OutOfResourcesException extends Exception { OutOfResourcesException()104 public OutOfResourcesException() { 105 } OutOfResourcesException(String name)106 public OutOfResourcesException(String name) { 107 super(name); 108 } 109 } 110 111 /** 112 * Construct a new SurfaceTexture to stream images to a given OpenGL texture. 113 * 114 * @param texName the OpenGL texture object name (e.g. generated via glGenTextures) 115 * 116 * @throws android.view.Surface.OutOfResourcesException If the SurfaceTexture cannot be created. 117 */ SurfaceTexture(int texName)118 public SurfaceTexture(int texName) { 119 this(texName, false); 120 } 121 122 /** 123 * Construct a new SurfaceTexture to stream images to a given OpenGL texture. 124 * <p> 125 * In single buffered mode the application is responsible for serializing access to the image 126 * content buffer. Each time the image content is to be updated, the 127 * {@link #releaseTexImage()} method must be called before the image content producer takes 128 * ownership of the buffer. For example, when producing image content with the NDK 129 * ANativeWindow_lock and ANativeWindow_unlockAndPost functions, {@link #releaseTexImage()} 130 * must be called before each ANativeWindow_lock, or that call will fail. When producing 131 * image content with OpenGL ES, {@link #releaseTexImage()} must be called before the first 132 * OpenGL ES function call each frame. 133 * 134 * @param texName the OpenGL texture object name (e.g. generated via glGenTextures) 135 * @param singleBufferMode whether the SurfaceTexture will be in single buffered mode. 136 * 137 * @throws android.view.Surface.OutOfResourcesException If the SurfaceTexture cannot be created. 138 */ SurfaceTexture(int texName, boolean singleBufferMode)139 public SurfaceTexture(int texName, boolean singleBufferMode) { 140 mCreatorLooper = Looper.myLooper(); 141 mIsSingleBuffered = singleBufferMode; 142 nativeInit(false, texName, singleBufferMode, new WeakReference<SurfaceTexture>(this)); 143 } 144 145 /** 146 * Construct a new SurfaceTexture to stream images to a given OpenGL texture. 147 * <p> 148 * In single buffered mode the application is responsible for serializing access to the image 149 * content buffer. Each time the image content is to be updated, the 150 * {@link #releaseTexImage()} method must be called before the image content producer takes 151 * ownership of the buffer. For example, when producing image content with the NDK 152 * ANativeWindow_lock and ANativeWindow_unlockAndPost functions, {@link #releaseTexImage()} 153 * must be called before each ANativeWindow_lock, or that call will fail. When producing 154 * image content with OpenGL ES, {@link #releaseTexImage()} must be called before the first 155 * OpenGL ES function call each frame. 156 * <p> 157 * Unlike {@link #SurfaceTexture(int, boolean)}, which takes an OpenGL texture object name, 158 * this constructor creates the SurfaceTexture in detached mode. A texture name must be passed 159 * in using {@link #attachToGLContext} before calling {@link #releaseTexImage()} and producing 160 * image content using OpenGL ES. 161 * 162 * @param singleBufferMode whether the SurfaceTexture will be in single buffered mode. 163 * 164 * @throws android.view.Surface.OutOfResourcesException If the SurfaceTexture cannot be created. 165 */ SurfaceTexture(boolean singleBufferMode)166 public SurfaceTexture(boolean singleBufferMode) { 167 mCreatorLooper = Looper.myLooper(); 168 mIsSingleBuffered = singleBufferMode; 169 nativeInit(true, 0, singleBufferMode, new WeakReference<SurfaceTexture>(this)); 170 } 171 172 /** 173 * Register a callback to be invoked when a new image frame becomes available to the 174 * SurfaceTexture. 175 * <p> 176 * The callback may be called on an arbitrary thread, so it is not 177 * safe to call {@link #updateTexImage} without first binding the OpenGL ES context to the 178 * thread invoking the callback. 179 * </p> 180 * 181 * @param listener The listener to use, or null to remove the listener. 182 */ setOnFrameAvailableListener(@ullable OnFrameAvailableListener listener)183 public void setOnFrameAvailableListener(@Nullable OnFrameAvailableListener listener) { 184 setOnFrameAvailableListener(listener, null); 185 } 186 187 /** 188 * Register a callback to be invoked when a new image frame becomes available to the 189 * SurfaceTexture. 190 * <p> 191 * If a handler is specified, the callback will be invoked on that handler's thread. 192 * If no handler is specified, then the callback may be called on an arbitrary thread, 193 * so it is not safe to call {@link #updateTexImage} without first binding the OpenGL ES 194 * context to the thread invoking the callback. 195 * </p> 196 * 197 * @param listener The listener to use, or null to remove the listener. 198 * @param handler The handler on which the listener should be invoked, or null 199 * to use an arbitrary thread. 200 */ setOnFrameAvailableListener(@ullable final OnFrameAvailableListener listener, @Nullable Handler handler)201 public void setOnFrameAvailableListener(@Nullable final OnFrameAvailableListener listener, 202 @Nullable Handler handler) { 203 if (listener != null) { 204 // Although we claim the thread is arbitrary, earlier implementation would 205 // prefer to send the callback on the creating looper or the main looper 206 // so we preserve this behavior here. 207 Looper looper = handler != null ? handler.getLooper() : 208 mCreatorLooper != null ? mCreatorLooper : Looper.getMainLooper(); 209 mOnFrameAvailableHandler = new Handler(looper, null, true /*async*/) { 210 @Override 211 public void handleMessage(Message msg) { 212 listener.onFrameAvailable(SurfaceTexture.this); 213 } 214 }; 215 } else { 216 mOnFrameAvailableHandler = null; 217 } 218 } 219 220 /** 221 * Set the default size of the image buffers. The image producer may override the buffer size, 222 * in which case the producer-set buffer size will be used, not the default size set by this 223 * method. Both video and camera based image producers do override the size. This method may 224 * be used to set the image size when producing images with {@link android.graphics.Canvas} (via 225 * {@link android.view.Surface#lockCanvas}), or OpenGL ES (via an EGLSurface). 226 * <p> 227 * The new default buffer size will take effect the next time the image producer requests a 228 * buffer to fill. For {@link android.graphics.Canvas} this will be the next time {@link 229 * android.view.Surface#lockCanvas} is called. For OpenGL ES, the EGLSurface should be 230 * destroyed (via eglDestroySurface), made not-current (via eglMakeCurrent), and then recreated 231 * (via {@code eglCreateWindowSurface}) to ensure that the new default size has taken effect. 232 * <p> 233 * The width and height parameters must be no greater than the minimum of 234 * {@code GL_MAX_VIEWPORT_DIMS} and {@code GL_MAX_TEXTURE_SIZE} (see 235 * {@link javax.microedition.khronos.opengles.GL10#glGetIntegerv glGetIntegerv}). 236 * An error due to invalid dimensions might not be reported until 237 * updateTexImage() is called. 238 */ setDefaultBufferSize(int width, int height)239 public void setDefaultBufferSize(int width, int height) { 240 nativeSetDefaultBufferSize(width, height); 241 } 242 243 /** 244 * Update the texture image to the most recent frame from the image stream. This may only be 245 * called while the OpenGL ES context that owns the texture is current on the calling thread. 246 * It will implicitly bind its texture to the {@code GL_TEXTURE_EXTERNAL_OES} texture target. 247 */ updateTexImage()248 public void updateTexImage() { 249 nativeUpdateTexImage(); 250 } 251 252 /** 253 * Releases the the texture content. This is needed in single buffered mode to allow the image 254 * content producer to take ownership of the image buffer. 255 * <p> 256 * For more information see {@link #SurfaceTexture(int, boolean)}. 257 */ releaseTexImage()258 public void releaseTexImage() { 259 nativeReleaseTexImage(); 260 } 261 262 /** 263 * Detach the SurfaceTexture from the OpenGL ES context that owns the OpenGL ES texture object. 264 * This call must be made with the OpenGL ES context current on the calling thread. The OpenGL 265 * ES texture object will be deleted as a result of this call. After calling this method all 266 * calls to {@link #updateTexImage} will throw an {@link java.lang.IllegalStateException} until 267 * a successful call to {@link #attachToGLContext} is made. 268 * <p> 269 * This can be used to access the SurfaceTexture image contents from multiple OpenGL ES 270 * contexts. Note, however, that the image contents are only accessible from one OpenGL ES 271 * context at a time. 272 */ detachFromGLContext()273 public void detachFromGLContext() { 274 int err = nativeDetachFromGLContext(); 275 if (err != 0) { 276 throw new RuntimeException("Error during detachFromGLContext (see logcat for details)"); 277 } 278 } 279 280 /** 281 * Attach the SurfaceTexture to the OpenGL ES context that is current on the calling thread. A 282 * new OpenGL ES texture object is created and populated with the SurfaceTexture image frame 283 * that was current at the time of the last call to {@link #detachFromGLContext}. This new 284 * texture is bound to the {@code GL_TEXTURE_EXTERNAL_OES} texture target. 285 * <p> 286 * This can be used to access the SurfaceTexture image contents from multiple OpenGL ES 287 * contexts. Note, however, that the image contents are only accessible from one OpenGL ES 288 * context at a time. 289 * 290 * @param texName The name of the OpenGL ES texture that will be created. This texture name 291 * must be unusued in the OpenGL ES context that is current on the calling thread. 292 */ attachToGLContext(int texName)293 public void attachToGLContext(int texName) { 294 int err = nativeAttachToGLContext(texName); 295 if (err != 0) { 296 throw new RuntimeException("Error during attachToGLContext (see logcat for details)"); 297 } 298 } 299 300 /** 301 * Retrieve the 4x4 texture coordinate transform matrix associated with the texture image set by 302 * the most recent call to {@link #updateTexImage}. 303 * <p> 304 * This transform matrix maps 2D homogeneous texture coordinates of the form (s, t, 0, 1) with s 305 * and t in the inclusive range [0, 1] to the texture coordinate that should be used to sample 306 * that location from the texture. Sampling the texture outside of the range of this transform 307 * is undefined. 308 * <p> 309 * The matrix is stored in column-major order so that it may be passed directly to OpenGL ES via 310 * the {@code glLoadMatrixf} or {@code glUniformMatrix4fv} functions. 311 * <p> 312 * If the underlying buffer has a crop associated with it, the transformation will also include 313 * a slight scale to cut off a 1-texel border around the edge of the crop. This ensures that 314 * when the texture is bilinear sampled that no texels outside of the buffer's valid region 315 * are accessed by the GPU, avoiding any sampling artifacts when scaling. 316 * 317 * @param mtx the array into which the 4x4 matrix will be stored. The array must have exactly 318 * 16 elements. 319 */ getTransformMatrix(float[] mtx)320 public void getTransformMatrix(float[] mtx) { 321 // Note we intentionally don't check mtx for null, so this will result in a 322 // NullPointerException. But it's safe because it happens before the call to native. 323 if (mtx.length != 16) { 324 throw new IllegalArgumentException(); 325 } 326 nativeGetTransformMatrix(mtx); 327 } 328 329 /** 330 * Retrieve the timestamp associated with the texture image set by the most recent call to 331 * {@link #updateTexImage}. 332 * 333 * <p>This timestamp is in nanoseconds, and is normally monotonically increasing. The timestamp 334 * should be unaffected by time-of-day adjustments. The specific meaning and zero point of the 335 * timestamp depends on the source providing images to the SurfaceTexture. Unless otherwise 336 * specified by the image source, timestamps cannot generally be compared across SurfaceTexture 337 * instances, or across multiple program invocations. It is mostly useful for determining time 338 * offsets between subsequent frames.</p> 339 * 340 * <p>For camera sources, timestamps should be strictly monotonic. Timestamps from MediaPlayer 341 * sources may be reset when the playback position is set. For EGL and Vulkan producers, the 342 * timestamp is the desired present time set with the {@code EGL_ANDROID_presentation_time} or 343 * {@code VK_GOOGLE_display_timing} extensions.</p> 344 */ 345 getTimestamp()346 public long getTimestamp() { 347 return nativeGetTimestamp(); 348 } 349 350 /** 351 * {@code release()} frees all the buffers and puts the SurfaceTexture into the 352 * 'abandoned' state. Once put in this state the SurfaceTexture can never 353 * leave it. When in the 'abandoned' state, all methods of the 354 * {@code IGraphicBufferProducer} interface will fail with the {@code NO_INIT} 355 * error. 356 * <p> 357 * Note that while calling this method causes all the buffers to be freed 358 * from the perspective of the the SurfaceTexture, if there are additional 359 * references on the buffers (e.g. if a buffer is referenced by a client or 360 * by OpenGL ES as a texture) then those buffer will remain allocated. 361 * <p> 362 * Always call this method when you are done with SurfaceTexture. Failing 363 * to do so may delay resource deallocation for a significant amount of 364 * time. 365 * 366 * @see #isReleased() 367 */ release()368 public void release() { 369 nativeRelease(); 370 } 371 372 /** 373 * Returns {@code true} if the SurfaceTexture was released. 374 * 375 * @see #release() 376 */ isReleased()377 public boolean isReleased() { 378 return nativeIsReleased(); 379 } 380 381 @Override finalize()382 protected void finalize() throws Throwable { 383 try { 384 nativeFinalize(); 385 } finally { 386 super.finalize(); 387 } 388 } 389 390 /** 391 * This method is invoked from native code only. 392 */ 393 @SuppressWarnings({"UnusedDeclaration"}) 394 @UnsupportedAppUsage(maxTargetSdk = Build.VERSION_CODES.R, trackingBug = 170729553) postEventFromNative(WeakReference<SurfaceTexture> weakSelf)395 private static void postEventFromNative(WeakReference<SurfaceTexture> weakSelf) { 396 SurfaceTexture st = weakSelf.get(); 397 if (st != null) { 398 Handler handler = st.mOnFrameAvailableHandler; 399 if (handler != null) { 400 handler.sendEmptyMessage(0); 401 } 402 } 403 } 404 405 /** 406 * Returns {@code true} if the SurfaceTexture is single-buffered. 407 * @hide 408 */ isSingleBuffered()409 public boolean isSingleBuffered() { 410 return mIsSingleBuffered; 411 } 412 nativeInit(boolean isDetached, int texName, boolean singleBufferMode, WeakReference<SurfaceTexture> weakSelf)413 private native void nativeInit(boolean isDetached, int texName, 414 boolean singleBufferMode, WeakReference<SurfaceTexture> weakSelf) 415 throws Surface.OutOfResourcesException; nativeFinalize()416 private native void nativeFinalize(); nativeGetTransformMatrix(float[] mtx)417 private native void nativeGetTransformMatrix(float[] mtx); nativeGetTimestamp()418 private native long nativeGetTimestamp(); nativeSetDefaultBufferSize(int width, int height)419 private native void nativeSetDefaultBufferSize(int width, int height); nativeUpdateTexImage()420 private native void nativeUpdateTexImage(); nativeReleaseTexImage()421 private native void nativeReleaseTexImage(); 422 @UnsupportedAppUsage nativeDetachFromGLContext()423 private native int nativeDetachFromGLContext(); nativeAttachToGLContext(int texName)424 private native int nativeAttachToGLContext(int texName); nativeRelease()425 private native void nativeRelease(); nativeIsReleased()426 private native boolean nativeIsReleased(); 427 } 428