1 /* 2 * Copyright (C) 2022 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.cts.verifier.camera.its; 18 19 import android.content.Context; 20 import android.graphics.Bitmap; 21 import android.graphics.SurfaceTexture; 22 import android.media.MediaCodec; 23 import android.media.MediaCodecList; 24 import android.media.MediaFormat; 25 import android.media.MediaMuxer; 26 import android.media.MediaRecorder; 27 import android.opengl.EGL14; 28 import android.opengl.EGLConfig; 29 import android.opengl.EGLContext; 30 import android.opengl.EGLDisplay; 31 import android.opengl.EGLExt; 32 import android.opengl.EGLSurface; 33 import android.opengl.GLES11Ext; 34 import android.opengl.GLES20; 35 import android.os.ConditionVariable; 36 import android.os.Handler; 37 import android.util.Size; 38 import android.view.Surface; 39 40 import java.io.IOException; 41 import java.io.OutputStream; 42 import java.nio.ByteBuffer; 43 import java.nio.ByteOrder; 44 import java.nio.FloatBuffer; 45 import java.nio.IntBuffer; 46 import java.util.ArrayList; 47 import java.util.List; 48 49 /** 50 * Class to record a preview like stream. It sets up a SurfaceTexture that the camera can write to, 51 * and copies over the camera frames to a MediaRecorder or MediaCodec surface. 52 */ 53 class PreviewRecorder implements AutoCloseable { 54 private static final String TAG = PreviewRecorder.class.getSimpleName(); 55 56 // Frame capture timeout duration in milliseconds. 57 private static final int FRAME_CAPTURE_TIMEOUT_MS = 2000; // 2 seconds 58 59 private static final int GREEN_PAINT = 1; 60 private static final int NO_PAINT = 0; 61 62 // Simple Vertex Shader that rotates the texture before passing it to Fragment shader. 63 private static final String VERTEX_SHADER = String.join( 64 "\n", 65 "", 66 "attribute vec4 vPosition;", 67 "uniform mat4 texMatrix;", // provided by SurfaceTexture 68 "uniform mat2 texRotMatrix;", // optional rotation matrix, from Sensor Orientation 69 "varying vec2 vTextureCoord;", 70 "void main() {", 71 " gl_Position = vPosition;", 72 " vec2 texCoords = texRotMatrix * vPosition.xy;", // rotate the coordinates before 73 // applying transform from 74 // SurfaceTexture 75 " texCoords = (texCoords + vec2(1.0, 1.0)) / 2.0;", // Texture coordinates 76 // have range [0, 1] 77 " vTextureCoord = (texMatrix * vec4(texCoords, 0.0, 1.0)).xy;", 78 "}", 79 "" 80 ); 81 82 // Simple Fragment Shader that samples the passed texture at a given coordinate. 83 private static final String FRAGMENT_SHADER = String.join( 84 "\n", 85 "", 86 "#extension GL_OES_EGL_image_external : require", 87 "precision mediump float;", 88 "varying vec2 vTextureCoord;", 89 "uniform samplerExternalOES sTexture;", // implicitly populated by SurfaceTexture 90 "uniform int paintIt;", 91 "void main() {", 92 " if (paintIt == 1) {", 93 " gl_FragColor = vec4(0.0, 1.0, 0.0, 1.0);", // green frame 94 " } else {", 95 " gl_FragColor = texture2D(sTexture, vTextureCoord);", // copy frame 96 " }", 97 "}", 98 "" 99 ); 100 101 // column-major vertices list of a rectangle that fills the entire screen 102 private static final float[] FULLSCREEN_VERTICES = { 103 -1, -1, // bottom left 104 1, -1, // bottom right 105 -1, 1, // top left 106 1, 1, // top right 107 }; 108 109 110 private boolean mRecordingStarted = false; // tracks if the MediaRecorder/MediaCodec instance 111 // was already used to record a video. 112 113 // Lock to protect reads/writes to the various Surfaces below. 114 private final Object mRecordLock = new Object(); 115 // Tracks if the mMediaRecorder/mMediaCodec is currently recording. Protected by mRecordLock. 116 private volatile boolean mIsRecording = false; 117 private boolean mIsPaintGreen = false; 118 119 private final Size mPreviewSize; 120 private final int mMaxFps; 121 private final Handler mHandler; 122 123 private Surface mRecordSurface; // MediaRecorder/MediaCodec source. EGL writes to this surface 124 125 private MediaRecorder mMediaRecorder; 126 127 private MediaCodec mMediaCodec; 128 private MediaMuxer mMediaMuxer; 129 private Object mMediaCodecCondition; 130 131 private SurfaceTexture mCameraTexture; // Handles writing frames from camera as texture to 132 // the GLSL program. 133 private Surface mCameraSurface; // Surface corresponding to mCameraTexture that the 134 // Camera HAL writes to 135 136 private int mGLShaderProgram = 0; 137 private EGLDisplay mEGLDisplay = EGL14.EGL_NO_DISPLAY; 138 private EGLContext mEGLContext = EGL14.EGL_NO_CONTEXT; 139 private EGLSurface mEGLRecorderSurface; // EGL Surface corresponding to mRecordSurface 140 141 private int mVPositionLoc; 142 private int mTexMatrixLoc; 143 private int mTexRotMatrixLoc; 144 private int mPaintItLoc; 145 146 147 private final float[] mTexRotMatrix; // length = 4 148 private final float[] mTransformMatrix = new float[16]; 149 150 // An offset applied to convert from camera timestamp to codec timestamp, due to potentially 151 // different time bases (elapsedRealtime vs uptime). 152 private long mEncoderTimestampOffset; 153 private List<Long> mFrameTimeStamps = new ArrayList(); 154 /** 155 * Initializes MediaRecorder/MediaCodec and EGL context. The result of recorded video will 156 * be stored in {@code outputFile}. 157 */ PreviewRecorder(int cameraId, Size previewSize, int maxFps, int sensorOrientation, String outputFile, Handler handler, boolean hlg10Enabled, long encoderTimestampOffset, Context context)158 PreviewRecorder(int cameraId, Size previewSize, int maxFps, int sensorOrientation, 159 String outputFile, Handler handler, boolean hlg10Enabled, long encoderTimestampOffset, 160 Context context) throws ItsException { 161 // Ensure that we can record the given size 162 int maxSupportedResolution = ItsUtils.RESOLUTION_TO_CAMCORDER_PROFILE 163 .stream() 164 .map(p -> p.first) 165 .max(Integer::compareTo) 166 .orElse(0); 167 int currentResolution = previewSize.getHeight() * previewSize.getWidth(); 168 if (currentResolution > maxSupportedResolution) { 169 throw new ItsException("Requested preview size is greater than maximum " 170 + "supported preview size."); 171 } 172 173 mHandler = handler; 174 mPreviewSize = previewSize; 175 mMaxFps = maxFps; 176 // rotate the texture as needed by the sensor orientation 177 mTexRotMatrix = getRotationMatrix(sensorOrientation); 178 mEncoderTimestampOffset = encoderTimestampOffset; 179 180 ConditionVariable cv = new ConditionVariable(); 181 cv.close(); 182 183 // Init fields in the passed handler to bind egl context to the handler thread. 184 mHandler.post(() -> { 185 try { 186 initPreviewRecorder(cameraId, outputFile, hlg10Enabled, context); 187 } catch (ItsException e) { 188 Logt.e(TAG, "Failed to init preview recorder", e); 189 throw new ItsRuntimeException("Failed to init preview recorder", e); 190 } finally { 191 cv.open(); 192 } 193 }); 194 // Wait for up to 1s for handler to finish initializing 195 if (!cv.block(1000)) { 196 throw new ItsException("Preview recorder did not initialize in 1000ms"); 197 } 198 199 } 200 initPreviewRecorder(int cameraId, String outputFile, boolean hlg10Enabled, Context context)201 private void initPreviewRecorder(int cameraId, String outputFile, 202 boolean hlg10Enabled, Context context) throws ItsException { 203 204 // order of initialization is important 205 if (hlg10Enabled) { 206 Logt.i(TAG, "HLG10 Enabled, using MediaCodec"); 207 setupMediaCodec(cameraId, outputFile, context); 208 } else { 209 Logt.i(TAG, "HLG10 Disabled, using MediaRecorder"); 210 setupMediaRecorder(cameraId, outputFile, context); 211 } 212 213 initEGL(hlg10Enabled); // requires recording surfaces to be set up 214 compileShaders(); // requires EGL context to be set up 215 setupCameraTexture(); // requires EGL context to be set up 216 217 218 mCameraTexture.setOnFrameAvailableListener(surfaceTexture -> { 219 // Synchronized on mRecordLock to ensure that all surface are valid while encoding 220 // frames. All surfaces should be valid for as long as mIsRecording is true. 221 synchronized (mRecordLock) { 222 if (surfaceTexture.isReleased()) { 223 return; // surface texture already cleaned up, do nothing. 224 } 225 226 // Bind EGL context to the current thread (just in case the 227 // executing thread changes) 228 EGL14.eglMakeCurrent(mEGLDisplay, mEGLRecorderSurface, 229 mEGLRecorderSurface, mEGLContext); 230 surfaceTexture.updateTexImage(); // update texture to the latest frame 231 232 // Only update the frame if the recorder is currently recording. 233 if (!mIsRecording) { 234 return; 235 } 236 try { 237 // Set EGL presentation time to camera timestamp 238 long timestamp = surfaceTexture.getTimestamp(); 239 EGLExt.eglPresentationTimeANDROID( 240 mEGLDisplay, mEGLRecorderSurface, 241 timestamp + mEncoderTimestampOffset); 242 copyFrameToRecordSurface(); 243 // Capture results are not collected for padded green frames 244 if (mIsPaintGreen) { 245 Logt.v(TAG, "Recorded frame# " + mFrameTimeStamps.size() 246 + " timestamp = " + timestamp 247 + " with color. mIsPaintGreen = " + mIsPaintGreen); 248 } else { 249 mFrameTimeStamps.add(timestamp); 250 Logt.v(TAG, "Recorded frame# " + mFrameTimeStamps.size() 251 + " timestamp = " + timestamp); 252 } 253 } catch (ItsException e) { 254 Logt.e(TAG, "Failed to copy texture to recorder.", e); 255 throw new ItsRuntimeException("Failed to copy texture to recorder.", e); 256 } 257 } 258 }, mHandler); 259 } 260 setupMediaRecorder(int cameraId, String outputFile, Context context)261 private void setupMediaRecorder(int cameraId, String outputFile, Context context) 262 throws ItsException { 263 mRecordSurface = MediaCodec.createPersistentInputSurface(); 264 265 mMediaRecorder = new MediaRecorder(context); 266 mMediaRecorder.setAudioSource(MediaRecorder.AudioSource.DEFAULT); 267 mMediaRecorder.setVideoSource(MediaRecorder.VideoSource.SURFACE); 268 mMediaRecorder.setOutputFormat(MediaRecorder.OutputFormat.DEFAULT); 269 270 mMediaRecorder.setVideoSize(mPreviewSize.getWidth(), mPreviewSize.getHeight()); 271 mMediaRecorder.setVideoEncoder(MediaRecorder.VideoEncoder.DEFAULT); 272 mMediaRecorder.setAudioEncoder(MediaRecorder.AudioEncoder.DEFAULT); 273 mMediaRecorder.setVideoEncodingBitRate( 274 ItsUtils.calculateBitrate(cameraId, mPreviewSize, mMaxFps)); 275 mMediaRecorder.setInputSurface(mRecordSurface); 276 mMediaRecorder.setVideoFrameRate(mMaxFps); 277 mMediaRecorder.setOutputFile(outputFile); 278 279 try { 280 mMediaRecorder.prepare(); 281 } catch (IOException e) { 282 throw new ItsException("Error preparing MediaRecorder", e); 283 } 284 } 285 setupMediaCodec(int cameraId, String outputFilePath, Context context)286 private void setupMediaCodec(int cameraId, String outputFilePath, Context context) 287 throws ItsException { 288 MediaCodecList list = new MediaCodecList(MediaCodecList.REGULAR_CODECS); 289 int videoBitRate = ItsUtils.calculateBitrate(cameraId, mPreviewSize, mMaxFps); 290 MediaFormat format = ItsUtils.initializeHLG10Format(mPreviewSize, videoBitRate, mMaxFps); 291 String codecName = list.findEncoderForFormat(format); 292 assert (codecName != null); 293 294 try { 295 mMediaMuxer = new MediaMuxer(outputFilePath, 296 MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4); 297 } catch (IOException e) { 298 throw new ItsException("Error preparing the MediaMuxer."); 299 } 300 301 try { 302 mMediaCodec = MediaCodec.createByCodecName(codecName); 303 } catch (IOException e) { 304 throw new ItsException("Error preparing the MediaCodec."); 305 } 306 307 mMediaCodec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); 308 mMediaCodecCondition = new Object(); 309 mMediaCodec.setCallback( 310 new ItsUtils.MediaCodecListener(mMediaMuxer, mMediaCodecCondition), mHandler); 311 312 mRecordSurface = mMediaCodec.createInputSurface(); 313 assert (mRecordSurface != null); 314 } 315 initEGL(boolean hlg10Enabled)316 private void initEGL(boolean hlg10Enabled) throws ItsException { 317 // set up EGL Display 318 mEGLDisplay = EGL14.eglGetDisplay(EGL14.EGL_DEFAULT_DISPLAY); 319 if (mEGLDisplay == EGL14.EGL_NO_DISPLAY) { 320 throw new ItsException("Unable to get EGL display"); 321 } 322 323 int[] version = {0, 0}; 324 if (!EGL14.eglInitialize(mEGLDisplay, version, /* majorOffset= */0, 325 version, /* minorOffset= */1)) { 326 mEGLDisplay = null; 327 throw new ItsException("unable to initialize EGL14"); 328 } 329 330 int colorBits = 8; 331 int alphaBits = 8; 332 if (hlg10Enabled) { 333 colorBits = 10; 334 alphaBits = 2; 335 } 336 int[] configAttribList = { 337 EGL14.EGL_RENDERABLE_TYPE, EGL14.EGL_OPENGL_ES2_BIT, 338 EGL14.EGL_RED_SIZE, colorBits, 339 EGL14.EGL_GREEN_SIZE, colorBits, 340 EGL14.EGL_BLUE_SIZE, colorBits, 341 EGL14.EGL_ALPHA_SIZE, alphaBits, 342 EGL14.EGL_DEPTH_SIZE, 0, 343 EGL14.EGL_STENCIL_SIZE, 0, 344 EGL14.EGL_NONE 345 }; 346 347 // set up EGL Config 348 EGLConfig[] configs = new EGLConfig[1]; 349 int[] numConfigs = {1}; 350 EGL14.eglChooseConfig(mEGLDisplay, configAttribList, 0, configs, 351 0, configs.length, numConfigs, 0); 352 if (configs[0] == null) { 353 throw new ItsException("Unable to initialize EGL config"); 354 } 355 356 EGLConfig EGLConfig = configs[0]; 357 358 int[] contextAttribList = { EGL14.EGL_CONTEXT_CLIENT_VERSION, 2, EGL14.EGL_NONE }; 359 360 mEGLContext = EGL14.eglCreateContext(mEGLDisplay, EGLConfig, EGL14.EGL_NO_CONTEXT, 361 contextAttribList, 0); 362 if (mEGLContext == EGL14.EGL_NO_CONTEXT) { 363 throw new ItsException("Failed to create EGL context"); 364 } 365 366 int[] clientVersion = {0}; 367 EGL14.eglQueryContext(mEGLDisplay, mEGLContext, EGL14.EGL_CONTEXT_CLIENT_VERSION, 368 clientVersion, /* offset= */0); 369 Logt.i(TAG, "EGLContext created, client version " + clientVersion[0]); 370 371 // Create EGL Surface to write to the recording Surface. 372 int[] surfaceAttribs = {EGL14.EGL_NONE}; 373 mEGLRecorderSurface = EGL14.eglCreateWindowSurface(mEGLDisplay, EGLConfig, mRecordSurface, 374 surfaceAttribs, /* offset= */0); 375 if (mEGLRecorderSurface == EGL14.EGL_NO_SURFACE) { 376 throw new ItsException("Failed to create EGL recorder surface"); 377 } 378 379 // Bind EGL context to the current (handler) thread. 380 EGL14.eglMakeCurrent(mEGLDisplay, mEGLRecorderSurface, mEGLRecorderSurface, mEGLContext); 381 } 382 setupCameraTexture()383 private void setupCameraTexture() throws ItsException { 384 mCameraTexture = new SurfaceTexture(createTexture()); 385 mCameraTexture.setDefaultBufferSize(mPreviewSize.getWidth(), mPreviewSize.getHeight()); 386 mCameraSurface = new Surface(mCameraTexture); 387 } 388 389 /** 390 * Compiles the vertex and fragment shader into a shader program, and sets up the location 391 * fields that will be written to later. 392 */ compileShaders()393 private void compileShaders() throws ItsException { 394 int vertexShader = createShader(GLES20.GL_VERTEX_SHADER, VERTEX_SHADER); 395 int fragmentShader = createShader(GLES20.GL_FRAGMENT_SHADER, FRAGMENT_SHADER); 396 397 mGLShaderProgram = GLES20.glCreateProgram(); 398 GLES20.glAttachShader(mGLShaderProgram, vertexShader); 399 GLES20.glAttachShader(mGLShaderProgram, fragmentShader); 400 GLES20.glLinkProgram(mGLShaderProgram); 401 402 int[] linkStatus = {0}; 403 GLES20.glGetProgramiv(mGLShaderProgram, GLES20.GL_LINK_STATUS, linkStatus, 0); 404 if (linkStatus[0] == 0) { 405 String msg = "Could not link program: " + GLES20.glGetProgramInfoLog(mGLShaderProgram); 406 GLES20.glDeleteProgram(mGLShaderProgram); 407 throw new ItsException(msg); 408 } 409 410 mVPositionLoc = GLES20.glGetAttribLocation(mGLShaderProgram, "vPosition"); 411 mTexMatrixLoc = GLES20.glGetUniformLocation(mGLShaderProgram, "texMatrix"); 412 mTexRotMatrixLoc = GLES20.glGetUniformLocation(mGLShaderProgram, "texRotMatrix"); 413 mPaintItLoc = GLES20.glGetUniformLocation(mGLShaderProgram, "paintIt"); 414 415 GLES20.glUseProgram(mGLShaderProgram); 416 assertNoGLError("glUseProgram"); 417 } 418 419 /** 420 * Creates a new GLSL texture that can be populated by {@link SurfaceTexture} and returns the 421 * corresponding ID. Throws {@link ItsException} if there is an error creating the textures. 422 */ createTexture()423 private int createTexture() throws ItsException { 424 IntBuffer buffer = IntBuffer.allocate(1); 425 GLES20.glGenTextures(1, buffer); 426 int texId = buffer.get(0); 427 428 // This flags the texture to be implicitly populated by SurfaceTexture 429 GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, texId); 430 431 GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER, 432 GLES20.GL_NEAREST); 433 GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER, 434 GLES20.GL_LINEAR); 435 GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_S, 436 GLES20.GL_CLAMP_TO_EDGE); 437 GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_T, 438 GLES20.GL_CLAMP_TO_EDGE); 439 440 boolean isTexture = GLES20.glIsTexture(texId); 441 if (!isTexture) { 442 throw new ItsException("Failed to create texture id. Returned texture id: " + texId); 443 } 444 445 return texId; 446 } 447 448 /** 449 * Compiles the gives {@code source} as a shader of the provided {@code type}. Throws an 450 * {@link ItsException} if there are errors while compiling the shader. 451 */ createShader(int type, String source)452 private int createShader(int type, String source) throws ItsException { 453 int shader = GLES20.glCreateShader(type); 454 GLES20.glShaderSource(shader, source); 455 GLES20.glCompileShader(shader); 456 457 int[] compiled = new int[]{0}; 458 GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compiled, 0); 459 if (compiled[0] == GLES20.GL_FALSE) { 460 String msg = "Could not compile shader " + type + ": " 461 + GLES20.glGetShaderInfoLog(shader); 462 GLES20.glDeleteShader(shader); 463 throw new ItsException(msg); 464 } 465 466 return shader; 467 } 468 469 /** 470 * Throws an {@link ItsException} if the previous GL call resulted in an error. No-op otherwise. 471 */ assertNoGLError(String op)472 private void assertNoGLError(String op) throws ItsException { 473 int error = GLES20.glGetError(); 474 if (error != GLES20.GL_NO_ERROR) { 475 String msg = op + ": glError 0x" + Integer.toHexString(error); 476 throw new ItsException(msg); 477 } 478 } 479 480 481 482 /** 483 * Copies a frame encoded as a texture by {@code mCameraTexture} to 484 * {@code mRecordSurface} by running our simple shader program for one frame that draws 485 * to {@code mEGLRecorderSurface}. 486 */ copyFrameToRecordSurface()487 private void copyFrameToRecordSurface() throws ItsException { 488 // Clear color buffer 489 GLES20.glClearColor(0f, 0f, 0f, 1f); 490 assertNoGLError("glClearColor"); 491 GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); 492 assertNoGLError("glClear"); 493 494 // read texture transformation matrix from SurfaceTexture and write it to GLSL program. 495 mCameraTexture.getTransformMatrix(mTransformMatrix); 496 GLES20.glUniformMatrix4fv(mTexMatrixLoc, /* count= */1, /* transpose= */false, 497 mTransformMatrix, /* offset= */0); 498 assertNoGLError("glUniformMatrix4fv"); 499 500 // write texture rotation matrix to GLSL program 501 GLES20.glUniformMatrix2fv(mTexRotMatrixLoc, /* count= */1, /* transpose= */false, 502 mTexRotMatrix, /* offset= */0); 503 assertNoGLError("glUniformMatrix2fv"); 504 505 GLES20.glUniform1i(mPaintItLoc, mIsPaintGreen ? GREEN_PAINT : NO_PAINT); 506 assertNoGLError("glUniform1i"); 507 508 // write vertices of the full-screen rectangle to the GLSL program 509 ByteBuffer nativeBuffer = ByteBuffer.allocateDirect( 510 FULLSCREEN_VERTICES.length * Float.BYTES); 511 nativeBuffer.order(ByteOrder.nativeOrder()); 512 FloatBuffer vertexBuffer = nativeBuffer.asFloatBuffer(); 513 vertexBuffer.put(FULLSCREEN_VERTICES); 514 nativeBuffer.position(0); 515 vertexBuffer.position(0); 516 517 GLES20.glEnableVertexAttribArray(mVPositionLoc); 518 assertNoGLError("glEnableVertexAttribArray"); 519 GLES20.glVertexAttribPointer(mVPositionLoc, /* size= */ 2, GLES20.GL_FLOAT, 520 /* normalized= */ false, /* stride= */ 8, vertexBuffer); 521 assertNoGLError("glVertexAttribPointer"); 522 523 524 // viewport size should match the frame dimensions to prevent stretching/cropping 525 GLES20.glViewport(0, 0, mPreviewSize.getWidth(), mPreviewSize.getHeight()); 526 assertNoGLError("glViewport"); 527 528 GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */0, /* count= */4); 529 assertNoGLError("glDrawArrays"); 530 531 if (!EGL14.eglSwapBuffers(mEGLDisplay, mEGLRecorderSurface)) { 532 throw new ItsException("EglSwapBuffers failed to copy buffer to recording surface"); 533 } 534 } 535 536 /** 537 * Returns column major 2D rotation matrix that can be fed directly to GLSL. 538 * This matrix rotates around the origin. 539 */ getRotationMatrix(int orientationDegrees)540 private static float[] getRotationMatrix(int orientationDegrees) { 541 double rads = orientationDegrees * Math.PI / 180; 542 // Rotate clockwise because sensor orientation assumes clockwise rotation 543 return new float[] { 544 (float) Math.cos(rads), (float) -Math.sin(rads), 545 (float) Math.sin(rads), (float) Math.cos(rads) 546 }; 547 } 548 getCameraSurface()549 Surface getCameraSurface() { 550 return mCameraSurface; 551 } 552 553 /** 554 * Copies a frame encoded as a texture by {@code mCameraTexture} to a Bitmap by running our 555 * simple shader program for one frame and then convert the frame to a JPEG and write to 556 * the JPEG bytes to the {@code outputStream}. 557 * 558 * This method should not be called while recording. 559 * 560 * @param outputStream The stream to which the captured JPEG image bytes are written to 561 */ getFrame(OutputStream outputStream)562 void getFrame(OutputStream outputStream) throws ItsException { 563 synchronized (mRecordLock) { 564 if (mIsRecording) { 565 throw new ItsException("Attempting to get frame while recording is active is an " 566 + "invalid combination."); 567 } 568 569 ConditionVariable cv = new ConditionVariable(); 570 cv.close(); 571 // GL copy texture to JPEG should happen on the thread EGL Context was bound to 572 mHandler.post(() -> { 573 try { 574 copyFrameToRecordSurface(); 575 576 ByteBuffer frameBuffer = ByteBuffer.allocateDirect( 577 mPreviewSize.getWidth() * mPreviewSize.getHeight() * 4); 578 frameBuffer.order(ByteOrder.nativeOrder()); 579 580 GLES20.glReadPixels( 581 0, 582 0, 583 mPreviewSize.getWidth(), 584 mPreviewSize.getHeight(), 585 GLES20.GL_RGBA, 586 GLES20.GL_UNSIGNED_BYTE, 587 frameBuffer); 588 Bitmap frame = Bitmap.createBitmap( 589 mPreviewSize.getWidth(), 590 mPreviewSize.getHeight(), 591 Bitmap.Config.ARGB_8888); 592 frame.copyPixelsFromBuffer(frameBuffer); 593 frame.compress(Bitmap.CompressFormat.JPEG, 100, outputStream); 594 } catch (ItsException e) { 595 Logt.e(TAG, "Could not get frame from texture", e); 596 throw new ItsRuntimeException("Failed to get frame from texture", e); 597 } finally { 598 cv.open(); 599 } 600 }); 601 602 // Wait for up to two seconds for jpeg frame capture. 603 if (!cv.block(FRAME_CAPTURE_TIMEOUT_MS)) { 604 throw new ItsException("Frame capture timed out"); 605 } 606 } 607 } 608 609 /** 610 * Starts recording frames from mCameraSurface. This method should 611 * only be called once. Throws {@link ItsException} on subsequent calls. 612 */ startRecording()613 void startRecording() throws ItsException { 614 if (mRecordingStarted) { 615 throw new ItsException("Attempting to record on a stale PreviewRecorder. " 616 + "Create a new instance instead."); 617 } 618 mRecordingStarted = true; 619 Logt.i(TAG, "Starting Preview Recording."); 620 synchronized (mRecordLock) { 621 mIsRecording = true; 622 if (mMediaRecorder != null) { 623 mMediaRecorder.start(); 624 } else { 625 mMediaCodec.start(); 626 } 627 } 628 } 629 630 /** 631 * Override camera frames with green frames, if recordGreenFrames 632 * parameter is true. Record Green frames as buffer to workaround 633 * MediaRecorder issue of missing frames at the end of recording. 634 */ overrideCameraFrames(boolean recordGreenFrames)635 void overrideCameraFrames(boolean recordGreenFrames) throws ItsException { 636 Logt.i(TAG, "Recording Camera frames. recordGreenFrames = " + recordGreenFrames); 637 synchronized (mRecordLock) { 638 mIsPaintGreen = recordGreenFrames; 639 } 640 } 641 642 /** 643 * Stops recording frames. 644 */ stopRecording()645 void stopRecording() throws ItsException { 646 Logt.i(TAG, "Stopping Preview Recording."); 647 synchronized (mRecordLock) { 648 stopRecordingLocked(); 649 } 650 } 651 stopRecordingLocked()652 private void stopRecordingLocked() throws ItsException { 653 mIsRecording = false; 654 if (mMediaRecorder != null) { 655 mMediaRecorder.stop(); 656 } else { 657 mMediaCodec.signalEndOfInputStream(); 658 659 synchronized (mMediaCodecCondition) { 660 try { 661 mMediaCodecCondition.wait(ItsUtils.SESSION_CLOSE_TIMEOUT_MS); 662 } catch (InterruptedException e) { 663 throw new ItsException("Unexpected InterruptedException: ", e); 664 } 665 } 666 667 mMediaMuxer.stop(); 668 mMediaCodec.stop(); 669 } 670 } 671 672 @Override close()673 public void close() throws ItsException { 674 // synchronized to prevent reads and writes to surfaces while they are being released. 675 synchronized (mRecordLock) { 676 if (mIsRecording) { 677 Logt.e(TAG, "Preview recording was not stopped before closing."); 678 stopRecordingLocked(); 679 } 680 mCameraSurface.release(); 681 mCameraTexture.release(); 682 if (mMediaRecorder != null) { 683 mMediaRecorder.release(); 684 } 685 if (mMediaCodec != null) { 686 mMediaCodec.release(); 687 } 688 if (mMediaMuxer != null) { 689 mMediaMuxer.release(); 690 } 691 mRecordSurface.release(); 692 693 ConditionVariable cv = new ConditionVariable(); 694 cv.close(); 695 // GL Cleanup should happen on the thread EGL Context was bound to 696 mHandler.post(() -> { 697 try { 698 cleanupEgl(); 699 } finally { 700 cv.open(); 701 } 702 }); 703 704 // Wait for up to a second for egl to clean up. 705 // Since this is clean up, do nothing if the handler takes longer than 1s. 706 cv.block(/*timeoutMs=*/ 1000); 707 } 708 } 709 cleanupEgl()710 private void cleanupEgl() { 711 if (mGLShaderProgram == 0) { 712 // egl program was never set up, no cleanup needed 713 return; 714 } 715 716 Logt.i(TAG, "Cleaning up EGL Context"); 717 GLES20.glDeleteProgram(mGLShaderProgram); 718 // Release the egl surfaces and context from the handler 719 EGL14.eglMakeCurrent(mEGLDisplay, EGL14.EGL_NO_SURFACE, 720 EGL14.EGL_NO_SURFACE, EGL14.EGL_NO_CONTEXT); 721 EGL14.eglDestroySurface(mEGLDisplay, mEGLRecorderSurface); 722 EGL14.eglDestroyContext(mEGLDisplay, mEGLContext); 723 724 EGL14.eglTerminate(mEGLDisplay); 725 } 726 727 /** 728 * Returns Camera frame's timestamps only after recording completes. 729 */ getFrameTimeStamps()730 public List<Long> getFrameTimeStamps() throws IllegalStateException { 731 synchronized (mRecordLock) { 732 if (mIsRecording) { 733 throw new IllegalStateException("Can't return timestamps during recording."); 734 } 735 return mFrameTimeStamps; 736 } 737 } 738 } 739