1 /* 2 * Copyright (C) 2023 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.tools.device.traces.monitors 18 19 import android.content.Context 20 import android.hardware.display.DisplayManager 21 import android.media.MediaCodec 22 import android.media.MediaCodecInfo 23 import android.media.MediaFormat 24 import android.media.MediaMuxer 25 import android.os.SystemClock 26 import android.tools.common.FLICKER_TAG 27 import android.tools.common.Logger 28 import android.tools.device.traces.deleteIfExists 29 import android.util.DisplayMetrics 30 import android.view.WindowManager 31 import java.io.File 32 import java.io.FileOutputStream 33 import java.nio.ByteBuffer 34 import java.nio.ByteOrder 35 import java.util.concurrent.TimeUnit 36 37 /** Runnable to record the screen contents and winscope metadata */ 38 class ScreenRecordingRunnable( 39 private val outputFile: File, 40 context: Context, 41 private val width: Int = 720, 42 private val height: Int = 1280 43 ) : Runnable { 44 private val displayManager = context.getSystemService(Context.DISPLAY_SERVICE) as DisplayManager 45 private val windowManager = context.getSystemService(Context.WINDOW_SERVICE) as WindowManager 46 private var finished = false 47 internal var isFrameRecorded = false 48 49 private val metrics: DisplayMetrics 50 get() { 51 val metrics = DisplayMetrics() 52 windowManager.defaultDisplay.getRealMetrics(metrics) 53 return metrics 54 } 55 56 private val encoder = createEncoder() 57 private val inputSurface = encoder.createInputSurface() 58 private val virtualDisplay = 59 displayManager.createVirtualDisplay( 60 "Recording Display", 61 width, 62 height, 63 metrics.densityDpi, 64 inputSurface, 65 DisplayManager.VIRTUAL_DISPLAY_FLAG_AUTO_MIRROR, 66 null, 67 null 68 ) 69 private val muxer = createMuxer() 70 private var metadataTrackIndex = -1 71 private var videoTrackIndex = -1 72 stopnull73 internal fun stop() { 74 encoder.signalEndOfInputStream() 75 finished = true 76 } 77 runnull78 override fun run() { 79 Logger.d(FLICKER_TAG, "Starting screen recording to file $outputFile") 80 81 val timestampsMonotonicUs = mutableListOf<Long>() 82 try { 83 // Start encoder and muxer 84 encoder.start() 85 val bufferInfo = MediaCodec.BufferInfo() 86 87 while (true) { 88 val bufferIndex = encoder.dequeueOutputBuffer(bufferInfo, TIMEOUT_MS) 89 if (bufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 90 prepareMuxer() 91 } else if (bufferIndex >= 0) { 92 val timestampMonotonicUs = writeSample(bufferIndex, bufferInfo) 93 val endOfStream = bufferInfo.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM 94 // end of the stream samples have 0 timestamp 95 if (endOfStream > 0) { 96 break 97 } else { 98 timestampsMonotonicUs.add(timestampMonotonicUs) 99 } 100 } 101 } 102 } finally { 103 writeMetadata(timestampsMonotonicUs) 104 encoder.stop() 105 muxer.stop() 106 muxer.release() 107 encoder.release() 108 inputSurface.release() 109 virtualDisplay.release() 110 } 111 } 112 113 /** 114 * Fetches a sample from the encoder and writes it to the video file 115 * 116 * @return sample timestamp (or 0 for invalid buffers) 117 */ writeSamplenull118 private fun writeSample(bufferIndex: Int, bufferInfo: MediaCodec.BufferInfo): Long { 119 val data = encoder.getOutputBuffer(bufferIndex) 120 return if (data != null) { 121 val endOfStream = bufferInfo.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM 122 123 if (endOfStream == 0) { 124 val outputBuffer = 125 encoder.getOutputBuffer(bufferIndex) ?: error("Unable to acquire next frame") 126 127 muxer.writeSampleData(videoTrackIndex, outputBuffer, bufferInfo) 128 isFrameRecorded = true 129 } 130 encoder.releaseOutputBuffer(bufferIndex, /* render */ false) 131 bufferInfo.presentationTimeUs 132 } else { 133 0 134 } 135 } 136 prepareMuxernull137 private fun prepareMuxer() { 138 videoTrackIndex = muxer.addTrack(encoder.outputFormat) 139 val metadataFormat = MediaFormat() 140 metadataFormat.setString(MediaFormat.KEY_MIME, MIME_TYPE_METADATA) 141 metadataTrackIndex = muxer.addTrack(metadataFormat) 142 muxer.start() 143 } 144 145 /** 146 * Saves metadata needed by Winscope to synchronize the screen recording playback with other 147 * traces. 148 * 149 * The metadata (version 2) is written as a binary array with the following format: 150 * - winscope magic string (#VV1NSC0PET1ME2#, 16B). 151 * - the metadata version number (4B little endian). 152 * - Realtime-to-elapsed time offset in nanoseconds (8B little endian). 153 * - the recorded frames count (4B little endian) 154 * - for each recorded frame: 155 * ``` 156 * - System time in elapsed clock timebase in nanoseconds (8B little endian). 157 * ``` 158 */ writeMetadatanull159 private fun writeMetadata(timestampsMonotonicUs: List<Long>) { 160 if (timestampsMonotonicUs.isEmpty()) { 161 Logger.v(FLICKER_TAG, "Not writing winscope metadata (no frames/timestamps)") 162 return 163 } 164 165 Logger.v( 166 FLICKER_TAG, 167 "Writing winscope metadata (size=${timestampsMonotonicUs.size} " + 168 ", monotonic timestamps range [us] = " + 169 "${timestampsMonotonicUs.first()}-${timestampsMonotonicUs.last()})" 170 ) 171 172 val monotonicTimeNs = TimeUnit.MILLISECONDS.toNanos(SystemClock.uptimeMillis()) 173 val elapsedTimeNs = SystemClock.elapsedRealtimeNanos() 174 val realTimeNs = TimeUnit.MILLISECONDS.toNanos(System.currentTimeMillis()) 175 176 val elapsedToMonotonicTimeOffsetNs = elapsedTimeNs - monotonicTimeNs 177 val realToElapsedTimeOffsetNs = realTimeNs - elapsedTimeNs 178 179 val bufferSize = 180 WINSCOPE_MAGIC_STRING.toByteArray().size + 181 Int.SIZE_BYTES + 182 Long.SIZE_BYTES + 183 Int.SIZE_BYTES + 184 (timestampsMonotonicUs.size * Long.SIZE_BYTES) 185 186 val buffer = 187 ByteBuffer.allocate(bufferSize) 188 .order(ByteOrder.LITTLE_ENDIAN) 189 .put(WINSCOPE_MAGIC_STRING.toByteArray()) 190 .putInt(WINSCOPE_METADATA_VERSION) 191 .putLong(realToElapsedTimeOffsetNs) 192 .putInt(timestampsMonotonicUs.size) 193 .apply { 194 timestampsMonotonicUs.forEach { 195 putLong(elapsedToMonotonicTimeOffsetNs + TimeUnit.MICROSECONDS.toNanos(it)) 196 } 197 } 198 199 val bufferInfo = MediaCodec.BufferInfo() 200 bufferInfo.size = bufferSize 201 bufferInfo.presentationTimeUs = timestampsMonotonicUs[0] 202 muxer.writeSampleData(metadataTrackIndex, buffer, bufferInfo) 203 } 204 205 /** 206 * Create and configure a MediaCodec encoder with [MIME_TYPE_VIDEO] format. 207 * 208 * @return a Surface that can be used to record 209 */ createEncodernull210 private fun createEncoder(): MediaCodec { 211 val format = MediaFormat.createVideoFormat(MIME_TYPE_VIDEO, width, height) 212 val displayMode = windowManager.defaultDisplay.mode 213 format.setInteger( 214 MediaFormat.KEY_COLOR_FORMAT, 215 MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface 216 ) 217 format.setInteger(MediaFormat.KEY_BIT_RATE, BIT_RATE) 218 format.setFloat(MediaFormat.KEY_FRAME_RATE, displayMode.refreshRate) 219 format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, IFRAME_INTERVAL) 220 format.setInteger(MediaFormat.KEY_WIDTH, width) 221 format.setInteger(MediaFormat.KEY_HEIGHT, height) 222 format.setString(MediaFormat.KEY_MIME, MIME_TYPE_VIDEO) 223 224 val mediaCodec = MediaCodec.createEncoderByType(MIME_TYPE_VIDEO) 225 mediaCodec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE) 226 return mediaCodec 227 } 228 createMuxernull229 private fun createMuxer(): MediaMuxer { 230 outputFile.deleteIfExists() 231 require(!outputFile.exists()) 232 outputFile.createNewFile() 233 val inputStream = FileOutputStream(outputFile) 234 return MediaMuxer(inputStream.fd, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4) 235 } 236 237 companion object { 238 private const val WINSCOPE_MAGIC_STRING = "#VV1NSC0PET1ME2#" 239 private const val WINSCOPE_METADATA_VERSION = 2 240 private const val MIME_TYPE_VIDEO = MediaFormat.MIMETYPE_VIDEO_AVC 241 private const val MIME_TYPE_METADATA = "application/octet-stream" 242 private const val BIT_RATE = 2000000 // 2Mbps 243 private const val IFRAME_INTERVAL = 2 // 2 second between I-frames 244 private const val TIMEOUT_MS = 100L 245 } 246 } 247