1 /* <lambda>null2 * Copyright (C) 2024 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package platform.test.motion.filmstrip 18 19 import android.media.MediaCodec 20 import android.media.MediaCodecInfo.CodecCapabilities 21 import android.media.MediaFormat 22 import android.media.MediaFormat.KEY_BIT_RATE 23 import android.media.MediaFormat.KEY_COLOR_FORMAT 24 import android.media.MediaFormat.KEY_FRAME_RATE 25 import android.media.MediaMuxer 26 import android.view.Surface 27 import platform.test.motion.golden.TimestampFrameId 28 29 /** Produces an MP4 based on the [screenshots]. */ 30 class VideoRenderer(private val screenshots: List<MotionScreenshot>) { 31 32 private var screenshotWidth = screenshots.maxOf { it.bitmap.width }.roundUpToNextMultipleOf16() 33 private var screenshotHeight = 34 screenshots.maxOf { it.bitmap.height }.roundUpToNextMultipleOf16() 35 36 /** 37 * Creates an MP4 file at [path], which will contain all screenshots. 38 * 39 * [bitsPerPixel] is used to estimate the bitrate needed. 40 */ 41 fun renderToFile(path: String, bitsPerPixel: Float = 0.25f) { 42 require(screenshots.isNotEmpty()) { "Filmstrip must have at least one screenshot" } 43 val muxer = MediaMuxer(path, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4) 44 45 val bitrate = (screenshotWidth * screenshotHeight * bitsPerPixel * FRAME_RATE).toInt() 46 val mime = "video/avc" 47 val format = MediaFormat.createVideoFormat(mime, screenshotWidth, screenshotHeight) 48 format.setInteger(KEY_BIT_RATE, bitrate) 49 format.setFloat(KEY_FRAME_RATE, FRAME_RATE) 50 format.setInteger(KEY_COLOR_FORMAT, CodecCapabilities.COLOR_FormatSurface) 51 format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1) 52 53 val codec = MediaCodec.createEncoderByType(mime) 54 codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE) 55 val surface = codec.createInputSurface() 56 codec.start() 57 58 encodeScreenshotsInVideo(codec, muxer, surface) 59 60 codec.stop() 61 codec.release() 62 muxer.stop() 63 muxer.release() 64 } 65 66 private fun encodeScreenshotsInVideo(encoder: MediaCodec, muxer: MediaMuxer, surface: Surface) { 67 val bufferInfo = MediaCodec.BufferInfo() 68 val screenshotIterator = screenshots.iterator() 69 var isEndOfStream = false 70 var videoTrackIndex = -1 71 72 // The encoder uses the system clock of the [unlockCanvasAndPost] call as frame time. 73 // However, this is arbitrary, as encoding happens as fast as possible. To avoid the extra 74 // complexity of video bitmap format conversion that would be required when using 75 // [queueInputBuffer] instead (which would allow specifying the presentation time), this 76 // will override the presentation time when muxing instead. 77 val framePresentationTimesUsIterator = 78 buildList { 79 add(0L) 80 var presentationTimeUs = 0L 81 screenshots 82 .zipWithNext { first, second -> 83 if ( 84 first.frameId is TimestampFrameId && 85 second.frameId is TimestampFrameId 86 ) { 87 second.frameId.milliseconds - first.frameId.milliseconds 88 } else { 89 // Exactly one frame for before / after 90 FRAME_DURATION 91 } 92 } 93 .forEach { frameDurationMillis -> 94 presentationTimeUs += frameDurationMillis * 1000L 95 add(presentationTimeUs) 96 } 97 } 98 .iterator() 99 100 while (true) { 101 if (screenshotIterator.hasNext()) { 102 surface.lockCanvas(null).also { canvas -> 103 val screenshot = screenshotIterator.next() 104 canvas.drawBitmap(screenshot.bitmap, 0.0f, 0.0f, null) 105 surface.unlockCanvasAndPost(canvas) 106 } 107 } else if (!isEndOfStream) { 108 encoder.signalEndOfInputStream() 109 isEndOfStream = true 110 } 111 112 val outputBufferIndex = encoder.dequeueOutputBuffer(bufferInfo, DEQUEUE_TIMEOUT_US) 113 if (outputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { 114 // No output available yet. 115 continue 116 } else if (outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 117 videoTrackIndex = muxer.addTrack(encoder.outputFormat) 118 muxer.start() 119 } else if (outputBufferIndex >= 0) { 120 val encodedDataBuffer = 121 encoder.getOutputBuffer(outputBufferIndex) 122 ?: throw RuntimeException("encoderOutputBuffer $outputBufferIndex was null") 123 124 if (bufferInfo.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG != 0) { 125 // Config was already sent to the muxer in INFO_OUTPUT_FORMAT_CHANGED. 126 bufferInfo.size = 0 127 } 128 129 if (bufferInfo.size != 0) { 130 encodedDataBuffer.position(bufferInfo.offset) 131 encodedDataBuffer.limit(bufferInfo.offset + bufferInfo.size) 132 check(framePresentationTimesUsIterator.hasNext()) { 133 "More than the number of input frames are sent to the output" 134 } 135 bufferInfo.presentationTimeUs = framePresentationTimesUsIterator.next() 136 muxer.writeSampleData(videoTrackIndex, encodedDataBuffer, bufferInfo) 137 } 138 139 encoder.releaseOutputBuffer(outputBufferIndex, false) 140 141 if (bufferInfo.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0) { 142 return 143 } 144 } else { 145 throw AssertionError("Unexpected dequeueOutputBuffer response $outputBufferIndex") 146 } 147 } 148 } 149 150 companion object { 151 // Tests produce a frame every 16ms (62.5fps) 152 const val FRAME_DURATION = 16L 153 const val FRAME_RATE = 1000f / FRAME_DURATION 154 const val DEQUEUE_TIMEOUT_US = 10_000L 155 156 private fun Int.roundUpToNextMultipleOf16(): Int = (this + 15) and 0xF.inv() 157 } 158 } 159