1 /* 2 * Copyright 2023 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.media.mediaediting.cts; 18 19 import static androidx.media3.common.C.MEDIA_CODEC_PRIORITY_NON_REALTIME; 20 import static androidx.media3.common.util.Assertions.checkNotNull; 21 import static androidx.media3.common.util.Assertions.checkState; 22 import static androidx.media3.common.util.Assertions.checkStateNotNull; 23 import static com.google.common.truth.Truth.assertThat; 24 25 import android.content.Context; 26 import android.content.res.AssetFileDescriptor; 27 import android.graphics.ImageFormat; 28 import android.media.Image; 29 import android.media.ImageReader; 30 import android.media.MediaCodec; 31 import android.media.MediaCodecInfo; 32 import android.media.MediaExtractor; 33 import android.media.MediaFormat; 34 import android.os.Handler; 35 import androidx.annotation.Nullable; 36 import androidx.annotation.RequiresApi; 37 import androidx.media3.common.MimeTypes; 38 import androidx.media3.common.util.ConditionVariable; 39 import androidx.media3.common.util.UnstableApi; 40 import androidx.media3.common.util.Util; 41 import java.io.IOException; 42 import java.nio.ByteBuffer; 43 44 /** A wrapper for decoding a video using {@link MediaCodec}. */ 45 @UnstableApi 46 @RequiresApi(21) 47 public final class VideoDecodingWrapper implements AutoCloseable { 48 49 private static final int IMAGE_AVAILABLE_TIMEOUT_MS = 10_000; 50 51 // Use ExoPlayer's 10ms timeout setting. In practise, the test durations from using timeouts of 52 // 1/10/100ms don't differ significantly. 53 private static final long DEQUEUE_TIMEOUT_US = 10_000; 54 // SSIM should be calculated using the luma (Y') channel, thus using the YUV color space. 55 private static final int IMAGE_READER_COLOR_SPACE = ImageFormat.YUV_420_888; 56 private static final int MEDIA_CODEC_COLOR_SPACE = 57 MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible; 58 private static final String ASSET_FILE_SCHEME = "asset:///"; 59 60 private final MediaFormat mediaFormat; 61 private final MediaCodec mediaCodec; 62 private final MediaExtractor mediaExtractor; 63 private final MediaCodec.BufferInfo bufferInfo; 64 private final ImageReader imageReader; 65 private final ConditionVariable imageAvailableConditionVariable; 66 private final int comparisonInterval; 67 68 private boolean isCurrentFrameComparisonFrame; 69 private boolean hasReadEndOfInputStream; 70 private boolean queuedEndOfStreamToDecoder; 71 private boolean dequeuedAllDecodedFrames; 72 private boolean isCodecStarted; 73 private int dequeuedFramesCount; 74 75 /** 76 * Creates a new instance. 77 * 78 * @param context The {@link Context}. 79 * @param filePath The path to the video file. 80 * @param comparisonInterval The number of frames between the frames selected for comparison. 81 * @param maxImagesAllowed The max number of images allowed in {@link ImageReader}. 82 * @throws IOException When failed to open the video file. 83 */ VideoDecodingWrapper( Context context, String filePath, int comparisonInterval, int maxImagesAllowed)84 public VideoDecodingWrapper( 85 Context context, String filePath, int comparisonInterval, int maxImagesAllowed) 86 throws IOException { 87 this.comparisonInterval = comparisonInterval; 88 mediaExtractor = new MediaExtractor(); 89 bufferInfo = new MediaCodec.BufferInfo(); 90 91 if (filePath.contains(ASSET_FILE_SCHEME)) { 92 AssetFileDescriptor assetFd = 93 context.getAssets().openFd(filePath.replace(ASSET_FILE_SCHEME, "")); 94 mediaExtractor.setDataSource( 95 assetFd.getFileDescriptor(), assetFd.getStartOffset(), assetFd.getLength()); 96 } else { 97 mediaExtractor.setDataSource(filePath); 98 } 99 100 @Nullable MediaFormat mediaFormat = null; 101 for (int i = 0; i < mediaExtractor.getTrackCount(); i++) { 102 if (MimeTypes.isVideo(mediaExtractor.getTrackFormat(i).getString(MediaFormat.KEY_MIME))) { 103 mediaFormat = mediaExtractor.getTrackFormat(i); 104 mediaExtractor.selectTrack(i); 105 break; 106 } 107 } 108 109 checkStateNotNull(mediaFormat); 110 checkState(mediaFormat.containsKey(MediaFormat.KEY_WIDTH)); 111 int width = mediaFormat.getInteger(MediaFormat.KEY_WIDTH); 112 checkState(mediaFormat.containsKey(MediaFormat.KEY_HEIGHT)); 113 int height = mediaFormat.getInteger(MediaFormat.KEY_HEIGHT); 114 115 // Create a handler for the main thread to receive image available notifications. The current 116 // (test) thread blocks until this callback is received. 117 Handler mainThreadHandler = Util.createHandlerForCurrentOrMainLooper(); 118 imageAvailableConditionVariable = new ConditionVariable(); 119 imageReader = 120 ImageReader.newInstance(width, height, IMAGE_READER_COLOR_SPACE, maxImagesAllowed); 121 imageReader.setOnImageAvailableListener( 122 imageReader -> imageAvailableConditionVariable.open(), mainThreadHandler); 123 124 String sampleMimeType = checkNotNull(mediaFormat.getString(MediaFormat.KEY_MIME)); 125 mediaFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, MEDIA_CODEC_COLOR_SPACE); 126 mediaFormat.setInteger(MediaFormat.KEY_PRIORITY, MEDIA_CODEC_PRIORITY_NON_REALTIME); 127 this.mediaFormat = mediaFormat; 128 mediaCodec = MediaCodec.createDecoderByType(sampleMimeType); 129 } 130 131 /** 132 * Returns the next decoded comparison frame, or {@code null} if the stream has ended. The caller 133 * takes ownership of any returned image and is responsible for closing it before calling this 134 * method again. 135 */ 136 @Nullable runUntilComparisonFrameOrEnded()137 public Image runUntilComparisonFrameOrEnded() throws InterruptedException { 138 if (!isCodecStarted) { 139 mediaCodec.configure( 140 mediaFormat, imageReader.getSurface(), /* crypto= */ null, /* flags= */ 0); 141 mediaCodec.start(); 142 isCodecStarted = true; 143 } 144 while (!hasEnded() && !isCurrentFrameComparisonFrame) { 145 while (dequeueOneFrameFromDecoder()) {} 146 while (queueOneFrameToDecoder()) {} 147 } 148 if (isCurrentFrameComparisonFrame && !hasEnded()) { 149 isCurrentFrameComparisonFrame = false; 150 assertThat(imageAvailableConditionVariable.block(IMAGE_AVAILABLE_TIMEOUT_MS)).isTrue(); 151 imageAvailableConditionVariable.close(); 152 return imageReader.acquireLatestImage(); 153 } 154 return null; 155 } 156 157 /** Returns whether decoding has ended. */ hasEnded()158 private boolean hasEnded() { 159 return dequeuedAllDecodedFrames; 160 } 161 162 /** Returns whether a frame is queued to the {@link MediaCodec decoder}. */ queueOneFrameToDecoder()163 private boolean queueOneFrameToDecoder() { 164 if (queuedEndOfStreamToDecoder) { 165 return false; 166 } 167 168 int inputBufferIndex = mediaCodec.dequeueInputBuffer(DEQUEUE_TIMEOUT_US); 169 if (inputBufferIndex < 0) { 170 return false; 171 } 172 173 if (hasReadEndOfInputStream) { 174 mediaCodec.queueInputBuffer( 175 inputBufferIndex, 176 /* offset= */ 0, 177 /* size= */ 0, 178 /* presentationTimeUs= */ 0, 179 MediaCodec.BUFFER_FLAG_END_OF_STREAM); 180 queuedEndOfStreamToDecoder = true; 181 return false; 182 } 183 184 ByteBuffer inputBuffer = checkNotNull(mediaCodec.getInputBuffer(inputBufferIndex)); 185 int sampleSize = mediaExtractor.readSampleData(inputBuffer, /* offset= */ 0); 186 mediaCodec.queueInputBuffer( 187 inputBufferIndex, 188 /* offset= */ 0, 189 sampleSize, 190 mediaExtractor.getSampleTime(), 191 mediaExtractor.getSampleFlags()); 192 // MediaExtractor.advance does not reliably return false for end-of-stream, so check sample 193 // metadata instead as a more reliable signal. See [internal: b/121204004]. 194 mediaExtractor.advance(); 195 hasReadEndOfInputStream = mediaExtractor.getSampleTime() == -1; 196 return true; 197 } 198 199 /** Returns whether a frame is decoded, renders the frame if the frame is a comparison frame. */ dequeueOneFrameFromDecoder()200 private boolean dequeueOneFrameFromDecoder() { 201 if (isCurrentFrameComparisonFrame) { 202 return false; 203 } 204 205 int outputBufferIndex = mediaCodec.dequeueOutputBuffer(bufferInfo, DEQUEUE_TIMEOUT_US); 206 if (outputBufferIndex < 0) { 207 return false; 208 } 209 isCurrentFrameComparisonFrame = dequeuedFramesCount % comparisonInterval == 0; 210 dequeuedFramesCount++; 211 mediaCodec.releaseOutputBuffer(outputBufferIndex, /* render= */ isCurrentFrameComparisonFrame); 212 213 if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { 214 dequeuedAllDecodedFrames = true; 215 } 216 return true; 217 } 218 219 @Override close()220 public void close() { 221 mediaExtractor.release(); 222 mediaCodec.release(); 223 imageReader.close(); 224 } 225 } 226